diff --git a/.env.example b/.env.example index 9527957b3d..470a6107d6 100644 --- a/.env.example +++ b/.env.example @@ -55,10 +55,11 @@ INITIAL_PASSWORD=CHANGEME # loader (bin/cli/plugins.mjs) at a package tree — this one drives the server-side scanner. # OMNIROUTE_PLUGINS_DIR=/opt/omniroute/plugins -# Escape hatch for the test-context DATA_DIR guard (#10428). A test run that never -# chose a DATA_DIR is redirected to a throwaway temp dir so it cannot open the -# operator's real database. Set to 1 only for a deliberate run against the real -# DATA_DIR — never for CI. Used by: src/lib/dataPaths.ts +# Escape hatch for the test/eval DATA_DIR guard (#10428). A test or node eval/print +# probe (-e/--eval/-p/--print, including --eval=/--print=) that never chose a DATA_DIR +# is redirected to a throwaway temp dir so it cannot open the operator's real database. +# Set to 1 only for a deliberate run against the real DATA_DIR — never for CI. +# Used by: src/lib/dataPaths.ts # OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1 # Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git @@ -96,9 +97,11 @@ STORAGE_ENCRYPTION_KEY= # Default: v1 | Increment when rotating STORAGE_ENCRYPTION_KEY. STORAGE_ENCRYPTION_KEY_VERSION=v1 -# Automatic SQLite backup on startup. -# Used by: src/lib/db/backup.ts — creates a timestamped backup before migrations. -# Default: false (backups enabled) | Set true to skip backup on every restart. +# Routine/pre-write SQLite backups. +# Used by: src/lib/db/backup.ts. Set true only when those backups are managed externally. +# This never disables the migration runner's mandatory, content-addressed safety snapshot +# or its mass-migration guard for an existing persistent database. +# Default: false (routine backups enabled). DISABLE_SQLITE_AUTO_BACKUP=false # ── Redis (Rate Limiting) ── diff --git a/CHANGELOG.md b/CHANGELOG.md index bfcad514f4..80abce64dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,10 @@ _Living section — cycle opened at the v3.8.50 freeze (parallel-cycle model). B ### 🐛 Bug Fixes +- **security(streaming):** sanitize generic mid-stream error messages before emitting OpenAI, + Responses, or Claude SSE failure frames and before diagnostic logging, while preserving raw + failures for internal classification and keeping client disconnects out of provider failure state. + ### 📝 Maintenance --- diff --git a/README.md b/README.md index eac646a990..fe42852adf 100644 --- a/README.md +++ b/README.md @@ -7,19 +7,19 @@ # 🚀 OmniRoute — The Free AI Gateway -OmniRoute — Never stop coding. Every AI tool → 357 providers — 150+ free — through one endpoint. Claude Code, Codex, Cursor, Cline, Copilot & Antigravity into FREE Claude / GPT / Gemini with auto-fallback. RTK + Caveman stacked compression saves 15–95% tokens (~89% avg) — never hit limits. 357 AI providers · 150+ free tiers · ~1.66B free tokens/mo · 19 routing strategies · $0 to start. +OmniRoute — Never stop coding. Every AI tool → 357 providers — 150+ free — through one endpoint. Claude Code, Codex, Cursor, Cline, Copilot & Antigravity into FREE Claude / GPT / Gemini with auto-fallback. RTK + Caveman stacked compression saves 15–95% tokens (~89% avg) — never hit limits. 357 AI providers · 150+ free tiers · ~1.63B free tokens/mo · 19 routing strategies · $0 to start.
-## 💰 ~1.66B Free Tokens / Month +## 💰 ~1.63B Free Tokens / Month
-> Stacking free tiers by hand is painful — dozens of SDKs, dozens of rate limits, and no idea how much you actually have. OmniRoute catalogs **476 free-tier entries across 39 recurring pool keys** and computes the token headline from the **21 pools with a published positive monthly budget**, deduplicated by shared pool. The result stays visible on the dashboard (`/dashboard/free-tiers`). +> Stacking free tiers by hand is painful — dozens of SDKs, dozens of rate limits, and no idea how much you actually have. OmniRoute catalogs **476 free-tier entries across 38 recurring pool keys** and computes the token headline from the **20 pools with a published positive monthly budget**, deduplicated by shared pool. The result stays visible on the dashboard (`/dashboard/free-tiers`). -OmniRoute free-tier budget card: ~1.66B free tokens per month steady, up to ~2.28B in the first month with signup credits, from 39 documented recurring pool keys covering 476 cataloged free-tier entries behind one endpoint. Honest pool-deduped math — each shared pool counted once, including 21 recurring pools with a published positive monthly token budget; 13 providers are marked avoid in the terms-risk catalog so you decide. Budget bar includes Mistral 1B, LLM7 150M, Nara 150M, Gemini 60M and smaller pools, plus first-month signup credits and permanently-free no-token-cap providers surfaced separately so they never inflate the headline. Live used/remaining on /dashboard/free-tiers. +OmniRoute free-tier budget card: ~1.63B free tokens per month steady, up to ~2.25B in the first month with signup credits, from 38 documented recurring pool keys covering 476 cataloged free-tier entries behind one endpoint. Honest pool-deduped math — each shared pool counted once, including 20 recurring pools with a published positive monthly token budget; 13 providers are marked avoid in the terms-risk catalog so you decide. Budget bar includes Mistral 1B, LLM7 150M, Nara 150M, xKiro 150M, Gemini 60M and smaller pools, plus first-month signup credits and permanently-free no-token-cap providers surfaced separately so they never inflate the headline. Live used/remaining on /dashboard/free-tiers. > Animated summary of the live `/dashboard/free-tiers` page. Full methodology (pool dedupe, credit tiers, provider terms): **[docs/reference/FREE_TIERS.md](docs/reference/FREE_TIERS.md)**. > @@ -209,7 +209,7 @@ curl http://localhost:20128/v1/chat/completions \ -The Promise — One endpoint and 357 providers. Automatic fallback keeps routing while another healthy target is available. Six pillars: resilient fallback across 357 providers · up to 95% token savings on eligible workloads · $0 to start with 150+ free tiers and 54 recurring/keyless free-forever providers · 36 CLI/agent integrations through one config · OpenAI, Claude, Gemini and Responses API compatibility at /v1 · production controls including circuit breakers, TLS stealth, MCP 110 tools, A2A, memory, guardrails, evals and 39,000+ static test declarations across 5,100+ tracked test files. +The Promise — One endpoint and 357 providers. Automatic fallback keeps routing while another healthy target is available. Six pillars: resilient fallback across 357 providers · up to 95% token savings on eligible workloads · $0 to start with 150+ free tiers and 53 recurring/keyless free-forever providers · 36 CLI/agent integrations through one config · OpenAI, Claude, Gemini and Responses API compatibility at /v1 · production controls including circuit breakers, TLS stealth, MCP 110 tools, A2A, memory, guardrails, evals and 39,000+ static test declarations across 5,100+ tracked test files.

@@ -518,9 +518,9 @@ Pix copia-e-cola: ## 📡 OmniRoute Radar -The main free-tier headline remains **~1.66B tokens/month** from the documented, +The main free-tier headline remains **~1.63B tokens/month** from the documented, pool-deduplicated catalog above. Temporary provider signup credits can separately lift the first -month to **~2.28B**. Radar is an optional, signed catalog overlay for people who want fresher +month to **~2.25B**. Radar is an optional, signed catalog overlay for people who want fresher free-model availability between OmniRoute releases; the community catalog and every existing free feature remain free. @@ -648,7 +648,7 @@ of your shell history. → [CLI Integrations](docs/guides/CLI-INTEGRATIONS.md) -> **357 registered providers** across the canonical chat, media, search, local, cloud-agent and system collections, including **152 carrying `hasFree: true` discovery metadata**. The chat model registry covers **229 providers / 2,554 distinct provider-model pairs / 1,283 raw model IDs**; the separate free-budget catalog has **476 per-model rows**, **39 recurring pools** and **54 recurring/keyless free-forever providers**. These are different denominators by design; definitions and pool-deduped calculations live in the [Provider Reference](docs/reference/PROVIDER_REFERENCE.md) and [Free Tiers](docs/reference/FREE_TIERS.md). +> **357 registered providers** across the canonical chat, media, search, local, cloud-agent and system collections, including **152 carrying `hasFree: true` discovery metadata**. The chat model registry covers **229 providers / 2,554 distinct provider-model pairs / 1,283 raw model IDs**; the separate free-budget catalog has **476 per-model rows**, **38 recurring pools** and **53 recurring/keyless free-forever providers**. These are different denominators by design; definitions and pool-deduped calculations live in the [Provider Reference](docs/reference/PROVIDER_REFERENCE.md) and [Free Tiers](docs/reference/FREE_TIERS.md).
@@ -1307,7 +1307,7 @@ Métricas canônicas em 2026-08-24: **1.029 vídeos únicos** · **11.132.922 vi Resilience GuideCircuit breakers, cooldowns, queue, anti-thundering herd, TLS spoofing Auto-Combo Engine16-factor scoring, mode packs, self-healing Proxy Guide3-level proxy system, 1proxy marketplace, registry CRUD - Free TiersConsolidated directory: 39 documented recurring pools / 476 cataloged free-tier entries + Free TiersConsolidated directory: 38 documented recurring pools / 476 cataloged free-tier entries Features GalleryVisual dashboard tour with screenshots Codebase DocumentationBeginner-friendly codebase walkthrough diff --git a/changelog.d/fixes/0000-huggingchat-transport-error-redaction.md b/changelog.d/fixes/0000-huggingchat-transport-error-redaction.md new file mode 100644 index 0000000000..3951c9dba8 --- /dev/null +++ b/changelog.d/fixes/0000-huggingchat-transport-error-redaction.md @@ -0,0 +1 @@ +- Sanitize HuggingChat conversation-creation and message-send transport failures before they reach client error bodies or provider logs. diff --git a/changelog.d/fixes/0000-public-error-boundary-hardening.md b/changelog.d/fixes/0000-public-error-boundary-hardening.md new file mode 100644 index 0000000000..c0531eba1d --- /dev/null +++ b/changelog.d/fixes/0000-public-error-boundary-hardening.md @@ -0,0 +1 @@ +- **fix(security):** Sanitize provider and runtime failures before public API, SSE and MCP responses and before persistent request, proxy and usage logs, preventing credentials, stack traces and host filesystem paths from crossing those boundaries while preserving stable error codes and useful diagnostics. diff --git a/changelog.d/fixes/11773-cerebras-free-tier.md b/changelog.d/fixes/11773-cerebras-free-tier.md new file mode 100644 index 0000000000..ecc67975de --- /dev/null +++ b/changelog.d/fixes/11773-cerebras-free-tier.md @@ -0,0 +1 @@ +- **fix(providers):** reclassify Cerebras as a one-time $5 signup credit (payment method required, 30-day validity), not a recurring no-card 1M tokens/day trial ([#11773](https://github.com/diegosouzapw/OmniRoute/issues/11773)) diff --git a/changelog.d/fixes/12304-cache-config-preserve-client-cache.md b/changelog.d/fixes/12304-cache-config-preserve-client-cache.md new file mode 100644 index 0000000000..1c4da7f4c7 --- /dev/null +++ b/changelog.d/fixes/12304-cache-config-preserve-client-cache.md @@ -0,0 +1 @@ +- **fix(settings):** `PUT /api/settings/cache-config` now persists `alwaysPreserveClientCache` to the flat general settings the runtime cache-control policy actually reads; previously the value landed in the databaseSettings "cache" section and was silently ignored, so the endpoint had no effect on `cache_control` passthrough ([#12304](https://github.com/diegosouzapw/OmniRoute/pull/12304)) — thanks @davidebaraldo diff --git a/changelog.d/fixes/PENDING-perplexity-stream-error-boundary.md b/changelog.d/fixes/PENDING-perplexity-stream-error-boundary.md new file mode 100644 index 0000000000..0d9a14a2db --- /dev/null +++ b/changelog.d/fixes/PENDING-perplexity-stream-error-boundary.md @@ -0,0 +1 @@ +- **fix(providers):** Perplexity Web no longer turns upstream stream failures into successful assistant text; pre-content failures remain eligible for fallback, partial output ends with a structured sanitized error, and failed sessions are not persisted diff --git a/changelog.d/fixes/PENDING-zai-web-stream-error-boundary.md b/changelog.d/fixes/PENDING-zai-web-stream-error-boundary.md new file mode 100644 index 0000000000..ba4d82521b --- /dev/null +++ b/changelog.d/fixes/PENDING-zai-web-stream-error-boundary.md @@ -0,0 +1 @@ +- **Z.ai Web:** HTTP 200 streams carrying an upstream error now terminate with a structured failure instead of assistant text plus a normal stop, preserving partial output while allowing pre-content combo fallback. diff --git a/changelog.d/fixes/adapta-nonstream-sse-error.md b/changelog.d/fixes/adapta-nonstream-sse-error.md new file mode 100644 index 0000000000..fb34658a4b --- /dev/null +++ b/changelog.d/fixes/adapta-nonstream-sse-error.md @@ -0,0 +1 @@ +- **fix(sse):** Treat Adapta Web `type:error` SSE events as sanitized non-stream failures instead of empty HTTP 200 completions. diff --git a/changelog.d/fixes/dashboard-request-failed-redaction.md b/changelog.d/fixes/dashboard-request-failed-redaction.md new file mode 100644 index 0000000000..a9efc53dc7 --- /dev/null +++ b/changelog.d/fixes/dashboard-request-failed-redaction.md @@ -0,0 +1 @@ +- **fix(security):** sanitize `request.failed` diagnostics before publishing them to live dashboard listeners and replay history, while keeping status, model, provider, latency, and internal call-log diagnostics intact. diff --git a/changelog.d/fixes/migration-151-152-safety.md b/changelog.d/fixes/migration-151-152-safety.md new file mode 100644 index 0000000000..752290573e --- /dev/null +++ b/changelog.d/fixes/migration-151-152-safety.md @@ -0,0 +1 @@ +- Harden SQLite upgrades around the historical migration-074 version collision: missing discovery and inspector tables are replayed atomically, pre-existing databases (including setup-created skeletons) receive reusable content-addressed safety snapshots, and Node test/eval probes without `DATA_DIR` are isolated from the operator database. diff --git a/changelog.d/fixes/pending-grok-web-stream-error-boundary.md b/changelog.d/fixes/pending-grok-web-stream-error-boundary.md new file mode 100644 index 0000000000..28e4292504 --- /dev/null +++ b/changelog.d/fixes/pending-grok-web-stream-error-boundary.md @@ -0,0 +1,4 @@ +- **fix(grok-web):** treat upstream streaming failures as failures instead of successful + assistant text: error-only streams now fail readiness with HTTP 502, while failures after + legitimate content preserve that partial output and terminate through the sanitized stream + failure path without a normal `stop` completion. diff --git a/changelog.d/fixes/pending-huggingchat-stream-error-boundary.md b/changelog.d/fixes/pending-huggingchat-stream-error-boundary.md new file mode 100644 index 0000000000..7285276d10 --- /dev/null +++ b/changelog.d/fixes/pending-huggingchat-stream-error-boundary.md @@ -0,0 +1 @@ +- HuggingChat now turns HTTP 200 JSONL generation failures into a sanitized 502 before content, or a fixed public stream failure after partial output, so fallback and request persistence no longer record a false successful stop. diff --git a/changelog.d/fixes/pending-onemin-stream-error-boundary.md b/changelog.d/fixes/pending-onemin-stream-error-boundary.md new file mode 100644 index 0000000000..818b124a2f --- /dev/null +++ b/changelog.d/fixes/pending-onemin-stream-error-boundary.md @@ -0,0 +1 @@ +- **fix(providers):** keep 1min.ai HTTP 200 stream errors out of assistant content, preserve partial output, and expose sanitized terminal errors so pre-content failures can fall back. diff --git a/changelog.d/fixes/zed-hosted-stream-error-boundary.md b/changelog.d/fixes/zed-hosted-stream-error-boundary.md new file mode 100644 index 0000000000..e15742f092 --- /dev/null +++ b/changelog.d/fixes/zed-hosted-stream-error-boundary.md @@ -0,0 +1 @@ +- **fix(providers):** Zed Hosted streaming failures now trigger fallback before content and end partial streams with a sanitized structured error instead of fake assistant text and a normal-success stop. diff --git a/changelog.d/maintenance/error-boundary-campaign-filesize.md b/changelog.d/maintenance/error-boundary-campaign-filesize.md new file mode 100644 index 0000000000..f57eef1c49 --- /dev/null +++ b/changelog.d/maintenance/error-boundary-campaign-filesize.md @@ -0,0 +1 @@ +- **chore(quality):** rebaseline the file-size caps the error-boundary campaign grew past (`open-sse/executors/codex.ts`, `open-sse/vendor/codex-chatgpt-web/bridge.ts`, both via [#12444](https://github.com/diegosouzapw/OmniRoute/pull/12444)) diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index c39d3d5d41..97b881c510 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -412,7 +412,7 @@ "open-sse/executors/antigravity.ts": 1665, "open-sse/executors/base.ts": 1751, "open-sse/executors/chatgpt-web.ts": 5056, - "open-sse/executors/codex.ts": 1499, + "open-sse/executors/codex.ts": 1505, "open-sse/executors/cursor.ts": 1759, "open-sse/executors/muse-spark-web.ts": 1405, "open-sse/handlers/chatCore.ts": 5984, @@ -428,7 +428,7 @@ "open-sse/utils/proxyFetch.ts": 1271, "open-sse/utils/stream.ts": 3072, "open-sse/vendor/codex-chatgpt-web/adapters/chatgpt-web/browser-worker.ts": 4398, - "open-sse/vendor/codex-chatgpt-web/bridge.ts": 1322, + "open-sse/vendor/codex-chatgpt-web/bridge.ts": 1335, "src/app/(dashboard)/dashboard/HomePageClient.tsx": 1344, "src/app/(dashboard)/dashboard/api-manager/ApiManagerPageClient.tsx": 3186, "src/app/(dashboard)/dashboard/combos/page.tsx": 5018, @@ -636,5 +636,6 @@ "_rebaseline_2026_09_02_12325_merge_v3851": "Merge of release/v3.8.51 into #12325. Both sides grew chatCore.ts at the same chokepoint: #12239 took it 5946->5976 upstream, and this PR adds its +9 non-Codex 429 branch on top. check-file-size.mjs counts split(\"\\\\n\").length (trailing-newline empty element), so the merged file is 5981. The cap is the merged LOC, not either side alone; no other entry moves.", "_rebaseline_2026_09_03_houminxi_batch_stacked": "Crescimento medido DEPOIS que os 9 PRs da leva HouMinXi entraram, quando cada um empilhou sobre o rebaseline do anterior: providers/page.tsx 2007->2025 (+18 = feedback de erro por linha do import CSV do #12504 somado a busca por nome/baseUrl do #12495, ambos no mesmo painel de conexoes); chatCore.ts 5981->5984 (+3 = o #12325 invalida o cache generico de quota no 429 upstream, ao lado do ramo Codex ja existente); accountFallback.ts 2461->2467 (+6 = o #12566 empilha a carve-out de familia Antigravity sobre o rebaseline 2422->2461 que o #12590 registrou para o carve-out credits_exhausted da Moonshot; os dois tocam checkFallbackError). Cada PR mediu certo isoladamente, mas nenhum enxergava o empilhamento. Fiacao em chokepoints existentes. NAO cobre codex.ts nem stream.ts, que ja violavam no tip antes desta leva (drift da base).", "_rebaseline_2026_09_03_12604_claude_code_2_1_258": "PR #12604 (bump da wire identity do Claude Code 2.1.220->2.1.258, commits do @ggiak vindos do #12402) crescimento proprio: src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx 1606->1607 (+1, a linha do seletor que acompanha a nova versao de identidade). Uma linha num painel de settings ja existente; nao ha o que extrair. Coberto por client-identity-profiles e claude-codex-identity-version-sync (138/138 focados).", - "_rebaseline_2026_09_03_hartmark_batch": "Leva hartmark (#12293 #12355 #12447 #12445 #12446 #12460 #12461 #12338 #12448) crescimento proprio, medido no tip com os nove mergeados: src/app/(dashboard)/dashboard/combos/page.tsx 5012->5018 (+6, #12355 impede que a falha de bundling do tiktoken de um provider sem relacao derrube /api/providers, e o painel passa a lidar com o estado degradado); open-sse/services/combo.ts 4023->4036 (+13, #12338 nos fixes do universal-handoff: nota de bare-fallback, escopo por mesma requisicao e log da falha silenciosa). Fiacao em chokepoints existentes do roteamento de combo. NAO cobre codex.ts nem stream.ts, ja violando no tip antes desta leva (drift da base)." + "_rebaseline_2026_09_03_hartmark_batch": "Leva hartmark (#12293 #12355 #12447 #12445 #12446 #12460 #12461 #12338 #12448) crescimento proprio, medido no tip com os nove mergeados: src/app/(dashboard)/dashboard/combos/page.tsx 5012->5018 (+6, #12355 impede que a falha de bundling do tiktoken de um provider sem relacao derrube /api/providers, e o painel passa a lidar com o estado degradado); open-sse/services/combo.ts 4023->4036 (+13, #12338 nos fixes do universal-handoff: nota de bare-fallback, escopo por mesma requisicao e log da falha silenciosa). Fiacao em chokepoints existentes do roteamento de combo. NAO cobre codex.ts nem stream.ts, ja violando no tip antes desta leva (drift da base).", + "_rebaseline_2026_09_03_error_boundary_campaign": "Campanha de error-boundary (#12431 #12438 #12444 #12454 #12455 #12456 #12457 #12458 #12459 #12465 #12466 #12467 #12469 #12435), medido no tip com os 14 mergeados. open-sse/executors/codex.ts 1499->1505: os primeiros 4 (1499->1503) sao DRIFT ANTERIOR a esta campanha, ja presente no tip antes dela; os 2 ultimos (1503->1505) sao do #12444, que fecha o boundary de falha da resposta do Codex. Absorver o drift junto foi inevitavel porque o cap e um numero so, mas fica registrado aqui que 4 das 6 linhas nao sao desta leva. open-sse/vendor/codex-chatgpt-web/bridge.ts 1322->1335 (+13): tambem do #12444, no mesmo caminho de falha. NAO cobre open-sse/utils/stream.ts, que segue violando por drift anterior e independente." } diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md index 7f79c63785..7f475ea9f6 100644 --- a/docs/diagrams/README.md +++ b/docs/diagrams/README.md @@ -34,7 +34,7 @@ inside GitHub's `` sandbox: | [combo-always-on.svg](./combo-always-on.svg) | style reference | Animated priority-combo fallback (4 layers, 16s loop). Edit the SVG directly — there is no `.mmd` source. | | [cli-terminal.svg](./cli-terminal.svg) | README.md (root) | Compact half-height animated terminal (1200×350): 3 real CLI commands cycling with typewriter + scrolling subcommand ticker; first frame = completed providers screen. Edit the SVG directly — there is no `.mmd` source. | | [compression-pipeline.svg](./compression-pipeline.svg) | README.md (root) | Animated 12-engine compression funnel (8s loop). Edit the SVG directly — there is no `.mmd` source. | -| [free-tier-budget.svg](./free-tier-budget.svg) | README.md (root) | Animated free-tier budget card (~1.66B/mo quantified headline, 21-pool budget bar, per-pool grid, signup credits, 10s loop). Edit the SVG directly — there is no `.mmd` source. | +| [free-tier-budget.svg](./free-tier-budget.svg) | README.md (root) | Animated free-tier budget card (~1.63B/mo quantified headline, 21-pool budget bar, per-pool grid, signup credits, 10s loop). Edit the SVG directly — there is no `.mmd` source. | | [readme-hero.svg](./readme-hero.svg) | README.md (root) | Animated hero card (tagline, live provider/free-access headline, full-width compression bar demo, 6 stat chips). Edit the SVG directly — there is no `.mmd` source. | | [promise-pillars.svg](./promise-pillars.svg) | README.md (root) | Animated "The Promise" 6-pillar card (12s border-highlight sweep). Edit the SVG directly — there is no `.mmd` source. | | [why-pain-fix.svg](./why-pain-fix.svg) | README.md (root) | Animated "Why OmniRoute" 10-row pain-vs-fix ledger (15s green row sweep). Edit the SVG directly — there is no `.mmd` source. | diff --git a/docs/diagrams/free-tier-budget.svg b/docs/diagrams/free-tier-budget.svg index de68a27ef6..7452b4bc7b 100644 --- a/docs/diagrams/free-tier-budget.svg +++ b/docs/diagrams/free-tier-budget.svg @@ -1,5 +1,5 @@ - - Pool-deduplicated chart of the 21 recurring free-token pools with positive published budgets, plus signup credits and uncapped providers shown separately. + + Pool-deduplicated chart of the 20 recurring free-token pools with positive published budgets, plus signup credits and uncapped providers shown separately. @@ -28,9 +28,9 @@ - - - + + + FREE-TIER BUDGET · LIVE ON /dashboard/free-tiers @@ -61,10 +61,10 @@ - ~1.66B + ~1.63B FREE TOKENS / MONTH · STEADY - up to ~2.28B in your first month — signup credits - documented free tiers · 39 recurring pools · 476 catalog entries · one endpoint + up to ~2.25B in your first month — signup credits + documented free tiers · 38 recurring pools · 476 catalog entries · one endpoint @@ -75,37 +75,36 @@ every rate limit · 24/7 we don't publish that - ~1.66B + ~1.63B each shared free pool counted once ✓ 13 providers ToS-flagged — we flag it · you decide - - WHERE IT COMES FROM · 21 QUANTIFIED RECURRING POOLS + + WHERE IT COMES FROM · 20 QUANTIFIED RECURRING POOLS - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + @@ -114,90 +113,89 @@ each segment = one recurring pool · widths floored so every pool shows · audited pool budgets below - + Mistral 1.00B LLM7 150M Nara 150M xKiro 150M Gemini 60M - Cerebras 30M - Cloudflare AI 30M - API Airforce 24M - Ollama Cloud 20M - Groq 15M - Bluesminds 7.2M - SambaNova 6M - Arcee 4.8M - Navy 4.5M - BazaarLink 3.6M - OpenRouter 1.2M - Cohere 800K - HuggingChat 500K - Morph 400K - Hugging Face 200K - Kiro 25K + Cloudflare AI 30M + API Airforce 24M + Ollama Cloud 20M + Groq 15M + Bluesminds 7.2M + SambaNova 6M + Arcee 4.8M + Navy 4.5M + BazaarLink 3.6M + OpenRouter 1.2M + Cohere 800K + HuggingChat 500K + Morph 400K + Hugging Face 200K + Kiro 25K - + FIRST MONTH · ONE-TIME SIGNUP CREDITS ~626M + + FIRST MONTH · ONE-TIME SIGNUP CREDITS ~626M - - vertex 300M - - agentrouter 200M - - predibase 25M - - together 25M - - glm-cn 20M - - doubao 15M - - ai21 10M - - deepseek 5M - - hyperbolic 5M - - longcat 10M - - … + + vertex 300M + + agentrouter 200M + + predibase 25M + + together 25M + + glm-cn 20M + + doubao 15M + + ai21 10M + + deepseek 5M + + hyperbolic 5M + + longcat 10M + + … - ∞ - PLUS THE UN-COUNTABLE — PERMANENTLY FREE · NO TOKEN CAP + ∞ + PLUS THE UN-COUNTABLE — PERMANENTLY FREE · NO TOKEN CAP - - SiliconFlow - - Z.AI GLM-Flash - - Kilo - - OpenCode Zen - - baidu - - … + + SiliconFlow + + Z.AI GLM-Flash + + Kilo + + OpenCode Zen + + baidu + + … - $10 OpenRouter top-up → +24M/mo - surfaced separately — never inflates the headline + $10 OpenRouter top-up → +24M/mo + surfaced separately — never inflates the headline - - CURRENT MONTH - - + + CURRENT MONTH + + - + - + - LIVE - used / remaining · per-model breakdown · transparent terms flag per provider + LIVE + used / remaining · per-model breakdown · transparent terms flag per provider diff --git a/docs/diagrams/promise-pillars.svg b/docs/diagrams/promise-pillars.svg index 024ec2423b..d16b99e994 100644 --- a/docs/diagrams/promise-pillars.svg +++ b/docs/diagrams/promise-pillars.svg @@ -1,4 +1,4 @@ - + Animated promise card: six pillar tiles fade in in reading order, then a soft colored border highlight sweeps from tile to tile in a continuous cycle. @@ -73,7 +73,7 @@ $0 to start - 150+ providers with a free tier, 54 free + 150+ providers with a free tier, 53 free forever — Qoder, Pollinations, Cloudflare, SiliconFlow… No card needed. diff --git a/docs/diagrams/readme-hero.svg b/docs/diagrams/readme-hero.svg index 5fc2bcaff5..0f0ab67b5f 100644 --- a/docs/diagrams/readme-hero.svg +++ b/docs/diagrams/readme-hero.svg @@ -1,4 +1,4 @@ - + Animated hero card: a pulse travels the divider line and a compression bar demo repeatedly shrinks a prompt by up to 95 percent; all headline content is static and readable on the first frame. @@ -72,7 +72,7 @@ 150+ FREE TIERS - ~1.66B + ~1.63B FREE TOKENS / MO 15–95% diff --git a/docs/getting-started/FREE-TIERS-GUIDE.md b/docs/getting-started/FREE-TIERS-GUIDE.md index fe7ebbcc49..a70e882019 100644 --- a/docs/getting-started/FREE-TIERS-GUIDE.md +++ b/docs/getting-started/FREE-TIERS-GUIDE.md @@ -1,6 +1,6 @@ # Free Tiers Guide: Understand and Combine Free AI Access -> **TL;DR**: OmniRoute registers 351 provider IDs, with **152 provider-catalog entries marked `hasFree`**. The stricter audited free-model catalog covers **39 recurring pool keys / 445 entries** (438 active + 7 discontinued). Connect several suitable providers for broader fallback capacity; every quota, approval rule, privacy policy, and paid-overage condition still applies. +> **TL;DR**: OmniRoute registers 351 provider IDs, with **152 provider-catalog entries marked `hasFree`**. The stricter audited free-model catalog covers **38 recurring pool keys / 476 entries** (469 active + 7 discontinued). Connect several suitable providers for broader fallback capacity; every quota, approval rule, privacy policy, and paid-overage condition still applies. --- @@ -161,9 +161,9 @@ The live, pool-deduplicated catalog currently reports: | Metric | Current audited value | Interpretation | | ---------------------------------------------------- | -----------------------------------------------: | ----------------------------------------------------------------------------------------- | -| Recurring quantified grant | **~1.51B tokens/month** | Shared pools counted once; excludes uncapped providers from the sum | -| First month with signup grants | **~2.13B tokens** | Recurring total plus one-time and recurring credits | -| Audited free-model inventory | **39 recurring pool keys / 445 catalog entries** | 438 active + 7 discontinued; distinct from the 351-provider catalog | +| Recurring quantified grant | **~1.63B tokens/month** | Shared pools counted once; excludes uncapped providers from the sum | +| First month with signup grants | **~2.25B tokens** | Recurring total plus one-time and recurring credits | +| Audited free-model inventory | **38 recurring pool keys / 476 catalog entries** | 469 active + 7 discontinued; distinct from the 351-provider catalog | | Recurring/keyless free-forever providers represented | **55** | Unique providers across recurring daily/monthly/credit/uncapped and keyless catalog types | | Provider catalog entries marked `hasFree` | **152 / 351** | Broader provider metadata; not all have a quantifiable recurring quota | diff --git a/docs/getting-started/PROVIDERS-GUIDE.md b/docs/getting-started/PROVIDERS-GUIDE.md index 1ac7bbfecf..c65b6ad647 100644 --- a/docs/getting-started/PROVIDERS-GUIDE.md +++ b/docs/getting-started/PROVIDERS-GUIDE.md @@ -183,7 +183,7 @@ These providers offer **free access** with no credit card: | **LongCat** | 10M one-time | LongCat-2.0 | API key + KYC | | **Cloudflare AI** | 10K neurons/day | 50+ models | No auth needed | | **NVIDIA NIM** | ~40 RPM | 129 models | API key needed | -| **Cerebras** | 1M tokens/day | Qwen3 235B, GPT-OSS 120B | API key needed | +| **Cerebras** | $5 signup credit | GLM 4.7, GPT-OSS 120B | API key + card | | **Qoder** | Unlimited | Kimi-K2, DeepSeek-R1, Qwen3-coder | No auth needed | **Tip**: Connect multiple free providers for **unlimited free AI** with automatic fallback! diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index 9c2b536ab6..7d5137f6f9 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -613,7 +613,7 @@ In-process density (compression off the HTTP isolate) is [#11023](https://github ## Important Notes - **SQLite WAL Mode:** `docker stop` should be allowed to finish so OmniRoute can checkpoint the latest changes back into `storage.sqlite`. The bundled Compose files already set a 40s stop grace period. If you run the image directly, keep `--stop-timeout 40`. -- **`DISABLE_SQLITE_AUTO_BACKUP`:** Set to `true` if backups are managed externally. +- **`DISABLE_SQLITE_AUTO_BACKUP`:** Set to `true` if routine/pre-write backups are managed externally. Existing-database migrations still require their own durable safety snapshot and mass-migration guard. - **Data Persistence:** Always mount a volume to `/app/data` to persist your database, keys, and configurations across container restarts. - **Port Configuration:** Override `PORT` environment variable to change the default `20128` port. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 056a37beb9..27bf863d21 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -86,7 +86,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | Variable | Default | Source File | Description | | -------------------------------------- | -------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `DATA_DIR` | `~/.omniroute/` | `src/lib/db/core.ts` | Root directory for SQLite DB, backups, and data files. Override for Docker volumes or custom paths. | -| `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR` | _(unset)_ | `src/lib/dataPaths.ts` | Escape hatch for the test-context DATA_DIR guard (#10428). Test runs with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory. | +| `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR` | _(unset)_ | `src/lib/dataPaths.ts` | Escape hatch for the test/eval DATA_DIR guard (#10428). Tests and Node eval/print probes (`-e`/`--eval`/`-p`/`--print`, including `--eval=`/`--print=` forms) with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory. | | `OMNIROUTE_BUILD_SHA` | _(unset)_ | `src/lib/monitoring/buildSha.ts` | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`. | | `OMNIROUTE_RELEASE_REF` | `origin/main` | `scripts/build/buildProvenance.ts` | Ref the pack-artifact provenance gate checks the build SHA against (#10427). | | `OMNIROUTE_ALLOW_CANARY_BUILD` | _(unset)_ | `scripts/build/buildProvenance.ts` | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427). | @@ -97,7 +97,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_PLUGINS_DIR` | _(unset)_ | `src/lib/plugins/scanner.ts` | Directory the **runtime plugin scanner** reads — and the root the plugin manager installs into — overriding the home-derived default (#11827). Point it at the bind-mounted plugin tree in Docker/K8s instead of moving HOME just to relocate the scan path (HOME governs every other home-relative behaviour too). Unset = `~/.omniroute/plugins`, or `/tmp/.omniroute/plugins` when the process exports no home at all — the silent non-discovery this variable removes. The resolved directory is logged once at startup as `scanner.dir_resolved` with the input that won. Server-side only: CLI command plugins keep their own `OMNIROUTE_PLUGIN_PATH` (section 9). | | `STORAGE_ENCRYPTION_KEY` | _(empty = disabled)_ | `src/lib/db/encryption.ts` | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`. | | `STORAGE_ENCRYPTION_KEY_VERSION` | `v1` | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups. | -| `DISABLE_SQLITE_AUTO_BACKUP` | `false` | `src/lib/db/backup.ts` | When `true`, skips automatic + pre-write SQLite file backups (startup, models.dev pricing save/clear, settings writes). Manual and pre-restore backups still run. Non-manual backups are also **throttled to at most once per 60 minutes** so hourly models.dev sync does not copy the whole DB on every pricing write. Dashboard **Settings → Storage** can disable auto-backup independently. | +| `DISABLE_SQLITE_AUTO_BACKUP` | `false` | `src/lib/db/backup.ts` | When `true`, skips routine/pre-write SQLite file backups (models.dev pricing save/clear, settings writes). Manual and pre-restore backups still run. It does **not** disable the migration runner's mandatory durable safety snapshot or mass-migration guard for an existing persistent DB. Non-manual backups are throttled to at most once per 60 minutes. Dashboard **Settings → Storage** can disable routine auto-backup independently. | | `OMNIROUTE_CRYPT_KEY` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** for `STORAGE_ENCRYPTION_KEY`. Accepted as a fallback when the primary variable is absent. | | `OMNIROUTE_API_KEY_BASE64` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** (Base64-encoded form) accepted as a fallback. Decoded automatically before use. | | `OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS` | _(unset)_ | `src/lib/db/core.ts` | Override the periodic SQLite healthcheck interval (ms). When unset, defaults are derived from `NODE_ENV`. | @@ -121,6 +121,16 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `BATCH_BACKOFF_MAX_MS` | `3600000` (1h) | `open-sse/services/batchProcessor.ts` | Cap (ms) for exponential backoff between batch item retries. | | `BATCH_MAX_CONCURRENT` | `1` | `open-sse/services/batchProcessor.ts` | Maximum number of batches processed concurrently. Raise to increase throughput; keep low to avoid rate-limit storms. | +> [!IMPORTANT] +> Before changing an existing persistent database, the migration runner publishes a complete, +> content-addressed snapshot under `DATA_DIR/db_backups/`. Publication requires a filesystem +> that supports same-filesystem, no-overwrite hard links plus durable file sync. POSIX hosts also +> require directory sync; on Windows, Node may reject directory handles, so OmniRoute flushes the +> published file and treats directory-entry sync as best effort. +> If the mounted `DATA_DIR` cannot provide those guarantees, startup fails closed before applying +> a migration. Move `DATA_DIR` to a volume with those primitives; do not use +> `DISABLE_SQLITE_AUTO_BACKUP` to bypass migration safety. + ### Scenarios | Scenario | Configuration | @@ -1321,8 +1331,8 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `TAILSCALED_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscaled` daemon binary. | | `TAILSCALE_AUTHKEY` | _(unset)_ | `src/lib/tailscaleTunnel.ts` | Pre-shared Tailscale auth key for non-interactive / headless `tailscale up` (passed via `--auth-key=`). When unset, login falls back to the interactive browser auth URL. | | `NGROK_AUTHTOKEN` | _(unset)_ | `src/lib/ngrokTunnel.ts` | Authenticates outbound ngrok tunnels. | -| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum SQLite backup files retained on disk. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | -| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. | +| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts` | Maximum SQLite backup files retained by manual/scheduled backup cleanup. Migration snapshots are content-addressed and reused for an identical DB state; they are not pruned inside the concurrent migration window. Overrides the value saved from Settings → Database backup retention. | +| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts` | Maximum age (days) retained by manual/scheduled backup cleanup. `0` disables age-based pruning. Migration snapshots are not pruned inside the concurrent migration window. Overrides the value saved from Settings → Database backup retention. | | `OMNIROUTE_BACKUP_SCHEDULE_JOB_INTERVAL_MS` | `30000` | `src/lib/jobs/backupScheduleJob.ts` | Tick interval (ms) of the server-side job that executes `backup-schedule.json`. Must stay well under the 1-minute cron granularity; values below `5000` or unparseable fall back to `30000`. | | `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` for any Podman topology. Because the container cannot determine whether the engine is local or reached through Podman Machine, the warning stays topology-neutral and points to `contrib/podman/README.md`. | | `QUOTA_STORE_DRIVER` | `sqlite` | `src/lib/quota/storeFactory.ts` | Quota-share consumption store backend: `sqlite` (default) or `redis`. | diff --git a/docs/reference/FREE_TIERS.md b/docs/reference/FREE_TIERS.md index ea05ef99ea..aeabc4e4c8 100644 --- a/docs/reference/FREE_TIERS.md +++ b/docs/reference/FREE_TIERS.md @@ -15,21 +15,23 @@ lastUpdated: 2026-09-02 | Metric | Tokens / month | Meaning | | ------------------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Documented recurring grant (steady)** | **~1.66B** | Free-tier **pools** (per-model catalog), each shared pool counted **once**. The live source behind `/api/free-tier/summary` and the dashboard's Free-Tier Budget page. **Use this number.** | -| **+ first month with signup credits** | **~2.28B** | Steady + one-time signup credits (Together $25, Z.AI 20M, DeepSeek 5M, …), deduped per account. **First month only** — does not recur. | +| **Documented recurring grant (steady)** | **~1.63B** | Free-tier **pools** (per-model catalog), each shared pool counted **once**. The live source behind `/api/free-tier/summary` and the dashboard's Free-Tier Budget page. **Use this number.** | +| **+ first month with signup credits** | **~2.25B** | Steady + one-time signup credits (Together $25, Z.AI 20M, DeepSeek 5M, …), deduped per account. **First month only** — does not recur. | | **+ permanently free, no published cap** | _un-quantifiable_ | `siliconflow`, `glm-cn` (GLM-4-Flash), `tencent`, `baidu`, `kilo-gateway`, `opencode-zen` — real recurring access, rate/concurrency-limited, **no token cap to count**. Listed, never summed (counting them at `RPM×24/7` is the inflation we reject). | | **+ deposit-unlock boost** | **+~24M** | A one-time **$10** OpenRouter top-up raises its free pool from 50 → 1000 req/day. Reported separately so it never inflates the steady number. | | Theoretical ceiling (all rate limits, 24/7) | ~10B | Sum of every provider rate limit extrapolated to non-stop use. **Not a guarantee** — do not headline this. | -**Honest headline:** _OmniRoute aggregates **~1.66B documented free tokens per month** (up to ~2.28B in your first month with signup credits) across 39 free-tier pools — plus a long tail of permanently-free, no-cap providers — and RTK + Caveman compression (15–95% token savings) stretches that further._ +**Honest headline:** _OmniRoute aggregates **~1.63B documented free tokens per month** (up to ~2.25B in your first month with signup credits) across 38 free-tier pools — plus a long tail of permanently-free, no-cap providers — and RTK + Caveman compression (15–95% token savings) stretches that further._ > **Why this dropped from the previous ~1.94B.** The 2026-06-17 refresh is an honesty correction, not a loss: `gemini` is now pool-deduped (was inflated by counting each Flash variant separately, 462M → 60M), `cloudflare-ai` corrected to its real 10k-Neurons/day (122M → 30M), `doubao` reclassified as a one-time signup credit (not recurring), and shut-down tiers removed (`chutes`/`phind`/`kluster` discontinued). Partly offset by `llm7` (correct 5M/day → 150M) and new free providers (Kilo, OpenCode Zen, Z.AI GLM-Flash). > > **Further corrected to ~1.37B in v3.8.42:** `longcat` was reclassified from a 150M/mo recurring grant to a one-time 10M signup credit after its free preview ended. Same honesty rule — no provider was dropped by mistake. > -> **Updated on 2026-09-02 after adding xKiro:** the source now reports 39 recurring pool keys (38 after the 2026-08-26 Felo Web retirement — Felo Web stays excluded while its GPL-derived provenance/licensing remains on HOLD). This is the live, CI-gated number (`check:docs-counts` fails the build if this drifts from `computeFreeModelTotals()`). +> **Corrected to ~1.48B on 2026-09-03 (#11773):** `cerebras` was reclassified from a 30M/mo recurring grant (old no-card 1M tokens/day trial) to a one-time $5 signup credit that requires a payment method. Same honesty rule as LongCat. +> +> **Updated on 2026-09-02 after adding xKiro:** the source now reports 38 recurring pool keys — 37 after the 2026-08-26 Felo Web retirement and the #11773 Cerebras reclassification, plus the new `xkiro-free` pool (150M/mo). Felo Web stays excluded while its GPL-derived provenance/licensing remains on HOLD. This is the live, CI-gated number (`check:docs-counts` fails the build if this drifts from `computeFreeModelTotals()`). -Biggest **documented** contributors: `mistral` 1.00B, `llm7` 150M, `xkiro` 150M, `nara` 150M, `gemini` 60M, `cerebras` 30M, `cloudflare-ai` 30M, `api-airforce` 24M. (`longcat` is excluded — its 10M LongCat-2.0 grant is a one-time, KYC-gated signup credit, not a recurring monthly budget.) +Biggest **documented** contributors: `mistral` 1.00B, `llm7` 150M, `xkiro` 150M, `nara` 150M, `gemini` 60M, `cloudflare-ai` 30M, `api-airforce` 24M. (`longcat` is excluded — its 10M LongCat-2.0 grant is a one-time, KYC-gated signup credit, not a recurring monthly budget.) > ⚠️ The theoretical ceiling (~10B) is inflated by rate-limit-only providers with **no published token cap** (`tencent`, `siliconflow`, `nvidia`, `baidu`, `glm-cn`, `sparkdesk`) whose figures would be `RPM/TPM × 24/7 × 30d` — a theoretical maximum no single account will sustain. They are **excluded** from the defensible number (shown in the "permanently free, no cap" row instead). This is the same inflation that makes competitors' multi-billion claims unreliable. @@ -69,7 +71,7 @@ purpose. ## Methodology & caveats - Numbers are **upper-bound estimates** from each provider's documented free-tier limits as of **2026-06-17**, gathered by web research. Free tiers change constantly — re-verify before relying on a figure. -- **What an entry actually vouches for.** No entry carries a per-row confidence rating, and the API serves none — treat every figure above as an estimate of the same, unstated quality. Two facts are different, because they are curated by hand rather than inferred: 46 entries carry an independently documented hard stop (39 of them the xKiro rows, which all share one daily allowance), and 13 entries carry a prompt-training disclosure. `hardStopGuaranteed` is set only when the provider's own terms say that exceeding the free allowance refuses the request rather than silently starting to bill you, with the source in a comment next to the entry; it is never defaulted to `true`, and an entry nobody has verified stays unset. So a missing hard-stop flag means "not established", not "known to bill you". +- **What an entry actually vouches for.** No entry carries a per-row confidence rating, and the API serves none — treat every figure above as an estimate of the same, unstated quality. Two facts are different, because they are curated by hand rather than inferred: 44 entries carry an independently documented hard stop (39 of them the xKiro rows, which all share one daily allowance), and 13 entries carry a prompt-training disclosure. `hardStopGuaranteed` is set only when the provider's own terms say that exceeding the free allowance refuses the request rather than silently starting to bill you, with the source in a comment next to the entry; it is never defaulted to `true`, and an entry nobody has verified stays unset. So a missing hard-stop flag means "not established", not "known to bill you". - `estMonthlyFreeTokens` = recurring monthly tokens only. **One-time signup credits do not recur** and count as 0. Discontinued tiers are also 0. - Daily token cap → `monthly = daily × 30`. Only RPD documented → `RPD × ~800 output tokens × 30`. Only RPM/TPM (no daily cap) → **uncapped** (see below). - **Permanently free, but no published token cap** (`siliconflow`, `glm-cn`, `tencent`, `baidu`, `kilo-gateway`, `opencode-zen`): these are real recurring free access, rate/concurrency-limited. We classify them `recurring-uncapped` and **never sum them** — multiplying `RPM × 24/7 × 30d` would produce a fantasy ceiling (the inflation we reject). They are listed so you know they exist. @@ -195,7 +197,7 @@ purpose. | `xkiro` | recurring | ~150M | — | caution | 39 | | `longcat` | one-time | — | 10M | caution | 1 | | `gemini` | recurring | ~60M | — | caution | 4 | -| `cerebras` | recurring | ~30M | — | caution | 2 | +| `cerebras` | one-time | — | $5 credit | caution | 2 | | `cloudflare-ai` | recurring | ~30M | — | caution | 9 | | `api-airforce` | recurring | ~24M | — | caution | 7 | | `ollama-cloud` | recurring | ~20M | — | ambiguous | 8 | @@ -278,7 +280,7 @@ purpose. - **`bluesminds`** — Our shipped freeNote was "(none)" — but BluesMinds does have a documented free tier: 500 pi credits, 20 RPM, 300 RPD, permanent free plan. The catalog significantly understates the offering. - **`brave-search`** — The catalog notes "(none)" suggesting no free tier was tracked, but in reality there was a free 5,000 queries/month tier (no card) until February 12, 2026, which has since been replaced by a $5/month… - **`byteplus`** — Our catalog shipped "(none)" but BytePlus ModelArk does have a free tier: a one-time trial credit of 500k tokens per LLM model for new accounts. The catalog underreports this. -- **`cerebras`** — TPM appears tightened from 60K to 30K on current documented models (gpt-oss-120b, zai-glm-4.7). RPM of 5 is now explicitly documented (was not in our shipped note). Daily token cap of 1M/day is uncha… +- **`cerebras`** — The no-card 1M tokens/day trial is gone. Live cerebras.ai/pricing (2026-09-03) is a one-time $5 signup credit, payment method required, 30-day validity. Reclassified as `one-time-initial` (LongCat-shaped); dropped from `LEGACY_FREE_PROVIDERS` and the recurring budget. - **`chutes`** — The shipped freeNote says "Free tier available" but as of March 15, 2026, the free tier has been officially discontinued. The catalog note is stale and should be updated to reflect that there is no r… - **`coze`** — The shipped note "Free ByteDance agent platform" is directionally accurate but omits that the free tier is now tightly credit-capped (10 credits/day ≈ 5–100 messages depending on model), a constraint… - **`deepinfra`** — Our shipped freeNote says "Free signup credits for API testing" — this appears stale. The official pricing page now requires card/prepayment with no documented general free signup credit. The free ti… diff --git a/docs/reference/PROVIDER_REFERENCE.md b/docs/reference/PROVIDER_REFERENCE.md index ade1fc7100..4cb619a42e 100644 --- a/docs/reference/PROVIDER_REFERENCE.md +++ b/docs/reference/PROVIDER_REFERENCE.md @@ -1,14 +1,14 @@ --- title: "Provider Reference" version: 3.8.51 -lastUpdated: 2026-09-03 +lastUpdated: 2026-09-04 --- # Provider Reference > **Auto-generated** from `src/shared/constants/providers.ts` — do not edit by hand. > Regenerate with: `npm run gen:provider-reference` -> **Last generated:** 2026-09-03 +> **Last generated:** 2026-09-04 Total providers: **357**. See category breakdown below. @@ -151,7 +151,7 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each | `bluesminds` | `bm` | BluesMinds | API key | [link](https://www.bluesminds.com) | Free daily pi credits — supports 200+ models including GPT-4o, GPT-4.1, Claude Sonnet 4.5, Gemini 2.0 Flash, DeepSeek V4, Qwen, Kimi K2 | | `byteplus` | `bpm` | BytePlus ModelArk | API key | [link](https://console.byteplus.com/ark) | — | | `bytez` | `bytez` | Bytez | API key | [link](https://bytez.com) | $1 free credits, refreshes every 4 weeks | -| `cerebras` | `cerebras` | Cerebras | API key | [link](https://inference.cerebras.ai) | Free Trial: 1M tokens/day, 30K TPM, 5 RPM — no credit card. | +| `cerebras` | `cerebras` | Cerebras | API key | [link](https://inference.cerebras.ai) | One-time $5 signup credit (30-day validity); a payment method is required. Not a recurring free tier. | | `charm-hyper` | `charm-hyper` | Charm Hyper | API key | [link](https://hyper.charm.land) | 100 free monthly Hypercredits on signup | | `chat-oripe` | `chat-oripe` | Chat Oripe | API key, aggregator | [link](https://api.oriper.com) | Official metadata advertises 2M tokens/month, but the public site and documentation were blocked during audit; treat the quota and brand mapping as unconfirmed. | | `chatanywhere` | `chatanywhere` | ChatAnywhere | API key, aggregator | [link](https://chatanywhere.tech) | Personal, educational or research use only: public documentation cites 10,000 points/day and 200 requests/day per IP/key; do not use for commercial traffic. | diff --git a/docs/screenshots/free-tier-budget-card.svg b/docs/screenshots/free-tier-budget-card.svg index c56452439a..9843ddc881 100644 --- a/docs/screenshots/free-tier-budget-card.svg +++ b/docs/screenshots/free-tier-budget-card.svg @@ -5,9 +5,9 @@ Monthly free-token budget 20 free pools · 446 models · one endpoint Steady / month -~1.51B +~1.63B First month (+ signup credits) -~2.13B +~2.25B ToS-flagged (you decide) 13 providers diff --git a/docs/security/ERROR_SANITIZATION.md b/docs/security/ERROR_SANITIZATION.md index 898ca209d9..e3ffe04c77 100644 --- a/docs/security/ERROR_SANITIZATION.md +++ b/docs/security/ERROR_SANITIZATION.md @@ -1,14 +1,16 @@ --- title: "Error Message Sanitization" -version: 3.8.40 -lastUpdated: 2026-06-28 +version: 3.8.51 +lastUpdated: 2026-09-02 --- # Error Message Sanitization -> **Source of truth:** `open-sse/utils/error.ts` — `sanitizeErrorMessage`, `buildErrorBody`, `createErrorResult` -> **Tests:** `tests/unit/error-message-sanitization.test.ts` -> **Last updated:** 2026-06-28 — v3.8.40 +> **Source of truth:** `open-sse/utils/errorSanitization.ts`, +> `open-sse/utils/errorPathRedaction.ts`, and the public builders in `open-sse/utils/error.ts` +> **Tests:** `tests/unit/error-message-sanitization.test.ts`, +> `tests/unit/error-public-boundaries-hardening.test.ts` +> **Last updated:** 2026-09-02 — v3.8.51 > **Audience:** Any engineer touching error responses (HTTP routes, SSE streams, executors, MCP handlers). > **Status:** **MANDATORY** for every code path that returns an error message to a client. @@ -20,10 +22,18 @@ CodeQL rule `js/stack-trace-exposure` (CWE-209) flags any code path where an err - Library / framework versions inferred from stack frames → targeted exploit selection. - Sensitive runtime values that may be string-interpolated into errors (DB queries, config values). -The `sanitizeErrorMessage` helper in `open-sse/utils/error.ts` strips both classes of leakage: +The `sanitizeErrorMessage` helper exported by `open-sse/utils/error.ts` strips these classes of +leakage: -1. Multi-line stack traces — only the first line (the actual error message) is kept. -2. Absolute paths (`/...*.{ts,js,tsx,jsx,mjs,cjs}[:line[:col]]` and `C:\...`) — replaced with ``. +1. Physical, serialized, and unambiguously inline JavaScript stack-frame tails. +2. Absolute POSIX, Windows, UNC, and `file://` filesystem paths, while preserving safe HTTPS URLs + and explicitly marked API routes. +3. Credential assignments, common provider token formats, private-key PEM blocks, and base64 data + URLs. + +The sanitizer caps input length and fails closed when a thrown value rejects string coercion. +Recursive upstream JSON sanitization also drops unsafe credential/path keys, session aliases, and +prototype-control keys before a response is serialized. ## The mandatory pattern @@ -59,7 +69,10 @@ import { } from "@omniroute/open-sse/utils/error.ts"; ``` -All of these route through `buildErrorBody` and therefore through `sanitizeErrorMessage`. **You never need to call `sanitizeErrorMessage` manually** when using these helpers. +All of these apply the canonical public-error boundary. `errorResponse`, `writeStreamError`, and +`createErrorResult` route through `buildErrorBody`; the three specialized retry/circuit helpers +project and sanitize their public context directly. **You never need to call +`sanitizeErrorMessage` manually** when using these helpers. ### 2. Custom error envelopes (rare) @@ -81,17 +94,25 @@ This is the only sanctioned way to assemble a custom error body. See `open-sse/e ### 3. Logging vs. responding -`sanitizeErrorMessage` should **only** wrap the value that crosses the network boundary. Internal logs (`pino`, `console`) should keep the full message, including stack, so operators can debug. Pattern: +Trusted internal exceptions may keep their full message and stack so operators can debug. Values +originating at provider, validation, browser-session, or credential-adjacent boundaries must be +sanitized before they enter console output, audit metadata, or persistent call logs. Pattern: ```ts try { // ... } catch (err) { - log.error({ err }, "handler failed"); // full err with stack — internal log + log.error({ err }, "handler failed"); // trusted internal exception only return errorResponse(500, getErrorMessage(err)); // sanitized — sent to client } ``` +For provider-controlled failures, project the logged value too: + +```ts +log.error({ message: sanitizeErrorMessage(err) || "Provider request failed" }); +``` + ### 4. Forbidden patterns ❌ **Never** put raw exception output in a Response body: @@ -112,7 +133,9 @@ const safe = String(err).split("\n")[0]; ❌ **Never** sanitize in the route and forget the SSE path. Anything that writes to a stream goes through `writeStreamError` (or its underlying `buildErrorBody`). -❌ **Never** include `process.cwd()`, `__filename`, `__dirname`, env-derived paths in error messages — they bypass the path regex and reveal the deployment topology. +❌ **Never** intentionally include `process.cwd()`, `__filename`, `__dirname`, or env-derived paths +in error messages. The sanitizer covers absolute paths as defense in depth, but callers must not +construct topology-bearing messages in the first place. ## Coverage in CI @@ -129,7 +152,9 @@ When adding a new route or executor, copy the assertion pattern from this file. ## Related controls - `js/stack-trace-exposure` CodeQL alerts in `.github/security` should always be **either** fixed via these helpers **or** dismissed with a comment citing this doc. -- The `pino` redaction config (`src/shared/utils/logRedaction.ts`) handles structured log redaction separately. This doc covers only the response-message surface. +- The `pino` redaction config (`src/shared/utils/logRedaction.ts`) handles trusted structured logs + separately. This document covers public response messages and provider-controlled values that + cross persistent call/proxy-log boundaries. - Upstream-header denylist (`src/shared/constants/upstreamHeaders.ts`) covers header leakage — keep both files aligned when adding a new exfiltration concern. ## Upstream details passthrough @@ -138,27 +163,39 @@ When adding a new route or executor, copy the assertion pattern from this file. parsed body from the upstream provider). When provided, it is sanitized by `sanitizeUpstreamDetails` before inclusion in the response as `upstream_details`. -An optional fourth argument `classification` (`{ type?: string; code?: string }`) -preserves an explicit error type/code instead of re-deriving both from the -status-code table — used when the caller already classified the failure (e.g. -HTTP 499 → `client_disconnected`). +An optional fourth argument `classification` +(`{ type?: string; code?: string; reason?: string }`) accepts an explicit public classification. +Every field is projected onto the bounded public-identifier vocabulary. Unsafe, credential-shaped, +control-character, or overlong values fall back to the status-derived type/code; an unsafe optional +reason is omitted. Three-digit HTTP status identifiers (`100` through `599`) remain valid for +provider contracts that expose the numeric upstream status as a machine-readable code. The same +bounded range is accepted in the locally generated HTTP-status placeholder form; arbitrary provider +numbers and names remain outside the vocabulary. + +Pass every explicit classification in that fourth argument. Never overwrite +`body.error.code`, `body.error.type`, or `body.error.reason` after `buildErrorBody()` returns; +post-builder mutation bypasses the public projection. Sanitization rules applied to `upstreamDetails`: 1. String leaves: run through `sanitizeErrorMessage` (strips stacks + absolute paths). -2. Key blocklist: keys matching `/stack|trace|path|file|cwd|dir|password|secret|token|key/i` - are removed. +2. Unsafe path, credential, session-alias, and prototype-control keys are removed. 3. Depth cap: nesting beyond 4 levels is replaced with the string `"[truncated]"`. 4. Arrays are capped at 32 elements. -Only the seven upstream-error `createErrorResult` call sites in `chatCore.ts` pass -`upstreamErrorBody`. Internal OmniRoute errors (SSE parse failures, empty content, -guardrail blocks) do not include `upstream_details`. +Only call sites with a parsed provider error body should pass `upstreamDetails`. Internal OmniRoute +errors (SSE parse failures, empty content, guardrail blocks) must not include it. Do NOT pass raw `err.stack`, `err.message`, or any string from a runtime exception to `upstreamDetails`. Those must still go through `errorResponse` / `buildErrorBody(code, msg)` without an upstream body. +Selective upstream 4xx passthrough preserves the provider's safe JSON shape and wording required by +client auto-recovery, but it is not byte-for-byte passthrough: the recursive sanitizer always runs +before serialization. Cyclic, BigInt-bearing, or hostile `toJSON()` bodies fail closed and are not +eligible for passthrough. OCR and moderation apply the same rule; non-JSON, blank, or mislabeled +upstream bodies are converted to the canonical OmniRoute JSON error envelope. + ## Known CodeQL limitation: custom sanitizers not recognized The CodeQL query [`js/stack-trace-exposure`](https://codeql.github.com/codeql-query-help/javascript/js-stack-trace-exposure/) uses a fixed allowlist of sanitizer patterns (e.g. inline `.split("\n")[0]`, `String#replace` with specific regex shapes, access to `.message` on `Error`). It does **not** recognize indirection through a custom helper like our `sanitizeErrorMessage()`. diff --git a/open-sse/config/freeModelCatalog.data.ts b/open-sse/config/freeModelCatalog.data.ts index c63b25ef53..6814a4e6af 100644 --- a/open-sse/config/freeModelCatalog.data.ts +++ b/open-sse/config/freeModelCatalog.data.ts @@ -106,9 +106,12 @@ export const FREE_MODEL_BUDGETS: FreeModelBudget[] = [ { provider: "bytez", modelId: "meta-llama/Llama-3.3-70B-Instruct", displayName: "meta-llama/Llama-3.3-70B-Instruct", monthlyTokens: 0, creditTokens: 1000000, freeType: "recurring-credit", poolKey: "bytez", tos: "ambiguous" }, { provider: "bytez", modelId: "mistralai/Mistral-7B-Instruct-v0.3", displayName: "mistralai/Mistral-7B-Instruct-v0.3", monthlyTokens: 0, creditTokens: 1000000, freeType: "recurring-credit", poolKey: "bytez", tos: "ambiguous" }, { provider: "bytez", modelId: "Qwen/Qwen2.5-72B-Instruct", displayName: "Qwen/Qwen2.5-72B-Instruct", monthlyTokens: 0, creditTokens: 1000000, freeType: "recurring-credit", poolKey: "bytez", tos: "ambiguous" }, - // hardStopGuaranteed: Cerebras pricing page states "Free Trial: 1M tokens/day... no credit card" (open-sse/services/../providers/apikey/inference-hosts.ts:74-84). - { provider: "cerebras", modelId: "zai-glm-4.7", displayName: "GLM 4.7", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cerebras", tos: "caution", hardStopGuaranteed: true }, - { provider: "cerebras", modelId: "gpt-oss-120b", displayName: "GPT OSS 120B", monthlyTokens: 30000000, creditTokens: 0, freeType: "recurring-daily", poolKey: "cerebras", tos: "caution", hardStopGuaranteed: true }, + // #11773: cerebras.ai/pricing (2026-09-03) is a one-time $5 signup credit + // gated on a payment method, 30-day expiry — not the old no-card 1M/day + // trial. creditTokens stays 0 because Cerebras publishes dollars, not a + // token grant. hardStopGuaranteed must stay unset: a stored card can bill. + { provider: "cerebras", modelId: "zai-glm-4.7", displayName: "GLM 4.7", monthlyTokens: 0, creditTokens: 0, freeType: "one-time-initial", poolKey: "cerebras", tos: "caution" }, + { provider: "cerebras", modelId: "gpt-oss-120b", displayName: "GPT OSS 120B", monthlyTokens: 0, creditTokens: 0, freeType: "one-time-initial", poolKey: "cerebras", tos: "caution" }, // #8717: drop dead Workers AI ids (400/403/410). Keep Neurons/day budget on fp8-fast. { provider: "cloudflare-ai", modelId: "@cf/mistral/mistral-7b-instruct-v0.2-lora", displayName: "Mistral 7B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" }, { provider: "cloudflare-ai", modelId: "@cf/qwen/qwen2.5-coder-32b-instruct", displayName: "Qwen 2.5 Coder 32B (🆓)", monthlyTokens: 0, creditTokens: 0, freeType: "recurring-daily", poolKey: "cloudflare-ai", tos: "caution" }, diff --git a/open-sse/config/freeTierCatalog.ts b/open-sse/config/freeTierCatalog.ts index 339f01a100..68cc240caa 100644 --- a/open-sse/config/freeTierCatalog.ts +++ b/open-sse/config/freeTierCatalog.ts @@ -16,7 +16,6 @@ export const FREE_TIER_BUDGETS: Record = { "cloudflare-ai": 122_000_000, gemini: 60_000_000, doubao: 60_000_000, - cerebras: 30_000_000, "api-airforce": 24_000_000, "ollama-cloud": 20_000_000, groq: 15_000_000, diff --git a/open-sse/config/searchRegistry.ts b/open-sse/config/searchRegistry.ts index e4d4546212..3e9105ad35 100644 --- a/open-sse/config/searchRegistry.ts +++ b/open-sse/config/searchRegistry.ts @@ -33,6 +33,17 @@ export interface SearchProviderConfig { */ fallbackOnly?: boolean; disabled?: boolean; + /** + * May a CALLER-supplied `provider_options.baseUrl` redirect this provider? + * + * Only ever true for a keyless, self-hosted provider. For anything with + * `authType: "apikey"` the builder attaches the OPERATOR's key to whatever + * host the base URL resolves to, so honoring a caller-chosen value hands that + * key to the caller's server (GHSA-3f8g-pfh9-j687). The invariant + * "never set alongside authType: apikey" is enforced by + * tests/unit/search-baseurl-client-override-3f8g.test.ts. + */ + allowClientBaseUrlOverride?: boolean; } export const SEARCH_PROVIDERS: Record = { @@ -232,6 +243,11 @@ export const SEARCH_PROVIDERS: Record = { timeoutMs: 10_000, cacheTTLMs: 3 * 60 * 1000, fallbackOnly: true, + // Keyless and self-hosted by definition: the caller names their own SearXNG + // instance and no operator credential travels with the request. This is a + // documented flow (tests/unit/search-route.test.ts). Still block-metadata + // guarded, so IMDS stays unreachable. + allowClientBaseUrlOverride: true, }, "ollama-search": { diff --git a/open-sse/executors/adapta-web.ts b/open-sse/executors/adapta-web.ts index da1b836002..619b952c3c 100644 --- a/open-sse/executors/adapta-web.ts +++ b/open-sse/executors/adapta-web.ts @@ -1,10 +1,11 @@ import { BaseExecutor, type ExecuteInput } from "./base.ts"; import { prepareToolMessages, buildToolAwareResult } from "../translator/webTools.ts"; -import { sanitizeErrorMessage } from "../utils/error.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; const ADAPTA_APP_URL = "https://agent.adapta.one"; const ADAPTA_CLERK_URL = "https://clerk.agent.adapta.one"; const ADAPTA_STREAM_URL = `${ADAPTA_APP_URL}/api/chat/stream/v1`; +const ADAPTA_PUBLIC_STREAM_ERROR = `\n\n[Erro: ${sanitizeErrorMessage("Adapta upstream error")}]`; // Default model ID in Adapta's internal system (corresponds to "ONE" / auto-select) const DEFAULT_AI_MODEL_ID = 14; @@ -321,10 +322,9 @@ function transformStream(adaptaStream: ReadableStream, model: string): ReadableS if (event.id === "quick-response") continue; // Real text ended — stream will send more events or close } else if (type === "error") { - const errText = String(event.errorText ?? "Adapta upstream error"); ensureRole(); - // Emit the error as content so the user sees it - chunk({ content: `\n\n[Erro: ${errText}]` }); + // Keep upstream diagnostics private: the transformed SSE is a public HTTP 200 body. + chunk({ content: ADAPTA_PUBLIC_STREAM_ERROR }); finalize(); return; } else if (type === "done" || type === "end") { @@ -480,9 +480,10 @@ export class AdaptaWebExecutor extends BaseExecutor { const reader = resp.body!.getReader(); let buf = ""; let fullText = ""; + let upstreamErrorMessage: string | null = null; try { - while (true) { + readLoop: while (true) { const { done, value } = await reader.read(); if (done) break; buf += decoder.decode(value, { stream: true }); @@ -494,6 +495,9 @@ export class AdaptaWebExecutor extends BaseExecutor { const ev = JSON.parse(line.slice(6)); if (ev.type === "text-delta" && ev.id !== "quick-response") { fullText += String(ev.delta ?? ""); + } else if (ev.type === "error") { + upstreamErrorMessage = "Adapta upstream error"; + break readLoop; } } catch { // skip @@ -501,9 +505,24 @@ export class AdaptaWebExecutor extends BaseExecutor { } } } finally { + if (upstreamErrorMessage) { + void reader.cancel("Adapta upstream SSE error").catch(() => undefined); + } reader.releaseLock(); } + if (upstreamErrorMessage) { + return { + response: new Response(JSON.stringify(buildErrorBody(502, upstreamErrorMessage)), { + status: 502, + headers: { "Content-Type": "application/json" }, + }), + url: ADAPTA_STREAM_URL, + headers, + transformedBody: requestPayload, + }; + } + if (hasTools) { const { content, toolCalls, finishReason } = buildToolAwareResult( fullText, diff --git a/open-sse/executors/claude-web.ts b/open-sse/executors/claude-web.ts index 5034b0da6c..77c06b5b14 100644 --- a/open-sse/executors/claude-web.ts +++ b/open-sse/executors/claude-web.ts @@ -216,9 +216,10 @@ function makeErrorResponse( extraHeaders?: Record; } ): Response { - const body = buildErrorBody(status, message, options?.details); - if (options?.type) body.error.type = options.type; - if (options?.code) body.error.code = options.code; + const body = buildErrorBody(status, message, options?.details, { + type: options?.type, + code: options?.code, + }); const headers: Record = { "Content-Type": "application/json" }; if (options?.extraHeaders) { for (const [key, value] of Object.entries(options.extraHeaders)) { diff --git a/open-sse/executors/claude-web/stream.ts b/open-sse/executors/claude-web/stream.ts index 264f8218e8..e3b6c724e9 100644 --- a/open-sse/executors/claude-web/stream.ts +++ b/open-sse/executors/claude-web/stream.ts @@ -453,9 +453,10 @@ function makeChunk( } function protocolErrorBody(): Record { - const body = buildErrorBody(502, "Claude Web stream protocol error"); - body.error.type = "upstream_protocol_error"; - body.error.code = "claude_web_protocol_error"; + const body = buildErrorBody(502, "Claude Web stream protocol error", undefined, { + type: "upstream_protocol_error", + code: "claude_web_protocol_error", + }); return body as unknown as Record; } diff --git a/open-sse/executors/codex.ts b/open-sse/executors/codex.ts index 4701f175c2..1194fe4dee 100644 --- a/open-sse/executors/codex.ts +++ b/open-sse/executors/codex.ts @@ -38,6 +38,7 @@ import { applyReasoningInputPolicy } from "../services/reasoningInputPolicy.ts"; import { normalizeCodexVerbosity } from "../services/codexVerbosity.ts"; import { getThinkingBudgetConfig, ThinkingMode } from "../services/thinkingBudget.ts"; import { CORS_HEADERS } from "../utils/cors.ts"; +import { projectCodexPublicError } from "../utils/codexPublicError.ts"; import { errorResponse } from "../utils/error.ts"; import { normalizeCodexResponsesInput } from "../utils/responsesInputNormalization.ts"; import * as prl from "../utils/providerRequestLogging.ts"; @@ -493,7 +494,6 @@ function toCodexResponseFailedEvent(parsed: Record): Record = { code, message }; const explicitStatus = toStatusCode(parsed.status_code) ?? toStatusCode(parsed.status) ?? @@ -503,8 +503,10 @@ function toCodexResponseFailedEvent(parsed: Record): Record = { + ...projectCodexPublicError({ status: statusCode, code, type }), + }; - if (type) error.type = type; if (statusCode !== null) error.status_code = statusCode; return { @@ -955,7 +957,7 @@ export class CodexExecutor extends BaseExecutor { } }; - const failController = (code: string, message: string) => { + const failController = (code: string, _message: string) => { if (closed) return; const controller = streamController; const payload = JSON.stringify({ @@ -963,7 +965,7 @@ export class CodexExecutor extends BaseExecutor { response: { id: null, status: "failed", - error: { code, message }, + error: projectCodexPublicError({ status: 502, code, type: "provider_error" }), }, }); try { diff --git a/open-sse/executors/grok-web.ts b/open-sse/executors/grok-web.ts index 939a86903c..011e81c32b 100644 --- a/open-sse/executors/grok-web.ts +++ b/open-sse/executors/grok-web.ts @@ -19,7 +19,7 @@ import { type ExecuteInput, type ExecutorLog, } from "./base.ts"; -import { FETCH_TIMEOUT_MS } from "../config/constants.ts"; +import { FETCH_TIMEOUT_MS, STREAM_READINESS_TIMEOUT_MS } from "../config/constants.ts"; import { buildGrokCookieHeader } from "@/lib/providers/webCookieAuth"; import { tlsFetchGrok, @@ -27,7 +27,8 @@ import { isCloudflareChallenge, type TlsFetchResult, } from "../services/grokTlsClient.ts"; -import { sanitizeErrorMessage } from "../utils/error.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; +import { ensureStreamReadiness } from "../utils/streamReadiness.ts"; import { shouldUseGrokBrowserBacked, acquireFreshGrokClearance, @@ -119,12 +120,29 @@ async function* readGrokNdjsonEvents( const reader = body.getReader(); const decoder = new TextDecoder(); let buffer = ""; + let reachedEnd = false; + let cancelRequested = false; + + const requestReaderCancel = (reason?: unknown) => { + if (cancelRequested || reachedEnd) return; + cancelRequested = true; + // Cancellation must release the upstream promptly even when a provider's + // underlying cancel promise never settles. + void reader.cancel(reason).catch(() => {}); + }; + const handleAbort = () => requestReaderCancel(signal?.reason); + + if (signal?.aborted) requestReaderCancel(signal.reason); + else signal?.addEventListener("abort", handleAbort, { once: true }); try { while (true) { if (signal?.aborted) return; const { value, done } = await reader.read(); - if (done) break; + if (done) { + reachedEnd = true; + break; + } buffer += decoder.decode(value, { stream: true }); while (true) { @@ -142,6 +160,8 @@ async function* readGrokNdjsonEvents( } } + if (signal?.aborted) return; + // Flush remaining buffer buffer += decoder.decode(); const remaining = buffer.trim(); @@ -153,7 +173,11 @@ async function* readGrokNdjsonEvents( } } } finally { - reader.releaseLock(); + signal?.removeEventListener("abort", handleAbort); + if (!reachedEnd) requestReaderCancel(signal?.reason ?? "Grok stream reader closed early"); + try { + reader.releaseLock(); + } catch {} } } @@ -271,6 +295,8 @@ async function* extractContent( } } + if (signal?.aborted) return; + const trailingThinking = suppressThinkingAfterVisibleContent && emittedVisibleContent ? "" : thinkingFilter.flush(); if (trailingThinking) { @@ -292,6 +318,25 @@ function sseChunk(data: unknown): string { return `data: ${JSON.stringify(data)}\n\n`; } +const GROK_STREAM_FAILURE_MESSAGE = "Grok upstream stream failed"; +const GROK_STREAM_FAILURE_CODE = "GROK_STREAM_ERROR"; + +function grokStreamErrorChunk(): string { + return sseChunk( + buildErrorBody(502, GROK_STREAM_FAILURE_MESSAGE, undefined, { + type: "upstream_error", + code: GROK_STREAM_FAILURE_CODE, + }) + ); +} + +function grokStreamFailure(): Error & { statusCode: number; code: string } { + return Object.assign(new Error(GROK_STREAM_FAILURE_MESSAGE), { + statusCode: 502, + code: GROK_STREAM_FAILURE_CODE, + }); +} + function enqueueStreamingToolCalls( controller: ReadableStreamDefaultController, encoder: TextEncoder, @@ -349,63 +394,77 @@ function buildStreamingResponse( signal?: AbortSignal | null ): ReadableStream { const encoder = new TextEncoder(); + const streamAbortController = new AbortController(); + const requestStreamCancel = (reason?: unknown) => { + if (!streamAbortController.signal.aborted) streamAbortController.abort(reason); + }; + const handleParentAbort = () => requestStreamCancel(signal?.reason); + + if (signal?.aborted) requestStreamCancel(signal.reason); + else signal?.addEventListener("abort", handleParentAbort, { once: true }); return new ReadableStream( { async start(controller) { + let roleSent = false; + let firstOutputHandedOff = false; try { - // Initial role chunk - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { index: 0, delta: { role: "assistant" }, finish_reason: null, logprobs: null }, - ], - }) - ) - ); - let fp = ""; let buffered = ""; + const enqueueRole = () => { + if (roleSent) return; + controller.enqueue( + encoder.encode( + sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: fp || null, + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }) + ) + ); + roleSent = true; + }; + + const handOffFirstOutput = async () => { + if (firstOutputHandedOff) return; + firstOutputHandedOff = true; + // Give readiness/finalization wrappers one turn to attach before a later + // upstream failure errors the stream and invalidates queued chunks. + await new Promise((resolve) => setImmediate(resolve)); + }; + for await (const chunk of extractContent( eventStream, isThinkingModel, toolRegistry, - signal, + streamAbortController.signal, true )) { if (chunk.fingerprint) fp = chunk.fingerprint; if (chunk.error) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: fp || null, - choices: [ - { - index: 0, - delta: { content: `[Error: ${chunk.error}]` }, - finish_reason: null, - logprobs: null, - }, - ], - }) - ) - ); - break; + if (roleSent) { + controller.error(grokStreamFailure()); + return; + } + controller.enqueue(encoder.encode(grokStreamErrorChunk())); + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + return; } if (chunk.thinking) { + enqueueRole(); controller.enqueue( encoder.encode( sseChunk({ @@ -425,10 +484,12 @@ function buildStreamingResponse( }) ) ); + await handOffFirstOutput(); continue; } if (chunk.toolCalls) { + enqueueRole(); enqueueStreamingToolCalls(controller, encoder, { id: cid, created, @@ -444,6 +505,7 @@ function buildStreamingResponse( if (chunk.fullMessage) { const toolCalls = parseClientToolCallMarkup(chunk.fullMessage, toolRegistry); if (toolCalls) { + enqueueRole(); enqueueStreamingToolCalls(controller, encoder, { id: cid, created, @@ -453,6 +515,30 @@ function buildStreamingResponse( }); return; } + if (!buffered) { + enqueueRole(); + buffered = chunk.fullMessage; + controller.enqueue( + encoder.encode( + sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: fp || null, + choices: [ + { + index: 0, + delta: { content: chunk.fullMessage }, + finish_reason: null, + logprobs: null, + }, + ], + }) + ) + ); + await handOffFirstOutput(); + } } if (chunk.delta) { @@ -469,6 +555,7 @@ function buildStreamingResponse( return; } if (hasOpenToolCallMarkup(buffered)) continue; + enqueueRole(); controller.enqueue( encoder.encode( sseChunk({ @@ -488,10 +575,13 @@ function buildStreamingResponse( }) ) ); + await handOffFirstOutput(); } } - // Stop chunk + if (streamAbortController.signal.aborted || !roleSent) return; + + // Stop chunk — only after legitimate content/reasoning/tool output. controller.enqueue( encoder.encode( sseChunk({ @@ -505,37 +595,24 @@ function buildStreamingResponse( ) ); controller.enqueue(encoder.encode("data: [DONE]\n\n")); - } catch (err) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { - index: 0, - delta: { - content: sanitizeErrorMessage( - `[Stream error: ${err instanceof Error ? err.message : String(err)}]` - ), - }, - finish_reason: "stop", - logprobs: null, - }, - ], - }) - ) - ); + } catch { + if (streamAbortController.signal.aborted) return; + if (roleSent) { + controller.error(grokStreamFailure()); + return; + } + controller.enqueue(encoder.encode(grokStreamErrorChunk())); controller.enqueue(encoder.encode("data: [DONE]\n\n")); } finally { + signal?.removeEventListener("abort", handleParentAbort); try { controller.close(); } catch {} } }, + cancel(reason) { + requestStreamCancel(reason); + }, }, { highWaterMark: 16384 } ); @@ -1026,6 +1103,13 @@ export class GrokWebExecutor extends BaseExecutor { "X-Accel-Buffering": "no", }, }); + const readiness = await ensureStreamReadiness(finalResponse, { + timeoutMs: STREAM_READINESS_TIMEOUT_MS, + provider: this.provider, + model, + log, + }); + finalResponse = readiness.response; } else { finalResponse = await buildNonStreamingResponse( tlsResult.body, diff --git a/open-sse/executors/huggingchat.ts b/open-sse/executors/huggingchat.ts index 7b7557ce02..30ec7da0ea 100644 --- a/open-sse/executors/huggingchat.ts +++ b/open-sse/executors/huggingchat.ts @@ -27,7 +27,11 @@ import { import { FETCH_TIMEOUT_MS } from "../config/constants.ts"; import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; import { normalizeSessionCookieHeader } from "@/lib/providers/webCookieAuth"; -import { streamJsonlToOpenAi, readJsonlResponse } from "./huggingchat/jsonlStream.ts"; +import { + HuggingChatStreamError, + readJsonlResponse, + streamJsonlToOpenAi, +} from "./huggingchat/jsonlStream.ts"; const HUGGINGFACE_BASE = "https://huggingface.co"; const CONVERSATION_URL = `${HUGGINGFACE_BASE}/chat/conversation`; @@ -38,6 +42,7 @@ const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"; const DEFAULT_MODEL = "baidu/ERNIE-4.5-VL-424B-A47B-Base-PT"; +const HUGGINGCHAT_PUBLIC_STREAM_ERROR = "HuggingChat generation failed"; // -- Helpers ----------------------------------------------------------------- @@ -400,13 +405,15 @@ export class HuggingChatExecutor extends BaseExecutor { }; } } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); log?.error?.("HUGGINGCHAT", `Conversation creation failed: ${message}`); return { response: new Response( - JSON.stringify({ - error: { message: `HuggingChat connection failed: ${message}`, type: "upstream_error" }, - }), + JSON.stringify( + buildErrorBody(502, `HuggingChat connection failed: ${message}`, undefined, { + type: "upstream_error", + }) + ), { status: 502, headers: { "Content-Type": "application/json" } } ), url: CONVERSATION_URL, @@ -463,13 +470,15 @@ export class HuggingChatExecutor extends BaseExecutor { signal: combinedSignal, }); } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); log?.error?.("HUGGINGCHAT", `Message send failed: ${message}`); return { response: new Response( - JSON.stringify({ - error: { message: `HuggingChat connection failed: ${message}`, type: "upstream_error" }, - }), + JSON.stringify( + buildErrorBody(502, `HuggingChat connection failed: ${message}`, undefined, { + type: "upstream_error", + }) + ), { status: 502, headers: { "Content-Type": "application/json" } } ), url: messageUrl, @@ -523,25 +532,80 @@ export class HuggingChatExecutor extends BaseExecutor { if (stream) { const encoder = new TextEncoder(); + const streamCancellationController = new AbortController(); const jsonlStream = streamJsonlToOpenAi( upstreamResponse.body, resolvedModel, id, created, - signal + signal, + streamCancellationController.signal ); - const sseStream = new ReadableStream({ - async start(controller) { - try { - for await (const chunk of jsonlStream) { - controller.enqueue(encoder.encode(chunk)); - } - } catch (err) { - log?.error?.("HUGGINGCHAT", `Stream error: ${err}`); - } finally { - controller.close(); + const primedChunks: string[] = []; + try { + const first = await jsonlStream.next(); + if (!first.done) { + primedChunks.push(first.value); + if (first.value.includes('"role":"assistant"')) { + const content = await jsonlStream.next(); + if (!content.done) primedChunks.push(content.value); } + } + } catch (err) { + if (!(err instanceof HuggingChatStreamError)) throw err; + const message = err instanceof Error ? err.message : String(err); + const safeMessage = sanitizeErrorMessage(message); + log?.error?.("HUGGINGCHAT", `Stream failed before content: ${safeMessage}`); + return { + response: new Response( + JSON.stringify( + buildErrorBody(502, message, undefined, { + type: "upstream_error", + code: "huggingchat_generation_error", + }) + ), + { status: 502, headers: { "Content-Type": "application/json" } } + ), + url: messageUrl, + headers: baseHeaders, + transformedBody: sendDataPayload, + }; + } + + let primedChunkIndex = 0; + let streamCancelled = false; + const sseStream = new ReadableStream({ + async pull(controller) { + if (streamCancelled) return; + if (primedChunkIndex < primedChunks.length) { + controller.enqueue(encoder.encode(primedChunks[primedChunkIndex])); + primedChunkIndex += 1; + return; + } + + try { + const chunk = await jsonlStream.next(); + if (streamCancelled) return; + if (chunk.done) { + controller.close(); + return; + } + controller.enqueue(encoder.encode(chunk.value)); + } catch (err) { + if (streamCancelled) return; + const message = err instanceof Error ? err.message : String(err); + const safeMessage = sanitizeErrorMessage(message); + log?.error?.("HUGGINGCHAT", `Stream error: ${safeMessage}`); + controller.error( + Object.assign(new Error(HUGGINGCHAT_PUBLIC_STREAM_ERROR), { statusCode: 502 }) + ); + } + }, + cancel() { + streamCancelled = true; + streamCancellationController.abort(); + void jsonlStream.return(undefined).catch(() => undefined); }, }); @@ -560,7 +624,29 @@ export class HuggingChatExecutor extends BaseExecutor { }; } - const fullText = await readJsonlResponse(upstreamResponse.body, signal); + let fullText: string; + try { + fullText = await readJsonlResponse(upstreamResponse.body, signal); + } catch (err) { + if (!(err instanceof HuggingChatStreamError)) throw err; + const message = err instanceof Error ? err.message : String(err); + const safeMessage = sanitizeErrorMessage(message); + log?.error?.("HUGGINGCHAT", `Generation error: ${safeMessage}`); + return { + response: new Response( + JSON.stringify( + buildErrorBody(502, message, undefined, { + type: "upstream_error", + code: "huggingchat_generation_error", + }) + ), + { status: 502, headers: { "Content-Type": "application/json" } } + ), + url: messageUrl, + headers: baseHeaders, + transformedBody: sendDataPayload, + }; + } const completionTokens = estimateTokens(fullText); return { diff --git a/open-sse/executors/huggingchat/jsonlStream.ts b/open-sse/executors/huggingchat/jsonlStream.ts index b09bcb2c30..3d4980aebb 100644 --- a/open-sse/executors/huggingchat/jsonlStream.ts +++ b/open-sse/executors/huggingchat/jsonlStream.ts @@ -1,5 +1,36 @@ // Pure JSONL stream translation (HuggingChat NDJSON -> OpenAI SSE). Verbatim from huggingchat.ts. +export class HuggingChatStreamError extends Error { + constructor(message: string) { + super(message); + this.name = "HuggingChatStreamError"; + } +} + +function cancelReader(reader: ReadableStreamDefaultReader): void { + try { + void reader.cancel().catch(() => undefined); + } catch { + // The error event is authoritative; transport cleanup is best effort. + } +} + +function bindReaderCancellation( + reader: ReadableStreamDefaultReader, + signal?: AbortSignal | null +): () => void { + if (!signal) return () => undefined; + + const cancel = () => cancelReader(reader); + if (signal.aborted) { + cancel(); + return () => undefined; + } + + signal.addEventListener("abort", cancel, { once: true }); + return () => signal.removeEventListener("abort", cancel); +} + export function sseChunk(data: unknown): string { return `data: ${JSON.stringify(data)}\n\n`; } @@ -42,9 +73,11 @@ export async function* streamJsonlToOpenAi( model: string, id: string, created: number, - signal?: AbortSignal | null + signal?: AbortSignal | null, + cancellationSignal?: AbortSignal | null ): AsyncGenerator { const reader = body.getReader(); + const unbindReaderCancellation = bindReaderCancellation(reader, cancellationSignal); const decoder = new TextDecoder(); let buffer = ""; let emittedRole = false; @@ -70,16 +103,8 @@ export async function* streamJsonlToOpenAi( const parsed = parseJsonlLine(trimmed); if (parsed.error) { - yield sseChunk({ - id, - object: "chat.completion.chunk", - created, - model, - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - }); - yield "data: [DONE]\n\n"; - finished = true; - return; + cancelReader(reader); + throw new HuggingChatStreamError(parsed.error); } if (parsed.token) { @@ -140,6 +165,9 @@ export async function* streamJsonlToOpenAi( if (!finished && buffer.trim()) { const parsed = parseJsonlLine(buffer.trim()); + if (parsed.error) { + throw new HuggingChatStreamError(parsed.error); + } if (parsed.token && !signal?.aborted) { if (!emittedRole) { emittedRole = true; @@ -161,10 +189,11 @@ export async function* streamJsonlToOpenAi( } } } finally { + unbindReaderCancellation(); reader.releaseLock(); } - if (!signal?.aborted) { + if (!signal?.aborted && !cancellationSignal?.aborted) { yield sseChunk({ id, object: "chat.completion.chunk", @@ -172,7 +201,9 @@ export async function* streamJsonlToOpenAi( model, choices: [{ index: 0, delta: {}, finish_reason: "stop" }], }); - yield "data: [DONE]\n\n"; + if (!signal?.aborted && !cancellationSignal?.aborted) { + yield "data: [DONE]\n\n"; + } } } @@ -204,7 +235,10 @@ export async function readJsonlResponse( const parsed = parseJsonlLine(trimmed); if (parsed.token) fullText += parsed.token; if (parsed.text) return parsed.text; - if (parsed.error) throw new Error(parsed.error); + if (parsed.error) { + cancelReader(reader); + throw new HuggingChatStreamError(parsed.error); + } } } @@ -212,6 +246,7 @@ export async function readJsonlResponse( const parsed = parseJsonlLine(buffer.trim()); if (parsed.text) return parsed.text; if (parsed.token) fullText += parsed.token; + if (parsed.error) throw new HuggingChatStreamError(parsed.error); } } finally { reader.releaseLock(); diff --git a/open-sse/executors/ninerouter.ts b/open-sse/executors/ninerouter.ts index 0c6bfd8bc6..74fa30a48e 100644 --- a/open-sse/executors/ninerouter.ts +++ b/open-sse/executors/ninerouter.ts @@ -72,8 +72,7 @@ export class NineRouterExecutor extends BaseExecutor { * Message goes through buildErrorBody to satisfy hard rule #12 (no raw err.message). */ private buildServiceUnavailableResponse(message: string): Response { - const body = buildErrorBody(503, message); - body.error.code = "service_not_running"; + const body = buildErrorBody(503, message, undefined, { code: "service_not_running" }); return new Response(JSON.stringify(body), { status: 503, headers: { diff --git a/open-sse/executors/oneminai.ts b/open-sse/executors/oneminai.ts index 023f6ce425..5b3e2f4dda 100644 --- a/open-sse/executors/oneminai.ts +++ b/open-sse/executors/oneminai.ts @@ -16,6 +16,8 @@ type OpenAIMessage = { }; const CHAT_URL = "https://api.1min.ai/api/chat-with-ai"; +const MAX_STREAM_ERROR_DATA_CHARS = 64 * 1024; +const STREAM_ERROR_FALLBACK = "1min.ai upstream stream failed"; const ROLE_LABELS: Record = { system: "System", developer: "System", @@ -69,7 +71,35 @@ function buildSseChunk(data: unknown): string { return `data: ${JSON.stringify(data)}\n\n`; } -function buildOpenAiJsonCompletion(content: string, model: string, id: string, created: number): Response { +function parseStreamErrorMessage(data: string): string { + if (!data || data.length > MAX_STREAM_ERROR_DATA_CHARS) return STREAM_ERROR_FALLBACK; + + try { + const parsed = asRecord(JSON.parse(data)); + const directMessage = typeof parsed.message === "string" ? parsed.message.trim() : ""; + if (directMessage) return directMessage; + + if (typeof parsed.error === "string") { + const errorMessage = parsed.error.trim(); + if (errorMessage) return errorMessage; + } + + const nestedError = asRecord(parsed.error); + const nestedMessage = typeof nestedError.message === "string" ? nestedError.message.trim() : ""; + if (nestedMessage) return nestedMessage; + } catch { + // Malformed and over-complex payloads use the fixed public fallback below. + } + + return STREAM_ERROR_FALLBACK; +} + +function buildOpenAiJsonCompletion( + content: string, + model: string, + id: string, + created: number +): Response { return new Response( JSON.stringify({ id, @@ -84,7 +114,11 @@ function buildOpenAiJsonCompletion(content: string, model: string, id: string, c ); } -function toOpenAiErrorResponse(status: number, message: string, upstreamDetails?: unknown): Response { +function toOpenAiErrorResponse( + status: number, + message: string, + upstreamDetails?: unknown +): Response { return new Response(JSON.stringify(buildErrorBody(status, message, upstreamDetails)), { status, headers: { "Content-Type": "application/json" }, @@ -96,109 +130,214 @@ function toOpenAiErrorResponse(status: number, message: string, upstreamDetails? * data: {...}) from the upstream Response body and re-emit them as standard * OpenAI chat.completion.chunk SSE. */ -function translateSseStream(upstreamBody: ReadableStream, model: string, id: string, created: number): ReadableStream { +function translateSseStream( + upstreamBody: ReadableStream, + model: string, + id: string, + created: number +): ReadableStream { const decoder = new TextDecoder(); const encoder = new TextEncoder(); + const reader = upstreamBody.getReader(); + const pendingChunks: Uint8Array[] = []; + let buffer = ""; + let finished = false; + let roleEmitted = false; + let terminalError: Error | null = null; + let upstreamCancelRequested = false; + let downstreamCancelled = false; + let readInFlight = false; + let readerReleased = false; + + const releaseReader = () => { + if (readerReleased) return; + readerReleased = true; + reader.releaseLock(); + }; + + const cancelUpstream = (reason: unknown) => { + if (upstreamCancelRequested) return; + upstreamCancelRequested = true; + try { + // Upstream cleanup is provider-controlled and may never settle. The + // translated stream owns the reader lock and releases it independently. + void reader.cancel(reason).catch(() => {}); + } catch { + // Cancellation is cleanup-only; the terminal state is already fixed. + } + }; + + const queueChunk = (text: string) => { + pendingChunks.push(encoder.encode(text)); + }; + + const emitRole = () => { + if (roleEmitted) return; + roleEmitted = true; + queueChunk( + buildSseChunk({ + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }) + ); + }; + + const finish = () => { + if (finished) return; + finished = true; + queueChunk( + buildSseChunk({ + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }) + ); + queueChunk("data: [DONE]\n\n"); + }; + + const emitContent = (text: string) => { + if (!text) return; + emitRole(); + queueChunk( + buildSseChunk({ + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { content: text }, finish_reason: null }], + }) + ); + }; + + const emitError = (data: string) => { + if (finished) return; + finished = true; + cancelUpstream("1min.ai upstream stream error"); + + if (!roleEmitted) { + const message = parseStreamErrorMessage(data); + queueChunk(buildSseChunk(buildErrorBody(502, message))); + queueChunk("data: [DONE]\n\n"); + return; + } + + // A bare `{ error }` frame is dropped by the OpenAI passthrough sanitizer. + // Preserve every content delta already queued, then error the source with + // a fixed public message. pipeWithDisconnect() converts it into a native + // terminal error frame and drives usage, call-log, and fallback finalizers. + terminalError = Object.assign(new Error(STREAM_ERROR_FALLBACK), { + statusCode: 502, + }); + }; + + // SSE event framing: "event:"/"data:" lines, blank-line separated records. + const processEvent = (eventText: string) => { + let eventType = "message"; + const dataLines: string[] = []; + for (const rawLine of eventText.split("\n")) { + if (rawLine.startsWith("event:")) { + eventType = rawLine.slice(6).trim(); + } else if (rawLine.startsWith("data:")) { + dataLines.push(rawLine.slice(5).trim()); + } + } + const data = dataLines.join("\n"); + if (eventType === "content") { + try { + const parsed = asRecord(JSON.parse(data)); + if (typeof parsed.content === "string") emitContent(parsed.content); + } catch { + // Ignore malformed content events rather than surfacing partial JSON. + } + } else if (eventType === "error") { + emitError(data); + } else if (eventType === "done") { + finish(); + } + // "result" carries the final full aiRecord, redundant with the content + // events already streamed — intentionally ignored. + }; + + const processBufferedEvents = () => { + let separatorIndex = buffer.indexOf("\n\n"); + while (separatorIndex !== -1 && !finished) { + processEvent(buffer.slice(0, separatorIndex)); + buffer = buffer.slice(separatorIndex + 2); + separatorIndex = buffer.indexOf("\n\n"); + } + }; return new ReadableStream({ - async start(controller) { - controller.enqueue( - encoder.encode( - buildSseChunk({ - id, - object: "chat.completion.chunk", - created, - model, - choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], - }) - ) - ); + async pull(controller) { + if (downstreamCancelled) return; - const reader = upstreamBody.getReader(); - let buffer = ""; - let finished = false; - - const finish = () => { - if (finished) return; - finished = true; - controller.enqueue( - encoder.encode( - buildSseChunk({ - id, - object: "chat.completion.chunk", - created, - model, - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - }) - ) - ); - controller.enqueue(encoder.encode("data: [DONE]\n\n")); - controller.close(); - }; - - const emitContent = (text: string) => { - if (!text) return; - controller.enqueue( - encoder.encode( - buildSseChunk({ - id, - object: "chat.completion.chunk", - created, - model, - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], - }) - ) - ); - }; - - // SSE event framing: "event:"/"data:" lines, blank-line separated records. - const processEvent = (eventText: string) => { - let eventType = "message"; - const dataLines: string[] = []; - for (const rawLine of eventText.split("\n")) { - if (rawLine.startsWith("event:")) { - eventType = rawLine.slice(6).trim(); - } else if (rawLine.startsWith("data:")) { - dataLines.push(rawLine.slice(5).trim()); - } - } - const data = dataLines.join("\n"); - if (eventType === "content") { - try { - const parsed = asRecord(JSON.parse(data)); - if (typeof parsed.content === "string") emitContent(parsed.content); - } catch { - // Ignore malformed content events rather than surfacing partial JSON. - } - } else if (eventType === "error") { - emitContent(`\n[1min.ai error: ${data}]`); - finish(); - } else if (eventType === "done") { - finish(); - } - // "result" carries the final full aiRecord, redundant with the content - // events already streamed — intentionally ignored. - }; - - try { - while (!finished) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - let separatorIndex = buffer.indexOf("\n\n"); - while (separatorIndex !== -1) { - processEvent(buffer.slice(0, separatorIndex)); - buffer = buffer.slice(separatorIndex + 2); - separatorIndex = buffer.indexOf("\n\n"); - } - } - if (!finished && buffer.trim()) processEvent(buffer); - finish(); - } catch (error) { - controller.error(error); - } finally { - reader.releaseLock(); + if (pendingChunks.length > 0) { + controller.enqueue(pendingChunks.shift()!); + return; } + + if (terminalError) { + releaseReader(); + controller.error(terminalError); + return; + } + + if (finished) { + releaseReader(); + controller.close(); + return; + } + + readInFlight = true; + try { + while (pendingChunks.length === 0 && !finished && !downstreamCancelled) { + const { done, value } = await reader.read(); + if (downstreamCancelled) return; + if (done) { + buffer += decoder.decode(); + if (buffer.trim()) processEvent(buffer); + finish(); + break; + } + + buffer += decoder.decode(value, { stream: true }); + // Process the complete upstream chunk, even after it queues output. + // One network read may contain multiple content events followed by + // an error; the internal queue preserves all of them in order. + processBufferedEvents(); + } + + if (downstreamCancelled) return; + if (pendingChunks.length > 0) { + controller.enqueue(pendingChunks.shift()!); + } else if (terminalError) { + releaseReader(); + controller.error(terminalError); + } else if (finished) { + releaseReader(); + controller.close(); + } + } catch (error) { + releaseReader(); + if (!downstreamCancelled) controller.error(error); + } finally { + readInFlight = false; + if (downstreamCancelled) releaseReader(); + } + }, + cancel(reason) { + downstreamCancelled = true; + pendingChunks.length = 0; + // A client disconnect must release the upstream reader even when its + // next pull never settles. Do not await provider cleanup here: the + // downstream cancellation contract must remain bounded. + cancelUpstream(reason ?? "1min.ai downstream cancelled"); + if (!readInFlight) releaseReader(); }, }); } @@ -290,7 +429,9 @@ export class OneMinAiExecutor extends BaseExecutor { const aiRecord = asRecord(json.aiRecord); const detail = asRecord(aiRecord.aiRecordDetail); const resultObject = Array.isArray(detail.resultObject) ? detail.resultObject : []; - const content = resultObject.filter((part): part is string => typeof part === "string").join(""); + const content = resultObject + .filter((part): part is string => typeof part === "string") + .join(""); return { response: buildOpenAiJsonCompletion(content, model, id, created), diff --git a/open-sse/executors/perplexity-web.ts b/open-sse/executors/perplexity-web.ts index 6d0dc6d859..16ae613c8e 100644 --- a/open-sse/executors/perplexity-web.ts +++ b/open-sse/executors/perplexity-web.ts @@ -17,6 +17,7 @@ import { prepareToolMessages } from "../translator/webTools.ts"; import { buildToolModeResponse } from "./chatgptWebTools.ts"; import { sanitizeErrorMessage } from "../utils/error.ts"; import { buildSessionCookieHeader, mergeRefreshedCookie } from "../utils/nextAuthCookie.ts"; +import { formatTranslatedStreamError } from "../utils/streamErrorFormat.ts"; import { PPLX_SSE_ENDPOINT, PPLX_USER_AGENT, @@ -35,6 +36,8 @@ import { const SESSION_MAX_AGE_MS = 3600_000; const SESSION_MAX_ENTRIES = 200; +const PPLX_STREAM_ERROR_MESSAGE = "Perplexity upstream stream failed"; +const PPLX_STREAM_ERROR_CODE = "PPLX_STREAM_ERROR"; interface SessionEntry { backendUuid: string; @@ -102,155 +105,223 @@ function buildStreamingResponse( signal?: AbortSignal | null ): ReadableStream { const encoder = new TextEncoder(); + const streamAbortController = new AbortController(); + const forwardInputAbort = () => + streamAbortController.abort(signal?.reason ?? "perplexity_request_aborted"); + if (signal?.aborted) forwardInputAbort(); + else signal?.addEventListener("abort", forwardInputAbort, { once: true }); + let inputAbortListenerAttached = Boolean(signal && !signal.aborted); + const removeInputAbortListener = () => { + if (!inputAbortListenerAttached) return; + inputAbortListenerAttached = false; + signal?.removeEventListener("abort", forwardInputAbort); + }; + const abortEventStream = (reason: unknown) => { + removeInputAbortListener(); + if (!streamAbortController.signal.aborted) streamAbortController.abort(reason); + }; + const contentIterator = extractContent(eventStream, streamAbortController.signal)[ + Symbol.asyncIterator + ](); + let fullAnswer = ""; + let respBackendUuid: string | null = null; + let roleEmitted = false; + let finished = false; + let pendingFailure: (Error & { statusCode: number }) | null = null; - return new ReadableStream( - { - async start(controller) { - try { - // Initial role chunk - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { index: 0, delta: { role: "assistant" }, finish_reason: null, logprobs: null }, - ], - }) - ) - ); + const enqueuePreContentFailure = (controller: ReadableStreamDefaultController) => { + controller.enqueue( + encoder.encode( + formatTranslatedStreamError({ + status: 502, + message: PPLX_STREAM_ERROR_MESSAGE, + type: "upstream_error", + code: PPLX_STREAM_ERROR_CODE, + }) + ) + ); + }; - let fullAnswer = ""; - let respBackendUuid: string | null = null; + const takeAssistantRoleChunk = (): string => { + if (roleEmitted) return ""; + roleEmitted = true; + return sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: null, + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }); + }; - for await (const chunk of extractContent(eventStream, signal)) { - if (chunk.backendUuid) respBackendUuid = chunk.backendUuid; + const completeStream = (controller: ReadableStreamDefaultController) => { + if (finished) return; + finished = true; + controller.enqueue( + encoder.encode( + sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: null, + choices: [{ index: 0, delta: {}, finish_reason: "stop", logprobs: null }], + }) + ) + ); + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + sessionStore(history, currentMsg, cleanResponse(fullAnswer), respBackendUuid); + removeInputAbortListener(); + controller.close(); + }; - if (chunk.error) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { - index: 0, - delta: { content: `[Error: ${chunk.error}]` }, - finish_reason: null, - logprobs: null, - }, - ], - }) - ) - ); - break; - } + const failStream = (controller: ReadableStreamDefaultController) => { + if (roleEmitted) { + pendingFailure = Object.assign(new Error(PPLX_STREAM_ERROR_MESSAGE), { + statusCode: 502, + }); + finished = true; + controller.close(); + return; + } + finished = true; + enqueuePreContentFailure(controller); + controller.close(); + }; - if (chunk.thinking) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { - index: 0, - delta: { reasoning_content: chunk.thinking + "\n" }, - finish_reason: null, - logprobs: null, - }, - ], - }) - ) - ); - continue; - } + const providerStream = new ReadableStream({ + async pull(controller) { + if (finished) return; - if (chunk.done) { - fullAnswer = chunk.answer || fullAnswer; - break; - } - - let dt = chunk.delta || ""; - if (dt) { - dt = cleanResponse(dt, false); - if (dt) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { index: 0, delta: { content: dt }, finish_reason: null, logprobs: null }, - ], - }) - ) - ); - } - } - if (chunk.answer) fullAnswer = chunk.answer; - } - - // Stop chunk - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [{ index: 0, delta: {}, finish_reason: "stop", logprobs: null }], - }) - ) - ); - controller.enqueue(encoder.encode("data: [DONE]\n\n")); - - sessionStore(history, currentMsg, cleanResponse(fullAnswer), respBackendUuid); - } catch (err) { - controller.enqueue( - encoder.encode( - sseChunk({ - id: cid, - object: "chat.completion.chunk", - created, - model, - system_fingerprint: null, - choices: [ - { - index: 0, - delta: { - content: `[Stream error: ${err instanceof Error ? err.message : String(err)}]`, - }, - finish_reason: "stop", - logprobs: null, - }, - ], - }) - ) - ); - controller.enqueue(encoder.encode("data: [DONE]\n\n")); - } finally { - try { - controller.close(); - } catch {} + try { + const next = await contentIterator.next(); + if (streamAbortController.signal.aborted) { + finished = true; + controller.close(); + return; } - }, + if (next.done === true) { + completeStream(controller); + return; + } + + const chunk = next.value; + if (chunk.backendUuid) respBackendUuid = chunk.backendUuid; + + if (chunk.error) { + failStream(controller); + removeInputAbortListener(); + void contentIterator.return?.(undefined).catch(() => undefined); + return; + } + + if (chunk.thinking) { + controller.enqueue( + encoder.encode( + takeAssistantRoleChunk() + + sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: null, + choices: [ + { + index: 0, + delta: { reasoning_content: chunk.thinking + "\n" }, + finish_reason: null, + logprobs: null, + }, + ], + }) + ) + ); + return; + } + + if (chunk.done) { + fullAnswer = chunk.answer || fullAnswer; + completeStream(controller); + await contentIterator.return?.(undefined); + return; + } + + let dt = chunk.delta || ""; + if (dt) { + dt = cleanResponse(dt, false); + if (dt) { + controller.enqueue( + encoder.encode( + takeAssistantRoleChunk() + + sseChunk({ + id: cid, + object: "chat.completion.chunk", + created, + model, + system_fingerprint: null, + choices: [ + { index: 0, delta: { content: dt }, finish_reason: null, logprobs: null }, + ], + }) + ) + ); + } + } + if (chunk.answer) fullAnswer = chunk.answer; + } catch { + failStream(controller); + removeInputAbortListener(); + void contentIterator.return?.(undefined).catch(() => undefined); + } }, - { highWaterMark: 16384 } - ); + + cancel(reason) { + finished = true; + abortEventStream(reason); + void contentIterator.return?.(undefined).catch(() => undefined); + }, + }); + + // Erroring the provider stream immediately would discard output buffered by the readiness + // handoff. Drain each provider chunk through a backpressure-aware reader, then reject only the + // read after the last legitimate chunk. The outer pipeline converts that fixed public error to + // the client's canonical terminal frame and records the stream failure. + const providerReader = providerStream.getReader(); + let cancelled = false; + return new ReadableStream({ + async pull(controller) { + try { + const next = await providerReader.read(); + if (cancelled) return; + if (next.done === false) { + controller.enqueue(next.value); + return; + } + if (pendingFailure) { + controller.error(pendingFailure); + return; + } + controller.close(); + } catch (error) { + if (!cancelled) controller.error(error); + } + }, + + cancel(reason) { + if (cancelled) return; + cancelled = true; + abortEventStream(reason); + void providerReader.cancel(reason).catch(() => undefined); + }, + }); } async function buildNonStreamingResponse( diff --git a/open-sse/executors/perplexity-web/protocol.ts b/open-sse/executors/perplexity-web/protocol.ts index 12e98ccdc4..f3f8a318c5 100644 --- a/open-sse/executors/perplexity-web/protocol.ts +++ b/open-sse/executors/perplexity-web/protocol.ts @@ -213,6 +213,19 @@ export async function* readPplxSseEvents( const decoder = new TextDecoder(); let buffer = ""; let dataLines: string[] = []; + let readerFinished = false; + let readerCancelRequested = false; + + const cancelReader = (reason: unknown) => { + if (readerFinished || readerCancelRequested) return; + readerCancelRequested = true; + // Cancellation is a client-facing latency boundary. Request upstream cleanup once, but never + // await a hostile underlying source whose cancel hook does not settle. + void reader.cancel(reason).catch(() => undefined); + }; + const handleAbort = () => cancelReader(signal?.reason ?? "perplexity_stream_aborted"); + if (signal?.aborted) handleAbort(); + else signal?.addEventListener("abort", handleAbort, { once: true }); function flush(): PplxStreamEvent | null | "done" { if (dataLines.length === 0) return null; @@ -231,7 +244,10 @@ export async function* readPplxSseEvents( while (true) { if (signal?.aborted) return; const { value, done } = await reader.read(); - if (done) break; + if (done) { + readerFinished = true; + break; + } buffer += decoder.decode(value, { stream: true }); while (true) { @@ -263,7 +279,13 @@ export async function* readPplxSseEvents( const tail = flush(); if (tail && tail !== "done") yield tail; } finally { - reader.releaseLock(); + signal?.removeEventListener("abort", handleAbort); + cancelReader(signal?.reason ?? "perplexity_stream_reader_closed"); + try { + reader.releaseLock(); + } catch { + // A hostile source may keep its cancel promise pending; the lock can be released later by GC. + } } } @@ -915,6 +937,10 @@ export async function* extractContent( } } + // Cancellation is not a successful terminal event. In particular, do not synthesize the final + // `done` chunk: streaming callers use that signal to emit stop/[DONE] and persist the session. + if (signal?.aborted) return; + // End-of-stream without a COMPLETED frame still try the last text blob. if (!fullAnswer.trim() && lastEventText) { const fromText = extractAnswerFromFinalText(lastEventText); diff --git a/open-sse/executors/zai-web/stream.ts b/open-sse/executors/zai-web/stream.ts index 48b99312dd..1c87321bf1 100644 --- a/open-sse/executors/zai-web/stream.ts +++ b/open-sse/executors/zai-web/stream.ts @@ -1,4 +1,4 @@ -import { sanitizeErrorMessage } from "../../utils/error.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "../../utils/error.ts"; export interface ZaiDelta { content: string; @@ -113,22 +113,73 @@ function parseSsePayload(data: string): ZaiDelta | null { } } +type ZaiDeltaSource = { + deltas: AsyncGenerator; + cancel: (reason?: unknown) => void; +}; + +function createZaiDeltaSource(sourceBody: ReadableStream): ZaiDeltaSource { + const decoder = new TextDecoder(); + const reader = sourceBody.getReader(); + const buffer = { text: "" }; + let upstreamDone = false; + let cancelRequested = false; + let readerReleased = false; + + const releaseReader = () => { + if (readerReleased) return; + readerReleased = true; + try { + reader.releaseLock(); + } catch { + // A concurrent read cancellation owns the final release. + } + }; + + const cancel = (reason?: unknown) => { + if (upstreamDone || cancelRequested) return; + cancelRequested = true; + try { + // Do not await an upstream cancel hook: a stalled provider is allowed to + // ignore cancellation, but it must never keep the client cancellation open. + void reader.cancel(reason).catch(() => {}); + } catch { + // The reader may already have closed or released concurrently. + } + }; + + async function* iterate(): AsyncGenerator { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + upstreamDone = true; + return; + } + const payloads = extractSseDataPayloads(buffer, decoder.decode(value, { stream: true })); + for (const raw of payloads) { + const delta = parseSsePayload(raw); + if (delta) yield delta; + } + } + } finally { + if (!upstreamDone && !cancelRequested) cancel("Z.ai delta iteration ended"); + releaseReader(); + } + } + + return { deltas: iterate(), cancel }; +} + async function drainSseDeltas( sourceBody: ReadableStream, onDelta: (delta: ZaiDelta) => boolean ): Promise { - const decoder = new TextDecoder(); - const reader = sourceBody.getReader(); - const buffer = { text: "" }; - while (true) { - const { done, value } = await reader.read(); - if (done) return false; - const payloads = extractSseDataPayloads(buffer, decoder.decode(value, { stream: true })); - for (const raw of payloads) { - const delta = parseSsePayload(raw); - if (delta && onDelta(delta)) return true; - } + const { deltas } = createZaiDeltaSource(sourceBody); + for await (const delta of deltas) { + if (onDelta(delta)) return true; } + return false; } function emitDeltaChunks( @@ -137,17 +188,32 @@ function emitDeltaChunks( emitChunk: ZaiChunkEmitter, roleState: { emitted: boolean } ): boolean { - if (!roleState.emitted && (delta.content || delta.reasoning || delta.error)) { + if (delta.error) { + const errorBody = buildErrorBody(502, `Z.ai stream failed: ${delta.error}`, undefined, { + type: "upstream_error", + code: "zai_stream_error", + }); + + if (!roleState.emitted) { + // Keep a pre-content failure as an error-only Chat frame. Stream readiness + // rejects it before response headers are committed, so fallback receives a 502. + controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(errorBody)}\n\n`)); + controller.close(); + } else { + // Once content is public, error the protocol-neutral producer. The shared + // pipeline preserves prior chunks, records the failure, and emits the terminal + // error in the client's native Chat, Claude, or Responses wire format. + controller.error(Object.assign(new Error(errorBody.error.message), { statusCode: 502 })); + } + return true; + } + + if (!roleState.emitted && (delta.content || delta.reasoning)) { roleState.emitted = true; emitChunk(controller, { role: "assistant", content: "" }); } if (delta.reasoning) emitChunk(controller, { reasoning_content: delta.reasoning }); if (delta.content) emitChunk(controller, { content: delta.content }); - // Surfaced as visible content, matching the other web executors' mid-stream - // error convention (see zed-hosted's createErrorChunk): the 200 is already on - // the wire, so the status cannot change — but the caller must not be left - // reading an empty success. Any content streamed before the failure is kept. - if (delta.error) emitChunk(controller, { content: `[Z.ai error] ${delta.error}` }); if (delta.done) { emitChunk(controller, {}, "stop"); controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")); @@ -162,19 +228,32 @@ export function buildZaiStreamingBody( emitChunk: ZaiChunkEmitter, signal: AbortSignal | null | undefined ): ReadableStream { + const deltaSource = createZaiDeltaSource(sourceBody); + const { deltas } = deltaSource; + const roleState = { emitted: false }; + let terminated = false; + return new ReadableStream({ - async start(controller) { - const roleState = { emitted: false }; + async pull(controller) { + if (terminated) return; try { - const ended = await drainSseDeltas(sourceBody, (delta) => - emitDeltaChunks(controller, delta, emitChunk, roleState) - ); - if (ended) return; + const next = await deltas.next(); + if (terminated) return; + if (next.done === false) { + if (emitDeltaChunks(controller, next.value, emitChunk, roleState)) { + terminated = true; + await deltas.return(undefined); + } + return; + } + + terminated = true; if (!roleState.emitted) emitChunk(controller, { role: "assistant", content: "" }); emitChunk(controller, {}, "stop"); controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")); controller.close(); } catch (error) { + terminated = true; if (!signal?.aborted) { try { controller.error(error); @@ -184,6 +263,11 @@ export function buildZaiStreamingBody( } } }, + cancel(reason) { + terminated = true; + deltaSource.cancel(reason); + void deltas.return(undefined).catch(() => {}); + }, }); } diff --git a/open-sse/executors/zed-hosted.ts b/open-sse/executors/zed-hosted.ts index ef1ae4ade6..ba66706f10 100644 --- a/open-sse/executors/zed-hosted.ts +++ b/open-sse/executors/zed-hosted.ts @@ -44,6 +44,8 @@ import { zedLlmFetch, type ZedCredentials, } from "../shared/zedAuth.ts"; +import { buildErrorBody } from "../utils/error.ts"; +import { hasUsefulStreamContent } from "../utils/streamReadiness.ts"; import { resolveSuppressThinkClose, THINKING_MARKER_HEADER } from "../utils/thinkCloseMarker.ts"; // Wire values for the `provider` field of POST /completions. These are NOT @@ -122,37 +124,72 @@ function convertProviderEvent( return event; } -function createErrorChunk(model: string, message: string): Record { - return { - id: `chatcmpl-zed-error-${Date.now()}`, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model, - choices: [{ index: 0, delta: { content: `[Zed error] ${message}` }, finish_reason: "stop" }], - }; +const MAX_ZED_FAILURE_MESSAGE_LENGTH = 512; +const MAX_PENDING_ZED_OUTPUT_LENGTH = 64 * 1024; +const ZED_STREAM_FAILURE_PUBLIC_MESSAGE = "Zed upstream stream failed"; + +function boundedFailureText(value: unknown): string | null { + if (typeof value !== "string" && typeof value !== "number") return null; + const text = String(value).trim(); + return text ? text.slice(0, MAX_ZED_FAILURE_MESSAGE_LENGTH) : null; +} + +function extractZedFailureMessage(failed: Record): string { + const nestedError = + failed.error && typeof failed.error === "object" && !Array.isArray(failed.error) + ? (failed.error as Record) + : null; + const candidates = [ + failed.message, + nestedError?.message, + typeof failed.error === "object" ? undefined : failed.error, + failed.code, + nestedError?.code, + ]; + for (const candidate of candidates) { + const text = boundedFailureText(candidate); + if (text) return text; + } + return "request failed"; +} + +function createErrorChunk(message: string): ReturnType { + return buildErrorBody(502, `Zed stream failed: ${message}`, undefined, { + type: "upstream_error", + code: "ZED_STREAM_FAILED", + }); } /** - * The single controller capability these SSE helpers use. They only ever enqueue — - * never `close()`, never read `desiredSize` — so typing them by that one method lets - * the same code serve both stream kinds. The wider + * The controller capabilities these SSE helpers use. Normal frames only enqueue; + * terminal failures also terminate so they do not depend on the upstream socket + * eventually reaching EOF. Narrow controller types keep the helpers honest. The wider * `ReadableStreamDefaultController` annotation rejected every call site, because the * helpers are driven from a TransformStream and `TransformStreamDefaultController` * has no `close()`. */ type SseEnqueueTarget = Pick, "enqueue">; +type SseProcessTarget = Pick, "enqueue" | "terminate">; + +function serializeSseObject(chunk: unknown): string { + if (!chunk) return ""; + let serialized = ""; + const items = Array.isArray(chunk) ? chunk : [chunk]; + for (const item of items) { + if (!item) continue; + serialized += `data: ${JSON.stringify(item)}\n\n`; + } + return serialized; +} function enqueueSseObject( controller: SseEnqueueTarget, encoder: TextEncoder, chunk: unknown ): void { - if (!chunk) return; - const items = Array.isArray(chunk) ? chunk : [chunk]; - for (const item of items) { - if (!item) continue; - controller.enqueue(encoder.encode(`data: ${JSON.stringify(item)}\n\n`)); - } + const serialized = serializeSseObject(chunk); + if (!serialized) return; + controller.enqueue(encoder.encode(serialized)); } type ZedLine = { done?: true; status?: unknown; event?: unknown } | null; @@ -226,16 +263,47 @@ function wrapZedCompletionStream( } let buffer = ""; let done = false; + let providerOutputForwarded = false; + let pendingProviderOutput = ""; + let pendingFailure: (Error & { statusCode: number }) | null = null; + + const forwardProviderOutput = (controller: SseEnqueueTarget, chunk: unknown) => { + const serialized = serializeSseObject(chunk); + if (!serialized) return; + if (providerOutputForwarded) { + controller.enqueue(encoder.encode(serialized)); + return; + } + + // A role/bootstrap-only chunk makes ensureStreamReadiness release the response before any + // model output exists. If the next chunk is status.failed, downstream read-ahead can discard + // the first real content while propagating the error. Hold structural frames until the first + // substantive text/reasoning/tool delta, then release them atomically with that output. + const outputWithBootstrap = pendingProviderOutput + serialized; + if (!hasUsefulStreamContent(outputWithBootstrap)) { + pendingProviderOutput = + outputWithBootstrap.length <= MAX_PENDING_ZED_OUTPUT_LENGTH + ? outputWithBootstrap + : serialized.length <= MAX_PENDING_ZED_OUTPUT_LENGTH + ? serialized + : ""; + return; + } + controller.enqueue(encoder.encode(outputWithBootstrap)); + pendingProviderOutput = ""; + providerOutputForwarded = true; + }; const finish = (controller: SseEnqueueTarget) => { if (done) return; const finalChunk = convertProviderEvent(provider, null, state); - enqueueSseObject(controller, encoder, finalChunk); - controller.enqueue(encoder.encode("data: [DONE]\n\n")); + const finalOutput = `${pendingProviderOutput}${serializeSseObject(finalChunk)}data: [DONE]\n\n`; + pendingProviderOutput = ""; + controller.enqueue(encoder.encode(finalOutput)); done = true; }; - const processLine = (line: string, controller: SseEnqueueTarget) => { + const processLine = (line: string, controller: SseProcessTarget) => { if (done) return; const payload = unwrapZedLine(line); if (!payload) return; @@ -246,17 +314,29 @@ function wrapZedCompletionStream( if (payload.status) { const status = normalizeStatus(payload.status); if (status?.type === "failed" || status?.failed) { - const failed = (status.failed as Record) || status; - const message = String(failed.message || failed.error || failed.code || "request failed"); - enqueueSseObject(controller, encoder, createErrorChunk(model, message)); - finish(controller); + const failed = + status.failed && typeof status.failed === "object" && !Array.isArray(status.failed) + ? (status.failed as Record) + : status; + if (providerOutputForwarded) { + pendingFailure = Object.assign(new Error(ZED_STREAM_FAILURE_PUBLIC_MESSAGE), { + statusCode: 502, + }); + done = true; + controller.terminate(); + return; + } + pendingProviderOutput = ""; + enqueueSseObject(controller, encoder, createErrorChunk(extractZedFailureMessage(failed))); + done = true; + controller.terminate(); } else if (status?.type === "stream_ended" || status === ("stream_ended" as unknown)) { finish(controller); } return; } const converted = convertProviderEvent(provider, payload.event, state); - enqueueSseObject(controller, encoder, converted); + forwardProviderOutput(controller, converted); }; const transformed = response.body.pipeThrough( @@ -281,7 +361,47 @@ function wrapZedCompletionStream( }) ); - return new Response(transformed, { + // `TransformStreamDefaultController.error()` discards already-enqueued output. A failed + // status can share one upstream network chunk with the last content delta, so erroring the + // transform immediately would erase that partial answer. Drain the transformed chunks through + // a backpressure-aware reader first, then reject the next read with the fixed public error. + // The normal chat pipeline turns that rejection into its client-format terminal frame and + // records the 502 through the existing failure finalizers. + const transformedReader = transformed.getReader(); + let guardedStreamCancelled = false; + const cancelTransformedReader = (reason: unknown) => { + if (guardedStreamCancelled) return; + guardedStreamCancelled = true; + // Client cancellation must settle independently of an upstream body whose cancel hook hangs. + // Request cancellation once, but do not await provider cleanup on the client-facing boundary. + void transformedReader.cancel(reason).catch(() => { + console.debug("[ZED] upstream stream cancellation rejected"); + }); + }; + const guardedStream = new ReadableStream({ + async pull(controller) { + try { + const next = await transformedReader.read(); + if (guardedStreamCancelled) return; + if (!next.done) { + controller.enqueue(next.value); + return; + } + if (pendingFailure) { + controller.error(pendingFailure); + return; + } + controller.close(); + } catch (error) { + if (!guardedStreamCancelled) controller.error(error); + } + }, + cancel(reason) { + cancelTransformedReader(reason); + }, + }); + + return new Response(guardedStream, { status: response.status, statusText: response.statusText, headers: { diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 65a986d186..622e934084 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -5,7 +5,8 @@ import { import { injectMemoryAndSkills } from "./chatCore/memorySkillsInjection.ts"; import { resolveChatCoreRequestSetup } from "./chatCore/requestSetup.ts"; import { normalizeOpenAICompatibleTools } from "./chatCore/openAICompatibleTools.ts"; -import { buildFailureUsageRecord } from "./chatCore/failureUsage.ts"; +import { buildFailureUsageRecord, projectFailureUsageErrorCode } from "./chatCore/failureUsage.ts"; +import { createTranslationFailureResult } from "./chatCore/translationFailure.ts"; import { estimateFinalInputTokens } from "./chatCore/contextEstimation.ts"; import { extractSystemRoleMessages, @@ -2513,35 +2514,11 @@ export async function handleChatCore({ : HTTP_STATUS.SERVER_ERROR; const message = error?.message || "Invalid request"; const errorType = typeof error?.errorType === "string" ? error.errorType : null; - - log?.warn?.("TRANSLATE", `Request translation failed: ${message}`); - - if (errorType) { - trackPendingRequest(model, provider, connectionId, false); - return { - success: false, - status: statusCode, - error: message, - response: new Response( - JSON.stringify({ - error: { - message, - type: errorType, - code: errorType, - }, - }), - { - status: statusCode, - headers: { - "Content-Type": "application/json", - }, - } - ), - }; - } + const result = createTranslationFailureResult(statusCode, message, errorType); + log?.warn?.("TRANSLATE", `Request translation failed: ${result.error}`); trackPendingRequest(model, provider, connectionId, false); - return createErrorResult(statusCode, message); + return result; } // The latest OmniGlyph release has protocol-native OpenAI transforms. Run @@ -3924,10 +3901,14 @@ export async function handleChatCore({ streamController.handleError(error); return createErrorResult(499, "Request aborted"); } - persistFailureUsage( - failureStatus, - upstreamErrorCode || (error instanceof Error && error.name ? error.name : "upstream_error") - ); + const persistentErrorCode = projectFailureUsageErrorCode({ + statusCode: failureStatus, + message: failureMessage, + errorCode: + upstreamErrorCode || (error instanceof Error && error.name ? error.name : "upstream_error"), + errorType: upstreamErrorType, + }); + persistFailureUsage(failureStatus, persistentErrorCode); console.log(`${COLORS.red}[ERROR] ${failureMessage}${COLORS.reset}`); if (stream && upstreamErrorCode) { const result = createStreamingErrorResult( @@ -4253,6 +4234,9 @@ export async function handleChatCore({ `${decision.kind} (model remaining: ${decision.snapshot.modelRemaining ?? "unknown"}, total remaining: ${decision.snapshot.totalRemaining ?? "unknown"})` ); } + // Classifiers and recovery paths above consume the raw provider wording. + // Project a separate value only at persistent connection-state boundaries. + const persistentMessage = sanitizeErrorMessage(message) || "Provider request failed"; const errorConnectionId = getCurrentConnectionId(); if (errorConnectionId && errorType) { try { @@ -4264,7 +4248,7 @@ export async function handleChatCore({ { testStatus: "banned", isActive: false, - lastError: message, + lastError: persistentMessage, lastErrorType: errorType, errorCode: String(statusCode), }, @@ -4295,7 +4279,7 @@ export async function handleChatCore({ ) { await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); console.warn( @@ -4308,7 +4292,7 @@ export async function handleChatCore({ { testStatus: "deactivated", isActive: false, - lastError: message, + lastError: persistentMessage, lastErrorType: errorType, errorCode: String(statusCode), }, @@ -4332,7 +4316,7 @@ export async function handleChatCore({ errorConnectionId, { testStatus: "credits_exhausted", - lastError: message, + lastError: persistentMessage, lastErrorType: errorType, errorCode: String(statusCode), }, @@ -4418,7 +4402,7 @@ export async function handleChatCore({ rateLimitedUntil: kimiRateLimitResetAt, backoffLevel: 0, lastErrorType: PROVIDER_ERROR_TYPES.RATE_LIMITED, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); console.warn( @@ -4447,7 +4431,7 @@ export async function handleChatCore({ errorConnectionId, { testStatus: "credits_exhausted", - lastError: message, + lastError: persistentMessage, lastErrorType: errorType, errorCode: String(statusCode), }, @@ -4463,14 +4447,14 @@ export async function handleChatCore({ // Normal 401 (token/session auth issue): keep account active for refresh/re-auth. await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); } else if (errorType === PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN) { // OAuth 401 with invalid credentials - token refresh can recover await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); console.warn( @@ -4480,7 +4464,7 @@ export async function handleChatCore({ // Cloud Code 403 with stale project: not a ban, keep account active. await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); console.warn( @@ -4496,7 +4480,7 @@ export async function handleChatCore({ const geoCooldownMs = COOLDOWN_MS.geoBlocked ?? 24 * 60 * 60 * 1000; await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); // T-PROBE: the 24h exclusion is a routing mutation — a probe must @@ -4521,7 +4505,7 @@ export async function handleChatCore({ const byopCooldownMs = COOLDOWN_MS.gcpProjectRequired ?? 24 * 60 * 60 * 1000; await updateProviderConnection(errorConnectionId, { lastErrorType: errorType, - lastError: message, + lastError: persistentMessage, errorCode: statusCode, }); try { @@ -5305,9 +5289,12 @@ export async function handleChatCore({ }).catch(() => {}); const malformed = describeMalformedNonStream(translatedResponse, malformedTranslatedReason); const malformedMessage = `[${provider}/${model}] ${malformed.message}`; - const malformedClientBody = buildErrorBody(HTTP_STATUS.BAD_GATEWAY, malformedMessage); - malformedClientBody.error.code = malformed.code; - malformedClientBody.error.type = malformed.type; + const malformedClientBody = buildErrorBody( + HTTP_STATUS.BAD_GATEWAY, + malformedMessage, + undefined, + { code: malformed.code, type: malformed.type } + ); persistAttemptLogs({ status: HTTP_STATUS.BAD_GATEWAY, tokens: usage, diff --git a/open-sse/handlers/chatCore/attemptLogging.ts b/open-sse/handlers/chatCore/attemptLogging.ts index 07de9f19b2..5ae0876f77 100644 --- a/open-sse/handlers/chatCore/attemptLogging.ts +++ b/open-sse/handlers/chatCore/attemptLogging.ts @@ -19,6 +19,7 @@ import { saveCallLog } from "@/lib/usageDb"; import type { VideoBridgeLogRedactionEntry } from "@/lib/guardrails/videoBridge"; import { FORMATS } from "../../translator/formats.ts"; import { takeEarlyKeepaliveBytes } from "../../utils/earlyKeepaliveByteBuffer.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; import { cloneBoundedChatLogPayload, truncateForLog } from "./logTruncation.ts"; import { attachLogMeta } from "./cacheUsageMeta.ts"; @@ -317,7 +318,9 @@ export function resolveRequestLifecycleEvent(input: { name: "request.failed", payload: { id: traceId, - error: error || `HTTP ${status}`, + // Dashboard listeners and event history cross a public WebSocket boundary. Keep the raw + // diagnostic in the call log/pipeline above, but expose only the canonical safe projection. + error: sanitizeErrorMessage(error || `HTTP ${status}`), statusCode: typeof status === "number" ? status : undefined, latencyMs, model: model || undefined, diff --git a/open-sse/handlers/chatCore/failureUsage.ts b/open-sse/handlers/chatCore/failureUsage.ts index 52f70fbfee..d9fff0ae33 100644 --- a/open-sse/handlers/chatCore/failureUsage.ts +++ b/open-sse/handlers/chatCore/failureUsage.ts @@ -8,6 +8,21 @@ * `latencyMs` (Date.now() - startTime) and fires the fire-and-forget saveRequestUsage(...).catch(). */ +import { buildErrorBody } from "../../utils/error.ts"; + +export function projectFailureUsageErrorCode(opts: { + statusCode: number; + message: string; + errorCode?: string | null; + errorType?: string | null; +}): string { + const errorBody = buildErrorBody(opts.statusCode, opts.message, undefined, { + code: opts.errorCode || undefined, + type: opts.errorType || undefined, + }); + return errorBody.error.code || String(opts.statusCode); +} + export function buildFailureUsageRecord(opts: { provider: string | null | undefined; model: string | null | undefined; diff --git a/open-sse/handlers/chatCore/streamErrorResult.ts b/open-sse/handlers/chatCore/streamErrorResult.ts index 77244b611d..04e041e55c 100644 --- a/open-sse/handlers/chatCore/streamErrorResult.ts +++ b/open-sse/handlers/chatCore/streamErrorResult.ts @@ -25,13 +25,7 @@ export function createStreamingErrorResult( code?: string, type?: string ) { - const errorBody = buildErrorBody(statusCode, message); - if (code) { - errorBody.error.code = code; - } - if (type) { - errorBody.error.type = type; - } + const errorBody = buildErrorBody(statusCode, message, undefined, { code, type }); const body = `data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`; diff --git a/open-sse/handlers/chatCore/translationFailure.ts b/open-sse/handlers/chatCore/translationFailure.ts new file mode 100644 index 0000000000..622b5c8c47 --- /dev/null +++ b/open-sse/handlers/chatCore/translationFailure.ts @@ -0,0 +1,24 @@ +import { buildErrorBody, createErrorResult } from "../../utils/error.ts"; + +export function createTranslationFailureResult( + status: number, + message: string, + errorType: string | null +) { + if (!errorType) return createErrorResult(status, message); + const body = buildErrorBody( + status, + message, + undefined, + { type: errorType, code: errorType } + ); + return { + success: false as const, + status, + error: body.error.message, + response: new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }), + }; +} diff --git a/open-sse/handlers/moderations.ts b/open-sse/handlers/moderations.ts index c153e10eea..fa5cb72a16 100644 --- a/open-sse/handlers/moderations.ts +++ b/open-sse/handlers/moderations.ts @@ -6,7 +6,8 @@ import { CORS_HEADERS } from "../utils/cors.ts"; */ import { getModerationProvider, parseModerationModel } from "../config/moderationRegistry.ts"; -import { errorResponse, redactSensitiveErrorText } from "../utils/error.ts"; +import { errorResponse, sanitizeErrorMessage } from "../utils/error.ts"; +import { buildSanitizedUpstreamErrorResponse } from "../utils/upstreamErrorResponse.ts"; import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta"; import { generateRequestId } from "@/shared/utils/requestId"; @@ -57,14 +58,11 @@ export async function handleModeration({ body, credentials }) { if (!res.ok) { const errText = await res.text(); - // secret-leak hardening: redact any credential the upstream echoed back - // before relaying the error body to the client (structure-preserving). - return new Response(redactSensitiveErrorText(errText), { + return buildSanitizedUpstreamErrorResponse({ status: res.status, - headers: { - "Content-Type": "application/json", - ...CORS_HEADERS, - }, + rawBody: errText, + fallbackMessage: `Moderation provider returned HTTP ${res.status}`, + headers: CORS_HEADERS, }); } @@ -79,6 +77,10 @@ export async function handleModeration({ body, credentials }) { }); return new Response(JSON.stringify(data), { status: 200, headers }); } catch (err) { - return errorResponse(500, `Moderation request failed: ${err.message}`); + const safeDetail = + sanitizeErrorMessage(err) + .replace(/^[A-Za-z]*Error:\s*/, "") + .trim() || "unknown upstream failure"; + return errorResponse(500, `Moderation request failed: ${safeDetail}`); } } diff --git a/open-sse/handlers/ocr.ts b/open-sse/handlers/ocr.ts index f5d52f0106..16538e08cc 100644 --- a/open-sse/handlers/ocr.ts +++ b/open-sse/handlers/ocr.ts @@ -11,7 +11,8 @@ import { parseOcrModel, OCR_PROVIDERS, } from "../config/ocrRegistry.ts"; -import { errorResponse, redactSensitiveErrorText } from "../utils/error.ts"; +import { errorResponse, sanitizeErrorMessage } from "../utils/error.ts"; +import { buildSanitizedUpstreamErrorResponse } from "../utils/upstreamErrorResponse.ts"; import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta"; import { generateRequestId } from "@/shared/utils/requestId"; import { @@ -151,15 +152,11 @@ export async function handleOcr({ if (!res.ok) { const errText = await res.text(); - // secret-leak hardening: an upstream OCR provider can echo the offending - // request (Authorization header / api key) inside its error text. Redact - // secret patterns (structure-preserving) before relaying to the client. - return new Response(redactSensitiveErrorText(errText), { + return buildSanitizedUpstreamErrorResponse({ status: res.status, - headers: { - "Content-Type": "application/json", - ...CORS_HEADERS, - }, + rawBody: errText, + fallbackMessage: `OCR provider returned HTTP ${res.status}`, + headers: CORS_HEADERS, }); } @@ -184,7 +181,8 @@ export async function handleOcr({ }); return new Response(JSON.stringify(parsed), { status: 200, headers }); } catch (err) { - console.error("[OCR]", err); + const safeErrorMessage = sanitizeErrorMessage(err).trim() || "OCR request failed"; + console.error("[OCR]", safeErrorMessage); return errorResponse(500, "OCR request failed"); } } diff --git a/open-sse/handlers/search.ts b/open-sse/handlers/search.ts index b1397f3c5d..e68603e459 100644 --- a/open-sse/handlers/search.ts +++ b/open-sse/handlers/search.ts @@ -20,6 +20,9 @@ import { randomUUID } from "crypto"; * } */ +export { resolveSearchBaseUrl, SearchBaseUrlOverrideError } from "./search/baseUrl.ts"; +import { resolveSearchBaseUrl } from "./search/baseUrl.ts"; + import { getSearchProvider, isUnconfiguredLoopbackSearchProvider, @@ -36,7 +39,6 @@ import * as anysearchSearch from "./search/anysearchSearch.ts"; import { freeWebSearch } from "../services/freeWebSearch.ts"; import { saveCallLog } from "@/lib/usageDb"; import { safeOutboundFetch } from "@/shared/network/safeOutboundFetch"; -import { parseAndValidateNonMetadataUrl } from "@/shared/network/outboundUrlGuard"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { z } from "zod"; @@ -319,25 +321,6 @@ function getProviderSettingString( return undefined; } -export function resolveSearchBaseUrl( - config: SearchProviderConfig, - params: SearchRequestParams -): string { - const override = getProviderSettingString(params, "baseUrl"); - if (override) { - // GHSA-j7j4-g9qc-q69c: the override is client-controlled (provider_options / - // providerSpecificData) and flows into a plain fetch() sink — validate it - // before any builder uses it as the server-side fetch target. Mode is - // block-metadata (NOT public-only): the primary searxng use case is a - // self-hosted instance on loopback/LAN, so private hosts keep working, - // while cloud-metadata endpoints (IMDS credential theft) are rejected. - // The catalog's own config.baseUrl is operator config and stays untouched. - parseAndValidateNonMetadataUrl(override); - return override.replace(/\/+$/, ""); - } - return config.baseUrl.replace(/\/+$/, ""); -} - function toSearchPageNumber(offset: number | undefined, maxResults: number): number | undefined { if (typeof offset !== "number" || offset <= 0 || maxResults <= 0) return undefined; return Math.floor(offset / maxResults) + 1; diff --git a/open-sse/handlers/search/baseUrl.ts b/open-sse/handlers/search/baseUrl.ts new file mode 100644 index 0000000000..ba1e1fe6b0 --- /dev/null +++ b/open-sse/handlers/search/baseUrl.ts @@ -0,0 +1,66 @@ +/** + * Base-URL resolution for /v1/search — the trust decision, on its own. + * + * The two override sources are NOT equally trusted, and reading them through + * one call is what produced GHSA-3f8g-pfh9-j687: + * + * - `providerSpecificData` is the stored provider connection + * (`credentials?.providerSpecificData`) — OPERATOR config. Honored under + * block-metadata, so a self-hosted SearXNG on loopback/LAN keeps working + * while cloud metadata (IMDS credential theft) stays rejected (GHSA-j7j4). + * - `providerOptions` is `body.provider_options` — CALLER input. Honored only + * by a provider that is keyless AND opted in via + * `allowClientBaseUrlOverride`. A keyed builder attaches the OPERATOR's key + * to whatever host this resolves to (`key=`/`api_key=` in the query for + * google-pse/searchapi, `X-API-Key`/`Authorization` for + * you.com/linkup/nimble/ollama), so a caller-chosen host would collect it — + * and a block-metadata check does nothing about that, because the attacker + * simply names their own public host. + * + * Full coverage: tests/unit/search-baseurl-client-override-3f8g.test.ts. + */ + +import { parseAndValidateNonMetadataUrl } from "@/shared/network/outboundUrlGuard"; +import type { SearchProviderConfig } from "../../config/searchRegistry.ts"; + +interface BaseUrlParams { + providerOptions?: Record; + providerSpecificData?: Record; +} + +/** Read one string setting from a SINGLE source, so callers can distinguish trust. */ +function readSetting(source: Record | undefined, key: string): string | undefined { + const value = source?.[key]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +/** Refusal for a caller-supplied `provider_options.baseUrl` (GHSA-3f8g-pfh9-j687). */ +export class SearchBaseUrlOverrideError extends Error { + readonly code = "SEARCH_BASE_URL_OVERRIDE_REFUSED"; + constructor(providerId: string) { + super( + `provider_options.baseUrl is not accepted for search provider "${providerId}". ` + + `Set the base URL on the provider connection instead.` + ); + this.name = "SearchBaseUrlOverrideError"; + } +} + +export function resolveSearchBaseUrl(config: SearchProviderConfig, params: BaseUrlParams): string { + const operatorOverride = readSetting(params.providerSpecificData, "baseUrl"); + if (operatorOverride) { + parseAndValidateNonMetadataUrl(operatorOverride); + return operatorOverride.replace(/\/+$/, ""); + } + + const callerOverride = readSetting(params.providerOptions, "baseUrl"); + if (callerOverride) { + if (!config.allowClientBaseUrlOverride || config.authType === "apikey") { + throw new SearchBaseUrlOverrideError(config.id); + } + parseAndValidateNonMetadataUrl(callerOverride); + return callerOverride.replace(/\/+$/, ""); + } + + return config.baseUrl.replace(/\/+$/, ""); +} diff --git a/open-sse/mcp-server/errorMessage.ts b/open-sse/mcp-server/errorMessage.ts new file mode 100644 index 0000000000..f90c570166 --- /dev/null +++ b/open-sse/mcp-server/errorMessage.ts @@ -0,0 +1,13 @@ +import { sanitizeErrorMessage } from "../utils/error.ts"; + +export function toSafeMcpErrorMessage( + value: unknown, + fallback = "MCP tool execution failed" +): string { + try { + const raw = value instanceof Error ? value.message : value; + return sanitizeErrorMessage(raw) || fallback; + } catch { + return fallback; + } +} diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 73e3a387dc..4c30e608f9 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -93,7 +93,7 @@ import { import { getDbInstance, ensureDbInitialized } from "../../src/lib/db/core.ts"; import { normalizeQuotaResponse } from "../../src/shared/contracts/quota.ts"; import { resolveOmniRouteBaseUrl } from "../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; -import { sanitizeErrorMessage } from "../utils/error.ts"; +import { toSafeMcpErrorMessage } from "./errorMessage.ts"; import { mcpFetchTimeoutSignal } from "./fetchTimeout.ts"; import { getMcpModelsCatalog } from "./catalog.ts"; import { registerRadarCatalogTool } from "./radarCatalog.ts"; @@ -328,9 +328,7 @@ async function handleGetHealth() { .filter(({ settled }) => settled.status === "rejected") .map(({ source, settled }) => ({ source, - error: sanitizeErrorMessage( - settled.status === "rejected" ? (settled as PromiseRejectedResult).reason : undefined - ), + error: toSafeMcpErrorMessage((settled as PromiseRejectedResult).reason, ""), })); const result = { @@ -378,7 +376,7 @@ async function handleGetHealth() { await logToolCall("omniroute_get_health", {}, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_get_health", {}, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -420,7 +418,7 @@ async function handleListCombos(args: { includeMetrics?: boolean }) { await logToolCall("omniroute_list_combos", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_list_combos", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -435,7 +433,7 @@ async function handleGetComboMetrics(args: { comboId: string }) { await logToolCall("omniroute_get_combo_metrics", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_get_combo_metrics", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -451,7 +449,7 @@ async function handleSwitchCombo(args: { comboId: string; active: boolean }) { await logToolCall("omniroute_switch_combo", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_switch_combo", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -472,7 +470,7 @@ async function handleCreateCombo(args: { await logToolCall("omniroute_create_combo", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_create_combo", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -493,7 +491,7 @@ async function handleCheckQuota(args: { provider?: string; connectionId?: string await logToolCall("omniroute_check_quota", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_check_quota", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -562,7 +560,7 @@ async function handleRouteRequest(args: { ); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall( "omniroute_route_request", { model: args.model }, @@ -611,7 +609,7 @@ async function handleCostReport(args: { period?: string }) { await logToolCall("omniroute_cost_report", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_cost_report", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -631,7 +629,7 @@ async function handleListModelsCatalog(args: { provider?: string; capability?: s ); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_list_models_catalog", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -660,7 +658,7 @@ async function handleWebSearch(args: { await logToolCall("omniroute_web_search", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_web_search", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -686,7 +684,7 @@ async function handleXSearch(args: { await logToolCall("omniroute_x_search", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_x_search", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -726,7 +724,7 @@ async function handleWebFetch(args: { await logToolCall("omniroute_web_fetch", args, result, Date.now() - start, true); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err); await logToolCall("omniroute_web_fetch", args, null, Date.now() - start, false, msg); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } @@ -1182,7 +1180,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Memory tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1209,7 +1207,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Skill tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1234,7 +1232,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Agent skill tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }) @@ -1259,7 +1257,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "GitHub skill tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1286,7 +1284,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Plugin tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1313,7 +1311,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Compression tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1350,7 +1348,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }], }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Pool tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1378,7 +1376,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Gamification tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1405,7 +1403,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Notion tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1432,8 +1430,9 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (error) { + const msg = toSafeMcpErrorMessage(error, "Local corpus tool execution failed"); return { - content: [{ type: "text" as const, text: `Error: ${sanitizeErrorMessage(error)}` }], + content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true, }; } @@ -1461,7 +1460,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { const result = await toolDef.handler(parsedArgs, extra); return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Obsidian tool execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; } }, @@ -1502,7 +1501,7 @@ export function createMcpServer(options?: CreateMcpServerOptions): McpServer { ], }; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = toSafeMcpErrorMessage(err, "Skill execution failed"); return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true, diff --git a/open-sse/services/__tests__/tierResolver.test.ts b/open-sse/services/__tests__/tierResolver.test.ts index fac0b24404..132b6e8561 100644 --- a/open-sse/services/__tests__/tierResolver.test.ts +++ b/open-sse/services/__tests__/tierResolver.test.ts @@ -60,10 +60,10 @@ describe("TierResolver", () => { expect(result.hasFreeTier).toBe(true); }); - it("classifies Cerebras as free", () => { + it("classifies Cerebras as not free after the no-card trial ended (#11773)", () => { const result = classifyTier("cerebras", "llama-3.1-70b"); - expect(result.tier).toBe(PROVIDER_TIER.FREE); - expect(result.hasFreeTier).toBe(true); + expect(result.tier).not.toBe(PROVIDER_TIER.FREE); + expect(result.hasFreeTier).toBe(false); }); it("classifies Groq as free", () => { @@ -228,7 +228,6 @@ describe("TierResolver", () => { "longcat", "cloudflare-ai", "nvidia-nim", - "cerebras", "groq", ]) { expect(LEGACY_FREE_PROVIDERS.includes(id), `expected ${id} in LEGACY_FREE_PROVIDERS`).toBe( diff --git a/open-sse/services/tierConfig.ts b/open-sse/services/tierConfig.ts index 2119029e13..b02234a6b0 100644 --- a/open-sse/services/tierConfig.ts +++ b/open-sse/services/tierConfig.ts @@ -52,7 +52,6 @@ export const LEGACY_FREE_PROVIDERS: readonly string[] = [ "longcat", "cloudflare-ai", "nvidia-nim", - "cerebras", "groq", ]; diff --git a/open-sse/services/tierDefaults.json b/open-sse/services/tierDefaults.json index 5e1e4a23cf..1e74212f3b 100644 --- a/open-sse/services/tierDefaults.json +++ b/open-sse/services/tierDefaults.json @@ -19,7 +19,6 @@ "longcat", "cloudflare-ai", "nvidia-nim", - "cerebras", "groq" ] } diff --git a/open-sse/translator/response/openai-responses.ts b/open-sse/translator/response/openai-responses.ts index a2244f01f5..43453c0b28 100644 --- a/open-sse/translator/response/openai-responses.ts +++ b/open-sse/translator/response/openai-responses.ts @@ -5,6 +5,7 @@ import { register } from "../registry.ts"; import { FORMATS } from "../formats.ts"; import { appendToolCallArgumentDelta } from "../../utils/toolCallArguments.ts"; +import { projectCompletedStreamError } from "../../utils/streamErrorFormat.ts"; import { fallbackToolCallId } from "../helpers/toolCallHelper.ts"; import { shouldParseTextualReasoningTags } from "../../handlers/responseSanitizer.ts"; import { getReadableReasoningValue } from "../../utils/reasoningFields.ts"; @@ -746,6 +747,7 @@ function sendCompleted(state, emit) { // translator or the OpenAI-Responses translator itself when the upstream // SSE stream emits a JSON error object after partial content. const upstreamErr = state.upstreamError; + const publicUpstreamError = projectCompletedStreamError(upstreamErr); const response: Record = { id: state.responseId, @@ -753,9 +755,7 @@ function sendCompleted(state, emit) { created_at: state.created, status: upstreamErr ? "failed" : "completed", background: false, - error: upstreamErr - ? { code: String(upstreamErr.status ?? ""), message: upstreamErr.message ?? "" } - : null, + error: publicUpstreamError, output, }; diff --git a/open-sse/utils/codexPublicError.ts b/open-sse/utils/codexPublicError.ts new file mode 100644 index 0000000000..6a8c92209c --- /dev/null +++ b/open-sse/utils/codexPublicError.ts @@ -0,0 +1,110 @@ +import { sanitizeErrorMessage } from "./error.ts"; + +export const CODEX_PUBLIC_ERROR_MESSAGE = sanitizeErrorMessage("Codex provider request failed"); + +export interface CodexPublicError { + message: string; + type: string; + code: string; +} + +interface CodexPublicErrorInput { + status?: number | null; + type?: unknown; + code?: unknown; +} + +interface CodexPublicErrorRule { + type: string; + allowsStatus: (status: number) => boolean; +} + +const exactStatuses = + (...statuses: number[]) => + (status: number): boolean => + statuses.includes(status); + +const CODEX_PUBLIC_ERROR_RULES = new Map([ + ["browser_stream_inconsistent", { type: "server_error", allowsStatus: exactStatuses(502) }], + ["chatgpt_session_expired", { type: "authentication_error", allowsStatus: exactStatuses(401) }], + ["chatgpt_submission_ambiguous", { type: "server_error", allowsStatus: exactStatuses(502) }], + ["chatgpt_submitted_turn_failed", { type: "server_error", allowsStatus: exactStatuses(502) }], + ["chatgpt_subscription_unavailable", { type: "server_error", allowsStatus: exactStatuses(503) }], + ["client_cancelled", { type: "invalid_request_error", allowsStatus: exactStatuses(499) }], + ["client_closed_request", { type: "invalid_request_error", allowsStatus: exactStatuses(499) }], + ["codex_app_server_turn_failed", { type: "provider_error", allowsStatus: exactStatuses(502) }], + [ + "compaction_control_unavailable", + { type: "invalid_request_error", allowsStatus: exactStatuses(409) }, + ], + [ + "compaction_handoff_failed", + { type: "invalid_request_error", allowsStatus: exactStatuses(409) }, + ], + [ + "compaction_source_unavailable", + { type: "invalid_request_error", allowsStatus: exactStatuses(409) }, + ], + ["connector_not_found", { type: "connector_error", allowsStatus: exactStatuses(424) }], + [ + "context_length_exceeded", + { type: "invalid_request_error", allowsStatus: exactStatuses(400, 413) }, + ], + ["insufficient_quota", { type: "insufficient_quota", allowsStatus: exactStatuses(429) }], + ["invalid_api_key", { type: "authentication_error", allowsStatus: exactStatuses(401) }], + ["invalid_output_schema", { type: "invalid_request_error", allowsStatus: exactStatuses(400) }], + ["invalid_request_error", { type: "invalid_request_error", allowsStatus: exactStatuses(400) }], + ["multipart_protocol_violation", { type: "server_error", allowsStatus: exactStatuses(502) }], + ["origin_rejected", { type: "invalid_request_error", allowsStatus: exactStatuses(403) }], + ["permission_denied", { type: "permission_error", allowsStatus: exactStatuses(403) }], + ["prompt_attachment_integrity", { type: "server_error", allowsStatus: exactStatuses(502) }], + ["rate_limit_exceeded", { type: "rate_limit_error", allowsStatus: exactStatuses(429) }], + ["server_is_overloaded", { type: "server_error", allowsStatus: exactStatuses(503) }], + [ + "structured_output_validation_failed", + { type: "server_error", allowsStatus: exactStatuses(502) }, + ], + ["subscription_required", { type: "permission_error", allowsStatus: exactStatuses(403) }], + [ + "upstream_server_error", + { + type: "server_error", + allowsStatus: (status) => status >= 500 && status <= 599 && status !== 503, + }, + ], + [ + "upstream_websocket_connect_failed", + { type: "provider_error", allowsStatus: exactStatuses(502) }, + ], + ["upstream_websocket_error", { type: "provider_error", allowsStatus: exactStatuses(502) }], + ["usage_limit_reached", { type: "rate_limit_error", allowsStatus: exactStatuses(429) }], +]); + +function defaultPublicClassification(status: number): Pick { + if (status === 429) return { type: "rate_limit_error", code: "rate_limit_exceeded" }; + if (status === 401) return { type: "authentication_error", code: "invalid_api_key" }; + if (status === 403) return { type: "permission_error", code: "permission_denied" }; + if (status === 499) return { type: "invalid_request_error", code: "client_closed_request" }; + if (status === 503) return { type: "server_error", code: "server_is_overloaded" }; + if (status >= 500) return { type: "server_error", code: "upstream_server_error" }; + return { type: "invalid_request_error", code: "invalid_request_error" }; +} + +/** + * Project an internally classified Codex failure onto its public Responses contract. + * + * Upstream message, code, and type fields are untrusted. The public message is fixed, + * while code/type retain only closed, protocol-level identifiers already produced by + * OmniRoute. Everything else falls back to the HTTP status classification. + */ +export function projectCodexPublicError(input: CodexPublicErrorInput): CodexPublicError { + const status = + typeof input.status === "number" && Number.isInteger(input.status) ? input.status : 502; + const fallback = defaultPublicClassification(status); + const rule = + typeof input.code === "string" ? CODEX_PUBLIC_ERROR_RULES.get(input.code) : undefined; + if (!rule || !rule.allowsStatus(status)) { + return { message: CODEX_PUBLIC_ERROR_MESSAGE, ...fallback }; + } + return { message: CODEX_PUBLIC_ERROR_MESSAGE, type: rule.type, code: input.code as string }; +} diff --git a/open-sse/utils/credentialPatterns.ts b/open-sse/utils/credentialPatterns.ts new file mode 100644 index 0000000000..02784a4ae5 --- /dev/null +++ b/open-sse/utils/credentialPatterns.ts @@ -0,0 +1,79 @@ +/** Pure credential signatures shared by guardrails and public error sanitization. */ +export interface CredentialPattern { + name: string; + regex: RegExp; + replacement: string; +} + +export const CREDENTIAL_PATTERNS: CredentialPattern[] = [ + { name: "openai_proj", regex: /sk-proj-[A-Za-z0-9_-]{20,}/g, replacement: "[REDACTED:openai]" }, + { name: "openai", regex: /\bsk-[A-Za-z0-9]{48}\b/g, replacement: "[REDACTED:openai]" }, + { + name: "anthropic", + regex: /sk-ant-api[0-9]?-[A-Za-z0-9_-]{20,}/g, + replacement: "[REDACTED:anthropic]", + }, + { + name: "anthropic_alt", + regex: /sk-ant-[A-Za-z0-9_-]{20,}/g, + replacement: "[REDACTED:anthropic]", + }, + { name: "google", regex: /AIza[0-9A-Za-z_-]{35}/g, replacement: "[REDACTED:google]" }, + { name: "huggingface", regex: /hf_[A-Za-z0-9]{34}/g, replacement: "[REDACTED:hf]" }, + { name: "replicate", regex: /r8_[A-Za-z0-9]{37}/g, replacement: "[REDACTED:replicate]" }, + { name: "github", regex: /gh[pousr]_[A-Za-z0-9]{36,}/g, replacement: "[REDACTED:github]" }, + { name: "slack", regex: /xox[bpoa]-[A-Za-z0-9-]{10,}/g, replacement: "[REDACTED:slack]" }, + { name: "linear", regex: /lin_api_[A-Za-z0-9]{40}/g, replacement: "[REDACTED:linear]" }, + { name: "notion", regex: /secret_[A-Za-z0-9]{43}/g, replacement: "[REDACTED:notion]" }, + { name: "npm", regex: /npm_[A-Za-z0-9]{36}/g, replacement: "[REDACTED:npm]" }, + { + name: "postman", + regex: /PMAK-[a-f0-9]{8}-[a-f0-9]{32}/g, + replacement: "[REDACTED:postman]", + }, + { + name: "discord", + regex: /\b[MN][A-Za-z0-9]{23}\.[A-Za-z0-9]{6}\.[A-Za-z0-9]{27}\b/g, + replacement: "[REDACTED:discord]", + }, + { + name: "stripe", + regex: /(?:sk|rk)_(?:live|test)_[0-9a-zA-Z]{24,}/g, + replacement: "[REDACTED:stripe]", + }, + { + name: "square", + regex: /sq0(?:atp-[0-9A-Za-z_-]{22}|csp-[0-9A-Za-z_-]{43})/g, + replacement: "[REDACTED:square]", + }, + { name: "aws_access_key", regex: /AKIA[0-9A-Z]{16}/g, replacement: "[REDACTED:aws]" }, + { name: "twilio", regex: /\bSK[0-9a-fA-F]{32}\b/g, replacement: "[REDACTED:twilio]" }, + { + name: "sendgrid", + regex: /SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}/g, + replacement: "[REDACTED:sendgrid]", + }, + { name: "mailgun", regex: /key-[a-f0-9]{32}/g, replacement: "[REDACTED:mailgun]" }, + { + name: "private_key", + regex: + /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/g, + replacement: "[REDACTED:private_key]", + }, + { + name: "jwt", + regex: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, + replacement: "[REDACTED:jwt]", + }, + { + name: "connection_string", + regex: /(?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|amqp):\/\/[^:/@\s"']+:[^:/@\s"']+@/g, + replacement: "[REDACTED:connection_string]", + }, + { + name: "auth_header", + regex: + /((?:["\x27]?(?:Authorization|x-api-key|api-key|apikey)["\x27]?\s*[:=]\s*["\x27]?)(?:(?:Bearer|Basic|Token)\s+)?)[A-Za-z0-9._~+/=-]{10,}/gi, + replacement: "$1[REDACTED:auth_header]", + }, +]; diff --git a/open-sse/utils/error.ts b/open-sse/utils/error.ts index 4fafd05b1a..5d7357eb1c 100644 --- a/open-sse/utils/error.ts +++ b/open-sse/utils/error.ts @@ -1,15 +1,18 @@ import { CORS_HEADERS } from "./cors.ts"; import { unwrapClinepassEnvelope } from "./clinepassEnvelope.ts"; +import { + redactSensitiveErrorText, + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "./errorSanitization.ts"; import { getDefaultErrorMessage, getErrorInfo } from "../config/errorConfig.ts"; import { normalizePayloadForLog } from "@/lib/logPayloads"; import type { ModelCooldownErrorPayload } from "@/types"; import { buildPassthroughErrorResponse } from "./upstreamErrorPassthrough.ts"; -/** - * Sanitize an error message to prevent stack trace exposure in API responses. - * Strips stack traces, file paths, and absolute Windows/POSIX paths from - * error messages before they reach the client. - */ +export { redactSensitiveErrorText, sanitizeErrorMessage, sanitizeUpstreamDetails }; + +/** Client-visible error shape; dynamic fields are projected through canonical boundaries. */ interface ErrorResponseBody { error: { message: string; @@ -20,91 +23,6 @@ interface ErrorResponseBody { upstream_details?: Record | null; // sanitized upstream provider body } -// Length cap protects against pathological inputs even before tokenization. -const MAX_ERROR_LEN = 4096; -const SOURCE_EXT = ["ts", "tsx", "js", "jsx", "mjs", "cjs"] as const; - -function looksLikeAbsolutePath(tok: string): boolean { - // POSIX: "/<...>.ts" (optionally followed by :line[:col]). - // Windows: "C:\<...>.ts" or "C:/<...>.ts". - if (tok.length < 4 || tok.length > 2048) return false; - const isPosix = tok.charCodeAt(0) === 0x2f; // '/' - const isWindows = tok.length > 2 && tok.charCodeAt(1) === 0x3a && /[A-Za-z]/.test(tok[0]); - if (!isPosix && !isWindows) return false; - const dot = tok.lastIndexOf("."); - if (dot <= 0 || dot === tok.length - 1) return false; - const ext = tok - .slice(dot + 1) - .split(":", 1)[0] - .toLowerCase(); - return (SOURCE_EXT as readonly string[]).includes(ext); -} - -export function redactSensitiveErrorText(value: string): string { - return value - .replace(/data:[^,\s]+;base64,[A-Za-z0-9+/=_-]+/gi, "[REDACTED_DATA_URL]") - .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "$1 [REDACTED]") - .replace( - /(["']?(?:api[_-]?key|access[_-]?token|authorization|cookie|secret)["']?\s*[:=]\s*["'])[^"']*(["'])/gi, - "$1[REDACTED]$2" - ) - .replace( - /(["']?(?:api[_-]?key|access[_-]?token|authorization|cookie|secret)["']?\s*[:=]\s*)[^"',\s}]+/gi, - "$1[REDACTED]" - ); -} - -/** - * Strip stack-trace tail and absolute source paths from error messages. - * - * Implemented via simple whitespace tokenization (linear time) instead of a - * single complex regex, so CodeQL `js/polynomial-redos` stays clean even when - * the runtime error message is attacker-controlled. - */ -export function sanitizeErrorMessage(message: unknown): string { - let str = typeof message === "string" ? message : String(message ?? ""); - if (str.length > MAX_ERROR_LEN) str = str.slice(0, MAX_ERROR_LEN); - const nl = str.indexOf("\n"); - const firstLine = nl >= 0 ? str.slice(0, nl) : str; - // Preserve original whitespace by splitting on captured separator. - const parts = firstLine.split(/(\s+)/); - for (let i = 0; i < parts.length; i++) { - if (looksLikeAbsolutePath(parts[i])) parts[i] = ""; - } - return redactSensitiveErrorText(parts.join("")); -} - -const BLOCKED_KEYS = - /stack|trace|path|file|cwd|dir|password|secret|token|key|authorization|cookie/i; -const MAX_DEPTH = 4; - -/** - * Recursively sanitize an arbitrary JSON value from an upstream provider body. - * - Strings: run through sanitizeErrorMessage (strips stacks + absolute paths). - * - Keys matching BLOCKED_KEYS are dropped (credential/path guards). - * - Depth capped at MAX_DEPTH to prevent pathological nesting. - * - Arrays capped at 32 elements. - * - Returns null for null/undefined/non-JSON-serializable values. - */ -export function sanitizeUpstreamDetails(value: unknown, depth = 0): unknown { - if (depth > MAX_DEPTH) return "[truncated]"; - if (value === null || value === undefined) return null; - if (typeof value === "string") return sanitizeErrorMessage(value); - if (typeof value === "number" || typeof value === "boolean") return value; - if (Array.isArray(value)) { - return value.slice(0, 32).map((v) => sanitizeUpstreamDetails(v, depth + 1)); - } - if (typeof value === "object") { - const out: Record = {}; - for (const [k, v] of Object.entries(value as Record)) { - if (BLOCKED_KEYS.test(k)) continue; - out[k] = sanitizeUpstreamDetails(v, depth + 1); - } - return out; - } - return null; -} - /** Optional caller classification; when set, wins over status-derived defaults. */ export type ErrorBodyClassification = { type?: string; @@ -112,6 +30,279 @@ export type ErrorBodyClassification = { reason?: string; }; +const PUBLIC_ERROR_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const SAFE_PUBLIC_ERROR_IDENTIFIERS = new Set([ + "abort", + "aborted", + "account_semaphore_capacity", + "acp_cancelled", + "acp_early_exit", + "acp_error", + "acp_output_too_large", + "acp_session_mismatch", + "acp_timeout", + "admission_aborted", + "admission_deadline", + "admission_lane_evicted", + "admission_oversized", + "admission_queue_full", + "admission_shutdown", + "admission_unavailable", + "all_accounts_inactive", + "all_targets_skipped", + "antigravity_pre_response_timeout", + "api_error", + "authentication_error", + "authentication_required", + "auth_error", + "bad_gateway", + "bad_request", + "bedrock_stream_error", + "billing_error", + "blackbox_auth_required", + "blackbox_rate_limit", + "blackbox_subscription_required", + "body_exceeds_budget", + "browser_stream_inconsistent", + "capability_mismatch", + "cf_mitigated_challenge", + "chat_admission_busy", + "chat_history_too_large", + "chatgpt_web_codex_error", + "chatgpt_web_codex_turn_failed", + "chatgpt_session_expired", + "chatgpt_submission_ambiguous", + "chatgpt_submitted_turn_failed", + "chatgpt_subscription_unavailable", + "client_cancelled", + "client_closed_request", + "client_disconnected", + "cli_not_found", + "cloudflare_challenge", + "cloudflare_or_bot", + "codex_app_server_unconfigured", + "codex_app_server_turn_failed", + "combo_target_timeout", + "combo_timeout", + "compaction_control_unavailable", + "compaction_handoff_failed", + "connector_error", + "connector_not_found", + "connection_error", + "context_length_exceeded", + "context_window", + "chipotle_error", + "devin_agentic_error", + "devin_cli_error", + "devin_desktop_error", + "devin_internal_tool_execution", + "duplicate_tool_use_id", + "direct_response_start_timeout", + "eai_again", + "econnrefused", + "econnreset", + "empty_acp_output", + "empty_content", + "empty_messages", + "empty_response", + "executor_contract_violation", + "error", + "etimedout", + "executor_error", + "feature_disabled", + "gateway_timeout", + "gemini_tpm_exhausted", + "gcp_project_required", + "grok_error", + "insufficient_quota", + "incompatible_reasoning_effort", + "internal_server_error", + "invalid_acp_frame", + "invalid_acp_upstream", + "invalid_api_key", + "invalid_kiro_tool_call", + "invalid_request", + "invalid_request_error", + "invalid_previous_response_binding", + "invalid_tool_arguments", + "invalid_tool_choice", + "invalid_tool_json", + "invalid_tool_name", + "invalid_tools", + "invalid_trailer", + "lease_action_invalid", + "lease_api_key_invalid", + "lease_authentication_required", + "lease_authorization_mismatch", + "lease_capacity_unavailable", + "lease_connection_mismatch", + "lease_content_type_required", + "lease_context_invalid", + "lease_context_required", + "lease_error", + "lease_fence_stale", + "lease_key_configuration_invalid", + "lease_key_policy_invalid", + "lease_model_invalid", + "lease_no_eligible_connection", + "lmarena_error", + "lease_required", + "lease_scope_required", + "lease_service_unavailable", + "lease_eligibility_unavailable", + "lease_unsupported_route", + "lease_unsupported_transport", + "message_limit", + "missing_credits", + "meta_ai_empty_response", + "meta_ai_mode_switch_failed", + "meta_ai_warmup_failed", + "meta_ai_ws_error", + "missing_tool_name", + "missing_tool_use_id", + "mixed_tool_narrative", + "missing_authorization", + "missing_cookie", + "missing_project_id", + "missing_credentials", + "missing_session_id", + "model_not_found", + "model_not_supported", + "model_shutdown", + "multipart_protocol_violation", + "multiple_tool_requests", + "native_codex_pinned_model_unavailable", + "network_error", + "no_free_eligible_connection", + "not_found", + "oauth_missing_project_id", + "orphan_tool_result", + "payload_too_large", + "payment_required", + "permission_error", + "premium_model_requires_key", + "prompt_attachment_integrity", + "provider_error", + "provider_retired", + "provider_unavailable", + "pplx_error", + "proxy_unavailable", + "proxy_family_unavailable", + "proxy_request_failed", + "proxy_unreachable", + "quota_exhausted", + "quota_not_allocated", + "quota_only", + "rate_limit_error", + "rate_limit_execution_timeout", + "rate_limit_exceeded", + "rate_limit_queue_full", + "rate_limit_queue_timeout", + "rate_limit_queue_wedged", + "rate_limit_longer_reached", + "rate_limit_reached", + "rate_limited", + "reached_limit", + "relay_timeout", + "resource_pressure", + "resource_exhausted", + "request_failed", + "risk_session_stale", + "server_error", + "semaphore_queue_full", + "semaphore_timeout", + "service_unavailable", + "service_not_running", + "session_expired", + "session_pool_exhausted", + "spawn_failed", + "stream_error", + "stream_disconnected", + "stream_early_eof", + "stream_idle_timeout", + "stream_pipeline_error", + "stream_readiness_timeout", + "stream_terminated", + "stream_timeout", + "storage_encryption_stale", + "structure_limit", + "structured_output", + "structured_output_validation_failed", + "timeout_error", + "timeout", + "token_limit_exceeded", + "token_required", + "tls_client_unavailable", + "tls_circuit_open", + "tls_fingerprint_failed", + "tls_session_capacity", + "tool_calling_not_supported", + "tools", + "undeclared_historical_tool", + "und_err_body_timeout", + "und_err_connect_timeout", + "und_err_headers_timeout", + "und_err_socket", + "unexpected_acp_response", + "unexecuted_tool_intent", + "unavailable", + "unknown_devin_model", + "unknown_tool", + "unverified_codex_client", + "unsafe_devin_home", + "unsupported_acp_version", + "unsupported_content_block", + "unsupported_control_for_provider", + "unsupported_endpoint", + "unsupported_image_block", + "unsupported_role", + "unsupported_system_block", + "upstream_error", + "upstream_access_denied", + "upstream_auth_error", + "upstream_empty_response", + "upstream_response_failed", + "upstream_response_error", + "upstream_server_error", + "upstream_protocol_error", + "upstream_timeout", + "upstream_websocket_connect_failed", + "upstream_websocket_error", + "usage_limit_reached", + "unsupported_feature", + "unsupported_runtime", + "video_artifact_content_type_invalid", + "video_artifact_download_failed", + "video_artifact_not_ready", + "video_artifact_signature_invalid", + "video_artifact_too_large", + "video_artifact_unavailable", + "video_artifact_url_blocked", + "video_artifact_url_invalid", + "vision", + "claude_web_protocol_error", + "wreq_unavailable", +]); + +function isSafePublicErrorIdentifier(value: string): boolean { + if (!PUBLIC_ERROR_IDENTIFIER.test(value)) return false; + if (/^[1-5]\d{2}$/.test(value)) return true; + if (/^HTTP_[1-5]\d{2}$/i.test(value)) return true; + return SAFE_PUBLIC_ERROR_IDENTIFIERS.has(value.toLowerCase()); +} + +/** Project an internal classification onto the bounded client-visible identifier vocabulary. */ +export function projectPublicErrorIdentifier(value: unknown, fallback: unknown): string { + const safeFallback = + fallback === "" + ? "" + : typeof fallback === "string" && isSafePublicErrorIdentifier(fallback) + ? fallback + : "error"; + if (typeof value !== "string") return safeFallback; + return isSafePublicErrorIdentifier(value) ? value : safeFallback; +} + /** * Build OpenAI-compatible error response body. Message is always sanitized * so callers do not need to remember to strip stack traces themselves. @@ -128,13 +319,17 @@ export function buildErrorBody( ): ErrorResponseBody { const errorInfo = getErrorInfo(statusCode); const safeMessage = sanitizeErrorMessage(message) || getDefaultErrorMessage(statusCode); + const safeReason = + typeof classification?.reason === "string" && isSafePublicErrorIdentifier(classification.reason) + ? classification.reason + : undefined; const body: ErrorResponseBody = { error: { message: safeMessage, - type: classification?.type ?? errorInfo.type, - code: classification?.code ?? errorInfo.code, - reason: classification?.reason, + type: projectPublicErrorIdentifier(classification?.type, errorInfo.type), + code: projectPublicErrorIdentifier(classification?.code, errorInfo.code), + reason: safeReason, }, }; @@ -183,7 +378,7 @@ export interface ComboRecoveryHint { action: ComboRecoveryAction; /** Seconds the client should wait before retrying. Only meaningful when action="wait". */ retry_after_seconds?: number; - /** Human-readable next step — included verbatim in the error body for non-MCP clients. */ + /** Human-readable next step — sanitized and length-capped for non-MCP clients. */ next_step: string; } @@ -203,21 +398,36 @@ export interface ComboDiagnostics { } function clampDiagStr(v: unknown, max = 128): string { - return typeof v === "string" ? v.slice(0, max).replace(/[\r\n]+/g, " ") : ""; + return typeof v === "string" ? sanitizeErrorMessage(v).slice(0, max) : ""; +} + +const RECOVERY_ROUTE_PLACEHOLDERS = [ + ["/dashboard/providers", "OMNIROUTE_SAFE_DASHBOARD_PROVIDERS_ROUTE"], +] as const; + +function clampRecoveryStr(value: unknown, max: number): string { + if (typeof value !== "string") return ""; + let projected = value; + for (const [route, placeholder] of RECOVERY_ROUTE_PLACEHOLDERS) { + projected = projected.replaceAll(route, placeholder); + } + projected = sanitizeErrorMessage(projected); + for (const [route, placeholder] of RECOVERY_ROUTE_PLACEHOLDERS) { + projected = projected.replaceAll(placeholder, route); + } + return projected.slice(0, max); } /** - * HTTP header values must be Latin1/ByteString (undici throws a TypeError - * otherwise — see #6612). Replace any codepoint outside the Latin1 range - * (0-255) with "?" so header construction never throws. Only used for the - * literal header value; the JSON body keeps the original, unsanitized - * readable text via `sanitizeComboDiagnostics`. + * HTTP header values must exclude controls and remain ByteString-compatible + * (undici throws a TypeError otherwise — see #6612). Replace every codepoint + * outside printable ASCII with "?" so header construction never throws. */ function toHeaderSafeAscii(v: string): string { let out = ""; for (let i = 0; i < v.length; i++) { const code = v.charCodeAt(i); - out += code > 255 ? "?" : v[i]; + out += code < 0x20 || code > 0x7e ? "?" : v[i]; } return out; } @@ -242,7 +452,7 @@ export function sanitizeRecoveryHint( if (!action || !RECOVERY_ACTIONS.has(action)) return undefined; // Reject empty OR whitespace-only next_step — the value must render usefully as a // header and as a body field. A whitespace-only string would print as a blank hint. - const next_step = clampDiagStr(r.next_step, 200).trim(); + const next_step = clampRecoveryStr(r.next_step, 200).trim(); if (!next_step) return undefined; const hint: ComboRecoveryHint = { action, next_step }; if (typeof r.retry_after_seconds === "number" && Number.isFinite(r.retry_after_seconds)) { @@ -293,12 +503,10 @@ export function errorResponseWithComboDiagnostics( opts: { code?: string; type?: string } = {} ): Response { const safe = sanitizeComboDiagnostics(diagnostics); - const body = buildErrorBody(statusCode, message) as ErrorResponseBody & { + const body = buildErrorBody(statusCode, message, undefined, opts) as ErrorResponseBody & { diagnostics?: ComboDiagnostics; recovery_hint?: ComboRecoveryHint; }; - if (opts.code) body.error.code = opts.code; - if (opts.type) body.error.type = opts.type; body.diagnostics = safe; if (safe.recovery) body.recovery_hint = safe.recovery; const excludedHeader = toHeaderSafeAscii( @@ -399,6 +607,29 @@ function normalizeRetryAfterSeconds(retryAfter?: string | number | Date | null): return 1; } +const MAX_PUBLIC_CONTEXT_LABEL_LENGTH = 256; + +function projectPublicContextLabel(value: unknown): string | null { + if (typeof value !== "string") return null; + const label = value.trim(); + if ( + label.length === 0 || + label.length > MAX_PUBLIC_CONTEXT_LABEL_LENGTH || + /[\u0000-\u001f\u007f]/.test(label) + ) { + return null; + } + return sanitizeErrorMessage(label) === label ? label : null; +} + +function projectPublicRetryTimestamp(value: unknown): string | null { + if (typeof value !== "string") return null; + const timestamp = value.trim(); + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(timestamp)) return null; + const parsed = Date.parse(timestamp); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === timestamp ? timestamp : null; +} + /** * Parse Antigravity error message to extract retry time * Example: "You have exhausted your capacity on this model. Your quota will reset after 2h7m23s." @@ -442,7 +673,7 @@ export function parseAntigravityRetryTime(message: unknown): number | null { * @returns {Promise<{statusCode: number, message: string, retryAfterMs: number|null, responseBody: unknown}>} */ export async function parseUpstreamError(response: Response, provider: string | null = null) { - let message: unknown = ""; + let message = ""; let retryAfterMs: number | null = null; let responseBody: unknown = null; let errorCode: unknown = undefined; @@ -462,9 +693,15 @@ export async function parseUpstreamError(response: Response, provider: string | // stack) — still routed through sanitizeErrorMessage/buildErrorBody by // every consumer below (Rule #12). const { error: clinepassEnvError } = unwrapClinepassEnvelope(json, provider); - message = clinepassEnvError + const extractedMessage = clinepassEnvError ? clinepassEnvError.message - : json.error?.message || json.message || json.error || text; + : json.error?.message || + json.message || + (typeof json.error === "string" ? json.error : null); + message = + typeof extractedMessage === "string" + ? extractedMessage + : `Upstream error: ${response.status}`; errorCode = json.error?.code || json.code; errorType = json.error?.type || json.type; } catch { @@ -475,7 +712,7 @@ export async function parseUpstreamError(response: Response, provider: string | responseBody = { _rawText: message }; } - const messageStr = typeof message === "string" ? message : JSON.stringify(message); + const messageStr = message; const retryAfterHeader = response.headers?.get?.("retry-after"); if (retryAfterHeader && !retryAfterMs) { @@ -545,13 +782,10 @@ export function createErrorResult( upstreamDetails?: unknown, opts?: { passthrough?: boolean } ) { - const body = buildErrorBody(statusCode, message, upstreamDetails); - if (errorCode) { - body.error.code = errorCode; - } - if (errorType) { - body.error.type = errorType; - } + const body = buildErrorBody(statusCode, message, upstreamDetails, { + code: errorCode, + type: errorType, + }); const result: { success: false; @@ -591,8 +825,8 @@ export function createErrorResult( result.retryAfterMs = retryAfterMs; } - // Opt-in relay of the verbatim upstream error body (Claude Code auto-recover - // contract — see upstreamErrorPassthrough.ts). Only swaps `result.response`; + // Opt-in relay of the recursively sanitized upstream JSON shape (Claude Code + // auto-recover contract — see upstreamErrorPassthrough.ts). Only swaps `result.response`; // `result.error`/`rawMessage`/`errorType`/`errorCode` stay untouched so // server-side classification (checkFallbackError, combo retry logic, etc.) // never sees a different value depending on this flag. @@ -625,7 +859,9 @@ export function unavailableResponse( retryAfterHuman?: string ) { const retryAfterSec = normalizeRetryAfterSeconds(retryAfter); - const msg = retryAfterHuman ? `${message} (${retryAfterHuman})` : message; + const safeMessage = sanitizeErrorMessage(message) || getDefaultErrorMessage(statusCode); + const safeRetryAfterHuman = retryAfterHuman ? sanitizeErrorMessage(retryAfterHuman) : ""; + const msg = safeRetryAfterHuman ? `${safeMessage} (${safeRetryAfterHuman})` : safeMessage; return new Response(JSON.stringify({ error: { message: msg } }), { status: statusCode, headers: { @@ -640,13 +876,14 @@ export function providerCircuitOpenResponse( retryAfter?: string | number | Date | null ) { const retryAfterSec = normalizeRetryAfterSeconds(retryAfter); + const safeProvider = projectPublicContextLabel(provider) ?? "unknown"; return new Response( JSON.stringify({ error: { - message: `Provider ${provider} circuit breaker is open`, + message: `Provider ${safeProvider} circuit breaker is open`, type: "server_error", code: "provider_circuit_open", - provider, + provider: safeProvider, retry_after: retryAfterSec, }, }), @@ -672,9 +909,10 @@ export function buildModelCooldownBody({ retryAfterAt?: string | null; credentialsCoolingCount?: number | null; }): ModelCooldownErrorPayload { - const resolvedModel = typeof model === "string" && model.trim().length > 0 ? model.trim() : null; - const resolvedRetryAfterAt = - typeof retryAfterAt === "string" && retryAfterAt.length > 0 ? retryAfterAt : null; + const resolvedModel = projectPublicContextLabel(model); + const resolvedRetryAfterAt = projectPublicRetryTimestamp(retryAfterAt); + const resolvedResetSeconds = + Number.isFinite(retryAfterSec) && retryAfterSec > 0 ? Math.max(Math.ceil(retryAfterSec), 1) : 1; const resolvedCoolingCount = typeof credentialsCoolingCount === "number" && Number.isFinite(credentialsCoolingCount) && @@ -690,7 +928,7 @@ export function buildModelCooldownBody({ type: "rate_limit_error", code: "model_cooldown", ...(resolvedModel ? { model: resolvedModel } : {}), - reset_seconds: Math.max(Math.ceil(retryAfterSec), 1), + reset_seconds: resolvedResetSeconds, ...(resolvedRetryAfterAt ? { retry_after: resolvedRetryAfterAt } : {}), ...(resolvedCoolingCount ? { credentials_cooling: resolvedCoolingCount } : {}), }, diff --git a/open-sse/utils/errorPathRedaction.ts b/open-sse/utils/errorPathRedaction.ts new file mode 100644 index 0000000000..2b372af583 --- /dev/null +++ b/open-sse/utils/errorPathRedaction.ts @@ -0,0 +1,905 @@ +const SOURCE_EXT = ["ts", "tsx", "js", "jsx", "mjs", "cjs", "mts", "cts"] as const; +const NATIVE_EXT = ["node", "so", "dylib", "dll"] as const; +const LEADING_PATH_PUNCTUATION = "'\"`([{<"; +const TRAILING_PATH_PUNCTUATION = "'\"`)]}>.,;:!?"; +const PATH_SPAN_END_PUNCTUATION = "'\"`)]}>.,;:!?"; +const FILE_URI_PREFIX = "file://"; +const HTTP_METHODS = [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "OPTIONS", + "HEAD", + "CONNECT", + "TRACE", +] as const; +const CLEAR_PROSE_BOUNDARIES = [ + "after", + "because", + "before", + "but", + "crashed", + "denied", + "eacces", + "enoent", + "expired", + "failed", + "rejected", + "retry", + "then", + "when", + "while", +] as const; +const POSIX_FILESYSTEM_ROOTS = [ + "/Users", + "/app", + "/boot", + "/data", + "/dev", + "/etc", + "/home", + "/media", + "/mnt", + "/nix", + "/opt", + "/private", + "/proc", + "/root", + "/run", + "/srv", + "/sys", + "/tmp", + "/usr", + "/var", + "/workspace", +] as const; +const WINDOWS_ROOT_RELATIVE_ROOTS = new Set([ + "program files", + "programdata", + "temp", + "users", + "windows", +]); + +function isWindowsAbsolutePathAt(value: string, start: number): boolean { + const remaining = value.length - start; + if (remaining > 2) { + const first = value.charCodeAt(start); + const second = value.charCodeAt(start + 1); + if ((first === 0x5c && second === 0x5c) || (first === 0x2f && second === 0x2f)) { + return true; + } + } + if (remaining < 3 || value.charCodeAt(start + 1) !== 0x3a) return false; + const driveLetter = value.charCodeAt(start); + const isAsciiLetter = + (driveLetter >= 0x41 && driveLetter <= 0x5a) || (driveLetter >= 0x61 && driveLetter <= 0x7a); + return ( + isAsciiLetter && (value.charCodeAt(start + 2) === 0x2f || value.charCodeAt(start + 2) === 0x5c) + ); +} + +function isWindowsAbsolutePath(value: string): boolean { + return isWindowsAbsolutePathAt(value, 0); +} + +function isWindowsRootRelativePathAt(value: string, start: number): boolean { + if ( + value.charCodeAt(start) !== 0x5c || + value.charCodeAt(start + 1) === 0x5c || + isWhitespace(value[start + 1]) + ) { + return false; + } + + const tokenEnd = findTokenEnd(value, start); + let firstSeparator = start + 1; + while (firstSeparator < tokenEnd && value.charCodeAt(firstSeparator) !== 0x5c) { + firstSeparator++; + } + const root = value.slice(start + 1, firstSeparator).toLowerCase(); + if (WINDOWS_ROOT_RELATIVE_ROOTS.has(root)) return true; + return ( + firstSeparator < tokenEnd - 1 || tokenContainsPathExtensionEvidence(value, start + 1, tokenEnd) + ); +} + +function hasAbsoluteFileUriAt(value: string, start: number): boolean { + const prefixEnd = start + FILE_URI_PREFIX.length; + return ( + value.length > prefixEnd && + value.slice(start, prefixEnd).toLowerCase() === FILE_URI_PREFIX && + !isWhitespace(value[prefixEnd]) + ); +} + +function hasAbsoluteFileUri(value: string): boolean { + return hasAbsoluteFileUriAt(value, 0); +} + +function isSyntacticallyAbsolutePathAt(value: string, start: number): boolean { + return ( + value.charCodeAt(start) === 0x2f || + isWindowsAbsolutePathAt(value, start) || + isWindowsRootRelativePathAt(value, start) || + hasAbsoluteFileUriAt(value, start) + ); +} + +function isAsciiDigit(code: number): boolean { + return code >= 0x30 && code <= 0x39; +} + +function isAsciiLetter(code: number): boolean { + return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a); +} + +function isAsciiAlphaNumeric(code: number): boolean { + return isAsciiDigit(code) || isAsciiLetter(code); +} + +function hasHttpUrlSchemeBefore(value: string, slashIndex: number): boolean { + for (const scheme of ["http:", "https:"]) { + const schemeStart = slashIndex - scheme.length; + if (schemeStart < 0 || value.slice(schemeStart, slashIndex).toLowerCase() !== scheme) continue; + if (schemeStart === 0 || !isAsciiAlphaNumeric(value.charCodeAt(schemeStart - 1))) return true; + } + return false; +} + +function isWhitespace(value: string): boolean { + return /\s/.test(value); +} + +function isRouteContextWord(value: string): boolean { + return value === "Route" || (HTTP_METHODS as readonly string[]).includes(value); +} + +function hasRouteContextBefore(value: string, candidateIndex: number): boolean { + let index = candidateIndex - 1; + while ( + index >= 0 && + (isWhitespace(value[index]) || + value.charCodeAt(index) === 0x28 || + value.charCodeAt(index) === 0x3a) + ) { + index--; + } + + const contextEnd = index + 1; + while (index >= 0 && isAsciiAlphaNumeric(value.charCodeAt(index))) index--; + return isRouteContextWord(value.slice(index + 1, contextEnd)); +} + +function isRouteContextToken(value: string): boolean { + let end = value.length; + while (end > 0 && !isAsciiAlphaNumeric(value.charCodeAt(end - 1))) end--; + let start = end; + while (start > 0 && isAsciiAlphaNumeric(value.charCodeAt(start - 1))) start--; + return isRouteContextWord(value.slice(start, end)); +} + +function matchesPosixFilesystemRootAt(value: string, start: number, root: string): boolean { + if (!value.startsWith(root, start)) return false; + const rootEnd = start + root.length; + return ( + rootEnd === value.length || + value.charCodeAt(rootEnd) === 0x2f || + PATH_SPAN_END_PUNCTUATION.includes(value[rootEnd]) + ); +} + +function isKnownPosixFilesystemPathAt(value: string, start: number): boolean { + return POSIX_FILESYSTEM_ROOTS.some((root) => matchesPosixFilesystemRootAt(value, start, root)); +} + +function isKnownPosixFilesystemPath(value: string): boolean { + return isKnownPosixFilesystemPathAt(value, 0); +} + +function looksLikeAbsolutePath(token: string): boolean { + // POSIX: common filesystem roots, with or without a source extension. + // Windows: drive-letter, UNC, or extended-length absolute paths. + // Source-file paths rooted elsewhere remain covered by SOURCE_EXT below. + if (token.length < 4 || token.length > 2048) return false; + const isPosix = token.charCodeAt(0) === 0x2f; + const isWindows = isWindowsAbsolutePath(token) || isWindowsRootRelativePathAt(token, 0); + if (!isPosix && !isWindows) return false; + if (isWindows) return true; + if (isKnownPosixFilesystemPath(token)) return true; + const dot = token.lastIndexOf("."); + if (dot <= 0 || dot === token.length - 1) return false; + const extension = token + .slice(dot + 1) + .split(":", 1)[0] + .toLowerCase(); + return ( + (SOURCE_EXT as readonly string[]).includes(extension) || + (NATIVE_EXT as readonly string[]).includes(extension) + ); +} + +function redactAbsolutePathToken(token: string, followsRouteContext: boolean): string { + let start = 0; + let end = token.length; + + while (start < end && LEADING_PATH_PUNCTUATION.includes(token[start])) start++; + while (end > start && TRAILING_PATH_PUNCTUATION.includes(token[end - 1])) end--; + + const candidate = token.slice(start, end); + const isFileUri = hasAbsoluteFileUri(candidate); + const pathCandidate = isFileUri ? candidate.slice(FILE_URI_PREFIX.length) : candidate; + + if ( + !isFileUri && + !isWindowsAbsolutePath(pathCandidate) && + !isWindowsRootRelativePathAt(pathCandidate, 0) && + pathCandidate.charCodeAt(0) === 0x2f && + followsRouteContext + ) { + return token; + } + if (!isFileUri && !looksLikeAbsolutePath(pathCandidate)) return token; + return `${token.slice(0, start)}${token.slice(end)}`; +} + +function findPathQuote(value: string, start: number, quote: string, takeFirst: boolean): number { + let candidate = value.indexOf(quote, start); + if (takeFirst || candidate < 0) return candidate < 0 ? value.length : candidate; + + while (candidate < value.length) { + const nextQuote = value.indexOf(quote, candidate + 1); + if (nextQuote < 0) return candidate; + // Two separately quoted absolute paths are unambiguous. Close the first + // candidate so the second one is scanned on its own; otherwise keep + // consuming quotes fail-closed because POSIX filenames may contain them. + if (isSyntacticallyAbsolutePathAt(value, nextQuote + 1)) return candidate; + candidate = nextQuote; + } + return value.length; +} + +function redactQuotedAbsolutePaths(value: string): string { + const parts: string[] = []; + let copyStart = 0; + let index = 0; + + while (index < value.length) { + const quote = value[index]; + if (quote !== "'" && quote !== '"' && quote !== "`") { + index++; + continue; + } + const candidateStart = index + 1; + if (!isSyntacticallyAbsolutePathAt(value, candidateStart)) { + index++; + continue; + } + + const isShieldedRoute = + value.charCodeAt(candidateStart) === 0x2f && + !isWindowsAbsolutePathAt(value, candidateStart) && + hasRouteContextBefore(value, index); + // Route/API contexts use their first closing quote so a later quoted + // filesystem path is still scanned independently. Filesystem candidates + // take the last matching quote on the line: POSIX filenames may themselves + // contain quote characters, whitespace, and punctuation, so earlier + // matches are ambiguous and must fail closed rather than expose a suffix. + const closingQuote = findPathQuote(value, candidateStart, quote, isShieldedRoute); + if (isShieldedRoute) { + if (closingQuote >= value.length) break; + index = closingQuote + 1; + continue; + } + parts.push(value.slice(copyStart, candidateStart), ""); + copyStart = closingQuote; + + if (closingQuote >= value.length) break; + index = closingQuote + 1; + } + + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +function findPathExtensionEnd(value: string, dot: number): number { + let end = dot + 1; + const maxExtensionEnd = Math.min(value.length, end + 16); + while (end < maxExtensionEnd && isAsciiAlphaNumeric(value.charCodeAt(end))) end++; + if (end === dot + 1 || (end === maxExtensionEnd && isAsciiAlphaNumeric(value.charCodeAt(end)))) { + return -1; + } + let hasLetter = false; + for (let index = dot + 1; index < end; index++) { + if (isAsciiLetter(value.charCodeAt(index))) hasLetter = true; + } + if (!hasLetter) return -1; + + while (value.charCodeAt(end) === 0x3a) { + let coordinateEnd = end + 1; + if (!isAsciiDigit(value.charCodeAt(coordinateEnd))) break; + while (coordinateEnd < value.length && isAsciiDigit(value.charCodeAt(coordinateEnd))) { + coordinateEnd++; + } + end = coordinateEnd; + } + + if ( + end === value.length || + isWhitespace(value[end]) || + PATH_SPAN_END_PUNCTUATION.includes(value[end]) + ) { + return end; + } + return -1; +} + +function findTokenEnd(value: string, start: number): number { + let end = start; + while (end < value.length && !isWhitespace(value[end])) end++; + return end; +} + +function findExtensionEndInToken(value: string, start: number, end: number): number { + let lastExtensionEnd = -1; + for (let index = start; index < end; index++) { + const code = value.charCodeAt(index); + if (code === 0x2f || code === 0x5c) { + lastExtensionEnd = -1; + continue; + } + if (code !== 0x2e) continue; + const extensionEnd = findPathExtensionEnd(value, index); + if (extensionEnd >= 0 && extensionEnd <= end) lastExtensionEnd = extensionEnd; + } + return lastExtensionEnd; +} + +function tokenContainsPathExtensionEvidence(value: string, start: number, end: number): boolean { + for (let dot = start; dot < end; dot++) { + if (value.charCodeAt(dot) !== 0x2e) continue; + let extensionEnd = dot + 1; + const maxExtensionEnd = Math.min(end, extensionEnd + 16); + let hasLetter = false; + while (extensionEnd < maxExtensionEnd && isAsciiAlphaNumeric(value.charCodeAt(extensionEnd))) { + if (isAsciiLetter(value.charCodeAt(extensionEnd))) hasLetter = true; + extensionEnd++; + } + if ( + extensionEnd === dot + 1 || + !hasLetter || + (extensionEnd === maxExtensionEnd && + extensionEnd < end && + isAsciiAlphaNumeric(value.charCodeAt(extensionEnd))) + ) { + continue; + } + if ( + extensionEnd === end || + value.charCodeAt(extensionEnd) === 0x2f || + value.charCodeAt(extensionEnd) === 0x5c || + PATH_SPAN_END_PUNCTUATION.includes(value[extensionEnd]) + ) { + return true; + } + } + return false; +} + +function tokenContainsPathSeparator(value: string, start: number, end: number): boolean { + for (let index = start; index < end; index++) { + const code = value.charCodeAt(index); + if (code === 0x2f || code === 0x5c) return true; + } + return false; +} + +function remainderContainsFilesystemSeparator(value: string, start: number): boolean { + let tokenStart = start; + let previousToken = ""; + while (tokenStart < value.length) { + while (tokenStart < value.length && isWhitespace(value[tokenStart])) tokenStart++; + if (tokenStart >= value.length) return false; + + const tokenEnd = findTokenEnd(value, tokenStart); + const token = value.slice(tokenStart, tokenEnd).toLowerCase(); + const isHttpUrl = token.includes("http://") || token.includes("https://"); + let separatorIndex = tokenStart; + while ( + separatorIndex < tokenEnd && + value.charCodeAt(separatorIndex) !== 0x2f && + value.charCodeAt(separatorIndex) !== 0x5c + ) { + separatorIndex++; + } + const precedingSeparatorCode = + separatorIndex > tokenStart ? value.charCodeAt(separatorIndex - 1) : -1; + const contextIndex = + precedingSeparatorCode === 0x27 || + precedingSeparatorCode === 0x22 || + precedingSeparatorCode === 0x60 + ? separatorIndex - 1 + : separatorIndex; + const isShieldedRoute = + separatorIndex < tokenEnd && + value.charCodeAt(separatorIndex) === 0x2f && + !isWindowsAbsolutePathAt(value, separatorIndex) && + (isRouteContextToken(previousToken) || hasRouteContextBefore(value, contextIndex)); + if (!isHttpUrl && separatorIndex < tokenEnd && !isShieldedRoute) return true; + previousToken = value.slice(tokenStart, tokenEnd); + tokenStart = tokenEnd; + } + return false; +} + +function trimPathSpanEnd(value: string, start: number, end: number): number { + while (end > start && PATH_SPAN_END_PUNCTUATION.includes(value[end - 1])) end--; + return end; +} + +function isClearProseBoundaryToken(value: string, start: number, end: number): boolean { + while (start < end && LEADING_PATH_PUNCTUATION.includes(value[start])) start++; + end = trimPathSpanEnd(value, start, end); + return (CLEAR_PROSE_BOUNDARIES as readonly string[]).includes( + value.slice(start, end).toLowerCase() + ); +} + +function findUnquotedPathEnd( + value: string, + start: number, + acceptFirstTokenPunctuation: boolean, + acceptEndpointBeforeAnotherAbsolute: boolean, + failClosedAmbiguity: boolean +): number { + let tokenStart = start; + let isFirstToken = true; + let firstTokenEnd = -1; + let firstTrimmedTokenEnd = -1; + let lastPathTokenEnd = -1; + let resolvedExtensionEnd = -1; + let hasFilesystemEvidence = false; + let hasUnresolvedFragments = false; + + const resolveEndpoint = (): number => { + if (hasUnresolvedFragments) { + return failClosedAmbiguity || hasFilesystemEvidence ? value.length : -1; + } + if (resolvedExtensionEnd >= 0) return resolvedExtensionEnd; + if (hasFilesystemEvidence && lastPathTokenEnd >= 0) return lastPathTokenEnd; + if ( + acceptFirstTokenPunctuation && + firstTrimmedTokenEnd >= 0 && + firstTrimmedTokenEnd < firstTokenEnd + ) { + return firstTrimmedTokenEnd; + } + return -1; + }; + + while (tokenStart < value.length) { + const tokenEnd = findTokenEnd(value, tokenStart); + const extensionEnd = findExtensionEndInToken(value, tokenStart, tokenEnd); + const trimmedTokenEnd = trimPathSpanEnd(value, tokenStart, tokenEnd); + + if (isFirstToken) { + firstTokenEnd = tokenEnd; + firstTrimmedTokenEnd = trimmedTokenEnd; + lastPathTokenEnd = trimmedTokenEnd; + // A prose-looking token may itself be a directory name. It is a safe + // boundary only when no later token carries path-separator evidence; + // otherwise keep scanning so a filesystem suffix cannot survive. + } else if ( + isClearProseBoundaryToken(value, tokenStart, tokenEnd) && + (!remainderContainsFilesystemSeparator(value, tokenEnd) || + (!failClosedAmbiguity && !hasFilesystemEvidence)) + ) { + return resolveEndpoint(); + } + + const containsSeparator = tokenContainsPathSeparator(value, tokenStart, tokenEnd); + const containsExtensionEvidence = tokenContainsPathExtensionEvidence( + value, + tokenStart, + tokenEnd + ); + if (containsSeparator) { + lastPathTokenEnd = trimmedTokenEnd; + hasFilesystemEvidence = true; + hasUnresolvedFragments = false; + resolvedExtensionEnd = extensionEnd >= 0 ? extensionEnd : -1; + if (extensionEnd < 0 && containsExtensionEvidence) { + resolvedExtensionEnd = trimmedTokenEnd; + } + } else if (extensionEnd >= 0) { + resolvedExtensionEnd = extensionEnd; + hasFilesystemEvidence = true; + hasUnresolvedFragments = false; + } else if (containsExtensionEvidence) { + resolvedExtensionEnd = trimmedTokenEnd; + hasFilesystemEvidence = true; + hasUnresolvedFragments = false; + } else if (!isFirstToken) { + hasUnresolvedFragments = true; + } + + let nextTokenStart = tokenEnd; + while (nextTokenStart < value.length && isWhitespace(value[nextTokenStart])) nextTokenStart++; + if (nextTokenStart >= value.length) return resolveEndpoint(); + if (isSyntacticallyAbsolutePathAt(value, nextTokenStart)) { + const endpoint = resolveEndpoint(); + if (endpoint >= 0) return endpoint; + return acceptEndpointBeforeAnotherAbsolute ? lastPathTokenEnd : -1; + } + + tokenStart = nextTokenStart; + isFirstToken = false; + } + return resolveEndpoint(); +} + +function isUnquotedPosixSpanCandidateAt(value: string, start: number): boolean { + const tokenEnd = findTokenEnd(value, start); + const token = value.slice(start, tokenEnd); + if (isKnownPosixFilesystemPath(token)) return true; + if ( + findExtensionEndInToken(value, start, tokenEnd) >= 0 || + tokenContainsPathExtensionEvidence(value, start, tokenEnd) + ) { + return true; + } + + let slashCount = 0; + for (let index = start; index < tokenEnd; index++) { + if (value.charCodeAt(index) === 0x2f) slashCount++; + } + // Any boundary-delimited absolute POSIX token is filesystem-sensitive by + // default. Explicit Route/HTTP context is shielded by the caller before this + // candidate check, so `/vault` is redacted while `Route /vault` is retained. + return slashCount >= 1 && token.length > 1; +} + +function redactUnquotedAbsolutePathSpans(value: string): string { + const parts: string[] = []; + let copyStart = 0; + let index = 0; + + while (index < value.length) { + const previous = index > 0 ? value[index - 1] : ""; + const followsQuote = previous === "'" || previous === '"' || previous === "`"; + const hasCommonBoundary = + index === 0 || + isWhitespace(previous) || + LEADING_PATH_PUNCTUATION.includes(previous) || + previous === "=" || + previous === ":" || + previous === "," || + previous === ";" || + previous === "." || + previous === ">" || + previous === "|"; + const startsForwardSlashUnc = + value.charCodeAt(index) === 0x2f && value.charCodeAt(index + 1) === 0x2f; + const startsHttpUrl = + startsForwardSlashUnc && previous === ":" && hasHttpUrlSchemeBefore(value, index); + const isWindowsPath = + !followsQuote && + (isWindowsAbsolutePathAt(value, index) || isWindowsRootRelativePathAt(value, index)) && + !startsHttpUrl; + const isFileUriPath = !followsQuote && hasAbsoluteFileUriAt(value, index); + const isPosixPath = + !followsQuote && + value.charCodeAt(index) === 0x2f && + value.charCodeAt(index + 1) !== 0x2f && + !hasRouteContextBefore(value, index) && + isUnquotedPosixSpanCandidateAt(value, index); + const hasBoundary = hasCommonBoundary || (isWindowsPath && previous === ":"); + if (!hasBoundary || (!isWindowsPath && !isFileUriPath && !isPosixPath)) { + index++; + continue; + } + + // Whitespace makes an unquoted path ambiguous. Extend through adjacent + // separator-bearing tokens or to a deterministic filename extension. + // Unequivocal Windows, file-URI, and known-root candidates fail closed; + // arbitrary extensionless POSIX text falls back to token-level handling so + // ordinary `/x/y` route text is not redacted indiscriminately. + const isKnownPosixPath = isKnownPosixFilesystemPathAt(value, index); + const pathEnd = findUnquotedPathEnd( + value, + index, + isWindowsPath || isFileUriPath || isKnownPosixPath, + isWindowsPath || isFileUriPath || isKnownPosixPath, + isWindowsPath || isFileUriPath || isKnownPosixPath + ); + if (pathEnd < 0) { + const mustFailClosed = isWindowsPath || isFileUriPath || isKnownPosixPath; + if (mustFailClosed) { + // An unequivocal filesystem prefix with an unknowable endpoint must + // fail closed over the rest of the first line rather than expose a + // suffix such as `Files\\secret` or `My Project`. + parts.push(value.slice(copyStart, index), ""); + copyStart = value.length; + index = value.length; + break; + } + index++; + continue; + } + parts.push(value.slice(copyStart, index), ""); + copyStart = pathEnd; + index = pathEnd; + } + + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +function isPhysicalLineSeparator(code: number): boolean { + return code === 0x0a || code === 0x0d || code === 0x2028 || code === 0x2029; +} + +function serializedLineSeparatorLengthAt(value: string, start: number): number { + if (value.charCodeAt(start) !== 0x5c) return 0; + const marker = value[start + 1]?.toLowerCase(); + if (marker === "n" || marker === "r") return 2; + const unicodeMarker = value.slice(start + 1, start + 6).toLowerCase(); + return unicodeMarker === "u000a" || + unicodeMarker === "u000d" || + unicodeMarker === "u2028" || + unicodeMarker === "u2029" + ? 6 + : 0; +} + +function looksLikeRelativeStackLocation(token: string): boolean { + if (token.length < 6 || token.length > 2048) return false; + + const lastForwardSlash = token.lastIndexOf("/"); + const lastBackslash = token.lastIndexOf("\\"); + const lastSeparator = Math.max(lastForwardSlash, lastBackslash); + if (lastSeparator === token.length - 1) return false; + + const columnSeparator = token.lastIndexOf(":"); + const lineSeparator = token.lastIndexOf(":", columnSeparator - 1); + if (lineSeparator < 0 || !hasNumericLineColumnSuffix(token, lineSeparator)) return false; + const queryIndex = token.indexOf("?", lastSeparator + 1); + const fragmentIndex = token.indexOf("#", lastSeparator + 1); + const metadataIndexes = [queryIndex, fragmentIndex].filter( + (index) => index >= 0 && index < lineSeparator + ); + const extensionEnd = metadataIndexes.length > 0 ? Math.min(...metadataIndexes) : lineSeparator; + const dot = token.lastIndexOf(".", extensionEnd - 1); + if (dot <= lastSeparator || dot === extensionEnd - 1) return false; + const extension = token.slice(dot + 1, extensionEnd).toLowerCase(); + if (!(SOURCE_EXT as readonly string[]).includes(extension)) return false; + return true; +} + +function looksLikeUrlStackLocation(token: string): boolean { + if (token.length < 12 || token.length > 2048) return false; + const lower = token.toLowerCase(); + if (!lower.startsWith("http://") && !lower.startsWith("https://")) return false; + const columnSeparator = token.lastIndexOf(":"); + const lineSeparator = token.lastIndexOf(":", columnSeparator - 1); + return lineSeparator > 0 && hasNumericLineColumnSuffix(token, lineSeparator); +} + +function hasNumericLineColumnSuffix(value: string, separator: number): boolean { + if (value.charCodeAt(separator) !== 0x3a) return false; + let index = separator + 1; + if (!isAsciiDigit(value.charCodeAt(index))) return false; + while (index < value.length && isAsciiDigit(value.charCodeAt(index))) index++; + if (value.charCodeAt(index) !== 0x3a) return false; + + index++; + if (!isAsciiDigit(value.charCodeAt(index))) return false; + while (index < value.length && isAsciiDigit(value.charCodeAt(index))) index++; + return index === value.length; +} + +function isNodeModulePathCode(code: number): boolean { + return ( + isAsciiAlphaNumeric(code) || code === 0x2e || code === 0x2f || code === 0x5f || code === 0x2d + ); +} + +function looksLikeNodeStackLocation(token: string): boolean { + if (token.length < 10 || token.length > 2048 || !token.startsWith("node:")) return false; + const columnSeparator = token.lastIndexOf(":"); + const lineSeparator = token.lastIndexOf(":", columnSeparator - 1); + if (lineSeparator <= 5 || !hasNumericLineColumnSuffix(token, lineSeparator)) return false; + for (let index = 5; index < lineSeparator; index++) { + if (!isNodeModulePathCode(token.charCodeAt(index))) return false; + } + return true; +} + +function looksLikeEvalStackLocation(token: string): boolean { + return token.length <= 64 && token.startsWith("[eval]") && hasNumericLineColumnSuffix(token, 6); +} + +function isRecognizedStackPathAt(value: string, start: number): boolean { + if (hasAbsoluteFileUriAt(value, start)) return true; + const tokenEnd = trimPathSpanEnd(value, start, findTokenEnd(value, start)); + const token = value.slice(start, tokenEnd); + return ( + looksLikeAbsolutePath(token) || + looksLikeRelativeStackLocation(token) || + looksLikeUrlStackLocation(token) || + looksLikeNodeStackLocation(token) || + looksLikeEvalStackLocation(token) + ); +} + +function isStackFrameLabel(value: string, start: number, end: number): boolean { + const label = value.slice(start, end).trim(); + if (label.length === 0 || label.length > 256) return false; + if (!/^[A-Za-z_$<]/.test(label) || /[^A-Za-z0-9_$.[\]<>:/ -]/.test(label)) return false; + if (!/\s/.test(label)) return true; + return /^(?:async|new)\s+\S+$/.test(label) || /^\S+\s+\[as\s+\S+\]$/.test(label); +} + +function skipAsyncStackPrefix(value: string, start: number): number { + if (value.slice(start, start + 5) !== "async" || !isWhitespace(value[start + 5])) return start; + let locationStart = start + 6; + while (locationStart < value.length && isWhitespace(value[locationStart])) locationStart++; + return locationStart; +} + +function isAggregateIndexLocationAt(value: string, start: number): boolean { + if (value.slice(start, start + 5) !== "index" || !isWhitespace(value[start + 5])) return false; + let index = start + 6; + while (index < value.length && isWhitespace(value[index])) index++; + if (!isAsciiDigit(value.charCodeAt(index))) return false; + while (index < value.length && isAsciiDigit(value.charCodeAt(index))) index++; + while (index < value.length && isWhitespace(value[index])) index++; + return value.charCodeAt(index) === 0x29; +} + +function looksLikeStackFrameAt(value: string, atIndex: number, allowDirectPath: boolean): boolean { + if (value.slice(atIndex, atIndex + 2).toLowerCase() !== "at") return false; + let labelStart = atIndex + 2; + if (!isWhitespace(value[labelStart])) return false; + while (labelStart < value.length && isWhitespace(value[labelStart])) labelStart++; + labelStart = skipAsyncStackPrefix(value, labelStart); + if (allowDirectPath && isRecognizedStackPathAt(value, labelStart)) return true; + + const openParen = value.indexOf("(", labelStart); + if (openParen < 0 || openParen - labelStart > 256) return false; + let pathStart = openParen + 1; + while (pathStart < value.length && isWhitespace(value[pathStart])) pathStart++; + return ( + isStackFrameLabel(value, labelStart, openParen) && + (isRecognizedStackPathAt(value, pathStart) || + (allowDirectPath && isAggregateIndexLocationAt(value, pathStart))) + ); +} + +function looksLikeAtSignStackFrameAt(value: string, frameStart: number): boolean { + const tokenEnd = trimPathSpanEnd(value, frameStart, findTokenEnd(value, frameStart)); + const atSign = value.indexOf("@", frameStart); + if (atSign <= frameStart || atSign >= tokenEnd || atSign - frameStart > 256) return false; + return isStackFrameLabel(value, frameStart, atSign) && isRecognizedStackPathAt(value, atSign + 1); +} + +function findSerializedStackFrameStart(value: string): number { + for (let index = 0; index < value.length; index++) { + const separatorLength = serializedLineSeparatorLengthAt(value, index); + if (separatorLength === 0) continue; + let frameStart = index + separatorLength; + while (frameStart < value.length) { + while (frameStart < value.length && isWhitespace(value[frameStart])) frameStart++; + const adjacentSeparatorLength = serializedLineSeparatorLengthAt(value, frameStart); + if (adjacentSeparatorLength === 0) break; + frameStart += adjacentSeparatorLength; + } + if ( + looksLikeStackFrameAt(value, frameStart, true) || + looksLikeAtSignStackFrameAt(value, frameStart) + ) { + let separatorStart = index; + while (separatorStart > 0 && value.charCodeAt(separatorStart - 1) === 0x5c) { + separatorStart--; + } + return separatorStart; + } + } + return -1; +} + +function findInlineStackFrameStart(value: string): number { + let marker = value.indexOf(" at "); + while (marker >= 0) { + if (looksLikeStackFrameAt(value, marker + 1, false)) return marker; + marker = value.indexOf(" at ", marker + 4); + } + return -1; +} + +function findInlineAtSignStackFrameStart(value: string): number { + let frameStart = 0; + while (frameStart < value.length) { + if (looksLikeAtSignStackFrameAt(value, frameStart)) { + return frameStart > 0 && isWhitespace(value[frameStart - 1]) ? frameStart - 1 : frameStart; + } + const tokenEnd = findTokenEnd(value, frameStart); + frameStart = tokenEnd; + while (frameStart < value.length && isWhitespace(value[frameStart])) frameStart++; + } + return -1; +} + +function physicalLineSeparatorLengthAt(value: string, start: number): number { + const code = value.charCodeAt(start); + if (!isPhysicalLineSeparator(code)) return 0; + return code === 0x0d && value.charCodeAt(start + 1) === 0x0a ? 2 : 1; +} + +function findPhysicalStackFrameStart(value: string): number { + for (let index = 0; index < value.length; index++) { + const separatorLength = physicalLineSeparatorLengthAt(value, index); + if (separatorLength === 0) continue; + let frameStart = index + separatorLength; + while (frameStart < value.length && isWhitespace(value[frameStart])) frameStart++; + if ( + looksLikeStackFrameAt(value, frameStart, true) || + looksLikeAtSignStackFrameAt(value, frameStart) + ) { + return index; + } + index += separatorLength - 1; + } + return -1; +} + +/** Strip only recognized physical, serialized, and inline JavaScript stack-frame tails. */ +export function stripRecognizedErrorStackTail(value: string): string { + const candidates = [ + findPhysicalStackFrameStart(value), + findSerializedStackFrameStart(value), + findInlineStackFrameStart(value), + findInlineAtSignStackFrameStart(value), + ].filter((candidate) => candidate >= 0); + if (candidates.length === 0) return value; + return value.slice(0, Math.min(...candidates)); +} + +/** + * Public exception messages remain fail-closed at the first physical line. + * Provider passthroughs that require multiline capability wording use the + * narrower recognized-frame helper above instead. + */ +export function stripErrorStackTail(value: string): string { + let firstLineEnd = value.length; + for (let index = 0; index < value.length; index++) { + if (isPhysicalLineSeparator(value.charCodeAt(index))) { + firstLineEnd = index; + break; + } + } + return stripRecognizedErrorStackTail(value.slice(0, firstLineEnd)); +} + +/** + * Redact absolute filesystem paths while preserving URLs, explicitly marked + * API routes, and punctuation around determinable endpoints. Unequivocal + * filesystem prefixes fail closed when an unquoted endpoint is ambiguous. + */ +export function redactErrorPaths(value: string): string { + const quotedPathsRedacted = redactQuotedAbsolutePaths(value); + const pathSpansRedacted = redactUnquotedAbsolutePathSpans(quotedPathsRedacted); + const parts = pathSpansRedacted.split(/(\s+)/); + let previousToken = ""; + for (let index = 0; index < parts.length; index++) { + const token = parts[index]; + if (isWhitespace(token)) continue; + parts[index] = redactAbsolutePathToken(token, isRouteContextToken(previousToken)); + previousToken = token; + } + return parts.join(""); +} diff --git a/open-sse/utils/errorSanitization.ts b/open-sse/utils/errorSanitization.ts new file mode 100644 index 0000000000..1b7601d4ee --- /dev/null +++ b/open-sse/utils/errorSanitization.ts @@ -0,0 +1,895 @@ +import { + redactErrorPaths, + stripErrorStackTail, + stripRecognizedErrorStackTail, +} from "./errorPathRedaction.ts"; +import { CREDENTIAL_PATTERNS } from "./credentialPatterns.ts"; + +// Length cap protects against pathological inputs even before tokenization. +const MAX_ERROR_LEN = 4096; +const MAX_ERROR_SCAN_HEADROOM = 512; +const MAX_SECURITY_ESCAPE_LAYERS = 3; +const STRONG_CREDENTIAL_TOKEN_SOURCE = + "(?:eyJ[A-Za-z0-9_-]{5,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}|" + + "github_pat_[A-Za-z0-9_]{20,}|ghp_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{20,}|" + + "xox[a-z]-[A-Za-z0-9-]{10,}|(?:AKIA|ASIA)[A-Z0-9]{16}|" + + "(?= 0x30 && code <= 0x39) || + (code >= 0x41 && code <= 0x5a) || + (code >= 0x61 && code <= 0x7a) + ); +} + +function asciiHexValue(code: number): number { + if (code >= 0x30 && code <= 0x39) return code - 0x30; + if (code >= 0x41 && code <= 0x46) return code - 0x41 + 10; + if (code >= 0x61 && code <= 0x66) return code - 0x61 + 10; + return -1; +} + +function unicodeEscapeCodeAt(value: string, start: number): number | null { + if ( + value.charCodeAt(start) !== 0x5c || + (value[start + 1] !== "u" && value[start + 1] !== "U") || + start + 5 >= value.length + ) { + return null; + } + + let decoded = 0; + for (let digit = start + 2; digit <= start + 5; digit++) { + const nibble = asciiHexValue(value.charCodeAt(digit)); + if (nibble < 0) return null; + decoded = decoded * 16 + nibble; + } + return decoded; +} + +function isPrintableAscii(code: number | null): code is number { + return code !== null && code >= 0x20 && code <= 0x7e; +} + +function isSecurityWhitespaceCode(code: number | null): boolean { + return code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d; +} + +function isEscapeTokenBoundary(code: number): boolean { + return !isAsciiAlphaNumericCode(code) && code !== 0x2e && code !== 0x5f && code !== 0x2d; +} + +function shouldPreserveUnicodeUncEvidence( + value: string, + runStart: number, + runEnd: number, + decoded: number +): boolean { + if ( + runEnd - runStart < 2 || + decoded === 0x2f || + decoded === 0x5c || + decoded === 0x3a || + (runStart > 0 && !isEscapeTokenBoundary(value.charCodeAt(runStart - 1))) + ) { + return false; + } + + const afterEscape = runEnd + 5; + let tokenEnd = afterEscape; + while (tokenEnd < value.length && !/\s/.test(value[tokenEnd])) tokenEnd++; + if (value.slice(afterEscape, tokenEnd).includes("=")) return false; + return afterEscape < tokenEnd; +} + +function decodeSecurityEscapesOnce( + value: string, + decodeQuotes: boolean, + maxLength: number +): string { + const output: string[] = []; + let changed = false; + + for (let index = 0; index < value.length; index++) { + if (value.charCodeAt(index) !== 0x5c) { + output.push(value[index]); + continue; + } + + const runStart = index; + while (index < value.length && value.charCodeAt(index) === 0x5c) index++; + const runEnd = index; + if (runEnd >= value.length) { + output.push(value.slice(runStart)); + break; + } + + const escaped = value[runEnd]; + if (escaped === "u" || escaped === "U") { + const decoded = unicodeEscapeCodeAt(value, runEnd - 1); + const isQuote = decoded === 0x22 || decoded === 0x27; + if (isSecurityWhitespaceCode(decoded)) { + output.push(" "); + index = runEnd + 4; + changed = true; + continue; + } + if ( + isPrintableAscii(decoded) && + (decodeQuotes || !isQuote) && + !shouldPreserveUnicodeUncEvidence(value, runStart, runEnd, decoded) + ) { + output.push(String.fromCharCode(decoded)); + index = runEnd + 4; + changed = true; + continue; + } + output.push(value.slice(runStart, runEnd + 5)); + index = runEnd + 4; + continue; + } + + if ( + escaped === "b" || + escaped === "f" || + escaped === "n" || + escaped === "r" || + escaped === "t" + ) { + output.push(" "); + index = runEnd; + changed = true; + continue; + } + + if (escaped === "/" || (decodeQuotes && (escaped === '"' || escaped === "'"))) { + output.push(escaped); + index = runEnd; + changed = true; + continue; + } + + output.push(value.slice(runStart, runEnd)); + index = runEnd - 1; + } + + return changed ? output.join("").slice(0, maxLength) : value; +} + +function hasResidualSecurityEscape(value: string): boolean { + for (let index = 0; index < value.length; index++) { + if (value.charCodeAt(index) !== 0x5c) continue; + while (index < value.length && value.charCodeAt(index) === 0x5c) index++; + if (index >= value.length) return false; + const escaped = value[index]; + if ( + escaped === "b" || + escaped === "f" || + escaped === "n" || + escaped === "r" || + escaped === "t" + ) { + return true; + } + if (escaped === "/" || escaped === '"' || escaped === "'") return true; + if (escaped === "u" || escaped === "U") { + const decoded = unicodeEscapeCodeAt(value, index - 1); + if (isPrintableAscii(decoded) || isSecurityWhitespaceCode(decoded)) return true; + } + } + return false; +} + +/** Decode bounded security ASCII/JSON escapes while never materializing arbitrary Unicode. */ +function normalizeSecurityEscapes( + value: string, + decodeQuotes: boolean, + maxLength = MAX_ERROR_LEN +): string { + let normalized = value.slice(0, maxLength); + for (let layer = 0; layer < MAX_SECURITY_ESCAPE_LAYERS; layer++) { + const decoded = decodeSecurityEscapesOnce(normalized, decodeQuotes, maxLength); + if (decoded === normalized) break; + normalized = decoded.slice(0, maxLength); + } + return normalized; +} + +function isCredentialLabelBoundary(code: number): boolean { + return !isAsciiAlphaNumericCode(code) && code !== 0x5f && code !== 0x2d; +} + +function matchCredentialAssignmentAt(value: string, start: number): CredentialAssignment | null { + const keyQuote = value[start] === '"' || value[start] === "'" ? value[start] : ""; + const labelStart = start + (keyQuote ? 1 : 0); + const cliFlag = + !keyQuote && + labelStart >= 2 && + value.slice(labelStart - 2, labelStart) === "--" && + (labelStart === 2 || isCredentialLabelBoundary(value.charCodeAt(labelStart - 3))); + + for (const [label, failClosed] of CREDENTIAL_LABELS) { + const labelEnd = labelStart + label.length; + if (value.slice(labelStart, labelEnd).toLowerCase() !== label) continue; + let index = labelEnd; + if ( + (label === "arena-auth-prod-v1" || label === "__secure-next-auth.session-token") && + value[index] === "." + ) { + const chunkStart = ++index; + while (index < value.length && /\d/.test(value[index])) index++; + if (index === chunkStart) continue; + } + if (keyQuote) { + if (value[index] !== keyQuote) continue; + index++; + } else if (!isCredentialLabelBoundary(value.charCodeAt(index))) { + continue; + } else if (value[index] === '"' || value[index] === "'") { + index++; + } + const separatorStart = index; + while (/\s/.test(value[index])) index++; + if (value[index] === ":" || value[index] === "=") { + index++; + while (/\s/.test(value[index])) index++; + } else if (!(cliFlag && index > separatorStart)) { + continue; + } + return { valueStart: index, failClosed }; + } + return null; +} + +function findQuotedCredentialEnd(value: string, start: number, quote: string): number { + let index = start + 1; + while (index < value.length) { + if (value.charCodeAt(index) === 0x5c) { + index += 2; + continue; + } + if (value[index] === quote) return index; + index++; + } + return -1; +} + +function findUnquotedCredentialEnd(value: string, start: number): number { + let end = start; + while (end < value.length) { + const char = value[end]; + if (/\s/.test(char) || char === '"' || char === "'" || char === "," || char === "}") break; + end++; + } + return end; +} + +function redactLabeledCredentialAssignments(value: string): string { + const parts: string[] = []; + let copyStart = 0; + let index = 0; + + while (index < value.length) { + const assignment = matchCredentialAssignmentAt(value, index); + if (!assignment) { + index++; + continue; + } + + const { valueStart, failClosed } = assignment; + const quote = value[valueStart] === '"' || value[valueStart] === "'" ? value[valueStart] : ""; + if (quote) { + const closingQuote = findQuotedCredentialEnd(value, valueStart, quote); + parts.push(value.slice(copyStart, valueStart + 1), "[REDACTED]"); + if (closingQuote < 0) { + copyStart = value.length; + index = value.length; + } else { + parts.push(quote); + copyStart = closingQuote + 1; + index = copyStart; + } + continue; + } + + // A leading backslash may be a serialized quote or another encoded + // delimiter. Do not redact only that prefix and leave the value behind. + const valueEnd = + failClosed || value.charCodeAt(valueStart) === 0x5c + ? value.length + : findUnquotedCredentialEnd(value, valueStart); + parts.push(value.slice(copyStart, valueStart), "[REDACTED]"); + copyStart = valueEnd; + index = Math.max(valueEnd, valueStart + 1); + } + + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +function redactPrivateKeyPemBlocks(value: string): string { + // ASCII-only fold keeps offsets aligned even when the surrounding message + // contains Unicode characters whose full uppercase form expands in length. + const upperValue = value.replace(/[a-z]/g, (char) => char.toUpperCase()); + const beginPrefix = "-----BEGIN "; + const parts: string[] = []; + let copyStart = 0; + let searchStart = 0; + + while (searchStart < value.length) { + const blockStart = upperValue.indexOf(beginPrefix, searchStart); + if (blockStart < 0) break; + const labelStart = blockStart + beginPrefix.length; + const headerEnd = upperValue.indexOf("-----", labelStart); + if (headerEnd < 0) break; + const label = upperValue.slice(labelStart, headerEnd).trim(); + if (!/^(?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?$/.test(label)) { + searchStart = headerEnd + 5; + continue; + } + + const endMarker = `-----END ${label}-----`; + const closingStart = upperValue.indexOf(endMarker, headerEnd + 5); + const blockEnd = closingStart < 0 ? value.length : closingStart + endMarker.length; + parts.push(value.slice(copyStart, blockStart), "[REDACTED]"); + copyStart = blockEnd; + searchStart = blockEnd; + } + + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +const DATA_URL_PREFIX = "data:"; +const BASE64_DATA_URL_MARKER = ";base64"; +const REDACTED_DATA_URL = "[REDACTED_DATA_URL]"; + +function matchesAsciiCaseInsensitiveAt(value: string, start: number, expected: string): boolean { + if (start < 0 || start + expected.length > value.length) return false; + for (let offset = 0; offset < expected.length; offset++) { + const code = value.charCodeAt(start + offset); + const foldedCode = code >= 0x41 && code <= 0x5a ? code + 0x20 : code; + if (foldedCode !== expected.charCodeAt(offset)) return false; + } + return true; +} + +function isBase64DataUrlPayloadCode(code: number): boolean { + return ( + isAsciiAlphaNumericCode(code) || + code === 0x2b || + code === 0x2f || + code === 0x3d || + code === 0x5f || + code === 0x2d + ); +} + +function isEcmaScriptWhitespaceCode(code: number): boolean { + return ( + (code >= 0x09 && code <= 0x0d) || + code === 0x20 || + code === 0xa0 || + code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || + code === 0x2028 || + code === 0x2029 || + code === 0x202f || + code === 0x205f || + code === 0x3000 || + code === 0xfeff + ); +} + +/** Redact base64 data URLs in one pass, including input with many repeated `data:` prefixes. */ +function redactBase64DataUrls(value: string): string { + const parts: string[] = []; + let copyStart = 0; + let index = 0; + + while (index < value.length) { + if (!matchesAsciiCaseInsensitiveAt(value, index, DATA_URL_PREFIX)) { + index++; + continue; + } + + const dataUrlStart = index; + const mediaTypeStart = dataUrlStart + DATA_URL_PREFIX.length; + let delimiter = mediaTypeStart; + while ( + delimiter < value.length && + value[delimiter] !== "," && + !isEcmaScriptWhitespaceCode(value.charCodeAt(delimiter)) + ) { + delimiter++; + } + + const markerStart = delimiter - BASE64_DATA_URL_MARKER.length; + const hasBase64Marker = + delimiter < value.length && + value[delimiter] === "," && + markerStart >= mediaTypeStart && + matchesAsciiCaseInsensitiveAt(value, markerStart, BASE64_DATA_URL_MARKER); + if (!hasBase64Marker) { + index = delimiter < value.length ? delimiter + 1 : value.length; + continue; + } + + let payloadEnd = delimiter + 1; + while (payloadEnd < value.length && isBase64DataUrlPayloadCode(value.charCodeAt(payloadEnd))) { + payloadEnd++; + } + if (payloadEnd === delimiter + 1) { + index = delimiter + 1; + continue; + } + + parts.push(value.slice(copyStart, dataUrlStart), REDACTED_DATA_URL); + copyStart = payloadEnd; + index = payloadEnd; + } + + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +const HTTP_URL_RE = /https?:\/\//gi; +const URL_QUERY_PARAM_RE = /([?&])([^=&#]+)=([^&#]*)/g; + +function isUrlTerminator(char: string): boolean { + return ( + /\s/.test(char) || + char === '"' || + char === "'" || + char === "`" || + char === "<" || + char === ">" || + char === ")" || + char === "]" || + char === "}" || + char === "," || + char === ";" + ); +} + +function normalizeUrlQueryKey(key: string): string { + let decoded = key.replace(/\+/g, " "); + try { + decoded = decodeURIComponent(decoded); + } catch { + // Malformed percent escapes stay visible to the conservative ASCII fold. + } + return decoded.replace(/[^A-Za-z0-9]/g, "").toLowerCase(); +} + +function isSensitiveUrlQueryKey(key: string): boolean { + const normalized = normalizeUrlQueryKey(key); + return ( + normalized === "sig" || + normalized === "signature" || + normalized === "key" || + normalized === "apikey" || + normalized === "token" || + normalized === "accesstoken" || + normalized === "refreshtoken" || + normalized === "credential" || + normalized === "password" || + normalized === "secret" || + normalized === "awsaccesskeyid" || + normalized === "googleaccessid" || + normalized === "xamzcredential" || + normalized === "xamzsignature" || + normalized === "xamzsecuritytoken" || + normalized === "xgoogcredential" || + normalized === "xgoogsignature" + ); +} + +function redactUrlSegment(segment: string): string { + const schemeEnd = segment.indexOf("//") + 2; + let authorityEnd = segment.length; + for (const delimiter of ["/", "?", "#"]) { + const candidate = segment.indexOf(delimiter, schemeEnd); + if (candidate >= 0) authorityEnd = Math.min(authorityEnd, candidate); + } + + let redacted = segment; + const userInfoEnd = segment.lastIndexOf("@", authorityEnd); + if (userInfoEnd >= schemeEnd) { + redacted = `${segment.slice(0, schemeEnd)}[REDACTED]@${segment.slice(userInfoEnd + 1)}`; + } + + URL_QUERY_PARAM_RE.lastIndex = 0; + return redacted.replace(URL_QUERY_PARAM_RE, (match, separator: string, key: string) => + isSensitiveUrlQueryKey(key) ? `${separator}redacted=[REDACTED]` : match + ); +} + +function redactSensitiveUrlCredentials(value: string): string { + HTTP_URL_RE.lastIndex = 0; + const parts: string[] = []; + let copyStart = 0; + let match = HTTP_URL_RE.exec(value); + while (match) { + const start = match.index; + let end = HTTP_URL_RE.lastIndex; + while (end < value.length && !isUrlTerminator(value[end])) end++; + const segment = value.slice(start, end); + const redacted = redactUrlSegment(segment); + if (redacted !== segment) { + parts.push(value.slice(copyStart, start), redacted); + copyStart = end; + } + HTTP_URL_RE.lastIndex = Math.max(end, HTTP_URL_RE.lastIndex); + match = HTTP_URL_RE.exec(value); + } + if (parts.length === 0) return value; + parts.push(value.slice(copyStart)); + return parts.join(""); +} + +function redactKnownCredentialPatterns(value: string): string { + let redacted = value; + for (const pattern of CREDENTIAL_PATTERNS) { + if (pattern.name === "auth_header") continue; + pattern.regex.lastIndex = 0; + redacted = redacted.replace(pattern.regex, "[REDACTED]"); + } + return redacted; +} + +export function redactSensitiveErrorText(value: string): string { + const normalized = normalizeSecurityEscapes( + value, + false, + MAX_ERROR_LEN + MAX_ERROR_SCAN_HEADROOM + ); + const catalogRedacted = redactKnownCredentialPatterns(redactSensitiveUrlCredentials(normalized)); + const commonCredentialsRedacted = redactBase64DataUrls(redactPrivateKeyPemBlocks(catalogRedacted)) + .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "$1 [REDACTED]") + .replace(STRONG_CREDENTIAL_TOKEN_GLOBAL, "[REDACTED]"); + return redactLabeledCredentialAssignments(commonCredentialsRedacted); +} + +export function containsSensitiveErrorCredential(value: string): boolean { + const normalized = normalizeSecurityEscapes( + value, + false, + MAX_ERROR_LEN + MAX_ERROR_SCAN_HEADROOM + ); + const directRedacted = redactKnownCredentialPatterns(redactSensitiveUrlCredentials(normalized)) + .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "$1 [REDACTED]") + .replace(STRONG_CREDENTIAL_TOKEN_GLOBAL, "[REDACTED]"); + if (directRedacted !== normalized) return true; + if ( + /(?:^|\s)--(?:api[-_]?key|token|password|secret)\s+(?:"[^"]*"|'[^']*'|\S+)/i.test(normalized) + ) { + return true; + } + return /(?:api[_-]?key|access[_-]?token|refresh[_-]?token|authorization|cookie|secret)["']?\s*[:=]\s*["']?[^"'\\,\s}]{6,}/i.test( + normalized + ); +} + +function coerceErrorText(value: unknown): string { + if (typeof value === "string") return value; + if (value === null || value === undefined) return ""; + try { + return String(value); + } catch { + // Fail closed when an attacker-controlled toString/valueOf accessor throws. + return ""; + } +} + +function truncateSanitizedErrorText(value: string): string { + if (value.length <= MAX_ERROR_LEN) return value; + const markerStart = value.lastIndexOf("[REDACTED", MAX_ERROR_LEN); + const markerEnd = markerStart >= 0 ? value.indexOf("]", markerStart) : -1; + if ( + markerStart >= 0 && + markerStart < MAX_ERROR_LEN && + markerEnd >= MAX_ERROR_LEN && + markerEnd - markerStart <= 128 + ) { + const marker = value.slice(markerStart, markerEnd + 1); + return `${value.slice(0, MAX_ERROR_LEN - marker.length)}${marker}`; + } + return value.slice(0, MAX_ERROR_LEN); +} + +/** + * Strip stack-trace tails, credentials, and absolute source paths from a + * client-visible error message. + */ +function sanitizeErrorMessageWithStackPolicy( + message: unknown, + stripStackTail: (value: string) => string +): string { + let str = coerceErrorText(message); + if (str.length > MAX_ERROR_LEN + MAX_ERROR_SCAN_HEADROOM) { + str = str.slice(0, MAX_ERROR_LEN + MAX_ERROR_SCAN_HEADROOM); + } + // Preserve quote provenance until hidden labels/delimiters have been + // exposed and redacted, then decode safe quote escapes in the clean text. + // Raw URI credentials must be projected before the path tokenizer consumes + // the URI tail; Windows path evidence still stays intact until after this + // credential-only pass and is redacted before escape normalization. + str = redactKnownCredentialPatterns(redactSensitiveUrlCredentials(stripStackTail(str))); + str = redactErrorPaths(str); + str = redactSensitiveErrorText(str); + str = truncateSanitizedErrorText(str); + str = normalizeSecurityEscapes(str, false); + str = redactSensitiveErrorText(redactErrorPaths(stripStackTail(str))); + str = normalizeSecurityEscapes(str, true); + str = redactSensitiveErrorText(redactErrorPaths(stripStackTail(str))); + return hasResidualSecurityEscape(str) ? "[REDACTED]" : str.trimEnd(); +} + +export function sanitizeErrorMessage(message: unknown): string { + return sanitizeErrorMessageWithStackPolicy(message, stripErrorStackTail); +} + +function sanitizePassthroughErrorMessage(message: unknown): string { + return sanitizeErrorMessageWithStackPolicy(message, stripRecognizedErrorStackTail); +} + +const BLOCKED_KEYS = + /stack|trace|path|file|cwd|dir|password|secret|token|key|authorization|cookie|credential|session(?!_?(?:count|status)$)/i; +const BLOCKED_CREDENTIAL_ALIAS_KEYS = + /^(?:cf_clearance|__cf_bm|_cfuvid|_puid|sso|sso-rw|arena-auth-prod-v1(?:\.\d+)?)$/i; +const PROTOTYPE_CONTROL_KEYS = new Set(["__proto__", "constructor", "prototype"]); +const MAX_DEPTH = 4; +const MAX_UPSTREAM_KEY_LEN = 256; +type UpstreamClassificationKey = "code" | "reason" | "status" | "type"; +const SAFE_UPSTREAM_STATUS_IDENTIFIERS = new Set([ + "ABORTED", + "ALREADY_EXISTS", + "CANCELLED", + "DATA_LOSS", + "DEADLINE_EXCEEDED", + "FAILED_PRECONDITION", + "INTERNAL", + "INVALID_ARGUMENT", + "NOT_FOUND", + "OK", + "OUT_OF_RANGE", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "UNAUTHENTICATED", + "UNAVAILABLE", + "UNIMPLEMENTED", + "UNKNOWN", +]); +const SAFE_UPSTREAM_ERROR_IDENTIFIERS = new Set([ + "api_error", + "auth_error", + "authentication_error", + "bad_gateway", + "bad_request", + "billing_error", + "context_length_exceeded", + "error", + "gateway_timeout", + "insufficient_quota", + "invalid_api_key", + "invalid_request", + "invalid_request_error", + "model_not_found", + "not_found", + "payment_required", + "permission_error", + "provider_error", + "quota_exhausted", + "rate_limit_error", + "rate_limit_exceeded", + "server_error", + "upstream_error", + "upstream_timeout", +]); + +function describeOpaqueBinaryDetail(value: ArrayBuffer | ArrayBufferView): string { + return `[binary ${value.byteLength} bytes]`; +} + +function normalizeUpstreamClassificationKey(key: string): UpstreamClassificationKey | null { + const normalized = key.replace(/[-_]/g, "").toLowerCase(); + if (normalized === "code" || normalized === "errorcode") return "code"; + if (normalized === "reason" || normalized === "errorreason") return "reason"; + if ( + normalized === "status" || + normalized === "statuscode" || + normalized === "errorstatus" || + normalized === "errorstatuscode" + ) { + return "status"; + } + if (normalized === "type" || normalized === "errortype" || normalized === "subtype") { + return "type"; + } + return null; +} + +function projectUpstreamErrorIdentifier(key: UpstreamClassificationKey, value: unknown): unknown { + if (typeof value === "number") { + if (!Number.isInteger(value)) return undefined; + if (key === "code" && value >= 0 && value <= 16) return value; + return (key === "code" || key === "status") && value >= 100 && value <= 599 ? value : undefined; + } + if (typeof value !== "string") return undefined; + if (key === "status" && SAFE_UPSTREAM_STATUS_IDENTIFIERS.has(value.toUpperCase())) { + return value; + } + if ( + /^[1-5]\d{2}$/.test(value) || + /^HTTP_[1-5]\d{2}$/i.test(value) || + SAFE_UPSTREAM_ERROR_IDENTIFIERS.has(value.toLowerCase()) + ) { + return value; + } + if (key === "type") return "upstream_error"; + if (key === "code") return ""; + return undefined; +} + +function isSafeUpstreamDetailKey(key: string): boolean { + if ( + key.length === 0 || + key.length > MAX_UPSTREAM_KEY_LEN || + BLOCKED_KEYS.test(key) || + BLOCKED_CREDENTIAL_ALIAS_KEYS.test(key) || + PROTOTYPE_CONTROL_KEYS.has(key.toLowerCase()) + ) { + return false; + } + return sanitizeErrorMessage(key) === key; +} + +/** + * Recursively sanitize an arbitrary JSON value from an upstream provider body. + * Unsafe keys are dropped rather than renamed so sanitized-key collisions + * cannot restore a secret under a public placeholder. + */ +function sanitizeUpstreamDetailsInternal( + value: unknown, + depth: number, + preserveSafeMultiline: boolean, + projectClassification: boolean +): unknown { + if (depth > MAX_DEPTH) return "[truncated]"; + if (value === null || value === undefined) return null; + if (typeof value === "string") { + return preserveSafeMultiline + ? sanitizePassthroughErrorMessage(value) + : sanitizeErrorMessage(value); + } + if (typeof value === "number" || typeof value === "boolean") return value; + if (typeof value === "object") { + try { + if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) { + return describeOpaqueBinaryDetail(value); + } + if (Array.isArray(value)) { + return value + .slice(0, 32) + .map((entry) => + sanitizeUpstreamDetailsInternal( + entry, + depth + 1, + preserveSafeMultiline, + projectClassification + ) + ); + } + const out = Object.create(null) as Record; + for (const [key, entryValue] of Object.entries(value as Record)) { + if (!isSafeUpstreamDetailKey(key)) continue; + const normalizedKey = key.toLowerCase(); + const classificationKey = normalizeUpstreamClassificationKey(normalizedKey); + if (projectClassification && classificationKey) { + const projected = projectUpstreamErrorIdentifier(classificationKey, entryValue); + if (projected !== undefined) out[key] = projected; + continue; + } + const childProjectsClassification = + normalizedKey === "error" || + normalizedKey === "errors" || + normalizedKey === "warning" || + normalizedKey === "warnings"; + out[key] = sanitizeUpstreamDetailsInternal( + entryValue, + depth + 1, + preserveSafeMultiline, + childProjectsClassification + ); + } + return out; + } catch { + return null; + } + } + return null; +} + +export function sanitizeUpstreamDetails(value: unknown, depth = 0): unknown { + return sanitizeUpstreamDetailsInternal(value, depth, false, depth === 0); +} + +/** Provider-only projection that preserves safe multiline capability wording. */ +export function sanitizePassthroughUpstreamDetails(value: unknown, depth = 0): unknown { + return sanitizeUpstreamDetailsInternal(value, depth, true, depth === 0); +} diff --git a/open-sse/utils/passthroughTailProcessor.ts b/open-sse/utils/passthroughTailProcessor.ts index ab45fb5474..3fa75e58c6 100644 --- a/open-sse/utils/passthroughTailProcessor.ts +++ b/open-sse/utils/passthroughTailProcessor.ts @@ -9,6 +9,7 @@ import { stripResponsesLifecycleEcho, } from "./responsesStreamHelpers.ts"; import { getAnyReasoningValue } from "./reasoningFields.ts"; +import { projectStreamFailureEvent, type StreamFailurePayload } from "./streamErrorFormat.ts"; type JsonRecord = Record; @@ -47,6 +48,7 @@ export type PassthroughTailProcessorContext = { hasPassthroughToolCalls: () => boolean; toResponsesCompletedWithToolCalls: (parsed: JsonRecord) => JsonRecord; restoreOpenAIToolNames: (parsed: JsonRecord) => boolean; + abortFailure: (failure: StreamFailurePayload, publicMessage: string) => void; }; function asRecord(value: unknown): JsonRecord { @@ -284,7 +286,13 @@ export function processBufferedPassthroughLine( context.updateClaudeEmptyResponseLifecycle(parsedPassthroughData); } - const parsed = parsedPassthroughData as JsonRecord; + const projectedFailure = projectStreamFailureEvent(parsedPassthroughData); + const parsed = projectedFailure + ? projectedFailure.publicPayload + : (parsedPassthroughData as JsonRecord); + if (projectedFailure) { + output = `data: ${JSON.stringify(parsed)}\n\n`; + } if (context.sanitizeUsagePayload(parsed)) { output = `data: ${JSON.stringify(parsed)}\n\n`; } @@ -301,6 +309,14 @@ export function processBufferedPassthroughLine( } context.pushClientPayload(parsed); + + output = context.passthroughEventPrefix.prefixData(output, line); + context.emitConvertedOutput(output); + if (projectedFailure) { + context.abortFailure(projectedFailure.internalFailure, projectedFailure.publicMessage); + return true; + } + return false; } output = context.passthroughEventPrefix.prefixData(output, line); diff --git a/open-sse/utils/responsesFailureOutput.ts b/open-sse/utils/responsesFailureOutput.ts new file mode 100644 index 0000000000..e085ba3b69 --- /dev/null +++ b/open-sse/utils/responsesFailureOutput.ts @@ -0,0 +1,70 @@ +type JsonRecord = Record; + +export type ResponsesFailureOutputStringField = "id" | "text" | "refusal"; + +export type ResponsesFailureOutputStringProjector = ( + field: ResponsesFailureOutputStringField, + value: string +) => string; + +function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +/** + * Retain only public assistant text/refusal output from a failed Responses payload. + * Failure envelopes may contain reasoning, tool arguments, annotations, commentary, + * or provider diagnostics, so every retained field is reconstructed explicitly. + */ +export function projectResponsesFailureOutput( + value: unknown, + projectString: ResponsesFailureOutputStringProjector +): JsonRecord[] { + if (!Array.isArray(value)) return []; + + const output: JsonRecord[] = []; + for (const item of value) { + const record = asRecord(item); + if (record.type !== "message" || record.role !== "assistant" || record.phase === "commentary") { + continue; + } + + const content: JsonRecord[] = []; + if (Array.isArray(record.content)) { + for (const part of record.content) { + const contentPart = asRecord(part); + if (contentPart.phase === "commentary") continue; + if (contentPart.type === "output_text" && typeof contentPart.text === "string") { + content.push({ + type: "output_text", + text: projectString("text", contentPart.text), + // Preserve the required Responses schema without forwarding any + // untrusted citation/file metadata supplied by the provider. + annotations: [], + }); + } else if (contentPart.type === "refusal" && typeof contentPart.refusal === "string") { + content.push({ + type: "refusal", + refusal: projectString("refusal", contentPart.refusal), + }); + } + } + } + + const projected: JsonRecord = { + type: "message", + role: "assistant", + content, + }; + if (typeof record.id === "string") projected.id = projectString("id", record.id); + if ( + record.status === "in_progress" || + record.status === "completed" || + record.status === "incomplete" + ) { + projected.status = record.status; + } + output.push(projected); + } + return output; +} diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index d33cc8a526..f7b01b1464 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -50,9 +50,11 @@ import { parseTextualToolCallCandidate, isValidToolCallHeaderPrefix } from "./te import { stripObfuscationZeroWidth } from "./zeroWidth.ts"; import { formatTranslatedStreamError, - normalizeStreamFailurePayload, + prepareTranslatedStreamFailure, + projectStreamFailureEvent, type StreamFailurePayload, } from "./streamErrorFormat.ts"; +import { createStreamFailureAborter } from "./streamFailureBoundary.ts"; import { recordToolLatency } from "../services/toolLatencyTracker.ts"; import { extractToolSchemaMap } from "../translator/response/openai-responses/toolSchemas.ts"; import { @@ -178,8 +180,6 @@ type StreamOptions = { * codex-compatible `namespace` + `name` fields. */ requestToolIdentityMap?: Map | null; - /** High water mark for the TransformStream internal buffer (default: 16384) */ - highWaterMark?: number; }; type TranslateState = ReturnType & { @@ -1175,7 +1175,39 @@ export function createSSEStream(options: StreamOptions = {}) { } }; - const highWaterMark = options.highWaterMark ?? 16384; + const abortStreamFailure = createStreamFailureAborter({ + onFailure, + onComplete, + getUsage: () => state?.usage, + timing, + buildProviderPayload: () => + providerPayloadCollector.build(providerPayloadCollector.getSummary(), { + includeEvents: false, + }), + buildClientPayload: (body) => clientPayloadCollector.build(body, { includeEvents: false }), + clearIdleTimer, + clearPendingRequest: clearPendingRequestFromStream, + markPendingRequestCleared, + model, + }); + + const emitTranslatedFailureAndAbort = ( + controller: TransformStreamDefaultController, + payload: unknown + ): boolean => { + const failure = prepareTranslatedStreamFailure(payload); + if (!failure) return false; + providerPayloadCollector.push(failure.providerPayload); + const output = formatTranslatedStreamError(failure.record, sourceFormat); + reqLogger?.appendConvertedChunk?.(output); + forward(controller, encoder.encode(output)); + upstreamErrorForwarded = true; + doneSent = true; + abortStreamFailure(controller, failure.internalFailure, failure.publicMessage, { + notifyComplete: true, + }); + return true; + }; return new TransformStream( { @@ -1241,6 +1273,7 @@ export function createSSEStream(options: StreamOptions = {}) { let injectedUsage = false; let clientPayload: unknown = null; let failurePayload: StreamFailurePayload | null = null; + let publicFailureMessage: string | null = null; if (skipPassthroughEvent) { if (!trimmed) { @@ -1328,6 +1361,14 @@ export function createSSEStream(options: StreamOptions = {}) { if (trimmed.startsWith("data:") && trimmed.slice(5).trim() !== "[DONE]") { try { let parsed = parsedPassthroughData ?? JSON.parse(trimmed.slice(5).trim()); + const projectedFailure = projectStreamFailureEvent(parsed); + if (projectedFailure) { + parsed = projectedFailure.publicPayload; + failurePayload = projectedFailure.internalFailure; + publicFailureMessage = projectedFailure.publicMessage; + output = `data: ${JSON.stringify(parsed)}\n\n`; + injectedUsage = true; + } // Some upstream Responses-compatible providers leak an initial Chat Completions // bootstrap chunk (assistant role + empty content) before emitting proper @@ -1484,9 +1525,6 @@ export function createSSEStream(options: StreamOptions = {}) { ); } } - if (parsed.type === "response.failed") { - failurePayload = normalizeStreamFailurePayload(parsed); - } if ( parsed.type === "response.reasoning_summary_text.delta" || parsed.type === "response.reasoning_summary_text.done" || @@ -1810,20 +1848,22 @@ export function createSSEStream(options: StreamOptions = {}) { const rawDelta = parsed.choices?.[0]?.delta; const hadReasoningAlias = hasUnsupportedReasoningSignal(rawDelta); - parsed = sanitizeStreamingChunk(parsed); - if ( - parsed && - typeof parsed === "object" && - !Array.isArray(parsed) && - (parsed as Record)[OMIT_STREAMING_CHUNK_MARKER] === true - ) { - continue; + if (!projectedFailure) { + parsed = sanitizeStreamingChunk(parsed); + if ( + parsed && + typeof parsed === "object" && + !Array.isArray(parsed) && + (parsed as Record)[OMIT_STREAMING_CHUNK_MARKER] === true + ) { + continue; + } } const restoredOpenAIToolName = restoreOpenAIToolNames(parsed, toolNameMap); const idFixed = hadNonStringTopLevelId ? false : fixInvalidId(parsed); - if (!hasValuableContent(parsed, FORMATS.OPENAI)) { + if (!projectedFailure && !hasValuableContent(parsed, FORMATS.OPENAI)) { continue; } @@ -2052,20 +2092,10 @@ export function createSSEStream(options: StreamOptions = {}) { reqLogger?.appendConvertedChunk?.(output); forward(controller, encoder.encode(output)); if (failurePayload) { - let failureHandled = false; - if (onFailure) { - try { - failureHandled = onFailure(failurePayload) === true; - } catch (e) { - console.debug(`[STREAM] onFailure callback error:`, e); - } - } - clearIdleTimer(); - if (!failureHandled) { - clearPendingRequestFromStream(); - } - controller.error( - markPendingRequestCleared(new Error(failurePayload.message || "Upstream failure")) + abortStreamFailure( + controller, + failurePayload, + publicFailureMessage || "Upstream failure" ); return; } @@ -2087,14 +2117,7 @@ export function createSSEStream(options: StreamOptions = {}) { if (upstreamErrorForwarded) continue; - if (parsed.error) { - const output = formatTranslatedStreamError(parsed, sourceFormat); - reqLogger?.appendConvertedChunk?.(output); - forward(controller, encoder.encode(output)); - upstreamErrorForwarded = true; - doneSent = true; - continue; - } + if (emitTranslatedFailureAndAbort(controller, parsed)) return; // #5786 — drop replayed Responses-API events (identical/lower sequence_number // re-sent on an upstream reconnect) so their deltas are not glued twice into @@ -2356,6 +2379,8 @@ export function createSSEStream(options: StreamOptions = {}) { ]) as JsonRecord, restoreOpenAIToolNames: (parsed: JsonRecord) => restoreOpenAIToolNames(parsed, toolNameMap), + abortFailure: (failure: StreamFailurePayload, publicMessage: string) => + abortStreamFailure(controller, failure, publicMessage), }; for (const line of normalizedTailLines) { @@ -2369,12 +2394,18 @@ export function createSSEStream(options: StreamOptions = {}) { clearPendingPassthroughEvent(); } else if (buffer) { let output = buffer; + let bufferedProjectedFailure: ReturnType = null; if (buffer.startsWith("data:") && !buffer.startsWith("data: ")) { output = "data: " + buffer.slice(5); } - const bufferedPayload = parseSSELine(bufferedLine); + let bufferedPayload = parseSSELine(bufferedLine); if (bufferedPayload) { providerPayloadCollector.push(bufferedPayload); + bufferedProjectedFailure = projectStreamFailureEvent(bufferedPayload); + if (bufferedProjectedFailure) { + bufferedPayload = bufferedProjectedFailure.publicPayload; + output = `data: ${JSON.stringify(bufferedPayload)}\n\n`; + } if (sanitizeUsagePayloadForRequest(bufferedPayload, body, clientResponseFormat)) output = `data: ${JSON.stringify(bufferedPayload)}\n\n`; if ( @@ -2423,6 +2454,14 @@ export function createSSEStream(options: StreamOptions = {}) { } reqLogger?.appendConvertedChunk?.(output); forward(controller, encoder.encode(output)); + if (bufferedProjectedFailure) { + abortStreamFailure( + controller, + bufferedProjectedFailure.internalFailure, + bufferedProjectedFailure.publicMessage + ); + return; + } } if (shouldInjectClaudeEmptyResponseOnFlush(claudeEmptyResponseLifecycle)) { @@ -2673,6 +2712,7 @@ export function createSSEStream(options: StreamOptions = {}) { if (buffer.trim()) { const parsed = parseSSELine(buffer.trim()); if (parsed && !parsed.done) { + if (emitTranslatedFailureAndAbort(controller, parsed)) return; providerPayloadCollector.push(parsed); // Extract usage from remaining buffer — if the usage-bearing event // (e.g. response.completed) is the last SSE line, it ends up here @@ -2737,58 +2777,9 @@ export function createSSEStream(options: StreamOptions = {}) { // terminal signal for the client. } - let failureHandled = false; - if (onFailure) { - try { - timing.markInterrupted(); - failureHandled = - onFailure({ - status: err.status, - message: err.message, - code: err.code, - type: err.type, - }) === true; - } catch (e) { - console.debug(`[STREAM] onFailure callback error (${model || "unknown"}):`, e); - } - } - const errorBody = buildErrorBody(err.status, err.message); - if (onComplete) { - try { - onComplete({ - status: err.status, - usage: state?.usage, - responseBody: errorBody, - ttft: timing.ttftMs(), - itlMs: timing.avgItlMs(), - interrupted: timing.interrupted, - error: err.message, - errorCode: err.code, - providerPayload: providerPayloadCollector.build( - providerPayloadCollector.getSummary(), - { includeEvents: false } - ), - clientPayload: clientPayloadCollector.build(errorBody, { - includeEvents: false, - }), - }); - failureHandled = true; - } catch (e) { - console.debug( - `[STREAM] onComplete callback error in error path (${model || "unknown"}):`, - e - ); - } - } - - clearIdleTimer(); - if (!failureHandled) { - clearPendingRequestFromStream(); - } - controller.error( - markPendingRequestCleared(new Error(err.message || "Upstream failure")) - ); + const publicErrorMessage = errorBody.error.message; + abortStreamFailure(controller, err, publicErrorMessage, { notifyComplete: true }); return; } @@ -2996,8 +2987,8 @@ export function createSSEStream(options: StreamOptions = {}) { clearIdleTimer(); }, }, - { highWaterMark }, - { highWaterMark } + { highWaterMark: 16384 }, + { highWaterMark: 16384 } ); } @@ -3019,8 +3010,7 @@ export function createSSETransformStreamWithLogger( copilotCompatibleReasoning = false, suppressThinkClose = false, customToolNames: ReadonlySet = new Set(), - requestToolIdentityMap: Map | null = null, - highWaterMark?: number + requestToolIdentityMap: Map | null = null ) { return createSSEStream({ mode: STREAM_MODE.TRANSLATE, @@ -3039,7 +3029,6 @@ export function createSSETransformStreamWithLogger( suppressThinkClose, customToolNames, requestToolIdentityMap, - highWaterMark, }); } @@ -3054,8 +3043,7 @@ export function createPassthroughStreamWithLogger( apiKeyInfo: unknown = null, onFailure: ((payload: StreamFailurePayload) => boolean | void | Promise) | null = null, clientResponseFormat: string | null = null, - requestToolIdentityMap: Map | null = null, - highWaterMark?: number + requestToolIdentityMap: Map | null = null ) { return createSSEStream({ mode: STREAM_MODE.PASSTHROUGH, @@ -3070,7 +3058,6 @@ export function createPassthroughStreamWithLogger( onFailure, clientResponseFormat, requestToolIdentityMap, - highWaterMark, }); } diff --git a/open-sse/utils/streamErrorFormat.ts b/open-sse/utils/streamErrorFormat.ts index 56b747f4e4..05a065a864 100644 --- a/open-sse/utils/streamErrorFormat.ts +++ b/open-sse/utils/streamErrorFormat.ts @@ -1,5 +1,6 @@ import { FORMATS } from "../translator/formats.ts"; -import { buildErrorBody } from "./error.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "./error.ts"; +import { projectResponsesFailureOutput } from "./responsesFailureOutput.ts"; /** * Upstream stream-failure normalization + client-format error framing. @@ -17,10 +18,125 @@ export type StreamFailurePayload = { type?: string; }; +export type ProjectedStreamFailureEvent = { + internalFailure: StreamFailurePayload; + publicMessage: string; + publicPayload: JsonRecord; +}; + +export type PreparedTranslatedStreamFailure = { + record: JsonRecord; + providerPayload: JsonRecord; + internalFailure: StreamFailurePayload; + publicMessage: string; +}; + +export function projectCompletedStreamError( + failure: StreamFailurePayload | null | undefined +): JsonRecord | null { + if (!failure) return null; + const status = Number.isInteger(failure.status) ? failure.status : 502; + return buildErrorBody(status, failure.message, undefined, { + type: failure.type ?? "server_error", + code: String(failure.status ?? 502), + }).error; +} + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } +const RESPONSES_FAILURE_SCALAR_FIELDS = [ + "id", + "object", + "created_at", + "completed_at", + "background", + "model", + "max_output_tokens", + "max_tool_calls", + "parallel_tool_calls", + "previous_response_id", + "service_tier", + "store", + "temperature", + "top_p", + "truncation", +] as const; + +const ABSOLUTE_PATH_SEGMENT = + /(?:^|[\\/])(?:Users|app|etc|home|opt|private|root|srv|tmp|usr|var|workspace)[\\/]/i; + +function projectResponsesFailureString(key: string, value: string): string { + const sanitized = sanitizeErrorMessage(value); + if (sanitized !== value || ABSOLUTE_PATH_SEGMENT.test(value)) return "[REDACTED]"; + if ( + (key === "id" || key === "previous_response_id") && + !/^[A-Za-z0-9][\w.:-]{0,511}$/.test(value) + ) { + return "[REDACTED]"; + } + if (key === "model" && !/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(value)) { + return "[REDACTED]"; + } + return sanitized; +} + +function projectResponsesFailureUsage(value: unknown): JsonRecord | null { + const usage = asRecord(value); + const projected: JsonRecord = {}; + for (const key of ["input_tokens", "output_tokens", "total_tokens"] as const) { + if (typeof usage[key] === "number" && Number.isFinite(usage[key])) { + projected[key] = usage[key]; + } + } + const allowedDetailFields = { + input_tokens_details: new Set(["cached_tokens"]), + output_tokens_details: new Set([ + "reasoning_tokens", + "accepted_prediction_tokens", + "rejected_prediction_tokens", + ]), + } as const; + for (const key of ["input_tokens_details", "output_tokens_details"] as const) { + const details = asRecord(usage[key]); + const projectedDetails = Object.fromEntries( + Object.entries(details).filter( + ([detailKey, detail]) => + allowedDetailFields[key].has(detailKey) && + typeof detail === "number" && + Number.isFinite(detail) + ) + ); + if (Object.keys(projectedDetails).length > 0) projected[key] = projectedDetails; + } + return Object.keys(projected).length > 0 ? projected : null; +} + +function projectResponsesFailureObject(response: JsonRecord, publicError: JsonRecord): JsonRecord { + const projected: JsonRecord = { status: "failed", error: publicError }; + + // A failed Responses event is an error boundary, so copy only documented protocol + // fields with their scalar shapes. Spreading the upstream object would also publish + // provider-only siblings such as diagnostics, settings, raw messages, or stack traces. + for (const key of RESPONSES_FAILURE_SCALAR_FIELDS) { + const value = response[key]; + if (typeof value === "string") projected[key] = projectResponsesFailureString(key, value); + else if (value === null || typeof value === "number" || typeof value === "boolean") + projected[key] = value; + } + if (Array.isArray(response.output)) { + projected.output = projectResponsesFailureOutput( + response.output, + projectResponsesFailureString + ); + } + const usage = projectResponsesFailureUsage(response.usage); + if (usage) projected.usage = usage; + if ("last_error" in response) projected.last_error = publicError; + return projected; +} + function toStreamFailureStatus(value: unknown): number | null { if (typeof value === "number" && Number.isInteger(value) && value >= 400 && value <= 599) { return value; @@ -48,19 +164,30 @@ function looksLikeStreamRateLimit(code: string, type: string, message: string): export function normalizeStreamFailurePayload(payload: unknown): StreamFailurePayload | null { const record = payload && typeof payload === "object" ? (payload as JsonRecord) : {}; const response = asRecord(record.response); - const error = Object.keys(asRecord(response.error)).length - ? asRecord(response.error) - : Object.keys(asRecord(record.error)).length - ? asRecord(record.error) - : record; + const responseError = response.error; + const responseLastError = response.last_error; + const rootError = record.error; + const error = Object.keys(asRecord(responseError)).length + ? asRecord(responseError) + : Object.keys(asRecord(responseLastError)).length + ? asRecord(responseLastError) + : Object.keys(asRecord(rootError)).length + ? asRecord(rootError) + : record; const code = typeof error.code === "string" ? error.code : "upstream_error"; const type = typeof error.type === "string" ? error.type : undefined; const message = typeof error.message === "string" && error.message.trim() ? error.message - : typeof record.message === "string" && record.message.trim() - ? record.message - : "Upstream failure"; + : typeof responseError === "string" && responseError.trim() + ? responseError + : typeof responseLastError === "string" && responseLastError.trim() + ? responseLastError + : typeof rootError === "string" && rootError.trim() + ? rootError + : typeof record.message === "string" && record.message.trim() + ? record.message + : "Upstream failure"; const status = toStreamFailureStatus(error.status_code) ?? toStreamFailureStatus(error.status) ?? @@ -78,6 +205,80 @@ export function normalizeStreamFailurePayload(payload: unknown): StreamFailurePa }; } +export function prepareTranslatedStreamFailure( + payload: unknown +): PreparedTranslatedStreamFailure | null { + const record = asRecord(payload); + const projected = projectStreamFailureEvent(record); + if (!projected && !record.error) return null; + return { + record, + providerPayload: projected?.publicPayload ?? record, + internalFailure: projected?.internalFailure ?? + normalizeStreamFailurePayload(record) ?? { + status: 502, + message: "Upstream failure", + code: "stream_error", + type: "server_error", + }, + publicMessage: projected?.publicMessage || "Upstream failure", + }; +} + +/** + * Project same-format upstream failure events before they cross the client/log boundary. + * + * `internalFailure` intentionally retains the raw provider wording: account fallback uses it + * to classify quota/reset hints before the persistence seam sanitizes the stored message. + * `publicPayload` is a separate protocol-preserving object whose failure subtrees are rebuilt by + * the canonical public boundary. Callers must never forward the raw payload for these events. + */ +export function projectStreamFailureEvent(payload: unknown): ProjectedStreamFailureEvent | null { + const record = asRecord(payload); + const response = asRecord(record.response); + const hasRootError = + Object.keys(asRecord(record.error)).length > 0 || + (typeof record.error === "string" && record.error.trim().length > 0); + const isResponsesFailure = + record.type === "response.failed" || + (record.type === "response.completed" && response.status === "failed"); + const isClaudeFailure = record.type === "error"; + if (!isResponsesFailure && !isClaudeFailure && !hasRootError) return null; + + const internalFailure = normalizeStreamFailurePayload(record); + if (!internalFailure) return null; + + const publicError = buildErrorBody(internalFailure.status, internalFailure.message, undefined, { + type: internalFailure.type ?? "server_error", + code: internalFailure.code ?? "stream_error", + }).error; + let publicPayload: JsonRecord; + if (isResponsesFailure) { + // Preserve protocol metadata and partial `output[].content[]` without passing output + // through a bounded-depth details sanitizer, while excluding arbitrary diagnostic siblings. + const publicResponse = projectResponsesFailureObject(response, publicError); + publicPayload = { + type: record.type, + response: publicResponse, + ...(typeof record.sequence_number === "number" + ? { sequence_number: record.sequence_number } + : {}), + }; + } else if (isClaudeFailure) { + publicPayload = { type: "error", error: publicError }; + } else { + // OpenAI-compatible HTTP-200 streams commonly emit a bare `{ error: ... }` frame. + // Rebuild the complete public envelope so provider-only fields cannot cross the wire. + publicPayload = { error: publicError }; + } + + return { + internalFailure, + publicMessage: publicError.message, + publicPayload, + }; +} + export function formatTranslatedStreamError(payload: unknown, sourceFormat?: string): string { const failure = normalizeStreamFailurePayload(payload) ?? { status: 502, diff --git a/open-sse/utils/streamFailureBoundary.ts b/open-sse/utils/streamFailureBoundary.ts new file mode 100644 index 0000000000..bb3da7c850 --- /dev/null +++ b/open-sse/utils/streamFailureBoundary.ts @@ -0,0 +1,76 @@ +import { buildErrorBody } from "./error.ts"; +import type { StreamFailurePayload } from "./streamErrorFormat.ts"; +import type { StreamTiming } from "./streamTiming.ts"; + +type CompletePayload = { + status: number; + usage: unknown; + responseBody: unknown; + providerPayload: unknown; + clientPayload: unknown; + error: string; + errorCode?: string; + ttft: number | null; + itlMs: number | null; + interrupted: boolean; +}; + +type AborterContext = { + onFailure?: ((payload: StreamFailurePayload) => boolean | void | Promise) | null; + onComplete?: ((payload: CompletePayload) => void) | null; + getUsage: () => unknown; + timing: StreamTiming; + buildProviderPayload: () => unknown; + buildClientPayload: (body: unknown) => unknown; + clearIdleTimer: () => void; + clearPendingRequest: () => void; + markPendingRequestCleared: (error: Error) => Error; + model?: string | null; +}; + +export function createStreamFailureAborter(context: AborterContext) { + return ( + controller: TransformStreamDefaultController, + failure: StreamFailurePayload, + publicMessage: string, + options: { notifyComplete?: boolean } = {} + ): void => { + let handled = false; + context.timing.markInterrupted(); + if (context.onFailure) { + try { + handled = context.onFailure(failure) === true; + } catch (error) { + console.debug("[STREAM] onFailure callback error:", error); + } + } + let safeMessage = publicMessage || "Upstream failure"; + if (options.notifyComplete && context.onComplete) { + const body = buildErrorBody(failure.status, failure.message); + safeMessage = body.error.message; + try { + context.onComplete({ + status: failure.status, + usage: context.getUsage(), + responseBody: body, + ttft: context.timing.ttftMs(), + itlMs: context.timing.avgItlMs(), + interrupted: context.timing.interrupted, + error: safeMessage, + errorCode: failure.code, + providerPayload: context.buildProviderPayload(), + clientPayload: context.buildClientPayload(body), + }); + handled = true; + } catch (error) { + console.debug( + `[STREAM] onComplete callback error in error path (${context.model || "unknown"}):`, + error + ); + } + } + context.clearIdleTimer(); + if (!handled) context.clearPendingRequest(); + controller.error(context.markPendingRequestCleared(new Error(safeMessage))); + }; +} diff --git a/open-sse/utils/streamFailureFinalization.ts b/open-sse/utils/streamFailureFinalization.ts index 7d4e57ffba..38a740d1fb 100644 --- a/open-sse/utils/streamFailureFinalization.ts +++ b/open-sse/utils/streamFailureFinalization.ts @@ -5,6 +5,7 @@ import { import { HTTP_STATUS } from "../config/constants.ts"; import { buildErrorBody } from "./error.ts"; +import { sanitizeErrorMessage } from "./errorSanitization.ts"; export type StreamCompletionPayload = { status: number; @@ -129,9 +130,7 @@ export function finalizeStreamRequestLog({ } else { console.warn( "finalizeMostRecentPendingRequest failed:", - error && typeof error === "object" && "message" in error - ? (error as { message?: unknown }).message - : error + sanitizeErrorMessage(error) || "Stream request finalization failed" ); } } catch {} @@ -158,12 +157,12 @@ export function createStreamFailureFinalizers({ const status = failure.status || HTTP_STATUS.BAD_GATEWAY; const message = failure.message || "Upstream stream error"; - const code = failure.code || failure.type || String(status); const classification = failure.code || failure.type ? { code: failure.code, type: failure.type } : undefined; + const errorBody = buildErrorBody(status, message, undefined, classification); + const projectedCode = errorBody.error.code || String(status); if (!isFailureCompletionRecorded()) { - const errorBody = buildErrorBody(status, message, undefined, classification); onStreamComplete({ status, usage: null, @@ -171,12 +170,12 @@ export function createStreamFailureFinalizers({ providerPayload: errorBody, clientPayload: errorBody, error: message, - errorCode: code, + errorCode: projectedCode, ttft: 0, }); } - persistFailureUsage(status, code); + persistFailureUsage(status, projectedCode); try { onStreamFailure?.(failure); } catch { diff --git a/open-sse/utils/streamHandler.ts b/open-sse/utils/streamHandler.ts index 7776f2e5e9..841346bcfe 100644 --- a/open-sse/utils/streamHandler.ts +++ b/open-sse/utils/streamHandler.ts @@ -1,6 +1,7 @@ import { trackPendingRequest } from "@/lib/usageDb"; import { STREAM_IDLE_TIMEOUT_MS } from "../config/constants.ts"; import { FORMATS } from "../translator/formats.ts"; +import { buildErrorBody } from "./error.ts"; import { PENDING_REQUEST_CLEARED_MARKER } from "./stream.ts"; import { createCompletedResponsesToolHandoffWatcher } from "./responsesToolHandoff.ts"; import { createStreamContentWatcher, type StreamContentWatcher } from "./streamReadiness.ts"; @@ -187,6 +188,10 @@ function getErrorStatusCode(error: unknown): number { return 502; } +function getPublicErrorMessage(errorMsg: string, statusCode: number): string { + return buildErrorBody(statusCode, errorMsg).error.message; +} + function isDeadlineAbortReason(reason: unknown): reason is Error { return ( reason instanceof Error && @@ -406,7 +411,7 @@ export function createStreamController({ } if (error instanceof Error) { - logStream(`error: ${error.message}`); + logStream(`error: ${getPublicErrorMessage(error.message, getErrorStatusCode(error))}`); return; } logStream("error: unknown"); @@ -452,6 +457,7 @@ export function buildStreamErrorChunks( clientResponseFormat?: string | null ) { const statusMapping = getStreamErrorStatusMapping(statusCode); + const publicErrorMessage = getPublicErrorMessage(errorMsg, statusCode); if (isResponsesClientFormat(clientResponseFormat)) { const errorEvent = { @@ -460,7 +466,7 @@ export function buildStreamErrorChunks( id: null, status: "failed", error: { - message: errorMsg, + message: publicErrorMessage, type: statusMapping.responses.type, code: statusMapping.responses.code, }, @@ -475,7 +481,7 @@ export function buildStreamErrorChunks( type: "error", error: { type: statusMapping.claude.type, - message: errorMsg, + message: publicErrorMessage, }, }; @@ -498,7 +504,7 @@ export function buildStreamErrorChunks( }, ], error: { - message: errorMsg, + message: publicErrorMessage, type: statusMapping.responses.type, code: statusMapping.responses.code, }, diff --git a/open-sse/utils/streamReadiness.ts b/open-sse/utils/streamReadiness.ts index 4bbeef1e0e..908c18725c 100644 --- a/open-sse/utils/streamReadiness.ts +++ b/open-sse/utils/streamReadiness.ts @@ -421,29 +421,67 @@ function prependBufferedChunks( chunks: Uint8Array[], reader: ReadableStreamDefaultReader ): ReadableStream { + let bufferedIndex = 0; + let readInFlight = false; + let cancelRequested = false; + let readerReleased = false; + + const releaseReader = () => { + if (readerReleased) return; + readerReleased = true; + reader.releaseLock(); + }; + + const cancelReader = (reason: unknown) => { + if (cancelRequested) return; + cancelRequested = true; + + try { + // The provider controls this promise and may never settle. Cancellation + // of the replay stream must remain bounded, so cleanup is deliberately + // fire-and-forget while the in-flight read releases the lock in `pull`. + void reader.cancel(reason).catch(() => {}); + } catch { + // A synchronous cancellation failure is cleanup-only; the downstream + // stream has already been cancelled by its consumer. + } + + if (!readInFlight) releaseReader(); + }; + return new ReadableStream({ - async start(controller) { + async pull(controller) { + if (cancelRequested) return; + + // Replay exactly one readiness chunk per demand. Reading the source + // eagerly here would let a subsequent source error clear this queue + // before the consumer has observed the buffered prefix. + if (bufferedIndex < chunks.length) { + controller.enqueue(chunks[bufferedIndex]); + bufferedIndex += 1; + return; + } + + readInFlight = true; try { - for (const chunk of chunks) { - controller.enqueue(chunk); + const { done, value } = await reader.read(); + if (cancelRequested) return; + if (done) { + releaseReader(); + controller.close(); + } else if (value) { + controller.enqueue(value); } - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (value) controller.enqueue(value); - } - - controller.close(); } catch (error) { - controller.error(error); + releaseReader(); + if (!cancelRequested) controller.error(error); } finally { - reader.releaseLock(); + readInFlight = false; + if (cancelRequested) releaseReader(); } }, - async cancel(reason) { - await reader.cancel(reason).catch(() => {}); - reader.releaseLock(); + cancel(reason) { + cancelReader(reason); }, }); } diff --git a/open-sse/utils/upstreamErrorPassthrough.ts b/open-sse/utils/upstreamErrorPassthrough.ts index 21d0c6c964..30fa4ffff2 100644 --- a/open-sse/utils/upstreamErrorPassthrough.ts +++ b/open-sse/utils/upstreamErrorPassthrough.ts @@ -1,12 +1,16 @@ +import { + containsSensitiveErrorCredential, + sanitizePassthroughUpstreamDetails, +} from "./errorSanitization.ts"; + /** * Selective upstream 4xx error passthrough (Claude Code auto-recover contract). * - * Claude Code matches the upstream error WORDING to auto-disable capabilities - * (thinking / output_config) for the rest of the conversation. Wrapping the body - * via buildErrorBody() truncates the message and breaks that recovery. For - * upstream-originated 4xx errors the body is the provider's public API message — - * not our internals — so it is safe and required to relay it verbatim. - * OmniRoute-generated errors MUST keep using buildErrorBody() (Hard Rule #12). + * Claude Code matches upstream error wording to auto-disable capabilities + * (thinking / output_config) for the rest of the conversation. This path keeps + * the wording and JSON shape required for that recovery after applying the + * canonical recursive sanitizer. OmniRoute-generated errors MUST keep using + * buildErrorBody() (Hard Rule #12). */ const PASSTHROUGH_MIN = 400; const PASSTHROUGH_MAX = 499; @@ -17,13 +21,10 @@ const EXCLUDED_STATUSES = new Set([401, 403, 407]); const INTERNAL_LEAK_RE = /\sat\s\/|node_modules|omniroute\//i; // #10898-sec / secret-in-error hardening: some providers echo the offending // request (including an Authorization header or api key) inside a 400/422/429 -// validation body. Passthrough relays the body VERBATIM (the Claude Code -// capability-recovery contract needs the exact wording), so we cannot key-drop -// via sanitizeUpstreamDetails without breaking that contract. Instead, if the -// body actually carries a credential pattern, REFUSE passthrough and let the -// caller fall back to the sanitized buildErrorBody path. Bodies without a -// secret (the overwhelming majority, carrying capability/quota wording) still -// relay verbatim. Mirrors the vocabulary of redactSensitiveErrorText in error.ts. +// validation body. If the body carries a credential pattern, REFUSE passthrough +// before the recursive sanitizer so the caller falls back to buildErrorBody. +// Eligible JSON retains its safe shape and capability/quota wording after the +// recursive projection. Mirrors redactSensitiveErrorText in errorSanitization.ts. const CREDENTIAL_LEAK_RE = /\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}|\bsk-[A-Za-z0-9._-]{8,}|(?:api[_-]?key|access[_-]?token|refresh[_-]?token|authorization|cookie|secret)\\?["']?\s*[:=]\s*\\?["']?[^"'\\,\s}]{6,}/i; @@ -31,10 +32,17 @@ export function shouldPassthroughUpstreamError(statusCode: number, upstreamBody: if (statusCode < PASSTHROUGH_MIN || statusCode > PASSTHROUGH_MAX) return false; if (EXCLUDED_STATUSES.has(statusCode)) return false; if (!upstreamBody || typeof upstreamBody !== "object") return false; - const text = JSON.stringify(upstreamBody); + let text: string | undefined; + try { + text = JSON.stringify(upstreamBody); + } catch { + // Relay only JSON-stable objects; cyclic/BigInt/hostile toJSON bodies fail closed. + return false; + } + if (typeof text !== "string") return false; if (INTERNAL_LEAK_RE.test(text)) return false; // Refuse passthrough when the provider echoed a credential back to us. - if (CREDENTIAL_LEAK_RE.test(text)) return false; + if (CREDENTIAL_LEAK_RE.test(text) || containsSensitiveErrorCredential(text)) return false; return true; } @@ -44,8 +52,18 @@ export function buildPassthroughErrorResponse( headers?: Record ): Response | null { if (!shouldPassthroughUpstreamError(statusCode, upstreamBody)) return null; - return new Response(JSON.stringify(upstreamBody), { - status: statusCode, - headers: { "Content-Type": "application/json", ...(headers || {}) }, - }); + try { + const sanitizedBody = sanitizePassthroughUpstreamDetails(upstreamBody); + const publicBody = + sanitizedBody && typeof sanitizedBody === "object" + ? sanitizedBody + : { error: { message: "Upstream error" } }; + return new Response(JSON.stringify(publicBody), { + status: statusCode, + headers: { "Content-Type": "application/json", ...(headers || {}) }, + }); + } catch { + // A proxy/getter may behave differently between eligibility and projection. + return null; + } } diff --git a/open-sse/utils/upstreamErrorResponse.ts b/open-sse/utils/upstreamErrorResponse.ts new file mode 100644 index 0000000000..1581160900 --- /dev/null +++ b/open-sse/utils/upstreamErrorResponse.ts @@ -0,0 +1,46 @@ +import { buildErrorBody, sanitizeUpstreamDetails } from "./error.ts"; + +interface SanitizedUpstreamErrorResponseOptions { + status: number; + rawBody: string; + fallbackMessage: string; + headers?: Record; +} + +/** + * Preserve a provider's JSON error shape while applying the canonical recursive sanitizer. + * Providers sometimes label plain text as JSON; those bodies use OmniRoute's canonical error + * envelope so the advertised content type always matches the response bytes. + */ +export function buildSanitizedUpstreamErrorResponse({ + status, + rawBody, + fallbackMessage, + headers, +}: SanitizedUpstreamErrorResponseOptions): Response { + const trimmedBody = rawBody.trim(); + + if (trimmedBody) { + try { + const parsedBody: unknown = JSON.parse(trimmedBody); + const serializedBody = JSON.stringify(sanitizeUpstreamDetails(parsedBody)); + if (serializedBody !== undefined) { + return new Response(serializedBody, { + status, + headers: { ...headers, "Content-Type": "application/json" }, + }); + } + } catch { + // Upstreams commonly return text or HTML despite an application/json response header. + // Treat it as an opaque message and use the canonical JSON envelope below. + } + } + + // Non-JSON is an opaque upstream body. Do not echo even sanitized fragments: + // provider HTML/plaintext can contain credentials or implementation details + // outside the patterns the canonical sanitizer knows about. + return new Response(JSON.stringify(buildErrorBody(status, fallbackMessage)), { + status, + headers: { ...headers, "Content-Type": "application/json" }, + }); +} diff --git a/open-sse/utils/upstreamResponseHeaders.ts b/open-sse/utils/upstreamResponseHeaders.ts index 834d354f18..9f6c16d1e6 100644 --- a/open-sse/utils/upstreamResponseHeaders.ts +++ b/open-sse/utils/upstreamResponseHeaders.ts @@ -45,3 +45,36 @@ export function filterUpstreamResponseHeaderEntries( } export const STRIP_UPSTREAM_HEADER_NAMES: ReadonlySet = STRIP_HEADER_NAMES; + +/** + * Response headers that must never be relayed back to a client. + * + * A relay sends its own credential upstream (the bifrost route sends + * `Authorization: Bearer ${BIFROST_API_KEY}` to the sidecar). If that upstream + * echoes the header back — or sets its own session cookie — copying the response + * headers wholesale hands it to whoever holds the relay token + * (GHSA-9m72-44hg-w32g). `set-cookie` matters as much as `authorization`: it is + * a session, and the browser would store it against OUR origin. + */ +const SENSITIVE_RESPONSE_HEADER_NAMES: ReadonlyArray = [ + "authorization", + "proxy-authorization", + "x-api-key", + "x-goog-api-key", + "api-key", + "cookie", + "set-cookie", +]; + +/** + * New Headers with the stale framing set AND any echoed credential/session + * header removed. Use this instead of `new Headers(upstream.headers)` on every + * path that relays an upstream response to a client. Does not mutate the input. + */ +export function stripSensitiveResponseHeaders(input: Headers): Headers { + return new Headers( + filterUpstreamResponseHeaderEntries(input.entries(), SENSITIVE_RESPONSE_HEADER_NAMES) + ); +} + +export { SENSITIVE_RESPONSE_HEADER_NAMES }; diff --git a/open-sse/vendor/codex-chatgpt-web/bridge.ts b/open-sse/vendor/codex-chatgpt-web/bridge.ts index 38eacd167c..a6f6244969 100644 --- a/open-sse/vendor/codex-chatgpt-web/bridge.ts +++ b/open-sse/vendor/codex-chatgpt-web/bridge.ts @@ -5,6 +5,7 @@ import type { CodexProviderContinuationState, CodexUsage, } from "./types"; +import { projectCodexPublicError } from "../../utils/codexPublicError"; import { adapterFailureFromMessage, classifyError, type CodexErrorPayload } from "./lib/errors"; import { encodeCompactionSummary } from "./responses/compaction"; import { encodeReasoningEnvelope, type ReasoningEnvelope } from "./responses/reasoning-envelope"; @@ -46,7 +47,8 @@ function responsesUsage(usage: CodexUsage | undefined): Record } function responseError(status: number, type: string, message: string): CodexErrorPayload { - return classifyError(status, type, message); + const classified = classifyError(status, type, message); + return projectCodexPublicError({ status, type: classified.type, code: classified.code }); } function adapterFailureFromEvent(event: Extract): { @@ -54,14 +56,25 @@ function adapterFailureFromEvent(event: Extract error: CodexErrorPayload; } { if (event.status === undefined && event.errorType === undefined && event.code === undefined) { - return adapterFailureFromMessage(event.message); + const fallback = adapterFailureFromMessage(event.message); + return { + httpStatus: fallback.httpStatus, + error: projectCodexPublicError({ + status: fallback.httpStatus, + type: fallback.error.type, + code: fallback.error.code, + }), + }; } const fallback = adapterFailureFromMessage(event.message); const httpStatus = event.status ?? fallback.httpStatus; const error = classifyError(httpStatus, event.errorType ?? fallback.error.type, event.message); if (event.errorType !== undefined) error.type = event.errorType; if (event.code !== undefined) error.code = event.code; - return { httpStatus, error }; + return { + httpStatus, + error: projectCodexPublicError({ status: httpStatus, type: error.type, code: error.code }), + }; } export { adapterFailureFromMessage } from "./lib/errors"; @@ -1314,7 +1327,7 @@ export function buildResponseJSON( } export function formatErrorResponse(status: number, type: string, message: string): Response { - return new Response(JSON.stringify({ error: classifyError(status, type, message) }), { + return new Response(JSON.stringify({ error: responseError(status, type, message) }), { status, headers: { "Content-Type": "application/json" }, }); diff --git a/src/app/api/logs/[id]/route.ts b/src/app/api/logs/[id]/route.ts index afdb7d2432..4fad3932a9 100644 --- a/src/app/api/logs/[id]/route.ts +++ b/src/app/api/logs/[id]/route.ts @@ -1,5 +1,7 @@ import { NextResponse } from "next/server"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; +import { sanitizeErrorFramesFromLogChunks } from "@/lib/logPayloads"; import { getCallLogById } from "@/lib/usageDb"; import { getCompletedDetails, getPendingById } from "@/lib/usage/usageHistory"; import { @@ -18,6 +20,29 @@ import { // before it's parsed. const CHUNK_LOG_TIMESTAMP_PREFIX = /^\[\d{2}:\d{2}:\d{2}\.\d{3}\]\s*/; +type ManagementStreamChunks = { + provider?: string[]; + openai?: string[]; + client?: string[]; +}; + +function projectManagementStreamChunks( + streamChunks: ManagementStreamChunks | null | undefined +): ManagementStreamChunks | null { + if (!streamChunks) return null; + return { + ...(streamChunks.provider + ? { provider: sanitizeErrorFramesFromLogChunks(streamChunks.provider) } + : {}), + ...(streamChunks.openai + ? { openai: sanitizeErrorFramesFromLogChunks(streamChunks.openai) } + : {}), + ...(streamChunks.client + ? { client: sanitizeErrorFramesFromLogChunks(streamChunks.client) } + : {}), + }; +} + // Best-effort parse of the accumulated SSE `data:` lines captured live for an // in-flight request (open-sse/utils/requestLogger.ts's appendConvertedChunk // mutates these arrays in place as chunks arrive, so this reflects "the reply @@ -77,12 +102,13 @@ export async function GET( try { const pendingRequestDetail = getPendingById().get(id); if (pendingRequestDetail) { + const safeStreamChunks = projectManagementStreamChunks(pendingRequestDetail.streamChunks); const pipelinePayloads: any = { clientRequest: pendingRequestDetail.clientRequest ?? null, providerRequest: pendingRequestDetail.providerRequest ?? null, providerResponse: pendingRequestDetail.providerResponse ?? null, clientResponse: pendingRequestDetail.clientResponse ?? null, - streamChunks: pendingRequestDetail.streamChunks ?? null, + streamChunks: safeStreamChunks, }; const activeEntry = { @@ -102,7 +128,7 @@ export async function GET( // The still-generating reply so far — the request's own context // panel renders this alongside its (already-complete) requestBody // instead of waiting for the stream to finish. - partialAssistantText: extractPartialAssistantText(pendingRequestDetail.streamChunks), + partialAssistantText: extractPartialAssistantText(safeStreamChunks), }; return NextResponse.json(activeEntry); @@ -123,12 +149,13 @@ export async function GET( const completed = getCompletedDetails(); const inMem = completed.get(id); if (inMem) { + const safeStreamChunks = projectManagementStreamChunks(inMem.streamChunks); const pipelinePayloads: any = { clientRequest: inMem.clientRequest ?? null, providerRequest: inMem.providerRequest ?? null, providerResponse: inMem.providerResponse ?? null, clientResponse: inMem.clientResponse ?? null, - streamChunks: inMem.streamChunks ?? null, + streamChunks: safeStreamChunks, }; const minimal = { @@ -142,7 +169,7 @@ export async function GET( duration: Date.now() - inMem.startedAt, detailState: "in-memory", active: false, - error: inMem.error || null, + error: sanitizeErrorMessage(inMem.error) || null, pipelinePayloads, hasPipelineDetails: true, }; diff --git a/src/app/api/providers/[id]/models/staleEncryptionGuard.ts b/src/app/api/providers/[id]/models/staleEncryptionGuard.ts index fc410b1a37..5f2384e921 100644 --- a/src/app/api/providers/[id]/models/staleEncryptionGuard.ts +++ b/src/app/api/providers/[id]/models/staleEncryptionGuard.ts @@ -40,9 +40,7 @@ export function buildStaleEncryptionKeyResponse( `(STORAGE_ENCRYPTION_KEY changed or unset). Re-authenticate this account, or verify ` + `STORAGE_ENCRYPTION_KEY matches the key used to store it.`; - // buildErrorBody sanitizes the message (Rule #12); override the type so the - // client can key off the specific stale-encryption cause. - const body = buildErrorBody(424, message); - body.error.type = "storage_encryption_stale"; + // buildErrorBody sanitizes the message and projects the client-visible classification. + const body = buildErrorBody(424, message, undefined, { type: "storage_encryption_stale" }); return NextResponse.json(body, { status: 424 }); } diff --git a/src/app/api/providers/[id]/test/publicErrorBoundary.ts b/src/app/api/providers/[id]/test/publicErrorBoundary.ts new file mode 100644 index 0000000000..30addfeffb --- /dev/null +++ b/src/app/api/providers/[id]/test/publicErrorBoundary.ts @@ -0,0 +1,155 @@ +import { projectProviderValidationResultForPublicResponse } from "@/lib/providers/validation/transport"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; +import { makeDiagnosis } from "./codexAppServerHealth"; +import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; + +export function toSafeMessage(value: unknown, fallback = "Unknown error"): string { + const safeMessage = sanitizeErrorMessage(value).trim(); + return safeMessage || fallback; +} + +/** + * A provider/account that the upstream has deactivated (vs. a revoked/expired token). + * #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT + * account is deactivated, in which case the API returns 401 — mislabeling that as + * "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the + * account-fallback classifier already trusts. + */ +export function isAccountDeactivatedMessage(text: string): boolean { + const normalized = (text || "").toLowerCase(); + return ( + normalized.includes("account_deactivated") || + (normalized.includes("deactivat") && normalized.includes("account")) + ); +} + +export function classifyFailure({ + error, + statusCode = null, + refreshFailed = false, + unsupported = false, + provider, +}: ClassifyFailureArgs) { + const message = toSafeMessage(error, "Connection test failed"); + const normalized = message.toLowerCase(); + const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null; + + if (unsupported) { + return makeDiagnosis("unsupported", "validation", message, "unsupported"); + } + + if (refreshFailed || normalized.includes("refresh failed")) { + return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed"); + } + + // #1444: a deactivated account is distinct from a revoked/expired token — surface it + // as account_deactivated (which the dashboard renders as "Account Deactivated") before + // the generic 401/403 branch below would mark it "upstream_auth_error". + if (isAccountDeactivatedMessage(normalized)) { + return makeDiagnosis("account_deactivated", "account", message, "account_deactivated"); + } + + if (numericStatus === 401 || numericStatus === 403) { + return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus); + } + + if (numericStatus === 429) { + return makeDiagnosis("upstream_rate_limited", "upstream", message, "429"); + } + + if (numericStatus && numericStatus >= 500) { + return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus)); + } + + if (normalized.includes("token expired") || normalized.includes("expired")) { + return makeDiagnosis("token_expired", "oauth", message, "token_expired"); + } + + if ( + normalized.includes("invalid api key") || + normalized.includes("token invalid") || + normalized.includes("revoked") || + normalized.includes("access denied") || + normalized.includes("unauthorized") || + normalized.includes("forbidden") + ) { + return makeDiagnosis( + "upstream_auth_error", + "upstream", + message, + numericStatus ? String(numericStatus) : "auth_failed" + ); + } + + if ( + normalized.includes("rate limit") || + normalized.includes("quota") || + normalized.includes("too many requests") + ) { + return makeDiagnosis( + "upstream_rate_limited", + "upstream", + message, + numericStatus ? String(numericStatus) : "rate_limited" + ); + } + + if ( + normalized.includes("fetch failed") || + normalized.includes("network") || + normalized.includes("timeout") || + normalized.includes("timed out") || + normalized.includes("econn") || + normalized.includes("enotfound") || + normalized.includes("socket") + ) { + return makeDiagnosis("network_error", "upstream", message, "network_error"); + } + + return makeDiagnosis( + "upstream_error", + "upstream", + message, + numericStatus ? String(numericStatus) : "upstream_error" + ); +} + +/** Allowlist the CLI health fields safe to expose outside the local runtime boundary. */ +export function projectProviderRuntimeForPublicResponse( + runtime: unknown +): Record | null { + if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) return null; + const record = runtime as Record; + const projected: Record = {}; + + for (const field of ["installed", "runnable", "requiresBinary"] as const) { + if (typeof record[field] === "boolean") projected[field] = record[field]; + } + for (const field of ["reason", "runtimeMode", "version", "command"] as const) { + if (typeof record[field] !== "string") continue; + const safeValue = sanitizeErrorMessage(record[field]).trim(); + if (safeValue) projected[field] = safeValue.slice(0, 512); + } + + return projected; +} + +/** Sanitize every connection-test result before health writes, logs, and HTTP responses. */ +export function projectConnectionTestResultForPublicResponse< + T extends { error?: unknown; warning?: unknown; diagnosis?: unknown }, +>(result: T) { + const projected = projectProviderValidationResultForPublicResponse(result); + if (!projected.diagnosis || typeof projected.diagnosis !== "object") return projected; + + const diagnosis = projected.diagnosis as Record; + return { + ...projected, + diagnosis: { + ...diagnosis, + message: + diagnosis.message === null || diagnosis.message === undefined + ? null + : toSafeMessage(diagnosis.message, "Connection test failed"), + }, + }; +} diff --git a/src/app/api/providers/[id]/test/route.ts b/src/app/api/providers/[id]/test/route.ts index cc663a95c8..1a81180358 100644 --- a/src/app/api/providers/[id]/test/route.ts +++ b/src/app/api/providers/[id]/test/route.ts @@ -7,6 +7,7 @@ import { isCloudEnabled, resolveProxyForConnection } from "@/lib/db/settings"; import { getConsistentMachineId } from "@/shared/utils/machineId"; import { syncToCloud } from "@/lib/cloudSync"; import { validateProviderApiKey } from "@/lib/providers/validation"; +import { projectProviderValidationResultForPublicResponse } from "@/lib/providers/validation/transport"; import { getCliRuntimeStatus } from "@/shared/services/cliRuntime"; import { buildQoderCliNotFoundHint } from "@omniroute/open-sse/services/qoderCliResolve.ts"; // Use the shared open-sse token refresh with built-in dedup/race-condition cache @@ -29,11 +30,19 @@ import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHea import { recoverKeyHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts"; import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits"; import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation"; -import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult"; import { classifyOAuthProbeInconclusive, OAUTH_TEST_CONFIG } from "./oauthTestConfig"; import { isGeoBlockedError } from "@omniroute/open-sse/services/errorClassifier.ts"; import * as retirement from "@/lib/providers/chatgptWebRetirementResponse"; +import { + classifyFailure, + isAccountDeactivatedMessage, + projectConnectionTestResultForPublicResponse, + projectProviderRuntimeForPublicResponse, + toSafeMessage, +} from "./publicErrorBoundary"; + +export { classifyFailure, projectProviderRuntimeForPublicResponse } from "./publicErrorBoundary"; // Match the API-key path's 30s timeout so a hung OAuth upstream cannot block the test queue. const OAUTH_TEST_TIMEOUT_MS = 30_000; @@ -45,115 +54,6 @@ const providerConnectionTestBodySchema = z.object({ validationModelId: z.string().max(500).optional(), }); -function toSafeMessage(value: any, fallback = "Unknown error"): string { - if (typeof value !== "string") return fallback; - const trimmed = value.trim(); - return trimmed || fallback; -} - -/** - * A provider/account that the upstream has deactivated (vs. a revoked/expired token). - * #1444: a Codex account can have a perfectly healthy OAuth refresh while its ChatGPT - * account is deactivated, in which case the API returns 401 — mislabeling that as - * "Token invalid or revoked" hides the real cause. Mirrors the deactivation phrases the - * account-fallback classifier already trusts. - */ -function isAccountDeactivatedMessage(text: string): boolean { - const n = (text || "").toLowerCase(); - return n.includes("account_deactivated") || (n.includes("deactivat") && n.includes("account")); -} - -export function classifyFailure({ - error, - statusCode = null, - refreshFailed = false, - unsupported = false, - provider, -}: ClassifyFailureArgs) { - const message = toSafeMessage(error, "Connection test failed"); - const normalized = message.toLowerCase(); - const numericStatus = Number.isFinite(statusCode) ? Number(statusCode) : null; - - if (unsupported) { - return makeDiagnosis("unsupported", "validation", message, "unsupported"); - } - - if (refreshFailed || normalized.includes("refresh failed")) { - return makeDiagnosis("token_refresh_failed", "oauth", message, "refresh_failed"); - } - - // #1444: a deactivated account is distinct from a revoked/expired token — surface it - // as account_deactivated (which the dashboard renders as "Account Deactivated") before - // the generic 401/403 branch below would mark it "upstream_auth_error". - if (isAccountDeactivatedMessage(normalized)) { - return makeDiagnosis("account_deactivated", "account", message, "account_deactivated"); - } - - if (numericStatus === 401 || numericStatus === 403) { - return classifyAmbiguousOrAuthError(provider, normalized, message, numericStatus); - } - - if (numericStatus === 429) { - return makeDiagnosis("upstream_rate_limited", "upstream", message, "429"); - } - - if (numericStatus && numericStatus >= 500) { - return makeDiagnosis("upstream_unavailable", "upstream", message, String(numericStatus)); - } - - if (normalized.includes("token expired") || normalized.includes("expired")) { - return makeDiagnosis("token_expired", "oauth", message, "token_expired"); - } - - if ( - normalized.includes("invalid api key") || - normalized.includes("token invalid") || - normalized.includes("revoked") || - normalized.includes("access denied") || - normalized.includes("unauthorized") || - normalized.includes("forbidden") - ) { - return makeDiagnosis( - "upstream_auth_error", - "upstream", - message, - numericStatus ? String(numericStatus) : "auth_failed" - ); - } - - if ( - normalized.includes("rate limit") || - normalized.includes("quota") || - normalized.includes("too many requests") - ) { - return makeDiagnosis( - "upstream_rate_limited", - "upstream", - message, - numericStatus ? String(numericStatus) : "rate_limited" - ); - } - - if ( - normalized.includes("fetch failed") || - normalized.includes("network") || - normalized.includes("timeout") || - normalized.includes("timed out") || - normalized.includes("econn") || - normalized.includes("enotfound") || - normalized.includes("socket") - ) { - return makeDiagnosis("network_error", "upstream", message, "network_error"); - } - - return makeDiagnosis( - "upstream_error", - "upstream", - message, - numericStatus ? String(numericStatus) : "upstream_error" - ); -} - function hasQoderToken(connection: any): boolean { if (typeof connection?.apiKey === "string" && connection.apiKey.trim().length > 0) return true; const psd = connection?.providerSpecificData; @@ -218,7 +118,10 @@ async function getProviderRuntimeStatus(connection: any) { error: runtimeMessage, }; } catch (error) { - const runtimeMessage = `Failed to check local CLI runtime: ${(error as any)?.message || "runtime_check_failed"}`; + const runtimeMessage = `Failed to check local CLI runtime: ${toSafeMessage( + error, + "runtime_check_failed" + )}`; return { installed: false, runnable: false, @@ -302,7 +205,10 @@ async function refreshOAuthToken(connection: any) { }); return result; // { accessToken, expiresIn, refreshToken } or null } catch (err) { - console.error(`Error refreshing ${provider} token:`, (err as any).message); + console.error( + `Error refreshing ${provider} token:`, + toSafeMessage(err, "Token refresh failed") + ); return null; } } @@ -376,7 +282,10 @@ async function syncToCloudIfEnabled() { const machineId = await getConsistentMachineId(); await syncToCloud(machineId); } catch (error) { - console.log("Error syncing to cloud after token refresh:", error); + console.log( + "Error syncing to cloud after token refresh:", + toSafeMessage(error, "Cloud sync failed") + ); } } @@ -934,11 +843,13 @@ async function testApiKeyConnection(connection: any) { }; } - const result = await validateProviderApiKey({ - provider: connection.provider, - apiKey: connection.apiKey, - providerSpecificData: connection.providerSpecificData, - }); + const result = projectProviderValidationResultForPublicResponse( + await validateProviderApiKey({ + provider: connection.provider, + apiKey: connection.apiKey, + providerSpecificData: connection.providerSpecificData, + }) + ); if (result.unsupported) { const error = "Provider test not supported"; @@ -1001,8 +912,11 @@ export async function testSingleConnection(connectionId: string, validationModel let proxyInfo: any = null; try { proxyInfo = await resolveProxyForConnection(connectionId); - } catch (proxyErr: any) { - console.log(`[ConnectionTest] Failed to resolve proxy for ${connectionId}:`, proxyErr?.message); + } catch (proxyErr: unknown) { + console.log( + `[ConnectionTest] Failed to resolve proxy for ${connectionId}:`, + toSafeMessage(proxyErr, "Proxy resolution failed") + ); } let result; @@ -1046,6 +960,12 @@ export async function testSingleConnection(connectionId: string, validationModel ); } + // Every runtime path converges here before any health-state write, diagnosis, + // persistent log, or public response. API-key validation is projected at its + // own seam above as well so future refactors cannot move it past this boundary. + result = projectConnectionTestResultForPublicResponse(result); + const publicRuntime = projectProviderRuntimeForPublicResponse(runtime); + const latencyMs = Date.now() - startTime; // Unsupported validation capability is neutral: the probe established that @@ -1063,14 +983,14 @@ export async function testSingleConnection(connectionId: string, validationModel } catch (activateError) { console.log( `[ConnectionTest] Failed to activate unverifiable connection ${connectionId}:`, - (activateError as any)?.message || activateError + toSafeMessage(activateError, "Connection activation failed") ); } } return { ...result, latencyMs, - runtime: runtime || null, + runtime: publicRuntime, testedAt: null, }; } @@ -1214,7 +1134,7 @@ export async function testSingleConnection(connectionId: string, validationModel diagnosis, latencyMs, statusCode: result.statusCode || null, - runtime: runtime || null, + runtime: publicRuntime, testedAt: now, }; } @@ -1245,7 +1165,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: } catch (error) { const retired = retirement.responseForError(error); if (retired) return retired; - console.log("Error testing connection:", error); + console.log("Error testing connection:", toSafeMessage(error, "Connection test failed")); return NextResponse.json({ error: "Test failed" }, { status: 500 }); } } diff --git a/src/app/api/providers/validate/route.ts b/src/app/api/providers/validate/route.ts index 7d92d4ac92..0992acde94 100644 --- a/src/app/api/providers/validate/route.ts +++ b/src/app/api/providers/validate/route.ts @@ -1,4 +1,5 @@ import { NextResponse } from "next/server"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; import { getAuditRequestContext, logAuditEvent } from "@/lib/compliance/index"; import { getProviderNodeById } from "@/models"; @@ -8,6 +9,7 @@ import { isAnthropicCompatibleProvider, } from "@/shared/constants/providers"; import { validateProviderApiKey } from "@/lib/providers/validation"; +import { projectProviderValidationResultForPublicResponse } from "@/lib/providers/validation/transport"; import { getProxyForLevel } from "@/lib/db/settings"; import { resolveProxyForProvider } from "@/lib/db/proxies"; import { validateProviderApiKeySchema } from "@/shared/validation/schemas"; @@ -123,12 +125,14 @@ export async function POST(request) { proxyToUse = providerProxy || globalProxy || null; } - const result = await runWithProxyContextOrDirect(proxyToUse || null, () => - validateProviderApiKey({ - provider, - apiKey, - providerSpecificData, - }) + const result = projectProviderValidationResultForPublicResponse( + await runWithProxyContextOrDirect(proxyToUse || null, () => + validateProviderApiKey({ + provider, + apiKey, + providerSpecificData, + }) + ) ); if (result.unsupported) { @@ -174,7 +178,7 @@ export async function POST(request) { providerSpecificData: result.providerSpecificData || null, }); } catch (error) { - console.log("Error validating API key:", error); + console.log("Error validating API key:", sanitizeErrorMessage(error) || "Validation failed"); return NextResponse.json({ error: "Validation failed" }, { status: 500 }); } } diff --git a/src/app/api/settings/cache-config/route.ts b/src/app/api/settings/cache-config/route.ts index fc5a6634aa..cbf777b634 100644 --- a/src/app/api/settings/cache-config/route.ts +++ b/src/app/api/settings/cache-config/route.ts @@ -58,8 +58,12 @@ export async function GET(request: NextRequest) { const flatSettings = await getSettings(); const config: Record = {}; for (const key of CACHE_CONFIG_KEYS) { - if (key === "idempotencyWindowMs") { - config[key] = flatSettings.idempotencyWindowMs ?? DEFAULTS[key]; + if (key === "idempotencyWindowMs" || key === "alwaysPreserveClientCache") { + // These live in the flat general settings (src/lib/db/settings.ts): + // idempotencyLayer and getCacheControlSettings() both read from there, + // so reporting the databaseSettings "cache" copy would show a value the + // runtime never uses. + config[key] = flatSettings[key] ?? DEFAULTS[key]; } else { config[key] = (cache as Record)[key] ?? DEFAULTS[key]; } @@ -106,9 +110,6 @@ export async function PUT(request: NextRequest) { if (body.promptCacheStrategy !== undefined) { updates.promptCacheStrategy = body.promptCacheStrategy; } - if (body.alwaysPreserveClientCache !== undefined) { - updates.alwaysPreserveClientCache = body.alwaysPreserveClientCache; - } if (body.modelCatalogCacheTtlMs !== undefined) { updates.modelCatalogCacheTtlMs = body.modelCatalogCacheTtlMs; } @@ -116,12 +117,23 @@ export async function PUT(request: NextRequest) { // updateDatabaseSettings() calls invalidateDbCache("settings") internally, // which bumps the model-catalog cache version so in-flight responses pick // up the fresh TTL — no separate version bump needed here. - updateDatabaseSettings({ cache: updates }); + if (Object.keys(updates).length > 0) { + updateDatabaseSettings({ cache: updates }); + } - // idempotencyWindowMs is not part of the databaseSettings "cache" section — - // persist it through the flat general settings module instead (see GET). + // idempotencyWindowMs and alwaysPreserveClientCache are read from the flat + // general settings (see GET) — persisting them into the databaseSettings + // "cache" section would be a silent no-op for the runtime, which is what + // made this endpoint's alwaysPreserveClientCache writes ineffective before. + const flatUpdates: Record = {}; if (body.idempotencyWindowMs !== undefined) { - await updateSettings({ idempotencyWindowMs: body.idempotencyWindowMs }); + flatUpdates.idempotencyWindowMs = body.idempotencyWindowMs; + } + if (body.alwaysPreserveClientCache !== undefined) { + flatUpdates.alwaysPreserveClientCache = body.alwaysPreserveClientCache; + } + if (Object.keys(flatUpdates).length > 0) { + await updateSettings(flatUpdates); } return NextResponse.json({ ok: true }); diff --git a/src/app/api/v1/relay/chat/completions/bifrost/route.ts b/src/app/api/v1/relay/chat/completions/bifrost/route.ts index 31b007959f..8674125c38 100644 --- a/src/app/api/v1/relay/chat/completions/bifrost/route.ts +++ b/src/app/api/v1/relay/chat/completions/bifrost/route.ts @@ -29,9 +29,14 @@ */ import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors"; +import { stripSensitiveResponseHeaders } from "@omniroute/open-sse/utils/upstreamResponseHeaders"; import { createInjectionGuard } from "@/middleware/promptInjectionGuard"; import { getRelayTokenByHash, checkRateLimit, recordRelayUsage } from "@/lib/db/relayProxies"; -import { buildErrorBody } from "@omniroute/open-sse/utils/error"; +import { + buildErrorBody, + parseUpstreamError, + sanitizeErrorMessage, +} from "@omniroute/open-sse/utils/error"; import { getProviderPluginManifestHeader } from "@omniroute/open-sse/config/providerPluginManifestUrl.ts"; import { z } from "zod"; import { @@ -299,7 +304,11 @@ export async function POST(request: Request) { }); }; - const newHeaders = new Headers(upstream.headers); + // Never copy the upstream headers wholesale: the sidecar receives our + // `Authorization: Bearer ${BIFROST_API_KEY}` and anything it echoes back — + // that header, its own set-cookie — would reach the relay-token holder + // (GHSA-9m72-44hg-w32g). + const newHeaders = stripSensitiveResponseHeaders(upstream.headers); newHeaders.set("X-Routed-By", "bifrost"); newHeaders.set("X-Relay-Token", token.tokenPrefix + "..."); if (!wantsStream) { @@ -321,6 +330,28 @@ export async function POST(request: Request) { } clearTimeout(tid); + + // Normalize non-2xx through parseUpstreamError + buildErrorBody instead of + // relaying the sidecar body verbatim — parity with the TS sibling route and + // Hard Rule #12 (GHSA-9m72-44hg-w32g). + if (!upstream.ok) { + const parsed = await parseUpstreamError(upstream, null); + const errorBody = buildErrorBody( + parsed.statusCode, + sanitizeErrorMessage(parsed.message), + parsed.responseBody + ); + newHeaders.set("Content-Type", "application/json"); + if (parsed.retryAfterMs && parsed.retryAfterMs > 0) { + newHeaders.set("Retry-After", String(Math.ceil(parsed.retryAfterMs / 1000))); + } + recordUsage("error", parsed.statusCode); + return new Response(JSON.stringify(errorBody), { + status: parsed.statusCode, + headers: newHeaders, + }); + } + recordUsage(upstream.status < 500 ? "success" : "error", upstream.status); return new Response(upstream.body, { diff --git a/src/app/api/v1/relay/chat/completions/route.ts b/src/app/api/v1/relay/chat/completions/route.ts index fa40d4a009..4e43bb7a13 100644 --- a/src/app/api/v1/relay/chat/completions/route.ts +++ b/src/app/api/v1/relay/chat/completions/route.ts @@ -7,6 +7,7 @@ */ import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors"; +import { stripSensitiveResponseHeaders } from "@omniroute/open-sse/utils/upstreamResponseHeaders"; import { handleChat } from "@/sse/handlers/chat"; import { withChatAdmission } from "@/shared/middleware/withChatAdmission"; import { createInjectionGuard } from "@/middleware/promptInjectionGuard"; @@ -109,7 +110,9 @@ async function forwardToBifrost( signal: ac.signal, }); - const headers = new Headers(upstream.headers); + // Same strip as the bifrost sibling: an echoed upstream credential or + // set-cookie must not reach the relay-token holder (GHSA-9m72-44hg-w32g). + const headers = stripSensitiveResponseHeaders(upstream.headers); headers.set("X-Routed-By", "bifrost"); headers.set("X-Routing-Backend", "bifrost"); headers.set("X-Relay-Token", token.tokenPrefix + "..."); diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index e9643f3511..89659dc23c 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -6139,7 +6139,7 @@ "bluesminds": "Get your API key at https://www.bluesminds.com — OpenAI-compatible endpoint at https://api.bluesminds.com/v1 with free daily credits. VIP models (Claude Opus 4.5, Gemini 2.5 Pro) consume pi credits.", "byteplus": "Connect BytePlus ModelArk with an API key.", "bytez": "$1 free credits, refreshes every 4 weeks", - "cerebras": "Free Trial: 1M tokens/day, 30K TPM, 5 RPM — no credit card.", + "cerebras": "One-time $5 signup credit (30-day validity); a payment method is required. Not a recurring free tier.", "charm-hyper": "Create an API key at https://hyper.charm.land, then paste it here as a Bearer token.", "chutes": "Bearer API key for the Chutes OpenAI-compatible gateway.", "clarifai": "Clarifai exposes OpenAI-compatible chat, responses and /models on /v2/ext/openai/v1. Public/community models typically require a PAT; app-scoped keys only work for resources inside that app.", diff --git a/src/i18n/messages/es.json b/src/i18n/messages/es.json index ba61b59daa..09b8296a82 100644 --- a/src/i18n/messages/es.json +++ b/src/i18n/messages/es.json @@ -6136,7 +6136,7 @@ "bluesminds": "Get your API key at https://www.bluesminds.com — OpenAI-compatible endpoint at https://api.bluesminds.com/v1 with free daily credits. VIP models (Claude Opus 4.5, Gemini 2.5 Pro) consume pi credits.", "byteplus": "Connect BytePlus ModelArk with an API key.", "bytez": "$1 free credits, refreshes every 4 weeks", - "cerebras": "Free Trial: 1M tokens/day, 30K TPM, 5 RPM — no credit card.", + "cerebras": "One-time $5 signup credit (30-day validity); a payment method is required. Not a recurring free tier.", "charm-hyper": "Create an API key at https://hyper.charm.land, then paste it here as a Bearer token.", "chutes": "Bearer API key for the Chutes OpenAI-compatible gateway.", "clarifai": "Clarifai exposes OpenAI-compatible chat, responses and /models on /v2/ext/openai/v1. Public/community models typically require a PAT; app-scoped keys only work for resources inside that app.", diff --git a/src/lib/dataPaths.ts b/src/lib/dataPaths.ts index 01c93c2bad..b4b9801501 100644 --- a/src/lib/dataPaths.ts +++ b/src/lib/dataPaths.ts @@ -101,30 +101,70 @@ export function isTestContext(): boolean { ); } +/** + * `node --eval` / `node -e` (and their print variants) are common shapes used by + * one-off import probes. + * Such a process has no application entry point from which to establish storage intent, + * so defaulting it to the operator's durable database is unsafe. A deliberate production + * inspection can still opt in with an explicit DATA_DIR (preferred) or + * OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1. + */ +function isEvalProbeContext(): boolean { + return process.execArgv.some( + (arg) => + arg === "--eval" || + arg === "-e" || + arg === "-pe" || + arg === "-ep" || + arg.startsWith("--eval=") || + arg === "--print" || + arg === "-p" || + arg.startsWith("--print=") + ); +} + /** Process-wide redirect target, so repeated calls share one DB instead of one per call. */ let testContextDataDir: string | null = null; +let testContextCleanupRegistered = false; export function resolveWritableDataDir({ isCloud = false }: { isCloud?: boolean } = {}): string { const resolved = resolveDataDir({ isCloud }); + const configured = normalizeConfiguredPath(process.env.DATA_DIR); // Cloud/serverless never owns a writable home dir; leave its sentinel alone. if (isCloud) return resolved; - // #10428: a test/ad-hoc run that never chose a DATA_DIR would otherwise open the + // #10428: a test/eval-probe run that never chose a DATA_DIR would otherwise open the // OPERATOR'S REAL database (~/.omniroute/storage.sqlite — live provider credentials). // Redirect to a throwaway dir instead of throwing: the documented single-file command // (`node --import tsx/esm --test tests/unit/x.test.ts`) does not load the isolation // setup, and a hard failure there would only teach people to disable the guard. // `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1` opts back in, so the intent is recorded. if ( - !process.env.DATA_DIR && - isTestContext() && + !configured && + (isTestContext() || isEvalProbeContext()) && process.env.OMNIROUTE_ALLOW_DEFAULT_DATA_DIR !== "1" ) { if (!testContextDataDir) { testContextDataDir = fs.mkdtempSync(path.join(os.tmpdir(), `${APP_NAME}-testctx-`)); + if (!testContextCleanupRegistered) { + testContextCleanupRegistered = true; + process.once("exit", () => { + if (!testContextDataDir) return; + try { + fs.rmSync(testContextDataDir, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 25, + }); + } catch { + // An unclean exit is left to the operating system's temp-directory policy. + } + }); + } console.warn( - `[DATA_DIR] test context without DATA_DIR → using '${testContextDataDir}' instead of ` + + `[DATA_DIR] test/eval context without DATA_DIR → using '${testContextDataDir}' instead of ` + `'${resolved}'. Set DATA_DIR explicitly (or load tests/_setup/isolateDataDir.ts) to silence this.` ); } @@ -132,7 +172,6 @@ export function resolveWritableDataDir({ isCloud = false }: { isCloud?: boolean } // No explicit override → already the default user dir; nothing to fall back to. - const configured = normalizeConfiguredPath(process.env.DATA_DIR); if (!configured) return resolved; try { diff --git a/src/lib/db/backup.ts b/src/lib/db/backup.ts index effbaf09c0..d170a5b764 100644 --- a/src/lib/db/backup.ts +++ b/src/lib/db/backup.ts @@ -101,6 +101,24 @@ function getBackupDir() { return DB_BACKUPS_DIR || path.join(DATA_DIR, "db_backups"); } +function listBackupFilesNewestFirst(backupDir: string) { + return fs + .readdirSync(backupDir) + .filter((filename) => filename.startsWith("db_") && filename.endsWith(".sqlite")) + .flatMap((filename) => { + try { + return [{ filename, stat: fs.statSync(path.join(backupDir, filename)) }]; + } catch { + // A concurrent retention pass may remove an entry after readdir. + return []; + } + }) + .sort( + (left, right) => + right.stat.mtimeMs - left.stat.mtimeMs || right.filename.localeCompare(left.filename) + ); +} + export function cleanupDbBackups(options?: { maxFiles?: number; retentionDays?: number; @@ -272,16 +290,26 @@ export function backupDbFile(reason = "auto") { if (reason !== "manual" && reason !== "pre-restore") { // Shrink detection is useful for automatic safety backups, but it should // never block an explicit operator action like manual backup or pre-restore. + // Only timestamp-named automatic/manual backups are shrink baselines. The + // content-addressed migration snapshots are restore points, not periodic size + // samples; excluding them also keeps this lookup to names only with a single stat + // even in legacy directories containing tens of thousands of timestamp backups. const existingBackups = fs .readdirSync(backupDir) - .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite")) + .filter((filename) => /^db_\d{4}-.*\.sqlite$/.test(filename)) .sort(); if (existingBackups.length > 0) { - const latestBackup = existingBackups[existingBackups.length - 1]; - const latestStat = fs.statSync(path.join(backupDir, latestBackup)); - if (latestStat.size > 4096 && stat.size < latestStat.size * 0.5) { - console.warn(`[DB] Backup SKIPPED — DB shrank from ${latestStat.size}B to ${stat.size}B`); - return null; + const latestBackup = existingBackups.at(-1)!; + try { + const latestStat = fs.statSync(path.join(backupDir, latestBackup)); + if (latestStat.size > 4096 && stat.size < latestStat.size * 0.5) { + console.warn( + `[DB] Backup SKIPPED — DB shrank from ${latestStat.size}B to ${stat.size}B` + ); + return null; + } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code !== "ENOENT") throw error; } } } @@ -316,16 +344,11 @@ export async function listDbBackups() { try { if (!fs.existsSync(backupDir)) return []; - const entries = fs - .readdirSync(backupDir) - .filter((f) => f.startsWith("db_") && f.endsWith(".sqlite")) - .sort() - .reverse(); + const entries = listBackupFilesNewestFirst(backupDir); const { tryOpenSync } = await import("@/lib/db/adapters/driverFactory"); - return entries.map((filename) => { + return entries.map(({ filename, stat }) => { const filePath = path.join(backupDir, filename); - const stat = fs.statSync(filePath); const match = filename.match(/^db_(.+?)_([^.]+)\.sqlite$/); const reason = match ? match[2] : "unknown"; diff --git a/src/lib/db/backupRetention.ts b/src/lib/db/backupRetention.ts index cbc9efeaa5..150bfe07d7 100644 --- a/src/lib/db/backupRetention.ts +++ b/src/lib/db/backupRetention.ts @@ -1,17 +1,12 @@ /** * Backup retention primitives — pure filesystem work, no `core.ts` dependency. * - * This module exists so BOTH backup call sites can share one retention policy: - * - * - `backup.ts` (manual/API/auto backups) — resolves the operator's settings from the - * database and delegates here. - * - `migrationRunner.ts` (pre-migration snapshots) — cannot import `backup.ts`, because - * `core.ts` already imports `migrationRunner.ts` and `backup.ts` imports `core.ts`; - * that edge would close a cycle. Keeping the policy here, free of `core`, lets the - * migration path prune without one. - * - * Before #10421 the migration path had no retention at all and `db_backups/` grew - * without bound (observed: 48.999 files / 204 GB against a 5,3 MB live database). + * `backup.ts` (manual/API/auto backups) resolves the operator's settings from the + * database and delegates pure family pruning here. The migration runner deliberately + * does not prune during its concurrent safety window: its snapshots are content-addressed + * and reused for an identical DB state, while manual/scheduled cleanup remains the single + * retention boundary. Before #10421, repeated failed starts created distinct timestamped + * snapshots and `db_backups/` grew without bound (observed: 48,999 files / 204 GB). */ import fs from "fs"; diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index d636899a32..c8ba9b1f18 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -1118,10 +1118,10 @@ export function getDbInstance(): SqliteDatabase { // This is needed so the migration runner skips the mass-migration safety abort // that would otherwise trigger because heuristic seeding marks some migrations // as applied, making the fresh DB look like a wiped existing DB (#1328). - // #9934: also classify as fresh a file that `omniroute setup` created with - // only the clipped skeleton schema (see the probe below) — even though the - // file exists, it has never had migrations run. - let isNewDb = !fs.existsSync(sqliteFile); + // #9934: also classify a setup-created skeleton as logically fresh for the mass guard, + // while tracking its pre-existing file independently for mandatory snapshot safety. + const databaseExistedBeforeInitialization = fs.existsSync(sqliteFile); + let isNewDb = !databaseExistedBeforeInitialization; // Detect and handle old schema format — preserve data when possible (#146) // Uses a single probe connection that becomes the real connection when possible. @@ -1310,7 +1310,7 @@ export function getDbInstance(): SqliteDatabase { VALUES ('001', 'initial_schema'); `); - runMigrations(db, { isNewDb }); + runMigrations(db, { isNewDb, databaseExistedBeforeInitialization }); // Fresh installs need the same post-migration index guarantee as upgraded // databases, including recovery from an interrupted migration 127 attempt. ensureUsageHistoryAccountIndex(db); diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index f53cd58070..58073518b8 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -21,37 +21,29 @@ import type { SqliteAdapter } from "./adapters/types"; import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings"; import { isAutomatedTestProcess } from "@/shared/utils/testProcess"; import { - RENAMED_MIGRATION_COMPATIBILITY, LEGACY_VERSION_SLOT_MIGRATIONS, - SUPERSEDED_DUPLICATE_MIGRATIONS, - PHYSICAL_SCHEMA_SENTINELS, - INITIAL_SCHEMA_SENTINELS, OPTIONAL_FTS5_MIGRATION_VERSIONS, + RENAMED_MIGRATION_COMPATIBILITY, + SUPERSEDED_DUPLICATE_MIGRATIONS, } from "./migrationRunner/constants"; import { getExtraMigrationFiles } from "./migrationRunner/extraDirs"; -// Retention primitives live in their own `core`-free module: `core.ts` imports this file, -// so importing `backup.ts` (which imports `core.ts`) here would close a dependency cycle. +import { migrationConsole as console } from "./migrationRunner/logger"; import { - MAX_DB_BACKUPS, - DEFAULT_DB_BACKUP_RETENTION_DAYS, - parsePositiveInt, - parseNonNegativeInt, - pruneBackupDirectory, -} from "./backupRetention"; - -const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string"; - -const console = { - log: (...args: unknown[]) => { - if (!isNodeTestRunnerChild) globalThis.console.log(...args); - }, - warn: (...args: unknown[]) => { - if (!isNodeTestRunnerChild) globalThis.console.warn(...args); - }, - error: (...args: unknown[]) => { - globalThis.console.error(...args); - }, -}; + createPreMigrationBackup, + hashFileSync, + type PreMigrationBackupReceipt, +} from "./migrationRunner/preMigrationBackup"; +import { + detectNameMismatches, + getPlausiblePendingCount, + hasColumn, + hasLedgerRepairCandidates, + hasPhysicalTable, + hasTable, + inferPhysicalSchemaBaseline, + reconcileRenumberedMigrations, + rehomeLegacyVersionSlotMigrations, +} from "./migrationRunner/schemaState"; /** * Resolve the migrations directory path safely across platforms. @@ -336,16 +328,96 @@ function getAppliedRecords(db: SqliteAdapter): Array<{ version: string; name: st }>; } -function hasTable(db: SqliteAdapter, tableName: string): boolean { - const row = db - .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?") - .get(tableName) as { name?: string } | undefined; - return Boolean(row?.name); +/** + * Reopen a narrowly selected migration when the table it creates is physically absent. + * + * Historical databases can carry `074_discovery_results` or the rehomed + * `081_inspector_custom_hosts` in the ledger without the table itself (for example after a + * version-slot collision or an incomplete manual recovery). Treating either marker as + * authoritative leaves an incomplete schema. A same-named view does not count as the table; + * replaying the owning migration fails closed instead of silently advancing. + * + * This intentionally detects table absence only. It is not a general schema-healing layer: + * column/rebuild migrations continue to use targeted idempotency checks elsewhere. + */ +const REQUIRED_PHYSICAL_MIGRATIONS = [ + { version: "074", name: "discovery_results", tableName: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts", tableName: "inspector_custom_hosts" }, +] as const; + +function validateRequiredPhysicalMigrationProvenance( + db: SqliteAdapter, + files: Array<{ version: string; name: string; path: string }> +): void { + for (const required of REQUIRED_PHYSICAL_MIGRATIONS) { + if (hasPhysicalTable(db, required.tableName)) continue; + + const migrationExists = files.some( + (file) => file.version === required.version && file.name === required.name + ); + if (!migrationExists) continue; + + const occupied = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(required.version) as { version: string; name: string } | undefined; + if (!occupied || occupied.name === required.name) continue; + + const knownRenumberedCollision = RENAMED_MIGRATION_COMPATIBILITY.some( + (compatibility) => + compatibility.fromVersion === occupied.version && + compatibility.fromName === occupied.name && + files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ) && + files.some( + (file) => + file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ) + ); + const knownLegacySlotCollision = LEGACY_VERSION_SLOT_MIGRATIONS.some( + (legacy) => + legacy.version === occupied.version && + legacy.name === occupied.name && + files.some((file) => file.version === legacy.version && file.name !== legacy.name) + ); + const knownRepairableCollision = knownRenumberedCollision || knownLegacySlotCollision; + if (knownRepairableCollision) continue; + + throw new Error( + `[Migration] Required table "${required.tableName}" is missing, but version ` + + `${required.version} is recorded as unknown migration "${occupied.name}" instead of ` + + `"${required.name}". Refusing to treat this database as current.` + ); + } } -function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean { - const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>; - return columns.some((column) => column.name === columnName); +function findAtomicPhysicalReplays( + db: SqliteAdapter, + files: Array<{ version: string; name: string; path: string }> +): Set { + const replayVersions = new Set(); + + for (const required of REQUIRED_PHYSICAL_MIGRATIONS) { + if (hasPhysicalTable(db, required.tableName)) continue; + + const migrationExists = files.some( + (file) => file.version === required.version && file.name === required.name + ); + if (!migrationExists) continue; + + const applied = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(required.version, required.name) as { version: string; name: string } | undefined; + if (!applied) continue; + + replayVersions.add(required.version); + console.warn( + `[Migration] Will atomically replay ${required.version}_${required.name}: ledger recorded ` + + `"${applied.name}" but required table "${required.tableName}" is missing.` + ); + } + + return replayVersions; } function ensureColumn(db: SqliteAdapter, tableName: string, columnName: string, ddl: string): void { @@ -651,276 +723,31 @@ function applyCompressionCombosMigration(db: SqliteAdapter, migrationPath: strin `); } -function inferPhysicalSchemaBaseline(db: SqliteAdapter): { - version: string; - description: string; -} | null { - for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) { - if (hasTable(db, sentinel.tableName)) { - return { - version: sentinel.version, - description: sentinel.description, - }; - } - } - - const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName)); - if (hasInitialSchema) { - return { - version: "001", - description: "initial schema tables", - }; - } - - return null; -} - -function getPlausiblePendingCount( - files: Array<{ version: string; name: string; path: string }>, - baselineVersion: string -): number { - const baseline = Number.parseInt(baselineVersion, 10); - return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length; -} - /** - * Detect migration name mismatches — when a migration version number - * has been reused/renumbered with a different name. This is a strong signal - * that the migration tracking is corrupted or migrations were renumbered. - */ -function detectNameMismatches( - appliedRecords: Array<{ version: string; name: string }>, - files: Array<{ version: string; name: string; path: string }> -): Array<{ version: string; appliedName: string; diskName: string }> { - const appliedByName = new Map(appliedRecords.map((r) => [r.version, r.name])); - const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = []; - - for (const file of files) { - const appliedName = appliedByName.get(file.version); - if (appliedName && appliedName !== file.name) { - mismatches.push({ - version: file.version, - appliedName, - diskName: file.name, - }); - } - } - - return mismatches; -} - -function reconcileRenumberedMigrations( - db: SqliteAdapter, - files: Array<{ version: string; name: string; path: string }> -): boolean { - let repaired = false; - - for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { - const hasTargetFile = files.some( - (file) => file.version === compatibility.toVersion && file.name === compatibility.toName - ); - const hasSourceFile = files.some( - (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName - ); - - if (!hasTargetFile || !hasSourceFile) { - continue; - } - - const legacyRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") - .get(compatibility.fromVersion, compatibility.fromName) as - { version: string; name: string } | undefined; - if (!legacyRow) { - continue; - } - - const targetRow = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") - .get(compatibility.toVersion) as { version: string } | undefined; - - const applyRepair = db.transaction(() => { - if (targetRow) { - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( - compatibility.fromVersion, - compatibility.fromName - ); - } else { - db.prepare( - "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" - ).run( - compatibility.toVersion, - compatibility.toName, - compatibility.fromVersion, - compatibility.fromName - ); - } - }); - - applyRepair(); - repaired = true; - console.warn( - `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` + - `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.` - ); - - // After the compat rewrite, verify the old version slot is now free. - // A residual row (from a failed prior run, manual intervention, or edge-case - // UPDATE conflict) at the old version would shadow a NEW migration file - // placed at that version number — e.g. 028_create_files_and_batches.sql - // would be skipped because getAppliedVersions() still sees version "028". - const residualRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") - .get(compatibility.fromVersion) as { version: string; name: string } | undefined; - if (residualRow) { - console.warn( - `[Migration] ⚠️ Residual row at version ${compatibility.fromVersion} ` + - `(name: "${residualRow.name}") still present after compat rewrite — ` + - `removing to unblock new migration at this version slot.` - ); - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run( - compatibility.fromVersion - ); - } - } - - return repaired; -} - -function rehomeLegacyVersionSlotMigrations( - db: SqliteAdapter, - files: Array<{ version: string; name: string; path: string }> -): boolean { - let repaired = false; - const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); - - for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { - const diskName = diskNamesByVersion.get(legacy.version); - if (!diskName || diskName === legacy.name) { - continue; - } - - const legacyRow = db - .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") - .get(legacy.version, legacy.name) as { version: string; name: string } | undefined; - if (!legacyRow) { - continue; - } - - const legacyVersion = `legacy-${legacy.version}-${legacy.name}`; - const applyRepair = db.transaction(() => { - const existingLegacyRow = db - .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") - .get(legacyVersion) as { version: string } | undefined; - - if (existingLegacyRow) { - db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( - legacy.version, - legacy.name - ); - return; - } - - db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run( - legacyVersion, - legacy.version, - legacy.name - ); - }); - - applyRepair(); - repaired = true; - console.warn( - `[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` + - `to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.` - ); - } - - return repaired; -} - -/** - * Read a persisted `dbBackup` retention setting through the adapter that is ALREADY open - * for this migration run. + * Run a callback while holding SQLite's IMMEDIATE writer transaction. * - * `backup.ts`'s equivalent goes through `getDbInstance()`, which is unsafe here: this - * code runs from inside database initialization, so asking for the singleton would - * re-enter it. Reading off `db` keeps the same stored values without that risk. A DB too - * old to have `key_value` yet simply falls back to the default. + * Production adapters expose `immediate()` directly. A small number of long-standing + * migration tests and external callers still pass a raw better-sqlite3 Database, whose + * transaction wrapper exposes `.immediate()` instead. Supporting both shapes here keeps + * the safety transaction real: this must never degrade to a plain callback invocation. */ -function readStoredBackupSetting(db: SqliteAdapter, key: string, min: number): number | undefined { - try { - const row = db - .prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?") - .get("dbBackup", key) as { value?: string } | undefined; - if (!row?.value) return undefined; - const parsed = JSON.parse(row.value); - return Number.isInteger(parsed) && parsed >= min ? parsed : undefined; - } catch { - return undefined; +function runImmediateTransaction(db: SqliteAdapter, fn: () => T): T { + const adapterImmediate = (db as Partial).immediate; + if (typeof adapterImmediate === "function") { + let result!: T; + adapterImmediate.call(db, () => { + result = fn(); + }); + return result; } -} -/** - * Enforce the backup retention budget after a pre-migration snapshot (#10421). - * - * Precedence matches `backup.ts`: env override → persisted operator setting → default. - * Never throws: a migration must not fail because housekeeping did. - */ -function pruneMigrationBackups(db: SqliteAdapter, backupDir: string): void { - try { - const maxFiles = process.env.DB_BACKUP_MAX_FILES - ? parsePositiveInt(process.env.DB_BACKUP_MAX_FILES, MAX_DB_BACKUPS) - : (readStoredBackupSetting(db, "maxFiles", 1) ?? MAX_DB_BACKUPS); - const retentionDays = process.env.DB_BACKUP_RETENTION_DAYS - ? parseNonNegativeInt(process.env.DB_BACKUP_RETENTION_DAYS, DEFAULT_DB_BACKUP_RETENTION_DAYS) - : (readStoredBackupSetting(db, "retentionDays", 0) ?? DEFAULT_DB_BACKUP_RETENTION_DAYS); - - const result = pruneBackupDirectory({ backupDir, maxFiles, retentionDays }); - if (result.deletedFiles > 0) { - console.log( - `[Migration] Pruned ${result.deletedFiles} old backup file(s) ` + - `(${result.keptBackupFamilies} kept, maxFiles=${maxFiles}, retentionDays=${retentionDays}).` - ); - } - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[Migration] Failed to prune old backups: ${message}`); - } -} - -/** - * Create a pre-migration backup of the SQLite database using VACUUM INTO. - * Returns the backup path on success, null on failure. - */ -function createPreMigrationBackup(db: SqliteAdapter): string | null { - try { - const sqliteFile = db.name; - if (!sqliteFile || sqliteFile === ":memory:") return null; - - const backupDir = path.join(path.dirname(sqliteFile), "db_backups"); - if (!fs.existsSync(backupDir)) { - fs.mkdirSync(backupDir, { recursive: true }); - } - - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const backupPath = path.join(backupDir, `db_${timestamp}_pre-migration.sqlite`); - const escapedBackupPath = backupPath.replace(/'/g, "''"); - - db.exec(`VACUUM INTO '${escapedBackupPath}'`); - console.log(`[Migration] Pre-migration backup created: ${backupPath}`); - - // #10421: apply the operator's retention budget right here. Without this the - // migration path was the one backup producer that never pruned, so every process - // start with a pending migration added ~5 MB forever (observed: 49k files / 204 GB). - pruneMigrationBackups(db, backupDir); - - return backupPath; - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[Migration] Failed to create pre-migration backup: ${message}`); - return null; + const rawTransaction = db.transaction(fn) as ReturnType & { + immediate?: () => T; + }; + if (typeof rawTransaction.immediate !== "function") { + throw new Error("[Migration] Database adapter does not support IMMEDIATE transactions."); } + return rawTransaction.immediate(); } /** @@ -932,15 +759,243 @@ function createPreMigrationBackup(db: SqliteAdapter): string | null { * 2. Aborts if too many pending migrations on an existing DB (likely wipe) * 3. Creates automatic backup before running any migrations */ -export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean }): number { +export function runMigrations( + db: SqliteAdapter, + options?: { isNewDb?: boolean; databaseExistedBeforeInitialization?: boolean } +): number { const isNewDb = options?.isNewDb === true; + // `isNewDb` also covers a setup-created skeleton so it can bypass the mass-migration + // false positive. Snapshot eligibility must use the independent physical-file fact: + // that skeleton can already contain provider credentials and other operator state. + const databaseExistedBeforeInitialization = + options?.databaseExistedBeforeInitialization ?? !isNewDb; ensureMigrationsTable(db); const files = filterSupersededDuplicateMigrations(getMigrationFiles()); - rehomeLegacyVersionSlotMigrations(db, files); - reconcileRenumberedMigrations(db, files); - const applied = getAppliedVersions(db); - const appliedRecords = getAppliedRecords(db); + validateRequiredPhysicalMigrationProvenance(db, files); + let preMigrationBackup: PreMigrationBackupReceipt | null = null; + let plan!: { + atomicPhysicalReplays: Set; + appliedRecords: Array<{ version: string; name: string }>; + pending: typeof files; + deferredUnsupported: typeof files; + highestAppliedBeforeMigrations: number; + }; + let count = 0; + + const preliminaryApplied = getAppliedVersions(db); + const preliminaryAtomicReplays = findAtomicPhysicalReplays(db, files); + const preliminaryPending = files.filter( + (file) => !preliminaryApplied.has(file.version) || preliminaryAtomicReplays.has(file.version) + ); + const preliminaryDeferred = preliminaryPending.filter((migration) => + isDeferredUnsupportedMigration(db, migration) + ); + const preliminaryActionable = preliminaryPending.filter( + (migration) => !preliminaryDeferred.some((deferred) => deferred.version === migration.version) + ); + const preliminaryHasRepairCandidates = hasLedgerRepairCandidates(db, files); + + // Preserve the historical read-only/no-op path. Merely checking an already-current + // database must not acquire a writer lock (or fail SQLITE_BUSY because another supported + // host currently owns one). Safety state is recomputed under IMMEDIATE whenever work exists. + if (preliminaryActionable.length === 0 && !preliminaryHasRepairCandidates) { + const numericApplied = Array.from(preliminaryApplied) + .map((version) => Number.parseInt(version, 10)) + .filter((version) => !Number.isNaN(version)); + plan = { + atomicPhysicalReplays: preliminaryAtomicReplays, + appliedRecords: getAppliedRecords(db), + pending: preliminaryPending, + deferredUnsupported: preliminaryDeferred, + highestAppliedBeforeMigrations: numericApplied.length > 0 ? Math.max(...numericApplied) : 0, + }; + } + + // sql.js export() finalizes its active SAVEPOINT, so exporting from inside + // `db.immediate()` would make a later safety throw unable to roll repairs back. + // Its adapter is synchronous and in-memory, so no JavaScript writer can interleave + // between this preflight/export and the immediately following savepoint. + if ( + !plan && + db.driver === "sql.js" && + (preliminaryActionable.length > 0 || preliminaryHasRepairCandidates) + ) { + const needsSnapshot = + (preliminaryActionable.length > 0 || preliminaryHasRepairCandidates) && + db.name !== ":memory:" && + databaseExistedBeforeInitialization; + + if (needsSnapshot) { + preMigrationBackup = createPreMigrationBackup(db); + if (!preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + } + } + + // Hold SQLite's native writer lock through snapshot selection, compatibility repairs, + // and the mass-safety decision. Native adapters open a separate read-only connection + // for VACUUM INTO while competing writers remain blocked. The outer transaction then + // commits before migrations so the repository's one-transaction-per-file contract stays + // intact: an earlier successful migration remains committed if a later file fails. + if (!plan) + runImmediateTransaction(db, () => { + const appliedBeforeRepair = getAppliedVersions(db); + const hadAppliedBeforeRepair = appliedBeforeRepair.size > 0; + const preliminaryAtomicReplays = findAtomicPhysicalReplays(db, files); + const preliminaryPending = files.filter( + (file) => + !appliedBeforeRepair.has(file.version) || preliminaryAtomicReplays.has(file.version) + ); + const preliminaryActionable = preliminaryPending.filter( + (migration) => !isDeferredUnsupportedMigration(db, migration) + ); + const mayWriteExistingDatabase = + preliminaryActionable.length > 0 || hasLedgerRepairCandidates(db, files); + const needsSnapshot = + mayWriteExistingDatabase && db.name !== ":memory:" && databaseExistedBeforeInitialization; + + if (needsSnapshot && !preMigrationBackup) { + if (db.driver === "sql.js") { + throw new Error( + "[Migration] sql.js safety state changed after its pre-transaction snapshot preflight; " + + "refusing to export from inside the rollback savepoint." + ); + } + preMigrationBackup = createPreMigrationBackup(db); + if (!preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + } + + rehomeLegacyVersionSlotMigrations(db, files); + reconcileRenumberedMigrations(db, files); + + const atomicPhysicalReplays = findAtomicPhysicalReplays(db, files); + const applied = getAppliedVersions(db); + const appliedRecords = getAppliedRecords(db); + const pending = files.filter( + (file) => !applied.has(file.version) || atomicPhysicalReplays.has(file.version) + ); + const deferredUnsupported = pending.filter((migration) => + isDeferredUnsupportedMigration(db, migration) + ); + const actionablePending = pending.filter( + (migration) => + !deferredUnsupported.some((deferred) => deferred.version === migration.version) + ); + const isFreshSeedOnly = + applied.size === 1 && + applied.has("001") && + inferPhysicalSchemaBaseline(db) === null && + hasTable(db, "provider_connections"); + const requiresDurableBackup = + actionablePending.length > 0 && + db.name !== ":memory:" && + databaseExistedBeforeInitialization; + + // Recompute under the same writer transaction as repairs and fail before any + // ledger mutation can commit if the durable-snapshot requirement is not met. + if (requiresDurableBackup && !preMigrationBackup) { + throw new Error( + "[Migration] Refusing to migrate an existing database without a durable snapshot. " + + "The DATA_DIR filesystem must support atomic hard-link publication." + ); + } + + const isTestEnvironment = isAutomatedTestProcess(); + const maxPendingMigrations = resolveMaxPendingMigrations(); + if ( + actionablePending.length > 0 && + !isTestEnvironment && + !isNewDb && + !isFreshSeedOnly && + maxPendingMigrations > 0 && + (applied.size > 0 || hadAppliedBeforeRepair) && + actionablePending.length > maxPendingMigrations + ) { + const physicalBaseline = inferPhysicalSchemaBaseline(db); + const plausiblePendingCount = physicalBaseline + ? getPlausiblePendingCount(files, physicalBaseline.version) + : null; + + if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) { + console.warn( + `[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` + + `because the physical schema only proves ${physicalBaseline?.version} ` + + `(${physicalBaseline?.description}).` + ); + } else { + const schemaHint = + physicalBaseline && plausiblePendingCount !== null + ? ` Physical schema already shows ${physicalBaseline.version} ` + + `(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` + + `migration(s) are expected from a legitimate upgrade.` + : ""; + const bypassHint = + ` To bypass this check (e.g. after restoring a backup where the migration ` + + `tracking table was wiped), set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 in your ` + + `server.env or DATA_DIR/.env and restart.`; + const msg = + `[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` + + `(threshold is ${maxPendingMigrations}). ` + + `This usually means the migration tracking table was accidentally wiped. ` + + `Running all migrations from scratch will cause data loss or schema errors.` + + schemaHint + + bypassHint; + + if (memoizedSafetyAbort && memoizedSafetyAbort.message === msg) { + console.error( + `[Migration] 🛑 ABORT (repeat — see earlier detail): ` + + `${actionablePending.length} pending > threshold ${maxPendingMigrations}. ` + + `Set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 to bypass.` + ); + throw memoizedSafetyAbort; + } + console.error(msg); + memoizedSafetyAbort = new MigrationSafetyAbortError(msg); + throw memoizedSafetyAbort; + } + } + + if ( + preMigrationBackup && + hashFileSync(preMigrationBackup.path) !== preMigrationBackup.sha256 + ) { + throw new Error( + "[Migration] Refusing to migrate because the pre-migration snapshot changed before use." + ); + } + + const numericApplied = Array.from(applied) + .map((version) => Number.parseInt(version, 10)) + .filter((version) => !Number.isNaN(version)); + const highestAppliedBeforeMigrations = + numericApplied.length > 0 ? Math.max(...numericApplied) : 0; + + plan = { + atomicPhysicalReplays, + appliedRecords, + pending, + deferredUnsupported, + highestAppliedBeforeMigrations, + }; + }); + + const { + atomicPhysicalReplays, + appliedRecords, + pending, + deferredUnsupported, + highestAppliedBeforeMigrations, + } = plan; // ── Safety Check 1: Detect migration name mismatches (renumbering) ── const mismatches = detectNameMismatches(appliedRecords, files); @@ -963,34 +1018,15 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } ); } - // ── Gap Reconciliation: Identify non-contiguous missing migrations ── - // Do not rely on any highest-version-applied heuristic. We must explicitly - // iterate through all missing files on disk and apply them if they are missing - // from the _omniroute_migrations table. - const numericApplied = Array.from(applied) - .map((v) => Number.parseInt(v, 10)) - .filter((n) => !Number.isNaN(n)); - const highestApplied = numericApplied.length > 0 ? Math.max(...numericApplied) : 0; - const pending = files.filter((f) => { - const isMissing = !applied.has(f.version); - if (isMissing && Number(f.version) < highestApplied) { + for (const migration of pending) { + if (Number(migration.version) < highestAppliedBeforeMigrations) { console.warn( `[Migration] 🔄 RECONCILIATION: Found missing intermediate migration ` + - `${f.version}_${f.name} (highest applied is ${highestApplied}). ` + + `${migration.version}_${migration.name} ` + + `(highest applied is ${highestAppliedBeforeMigrations}). ` + `This gap will be back-filled to ensure schema integrity.` ); } - return isMissing; - }); - const deferredUnsupported = pending.filter((migration) => - isDeferredUnsupportedMigration(db, migration) - ); - const actionablePending = pending.filter( - (migration) => !deferredUnsupported.some((deferred) => deferred.version === migration.version) - ); - - if (pending.length === 0) { - return 0; // Nothing to do } if (deferredUnsupported.length > 0) { @@ -1003,101 +1039,28 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } ); } - // ── Safety Check 2: Mass-migration detection (abort if existing DB + many migrations) ── - // Skip in test environments where fresh DBs legitimately have many pending migrations. - const isTestEnvironment = isAutomatedTestProcess(); - - // #3416: resolve the threshold at call time so OMNIROUTE_MAX_PENDING_MIGRATIONS - // can override the default (0 disables the check). The abort message below - // interpolates this resolved value, so it auto-reflects any override. - const maxPendingMigrations = resolveMaxPendingMigrations(); - - // #9934: `omniroute setup`'s openOmniRouteDb writes a partial skeleton file - // (provider_connections + key_value) that has never had migrations run. When - // the first `serve` opens it and auto-seeds only the 001 marker, the applied - // set is exactly {001} — which would otherwise look like a wiped existing DB - // and trip this abort on a brand-new install. This is distinct from a real - // wiped/backup-restored database: that case has a non-trivial physical schema - // (baseline inference is non-null) and full data tables, so it still aborts. - // The 001-marker-only state on a provider_connections skeleton is the fresh - // auto-seed — let it through. A genuinely empty table is already exempt via - // `applied.size > 0`, and an upgraded DB has a non-trivial applied set. - const isFreshSeedOnly = - applied.size === 1 && - applied.has("001") && - inferPhysicalSchemaBaseline(db) === null && - hasTable(db, "provider_connections"); - - if ( - !isTestEnvironment && - !isNewDb && - !isFreshSeedOnly && - process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true" && - maxPendingMigrations > 0 && - applied.size > 0 && - actionablePending.length > maxPendingMigrations - ) { - const physicalBaseline = inferPhysicalSchemaBaseline(db); - const plausiblePendingCount = physicalBaseline - ? getPlausiblePendingCount(files, physicalBaseline.version) - : null; - - if (plausiblePendingCount !== null && actionablePending.length <= plausiblePendingCount) { - console.warn( - `[Migration] Allowing ${actionablePending.length} pending migrations on an existing database ` + - `because the physical schema only proves ${physicalBaseline?.version} ` + - `(${physicalBaseline?.description}).` - ); - } else { - const schemaHint = - physicalBaseline && plausiblePendingCount !== null - ? ` Physical schema already shows ${physicalBaseline.version} ` + - `(${physicalBaseline.description}), so at most ${plausiblePendingCount} pending ` + - `migration(s) are expected from a legitimate upgrade.` - : ""; - const bypassHint = - ` To bypass this check (e.g. after restoring a backup where the migration ` + - `tracking table was wiped), set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 in your ` + - `server.env or DATA_DIR/.env and restart.`; - const msg = - `[Migration] 🛑 ABORT: Detected ${actionablePending.length} pending migrations on an existing database ` + - `(threshold is ${maxPendingMigrations}). ` + - `This usually means the migration tracking table was accidentally wiped. ` + - `Running all migrations from scratch will cause data loss or schema errors.` + - schemaHint + - bypassHint; - - // #6260: memoize so the cascade of downstream ensureDbInitialized() calls - // that re-open the DB throw the SAME instance and only log once. - if (memoizedSafetyAbort && memoizedSafetyAbort.message === msg) { - console.error( - `[Migration] 🛑 ABORT (repeat — see earlier detail): ` + - `${actionablePending.length} pending > threshold ${maxPendingMigrations}. ` + - `Set OMNIROUTE_MAX_PENDING_MIGRATIONS=0 to bypass.` - ); - throw memoizedSafetyAbort; - } - console.error(msg); - memoizedSafetyAbort = new MigrationSafetyAbortError(msg); - throw memoizedSafetyAbort; - } + if (preMigrationBackup && hashFileSync(preMigrationBackup.path) !== preMigrationBackup.sha256) { + throw new Error( + "[Migration] Refusing to migrate because the pre-migration snapshot changed before use." + ); } - // ── Safety Check 3: Pre-migration backup ── - // Skip backup if it's a completely fresh database (0 applied and all pending) - // or if running in tests (where AUTO_BACKUP might be disabled) - if (applied.size > 0 && process.env.DISABLE_SQLITE_AUTO_BACKUP !== "true") { - createPreMigrationBackup(db); - } - - let count = 0; - for (const migration of pending) { - if (isDeferredUnsupportedMigration(db, migration)) { - continue; - } + if (isDeferredUnsupportedMigration(db, migration)) continue; const applyMigration = db.transaction(() => { + if (atomicPhysicalReplays.has(migration.version)) { + const removed = db + .prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?") + .run(migration.version, migration.name); + if (removed.changes !== 1) { + throw new Error( + `[Migration] Atomic replay lost its expected ledger marker for ` + + `${migration.version}_${migration.name}.` + ); + } + } + if (isSchemaAlreadyApplied(db, migration)) { console.warn( `[Migration] Skipped executing ${migration.version}_${migration.name} as schema changes are already present (Idempotency check).` @@ -1120,29 +1083,36 @@ export function runMigrations(db: SqliteAdapter, options?: { isNewDb?: boolean } try { applyMigration(); - count++; + count += 1; console.log(`[Migration] Applied: ${migration.version}_${migration.name}`); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); - // "duplicate column name" means the column already exists — end state achieved, mark applied. - if (message.includes("duplicate column name")) { + if ( + message.includes("duplicate column name") && + !atomicPhysicalReplays.has(migration.version) + ) { const applyMarkerOnly = db.transaction(() => { db.prepare( "INSERT OR IGNORE INTO _omniroute_migrations (version, name) VALUES (?, ?)" ).run(migration.version, migration.name); }); applyMarkerOnly(); - count++; + count += 1; console.log( `[Migration] Applied (column pre-exists): ${migration.version}_${migration.name}` ); } else { console.error(`[Migration] FAILED: ${migration.version}_${migration.name} — ${message}`); - throw err; // Re-throw to prevent DB from starting in inconsistent state + throw err; } } } + // Retention intentionally does not run inside the migration window. Another process + // may still be using a different snapshot as its in-flight restore point. Manual and + // scheduled backup paths continue to enforce the operator's retention policy; retries + // here are bounded by the deterministic content address instead of destructive pruning. + if (count > 0) { console.log(`[Migration] ${count} migration(s) applied successfully.`); } @@ -1175,7 +1145,7 @@ function insertDefaultDatabaseSettings(db: SqliteAdapter) { // Run in an immediate transaction to avoid nested transactions try { - db.immediate(() => { + runImmediateTransaction(db, () => { tx(); }); } catch (error) { diff --git a/src/lib/db/migrationRunner/constants.ts b/src/lib/db/migrationRunner/constants.ts index 773f6e8c12..089837f93f 100644 --- a/src/lib/db/migrationRunner/constants.ts +++ b/src/lib/db/migrationRunner/constants.ts @@ -158,6 +158,14 @@ export const RENAMED_MIGRATION_COMPATIBILITY = [ toVersion: "151", toName: "windsurf_to_devin_desktop", }, + { + // inspector_custom_hosts was once published in slot 074, now occupied by + // discovery_results. Its canonical idempotent migration lives at 081. + fromVersion: "074", + fromName: "inspector_custom_hosts", + toVersion: "081", + toName: "inspector_custom_hosts", + }, { fromVersion: "134", fromName: "ccr_blocks", diff --git a/src/lib/db/migrationRunner/logger.ts b/src/lib/db/migrationRunner/logger.ts new file mode 100644 index 0000000000..c9f9b0d2e7 --- /dev/null +++ b/src/lib/db/migrationRunner/logger.ts @@ -0,0 +1,13 @@ +const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string"; + +export const migrationConsole = { + log: (...args: unknown[]) => { + if (!isNodeTestRunnerChild) globalThis.console.log(...args); + }, + warn: (...args: unknown[]) => { + if (!isNodeTestRunnerChild) globalThis.console.warn(...args); + }, + error: (...args: unknown[]) => { + globalThis.console.error(...args); + }, +}; diff --git a/src/lib/db/migrationRunner/preMigrationBackup.ts b/src/lib/db/migrationRunner/preMigrationBackup.ts new file mode 100644 index 0000000000..3e2eea3caf --- /dev/null +++ b/src/lib/db/migrationRunner/preMigrationBackup.ts @@ -0,0 +1,293 @@ +import { createHash } from "crypto"; +import fs from "fs"; +import path from "path"; + +import type { SqliteAdapter } from "../adapters/types"; +import { tryOpenSync } from "../adapters/driverFactory"; +import { migrationConsole as console } from "./logger"; + +export type PreMigrationBackupReceipt = { + path: string; + sha256: string; +}; + +function fsyncDirectoryEntry(directory: string): void { + let fd: number | null = null; + try { + fd = fs.openSync(directory, "r"); + fs.fsyncSync(fd); + } catch (error: unknown) { + const code = (error as NodeJS.ErrnoException | null)?.code; + const windowsDirectoryHandleUnsupported = + process.platform === "win32" && + (code === "EACCES" || code === "EPERM" || code === "EISDIR" || code === "EINVAL"); + if (!windowsDirectoryHandleUnsupported) throw error; + } finally { + if (fd !== null) fs.closeSync(fd); + } +} + +export function hashFileSync(filePath: string): string { + const hash = createHash("sha256"); + const fd = fs.openSync(filePath, "r"); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let position = 0; + + try { + while (true) { + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + fs.closeSync(fd); + } + + return hash.digest("hex"); +} + +function getReusablePreMigrationBackup( + candidatePath: string, + expectedSha256: string +): PreMigrationBackupReceipt | null { + if (!fs.existsSync(candidatePath)) return null; + + const before = fs.lstatSync(candidatePath); + if (!before.isFile() || hashFileSync(candidatePath) !== expectedSha256) { + throw new Error( + `[Migration] Content-addressed snapshot path exists with unexpected content: ${candidatePath}` + ); + } + const after = fs.lstatSync(candidatePath); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs + ) { + throw new Error( + `[Migration] Content-addressed snapshot changed while it was being validated: ${candidatePath}` + ); + } + + return { path: candidatePath, sha256: expectedSha256 }; +} + +function publishSnapshotWithoutOverwrite(tempPath: string, destination: string): void { + // link() publishes a complete same-filesystem image atomically and, unlike rename(), + // fails with EEXIST instead of overwriting a path created by another process. There is + // deliberately no copy/rename fallback: filesystems without this primitive fail closed + // instead of exposing a partial canonical `.sqlite` file after a crash. + fs.linkSync(tempPath, destination); + const publishedFd = fs.openSync(destination, "r+"); + try { + // Flush through the published name as well as the already-fsynced temp handle. + // On Windows this maps to FlushFileBuffers and is the strongest file-level + // durability proof available when directory handles are unsupported by Node. + fs.fsyncSync(publishedFd); + } finally { + fs.closeSync(publishedFd); + } + fsyncDirectoryEntry(path.dirname(destination)); +} + +function fsyncReusableSnapshot(snapshotPath: string): void { + const fd = fs.openSync(snapshotPath, "r+"); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +type SqlJsSnapshotClone = { + run(sql: string): void; + export(): Uint8Array; + close(): void; +}; + +const SQLITE_HEADER_MIN_BYTES = 100; +const SQLITE_HEADER_MAGIC = "SQLite format 3\0"; +const SQLITE_CHANGE_COUNTER_OFFSET = 24; +const SQLITE_VERSION_VALID_FOR_OFFSET = 92; +const SQLITE_STANDALONE_CHANGE_COUNTER = 1; + +function exportCanonicalSqlJsSnapshot(raw: { export: () => Uint8Array }): Buffer { + const RawDatabase = ( + raw as unknown as { constructor: new (data: Uint8Array) => SqlJsSnapshotClone } + ).constructor; + let clone: SqlJsSnapshotClone | null = null; + + try { + // A rolled-back sql.js SAVEPOINT can leave SQLite's physical change counter advanced + // even though every logical row/schema change was undone. Canonicalize only a detached + // clone: VACUUM removes rollback-only page artifacts without touching the live database. + clone = new RawDatabase(raw.export()); + clone.run("VACUUM"); + const canonical = Buffer.from(clone.export()); + + if ( + canonical.length < SQLITE_HEADER_MIN_BYTES || + canonical.subarray(0, SQLITE_HEADER_MAGIC.length).toString("binary") !== SQLITE_HEADER_MAGIC + ) { + throw new Error("sql.js export did not produce a valid SQLite file header"); + } + + // SQLite file-header offsets 24 and 92 are the change counter and + // version-valid-for number. VACUUM keeps the two equal, but seeds them from the + // source image, so an otherwise identical rolled-back retry still gets a different + // byte hash. A standalone snapshot has no open readers to invalidate; assigning the + // same stable value to both fields preserves a valid/restorable header while making + // the complete canonical image deterministic. + canonical.writeUInt32BE(SQLITE_STANDALONE_CHANGE_COUNTER, SQLITE_CHANGE_COUNTER_OFFSET); + canonical.writeUInt32BE(SQLITE_STANDALONE_CHANGE_COUNTER, SQLITE_VERSION_VALID_FOR_OFFSET); + return canonical; + } finally { + clone?.close(); + } +} + +function writeSqlJsSnapshot(raw: { export: () => Uint8Array }, tempPath: string): void { + let fd: number | null = null; + + try { + fd = fs.openSync(tempPath, "wx"); + fs.writeFileSync(fd, exportCanonicalSqlJsSnapshot(raw)); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = null; + } catch (error: unknown) { + if (fd !== null) { + try { + fs.closeSync(fd); + } catch { + // The original snapshot error remains authoritative. + } + } + throw error; + } +} + +function cleanupOwnedSnapshotTemp(tempDir: string | null, tempPath: string | null): void { + if (!tempDir || !fs.existsSync(tempDir)) return; + + try { + // `tempDir` comes only from mkdtempSync below. Removing that exact owned directory + // lets Node retry Windows/AV EBUSY and EPERM failures without touching canonical backups. + fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 25 }); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[Migration] Failed to remove owned snapshot temp directory` + + `${tempPath ? ` (${tempPath})` : ""}: ${message}` + ); + } +} + +/** + * Create a synchronous pre-migration snapshot. + * + * Native SQLite drivers use VACUUM INTO. sql.js has an in-memory VFS, so a host + * path passed to VACUUM INTO is not writable; export its current database image + * directly instead. The SHA-256 content address lives in the first portion of the + * canonical `db__.sqlite` shape, preserving reason parsing while + * making unchanged retries an O(1) lookup even with tens of thousands of old backups. + * Work happens inside an exclusively-created + * temp directory, so failure cleanup has exact ownership. Publication uses an atomic, + * no-overwrite hard link. If the filesystem cannot provide that primitive, the caller + * fails closed instead of exposing a partial canonical `.sqlite` file. A content hash + * reuses an identical prior snapshot, so repeated zero-progress startups retain one + * restore point for that database state without ever deleting a published backup. + */ +export function createPreMigrationBackup(db: SqliteAdapter): PreMigrationBackupReceipt | null { + let backupPath: string | null = null; + let tempPath: string | null = null; + let tempDir: string | null = null; + + try { + const sqliteFile = db.name; + if (!sqliteFile || sqliteFile === ":memory:") return null; + + const backupDir = path.join(path.dirname(sqliteFile), "db_backups"); + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir, { recursive: true }); + fsyncDirectoryEntry(path.dirname(backupDir)); + } + + tempDir = fs.mkdtempSync(path.join(backupDir, ".migration-snapshot-")); + tempPath = path.join(tempDir, "snapshot.sqlite"); + + if (db.driver === "sql.js") { + const raw = db.raw as { export?: () => Uint8Array } | null; + if (!raw || typeof raw.export !== "function") { + throw new Error("sql.js adapter does not expose database export()"); + } + writeSqlJsSnapshot(raw as { export: () => Uint8Array }, tempPath); + } else { + const escapedTempPath = tempPath.replace(/'/g, "''"); + const snapshotDb = tryOpenSync(sqliteFile, { readonly: true, fileMustExist: true }); + if (!snapshotDb) { + throw new Error("no synchronous read-only SQLite driver is available for snapshotting"); + } + try { + snapshotDb.exec(`VACUUM INTO '${escapedTempPath}'`); + } finally { + snapshotDb.close(); + } + const fd = fs.openSync(tempPath, "r+"); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + } + + const sha256 = hashFileSync(tempPath); + backupPath = path.join(backupDir, `db_state-${sha256}_pre-migration.sqlite`); + const reusable = getReusablePreMigrationBackup(backupPath, sha256); + if (reusable) { + fsyncReusableSnapshot(reusable.path); + fsyncDirectoryEntry(backupDir); + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Reusing identical pre-migration backup: ${reusable.path}`); + return reusable; + } + + try { + publishSnapshotWithoutOverwrite(tempPath, backupPath); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code !== "EEXIST") throw error; + const racedReusable = getReusablePreMigrationBackup(backupPath, sha256); + if (!racedReusable) throw error; + fsyncReusableSnapshot(racedReusable.path); + fsyncDirectoryEntry(backupDir); + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Reusing concurrently published backup: ${racedReusable.path}`); + return racedReusable; + } + cleanupOwnedSnapshotTemp(tempDir, tempPath); + tempDir = null; + tempPath = null; + console.log(`[Migration] Pre-migration backup created: ${backupPath}`); + + return { path: backupPath, sha256 }; + } catch (error: unknown) { + // Never unlink a canonical backup here: publication may have failed because another + // actor created it first. The exclusive temp directory is the only cleanup authority. + cleanupOwnedSnapshotTemp(tempDir, tempPath); + const message = error instanceof Error ? error.message : String(error); + console.warn(`[Migration] Failed to create pre-migration backup: ${message}`); + throw new Error( + `[Migration] Refusing to migrate an existing database without a durable snapshot. ` + + `Snapshot creation failed: ${message}. The DATA_DIR filesystem must support atomic ` + + `no-overwrite hard links, durable file synchronization, and directory synchronization ` + + `where the platform exposes it.`, + { cause: error instanceof Error ? error : undefined } + ); + } +} diff --git a/src/lib/db/migrationRunner/schemaState.ts b/src/lib/db/migrationRunner/schemaState.ts new file mode 100644 index 0000000000..f38e28d0b1 --- /dev/null +++ b/src/lib/db/migrationRunner/schemaState.ts @@ -0,0 +1,248 @@ +import type { SqliteAdapter } from "../adapters/types"; +import { + INITIAL_SCHEMA_SENTINELS, + LEGACY_VERSION_SLOT_MIGRATIONS, + PHYSICAL_SCHEMA_SENTINELS, + RENAMED_MIGRATION_COMPATIBILITY, +} from "./constants"; +import { migrationConsole as console } from "./logger"; + +type MigrationFile = { version: string; name: string; path: string }; + +export function hasTable(db: SqliteAdapter, tableName: string): boolean { + const row = db + .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?") + .get(tableName) as { name?: string } | undefined; + return Boolean(row?.name); +} + +export function hasPhysicalTable(db: SqliteAdapter, tableName: string): boolean { + const row = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(tableName) as { name?: string } | undefined; + return Boolean(row?.name); +} + +export function hasColumn(db: SqliteAdapter, tableName: string, columnName: string): boolean { + const columns = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{ name?: string }>; + return columns.some((column) => column.name === columnName); +} + +export function inferPhysicalSchemaBaseline(db: SqliteAdapter): { + version: string; + description: string; +} | null { + for (const sentinel of PHYSICAL_SCHEMA_SENTINELS) { + if (hasTable(db, sentinel.tableName)) { + return { + version: sentinel.version, + description: sentinel.description, + }; + } + } + + const hasInitialSchema = INITIAL_SCHEMA_SENTINELS.every((tableName) => hasTable(db, tableName)); + if (hasInitialSchema) { + return { + version: "001", + description: "initial schema tables", + }; + } + + return null; +} + +export function getPlausiblePendingCount(files: MigrationFile[], baselineVersion: string): number { + const baseline = Number.parseInt(baselineVersion, 10); + return files.filter((file) => Number.parseInt(file.version, 10) > baseline).length; +} + +/** + * Detect migration name mismatches — when a migration version number + * has been reused/renumbered with a different name. This is a strong signal + * that the migration tracking is corrupted or migrations were renumbered. + */ +export function detectNameMismatches( + appliedRecords: Array<{ version: string; name: string }>, + files: MigrationFile[] +): Array<{ version: string; appliedName: string; diskName: string }> { + const appliedByName = new Map(appliedRecords.map((record) => [record.version, record.name])); + const mismatches: Array<{ version: string; appliedName: string; diskName: string }> = []; + + for (const file of files) { + const appliedName = appliedByName.get(file.version); + if (appliedName && appliedName !== file.name) { + mismatches.push({ + version: file.version, + appliedName, + diskName: file.name, + }); + } + } + + return mismatches; +} + +export function reconcileRenumberedMigrations(db: SqliteAdapter, files: MigrationFile[]): boolean { + let repaired = false; + + for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { + const hasTargetFile = files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ); + const hasSourceFile = files.some( + (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ); + + if (!hasTargetFile || !hasSourceFile) { + continue; + } + + const legacyRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(compatibility.fromVersion, compatibility.fromName) as + { version: string; name: string } | undefined; + if (!legacyRow) { + continue; + } + + const targetRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(compatibility.toVersion) as { version: string; name: string } | undefined; + + const isSameSlotReplacement = compatibility.fromVersion === compatibility.toVersion; + if (targetRow && !isSameSlotReplacement && targetRow.name !== compatibility.toName) { + throw new Error( + `[Migration] Cannot reconcile ${compatibility.fromVersion}_${compatibility.fromName}: ` + + `target version ${compatibility.toVersion} is occupied by unknown migration ` + + `"${targetRow.name}" (expected "${compatibility.toName}").` + ); + } + + const applyRepair = db.transaction(() => { + if (targetRow) { + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( + compatibility.fromVersion, + compatibility.fromName + ); + } else { + db.prepare( + "UPDATE _omniroute_migrations SET version = ?, name = ? WHERE version = ? AND name = ?" + ).run( + compatibility.toVersion, + compatibility.toName, + compatibility.fromVersion, + compatibility.fromName + ); + } + }); + + applyRepair(); + repaired = true; + console.warn( + `[Migration] Reconciled renamed migration ${compatibility.fromVersion}_${compatibility.fromName} ` + + `to ${compatibility.toVersion}_${compatibility.toName} to preserve pending migrations.` + ); + + // After the compat rewrite, verify the old version slot is now free. + // A residual row (from a failed prior run, manual intervention, or edge-case + // UPDATE conflict) at the old version would shadow a NEW migration file + // placed at that version number — e.g. 028_create_files_and_batches.sql + // would be skipped because getAppliedVersions() still sees version "028". + const residualRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ?") + .get(compatibility.fromVersion) as { version: string; name: string } | undefined; + if (residualRow) { + console.warn( + `[Migration] ⚠️ Residual row at version ${compatibility.fromVersion} ` + + `(name: "${residualRow.name}") still present after compat rewrite — ` + + `removing to unblock new migration at this version slot.` + ); + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ?").run( + compatibility.fromVersion + ); + } + } + + return repaired; +} + +export function rehomeLegacyVersionSlotMigrations( + db: SqliteAdapter, + files: MigrationFile[] +): boolean { + let repaired = false; + const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); + + for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { + const diskName = diskNamesByVersion.get(legacy.version); + if (!diskName || diskName === legacy.name) { + continue; + } + + const legacyRow = db + .prepare("SELECT version, name FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(legacy.version, legacy.name) as { version: string; name: string } | undefined; + if (!legacyRow) { + continue; + } + + const legacyVersion = `legacy-${legacy.version}-${legacy.name}`; + const applyRepair = db.transaction(() => { + const existingLegacyRow = db + .prepare("SELECT version FROM _omniroute_migrations WHERE version = ?") + .get(legacyVersion) as { version: string } | undefined; + + if (existingLegacyRow) { + db.prepare("DELETE FROM _omniroute_migrations WHERE version = ? AND name = ?").run( + legacy.version, + legacy.name + ); + return; + } + + db.prepare("UPDATE _omniroute_migrations SET version = ? WHERE version = ? AND name = ?").run( + legacyVersion, + legacy.version, + legacy.name + ); + }); + + applyRepair(); + repaired = true; + console.warn( + `[Migration] Rehomed legacy migration ${legacy.version}_${legacy.name} ` + + `to ${legacyVersion} so current ${legacy.version}_${diskName} can apply.` + ); + } + + return repaired; +} + +export function hasLedgerRepairCandidates(db: SqliteAdapter, files: MigrationFile[]): boolean { + const diskNamesByVersion = new Map(files.map((file) => [file.version, file.name])); + for (const legacy of LEGACY_VERSION_SLOT_MIGRATIONS) { + const diskName = diskNamesByVersion.get(legacy.version); + if (!diskName || diskName === legacy.name) continue; + const row = db + .prepare("SELECT 1 FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(legacy.version, legacy.name); + if (row) return true; + } + + for (const compatibility of RENAMED_MIGRATION_COMPATIBILITY) { + const hasTargetFile = files.some( + (file) => file.version === compatibility.toVersion && file.name === compatibility.toName + ); + const hasSourceFile = files.some( + (file) => file.version === compatibility.fromVersion && file.name !== compatibility.fromName + ); + if (!hasTargetFile || !hasSourceFile) continue; + const row = db + .prepare("SELECT 1 FROM _omniroute_migrations WHERE version = ? AND name = ?") + .get(compatibility.fromVersion, compatibility.fromName); + if (row) return true; + } + + return false; +} diff --git a/src/lib/guardrails/credentialMasker.ts b/src/lib/guardrails/credentialMasker.ts index d5529f84f6..6ac88f8fb3 100644 --- a/src/lib/guardrails/credentialMasker.ts +++ b/src/lib/guardrails/credentialMasker.ts @@ -1,5 +1,9 @@ -import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base"; +import { CREDENTIAL_PATTERNS } from "@omniroute/open-sse/utils/credentialPatterns.ts"; import { getSettings } from "@/lib/db/settings"; +import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base"; + +export { CREDENTIAL_PATTERNS }; +export type { CredentialPattern } from "@omniroute/open-sse/utils/credentialPatterns.ts"; /** * CredentialMaskerGuardrail — redacts well-known API-key / secret-token patterns @@ -11,88 +15,6 @@ import { getSettings } from "@/lib/db/settings"; * Future: per-pipeline / per-provider scoping via GuardrailContext. */ -export interface CredentialPattern { - name: string; - regex: RegExp; - replacement: string; -} - -export const CREDENTIAL_PATTERNS: CredentialPattern[] = [ - // ── LLM provider keys ────────────────────────────────────────────────── - { name: "openai_proj", regex: /sk-proj-[A-Za-z0-9_-]{20,}/g, replacement: "[REDACTED:openai]" }, - { name: "openai", regex: /\bsk-[A-Za-z0-9]{48}\b/g, replacement: "[REDACTED:openai]" }, - { - name: "anthropic", - regex: /sk-ant-api[0-9]?-[A-Za-z0-9_-]{20,}/g, - replacement: "[REDACTED:anthropic]", - }, - { - name: "anthropic_alt", - regex: /sk-ant-[A-Za-z0-9_-]{20,}/g, - replacement: "[REDACTED:anthropic]", - }, - { name: "google", regex: /AIza[0-9A-Za-z_-]{35}/g, replacement: "[REDACTED:google]" }, - { name: "huggingface", regex: /hf_[A-Za-z0-9]{34}/g, replacement: "[REDACTED:hf]" }, - { name: "replicate", regex: /r8_[A-Za-z0-9]{37}/g, replacement: "[REDACTED:replicate]" }, - // ── VCS / SaaS tokens ────────────────────────────────────────────────── - { name: "github", regex: /gh[pousr]_[A-Za-z0-9]{36,}/g, replacement: "[REDACTED:github]" }, - { name: "slack", regex: /xox[bpoa]-[A-Za-z0-9-]{10,}/g, replacement: "[REDACTED:slack]" }, - { name: "linear", regex: /lin_api_[A-Za-z0-9]{40}/g, replacement: "[REDACTED:linear]" }, - { name: "notion", regex: /secret_[A-Za-z0-9]{43}/g, replacement: "[REDACTED:notion]" }, - { name: "npm", regex: /npm_[A-Za-z0-9]{36}/g, replacement: "[REDACTED:npm]" }, - { name: "postman", regex: /PMAK-[a-f0-9]{8}-[a-f0-9]{32}/g, replacement: "[REDACTED:postman]" }, - { - name: "discord", - regex: /\b[MN][A-Za-z0-9]{23}\.[A-Za-z0-9]{6}\.[A-Za-z0-9]{27}\b/g, - replacement: "[REDACTED:discord]", - }, - // ── Payments ─────────────────────────────────────────────────────────── - { - name: "stripe", - regex: /(?:sk|rk)_(?:live|test)_[0-9a-zA-Z]{24,}/g, - replacement: "[REDACTED:stripe]", - }, - { - name: "square", - regex: /sq0(?:atp-[0-9A-Za-z_-]{22}|csp-[0-9A-Za-z_-]{43})/g, - replacement: "[REDACTED:square]", - }, - // ── Cloud / infra ────────────────────────────────────────────────────── - { name: "aws_access_key", regex: /AKIA[0-9A-Z]{16}/g, replacement: "[REDACTED:aws]" }, - { name: "twilio", regex: /\bSK[0-9a-fA-F]{32}\b/g, replacement: "[REDACTED:twilio]" }, - { - name: "sendgrid", - regex: /SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}/g, - replacement: "[REDACTED:sendgrid]", - }, - { name: "mailgun", regex: /key-[a-f0-9]{32}/g, replacement: "[REDACTED:mailgun]" }, - // ── Crypto / identity ────────────────────────────────────────────────── - { - name: "private_key", - regex: - /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/g, - replacement: "[REDACTED:private_key]", - }, - { - name: "jwt", - regex: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, - replacement: "[REDACTED:jwt]", - }, - // ── Connection strings (creds embedded in URI) ───────────────────────── - { - name: "connection_string", - regex: /(?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|amqp):\/\/[^:/@\s"']+:[^:/@\s"']+@/g, - replacement: "[REDACTED:connection_string]", - }, - // ── Header-style secrets ─────────────────────────────────────────────── - { - name: "auth_header", - regex: - /((?:["\x27]?(?:Authorization|x-api-key|api-key|apikey)["\x27]?\s*[:=]\s*["\x27]?)(?:(?:Bearer|Basic|Token)\s+)?)[A-Za-z0-9._~+/=-]{10,}/gi, - replacement: "$1[REDACTED:auth_header]", - }, -]; - export interface CredentialRedactionResult { text: string; detections: Array<{ type: string; count: number }>; diff --git a/src/lib/logPayloads.ts b/src/lib/logPayloads.ts index 5aa2f9eb9c..5fdef8c675 100644 --- a/src/lib/logPayloads.ts +++ b/src/lib/logPayloads.ts @@ -1,3 +1,8 @@ +import { + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "@omniroute/open-sse/utils/errorSanitization.ts"; +import { projectResponsesFailureOutput } from "@omniroute/open-sse/utils/responsesFailureOutput.ts"; import { sanitizePII } from "./piiSanitizer"; const SENSITIVE_KEYS = new Set([ @@ -35,6 +40,21 @@ const SENSITIVE_KEYS = new Set([ "runtimeKey", ]); +const SENSITIVE_CHALLENGE_KEYS = new Set([ + "recaptchav3token", + "recaptchatoken", + "turnstiletoken", + "prooftoken", + "resumetoken", + "preparetoken", +]); + +function isSensitivePayloadKey(key: string): boolean { + if (SENSITIVE_KEYS.has(key)) return true; + const normalizedKey = key.replace(/[-_]/g, "").toLowerCase(); + return SENSITIVE_CHALLENGE_KEYS.has(normalizedKey); +} + type JsonRecord = Record; const ENCRYPTED_REASONING_KEY = "encrypted_content"; @@ -60,6 +80,283 @@ export function omitEncryptedReasoningFromLogChunks(chunks: string[]): string[] return found ? [omitted] : chunks; } +const ERROR_SUBTREE_KEYS = new Set([ + "error", + "errors", + "warning", + "warnings", + "errormessage", + "warningmessage", + "errordescription", + "warningdescription", + "lasterror", +]); + +function isErrorSubtreeKey(key: string): boolean { + return ERROR_SUBTREE_KEYS.has(key.replace(/[-_]/g, "").toLowerCase()); +} + +function sanitizeErrorSubtreeValue(value: unknown): unknown { + if (typeof value === "string") return sanitizeErrorMessage(value); + try { + if (value instanceof Error) { + return { + name: sanitizeErrorMessage(value.name) || "Error", + message: sanitizeErrorMessage(value.message), + }; + } + return sanitizeUpstreamDetails(value); + } catch { + return "[REDACTED]"; + } +} + +type ErrorSubtreeProjection = { value: unknown; found: boolean }; + +function projectErrorSubtreesForLog( + value: unknown, + seen = new WeakSet(), + forceResponsesFailure = false, + protocolResponseObject = false +): ErrorSubtreeProjection { + if (forceResponsesFailure && typeof value === "string") { + return { value: sanitizeErrorMessage(value) || "[REDACTED]", found: true }; + } + if (typeof value === "string") { + const trimmed = value.trim(); + if ( + (trimmed.startsWith("{") || trimmed.startsWith("[")) && + STREAM_ERROR_ENVELOPE_RE.test(trimmed) + ) { + try { + const parsed: unknown = JSON.parse(trimmed); + const projected = isDiscriminatedStreamError(parsed) + ? { value: sanitizeErrorSubtreeValue(parsed), found: true } + : projectErrorSubtreesForLog(parsed, seen); + if (projected.found) { + const serialized = JSON.stringify(projected.value); + if (typeof serialized === "string") return { value: serialized, found: true }; + } + } catch { + return { value: sanitizeErrorMessage(value) || "[REDACTED]", found: true }; + } + } + return { value, found: false }; + } + if (value === null || value === undefined || typeof value !== "object") { + return { value, found: false }; + } + if (isOpaqueBinary(value)) return { value, found: false }; + if (isDiscriminatedStreamError(value)) { + return { value: sanitizeErrorSubtreeValue(value), found: true }; + } + const declaresResponsesFailure = isResponsesFailureEvent(value); + const responsesFailure = forceResponsesFailure || declaresResponsesFailure; + if (seen.has(value)) return { value: "[circular]", found: false }; + seen.add(value); + + if (Array.isArray(value)) { + try { + let found = false; + const projected = value.map((entry) => { + const result = projectErrorSubtreesForLog(entry, seen, responsesFailure, false); + found ||= result.found; + return result.value; + }); + return { value: projected, found }; + } finally { + seen.delete(value); + } + } + + try { + let found = responsesFailure; + const projected: JsonRecord = {}; + for (const [key, entryValue] of Object.entries(value)) { + if (isErrorSubtreeKey(key) || (responsesFailure && isResponseFailureMessageKey(key))) { + projected[key] = sanitizeErrorSubtreeValue(entryValue); + found = true; + continue; + } + // Responses failures may attach diagnostics under neutral key names. Keep + // projecting through that envelope, while preserving partial model output + // as content rather than treating it as an error message. + const normalizedKey = key.replace(/[-_]/g, "").toLowerCase(); + const preservePartialOutput = + responsesFailure && + normalizedKey === "output" && + (protocolResponseObject || declaresResponsesFailure); + if (preservePartialOutput) { + projected[key] = projectResponsesFailureOutput( + entryValue, + (_field, stringValue) => sanitizeErrorMessage(stringValue) || "[REDACTED]" + ); + found = true; + continue; + } + const childIsProtocolResponse = + normalizedKey === "response" && + (declaresResponsesFailure || (forceResponsesFailure && !protocolResponseObject)); + const result = projectErrorSubtreesForLog( + entryValue, + seen, + responsesFailure, + childIsProtocolResponse + ); + projected[key] = result.value; + found ||= result.found; + } + return { value: projected, found }; + } catch { + return { value: "[REDACTED]", found: false }; + } finally { + seen.delete(value); + } +} + +const STREAM_ERROR_DISCRIMINATOR_KEYS = ["type", "event", "kind", "status"] as const; +const STREAM_ERROR_DISCRIMINATORS = new Set(["error", "warning"]); +const RESPONSES_FAILURE_DISCRIMINATORS = new Set(["response.failed"]); +const RESPONSE_FAILURE_MESSAGE_KEYS = new Set(["message", "detail", "details", "description"]); +const STREAM_ERROR_ENVELOPE_RE = + /["'](?:error|errors|warning|warnings|last_error|lastError|errorMessage|warningMessage)["']\s*:|["'](?:type|event|kind)["']\s*:\s*["'](?:error|warning|response\.(?:failed|completed))["']|["']status["']\s*:\s*["']failed["']/i; + +function isResponseFailureMessageKey(key: string): boolean { + return RESPONSE_FAILURE_MESSAGE_KEYS.has(key.replace(/[-_]/g, "").toLowerCase()); +} + +function isResponsesFailureEvent(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + try { + const record = value as JsonRecord; + const directFailure = STREAM_ERROR_DISCRIMINATOR_KEYS.some((key) => { + const discriminator = record[key]; + return ( + typeof discriminator === "string" && + RESPONSES_FAILURE_DISCRIMINATORS.has(discriminator.trim().toLowerCase()) + ); + }); + if (directFailure) return true; + + const status = record.status; + if (typeof status === "string" && status.trim().toLowerCase() === "failed") return true; + + const nestedResponse = record.response; + if (!nestedResponse || typeof nestedResponse !== "object" || Array.isArray(nestedResponse)) { + return false; + } + const nestedStatus = (nestedResponse as JsonRecord).status; + return typeof nestedStatus === "string" && nestedStatus.trim().toLowerCase() === "failed"; + } catch { + return true; + } +} + +function isDiscriminatedStreamError(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + try { + const record = value as JsonRecord; + return STREAM_ERROR_DISCRIMINATOR_KEYS.some((key) => { + const discriminator = record[key]; + return ( + typeof discriminator === "string" && + STREAM_ERROR_DISCRIMINATORS.has(discriminator.trim().toLowerCase()) + ); + }); + } catch { + return true; + } +} + +function sanitizeStreamErrorPayload( + rawPayload: string, + forceError: boolean, + forceResponsesFailure = false +): { found: boolean; value: string } { + try { + const parsed: unknown = JSON.parse(rawPayload); + if (forceError || isDiscriminatedStreamError(parsed)) { + const projected = sanitizeErrorSubtreeValue(parsed); + const serialized = JSON.stringify(projected); + return { + found: true, + value: typeof serialized === "string" ? serialized : "[REDACTED]", + }; + } + + const projected = projectErrorSubtreesForLog( + parsed, + new WeakSet(), + forceResponsesFailure + ); + if (!projected.found) return { found: false, value: rawPayload }; + return { found: true, value: JSON.stringify(projected.value) }; + } catch { + if (!forceError && !forceResponsesFailure && !STREAM_ERROR_ENVELOPE_RE.test(rawPayload)) { + return { found: false, value: rawPayload }; + } + return { + found: true, + value: sanitizeErrorMessage(rawPayload) || "[REDACTED]", + }; + } +} + +/** + * Sanitize error/warning records captured as fragmented SSE or NDJSON text. + * Prefixes are matched at the start of a line so unrelated `metadata:` fields + * cannot be mistaken for SSE `data:` frames. + */ +export function sanitizeErrorFramesFromLogChunks(chunks: string[]): string[] { + const combined = chunks.map((chunk) => chunk.replace(STREAM_CHUNK_TIMESTAMP_RE, "")).join(""); + let found = false; + let errorEventActive = false; + let responsesFailureEventActive = false; + const projectedLines = combined.split("\n").map((line) => { + if (line.trim().length === 0) { + errorEventActive = false; + responsesFailureEventActive = false; + return line; + } + + const eventMatch = line.match(/^\s*event:\s*([^\s]+)\s*$/i); + if (eventMatch) { + const eventName = eventMatch[1].toLowerCase(); + errorEventActive = STREAM_ERROR_DISCRIMINATORS.has(eventName); + responsesFailureEventActive = RESPONSES_FAILURE_DISCRIMINATORS.has(eventName); + return line; + } + + const dataMatch = line.match(/^(\s*data:)([ \t]?)(.*)$/); + if (dataMatch) { + const rawPayload = dataMatch[3].trim(); + if (!rawPayload || rawPayload === "[DONE]") return line; + const projected = sanitizeStreamErrorPayload( + rawPayload, + errorEventActive, + responsesFailureEventActive + ); + if (!projected.found) return line; + found = true; + return `${dataMatch[1]}${dataMatch[2]}${projected.value}`; + } + + if (errorEventActive || responsesFailureEventActive) { + found = true; + return sanitizeErrorMessage(line) || "[REDACTED]"; + } + + const trimmed = line.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return line; + const projected = sanitizeStreamErrorPayload(trimmed, false); + if (!projected.found) return line; + found = true; + return `${line.slice(0, line.length - line.trimStart().length)}${projected.value}`; + }); + + return found ? [projectedLines.join("\n")] : chunks; +} + /** * True for any binary/opaque byte view (Uint8Array, Buffer, DataView, other * typed arrays). `Array.isArray()` returns false for these, so callers that @@ -125,7 +422,7 @@ export function redactPayload(payload: unknown): unknown { const redacted: JsonRecord = {}; for (const [key, value] of Object.entries(payload)) { - if (SENSITIVE_KEYS.has(key)) { + if (isSensitivePayloadKey(key)) { redacted[key] = "[REDACTED]"; } else if (typeof value === "string" && value.startsWith("Bearer ")) { redacted[key] = "Bearer [REDACTED]"; @@ -162,7 +459,19 @@ export function sanitizePayloadPII(payload: unknown): unknown { export function protectPayloadForLog(payload: unknown): unknown { if (payload === null || payload === undefined) return null; const normalized = normalizePayloadForLog(payload); - const reasoningOmitted = omitEncryptedReasoningForLog(normalized); + const errorProjected = projectErrorSubtreesForLog(normalized).value; + const reasoningOmitted = omitEncryptedReasoningForLog(errorProjected); + const piiSanitized = sanitizePayloadPII(reasoningOmitted); + return redactPayload(piiSanitized); +} + +/** Project every string leaf because the payload is known to represent a failed response. */ +export function protectErrorPayloadForLog(payload: unknown): unknown { + if (payload === null || payload === undefined) return null; + const normalized = normalizePayloadForLog(payload); + if (isOpaqueBinary(normalized)) return describeOpaqueBinary(normalized); + const errorProjected = sanitizeErrorSubtreeValue(normalized); + const reasoningOmitted = omitEncryptedReasoningForLog(errorProjected); const piiSanitized = sanitizePayloadPII(reasoningOmitted); return redactPayload(piiSanitized); } diff --git a/src/lib/providers/validation/transport.ts b/src/lib/providers/validation/transport.ts index cbf6686aa9..c9472feb5c 100644 --- a/src/lib/providers/validation/transport.ts +++ b/src/lib/providers/validation/transport.ts @@ -1,6 +1,7 @@ // Outbound fetch wrappers for provider validation: proxy-fallback, SSRF-aware proxy targeting, and -// error→result mapping. Extracted from validation.ts (god-file decomposition). Behavior is -// byte-identical to the original inline defs. +// error→result mapping. Extracted from validation.ts (god-file decomposition) and kept as the +// common boundary for sanitizing validation failures. +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { SAFE_OUTBOUND_FETCH_PRESETS, SafeOutboundFetchError, @@ -11,6 +12,28 @@ import { isPrivateHost } from "@/shared/network/outboundUrlGuard"; import { getProviderValidationGuard } from "@/shared/network/outboundUrlGuardPolicy"; import { selectProxyForValidation } from "@omniroute/open-sse/services/proxyAutoSelector.ts"; +export type ProjectedProviderValidationResult = { + [K in keyof T]: K extends "error" | "warning" ? string | null : T[K]; +} & { + error?: string | null; + warning?: string | null; +}; + +export function projectProviderValidationResultForPublicResponse< + T extends { error?: unknown; warning?: unknown }, +>(result: T): ProjectedProviderValidationResult; +export function projectProviderValidationResultForPublicResponse( + result: Record +): Record { + const projected: Record = { ...result }; + for (const field of ["error", "warning"] as const) { + if (!Object.prototype.hasOwnProperty.call(result, field)) continue; + const value = result[field]; + projected[field] = value === null || value === undefined ? null : sanitizeErrorMessage(value); + } + return projected; +} + /** * Wrapped fetch call that auto-retries with a proxy when the direct connection * fails. This happens transparently so individual validators don't need to @@ -156,17 +179,30 @@ export function toWebCookieValidationErrorResult(provider: string, error: unknow } export function toValidationErrorResult(error: unknown) { - const message = error instanceof Error ? error.message : String(error || "Validation failed"); - const statusCode = getSafeOutboundFetchErrorStatus(error); + let rawMessage: unknown = error || "Validation failed"; + try { + if (error instanceof Error) rawMessage = error.message; + } catch { + rawMessage = "Validation failed"; + } + const message = sanitizeErrorMessage(rawMessage); + let statusCode: number | null = null; + let timeout = false; + let securityBlocked = false; + try { + statusCode = getSafeOutboundFetchErrorStatus(error); + timeout = error instanceof SafeOutboundFetchError && error.code === "TIMEOUT"; + securityBlocked = isSecurityBlockError(error); + } catch { + // Classification is advisory; hostile accessors must not escape the safe error boundary. + } return { valid: false, error: message || "Validation failed", unsupported: false as const, ...(statusCode ? { statusCode } : {}), - ...(error instanceof SafeOutboundFetchError && error.code === "TIMEOUT" - ? { timeout: true } - : {}), - ...(isSecurityBlockError(error) ? { securityBlocked: true } : {}), + ...(timeout ? { timeout: true } : {}), + ...(securityBlocked ? { securityBlocked: true } : {}), }; } diff --git a/src/lib/proxyLogger.ts b/src/lib/proxyLogger.ts index 8665e20c75..0bb278aa0f 100644 --- a/src/lib/proxyLogger.ts +++ b/src/lib/proxyLogger.ts @@ -7,6 +7,7 @@ * Pattern follows callLogs.js (T-15 decomposition). */ import { v4 as uuidv4 } from "uuid"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { getDbInstance, isCloud, isBuildPhase } from "./db/core"; import { ensureProxyLogsColumns } from "./db/schemaColumns"; @@ -99,7 +100,10 @@ function loadFromDb() { console.log(`[proxyLogger] Loaded ${proxyLogs.length} proxy logs from SQLite`); } } catch (err: any) { - console.warn("[proxyLogger] Failed to load from DB:", err.message); + console.warn( + "[proxyLogger] Failed to load from DB:", + sanitizeErrorMessage(err) || "Proxy log hydration failed" + ); } } @@ -113,10 +117,7 @@ loadFromDb(); /** Read at call time so tests can toggle it between imports. */ export function isProxyLogIncludeIps(): boolean { - return ( - process.env.PROXY_LOG_INCLUDE_IPS === "true" || - process.env.PROXY_LOG_INCLUDE_IPS === "1" - ); + return process.env.PROXY_LOG_INCLUDE_IPS === "true" || process.env.PROXY_LOG_INCLUDE_IPS === "1"; } /** @@ -152,6 +153,10 @@ export function formatProxyEgressConsoleLine(params: { // ──────────────── Log a proxy event ──────────────── export function logProxyEvent(entry: ProxyLogInput) { + const safeError = + entry.error === null || entry.error === undefined || entry.error === "" + ? null + : sanitizeErrorMessage(entry.error) || "Proxy request failed"; const log: ProxyLogEntry = { id: uuidv4(), timestamp: new Date().toISOString(), @@ -164,7 +169,7 @@ export function logProxyEvent(entry: ProxyLogInput) { clientIp: entry.clientIp ?? entry.publicIp ?? null, egressIp: entry.egressIp ?? null, latencyMs: entry.latencyMs || 0, - error: entry.error || null, + error: safeError, connectionId: entry.connectionId || null, comboId: entry.comboId || null, account: entry.account || null, @@ -236,15 +241,17 @@ export function flushProxyLogsSync() { // 1. If Redis driver is active, asynchronously publish batch to Redis Stream/Channel if (process.env.QUOTA_STORE_DRIVER === "redis" || process.env.QUOTA_STORE_REDIS_URL) { try { - import("@/lib/quota/redisQuotaStore").then(({ getRedisQuotaStore }) => { - const store = getRedisQuotaStore(process.env.QUOTA_STORE_REDIS_URL || ""); - const client = (store as any)?.client; - if (client && typeof client.publish === "function") { - for (const entry of batch) { - client.publish("omniroute:proxy_logs", JSON.stringify(entry)).catch(() => {}); + import("@/lib/quota/redisQuotaStore") + .then(({ getRedisQuotaStore }) => { + const store = getRedisQuotaStore(process.env.QUOTA_STORE_REDIS_URL || ""); + const client = (store as any)?.client; + if (client && typeof client.publish === "function") { + for (const entry of batch) { + client.publish("omniroute:proxy_logs", JSON.stringify(entry)).catch(() => {}); + } } - } - }).catch(() => {}); + }) + .catch(() => {}); } catch { /* ignore redis pub errors */ } @@ -289,7 +296,10 @@ export function flushProxyLogsSync() { transaction(batch); } catch (err: any) { - console.warn("[proxyLogger] Failed to write proxy log batch to disk:", err?.message || err); + console.warn( + "[proxyLogger] Failed to write proxy log batch to disk:", + sanitizeErrorMessage(err) || "Proxy log persistence failed" + ); } } @@ -351,7 +361,10 @@ export function clearProxyLogs() { const db = getDbInstance(); db.prepare("DELETE FROM proxy_logs").run(); } catch (err: any) { - console.warn("[proxyLogger] Failed to clear DB:", err.message); + console.warn( + "[proxyLogger] Failed to clear DB:", + sanitizeErrorMessage(err) || "Proxy log cleanup failed" + ); } } } diff --git a/src/lib/skills/executor.ts b/src/lib/skills/executor.ts index 692716d485..ac958f1a54 100644 --- a/src/lib/skills/executor.ts +++ b/src/lib/skills/executor.ts @@ -1,3 +1,8 @@ +import { + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "@omniroute/open-sse/utils/errorSanitization.ts"; + import { skillRegistry } from "./registry"; import { SkillExecution, SkillStatus, SkillHandler } from "./types"; import { builtinSkills } from "./builtins"; @@ -8,6 +13,169 @@ import { logger } from "../../../open-sse/utils/logger.ts"; const log = logger("SKILLS_EXECUTOR"); +function toSafeSkillErrorMessage(value: unknown): string { + try { + const raw = value instanceof Error ? value.message : value; + return sanitizeErrorMessage(raw) || "Skill execution failed"; + } catch { + return "Skill execution failed"; + } +} + +const SKILL_FAILURE_DISCRIMINATORS = new Set(["error", "failed", "failure"]); + +function isSkillErrorKey(key: string): boolean { + const normalizedKey = key.replace(/[-_]/g, "").toLowerCase(); + return ( + normalizedKey === "error" || + normalizedKey === "errors" || + normalizedKey === "warning" || + normalizedKey === "warnings" + ); +} + +function isFailureDiscriminator(value: unknown): boolean { + return typeof value === "string" && SKILL_FAILURE_DISCRIMINATORS.has(value.trim().toLowerCase()); +} + +function isSkillFailureOutput(output: Record): boolean { + try { + const status = output.status; + return ( + output.success === false || + (typeof status === "number" && Number.isFinite(status) && status >= 400) || + isFailureDiscriminator(status) || + isFailureDiscriminator(output.type) || + isFailureDiscriminator(output.event) || + isFailureDiscriminator(output.kind) + ); + } catch { + return true; + } +} + +type SensitiveSkillReferences = { + objects: WeakSet; + strings: Set; +}; + +function markSensitiveSkillReference(value: unknown, sensitive: SensitiveSkillReferences): void { + if (typeof value === "string") { + sensitive.strings.add(value); + return; + } + if (!value || typeof value !== "object" || sensitive.objects.has(value)) return; + + sensitive.objects.add(value); + try { + for (const entry of Object.values(value as Record)) { + markSensitiveSkillReference(entry, sensitive); + } + } catch { + // A revoked proxy or throwing getter is unsafe to expose at the boundary. + } +} + +function collectSensitiveSkillReferences( + value: unknown, + sensitive: SensitiveSkillReferences, + visited: WeakSet +): void { + if (!value || typeof value !== "object" || visited.has(value)) return; + visited.add(value); + + try { + for (const [key, entry] of Object.entries(value as Record)) { + if (isSkillErrorKey(key)) { + markSensitiveSkillReference(entry, sensitive); + } else { + collectSensitiveSkillReferences(entry, sensitive, visited); + } + } + } catch { + markSensitiveSkillReference(value, sensitive); + } +} + +type SkillProjectionContext = { + active: WeakSet; + projected: WeakMap; + sensitive: SensitiveSkillReferences; +}; + +function projectNestedSkillErrorSubtrees(value: unknown, context: SkillProjectionContext): unknown { + if (typeof value === "string") { + return context.sensitive.strings.has(value) ? sanitizeErrorMessage(value) : value; + } + if (!value || typeof value !== "object") return value; + if (context.active.has(value)) return "[circular]"; + if (context.projected.has(value)) return context.projected.get(value); + + if (context.sensitive.objects.has(value)) { + const safeValue = sanitizeUpstreamDetails(value); + context.projected.set(value, safeValue); + return safeValue; + } + + context.active.add(value); + if (Array.isArray(value)) { + const projected: unknown[] = []; + context.projected.set(value, projected); + for (const entry of value) projected.push(projectNestedSkillErrorSubtrees(entry, context)); + context.active.delete(value); + return projected; + } + + const projected: Record = {}; + context.projected.set(value, projected); + for (const [key, entry] of Object.entries(value as Record)) { + projected[key] = isSkillErrorKey(key) + ? sanitizeUpstreamDetails(entry) + : projectNestedSkillErrorSubtrees(entry, context); + } + context.active.delete(value); + return projected; +} + +function skillFailureMessage(output: Record): string { + try { + for (const candidate of [output.message, output.reason, output.statusText, output.error]) { + if (typeof candidate === "string" || candidate instanceof Error) { + return toSafeSkillErrorMessage(candidate); + } + } + } catch { + // Fall through to the stable public message. + } + return "Skill execution failed"; +} + +export function projectSkillOutputForBoundary( + output: Record +): Record { + try { + if (isSkillFailureOutput(output)) { + const projected = sanitizeUpstreamDetails(output); + return projected && typeof projected === "object" && !Array.isArray(projected) + ? (projected as Record) + : { success: false, error: "Skill execution failed" }; + } + + const sensitive: SensitiveSkillReferences = { + objects: new WeakSet(), + strings: new Set(), + }; + collectSensitiveSkillReferences(output, sensitive, new WeakSet()); + return projectNestedSkillErrorSubtrees(output, { + active: new WeakSet(), + projected: new WeakMap(), + sensitive, + }) as Record; + } catch { + return { success: false, error: "Skill execution failed" }; + } +} + class SkillExecutor { private static instance: SkillExecutor; private handlers: Map = new Map(); @@ -99,9 +267,14 @@ class SkillExecutor { const result = await this.executeWithTimeout( handler(input, { apiKeyId: context.apiKeyId, sessionId: context.sessionId || "" }) ); - output = result; + const resultIsFailure = isSkillFailureOutput(result); + output = projectSkillOutputForBoundary(result); + if (resultIsFailure) { + errorMessage = skillFailureMessage(result); + status = SkillStatus.ERROR; + } } catch (err) { - errorMessage = err instanceof Error ? err.message : String(err); + errorMessage = toSafeSkillErrorMessage(err); status = SkillStatus.ERROR; } @@ -131,7 +304,7 @@ class SkillExecutor { }; } catch (err) { const durationMs = Date.now() - startTime; - const errorMessage = err instanceof Error ? err.message : String(err); + const errorMessage = toSafeSkillErrorMessage(err); db.prepare( `UPDATE skill_executions SET status = ?, error_message = ?, duration_ms = ? WHERE id = ?` diff --git a/src/lib/skills/interception.ts b/src/lib/skills/interception.ts index 16b0146728..43c83c2d25 100644 --- a/src/lib/skills/interception.ts +++ b/src/lib/skills/interception.ts @@ -1,14 +1,29 @@ -import { skillExecutor } from "./executor"; +import { projectSkillOutputForBoundary, skillExecutor } from "./executor"; import { skillRegistry } from "./registry"; import { builtinSkills } from "./builtins"; import { memoryBuiltinHandlers, MEMORY_BUILTIN_TOOL_NAMES } from "./memoryBuiltins"; import { detectProvider, decodeSkillToolName } from "./injection"; import { OMNIROUTE_WEB_SEARCH_FALLBACK_TOOL_NAME } from "@omniroute/open-sse/services/webSearchFallback.ts"; import { OMNIROUTE_WEB_FETCH_FALLBACK_TOOL_NAME } from "@omniroute/open-sse/services/webFetchInterception.ts"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { logger } from "../../../open-sse/utils/logger.ts"; const log = logger("SKILLS_INTERCEPTION"); +function toSafeSkillErrorMessage(value: unknown): string { + try { + const raw = value instanceof Error ? value.message : value; + return sanitizeErrorMessage(raw) || "Skill execution failed"; + } catch { + return "Skill execution failed"; + } +} + +function projectSkillResultForPublicResponse(result: unknown): unknown { + if (!result || typeof result !== "object" || Array.isArray(result)) return result; + return projectSkillOutputForBoundary(result as Record); +} + interface ToolCall { id: string; name: string; @@ -130,7 +145,7 @@ export async function interceptToolCalls( return { id: call.id, - result, + result: projectSkillResultForPublicResponse(result), }; } @@ -151,11 +166,12 @@ export async function interceptToolCalls( sessionId: context.sessionId, }); - const result = + const result = projectSkillResultForPublicResponse( execution.output ?? - (execution.errorMessage - ? { error: execution.errorMessage } - : { error: "Skill execution returned no output" }); + (execution.errorMessage + ? { error: toSafeSkillErrorMessage(execution.errorMessage) } + : { error: "Skill execution returned no output" }) + ); log.info("skills.interception.execution_complete", { toolName: call.name, @@ -167,14 +183,15 @@ export async function interceptToolCalls( result, }; } catch (err) { + const safeError = toSafeSkillErrorMessage(err); log.error("skills.interception.execution_failed", { toolName: call.name, callId: call.id, - err: err instanceof Error ? err.message : String(err), + err: safeError, }); return { id: call.id, - result: { error: err instanceof Error ? err.message : String(err) }, + result: { error: safeError }, }; } }) diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index 43a5028b9d..5f0e3a03fc 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -8,6 +8,7 @@ import fs from "node:fs"; import path from "node:path"; import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { getDbInstance } from "../db/core"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; import { shouldPersistToDisk } from "./migrations"; @@ -21,7 +22,11 @@ import { getObservedReasoning, } from "./tokenAccounting"; import { isNoLog } from "../compliance/noLog"; -import { protectPayloadForLog, parseStoredPayload } from "../logPayloads"; +import { + parseStoredPayload, + protectErrorPayloadForLog, + protectPayloadForLog, +} from "../logPayloads"; import { pickDisplayValue } from "@/shared/utils/maskEmail"; import { CALL_LOGS_DIR, @@ -335,7 +340,10 @@ function readLegacyLogFromDisk(entry: { return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8")); } } catch (error) { - console.error("[callLogs] Failed to read legacy disk log:", (error as Error).message); + console.error( + "[callLogs] Failed to read legacy disk log:", + sanitizeErrorMessage(error) || "Legacy call log read failed" + ); } return null; @@ -447,10 +455,19 @@ async function saveCallLogOperation(entry: any): Promise { const noLogEnabled = Boolean(entry.noLog) || (apiKeyId ? isNoLog(apiKeyId) : false); const protectedRequestBody = noLogEnabled ? null : protectPayloadForLog(entry.requestBody); - const protectedResponseBody = noLogEnabled ? null : protectPayloadForLog(entry.responseBody); + const responseStatus = Number(entry.status); + const failedResponse = Number.isFinite(responseStatus) && responseStatus >= 400; + const protectedResponseBody = noLogEnabled + ? null + : failedResponse + ? protectErrorPayloadForLog(entry.responseBody) + : protectPayloadForLog(entry.responseBody); const protectedPipelinePayloads = noLogEnabled ? null - : protectPipelinePayloads(entry.pipelinePayloads ?? entry.pipeline ?? null); + : protectPipelinePayloads( + entry.pipelinePayloads ?? entry.pipeline ?? null, + failedResponse ? responseStatus : undefined + ); const protectedError = sanitizeErrorForLog(entry.error); const account = await resolveAccountName(entry.connectionId || null); @@ -582,7 +599,10 @@ async function saveCallLogOperation(entry: any): Promise { scheduleCallLogRotation(); } catch (error) { - console.error("[callLogs] Failed to save call log:", (error as Error).message); + console.error( + "[callLogs] Failed to save call log:", + sanitizeErrorMessage(error) || "Call log persistence failed" + ); } } diff --git a/src/lib/usage/callLogs/format.ts b/src/lib/usage/callLogs/format.ts index 40054c3a37..63068e73bc 100644 --- a/src/lib/usage/callLogs/format.ts +++ b/src/lib/usage/callLogs/format.ts @@ -1,7 +1,16 @@ import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { classifyProviderError } from "@omniroute/open-sse/services/errorClassifier.ts"; +import { + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "@omniroute/open-sse/utils/errorSanitization.ts"; import { sanitizePII } from "../../piiSanitizer"; -import { omitEncryptedReasoningFromLogChunks, protectPayloadForLog } from "../../logPayloads"; +import { + omitEncryptedReasoningFromLogChunks, + protectErrorPayloadForLog, + protectPayloadForLog, + sanitizeErrorFramesFromLogChunks, +} from "../../logPayloads"; import type { CallLogDetailState } from "../callLogArtifacts"; // #7879: re-export the canonical helper so existing consumers of this module // keep importing `toNumber` from here unchanged. @@ -44,15 +53,24 @@ export function normalizeDetailState(value: unknown): CallLogDetailState { export function sanitizeErrorForLog(error: unknown): unknown { if (error === null || error === undefined) return null; - if (typeof error === "string") return sanitizePII(error).text; - if (error instanceof Error) { - return { - message: sanitizePII(error.message).text, - stack: sanitizePII(error.stack || "").text || undefined, - name: error.name, - }; + if (typeof error === "string") { + return sanitizePII(sanitizeErrorMessage(error)).text; + } + try { + if (error instanceof Error) { + const message = sanitizePII(sanitizeErrorMessage(error.message)).text; + const stack = sanitizePII(sanitizeErrorMessage(error.stack || "")).text; + const name = sanitizeErrorMessage(error.name) || "Error"; + return { + message, + ...(stack ? { stack } : {}), + name, + }; + } + return protectPayloadForLog(sanitizeUpstreamDetails(error)); + } catch { + return "[REDACTED]"; } - return protectPayloadForLog(error); } export function toStoredErrorSummary(error: unknown): string | null { @@ -70,7 +88,10 @@ export function toStoredErrorSummary(error: unknown): string | null { } } -export function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | null { +export function protectPipelinePayloads( + payloads: unknown, + responseStatus?: unknown +): RequestPipelinePayloads | null { if (!payloads || typeof payloads !== "object") return null; const protectedPayloads: RequestPipelinePayloads = {}; @@ -84,7 +105,9 @@ export function protectPipelinePayloads(payloads: unknown): RequestPipelinePaylo .filter(([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0) .map(([stage, chunkValue]) => [ stage, - omitEncryptedReasoningFromLogChunks(chunkValue as string[]), + sanitizeErrorFramesFromLogChunks( + omitEncryptedReasoningFromLogChunks(chunkValue as string[]) + ), ]) ); if (Object.keys(compacted).length > 0) { @@ -95,6 +118,21 @@ export function protectPipelinePayloads(payloads: unknown): RequestPipelinePaylo continue; } + if (key === "providerResponse" || key === "clientResponse") { + const response = asRecord(value); + const status = Number(response.status ?? responseStatus); + if (Number.isFinite(status) && status >= 400 && status <= 599) { + const projectedResponse = + "body" in response + ? { ...response, body: protectErrorPayloadForLog(response.body) } + : protectErrorPayloadForLog(value); + protectedPayloads[key as "providerResponse" | "clientResponse"] = protectPayloadForLog( + projectedResponse + ) as RequestPipelinePayloads["providerResponse"]; + continue; + } + } + protectedPayloads[key as keyof RequestPipelinePayloads] = protectPayloadForLog(value) as never; } diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index 3d9b0dfa68..4a1a9f9216 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -9,6 +9,7 @@ import { getDbInstance } from "../db/core"; import { protectPayloadForLog } from "../logPayloads"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { resolveOrphanedUsageAccountIdentity, resolveUsageAccountIdentity, @@ -128,7 +129,7 @@ function normalizePendingMetadata(metadata?: PendingRequestMetadata): PendingReq normalized.status = Number.isFinite(status) ? status : null; } if (metadata.error !== undefined) { - normalized.error = toStringOrNull(metadata.error) || null; + normalized.error = sanitizeErrorMessage(toStringOrNull(metadata.error)) || null; } if (metadata.errorCode !== undefined) { normalized.errorCode = toStringOrNull(metadata.errorCode) || null; diff --git a/src/lib/usage/usageStats.ts b/src/lib/usage/usageStats.ts index 2bc459fef7..833eb6ef31 100644 --- a/src/lib/usage/usageStats.ts +++ b/src/lib/usage/usageStats.ts @@ -318,6 +318,10 @@ export async function getUsageStats() { } const pendingRequests = getPendingRequests(); + const publicPendingRequests = { + byModel: pendingRequests.byModel, + byAccount: pendingRequests.byAccount, + }; const stats: { totalRequests: number; @@ -329,7 +333,7 @@ export async function getUsageStats() { byAccount: Record; byApiKey: Record; last10Minutes: UsageBucket[]; - pending: ReturnType; + pending: Pick, "byModel" | "byAccount">; activeRequests: ActiveRequest[]; } = { totalRequests: 0, @@ -341,7 +345,7 @@ export async function getUsageStats() { byAccount: {}, byApiKey: {}, last10Minutes: [], - pending: pendingRequests, + pending: publicPendingRequests, activeRequests: [], }; diff --git a/src/shared/constants/pricing/inference-hosts.ts b/src/shared/constants/pricing/inference-hosts.ts index e09715b942..3551548bdd 100644 --- a/src/shared/constants/pricing/inference-hosts.ts +++ b/src/shared/constants/pricing/inference-hosts.ts @@ -340,26 +340,29 @@ export const DEFAULT_PRICING_INFERENCE = { cache_creation: 0, }, }, + // #11773: Developer-tier $/1M from cerebras.ai/pricing (2026-09-03). + // Signup is a one-time $5 credit, not a $0 token grant — keep paid rates + // so classifyTier cannot treat Cerebras as the free routing tier. cerebras: { - "gpt-oss-120b": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, - "gemma-4-31b": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, - "zai-glm-4.7": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, - "llama-3.3-70b": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, + "gpt-oss-120b": { input: 0.35, output: 0.75, cached: 0, reasoning: 0, cache_creation: 0 }, + "gemma-4-31b": { input: 0.4, output: 0.8, cached: 0, reasoning: 0, cache_creation: 0 }, + "zai-glm-4.7": { input: 2.25, output: 2.75, cached: 0, reasoning: 0, cache_creation: 0 }, + "llama-3.3-70b": { input: 0.85, output: 1.2, cached: 0, reasoning: 0, cache_creation: 0 }, "llama-4-scout-17b-16e-instruct": { - input: 0, - output: 0, + input: 0.2, + output: 0.2, cached: 0, reasoning: 0, cache_creation: 0, }, "qwen-3-235b-a22b-instruct-2507": { - input: 0, - output: 0, + input: 0.6, + output: 1.2, cached: 0, reasoning: 0, cache_creation: 0, }, - "qwen-3-32b": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, + "qwen-3-32b": { input: 0.4, output: 0.8, cached: 0, reasoning: 0, cache_creation: 0 }, }, nvidia: { "nvidia/gpt-oss-120b": { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }, diff --git a/src/shared/constants/providers/apikey/inference-hosts.ts b/src/shared/constants/providers/apikey/inference-hosts.ts index 84cd65ad41..730ccd64f3 100644 --- a/src/shared/constants/providers/apikey/inference-hosts.ts +++ b/src/shared/constants/providers/apikey/inference-hosts.ts @@ -86,7 +86,12 @@ export const APIKEY_PROVIDERS_INFERENCE = { textIcon: "CB", website: "https://inference.cerebras.ai", hasFree: true, - freeNote: "Free Trial: 1M tokens/day, 30K TPM, 5 RPM — no credit card.", + // #11773: Cerebras retired the no-card 1M tokens/day trial. Live + // cerebras.ai/pricing (2026-09-03) is a one-time $5 signup credit that + // requires a payment method and expires after 30 days — LongCat-shaped + // (hasFree stays true; not a recurring grant). + freeNote: + "One-time $5 signup credit (30-day validity); a payment method is required. Not a recurring free tier.", }, nvidia: { id: "nvidia", diff --git a/src/shared/utils/apiKeyPolicy.ts b/src/shared/utils/apiKeyPolicy.ts index 49cb628bb0..67d9b4b70e 100644 --- a/src/shared/utils/apiKeyPolicy.ts +++ b/src/shared/utils/apiKeyPolicy.ts @@ -254,8 +254,7 @@ async function isComboAllowedForKey( } function quotaPolicyResponse(message: string, code: string): Response { - const body = buildErrorBody(HTTP_STATUS.FORBIDDEN, message); - body.error.code = code; + const body = buildErrorBody(HTTP_STATUS.FORBIDDEN, message, undefined, { code }); return new Response(JSON.stringify(body), { status: HTTP_STATUS.FORBIDDEN, headers: { "Content-Type": "application/json" }, diff --git a/src/shared/utils/terminalStatus.ts b/src/shared/utils/terminalStatus.ts index 1b74768b9a..b2e46ed614 100644 --- a/src/shared/utils/terminalStatus.ts +++ b/src/shared/utils/terminalStatus.ts @@ -1,17 +1,33 @@ import { updateProviderConnection } from "@/lib/db/providers"; import { shouldIsolateProbeFailures } from "@/shared/utils/probeOrigin"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; -type Patch = { testStatus: string; isActive?: boolean; lastError?: string | null; errorCode?: string | null; lastErrorType?: string | null; lastErrorAt?: string | null }; -const TERMINAL = new Set(["banned","expired","deactivated","credits_exhausted"]); +type Patch = { + testStatus: string; + isActive?: boolean; + lastError?: string | null; + errorCode?: string | null; + lastErrorType?: string | null; + lastErrorAt?: string | null; +}; +const TERMINAL = new Set(["banned", "expired", "deactivated", "credits_exhausted"]); -export async function writeTerminalStatus(connectionId: string, patch: Patch, origin: "probe" | "production"): Promise { +export async function writeTerminalStatus( + connectionId: string, + patch: Patch, + origin: "probe" | "production" +): Promise { const isTerminal = TERMINAL.has(patch.testStatus.toLowerCase()); + const persistedLastError = + patch.lastError == null + ? null + : sanitizeErrorMessage(patch.lastError) || "Provider request failed"; // Double gate: AsyncLocalStorage probe + explicit origin "probe" — fail-safe ON const probeIsolated = await shouldIsolateProbeFailures(); if ((origin === "probe" || probeIsolated) && isTerminal) { // record-only: never remove from pool await updateProviderConnection(connectionId, { - lastError: patch.lastError ?? null, + lastError: persistedLastError, lastErrorAt: new Date().toISOString(), lastErrorType: patch.lastErrorType ?? null, errorCode: patch.errorCode ?? null, @@ -21,7 +37,7 @@ export async function writeTerminalStatus(connectionId: string, patch: Patch, or await updateProviderConnection(connectionId, { isActive: patch.isActive ?? (isTerminal ? false : undefined), testStatus: patch.testStatus, - lastError: patch.lastError ?? null, + lastError: persistedLastError, lastErrorAt: new Date().toISOString(), lastErrorType: patch.lastErrorType ?? null, errorCode: patch.errorCode ?? null, diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index b121876376..37029593af 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -73,6 +73,7 @@ import { } from "@omniroute/open-sse/services/accountFallback.ts"; import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts"; import { COOLDOWN_MS, RateLimitReason } from "@omniroute/open-sse/config/constants.ts"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/errorSanitization.ts"; import { honorsRuleLockScope, isEgressBucketedLockScope, @@ -2717,14 +2718,13 @@ export async function markAccountUnavailable( // the opt-in setting probeCanDisable restores the historical behavior. if (await shouldIsolateProbeFailures()) { await updateProviderConnection(connectionId, { - // lastError kept RAW (full text) — maximal probe visibility; the - // divergence vs the normal path's slice(0,100) is intentional. + // Persist safe wording only after classification has consumed the raw provider text. // backoffLevel is deliberately NOT written: a positive backoff // triggers the selection-time auto-decay (resetConnectionBackoff, // auth.ts getProviderCredentials) which wipes lastError back to // NULL on the next attempt — silently destroying the probe record. // The backoff is also routing state a probe must not touch (#9817). - lastError: errorText, + lastError: sanitizeErrorMessage(errorText) || "Provider request failed", lastErrorType: fallbackResult.reason || null, errorCode: status, lastErrorAt: new Date().toISOString(), @@ -3140,8 +3140,8 @@ export async function markAccountUnavailable( ); return { shouldFallback: true, cooldownMs: lockout.cooldownMs }; } - - const errorMsg = describeUpstreamFailure(errorText); + const errorMsg = + sanitizeErrorMessage(describeUpstreamFailure(errorText)) || "Provider request failed"; // T09: Codex per-scope lockout (do not block the whole account globally). if ( diff --git a/stryker.conf.json b/stryker.conf.json index 96f421f624..27e2825662 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -244,6 +244,7 @@ "tests/unit/embeddings-auth.test.ts", "tests/unit/error-classification.test.ts", "tests/unit/error-message-sanitization.test.ts", + "tests/unit/error-sanitizer-sk-key-qv45.test.ts", "tests/unit/error-sensitive-redaction.test.ts", "tests/unit/execute-chat-resource-pressure-breaker.test.ts", "tests/unit/executor-antigravity.test.ts", diff --git a/tests/fixtures/adapta-web-nonstream-error-boundary.fixture.ts b/tests/fixtures/adapta-web-nonstream-error-boundary.fixture.ts new file mode 100644 index 0000000000..2e58eb87ea --- /dev/null +++ b/tests/fixtures/adapta-web-nonstream-error-boundary.fixture.ts @@ -0,0 +1,129 @@ +// This suite owns process-wide DATA_DIR, plugin, fetch, and DB state. It must run only inside +// the subprocess launched by tests/unit/adapta-web-nonstream-error-boundary.test.ts. +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, afterEach, describe, it } from "node:test"; + +const TEST_DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-adapta-nonstream-error-")); +const TEST_PLUGINS_DIR = join(TEST_DATA_DIR, "plugins"); +mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; + +const originalFetch = globalThis.fetch; +const { AdaptaWebExecutor } = await import("../../open-sse/executors/adapta-web.ts"); + +interface ErrorEnvelope { + error?: { + message?: string; + type?: string; + code?: string; + }; + choices?: unknown[]; +} + +interface CompletionEnvelope { + choices?: Array<{ + message?: { + content?: string; + }; + finish_reason?: string; + }>; +} + +function installAdaptaFetch(upstreamBody: string): void { + const mockFetch = async (input: string | URL | Request): Promise => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + + if (url === "https://clerk.agent.adapta.one/v1/client") { + return Response.json({ response: { sessions: [{ id: "sess-fixture", status: "active" }] } }); + } + + if (url === "https://clerk.agent.adapta.one/v1/client/sessions/sess-fixture/tokens") { + return Response.json({ jwt: "eyJ.fixture.signature" }); + } + + if (url === "https://agent.adapta.one/api/chat/stream/v1") { + return new Response(upstreamBody, { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }); + } + + throw new Error(`Unexpected test fetch URL: ${url}`); + }; + + globalThis.fetch = mockFetch as typeof fetch; +} + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +after(async () => { + const { resetDbInstance } = await import("../../src/lib/db/core.ts"); + resetDbInstance(); + rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +describe("Adapta Web non-stream error boundary", () => { + it("returns a sanitized 502 when an HTTP 200 SSE body contains type:error", async () => { + installAdaptaFetch( + `data: ${JSON.stringify({ + type: "error", + errorText: + "SQLSTATE 42P01 private detail at /srv/omniroute/open-sse/executors/adapta-web.ts:481:9 Authorization: Bearer secret-token\n at secret (/srv/omniroute/internal.ts:1:1)", + })}\n\n` + ); + + const executor = new AdaptaWebExecutor(); + const result = await executor.execute({ + model: "adapta-one", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: false, + credentials: { apiKey: "fixture-client-error" }, + signal: null, + }); + + assert.equal(result.response.status, 502); + const payload = (await result.response.json()) as ErrorEnvelope; + assert.equal(payload.error?.type, "server_error"); + assert.equal(payload.error?.code, "bad_gateway"); + assert.equal(payload.error?.message, "Adapta upstream error"); + assert.ok(!payload.error?.message?.includes("SQLSTATE")); + assert.ok(!payload.error?.message?.includes("private detail")); + assert.ok(!payload.error?.message?.includes("/srv/omniroute")); + assert.ok(!payload.error?.message?.includes("secret-token")); + assert.ok(!payload.error?.message?.includes("\n")); + assert.equal(payload.choices, undefined); + }); + + it("preserves a normal non-stream completion assembled from text-delta events", async () => { + installAdaptaFetch( + [ + `data: ${JSON.stringify({ type: "text-delta", id: "quick-response", delta: "Loading" })}`, + `data: ${JSON.stringify({ type: "text-delta", id: "answer", delta: "Hello" })}`, + `data: ${JSON.stringify({ type: "text-delta", id: "answer", delta: " world" })}`, + `data: ${JSON.stringify({ type: "done" })}`, + "", + ].join("\n\n") + ); + + const executor = new AdaptaWebExecutor(); + const result = await executor.execute({ + model: "adapta-one", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: false, + credentials: { apiKey: "fixture-client-success" }, + signal: null, + }); + + assert.equal(result.response.status, 200); + const payload = (await result.response.json()) as CompletionEnvelope; + assert.equal(payload.choices?.[0]?.message?.content, "Hello world"); + assert.equal(payload.choices?.[0]?.finish_reason, "stop"); + }); +}); diff --git a/tests/fixtures/adapta-web-stream-error-boundary.fixture.ts b/tests/fixtures/adapta-web-stream-error-boundary.fixture.ts new file mode 100644 index 0000000000..02c7e6f07e --- /dev/null +++ b/tests/fixtures/adapta-web-stream-error-boundary.fixture.ts @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +assert.ok(process.env.DATA_DIR, "the subprocess fixture requires an isolated DATA_DIR"); +assert.ok( + process.env.OMNIROUTE_PLUGINS_DIR, + "the subprocess fixture requires an isolated OMNIROUTE_PLUGINS_DIR" +); +assert.equal(process.env.HOME, undefined, "the subprocess must not inherit HOME"); +assert.equal(process.env.CODEX_HOME, undefined, "the subprocess must not inherit CODEX_HOME"); + +const { AdaptaWebExecutor } = await import("../../open-sse/executors/adapta-web.ts"); + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +test("terminates an upstream error event without exposing its text in the public SSE", async () => { + const hostileError = + "SQLSTATE 42P01 at /srv/omniroute/private.ts:91 — Authorization: Bearer secret-token"; + const requestedUrls: string[] = []; + const logMessages: string[] = []; + + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + requestedUrls.push(url); + + if (url.endsWith("/v1/client")) { + return Response.json({ + response: { sessions: [{ id: "session-stream-error", status: "active" }] }, + }); + } + + if (url.includes("/tokens")) { + return Response.json({ jwt: "eyJ.test-session.jwt" }); + } + + return new Response(`data: ${JSON.stringify({ type: "error", errorText: hostileError })}\n\n`, { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }); + }) as typeof fetch; + + const executor = new AdaptaWebExecutor(); + const result = await executor.execute({ + model: "adapta-one", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "__client=unique-stream-error-cookie" }, + signal: null, + log: { + info: (_tag, message) => logMessages.push(message), + warn: (_tag, message) => logMessages.push(message), + }, + }); + + assert.equal(result.response.status, 200); + assert.equal(result.response.headers.get("content-type"), "text/event-stream"); + + const publicSse = await result.response.text(); + assert.equal(requestedUrls.length, 3); + assert.match(publicSse, /"content":"\\n\\n\[Erro: Adapta upstream error\]"/); + assert.match(publicSse, /"finish_reason":"stop"/); + assert.match(publicSse, /data: \[DONE\]/); + assert.doesNotMatch(publicSse, /SQLSTATE|\/srv\/omniroute|secret-token/); + assert.doesNotMatch(logMessages.join("\n"), /SQLSTATE|\/srv\/omniroute|secret-token/); +}); diff --git a/tests/fixtures/codex-response-failed-boundary.fixture.ts b/tests/fixtures/codex-response-failed-boundary.fixture.ts new file mode 100644 index 0000000000..eba763b40a --- /dev/null +++ b/tests/fixtures/codex-response-failed-boundary.fixture.ts @@ -0,0 +1,376 @@ +// This suite intentionally owns process-wide DATA_DIR, plugin, and DB state. It must run only +// inside the subprocess launched by tests/unit/codex-response-failed-boundary.test.ts. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { CodexWreqWebSocket } from "../../open-sse/executors/codex/appServerClient.ts"; +import type { AdapterEvent } from "../../open-sse/vendor/codex-chatgpt-web/types.ts"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-codex-boundary-data-")); +const TEST_PLUGINS_DIR = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-codex-boundary-plugins-") +); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; +process.env.APP_LOG_TO_FILE = "false"; + +const { CodexExecutor, __setCodexWebSocketTransportForTesting, encodeResponseSseEvent } = + await import("../../open-sse/executors/codex.ts"); +const { CodexAppServerExecutor } = await import("../../open-sse/executors/codex-app-server.ts"); +const { bridgeToResponsesSSE, buildResponseJSON } = + await import("../../open-sse/vendor/codex-chatgpt-web/bridge.ts"); +const { resetDbInstance } = await import("../../src/lib/db/core.ts"); + +const PUBLIC_MESSAGE = "Codex provider request failed"; +const HOSTILE_MESSAGE = + "token=codex-secret-value at /srv/omniroute/private/config.json\nforged-log: admin=true"; + +type FailedPayload = { + type: "response.failed"; + response: { + error: { + code: string | null; + message: string; + status_code?: number; + type?: string; + }; + }; +}; + +function responseFailedPayload(sse: string): FailedPayload { + for (const line of sse.split("\n")) { + if (!line.startsWith("data: ") || line === "data: [DONE]") continue; + const parsed = JSON.parse(line.slice("data: ".length)) as Record; + if (parsed.type === "response.failed") return parsed as FailedPayload; + } + assert.fail(`response.failed frame missing from: ${sse}`); +} + +function assertPublicFailure( + payload: FailedPayload, + expected: { code: string; type: string; statusCode?: number } +): void { + assert.equal(payload.response.error.message, PUBLIC_MESSAGE); + assert.equal(payload.response.error.code, expected.code); + assert.equal(payload.response.error.type, expected.type); + if (expected.statusCode !== undefined) { + assert.equal(payload.response.error.status_code, expected.statusCode); + } + assert.ok(!JSON.stringify(payload).includes(HOSTILE_MESSAGE)); + assert.ok(!JSON.stringify(payload).includes("codex-secret-value")); + assert.ok(!JSON.stringify(payload).includes("/srv/omniroute/private")); +} + +async function executeCodexWebSocketFailure( + websocket: Parameters[0] +): Promise { + __setCodexWebSocketTransportForTesting(websocket); + try { + const result = await new CodexExecutor().execute({ + model: "gpt-5.5", + body: { model: "gpt-5.5", input: "hello" }, + stream: true, + credentials: { + accessToken: "test-token", + providerSpecificData: { codexTransport: "websocket" }, + }, + }); + return await result.response.text(); + } finally { + __setCodexWebSocketTransportForTesting(undefined); + } +} + +async function executeAppServerFailure(stream: boolean): Promise { + const socket: CodexWreqWebSocket = { + send(data: string) { + const frame = JSON.parse(data) as Record; + if (frame.id == null || typeof frame.method !== "string") return; + queueMicrotask(() => { + if (frame.method === "thread/start") { + socket.onmessage?.({ + data: JSON.stringify({ + jsonrpc: "2.0", + id: frame.id, + result: { thread: { id: "thread-public-boundary" } }, + }), + }); + return; + } + if (frame.method === "turn/start") { + socket.onmessage?.({ + data: JSON.stringify({ + jsonrpc: "2.0", + id: frame.id, + result: { turn: { id: "turn-public-boundary", status: "inProgress" } }, + }), + }); + setTimeout(() => { + socket.onmessage?.({ + data: JSON.stringify({ + jsonrpc: "2.0", + method: "error", + params: { error: { message: HOSTILE_MESSAGE } }, + }), + }); + }, 0); + return; + } + socket.onmessage?.({ + data: JSON.stringify({ jsonrpc: "2.0", id: frame.id, result: {} }), + }); + }); + }, + close() {}, + onmessage: null, + onerror: null, + onclose: null, + }; + const executor = new CodexAppServerExecutor({ websocketFn: async () => socket }); + const result = await executor.execute({ + model: "gpt-5.5", + body: { input: "hello" }, + stream, + credentials: { + providerSpecificData: { + codexTransport: "app-server", + codexAppServerUrl: "ws://codex-app-server.test:1456", + codexAppServerToken: "test-app-server-token", + }, + }, + }); + return result.response; +} + +test.after(() => { + __setCodexWebSocketTransportForTesting(undefined); + resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.rmSync(TEST_PLUGINS_DIR, { recursive: true, force: true }); +}); + +test("Codex same-format error event emits only a fixed public failure contract", () => { + const result = encodeResponseSseEvent( + JSON.stringify({ + type: "error", + status_code: 502, + error: { + code: "secret_backend_code_9182", + type: "secret_backend_type_7731", + message: HOSTILE_MESSAGE, + }, + }) + ); + + assert.equal(result.terminal, true); + assertPublicFailure(responseFailedPayload(result.sse), { + code: "upstream_server_error", + type: "server_error", + statusCode: 502, + }); +}); + +test("Codex same-format quota classification survives while its raw message does not", () => { + const result = encodeResponseSseEvent( + JSON.stringify({ + type: "response.failed", + response: { + status: "failed", + error: { code: "usage_limit_reached", message: HOSTILE_MESSAGE }, + }, + }) + ); + + assertPublicFailure(responseFailedPayload(result.sse), { + code: "usage_limit_reached", + type: "rate_limit_error", + statusCode: 429, + }); +}); + +test("Codex same-format failures reject contradictory allowlisted status, code and type", () => { + const wrongStatus = responseFailedPayload( + encodeResponseSseEvent( + JSON.stringify({ + type: "response.failed", + status_code: 502, + response: { + status: "failed", + error: { + code: "invalid_api_key", + type: "rate_limit_error", + message: HOSTILE_MESSAGE, + }, + }, + }) + ).sse + ); + assertPublicFailure(wrongStatus, { + code: "upstream_server_error", + type: "server_error", + statusCode: 502, + }); + + const wrongType = responseFailedPayload( + encodeResponseSseEvent( + JSON.stringify({ + type: "response.failed", + status_code: 401, + response: { + status: "failed", + error: { + code: "invalid_api_key", + type: "rate_limit_error", + message: HOSTILE_MESSAGE, + }, + }, + }) + ).sse + ); + assertPublicFailure(wrongType, { + code: "invalid_api_key", + type: "authentication_error", + statusCode: 401, + }); +}); + +test("Codex WebSocket in-flight error event cannot expose transport details", async () => { + const socket = { + send() { + queueMicrotask(() => socket.onerror?.({ message: HOSTILE_MESSAGE })); + }, + close() {}, + onmessage: null as ((event: { data: unknown }) => void) | null, + onerror: null as ((event: { message?: string }) => void) | null, + onclose: null as (() => void) | null, + }; + const sse = await executeCodexWebSocketFailure(async () => socket); + + assertPublicFailure(responseFailedPayload(sse), { + code: "upstream_websocket_error", + type: "provider_error", + }); +}); + +test("Codex WebSocket connection failure cannot expose exception details", async () => { + const sse = await executeCodexWebSocketFailure(async () => { + throw new Error(HOSTILE_MESSAGE); + }); + + assertPublicFailure(responseFailedPayload(sse), { + code: "upstream_websocket_connect_failed", + type: "provider_error", + }); +}); + +test("Codex App Server streaming failure is projected before the HTTP 200 SSE boundary", async () => { + const response = await executeAppServerFailure(true); + assert.equal(response.status, 200); + + assertPublicFailure(responseFailedPayload(await response.text()), { + code: "codex_app_server_turn_failed", + type: "provider_error", + }); +}); + +test("Codex App Server non-streaming failure is projected before the HTTP 200 JSON boundary", async () => { + const response = await executeAppServerFailure(false); + assert.equal(response.status, 200); + const body = (await response.json()) as FailedPayload["response"] & { status: string }; + + assert.equal(body.status, "failed"); + assertPublicFailure( + { type: "response.failed", response: body }, + { + code: "codex_app_server_turn_failed", + type: "provider_error", + } + ); +}); + +test("ChatGPT Web Playwright adapter failures keep safe routing metadata without raw text", async () => { + async function* browserEvents(): AsyncGenerator { + yield { + type: "error", + message: HOSTILE_MESSAGE, + status: 502, + errorType: "server_error", + code: "chatgpt_submission_ambiguous", + retryable: false, + }; + } + + const sse = await new Response(bridgeToResponsesSSE(browserEvents(), "gpt-5.5")).text(); + assertPublicFailure(responseFailedPayload(sse), { + code: "chatgpt_submission_ambiguous", + type: "server_error", + }); +}); + +test("Codex bridge projects message-only adapter failures before SSE serialization", async () => { + async function* messageOnlyEvents(): AsyncGenerator { + yield { type: "error", message: HOSTILE_MESSAGE }; + } + + const sse = await new Response(bridgeToResponsesSSE(messageOnlyEvents(), "gpt-5.5")).text(); + assertPublicFailure(responseFailedPayload(sse), { + code: "upstream_server_error", + type: "server_error", + }); +}); + +test("Codex batch bridge projects message-only adapter failures before JSON serialization", () => { + const body = buildResponseJSON( + [{ type: "error", message: HOSTILE_MESSAGE }], + "gpt-5.5" + ) as FailedPayload["response"] & { status: string }; + + assert.equal(body.status, "failed"); + assertPublicFailure( + { type: "response.failed", response: body }, + { + code: "upstream_server_error", + type: "server_error", + } + ); +}); + +test("Codex bridge exceptions cannot serialize raw exception messages", async () => { + async function* throwingEvents(): AsyncGenerator { + throw new Error(HOSTILE_MESSAGE); + } + + const sse = await new Response(bridgeToResponsesSSE(throwingEvents(), "gpt-5.5")).text(); + assertPublicFailure(responseFailedPayload(sse), { + code: "upstream_server_error", + type: "server_error", + }); +}); + +test("Codex batch bridge applies the same public failure projector", () => { + const body = buildResponseJSON( + [ + { + type: "error", + message: HOSTILE_MESSAGE, + status: 502, + errorType: "server_error", + code: "chatgpt_submitted_turn_failed", + retryable: false, + }, + ], + "gpt-5.5" + ) as FailedPayload["response"] & { status: string }; + + assert.equal(body.status, "failed"); + assertPublicFailure( + { type: "response.failed", response: body }, + { + code: "chatgpt_submitted_turn_failed", + type: "server_error", + } + ); +}); diff --git a/tests/fixtures/dashboard-request-failed-redaction-probe.ts b/tests/fixtures/dashboard-request-failed-redaction-probe.ts new file mode 100644 index 0000000000..bf740c2220 --- /dev/null +++ b/tests/fixtures/dashboard-request-failed-redaction-probe.ts @@ -0,0 +1,125 @@ +import assert from "node:assert/strict"; + +import type { RequestFailedPayload } from "../../src/lib/events/types.ts"; + +const RESULT_PREFIX = "DASHBOARD_FAILURE_PROBE_RESULT="; + +async function main(): Promise { + assert.ok(process.env.DATA_DIR, "probe requires an isolated DATA_DIR"); + assert.ok(process.env.OMNIROUTE_PLUGINS_DIR, "probe requires an isolated plugins directory"); + assert.ok(process.env.API_KEY_SECRET, "probe requires a synthetic API_KEY_SECRET"); + + const { persistAttemptLogs } = await import("../../open-sse/handlers/chatCore/attemptLogging.ts"); + const eventBus = await import("../../src/lib/events/eventBus.ts"); + const dbCore = await import("../../src/lib/db/core.ts"); + const callLogs = await import("../../src/lib/usage/callLogs.ts"); + + let unsubscribe: (() => void) | undefined; + try { + globalThis.__omnirouteEventBus = undefined; + const hostileError = new Error( + "Provider failed in /srv/omniroute/src/private/provider.ts:42:7 with " + + "api_key='sk-live-dashboard-secret'" + ); + hostileError.stack = + `${hostileError.name}: ${hostileError.message}\n` + + " at dispatch (/srv/omniroute/src/private/transport.ts:91:3)"; + const rawDiagnostic = hostileError.stack; + const traceId = "trace-dashboard-redaction"; + const callLogId = "call-log-dashboard-redaction"; + + const deliveredPromise = new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + unsubscribe?.(); + reject(new Error("timed out waiting for persistAttemptLogs request.failed event")); + }, 10_000); + unsubscribe = eventBus.on("request.failed", (payload) => { + if (payload.id !== traceId) return; + clearTimeout(timeout); + unsubscribe?.(); + unsubscribe = undefined; + resolve(payload); + }); + }); + + persistAttemptLogs( + { + status: 502, + tokens: {}, + responseBody: null, + error: rawDiagnostic, + }, + { + traceId, + provider: "private-provider", + connectionId: null, + model: "private-model", + skillRequestId: "skill-dashboard-redaction", + detailedLoggingEnabled: false, + reqLogger: null, + pendingRequestId: callLogId, + clientRawRequest: { endpoint: "/v1/chat/completions" }, + requestedModel: "private-model", + credentials: null, + startTime: Date.now() - 37, + body: { model: "private-model", messages: [] }, + sourceFormat: "openai", + targetFormat: "openai", + comboName: null, + comboStepId: null, + comboExecutionKey: null, + tokensCompressed: null, + apiKeyInfo: null, + noLogEnabled: false, + correlationId: null, + modelPinned: false, + sessionTag: null, + } + ); + + const delivered = await deliveredPromise; + assert.equal(delivered.id, traceId); + assert.equal(delivered.statusCode, 502); + assert.equal(delivered.model, "private-model"); + assert.equal(delivered.provider, "private-provider"); + assert.ok(delivered.latencyMs >= 0); + assert.equal(delivered.error, "Error: Provider failed in with api_key='[REDACTED]'"); + assert.doesNotMatch(delivered.error, /sk-live-dashboard-secret|\/srv\/omniroute|\n/); + + const replayed = eventBus + .getEventHistory(undefined, 10) + .find( + (entry) => + entry.event === "request.failed" && + (entry.payload as RequestFailedPayload | undefined)?.id === traceId + ); + assert.ok(replayed, "late subscribers must have the safe request.failed history entry"); + assert.deepEqual(replayed.payload, delivered); + + const writerDrained = await callLogs.waitForCallLogSaves(10_000); + assert.equal(writerDrained, true, "call-log write must drain"); + const persisted = await callLogs.getCallLogById(callLogId); + assert.ok(persisted, "failed attempt must still be available to internal diagnostics"); + assert.equal(persisted.error, rawDiagnostic); + + console.log( + RESULT_PREFIX + + JSON.stringify({ + delivered, + replayMatches: JSON.stringify(replayed.payload) === JSON.stringify(delivered), + internalRawPreserved: persisted.error === rawDiagnostic, + writerDrained, + }) + ); + } finally { + unsubscribe?.(); + try { + await callLogs.waitForCallLogSaves(10_000); + await callLogs.closeCallLogSaves(10_000); + } finally { + dbCore.resetDbInstance(); + } + } +} + +await main(); diff --git a/tests/fixtures/grok-web-stream-error-boundary-child.ts b/tests/fixtures/grok-web-stream-error-boundary-child.ts new file mode 100644 index 0000000000..b3ad226d52 --- /dev/null +++ b/tests/fixtures/grok-web-stream-error-boundary-child.ts @@ -0,0 +1,517 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +// This file is executed only by the process-isolated unit-test wrapper. State +// mutations and repository imports must remain here, never in the parent test. +const originalDataDir = process.env.DATA_DIR; +const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; +const originalFetch = globalThis.fetch; +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-grok-web-stream-error-")); + +process.env.DATA_DIR = path.join(testRoot, "data"); +process.env.OMNIROUTE_PLUGINS_DIR = path.join(testRoot, "plugins"); +fs.mkdirSync(process.env.DATA_DIR, { recursive: true }); +fs.mkdirSync(process.env.OMNIROUTE_PLUGINS_DIR, { recursive: true }); +globalThis.fetch = async () => { + throw new Error("Unexpected network request in Grok stream error boundary test"); +}; + +const [ + { GrokWebExecutor }, + { __setTlsFetchOverrideForTesting }, + dbCore, + settingsDb, + callLogs, + artifactWriter, + { handleChatCore }, + usageHistory, + accountSemaphore, + requestDedup, + accountFallback, + loggerResource, +] = await Promise.all([ + import("../../open-sse/executors/grok-web.ts"), + import("../../open-sse/services/grokTlsClient.ts"), + import("../../src/lib/db/core.ts"), + import("../../src/lib/db/settings.ts"), + import("../../src/lib/usage/callLogs.ts"), + import("../../src/lib/usage/callLogArtifactWriter.ts"), + import("../../open-sse/handlers/chatCore.ts"), + import("../../src/lib/usage/usageHistory.ts"), + import("../../open-sse/services/accountSemaphore.ts"), + import("../../open-sse/services/requestDedup.ts"), + import("../../open-sse/services/accountFallback.ts"), + import("../../src/shared/utils/loggerResource.ts"), +]); + +function grokEventStream(events: unknown[]): ReadableStream { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(`${events.map((event) => JSON.stringify(event)).join("\n")}\n`) + ); + controller.close(); + }, + }); +} + +function stalledGrokEventStream( + events: unknown[], + onCancel: () => void +): ReadableStream { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(`${events.map((event) => JSON.stringify(event)).join("\n")}\n`) + ); + }, + pull() { + return new Promise(() => {}); + }, + cancel() { + onCancel(); + return new Promise(() => {}); + }, + }); +} + +type TestExecutorLog = { + debug?: (tag: string, message: string) => void; + info?: (tag: string, message: string) => void; + warn?: (tag: string, message: string) => void; + error?: (tag: string, message: string) => void; +}; + +async function executeStreamingBody( + upstreamBody: ReadableStream, + requestBody: Record = { + messages: [{ role: "user", content: "hello" }], + stream: true, + }, + options: { log?: TestExecutorLog | null; signal?: AbortSignal | null } = {} +): Promise { + __setTlsFetchOverrideForTesting(async () => ({ + status: 200, + headers: new Headers({ "Content-Type": "application/x-ndjson" }), + text: null, + body: upstreamBody, + })); + + const result = await new GrokWebExecutor().execute({ + model: "grok-4.1-fast", + body: requestBody, + stream: true, + credentials: { apiKey: "sso=test-only-cookie" }, + signal: options.signal ?? AbortSignal.timeout(10_000), + log: options.log ?? null, + }); + return result.response; +} + +function executeStreaming(events: unknown[]): Promise { + return executeStreamingBody(grokEventStream(events)); +} + +function parseSseData(text: string): unknown[] { + return text + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice("data: ".length)) as unknown); +} + +async function readUntilFailure(response: Response): Promise<{ text: string; error: unknown }> { + assert.ok(response.body, "expected a streaming response body"); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return { text, error: null }; + text += decoder.decode(value, { stream: true }); + } + } catch (error) { + text += decoder.decode(); + return { text, error }; + } +} + +async function waitFor(read: () => Promise, timeoutMs = 3_000): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const value = await read(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return null; +} + +async function settlesWithin(promise: Promise, timeoutMs = 500): Promise { + let timeout: ReturnType | undefined; + const settled = await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + if (timeout) clearTimeout(timeout); + return settled; +} + +test.afterEach(() => { + __setTlsFetchOverrideForTesting(null); + usageHistory.clearPendingRequests(); + accountSemaphore.resetAll(); + requestDedup.clearInflight(); + accountFallback.clearModelLock(); +}); + +test.after(async () => { + __setTlsFetchOverrideForTesting(null); + assert.equal(await callLogs.waitForCallLogSaves(3_000), true); + await artifactWriter.closeCallLogArtifactWriter(); + usageHistory.clearPendingRequests(); + accountSemaphore.resetAll(); + requestDedup.clearInflight(); + accountFallback.clearModelLock(); + dbCore.resetDbInstance(); + await loggerResource.closeSharedLoggerResource(); + globalThis.fetch = originalFetch; + + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalPluginsDir === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = originalPluginsDir; + + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("Grok Web rejects an error-only upstream stream before advertising HTTP 200 success", async () => { + const response = await executeStreaming([ + { + error: { + code: "UPSTREAM_PRIVATE_CODE", + message: + "UPSTREAM_PRIVATE_DETAIL Bearer top-secret-token /srv/grok/handler.ts:42\n" + + " at internal (/srv/grok/handler.ts:42:7)", + }, + }, + ]); + + assert.equal(response.status, 502); + assert.match(response.headers.get("Content-Type") ?? "", /application\/json/); + + const body = (await response.json()) as { + error: { message: string; type?: string; code?: string }; + upstream_details?: { error?: { message?: string } }; + }; + assert.equal(body.error.code, "STREAM_EARLY_EOF"); + assert.equal(body.error.type, "stream_early_eof"); + assert.equal(body.upstream_details?.error?.message, "Grok upstream stream failed"); + + const publicBody = JSON.stringify(body); + assert.doesNotMatch(publicBody, /UPSTREAM_PRIVATE/); + assert.doesNotMatch(publicBody, /top-secret-token/); + assert.doesNotMatch(publicBody, /\/srv\/grok/); + assert.doesNotMatch(publicBody, /\bat internal\b/); +}); + +test("Grok Web preserves partial content then rejects with a fixed public error", async () => { + let upstreamCancelCalls = 0; + const response = await executeStreamingBody( + stalledGrokEventStream( + [ + { result: { response: { token: "partial answer" } } }, + { + error: { + code: "UPSTREAM_PRIVATE_CODE", + message: "UPSTREAM_PRIVATE_DETAIL secret=never-public /srv/grok/stream.ts:99", + }, + }, + ], + () => { + upstreamCancelCalls += 1; + } + ) + ); + + assert.equal(response.status, 200); + const { text, error } = await readUntilFailure(response); + assert.ok(error instanceof Error); + assert.equal(error.message, "Grok upstream stream failed"); + const payloads = parseSseData(text) as Array>; + const content = payloads.find((payload) => { + const choices = payload.choices as Array<{ delta?: { content?: string } }> | undefined; + return choices?.[0]?.delta?.content === "partial answer"; + }); + assert.ok(content, "the valid content preceding the upstream failure must be retained"); + + assert.doesNotMatch(text, /UPSTREAM_PRIVATE/); + assert.doesNotMatch(text, /never-public/); + assert.doesNotMatch(text, /\/srv\/grok/); + assert.doesNotMatch(text, /\[Error:/); + assert.doesNotMatch(text, /"finish_reason":"stop"/); + assert.equal(upstreamCancelCalls, 1); +}); + +test("Grok Web converts a reader failure after content into the same safe terminal error", async () => { + const encoder = new TextEncoder(); + const upstreamBody = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(`${JSON.stringify({ result: { response: { token: "kept" } } })}\n`) + ); + setTimeout(() => { + controller.error( + new Error("READER_PRIVATE_DETAIL Bearer stream-token /srv/grok/reader.ts:12") + ); + }, 0); + }, + }); + + const response = await executeStreamingBody(upstreamBody); + assert.equal(response.status, 200); + const { text, error } = await readUntilFailure(response); + assert.ok(error instanceof Error); + assert.equal(error.message, "Grok upstream stream failed"); + const payloads = parseSseData(text) as Array>; + assert.ok( + payloads.some((payload) => { + const choices = payload.choices as Array<{ delta?: { content?: string } }> | undefined; + return choices?.[0]?.delta?.content === "kept"; + }) + ); + assert.doesNotMatch(text, /READER_PRIVATE/); + assert.doesNotMatch(text, /stream-token/); + assert.doesNotMatch(text, /\/srv\/grok/); + assert.doesNotMatch(text, /"finish_reason":"stop"/); +}); + +test("Grok Web propagates downstream cancellation once without awaiting a stuck upstream", async () => { + const encoder = new TextEncoder(); + let upstreamCancelCalls = 0; + const upstreamBody = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `${JSON.stringify({ result: { response: { token: "cancel-safe partial" } } })}\n` + ) + ); + }, + pull() { + return new Promise(() => {}); + }, + cancel() { + upstreamCancelCalls += 1; + return new Promise(() => {}); + }, + }); + const logMessages: string[] = []; + const recordLog = (tag: string, message: string) => { + logMessages.push(`${tag}: ${message}`); + }; + + const response = await executeStreamingBody(upstreamBody, undefined, { + log: { debug: recordLog, info: recordLog, warn: recordLog, error: recordLog }, + }); + assert.equal(response.status, 200); + assert.ok(response.body); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (!text.includes("cancel-safe partial")) { + const { done, value } = await reader.read(); + assert.equal(done, false); + if (value) text += decoder.decode(value, { stream: true }); + } + const logCountBeforeCancel = logMessages.length; + + assert.equal(await settlesWithin(reader.cancel("client stopped reading")), true); + assert.equal(await settlesWithin(reader.cancel("duplicate cancel")), true); + await new Promise((resolve) => setImmediate(resolve)); + + assert.equal(upstreamCancelCalls, 1); + assert.doesNotMatch(text, /"finish_reason":"stop"|data: \[DONE\]/); + assert.equal(logMessages.length, logCountBeforeCancel); +}); + +test("chatCore returns a pre-content Grok failure to the outer fallback contract", async () => { + const streamFailures: Array> = []; + let requestSucceeded = false; + const requestBody = { + model: "grok-4.1-fast", + messages: [{ role: "user", content: "fallback proof" }], + stream: true, + }; + + __setTlsFetchOverrideForTesting(async () => ({ + status: 200, + headers: new Headers({ "Content-Type": "application/x-ndjson" }), + text: null, + body: grokEventStream([ + { + error: { + code: "FALLBACK_PRIVATE_CODE", + message: "FALLBACK_PRIVATE_DETAIL secret=never-public /srv/grok/fallback.ts:5", + }, + }, + ]), + })); + + const result = await handleChatCore({ + body: structuredClone(requestBody), + modelInfo: { provider: "grok-web", model: "grok-4.1-fast", extendedContext: false }, + credentials: { apiKey: "sso=test-only-cookie", providerSpecificData: {} }, + connectionId: "grok-stream-error-fallback", + log: { debug() {}, info() {}, warn() {}, error() {} }, + clientRawRequest: { + endpoint: "/v1/chat/completions", + body: structuredClone(requestBody), + headers: new Headers({ accept: "text/event-stream" }), + }, + userAgent: "grok-stream-error-boundary-test", + onRequestSuccess() { + requestSucceeded = true; + }, + onStreamFailure(failure: Record) { + streamFailures.push(failure); + }, + } as never); + + assert.equal(result.success, false); + assert.equal(result.status, 502); + assert.equal(requestSucceeded, false); + assert.deepEqual(streamFailures, []); + + const publicBody = await result.response.text(); + assert.match(publicBody, /Grok upstream stream failed/); + assert.doesNotMatch(publicBody, /FALLBACK_PRIVATE|never-public|\/srv\/grok/); + assert.doesNotMatch(publicBody, /"role":"assistant"|"finish_reason":"stop"/); +}); + +test("chatCore converts a Grok post-content failure into terminal wire error and failed persistence", async () => { + await settingsDb.updateSettings({ call_log_pipeline_enabled: true }); + const streamFailures: Array> = []; + const requestBody = { + model: "grok-4.1-fast", + messages: [{ role: "user", content: "pipeline proof" }], + stream: true, + }; + + __setTlsFetchOverrideForTesting(async () => ({ + status: 200, + headers: new Headers({ "Content-Type": "application/x-ndjson" }), + text: null, + body: grokEventStream([ + { result: { response: { token: "pipeline partial" } } }, + { + error: { + code: "PIPELINE_PRIVATE_CODE", + message: "PIPELINE_PRIVATE_DETAIL secret=never-public /srv/grok/pipeline.ts:7", + }, + }, + ]), + })); + + const result = await handleChatCore({ + body: structuredClone(requestBody), + modelInfo: { provider: "grok-web", model: "grok-4.1-fast", extendedContext: false }, + credentials: { apiKey: "sso=test-only-cookie", providerSpecificData: {} }, + connectionId: "grok-stream-error-boundary", + log: { debug() {}, info() {}, warn() {}, error() {} }, + clientRawRequest: { + endpoint: "/v1/chat/completions", + body: structuredClone(requestBody), + headers: new Headers({ accept: "text/event-stream" }), + }, + userAgent: "grok-stream-error-boundary-test", + onStreamFailure(failure: Record) { + streamFailures.push(failure); + }, + } as never); + + assert.equal(result.success, true); + const wire = await result.response.text(); + assert.match(wire, /"content":"pipeline partial"/); + assert.match(wire, /"finish_reason":"error"/); + assert.match(wire, /"message":"Grok upstream stream failed"/); + assert.match(wire, /"type":"server_error"/); + assert.match(wire, /"code":"server_error"/); + assert.match(wire, /data: \[DONE\]/); + assert.doesNotMatch(wire, /"finish_reason":"stop"/); + assert.doesNotMatch(wire, /PIPELINE_PRIVATE|never-public|\/srv\/grok/); + + assert.equal(streamFailures.length, 1); + assert.deepEqual(streamFailures[0], { + status: 502, + message: "Grok upstream stream failed", + code: "stream_pipeline_error", + type: "stream_error", + }); + + assert.equal(await callLogs.waitForCallLogSaves(3_000), true); + const persisted = await waitFor(async () => { + const rows = await callLogs.getCallLogs({ provider: "grok-web", status: "error", limit: 5 }); + return rows.find((row) => row.connectionId === "grok-stream-error-boundary") ?? null; + }); + assert.ok(persisted, "expected the pipeline failure to be persisted"); + assert.equal(persisted.status, 502); + assert.equal(persisted.error, "Grok upstream stream failed"); + + const detail = await callLogs.getCallLogById(persisted.id); + assert.ok(detail?.pipelinePayloads, "expected failed pipeline payloads in the call log"); + const persistedPayload = JSON.stringify(detail.pipelinePayloads); + assert.match(persistedPayload, /Grok upstream stream failed/); + assert.doesNotMatch(persistedPayload, /PIPELINE_PRIVATE|never-public|\/srv\/grok/); +}); + +test("Grok Web still emits streaming tool calls after delaying the assistant role", async () => { + let upstreamCancelCalls = 0; + const response = await executeStreamingBody( + stalledGrokEventStream( + [ + { + result: { + response: { + modelResponse: { + message: + '{"name":"memory_context_tool","arguments":{"query":"grok"}}', + }, + }, + }, + }, + ], + () => { + upstreamCancelCalls += 1; + } + ), + { + messages: [{ role: "user", content: "search memory" }], + stream: true, + tools: [ + { + type: "function", + function: { + name: "memory_context_tool", + parameters: { type: "object", properties: { query: { type: "string" } } }, + }, + }, + ], + } + ); + + assert.equal(response.status, 200); + const text = await response.text(); + assert.match(text, /"role":"assistant"/); + assert.match(text, /"tool_calls"/); + assert.match(text, /"name":"memory_context_tool"/); + assert.match(text, /"finish_reason":"tool_calls"/); + assert.doesNotMatch(text, /"error"/); + assert.equal(upstreamCancelCalls, 1); +}); diff --git a/tests/fixtures/oneminai-stream-error-boundary.fixture.ts b/tests/fixtures/oneminai-stream-error-boundary.fixture.ts new file mode 100644 index 0000000000..5af1b6be9c --- /dev/null +++ b/tests/fixtures/oneminai-stream-error-boundary.fixture.ts @@ -0,0 +1,620 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +assert.ok(process.env.DATA_DIR, "the parent harness must provide an isolated DATA_DIR"); +assert.ok( + process.env.OMNIROUTE_PLUGINS_DIR, + "the parent harness must provide an isolated OMNIROUTE_PLUGINS_DIR" +); + +const [ + { OneMinAiExecutor }, + { ensureStreamReadiness }, + dbCore, + settingsDb, + callLogs, + usageHistory, + accountSemaphore, + readCache, + { handleChatCore }, +] = await Promise.all([ + import("../../open-sse/executors/oneminai.ts"), + import("../../open-sse/utils/streamReadiness.ts"), + import("../../src/lib/db/core.ts"), + import("../../src/lib/db/settings.ts"), + import("../../src/lib/usage/callLogs.ts"), + import("../../src/lib/usage/usageHistory.ts"), + import("../../open-sse/services/accountSemaphore.ts"), + import("../../src/lib/db/readCache.ts"), + import("../../open-sse/handlers/chatCore.ts"), +]); + +const originalFetch = globalThis.fetch; +const encoder = new TextEncoder(); +const STREAM_URL = "https://api.1min.ai/api/chat-with-ai?isStreaming=true"; + +type PersistenceIdentity = { + model: string; + connectionId: string; +}; + +const PRE_CONTENT_IDENTITY: PersistenceIdentity = { + model: "gpt-4o-mini-onemin-pre-content-boundary", + connectionId: "onemin-stream-pre-content-boundary", +}; +const BATCHED_IDENTITY: PersistenceIdentity = { + model: "gpt-4o-mini-onemin-batched-boundary", + connectionId: "onemin-stream-batched-boundary", +}; +const PARTIAL_IDENTITY: PersistenceIdentity = { + model: "gpt-4o-mini-onemin-partial-boundary", + connectionId: "onemin-stream-partial-boundary", +}; + +function installFetchFactory(responseFactory: () => Response): () => number { + let calls = 0; + globalThis.fetch = async (input, init = {}) => { + calls += 1; + assert.equal(String(input), STREAM_URL, "the test must never permit another network target"); + assert.equal(init.method, "POST"); + assert.equal((init.headers as Record)["API-KEY"], "unit-test-key"); + + return responseFactory(); + }; + return () => calls; +} + +function createStreamingResponse(events: string[]): Response { + return new Response( + new ReadableStream({ + start(controller) { + for (const event of events) controller.enqueue(encoder.encode(event)); + controller.close(); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); +} + +function installStreamingFetch(events: string[]): () => number { + return installFetchFactory(() => createStreamingResponse(events)); +} + +async function executeStreaming(events: string[]): Promise { + const getCalls = installStreamingFetch(events); + const result = await new OneMinAiExecutor().execute({ + model: "gpt-4o-mini", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "unit-test-key" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + assert.equal(getCalls(), 1); + return result.response; +} + +function noopLog() { + return { debug() {}, info() {}, warn() {}, error() {} }; +} + +async function invokeStreamingChatCore( + identity: PersistenceIdentity, + onStreamFailure?: (failure: { + status: number; + message: string; + code?: string; + type?: string; + }) => void, + onRequestSuccess?: () => Promise | void +) { + await settingsDb.updateSettings({ call_log_pipeline_enabled: true }); + readCache.invalidateDbCache("settings"); + const body = { + model: identity.model, + stream: true, + messages: [{ role: "user", content: "hello" }], + }; + + return handleChatCore({ + body: structuredClone(body), + modelInfo: { provider: "oneminai", model: identity.model, extendedContext: false }, + credentials: { + apiKey: "unit-test-key", + connectionId: identity.connectionId, + providerSpecificData: {}, + }, + connectionId: identity.connectionId, + log: noopLog(), + clientRawRequest: { + endpoint: "/v1/chat/completions", + body: structuredClone(body), + headers: new Headers({ + accept: "text/event-stream", + "x-omniroute-session-id": identity.connectionId, + }), + }, + userAgent: identity.connectionId, + onRequestSuccess, + onStreamFailure, + } as never); +} + +async function waitFor(read: () => Promise, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = await read(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return null; +} + +async function getOneMinCallLog(identity: PersistenceIdentity) { + assert.equal( + await callLogs.waitForCallLogSaves(5_000), + true, + "call-log persistence must drain before inspection" + ); + const rows = await callLogs.getCallLogs({ + provider: "oneminai", + model: identity.model, + limit: 20, + }); + const row = Array.isArray(rows) + ? rows.find( + (candidate) => + candidate.connectionId === identity.connectionId && + (candidate.model === identity.model || candidate.requestedModel === identity.model) + ) + : null; + return row ? callLogs.getCallLogById(row.id) : null; +} + +async function getOneMinUsage(identity: PersistenceIdentity) { + const rows = await usageHistory.getUsageHistory({ + provider: "oneminai", + model: identity.model, + }); + return rows.find((row) => row.connectionId === identity.connectionId) ?? null; +} + +async function assertUnusedPersistenceIdentity(identity: PersistenceIdentity) { + assert.equal( + await getOneMinCallLog(identity), + null, + `call-log identity must be unused before scenario: ${identity.connectionId}` + ); + assert.equal( + await getOneMinUsage(identity), + null, + `usage identity must be unused before scenario: ${identity.connectionId}` + ); +} + +async function readUntil( + reader: ReadableStreamDefaultReader, + marker: string +): Promise { + const decoder = new TextDecoder(); + let text = ""; + while (!text.includes(marker)) { + const { done, value } = await reader.read(); + assert.equal(done, false, `stream ended before ${marker}`); + if (value) text += decoder.decode(value, { stream: true }); + } + return text; +} + +async function readRemaining(reader: ReadableStreamDefaultReader): Promise { + const decoder = new TextDecoder(); + let text = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return text + decoder.decode(); + if (value) text += decoder.decode(value, { stream: true }); + } +} + +test.afterEach(async () => { + const drained = await callLogs.waitForCallLogSaves(5_000); + globalThis.fetch = originalFetch; + usageHistory.clearPendingRequests(); + accountSemaphore.resetAll(); + assert.equal(drained, true, "all call-log saves must drain before the next test"); +}); + +test.after(async () => { + const drained = await callLogs.waitForCallLogSaves(5_000); + try { + await callLogs.closeCallLogSaves(5_000); + } finally { + globalThis.fetch = originalFetch; + usageHistory.clearPendingRequests(); + accountSemaphore.resetAll(); + dbCore.resetDbInstance(); + } + assert.equal(drained, true, "all call-log saves must drain before teardown"); +}); + +test("1min.ai pre-content stream errors stay errors and permit readiness fallback", async () => { + const rawMessage = + "quota lookup failed at /srv/omniroute/open-sse/executors/oneminai.ts:170\n" + + " at translateSseStream (/srv/omniroute/open-sse/executors/oneminai.ts:99:5)"; + const response = await executeStreaming([ + `event: error\ndata: ${JSON.stringify({ error: { message: rawMessage } })}\n\n`, + ]); + const clientCopy = response.clone(); + + const readiness = await ensureStreamReadiness(response, { + timeoutMs: 2_000, + provider: "oneminai", + model: "gpt-4o-mini", + }); + assert.equal(readiness.ok, false); + if (readiness.ok) assert.fail("an error-only stream must not become ready"); + assert.equal(readiness.response.status, 502); + const fallbackBody = await readiness.response.text(); + assert.match(fallbackBody, /STREAM_EARLY_EOF/); + assert.doesNotMatch(fallbackBody, /\/srv\/omniroute/); + assert.doesNotMatch(fallbackBody, /translateSseStream/); + + const clientText = await clientCopy.text(); + assert.match(clientText, /^data: \{"error":/); + assert.match(clientText, /quota lookup failed at /); + assert.match(clientText, /data: \[DONE\]/); + assert.doesNotMatch(clientText, /"role":"assistant"/); + assert.doesNotMatch(clientText, /"finish_reason":"stop"/); + assert.doesNotMatch(clientText, /\/srv\/omniroute/); + assert.doesNotMatch(clientText, /translateSseStream/); +}); + +test("chatCore turns a pre-content 1min.ai stream error into persisted HTTP 502", async () => { + await assertUnusedPersistenceIdentity(PRE_CONTENT_IDENTITY); + installStreamingFetch([ + `event: error\ndata: ${JSON.stringify({ + error: { + message: + "quota lookup failed at /srv/omniroute/open-sse/executors/oneminai.ts:230 api_key=pre-content-secret\nstack tail", + }, + })}\n\n`, + ]); + + const result = await invokeStreamingChatCore(PRE_CONTENT_IDENTITY); + assert.equal(result.success, false); + if (result.success) assert.fail("a pre-content error must not commit HTTP 200"); + assert.equal(result.status, 502); + assert.equal(result.response.status, 502); + const clientBody = await result.response.text(); + assert.match(clientBody, /STREAM_EARLY_EOF/); + assert.doesNotMatch(clientBody, /pre-content-secret/); + assert.doesNotMatch(clientBody, /\/srv\/omniroute/); + assert.doesNotMatch(clientBody, /stack tail/); + + const detail = await waitFor(() => getOneMinCallLog(PRE_CONTENT_IDENTITY)); + assert.ok(detail, "the failed pre-content attempt must be persisted"); + assert.equal(detail.status, 502); + const persisted = JSON.stringify(detail); + assert.doesNotMatch(persisted, /pre-content-secret/); + assert.doesNotMatch(persisted, /\/srv\/omniroute/); + assert.doesNotMatch(persisted, /stack tail/); + + const usage = await waitFor(() => getOneMinUsage(PRE_CONTENT_IDENTITY)); + assert.ok(usage, "the failed pre-content usage record must be persisted"); + assert.equal(usage.success, false); + assert.equal(usage.status, "502"); + assert.equal(usage.errorCode, "STREAM_EARLY_EOF"); +}); + +test("chatCore preserves batched 1min.ai content before its terminal stream error", async () => { + await assertUnusedPersistenceIdentity(BATCHED_IDENTITY); + installStreamingFetch([ + 'event: content\ndata: {"content":"batched partial one"}\n\n' + + 'event: content\ndata: {"content":"batched partial two"}\n\n' + + `event: error\ndata: ${JSON.stringify({ + message: + "provider failed at /srv/omniroute/open-sse/executors/oneminai.ts:230 api_key=batched-secret", + })}\n\n`, + ]); + const failures: Array<{ + status: number; + message: string; + code?: string; + type?: string; + }> = []; + const requestSuccessPhases: string[] = []; + + const result = await invokeStreamingChatCore( + BATCHED_IDENTITY, + (failure) => failures.push(failure), + async () => { + requestSuccessPhases.push("started"); + await new Promise((resolve) => setTimeout(resolve, 30)); + requestSuccessPhases.push("finished"); + } + ); + assert.equal(result.success, true, "batched real content must cross the readiness boundary"); + assert.deepEqual(requestSuccessPhases, ["started", "finished"]); + assert.ok(result.response.body); + const clientText = await result.response.text(); + const firstContentIndex = clientText.indexOf("batched partial one"); + const secondContentIndex = clientText.indexOf("batched partial two"); + const errorIndex = clientText.indexOf('"error":'); + const doneIndex = clientText.indexOf("data: [DONE]"); + + assert.ok(firstContentIndex >= 0, "the first queued content delta must not be discarded"); + assert.ok(secondContentIndex >= 0, "the second queued content delta must not be discarded"); + assert.ok(firstContentIndex < secondContentIndex, "batched content must retain upstream order"); + assert.ok(secondContentIndex < errorIndex, "all batched content must precede its terminal error"); + assert.ok( + errorIndex < doneIndex, + `the terminal error must precede [DONE]: ${JSON.stringify(clientText)}` + ); + assert.match(clientText, /"finish_reason":"error"/); + assert.doesNotMatch(clientText, /"finish_reason":"stop"/); + assert.doesNotMatch(clientText, /response\.failed/); + assert.doesNotMatch(clientText, /batched-secret/); + assert.doesNotMatch(clientText, /\/srv\/omniroute/); + assert.deepEqual(failures, [ + { + status: 502, + message: "1min.ai upstream stream failed", + code: "stream_pipeline_error", + type: "stream_error", + }, + ]); + + const pending = usageHistory.getPendingRequests(); + assert.deepEqual(Object.keys(pending.byModel), []); + assert.deepEqual(Object.keys(pending.byAccount), []); + + const completed = [...usageHistory.getCompletedDetails().values()]; + assert.equal(completed.length, 1); + assert.equal(completed[0].status, 502); + assert.equal(completed[0].error, "1min.ai upstream stream failed"); + assert.equal(completed[0].errorCode, "stream_pipeline_error"); + + const detail = await waitFor(() => getOneMinCallLog(BATCHED_IDENTITY)); + assert.ok(detail, "the batched terminal stream failure must be persisted"); + assert.equal(detail.status, 502); + assert.equal(detail.error, "1min.ai upstream stream failed"); + const persisted = JSON.stringify(detail); + assert.doesNotMatch(persisted, /batched-secret/); + assert.doesNotMatch(persisted, /\/srv\/omniroute/); + + const usage = await waitFor(() => getOneMinUsage(BATCHED_IDENTITY)); + assert.ok(usage, "the batched terminal failure usage record must be persisted"); + assert.equal(usage.success, false); + assert.equal(usage.status, "502"); + assert.equal(usage.errorCode, "stream_pipeline_error"); +}); + +test("chatCore preserves partial 1min.ai content then finalizes and persists a stream failure", async () => { + await assertUnusedPersistenceIdentity(PARTIAL_IDENTITY); + let upstreamController: ReadableStreamDefaultController | null = null; + let cancelCalls = 0; + const getCalls = installFetchFactory( + () => + new Response( + new ReadableStream({ + start(controller) { + upstreamController = controller; + controller.enqueue( + encoder.encode('event: content\ndata: {"content":"partial answer"}\n\n') + ); + }, + cancel() { + cancelCalls += 1; + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ) + ); + const failures: Array<{ + status: number; + message: string; + code?: string; + type?: string; + }> = []; + + const result = await invokeStreamingChatCore(PARTIAL_IDENTITY, (failure) => + failures.push(failure) + ); + assert.equal(getCalls(), 1); + assert.equal(result.success, true, "real content must cross the readiness boundary"); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + let clientText = await readUntil(reader, "partial answer"); + + assert.ok(upstreamController); + upstreamController.enqueue( + encoder.encode( + `event: error\ndata: ${JSON.stringify({ + message: + "provider failed at /srv/omniroute/open-sse/executors/oneminai.ts:230 api_key=post-content-secret\nstack tail", + })}\n\n` + ) + ); + clientText += await readRemaining(reader); + + const roleIndex = clientText.indexOf('"role":"assistant"'); + const contentIndex = clientText.indexOf("partial answer"); + const errorIndex = clientText.indexOf('"error":'); + const doneIndex = clientText.indexOf("data: [DONE]"); + + assert.ok(roleIndex >= 0 && roleIndex < contentIndex, "the role must precede real content"); + assert.ok(contentIndex < errorIndex, "partial content must remain before the terminal error"); + assert.ok(errorIndex < doneIndex, "the pipeline error must precede [DONE]"); + assert.equal(clientText.match(/"role":"assistant"/g)?.length, 1); + assert.match(clientText, /"finish_reason":"error"/); + assert.match(clientText, /1min\.ai upstream stream failed/); + assert.doesNotMatch(clientText, /"finish_reason":"stop"/); + assert.doesNotMatch(clientText, /response\.failed/); + assert.doesNotMatch(clientText, /post-content-secret/); + assert.doesNotMatch(clientText, /\/srv\/omniroute/); + assert.doesNotMatch(clientText, /stack tail/); + + assert.equal(cancelCalls, 1, "the upstream source must be cancelled after its terminal error"); + assert.equal(failures.length, 1); + assert.deepEqual(failures[0], { + status: 502, + message: "1min.ai upstream stream failed", + code: "stream_pipeline_error", + type: "stream_error", + }); + const pending = usageHistory.getPendingRequests(); + assert.deepEqual(Object.keys(pending.byModel), []); + assert.deepEqual(Object.keys(pending.byAccount), []); + + const completed = [...usageHistory.getCompletedDetails().values()]; + assert.equal(completed.length, 1); + assert.equal(completed[0].status, 502); + assert.equal(completed[0].error, "1min.ai upstream stream failed"); + assert.equal(completed[0].errorCode, "stream_pipeline_error"); + + const detail = await waitFor(() => getOneMinCallLog(PARTIAL_IDENTITY)); + assert.ok(detail, "the post-content stream failure must be persisted"); + assert.equal(detail.status, 502); + assert.equal(detail.error, "1min.ai upstream stream failed"); + const persisted = JSON.stringify(detail); + assert.match(persisted, /1min\.ai upstream stream failed/); + assert.doesNotMatch(persisted, /post-content-secret/); + assert.doesNotMatch(persisted, /\/srv\/omniroute/); + assert.doesNotMatch(persisted, /stack tail/); + + const usage = await waitFor(() => getOneMinUsage(PARTIAL_IDENTITY)); + assert.ok(usage, "the post-content failure usage record must be persisted"); + assert.equal(usage.success, false); + assert.equal(usage.status, "502"); + assert.equal(usage.errorCode, "stream_pipeline_error"); +}); + +test("1min.ai error completion does not wait for an upstream cancel promise", async () => { + let cancelCalls = 0; + const getCalls = installFetchFactory( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode('event: error\ndata: {"message":"capacity unavailable"}\n\n') + ); + }, + cancel() { + cancelCalls += 1; + return new Promise(() => {}); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ) + ); + + const result = await new OneMinAiExecutor().execute({ + model: "gpt-4o-mini", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "unit-test-key" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + const clientText = await Promise.race([ + result.response.text(), + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error("translated stream stayed pending on cancel")), 500) + ), + ]); + + assert.equal(getCalls(), 1); + assert.equal(cancelCalls, 1); + assert.match(clientText, /capacity unavailable/); + assert.match(clientText, /data: \[DONE\]/); +}); + +test("1min.ai propagates downstream cancellation without awaiting upstream cleanup", async () => { + let upstreamController: ReadableStreamDefaultController | null = null; + let cancelCalls = 0; + let markPullStarted: (() => void) | null = null; + const pullStarted = new Promise((resolve) => { + markPullStarted = resolve; + }); + const getCalls = installFetchFactory( + () => + new Response( + new ReadableStream({ + start(controller) { + upstreamController = controller; + controller.enqueue( + encoder.encode('event: content\ndata: {"content":"partial answer"}\n\n') + ); + }, + pull() { + markPullStarted?.(); + return new Promise(() => {}); + }, + cancel() { + cancelCalls += 1; + return new Promise(() => {}); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ) + ); + + const result = await new OneMinAiExecutor().execute({ + model: "gpt-4o-mini", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "unit-test-key" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + + try { + const clientText = await readUntil(reader, "partial answer"); + assert.match(clientText, /"role":"assistant"/); + await pullStarted; + await Promise.race([ + reader.cancel("client disconnected"), + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error("downstream cancellation stayed pending")), 500) + ), + ]); + + assert.equal(getCalls(), 1); + assert.equal(cancelCalls, 1, "downstream cancellation must reach the upstream reader once"); + assert.deepEqual(await reader.read(), { value: undefined, done: true }); + } finally { + try { + upstreamController?.close(); + } catch { + // The fixed path has already cancelled and closed the upstream stream. + } + } +}); + +test("1min.ai accepts the bounded error-string shape without exposing a success chunk", async () => { + const response = await executeStreaming([ + 'event: error\ndata: {"error":"billing temporarily unavailable"}\n\n', + ]); + const clientText = await response.text(); + + assert.match(clientText, /"error":\{"message":"billing temporarily unavailable"/); + assert.doesNotMatch(clientText, /"role":"assistant"/); + assert.doesNotMatch(clientText, /"finish_reason":"stop"/); +}); + +test("1min.ai replaces oversized stream-error payloads with a fixed public fallback", async () => { + const oversizedMessage = `private-prefix-${"x".repeat(70 * 1024)}`; + const response = await executeStreaming([ + `event: error\ndata: ${JSON.stringify({ message: oversizedMessage })}\n\n`, + ]); + const clientText = await response.text(); + + assert.match(clientText, /1min\.ai upstream stream failed/); + assert.ok(clientText.length < 1_024, "the oversized upstream payload must not be reflected"); + assert.doesNotMatch(clientText, /private-prefix/); + assert.doesNotMatch(clientText, /"role":"assistant"/); + assert.doesNotMatch(clientText, /"finish_reason":"stop"/); +}); diff --git a/tests/fixtures/perplexity-web-stream-error-boundary.fixture.ts b/tests/fixtures/perplexity-web-stream-error-boundary.fixture.ts new file mode 100644 index 0000000000..8d02adbbad --- /dev/null +++ b/tests/fixtures/perplexity-web-stream-error-boundary.fixture.ts @@ -0,0 +1,648 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +assert.ok(process.env.DATA_DIR, "the subprocess fixture requires an isolated DATA_DIR"); +assert.ok( + process.env.OMNIROUTE_PLUGINS_DIR, + "the subprocess fixture requires an isolated OMNIROUTE_PLUGINS_DIR" +); + +const core = await import("../../src/lib/db/core.ts"); +const { getUsageHistory } = await import("../../src/lib/usage/usageHistory.ts"); +const { waitForCallLogSaves } = await import("../../src/lib/usage/callLogs.ts"); +const { closeCallLogArtifactWriter } = await import("../../src/lib/usage/callLogArtifactWriter.ts"); +const { PerplexityWebExecutor } = await import("../../open-sse/executors/perplexity-web.ts"); +const { __setTlsFetchOverrideForTesting } = + await import("../../open-sse/services/perplexityTlsClient.ts"); +const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); + +type StreamFailure = { status: number; message: string; code?: string; type?: string }; + +function createPerplexityStream( + events: Array> +): ReadableStream { + const encoder = new TextEncoder(); + const payload = + events.map((event) => `event: message\r\ndata: ${JSON.stringify(event)}\r\n\r\n`).join("") + + "event: end_of_stream\r\n\r\n"; + return new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(payload)); + controller.close(); + }, + }); +} + +async function executeWithUpstreamBody( + body: ReadableStream, + prompt = "hi" +): Promise { + __setTlsFetchOverrideForTesting(async () => ({ + status: 200, + headers: new Headers({ "Content-Type": "text/event-stream" }), + text: null, + body, + })); + + const executor = new PerplexityWebExecutor(); + const result = await executor.execute({ + model: "pplx-auto", + body: { messages: [{ role: "user", content: prompt }], stream: true }, + stream: true, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + return result.response; +} + +function executeStreaming( + events: Array>, + prompt = "hi" +): Promise { + return executeWithUpstreamBody(createPerplexityStream(events), prompt); +} + +async function executeThroughChatCore( + events: Array>, + prompt = "hi", + onStreamFailure?: (failure: StreamFailure) => void, + onRequestSuccess?: () => Promise, + model = "pplx-auto" +) { + return executeBodyThroughChatCore( + createPerplexityStream(events), + prompt, + onStreamFailure, + onRequestSuccess, + model + ); +} + +async function executeBodyThroughChatCore( + upstreamBody: ReadableStream, + prompt = "hi", + onStreamFailure?: (failure: StreamFailure) => void, + onRequestSuccess?: () => Promise, + model = "pplx-auto" +) { + __setTlsFetchOverrideForTesting(async () => ({ + status: 200, + headers: new Headers({ "Content-Type": "text/event-stream" }), + text: null, + body: upstreamBody, + })); + const body = { + model, + messages: [{ role: "user", content: prompt }], + stream: true, + }; + return handleChatCore({ + body: structuredClone(body), + modelInfo: { provider: "perplexity-web", model, extendedContext: false }, + credentials: { apiKey: "test-cookie", providerSpecificData: {} }, + log: { debug() {}, info() {}, warn() {}, error() {} }, + onRequestSuccess, + onStreamFailure, + clientRawRequest: { + endpoint: "/v1/chat/completions", + body: structuredClone(body), + headers: new Headers({ accept: "text/event-stream" }), + }, + userAgent: "perplexity-stream-error-boundary-test", + skipResourcePressureGuard: true, + }); +} + +function assertNoSensitiveDetail(value: string): void { + assert.doesNotMatch(value, /private-runtime\.ts/); + assert.doesNotMatch(value, /sk-pplx-secret/); + assert.doesNotMatch(value, /api_key/); +} + +function assertChatCompletionWire(value: string): void { + assert.doesNotMatch(value, /^event:/m, "Chat Completions must not receive Responses framing"); + const payloads = value + .split(/\r?\n/) + .filter((line) => line.startsWith("data:") && line.slice(5).trim() !== "[DONE]") + .map((line) => JSON.parse(line.slice(5).trim()) as Record); + assert.ok(payloads.length > 0); + for (const payload of payloads) { + assert.equal(payload.object, "chat.completion.chunk"); + assert.ok(Array.isArray(payload.choices)); + for (const choice of payload.choices as Array>) { + assert.equal(choice.index, 0); + assert.equal(typeof choice.delta, "object"); + assert.ok( + choice.finish_reason === null || typeof choice.finish_reason === "string", + "finish_reason must remain Chat Completions-compatible" + ); + } + } + assert.match(value, /data: \[DONE\]/); +} + +async function waitForPersistedStreamFailure(startedAt: Date, model = "pplx-auto") { + for (let attempt = 0; attempt < 80; attempt++) { + const rows = await getUsageHistory({ + provider: "perplexity-web", + model, + startDate: startedAt, + }); + const failure = rows.find( + (row) => + row.success === false && row.status === "502" && row.errorCode === "stream_pipeline_error" + ); + if (failure) return failure; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return null; +} + +async function waitForCondition(predicate: () => boolean, timeoutMs = 500): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return true; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + return predicate(); +} + +test.afterEach(() => { + __setTlsFetchOverrideForTesting(null); +}); + +test.after(async () => { + __setTlsFetchOverrideForTesting(null); + assert.equal( + await waitForCallLogSaves(3_000), + true, + "call-log writes must drain before the isolated DATA_DIR is removed" + ); + await closeCallLogArtifactWriter(); + core.resetDbInstance(); +}); + +test("pre-content Perplexity failures remain unready and return a sanitized 502", async () => { + const result = await executeThroughChatCore([ + { + error_code: "PPLX_ERROR", + error_message: + "failed at /srv/omniroute/private-runtime.ts:42:7 token=sk-pplx-secret-123456 api_key=hidden", + }, + ]); + + assert.equal(result.success, false, "the handler must expose a fallback-eligible failure"); + assert.equal(result.status, 502); + assert.equal(result.response.status, 502); + assert.equal(result.errorCode, "STREAM_EARLY_EOF"); + const body = await result.response.text(); + assert.match(body, /"error"/); + assertNoSensitiveDetail(body); +}); + +test("thrown Perplexity stream failures remain unready and return a sanitized 502", async () => { + const result = await executeBodyThroughChatCore( + new ReadableStream({ + start(controller) { + controller.error( + new Error( + "socket failed at /srv/omniroute/private-runtime.ts:51:9 token=sk-pplx-secret-catch" + ) + ); + }, + }), + "throw before content" + ); + + assert.equal(result.success, false, "the handler must expose a fallback-eligible failure"); + assert.equal(result.status, 502); + assert.equal(result.response.status, 502); + const responseBody = await result.response.text(); + assert.match(responseBody, /"error"/); + assertNoSensitiveDetail(responseBody); +}); + +test("thrown failures after content preserve the prefix and terminate as a safe error", async () => { + const encoder = new TextEncoder(); + const partialAnswer = "partial before transport failure"; + let upstreamRead = false; + let finalizedFailure: StreamFailure | null = null; + const upstreamBody = new ReadableStream({ + pull(controller) { + if (!upstreamRead) { + upstreamRead = true; + controller.enqueue( + encoder.encode( + `event: message\r\ndata: ${JSON.stringify({ + backend_uuid: "uuid-thrown-must-not-store", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [partialAnswer], progress: "IN_PROGRESS" }, + }, + ], + status: "PENDING", + })}\r\n\r\n` + ) + ); + return; + } + controller.error( + new Error( + "transport failed at /srv/omniroute/private-runtime.ts:79 token=sk-pplx-secret-after" + ) + ); + }, + }); + + const result = await executeBodyThroughChatCore( + upstreamBody, + "post-content transport failure", + (failure) => { + finalizedFailure = failure; + } + ); + + assert.equal(result.success, true); + const output = await result.response.text(); + assert.match(output, new RegExp(partialAnswer)); + assert.match(output, /"finish_reason":"error"/); + assert.doesNotMatch(output, /"finish_reason":"stop"/); + assert.doesNotMatch(output, /response\.failed/); + assertNoSensitiveDetail(output); + assertChatCompletionWire(output); + assert.ok(finalizedFailure); + assert.equal(finalizedFailure.status, 502); + assert.equal(finalizedFailure.message, "Perplexity upstream stream failed"); +}); + +test("partial content is preserved before a terminal error and the failed session is not stored", async () => { + const firstPrompt = "partial-boundary-first-prompt"; + const partialAnswer = "safe partial answer"; + const requestStartedAt = new Date(Date.now() - 1_000); + let finalizedFailure: StreamFailure | null = null; + const firstResult = await executeThroughChatCore( + [ + { + backend_uuid: "uuid-must-not-be-stored", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [partialAnswer], progress: "IN_PROGRESS" }, + }, + ], + status: "PENDING", + }, + { + error_code: "PPLX_ERROR", + error_message: + "later failure at /srv/omniroute/private-runtime.ts:66:2 token=sk-pplx-secret-partial", + }, + ], + firstPrompt, + (failure) => { + finalizedFailure = failure; + } + ); + + assert.equal(firstResult.success, true, "legitimate partial output must satisfy readiness"); + const output = await firstResult.response.text(); + assert.match(output, /"role":"assistant"/); + assert.match(output, new RegExp(partialAnswer)); + assert.match(output, /"error":\{/); + assert.match(output, /"finish_reason":"error"/); + assert.doesNotMatch(output, /"finish_reason":"stop"/); + assert.doesNotMatch(output, /response\.failed/); + assert.doesNotMatch(output, /event:\s*response\.failed/); + assert.doesNotMatch(output, /"type":"response\.failed"/); + assert.doesNotMatch(output, /\[Error:/); + assertNoSensitiveDetail(output); + assertChatCompletionWire(output); + assert.ok(finalizedFailure, "the downstream pipeline must finalize the stream failure"); + assert.equal(finalizedFailure.status, 502); + assert.equal(finalizedFailure.message, "Perplexity upstream stream failed"); + const persistedFailure = await waitForPersistedStreamFailure(requestStartedAt); + assert.ok(persistedFailure, "the handler must persist the terminal stream failure"); + + let followUpRequestBody: string | undefined; + __setTlsFetchOverrideForTesting(async (_url, options) => { + followUpRequestBody = String(options.body ?? ""); + return { + status: 200, + headers: new Headers({ "Content-Type": "text/event-stream" }), + text: null, + body: createPerplexityStream([ + { + backend_uuid: "next-success-uuid", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: ["next answer"], progress: "DONE" }, + }, + ], + status: "COMPLETED", + }, + ]), + }; + }); + + const executor = new PerplexityWebExecutor(); + const followUp = await executor.execute({ + model: "pplx-auto", + body: { + messages: [ + { role: "user", content: firstPrompt }, + { role: "assistant", content: partialAnswer }, + { role: "user", content: "continue" }, + ], + stream: false, + }, + stream: false, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + assert.equal(followUp.response.status, 200); + assert.ok(followUpRequestBody); + const sent = JSON.parse(followUpRequestBody) as { params?: Record }; + assert.equal( + sent.params?.last_backend_uuid, + undefined, + "a failed partial response must not create a reusable session" + ); +}); + +test("same-packet content and error preserve the prefix across repeated readiness handoffs", async () => { + for (let attempt = 0; attempt < 8; attempt++) { + const partialAnswer = `same-packet partial ${attempt}`; + let finalizedFailure: StreamFailure | null = null; + const result = await executeThroughChatCore( + [ + { + backend_uuid: `uuid-same-packet-${attempt}`, + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [partialAnswer], progress: "IN_PROGRESS" }, + }, + ], + status: "PENDING", + }, + { + error_code: "PPLX_ERROR", + error_message: `same packet private failure ${attempt} token=sk-pplx-secret-repeat`, + }, + ], + `same-packet prompt ${attempt}`, + (failure) => { + finalizedFailure = failure; + } + ); + + assert.equal(result.success, true); + const output = await result.response.text(); + assert.match(output, new RegExp(partialAnswer)); + assert.match(output, /"finish_reason":"error"/); + assert.doesNotMatch(output, /"finish_reason":"stop"/); + assert.doesNotMatch(output, /response\.failed/); + assertNoSensitiveDetail(output); + assertChatCompletionWire(output); + assert.ok(finalizedFailure); + assert.equal(finalizedFailure.status, 502); + } +}); + +test("a delayed success hook cannot erase same-packet content before terminal failure", async () => { + const partialAnswer = "prefix must survive delayed success bookkeeping"; + const model = "pplx-auto-delayed-success-proof"; + const requestStartedAt = new Date(Date.now() - 1_000); + let finalizedFailure: StreamFailure | null = null; + const result = await executeThroughChatCore( + [ + { + backend_uuid: "uuid-delayed-success-hook-must-not-store", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [partialAnswer], progress: "IN_PROGRESS" }, + }, + ], + status: "PENDING", + }, + { + error_code: "PPLX_ERROR", + error_message: "private same-packet failure token=sk-pplx-secret-delayed-hook", + }, + ], + "delayed success hook prompt", + (failure) => { + finalizedFailure = failure; + }, + async () => { + await new Promise((resolve) => setTimeout(resolve, 25)); + }, + model + ); + + assert.equal(result.success, true, "the legitimate prefix must satisfy readiness"); + const output = await result.response.text(); + assert.match(output, new RegExp(partialAnswer)); + assert.match(output, /"finish_reason":"error"/); + assert.doesNotMatch(output, /"finish_reason":"stop"/); + assert.doesNotMatch(output, /response\.failed/); + assertNoSensitiveDetail(output); + assertChatCompletionWire(output); + assert.ok(finalizedFailure); + assert.equal(finalizedFailure.status, 502); + assert.equal(finalizedFailure.message, "Perplexity upstream stream failed"); + assert.ok( + await waitForPersistedStreamFailure(requestStartedAt, model), + "the delayed handoff must still persist the terminal stream failure" + ); +}); + +test("successful streamed completions still store their Perplexity session", async () => { + const firstPrompt = "successful-session-first-prompt"; + const firstAnswer = "successful session answer"; + const firstResponse = await executeStreaming( + [ + { + backend_uuid: "uuid-success-is-stored", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [firstAnswer], progress: "DONE" }, + }, + ], + status: "COMPLETED", + }, + ], + firstPrompt + ); + const firstOutput = await firstResponse.text(); + assert.match(firstOutput, new RegExp(firstAnswer)); + assert.match(firstOutput, /"finish_reason":"stop"/); + + let followUpRequestBody: string | undefined; + __setTlsFetchOverrideForTesting(async (_url, options) => { + followUpRequestBody = String(options.body ?? ""); + return { + status: 200, + headers: new Headers({ "Content-Type": "text/event-stream" }), + text: null, + body: createPerplexityStream([ + { + backend_uuid: "uuid-next-success", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: ["continued"], progress: "DONE" }, + }, + ], + status: "COMPLETED", + }, + ]), + }; + }); + + const executor = new PerplexityWebExecutor(); + const followUp = await executor.execute({ + model: "pplx-auto", + body: { + messages: [ + { role: "user", content: firstPrompt }, + { role: "assistant", content: firstAnswer }, + { role: "user", content: "continue successful session" }, + ], + stream: false, + }, + stream: false, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + assert.equal(followUp.response.status, 200); + assert.ok(followUpRequestBody); + const sent = JSON.parse(followUpRequestBody) as { params?: Record }; + assert.equal(sent.params?.last_backend_uuid, "uuid-success-is-stored"); +}); + +test("downstream cancellation reaches a stalled Perplexity reader exactly once", async () => { + const encoder = new TextEncoder(); + const partialAnswer = "cancel after this prefix"; + let firstPull = true; + let upstreamCancelCount = 0; + let finalizedFailure: StreamFailure | null = null; + const upstreamBody = new ReadableStream({ + pull(controller) { + if (firstPull) { + firstPull = false; + controller.enqueue( + encoder.encode( + `event: message\r\ndata: ${JSON.stringify({ + backend_uuid: "uuid-cancel-must-not-store", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: [partialAnswer], progress: "IN_PROGRESS" }, + }, + ], + status: "PENDING", + })}\r\n\r\n` + ) + ); + return; + } + return new Promise(() => {}); + }, + cancel() { + upstreamCancelCount += 1; + return new Promise(() => {}); + }, + }); + + const result = await executeBodyThroughChatCore( + upstreamBody, + "cancel stalled stream", + (failure) => { + finalizedFailure = failure; + } + ); + assert.equal(result.success, true); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + const decoder = new TextDecoder(); + let prefix = ""; + for (let readCount = 0; readCount < 4 && !prefix.includes(partialAnswer); readCount += 1) { + const next = await Promise.race([ + reader.read(), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 500)), + ]); + if (next === "timeout" || next.done) break; + prefix += decoder.decode(next.value, { stream: true }); + } + + const cancelResult = await Promise.race([ + reader.cancel("client stopped reading").then(() => "settled"), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 500)), + ]); + assert.match(prefix, new RegExp(partialAnswer)); + assert.doesNotMatch(prefix, /"finish_reason":"stop"/); + assert.doesNotMatch(prefix, /data: \[DONE\]/); + assert.equal(cancelResult, "settled", "client cancellation must not await a hostile upstream"); + assert.equal( + await waitForCondition(() => upstreamCancelCount === 1), + true, + "cancellation must reach the real upstream reader" + ); + assert.equal(upstreamCancelCount, 1); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(finalizedFailure, null, "client cancellation must not finalize as provider failure"); + + let followUpRequestBody: string | undefined; + __setTlsFetchOverrideForTesting(async (_url, options) => { + followUpRequestBody = String(options.body ?? ""); + return { + status: 200, + headers: new Headers({ "Content-Type": "text/event-stream" }), + text: null, + body: createPerplexityStream([ + { + backend_uuid: "uuid-after-cancel", + blocks: [ + { + intended_usage: "markdown", + markdown_block: { chunks: ["answer after cancel"], progress: "DONE" }, + }, + ], + status: "COMPLETED", + }, + ]), + }; + }); + const executor = new PerplexityWebExecutor(); + const followUp = await executor.execute({ + model: "pplx-auto", + body: { + messages: [ + { role: "user", content: "cancel stalled stream" }, + { role: "assistant", content: partialAnswer }, + { role: "user", content: "continue after cancellation" }, + ], + stream: false, + }, + stream: false, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); + assert.equal(followUp.response.status, 200); + assert.ok(followUpRequestBody); + const sent = JSON.parse(followUpRequestBody) as { params?: Record }; + assert.equal( + sent.params?.last_backend_uuid, + undefined, + "a cancelled response must not create a reusable session" + ); +}); diff --git a/tests/fixtures/stream-handler-public-error-boundary.fixture.ts b/tests/fixtures/stream-handler-public-error-boundary.fixture.ts new file mode 100644 index 0000000000..de48e9bdf4 --- /dev/null +++ b/tests/fixtures/stream-handler-public-error-boundary.fixture.ts @@ -0,0 +1,211 @@ +// This suite owns process-wide DATA_DIR, plugin, logger, and DB state. It must run only inside +// the subprocess launched by tests/unit/stream-handler-public-error-boundary.test.ts. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const originalDataDir = process.env.DATA_DIR; +const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-stream-public-error-")); +const TEST_DATA_DIR = path.join(testRoot, "data"); +const TEST_PLUGINS_DIR = path.join(testRoot, "plugins"); +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; + +const [core, callLogs, artifactWriter, loggerResource, streamHandler, { FORMATS }] = + await Promise.all([ + import("../../src/lib/db/core.ts"), + import("../../src/lib/usage/callLogs.ts"), + import("../../src/lib/usage/callLogArtifactWriter.ts"), + import("../../src/shared/utils/loggerResource.ts"), + import("../../open-sse/utils/streamHandler.ts"), + import("../../open-sse/translator/formats.ts"), + ]); +const { createStreamController, pipeWithDisconnect } = streamHandler; + +const SECRET = "sk-live-streamhandler-secret-123456"; +const API_KEY = "provider-key-streamhandler-654321"; +const PRIVATE_PATH = "/srv/omniroute/private/provider.ts:42:9"; +const RAW_MESSAGE = + `Upstream failed at ${PRIVATE_PATH} Authorization: Bearer ${SECRET} api_key=${API_KEY}` + + `\n at dispatch (/srv/omniroute/private/dispatcher.ts:88:3)`; + +test.after(async () => { + assert.equal(await callLogs.waitForCallLogSaves(3_000), true); + await artifactWriter.closeCallLogArtifactWriter(); + core.resetDbInstance(); + await loggerResource.closeSharedLoggerResource(); + + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalPluginsDir === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = originalPluginsDir; + + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("fixture binds all persistent state to its process-owned directories", () => { + assert.equal(core.DATA_DIR, TEST_DATA_DIR); + assert.equal(core.SQLITE_FILE, path.join(TEST_DATA_DIR, "storage.sqlite")); + assert.equal(process.env.DATA_DIR, TEST_DATA_DIR); + assert.equal(process.env.OMNIROUTE_PLUGINS_DIR, TEST_PLUGINS_DIR); + assert.equal(fs.existsSync(TEST_DATA_DIR), true); + assert.equal(fs.existsSync(TEST_PLUGINS_DIR), true); +}); + +test("OpenAI stream failures keep raw diagnostics internal and sanitize the public wire", async () => { + const upstreamError = Object.assign(new Error(RAW_MESSAGE), { statusCode: 502 }); + const source = new ReadableStream({ + start(controller) { + controller.error(upstreamError); + }, + }); + let internalMessage = ""; + + const stream = pipeWithDisconnect( + new Response(source), + new TransformStream(), + createStreamController({ + clientResponseFormat: FORMATS.OPENAI, + onError(event) { + internalMessage = event.message; + return true; + }, + }), + { stallTimeoutMs: 0 } + ); + const publicWire = await new Response(stream).text(); + + assert.equal(internalMessage, RAW_MESSAGE, "failure classification must retain the raw message"); + assert.match(publicWire, /"finish_reason":"error"/); + assert.match(publicWire, /"code":"server_error"/); + assert.match(publicWire, /\[DONE\]/); + assert.doesNotMatch(publicWire, new RegExp(SECRET)); + assert.doesNotMatch(publicWire, new RegExp(API_KEY)); + assert.doesNotMatch(publicWire, /\/srv\/omniroute\/private/); + assert.doesNotMatch(publicWire, /dispatcher\.ts/); + assert.match(publicWire, /Authorization: \[REDACTED\]/); + assert.match(publicWire, //); +}); + +test("Responses stream failures preserve the failure event shape without leaking diagnostics", async () => { + const upstreamError = Object.assign(new Error(RAW_MESSAGE), { statusCode: 429 }); + const source = new ReadableStream({ + start(controller) { + controller.error(upstreamError); + }, + }); + let internalError: unknown; + + const stream = pipeWithDisconnect( + new Response(source), + new TransformStream(), + createStreamController({ + clientResponseFormat: FORMATS.OPENAI_RESPONSES, + onError(event) { + internalError = event.error; + return true; + }, + }), + { stallTimeoutMs: 0 } + ); + const publicWire = await new Response(stream).text(); + + assert.equal(internalError, upstreamError, "the original error object must reach classification"); + assert.match(publicWire, /event: response\.failed/); + assert.match(publicWire, /"type":"response\.failed"/); + assert.match(publicWire, /"type":"rate_limit_error"/); + assert.match(publicWire, /"code":"rate_limit_exceeded"/); + assert.doesNotMatch(publicWire, new RegExp(SECRET)); + assert.doesNotMatch(publicWire, new RegExp(API_KEY)); + assert.doesNotMatch(publicWire, /\/srv\/omniroute\/private/); + assert.doesNotMatch(publicWire, /dispatcher\.ts/); + assert.match(publicWire, /Authorization: \[REDACTED\]/); + assert.match(publicWire, //); +}); + +test("Claude stream failures preserve error and stop events without leaking diagnostics", async () => { + const upstreamError = Object.assign(new Error(RAW_MESSAGE), { statusCode: 403 }); + const source = new ReadableStream({ + start(controller) { + controller.error(upstreamError); + }, + }); + let internalStatusCode = 0; + + const stream = pipeWithDisconnect( + new Response(source), + new TransformStream(), + createStreamController({ + clientResponseFormat: FORMATS.CLAUDE, + onError(event) { + internalStatusCode = event.statusCode; + return true; + }, + }), + { stallTimeoutMs: 0 } + ); + const publicWire = await new Response(stream).text(); + + assert.equal(internalStatusCode, 403); + assert.match(publicWire, /event: error/); + assert.match(publicWire, /"type":"permission_error"/); + assert.match(publicWire, /event: message_stop/); + assert.doesNotMatch(publicWire, new RegExp(SECRET)); + assert.doesNotMatch(publicWire, new RegExp(API_KEY)); + assert.doesNotMatch(publicWire, /\/srv\/omniroute\/private/); + assert.doesNotMatch(publicWire, /dispatcher\.ts/); + assert.match(publicWire, /Authorization: \[REDACTED\]/); + assert.match(publicWire, //); +}); + +test("stream diagnostics sanitize logs while callbacks retain the original failure", () => { + const upstreamError = Object.assign(new Error(RAW_MESSAGE), { statusCode: 502 }); + const originalLog = console.log; + const logLines: string[] = []; + let internalError: unknown; + console.log = (...args: unknown[]) => { + logLines.push(args.map(String).join(" ")); + }; + + try { + createStreamController({ + provider: "test-provider", + model: "test-model", + onError(event) { + internalError = event.error; + return true; + }, + }).handleError(upstreamError); + } finally { + console.log = originalLog; + } + + const logs = logLines.join("\n"); + assert.equal(internalError, upstreamError); + assert.match(logs, /error: Upstream failed at /); + assert.match(logs, /Authorization: \[REDACTED\]/); + assert.doesNotMatch(logs, new RegExp(SECRET)); + assert.doesNotMatch(logs, new RegExp(API_KEY)); + assert.doesNotMatch(logs, /\/srv\/omniroute\/private/); + assert.doesNotMatch(logs, /dispatcher\.ts/); +}); + +test("client disconnects stay outside the provider-failure callback", () => { + let providerFailureRecorded = false; + const controller = createStreamController({ + onError() { + providerFailureRecorded = true; + return true; + }, + }); + + controller.handleError(new DOMException("request_signal_aborted", "AbortError")); + + assert.equal(providerFailureRecorded, false); + assert.equal(controller.signal.aborted, false); +}); diff --git a/tests/fixtures/zai-web-stream-error-boundary.fixture.ts b/tests/fixtures/zai-web-stream-error-boundary.fixture.ts new file mode 100644 index 0000000000..79afeef0eb --- /dev/null +++ b/tests/fixtures/zai-web-stream-error-boundary.fixture.ts @@ -0,0 +1,253 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import test from "node:test"; + +assert.ok(process.env.DATA_DIR, "the parent wrapper must provide a synthetic DATA_DIR"); +assert.ok( + process.env.OMNIROUTE_PLUGINS_DIR, + "the parent wrapper must provide a synthetic plugin directory" +); +assert.ok(process.env.API_KEY_SECRET, "the parent wrapper must provide a synthetic API secret"); + +fs.mkdirSync(process.env.DATA_DIR, { recursive: true }); +fs.mkdirSync(process.env.OMNIROUTE_PLUGINS_DIR, { recursive: true }); + +const [ + { buildZaiStreamingBody }, + { ensureStreamReadiness }, + { createSSEStream }, + { createStreamController, pipeWithDisconnect }, + { createStreamFailureFinalizers }, + { FORMATS }, + dbCore, + { closeSharedLoggerResource }, +] = await Promise.all([ + import("../../open-sse/executors/zai-web/stream.ts"), + import("../../open-sse/utils/streamReadiness.ts"), + import("../../open-sse/utils/stream.ts"), + import("../../open-sse/utils/streamHandler.ts"), + import("../../open-sse/utils/streamFailureFinalization.ts"), + import("../../open-sse/translator/formats.ts"), + import("../../src/lib/db/core.ts"), + import("../../src/shared/utils/loggerResource.ts"), +]); + +test.after(async () => { + await closeSharedLoggerResource(); + dbCore.resetDbInstance(); +}); + +const encoder = new TextEncoder(); + +function upstreamSse(...payloads: Record[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const payload of payloads) { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(payload)}\n\n`)); + } + controller.close(); + }, + }); +} + +function emitOpenAiChunk( + controller: ReadableStreamDefaultController, + delta: Record, + finish: string | null = null +): void { + const chunk = { + id: "chatcmpl-zai-test", + object: "chat.completion.chunk", + created: 1, + model: "glm-5.2", + choices: [{ index: 0, delta, finish_reason: finish }], + }; + controller.enqueue(encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`)); +} + +type PipelineResult = { + output: string; + completions: Array<{ status: number; errorCode?: string | null; error?: string | null }>; + persisted: Array<{ status: number; errorCode?: string }>; + failures: Array<{ status: number; message: string; code?: string; type?: string }>; +}; + +function jsonDataPayloads(output: string): Array> { + return output + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6)) as Record); +} + +async function runPartialFailurePipeline(clientResponseFormat: string): Promise { + const completions: PipelineResult["completions"] = []; + const persisted: PipelineResult["persisted"] = []; + const failures: PipelineResult["failures"] = []; + const { onPipelineStreamError } = createStreamFailureFinalizers({ + isFailureCompletionRecorded: () => false, + isStreamCompletionRecorded: () => false, + onStreamComplete(payload) { + completions.push({ + status: payload.status, + errorCode: payload.errorCode, + error: payload.error, + }); + }, + persistFailureUsage(status, errorCode) { + persisted.push({ status, errorCode }); + }, + onStreamFailure(failure) { + failures.push(failure); + }, + }); + + const zaiStream = buildZaiStreamingBody( + upstreamSse( + { + type: "chat:completion", + data: { delta_content: "partial answer", phase: "answer" }, + }, + { error: { message: "stream aborted upstream" } } + ), + emitOpenAiChunk, + null + ); + const passthrough = clientResponseFormat === FORMATS.OPENAI; + const transform = createSSEStream({ + mode: passthrough ? "passthrough" : "translate", + targetFormat: FORMATS.OPENAI, + sourceFormat: passthrough ? FORMATS.OPENAI : clientResponseFormat, + clientResponseFormat, + provider: "zai-web", + model: "glm-5.2", + body: { messages: [{ role: "user", content: "hello" }] }, + }); + const streamController = createStreamController({ + provider: "zai-web", + model: "glm-5.2", + clientResponseFormat, + onError: onPipelineStreamError, + }); + const output = await new Response( + pipeWithDisconnect( + new Response(zaiStream, { headers: { "Content-Type": "text/event-stream" } }), + transform, + streamController, + { stallTimeoutMs: 0 } + ) + ).text(); + + return { output, completions, persisted, failures }; +} + +function assertFailureWasPersisted(result: PipelineResult): void { + assert.deepEqual(result.completions, [ + { + status: 502, + errorCode: "stream_pipeline_error", + error: "Z.ai stream failed: stream aborted upstream", + }, + ]); + assert.deepEqual(result.persisted, [{ status: 502, errorCode: "stream_pipeline_error" }]); + assert.deepEqual(result.failures, [ + { + status: 502, + message: "Z.ai stream failed: stream aborted upstream", + code: "stream_pipeline_error", + type: "stream_error", + }, + ]); +} + +test("a pre-content Z.ai error fails stream readiness with a sanitized 502", async () => { + const rawFailure = + 'signature invalid at /srv/omniroute/open-sse/auth.ts:17:9 api_key="sk-private"\n' + + " at verify (/srv/omniroute/open-sse/auth.ts:17:9)"; + const stream = buildZaiStreamingBody( + upstreamSse({ error: { detail: rawFailure } }), + emitOpenAiChunk, + null + ); + + const readiness = await ensureStreamReadiness( + new Response(stream, { headers: { "Content-Type": "text/event-stream" } }), + { timeoutMs: 100, provider: "zai-web", model: "glm-5.2" } + ); + + if (readiness.ok) { + await readiness.response.body?.cancel(); + assert.fail("an error-only Z.ai stream must not be accepted as ready model output"); + } + + assert.equal(readiness.response.status, 502); + assert.equal(readiness.code, "STREAM_EARLY_EOF"); + assert.match(readiness.upstreamDiagnostic ?? "", /Z\.ai stream failed: signature invalid/); + + const publicBody = JSON.stringify(await readiness.response.json()); + assert.doesNotMatch(publicBody, /sk-private|\/srv\/omniroute|auth\.ts/); + assert.match(publicBody, //); +}); + +test("a partial Z.ai failure stays strict Chat and persists as pipeline failure", async () => { + const result = await runPartialFailurePipeline(FORMATS.OPENAI); + const payloads = jsonDataPayloads(result.output); + const terminal = payloads.find((payload) => "error" in payload); + + assert.match(result.output, /partial answer/, "content before the failure is preserved"); + assert.ok(terminal, "the Chat client receives a terminal structured error chunk"); + assert.equal(terminal.object, "chat.completion.chunk"); + assert.deepEqual(Object.keys(terminal).sort(), ["choices", "error", "object"]); + assert.deepEqual(terminal.choices, [{ index: 0, delta: {}, finish_reason: "error" }]); + assert.deepEqual(terminal.error, { + message: "Z.ai stream failed: stream aborted upstream", + type: "server_error", + code: "server_error", + }); + assert.match(result.output, /data: \[DONE\]/); + assert.doesNotMatch(result.output, /response\.failed|event: response\.failed/); + assert.doesNotMatch(result.output, /"finish_reason":"stop"/); + assertFailureWasPersisted(result); +}); + +test("a partial Z.ai failure is translated to Claude and persists as failure", async () => { + const result = await runPartialFailurePipeline(FORMATS.CLAUDE); + + assert.match(result.output, /partial answer/, "translated partial content is preserved"); + assert.match(result.output, /event: error\r?\n/); + assert.match(result.output, /"type":"error"/); + assert.match(result.output, /"message":"Z\.ai stream failed: stream aborted upstream"/); + assert.match(result.output, /event: message_stop\r?\n/); + assert.doesNotMatch(result.output, /response\.failed|event: response\.failed/); + assert.doesNotMatch(result.output, /finish_reason|data: \[DONE\]/); + assertFailureWasPersisted(result); +}); + +test("client cancellation stays non-blocking and cancels a stalled Z.ai body", async () => { + let markPullStarted: (() => void) | null = null; + const pullStarted = new Promise((resolve) => { + markPullStarted = resolve; + }); + let upstreamCancelCalls = 0; + const stalledUpstream = new ReadableStream({ + pull() { + markPullStarted?.(); + return new Promise(() => {}); + }, + cancel() { + upstreamCancelCalls += 1; + return new Promise(() => {}); + }, + }); + const reader = buildZaiStreamingBody(stalledUpstream, emitOpenAiChunk, null).getReader(); + const pendingRead = reader.read(); + void pendingRead.catch(() => {}); + await pullStarted; + + const outcome = await Promise.race([ + reader.cancel("client closed").then(() => "resolved" as const), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 100)), + ]); + + assert.equal(outcome, "resolved", "consumer cancel cannot wait for a stalled upstream body"); + assert.equal(upstreamCancelCalls, 1, "the locked upstream reader receives one cancel request"); +}); diff --git a/tests/fixtures/zed-hosted-stream-error-boundary-child.ts b/tests/fixtures/zed-hosted-stream-error-boundary-child.ts new file mode 100644 index 0000000000..c5c8f8352d --- /dev/null +++ b/tests/fixtures/zed-hosted-stream-error-boundary-child.ts @@ -0,0 +1,341 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// This file is executed only by the process-isolated unit-test wrapper. State +// mutations and repository imports must remain here, never in the parent test. +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-zed-stream-data-")); +const TEST_PLUGINS_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-zed-stream-plugins-")); +const originalDataDir = process.env.DATA_DIR; +const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; +const originalFetch = globalThis.fetch; +let networkCalls = 0; + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; +globalThis.fetch = async () => { + networkCalls += 1; + throw new Error("Unexpected network access in Zed stream boundary test"); +}; + +const core = await import("../../src/lib/db/core.ts"); +const loggerResource = await import("../../src/shared/utils/loggerResource.ts"); +const { __test__ } = await import("../../open-sse/executors/zed-hosted.ts"); +const { FORMATS } = await import("../../open-sse/translator/formats.ts"); +const { assembleStreamingPipeline } = + await import("../../open-sse/handlers/chatCore/streamingPipeline.ts"); +const { createPassthroughStreamWithLogger } = await import("../../open-sse/utils/stream.ts"); +const { createStreamFailureFinalizers } = + await import("../../open-sse/utils/streamFailureFinalization.ts"); +const { createStreamController } = await import("../../open-sse/utils/streamHandler.ts"); +const { ensureStreamReadiness } = await import("../../open-sse/utils/streamReadiness.ts"); +const { wrapZedCompletionStream } = __test__; + +type StreamCompletionEvent = Parameters< + Parameters[0]["onStreamComplete"] +>[0]; + +const RAW_FAILURE = "Bearer TOP_SECRET /srv/omniroute/zed-handler.ts:42 api_key=zed-secret"; +const TEST_MODEL = "grok-test-zed-stream-boundary"; +const TEST_CONNECTION_ID = "zed-stream-boundary-partial-connection"; + +function failedStatusLine(): string { + return JSON.stringify({ status: { failed: { message: RAW_FAILURE } } }); +} + +function nestedFailedStatusLine(): string { + return JSON.stringify({ + status: { + type: "failed", + error: { message: `${RAW_FAILURE} ${"x".repeat(2_000)}` }, + }, + }); +} + +function wrapOpenNdjson(lines: unknown[]): Response { + const encoder = new TextEncoder(); + const body = lines + .map((line) => (typeof line === "string" ? line : JSON.stringify(line))) + .join("\n"); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(`${body}\n`)); + // Keep the upstream open: status.failed must terminate the wrapped stream itself. + }, + cancel() {}, + }); + return wrapZedCompletionStream( + new Response(stream, { + status: 200, + headers: { "Content-Type": "application/x-ndjson" }, + }), + "x_ai", + TEST_MODEL + ); +} + +function wrapOpenFailedNdjson(): Response { + const encoder = new TextEncoder(); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(`${failedStatusLine()}\n`)); + // Deliberately stay open: status.failed is terminal by itself and must not + // depend on the upstream socket eventually reaching EOF. + }, + cancel() {}, + }); + return wrapZedCompletionStream( + new Response(body, { + status: 200, + headers: { "Content-Type": "application/x-ndjson" }, + }), + "x_ai", + TEST_MODEL + ); +} + +function wrapStalledNdjson(onCancel: () => void): Response { + const body = new ReadableStream({ + cancel() { + onCancel(); + return new Promise(() => {}); + }, + }); + return wrapZedCompletionStream( + new Response(body, { + status: 200, + headers: { "Content-Type": "application/x-ndjson" }, + }), + "x_ai", + TEST_MODEL + ); +} + +async function resolvesWithin(promise: Promise, timeoutMs: number): Promise { + let timeout: ReturnType | undefined; + try { + await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error(`operation exceeded ${timeoutMs}ms`)), + timeoutMs + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +async function waitFor(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + assert.equal(predicate(), true, `condition was not met within ${timeoutMs}ms`); +} + +function parseSsePayloads(text: string): Array> { + return text + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line.slice(6) !== "[DONE]") + .map((line) => JSON.parse(line.slice(6)) as Record); +} + +function assertNoSensitiveFailureText(text: string): void { + assert.doesNotMatch(text, /TOP_SECRET|zed-secret|\/srv\/omniroute\/zed-handler\.ts/); +} + +test.after(async () => { + core.resetDbInstance(); + await loggerResource.closeSharedLoggerResource(); + globalThis.fetch = originalFetch; + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalPluginsDir === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = originalPluginsDir; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.rmSync(TEST_PLUGINS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("zed-hosted pre-content status.failed becomes a sanitized 502 readiness failure", async () => { + const readiness = await ensureStreamReadiness(wrapOpenFailedNdjson(), { + timeoutMs: 100, + provider: "zed-hosted", + model: TEST_MODEL, + }); + + assert.equal(readiness.ok, false, "the structured error must remain eligible for fallback"); + if (readiness.ok) assert.fail("pre-content Zed failure must not make the stream ready"); + assert.equal(readiness.response.status, 502); + assert.equal(readiness.code, "STREAM_EARLY_EOF"); + + const bodyText = await readiness.response.text(); + const body = JSON.parse(bodyText) as { + error: { message: string; type: string; code: string }; + upstream_details?: { error?: { message?: string } }; + }; + assert.equal(body.error.type, "stream_early_eof"); + assert.equal(body.error.code, "STREAM_EARLY_EOF"); + assert.match(body.upstream_details?.error?.message ?? "", /Zed stream failed/i); + assertNoSensitiveFailureText(bodyText); + assert.equal(networkCalls, 0); +}); + +test("zed-hosted partial failure reaches stream finalization and persistence as 502", async () => { + const roleChunk = { + event: { + id: "chatcmpl-zed-partial", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }, + }; + const contentChunk = { + event: { + id: "chatcmpl-zed-partial", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { content: "partial answer" }, finish_reason: null }], + }, + }; + const readiness = await ensureStreamReadiness( + wrapOpenNdjson([ + roleChunk, + contentChunk, + nestedFailedStatusLine(), + { event: { ignored: "after failure" } }, + ]), + { + timeoutMs: 100, + provider: "zed-hosted", + model: TEST_MODEL, + } + ); + + assert.equal(readiness.ok, true, "partial model output must remain deliverable"); + const completionEvents: StreamCompletionEvent[] = []; + const persistedFailures: Array<{ + connectionId: string; + model: string; + status: number; + code?: string; + }> = []; + const streamFailures: Array<{ status: number; message: string; code?: string; type?: string }> = + []; + const pipelineErrors: Array<{ message: string; statusCode: number }> = []; + let streamCompletionRecorded = false; + let failureCompletionRecorded = false; + + const recordCompletion = (payload: StreamCompletionEvent): void => { + if (streamCompletionRecorded) return; + streamCompletionRecorded = true; + if (payload.status !== 200) failureCompletionRecorded = true; + completionEvents.push(payload); + }; + const finalizers = createStreamFailureFinalizers({ + isFailureCompletionRecorded: () => failureCompletionRecorded, + isStreamCompletionRecorded: () => streamCompletionRecorded, + onStreamComplete: recordCompletion, + persistFailureUsage: (status, code) => + persistedFailures.push({ + connectionId: TEST_CONNECTION_ID, + model: TEST_MODEL, + status, + code, + }), + onStreamFailure: (failure) => streamFailures.push(failure), + }); + const streamController = createStreamController({ + onError: (event) => { + pipelineErrors.push({ message: event.message, statusCode: event.statusCode }); + return finalizers.onPipelineStreamError(event); + }, + provider: "zed-hosted", + model: TEST_MODEL, + connectionId: TEST_CONNECTION_ID, + clientResponseFormat: FORMATS.OPENAI, + }); + const transformStream = createPassthroughStreamWithLogger( + "zed-hosted", + null, + null, + TEST_MODEL, + TEST_CONNECTION_ID, + { messages: [{ role: "user", content: "test" }] }, + recordCompletion, + null, + finalizers.handleStreamFailure, + FORMATS.OPENAI + ); + const responseHeaders: Record = {}; + const finalStream = assembleStreamingPipeline({ + providerResponse: readiness.response, + transformStream, + streamController, + createPiiTransform: null, + clientRawRequestHeaders: null, + clientResponseFormat: FORMATS.OPENAI, + echoModel: null, + responseHeaders, + }); + const text = await new Response(finalStream, { headers: responseHeaders }).text(); + const payloads = parseSsePayloads(text); + const errorPayload = payloads.find((payload) => "error" in payload) as + { error: { message: string; type: string; code: string } } | undefined; + + assert.match(text, /partial answer/); + assert.ok(errorPayload, "the stream handler must emit its format-safe terminal error"); + assert.equal(errorPayload.error.type, "server_error"); + assert.equal(errorPayload.error.code, "server_error"); + assert.equal(errorPayload.error.message, "Zed upstream stream failed"); + assert.match(text, /"finish_reason":"error"/); + assert.doesNotMatch(text, /\[Zed error\]|"finish_reason":"stop"|response\.failed/); + assert.doesNotMatch(text, /"ignored":"after failure"/); + assertNoSensitiveFailureText(text); + + assert.equal(completionEvents.length, 1, "the failure must finalize exactly once"); + assert.equal(completionEvents[0].status, 502); + assert.equal(completionEvents[0].error, "Zed upstream stream failed"); + assert.equal(completionEvents[0].errorCode, "stream_pipeline_error"); + assert.deepEqual(persistedFailures, [ + { + connectionId: TEST_CONNECTION_ID, + model: TEST_MODEL, + status: 502, + code: "stream_pipeline_error", + }, + ]); + assert.deepEqual(streamFailures, [ + { + status: 502, + message: "Zed upstream stream failed", + code: "stream_pipeline_error", + type: "stream_error", + }, + ]); + assert.deepEqual(pipelineErrors, [{ message: "Zed upstream stream failed", statusCode: 502 }]); + assertNoSensitiveFailureText(JSON.stringify(completionEvents)); + assert.equal(networkCalls, 0); +}); + +test("zed-hosted client cancellation does not await a stalled upstream cancel hook", async () => { + let upstreamCancelCalls = 0; + const response = wrapStalledNdjson(() => { + upstreamCancelCalls += 1; + }); + assert.ok(response.body); + + const reader = response.body.getReader(); + const pendingRead = reader.read(); + await resolvesWithin(reader.cancel("client disconnected"), 100); + const readResult = await pendingRead; + assert.equal(readResult.done, true); + await waitFor(() => upstreamCancelCalls === 1, 100); + + await resolvesWithin(reader.cancel("duplicate cancel"), 100); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(upstreamCancelCalls, 1, "the upstream cancel hook must be requested exactly once"); + assert.equal(networkCalls, 0); +}); diff --git a/tests/unit/_fixtures/huggingchat-stream-error-boundary.fixture.ts b/tests/unit/_fixtures/huggingchat-stream-error-boundary.fixture.ts new file mode 100644 index 0000000000..fc901496d2 --- /dev/null +++ b/tests/unit/_fixtures/huggingchat-stream-error-boundary.fixture.ts @@ -0,0 +1,592 @@ +import assert from "node:assert/strict"; +import { isAbsolute, relative } from "node:path"; +import { after, test } from "node:test"; + +function requiredEnv(name: string): string { + const value = process.env[name]; + assert.ok(value, `${name} must be supplied by the isolated parent wrapper`); + return value; +} + +const testRoot = requiredEnv("OMNIROUTE_HUGGINGCHAT_TEST_ROOT"); +const fixtureRunId = requiredEnv("OMNIROUTE_HUGGINGCHAT_TEST_RUN_ID"); +const testDataDir = requiredEnv("DATA_DIR"); +const testPluginsDir = requiredEnv("OMNIROUTE_PLUGINS_DIR"); +const xdgConfigDir = requiredEnv("XDG_CONFIG_HOME"); + +for (const [name, candidate] of [ + ["DATA_DIR", testDataDir], + ["OMNIROUTE_PLUGINS_DIR", testPluginsDir], + ["XDG_CONFIG_HOME", xdgConfigDir], +] as const) { + const fromRoot = relative(testRoot, candidate); + assert.equal( + isAbsolute(fromRoot) || fromRoot.startsWith(".."), + false, + `${name} escaped test root` + ); +} +assert.equal( + process.env.NODE_TEST_CONTEXT, + undefined, + "nested node:test state must not be inherited" +); +assert.equal(process.env.HOME, undefined, "the child must not inherit the operator HOME"); +assert.equal(process.env.CODEX_HOME, undefined, "the child must not inherit CODEX_HOME"); +assert.match(requiredEnv("API_KEY_SECRET"), /^[0-9a-f]{64}$/); + +const [ + { HuggingChatExecutor }, + { HuggingChatStreamError, streamJsonlToOpenAi }, + { createPassthroughStreamWithLogger }, + { createStreamController, pipeWithDisconnect }, + { createStreamFailureFinalizers, finalizeStreamRequestLog }, + { ensureStreamReadiness }, + { FORMATS }, + usageHistory, + coreDb, + callLogs, + callLogArtifactWriter, + loggerResource, +] = await Promise.all([ + import("../../../open-sse/executors/huggingchat.ts"), + import("../../../open-sse/executors/huggingchat/jsonlStream.ts"), + import("../../../open-sse/utils/stream.ts"), + import("../../../open-sse/utils/streamHandler.ts"), + import("../../../open-sse/utils/streamFailureFinalization.ts"), + import("../../../open-sse/utils/streamReadiness.ts"), + import("../../../open-sse/translator/formats.ts"), + import("../../../src/lib/usage/usageHistory.ts"), + import("../../../src/lib/db/core.ts"), + import("../../../src/lib/usage/callLogs.ts"), + import("../../../src/lib/usage/callLogArtifactWriter.ts"), + import("../../../src/shared/utils/loggerResource.ts"), +]); + +after(async () => { + assert.equal( + await callLogs.waitForCallLogSaves(10_000), + true, + "all asynchronous call-log writes must drain before DB teardown" + ); + await callLogArtifactWriter.closeCallLogArtifactWriter(); + usageHistory.clearPendingRequests(); + await loggerResource.closeSharedLoggerResource(); + coreDb.resetDbInstance(); +}); + +function jsonlBody(lines: string[], trailingNewline = true): ReadableStream { + const encoded = new TextEncoder().encode(`${lines.join("\n")}${trailingNewline ? "\n" : ""}`); + return new ReadableStream({ + start(controller) { + controller.enqueue(encoded); + controller.close(); + }, + }); +} + +async function collectStream(body: ReadableStream): Promise { + const chunks: string[] = []; + for await (const chunk of streamJsonlToOpenAi( + body, + "test/huggingchat-model", + "chatcmpl-huggingchat-test", + 1_725_000_000 + )) { + chunks.push(chunk); + } + return chunks.join(""); +} + +test("HuggingChat turns a pre-content JSONL generation error into a sanitized 502", async () => { + const rawError = + "generation failed at /srv/omniroute/providers/huggingchat.ts:44:9 api_key=super-secret\n" + + " at provider (/srv/omniroute/runtime.ts:1:1)"; + const realFetch = globalThis.fetch; + let callCount = 0; + const errorLogs: string[] = []; + + globalThis.fetch = (async () => { + callCount += 1; + if (callCount === 1) { + return Response.json({ conversationId: "conversation-test" }); + } + if (callCount === 2) { + return Response.json({ rootMessageId: "root-message-test" }); + } + if (callCount === 3) { + return new Response( + jsonlBody( + [ + JSON.stringify({ type: "status", status: "started" }), + JSON.stringify({ type: "status", status: "error", message: rawError }), + ], + false + ), + { status: 200, headers: { "Content-Type": "application/jsonl" } } + ); + } + throw new Error(`Unexpected fetch call ${callCount}`); + }) as typeof globalThis.fetch; + + try { + const result = await new HuggingChatExecutor().execute({ + model: "test/huggingchat-model", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "hf-chat=fake-cookie" }, + signal: null, + log: { error: (_tag, message) => errorLogs.push(message) }, + }); + + assert.equal(callCount, 3, "the test must intercept every HuggingChat request"); + assert.equal(result.response.status, 502); + assert.match(result.response.headers.get("content-type") || "", /application\/json/); + + const payload = (await result.response.json()) as { + error: { message: string; type?: string; code?: string }; + }; + assert.equal(payload.error.type, "upstream_error"); + assert.equal(payload.error.code, "huggingchat_generation_error"); + assert.match(payload.error.message, /generation failed/); + assert.doesNotMatch(payload.error.message, /\/srv\/omniroute/); + assert.doesNotMatch(payload.error.message, /super-secret/); + assert.doesNotMatch(payload.error.message, /\n\s*at /); + assert.equal(errorLogs.length, 1); + assert.doesNotMatch(errorLogs[0], /\/srv\/omniroute/); + assert.doesNotMatch(errorLogs[0], /super-secret/); + assert.doesNotMatch(errorLogs[0], /\n\s*at /); + } finally { + globalThis.fetch = realFetch; + } +}); + +test("HuggingChat turns a terminal non-stream JSONL error into a sanitized 502", async () => { + const rawError = + "generation failed at /srv/omniroute/providers/huggingchat.ts:55:2 cookie=super-secret\n" + + " at provider (/srv/omniroute/runtime.ts:1:1)"; + const realFetch = globalThis.fetch; + let callCount = 0; + + globalThis.fetch = (async () => { + callCount += 1; + if (callCount === 1) return Response.json({ conversationId: "conversation-test" }); + if (callCount === 2) return Response.json({ rootMessageId: "root-message-test" }); + if (callCount === 3) { + return new Response( + jsonlBody([JSON.stringify({ type: "status", status: "error", message: rawError })], false), + { status: 200, headers: { "Content-Type": "application/jsonl" } } + ); + } + throw new Error(`Unexpected fetch call ${callCount}`); + }) as typeof globalThis.fetch; + + try { + const result = await new HuggingChatExecutor().execute({ + model: "test/huggingchat-model", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: false, + credentials: { apiKey: "hf-chat=fake-cookie" }, + signal: null, + }); + + assert.equal(callCount, 3, "the test must intercept every HuggingChat request"); + assert.equal(result.response.status, 502); + const payload = (await result.response.json()) as { + error: { message: string; type?: string; code?: string }; + }; + assert.equal(payload.error.type, "upstream_error"); + assert.equal(payload.error.code, "huggingchat_generation_error"); + assert.doesNotMatch(payload.error.message, /\/srv\/omniroute/); + assert.doesNotMatch(payload.error.message, /super-secret/); + assert.doesNotMatch(payload.error.message, /\n\s*at /); + } finally { + globalThis.fetch = realFetch; + } +}); + +test("HuggingChat rejects its JSONL generator after partial content instead of faking success", async () => { + const rawError = + "generation failed at /srv/omniroute/providers/huggingchat.ts:44:9 access_token=super-secret\n" + + " at provider (/srv/omniroute/runtime.ts:1:1)"; + const stream = streamJsonlToOpenAi( + jsonlBody([ + JSON.stringify({ type: "stream", token: "partial answer" }), + JSON.stringify({ type: "status", status: "error", message: rawError }), + ]), + "test/huggingchat-model", + "chatcmpl-huggingchat-test", + 1_725_000_000 + ); + + const roleChunk = await stream.next(); + const contentChunk = await stream.next(); + + assert.equal(roleChunk.done, false); + assert.match(roleChunk.value || "", /"role":"assistant"/); + assert.equal(contentChunk.done, false); + assert.match(contentChunk.value || "", /partial answer/); + await assert.rejects(() => stream.next(), HuggingChatStreamError); +}); + +test("HuggingChat partial failures reach stream finalization, persistence, and fallback", async () => { + const model = `test/huggingchat-model-${fixtureRunId}`; + const provider = "huggingchat"; + const connectionId = `huggingchat-stream-error-boundary-${fixtureRunId}`; + const publicErrorMessage = "HuggingChat generation failed"; + const rawError = + "generation failed at /srv/omniroute/providers/huggingchat.ts:44:9 access_token=super-secret\n" + + " at provider (/srv/omniroute/runtime.ts:1:1)"; + const realFetch = globalThis.fetch; + let callCount = 0; + const errorLogs: string[] = []; + + globalThis.fetch = (async () => { + callCount += 1; + if (callCount === 1) return Response.json({ conversationId: "conversation-test" }); + if (callCount === 2) return Response.json({ rootMessageId: "root-message-test" }); + if (callCount === 3) { + return new Response( + jsonlBody([ + JSON.stringify({ type: "stream", token: "partial answer" }), + JSON.stringify({ type: "status", status: "error", message: rawError }), + ]), + { status: 200, headers: { "Content-Type": "application/jsonl" } } + ); + } + throw new Error(`Unexpected fetch call ${callCount}`); + }) as typeof globalThis.fetch; + + usageHistory.clearPendingRequests(); + assert.equal(usageHistory.getPendingById().size, 0, "the child must start without pending state"); + assert.equal( + usageHistory.getCompletedDetails().size, + 0, + "the child must start without completed state" + ); + const previousPersistence = coreDb + .getDbInstance() + .prepare("SELECT COUNT(*) AS count FROM call_logs WHERE connection_id = ? AND model = ?") + .get(connectionId, model) as { count: number }; + assert.equal(previousPersistence.count, 0, "the child must not reuse a prior persisted identity"); + const requestId = usageHistory.trackPendingRequest(model, provider, connectionId, true); + assert.ok(requestId, "the full-pipeline test must own a real pending request"); + + type CompletionPayload = { + status: number; + usage: unknown; + providerPayload?: unknown; + clientPayload?: unknown; + error?: string | null; + errorCode?: string | null; + }; + type FailurePayload = { + status: number; + message: string; + code?: string; + type?: string; + }; + + let completionPayload: CompletionPayload | null = null; + let streamCompletionRecorded = false; + let streamFailureCompletionRecorded = false; + const persistedFailures: Array<{ status: number; errorCode?: string }> = []; + const fallbackFailures: FailurePayload[] = []; + + const onStreamComplete = (payload: CompletionPayload) => { + const normalizedStatus = payload.status || 200; + if (streamCompletionRecorded) return; + streamCompletionRecorded = true; + if (normalizedStatus !== 200) { + if (streamFailureCompletionRecorded) return; + streamFailureCompletionRecorded = true; + } + completionPayload = payload; + finalizeStreamRequestLog({ + pendingRequestId: requestId, + model, + provider, + connectionId, + providerResponse: payload.providerPayload, + clientResponse: payload.clientPayload, + status: normalizedStatus, + error: payload.error, + errorCode: payload.errorCode, + }); + }; + + const { handleStreamFailure, onPipelineStreamError } = createStreamFailureFinalizers({ + isFailureCompletionRecorded: () => streamFailureCompletionRecorded, + isStreamCompletionRecorded: () => streamCompletionRecorded, + onStreamComplete, + persistFailureUsage: (status, errorCode) => { + persistedFailures.push({ status, errorCode }); + }, + onStreamFailure: (failure) => { + fallbackFailures.push(failure); + }, + }); + + try { + const result = await new HuggingChatExecutor().execute({ + model, + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "hf-chat=fake-cookie" }, + signal: null, + log: { error: (_tag, message) => errorLogs.push(message) }, + }); + + assert.equal(callCount, 3, "the test must intercept every HuggingChat request"); + assert.equal(result.response.status, 200, "partial output has already committed HTTP 200"); + const readiness = await ensureStreamReadiness(result.response, { + timeoutMs: 1_000, + provider, + model, + }); + if (!readiness.ok) assert.fail(`unexpected readiness failure: ${readiness.reason}`); + + const transform = createPassthroughStreamWithLogger( + provider, + null, + null, + model, + connectionId, + { messages: [{ role: "user", content: "hello" }] }, + onStreamComplete, + null, + handleStreamFailure, + FORMATS.OPENAI + ); + const streamController = createStreamController({ + onError: onPipelineStreamError, + provider, + model, + connectionId, + clientResponseFormat: FORMATS.OPENAI, + }); + const clientStream = pipeWithDisconnect(readiness.response, transform, streamController, { + stallTimeoutMs: 0, + }); + const wire = await new Response(clientStream).text(); + + assert.match(wire, /partial answer/); + assert.match(wire, /"finish_reason":"error"/); + assert.match(wire, new RegExp(publicErrorMessage)); + assert.match(wire, /data: \[DONE\]/); + assert.doesNotMatch(wire, /"finish_reason":"stop"/); + assert.doesNotMatch(wire, /\/srv\/omniroute/); + assert.doesNotMatch(wire, /super-secret/); + assert.equal(errorLogs.length, 1); + assert.doesNotMatch(errorLogs[0], /\/srv\/omniroute/); + assert.doesNotMatch(errorLogs[0], /super-secret/); + assert.doesNotMatch(errorLogs[0], /\n\s*at /); + + assert.ok(completionPayload, "the pipeline must record a terminal failure"); + assert.equal(completionPayload.status, 502); + assert.equal(completionPayload.error, publicErrorMessage); + assert.equal(completionPayload.errorCode, "stream_pipeline_error"); + assert.deepEqual(persistedFailures, [{ status: 502, errorCode: "stream_pipeline_error" }]); + assert.deepEqual(fallbackFailures, [ + { + status: 502, + message: publicErrorMessage, + code: "stream_pipeline_error", + type: "stream_error", + }, + ]); + + assert.equal(usageHistory.getPendingById().has(requestId), false); + const completedDetail = usageHistory.getCompletedDetails().get(requestId); + assert.ok(completedDetail, "failure finalization must persist the completed request detail"); + assert.equal(completedDetail.status, 502); + assert.equal(completedDetail.error, publicErrorMessage); + assert.equal(completedDetail.errorCode, "stream_pipeline_error"); + assert.doesNotMatch(JSON.stringify(completedDetail), /\/srv\/omniroute|super-secret/); + assert.deepEqual( + [...usageHistory.getCompletedDetails().keys()], + [requestId], + "only this child run may own completed usage state" + ); + } finally { + globalThis.fetch = realFetch; + usageHistory.clearPendingRequests(); + } +}); + +test("HuggingChat reports an authoritative error without waiting for transport cancellation", async () => { + let cancelCalled = false; + const encoded = new TextEncoder().encode( + `${JSON.stringify({ type: "status", status: "error", message: "provider failed" })}\n` + ); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoded); + }, + cancel() { + cancelCalled = true; + return new Promise(() => undefined); + }, + }); + const stream = streamJsonlToOpenAi( + body, + "test/huggingchat-model", + "chatcmpl-huggingchat-test", + 1_725_000_000 + ); + + const outcome = await Promise.race([ + stream.next().then( + () => ({ kind: "resolved" as const }), + (error: unknown) => ({ kind: "rejected" as const, error }) + ), + new Promise<{ kind: "hung" }>((resolve) => { + setImmediate(() => resolve({ kind: "hung" })); + }), + ]); + + assert.equal(cancelCalled, true); + assert.equal(outcome.kind, "rejected", "transport cleanup must not delay error delivery"); + assert.ok( + outcome.kind === "rejected" && outcome.error instanceof HuggingChatStreamError, + "the authoritative HuggingChat error must remain classifiable" + ); +}); + +test("HuggingChat cancellation suppresses final chunks after a pending JSONL read", async () => { + let upstreamCancelCalled = false; + let upstreamPullCount = 0; + const cancellationController = new AbortController(); + const token = new TextEncoder().encode( + `${JSON.stringify({ type: "stream", token: "partial answer" })}\n` + ); + const body = new ReadableStream({ + pull(controller) { + upstreamPullCount += 1; + if (upstreamPullCount === 1) { + controller.enqueue(token); + return; + } + return new Promise(() => undefined); + }, + cancel() { + upstreamCancelCalled = true; + return new Promise(() => undefined); + }, + }); + const stream = streamJsonlToOpenAi( + body, + "test/huggingchat-model", + "chatcmpl-huggingchat-test", + 1_725_000_000, + null, + cancellationController.signal + ); + + assert.match((await stream.next()).value || "", /"role":"assistant"/); + assert.match((await stream.next()).value || "", /partial answer/); + const pendingNext = stream.next(); + await new Promise((resolve) => setImmediate(resolve)); + cancellationController.abort(); + + const outcome = await Promise.race([ + pendingNext.then((result) => ({ kind: "settled" as const, result })), + new Promise<{ kind: "hung" }>((resolve) => setImmediate(() => resolve({ kind: "hung" }))), + ]); + + assert.equal(upstreamCancelCalled, true); + assert.equal(outcome.kind, "settled", "cancellation must settle the pending generator read"); + assert.equal( + outcome.kind === "settled" ? outcome.result.done : false, + true, + "a cancelled generator must not emit stop or [DONE]" + ); + void stream.return(undefined).catch(() => undefined); +}); + +test("HuggingChat client cancellation reaches a blocked upstream reader without waiting", async () => { + const realFetch = globalThis.fetch; + let callCount = 0; + let upstreamCancelCalled = false; + let upstreamPullCount = 0; + const errorLogs: string[] = []; + const token = new TextEncoder().encode( + `${JSON.stringify({ type: "stream", token: "partial answer" })}\n` + ); + const blockedBody = new ReadableStream({ + pull(controller) { + upstreamPullCount += 1; + if (upstreamPullCount === 1) { + controller.enqueue(token); + return; + } + return new Promise(() => undefined); + }, + cancel() { + upstreamCancelCalled = true; + return new Promise(() => undefined); + }, + }); + + globalThis.fetch = (async () => { + callCount += 1; + if (callCount === 1) return Response.json({ conversationId: "conversation-test" }); + if (callCount === 2) return Response.json({ rootMessageId: "root-message-test" }); + if (callCount === 3) { + return new Response(blockedBody, { + status: 200, + headers: { "Content-Type": "application/jsonl" }, + }); + } + throw new Error(`Unexpected fetch call ${callCount}`); + }) as typeof globalThis.fetch; + + try { + const result = await new HuggingChatExecutor().execute({ + model: "test/huggingchat-model", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: true, + credentials: { apiKey: "hf-chat=fake-cookie" }, + signal: null, + log: { error: (_tag, message) => errorLogs.push(message) }, + }); + + assert.equal(callCount, 3, "the test must intercept every HuggingChat request"); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + const roleChunk = await reader.read(); + const contentChunk = await reader.read(); + assert.match(new TextDecoder().decode(roleChunk.value), /"role":"assistant"/); + assert.match(new TextDecoder().decode(contentChunk.value), /partial answer/); + + const blockedRead = reader.read(); + await new Promise((resolve) => setImmediate(resolve)); + const cancelOutcome = await Promise.race([ + reader.cancel("client disconnected").then(() => "resolved" as const), + new Promise<"hung">((resolve) => setImmediate(() => resolve("hung"))), + ]); + void blockedRead.catch(() => undefined); + + assert.equal(cancelOutcome, "resolved", "downstream cancellation must remain non-blocking"); + assert.equal(upstreamCancelCalled, true, "cancellation must reach the locked upstream reader"); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(errorLogs, [], "client cancellation must not log a provider stream failure"); + } finally { + globalThis.fetch = realFetch; + } +}); + +test("HuggingChat keeps the normal JSONL completion contract unchanged", async () => { + const output = await collectStream( + jsonlBody([ + JSON.stringify({ type: "stream", token: "complete answer" }), + JSON.stringify({ type: "status", status: "finished" }), + ]) + ); + + assert.match(output, /"role":"assistant"/); + assert.match(output, /complete answer/); + assert.match(output, /"finish_reason":"stop"/); + assert.match(output, /data: \[DONE\]/); + assert.doesNotMatch(output, /"error":\{/); +}); diff --git a/tests/unit/_fixtures/huggingchat-transport-error-child.ts b/tests/unit/_fixtures/huggingchat-transport-error-child.ts new file mode 100644 index 0000000000..2a904451d9 --- /dev/null +++ b/tests/unit/_fixtures/huggingchat-transport-error-child.ts @@ -0,0 +1,85 @@ +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const CHILD_RESULT_PREFIX = "HUGGINGCHAT_TRANSPORT_RESULT="; +const scenario = process.argv[2]; + +if (scenario !== "conversation-creation" && scenario !== "message-send") { + throw new Error(`Unknown HuggingChat transport scenario: ${String(scenario)}`); +} + +const testRoot = mkdtempSync(join(tmpdir(), "omniroute-huggingchat-transport-child-")); +const testDataDir = join(testRoot, "data"); +const testPluginsDir = join(testRoot, "plugins"); +const testConfigDir = join(testRoot, "config"); + +mkdirSync(testDataDir, { recursive: true }); +mkdirSync(testPluginsDir, { recursive: true }); +mkdirSync(testConfigDir, { recursive: true }); +process.env.DATA_DIR = testDataDir; +process.env.OMNIROUTE_PLUGINS_DIR = testPluginsDir; +process.env.XDG_CONFIG_HOME = testConfigDir; +process.env.APP_LOG_TO_FILE = "false"; +process.env.API_KEY_SECRET = "synthetic-huggingchat-transport-test-key"; + +const hostileTransportMessage = + "TLS request failed at /srv/omniroute/providers/huggingchat/client.ts:44:9 " + + "access_token=transport-secret\n" + + " at sendRequest (/srv/omniroute/runtime/fetch.ts:12:3)"; + +const originalFetch = globalThis.fetch; +let fetchCalls = 0; +const errorLogs: string[] = []; +let childResult: Record | null = null; + +try { + const { HuggingChatExecutor } = await import("../../../open-sse/executors/huggingchat.ts"); + + globalThis.fetch = (async () => { + fetchCalls += 1; + + if (scenario === "conversation-creation") { + throw new Error(hostileTransportMessage); + } + + if (fetchCalls === 1) { + return Response.json({ conversationId: "conversation-test" }); + } + if (fetchCalls === 2) { + return Response.json({ rootMessageId: "root-message-test" }); + } + if (fetchCalls === 3) { + throw new Error(hostileTransportMessage); + } + throw new Error(`Unexpected fetch call ${fetchCalls}`); + }) as typeof globalThis.fetch; + + const result = await new HuggingChatExecutor().execute({ + model: "test/huggingchat-model", + body: { messages: [{ role: "user", content: "hello" }] }, + stream: false, + credentials: { apiKey: "hf-chat=fake-cookie" }, + signal: null, + log: { error: (_tag, message) => errorLogs.push(message) }, + }); + + childResult = { + fetchCalls, + status: result.response.status, + contentType: result.response.headers.get("content-type") || "", + errorLogs, + payload: await result.response.json(), + }; +} finally { + globalThis.fetch = originalFetch; + const coreDb = await import("../../../src/lib/db/core.ts"); + coreDb.resetDbInstance(); + rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +} + +if (!childResult) { + throw new Error(`HuggingChat ${scenario} probe did not produce a result`); +} + +process.stdout.write(`${CHILD_RESULT_PREFIX}${JSON.stringify(childResult)}\n`); diff --git a/tests/unit/adapta-web-nonstream-error-boundary.test.ts b/tests/unit/adapta-web-nonstream-error-boundary.test.ts new file mode 100644 index 0000000000..ef5c5894f4 --- /dev/null +++ b/tests/unit/adapta-web-nonstream-error-boundary.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const FIXTURE = fileURLToPath( + new URL("../fixtures/adapta-web-nonstream-error-boundary.fixture.ts", import.meta.url) +); + +const CHILD_RUNTIME_ENV_KEYS = [ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "SystemRoot", + "ComSpec", + "PATHEXT", + "LANG", + "LC_ALL", + "TZ", +] as const; + +function buildFixtureEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + NODE_ENV: "test", + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: "adapta-boundary-fixture-api-key-secret-20260902", + DISABLE_SQLITE_AUTO_BACKUP: "true", + }; + + for (const key of CHILD_RUNTIME_ENV_KEYS) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + + // A nested test runner must receive its own context instead of inheriting the parent's. + delete env.NODE_TEST_CONTEXT; + return env; +} + +test("Adapta Web non-stream error boundaries pass in an isolated process", () => { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx/esm", + "--import", + "./open-sse/utils/setupPolyfill.ts", + "--test", + "--test-force-exit", + FIXTURE, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: buildFixtureEnv(), + timeout: 60_000, + } + ); + + assert.ifError(result.error); + assert.equal( + result.signal, + null, + `isolated Adapta boundary fixture terminated by ${result.signal}\n${result.stdout}\n${result.stderr}` + ); + assert.equal( + result.status, + 0, + `isolated Adapta boundary fixture failed\n${result.stdout}\n${result.stderr}` + ); +}); diff --git a/tests/unit/bifrost-relay-response-leak-9m72.test.ts b/tests/unit/bifrost-relay-response-leak-9m72.test.ts new file mode 100644 index 0000000000..c5affaf655 --- /dev/null +++ b/tests/unit/bifrost-relay-response-leak-9m72.test.ts @@ -0,0 +1,103 @@ +/** + * GHSA-9m72-44hg-w32g — the standalone bifrost relay route copied ALL upstream + * response headers (`new Headers(upstream.headers)`) and returned non-2xx bodies + * verbatim, while its TypeScript sibling routed non-2xx through + * parseUpstreamError + buildErrorBody + stripStaleEncodingHeaders. + * + * The relay sends `Authorization: Bearer ${BIFROST_API_KEY}` to the sidecar, so + * anything the sidecar (or a further upstream) echoes back — that header, its own + * `set-cookie`, an `x-api-key` — reached the relay-token holder untouched. + * + * Both relay routes copy upstream headers, so the strip is a shared helper used + * by both rather than a fix in one and a second copy waiting to drift (the + * failure mode of GHSA-v7g9 and GHSA-qv45). + * + * Run with: + * node --import tsx/esm --test tests/unit/bifrost-relay-response-leak-9m72.test.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { stripSensitiveResponseHeaders } from "../../open-sse/utils/upstreamResponseHeaders.ts"; + +const read = (rel: string) => + readFileSync(fileURLToPath(new URL(`../../${rel}`, import.meta.url)), "utf8"); + +const BIFROST_ROUTE = "src/app/api/v1/relay/chat/completions/bifrost/route.ts"; +const TS_ROUTE = "src/app/api/v1/relay/chat/completions/route.ts"; + +describe("stripSensitiveResponseHeaders", () => { + it("drops credentials and cookies the upstream echoed back", () => { + const upstream = new Headers([ + ["authorization", "Bearer sidecar-secret"], + ["x-api-key", "op-key"], + ["x-goog-api-key", "goog-key"], + ["api-key", "azure-key"], + ["cookie", "session=abc"], + ["set-cookie", "session=abc; HttpOnly"], + ["proxy-authorization", "Basic zzz"], + ["content-type", "application/json"], + ["x-request-id", "keep-me"], + ]); + const out = stripSensitiveResponseHeaders(upstream); + for (const gone of [ + "authorization", + "x-api-key", + "x-goog-api-key", + "api-key", + "cookie", + "set-cookie", + "proxy-authorization", + ]) { + assert.equal(out.get(gone), null, `${gone} survived`); + } + assert.equal(out.get("content-type"), "application/json"); + assert.equal(out.get("x-request-id"), "keep-me"); + }); + + it("also drops the stale framing headers", () => { + const out = stripSensitiveResponseHeaders( + new Headers([ + ["content-encoding", "gzip"], + ["content-length", "123"], + ["transfer-encoding", "chunked"], + ["x-keep", "yes"], + ]) + ); + assert.equal(out.get("content-encoding"), null); + assert.equal(out.get("content-length"), null); + assert.equal(out.get("transfer-encoding"), null); + assert.equal(out.get("x-keep"), "yes"); + }); + + it("does not mutate the input Headers", () => { + const input = new Headers([["authorization", "Bearer x"]]); + stripSensitiveResponseHeaders(input); + assert.equal(input.get("authorization"), "Bearer x"); + }); +}); + +describe("both relay routes use the shared strip (GHSA-9m72-44hg-w32g)", () => { + for (const route of [BIFROST_ROUTE, TS_ROUTE]) { + it(`${route} strips sensitive upstream response headers`, () => { + const src = read(route); + assert.ok( + src.includes("stripSensitiveResponseHeaders"), + `${route} relays upstream headers verbatim — a sidecar-echoed credential reaches the caller` + ); + assert.ok( + !/new Headers\(upstream\.headers\)/.test(src), + `${route} still copies upstream headers wholesale` + ); + }); + } + + it("the bifrost route normalizes non-2xx through the error sanitizer", () => { + const src = read(BIFROST_ROUTE); + assert.ok(src.includes("buildErrorBody"), "bifrost non-2xx must not be relayed verbatim"); + assert.ok(src.includes("sanitizeErrorMessage"), "bifrost non-2xx must be sanitized (HR#12)"); + }); +}); diff --git a/tests/unit/cache-config-preserve-client-cache.test.ts b/tests/unit/cache-config-preserve-client-cache.test.ts new file mode 100644 index 0000000000..ab2b3c825e --- /dev/null +++ b/tests/unit/cache-config-preserve-client-cache.test.ts @@ -0,0 +1,82 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// Regression guard: PUT /api/settings/cache-config persisted +// `alwaysPreserveClientCache` into the databaseSettings "cache" section, but +// the runtime (getCacheControlSettings → getSettings) reads the FLAT general +// settings key — so the endpoint accepted the value, GET echoed it back, and +// the router never changed behavior. This test proves the value written +// through the route is the one the cache-control policy actually consumes. + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cache-config-flat-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../src/lib/db/core.ts"); + +function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +function makeJsonRequest(method: string, body?: unknown): Request { + return new Request("http://localhost/api/settings/cache-config", { + method, + headers: { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); +} + +test.beforeEach(() => { + resetStorage(); +}); + +test.after(() => { + resetStorage(); +}); + +test("alwaysPreserveClientCache set via cache-config reaches the runtime read path", async (t) => { + const cacheConfigRoute = await import("../../src/app/api/settings/cache-config/route.ts"); + const { getSettings } = await import("../../src/lib/db/settings.ts"); + const { getCacheControlSettings, invalidateCacheControlSettingsCache } = + await import("../../src/lib/cacheControlSettings.ts"); + + await t.test("PUT persists to the flat settings the runtime reads", async () => { + const putResponse = await cacheConfigRoute.PUT( + makeJsonRequest("PUT", { alwaysPreserveClientCache: "always" }) as never + ); + assert.equal(putResponse.status, 200); + + // The runtime read path: getCacheControlSettings() → getSettings() (flat). + // RED before the fix: the route wrote databaseSettings "cache" instead, + // so both of these still reported the default "auto". + const flatSettings = await getSettings(); + assert.equal(flatSettings.alwaysPreserveClientCache, "always"); + + invalidateCacheControlSettingsCache(); + assert.equal(await getCacheControlSettings(), "always"); + }); + + await t.test("GET reports the flat value, not the ignored cache-section copy", async () => { + // Seed a stale value in the databaseSettings "cache" section — the store + // the runtime never reads. GET must not surface it. + const { updateDatabaseSettings } = await import("../../src/lib/db/databaseSettings.ts"); + updateDatabaseSettings({ + cache: { alwaysPreserveClientCache: "never" }, + } as Parameters[0]); + + const putResponse = await cacheConfigRoute.PUT( + makeJsonRequest("PUT", { alwaysPreserveClientCache: "always" }) as never + ); + assert.equal(putResponse.status, 200); + + const getResponse = await cacheConfigRoute.GET(makeJsonRequest("GET") as never); + const body = await getResponse.json(); + assert.equal(getResponse.status, 200); + assert.equal(body.alwaysPreserveClientCache, "always"); + }); +}); diff --git a/tests/unit/calllogs-format-split.test.ts b/tests/unit/calllogs-format-split.test.ts index a88f86c960..4bb267bfab 100644 --- a/tests/unit/calllogs-format-split.test.ts +++ b/tests/unit/calllogs-format-split.test.ts @@ -17,7 +17,6 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; - import { asRecord, toNumber, @@ -96,6 +95,15 @@ describe("callLogs/format — toStoredErrorSummary", () => { assert.ok(out.includes("kaboom")); assert.ok(out.includes("message")); }); + it("removes credentials, filesystem paths, and stack frames before persistence", () => { + const out = toStoredErrorSummary( + "Provider failed access_token=persisted-secret at /srv/private/provider.json\n" + + " at dispatch (/srv/private/dispatcher.ts:42:7)" + ); + + assert.equal(typeof out, "string"); + assert.doesNotMatch(out, /persisted-secret|srv\/private|dispatcher\.ts|\bat dispatch\b/i); + }); }); describe("callLogs/format — buildRequestSummary", () => { diff --git a/tests/unit/cerebras-free-tier-11773.test.ts b/tests/unit/cerebras-free-tier-11773.test.ts new file mode 100644 index 0000000000..1d1f0881b7 --- /dev/null +++ b/tests/unit/cerebras-free-tier-11773.test.ts @@ -0,0 +1,46 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { APIKEY_PROVIDERS } from "../../src/shared/constants/providers/apikey/index.ts"; +import { getProviderById } from "../../src/shared/constants/providers.ts"; +import { FREE_MODEL_BUDGETS } from "../../open-sse/config/freeModelCatalog.ts"; +import { FREE_TIER_BUDGETS } from "../../open-sse/config/freeTierCatalog.ts"; +import { LEGACY_FREE_PROVIDERS } from "../../open-sse/services/tierConfig.ts"; +import { classifyTier, clearTierCache } from "../../open-sse/services/tierResolver.ts"; +import { PROVIDER_TIER } from "../../open-sse/services/tierTypes.ts"; + +const CEREBRAS_MODELS = ["zai-glm-4.7", "gpt-oss-120b"] as const; + +test("#11773 cerebras stays catalogued, but not as a recurring zero-cost tier", () => { + const entry = APIKEY_PROVIDERS.cerebras; + assert.ok(entry, "APIKEY_PROVIDERS.cerebras must remain registered"); + assert.equal(entry.hasFree, true); + assert.equal(Object.hasOwn(FREE_TIER_BUDGETS, "cerebras"), false); + assert.equal(LEGACY_FREE_PROVIDERS.includes("cerebras"), false); +}); + +test("#11773 cerebras freeNote describes the $5 card-gated signup credit", () => { + const note = getProviderById("cerebras")?.freeNote ?? ""; + assert.match(note, /\$5/); + assert.match(note, /30.?day|30 days/i); + assert.match(note, /payment method|credit card/i); + assert.equal(/1M tokens\/day|30K TPM/.test(note), false); +}); + +test("#11773 cerebras catalog rows are one-time signup credits, not a hard-stop free trial", () => { + const rows = FREE_MODEL_BUDGETS.filter((row) => row.provider === "cerebras"); + assert.ok(rows.length >= CEREBRAS_MODELS.length, "catalog must keep the live Cerebras models"); + for (const modelId of CEREBRAS_MODELS) { + const row = rows.find((entry) => entry.modelId === modelId); + assert.ok(row, `missing catalog row for ${modelId}`); + assert.equal(row.freeType, "one-time-initial"); + assert.equal(row.monthlyTokens, 0); + assert.notEqual(row.hardStopGuaranteed, true); + } +}); + +test("#11773 cerebras is not classified as the free routing tier", () => { + clearTierCache(); + const result = classifyTier("cerebras", "zai-glm-4.7"); + assert.notEqual(result.tier, PROVIDER_TIER.FREE); +}); diff --git a/tests/unit/chatcore-stream-error-result.test.ts b/tests/unit/chatcore-stream-error-result.test.ts index 352877046a..3ebe5821e7 100644 --- a/tests/unit/chatcore-stream-error-result.test.ts +++ b/tests/unit/chatcore-stream-error-result.test.ts @@ -43,6 +43,23 @@ test("createStreamingErrorResult attaches optional code and type", async () => { assert.equal(json.error.type, "rate_limit_error"); }); +test("createStreamingErrorResult sanitizes code and type at the SSE boundary", async () => { + const result = createStreamingErrorResult( + 502, + "upstream failed", + "sk-live-secret-value", + "server_error\nX-Leak: yes" + ); + const body = await result.response.text(); + const json = JSON.parse(body.slice("data: ".length, body.indexOf("\n\n"))) as { + error: { code: string; type: string }; + }; + + assert.equal(json.error.code, "bad_gateway"); + assert.equal(json.error.type, "server_error"); + assert.doesNotMatch(body, /sk-live-secret-value|X-Leak/); +}); + test("getUpstreamErrorIdentifier returns a non-empty string code or undefined", () => { assert.equal(getUpstreamErrorIdentifier({ code: "ECONNRESET" }), "ECONNRESET"); assert.equal(getUpstreamErrorIdentifier({ code: "" }), undefined); diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index 861f716251..c0125fff83 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -4,8 +4,15 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-chatcore-translation-")); +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-chatcore-translation-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const settingsDb = await import("../../src/lib/db/settings.ts"); @@ -448,7 +455,11 @@ test.after(async () => { resetAccountSemaphores(); await flushAsyncSideEffects(); await resetStorage(); - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); test("chatCore times out upstream execution before provider response headers", async () => { // This test asserts pendingDetail.providerRequest — only attached when the @@ -1938,35 +1949,12 @@ test("chatCore surfaces translation errors with explicit status codes", async () FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, () => { - const error = new Error("responses translator rejected the payload"); - error.statusCode = 409; - throw error; - }, - null - ); - - const { result } = await invokeChatCore({ - provider: "openai", - model: "gpt-4o-mini", - endpoint: "/v1/responses", - body: { - model: "gpt-4o-mini", - input: "hello", - }, - }); - - assert.equal(result.success, false); - assert.equal(result.status, 409); - assert.equal(result.error, "responses translator rejected the payload"); -}); -test("chatCore surfaces typed translation errors with the declared error type", async () => { - register( - FORMATS.OPENAI_RESPONSES, - FORMATS.OPENAI, - () => { - const error = new Error("typed translator failure"); + const error = new Error( + "translator rejected access_token=translation-secret at /srv/private/translator.ts\n" + + " at translate (/srv/private/translator.ts:41:8)" + ); error.statusCode = 422; - error.errorType = "unsupported_feature"; + error.errorType = "unsupported_feature access_token=type-secret /srv/private/type.ts"; throw error; }, null @@ -1984,10 +1972,16 @@ test("chatCore surfaces typed translation errors with the declared error type", assert.equal(result.success, false); assert.equal(result.status, 422); - - const payload = (await result.response.json()) as any; - assert.equal(payload.error.type, "unsupported_feature"); - assert.equal(payload.error.code, "unsupported_feature"); + const payload = (await result.response.json()) as { + error: { message: string; type: string; code: string }; + }; + assert.equal(payload.error.type, "invalid_request_error"); + assert.equal(payload.error.code, ""); + assert.match(payload.error.message, /translator rejected/); + assert.doesNotMatch( + JSON.stringify({ payload, internalError: result.error }), + /translation-secret|type-secret|srv\/private|translator\.ts|type\.ts|\bat translate\b/i + ); }); test("chatCore returns 500 when translation throws a generic error", async () => { register( diff --git a/tests/unit/check-docs-counts-sync.test.ts b/tests/unit/check-docs-counts-sync.test.ts index 5f786f327a..2cea8b52db 100644 --- a/tests/unit/check-docs-counts-sync.test.ts +++ b/tests/unit/check-docs-counts-sync.test.ts @@ -485,8 +485,8 @@ const TRAINING_CLAIM = { }; test("the hard-stop claim passes on the real sentence and fails on a stale count", () => { - const v = makeValidator(7, HARD_STOP_CLAIM); - assert.equal(v("7 entries carry an independently documented hard stop, and").ok, true); + const v = makeValidator(5, HARD_STOP_CLAIM); + assert.equal(v("5 entries carry an independently documented hard stop, and").ok, true); assert.equal(v("99 entries carry an independently documented hard stop, and").ok, false); }); @@ -500,10 +500,10 @@ test("the training claim passes on the real sentence and fails on a stale count" test("a reworded or deleted sentence fails, instead of passing as absent", () => { // The gate's real failure mode is not a stale number, it is silence: reword the // sentence past the pattern and "no claim in this file" used to read green. - const required = makeValidator(7, { ...HARD_STOP_CLAIM, requireClaim: true }); - assert.equal(required("7 entries have a provider-documented hard-stop guarantee.").ok, false); + const required = makeValidator(5, { ...HARD_STOP_CLAIM, requireClaim: true }); + assert.equal(required("5 entries have a provider-documented hard-stop guarantee.").ok, false); assert.equal(required("the page no longer mentions it at all").ok, false); - assert.equal(required("7 entries carry an independently documented hard stop.").ok, true); + assert.equal(required("5 entries carry an independently documented hard stop.").ok, true); const trainingRequired = makeValidator(13, { ...TRAINING_CLAIM, requireClaim: true }); assert.equal(trainingRequired("13 entries disclose training use.").ok, false); @@ -514,7 +514,7 @@ test("the live page actually satisfies both required gates", () => { // A unit test on synthetic strings proves the validator; this one proves the // document. Without it, the two could drift apart and both stay green. const page = readFileSync(path.resolve(here, "../../docs/reference/FREE_TIERS.md"), "utf8"); - assert.equal(makeValidator(7, { ...HARD_STOP_CLAIM, requireClaim: true })(page).ok, true); + assert.equal(makeValidator(5, { ...HARD_STOP_CLAIM, requireClaim: true })(page).ok, true); assert.equal(makeValidator(13, { ...TRAINING_CLAIM, requireClaim: true })(page).ok, true); }); diff --git a/tests/unit/codex-response-failed-boundary.test.ts b/tests/unit/codex-response-failed-boundary.test.ts new file mode 100644 index 0000000000..51f8ffe59e --- /dev/null +++ b/tests/unit/codex-response-failed-boundary.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const FIXTURE = fileURLToPath( + new URL("../fixtures/codex-response-failed-boundary.fixture.ts", import.meta.url) +); + +const CHILD_RUNTIME_ENV_KEYS = [ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "SystemRoot", + "ComSpec", + "PATHEXT", + "LANG", + "LC_ALL", + "TZ", +] as const; + +function buildFixtureEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + NODE_ENV: "test", + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: "codex-boundary-fixture-api-key-secret-20260902", + DISABLE_SQLITE_AUTO_BACKUP: "true", + }; + + for (const key of CHILD_RUNTIME_ENV_KEYS) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + + // A nested test runner must receive its own context instead of inheriting the parent's. + delete env.NODE_TEST_CONTEXT; + return env; +} + +test("Codex public failure boundaries pass in an isolated process", () => { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx/esm", + "--import", + "./open-sse/utils/setupPolyfill.ts", + "--test", + "--test-force-exit", + FIXTURE, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: buildFixtureEnv(), + timeout: 60_000, + } + ); + + assert.ifError(result.error); + assert.equal( + result.signal, + null, + `isolated Codex boundary fixture terminated by ${result.signal}\n${result.stdout}\n${result.stderr}` + ); + assert.equal( + result.status, + 0, + `isolated Codex boundary fixture failed\n${result.stdout}\n${result.stderr}` + ); +}); diff --git a/tests/unit/combo-diagnostics-trace.test.ts b/tests/unit/combo-diagnostics-trace.test.ts index fe8bb546a4..dc1354f7ca 100644 --- a/tests/unit/combo-diagnostics-trace.test.ts +++ b/tests/unit/combo-diagnostics-trace.test.ts @@ -9,9 +9,9 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { errorResponseWithComboDiagnostics, sanitizeComboDiagnostics } = await import( - "../../open-sse/utils/error.ts" -); +const { errorResponseWithComboDiagnostics, sanitizeComboDiagnostics } = + await import("../../open-sse/utils/error.ts"); +const { buildRecoveryHint } = await import("../../open-sse/services/combo/pinRecovery.ts"); test("combo diagnostics: headers + body carry the sanitized trace (code override preserved)", async () => { const res = errorResponseWithComboDiagnostics( @@ -89,7 +89,9 @@ test("combo diagnostics: terminalReason with a non-Latin1 char (em dash) must no { poolSize: 4, attempted: 1, - excluded: [{ provider: "deepseek", model: "deepseek-v4-flash-free", reason: "quality — bad" }], + excluded: [ + { provider: "deepseek", model: "deepseek-v4-flash-free", reason: "quality — bad" }, + ], attemptOrder: [{ provider: "deepseek", model: "deepseek-v4-flash-free" }], terminalReason, } @@ -112,8 +114,43 @@ test("combo diagnostics: JSON body keeps the original non-Latin1 text even thoug } ); // Header value must be a valid Latin1 ByteString — em dash (U+2014) replaced. - assert.equal(res.headers.get("x-omniroute-combo-terminal-reason"), terminalReason.replace("—", "?")); + assert.equal( + res.headers.get("x-omniroute-combo-terminal-reason"), + terminalReason.replace("—", "?") + ); const body = await res.json(); // JSON body keeps the original, readable (unsanitized) em dash. assert.equal(body.diagnostics.terminalReason, terminalReason); }); + +test("combo diagnostics preserve every canonical recovery hint up to the existing cap", async () => { + const reasons = [ + "reasoning_budget_exhausted", + "max_attempts_exceeded", + "all_accounts_inactive", + "quota_exhausted", + "all_models_failed", + "no_executable_targets", + "context_requirements_exhausted", + "all_targets_skipped", + "unknown_reason", + ]; + + for (const reason of reasons) { + const recovery = buildRecoveryHint(reason, 30); + const response = errorResponseWithComboDiagnostics(503, "combo failed", { + poolSize: 1, + attempted: 1, + excluded: [], + attemptOrder: [], + terminalReason: reason, + recovery, + }); + const body = (await response.json()) as { + recovery_hint?: { action: string; next_step: string }; + }; + + assert.equal(body.recovery_hint?.action, recovery.action, reason); + assert.equal(body.recovery_hint?.next_step, recovery.next_step.slice(0, 200), reason); + } +}); diff --git a/tests/unit/dashboard-request-failed-redaction.test.ts b/tests/unit/dashboard-request-failed-redaction.test.ts new file mode 100644 index 0000000000..e039f7c0dc --- /dev/null +++ b/tests/unit/dashboard-request-failed-redaction.test.ts @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +import type { RequestFailedPayload } from "../../src/lib/events/types.ts"; + +const RESULT_PREFIX = "DASHBOARD_FAILURE_PROBE_RESULT="; +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); +const probePath = fileURLToPath( + new URL("../fixtures/dashboard-request-failed-redaction-probe.ts", import.meta.url) +); + +type ProbeResult = { + delivered: RequestFailedPayload; + replayMatches: boolean; + internalRawPreserved: boolean; + writerDrained: boolean; +}; + +function runProbe(env: NodeJS.ProcessEnv): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile( + process.execPath, + ["--import", "tsx/esm", probePath], + { + cwd: repoRoot, + env, + timeout: 30_000, + maxBuffer: 8 * 1024 * 1024, + }, + (error, stdout, stderr) => { + if (error) { + reject( + new Error( + `dashboard failure probe exited unsuccessfully: ${error.message}\n` + + `stdout:\n${stdout}\nstderr:\n${stderr}` + ) + ); + return; + } + resolve({ stdout, stderr }); + } + ); + }); +} + +test("persistAttemptLogs redacts request.failed delivery/replay but keeps its internal log", async () => { + const isolationRoot = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-dashboard-failure-redaction-") + ); + const dataDir = path.join(isolationRoot, "data"); + const pluginsDir = path.join(isolationRoot, "plugins"); + fs.mkdirSync(dataDir, { recursive: true }); + fs.mkdirSync(pluginsDir, { recursive: true }); + + try { + // The subprocess receives only process/runtime basics plus synthetic OmniRoute settings: no + // provider credentials are inherited and no parent singleton/env/global state is mutated. + const { stdout, stderr } = await runProbe({ + PATH: process.env.PATH, + NODE_PATH: process.env.NODE_PATH, + LANG: process.env.LANG, + LC_ALL: process.env.LC_ALL, + TZ: process.env.TZ, + TMPDIR: process.env.TMPDIR, + NODE_ENV: "test", + DATA_DIR: dataDir, + OMNIROUTE_PLUGINS_DIR: pluginsDir, + API_KEY_SECRET: "test-dashboard-failure-redaction-secret", + PII_RESPONSE_SANITIZATION: "false", + OMNIROUTE_ENABLE_LIVE_WS: "0", + }); + + assert.doesNotMatch(stderr, /sk-live-dashboard-secret|\/srv\/omniroute/); + const resultLine = stdout.split(/\r?\n/).find((line) => line.startsWith(RESULT_PREFIX)); + assert.ok(resultLine, `probe did not emit its result marker; stdout:\n${stdout}`); + const result = JSON.parse(resultLine.slice(RESULT_PREFIX.length)) as ProbeResult; + + assert.equal(result.delivered.id, "trace-dashboard-redaction"); + assert.equal(result.delivered.statusCode, 502); + assert.equal(result.delivered.model, "private-model"); + assert.equal(result.delivered.provider, "private-provider"); + assert.equal( + result.delivered.error, + "Error: Provider failed in with api_key='[REDACTED]'" + ); + assert.equal(result.replayMatches, true); + assert.equal(result.internalRawPreserved, true); + assert.equal(result.writerDrained, true); + } finally { + // The probe exits only after draining/closing its writer and resetting its DB singleton. + fs.rmSync(isolationRoot, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + } +}); + +test("the private LiveWS bridge forwards the already-safe event into its backlog unchanged", () => { + const source = fs.readFileSync( + fileURLToPath(new URL("../../src/server/ws/liveServer.ts", import.meta.url)), + "utf8" + ); + + // publishDashboardEvent/eventHistoryBacklog are module-private. This bounded source-chain + // assertion avoids opening a server while proving the bus payload is what live delivery and + // welcome/backlog replay store. The behavioral safety assertion lives in the subprocess above. + assert.match( + source, + /eventHistoryBacklog\.push\(\{ event, payload, timestamp \}\)/, + "the LiveWS backlog must store the event-bus payload" + ); + assert.match( + source, + /data:\s*h\.payload/, + "welcome replay must forward the stored backlog payload" + ); + assert.match( + source, + /onAny\(\(event:[^\n]+payload:[^\n]+\)\s*=>\s*\{\s*publishDashboardEvent\(event, payload\)/, + "the LiveWS bridge must publish the same event-bus payload" + ); +}); diff --git a/tests/unit/datadir-test-context-guard-10428.test.ts b/tests/unit/datadir-test-context-guard-10428.test.ts index 93fad86c73..b7bfe3ccd9 100644 --- a/tests/unit/datadir-test-context-guard-10428.test.ts +++ b/tests/unit/datadir-test-context-guard-10428.test.ts @@ -1,5 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; import fs from "node:fs"; @@ -21,6 +22,30 @@ import fs from "node:fs"; */ const { resolveWritableDataDir, getDefaultDataDir } = await import("../../src/lib/dataPaths.ts"); +const redirectedDirs = new Set(); + +function assertOwnedRedirectDir(candidate: string): string { + const resolved = path.resolve(candidate); + const tempRoot = path.resolve(os.tmpdir()); + assert.ok( + resolved.startsWith(`${tempRoot}${path.sep}`) && + path.basename(resolved).startsWith("omniroute-testctx-"), + `refusing to treat a non-owned path as a test redirect: ${resolved}` + ); + return resolved; +} + +function rememberRedirectDir(candidate: string): string { + const resolved = assertOwnedRedirectDir(candidate); + redirectedDirs.add(resolved); + return resolved; +} + +test.after(() => { + for (const redirected of redirectedDirs) { + fs.rmSync(redirected, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); function withEnv(overrides: Record, run: () => void) { const saved: Record = {}; @@ -39,11 +64,58 @@ function withEnv(overrides: Record, run: () => void) } } +const EVAL_PROBE_SCRIPT = + "import('./src/lib/dataPaths.ts').then(({ resolveWritableDataDir }) => " + + "console.log('OMNIROUTE_TEST_DATA_DIR=' + resolveWritableDataDir()))"; + +function assertEvalProbeIsIsolated(evalArgs: string[], configuredDataDir = "") { + const result = spawnSync(process.execPath, ["--import", "tsx/esm", ...evalArgs], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + DATA_DIR: configuredDataDir, + XDG_CONFIG_HOME: "", + NODE_ENV: "production", + NODE_TEST_CONTEXT: "", + VITEST: "", + OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: "", + }, + }); + + assert.equal(result.status, 0, result.stderr); + const outputLine = result.stdout + .trim() + .split("\n") + .find((line) => line.startsWith("OMNIROUTE_TEST_DATA_DIR=")); + const resolved = outputLine?.slice("OMNIROUTE_TEST_DATA_DIR=".length) ?? ""; + const ownedRedirect = assertOwnedRedirectDir(resolved); + try { + assert.notEqual( + ownedRedirect, + path.join(os.homedir(), ".omniroute"), + "an eval/import probe must not inherit the normal server's default database" + ); + assert.equal( + fs.existsSync(ownedRedirect), + false, + "the child exit handler must remove its exact redirected DATA_DIR" + ); + } finally { + fs.rmSync(ownedRedirect, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + } +} + test("G1: a test context with no DATA_DIR never resolves to the operator's real data dir", () => { withEnv( { DATA_DIR: undefined, NODE_ENV: "test", OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: undefined }, () => { - const resolved = resolveWritableDataDir(); + const resolved = rememberRedirectDir(resolveWritableDataDir()); assert.notEqual( resolved, getDefaultDataDir(), @@ -101,7 +173,7 @@ test("G5: node:test subprocesses are detected through NODE_TEST_CONTEXT too", () OMNIROUTE_ALLOW_DEFAULT_DATA_DIR: undefined, }, () => { - const resolved = resolveWritableDataDir(); + const resolved = rememberRedirectDir(resolveWritableDataDir()); assert.notEqual(resolved, getDefaultDataDir()); assert.ok(resolved.startsWith(os.tmpdir())); } @@ -110,8 +182,24 @@ test("G5: node:test subprocesses are detected through NODE_TEST_CONTEXT too", () test("G6: the redirect is stable within a process (same dir on repeated calls)", () => { withEnv({ DATA_DIR: undefined, NODE_ENV: "test" }, () => { - const first = resolveWritableDataDir(); - const second = resolveWritableDataDir(); + const first = rememberRedirectDir(resolveWritableDataDir()); + const second = rememberRedirectDir(resolveWritableDataDir()); assert.equal(first, second, "a per-call temp dir would split the DB across handles"); }); }); + +test("G7: a node --eval probe without DATA_DIR is isolated from the operator home", () => { + assertEvalProbeIsIsolated(["--eval", EVAL_PROBE_SCRIPT]); +}); + +test("G8: the single-argument --eval= form is isolated too", () => { + assertEvalProbeIsIsolated([`--eval=${EVAL_PROBE_SCRIPT}`]); +}); + +test("G9: whitespace DATA_DIR is absent for a node -e probe", () => { + assertEvalProbeIsIsolated(["-e", EVAL_PROBE_SCRIPT], " "); +}); + +test("G10: a combined node -pe probe is isolated too", () => { + assertEvalProbeIsIsolated(["-pe", EVAL_PROBE_SCRIPT]); +}); diff --git a/tests/unit/db-backup-extended.test.ts b/tests/unit/db-backup-extended.test.ts index 4ed9a08064..b0ac9e4f6b 100644 --- a/tests/unit/db-backup-extended.test.ts +++ b/tests/unit/db-backup-extended.test.ts @@ -97,6 +97,32 @@ test("backupDbFile creates manual backups and listDbBackups returns metadata", a assert.equal(fs.existsSync(backupPath), true); }); +test("listDbBackups orders mixed timestamp and content-addressed names by mtime", async () => { + seedConnections(2); + fs.mkdirSync(core.DB_BACKUPS_DIR, { recursive: true }); + + const lexicallyFutureButOld = "db_2099-01-01T00-00-00-000Z_manual.sqlite"; + const timestampMiddle = "db_2026-09-02T00-00-00-000Z_manual.sqlite"; + const contentAddressedNewest = `db_state-${"a".repeat(64)}_pre-migration.sqlite`; + for (const filename of [lexicallyFutureButOld, timestampMiddle, contentAddressedNewest]) { + await core.getDbInstance().backup(path.join(core.DB_BACKUPS_DIR, filename)); + } + + const now = Date.now() / 1000; + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, lexicallyFutureButOld), now - 120, now - 120); + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, timestampMiddle), now - 60, now - 60); + fs.utimesSync(path.join(core.DB_BACKUPS_DIR, contentAddressedNewest), now, now); + + const backups = await backupDb.listDbBackups(); + assert.deepEqual( + backups.map((backup) => backup.id), + [contentAddressedNewest, timestampMiddle, lexicallyFutureButOld], + "content-addressed migration snapshots must not make filename order masquerade as recency" + ); + assert.equal(backups[0]?.reason, "pre-migration"); + assert.equal(backups[0]?.connectionCount, 2); +}); + test("listDbBackups returns an empty list when the backup directory is missing", async () => { fs.rmSync(core.DB_BACKUPS_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); const backups = await backupDb.listDbBackups(); diff --git a/tests/unit/db-fresh-setup-9934.test.ts b/tests/unit/db-fresh-setup-9934.test.ts index 2330eedb6e..984c3a9f10 100644 --- a/tests/unit/db-fresh-setup-9934.test.ts +++ b/tests/unit/db-fresh-setup-9934.test.ts @@ -101,6 +101,13 @@ test( const cli = await importFresh("bin/cli/sqlite.mjs"); const setup = await cli.openOmniRouteDb(); assert.ok(fs.existsSync(setup.dbPath), "setup created storage.sqlite"); + setup.db + .prepare( + `INSERT INTO provider_connections + (id, provider, created_at, updated_at) + VALUES (?, ?, ?, ?)` + ) + .run("setup-provider", "openai", "2026-09-02T00:00:00.000Z", "2026-09-02T00:00:00.000Z"); setup.db.close(); const onDisk = new Database(setup.dbPath, { readonly: true }); @@ -139,6 +146,27 @@ test( (maxRow?.maxV ?? 0) > 1, `expected migrations beyond 001 to run, got max=${maxRow?.maxV}` ); + + const backupDir = path.join(dataDir, "db_backups"); + const snapshots = fs + .readdirSync(backupDir) + .filter((name) => /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/.test(name)); + assert.equal( + snapshots.length, + 1, + "a setup-created file is logically fresh for the mass guard but physically existing for snapshot safety" + ); + + const snapshot = new Database(path.join(backupDir, snapshots[0]!), { readonly: true }); + try { + assert.deepEqual( + snapshot.prepare("SELECT id, provider FROM provider_connections").get(), + { id: "setup-provider", provider: "openai" }, + "the mandatory snapshot must preserve setup-created provider state" + ); + } finally { + snapshot.close(); + } } finally { if (originalDataDir === undefined) delete process.env.DATA_DIR; else process.env.DATA_DIR = originalDataDir; diff --git a/tests/unit/db-migration-missing-physical-schema.test.ts b/tests/unit/db-migration-missing-physical-schema.test.ts new file mode 100644 index 0000000000..7f3c86d6da --- /dev/null +++ b/tests/unit/db-migration-missing-physical-schema.test.ts @@ -0,0 +1,839 @@ +// ENVIRONMENT NOTE (sandbox better-sqlite3 / glibc limitation, not a code defect): +// This test constructs a real better-sqlite3 database. Production and CI load the +// native addon normally; see tests/unit/_helpers/betterSqlite3Availability.ts for +// the documented fallback context on older sandboxes. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import Database from "better-sqlite3"; + +const isIsolatedChild = process.env.OMNIROUTE_DB_MIGRATION_SAFETY_CHILD === "1"; + +if (!isIsolatedChild) { + test("historical migration repair scenarios pass in an isolated process", () => { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-schema-repair-data-")); + const migrationsDir = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-schema-repair-migrations-") + ); + + try { + const childEnv = { + ...process.env, + DATA_DIR: dataDir, + OMNIROUTE_DB_MIGRATION_SAFETY_CHILD: "1", + OMNIROUTE_MAX_PENDING_MIGRATIONS: "", + OMNIROUTE_MIGRATIONS_DIR: migrationsDir, + }; + // Node's test runner exports this only to the current test worker. Passing it into + // another `node --test` process makes Node classify the nested file as recursive and + // skip every subtest while returning exit 0 — a dangerous false green. + delete childEnv.NODE_TEST_CONTEXT; + + const result = spawnSync( + process.execPath, + ["--import", "tsx/esm", "--test", fileURLToPath(import.meta.url)], + { + cwd: process.cwd(), + encoding: "utf8", + env: childEnv, + } + ); + + assert.equal( + result.status, + 0, + `isolated migration regressions failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ); + assert.match(result.stdout, /\btests 13\b/, "the isolated child must execute all subtests"); + assert.match(result.stdout, /\bpass 13\b/, "the isolated child must pass all subtests"); + } finally { + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.rmSync(migrationsDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); +} else { + const dataDir = process.env.DATA_DIR; + const migrationsDir = process.env.OMNIROUTE_MIGRATIONS_DIR; + assert.ok(dataDir, "isolated child requires an explicit DATA_DIR"); + assert.ok(migrationsDir, "isolated child requires an explicit migrations directory"); + const discoveryMigrationSql = fs.readFileSync( + path.resolve("src/lib/db/migrations/074_discovery_results.sql"), + "utf8" + ); + + fs.writeFileSync( + path.join(migrationsDir, "074_discovery_results.sql"), + discoveryMigrationSql, + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "081_inspector_custom_hosts.sql"), + ` + CREATE TABLE IF NOT EXISTS inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX IF NOT EXISTS idx_inspector_custom_hosts_enabled + ON inspector_custom_hosts(enabled); + `, + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "151_windsurf_to_devin_desktop.sql"), + "UPDATE discovery_results SET provider_id = 'devin-desktop' WHERE provider_id = 'windsurf';", + "utf8" + ); + fs.writeFileSync( + path.join(migrationsDir, "152_remove_puter_provider.sql"), + "DELETE FROM discovery_results WHERE provider_id = 'puter';", + "utf8" + ); + + const { runMigrations } = await import("../../src/lib/db/migrationRunner.ts"); + + function listPreMigrationBackups(): string[] { + const backupDir = path.join(dataDir, "db_backups"); + if (!fs.existsSync(backupDir)) return []; + return fs + .readdirSync(backupDir) + .filter((name) => name.endsWith("_pre-migration.sqlite")) + .sort(); + } + + function withNonTestEnvironment(fn: () => T): T { + const previousNodeEnv = process.env.NODE_ENV; + const previousVitest = process.env.VITEST; + const previousArgv = [...process.argv]; + const previousExecArgv = [...process.execArgv]; + + delete process.env.NODE_ENV; + delete process.env.VITEST; + process.argv = process.argv.filter((arg) => !arg.includes("test")); + process.execArgv = process.execArgv.filter((arg) => !arg.includes("test")); + + try { + return fn(); + } finally { + process.argv = previousArgv; + process.execArgv = previousExecArgv; + if (previousNodeEnv === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = previousNodeEnv; + if (previousVitest === undefined) delete process.env.VITEST; + else process.env.VITEST = previousVitest; + } + } + + test.after(() => { + // The parent owns both explicit temp directories and removes them after this + // process exits. Keeping ownership there also covers child startup failures. + }); + + test("runner repairs the 074 inspector collision before migrations 151 and 152", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + INSERT INTO inspector_custom_hosts (host, enabled) + VALUES ('api.example.test', 1); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "precondition: the collided 074 marker hides the missing discovery_results table" + ); + + assert.equal(runMigrations(db as never), 3); + + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + "074 must be replayed before migrations 151 and 152 reference discovery_results" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + assert.deepEqual( + db.prepare("SELECT host, enabled FROM inspector_custom_hosts").get(), + { host: "api.example.test", enabled: 1 }, + "re-homing the inspector marker to 081 must preserve the existing table data" + ); + assert.equal(runMigrations(db as never), 0, "the repaired state must be idempotent"); + } finally { + db.close(); + } + }); + + test("runner rehomes a collided 074 inspector marker even when both tables exist", () => { + const db = new Database(":memory:"); + + try { + db.exec(discoveryMigrationSql); + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + INSERT INTO inspector_custom_hosts (host, enabled) + VALUES ('api.example.test', 1); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 3); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ], + "the old 074 name must not remain as a permanent CRITICAL mismatch" + ); + assert.deepEqual(db.prepare("SELECT host FROM inspector_custom_hosts").get(), { + host: "api.example.test", + }); + } finally { + db.close(); + } + }); + + test("runner rebuilds both collided tables when neither physical table survived", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 4); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + "the canonical 074 table must be restored" + ); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'inspector_custom_hosts'" + ) + .get(), + "the rehomed 081 marker must not hide a missing inspector table" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner atomically replays 081 when its marker exists without the inspector table", () => { + const db = new Database(":memory:"); + + try { + db.exec(discoveryMigrationSql); + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'inspector_custom_hosts'); + `); + + assert.equal(runMigrations(db as never), 3); + assert.ok( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'inspector_custom_hosts'" + ) + .get(), + "a valid 081 marker must be replayed when its physical table is absent" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner fails closed when target 081 has unknown provenance", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'unknown_historical_migration'); + `); + + assert.throws( + () => runMigrations(db as never), + /target version 081 is occupied by unknown migration/i + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "inspector_custom_hosts" }, + { version: "081", name: "unknown_historical_migration" }, + ], + "a target collision must preserve both provenance records" + ); + } finally { + db.close(); + } + }); + + test("runner rejects an unknown 074 marker even when all later migrations are marked", () => { + const db = new Database(":memory:"); + + try { + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'unknown_historical_migration'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('081', 'inspector_custom_hosts'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('151', 'windsurf_to_devin_desktop'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('152', 'remove_puter_provider'); + `); + + assert.throws( + () => runMigrations(db as never), + /required table "discovery_results" is missing.*unknown migration/i, + "unknown provenance must fail closed instead of being silently rewritten" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [ + { version: "074", name: "unknown_historical_migration" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + } finally { + db.close(); + } + }); + + test("runner backs up an existing DB before reopening its only applied marker", () => { + const sqlitePath = path.join(dataDir, "only-marker.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + + assert.equal(runMigrations(db as never), 4); + + const backupDir = path.join(dataDir, "db_backups"); + const backups = fs + .readdirSync(backupDir) + .filter((name) => name.endsWith("_pre-migration.sqlite")); + assert.equal( + backups.length, + 1, + "removing the only marker must not make an existing DB look fresh and skip its snapshot" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("snapshot publication never deletes a raced final path", () => { + const sqlitePath = path.join(dataDir, "snapshot-publish-race.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const originalLinkSync = fs.linkSync; + let racedFinalPath: string | null = null; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + + fs.linkSync = ((_existingPath: fs.PathLike, newPath: fs.PathLike) => { + racedFinalPath = String(newPath); + fs.writeFileSync(racedFinalPath, "third-party-sentinel"); + throw Object.assign(new Error("destination already exists"), { code: "EEXIST" }); + }) as typeof fs.linkSync; + + assert.throws( + () => runMigrations(db as never), + /without a durable snapshot/, + "a raced final name must fail closed before atomic replay" + ); + assert.ok(racedFinalPath); + assert.equal( + fs.readFileSync(racedFinalPath, "utf8"), + "third-party-sentinel", + "snapshot failure cleanup must never unlink another actor's final path" + ); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + } finally { + fs.linkSync = originalLinkSync; + if (racedFinalPath && fs.existsSync(racedFinalPath)) fs.unlinkSync(racedFinalPath); + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("snapshot publication fails closed when hard links are unsupported", () => { + const sqlitePath = path.join(dataDir, "snapshot-publish-fallback.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const originalLinkSync = fs.linkSync; + const backupsBefore = listPreMigrationBackups(); + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + `); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" }); + }) as typeof fs.linkSync; + + assert.throws( + () => runMigrations(db as never), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined + ); + assert.deepEqual(listPreMigrationBackups(), backupsBefore); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("an only-marker repair cannot disarm the mass-migration barrier on retry", () => { + const sqlitePath = path.join(dataDir, "only-marker-mass-safety.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const previousMaxPending = process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + + try { + process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = "1"; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + const runOnce = () => withNonTestEnvironment(() => runMigrations(db as never)); + const backupsBefore = listPreMigrationBackups(); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "inspector_custom_hosts" }], + "an abort must restore the marker that was rehomed to calculate the real pending set" + ); + const afterFirstAbort = listPreMigrationBackups(); + const created = afterFirstAbort.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first abort must retain one restore point"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "inspector_custom_hosts" }], + "the second startup must hit the same barrier instead of treating the DB as fresh" + ); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstAbort, + "the identical retry must reuse the first content-addressed snapshot" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + if (previousMaxPending === undefined) delete process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + else process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = previousMaxPending; + } + }); + + test("a failed atomic 074 replay restores its marker and does not churn snapshots", () => { + const sqlitePath = path.join(dataDir, "failed-atomic-replay.sqlite"); + const db = new Database(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + CREATE TRIGGER block_migration_ledger_replay + BEFORE INSERT ON _omniroute_migrations + WHEN NEW.version = '074' + BEGIN + SELECT RAISE(ABORT, 'ledger replay blocked'); + END; + `); + + const runOnce = () => runMigrations(db as never); + const backupsBefore = listPreMigrationBackups(); + + assert.throws(runOnce, /ledger replay blocked/); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "the table creation and marker replacement must roll back together" + ); + const afterFirstFailure = listPreMigrationBackups(); + const created = afterFirstFailure.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first failed replay must retain one restore point"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /ledger replay blocked/); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstFailure, + "the identical failed replay must reuse its content-addressed restore point" + ); + } finally { + db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); + + test("sql.js rolls ledger repairs back when the mass-migration barrier aborts", async () => { + const sqlitePath = path.join(dataDir, "sqljs-mass-safety.sqlite"); + const { createSqlJsAdapter } = await import("../../src/lib/db/adapters/sqljsAdapter.ts"); + const db = await createSqlJsAdapter(sqlitePath); + const previousMaxPending = process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + const backupsBefore = listPreMigrationBackups(); + + try { + process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = "1"; + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE inspector_custom_hosts ( + host TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO inspector_custom_hosts (host) VALUES ('api.example.test'); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'inspector_custom_hosts'); + `); + + const runOnce = () => withNonTestEnvironment(() => runMigrations(db)); + const expectedLedger = [{ version: "074", name: "inspector_custom_hosts" }]; + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + expectedLedger, + "sql.js must roll the compatibility repair back with the safety savepoint" + ); + const afterFirstAbort = listPreMigrationBackups(); + const created = afterFirstAbort.filter((name) => !backupsBefore.includes(name)); + assert.equal(created.length, 1, "the first sql.js abort must retain one host snapshot"); + assert.match(created[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + + assert.throws(runOnce, /threshold is 1/i); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + expectedLedger, + "a retry must see the same original ledger rather than committed repair residue" + ); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstAbort, + "the identical sql.js abort must reuse its content-addressed snapshot" + ); + } finally { + db.close(); + if (previousMaxPending === undefined) delete process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS; + else process.env.OMNIROUTE_MAX_PENDING_MIGRATIONS = previousMaxPending; + } + }); + + test("sql.js exports a real host snapshot before replaying 074", async () => { + const sqlitePath = path.join(dataDir, "sqljs-physical-replay.sqlite"); + const { createSqlJsAdapter } = await import("../../src/lib/db/adapters/sqljsAdapter.ts"); + const db = await createSqlJsAdapter(sqlitePath); + const previousDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; + const backupsBefore = listPreMigrationBackups(); + + try { + delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + db.exec(` + PRAGMA user_version = 42; + PRAGMA application_id = 1337; + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO _omniroute_migrations (version, name) + VALUES ('074', 'discovery_results'); + CREATE TRIGGER block_sqljs_ledger_replay + BEFORE INSERT ON _omniroute_migrations + WHEN NEW.version = '074' + BEGIN + SELECT RAISE(ABORT, 'sqljs ledger replay blocked'); + END; + `); + + assert.throws(() => runMigrations(db), /sqljs ledger replay blocked/); + assert.deepEqual(db.prepare("SELECT version, name FROM _omniroute_migrations").all(), [ + { version: "074", name: "discovery_results" }, + ]); + assert.equal( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "sql.js must roll back the table and marker replacement together" + ); + const afterFirstFailure = listPreMigrationBackups(); + const firstCreated = afterFirstFailure.filter((name) => !backupsBefore.includes(name)); + assert.equal(firstCreated.length, 1, "sql.js must retain one host restore point"); + + assert.throws(() => runMigrations(db), /sqljs ledger replay blocked/); + assert.deepEqual( + listPreMigrationBackups(), + afterFirstFailure, + "the identical sql.js failure must reuse its content-addressed snapshot" + ); + + db.exec("DROP TRIGGER block_sqljs_ledger_replay"); + assert.equal(runMigrations(db), 4); + + const created = listPreMigrationBackups().filter((name) => !backupsBefore.includes(name)); + assert.equal( + created.length, + 2, + `dropping the trigger changes the DB state and must create a second snapshot: ${created}` + ); + + const snapshot = new Database(path.join(dataDir, "db_backups", created[0]!), { + readonly: true, + }); + try { + assert.equal(snapshot.pragma("integrity_check", { simple: true }), "ok"); + assert.equal(snapshot.pragma("user_version", { simple: true }), 42); + assert.equal(snapshot.pragma("application_id", { simple: true }), 1337); + const snapshotBytes = fs.readFileSync(path.join(dataDir, "db_backups", created[0]!)); + assert.equal(snapshotBytes.readUInt32BE(24), 1); + assert.equal( + snapshotBytes.readUInt32BE(92), + snapshotBytes.readUInt32BE(24), + "the normalized SQLite change counter and version-valid-for fields must agree" + ); + assert.deepEqual( + snapshot.prepare("SELECT version, name FROM _omniroute_migrations").all(), + [{ version: "074", name: "discovery_results" }] + ); + assert.equal( + snapshot + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'discovery_results'" + ) + .get(), + undefined, + "the snapshot must contain the complete pre-replay image" + ); + } finally { + snapshot.close(); + } + + const { listDbBackups } = await import("../../src/lib/db/backup.ts"); + const listed = await listDbBackups(); + assert.equal( + listed.find((backup) => backup.id === created[0])?.reason, + "pre-migration", + "the content address must not change the public backup reason" + ); + + db.close(); + const reopened = await createSqlJsAdapter(sqlitePath); + try { + assert.deepEqual( + reopened + .prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version") + .all(), + [ + { version: "074", name: "discovery_results" }, + { version: "081", name: "inspector_custom_hosts" }, + { version: "151", name: "windsurf_to_devin_desktop" }, + { version: "152", name: "remove_puter_provider" }, + ] + ); + assert.ok( + reopened + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_discovery_results_provider'" + ) + .get() + ); + assert.ok( + reopened + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_discovery_results_status'" + ) + .get() + ); + } finally { + reopened.close(); + } + } finally { + if (db.open) db.close(); + if (previousDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = previousDisableBackup; + } + }); +} diff --git a/tests/unit/db-migrationrunner-constants-split.test.ts b/tests/unit/db-migrationrunner-constants-split.test.ts index d8cebe6f1f..eb932cbb13 100644 --- a/tests/unit/db-migrationrunner-constants-split.test.ts +++ b/tests/unit/db-migrationrunner-constants-split.test.ts @@ -70,8 +70,8 @@ describe("migrationRunner/constants — exact small-table snapshots", () => { // ── large tables — count + shape + spot-checks (corruption guard) ───────────── describe("migrationRunner/constants — large-table integrity", () => { - it("RENAMED_MIGRATION_COMPATIBILITY has 31 well-formed entries", () => { - assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 31); + it("RENAMED_MIGRATION_COMPATIBILITY has 32 well-formed entries", () => { + assert.equal(RENAMED_MIGRATION_COMPATIBILITY.length, 32); for (const e of RENAMED_MIGRATION_COMPATIBILITY) { assert.equal(typeof e.fromVersion, "string"); assert.equal(typeof e.fromName, "string"); @@ -113,6 +113,17 @@ describe("migrationRunner/constants — large-table integrity", () => { "144", ] ); + assert.deepEqual( + RENAMED_MIGRATION_COMPATIBILITY.find( + (e) => e.fromVersion === "074" && e.fromName === "inspector_custom_hosts" + ), + { + fromVersion: "074", + fromName: "inspector_custom_hosts", + toVersion: "081", + toName: "inspector_custom_hosts", + } + ); assert.deepEqual(RENAMED_MIGRATION_COMPATIBILITY.at(-7), { fromVersion: "134", fromName: "ccr_blocks", diff --git a/tests/unit/db-pre-migration-backup-retention-10421.test.ts b/tests/unit/db-pre-migration-backup-retention-10421.test.ts index 798bb3cb66..836043d036 100644 --- a/tests/unit/db-pre-migration-backup-retention-10421.test.ts +++ b/tests/unit/db-pre-migration-backup-retention-10421.test.ts @@ -1,27 +1,24 @@ // ENVIRONMENT NOTE (sandbox better-sqlite3 / glibc limitation, not a code defect): -// This test constructs or exercises a real better-sqlite3-backed SQLite database. -// better-sqlite3 is a native addon; production and CI load it normally, but some -// sandboxes/dev boxes ship a system glibc older than the prebuilt binary requires -// ("GLIBC_2.29 not found"), so the native module fails to dlopen and any test that -// reaches better-sqlite3 directly (or asserts stdout that the load-failure warning -// would pollute) fails HERE while passing in CI. This is a known environment -// limitation, not a defect in the code under test: the OmniRoute runtime itself -// cascades to node:sqlite/sql.js when better-sqlite3 is unavailable. See -// tests/unit/_helpers/betterSqlite3Availability.ts for a guard helper. -// #10421 — pre-migration backups were created on every migration run and never pruned, -// so `db_backups/` grew without bound (observed: 48.999 files / 204 GB against a 5,3 MB -// live database). The pruning logic already existed in `cleanupDbBackups()` but nothing -// on the migration path ever reached it. These tests pin the retention step to the -// backup call site so the operator's maxFiles/retentionDays budget is honored there too. +// This suite uses a real on-disk better-sqlite3 database because migration snapshots +// must exercise SQLite's native read-only VACUUM path. Production and CI load the native +// addon normally; see tests/unit/_helpers/betterSqlite3Availability.ts for older sandboxes. +// +// #10421 — repeated failed startups once created a fresh timestamped snapshot every time +// and pruned unrelated restore points. Migration safety now publishes a content-addressed +// snapshot once per database state, never deletes a published snapshot, and leaves retention +// to the manual/scheduled backup paths outside the migration window. -import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import test from "node:test"; import { pathToFileURL } from "node:url"; + import Database from "better-sqlite3"; +import { createBetterSqliteAdapter } from "../../src/lib/db/adapters/betterSqliteAdapter.ts"; + const serial = { concurrency: false }; async function importFresh(modulePath: string) { @@ -29,27 +26,23 @@ async function importFresh(modulePath: string) { return import(`${url}?test=${Date.now()}-${Math.random().toString(16).slice(2)}`); } -function withMockedMigrationFs(files: Record, fn: () => void) { +function withMockedMigrationFs(files: Record, fn: () => T): T { const originalExistsSync = fs.existsSync; const originalReaddirSync = fs.readdirSync; const originalReadFileSync = fs.readFileSync; - const isMigrationDir = (target: unknown) => String(target).replaceAll("\\", "/").endsWith("/src/lib/db/migrations") || String(target).replaceAll("\\", "/").endsWith("/migrations"); fs.existsSync = ((target: unknown) => { if (isMigrationDir(target)) return true; - const fileName = path.basename(String(target)); - if (Object.hasOwn(files, fileName)) return true; + if (Object.hasOwn(files, path.basename(String(target)))) return true; return originalExistsSync(target as string); }) as typeof fs.existsSync; - fs.readdirSync = ((target: string, options?: unknown) => { if (isMigrationDir(target)) return Object.keys(files); return originalReaddirSync(target, options as never); }) as typeof fs.readdirSync; - fs.readFileSync = ((target: unknown, options?: unknown) => { const fileName = path.basename(String(target)); if (Object.hasOwn(files, fileName)) return files[fileName]; @@ -65,149 +58,264 @@ function withMockedMigrationFs(files: Record, fn: () => void) { } } -/** Minimal SqliteAdapter over a real on-disk file (VACUUM INTO needs a file, not :memory:). */ function createFileDb(sqlitePath: string) { - const db = new Database(sqlitePath); - - return { - driver: "better-sqlite3", - get open() { - return db.open; - }, - get name() { - return db.name; - }, - prepare: (sql: string) => db.prepare(sql), - exec: (sql: string) => db.exec(sql), - pragma: (str: string, options?: unknown) => db.pragma(str, options as never), - transaction: (fn: (...args: unknown[]) => unknown) => { - const tx = db.transaction((...args: unknown[]) => fn(...args)); - return (...args: unknown[]) => tx(...args); - }, - immediate: (fn: () => void) => fn(), - async backup() {}, - checkpoint() {}, - close: () => db.close(), - get raw() { - return db; - }, - }; + return createBetterSqliteAdapter(new Database(sqlitePath)); } -/** - * Build a DB that already has migrations applied (so the pre-migration backup path is - * reached: it requires `applied.size > 0`) plus one pending migration to trigger a run. - */ -function seedAppliedDb(db: ReturnType) { +function seedExistingDb(db: ReturnType): void { db.exec(` CREATE TABLE provider_connections (id TEXT PRIMARY KEY); CREATE TABLE combos (id TEXT PRIMARY KEY); CREATE TABLE call_logs (id TEXT PRIMARY KEY); - `); -} - -/** - * Record 001 as applied in the runner's own ledger table. `runMigrations` only takes a - * pre-migration backup when `applied.size > 0`, so this is what puts the test on the - * code path under exercise. - */ -function seedAppliedMigration(db: ReturnType) { - db.exec(` - CREATE TABLE IF NOT EXISTS _omniroute_migrations ( + CREATE TABLE _omniroute_migrations ( version TEXT PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')) ); + INSERT INTO provider_connections (id) VALUES ('existing-data'); + INSERT INTO _omniroute_migrations (version, name) VALUES ('001', 'initial_schema'); `); - db.prepare( - "INSERT OR REPLACE INTO _omniroute_migrations (version, name, applied_at) VALUES (?, ?, ?)" - ).run("001", "initial_schema", new Date().toISOString()); } -function makeTempDataDir() { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-backup-retention-")); +function seedSetupSkeleton(db: ReturnType): void { + db.exec(` + CREATE TABLE provider_connections (id TEXT PRIMARY KEY); + CREATE TABLE _omniroute_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + INSERT INTO provider_connections (id) VALUES ('setup-preserved-data'); + INSERT INTO _omniroute_migrations (version, name) VALUES ('001', 'initial_schema'); + `); +} + +function makeTempDataDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-migration-snapshot-")); fs.mkdirSync(path.join(dir, "db_backups"), { recursive: true }); return dir; } -/** Pre-existing backups, oldest first, with distinct mtimes so retention ordering is stable. */ -function seedBackups(backupDir: string, count: number) { +function seedTraditionalBackups(backupDir: string, count: number): string[] { const names: string[] = []; - for (let i = 0; i < count; i++) { - const name = `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`; - const filePath = path.join(backupDir, name); - fs.writeFileSync(filePath, "x"); - const t = new Date(2026, 7, i + 1).getTime() / 1000; - fs.utimesSync(filePath, t, t); + for (let index = 0; index < count; index += 1) { + const name = + `db_2026-08-${String(index + 1).padStart(2, "0")}` + "T00-00-00-000Z_pre-migration.sqlite"; + fs.writeFileSync(path.join(backupDir, name), `seed-${index}`); names.push(name); } return names; } -function countBackups(backupDir: string) { - return fs.readdirSync(backupDir).filter((n) => n.startsWith("db_")).length; +function listCanonicalBackups(backupDir: string): string[] { + if (!fs.existsSync(backupDir)) return []; + return fs + .readdirSync(backupDir) + .filter((name) => name.startsWith("db_") && name.endsWith(".sqlite")) + .sort(); } -function withEnv(vars: Record, fn: () => void) { - const saved: Record = {}; - for (const [k, v] of Object.entries(vars)) { - saved[k] = process.env[k]; - if (v === undefined) delete process.env[k]; - else process.env[k] = v; +function listOwnedTempDirs(backupDir: string): string[] { + if (!fs.existsSync(backupDir)) return []; + return fs.readdirSync(backupDir).filter((name) => name.startsWith(".migration-snapshot-")); +} + +function withEnv(vars: Record, fn: () => T): T { + const saved = new Map(); + for (const [key, value] of Object.entries(vars)) { + saved.set(key, process.env[key]); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; } try { return fn(); } finally { - for (const [k, v] of Object.entries(saved)) { - if (v === undefined) delete process.env[k]; - else process.env[k] = v; + for (const [key, value] of saved) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; } } } test( - "#10421 runMigrations prunes pre-migration backups to the configured maxFiles", + "repeated zero-progress failures reuse one content-addressed snapshot without pruning", serial, async () => { const dataDir = makeTempDataDir(); const backupDir = path.join(dataDir, "db_backups"); - const sqlitePath = path.join(dataDir, "storage.sqlite"); - const db = createFileDb(sqlitePath); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); try { - seedAppliedDb(db); - seedBackups(backupDir, 30); - assert.equal(countBackups(backupDir), 30, "precondition: 30 stale backups on disk"); - + seedExistingDb(db); + const seeded = seedTraditionalBackups(backupDir, 6); const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + const files = { + "001_initial_schema.sql": "SELECT 1;", + "002_broken_probe.sql": "INSERT INTO table_that_does_not_exist VALUES (1);", + }; + const fail = () => withMockedMigrationFs(files, () => runMigrations(db)); - withEnv( - { - DB_BACKUP_MAX_FILES: "5", - DB_BACKUP_RETENTION_DAYS: "0", - DISABLE_SQLITE_AUTO_BACKUP: undefined, - }, - () => { + assert.throws(fail, /table_that_does_not_exist/); + const afterFirst = listCanonicalBackups(backupDir); + const contentAddressed = afterFirst.filter((name) => name.startsWith("db_state-")); + assert.equal(contentAddressed.length, 1); + assert.match(contentAddressed[0]!, /^db_state-[a-f0-9]{64}_pre-migration\.sqlite$/); + assert.equal( + seeded.every((name) => afterFirst.includes(name)), + true, + "migration failure must not prune pre-existing restore points" + ); + + assert.throws(fail, /table_that_does_not_exist/); + assert.deepEqual( + listCanonicalBackups(backupDir), + afterFirst, + "an unchanged failed startup must reuse the exact content-addressed snapshot" + ); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); + +test( + "an existing DB fails closed when hard-link publication is unavailable even with auto backup disabled", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + const originalLinkSync = fs.linkSync; + + try { + seedExistingDb(db); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported by this filesystem"), { + code: "ENOTSUP", + }); + }) as typeof fs.linkSync; + + assert.throws( + () => + withEnv({ DISABLE_SQLITE_AUTO_BACKUP: "true" }, () => + withMockedMigrationFs( + { + "001_initial_schema.sql": "SELECT 1;", + "002_ordinary_pending.sql": "CREATE TABLE must_not_apply (id INTEGER);", + }, + () => runMigrations(db) + ) + ), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE name = 'must_not_apply'").get(), + undefined, + "an ordinary pending migration must not run without its mandatory snapshot" + ); + assert.deepEqual( + db.prepare("SELECT version, name FROM _omniroute_migrations ORDER BY version").all(), + [{ version: "001", name: "initial_schema" }] + ); + assert.deepEqual(listCanonicalBackups(backupDir), []); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); + +test( + "a pre-existing setup skeleton requires a snapshot even when mass-migration safety treats it as fresh", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + const originalLinkSync = fs.linkSync; + + try { + seedSetupSkeleton(db); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + fs.linkSync = (() => { + throw Object.assign(new Error("hard links unsupported by this filesystem"), { + code: "ENOTSUP", + }); + }) as typeof fs.linkSync; + + assert.throws( + () => withMockedMigrationFs( { "001_initial_schema.sql": "SELECT 1;", - "002_retention_probe.sql": "CREATE TABLE retention_probe_10421 (id INTEGER);", + "002_ordinary_pending.sql": "CREATE TABLE must_not_apply (id INTEGER);", }, - () => { - // Mark 001 as applied so `applied.size > 0` and the backup path is reached. - seedAppliedMigration(db); + () => + runMigrations(db, { + isNewDb: true, + databaseExistedBeforeInitialization: true, + }) + ), + /durable snapshot.*hard links unsupported.*hard links.*synchronization/is + ); + assert.equal( + db.prepare("SELECT name FROM sqlite_master WHERE name = 'must_not_apply'").get(), + undefined, + "a setup-created persistent DB must not change when its safety snapshot cannot publish" + ); + assert.deepEqual( + db.prepare("SELECT id FROM provider_connections").all(), + [{ id: "setup-preserved-data" }], + "the setup-created provider state must remain untouched" + ); + assert.deepEqual(listCanonicalBackups(backupDir), []); + assert.deepEqual(listOwnedTempDirs(backupDir), []); + } finally { + fs.linkSync = originalLinkSync; + db.close(); + fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +); - runMigrations(db); - } - ); - } +test( + "successful migrations retain existing backups and do not prune inside the migration window", + serial, + async () => { + const dataDir = makeTempDataDir(); + const backupDir = path.join(dataDir, "db_backups"); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); + + try { + seedExistingDb(db); + const seeded = seedTraditionalBackups(backupDir, 6); + const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); + + const count = withEnv({ DB_BACKUP_MAX_FILES: "1", DB_BACKUP_RETENTION_DAYS: "0" }, () => + withMockedMigrationFs( + { + "001_initial_schema.sql": "SELECT 1;", + "002_success.sql": "CREATE TABLE migration_success (id INTEGER);", + }, + () => runMigrations(db) + ) ); - const remaining = countBackups(backupDir); + assert.equal(count, 1); assert.ok( - remaining <= 5, - `expected retention to cap db_backups at 5 files, found ${remaining} — ` + - `pre-migration backups are accumulating unbounded (#10421)` + db.prepare("SELECT name FROM sqlite_master WHERE name = 'migration_success'").get() + ); + const after = listCanonicalBackups(backupDir); + assert.equal(after.filter((name) => name.startsWith("db_state-")).length, 1); + assert.equal( + seeded.every((name) => after.includes(name)), + true, + "retention must remain outside the concurrent migration window" ); } finally { db.close(); @@ -216,51 +324,26 @@ test( } ); -test("#10421 the newest pre-migration backup survives pruning", serial, async () => { +test("an already-current DB does not acquire an IMMEDIATE writer lock", serial, async () => { const dataDir = makeTempDataDir(); - const backupDir = path.join(dataDir, "db_backups"); - const sqlitePath = path.join(dataDir, "storage.sqlite"); - const db = createFileDb(sqlitePath); + const db = createFileDb(path.join(dataDir, "storage.sqlite")); try { - seedAppliedDb(db); - seedBackups(backupDir, 10); - + seedExistingDb(db); const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts"); - - withEnv( - { - DB_BACKUP_MAX_FILES: "3", - DB_BACKUP_RETENTION_DAYS: "0", - DISABLE_SQLITE_AUTO_BACKUP: undefined, + const noWriterAdapter = { + ...db, + immediate: () => { + throw new Error("unexpected IMMEDIATE writer lock"); }, - () => { - withMockedMigrationFs( - { - "001_initial_schema.sql": "SELECT 1;", - "002_retention_probe.sql": "CREATE TABLE retention_probe_10421b (id INTEGER);", - }, - () => { - seedAppliedMigration(db); + }; - runMigrations(db); - } - ); - } + assert.equal( + withMockedMigrationFs({ "001_initial_schema.sql": "SELECT 1;" }, () => + runMigrations(noWriterAdapter) + ), + 0 ); - - const remaining = fs.readdirSync(backupDir).filter((n) => n.startsWith("db_")); - assert.ok(remaining.length <= 3, `expected <=3 backups, found ${remaining.length}`); - - // The backup written by THIS run must be among the survivors — pruning must never - // discard the snapshot that protects the migration it was taken for. - const seededNames = new Set( - Array.from({ length: 10 }, (_, i) => { - return `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`; - }) - ); - const fresh = remaining.filter((n) => !seededNames.has(n)); - assert.equal(fresh.length, 1, `expected the run's own backup to survive, got ${fresh.length}`); } finally { db.close(); fs.rmSync(dataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); diff --git a/tests/unit/error-message-sanitization.test.ts b/tests/unit/error-message-sanitization.test.ts index f33a74a591..8813e7ac71 100644 --- a/tests/unit/error-message-sanitization.test.ts +++ b/tests/unit/error-message-sanitization.test.ts @@ -8,8 +8,16 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-err-sanitize-")); +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-err-sanitize-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +const ORIGINAL_API_KEY_SECRET = process.env.API_KEY_SECRET; +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; process.env.API_KEY_SECRET = "test-api-key-secret-32chars-long!!"; const core = await import("../../src/lib/db/core.ts"); @@ -42,7 +50,13 @@ test.beforeEach(async () => { test.after(() => { core.resetDbInstance(); - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + if (ORIGINAL_API_KEY_SECRET === undefined) delete process.env.API_KEY_SECRET; + else process.env.API_KEY_SECRET = ORIGINAL_API_KEY_SECRET; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); async function createCombo(name: string, model: string) { @@ -338,7 +352,8 @@ test("buildErrorBody — upstream details with stack key are stripped", async () !("stack" in (body.upstream_details as any)), "stack must be stripped from upstream_details" ); - assert.equal((body.upstream_details as any).code, "internal"); + assert.equal((body.upstream_details as any).code, ""); + assert.doesNotMatch(JSON.stringify(body.upstream_details), /internal/); }); // ── createErrorResult with upstreamDetails ─────────────────────────────────── diff --git a/tests/unit/error-public-boundaries-hardening.test.ts b/tests/unit/error-public-boundaries-hardening.test.ts new file mode 100644 index 0000000000..8e0e202b5b --- /dev/null +++ b/tests/unit/error-public-boundaries-hardening.test.ts @@ -0,0 +1,11 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("public error boundaries pass in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL("./fixtures/error-public-boundaries-hardening.fixture.ts", import.meta.url), + expectedTests: 23, + label: "public error boundaries", + }); +}); diff --git a/tests/unit/error-sanitizer-sk-key-qv45.test.ts b/tests/unit/error-sanitizer-sk-key-qv45.test.ts new file mode 100644 index 0000000000..b92d5ebbb1 --- /dev/null +++ b/tests/unit/error-sanitizer-sk-key-qv45.test.ts @@ -0,0 +1,93 @@ +/** + * GHSA-qv45-56jc-4wmj — two copies of one redaction rule, and only one got the + * `sk-` pattern. + * + * `upstreamErrorPassthrough.ts`'s CREDENTIAL_LEAK_RE matches `\bsk-[…]{8,}` and + * REFUSES verbatim passthrough when an upstream 4xx echoes a key — correctly + * treating it as a leak. The body then falls through to `buildErrorBody` → + * `sanitizeErrorMessage` → `redactSensitiveErrorText`, which had no `sk-` + * pattern at all. So the layer that recognized the credential handed it to a + * layer that did not, and OpenAI-style `Incorrect API key provided: sk-proj-…` + * bodies were returned to the caller verbatim. + * + * The passthrough file's own comment says it "mirrors the vocabulary of + * redactSensitiveErrorText" — the mirror had diverged. This suite pins both + * directions so it cannot diverge again. + * + * Run with: + * node --import tsx/esm --test tests/unit/error-sanitizer-sk-key-qv45.test.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { redactSensitiveErrorText, sanitizeErrorMessage } from "../../open-sse/utils/error.ts"; + +const LEAKY_BODIES = [ + "Incorrect API key provided: sk-proj-AbCdEfGhIjKlMnOpQrStUv. You can find your API key at …", + "401 Unauthorized: sk-ant-api03-abcdefghijklmnopqrstuvwxyz-1234567890", + "invalid key sk_live_51H8xKzAbCdEfGhIjKlMn", + "Bad credentials for AIzaSyA1B2C3D4E5F6G7H8I9J0KaLbMcNdOeP", + "token rejected: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", +]; + +describe("redactSensitiveErrorText — raw credential patterns (GHSA-qv45-56jc-4wmj)", () => { + for (const body of LEAKY_BODIES) { + it(`redacts the raw credential in: ${body.slice(0, 42)}…`, () => { + const out = redactSensitiveErrorText(body); + assert.ok(!/\bsk[-_][A-Za-z0-9._-]{8,}/.test(out), `sk- survived: ${out}`); + assert.ok(!/\bAIza[A-Za-z0-9_-]{20,}/.test(out), `Google key survived: ${out}`); + assert.ok(!/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\./.test(out), `JWT survived: ${out}`); + assert.ok(out.includes("[REDACTED"), `nothing was redacted: ${out}`); + }); + } + + it("redacts through sanitizeErrorMessage, the path buildErrorBody actually uses", () => { + const out = sanitizeErrorMessage("Incorrect API key provided: sk-proj-AbCdEfGhIjKlMnOpQr."); + assert.ok(!out.includes("sk-proj-AbCdEfGhIjKlMnOpQr"), out); + }); + + it("keeps the pre-existing redactions working", () => { + assert.match(redactSensitiveErrorText("401: Bearer abc123def456"), /Bearer \[REDACTED\]/); + assert.match( + redactSensitiveErrorText('{"api_key":"secret-value","detail":"bad"}'), + /\[REDACTED\]/ + ); + assert.match( + redactSensitiveErrorText("data:image/png;base64,AAAABBBBCCCC"), + /\[REDACTED_DATA_URL\]/ + ); + }); + + it("does not maul ordinary error prose that merely contains 'sk'", () => { + for (const benign of [ + "Model gpt-5 is not available on this plan", + "risk score too high", + "task sk failed", // short, no credential shape + "Rate limit reached for requests", + ]) { + assert.equal(redactSensitiveErrorText(benign), benign); + } + }); +}); + +describe("the two redaction layers stay in step", () => { + it("every credential shape the passthrough layer refuses is also redacted here", async () => { + // If passthrough REFUSES a body as leaky, the fallback sanitizer is the only + // thing standing between that body and the caller. Anything the first layer + // calls a credential, the second must scrub. + const { shouldPassthroughUpstreamError } = + await import("../../open-sse/utils/upstreamErrorPassthrough.ts"); + for (const body of LEAKY_BODIES) { + const payload = { error: { message: body } }; + const relayedVerbatim = shouldPassthroughUpstreamError(401, payload); + if (relayedVerbatim) continue; // not classified as a leak — nothing to assert + const scrubbed = redactSensitiveErrorText(body); + assert.notEqual( + scrubbed, + body, + `passthrough refused this body as leaky but the sanitizer left it untouched: ${body}` + ); + } + }); +}); diff --git a/tests/unit/error-sensitive-redaction.test.ts b/tests/unit/error-sensitive-redaction.test.ts index 1c7b5129fb..eb2e7e2392 100644 --- a/tests/unit/error-sensitive-redaction.test.ts +++ b/tests/unit/error-sensitive-redaction.test.ts @@ -1,8 +1,11 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { sanitizeErrorMessage, sanitizeUpstreamDetails } from "../../open-sse/utils/error.ts"; +import { + sanitizeErrorMessage, + sanitizeUpstreamDetails, +} from "../../open-sse/utils/errorSanitization.ts"; -test("sanitizeErrorMessage redacts bearer credentials and image data URLs", () => { +test("sanitizeErrorMessage removes bearer credentials and image data URLs", () => { const raw = "upstream echoed Authorization: Bearer eyJ.secret.token and data:image/png;charset=utf-8;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="; const safe = sanitizeErrorMessage(raw); @@ -10,7 +13,10 @@ test("sanitizeErrorMessage redacts bearer credentials and image data URLs", () = assert.doesNotMatch(safe, /eyJ\.secret\.token/); assert.doesNotMatch(safe, /iVBORw0KGgo/); assert.match(safe, /\[REDACTED\]/); - assert.match(safe, /\[REDACTED_DATA_URL\]/); + // Authorization labels are fail-closed: once a credential label is seen, + // the sanitizer may discard the remaining untrusted tail instead of + // preserving a marker for each later secret. + assert.equal(safe, "upstream echoed Authorization: [REDACTED]"); }); test("sanitizeErrorMessage redacts common JSON credential fields", () => { @@ -25,6 +31,122 @@ test("sanitizeErrorMessage redacts common JSON credential fields", () => { assert.match(safe, /\[REDACTED\]/); }); +test("sanitizeErrorMessage redacts URL credentials while preserving safe URLs", () => { + const safeUrl = "https://example.com/docs/error?lang=en#recovery"; + const projected = sanitizeErrorMessage( + "proxy failed https://svc-user:p4ss-opaque-9382@internal.example/v1 " + + "then https://storage.example/blob?X-Amz-Credential=AKIAOPAQUE%2Fscope&" + + "X-Amz-Signature=signature-secret&X-Amz-Expires=60 " + + "and https://account.blob.core.windows.net/c?sv=2025-01-05&sig=sas-secret&se=soon " + + "then https://vertex.example/predict?key=vertex-key-secret&mode=express " + + "plus https://gateway.example/v1?api_key=query-api-secret&token=query-token-secret " + + `see ${safeUrl}` + ); + + assert.doesNotMatch( + projected, + /svc-user|p4ss-opaque|AKIAOPAQUE|signature-secret|sas-secret|vertex-key-secret|query-api-secret|query-token-secret/i + ); + assert.match(projected, /\[REDACTED\]/); + assert.match(projected, /X-Amz-Expires=60/); + assert.match(projected, /sv=2025-01-05/); + assert.match(projected, /se=soon/); + assert.match(projected, new RegExp(safeUrl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); +}); + +test("sanitizeErrorMessage redacts credentials hidden behind serialized whitespace", () => { + const inputs = [ + String.raw`api_key\t=opaque-tab-secret-9382746`, + String.raw`api_key\u0009=opaque-unicode-tab-9382746`, + String.raw`Bearer\topaque-bearer-secret-9382746`, + String.raw`api_key\\t=opaque-double-tab-secret-9382746`, + ]; + + for (const input of inputs) { + const projected = sanitizeErrorMessage(input); + assert.doesNotMatch(projected, /opaque-(?:tab|unicode-tab|bearer|double-tab)-secret/i); + assert.match(projected, /\[REDACTED\]/); + } +}); + +test("sanitizeErrorMessage redacts CLI credential flag values", () => { + const inputs = [ + "spawn failed: helper --api-key opaque-cli-key-9382746 --mode check", + 'spawn failed: helper --token "opaque cli token 9382746" --mode check', + "spawn failed: helper --password 'opaque-cli-password-9382746' --mode check", + ]; + + for (const input of inputs) { + const projected = sanitizeErrorMessage(input); + assert.doesNotMatch(projected, /opaque(?: cli|-cli)/i); + assert.match(projected, /\[REDACTED\]/); + } +}); + +test("sanitizeErrorMessage covers the canonical credential pattern catalog", () => { + const credentials = [ + `AIza${"A".repeat(35)}`, + `hf_${"A".repeat(34)}`, + `r8_${"A".repeat(37)}`, + `gho_${"A".repeat(36)}`, + `ghu_${"A".repeat(36)}`, + `ghs_${"A".repeat(36)}`, + `ghr_${"A".repeat(36)}`, + `lin_api_${"A".repeat(40)}`, + `secret_${"A".repeat(43)}`, + `npm_${"A".repeat(36)}`, + `PMAK-1234abcd-${"a".repeat(32)}`, + `rk_live_${"A".repeat(24)}`, + `sq0atp-${"A".repeat(22)}`, + `SK${"a".repeat(32)}`, + `SG.${"A".repeat(22)}.${"B".repeat(43)}`, + `key-${"a".repeat(32)}`, + `M${"A".repeat(23)}.${"B".repeat(6)}.${"C".repeat(27)}`, + "postgresql://db-user:db-password@db.internal.example/app", + ]; + + for (const credential of credentials) { + const projected = sanitizeErrorMessage(`upstream echoed ${credential}`); + assert.equal(projected.includes(credential), false, credential.slice(0, 16)); + assert.match(projected, /\[REDACTED(?::[^\]]+)?\]/); + } +}); + +test("sanitizeErrorMessage redacts credentials that cross the public length boundary", () => { + const credential = `hf_${"A".repeat(34)}`; + const projected = sanitizeErrorMessage(`${"x".repeat(4088)}${credential}`); + const escapedPrefixProjected = sanitizeErrorMessage( + `${String.raw`\t`}${"x".repeat(4088)}${credential}` + ); + + for (const output of [projected, escapedPrefixProjected]) { + assert.equal(output.includes("hf_"), false); + assert.equal(output.includes(credential), false); + assert.match(output, /\[REDACTED(?::[^\]]+)?\]$/); + assert.ok(output.length <= 4096); + } +}); + +test("sanitizeErrorMessage redacts closed and unterminated PGP private-key armor", () => { + const closed = sanitizeErrorMessage( + "provider returned -----BEGIN PGP PRIVATE KEY BLOCK-----\n" + + "Version: test\n\npgp-private-material\n" + + "-----END PGP PRIVATE KEY BLOCK----- after" + ); + const unterminated = sanitizeErrorMessage( + "provider returned -----BEGIN PGP PRIVATE KEY BLOCK-----\npgp-unterminated-material" + ); + + // Public exception messages fail closed at the first physical line; the + // post-block suffix is intentionally not recovered from a multiline secret. + assert.equal(closed, "provider returned [REDACTED]"); + assert.equal(unterminated, "provider returned [REDACTED]"); + assert.doesNotMatch( + `${closed} ${unterminated}`, + /pgp-private-material|pgp-unterminated-material/ + ); +}); + test("sanitizeUpstreamDetails drops credential headers and redacts data URLs", () => { const safe = sanitizeUpstreamDetails({ authorization: "Bearer sensitive", diff --git a/tests/unit/executor-adapta-web-stream-error.test.ts b/tests/unit/executor-adapta-web-stream-error.test.ts new file mode 100644 index 0000000000..f17667b92a --- /dev/null +++ b/tests/unit/executor-adapta-web-stream-error.test.ts @@ -0,0 +1,97 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const FIXTURE = fileURLToPath( + new URL("../fixtures/adapta-web-stream-error-boundary.fixture.ts", import.meta.url) +); +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const SYNTHETIC_API_KEY_SECRET = + "adapta-stream-boundary-test-secret-00000000000000000000000000000000"; + +type FixtureResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function runIsolatedFixture(testRoot: string): Promise { + const dataDir = path.join(testRoot, "data"); + const pluginsDir = path.join(testRoot, "plugins"); + fs.mkdirSync(dataDir, { recursive: true }); + fs.mkdirSync(pluginsDir, { recursive: true }); + + const childEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + NODE_PATH: process.env.NODE_PATH, + LANG: process.env.LANG ?? "C.UTF-8", + LC_ALL: process.env.LC_ALL, + TZ: process.env.TZ ?? "UTC", + TMPDIR: process.env.TMPDIR ?? os.tmpdir(), + NODE_ENV: "test", + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: SYNTHETIC_API_KEY_SECRET, + DATA_DIR: dataDir, + OMNIROUTE_PLUGINS_DIR: pluginsDir, + }; + delete childEnv.NODE_TEST_CONTEXT; + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx/esm", "--test", FIXTURE], { + cwd: REPO_ROOT, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + timeout: 90_000, + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +test("Adapta stream errors are sanitized in an isolated executor fixture", async () => { + const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-adapta-boundary-parent-")); + const originalDataDir = process.env.DATA_DIR; + const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; + const originalFetch = globalThis.fetch; + const eventBusOwner = globalThis as { __omnirouteEventBus?: unknown }; + const originalEventBus = eventBusOwner.__omnirouteEventBus; + + try { + const result = await runIsolatedFixture(testRoot); + assert.equal( + result.code, + 0, + `isolated Adapta fixture failed (signal=${result.signal ?? "none"})\n` + + `stdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ); + assert.equal(result.signal, null); + assert.match(result.stdout, /ℹ tests 1/); + assert.match(result.stdout, /ℹ pass 1/); + assert.match(result.stdout, /ℹ fail 0/); + assert.doesNotMatch(result.stdout + result.stderr, new RegExp(SYNTHETIC_API_KEY_SECRET)); + + assert.equal(process.env.DATA_DIR, originalDataDir); + assert.equal(process.env.OMNIROUTE_PLUGINS_DIR, originalPluginsDir); + assert.equal(globalThis.fetch, originalFetch); + assert.equal( + eventBusOwner.__omnirouteEventBus, + originalEventBus, + "the subprocess fixture must not replace the parent event bus singleton" + ); + } finally { + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); diff --git a/tests/unit/executor-adapta-web.test.ts b/tests/unit/executor-adapta-web.test.ts index 46105774ee..fc6538f8cf 100644 --- a/tests/unit/executor-adapta-web.test.ts +++ b/tests/unit/executor-adapta-web.test.ts @@ -36,18 +36,25 @@ describe("AdaptaWebExecutor", () => { }); it("execute returns proper result shape on auth failure", async () => { - const executor = new mod.AdaptaWebExecutor(); - const result = await executor.execute({ - model: "adapta-one", - body: { messages: [{ role: "user", content: "hi" }] }, - stream: false, - credentials: { apiKey: "invalid-jwt" }, - signal: null, - }); - assert.ok(result.response instanceof Response); - assert.ok(typeof result.url === "string"); - assert.ok(typeof result.headers === "object"); - assert.ok(result.transformedBody !== undefined); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => new Response(null, { status: 401 })) as typeof fetch; + + try { + const executor = new mod.AdaptaWebExecutor(); + const result = await executor.execute({ + model: "adapta-one", + body: { messages: [{ role: "user", content: "hi" }] }, + stream: false, + credentials: { apiKey: "invalid-jwt" }, + signal: null, + }); + assert.ok(result.response instanceof Response); + assert.ok(typeof result.url === "string"); + assert.ok(typeof result.headers === "object"); + assert.ok(result.transformedBody !== undefined); + } finally { + globalThis.fetch = originalFetch; + } }); it("testConnection returns false for invalid credentials", async () => { diff --git a/tests/unit/fixtures/error-public-boundaries-hardening.fixture.ts b/tests/unit/fixtures/error-public-boundaries-hardening.fixture.ts new file mode 100644 index 0000000000..2b953ebd9e --- /dev/null +++ b/tests/unit/fixtures/error-public-boundaries-hardening.fixture.ts @@ -0,0 +1,608 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-public-errors-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); + +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; + +const core = await import("../../../src/lib/db/core.ts"); +const { + buildErrorBody, + buildModelCooldownBody, + createErrorResult, + parseUpstreamError, + projectPublicErrorIdentifier, + providerCircuitOpenResponse, + sanitizeErrorMessage, + sanitizeUpstreamDetails, + unavailableResponse, +} = await import("../../../open-sse/utils/error.ts"); +const { buildPassthroughErrorResponse, shouldPassthroughUpstreamError } = + await import("../../../open-sse/utils/upstreamErrorPassthrough.ts"); + +test.after(() => { + core.resetDbInstance(); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("sanitizeErrorMessage removes non-source paths, credentials, and serialized stacks", () => { + const raw = String.raw`Provider failed at /srv/private/provider-key.json access_token=provider-secret\n at validate (C:\Users\admin\private\validator.ts:42:7)`; + const safe = sanitizeErrorMessage(raw); + + assert.match(safe, /Provider failed/i); + assert.doesNotMatch(safe, /srv\/private|provider-secret|C:\\Users|validator\.ts/i); + assert.doesNotMatch(safe, /\\n\s*at validate/i); +}); + +test("sanitizeErrorMessage redacts Windows drive-root-relative filesystem paths", () => { + const plain = sanitizeErrorMessage( + String.raw`Provider failed at \Users\admin\private\secret.txt` + ); + const quoted = sanitizeErrorMessage( + String.raw`Provider failed opening "\Windows\Temp\native.dll"` + ); + const singleSegment = sanitizeErrorMessage(String.raw`Provider failed opening \private.db`); + const prose = sanitizeErrorMessage(String.raw`Provider reported \offline without a path`); + const escapedInitialPaths = [ + String.raw`Provider failed at \bin\private.db`, + String.raw`Provider failed at \folder\private.db`, + String.raw`Provider failed at \new\private.db`, + String.raw`Provider failed at \root\private.db`, + String.raw`Provider failed at \temp\private.db`, + String.raw`Provider failed at C:\temp\private.db`, + ].map((message) => sanitizeErrorMessage(message)); + + assert.equal(plain, "Provider failed at "); + assert.equal(quoted, 'Provider failed opening ""'); + assert.equal(singleSegment, "Provider failed opening "); + assert.equal(prose, String.raw`Provider reported \offline without a path`); + for (const projected of escapedInitialPaths) { + assert.equal(projected, "Provider failed at "); + } +}); + +test("sanitizeErrorMessage redacts extensionless POSIX paths without hiding explicit routes", () => { + const compact = sanitizeErrorMessage("Provider failed at /custom/internal/secret"); + const spaced = sanitizeErrorMessage("Provider failed at /custom/internal secret directory"); + const route = sanitizeErrorMessage("Route /dashboard/providers is unavailable"); + const singleSegment = sanitizeErrorMessage("Provider failed opening /vault"); + const singleSegmentRoute = sanitizeErrorMessage("Route /vault is unavailable"); + const compoundPathAndRoute = sanitizeErrorMessage( + "Failed /vault then GET /home/profile returned 404" + ); + const knownRootRoutes = [ + sanitizeErrorMessage("GET /home returned 404"), + sanitizeErrorMessage("Route /run is unavailable"), + sanitizeErrorMessage("POST /data returned 409"), + sanitizeErrorMessage("Route /var is unavailable"), + ]; + const body = buildErrorBody(500, "Provider failed at /custom/internal/secret"); + + assert.doesNotMatch(compact, /custom\/internal\/secret/); + assert.doesNotMatch(spaced, /custom\/internal|secret directory/); + assert.doesNotMatch(body.error.message, /custom\/internal\/secret/); + assert.match(compact, //); + assert.equal(route, "Route /dashboard/providers is unavailable"); + assert.equal(singleSegment, "Provider failed opening "); + assert.equal(singleSegmentRoute, "Route /vault is unavailable"); + assert.equal(compoundPathAndRoute, "Failed then GET /home/profile returned 404"); + assert.deepEqual(knownRootRoutes, [ + "GET /home returned 404", + "Route /run is unavailable", + "POST /data returned 409", + "Route /var is unavailable", + ]); +}); + +test("sanitizeErrorMessage fails closed when string coercion is hostile", () => { + const hostile = { + toString(): never { + throw new Error("access_token=hostile-secret at /srv/private/hostile.ts:1:2"); + }, + }; + + assert.equal(sanitizeErrorMessage(hostile), ""); +}); + +test("buildErrorBody projects untrusted error classifications onto safe identifiers", () => { + const body = buildErrorBody(502, "upstream failed", undefined, { + type: "server_error\nX-Leak: yes", + code: "sk-live-secret-value", + reason: "access_token=reason-secret", + }); + + assert.equal(body.error.type, "server_error"); + assert.equal(body.error.code, "bad_gateway"); + assert.equal(body.error.reason, undefined); +}); + +test("createErrorResult rejects opaque upstream identifiers that could be echoed credentials", async () => { + const opaqueCredential = "AbC9xY7pQ2mN8vR4kL6z"; + const result = createErrorResult( + 502, + "upstream failed", + null, + opaqueCredential, + opaqueCredential + ); + const body = (await result.response.json()) as { + error: { code: string; type: string }; + }; + + assert.equal(body.error.code, "bad_gateway"); + assert.equal(body.error.type, "server_error"); + assert.doesNotMatch(JSON.stringify(body), new RegExp(opaqueCredential)); +}); + +test("parseUpstreamError never stringifies an untrusted error object into the public message", async () => { + const opaqueIdentifier = "AbC9xY7pQ2mN8vR4kL6z"; + const parsed = await parseUpstreamError( + Response.json( + { + error: { + code: opaqueIdentifier, + type: opaqueIdentifier, + reason: opaqueIdentifier, + }, + }, + { status: 502 } + ), + "openai" + ); + const result = createErrorResult( + parsed.statusCode, + parsed.message, + parsed.retryAfterMs, + parsed.errorCode as string, + parsed.errorType as string, + parsed.responseBody + ); + const bodyText = await result.response.text(); + + assert.equal(parsed.message, "Upstream error: 502"); + assert.doesNotMatch(bodyText, new RegExp(opaqueIdentifier)); +}); + +test("buildErrorBody preserves the configured empty code for unmapped client statuses", () => { + const body = buildErrorBody(424, "Dependency failed"); + + assert.equal(body.error.type, "invalid_request_error"); + assert.equal(body.error.code, ""); +}); + +test("public identifier vocabulary preserves current internal machine-readable contracts", () => { + const identifiers = [ + "context_length_exceeded", + "tool_calling_not_supported", + "vision", + "tools", + "structured_output", + "context_window", + "unsupported_endpoint", + "unverified_codex_client", + "invalid_previous_response_binding", + "incompatible_reasoning_effort", + "STREAM_READINESS_TIMEOUT", + "stream_timeout", + "STREAM_EARLY_EOF", + "stream_early_eof", + "LEASE_NO_ELIGIBLE_CONNECTION", + "LEASE_ELIGIBILITY_UNAVAILABLE", + "LEASE_UNSUPPORTED_ROUTE", + "LEASE_UNSUPPORTED_TRANSPORT", + "DIRECT_RESPONSE_START_TIMEOUT", + "PROXY_FAMILY_UNAVAILABLE", + "RELAY_TIMEOUT", + "TLS_FINGERPRINT_FAILED", + "PROXY_REQUEST_FAILED", + "TLS_SESSION_CAPACITY", + "TLS_CIRCUIT_OPEN", + "PROVIDER_RETIRED", + "upstream_empty_response", + "upstream_response_error", + "upstream_response_failed", + "stream_pipeline_error", + "stream_terminated", + "rate_limited", + "usage_limit_reached", + "timeout", + "semaphore_timeout", + "semaphore_queue_full", + "RATE_LIMIT_EXECUTION_TIMEOUT", + "RATE_LIMIT_QUEUE_FULL", + "RATE_LIMIT_QUEUE_WEDGED", + "RATE_LIMIT_QUEUE_TIMEOUT", + "rate_limit_queue_wedged", + "429", + "empty_response", + "stream_idle_timeout", + "empty_content", + "UNAVAILABLE", + "RESOURCE_EXHAUSTED", + "provider_unavailable", + "unsupported_feature", + "missing_project_id", + "oauth_missing_project_id", + "gcp_project_required", + "QUOTA_ONLY", + "QUOTA_NOT_ALLOCATED", + "cloudflare_challenge", + "cf_mitigated_challenge", + "upstream_protocol_error", + "claude_web_protocol_error", + "service_not_running", + "storage_encryption_stale", + "HTTP_429", + "BLACKBOX_SUBSCRIPTION_REQUIRED", + "BLACKBOX_AUTH_REQUIRED", + "BLACKBOX_RATE_LIMIT", + "abort", + "ABORTED", + "CHIPOTLE_ERROR", + "premium_model_requires_key", + "GROK_ERROR", + "TLS_CLIENT_UNAVAILABLE", + "upstream_access_denied", + "proxy_unavailable", + "EXECUTOR_ERROR", + "executor_contract_violation", + "orphan_tool_result", + "bedrock_stream_error", + "invalid_kiro_tool_call", + "devin_cli_error", + "upstream_websocket_error", + "upstream_websocket_connect_failed", + "codex_app_server_turn_failed", + "missing_credits", + "reached_limit", + "rate_limit_reached", + "rate_limit_longer_reached", + "client_cancelled", + "client_closed_request", + "compaction_control_unavailable", + "compaction_handoff_failed", + "connector_not_found", + "connector_error", + "prompt_attachment_integrity", + "chatgpt_session_expired", + "chatgpt_subscription_unavailable", + "upstream_server_error", + "multipart_protocol_violation", + "browser_stream_inconsistent", + "structured_output_validation_failed", + "chatgpt_submission_ambiguous", + "chatgpt_submitted_turn_failed", + "cli_not_found", + "upstream_auth_error", + "wreq_unavailable", + "api_error", + "connection_error", + "unsupported_runtime", + "VIDEO_ARTIFACT_URL_INVALID", + "VIDEO_ARTIFACT_URL_BLOCKED", + "VIDEO_ARTIFACT_DOWNLOAD_FAILED", + "VIDEO_ARTIFACT_TOO_LARGE", + "VIDEO_ARTIFACT_SIGNATURE_INVALID", + "VIDEO_ARTIFACT_NOT_READY", + "VIDEO_ARTIFACT_UNAVAILABLE", + "VIDEO_ARTIFACT_CONTENT_TYPE_INVALID", + "codex_app_server_unconfigured", + "meta_ai_warmup_failed", + "meta_ai_mode_switch_failed", + "meta_ai_ws_error", + "meta_ai_empty_response", + "PPLX_ERROR", + "cloudflare_or_bot", + "request_failed", + "lmarena_error", + "network_error", + ]; + + for (const identifier of identifiers) { + assert.equal(projectPublicErrorIdentifier(identifier, "bad_request"), identifier, identifier); + } +}); + +test("public numeric identifiers are limited to three-digit HTTP status codes", () => { + assert.equal(projectPublicErrorIdentifier("100", "bad_request"), "100"); + assert.equal(projectPublicErrorIdentifier("599", "bad_request"), "599"); + assert.equal(projectPublicErrorIdentifier("099", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("600", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("5000", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("40002", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("HTTP_600", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("HTTP_40002", "bad_request"), "bad_request"); + assert.equal(projectPublicErrorIdentifier("weird_error", "bad_gateway"), "bad_gateway"); +}); + +test("buildErrorBody callers never overwrite a projected public classification", () => { + const productionFiles: string[] = []; + const collectTypeScriptFiles = (directory: string): void => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (entry.name === "__tests__") continue; + collectTypeScriptFiles(entryPath); + } else if (entry.isFile() && /\.tsx?$/.test(entry.name)) { + productionFiles.push(entryPath); + } + } + }; + + collectTypeScriptFiles(path.join(REPO_ROOT, "open-sse")); + collectTypeScriptFiles(path.join(REPO_ROOT, "src")); + + const mutationPattern = /\b[A-Za-z_$][A-Za-z0-9_$]*\.error\.(?:code|type|reason)\s*=(?!=)/g; + const violations: string[] = []; + for (const filePath of productionFiles) { + const source = fs.readFileSync(filePath, "utf8"); + if (!source.includes("buildErrorBody")) continue; + for (const match of source.matchAll(mutationPattern)) { + const line = source.slice(0, match.index).split("\n").length; + violations.push(`${path.relative(REPO_ROOT, filePath)}:${line}`); + } + } + + assert.deepEqual(violations, []); + + const chatCoreSource = fs.readFileSync( + path.join(REPO_ROOT, "open-sse/handlers/chatCore.ts"), + "utf8" + ); + assert.doesNotMatch( + chatCoreSource, + /JSON\.stringify\(\s*\{\s*error\s*:\s*\{/, + "chatCore must not bypass buildErrorBody with a manually assembled error envelope" + ); +}); + +test("operational log persistence catches use the canonical sanitizer", () => { + const callLogsSource = fs.readFileSync(path.join(REPO_ROOT, "src/lib/usage/callLogs.ts"), "utf8"); + const proxyLoggerSource = fs.readFileSync(path.join(REPO_ROOT, "src/lib/proxyLogger.ts"), "utf8"); + + assert.match(callLogsSource, /sanitizeErrorMessage\(error\)/); + assert.doesNotMatch(callLogsSource, /\(error as Error\)\.message/); + assert.match(proxyLoggerSource, /sanitizeErrorMessage\(err\)/); + assert.doesNotMatch(proxyLoggerSource, /err\?\.message\s*\|\|\s*err/); +}); + +test("stream request finalization never warns with a raw error object", () => { + const source = fs.readFileSync( + path.join(REPO_ROOT, "open-sse/utils/streamFailureFinalization.ts"), + "utf8" + ); + + assert.match(source, /sanitizeErrorMessage\(error\)/); + assert.doesNotMatch(source, /"message" in error[\s\S]{0,160}: error/); +}); + +test("chatCore provider-failure writes use the projected persistent message", () => { + const source = fs.readFileSync(path.join(REPO_ROOT, "open-sse/handlers/chatCore.ts"), "utf8"); + const failureStart = source.indexOf("providerFailure: if (!providerResponse.ok)"); + const failureEnd = source.indexOf("// Non-streaming response", failureStart); + assert.ok(failureStart >= 0 && failureEnd > failureStart, "providerFailure block must exist"); + const failureBlock = source.slice(failureStart, failureEnd); + + assert.doesNotMatch(failureBlock, /lastError:\s*message\b/); + assert.ok( + (failureBlock.match(/lastError:\s*persistentMessage\b/g) || []).length >= 11, + "every providerFailure persistence branch must use persistentMessage" + ); +}); + +test("public cooldown and circuit responses sanitize dynamic context", async () => { + const unavailable = unavailableResponse( + 503, + "Provider failed at /srv/private/state.sqlite access_token=unavailable-secret", + 5, + "retry after reading C:\\Users\\admin\\private\\state.json" + ); + const unavailableBody = (await unavailable.json()) as { error: { message: string } }; + assert.doesNotMatch(unavailableBody.error.message, /srv\/private|unavailable-secret|C:\\Users/i); + + const circuit = providerCircuitOpenResponse( + "provider access_token=circuit-secret /home/service/provider.json", + 5 + ); + const circuitBody = (await circuit.json()) as { + error: { message: string; provider: string }; + }; + assert.equal(circuitBody.error.provider, "unknown"); + assert.doesNotMatch(JSON.stringify(circuitBody), /circuit-secret|\/home\/service/i); + + const cooldown = buildModelCooldownBody({ + model: "model access_token=model-secret /opt/models/private.json", + retryAfterSec: Number.NaN, + retryAfterAt: "not-a-timestamp access_token=timestamp-secret", + }); + assert.equal(cooldown.error.model, undefined); + assert.equal(cooldown.error.retry_after, undefined); + assert.equal(cooldown.error.reset_seconds, 1); + assert.doesNotMatch(JSON.stringify(cooldown), /model-secret|timestamp-secret|\/opt\/models/i); +}); + +test("sanitizeUpstreamDetails drops credential aliases and prototype-control keys", () => { + const input = Object.create(null) as Record; + input.error = { + message: "quota metadata at /srv/provider/private.json", + credential: "credential-secret", + sessionId: "session-secret", + session_count: 2, + }; + input.__proto__ = { leaked: true }; + + const safe = sanitizeUpstreamDetails(input) as Record; + const serialized = JSON.stringify(safe); + + assert.doesNotMatch(serialized, /credential-secret|session-secret|srv\/provider|__proto__/i); + assert.match(serialized, /"session_count":2/); +}); + +test("buildErrorBody fails closed for hostile upstream detail accessors", () => { + const hostile = new Proxy( + {}, + { + ownKeys(): never { + throw new Error("access_token=hostile-detail at /srv/private/detail.ts:1:2"); + }, + } + ); + + let body: ReturnType | undefined; + assert.doesNotThrow(() => { + body = buildErrorBody(502, "upstream failed", hostile); + }); + assert.equal(body?.upstream_details, undefined); + assert.doesNotMatch(JSON.stringify(body), /hostile-detail|srv\/private|detail\.ts/i); +}); + +test("upstream passthrough preserves safe wording but recursively sanitizes the JSON body", async () => { + const opaqueIdentifier = "AbC9xY7pQ2mN8vR4kL6z"; + const upstream = { + type: "error", + error: { + type: "invalid_request_error", + code: opaqueIdentifier, + reason: opaqueIdentifier, + message: "quota metadata from /srv/provider/private.json", + credential: "credential-secret", + session_count: 2, + details: [{ type: "integer", reason: "must be positive" }], + }, + }; + + assert.equal(shouldPassthroughUpstreamError(422, upstream), true); + const response = buildPassthroughErrorResponse(422, upstream); + assert.ok(response); + const serialized = JSON.stringify(await response.json()); + + assert.match(serialized, /invalid_request_error/); + assert.match(serialized, /"session_count":2/); + assert.match(serialized, /"type":"integer","reason":"must be positive"/); + assert.doesNotMatch( + serialized, + new RegExp(`credential-secret|srv/provider|${opaqueIdentifier}`, "i") + ); +}); + +test("upstream classification projection preserves HTTP numbers and rejects opaque aliases", () => { + const opaqueIdentifier = "AbC9xY7pQ2mN8vR4kL6z"; + const projected = sanitizeUpstreamDetails({ + code: 400, + status: "UNAVAILABLE", + oversizedCode: 40002, + error: { + code: 40002, + error_code: opaqueIdentifier, + errorCode: opaqueIdentifier, + error_type: opaqueIdentifier, + errorType: opaqueIdentifier, + sub_type: opaqueIdentifier, + subType: opaqueIdentifier, + status: opaqueIdentifier, + status_code: opaqueIdentifier, + statusCode: opaqueIdentifier, + message: "safe provider wording", + }, + }) as { + code?: unknown; + status?: unknown; + oversizedCode?: unknown; + error?: Record; + }; + + assert.equal(projected.code, 400); + assert.equal(projected.status, "UNAVAILABLE"); + assert.equal(projected.oversizedCode, 40002); + assert.equal(projected.error?.code, undefined); + assert.equal(projected.error?.error_code, ""); + assert.equal(projected.error?.errorCode, ""); + assert.equal(projected.error?.error_type, "upstream_error"); + assert.equal(projected.error?.errorType, "upstream_error"); + assert.equal(projected.error?.sub_type, "upstream_error"); + assert.equal(projected.error?.subType, "upstream_error"); + assert.equal(projected.error?.status, undefined); + assert.equal(projected.error?.status_code, undefined); + assert.equal(projected.error?.statusCode, undefined); + assert.equal(projected.error?.message, "safe provider wording"); + assert.doesNotMatch(JSON.stringify(projected), new RegExp(opaqueIdentifier)); +}); + +test("upstream classification projection preserves only real gRPC numeric codes", () => { + const projected = sanitizeUpstreamDetails({ + error: { code: 7 }, + errors: [{ code: 16 }, { code: 17 }, { code: 40002 }], + status: 7, + warning: { code: "model_capacity", type: "unknown" }, + }) as { + error?: { code?: unknown }; + errors?: Array<{ code?: unknown }>; + status?: unknown; + warning?: { code?: unknown; type?: unknown }; + }; + + assert.equal(projected.error?.code, 7); + assert.equal(projected.errors?.[0]?.code, 16); + assert.equal(projected.errors?.[1]?.code, undefined); + assert.equal(projected.errors?.[2]?.code, undefined); + assert.equal(projected.status, undefined); + assert.equal(projected.warning?.code, ""); + assert.equal(projected.warning?.type, "upstream_error"); +}); + +test("sanitizeUpstreamDetails fails closed for hostile prototype access", () => { + const hostile = new Proxy( + {}, + { + getPrototypeOf(): never { + throw new Error("access_token=prototype-secret at /srv/private/prototype.ts"); + }, + } + ); + + let projected: unknown; + assert.doesNotThrow(() => { + projected = sanitizeUpstreamDetails(hostile); + }); + assert.doesNotMatch(JSON.stringify(projected), /prototype-secret|srv\/private|prototype\.ts/i); +}); + +test("upstream passthrough fails closed for non-serializable bodies", () => { + const cyclic: Record = { error: { message: "safe" } }; + cyclic.self = cyclic; + + assert.equal(shouldPassthroughUpstreamError(400, cyclic), false); + assert.equal(buildPassthroughErrorResponse(400, cyclic), null); +}); + +test("upstream passthrough fails closed when getters change after eligibility", () => { + let reads = 0; + const upstream = Object.create(null) as Record; + Object.defineProperty(upstream, "error", { + enumerable: true, + get(): unknown { + reads += 1; + if (reads === 1) return { message: "safe capability error" }; + throw new Error("access_token=second-read-secret at /srv/private/getter.ts:1:2"); + }, + }); + + assert.doesNotThrow(() => buildPassthroughErrorResponse(400, upstream)); + assert.equal(buildPassthroughErrorResponse(400, upstream), null); +}); diff --git a/tests/unit/fixtures/mcp-public-error-boundaries.fixture.ts b/tests/unit/fixtures/mcp-public-error-boundaries.fixture.ts new file mode 100644 index 0000000000..1eaad34050 --- /dev/null +++ b/tests/unit/fixtures/mcp-public-error-boundaries.fixture.ts @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-mcp-error-boundaries-")); +const repoRoot = fileURLToPath(new URL("../../..", import.meta.url)); +const originalDataDir = process.env.DATA_DIR; +const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; +const originalApiKey = process.env.OMNIROUTE_API_KEY; +const originalApiKeyId = process.env.OMNIROUTE_API_KEY_ID; +const originalInternalToken = process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN; +const originalInternalTokenFile = process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN_FILE; +const originalBaseUrl = process.env.OMNIROUTE_BASE_URL; +process.env.DATA_DIR = path.join(testRoot, "data"); +process.env.OMNIROUTE_PLUGINS_DIR = path.join(testRoot, "plugins"); +process.env.OMNIROUTE_API_KEY = "mcp-boundary-test-key"; +process.env.OMNIROUTE_API_KEY_ID = "mcp-boundary-test-key-id"; +process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN = "mcp-boundary-internal-test-token"; +process.env.OMNIROUTE_BASE_URL = "http://localhost:20128"; +delete process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN_FILE; +fs.mkdirSync(process.env.DATA_DIR, { recursive: true }); +fs.mkdirSync(process.env.OMNIROUTE_PLUGINS_DIR, { recursive: true }); + +const { createMcpServer } = await import("../../../open-sse/mcp-server/server.ts"); +const { closeAuditDb, queryAuditEntries } = await import("../../../open-sse/mcp-server/audit.ts"); +const { obsidianTools } = await import("../../../open-sse/mcp-server/tools/obsidianTools.ts"); +const { skillTools } = await import("../../../open-sse/mcp-server/tools/skillTools.ts"); +const { skillRegistry } = await import("../../../src/lib/skills/registry.ts"); +const { skillExecutor } = await import("../../../src/lib/skills/executor.ts"); +const core = await import("../../../src/lib/db/core.ts"); + +type McpResult = { + content?: Array<{ type: string; text: string }>; + isError?: boolean; +}; + +type RegisteredTool = { + handler: (args: unknown, extra?: unknown) => Promise; +}; + +function getRegisteredHandler(server: unknown, toolName: string): RegisteredTool["handler"] { + const registry = (server as { _registeredTools?: Record }) + ._registeredTools; + assert.ok(registry, "McpServer should expose _registeredTools"); + const tool = registry[toolName]; + assert.ok(tool, `${toolName} must be registered`); + return tool.handler; +} + +function assertPublicMcpError(result: McpResult): void { + const text = result.content?.[0]?.text ?? ""; + assert.equal(result.isError, true); + assert.match(text, /Error:/); + assert.doesNotMatch(text, /mcp-boundary-secret|srv\/private|mcp-boundary\.ts|\bat execute\b/i); +} + +test.after(() => { + closeAuditDb(); + core.resetDbInstance(); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalPluginsDir === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = originalPluginsDir; + if (originalApiKey === undefined) delete process.env.OMNIROUTE_API_KEY; + else process.env.OMNIROUTE_API_KEY = originalApiKey; + if (originalApiKeyId === undefined) delete process.env.OMNIROUTE_API_KEY_ID; + else process.env.OMNIROUTE_API_KEY_ID = originalApiKeyId; + if (originalInternalToken === undefined) delete process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN; + else process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN = originalInternalToken; + if (originalInternalTokenFile === undefined) { + delete process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN_FILE; + } else { + process.env.OMNIROUTE_INTERNAL_SERVICE_TOKEN_FILE = originalInternalTokenFile; + } + if (originalBaseUrl === undefined) delete process.env.OMNIROUTE_BASE_URL; + else process.env.OMNIROUTE_BASE_URL = originalBaseUrl; + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("core MCP handlers sanitize upstream bodies before public and audit boundaries", async () => { + const hostile = "Bearer mcp-fetch-boundary-secret at /srv/private/mcp-fetch-boundary.ts:9:3"; + const originalFetch = globalThis.fetch; + const calledUrls: string[] = []; + globalThis.fetch = async (input) => { + calledUrls.push(String(input)); + return new Response(hostile, { status: 500 }); + }; + + try { + const handler = getRegisteredHandler(createMcpServer(), "omniroute_list_combos"); + const result = await handler({ includeMetrics: false }); + const publicText = result.content?.[0]?.text ?? ""; + assert.equal(result.isError, true); + assert.doesNotMatch( + publicText, + /mcp-fetch-boundary-secret|srv\/private|mcp-fetch-boundary\.ts/i + ); + assert.deepEqual(calledUrls, ["http://localhost:20128/api/combos"]); + + const audit = await queryAuditEntries({ tool: "omniroute_list_combos", success: false }); + assert.ok(audit.entries.length >= 1); + assert.doesNotMatch( + JSON.stringify(audit.entries), + /mcp-fetch-boundary-secret|srv\/private|mcp-fetch-boundary\.ts/i + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("every MCP public catch uses the canonical fail-closed projector", () => { + const source = fs.readFileSync(path.join(repoRoot, "open-sse/mcp-server/server.ts"), "utf8"); + assert.doesNotMatch(source, /err instanceof Error \? err\.message : String\(err\)/); +}); + +test("Obsidian and dynamic-skill MCP wrappers sanitize thrown errors", async () => { + const hostile = new Error( + "MCP failed access_token=mcp-boundary-secret at /srv/private/mcp-boundary.ts\n" + + " at execute (/srv/private/mcp-boundary.ts:9:3)" + ); + const mutableObsidianTool = obsidianTools[0] as unknown as { + name: string; + handler: (args: unknown, extra?: unknown) => Promise; + }; + const originalObsidianHandler = mutableObsidianTool.handler; + try { + mutableObsidianTool.handler = async () => { + throw hostile; + }; + const obsidianHandler = getRegisteredHandler(createMcpServer(), mutableObsidianTool.name); + assertPublicMcpError(await obsidianHandler({}, { authInfo: { scopes: ["read:obsidian"] } })); + } finally { + mutableObsidianTool.handler = originalObsidianHandler; + } + + const mutableRegistry = skillRegistry as unknown as { + list: () => Array<{ name: string; description: string; enabled: boolean }>; + }; + const mutableExecutor = skillExecutor as unknown as { + execute: (...args: unknown[]) => Promise; + }; + const originalList = mutableRegistry.list; + const originalExecute = mutableExecutor.execute; + try { + mutableRegistry.list = () => [ + { name: "mcp_boundary_skill", description: "boundary test", enabled: true }, + ]; + const dynamicHandler = getRegisteredHandler(createMcpServer(), "skill_mcp_boundary_skill"); + mutableExecutor.execute = async () => { + throw hostile; + }; + assertPublicMcpError( + await dynamicHandler({}, { authInfo: { clientId: "test", scopes: ["execute:skills"] } }) + ); + } finally { + mutableRegistry.list = originalList; + mutableExecutor.execute = originalExecute; + } +}); + +test("skill-tool MCP wrapper uses its own fail-closed fallback for hostile thrown values", async () => { + const mutableSkillTool = Object.values(skillTools)[0] as unknown as { + name: string; + handler: (args: unknown, extra?: unknown) => Promise; + }; + const originalHandler = mutableSkillTool.handler; + const revocable = Proxy.revocable({}, {}); + revocable.revoke(); + + try { + mutableSkillTool.handler = async () => { + throw revocable.proxy; + }; + const handler = getRegisteredHandler(createMcpServer(), mutableSkillTool.name); + const result = await handler({}, { authInfo: { scopes: ["read:skills"] } }); + assert.equal(result.isError, true); + assert.equal(result.content?.[0]?.text, "Error: Skill tool execution failed"); + } finally { + mutableSkillTool.handler = originalHandler; + } +}); diff --git a/tests/unit/fixtures/provider-connection-test-error-boundaries.fixture.ts b/tests/unit/fixtures/provider-connection-test-error-boundaries.fixture.ts new file mode 100644 index 0000000000..3ab084243b --- /dev/null +++ b/tests/unit/fixtures/provider-connection-test-error-boundaries.fixture.ts @@ -0,0 +1,262 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-provider-errors-")); +const originalDataDir = process.env.DATA_DIR; +const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; +const originalApiKeySecret = process.env.API_KEY_SECRET; +const originalDisableBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; +const originalDisableHealthCheck = process.env.OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK; +const pluginsDir = path.join(testRoot, "plugins"); +const testDataDir = path.join(testRoot, "data"); +fs.mkdirSync(pluginsDir, { recursive: true }); +fs.mkdirSync(testDataDir, { recursive: true }); +process.env.OMNIROUTE_PLUGINS_DIR = pluginsDir; +process.env.DATA_DIR = testDataDir; +assert.notEqual(fs.realpathSync(testDataDir), "/home/diegosouzapw/.omniroute"); +assert.notEqual(fs.realpathSync(pluginsDir), "/home/diegosouzapw/.omniroute/plugins"); + +process.env.API_KEY_SECRET = "provider-error-boundary-test-secret"; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; +process.env.OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK = "true"; + +// Connection tests suppress their call-log entry under node --test. This file +// exercises the real persistent boundary, so present a normal runtime identity +// before importing the route and its logging modules. +const originalArgv = process.argv; +const originalExecArgv = process.execArgv; +const originalNodeEnv = process.env.NODE_ENV; +const originalVitest = process.env.VITEST; +process.argv = [ + process.execPath, + path.join(process.cwd(), "scripts/ad-hoc/omniroute-boundary-harness.mjs"), +]; +process.execArgv = []; +process.env.NODE_ENV = "development"; +delete process.env.VITEST; + +const hostileValidationMessage = + "Jules failed access_token=jules-boundary-secret at /srv/private/validator.ts\n" + + " at probe (/srv/private/validator.ts:42:7)"; +const julesValidationUrl = "https://jules.googleapis.com/v1alpha/sources"; +const originalFetch = globalThis.fetch; +let validationFetchCalls = 0; +const boundaryFetch = (async (input: string | URL | Request) => { + const url = + typeof input === "string" ? input : input instanceof Request ? input.url : input.toString(); + assert.equal(url, julesValidationUrl, `unexpected outbound request: ${url}`); + validationFetchCalls += 1; + return new Response(hostileValidationMessage, { status: 500 }); +}) as typeof fetch; +globalThis.fetch = boundaryFetch; + +const core = await import("../../../src/lib/db/core.ts"); +const providersDb = await import("../../../src/lib/db/providers.ts"); +const { saveCallLog, waitForCallLogSaves, closeCallLogSaves } = + await import("../../../src/lib/usage/callLogs.ts"); +const { flushProxyLogsSync } = await import("../../../src/lib/proxyLogger.ts"); +const { projectProviderRuntimeForPublicResponse, testSingleConnection } = + await import("../../../src/app/api/providers/[id]/test/route.ts"); +// proxyFetch installs its global dispatcher while the imports above load. Put +// the deterministic stub back at the final fetch seam so this test can never +// reach Jules over the network. +globalThis.fetch = boundaryFetch; + +type ArtifactRow = { artifact_relpath: string | null; error_summary: string | null }; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function readArtifact(relativePath: string | null): Record { + assert.ok(relativePath, "call log must have a persisted detail artifact"); + const absolutePath = path.join(testDataDir, "call_logs", relativePath); + return JSON.parse(fs.readFileSync(absolutePath, "utf8")) as Record; +} + +test.after(async () => { + await closeCallLogSaves(2_000); + flushProxyLogsSync(); + globalThis.fetch = originalFetch; + process.argv = originalArgv; + process.execArgv = originalExecArgv; + if (originalNodeEnv === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = originalNodeEnv; + if (originalVitest === undefined) delete process.env.VITEST; + else process.env.VITEST = originalVitest; + if (originalPluginsDir === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = originalPluginsDir; + core.resetDbInstance(); + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + if (originalApiKeySecret === undefined) delete process.env.API_KEY_SECRET; + else process.env.API_KEY_SECRET = originalApiKeySecret; + if (originalDisableBackup === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = originalDisableBackup; + if (originalDisableHealthCheck === undefined) { + delete process.env.OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK; + } else { + process.env.OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK = originalDisableHealthCheck; + } + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("public runtime projection omits host paths and internal error envelopes", () => { + const projected = projectProviderRuntimeForPublicResponse({ + installed: true, + runnable: false, + requiresBinary: true, + reason: "not_executable", + runtimeMode: "local", + version: "v1 from /srv/private/bin/tool", + command: "/srv/private/bin/tool", + commandPath: "/srv/private/bin/tool", + settingsPath: "C:\\Users\\admin\\.config\\tool.json", + error: "access_token=runtime-secret at /srv/private/runtime.json", + diagnosis: { message: "runtime-secret at /srv/private/runtime.ts" }, + }); + const serialized = JSON.stringify(projected); + + assert.equal(projected?.installed, true); + assert.equal(projected?.runnable, false); + assert.equal("commandPath" in (projected || {}), false); + assert.equal("settingsPath" in (projected || {}), false); + assert.equal("error" in (projected || {}), false); + assert.equal("diagnosis" in (projected || {}), false); + assert.doesNotMatch(serialized, /runtime-secret|srv\/private|C:\\\\Users/i); +}); + +test("connection validation projects hostile errors before public and persistent boundaries", async () => { + const connection = await providersDb.createProviderConnection({ + provider: "jules", + authType: "apikey", + name: "Jules Error Boundary", + apiKey: "jules-test-key", + isActive: true, + testStatus: "active", + }); + assert.ok(connection?.id); + + const result = await testSingleConnection(connection.id); + assert.equal(result.valid, false); + assert.ok(validationFetchCalls > 0, "the deterministic Jules stub must handle the probe"); + assert.match(String(result.error), /Jules failed/i); + assert.equal(await waitForCallLogSaves(10_000), true, "call-log write must drain"); + flushProxyLogsSync(); + + const db = core.getDbInstance(); + const providerRow = db + .prepare("SELECT last_error FROM provider_connections WHERE id = ?") + .get(connection.id) as { last_error: string | null }; + const callLogRow = db + .prepare( + `SELECT error_summary, artifact_relpath + FROM call_logs + WHERE connection_id = ? AND model = 'connection-test' + ORDER BY rowid DESC LIMIT 1` + ) + .get(connection.id) as ArtifactRow; + const proxyLogRow = db + .prepare( + `SELECT error + FROM proxy_logs + WHERE connection_id = ? AND provider = 'jules' + AND target_url = 'jules/connection-test' + ORDER BY rowid DESC LIMIT 1` + ) + .get(connection.id) as { error: string | null }; + assert.ok(callLogRow, "connection test must write call_logs"); + assert.ok(proxyLogRow, "connection test must write proxy_logs"); + const artifact = readArtifact(callLogRow.artifact_relpath); + + const boundaries = { + publicResult: result, + providerLastError: providerRow.last_error, + callLogSummary: callLogRow.error_summary, + callLogArtifactError: artifact.error, + proxyLogError: proxyLogRow.error, + }; + const leakPattern = /jules-boundary-secret|srv\/private|validator\.ts|\bat probe\b/i; + const leakingBoundaries = Object.entries(boundaries) + .filter(([, value]) => leakPattern.test(JSON.stringify(value))) + .map(([name]) => name); + assert.deepEqual(leakingBoundaries, []); +}); + +test("failed call logs sanitize response-body copies while successful bodies stay unchanged", async () => { + const hostileBody = { + message: "access_token=call-body-secret at /srv/private/upstream.json", + detail: "Error: api_key=call-detail-secret\n at dispatch (/srv/private/rerank.ts:7:2)", + }; + const successBody = { + message: "Successful output mentions /tmp/public-example.ts and remains unchanged", + usage: { total_tokens: 4 }, + }; + + await saveCallLog({ + id: "error-body-json", + status: 502, + provider: "rerank-test", + model: "rerank-test", + responseBody: hostileBody, + pipelinePayloads: { + providerResponse: { body: hostileBody }, + clientResponse: { body: hostileBody }, + }, + }); + await saveCallLog({ + id: "error-body-text", + status: 503, + provider: "rerank-test", + model: "rerank-test", + responseBody: "Bearer plaintext-body-secret at C:\\Users\\admin\\upstream.txt", + }); + await saveCallLog({ + id: "success-body-control", + status: 200, + provider: "rerank-test", + model: "rerank-test", + responseBody: successBody, + pipelinePayloads: { + providerResponse: { body: successBody }, + clientResponse: { body: successBody }, + }, + }); + await saveCallLog({ + id: "error-body-binary", + status: 500, + provider: "rerank-test", + model: "rerank-test", + responseBody: Buffer.from([1, 2, 3, 4]), + }); + assert.equal(await waitForCallLogSaves(2_000), true, "call-log writes must drain"); + + const db = core.getDbInstance(); + const rows = db + .prepare( + `SELECT id, artifact_relpath FROM call_logs + WHERE id IN ( + 'error-body-json', 'error-body-text', 'success-body-control', 'error-body-binary' + )` + ) + .all() as Array<{ id: string; artifact_relpath: string | null }>; + const artifacts = Object.fromEntries( + rows.map((row) => [row.id, readArtifact(row.artifact_relpath)]) + ) as Record>; + + assert.doesNotMatch( + JSON.stringify({ json: artifacts["error-body-json"], text: artifacts["error-body-text"] }), + /call-body-secret|call-detail-secret|plaintext-body-secret|srv\/private|C:\\\\Users|\bat dispatch\b/i + ); + assert.deepEqual(artifacts["success-body-control"].responseBody, successBody); + assert.equal(artifacts["error-body-binary"].responseBody, "[binary 4 bytes]"); + const pipeline = artifacts["success-body-control"].pipeline; + assert.ok(isRecord(pipeline)); + assert.ok(isRecord(pipeline.providerResponse)); + assert.ok(isRecord(pipeline.clientResponse)); + assert.deepEqual(pipeline.providerResponse.body, successBody); + assert.deepEqual(pipeline.clientResponse.body, successBody); +}); diff --git a/tests/unit/fixtures/provider-last-error-sanitization.fixture.ts b/tests/unit/fixtures/provider-last-error-sanitization.fixture.ts new file mode 100644 index 0000000000..b131bdc67d --- /dev/null +++ b/tests/unit/fixtures/provider-last-error-sanitization.fixture.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-provider-last-error-")); +const testDataDir = path.join(testRoot, "data"); +const testPluginsDir = path.join(testRoot, "plugins"); +const originalEnv = { + DATA_DIR: process.env.DATA_DIR, + OMNIROUTE_PLUGINS_DIR: process.env.OMNIROUTE_PLUGINS_DIR, + API_KEY_SECRET: process.env.API_KEY_SECRET, + DISABLE_SQLITE_AUTO_BACKUP: process.env.DISABLE_SQLITE_AUTO_BACKUP, +}; +fs.mkdirSync(testDataDir, { recursive: true }); +fs.mkdirSync(testPluginsDir, { recursive: true }); +process.env.DATA_DIR = testDataDir; +process.env.OMNIROUTE_PLUGINS_DIR = testPluginsDir; +process.env.API_KEY_SECRET = "provider-last-error-test-secret"; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../../src/lib/db/core.ts"); +const providersDb = await import("../../../src/lib/db/providers.ts"); +const loggerResource = await import("../../../src/shared/utils/loggerResource.ts"); +const { runAsProbe } = await import("../../../src/shared/utils/probeOrigin.ts"); +const { writeTerminalStatus } = await import("../../../src/shared/utils/terminalStatus.ts"); +const { markAccountUnavailable } = await import("../../../src/sse/services/auth.ts"); + +function restoreEnv(name: keyof typeof originalEnv): void { + const original = originalEnv[name]; + if (original === undefined) delete process.env[name]; + else process.env[name] = original; +} + +function readLastError(connectionId: string): string | null { + const row = core + .getDbInstance() + .prepare("SELECT last_error FROM provider_connections WHERE id = ?") + .get(connectionId) as { last_error: string | null } | undefined; + return row?.last_error ?? null; +} + +test.after(async () => { + core.resetDbInstance(); + await loggerResource.closeSharedLoggerResource(); + restoreEnv("DATA_DIR"); + restoreEnv("OMNIROUTE_PLUGINS_DIR"); + restoreEnv("API_KEY_SECRET"); + restoreEnv("DISABLE_SQLITE_AUTO_BACKUP"); + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("normal and probe failures sanitize provider_connections.lastError at the write seam", async () => { + const hostile = + "provider failed access_token=provider-last-error-secret at /srv/private/provider.ts\n" + + " at dispatch (/srv/private/provider.ts:12:4)"; + const normal = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "normal last-error boundary", + apiKey: "normal-last-error-test-key", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + const probe = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "probe last-error boundary", + apiKey: "probe-last-error-test-key", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + const terminal = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "terminal last-error boundary", + apiKey: "terminal-last-error-test-key", // pragma: allowlist secret + isActive: true, + testStatus: "active", + }); + + await markAccountUnavailable(normal.id, 500, hostile, "openai"); + await runAsProbe(() => markAccountUnavailable(probe.id, 500, hostile, "openai")); + await writeTerminalStatus( + terminal.id, + { + testStatus: "banned", + isActive: false, + lastError: hostile, + lastErrorType: "forbidden", + errorCode: "403", + }, + "production" + ); + + const persisted = { + normal: readLastError(normal.id), + probe: readLastError(probe.id), + terminal: readLastError(terminal.id), + }; + assert.match(String(persisted.normal), /provider failed/i); + assert.match(String(persisted.probe), /provider failed/i); + assert.match(String(persisted.terminal), /provider failed/i); + assert.doesNotMatch( + JSON.stringify(persisted), + /provider-last-error-secret|srv\/private|provider\.ts|\bat dispatch\b/i + ); +}); diff --git a/tests/unit/fixtures/request-log-management-boundary.fixture.ts b/tests/unit/fixtures/request-log-management-boundary.fixture.ts new file mode 100644 index 0000000000..e3b75b22a9 --- /dev/null +++ b/tests/unit/fixtures/request-log-management-boundary.fixture.ts @@ -0,0 +1,108 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-management-boundary-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +const ORIGINAL_DISABLE_BACKUP = process.env.DISABLE_SQLITE_AUTO_BACKUP; + +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "1"; + +const core = await import("../../../src/lib/db/core.ts"); +const usageHistory = await import("../../../src/lib/usage/usageHistory.ts"); +const logsRoute = await import("../../../src/app/api/logs/[id]/route.ts"); +const usageHistoryRoute = await import("../../../src/app/api/usage/history/route.ts"); + +test.afterEach(() => { + usageHistory.clearPendingRequests(); +}); + +test.after(() => { + usageHistory.clearPendingRequests(); + core.resetDbInstance(); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + if (ORIGINAL_DISABLE_BACKUP === undefined) delete process.env.DISABLE_SQLITE_AUTO_BACKUP; + else process.env.DISABLE_SQLITE_AUTO_BACKUP = ORIGINAL_DISABLE_BACKUP; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +const HOSTILE = + "Bearer management-cache-secret at /srv/private/completed-request.ts:12:3\n" + + " at finalize (/srv/private/finalize.ts:4:2)"; + +async function readManagementDetail(id: string): Promise> { + const response = await logsRoute.GET(undefined as unknown as Request, { params: { id } }); + assert.equal(response.status, 200); + return (await response.json()) as Record; +} + +function serializedDetail(detail: Record): string { + return JSON.stringify(detail); +} + +test("management detail sanitizes in-flight failure chunks at the endpoint boundary", async () => { + const requestId = usageHistory.trackPendingRequest("model", "provider", "conn-inflight", true); + assert.ok(requestId); + usageHistory.updatePendingRequestStreamChunks("model", "provider", "conn-inflight", { + provider: [`event: error\ndata: ${HOSTILE}\n\n`], + openai: [], + client: [], + }); + + const detail = await readManagementDetail(requestId); + assert.doesNotMatch( + serializedDetail(detail), + /management-cache-secret|srv\/private|completed-request\.ts|\bat finalize\b/i + ); +}); + +test("management detail sanitizes completed error metadata and cached chunks", async () => { + const requestId = usageHistory.trackPendingRequest("model", "provider", "conn-completed", true); + assert.ok(requestId); + usageHistory.updatePendingRequestStreamChunks("model", "provider", "conn-completed", { + provider: [`data: ${JSON.stringify({ type: "error", message: HOSTILE })}\n\n`], + openai: [], + client: [], + }); + assert.equal( + usageHistory.finalizePendingRequestById(requestId, { status: 502, error: HOSTILE }), + true + ); + + const detail = await readManagementDetail(requestId); + assert.doesNotMatch( + serializedDetail(detail), + /management-cache-secret|srv\/private|completed-request\.ts|\bat finalize\b/i + ); +}); + +test("usage history endpoint exposes pending counters without raw request details", async () => { + const requestId = usageHistory.trackPendingRequest("model", "provider", "conn-usage", true); + assert.ok(requestId); + usageHistory.updatePendingRequestStreamChunks("model", "provider", "conn-usage", { + provider: [`event: error\ndata: ${HOSTILE}\n\n`], + openai: [], + client: [], + }); + + const response = await usageHistoryRoute.GET(undefined as unknown as Request); + assert.equal(response.status, 200); + const body = (await response.json()) as { + pending?: { byModel?: Record; details?: unknown }; + }; + assert.equal(body.pending?.byModel?.["model (provider)"], 1); + assert.equal("details" in (body.pending ?? {}), false); + assert.doesNotMatch(JSON.stringify(body), /management-cache-secret|srv\/private/i); +}); diff --git a/tests/unit/fixtures/stream-failure-persistent-classification.fixture.ts b/tests/unit/fixtures/stream-failure-persistent-classification.fixture.ts new file mode 100644 index 0000000000..c589a4bd02 --- /dev/null +++ b/tests/unit/fixtures/stream-failure-persistent-classification.fixture.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-stream-failure-code-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; + +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; + +const core = await import("../../../src/lib/db/core.ts"); +const failureUsage = await import("../../../open-sse/handlers/chatCore/failureUsage.ts"); +const usageHistory = await import("../../../src/lib/usage/usageHistory.ts"); +const { createStreamFailureFinalizers } = + await import("../../../open-sse/utils/streamFailureFinalization.ts"); + +test.after(() => { + core.resetDbInstance(); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("stream failure persists only the projected public classification", () => { + const opaqueCode = "opaque-stream-code-secret-9382746"; + let completionCode: string | null | undefined; + let persistedCode: string | undefined; + let classifierCode: string | undefined; + const { handleStreamFailure } = createStreamFailureFinalizers({ + isFailureCompletionRecorded: () => false, + onStreamComplete: (payload) => { + completionCode = payload.errorCode; + }, + persistFailureUsage: (_status, errorCode) => { + persistedCode = errorCode; + }, + onStreamFailure: (failure) => { + classifierCode = failure.code; + }, + }); + + assert.equal( + handleStreamFailure({ status: 502, message: "upstream failed", code: opaqueCode }), + true + ); + assert.equal(completionCode, "bad_gateway"); + assert.equal(persistedCode, "bad_gateway"); + assert.equal(classifierCode, opaqueCode); +}); + +test("pre-response failures persist only the projected public classification", async () => { + const opaqueCode = "opaque-pre-response-code-secret-6382951"; + const projectedCode = failureUsage.projectFailureUsageErrorCode({ + statusCode: 502, + message: "upstream request failed", + errorCode: opaqueCode, + errorType: "opaque-pre-response-type-secret-9472013", + }); + + assert.equal(projectedCode, "bad_gateway"); + + const provider = "persistent-error-code-boundary"; + await usageHistory.saveRequestUsage( + failureUsage.buildFailureUsageRecord({ + provider, + model: "model", + connectionId: null, + apiKeyInfo: null, + effectiveServiceTier: "standard", + isCombo: false, + comboStrategy: null, + statusCode: 502, + errorCode: projectedCode, + latencyMs: 1, + }) + ); + + const rows = await usageHistory.getUsageHistory({ provider }); + assert.equal(rows.length, 1); + assert.equal(rows[0]?.errorCode, "bad_gateway"); + assert.doesNotMatch(JSON.stringify(rows), /opaque-pre-response|6382951|9472013/); +}); diff --git a/tests/unit/free-note-freshness.test.ts b/tests/unit/free-note-freshness.test.ts index 1f24629218..67769aa00b 100644 --- a/tests/unit/free-note-freshness.test.ts +++ b/tests/unit/free-note-freshness.test.ts @@ -14,6 +14,9 @@ test("longcat freeNote reflects the post-2026-05-29 5M tokens/day reality", () = assert.match(note("longcat"), /5M tokens\/day|LongCat-2\.0/i); }); -test("cerebras freeNote reflects the tightened 30K TPM", () => { - assert.match(note("cerebras"), /30K TPM|1M tokens\/day/i); +test("cerebras freeNote reflects the $5 card-gated signup credit (#11773)", () => { + const n = note("cerebras"); + assert.match(n, /\$5/); + assert.match(n, /payment method|credit card/i); + assert.equal(/1M tokens\/day|30K TPM/.test(n), false); }); diff --git a/tests/unit/free-tier-catalog.test.ts b/tests/unit/free-tier-catalog.test.ts index 6f4b23e356..25282d5ef1 100644 --- a/tests/unit/free-tier-catalog.test.ts +++ b/tests/unit/free-tier-catalog.test.ts @@ -7,13 +7,14 @@ import { } from "../../open-sse/config/freeTierCatalog.ts"; test("FREE_TIER_BUDGETS holds positive integer monthly-token budgets", () => { - assert.ok(Object.keys(FREE_TIER_BUDGETS).length >= 19); + assert.ok(Object.keys(FREE_TIER_BUDGETS).length >= 18); for (const [id, tokens] of Object.entries(FREE_TIER_BUDGETS)) { assert.ok(Number.isInteger(tokens) && tokens > 0, `${id} must be a positive integer`); } assert.equal(FREE_TIER_BUDGETS.mistral, 1_000_000_000); assert.equal(FREE_TIER_BUDGETS["cloudflare-ai"], 122_000_000); - assert.equal(FREE_TIER_BUDGETS.cerebras, 30_000_000); + // #11773: Cerebras is a one-time $5 signup credit, not a recurring monthly grant. + assert.equal(FREE_TIER_BUDGETS.cerebras, undefined); // LongCat is excluded from this recurring-monthly catalog: its free tier is a // one-time 10M-token signup grant (not recurring), so it must not appear here. assert.equal(FREE_TIER_BUDGETS.longcat, undefined); @@ -27,9 +28,9 @@ test("FREE_TIER_TOS marks proxy-prohibited providers as avoid", () => { test("computeFreeTierTotals sums the documented budgets", () => { const t = computeFreeTierTotals(); - assert.equal(t.providerCount, 19); - assert.ok(t.documentedMonthlyTokens >= 1_350_000_000); - assert.ok(t.documentedMonthlyTokens <= 1_450_000_000); + assert.equal(t.providerCount, 18); + assert.ok(t.documentedMonthlyTokens >= 1_320_000_000); + assert.ok(t.documentedMonthlyTokens <= 1_420_000_000); assert.equal(typeof t.headline, "string"); assert.match(t.headline, /1\.3/); }); @@ -38,5 +39,5 @@ test("computeFreeTierTotals can exclude ToS-avoid providers", () => { const all = computeFreeTierTotals(); const clean = computeFreeTierTotals({ excludeTosAvoid: true }); assert.equal(all.documentedMonthlyTokens - clean.documentedMonthlyTokens, 25_000); - assert.equal(clean.providerCount, 18); + assert.equal(clean.providerCount, 17); }); diff --git a/tests/unit/gemini-responses-error-redaction.test.ts b/tests/unit/gemini-responses-error-redaction.test.ts new file mode 100644 index 0000000000..f8814b3e8c --- /dev/null +++ b/tests/unit/gemini-responses-error-redaction.test.ts @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { translateResponse, initState } from "../../open-sse/translator/index.ts"; +import { FORMATS } from "../../open-sse/translator/formats.ts"; + +test("Gemini keeps raw failure wording internal but projects response.completed.error", () => { + const state = initState(FORMATS.OPENAI_RESPONSES); + const hostileMessage = + "Gemini failed at /srv/omniroute/private-runtime.ts:71:3 token=sk-gemini-secret-123456"; + + const translated = translateResponse( + FORMATS.GEMINI, + FORMATS.OPENAI_RESPONSES, + { + response: { + error: { + code: 503, + status: "UNAVAILABLE", + message: hostileMessage, + api_key: "sk-gemini-secret-abcdef", + }, + }, + }, + state + ); + assert.equal(translated?.length ?? 0, 0); + assert.match(state.upstreamError?.message ?? "", /private-runtime\.ts/); + + const flushed = translateResponse(FORMATS.GEMINI, FORMATS.OPENAI_RESPONSES, null, state); + const completed = flushed.find((event) => event?.data?.type === "response.completed"); + assert.ok(completed); + assert.equal(completed.data.response.status, "failed"); + + const publicError = JSON.stringify(completed.data.response.error); + assert.doesNotMatch(publicError, /private-runtime\.ts/); + assert.doesNotMatch(publicError, /sk-gemini-secret/); + assert.doesNotMatch(publicError, /api_key/); + assert.equal(completed.data.response.error.code, "503"); +}); diff --git a/tests/unit/grok-web-stream-error-boundary.test.ts b/tests/unit/grok-web-stream-error-boundary.test.ts new file mode 100644 index 0000000000..afc9eb5e37 --- /dev/null +++ b/tests/unit/grok-web-stream-error-boundary.test.ts @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); +const fixturePath = fileURLToPath( + new URL("../fixtures/grok-web-stream-error-boundary-child.ts", import.meta.url) +); + +type FixtureResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function runFixture(): Promise { + // Keep the parent process pristine: test:unit:fast runs files with + // --test-isolation=none, so repository imports or env/DB mutations here can + // collide with unrelated tests. All stateful coverage lives in the child. + const childEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + NODE_PATH: process.env.NODE_PATH, + LANG: process.env.LANG, + LC_ALL: process.env.LC_ALL, + TZ: process.env.TZ, + TMPDIR: process.env.TMPDIR, + NODE_ENV: "test", + API_KEY_SECRET: "grok-boundary-test-only-secret-with-32-plus-characters", + DISABLE_SQLITE_AUTO_BACKUP: "true", + NO_COLOR: "1", + }; + // An inherited marker makes Node treat this nested --test run as recursive + // and silently skip the fixture instead of executing its seven regressions. + delete childEnv.NODE_TEST_CONTEXT; + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx/esm", "--test", fixturePath], { + cwd: repoRoot, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + + const timeout = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, 120_000); + + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("close", (code, signal) => { + clearTimeout(timeout); + if (timedOut) { + reject(new Error("Grok Web stream error boundary fixture timed out after 120 seconds")); + return; + } + resolve({ code, signal, stdout, stderr }); + }); + }); +} + +test("Grok Web stream error boundary passes in a process-isolated runtime", async () => { + const result = await runFixture(); + const output = `${result.stdout}\n${result.stderr}`; + + assert.equal(result.signal, null, output.slice(-12_000)); + assert.equal(result.code, 0, output.slice(-12_000)); + assert.match(output, /(?:^|\s)tests\s+7(?:\s|$)/m); + assert.match(output, /(?:^|\s)pass\s+7(?:\s|$)/m); + assert.match(output, /(?:^|\s)fail\s+0(?:\s|$)/m); +}); diff --git a/tests/unit/helpers/runIsolatedBoundaryFixture.ts b/tests/unit/helpers/runIsolatedBoundaryFixture.ts new file mode 100644 index 0000000000..07cc672e28 --- /dev/null +++ b/tests/unit/helpers/runIsolatedBoundaryFixture.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); +const CHILD_PATH = "/usr/local/bin:/usr/bin:/bin"; +const CHILD_MAX_BUFFER_BYTES = 10 * 1024 * 1024; + +type IsolatedBoundaryFixtureOptions = { + fixtureUrl: URL; + expectedTests: number; + label: string; + timeoutMs?: number; +}; + +export function runIsolatedBoundaryFixture({ + fixtureUrl, + expectedTests, + label, + timeoutMs = 180_000, +}: IsolatedBoundaryFixtureOptions): void { + const root = mkdtempSync(join(tmpdir(), "omniroute-public-error-child-")); + const dataDir = join(root, "data"); + const pluginsDir = join(root, "plugins"); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(pluginsDir, { recursive: true }); + + try { + const result = spawnSync( + process.execPath, + ["--import", "tsx/esm", "--test", "--test-reporter=tap", fileURLToPath(fixtureUrl)], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: "public-error-boundary-fixture-secret", + DATA_DIR: dataDir, + DISABLE_SQLITE_AUTO_BACKUP: "true", + LANG: "C.UTF-8", + LC_ALL: "C.UTF-8", + NODE_ENV: "test", + OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK: "true", + OMNIROUTE_PLUGINS_DIR: pluginsDir, + PATH: CHILD_PATH, + TZ: "UTC", + }, + maxBuffer: CHILD_MAX_BUFFER_BYTES, + timeout: timeoutMs, + } + ); + const diagnostics = [ + `${label} child status=${String(result.status)} signal=${String(result.signal)}`, + result.error ? `error=${String(result.error)}` : "", + `stdout:\n${result.stdout}`, + `stderr:\n${result.stderr}`, + ] + .filter(Boolean) + .join("\n"); + + assert.equal(result.error, undefined, diagnostics); + assert.equal(result.signal, null, diagnostics); + assert.equal(result.status, 0, diagnostics); + assert.match(result.stdout, new RegExp(`# tests ${expectedTests}(?:\\r?\\n|$)`), diagnostics); + assert.match(result.stdout, new RegExp(`# pass ${expectedTests}(?:\\r?\\n|$)`), diagnostics); + assert.match(result.stdout, /# fail 0(?:\r?\n|$)/, diagnostics); + } finally { + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +} diff --git a/tests/unit/huggingchat-stream-error-boundary.test.ts b/tests/unit/huggingchat-stream-error-boundary.test.ts new file mode 100644 index 0000000000..5fe1e9f545 --- /dev/null +++ b/tests/unit/huggingchat-stream-error-boundary.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const FIXTURE = fileURLToPath( + new URL("./_fixtures/huggingchat-stream-error-boundary.fixture.ts", import.meta.url) +); +const SYNTHETIC_API_KEY_SECRET = "0".repeat(64); + +type ChildResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function runFixture(testRoot: string): Promise { + const dataDir = join(testRoot, "data"); + const pluginsDir = join(testRoot, "plugins"); + // Keep config fallbacks inside the fixture root without inheriting or repurposing HOME. + const xdgConfigDir = join(testRoot, "xdg-config"); + + for (const dir of [dataDir, pluginsDir, xdgConfigDir]) { + mkdirSync(dir, { recursive: true }); + } + + const childEnv: NodeJS.ProcessEnv = { + API_KEY_SECRET: SYNTHETIC_API_KEY_SECRET, + APP_LOG_TO_FILE: "false", + DATA_DIR: dataDir, + DISABLE_SQLITE_AUTO_BACKUP: "true", + FORCE_COLOR: "0", + LANG: "C.UTF-8", + NODE_ENV: "test", + OMNIROUTE_HUGGINGCHAT_TEST_ROOT: testRoot, + OMNIROUTE_HUGGINGCHAT_TEST_RUN_ID: basename(testRoot), + OMNIROUTE_PLUGINS_DIR: pluginsDir, + TZ: "UTC", + XDG_CONFIG_HOME: xdgConfigDir, + }; + if (process.env.PATH) childEnv.PATH = process.env.PATH; + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx/esm", FIXTURE], { + cwd: REPO_ROOT, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk)); + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +function childDiagnostics(result: ChildResult): string { + return [ + `exit=${String(result.code)} signal=${String(result.signal)}`, + "--- stdout ---", + result.stdout, + "--- stderr ---", + result.stderr, + ].join("\n"); +} + +test("HuggingChat stream error boundaries stay isolated from shared DB and usage state", async () => { + const testRoot = mkdtempSync(join(tmpdir(), "omniroute-huggingchat-boundary-child-")); + try { + const result = await runFixture(testRoot); + assert.equal(result.signal, null, childDiagnostics(result)); + assert.equal(result.code, 0, childDiagnostics(result)); + assert.match(result.stdout, /(?:#|ℹ) pass 8\b/, childDiagnostics(result)); + assert.match(result.stdout, /(?:#|ℹ) fail 0\b/, childDiagnostics(result)); + assert.doesNotMatch(result.stdout + result.stderr, /super-secret/); + } finally { + rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); diff --git a/tests/unit/huggingchat-transport-error-redaction.test.ts b/tests/unit/huggingchat-transport-error-redaction.test.ts new file mode 100644 index 0000000000..e13253a588 --- /dev/null +++ b/tests/unit/huggingchat-transport-error-redaction.test.ts @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; + +const CHILD_RESULT_PREFIX = "HUGGINGCHAT_TRANSPORT_RESULT="; +const childFixture = fileURLToPath( + new URL("./_fixtures/huggingchat-transport-error-child.ts", import.meta.url) +); + +type TransportScenario = "conversation-creation" | "message-send"; + +type TransportFailureResult = { + fetchCalls: number; + status: number; + contentType: string; + errorLogs: string[]; + payload: { + error: { + message: string; + type?: string; + code?: string; + }; + }; +}; + +function runTransportScenario(scenario: TransportScenario): TransportFailureResult { + const result = spawnSync(process.execPath, ["--import", "tsx/esm", childFixture, scenario], { + cwd: process.cwd(), + encoding: "utf8", + timeout: 60_000, + env: { + NODE_ENV: "test", + NO_COLOR: "1", + DISABLE_SQLITE_AUTO_BACKUP: "true", + }, + }); + + assert.equal( + result.status, + 0, + `isolated ${scenario} probe failed: ${String(result.stderr).slice(0, 2_000)}` + ); + + const resultLine = String(result.stdout) + .split("\n") + .findLast((line) => line.startsWith(CHILD_RESULT_PREFIX)); + assert.ok(resultLine, `isolated ${scenario} probe did not emit its result`); + + return JSON.parse(resultLine.slice(CHILD_RESULT_PREFIX.length)) as TransportFailureResult; +} + +function assertPublicFailureIsSanitized(result: TransportFailureResult): void { + assert.equal(result.status, 502); + assert.match(result.contentType, /application\/json/); + assert.equal(result.payload.error.type, "upstream_error"); + assert.match(result.payload.error.message, /^HuggingChat connection failed:/); + assert.match(result.payload.error.message, //); + assert.match(result.payload.error.message, /access_token=\[REDACTED\]/); + + const publicText = JSON.stringify({ payload: result.payload, errorLogs: result.errorLogs }); + assert.doesNotMatch(publicText, /transport-secret/); + assert.doesNotMatch(publicText, /\/srv\/omniroute/); + assert.doesNotMatch(publicText, /sendRequest/); + assert.doesNotMatch(publicText, /\n\s*at /); +} + +test("HuggingChat sanitizes conversation-creation transport failures in body and log", () => { + const result = runTransportScenario("conversation-creation"); + + assert.equal(result.fetchCalls, 1, "the probe must intercept the conversation creation request"); + assert.equal(result.errorLogs.length, 1); + assert.match(result.errorLogs[0], /^Conversation creation failed:/); + assertPublicFailureIsSanitized(result); +}); + +test("HuggingChat sanitizes message-send transport failures in body and log", () => { + const result = runTransportScenario("message-send"); + + assert.equal(result.fetchCalls, 3, "the probe must intercept creation, parent lookup, and send"); + assert.equal(result.errorLogs.length, 1); + assert.match(result.errorLogs[0], /^Message send failed:/); + assertPublicFailureIsSanitized(result); +}); diff --git a/tests/unit/mcp-public-error-boundaries.test.ts b/tests/unit/mcp-public-error-boundaries.test.ts new file mode 100644 index 0000000000..92095c8f85 --- /dev/null +++ b/tests/unit/mcp-public-error-boundaries.test.ts @@ -0,0 +1,11 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("MCP public error boundaries pass in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL("./fixtures/mcp-public-error-boundaries.fixture.ts", import.meta.url), + expectedTests: 4, + label: "MCP public error boundaries", + }); +}); diff --git a/tests/unit/moderations-handler.test.ts b/tests/unit/moderations-handler.test.ts index 68ec32847a..640d0a68bc 100644 --- a/tests/unit/moderations-handler.test.ts +++ b/tests/unit/moderations-handler.test.ts @@ -2,9 +2,8 @@ import test from "node:test"; import assert from "node:assert/strict"; const { handleModeration } = await import("../../open-sse/handlers/moderations.ts"); -const { MODERATION_PROVIDERS, getModerationProvider, parseModerationModel } = await import( - "../../open-sse/config/moderationRegistry.ts" -); +const { MODERATION_PROVIDERS, getModerationProvider, parseModerationModel } = + await import("../../open-sse/config/moderationRegistry.ts"); const originalFetch = globalThis.fetch; @@ -136,6 +135,76 @@ test("handleModeration returns upstream error payloads with CORS headers", async assert.match(response.headers.get("access-control-allow-methods") || "", /OPTIONS/); }); +test("handleModeration sanitizes structured upstream error bodies", async () => { + globalThis.fetch = async () => + Response.json( + { + error: { + message: "quota metadata at /srv/provider/private.json", + api_key: "credential-value-12345", + }, + }, + { status: 429 } + ); + + const response = await handleModeration({ + body: { model: "openai/text-moderation-latest", input: "check this" }, + credentials: { apiKey: "sk-test" }, + }); + const payload = (await response.json()) as { + error: { message: string; api_key?: string }; + }; + + assert.equal(response.status, 429); + assert.equal(payload.error.api_key, undefined); + assert.doesNotMatch(payload.error.message, /srv\/provider/i); + assert.doesNotMatch(JSON.stringify(payload), /credential-value-12345/i); +}); + +test("handleModeration canonicalizes blank, plaintext, and mislabeled upstream failures", async () => { + const scenarios = [ + { name: "blank", body: " ", contentType: "application/json" }, + { + name: "plaintext", + body: "access_token=moderation-plain-secret at /srv/private/moderation.txt", + contentType: "text/plain", + }, + { + name: "mislabeled", + body: "api_key=moderation-html-secret at /srv/private/error.html", + contentType: "application/json", + }, + ]; + + for (const scenario of scenarios) { + globalThis.fetch = async () => + new Response(scenario.body, { + status: 502, + headers: { "content-type": scenario.contentType }, + }); + const response = await handleModeration({ + body: { model: "openai/text-moderation-latest", input: "check this" }, + credentials: { apiKey: "sk-test" }, + }); + const text = await response.text(); + const payload = JSON.parse(text) as { error: { message: string } }; + + assert.equal(response.status, 502, scenario.name); + assert.match(response.headers.get("content-type") || "", /application\/json/i, scenario.name); + assert.match( + response.headers.get("access-control-allow-methods") || "", + /OPTIONS/, + scenario.name + ); + assert.equal(typeof payload.error.message, "string", scenario.name); + assert.doesNotMatch( + text, + /moderation-plain-secret|moderation-html-secret|srv\/private|/i, + scenario.name + ); + } +}); + test("handleModeration returns a 500 when the upstream request throws", async () => { globalThis.fetch = async () => { throw new Error("socket closed"); diff --git a/tests/unit/ocr-handler-dispatch.test.ts b/tests/unit/ocr-handler-dispatch.test.ts index 2474f6b0b2..df8495b095 100644 --- a/tests/unit/ocr-handler-dispatch.test.ts +++ b/tests/unit/ocr-handler-dispatch.test.ts @@ -38,6 +38,86 @@ test("mistral path posts once and returns the upstream body", async () => { assert.equal(data.pages[0].markdown, "ok"); }); +test("OCR sanitizes structured upstream error bodies", async () => { + const opaqueIdentifier = "AbC9xY7pQ2mN8vR4kL6z"; + const res = await handleOcr({ + body: { + model: "mistral/mistral-ocr-latest", + document: { type: "image_url", image_url: "https://x/y.png" }, + }, + credentials: { apiKey: "sk" }, + fetchImpl: async () => + Response.json( + { + error: { + message: "quota metadata at /srv/provider/private.json", + type: opaqueIdentifier, + code: opaqueIdentifier, + reason: opaqueIdentifier, + api_key: "credential-value-12345", + }, + }, + { status: 429 } + ), + sleepImpl: noSleep, + }); + const payload = (await res.json()) as { + error: { message: string; type?: string; code?: string; reason?: string; api_key?: string }; + }; + + assert.equal(res.status, 429); + assert.equal(payload.error.api_key, undefined); + assert.doesNotMatch(payload.error.message, /srv\/provider/i); + assert.doesNotMatch( + JSON.stringify(payload), + new RegExp(`credential-value-12345|${opaqueIdentifier}`, "i") + ); +}); + +test("OCR canonicalizes blank, plaintext, and mislabeled upstream failures", async () => { + const scenarios = [ + { name: "blank", body: " ", contentType: "application/json" }, + { + name: "plaintext", + body: "access_token=ocr-plain-secret at /srv/private/ocr.txt", + contentType: "text/plain", + }, + { + name: "mislabeled", + body: "api_key=ocr-html-secret at /srv/private/ocr.html", + contentType: "application/json", + }, + ]; + + for (const scenario of scenarios) { + const res = await handleOcr({ + body: { + model: "mistral/mistral-ocr-latest", + document: { type: "image_url", image_url: "https://x/y.png" }, + }, + credentials: { apiKey: "sk" }, + fetchImpl: async () => + new Response(scenario.body, { + status: 502, + headers: { "content-type": scenario.contentType }, + }), + sleepImpl: noSleep, + }); + const text = await res.text(); + const payload = JSON.parse(text) as { error: { message: string } }; + + assert.equal(res.status, 502, scenario.name); + assert.match(res.headers.get("content-type") || "", /application\/json/i, scenario.name); + assert.match(res.headers.get("access-control-allow-methods") || "", /OPTIONS/, scenario.name); + assert.equal(typeof payload.error.message, "string", scenario.name); + assert.doesNotMatch( + text, + /ocr-plain-secret|ocr-html-secret|srv\/private|/i, + scenario.name + ); + } +}); + test("azure DI path polls Operation-Location until succeeded", async () => { const { impl, calls } = fetchStub([ { status: 202, headers: { "Operation-Location": "https://poll/op/1" } }, diff --git a/tests/unit/oneminai-stream-error-boundary.test.ts b/tests/unit/oneminai-stream-error-boundary.test.ts new file mode 100644 index 0000000000..88cc402e6e --- /dev/null +++ b/tests/unit/oneminai-stream-error-boundary.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const fixturePath = fileURLToPath( + new URL("../fixtures/oneminai-stream-error-boundary.fixture.ts", import.meta.url) +); + +type ChildFailure = Error & { + stdout?: string | Buffer; + stderr?: string | Buffer; +}; + +test( + "1min.ai stream-error boundary passes in an isolated persistence subprocess", + { timeout: 180_000 }, + async () => { + const originalDataDir = process.env.DATA_DIR; + const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; + const originalFetch = globalThis.fetch; + const testRoot = mkdtempSync(join(tmpdir(), "omniroute-onemin-stream-error-child-")); + const testDataDir = join(testRoot, "data"); + const testPluginsDir = join(testRoot, "plugins"); + + mkdirSync(testDataDir, { recursive: true }); + mkdirSync(testPluginsDir, { recursive: true }); + const childEnv: NodeJS.ProcessEnv = { + APP_LOG_TO_FILE: "false", + DATA_DIR: testDataDir, + DISABLE_SQLITE_AUTO_BACKUP: "true", + NODE_ENV: "test", + OMNIROUTE_PLUGINS_DIR: testPluginsDir, + }; + for (const name of ["PATH", "NODE_PATH", "LANG", "LC_ALL", "TZ", "TMPDIR"] as const) { + const value = process.env[name]; + if (value !== undefined) childEnv[name] = value; + } + // A nested `node --test` must create its own runner context instead of + // inheriting the parent's private reporter channel. + delete childEnv.NODE_TEST_CONTEXT; + + try { + let stdout = ""; + let stderr = ""; + try { + const child = await execFileAsync( + process.execPath, + ["--import", "tsx/esm", "--test", "--test-concurrency=1", fixturePath], + { + cwd: process.cwd(), + encoding: "utf8", + env: childEnv, + maxBuffer: 2 * 1024 * 1024, + timeout: 170_000, + } + ); + stdout = child.stdout; + stderr = child.stderr; + } catch (error) { + const failure = error as ChildFailure; + assert.fail( + [ + `isolated 1min.ai fixture failed: ${failure.message}`, + failure.stdout ? String(failure.stdout) : "", + failure.stderr ? String(failure.stderr) : "", + ] + .filter(Boolean) + .join("\n") + ); + } + + const childOutput = `${stdout}\n${stderr}`; + assert.match(childOutput, /tests 8/); + assert.match(childOutput, /pass 8/); + assert.match(childOutput, /fail 0/); + assert.doesNotMatch(childOutput, /not ok|failed to drain|stayed pending/i); + } finally { + rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + + assert.equal(process.env.DATA_DIR, originalDataDir); + assert.equal(process.env.OMNIROUTE_PLUGINS_DIR, originalPluginsDir); + assert.equal(globalThis.fetch, originalFetch); + } +); diff --git a/tests/unit/perplexity-web-stream-error-boundary.test.ts b/tests/unit/perplexity-web-stream-error-boundary.test.ts new file mode 100644 index 0000000000..57daec0c15 --- /dev/null +++ b/tests/unit/perplexity-web-stream-error-boundary.test.ts @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const FIXTURE = fileURLToPath( + new URL("../fixtures/perplexity-web-stream-error-boundary.fixture.ts", import.meta.url) +); + +type FixtureResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function runIsolatedFixture(testRoot: string): Promise { + const dataDir = path.join(testRoot, "data"); + const pluginsDir = path.join(testRoot, "plugins"); + fs.mkdirSync(dataDir, { recursive: true }); + fs.mkdirSync(pluginsDir, { recursive: true }); + const childEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + NODE_PATH: process.env.NODE_PATH, + LANG: process.env.LANG ?? "C.UTF-8", + LC_ALL: process.env.LC_ALL, + TZ: process.env.TZ ?? "UTC", + TMPDIR: process.env.TMPDIR ?? os.tmpdir(), + NODE_ENV: "test", + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: "perplexity-stream-boundary-test-secret-00000000000000000000000000000000", + DATA_DIR: dataDir, + OMNIROUTE_PLUGINS_DIR: pluginsDir, + }; + delete childEnv.NODE_TEST_CONTEXT; + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx/esm", "--test", FIXTURE], { + cwd: process.cwd(), + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + timeout: 90_000, + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +test("Perplexity stream failures preserve protocol semantics in an isolated full pipeline", async () => { + const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-pplx-boundary-parent-")); + const originalDataDir = process.env.DATA_DIR; + const originalPluginsDir = process.env.OMNIROUTE_PLUGINS_DIR; + const eventBusOwner = globalThis as { __omnirouteEventBus?: unknown }; + const originalEventBus = eventBusOwner.__omnirouteEventBus; + + try { + const result = await runIsolatedFixture(testRoot); + assert.equal( + result.code, + 0, + `isolated Perplexity fixture failed (signal=${result.signal ?? "none"})\n` + + `stdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ); + assert.equal(result.signal, null); + assert.match(result.stdout, /ℹ tests 8/); + assert.match(result.stdout, /ℹ pass 8/); + assert.match(result.stdout, /ℹ fail 0/); + + assert.equal(process.env.DATA_DIR, originalDataDir); + assert.equal(process.env.OMNIROUTE_PLUGINS_DIR, originalPluginsDir); + assert.equal( + eventBusOwner.__omnirouteEventBus, + originalEventBus, + "the subprocess fixture must not replace the parent event bus singleton" + ); + } finally { + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); diff --git a/tests/unit/provider-connection-test-error-boundaries.test.ts b/tests/unit/provider-connection-test-error-boundaries.test.ts new file mode 100644 index 0000000000..4db74062b0 --- /dev/null +++ b/tests/unit/provider-connection-test-error-boundaries.test.ts @@ -0,0 +1,14 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("provider connection error boundaries pass in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL( + "./fixtures/provider-connection-test-error-boundaries.fixture.ts", + import.meta.url + ), + expectedTests: 3, + label: "provider connection error boundaries", + }); +}); diff --git a/tests/unit/provider-last-error-sanitization.test.ts b/tests/unit/provider-last-error-sanitization.test.ts new file mode 100644 index 0000000000..e7dee5071a --- /dev/null +++ b/tests/unit/provider-last-error-sanitization.test.ts @@ -0,0 +1,11 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("provider last-error persistence passes in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL("./fixtures/provider-last-error-sanitization.fixture.ts", import.meta.url), + expectedTests: 1, + label: "provider last-error persistence", + }); +}); diff --git a/tests/unit/provider-validation-error-sanitization.test.ts b/tests/unit/provider-validation-error-sanitization.test.ts new file mode 100644 index 0000000000..4a725216b9 --- /dev/null +++ b/tests/unit/provider-validation-error-sanitization.test.ts @@ -0,0 +1,101 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import test from "node:test"; +import { + projectProviderValidationResultForPublicResponse, + toValidationErrorResult, +} from "../../src/lib/providers/validation/transport.ts"; + +test("provider validation sanitizes thrown error details", () => { + const result = toValidationErrorResult( + new Error( + "Provider probe failed at /srv/private/provider-key.json " + + "access_token=provider-secret\n at validate (/srv/private/validator.ts:42:7)" + ) + ); + + assert.equal(result.valid, false); + assert.match(result.error, /Provider probe failed/i); + assert.doesNotMatch(result.error, /srv\/private|provider-secret|validator\.ts|\bat validate\b/i); + assert.equal(result.unsupported, false); +}); + +test("provider validation fails closed for hostile thrown values", () => { + const hostile = new Proxy( + {}, + { + getPrototypeOf(): never { + throw new Error("access_token=prototype-secret at /srv/private/prototype.ts:1:2"); + }, + get(_target, property): unknown { + if (property === "code" || property === "isRetryable") { + throw new Error("access_token=metadata-secret at /srv/private/metadata.ts:1:2"); + } + if (property === "toString") { + return () => { + throw new Error("access_token=coercion-secret at /srv/private/coercion.ts:1:2"); + }; + } + return undefined; + }, + } + ); + + assert.deepEqual(toValidationErrorResult(hostile), { + valid: false, + error: "Validation failed", + unsupported: false, + }); +}); + +test("provider validation route sanitizes unexpected failures before persistent logging", () => { + const routeSource = fs.readFileSync( + new URL("../../src/app/api/providers/validate/route.ts", import.meta.url), + "utf8" + ); + + assert.match( + routeSource, + /console\.log\(\s*"Error validating API key:",\s*sanitizeErrorMessage\(error\) \|\| "Validation failed"\s*\)/ + ); + assert.doesNotMatch(routeSource, /console\.log\(\s*"Error validating API key:",\s*error\s*\)/); +}); + +test("provider validation final response projection sanitizes validator errors and warnings", () => { + const projected = projectProviderValidationResultForPublicResponse({ + valid: false, + error: + "Provider echoed access_token=response-secret at /srv/private/provider.json\n" + + " at validate (/srv/private/validator.ts:42:7)", + warning: "Retry after reading C:\\Users\\admin\\private\\warning.json", + method: "probe", + }); + const serialized = JSON.stringify(projected); + + assert.equal(projected.valid, false); + assert.equal(projected.method, "probe"); + assert.doesNotMatch( + serialized, + /response-secret|srv\/private|validator\.ts|C:\\Users|warning\.json/i + ); +}); + +test("provider validation projection preserves intentionally empty fields without synthetic text", () => { + const projected = projectProviderValidationResultForPublicResponse({ + valid: false, + error: "", + warning: "", + }); + + assert.equal(projected.error, ""); + assert.equal(projected.warning, ""); +}); + +test("provider validation route applies the final response projection", () => { + const routeSource = fs.readFileSync( + new URL("../../src/app/api/providers/validate/route.ts", import.meta.url), + "utf8" + ); + + assert.match(routeSource, /projectProviderValidationResultForPublicResponse\(/); +}); diff --git a/tests/unit/request-log-management-boundary.test.ts b/tests/unit/request-log-management-boundary.test.ts new file mode 100644 index 0000000000..2b619774db --- /dev/null +++ b/tests/unit/request-log-management-boundary.test.ts @@ -0,0 +1,11 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("request-log management boundaries pass in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL("./fixtures/request-log-management-boundary.fixture.ts", import.meta.url), + expectedTests: 3, + label: "request-log management boundaries", + }); +}); diff --git a/tests/unit/request-log-payloads.test.ts b/tests/unit/request-log-payloads.test.ts index 46aa84792d..098eaf12e8 100644 --- a/tests/unit/request-log-payloads.test.ts +++ b/tests/unit/request-log-payloads.test.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; const { normalizePayloadForLog, + protectErrorPayloadForLog, protectPayloadForLog, serializePayloadForStorage, parseStoredPayload, @@ -65,6 +66,426 @@ test("redacts web-impersonation body credentials but preserves non-secret 'capab }); }); +test("redacts challenge and handoff credentials from persistent request logs", () => { + const protectedPayload = protectPipelinePayloads({ + providerRequest: { + model: "browser-session-model", + recaptchaV3Token: "recaptcha-secret", + nested: { + recaptchaToken: "recaptcha-alias-secret", + turnstileToken: "turnstile-secret", + proofToken: "proof-secret", + resumeToken: "resume-secret", + prepare_token: "prepare-secret", + }, + }, + }); + + assert.deepEqual(protectedPayload?.providerRequest, { + model: "browser-session-model", + recaptchaV3Token: "[REDACTED]", + nested: { + recaptchaToken: "[REDACTED]", + turnstileToken: "[REDACTED]", + proofToken: "[REDACTED]", + resumeToken: "[REDACTED]", + prepare_token: "[REDACTED]", + }, + }); +}); + +test("sanitizes pipeline error messages before persistent request logs", () => { + const protectedPayload = protectPipelinePayloads({ + error: { + timestamp: "2026-09-02T00:00:00.000Z", + error: + "Provider failed access_token=pipeline-secret at /srv/private/provider.json\n" + + " at dispatch (/srv/private/dispatcher.ts:42:7)", + requestBody: { + max_tokens: 512, + temperature: 0.2, + prompt: "Inspect /tmp/example.ts without changing it", + }, + }, + }); + const serialized = JSON.stringify(protectedPayload); + + assert.doesNotMatch(serialized, /pipeline-secret|srv\/private|dispatcher\.ts|\bat dispatch\b/i); + assert.deepEqual(protectedPayload?.error?.requestBody, { + max_tokens: 512, + temperature: 0.2, + prompt: "Inspect /tmp/example.ts without changing it", + }); +}); + +test("sanitizes only nested error and warning subtrees in persisted response bodies", () => { + const payload = { + content: "Normal output mentions /tmp/public-example.ts and must remain intact", + usage: { completion_tokens: 7 }, + error: { + message: "access_token=response-secret at /srv/private/provider.json", + stack: "Error: response-secret\n at dispatch (/srv/private/dispatcher.ts:42:7)", + }, + warning: "Retry after reading C:\\Users\\admin\\private\\warning.json", + }; + + const protectedLegacyPayload = protectPayloadForLog(payload) as typeof payload; + const protectedPipeline = protectPipelinePayloads({ + providerResponse: { body: payload }, + clientResponse: { body: payload }, + }); + const serialized = JSON.stringify({ protectedLegacyPayload, protectedPipeline }); + + assert.doesNotMatch( + serialized, + /response-secret|srv\/private|dispatcher\.ts|C:\\Users|warning\.json/i + ); + assert.equal(protectedLegacyPayload.content, payload.content); + assert.deepEqual(protectedLegacyPayload.usage, payload.usage); + assert.equal(protectedPipeline?.providerResponse?.body?.content, payload.content); + assert.equal(protectedPipeline?.clientResponse?.body?.content, payload.content); +}); + +test("sanitizes in-band error marker objects even when an upstream uses HTTP 200", () => { + const protectedPayload = protectPayloadForLog({ + events: [ + { + type: "error", + content: + "access_token=in-band-secret at /srv/private/in-band.json\n" + + " at dispatch (/srv/private/in-band.ts:3:2)", + }, + ], + content: "Normal sibling content stays available", + }) as { events: Array<{ type: string; content: string }>; content: string }; + + assert.doesNotMatch( + JSON.stringify(protectedPayload.events), + /in-band-secret|srv\/private|in-band\.ts|\bat dispatch\b/i + ); + assert.equal(protectedPayload.content, "Normal sibling content stays available"); +}); + +test("sanitizes serialized error JSON nested below a neutral payload key", () => { + const protectedPayload = protectPayloadForLog({ + payload: JSON.stringify({ + type: "error", + message: "access_token=serialized-secret at /srv/private/serialized.json", + }), + }) as { payload: string }; + + assert.doesNotMatch(protectedPayload.payload, /serialized-secret|srv\/private/i); + assert.equal((JSON.parse(protectedPayload.payload) as { type: string }).type, "error"); +}); + +test("preserves deep successful payloads and still sanitizes deep error leaves", () => { + const successLeaf = { content: "deep successful content", usage: { total_tokens: 2 } }; + const errorLeaf = { + error: { + message: "access_token=deep-error-secret at /srv/private/deep.json", + }, + }; + let deepSuccess: Record = successLeaf; + let deepError: Record = errorLeaf; + for (let depth = 0; depth < 18; depth += 1) { + deepSuccess = { [`level_${depth}`]: deepSuccess }; + deepError = { [`level_${depth}`]: deepError }; + } + + assert.deepEqual(protectPayloadForLog(deepSuccess), deepSuccess); + assert.doesNotMatch( + JSON.stringify(protectPayloadForLog(deepError)), + /deep-error-secret|srv\/private/i + ); +}); + +test("error-mode log protection summarizes opaque binary bodies without enumerating bytes", () => { + assert.equal(protectErrorPayloadForLog(new Uint8Array([1, 2, 3, 4])), "[binary 4 bytes]"); + assert.equal(protectErrorPayloadForLog(Buffer.from([5, 6, 7])), "[binary 3 bytes]"); +}); + +test("error-mode log protection summarizes nested binary bodies without enumerating bytes", () => { + assert.deepEqual( + protectErrorPayloadForLog({ + data: new Uint8Array([11, 22, 33, 44]), + nested: { + body: Buffer.from([55, 66, 77]), + raw: new Uint8Array([88, 99]).buffer, + }, + }), + { + data: "[binary 4 bytes]", + nested: { + body: "[binary 3 bytes]", + raw: "[binary 2 bytes]", + }, + } + ); +}); + +test("sanitizes error frames split across persisted SSE chunks", () => { + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + '[12:00:00.000] data: {"error":{"message":"access_token=stream-secret at /srv/private/', + 'provider.json","stack":"Error: stream-secret\\n at dispatch (/srv/private/dispatcher.ts:42:7)"}}\n\n', + ], + }, + }); + const storedChunks = protectedPipeline?.streamChunks?.provider ?? []; + const serialized = JSON.stringify(storedChunks); + + assert.doesNotMatch(serialized, /stream-secret|srv\/private|dispatcher\.ts|\bat dispatch\b/i); + assert.match(serialized, /error/); +}); + +test("sanitizes plaintext SSE error events without treating metadata as data frames", () => { + const metadata = 'metadata: {"error":{"message":"healthy diagnostic"}}'; + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `${metadata}\nevent: error\ndata: access_token=plain-sse-secret at /srv/private/plain.txt\n\n`, + ], + }, + }); + const storedChunks = protectedPipeline?.streamChunks?.provider ?? []; + const serialized = JSON.stringify(storedChunks); + + assert.doesNotMatch(serialized, /plain-sse-secret|srv\/private|plain\.txt/i); + assert.match(serialized, /event: error/); + assert.equal(storedChunks[0].includes(metadata), true); +}); + +test("sanitizes discriminated SSE and raw NDJSON error records", () => { + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + 'data: {"type":"error","message":"access_token=sse-json-secret at /srv/private/sse.json"}\n\n', + '{"type":"error","subType":"upstream","message":"Bearer ndjson-secret at C:\\\\Users\\\\admin\\\\private.json"}\n', + '{"type":"error","content":"Error: api_key=lmarena-secret\\n at dispatch (/srv/private/lmarena.ts:8:2)"}\n', + ], + }, + }); + const storedChunks = protectedPipeline?.streamChunks?.provider ?? []; + const serialized = JSON.stringify(storedChunks); + + assert.doesNotMatch( + serialized, + /sse-json-secret|ndjson-secret|lmarena-secret|srv\/private|C:\\\\Users|\bat dispatch\b/i + ); + assert.equal(storedChunks[0].includes('"type":"error"'), true); +}); + +test("sanitizes response last_error aliases in objects, SSE, and NDJSON", () => { + const hostile = "access_token=last-error-secret at /srv/private/last-error.ts"; + const objectPayload = protectPayloadForLog({ + response: { status: "failed", last_error: { message: hostile } }, + }); + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `data: ${JSON.stringify({ response: { status: "failed", last_error: { message: hostile } } })}\n\n`, + `${JSON.stringify({ response: { status: "failed", lastError: { message: hostile } } })}\n`, + ], + }, + }); + const serialized = JSON.stringify({ objectPayload, protectedPipeline }); + + assert.doesNotMatch(serialized, /last-error-secret|srv\/private|last-error\.ts/i); + assert.match(serialized, /last_error|lastError/); +}); + +test("sanitizes response.failed messages without rewriting unrelated deep diagnostics", () => { + const hostile = "Bearer response-failed-secret at /srv/private/response-failed.ts:8:2"; + const diagnostics = { + trace: hostile, + output: { trace: hostile }, + level1: { level2: { level3: { level4: { level5: { label: "legitimate diagnostic" } } } } }, + }; + const output = [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe direct partial output" }], + }, + { type: "reasoning", reasoning_content: "private direct reasoning" }, + ]; + const objectPayload = protectPayloadForLog({ + type: "response.failed", + message: hostile, + diagnostics, + output, + }); + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `event: response.failed\ndata: ${JSON.stringify({ message: hostile, diagnostics })}\n\n`, + `${JSON.stringify({ type: "response.failed", message: hostile, diagnostics })}\n`, + ], + }, + }); + const serialized = JSON.stringify({ objectPayload, protectedPipeline }); + + assert.doesNotMatch(serialized, /response-failed-secret|srv\/private|response-failed\.ts/i); + assert.deepEqual( + (objectPayload as { diagnostics: typeof diagnostics }).diagnostics.level1, + diagnostics.level1 + ); + assert.deepEqual((objectPayload as { output: unknown }).output, [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe direct partial output", annotations: [] }], + }, + ]); + assert.doesNotMatch(serialized, /private direct reasoning/); + assert.match(serialized, /response\.failed/); +}); + +test("projects nested output when the SSE event alone marks response.failed", () => { + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `event: response.failed\ndata: ${JSON.stringify({ + response: { + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "safe event partial output" }], + }, + { type: "reasoning", reasoning_content: "private event reasoning" }, + ], + }, + })}\n\n`, + ], + }, + }); + const serialized = JSON.stringify(protectedPipeline); + + assert.match(serialized, /safe event partial output/); + assert.doesNotMatch(serialized, /private event reasoning|"reasoning"/); +}); + +test("sanitizes response.completed failed siblings in objects, SSE, and NDJSON", () => { + const hostile = "Bearer completed-failed-secret at /srv/private/completed-failed.ts:8:2"; + const partialOutput = [ + { + id: "msg_partial", + type: "message", + role: "assistant", + status: "in_progress", + diagnostics: { trace: hostile }, + content: [ + { + type: "output_text", + text: "partial safe output", + annotations: [{ type: "url_citation", url: "file:///srv/private/citation" }], + }, + { type: "output_text", phase: "commentary", text: "private commentary" }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, + { + id: "msg_roleless", + type: "message", + content: [{ type: "output_text", text: "private roleless output" }], + }, + { + type: "reasoning", + reasoning_content: "private chain of thought", + encrypted_content: "private encrypted reasoning", + }, + { + type: "function_call", + name: "read_private_file", + arguments: '{"api_key":"private tool argument"}', + }, + ]; + const projectedOutput = [ + { + id: "msg_partial", + type: "message", + role: "assistant", + status: "in_progress", + content: [ + { type: "output_text", text: "partial safe output", annotations: [] }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, + ]; + const completedFailure = { + type: "response.completed", + message: hostile, + response: { + status: "failed", + detail: hostile, + description: hostile, + error: { message: "Upstream request failed" }, + output: partialOutput, + }, + }; + const objectPayload = protectPayloadForLog(completedFailure); + const protectedPipeline = protectPipelinePayloads({ + streamChunks: { + provider: [ + `event: response.completed\ndata: ${JSON.stringify({ message: hostile, response: completedFailure.response })}\n\n`, + `${JSON.stringify(completedFailure)}\n`, + ], + }, + }); + const serialized = JSON.stringify({ objectPayload, protectedPipeline }); + + assert.doesNotMatch( + serialized, + /completed-failed-secret|srv\/private|completed-failed\.ts|private commentary|private roleless|private chain|private encrypted|private tool/i + ); + assert.match(serialized, /"annotations":\[\]/); + assert.doesNotMatch(serialized, /"url_citation"|"diagnostics"|"function_call"|"reasoning"/); + assert.match(serialized, /partial safe output/); + assert.match(serialized, /safe refusal/); + assert.deepEqual( + (objectPayload as { response: { output: typeof projectedOutput } }).response.output, + projectedOutput + ); +}); + +test("sanitizes upstream error bodies by status while preserving successful response bodies", () => { + const successBody = { + message: "Normal response mentions /tmp/public-example.ts and remains diagnostic content", + usage: { total_tokens: 3 }, + }; + const protectedJsonError = protectPipelinePayloads({ + providerResponse: { + status: 502, + statusText: "Bad Gateway", + headers: { "content-type": "application/json" }, + body: { + message: "access_token=json-body-secret at /srv/private/upstream.json", + detail: "Error: api_key=body-stack-secret\n at dispatch (/srv/private/body.ts:4:2)", + }, + }, + }); + const protectedPlaintextError = protectPipelinePayloads({ + providerResponse: { + status: 503, + body: "Bearer plaintext-body-secret at C:\\Users\\admin\\upstream.txt", + }, + }); + const protectedSuccess = protectPipelinePayloads({ + providerResponse: { status: 200, body: successBody }, + }); + const serialized = JSON.stringify({ protectedJsonError, protectedPlaintextError }); + + assert.doesNotMatch( + serialized, + /json-body-secret|body-stack-secret|plaintext-body-secret|srv\/private|C:\\\\Users|\bat dispatch\b/i + ); + assert.equal(protectedJsonError?.providerResponse?.status, 502); + assert.equal(protectedPlaintextError?.providerResponse?.status, 503); + assert.deepEqual(protectedSuccess?.providerResponse?.body, successBody); +}); + test("omits encrypted reasoning values from structured log payloads", () => { const encryptedContent = "encrypted".repeat(128); const payload = { diff --git a/tests/unit/search-baseurl-client-override-3f8g.test.ts b/tests/unit/search-baseurl-client-override-3f8g.test.ts new file mode 100644 index 0000000000..17ada75dce --- /dev/null +++ b/tests/unit/search-baseurl-client-override-3f8g.test.ts @@ -0,0 +1,159 @@ +/** + * GHSA-3f8g-pfh9-j687 — a tenant-supplied `provider_options.baseUrl` redirected + * the SaaS search builders, so the OPERATOR's search-provider API key was sent + * to a tenant-chosen host (query string for google-pse/searchapi, header for + * you.com/linkup/nimble/ollama). + * + * This is the same override the GHSA-j7j4-g9qc-q69c fix hardened — and that fix + * only added a block-metadata check, which does nothing here: the attacker + * points at their own PUBLIC host and collects the key. The j7j4 regression test + * missed it because its fixture was `searxng-search`, the one keyless provider + * where redirecting the base URL leaks no credential. + * + * The two override sources have different trust: + * - `providerSpecificData` comes from the stored provider connection + * (`credentials?.providerSpecificData`, search.ts:1510) — OPERATOR config. + * This is how an operator points at their self-hosted searxng, so it keeps + * working on loopback/LAN under the block-metadata policy. + * - `providerOptions` comes straight off the request body + * (`body.provider_options`, v1/search/route.ts:366) — TENANT input. It is + * refused outright: no provider needs a per-request caller-chosen fetch + * target, and honoring one is credential exfiltration for keyed providers + * and SSRF-with-readback for every provider. + * + * Run with: + * node --import tsx/esm --test tests/unit/search-baseurl-client-override-3f8g.test.ts + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { resolveSearchBaseUrl } from "../../open-sse/handlers/search.ts"; +import type { SearchProviderConfig } from "../../open-sse/config/searchRegistry.ts"; +import { SEARCH_PROVIDERS } from "../../open-sse/config/searchRegistry.ts"; + +const base = { query: "test", searchType: "web", maxResults: 5 }; + +/** Every provider whose builder attaches an operator-held key. */ +const KEYED_PROVIDER_IDS = Object.values(SEARCH_PROVIDERS) + .filter((p) => p.authType === "apikey") + .map((p) => p.id); + +describe("resolveSearchBaseUrl — tenant-supplied baseUrl (GHSA-3f8g-pfh9-j687)", () => { + it("covers every keyed provider in the registry, not one hand-picked fixture", () => { + // The j7j4 test only exercised searxng. Assert the registry actually has + // keyed providers so this suite cannot silently degrade to zero coverage. + assert.ok( + KEYED_PROVIDER_IDS.length >= 5, + `expected keyed providers, got ${KEYED_PROVIDER_IDS.length}` + ); + }); + + for (const id of KEYED_PROVIDER_IDS) { + it(`refuses a tenant baseUrl for the keyed provider ${id}`, () => { + const config = SEARCH_PROVIDERS[id]; + assert.throws( + () => + resolveSearchBaseUrl(config, { + ...base, + providerOptions: { baseUrl: "https://attacker.example" }, + }), + `${id} honored a tenant baseUrl — the operator key would be sent there` + ); + }); + } + + it("the opt-in flag is NEVER set on a provider that carries an operator key", () => { + // The whole invariant in one assertion: if this ever pairs with authType + // "apikey", a caller can redirect the operator's key again. + for (const p of Object.values(SEARCH_PROVIDERS)) { + if (p.allowClientBaseUrlOverride) { + assert.equal(p.authType, "none", `${p.id} opts into a caller baseUrl while holding a key`); + } + } + }); + + it("refuses a caller baseUrl for keyless providers that did NOT opt in", () => { + for (const id of ["context7", "duckduckgo-free"]) { + const config = SEARCH_PROVIDERS[id]; + if (!config || config.allowClientBaseUrlOverride) continue; + assert.throws( + () => + resolveSearchBaseUrl(config, { + ...base, + providerOptions: { baseUrl: "https://attacker.example" }, + }), + `${id} honored a caller baseUrl without opting in` + ); + } + }); + + it("SearXNG keeps its documented self-hosted caller override, IMDS still blocked", () => { + // Opted in: keyless, so no operator credential travels with the request. + // Loopback/LAN is the point of a self-hosted instance (search-route.test.ts + // covers the route-level flow); cloud metadata stays rejected. + const config = SEARCH_PROVIDERS["searxng-search"]; + assert.equal(config.allowClientBaseUrlOverride, true); + assert.equal( + resolveSearchBaseUrl(config, { + ...base, + providerOptions: { baseUrl: "http://127.0.0.1:8888/search" }, + }), + "http://127.0.0.1:8888/search" + ); + assert.throws(() => + resolveSearchBaseUrl(config, { + ...base, + providerOptions: { baseUrl: "http://169.254.169.254/latest/meta-data/" }, + }) + ); + }); + + it("keeps the operator-configured override working, including loopback/LAN", () => { + const config = SEARCH_PROVIDERS["searxng-search"]; + assert.equal( + resolveSearchBaseUrl(config, { + ...base, + providerSpecificData: { baseUrl: "http://127.0.0.1:8888/search" }, + }), + "http://127.0.0.1:8888/search" + ); + assert.equal( + resolveSearchBaseUrl(config, { + ...base, + providerSpecificData: { baseUrl: "http://10.0.0.5:8888" }, + }), + "http://10.0.0.5:8888" + ); + }); + + it("still blocks cloud metadata from the operator source (j7j4 must not regress)", () => { + const config = SEARCH_PROVIDERS["searxng-search"]; + assert.throws(() => + resolveSearchBaseUrl(config, { + ...base, + providerSpecificData: { baseUrl: "http://169.254.169.254/latest/meta-data/" }, + }) + ); + }); + + it("the operator source wins over a tenant one instead of the tenant shadowing it", () => { + const config = SEARCH_PROVIDERS["searxng-search"]; + assert.equal( + resolveSearchBaseUrl(config, { + ...base, + providerOptions: { baseUrl: "https://attacker.example" }, + providerSpecificData: { baseUrl: "http://127.0.0.1:8888" }, + }), + "http://127.0.0.1:8888" + ); + }); + + it("falls back to the catalog baseUrl when neither source supplies one", () => { + const config: SearchProviderConfig = { + ...SEARCH_PROVIDERS["searxng-search"], + baseUrl: "http://localhost:8888/search", + }; + assert.equal(resolveSearchBaseUrl(config, base), "http://localhost:8888/search"); + }); +}); diff --git a/tests/unit/search-baseurl-ssrf-guard.test.ts b/tests/unit/search-baseurl-ssrf-guard.test.ts index f42335edd3..5e4e1a6217 100644 --- a/tests/unit/search-baseurl-ssrf-guard.test.ts +++ b/tests/unit/search-baseurl-ssrf-guard.test.ts @@ -58,18 +58,26 @@ describe("resolveSearchBaseUrl — SSRF guard on client-controlled baseUrl (GHSA }); } - it("still allows a self-hosted loopback/LAN override (block-metadata, not public-only)", () => { + // CORRECTED for GHSA-3f8g-pfh9-j687. This case originally asserted the + // loopback/LAN override via `providerOptions` — i.e. via TENANT input — which + // is the SSRF-with-readback half of that advisory. The self-hosted use case + // this test was written to protect is operator configuration, and it arrives + // on `providerSpecificData` (search.ts reads it from the stored connection's + // `credentials?.providerSpecificData`), so that is where it is asserted now. + // Tenant-supplied overrides are refused outright — see + // search-baseurl-client-override-3f8g.test.ts. + it("still allows a self-hosted loopback/LAN override from OPERATOR config", () => { assert.equal( resolveSearchBaseUrl(config, { ...base, - providerOptions: { baseUrl: "http://127.0.0.1:9999" }, + providerSpecificData: { baseUrl: "http://127.0.0.1:9999" }, }), "http://127.0.0.1:9999" ); assert.equal( resolveSearchBaseUrl(config, { ...base, - providerOptions: { baseUrl: "http://10.0.0.5:8080" }, + providerSpecificData: { baseUrl: "http://10.0.0.5:8080" }, }), "http://10.0.0.5:8080" ); diff --git a/tests/unit/skills-executor.test.ts b/tests/unit/skills-executor.test.ts index 99975c06b1..17349e84b2 100644 --- a/tests/unit/skills-executor.test.ts +++ b/tests/unit/skills-executor.test.ts @@ -4,8 +4,15 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-skills-executor-")); +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-skills-executor-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; const coreDb = await import("../../src/lib/db/core.ts"); const settingsDb = await import("../../src/lib/db/settings.ts"); @@ -47,7 +54,11 @@ test.beforeEach(async () => { test.after(() => { resetSkillsRuntime(); coreDb.resetDbInstance(); - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); test("skillExecutor executes a registered handler and persists execution history", async () => { @@ -78,6 +89,119 @@ test("skillExecutor executes a registered handler and persists execution history assert.equal(listed[0].id, execution.id); }); +test("skillExecutor sanitizes failed outputs and nested error subtrees before persistence", async () => { + await registerEchoSkill(); + const hostile = + "tool failed access_token=skill-output-secret at /srv/private/skill-output.ts\n" + + " at run (/srv/private/skill-output.ts:8:2)"; + + skillExecutor.registerHandler("echo-handler", async () => ({ + success: false, + status: 502, + statusText: hostile, + headers: { authorization: "Bearer skill-output-secret" }, + body: hostile, + stdout: hostile, + stderr: hostile, + })); + + const failedOutput = await skillExecutor.execute( + "echo@1.0.0", + { value: "failure" }, + { apiKeyId: "key-a", sessionId: "session-output" } + ); + const storedFailure = skillExecutor.getExecution(failedOutput.id); + const failureSerialized = JSON.stringify({ failedOutput, storedFailure }); + + assert.equal((failedOutput.output as Record)?.status, 502); + assert.doesNotMatch( + failureSerialized, + /skill-output-secret|srv\/private|skill-output\.ts|\bat run\b/i + ); + + skillExecutor.registerHandler("echo-handler", async () => ({ + success: true, + payload: { + value: "preserve me", + error: { message: hostile }, + }, + warning: hostile, + })); + const successfulOutput = await skillExecutor.execute( + "echo@1.0.0", + { value: "success" }, + { apiKeyId: "key-a", sessionId: "session-success" } + ); + const storedSuccess = skillExecutor.getExecution(successfulOutput.id); + const successSerialized = JSON.stringify({ successfulOutput, storedSuccess }); + + assert.equal( + ((successfulOutput.output as Record)?.payload as Record) + ?.value, + "preserve me" + ); + assert.doesNotMatch( + successSerialized, + /skill-output-secret|srv\/private|skill-output\.ts|\bat run\b/i + ); +}); + +test("skillExecutor treats failure discriminators and aliased error objects as boundary failures", async () => { + await registerEchoSkill(); + const hostile = "Bearer skill-discriminator-secret at /srv/private/skill-discriminator.ts:8:2"; + + for (const result of [ + { type: "error", message: hostile }, + { status: "failed", reason: hostile }, + ]) { + skillExecutor.registerHandler("echo-handler", async () => result); + const execution = await skillExecutor.execute( + "echo@1.0.0", + { value: "discriminated-failure" }, + { apiKeyId: "key-a", sessionId: "session-discriminated" } + ); + const stored = skillExecutor.getExecution(execution.id); + assert.equal(execution.status, "error"); + assert.equal(stored?.status, "error"); + assert.doesNotMatch( + JSON.stringify({ execution, stored }), + /skill-discriminator-secret|srv\/private|skill-discriminator\.ts/i + ); + } + + const shared = { message: hostile }; + skillExecutor.registerHandler("echo-handler", async () => ({ + success: true, + payload: { error: shared }, + alias: shared, + })); + const aliased = await skillExecutor.execute( + "echo@1.0.0", + { value: "alias" }, + { apiKeyId: "key-a", sessionId: "session-alias" } + ); + assert.equal(aliased.status, "success"); + assert.doesNotMatch( + JSON.stringify({ aliased, stored: skillExecutor.getExecution(aliased.id) }), + /skill-discriminator-secret|srv\/private|skill-discriminator\.ts/i + ); + + const cyclic: Record = { success: true, error: shared }; + cyclic.self = cyclic; + skillExecutor.registerHandler("echo-handler", async () => cyclic); + const cycleSafe = await skillExecutor.execute( + "echo@1.0.0", + { value: "cycle" }, + { apiKeyId: "key-a", sessionId: "session-cycle" } + ); + assert.equal(cycleSafe.status, "success"); + assert.doesNotThrow(() => JSON.stringify(cycleSafe.output)); + assert.doesNotMatch( + JSON.stringify({ cycleSafe, stored: skillExecutor.getExecution(cycleSafe.id) }), + /skill-discriminator-secret|srv\/private|skill-discriminator\.ts/i + ); +}); + test("skillExecutor blocks execution when Skills are disabled in settings", async () => { await registerEchoSkill(); await settingsDb.updateSettings({ skillsEnabled: false }); @@ -122,7 +246,10 @@ test("skillExecutor turns handler errors and timeouts into error executions", as await registerEchoSkill(); skillExecutor.registerHandler("echo-handler", async () => { - throw new Error("handler exploded"); + throw new Error( + "handler exploded access_token=skill-db-secret at /srv/private/skill-executor.ts\n" + + " at execute (/srv/private/skill-executor.ts:21:5)" + ); }); const failed = await skillExecutor.execute( @@ -134,6 +261,16 @@ test("skillExecutor turns handler errors and timeouts into error executions", as assert.equal(failed.status, "error"); assert.equal(failed.output, null); assert.match(failed.errorMessage, /handler exploded/); + assert.doesNotMatch( + String(failed.errorMessage), + /skill-db-secret|srv\/private|skill-executor\.ts|\bat execute\b/i + ); + const storedFailure = skillExecutor.getExecution(failed.id); + assert.match(String(storedFailure?.errorMessage), /handler exploded/); + assert.doesNotMatch( + String(storedFailure?.errorMessage), + /skill-db-secret|srv\/private|skill-executor\.ts|\bat execute\b/i + ); skillExecutor.registerHandler( "echo-handler", diff --git a/tests/unit/skills-interception.test.ts b/tests/unit/skills-interception.test.ts index f6c6e600f2..2cc8c6acc8 100644 --- a/tests/unit/skills-interception.test.ts +++ b/tests/unit/skills-interception.test.ts @@ -4,12 +4,20 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-skills-interception-")); +const TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-skills-interception-")); +const TEST_DATA_DIR = path.join(TEST_ROOT, "data"); +const TEST_PLUGINS_DIR = path.join(TEST_ROOT, "plugins"); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +const ORIGINAL_PLUGINS_DIR = process.env.OMNIROUTE_PLUGINS_DIR; +fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +fs.mkdirSync(TEST_PLUGINS_DIR, { recursive: true }); process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_PLUGINS_DIR = TEST_PLUGINS_DIR; const coreDb = await import("../../src/lib/db/core.ts"); const { skillRegistry } = await import("../../src/lib/skills/registry.ts"); const { skillExecutor } = await import("../../src/lib/skills/executor.ts"); +const { builtinSkills } = await import("../../src/lib/skills/builtins.ts"); const { interceptToolCalls, extractToolCalls, handleToolCallExecution, buildWebSearchCallItem } = await import("../../src/lib/skills/interception.ts"); const { OMNIROUTE_WEB_SEARCH_FALLBACK_TOOL_NAME } = @@ -71,7 +79,11 @@ test.beforeEach(async () => { test.after(() => { resetRuntime(); coreDb.resetDbInstance(); - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + if (ORIGINAL_PLUGINS_DIR === undefined) delete process.env.OMNIROUTE_PLUGINS_DIR; + else process.env.OMNIROUTE_PLUGINS_DIR = ORIGINAL_PLUGINS_DIR; + fs.rmSync(TEST_ROOT, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); test("buildWebSearchCallItem emits a native web_search_call item only for successful web-search fallback results", () => { @@ -231,6 +243,91 @@ test("interceptToolCalls returns outputs, execution errors and missing-skill err ]); }); +test("skill errors are sanitized before OpenAI tool-result response shapes", async () => { + const hostileMessage = + "skill failure access_token=skill-public-secret at /srv/private/skill-handler.ts\n" + + " at execute (/srv/private/skill-handler.ts:17:4)"; + skillExecutor.registerHandler("broken-handler", async () => { + throw new Error(hostileMessage); + }); + + const chatResult = await handleToolCallExecution( + { + choices: [ + { + message: { + tool_calls: [{ id: "chat-error", function: { name: "broken@1.0.0", arguments: "{}" } }], + }, + }, + ], + }, + "gpt-4o-mini", + executionContext + ); + const responsesResult = await handleToolCallExecution( + { + object: "response", + output: [ + { + type: "function_call", + call_id: "responses-error", + name: "broken@1.0.0", + arguments: "{}", + }, + ], + }, + "openai", + executionContext + ); + const thrownResult = await interceptToolCalls( + [{ id: "thrown-error", name: "/srv/private/missing.ts", arguments: {} }], + executionContext + ); + const serialized = JSON.stringify({ chatResult, responsesResult, thrownResult }); + + assert.match(serialized, /skill failure|Skill not found/i); + assert.doesNotMatch( + serialized, + /skill-public-secret|srv\/private|skill-handler\.ts|\bat execute\b/i + ); +}); + +test("failed builtin outputs are sanitized before public tool results", async () => { + const hostile = + "builtin failed access_token=builtin-output-secret at /srv/private/builtin-output.ts\n" + + " at run (/srv/private/builtin-output.ts:9:4)"; + const mutableBuiltins = builtinSkills as unknown as Record< + string, + ( + input: Record, + context: Record + ) => Promise> + >; + const originalHttpRequest = mutableBuiltins.http_request; + + try { + mutableBuiltins.http_request = async () => ({ + success: false, + status: 502, + headers: { authorization: "Bearer builtin-output-secret" }, + body: hostile, + }); + const results = await interceptToolCalls( + [{ id: "builtin-failure", name: "http_request", arguments: { url: "https://example.com" } }], + { ...executionContext, builtinToolNames: ["http_request"] } + ); + const serialized = JSON.stringify(results); + + assert.equal((results[0]?.result as Record)?.status, 502); + assert.doesNotMatch( + serialized, + /builtin-output-secret|srv\/private|builtin-output\.ts|\bat run\b/i + ); + } finally { + mutableBuiltins.http_request = originalHttpRequest; + } +}); + test("handleToolCallExecution appends OpenAI tool results and leaves empty responses untouched", async () => { const openaiResponse = await handleToolCallExecution( { diff --git a/tests/unit/stream-failure-persistent-classification.test.ts b/tests/unit/stream-failure-persistent-classification.test.ts new file mode 100644 index 0000000000..481cf8cc6f --- /dev/null +++ b/tests/unit/stream-failure-persistent-classification.test.ts @@ -0,0 +1,14 @@ +import test from "node:test"; + +import { runIsolatedBoundaryFixture } from "./helpers/runIsolatedBoundaryFixture.ts"; + +test("stream failure persistence boundaries pass in an isolated child process", () => { + runIsolatedBoundaryFixture({ + fixtureUrl: new URL( + "./fixtures/stream-failure-persistent-classification.fixture.ts", + import.meta.url + ), + expectedTests: 2, + label: "stream failure persistence boundaries", + }); +}); diff --git a/tests/unit/stream-handler-public-error-boundary.test.ts b/tests/unit/stream-handler-public-error-boundary.test.ts new file mode 100644 index 0000000000..41aa796b19 --- /dev/null +++ b/tests/unit/stream-handler-public-error-boundary.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const FIXTURE = fileURLToPath( + new URL("../fixtures/stream-handler-public-error-boundary.fixture.ts", import.meta.url) +); + +const CHILD_RUNTIME_ENV_KEYS = [ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "SystemRoot", + "ComSpec", + "PATHEXT", + "LANG", + "LC_ALL", + "TZ", +] as const; + +function buildFixtureEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + NODE_ENV: "test", + APP_LOG_TO_FILE: "false", + API_KEY_SECRET: "stream-handler-boundary-fixture-secret-20260902", + DISABLE_SQLITE_AUTO_BACKUP: "true", + NO_COLOR: "1", + }; + + for (const key of CHILD_RUNTIME_ENV_KEYS) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + + // Nested test runners must not inherit the parent runner's recursion marker. + delete env.NODE_TEST_CONTEXT; + return env; +} + +test("generic stream public error boundaries pass in an isolated process", () => { + const result = spawnSync( + process.execPath, + ["--import", "tsx/esm", "--import", "./open-sse/utils/setupPolyfill.ts", "--test", FIXTURE], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: buildFixtureEnv(), + timeout: 120_000, + } + ); + const output = `${result.stdout}\n${result.stderr}`; + + assert.ifError(result.error); + assert.equal(result.signal, null, output.slice(-12_000)); + assert.equal(result.status, 0, output.slice(-12_000)); + assert.match(output, /(?:^|\s)tests\s+6(?:\s|$)/m); + assert.match(output, /(?:^|\s)pass\s+6(?:\s|$)/m); + assert.match(output, /(?:^|\s)fail\s+0(?:\s|$)/m); +}); diff --git a/tests/unit/stream-handler.test.ts b/tests/unit/stream-handler.test.ts index 8c19c08802..c358ff7c2a 100644 --- a/tests/unit/stream-handler.test.ts +++ b/tests/unit/stream-handler.test.ts @@ -256,7 +256,8 @@ test("createDisconnectAwareStream emits Responses API failure events for Respons assert.match(text, /event: response\.failed/); assert.match(text, /"type":"response\.failed"/); - assert.match(text, /"message":"responses stream\\ndied"/); + assert.match(text, /"message":"responses stream"/); + assert.doesNotMatch(text, /died/); assert.match(text, /"type":"server_error"/); assert.match(text, /"code":"server_error"/); assert.doesNotMatch(text, /chat\.completion\.chunk/); @@ -264,7 +265,7 @@ test("createDisconnectAwareStream emits Responses API failure events for Respons assert.doesNotMatch(text, /\[DONE\]/); }); -test("createDisconnectAwareStream keeps newlines escaped inside SSE data fields", async () => { +test("createDisconnectAwareStream strips multiline diagnostic tails from Responses errors", async () => { const upstreamError = Object.assign(new Error("line one\nline two\rline three"), { statusCode: 400, }); @@ -290,9 +291,9 @@ test("createDisconnectAwareStream keeps newlines escaped inside SSE data fields" const text = await readStreamText(stream); assert.match(text, /^event: response\.failed\ndata: \{"type":"response\.failed"/); - assert.match(text, /"message":"line one\\nline two\\rline three"/); - assert.doesNotMatch(text, /^line two/m); - assert.doesNotMatch(text, /^line three/m); + assert.match(text, /"message":"line one"/); + assert.doesNotMatch(text, /line two/); + assert.doesNotMatch(text, /line three/); }); test("createDisconnectAwareStream treats legacy OpenAI response format alias as Responses", async () => { @@ -360,7 +361,7 @@ test("createDisconnectAwareStream emits Claude SSE errors for Claude clients", a assert.doesNotMatch(text, /\[DONE\]/); }); -test("createDisconnectAwareStream keeps newlines escaped for Claude SSE errors", async () => { +test("createDisconnectAwareStream strips multiline diagnostic tails from Claude errors", async () => { const upstreamError = Object.assign(new Error("claude line one\nclaude line two"), { statusCode: 502, }); @@ -386,8 +387,8 @@ test("createDisconnectAwareStream keeps newlines escaped for Claude SSE errors", const text = await readStreamText(stream); assert.match(text, /^event: error\ndata: \{"type":"error"/); - assert.match(text, /"message":"claude line one\\nclaude line two"/); - assert.doesNotMatch(text, /^claude line two/m); + assert.match(text, /"message":"claude line one"/); + assert.doesNotMatch(text, /claude line two/); }); // #7699/#7816 — heuristic is scoped to FORMATS.CLAUDE (/v1/messages); a diff --git a/tests/unit/stream-passthrough-error-redaction.test.ts b/tests/unit/stream-passthrough-error-redaction.test.ts new file mode 100644 index 0000000000..9da6457def --- /dev/null +++ b/tests/unit/stream-passthrough-error-redaction.test.ts @@ -0,0 +1,446 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createSSEStream } from "../../open-sse/utils/stream.ts"; +import { FORMATS } from "../../open-sse/translator/formats.ts"; + +type Failure = { status: number; message: string; code?: string; type?: string }; + +async function collectUntilFailure( + chunks: string[], + sourceFormat: string, + convertedLog: string[], + mode: "passthrough" | "translate" = "passthrough", + targetFormat: string = FORMATS.OPENAI +): Promise<{ output: string; error: unknown; failure: Failure | null }> { + let failure: Failure | null = null; + const source = new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(new TextEncoder().encode(chunk)); + controller.close(); + }, + }); + const reader = source + .pipeThrough( + createSSEStream({ + mode, + ...(mode === "translate" ? { targetFormat } : {}), + sourceFormat, + ...(mode === "passthrough" ? { clientResponseFormat: sourceFormat } : {}), + provider: "hostile-upstream", + model: "hostile-model", + body: { input: "hello" }, + reqLogger: { + appendConvertedChunk(value: string) { + convertedLog.push(value); + }, + }, + onFailure(payload) { + failure = payload; + return true; + }, + }) + ) + .getReader(); + + let output = ""; + let error: unknown = null; + try { + while (true) { + const result = await reader.read(); + if (result.done) break; + output += new TextDecoder().decode(result.value); + } + } catch (caught) { + error = caught; + } + return { output, error, failure }; +} + +function assertNoHostileDetail(value: string): void { + assert.doesNotMatch(value, /private-runtime\.ts/); + assert.doesNotMatch(value, /sk-stream-secret/); + assert.doesNotMatch(value, /api_key/); +} + +test("translated root error frames notify onFailure and terminate with a public-safe error", async () => { + const convertedLog: string[] = []; + const raw = { + error: { + type: "server_error", + code: "opaque-provider-code", + message: + "translated failure at /srv/omniroute/private-runtime.ts:47:6 token=sk-stream-secret-xlate", + api_key: "sk-stream-secret-abcdef", + }, + }; + const result = await collectUntilFailure( + [`data: ${JSON.stringify(raw)}\n\n`], + FORMATS.CLAUDE, + convertedLog, + "translate" + ); + + assert.ok(result.error, "a translated upstream error must terminate the stream"); + assert.match(result.output, /event: error/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure, "translated failures must reach the internal classifier"); + assert.match(result.failure.message, /private-runtime\.ts/); + assert.equal(result.failure.code, "opaque-provider-code"); + assertNoHostileDetail(String(result.error)); +}); + +test("translated failed response.completed events cannot become successful Chat completions", async () => { + const convertedLog: string[] = []; + const raw = { + type: "response.completed", + response: { + id: "resp_translate_failed", + status: "failed", + output: [], + error: { + type: "server_error", + code: "translated_completed_failure", + message: + "completed translate failure at /srv/omniroute/private-runtime.ts:58:4 token=sk-stream-secret-completed-translate", + }, + }, + }; + const result = await collectUntilFailure( + [`data: ${JSON.stringify(raw)}\n\n`], + FORMATS.OPENAI, + convertedLog, + "translate", + FORMATS.OPENAI_RESPONSES + ); + + assert.ok(result.error, "a failed Responses completion must terminate translated Chat output"); + assert.match(result.output, /"error"/); + assert.doesNotMatch(result.output, /"finish_reason":"stop"/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure, "the translated failure must reach fallback classification"); + assert.equal(result.failure.code, "translated_completed_failure"); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("a translated failed response.completed tail without a newline still terminates", async () => { + const convertedLog: string[] = []; + const raw = { + type: "response.completed", + response: { + id: "resp_translate_failed_tail", + status: "failed", + output: [], + error: { + code: "translated_completed_tail_failure", + message: + "completed tail failure at /srv/omniroute/private-runtime.ts:59:4 token=sk-stream-secret-completed-tail", + }, + }, + }; + const result = await collectUntilFailure( + [`data: ${JSON.stringify(raw)}`], + FORMATS.OPENAI, + convertedLog, + "translate", + FORMATS.OPENAI_RESPONSES + ); + + assert.ok(result.error, "a buffered failed Responses completion must terminate in flush"); + assert.match(result.output, /"error"/); + assert.doesNotMatch(result.output, /"finish_reason":"stop"/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure); + assert.equal(result.failure.code, "translated_completed_tail_failure"); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("Responses response.failed is projected before forwarding, logging, and onFailure", async () => { + const convertedLog: string[] = []; + const raw = { + type: "response.failed", + response: { + id: "resp_hostile-/srv/omniroute/private-runtime.ts-token=sk-stream-secret-id", + model: "provider-model token=sk-stream-secret-model", + status: "failed", + output: [ + { + id: "msg_partial", + type: "message", + role: "assistant", + status: "in_progress", + diagnostics: { + stack: "at /srv/omniroute/private-runtime.ts:47:2", + api_key: "sk-stream-secret-output-diagnostics", + }, + content: [ + { + type: "output_text", + text: "safe partial output", + annotations: [ + { + type: "url_citation", + url: "https://example.invalid/?token=sk-stream-secret-annotation", + title: "at /srv/omniroute/private-runtime.ts:48:2", + }, + ], + }, + { + type: "output_text", + phase: "commentary", + text: "hidden nested commentary must not be public", + }, + { type: "refusal", refusal: "safe refusal" }, + ], + }, + { + id: "msg_commentary", + type: "message", + role: "assistant", + phase: "commentary", + content: [ + { + type: "output_text", + text: "hidden commentary at /srv/omniroute/private-runtime.ts:49:2", + }, + ], + }, + { + id: "msg_roleless", + type: "message", + content: [ + { + type: "output_text", + text: "roleless output must not be public", + annotations: [], + }, + ], + }, + { + id: "reasoning_private", + type: "reasoning", + encrypted_content: "sk-stream-secret-encrypted-reasoning", + summary: [ + { + type: "summary_text", + text: "at /srv/omniroute/private-runtime.ts:50:2", + }, + ], + }, + { + id: "call_private", + type: "function_call", + call_id: "call_private", + name: "read_private_file", + arguments: + '{"path":"/srv/omniroute/private-runtime.ts","api_key":"sk-stream-secret-tool"}', + }, + { + id: "provider_private", + type: "provider_diagnostics", + diagnostics: { + stack: "at /srv/omniroute/private-runtime.ts:51:2", + api_key: "sk-stream-secret-unknown-item", + }, + }, + ], + error: { + type: "server_error", + code: "server_error", + message: "failed at /srv/omniroute/private-runtime.ts:44:2 token=sk-stream-secret-123456", + api_key: "sk-stream-secret-abcdef", + }, + last_error: { + code: "server_error", + message: + "last failure at /srv/omniroute/private-runtime.ts:45:2 token=sk-stream-secret-last", + }, + message: + "sibling failure at /srv/omniroute/private-runtime.ts:46:2 token=sk-stream-secret-sibling", + diagnosis: { stack: "at /srv/omniroute/private-runtime.ts:46:2" }, + settings: { api_key: "sk-stream-secret-response-setting" }, + usage: { + input_tokens: 4, + output_tokens: 2, + total_tokens: 6, + input_tokens_details: { + cached_tokens: 1, + "sk-stream-secret-detail-key": 99, + }, + }, + }, + }; + const result = await collectUntilFailure( + [`event: response.failed\ndata: ${JSON.stringify(raw)}\n\n`], + FORMATS.OPENAI_RESPONSES, + convertedLog + ); + + assert.ok(result.error, "a failed Responses event must terminate the stream"); + assert.match(result.output, /response\.failed/); + assert.match(result.output, /"last_error":\{/); + assert.match(result.output, /safe partial output/); + assert.match(result.output, /safe refusal/); + assert.match(result.output, /"annotations":\[\]/); + assert.doesNotMatch(result.output, /hidden nested commentary must not be public/); + assert.doesNotMatch(result.output, /roleless output must not be public/); + assert.match(result.output, /"cached_tokens":1/); + assert.doesNotMatch(result.output, /\[truncated\]/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.doesNotMatch( + result.output, + /"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/ + ); + assert.doesNotMatch( + convertedLog.join("\n"), + /"diagnosis"|"diagnostics"|"settings"|"encrypted_content"|"function_call"|"provider_diagnostics"|"phase"|"url_citation"/ + ); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("failed response.completed events omit provider-only diagnostic siblings", async () => { + const convertedLog: string[] = []; + const raw = { + type: "response.completed", + response: { + id: "resp_failed_completed", + object: "response", + created_at: 1_777_777_777, + completed_at: 1_777_777_778, + status: "failed", + output: [], + error: { + code: "server_error", + message: + "completed failure at /srv/omniroute/private-runtime.ts:55:2 token=sk-stream-secret-completed", + }, + diagnosis: { stack: "at /srv/omniroute/private-runtime.ts:55:2" }, + settings: { api_key: "sk-stream-secret-completed-setting" }, + }, + }; + const result = await collectUntilFailure( + [`event: response.completed\ndata: ${JSON.stringify(raw)}\n\n`], + FORMATS.OPENAI_RESPONSES, + convertedLog + ); + + assert.ok(result.error, "a failed response.completed event must terminate the stream"); + assert.match(result.output, /"type":"response\.completed"/); + assert.match(result.output, /"id":"resp_failed_completed"/); + assert.match(result.output, /"created_at":1777777777/); + assert.match(result.output, /"completed_at":1777777778/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.doesNotMatch(result.output, /"diagnosis"|"settings"/); + assert.doesNotMatch(convertedLog.join("\n"), /"diagnosis"|"settings"/); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("OpenAI root error frames without a top-level type remain failures after projection", async () => { + const convertedLog: string[] = []; + const raw = { + error: { + type: "server_error", + code: "server_error", + message: "root failed at /srv/omniroute/private-runtime.ts:48:7 token=sk-stream-secret-root", + api_key: "sk-stream-secret-abcdef", + }, + }; + const result = await collectUntilFailure( + [`data: ${JSON.stringify(raw)}\n\n`], + FORMATS.OPENAI, + convertedLog + ); + + assert.ok(result.error, "an OpenAI error envelope must terminate the stream"); + assert.match(result.output, /"error"/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("OpenAI string error frames preserve raw classification but publish only safe text", async () => { + const convertedLog: string[] = []; + const raw = { + error: "string failure at /srv/omniroute/private-runtime.ts:49:8 token=sk-stream-secret-string", + }; + const result = await collectUntilFailure( + [`data: ${JSON.stringify(raw)}\n\n`], + FORMATS.OPENAI, + convertedLog + ); + + assert.ok(result.error, "a string OpenAI error must terminate the stream"); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("Claude type:error is projected before forwarding and terminates the stream", async () => { + const convertedLog: string[] = []; + const raw = { + type: "error", + error: { + type: "server_error", + code: "server_error", + message: + "claude failed at /srv/omniroute/private-runtime.ts:51:3 token=sk-stream-secret-123456", + api_key: "sk-stream-secret-abcdef", + }, + }; + const result = await collectUntilFailure( + [`event: error\ndata: ${JSON.stringify(raw)}\n\n`], + FORMATS.CLAUDE, + convertedLog + ); + + assert.ok(result.error, "a Claude error event must terminate the stream"); + assert.match(result.output, /event: error/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); + +test("a final response.failed frame without a trailing newline is projected before flush", async () => { + const convertedLog: string[] = []; + const raw = { + type: "response.failed", + response: { + status: "failed", + error: { + code: "server_error", + message: + "tail failed at /srv/omniroute/private-runtime.ts:61:8 token=sk-stream-secret-123456", + api_key: "sk-stream-secret-abcdef", + }, + }, + }; + const result = await collectUntilFailure( + [`event: response.failed\ndata: ${JSON.stringify(raw)}`], + FORMATS.OPENAI_RESPONSES, + convertedLog + ); + + assert.ok(result.error, "a buffered failed event must terminate during flush"); + assert.match(result.output, /response\.failed/); + assertNoHostileDetail(result.output); + assertNoHostileDetail(convertedLog.join("\n")); + assert.ok(result.failure); + assert.match(result.failure.message, /private-runtime\.ts/); + assertNoHostileDetail(String(result.error)); +}); diff --git a/tests/unit/stream-readiness.test.ts b/tests/unit/stream-readiness.test.ts index b2ea196818..04432fc62c 100644 --- a/tests/unit/stream-readiness.test.ts +++ b/tests/unit/stream-readiness.test.ts @@ -451,6 +451,123 @@ test("ensureStreamReadiness preserves buffered chunks when stream starts", async assert.match(text, / world/); }); +test("ensureStreamReadiness replays buffered chunks before a subsequent source error", async () => { + let reads = 0; + const response = new Response( + new ReadableStream({ + pull(controller) { + reads += 1; + if (reads === 1) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content: "prefix" } }], + })}\n\n` + ) + ); + return; + } + controller.error(Object.assign(new Error("terminal source failure"), { statusCode: 502 })); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + + const result = await ensureStreamReadiness(response, { timeoutMs: 100 }); + assert.equal(result.ok, true); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + const first = await reader.read(); + + assert.equal(first.done, false); + assert.match(new TextDecoder().decode(first.value), /prefix/); + await assert.rejects(reader.read(), /terminal source failure/); +}); + +test("ensureStreamReadiness replays multiple buffered chunks in order before an error", async () => { + let reads = 0; + const response = new Response( + new ReadableStream({ + pull(controller) { + reads += 1; + if (reads === 1) { + controller.enqueue(encoder.encode(": keepalive\n\n")); + return; + } + if (reads === 2) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content: "ready" } }], + })}\n\n` + ) + ); + return; + } + controller.error(new Error("failure after buffered prefix")); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + + const result = await ensureStreamReadiness(response, { timeoutMs: 100 }); + assert.equal(result.ok, true); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + const first = await reader.read(); + const second = await reader.read(); + + assert.equal(first.done, false); + assert.equal(second.done, false); + assert.match(new TextDecoder().decode(first.value), /keepalive/); + assert.match(new TextDecoder().decode(second.value), /ready/); + await assert.rejects(reader.read(), /failure after buffered prefix/); +}); + +test("ensureStreamReadiness cancellation is bounded when upstream cancel never settles", async () => { + let cancelCalls = 0; + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content: "prefix" } }], + })}\n\n` + ) + ); + }, + pull() { + return new Promise(() => {}); + }, + cancel() { + cancelCalls += 1; + return new Promise(() => {}); + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + + const result = await ensureStreamReadiness(response, { timeoutMs: 100 }); + assert.equal(result.ok, true); + assert.ok(result.response.body); + const reader = result.response.body.getReader(); + const first = await reader.read(); + assert.equal(first.done, false); + + await Promise.race([ + reader.cancel("client disconnected"), + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error("readiness cancellation stayed pending")), 500) + ), + ]); + await reader.cancel("duplicate cancellation"); + assert.equal(cancelCalls, 1); +}); + test("ensureStreamReadiness honors configured timeouts above 2000ms", async () => { const response = new Response( streamFromChunks( @@ -616,10 +733,7 @@ test("ensureStreamReadiness preserves sanitized error-only diagnostics on early assert.equal(result.response.status, 502); assert.equal(result.code, "STREAM_EARLY_EOF"); assert.equal(result.type, "stream_early_eof"); - assert.equal( - result.classificationReason, - "Stream ended before producing a non-ping SSE event" - ); + assert.equal(result.classificationReason, "Stream ended before producing a non-ping SSE event"); assert.equal( result.upstreamDiagnostic, "UPSTREAM_DETAIL quota exhausted; retry after 2s; empty content Bearer [REDACTED] " @@ -636,16 +750,9 @@ test("ensureStreamReadiness preserves sanitized error-only diagnostics on early assert.equal(body.upstream_details.error.message, result.upstreamDiagnostic); assert.equal(warnings.length, 1); - for (const surfaced of [ - result.reason, - body.upstream_details.error.message, - warnings[0], - ]) { + for (const surfaced of [result.reason, body.upstream_details.error.message, warnings[0]]) { assert.match(surfaced, /UPSTREAM_DETAIL/); - assert.doesNotMatch( - surfaced, - /SECOND_DETAIL|TOP_SECRET|\/srv\/omniroute\/handler\.ts/ - ); + assert.doesNotMatch(surfaced, /SECOND_DETAIL|TOP_SECRET|\/srv\/omniroute\/handler\.ts/); } }); diff --git a/tests/unit/upstream-error-passthrough.test.ts b/tests/unit/upstream-error-passthrough.test.ts index 84458df5f7..a89303c964 100644 --- a/tests/unit/upstream-error-passthrough.test.ts +++ b/tests/unit/upstream-error-passthrough.test.ts @@ -4,6 +4,7 @@ import { shouldPassthroughUpstreamError, buildPassthroughErrorResponse, } from "../../open-sse/utils/upstreamErrorPassthrough.ts"; +import { buildSanitizedUpstreamErrorResponse } from "../../open-sse/utils/upstreamErrorResponse.ts"; test("upstream error passthrough", async (t) => { await t.test("4xx com corpo JSON de erro do provider é elegível", () => { @@ -53,13 +54,24 @@ test("upstream error passthrough", async (t) => { }), false ); + for (const message of [ + String.raw`rejected api_key\t=opaque-tab-secret-9382746`, + String.raw`rejected api_key\u0009=opaque-unicode-tab-9382746`, + String.raw`rejected Bearer\\topaque-bearer-secret-9382746`, + "spawn failed: helper --api-key opaque-cli-key-9382746", + 'spawn failed: helper --token "opaque cli token 9382746"', + "spawn failed: helper --password 'opaque-cli-password-9382746'", + `upstream echoed hf_${"A".repeat(34)}`, + ]) { + assert.equal(shouldPassthroughUpstreamError(422, { error: { message } }), false, message); + } } ); await t.test( "corpo de capacidade/quota sem segredo continua elegível (contrato Claude Code preservado)", () => { - // The common case must still relay verbatim so Claude Code can match the - // wording to auto-disable capabilities. + // The common safe case must preserve wording so Claude Code can match it + // after recursive sanitization and auto-disable capabilities. assert.equal( shouldPassthroughUpstreamError(400, { error: { message: "thinking.type: adaptive is not supported" }, @@ -74,7 +86,7 @@ test("upstream error passthrough", async (t) => { ); } ); - await t.test("buildPassthroughErrorResponse preserva corpo byte-a-byte", async () => { + await t.test("buildPassthroughErrorResponse preserves an already-safe JSON body", async () => { const body = { type: "error", error: { type: "invalid_request_error", message: "thinking.type: nope" }, @@ -89,9 +101,85 @@ test("upstream error passthrough", async (t) => { }); }); +test("passthrough preserves multiline capability wording without stack frames", async () => { + const message = "validation failed\nthinking.type: adaptive is not supported"; + const res = buildPassthroughErrorResponse(400, { + type: "error", + error: { type: "invalid_request_error", message }, + }); + assert.ok(res); + const body = (await res.json()) as { error?: { message?: string } }; + assert.equal(body.error?.message, message); +}); + +test("passthrough removes basename and URL stack frames while preserving prose URLs", async () => { + const hostileMessages = [ + "boom\n at handler (server.js:12:3)", + String.raw`boom\n at handler (server.js:12:3)`, + "boom at handler (http://127.0.0.1:3000/_next/server.js:12:3)", + "boom at handler (webpack-internal:///app/server.js:12:3)", + "boom\n at handler (http://127.0.0.1:3000/_next/server.js?build=abc:12:3)", + String.raw`boom\n at handler (webpack-internal:///app/server.js#chunk:12:3)`, + String.raw`boom at handler (\Windows\Temp\server.js:12:3)`, + "boom\nhandler@file:///home/runner/private.js:12:3", + String.raw`boom\nhandler@/home/runner/private.cts:12:3`, + "boom\nhandler@https://127.0.0.1:3000/_next/server.mts?build=abc:12:3", + "boom at handler (http://127.0.0.1:3000/_next/chunks/route:12:3)", + ]; + + for (const message of hostileMessages) { + const response = buildPassthroughErrorResponse(400, { + type: "error", + error: { type: "invalid_request_error", message }, + }); + assert.ok(response); + const body = (await response.json()) as { error?: { message?: string } }; + assert.equal(body.error?.message, "boom"); + } + + const prose = "See https://example.com/docs/error for recovery guidance"; + const proseResponse = buildPassthroughErrorResponse(400, { + type: "error", + error: { type: "invalid_request_error", message: prose }, + }); + assert.ok(proseResponse); + const proseBody = (await proseResponse.json()) as { error?: { message?: string } }; + assert.equal(proseBody.error?.message, prose); + + const proseWithCoordinates = "See https://example.com/docs/error:12:3 for recovery guidance"; + const proseWithCoordinatesResponse = buildPassthroughErrorResponse(400, { + type: "error", + error: { type: "invalid_request_error", message: proseWithCoordinates }, + }); + assert.ok(proseWithCoordinatesResponse); + const proseWithCoordinatesBody = (await proseWithCoordinatesResponse.json()) as { + error?: { message?: string }; + }; + assert.equal(proseWithCoordinatesBody.error?.message, proseWithCoordinates); +}); + +test("canonical upstream JSON projection redacts URL credentials", async () => { + const response = buildSanitizedUpstreamErrorResponse({ + status: 422, + rawBody: JSON.stringify({ + error: { + message: + "proxy failed https://svc-user:p4ss-opaque-9382@internal.example/v1?" + + "X-Amz-Signature=amz-secret&sig=sas-secret", + }, + }), + fallbackMessage: "Upstream validation failed", + }); + const serialized = await response.text(); + + assert.equal(response.status, 422); + assert.doesNotMatch(serialized, /svc-user|p4ss-opaque|amz-secret|sas-secret/i); + assert.match(serialized, /\[REDACTED\]/); +}); + test("createErrorResult opt-in passthrough (opts.passthrough)", async (t) => { await t.test( - "com opts.passthrough e corpo elegível, result.response é o corpo upstream verbatim", + "com opts.passthrough e corpo elegível, result.response preserva o JSON upstream seguro", async () => { const { createErrorResult } = await import("../../open-sse/utils/error.ts"); const upstreamBody = { diff --git a/tests/unit/web-session-credentials.test.ts b/tests/unit/web-session-credentials.test.ts index 68c1a64565..12c38a5c06 100644 --- a/tests/unit/web-session-credentials.test.ts +++ b/tests/unit/web-session-credentials.test.ts @@ -104,6 +104,33 @@ test("web session credential metadata identifies cookie, token, and no-auth prov }); }); +test("MaxAI and UC keep independent top-level credential contracts", () => { + assert.deepEqual(webSessionCredentials.getWebSessionCredentialRequirement("maxai"), { + kind: "token", + credentialName: "MaxAI access token (Bearer) + device id", + placeholder: + "Use browser sign-in — OmniRoute mints the MaxAI access token, device id, and user id for you", + acceptsFullCookieHeader: false, + storageKeys: [ + "accessToken", + "access_token", + "maxaiAccessToken", + "deviceId", + "maxaiDeviceId", + "userId", + "maxaiUserId", + ], + }); + + const uc = webSessionCredentials.getWebSessionCredentialRequirement("uc"); + assert.ok(uc && uc.kind === "cookie"); + assert.equal(uc.credentialName, "Clerk __client cookie + session id + user id"); + assert.equal(uc.acceptsFullCookieHeader, true); + assert.ok(uc.storageKeys.includes("__client")); + assert.ok(uc.storageKeys.includes("sid")); + assert.ok(uc.storageKeys.includes("uid")); +}); + test("web session credential validator requires provider-specific non-empty values", () => { assert.equal( webSessionCredentials.hasUsableWebSessionCredential("kimi-web", { token: "kimi-token" }), diff --git a/tests/unit/zai-web-silent-empty-repro.test.ts b/tests/unit/zai-web-silent-empty-repro.test.ts index 60e2be4d64..01fb742455 100644 --- a/tests/unit/zai-web-silent-empty-repro.test.ts +++ b/tests/unit/zai-web-silent-empty-repro.test.ts @@ -1,9 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { buildZaiStreamingBody, parseZaiFrame, collectZaiNonStreaming } = await import( - "../../open-sse/executors/zai-web/stream.ts" -); +const { buildZaiStreamingBody, parseZaiFrame, collectZaiNonStreaming } = + await import("../../open-sse/executors/zai-web/stream.ts"); /** * Hard Rule #6 — "never silently swallow errors in SSE streams". @@ -47,6 +46,23 @@ async function readAll(stream: ReadableStream): Promise { return out; } +async function readUntilError(stream: ReadableStream): Promise<{ output: string; error: unknown }> { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let output = ""; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return { output, error: null }; + output += decoder.decode(value as Uint8Array, { stream: true }); + } + } catch (error) { + return { output, error }; + } finally { + reader.releaseLock(); + } +} + const emitChunk = ( controller: ReadableStreamDefaultController, delta: Record, @@ -61,6 +77,14 @@ const emitChunk = ( const contentOf = (sse: string) => [...sse.matchAll(/"content":"([^"]*)"/g)].map((m) => m[1]).join(""); +function errorPayloads(sse: string): Array> { + return sse + .split(/\r?\n/) + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6)) as Record) + .filter((payload) => "error" in payload); +} + test("parseZaiFrame classifies an error-shaped frame instead of discarding it", () => { assert.equal(parseZaiFrame({ error: "captcha expired" })?.error, "captcha expired"); assert.equal( @@ -88,25 +112,42 @@ test("REGRESSION GUARD: contentless frames are still skipped, not reported as er assert.equal(parseZaiFrame("not-an-object"), null); }); -test("a 200 stream carrying an error frame surfaces it instead of finishing empty", async () => { +test("a 200 stream carrying an error frame emits a terminal error instead of false success", async () => { const upstream = sseStream(JSON.stringify({ error: { detail: "signature invalid" } })); const out = await readAll(buildZaiStreamingBody(upstream, emitChunk, null)); - assert.match(contentOf(out), /signature invalid/, "the upstream's diagnosis must reach the caller"); - assert.match(contentOf(out), /\[Z\.ai error\]/, "tagged like the other web executors"); - assert.ok(out.includes('"finish_reason":"stop"')); - assert.ok(out.includes("[DONE]"), "the stream still terminates cleanly for the client"); + assert.equal(contentOf(out), "", "an upstream failure must not become assistant content"); + assert.deepEqual(errorPayloads(out), [ + { + error: { + message: "Z.ai stream failed: signature invalid", + type: "upstream_error", + code: "zai_stream_error", + }, + }, + ]); + assert.ok(!out.includes("response.failed"), "Chat streams cannot emit Responses events"); + assert.ok(!out.includes('"finish_reason":"stop"'), "a failure must not report a normal stop"); + assert.ok(!out.includes("[DONE]"), "readiness must see an error-only pre-content stream"); }); -test("an error frame after partial content still surfaces, keeping what was streamed", async () => { +test("an error after partial content preserves it, then errors the producer stream", async () => { const upstream = sseStream( - JSON.stringify({ type: "chat:completion", data: { delta_content: "partial", phase: "answer" } }), + JSON.stringify({ + type: "chat:completion", + data: { delta_content: "partial", phase: "answer" }, + }), JSON.stringify({ error: "stream aborted upstream" }) ); - const out = await readAll(buildZaiStreamingBody(upstream, emitChunk, null)); + const { output, error } = await readUntilError(buildZaiStreamingBody(upstream, emitChunk, null)); - assert.match(contentOf(out), /partial/, "already-streamed content is preserved"); - assert.match(contentOf(out), /stream aborted upstream/, "and the failure is appended, not dropped"); + assert.match(contentOf(output), /partial/, "already-streamed content is preserved"); + assert.match(String(error), /Z\.ai stream failed: stream aborted upstream/); + assert.ok(!output.includes("response.failed"), "the producer stays protocol-neutral"); + assert.ok( + !output.includes('"finish_reason":"stop"'), + "partial output does not make failure success" + ); }); test("control: a well-formed stream is untouched", async () => { diff --git a/tests/unit/zai-web-stream-error-boundary.test.ts b/tests/unit/zai-web-stream-error-boundary.test.ts new file mode 100644 index 0000000000..ee0aa5a52d --- /dev/null +++ b/tests/unit/zai-web-stream-error-boundary.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const fixture = fileURLToPath( + new URL("../fixtures/zai-web-stream-error-boundary.fixture.ts", import.meta.url) +); + +test("Z.ai stream error boundaries pass in a process-isolated fixture", () => { + const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-zai-stream-boundary-")); + const childEnv: NodeJS.ProcessEnv = { + API_KEY_SECRET: "zai-stream-boundary-test-only-secret", + DATA_DIR: path.join(testRoot, "data"), + OMNIROUTE_PLUGINS_DIR: path.join(testRoot, "plugins"), + }; + + // The parent itself is a node:test process. Never forward its runner identity to the child; + // `node --test` owns the child context and creates a fresh value for its fixture process. + delete childEnv.NODE_TEST_CONTEXT; + + try { + const result = spawnSync(process.execPath, ["--import", "tsx/esm", "--test", fixture], { + cwd: fileURLToPath(new URL("../..", import.meta.url)), + encoding: "utf8", + env: childEnv, + timeout: 60_000, + }); + const diagnostics = [result.stdout, result.stderr].filter(Boolean).join("\n"); + + assert.equal(result.error, undefined, diagnostics); + assert.equal(result.signal, null, diagnostics); + assert.equal(result.status, 0, diagnostics); + assert.match(result.stdout, /tests 4\b/); + assert.match(result.stdout, /pass 4\b/); + assert.match(result.stdout, /fail 0\b/); + } finally { + fs.rmSync(testRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } +}); diff --git a/tests/unit/zed-hosted-stream-error-boundary.test.ts b/tests/unit/zed-hosted-stream-error-boundary.test.ts new file mode 100644 index 0000000000..1565c1c1bd --- /dev/null +++ b/tests/unit/zed-hosted-stream-error-boundary.test.ts @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); +const fixturePath = fileURLToPath( + new URL("../fixtures/zed-hosted-stream-error-boundary-child.ts", import.meta.url) +); + +type FixtureResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +function runFixture(): Promise { + // Keep the parent process pristine: the fast unit suite can run files with + // --test-isolation=none, so all stateful imports and mutations live in the child. + const childEnv: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + NODE_PATH: process.env.NODE_PATH, + LANG: process.env.LANG, + LC_ALL: process.env.LC_ALL, + TZ: process.env.TZ, + TMPDIR: process.env.TMPDIR, + NODE_ENV: "test", + API_KEY_SECRET: "zed-boundary-test-only-secret-with-32-plus-characters", + DISABLE_SQLITE_AUTO_BACKUP: "true", + NO_COLOR: "1", + }; + // Inheriting this marker makes Node silently skip the nested --test run. + delete childEnv.NODE_TEST_CONTEXT; + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx/esm", "--test", fixturePath], { + cwd: repoRoot, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + + const timeout = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, 120_000); + + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("close", (code, signal) => { + clearTimeout(timeout); + if (timedOut) { + reject(new Error("Zed stream error boundary fixture timed out after 120 seconds")); + return; + } + resolve({ code, signal, stdout, stderr }); + }); + }); +} + +test("Zed stream error boundary passes in a process-isolated runtime", async () => { + const result = await runFixture(); + const output = `${result.stdout}\n${result.stderr}`; + + assert.equal(result.signal, null, output.slice(-12_000)); + assert.equal(result.code, 0, output.slice(-12_000)); + assert.match(output, /(?:^|\s)tests\s+3(?:\s|$)/m); + assert.match(output, /(?:^|\s)pass\s+3(?:\s|$)/m); + assert.match(output, /(?:^|\s)fail\s+0(?:\s|$)/m); +});