diff --git a/.env.example b/.env.example index 6375a5a4b8..b0fbf5ae5a 100644 --- a/.env.example +++ b/.env.example @@ -1086,9 +1086,11 @@ APP_LOG_TO_FILE=true # ═══════════════════════════════════════════════════════════════════════════════ # 18b. ARENA ELO SYNC # ═══════════════════════════════════════════════════════════════════════════════ -# Enable auto-updating model intelligence from Arena AI leaderboard ELO scores. +# Auto-update model intelligence from Arena AI leaderboard ELO scores (powers the +# Free Provider Rankings page). ON by default — fetches from api.wulong.dev on startup +# (non-blocking, never fatal). Set to false to opt out of the outbound sync. # Used by: src/lib/arenaEloSync.ts -# ARENA_ELO_SYNC_ENABLED=false +# ARENA_ELO_SYNC_ENABLED=true # Sync interval in seconds. Default: 86400 (24 hours). # ARENA_ELO_SYNC_INTERVAL=86400 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b479d2f5a..166ce8c171 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ ### 📝 Maintenance +- **feat(intelligence): enable Arena ELO sync by default** — the Arena AI leaderboard ELO sync (`ARENA_ELO_SYNC_ENABLED`) that powers the new Free Provider Rankings page ([#3799](https://github.com/diegosouzapw/OmniRoute/pull/3799)) is now **on by default** (was opt-in). It fetches from `api.wulong.dev` on startup (non-blocking, never fatal) and refreshes daily; set `ARENA_ELO_SYNC_ENABLED=false` to opt out of the outbound sync. Without it the rankings page renders empty. (thanks @diegosouzapw) - **chore(quality): Quality Gates → 100%** — completes Fase 6A (systemic hardening: a `stale-allowlist` helper applied across ~10 gates, `docs/architecture/QUALITY_GATES.md`, and ratchet engine v2 with `--require-tighten` + per-metric `eps`) and the entire Fase 7 (20 security/dead-code/mutation/tooling gates), with the Fase 8 plan documented. ([#3757](https://github.com/diegosouzapw/OmniRoute/pull/3757)) - **docs: close the remaining documentation gaps** for proxy operations, skills internals, the memory engine, RTK customization, and compression extensibility (post-#3438 audit, 5 areas in one PR). ([#3453](https://github.com/diegosouzapw/OmniRoute/pull/3453) — thanks @oyi77) - **chore(quality): re-baseline `providerRegistry.ts` file-size** (4692→4703) after #3768's Ollama Cloud `kimi-k2.7-code` capability fix grew the file past the frozen baseline, turning the release's own Fast Quality Gates red. No source file is touched. ([#3770](https://github.com/diegosouzapw/OmniRoute/pull/3770)) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 65afb247da..9712fbc4e9 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -269,27 +269,27 @@ OmniRoute provides a two-layer defense: request-side injection scanning and resp Route upstream LLM provider calls through an HTTP or SOCKS5 proxy for egress control, geo-routing, or IP masking. -| Variable | Default | Source File | Description | -| --------------------------------------- | --------- | -------------------------------------------- | ----------------------------------------------------------------------------------------- | -| `ENABLE_SOCKS5_PROXY` | `true` | `open-sse/executors` | Enable SOCKS5 proxy agent for upstream calls. Opt-out with `false`. | -| `NEXT_PUBLIC_ENABLE_SOCKS5_PROXY` | `true` | Client-side | Client-side awareness of SOCKS5 availability. | -| `HTTP_PROXY` | _(unset)_ | Node.js standard | HTTP proxy for upstream calls. | -| `HTTPS_PROXY` | _(unset)_ | Node.js standard | HTTPS proxy for upstream calls. | -| `ALL_PROXY` | _(unset)_ | Node.js standard | Universal proxy (supports `socks5://`). | -| `NO_PROXY` | _(unset)_ | Node.js standard | Comma-separated hostnames/IPs to bypass the proxy. | +| Variable | Default | Source File | Description | +| --------------------------------------- | --------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ENABLE_SOCKS5_PROXY` | `true` | `open-sse/executors` | Enable SOCKS5 proxy agent for upstream calls. Opt-out with `false`. | +| `NEXT_PUBLIC_ENABLE_SOCKS5_PROXY` | `true` | Client-side | Client-side awareness of SOCKS5 availability. | +| `HTTP_PROXY` | _(unset)_ | Node.js standard | HTTP proxy for upstream calls. | +| `HTTPS_PROXY` | _(unset)_ | Node.js standard | HTTPS proxy for upstream calls. | +| `ALL_PROXY` | _(unset)_ | Node.js standard | Universal proxy (supports `socks5://`). | +| `NO_PROXY` | _(unset)_ | Node.js standard | Comma-separated hostnames/IPs to bypass the proxy. | | `PROXY_FAIL_OPEN` | `false` | `src/sse/handlers/chatHelpers.ts` | When `false` (default), a request whose assigned proxy fails to resolve is **refused (fail-closed)** rather than falling back to a direct connection — prevents real-IP leaks. Set `true` to restore the legacy DIRECT fallback. | -| `ENABLE_TLS_FINGERPRINT` | `false` | `open-sse/executors` | Spoof TLS fingerprint using wreq-js (mimics Chrome 124). Counters JA3/JA4 blocking. | -| `OMNIROUTE_TURNSTILE_IGNORE_TLS_ERRORS` | `false` | `open-sse/services/claudeTurnstileSolver.ts` | Allow the Claude Turnstile Playwright browser context to ignore HTTPS certificate errors. | +| `ENABLE_TLS_FINGERPRINT` | `false` | `open-sse/executors` | Spoof TLS fingerprint using wreq-js (mimics Chrome 124). Counters JA3/JA4 blocking. | +| `OMNIROUTE_TURNSTILE_IGNORE_TLS_ERRORS` | `false` | `open-sse/services/claudeTurnstileSolver.ts` | Allow the Claude Turnstile Playwright browser context to ignore HTTPS certificate errors. | ### Scenarios -| Scenario | Configuration | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| **SOCKS5 through SSH tunnel** | `ALL_PROXY=socks5://127.0.0.1:7890`, `ENABLE_SOCKS5_PROXY=true` | -| **Corporate HTTP proxy** | `HTTP_PROXY=http://proxy.corp.com:3128`, `HTTPS_PROXY=http://proxy.corp.com:3128`, `NO_PROXY=localhost,internal.corp.com` | -| **Anti-fingerprint** | `ENABLE_TLS_FINGERPRINT=true` — requires `wreq-js` (included) | -| **Egress-controlled / no direct access** | Leave `PROXY_FAIL_OPEN=false` (default). Requests fail hard when the proxy is unavailable instead of leaking via direct. | -| **Legacy / dev — allow direct fallback** | `PROXY_FAIL_OPEN=true`. Restores pre-hardening behaviour: direct connection used when proxy resolution fails. | +| Scenario | Configuration | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| **SOCKS5 through SSH tunnel** | `ALL_PROXY=socks5://127.0.0.1:7890`, `ENABLE_SOCKS5_PROXY=true` | +| **Corporate HTTP proxy** | `HTTP_PROXY=http://proxy.corp.com:3128`, `HTTPS_PROXY=http://proxy.corp.com:3128`, `NO_PROXY=localhost,internal.corp.com` | +| **Anti-fingerprint** | `ENABLE_TLS_FINGERPRINT=true` — requires `wreq-js` (included) | +| **Egress-controlled / no direct access** | Leave `PROXY_FAIL_OPEN=false` (default). Requests fail hard when the proxy is unavailable instead of leaking via direct. | +| **Legacy / dev — allow direct fallback** | `PROXY_FAIL_OPEN=true`. Restores pre-hardening behaviour: direct connection used when proxy resolution fails. | > **Note (NVIDIA validation bypass — #3226):** NVIDIA's API-key validation endpoint > stalls when routed through the global proxy/TLS-patched fetch (undici dispatcher → 504). @@ -697,10 +697,10 @@ Automatic model pricing data synchronization from external sources. ## Arena ELO Sync -| Variable | Default | Source File | Description | -| --------------------------- | ------------- | -------------------------- | ------------------------------------------------------------- | -| `ARENA_ELO_SYNC_ENABLED` | `false` | `src/lib/arenaEloSync.ts` | Opt-in periodic Arena AI leaderboard ELO sync. | -| `ARENA_ELO_SYNC_INTERVAL` | `86400` (24h) | `src/lib/arenaEloSync.ts` | Sync interval in seconds. | +| Variable | Default | Source File | Description | +| ------------------------- | ------------- | ------------------------- | --------------------------------------------------------------------------- | +| `ARENA_ELO_SYNC_ENABLED` | `true` | `src/lib/arenaEloSync.ts` | Periodic Arena AI leaderboard ELO sync (on by default; `false` to opt out). | +| `ARENA_ELO_SYNC_INTERVAL` | `86400` (24h) | `src/lib/arenaEloSync.ts` | Sync interval in seconds. | --- @@ -877,58 +877,58 @@ Limits and safety knobs applied when the Skills framework (`src/lib/skills/`) ex Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), the 1Proxy egress pool, database backups and small per-feature overrides referenced by the executor layer or scripts. -| Variable | Default | Source File | Description | -| ------------------------------------------ | --------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `REDIS_URL` | `redis://localhost:6379` | `src/shared/utils/rateLimiter.ts` | Redis connection string for the rate limiter backend. | -| `ALIBABA_CODING_PLAN_HOST` | _(production host)_ | `open-sse/services/bailianQuotaFetcher.ts` | Override the host used to fetch Alibaba Bailian coding-plan quotas. | -| `ALIBABA_CODING_PLAN_QUOTA_URL` | derived from host | `open-sse/services/bailianQuotaFetcher.ts` | Full quota URL override for Alibaba Bailian. | -| `CONTEXT_RESERVE_TOKENS` | `1024` | `open-sse/services/contextManager.ts` | Tokens reserved for completion output when computing prompt budgets. | -| `MODEL_ALIAS_COMPAT_ENABLED` | enabled | `open-sse/services/model.ts` | Toggle the legacy model-alias compatibility layer used by older clients. | +| Variable | Default | Source File | Description | +| ------------------------------------------ | --------------------------------------------------------------------------- | ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REDIS_URL` | `redis://localhost:6379` | `src/shared/utils/rateLimiter.ts` | Redis connection string for the rate limiter backend. | +| `ALIBABA_CODING_PLAN_HOST` | _(production host)_ | `open-sse/services/bailianQuotaFetcher.ts` | Override the host used to fetch Alibaba Bailian coding-plan quotas. | +| `ALIBABA_CODING_PLAN_QUOTA_URL` | derived from host | `open-sse/services/bailianQuotaFetcher.ts` | Full quota URL override for Alibaba Bailian. | +| `CONTEXT_RESERVE_TOKENS` | `1024` | `open-sse/services/contextManager.ts` | Tokens reserved for completion output when computing prompt budgets. | +| `MODEL_ALIAS_COMPAT_ENABLED` | enabled | `open-sse/services/model.ts` | Toggle the legacy model-alias compatibility layer used by older clients. | | `OMNIROUTE_EMERGENCY_FALLBACK` | enabled | `open-sse/services/emergencyFallback.ts` | Set `false` (or `0`) to disable the emergency budget-exhaustion fallback that reroutes failed requests to the free `nvidia`/`openai/gpt-oss-120b` model. Effective precedence is Feature Flags DB override > env var > default; if unavailable, the service falls back to the raw env value. | -| `COMMAND_CODE_CALLBACK_PORT` | _(unset)_ | `src/app/api/providers/command-code/auth/shared.ts` | Local port used for OAuth-style callbacks from the Command Code CLI helper. | -| `COMMAND_CODE_VERSION` | `0.33.2` | `open-sse/executors/commandCode.ts` | Value sent as the `x-command-code-version` header to the Command Code upstream. Override to bump the CLI version. | -| `MITM_LOCAL_PORT` | `443` | `src/mitm/server.cjs` | Local bind port for the MITM debug proxy. | -| `MITM_DISABLE_TLS_VERIFY` | `0` | `src/mitm/server.cjs` | Set `1` to disable upstream TLS verification (development only). | -| `ONEPROXY_ENABLED` | `true` | `src/lib/oneproxySync.ts` | Enable the 1Proxy egress pool sync. | -| `ONEPROXY_API_URL` | `https://1proxy-api.aitradepulse.com` | `src/lib/oneproxySync.ts` | 1Proxy service API URL override. | -| `ONEPROXY_MAX_PROXIES` | `500` | `src/lib/oneproxySync.ts` | Maximum proxies imported per sync. | -| `ONEPROXY_MIN_QUALITY_THRESHOLD` | `50` | `src/lib/oneproxySync.ts` | Minimum quality score for imported proxies. | -| `FREE_PROXY_1PROXY_ENABLED` | `true` | `src/lib/freeProxyProviders/oneproxy.ts` | Enable the 1proxy free proxy source. Set to `false` to disable. | -| `FREE_PROXY_1PROXY_API_URL` | _(see oneproxy.ts)_ | `src/lib/freeProxyProviders/oneproxy.ts` | 1proxy API URL override. | -| `FREE_PROXY_1PROXY_MAX` | `500` | `src/lib/freeProxyProviders/oneproxy.ts` | Maximum proxies fetched per sync from 1proxy. | -| `FREE_PROXY_1PROXY_MIN_QUALITY` | `50` | `src/lib/freeProxyProviders/oneproxy.ts` | Minimum quality score threshold for 1proxy imports. | -| `FREE_PROXY_PROXIFLY_ENABLED` | `true` | `src/lib/freeProxyProviders/proxifly.ts` | Enable the Proxifly free proxy source. Set to `false` to disable. | -| `FREE_PROXY_PROXIFLY_QUANTITY` | `100` | `src/lib/freeProxyProviders/proxifly.ts` | Number of proxies to fetch per Proxifly sync. | -| `FREE_PROXY_PROXIFLY_ANONYMITY` | `elite` | `src/lib/freeProxyProviders/proxifly.ts` | Anonymity level filter for Proxifly (`elite`, `anonymous`, `transparent`). | -| `FREE_PROXY_IPLOCATE_ENABLED` | `false` | `src/lib/freeProxyProviders/iplocate.ts` | Enable the IPLocate free proxy source. Opt-in only. | -| `FREE_PROXY_IPLOCATE_BASE_URL` | `https://raw.githubusercontent.com/iplocate/free-proxy-list/main/protocols` | `src/lib/freeProxyProviders/iplocate.ts` | IPLocate proxy list base URL override. | -| `NEXT_PUBLIC_VERCEL_RELAY_ENABLED` | `true` | `src/app/(dashboard)/…/ProxyPoolTab.tsx` | Show/hide the Deploy Vercel Relay button in the Proxy Pool tab. | -| `VERCEL_API_BASE` | `https://api.vercel.com` | `src/app/api/settings/proxy/vercel-deploy/route.ts` | Vercel API base URL override (for testing). | -| `NEXT_PUBLIC_VERCEL_RELAY_DEFAULT_PROJECT` | `omniroute-relay` | `src/app/(dashboard)/…/VercelRelayModal.tsx` | Default project name pre-filled in the Vercel Relay deploy modal. | -| `TAILSCALE_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscale` binary. | -| `TAILSCALED_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscaled` daemon binary. | -| `NGROK_AUTHTOKEN` | _(unset)_ | `src/lib/ngrokTunnel.ts` | Authenticates outbound ngrok tunnels. | -| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts` | Maximum SQLite backup files retained on disk. | -| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. | -| `OMNIROUTE_TLS_PROXY_URL` | _(unset)_ | `open-sse/services/chatgptTlsClient.ts` | Override the TLS sidecar URL for tests. Production should leave unset. | -| `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` under rootless Podman so the fix instructions use `podman unshare chown` instead of `sudo chown`. | -| `QUOTA_STORE_DRIVER` | `sqlite` | `src/lib/quota/storeFactory.ts` | Quota-share consumption store backend: `sqlite` (default) or `redis`. | -| `QUOTA_STORE_REDIS_URL` | _(unset)_ | `src/lib/quota/storeFactory.ts` | Redis connection string used when `QUOTA_STORE_DRIVER=redis` (e.g. `redis://localhost:6379`). | -| `QUOTA_SATURATION_THRESHOLD` | `0.5` | `src/lib/quota/enforce.ts` | Pool saturation ratio (0..1); at/above it the pool enters strict mode (no borrowing). | -| `QUOTA_SOFT_DEPRIORITIZE_FACTOR` | `0.7` | `open-sse/services/combo.ts` | Score multiplier (0..1) applied to a target when the soft quota policy deprioritizes it. | -| `QUOTA_CONSUMPTION_RETENTION_DAYS` | `14` | `src/lib/db/quotaConsumption.ts` | Retention window (days) for `quota_consumption` buckets before GC (`gcQuotaConsumption`). | -| `AGENTBRIDGE_UPSTREAM_CA_CERT` | _(unset)_ | `src/mitm/manager.ts` | Extra CA certificate (PEM) trusted for AgentBridge upstream TLS connections. | -| `INSPECTOR_BUFFER_SIZE` | `1000` | `src/mitm/inspector/buffer.ts` | Max captured requests held in the Traffic Inspector ring buffer. | -| `INSPECTOR_MAX_BODY_KB` | `1024` | `src/mitm/inspector/buffer.ts` | Max captured request/response body size (KB) before truncation. | -| `INSPECTOR_HTTP_PROXY_PORT` | `8080` | `src/mitm/inspector/httpProxyServer.ts` | Local port for the Traffic Inspector HTTP proxy. | -| `INSPECTOR_HTTP_PROXY_AUTOSTART` | `false` | `src/mitm/inspector/httpProxyServer.ts` | Auto-start the inspector HTTP proxy on boot. | -| `INSPECTOR_TLS_INTERCEPT` | `false` | `src/lib/inspector/captureState.ts` | Enable TLS interception (MITM) for captured HTTPS traffic. | -| `INSPECTOR_LLM_HOSTS_EXTRA` | _(unset)_ | `src/lib/inspector/captureState.ts` | Extra hostnames (comma-separated) treated as LLM endpoints for capture. | -| `INSPECTOR_MASK_SECRETS` | `true` | `src/mitm/inspector/buffer.ts` | Mask secrets (auth headers / API keys) in captured traffic. | -| `INSPECTOR_SYSTEM_PROXY_GUARD_MINUTES` | `30` | `src/app/api/tools/traffic-inspector/capture-modes/system-proxy/route.ts` | Minutes before the system-proxy guard auto-reverts OS proxy settings. | -| `INSPECTOR_INTERNAL_INGEST_TOKEN` | _(auto)_ | `src/app/api/tools/traffic-inspector/internal/ingest/route.ts` | Token authenticating internal capture ingest into the inspector. | -| `PLAYGROUND_COMPARE_MAX_COLUMNS` | `4` | `src/app/(dashboard)/dashboard/playground/` | Max number of side-by-side columns in the Playground compare mode. | -| `PLAYGROUND_IMPROVE_PROMPT_DEFAULT_MODEL` | _(unset)_ | `src/app/(dashboard)/dashboard/playground/` | Default model for the Playground 'improve prompt' action (falls back to the active model when unset). | +| `COMMAND_CODE_CALLBACK_PORT` | _(unset)_ | `src/app/api/providers/command-code/auth/shared.ts` | Local port used for OAuth-style callbacks from the Command Code CLI helper. | +| `COMMAND_CODE_VERSION` | `0.33.2` | `open-sse/executors/commandCode.ts` | Value sent as the `x-command-code-version` header to the Command Code upstream. Override to bump the CLI version. | +| `MITM_LOCAL_PORT` | `443` | `src/mitm/server.cjs` | Local bind port for the MITM debug proxy. | +| `MITM_DISABLE_TLS_VERIFY` | `0` | `src/mitm/server.cjs` | Set `1` to disable upstream TLS verification (development only). | +| `ONEPROXY_ENABLED` | `true` | `src/lib/oneproxySync.ts` | Enable the 1Proxy egress pool sync. | +| `ONEPROXY_API_URL` | `https://1proxy-api.aitradepulse.com` | `src/lib/oneproxySync.ts` | 1Proxy service API URL override. | +| `ONEPROXY_MAX_PROXIES` | `500` | `src/lib/oneproxySync.ts` | Maximum proxies imported per sync. | +| `ONEPROXY_MIN_QUALITY_THRESHOLD` | `50` | `src/lib/oneproxySync.ts` | Minimum quality score for imported proxies. | +| `FREE_PROXY_1PROXY_ENABLED` | `true` | `src/lib/freeProxyProviders/oneproxy.ts` | Enable the 1proxy free proxy source. Set to `false` to disable. | +| `FREE_PROXY_1PROXY_API_URL` | _(see oneproxy.ts)_ | `src/lib/freeProxyProviders/oneproxy.ts` | 1proxy API URL override. | +| `FREE_PROXY_1PROXY_MAX` | `500` | `src/lib/freeProxyProviders/oneproxy.ts` | Maximum proxies fetched per sync from 1proxy. | +| `FREE_PROXY_1PROXY_MIN_QUALITY` | `50` | `src/lib/freeProxyProviders/oneproxy.ts` | Minimum quality score threshold for 1proxy imports. | +| `FREE_PROXY_PROXIFLY_ENABLED` | `true` | `src/lib/freeProxyProviders/proxifly.ts` | Enable the Proxifly free proxy source. Set to `false` to disable. | +| `FREE_PROXY_PROXIFLY_QUANTITY` | `100` | `src/lib/freeProxyProviders/proxifly.ts` | Number of proxies to fetch per Proxifly sync. | +| `FREE_PROXY_PROXIFLY_ANONYMITY` | `elite` | `src/lib/freeProxyProviders/proxifly.ts` | Anonymity level filter for Proxifly (`elite`, `anonymous`, `transparent`). | +| `FREE_PROXY_IPLOCATE_ENABLED` | `false` | `src/lib/freeProxyProviders/iplocate.ts` | Enable the IPLocate free proxy source. Opt-in only. | +| `FREE_PROXY_IPLOCATE_BASE_URL` | `https://raw.githubusercontent.com/iplocate/free-proxy-list/main/protocols` | `src/lib/freeProxyProviders/iplocate.ts` | IPLocate proxy list base URL override. | +| `NEXT_PUBLIC_VERCEL_RELAY_ENABLED` | `true` | `src/app/(dashboard)/…/ProxyPoolTab.tsx` | Show/hide the Deploy Vercel Relay button in the Proxy Pool tab. | +| `VERCEL_API_BASE` | `https://api.vercel.com` | `src/app/api/settings/proxy/vercel-deploy/route.ts` | Vercel API base URL override (for testing). | +| `NEXT_PUBLIC_VERCEL_RELAY_DEFAULT_PROJECT` | `omniroute-relay` | `src/app/(dashboard)/…/VercelRelayModal.tsx` | Default project name pre-filled in the Vercel Relay deploy modal. | +| `TAILSCALE_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscale` binary. | +| `TAILSCALED_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscaled` daemon binary. | +| `NGROK_AUTHTOKEN` | _(unset)_ | `src/lib/ngrokTunnel.ts` | Authenticates outbound ngrok tunnels. | +| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts` | Maximum SQLite backup files retained on disk. | +| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. | +| `OMNIROUTE_TLS_PROXY_URL` | _(unset)_ | `open-sse/services/chatgptTlsClient.ts` | Override the TLS sidecar URL for tests. Production should leave unset. | +| `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` under rootless Podman so the fix instructions use `podman unshare chown` instead of `sudo chown`. | +| `QUOTA_STORE_DRIVER` | `sqlite` | `src/lib/quota/storeFactory.ts` | Quota-share consumption store backend: `sqlite` (default) or `redis`. | +| `QUOTA_STORE_REDIS_URL` | _(unset)_ | `src/lib/quota/storeFactory.ts` | Redis connection string used when `QUOTA_STORE_DRIVER=redis` (e.g. `redis://localhost:6379`). | +| `QUOTA_SATURATION_THRESHOLD` | `0.5` | `src/lib/quota/enforce.ts` | Pool saturation ratio (0..1); at/above it the pool enters strict mode (no borrowing). | +| `QUOTA_SOFT_DEPRIORITIZE_FACTOR` | `0.7` | `open-sse/services/combo.ts` | Score multiplier (0..1) applied to a target when the soft quota policy deprioritizes it. | +| `QUOTA_CONSUMPTION_RETENTION_DAYS` | `14` | `src/lib/db/quotaConsumption.ts` | Retention window (days) for `quota_consumption` buckets before GC (`gcQuotaConsumption`). | +| `AGENTBRIDGE_UPSTREAM_CA_CERT` | _(unset)_ | `src/mitm/manager.ts` | Extra CA certificate (PEM) trusted for AgentBridge upstream TLS connections. | +| `INSPECTOR_BUFFER_SIZE` | `1000` | `src/mitm/inspector/buffer.ts` | Max captured requests held in the Traffic Inspector ring buffer. | +| `INSPECTOR_MAX_BODY_KB` | `1024` | `src/mitm/inspector/buffer.ts` | Max captured request/response body size (KB) before truncation. | +| `INSPECTOR_HTTP_PROXY_PORT` | `8080` | `src/mitm/inspector/httpProxyServer.ts` | Local port for the Traffic Inspector HTTP proxy. | +| `INSPECTOR_HTTP_PROXY_AUTOSTART` | `false` | `src/mitm/inspector/httpProxyServer.ts` | Auto-start the inspector HTTP proxy on boot. | +| `INSPECTOR_TLS_INTERCEPT` | `false` | `src/lib/inspector/captureState.ts` | Enable TLS interception (MITM) for captured HTTPS traffic. | +| `INSPECTOR_LLM_HOSTS_EXTRA` | _(unset)_ | `src/lib/inspector/captureState.ts` | Extra hostnames (comma-separated) treated as LLM endpoints for capture. | +| `INSPECTOR_MASK_SECRETS` | `true` | `src/mitm/inspector/buffer.ts` | Mask secrets (auth headers / API keys) in captured traffic. | +| `INSPECTOR_SYSTEM_PROXY_GUARD_MINUTES` | `30` | `src/app/api/tools/traffic-inspector/capture-modes/system-proxy/route.ts` | Minutes before the system-proxy guard auto-reverts OS proxy settings. | +| `INSPECTOR_INTERNAL_INGEST_TOKEN` | _(auto)_ | `src/app/api/tools/traffic-inspector/internal/ingest/route.ts` | Token authenticating internal capture ingest into the inspector. | +| `PLAYGROUND_COMPARE_MAX_COLUMNS` | `4` | `src/app/(dashboard)/dashboard/playground/` | Max number of side-by-side columns in the Playground compare mode. | +| `PLAYGROUND_IMPROVE_PROMPT_DEFAULT_MODEL` | _(unset)_ | `src/app/(dashboard)/dashboard/playground/` | Default model for the Playground 'improve prompt' action (falls back to the active model when unset). | --- diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 144ea23f23..ca7539d859 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -8720,7 +8720,7 @@ "subtitle": "Best free providers ranked by model ELO scores from Arena AI leaderboards", "loading": "Loading rankings...", "errorLoading": "Failed to load rankings", - "emptyState": "No rankings available. Ensure Arena ELO sync is enabled (ARENA_ELO_SYNC_ENABLED=true).", + "emptyState": "No rankings available yet. Arena ELO data syncs on startup (daily) — check back shortly, or trigger a manual sync. Set ARENA_ELO_SYNC_ENABLED=false to opt out.", "bestModel": "Best", "allCategories": "All Categories", "categoryDefault": "Default", diff --git a/src/lib/arenaEloSync.ts b/src/lib/arenaEloSync.ts index 587370931c..dc6b9b2bd9 100644 --- a/src/lib/arenaEloSync.ts +++ b/src/lib/arenaEloSync.ts @@ -7,7 +7,7 @@ * * Resolution order: user overrides > synced arena ELO > defaults * - * Opt-in via ARENA_ELO_SYNC_ENABLED=true (default: false). + * On by default; opt out via ARENA_ELO_SYNC_ENABLED=false. */ import { backupDbFile } from "./db/backup"; @@ -96,8 +96,7 @@ export interface SyncStatus { // ─── Configuration ─────────────────────────────────────── -const ARENA_ELO_API_BASE = - "https://api.wulong.dev/arena-ai-leaderboards/v1/leaderboard"; +const ARENA_ELO_API_BASE = "https://api.wulong.dev/arena-ai-leaderboards/v1/leaderboard"; /** Leaderboard categories to fetch from the Arena API. */ const FETCH_CATEGORIES = ["text", "code"] as const; @@ -166,9 +165,7 @@ const EXPIRY_DAYS = 7; const parsedInterval = parseInt(process.env.ARENA_ELO_SYNC_INTERVAL || "86400", 10); const SYNC_INTERVAL_MS = - Number.isFinite(parsedInterval) && parsedInterval > 0 - ? parsedInterval * 1000 - : 86400 * 1000; + Number.isFinite(parsedInterval) && parsedInterval > 0 ? parsedInterval * 1000 : 86400 * 1000; // ─── Periodic sync state ───────────────────────────────── @@ -245,9 +242,7 @@ export async function fetchArenaLeaderboards(): Promise { await Promise.all(fetches); if (Object.keys(result).length === 0) { - throw new Error( - `All Arena leaderboard fetches failed: ${errors.join("; ")}` - ); + throw new Error(`All Arena leaderboard fetches failed: ${errors.join("; ")}`); } return result; @@ -291,11 +286,9 @@ export function transformToModelIntelligence( data: ArenaLeaderboardMap ): Array> { const entries: Array> = []; - const expiresAt = new Date( - Date.now() + EXPIRY_DAYS * 24 * 60 * 60 * 1000 - ).toISOString(); + const expiresAt = new Date(Date.now() + EXPIRY_DAYS * 24 * 60 * 60 * 1000).toISOString(); - for (const [category, leaderboard] of Object.entries(data)) { + for (const [category, leaderboard] of Object.entries(data)) { const taskCategories = CATEGORY_TASK_MAP[category]; if (!taskCategories) continue; @@ -376,9 +369,7 @@ export async function syncArenaElo(dryRun = false): Promise { deleteExpiredIntelligence(); } catch (err) { const message = err instanceof Error ? err.message : String(err); - console.warn( - `[ARENA_ELO_SYNC] Failed to delete expired intelligence: ${message}` - ); + console.warn(`[ARENA_ELO_SYNC] Failed to delete expired intelligence: ${message}`); } } @@ -390,9 +381,7 @@ export async function syncArenaElo(dryRun = false): Promise { bulkUpsertModelIntelligence(entries); } catch (err) { const message = err instanceof Error ? err.message : String(err); - console.warn( - `[ARENA_ELO_SYNC] Failed to bulk upsert intelligence: ${message}` - ); + console.warn(`[ARENA_ELO_SYNC] Failed to bulk upsert intelligence: ${message}`); return { success: false, modelCount: 0, @@ -461,9 +450,7 @@ function startPeriodicSync(intervalMs?: number): void { const interval = intervalMs ?? SYNC_INTERVAL_MS; activeSyncIntervalMs = interval; - console.log( - `[ARENA_ELO_SYNC] Starting periodic sync every ${interval / 1000}s` - ); + console.log(`[ARENA_ELO_SYNC] Starting periodic sync every ${interval / 1000}s`); // Initial sync (non-blocking) syncArenaElo() @@ -485,9 +472,7 @@ function startPeriodicSync(intervalMs?: number): void { syncArenaElo() .then((result) => { if (result.success) { - console.log( - `[ARENA_ELO_SYNC] Periodic sync complete: ${result.modelCount} entries` - ); + console.log(`[ARENA_ELO_SYNC] Periodic sync complete: ${result.modelCount} entries`); } }) .catch((err) => { @@ -522,16 +507,14 @@ export function stopArenaEloSync(): void { * next scheduled sync time, interval, and active sources. */ export function getArenaEloSyncStatus(): SyncStatus { - const enabled = process.env.ARENA_ELO_SYNC_ENABLED === "true"; + const enabled = process.env.ARENA_ELO_SYNC_ENABLED !== "false"; return { enabled, lastSync: lastSyncTime, lastSyncModelCount, nextSync: syncTimer && lastSyncTime - ? new Date( - new Date(lastSyncTime).getTime() + activeSyncIntervalMs - ).toISOString() + ? new Date(new Date(lastSyncTime).getTime() + activeSyncIntervalMs).toISOString() : null, intervalMs: activeSyncIntervalMs, sources: ["arena_elo"], @@ -543,17 +526,17 @@ export function getArenaEloSyncStatus(): SyncStatus { /** * Initialize Arena ELO sync if enabled via environment variable. * - * Reads `ARENA_ELO_SYNC_ENABLED` (default: false). When enabled, - * starts periodic sync with the interval from `ARENA_ELO_SYNC_INTERVAL` + * Reads `ARENA_ELO_SYNC_ENABLED` (default: true; set to `false` to opt out). + * When enabled, starts periodic sync with the interval from `ARENA_ELO_SYNC_INTERVAL` * (default: 86400 seconds / daily). * * All errors during initialization or the initial sync are caught and logged * — initialization is never fatal. */ export async function initArenaEloSync(): Promise { - if (process.env.ARENA_ELO_SYNC_ENABLED !== "true") { + if (process.env.ARENA_ELO_SYNC_ENABLED === "false") { console.log( - "[ARENA_ELO_SYNC] Disabled (set ARENA_ELO_SYNC_ENABLED=true to enable)" + "[ARENA_ELO_SYNC] Disabled (ARENA_ELO_SYNC_ENABLED=false). Unset or =true to enable." ); return; } diff --git a/src/server-init.ts b/src/server-init.ts index bf04fa3063..8321226eb4 100644 --- a/src/server-init.ts +++ b/src/server-init.ts @@ -132,8 +132,9 @@ async function startServer() { } } - // Arena ELO sync: opt-in model intelligence from leaderboard data (non-blocking, never fatal) - if (process.env.ARENA_ELO_SYNC_ENABLED === "true") { + // Arena ELO sync: model intelligence from leaderboard data (non-blocking, never fatal). + // On by default; opt out with ARENA_ELO_SYNC_ENABLED=false. + if (process.env.ARENA_ELO_SYNC_ENABLED !== "false") { try { const { initArenaEloSync } = await import("./lib/arenaEloSync"); await initArenaEloSync(); diff --git a/tests/unit/arena-elo-sync.test.ts b/tests/unit/arena-elo-sync.test.ts index 2c13923428..0611c384ea 100644 --- a/tests/unit/arena-elo-sync.test.ts +++ b/tests/unit/arena-elo-sync.test.ts @@ -13,17 +13,15 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync( - path.join(os.tmpdir(), "omniroute-arena-elo-test-"), -); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-arena-elo-test-")); process.env.DATA_DIR = TEST_DATA_DIR; const MIGRATION_SQL = fs.readFileSync( path.resolve( import.meta.dirname ?? __dirname, - "../../src/lib/db/migrations/097_model_intelligence.sql", + "../../src/lib/db/migrations/097_model_intelligence.sql" ), - "utf8", + "utf8" ); import { tryOpenSync } from "../../src/lib/db/adapters/driverFactory"; @@ -48,9 +46,7 @@ import type { const originalFetch = globalThis.fetch; -function mockFetch( - impl: (url: string, opts?: RequestInit) => Promise, -): void { +function mockFetch(impl: (url: string, opts?: RequestInit) => Promise): void { globalThis.fetch = impl as typeof fetch; } @@ -80,7 +76,7 @@ function makeModelEntry(overrides: Partial = {}): ArenaModelEnt function makeLeaderboardData( models: ArenaModelEntry[] = [], - category = "text", + category = "text" ): ArenaLeaderboardData { return { meta: { leaderboard: category, model_count: models.length }, @@ -89,7 +85,7 @@ function makeLeaderboardData( } function makeLeaderboardMap( - categories: Partial>, + categories: Partial> ): ArenaLeaderboardMap { const map: ArenaLeaderboardMap = {}; for (const [cat, models] of Object.entries(categories)) { @@ -103,7 +99,7 @@ let testAdapter: SqliteAdapter; function createTestAdapter(): SqliteAdapter { const patchedSql = MIGRATION_SQL.replace( /\n\s*synced_at TEXT NOT NULL DEFAULT \(datetime\('now'\)\)/, - "\n synced_at TEXT NOT NULL", + "\n synced_at TEXT NOT NULL" ); const adapter = tryOpenSync(":memory:")!; adapter.exec(patchedSql); @@ -144,7 +140,7 @@ describe("normalizeModelName()", () => { it("strips 'anthropic/' vendor prefix", () => { assert.strictEqual( normalizeModelName("anthropic/claude-opus-4-6-thinking"), - "claude-opus-4-6-thinking", + "claude-opus-4-6-thinking" ); }); @@ -195,9 +191,7 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const topEntry = entries.find( - (e) => e.model === "top-model" && e.category === "default", - ); + const topEntry = entries.find((e) => e.model === "top-model" && e.category === "default"); assert.ok(topEntry); // taskFit = 0.4 + 0.58 * ((1500-1000) / 500) = 0.98 @@ -213,9 +207,7 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const lowEntry = entries.find( - (e) => e.model === "low-model" && e.category === "default", - ); + const lowEntry = entries.find((e) => e.model === "low-model" && e.category === "default"); assert.ok(lowEntry); // taskFit = 0.4 + 0.58 * ((1000-1000) / 500) = 0.4 @@ -231,9 +223,7 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const entry = entries.find( - (e) => e.model === "sparse-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "sparse-model" && e.category === "default"); assert.ok(entry); assert.strictEqual(entry.confidence, "low"); @@ -248,9 +238,7 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const entry = entries.find( - (e) => e.model === "mid-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "mid-model" && e.category === "default"); assert.ok(entry); assert.strictEqual(entry.confidence, "medium"); @@ -265,9 +253,7 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const entry = entries.find( - (e) => e.model === "popular-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "popular-model" && e.category === "default"); assert.ok(entry); assert.strictEqual(entry.confidence, "high"); @@ -275,9 +261,7 @@ describe("transformToModelIntelligence()", () => { it("category mapping: 'text' → [default, review, documentation, debugging]", () => { const data = makeLeaderboardMap({ - text: [ - makeModelEntry({ model: "text-model", score: 1200, votes: 5000, rank: 1 }), - ], + text: [makeModelEntry({ model: "text-model", score: 1200, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); @@ -286,25 +270,16 @@ describe("transformToModelIntelligence()", () => { .map((e) => e.category) .sort(); - assert.deepStrictEqual(categories, [ - "debugging", - "default", - "documentation", - "review", - ]); + assert.deepStrictEqual(categories, ["debugging", "default", "documentation", "review"]); }); it("category mapping: 'code' → [coding]", () => { const data = makeLeaderboardMap({ - code: [ - makeModelEntry({ model: "code-model", score: 1300, votes: 5000, rank: 1 }), - ], + code: [makeModelEntry({ model: "code-model", score: 1300, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); - const categories = entries - .filter((e) => e.model === "code-model") - .map((e) => e.category); + const categories = entries.filter((e) => e.model === "code-model").map((e) => e.category); assert.deepStrictEqual(categories, ["coding"]); }); @@ -312,38 +287,26 @@ describe("transformToModelIntelligence()", () => { it("expires_at is set to ~7 days in the future", () => { const before = Date.now(); const data = makeLeaderboardMap({ - text: [ - makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 }), - ], + text: [makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); const after = Date.now(); - const entry = entries.find( - (e) => e.model === "test-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "test-model" && e.category === "default"); assert.ok(entry); assert.ok(entry.expiresAt); const expiresMs = new Date(entry.expiresAt).getTime(); const sevenDaysMs = 7 * 24 * 60 * 60 * 1000; - assert.ok( - expiresMs >= before + sevenDaysMs - 2000, - `expiresAt too early: ${entry.expiresAt}`, - ); - assert.ok( - expiresMs <= after + sevenDaysMs + 2000, - `expiresAt too late: ${entry.expiresAt}`, - ); + assert.ok(expiresMs >= before + sevenDaysMs - 2000, `expiresAt too early: ${entry.expiresAt}`); + assert.ok(expiresMs <= after + sevenDaysMs + 2000, `expiresAt too late: ${entry.expiresAt}`); }); it("source is 'arena_elo' for all entries", () => { const data = makeLeaderboardMap({ - text: [ - makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 }), - ], + text: [makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); @@ -380,9 +343,7 @@ describe("transformToModelIntelligence()", () => { it("skips unknown leaderboard categories (e.g. vision)", () => { const data = makeLeaderboardMap({ - vision: [ - makeModelEntry({ model: "vision-model", score: 1300, votes: 5000, rank: 1 }), - ], + vision: [makeModelEntry({ model: "vision-model", score: 1300, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); @@ -398,26 +359,20 @@ describe("transformToModelIntelligence()", () => { }); const entries = transformToModelIntelligence(data); - const entry = entries.find( - (e) => e.model === "test-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "test-model" && e.category === "default"); assert.ok(entry); assert.strictEqual(entry.eloRaw, 1337); }); it("handles single-model leaderboard (eloRange = 1, avoids division by zero)", () => { const data = makeLeaderboardMap({ - text: [ - makeModelEntry({ model: "only-model", score: 1200, votes: 5000, rank: 1 }), - ], + text: [makeModelEntry({ model: "only-model", score: 1200, votes: 5000, rank: 1 })], }); const entries = transformToModelIntelligence(data); assert.ok(entries.length > 0); - const entry = entries.find( - (e) => e.model === "only-model" && e.category === "default", - ); + const entry = entries.find((e) => e.model === "only-model" && e.category === "default"); assert.ok(entry); assert.ok(Math.abs(entry.score - 0.4) < 0.001); }); @@ -449,11 +404,11 @@ describe("fetchArenaLeaderboards()", () => { it("successful fetch with valid JSON returns both leaderboards", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "text-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); const codeData = makeLeaderboardData( [makeModelEntry({ model: "code-model", score: 1300, votes: 5000, rank: 1 })], - "code", + "code" ); mockFetch(async (url: string) => { @@ -481,7 +436,7 @@ describe("fetchArenaLeaderboards()", () => { assert.ok(err instanceof Error); assert.ok(err.message.includes("All Arena leaderboard fetches failed")); return true; - }, + } ); }); @@ -496,20 +451,19 @@ describe("fetchArenaLeaderboards()", () => { assert.ok(err instanceof Error); assert.ok(err.message.includes("All Arena leaderboard fetches failed")); return true; - }, + } ); }); it("succeeds when one category fails but another succeeds", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "text-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return new Response("Error", { status: 500 }); + if (url.includes("name=code")) return new Response("Error", { status: 500 }); return new Response("Not found", { status: 404 }); }); @@ -532,7 +486,7 @@ describe("fetchArenaLeaderboards()", () => { assert.ok(err instanceof Error); assert.ok(err.message.includes("All Arena leaderboard fetches failed")); return true; - }, + } ); }); }); @@ -545,11 +499,11 @@ describe("syncArenaElo()", () => { it("happy path: returns success=true with correct modelCount", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "gpt-5.5", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); const codeData = makeLeaderboardData( [makeModelEntry({ model: "deepseek-r1", score: 1300, votes: 5000, rank: 1 })], - "code", + "code" ); mockFetch(async (url: string) => { @@ -572,13 +526,12 @@ describe("syncArenaElo()", () => { it("happy path: entries in DB have correct source and categories", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "unique-test-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -586,12 +539,7 @@ describe("syncArenaElo()", () => { const entries = getAllEntries().filter((e) => String(e.model) === "unique-test-model"); const categories = entries.map((e) => String(e.category)).sort(); - assert.deepStrictEqual(categories, [ - "debugging", - "default", - "documentation", - "review", - ]); + assert.deepStrictEqual(categories, ["debugging", "default", "documentation", "review"]); for (const entry of entries) { assert.strictEqual(entry.source, "arena_elo"); @@ -601,13 +549,12 @@ describe("syncArenaElo()", () => { it("dryRun=true → does not call bulkUpsertModelIntelligence", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -629,13 +576,12 @@ describe("syncArenaElo()", () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -659,26 +605,36 @@ describe("syncArenaElo()", () => { assert.ok(result.error); assert.ok( result.error!.includes("All Arena leaderboard fetches failed"), - `unexpected: ${result.error}`, + `unexpected: ${result.error}` ); }); it("calls deleteExpiredIntelligence before writing new entries", async () => { - testAdapter.prepare( - "INSERT INTO model_intelligence (model, source, category, score, elo_raw, confidence, synced_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - ).run("old-model", "arena_elo", "default", 0.5, 1000, "low", "2025-01-01T00:00:00Z", "2020-01-01T00:00:00Z"); + testAdapter + .prepare( + "INSERT INTO model_intelligence (model, source, category, score, elo_raw, confidence, synced_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) + .run( + "old-model", + "arena_elo", + "default", + 0.5, + 1000, + "low", + "2025-01-01T00:00:00Z", + "2020-01-01T00:00:00Z" + ); assert.strictEqual(countArenaEloEntries(), 1); const textData = makeLeaderboardData( [makeModelEntry({ model: "new-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -693,10 +649,8 @@ describe("syncArenaElo()", () => { it("handles empty leaderboard gracefully (0 entries, no DB write)", async () => { mockFetch(async (url: string) => { - if (url.includes("name=text")) - return jsonResponse(makeLeaderboardData([], "text")); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=text")) return jsonResponse(makeLeaderboardData([], "text")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -710,13 +664,12 @@ describe("syncArenaElo()", () => { it("updates lastSyncTime after successful sync", async () => { const textData = makeLeaderboardData( [makeModelEntry({ model: "test-model", score: 1200, votes: 5000, rank: 1 })], - "text", + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -729,19 +682,20 @@ describe("syncArenaElo()", () => { it("model aliases are stored in DB alongside canonical names", async () => { const textData = makeLeaderboardData( - [makeModelEntry({ - model: "anthropic/claude-opus-4-6-thinking", - score: 1400, - votes: 5000, - rank: 1, - })], - "text", + [ + makeModelEntry({ + model: "anthropic/claude-opus-4-6-thinking", + score: 1400, + votes: 5000, + rank: 1, + }), + ], + "text" ); mockFetch(async (url: string) => { if (url.includes("name=text")) return jsonResponse(textData); - if (url.includes("name=code")) - return jsonResponse(makeLeaderboardData([], "code")); + if (url.includes("name=code")) return jsonResponse(makeLeaderboardData([], "code")); return new Response("Not found", { status: 404 }); }); @@ -791,7 +745,7 @@ describe("getArenaEloSyncStatus()", () => { assert.ok(typeof status.nextSync === "string" || status.nextSync === null); }); - it("reflects ARENA_ELO_SYNC_ENABLED env var", () => { + it("reflects ARENA_ELO_SYNC_ENABLED env var (on by default, opt-out)", () => { const original = process.env.ARENA_ELO_SYNC_ENABLED; process.env.ARENA_ELO_SYNC_ENABLED = "true"; @@ -802,6 +756,11 @@ describe("getArenaEloSyncStatus()", () => { const disabledStatus = getArenaEloSyncStatus(); assert.strictEqual(disabledStatus.enabled, false); + // Default (unset) is now enabled — only an explicit "false" opts out. + delete process.env.ARENA_ELO_SYNC_ENABLED; + const defaultStatus = getArenaEloSyncStatus(); + assert.strictEqual(defaultStatus.enabled, true); + if (original !== undefined) { process.env.ARENA_ELO_SYNC_ENABLED = original; } else {