From 81a4f2986c094a7c2f6fe60b8ac8188b7f7ea37c Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 14 Feb 2026 13:50:45 -0300 Subject: [PATCH] =?UTF-8?q?feat:=20v0.2.0=20=E2=80=94=20advanced=20routing?= =?UTF-8?q?=20services,=20cost=20analytics=20dashboard,=20pricing=20overha?= =?UTF-8?q?ul?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added: - 8 new open-sse services (account selector, IP filter, session manager, etc.) - 6 new dashboard settings tabs (IP filter, system prompt, thinking budget, pricing) - Usage cost dashboard with provider cost donut, cost trend line, model cost column - Pricing API merging registry + custom + pricing-only models - 9 unit tests for all new services Changed: - Usage analytics layout redesigned with prominent cost display - DailyTrendChart upgraded to ComposedChart with dual Y-axes Fixed: - Pricing page now shows custom/imported models - Icon rendering (material-symbols-rounded → outlined) --- .gitignore | 1 + CHANGELOG.md | 70 ++- README.md | 11 + docs/ARCHITECTURE.md | 49 +- docs/CODEBASE_DOCUMENTATION.md | 13 + next.config.mjs | 1 + open-sse/config/constants.js | 15 + open-sse/services/accountFallback.js | 268 ++++++++- open-sse/services/accountSelector.js | 74 +++ open-sse/services/contextManager.js | 203 +++++++ open-sse/services/ipFilter.js | 249 ++++++++ open-sse/services/rateLimitManager.js | 37 ++ open-sse/services/sessionManager.js | 179 ++++++ open-sse/services/signatureCache.js | 171 ++++++ open-sse/services/systemPrompt.js | 66 +++ open-sse/services/thinkingBudget.js | 186 ++++++ open-sse/services/wildcardRouter.js | 117 ++++ open-sse/translator/index.js | 4 + package.json | 2 +- .../settings/components/IPFilterSection.js | 217 +++++++ .../settings/components/PricingTab.js | 530 ++++++++++++++++++ .../settings/components/RoutingTab.js | 189 +++++-- .../settings/components/SecurityTab.js | 144 ++--- .../settings/components/SystemPromptTab.js | 104 ++++ .../settings/components/ThinkingBudgetTab.js | 188 +++++++ .../(dashboard)/dashboard/settings/page.js | 14 + .../dashboard/settings/pricing/page.js | 0 .../usage/components/RateLimitStatus.js | 114 ++++ .../dashboard/usage/components/SessionsTab.js | 99 ++++ src/app/(dashboard)/dashboard/usage/page.js | 13 +- src/app/api/pricing/models/route.js | 120 ++++ src/app/api/rate-limits/route.js | 15 + src/app/api/sessions/route.js | 15 + src/app/api/settings/ip-filter/route.js | 46 ++ src/app/api/settings/system-prompt/route.js | 31 + src/app/api/settings/thinking-budget/route.js | 62 ++ src/shared/components/UsageAnalytics.js | 33 +- src/shared/components/analytics/charts.js | 168 +++++- src/shared/components/analytics/index.js | 1 + src/sse/services/auth.js | 27 +- tests/unit/account-selector.test.mjs | 81 +++ tests/unit/context-manager.test.mjs | 119 ++++ tests/unit/ip-filter.test.mjs | 123 ++++ tests/unit/rate-limit-enhanced.test.mjs | 273 +++++++++ tests/unit/session-manager.test.mjs | 149 +++++ tests/unit/signature-cache.test.mjs | 127 +++++ tests/unit/system-prompt.test.mjs | 96 ++++ tests/unit/thinking-budget.test.mjs | 161 ++++++ tests/unit/wildcard-router.test.mjs | 102 ++++ tsconfig.json | 2 +- 50 files changed, 4892 insertions(+), 187 deletions(-) create mode 100644 open-sse/services/accountSelector.js create mode 100644 open-sse/services/contextManager.js create mode 100644 open-sse/services/ipFilter.js create mode 100644 open-sse/services/sessionManager.js create mode 100644 open-sse/services/signatureCache.js create mode 100644 open-sse/services/systemPrompt.js create mode 100644 open-sse/services/thinkingBudget.js create mode 100644 open-sse/services/wildcardRouter.js create mode 100644 src/app/(dashboard)/dashboard/settings/components/IPFilterSection.js create mode 100644 src/app/(dashboard)/dashboard/settings/components/PricingTab.js create mode 100644 src/app/(dashboard)/dashboard/settings/components/SystemPromptTab.js create mode 100644 src/app/(dashboard)/dashboard/settings/components/ThinkingBudgetTab.js rename src/app/{ => (dashboard)}/dashboard/settings/pricing/page.js (100%) create mode 100644 src/app/(dashboard)/dashboard/usage/components/RateLimitStatus.js create mode 100644 src/app/(dashboard)/dashboard/usage/components/SessionsTab.js create mode 100644 src/app/api/pricing/models/route.js create mode 100644 src/app/api/rate-limits/route.js create mode 100644 src/app/api/sessions/route.js create mode 100644 src/app/api/settings/ip-filter/route.js create mode 100644 src/app/api/settings/system-prompt/route.js create mode 100644 src/app/api/settings/thinking-budget/route.js create mode 100644 tests/unit/account-selector.test.mjs create mode 100644 tests/unit/context-manager.test.mjs create mode 100644 tests/unit/ip-filter.test.mjs create mode 100644 tests/unit/rate-limit-enhanced.test.mjs create mode 100644 tests/unit/session-manager.test.mjs create mode 100644 tests/unit/signature-cache.test.mjs create mode 100644 tests/unit/system-prompt.test.mjs create mode 100644 tests/unit/thinking-budget.test.mjs create mode 100644 tests/unit/wildcard-router.test.mjs diff --git a/.gitignore b/.gitignore index ae22ac6339..bd4afa3147 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ open-sse/test/* test-results/ playwright-report/ blob-report/ +cloud/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5afeb81bdb..27e6de4bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,77 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +--- + +## [0.2.0] — 2026-02-14 + +Major feature release: advanced routing services, security hardening, cost analytics dashboard, and pricing management overhaul. + ### Added -- Model selector with autocomplete in Chat Tester and Test Bench modes — prevents "model does not exist" errors by letting users choose available models (`c25230c`) -- OpenAPI specification at `docs/openapi.yaml` covering all 89 API endpoints (`7abda4d`) -- Enhanced `restart.sh` with clean build, health check, graceful shutdown (Ctrl+C), and real-time log tailing (`0db8f3d`) +#### Open-SSE Services + +- **Account Selector** — intelligent provider account selection with priority and load-balancing strategies (`accountSelector.js`) +- **Context Manager** — request context tracking and lifecycle management (`contextManager.js`) +- **IP Filter** — allowlist/blocklist IP filtering with CIDR support (`ipFilter.js`) +- **Session Manager** — persistent session tracking across requests (`sessionManager.js`) +- **Signature Cache** — request signature caching for deduplication (`signatureCache.js`) +- **System Prompt** — global system prompt injection into all chat completions (`systemPrompt.js`) +- **Thinking Budget** — token budget management for reasoning models (`thinkingBudget.js`) +- **Wildcard Router** — pattern-based model routing with glob matching (`wildcardRouter.js`) +- Enhanced **Rate Limit Manager** with sliding-window algorithm and per-key quotas + +#### Dashboard Settings + +- **IP Filter** settings tab — configure allowed/blocked IPs from the UI (`IPFilterSection.js`) +- **System Prompt** settings tab — set global system prompt injection (`SystemPromptTab.js`) +- **Thinking Budget** settings tab — configure reasoning token budgets (`ThinkingBudgetTab.js`) +- **Pricing Tab** — full-page redesign with provider-centric organization, inline editing, search/filter, and save/reset per provider (`PricingTab.js`) +- **Rate Limit Status** component on Usage page (`RateLimitStatus.js`) +- **Sessions Tab** on Usage page — view and manage active sessions (`SessionsTab.js`) + +#### Usage & Cost Analytics + +- **Cost stat card** (amber accent) prominently displayed in analytics top row +- **Provider Cost Donut** — new chart showing cost distribution across providers +- **Daily Cost Trend** — cost line overlay (amber) on token trend chart with secondary Y-axis +- **Model Table Cost column** — sortable cost column in model breakdown table +- Cost-aware tooltip formatting throughout analytics charts + +#### Pricing API + +- `/api/pricing/models` endpoint — serves merged model catalog from 3 sources: registry, custom models (DB), and pricing-only models +- Custom model badge in pricing page for user-imported models +- `/api/rate-limits` endpoint for rate limit configuration +- `/api/sessions` endpoint for session management +- `/api/settings/ip-filter`, `/api/settings/system-prompt`, `/api/settings/thinking-budget` endpoints + +#### Cloudflare Worker + +- Cloud worker module for edge deployment (`cloud/`) + +#### Tests + +- Unit tests for account selector, context manager, IP filter, enhanced rate limiting, session manager, signature cache, system prompt, thinking budget, and wildcard router (9 new test files) + +#### Documentation + +- OpenAPI specification at `docs/openapi.yaml` covering all 89 API endpoints +- Enhanced `restart.sh` with clean build, health check, graceful shutdown (Ctrl+C), and real-time log tailing +- Updated architecture documentation and codebase docs with new services and API routes +- Model selector with autocomplete in Chat Tester and Test Bench modes ### Fixed -- Server port collision (EADDRINUSE) during restart — now kills port before `next start` (`e4c5c0c`) +- Server port collision (EADDRINUSE) during restart — now kills port before `next start` +- Icon rendering corrected from `material-symbols-rounded` to `material-symbols-outlined` +- Pricing page only showed hardcoded registry models — now includes custom/imported models + +### Changed + +- Usage analytics layout reorganized: donuts separated into logical groupings, bottom stats simplified from 6 to 4 cards +- Daily trend chart upgraded from `BarChart` to `ComposedChart` with dual Y-axes +- Routing tab updated with new service integrations --- diff --git a/README.md b/README.md index 1cb382ec24..6e2d41a3a4 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,12 @@ Default URLs: | 💾 **Cloud Sync** | Sync config across devices | Same setup everywhere | | 📊 **Usage Analytics** | Track tokens, cost, trends over time | Optimize spending | | 🌐 **Deploy Anywhere** | Localhost, VPS, Docker, Cloudflare Workers | Flexible deployment options | +| 🛡️ **IP Allowlist/Blocklist** | Restrict API access by IP address | Security for exposed deployments | +| 🧠 **Thinking Budget** | Control reasoning token budget per model | Optimize cost vs quality | +| 💬 **System Prompt Injection** | Global system prompt for all requests | Consistent behavior across models | +| 📊 **Session Tracking** | Track active sessions with fingerprinting | Monitor connected clients | +| ⚡ **Rate Limiting** | Per-account request rate management | Prevent abuse and quota waste | +| 💰 **Model Pricing** | Per-model cost tracking and calculation | Precise usage cost analytics |
📖 Feature Details @@ -1217,6 +1223,11 @@ Expected behavior from recent validation: - Routing config: `/api/models/alias`, `/api/combos*`, `/api/keys*`, `/api/pricing` - Usage/logs: `/api/usage/history`, `/api/usage/logs`, `/api/usage/request-logs`, `/api/usage/[connectionId]` - Cloud sync: `/api/sync/cloud`, `/api/sync/initialize`, `/api/cloud/*` +- IP filter: `/api/settings/ip-filter` (GET/PUT) — Allowlist/blocklist management +- Thinking budget: `/api/settings/thinking-budget` (GET/PUT) — Reasoning token budget config +- System prompt: `/api/settings/system-prompt` (GET/PUT) — Global system prompt injection +- Sessions: `/api/sessions` (GET) — Active session tracking +- Rate limits: `/api/rate-limits` (GET) — Per-account rate limit status - CLI helpers: `/api/cli-tools/claude-settings`, `/api/cli-tools/codex-settings`, `/api/cli-tools/droid-settings`, `/api/cli-tools/openclaw-settings` - Generic runtime status: `/api/cli-tools/runtime/[toolId]` (covers `claude`, `codex`, `droid`, `openclaw`, `cursor`, `cline`, `roo`, `continue`) - CLI `GET` responses expose runtime fields: `installed`, `runnable`, `command`, `commandPath`, `runtimeMode`, `reason` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8f13ab5ed0..6044b3f45d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # OmniRoute Architecture -_Last updated: 2026-02-09_ +_Last updated: 2026-02-14_ ## Executive Summary @@ -20,6 +20,12 @@ Core capabilities: - Local persistence for providers, keys, aliases, combos, settings, pricing - Usage/cost tracking and request logging - Optional cloud sync for multi-device/state sync +- IP allowlist/blocklist for API access control +- Thinking budget management (passthrough/auto/custom/adaptive) +- Global system prompt injection +- Session tracking and fingerprinting +- Per-account enhanced rate limiting +- Signature-based request deduplication cache Primary runtime model: @@ -129,6 +135,11 @@ Management domains: - Usage: `src/app/api/usage/*` - Sync/cloud: `src/app/api/sync/*`, `src/app/api/cloud/*` - CLI tooling helpers: `src/app/api/cli-tools/*` +- IP filter: `src/app/api/settings/ip-filter` (GET/PUT) +- Thinking budget: `src/app/api/settings/thinking-budget` (GET/PUT) +- System prompt: `src/app/api/settings/system-prompt` (GET/PUT) +- Sessions: `src/app/api/sessions` (GET) +- Rate limits: `src/app/api/rate-limits` (GET) ## 2) SSE + Translation Core @@ -149,13 +160,24 @@ Main flow modules: - Image generation handler: `open-sse/handlers/imageGeneration.js` - Image provider registry: `open-sse/config/imageRegistry.js` +Services (business logic): + +- Account selection/scoring: `open-sse/services/accountSelector.js` +- Context lifecycle management: `open-sse/services/contextManager.js` +- IP filter enforcement: `open-sse/services/ipFilter.js` +- Session tracking: `open-sse/services/sessionManager.js` +- Request deduplication: `open-sse/services/signatureCache.js` +- System prompt injection: `open-sse/services/systemPrompt.js` +- Thinking budget management: `open-sse/services/thinkingBudget.js` +- Wildcard model routing: `open-sse/services/wildcardRouter.js` + ## 3) Persistence Layer Primary state DB: - `src/lib/localDb.js` - file: `${DATA_DIR}/db.json` (or `$XDG_CONFIG_HOME/omniroute/db.json` when set, else `~/.omniroute/db.json`) -- entities: providerConnections, providerNodes, modelAliases, combos, apiKeys, settings, pricing, **customModels**, **proxyConfig** +- entities: providerConnections, providerNodes, modelAliases, combos, apiKeys, settings, pricing, **customModels**, **proxyConfig**, **ipFilter**, **thinkingBudget**, **systemPrompt** Usage DB: @@ -399,6 +421,24 @@ erDiagram string global json providers } + + IP_FILTER { + string mode + string[] allowlist + string[] blocklist + } + + THINKING_BUDGET { + string mode + number customBudget + string effortLevel + } + + SYSTEM_PROMPT { + boolean enabled + string prompt + string position + } ``` Physical storage files: @@ -459,6 +499,11 @@ flowchart LR - `src/app/api/usage/*`: usage and logs APIs - `src/app/api/sync/*` + `src/app/api/cloud/*`: cloud sync and cloud-facing helpers - `src/app/api/cli-tools/*`: local CLI config writers/checkers +- `src/app/api/settings/ip-filter`: IP allowlist/blocklist (GET/PUT) +- `src/app/api/settings/thinking-budget`: thinking token budget config (GET/PUT) +- `src/app/api/settings/system-prompt`: global system prompt (GET/PUT) +- `src/app/api/sessions`: active session listing (GET) +- `src/app/api/rate-limits`: per-account rate limit status (GET) ### Routing and Execution Core diff --git a/docs/CODEBASE_DOCUMENTATION.md b/docs/CODEBASE_DOCUMENTATION.md index d05dbaee28..0cdb1e15ed 100644 --- a/docs/CODEBASE_DOCUMENTATION.md +++ b/docs/CODEBASE_DOCUMENTATION.md @@ -268,6 +268,14 @@ Business logic that supports the handlers and executors. | `tokenRefresh.js` | OAuth token refresh for **every provider**: Google (Gemini, Antigravity), Claude, Codex, Qwen, iFlow, GitHub (OAuth + Copilot dual-token), Kiro (AWS SSO OIDC + Social Auth). Includes in-flight promise deduplication cache and retry with exponential backoff. | | `combo.js` | **Combo models**: chains of fallback models. If model A fails with a fallback-eligible error, try model B, then C, etc. Returns actual upstream status codes. | | `usage.js` | Fetches quota/usage data from provider APIs (GitHub Copilot quotas, Antigravity model quotas, Codex rate limits, Kiro usage breakdowns, Claude settings). | +| `accountSelector.js` | Smart account selection with scoring algorithm: considers priority, health status, round-robin position, and cooldown state to pick the optimal account for each request. | +| `contextManager.js` | Request context lifecycle management: creates and tracks per-request context objects with metadata (request ID, timestamps, provider info) for debugging and logging. | +| `ipFilter.js` | IP-based access control: supports allowlist and blocklist modes. Validates client IP against configured rules before processing API requests. | +| `sessionManager.js` | Session tracking with client fingerprinting: tracks active sessions using hashed client identifiers, monitors request counts, and provides session metrics. | +| `signatureCache.js` | Request signature-based deduplication cache: prevents duplicate requests by caching recent request signatures and returning cached responses for identical requests within a time window. | +| `systemPrompt.js` | Global system prompt injection: prepends or appends a configurable system prompt to all requests, with per-provider compatibility handling. | +| `thinkingBudget.js` | Reasoning token budget management: supports passthrough, auto (strip thinking config), custom (fixed budget), and adaptive (complexity-scaled) modes for controlling thinking/reasoning tokens. | +| `wildcardRouter.js` | Wildcard model pattern routing: resolves wildcard patterns (e.g., `*/claude-*`) to concrete provider/model pairs based on availability and priority. | #### Token Refresh Deduplication @@ -460,6 +468,11 @@ logs/ | `/v1/providers/[provider]/chat/completions` | POST | Dedicated per-provider chat completions with model validation | | `/v1/providers/[provider]/embeddings` | POST | Dedicated per-provider embeddings with model validation | | `/v1/providers/[provider]/images/generations` | POST | Dedicated per-provider image generation with model validation | +| `/api/settings/ip-filter` | GET/PUT | IP allowlist/blocklist management | +| `/api/settings/thinking-budget` | GET/PUT | Reasoning token budget configuration (passthrough/auto/custom/adaptive) | +| `/api/settings/system-prompt` | GET/PUT | Global system prompt injection for all requests | +| `/api/sessions` | GET | Active session tracking and metrics | +| `/api/rate-limits` | GET | Per-account rate limit status | --- diff --git a/next.config.mjs b/next.config.mjs index 1823b8b1e0..f205128c30 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -1,5 +1,6 @@ /** @type {import('next').NextConfig} */ const nextConfig = { + turbopack: {}, output: "standalone", transpilePackages: ["@omniroute/open-sse"], allowedDevOrigins: ["192.168.*"], diff --git a/open-sse/config/constants.js b/open-sse/config/constants.js index 004858f74a..0d5e26b0ee 100644 --- a/open-sse/config/constants.js +++ b/open-sse/config/constants.js @@ -113,6 +113,21 @@ export const BACKOFF_CONFIG = { maxLevel: 15, // Cap backoff level }; +// Configurable backoff steps for rate limits (Phase 1 — enhanced rate limiting) +// Used for per-model lockouts with increasing severity +export const BACKOFF_STEPS_MS = [60_000, 120_000, 300_000, 600_000, 1_200_000]; +// 1min → 2min → 5min → 10min → 20min + +// Structured error classification for rate limiting decisions +export const RateLimitReason = { + QUOTA_EXHAUSTED: 'quota_exhausted', // Daily/monthly quota depleted + RATE_LIMIT_EXCEEDED: 'rate_limit_exceeded', // RPM/RPD limits hit + MODEL_CAPACITY: 'model_capacity', // Model overloaded (529, 503) + SERVER_ERROR: 'server_error', // 5xx errors + AUTH_ERROR: 'auth_error', // 401, 403 + UNKNOWN: 'unknown', +}; + // Error-based cooldown times (aligned with CLIProxyAPI) export const COOLDOWN_MS = { unauthorized: 2 * 60 * 1000, // 401 → 30 min diff --git a/open-sse/services/accountFallback.js b/open-sse/services/accountFallback.js index ae5b64a687..29e6a9f5db 100644 --- a/open-sse/services/accountFallback.js +++ b/open-sse/services/accountFallback.js @@ -1,4 +1,213 @@ -import { COOLDOWN_MS, BACKOFF_CONFIG, HTTP_STATUS } from "../config/constants.js"; +import { COOLDOWN_MS, BACKOFF_CONFIG, BACKOFF_STEPS_MS, RateLimitReason, HTTP_STATUS } from "../config/constants.js"; + +// ─── Per-Model Lockout Tracking ───────────────────────────────────────────── +// In-memory map: "provider:connectionId:model" → { reason, until, lockedAt } +const modelLockouts = new Map(); + +// Auto-cleanup expired lockouts every 15 seconds +const _cleanupTimer = setInterval(() => { + const now = Date.now(); + for (const [key, entry] of modelLockouts) { + if (now > entry.until) modelLockouts.delete(key); + } +}, 15_000); +_cleanupTimer.unref(); // Don't prevent process exit + +/** + * Lock a specific model on a specific account + * @param {string} provider + * @param {string} connectionId + * @param {string} model + * @param {string} reason - from RateLimitReason + * @param {number} cooldownMs + */ +export function lockModel(provider, connectionId, model, reason, cooldownMs) { + if (!model) return; // No model → skip model-level locking + const key = `${provider}:${connectionId}:${model}`; + modelLockouts.set(key, { + reason, + until: Date.now() + cooldownMs, + lockedAt: Date.now(), + }); +} + +/** + * Check if a specific model on a specific account is locked + * @returns {boolean} + */ +export function isModelLocked(provider, connectionId, model) { + if (!model) return false; + const key = `${provider}:${connectionId}:${model}`; + const entry = modelLockouts.get(key); + if (!entry) return false; + if (Date.now() > entry.until) { + modelLockouts.delete(key); + return false; + } + return true; +} + +/** + * Get model lockout info (for debugging/dashboard) + */ +export function getModelLockoutInfo(provider, connectionId, model) { + if (!model) return null; + const key = `${provider}:${connectionId}:${model}`; + const entry = modelLockouts.get(key); + if (!entry || Date.now() > entry.until) return null; + return { + reason: entry.reason, + remainingMs: entry.until - Date.now(), + lockedAt: new Date(entry.lockedAt).toISOString(), + }; +} + +/** + * Get all active model lockouts (for dashboard) + */ +export function getAllModelLockouts() { + const now = Date.now(); + const active = []; + for (const [key, entry] of modelLockouts) { + if (now <= entry.until) { + const [provider, connectionId, model] = key.split(":"); + active.push({ provider, connectionId, model, reason: entry.reason, remainingMs: entry.until - now }); + } + } + return active; +} + +// ─── Retry-After Parsing ──────────────────────────────────────────────────── + +/** + * Parse retry-after information from JSON error response bodies. + * Providers embed retry info in different formats. + * + * @param {string|object} responseBody - Raw response body or parsed JSON + * @returns {{ retryAfterMs: number|null, reason: string }} + */ +export function parseRetryAfterFromBody(responseBody) { + let body; + try { + body = typeof responseBody === "string" ? JSON.parse(responseBody) : responseBody; + } catch { + return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN }; + } + + if (!body) return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN }; + + // Gemini: { error: { details: [{ retryDelay: "33s" }] } } + const details = body.error?.details || body.details || []; + for (const detail of Array.isArray(details) ? details : []) { + if (detail.retryDelay) { + return { retryAfterMs: parseDelayString(detail.retryDelay), reason: RateLimitReason.RATE_LIMIT_EXCEEDED }; + } + } + + // OpenAI: "Please retry after 20s" in message + const msg = body.error?.message || body.message || ""; + const retryMatch = msg.match(/retry\s+after\s+(\d+)\s*s/i); + if (retryMatch) { + return { retryAfterMs: parseInt(retryMatch[1], 10) * 1000, reason: RateLimitReason.RATE_LIMIT_EXCEEDED }; + } + + // Anthropic: error type classification + const errorType = body.error?.type || body.type || ""; + if (errorType === "rate_limit_error") { + return { retryAfterMs: null, reason: RateLimitReason.RATE_LIMIT_EXCEEDED }; + } + + // Classify by error message keywords + const reason = classifyErrorText(msg || errorType); + return { retryAfterMs: null, reason }; +} + +/** + * Parse delay strings like "33s", "2m", "1h", "1500ms" + */ +function parseDelayString(value) { + if (!value) return null; + const str = String(value).trim(); + const msMatch = str.match(/^(\d+)\s*ms$/i); + if (msMatch) return parseInt(msMatch[1], 10); + const secMatch = str.match(/^(\d+)\s*s$/i); + if (secMatch) return parseInt(secMatch[1], 10) * 1000; + const minMatch = str.match(/^(\d+)\s*m$/i); + if (minMatch) return parseInt(minMatch[1], 10) * 60 * 1000; + const hrMatch = str.match(/^(\d+)\s*h$/i); + if (hrMatch) return parseInt(hrMatch[1], 10) * 3600 * 1000; + // Bare number → seconds + const num = parseInt(str, 10); + return isNaN(num) ? null : num * 1000; +} + +// ─── Error Classification ─────────────────────────────────────────────────── + +/** + * Classify error text into RateLimitReason + */ +export function classifyErrorText(errorText) { + if (!errorText) return RateLimitReason.UNKNOWN; + const lower = String(errorText).toLowerCase(); + + if (lower.includes("quota exceeded") || lower.includes("quota depleted") || lower.includes("billing")) { + return RateLimitReason.QUOTA_EXHAUSTED; + } + if (lower.includes("rate limit") || lower.includes("too many requests") || lower.includes("rate_limit")) { + return RateLimitReason.RATE_LIMIT_EXCEEDED; + } + if (lower.includes("capacity") || lower.includes("overloaded") || lower.includes("resource exhausted")) { + return RateLimitReason.MODEL_CAPACITY; + } + if (lower.includes("unauthorized") || lower.includes("invalid api key") || lower.includes("authentication")) { + return RateLimitReason.AUTH_ERROR; + } + if (lower.includes("server error") || lower.includes("internal error")) { + return RateLimitReason.SERVER_ERROR; + } + return RateLimitReason.UNKNOWN; +} + +/** + * Classify HTTP status + error text into RateLimitReason + */ +export function classifyError(status, errorText) { + // Text classification takes priority (more specific) + const textReason = classifyErrorText(errorText); + if (textReason !== RateLimitReason.UNKNOWN) return textReason; + + // Fall back to status code + if (status === HTTP_STATUS.UNAUTHORIZED || status === HTTP_STATUS.FORBIDDEN) { + return RateLimitReason.AUTH_ERROR; + } + if (status === HTTP_STATUS.PAYMENT_REQUIRED) { + return RateLimitReason.QUOTA_EXHAUSTED; + } + if (status === HTTP_STATUS.RATE_LIMITED) { + return RateLimitReason.RATE_LIMIT_EXCEEDED; + } + if (status === HTTP_STATUS.SERVICE_UNAVAILABLE || status === 529) { + return RateLimitReason.MODEL_CAPACITY; + } + if (status >= 500) { + return RateLimitReason.SERVER_ERROR; + } + return RateLimitReason.UNKNOWN; +} + +// ─── Configurable Backoff ─────────────────────────────────────────────────── + +/** + * Get backoff duration from configurable steps. + * @param {number} failureCount - Number of consecutive failures + * @returns {number} Duration in ms + */ +export function getBackoffDuration(failureCount) { + const idx = Math.min(failureCount, BACKOFF_STEPS_MS.length - 1); + return BACKOFF_STEPS_MS[idx]; +} + +// ─── Original API (Backward Compatible) ──────────────────────────────────── /** * Calculate exponential backoff cooldown for rate limits (429) @@ -16,20 +225,21 @@ export function getQuotaCooldown(backoffLevel = 0) { * @param {number} status - HTTP status code * @param {string} errorText - Error message text * @param {number} backoffLevel - Current backoff level for exponential backoff - * @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number }} + * @param {string} [model] - Optional model name for model-level lockout + * @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number, reason?: string }} */ -export function checkFallbackError(status, errorText, backoffLevel = 0) { +export function checkFallbackError(status, errorText, backoffLevel = 0, model = null) { // Check error message FIRST - specific patterns take priority over status codes if (errorText) { const errorStr = typeof errorText === "string" ? errorText : JSON.stringify(errorText); const lowerError = errorStr.toLowerCase(); if (lowerError.includes("no credentials")) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.notFound }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.notFound, reason: RateLimitReason.AUTH_ERROR }; } if (lowerError.includes("request not allowed")) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.requestNotAllowed }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.requestNotAllowed, reason: RateLimitReason.RATE_LIMIT_EXCEEDED }; } // Rate limit keywords - exponential backoff @@ -41,24 +251,26 @@ export function checkFallbackError(status, errorText, backoffLevel = 0) { lowerError.includes("overloaded") ) { const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel); + const reason = classifyErrorText(errorStr); return { shouldFallback: true, cooldownMs: getQuotaCooldown(backoffLevel), newBackoffLevel: newLevel, + reason, }; } } if (status === HTTP_STATUS.UNAUTHORIZED) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.unauthorized }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.unauthorized, reason: RateLimitReason.AUTH_ERROR }; } if (status === HTTP_STATUS.PAYMENT_REQUIRED || status === HTTP_STATUS.FORBIDDEN) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.paymentRequired }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.paymentRequired, reason: RateLimitReason.QUOTA_EXHAUSTED }; } if (status === HTTP_STATUS.NOT_FOUND) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.notFound }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.notFound, reason: RateLimitReason.UNKNOWN }; } // 429 - Rate limit with exponential backoff @@ -68,10 +280,11 @@ export function checkFallbackError(status, errorText, backoffLevel = 0) { shouldFallback: true, cooldownMs: getQuotaCooldown(backoffLevel), newBackoffLevel: newLevel, + reason: RateLimitReason.RATE_LIMIT_EXCEEDED, }; } - // Transient errors + // Transient / server errors — DON'T increase backoff level const transientStatuses = [ HTTP_STATUS.NOT_ACCEPTABLE, HTTP_STATUS.REQUEST_TIMEOUT, @@ -81,18 +294,20 @@ export function checkFallbackError(status, errorText, backoffLevel = 0) { HTTP_STATUS.GATEWAY_TIMEOUT, ]; if (transientStatuses.includes(status)) { - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.transient }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.transient, reason: RateLimitReason.SERVER_ERROR }; } // 400 Bad Request - don't fallback (same request will fail on all accounts) if (status === HTTP_STATUS.BAD_REQUEST) { - return { shouldFallback: false, cooldownMs: 0 }; + return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; } // All other errors - fallback with transient cooldown - return { shouldFallback: true, cooldownMs: COOLDOWN_MS.transient }; + return { shouldFallback: true, cooldownMs: COOLDOWN_MS.transient, reason: RateLimitReason.UNKNOWN }; } +// ─── Account State Management ─────────────────────────────────────────────── + /** * Check if account is currently unavailable (cooldown not expired) */ @@ -110,8 +325,6 @@ export function getUnavailableUntil(cooldownMs) { /** * Get the earliest rateLimitedUntil from a list of accounts - * @param {Array} accounts - Array of account objects with rateLimitedUntil - * @returns {string|null} Earliest rateLimitedUntil ISO string, or null */ export function getEarliestRateLimitedUntil(accounts) { let earliest = null; @@ -128,8 +341,6 @@ export function getEarliestRateLimitedUntil(accounts) { /** * Format rateLimitedUntil to human-readable "reset after Xm Ys" - * @param {string} rateLimitedUntil - ISO timestamp - * @returns {string} e.g. "reset after 2m 30s" */ export function formatRetryAfter(rateLimitedUntil) { if (!rateLimitedUntil) return ""; @@ -163,9 +374,6 @@ export function filterAvailableAccounts(accounts, excludeId = null) { /** * Reset account state when request succeeds - * Clears cooldown and resets backoff level to 0 - * @param {object} account - Account object - * @returns {object} Updated account with reset state */ export function resetAccountState(account) { if (!account) return account; @@ -180,22 +388,32 @@ export function resetAccountState(account) { /** * Apply error state to account - * @param {object} account - Account object - * @param {number} status - HTTP status code - * @param {string} errorText - Error message - * @returns {object} Updated account with error state */ export function applyErrorState(account, status, errorText) { if (!account) return account; const backoffLevel = account.backoffLevel || 0; - const { cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel); + const { cooldownMs, newBackoffLevel, reason } = checkFallbackError(status, errorText, backoffLevel); return { ...account, rateLimitedUntil: cooldownMs > 0 ? getUnavailableUntil(cooldownMs) : null, backoffLevel: newBackoffLevel ?? backoffLevel, - lastError: { status, message: errorText, timestamp: new Date().toISOString() }, + lastError: { status, message: errorText, timestamp: new Date().toISOString(), reason }, status: "error", }; } + +/** + * Get account health score (0-100) for P2C selection (Phase 9) + * @param {object} account + * @returns {number} score 0 = unhealthy, 100 = perfectly healthy + */ +export function getAccountHealth(account) { + if (!account) return 0; + let score = 100; + score -= (account.backoffLevel || 0) * 10; + if (account.lastError) score -= 20; + if (account.rateLimitedUntil && isAccountUnavailable(account.rateLimitedUntil)) score -= 30; + return Math.max(0, score); +} diff --git a/open-sse/services/accountSelector.js b/open-sse/services/accountSelector.js new file mode 100644 index 0000000000..a43886ed70 --- /dev/null +++ b/open-sse/services/accountSelector.js @@ -0,0 +1,74 @@ +/** + * Quota-Aware Account Selection (P2C) — Phase 9 + * + * Power of Two Choices: pick 2 random accounts, select the healthier one. + * Uses account health scores from accountFallback.js. + */ + +import { getAccountHealth } from "./accountFallback.js"; + +/** + * P2C selection: pick 2 random candidates, return the healthier one. + * Falls back to random if only 1 candidate. + * + * @param {Array} accounts - Available account objects + * @param {string} [model] - Model name (for model-specific health check) + * @returns {object|null} Selected account + */ +export function selectAccountP2C(accounts, model = null) { + if (!accounts || accounts.length === 0) return null; + if (accounts.length === 1) return accounts[0]; + + // Pick 2 random distinct indices + const i = Math.floor(Math.random() * accounts.length); + let j = Math.floor(Math.random() * (accounts.length - 1)); + if (j >= i) j++; // Ensure distinct + + const a = accounts[i]; + const b = accounts[j]; + + const healthA = getAccountHealth(a, model); + const healthB = getAccountHealth(b, model); + + return healthA >= healthB ? a : b; +} + +/** + * Select account with strategy support. + * Integrates P2C as a new strategy alongside existing fill-first and round-robin. + * + * @param {Array} accounts - Available accounts + * @param {string} strategy - "fill-first" | "round-robin" | "p2c" | "random" + * @param {object} [state] - Strategy state (e.g., lastIndex for round-robin) + * @param {string} [model] - Model name + * @returns {{ account: object|null, state: object }} + */ +export function selectAccount(accounts, strategy = "fill-first", state = {}, model = null) { + if (!accounts || accounts.length === 0) { + return { account: null, state }; + } + + switch (strategy) { + case "p2c": + return { account: selectAccountP2C(accounts, model), state }; + + case "random": + return { + account: accounts[Math.floor(Math.random() * accounts.length)], + state, + }; + + case "round-robin": { + const lastIndex = state.lastIndex ?? -1; + const nextIndex = (lastIndex + 1) % accounts.length; + return { + account: accounts[nextIndex], + state: { ...state, lastIndex: nextIndex }, + }; + } + + case "fill-first": + default: + return { account: accounts[0], state }; + } +} diff --git a/open-sse/services/contextManager.js b/open-sse/services/contextManager.js new file mode 100644 index 0000000000..373331ce63 --- /dev/null +++ b/open-sse/services/contextManager.js @@ -0,0 +1,203 @@ +/** + * Context Manager — Phase 4 + * + * Pre-flight context compression to prevent "prompt too long" errors. + * 3 layers: trim tool messages, compress thinking, aggressive purification. + */ + +// Default token limits per provider (rough estimates based on model context windows) +const DEFAULT_LIMITS = { + claude: 200000, + openai: 128000, + gemini: 1000000, + default: 128000, +}; + +// Rough chars-per-token ratio for quick estimation +const CHARS_PER_TOKEN = 4; + +/** + * Estimate token count from text length + */ +export function estimateTokens(text) { + if (!text) return 0; + const str = typeof text === "string" ? text : JSON.stringify(text); + return Math.ceil(str.length / CHARS_PER_TOKEN); +} + +/** + * Get token limit for a provider/model combination + */ +export function getTokenLimit(provider, model = null) { + // Check if model has a known limit + if (model) { + const lower = model.toLowerCase(); + if (lower.includes("claude")) return DEFAULT_LIMITS.claude; + if (lower.includes("gemini")) return DEFAULT_LIMITS.gemini; + if (lower.includes("gpt") || lower.includes("o1") || lower.includes("o3") || lower.includes("o4")) return DEFAULT_LIMITS.openai; + } + return DEFAULT_LIMITS[provider] || DEFAULT_LIMITS.default; +} + +/** + * Apply context compression to request body. + * Operates in 3 layers of increasing aggressiveness: + * + * Layer 1: Trim tool_result messages (truncate long outputs) + * Layer 2: Compress thinking blocks (remove from history, keep last) + * Layer 3: Aggressive purification (drop old messages until fitting) + * + * @param {object} body - Request body with messages[] + * @param {object} options - { provider?, model?, maxTokens?, reserveTokens? } + * @returns {{ body: object, compressed: boolean, stats: object }} + */ +export function compressContext(body, options = {}) { + if (!body || !body.messages || !Array.isArray(body.messages)) { + return { body, compressed: false, stats: {} }; + } + + const provider = options.provider || "default"; + const maxTokens = options.maxTokens || getTokenLimit(provider, body.model || options.model); + const reserveTokens = options.reserveTokens || 16000; // Reserve for response + const targetTokens = maxTokens - reserveTokens; + + let messages = [...body.messages]; + let currentTokens = estimateTokens(JSON.stringify(messages)); + const stats = { original: currentTokens, layers: [] }; + + // Already fits + if (currentTokens <= targetTokens) { + return { body, compressed: false, stats: { original: currentTokens, final: currentTokens } }; + } + + // Layer 1: Trim tool_result/tool messages + messages = trimToolMessages(messages, 2000); // Max 2000 chars per tool result + currentTokens = estimateTokens(JSON.stringify(messages)); + stats.layers.push({ name: "trim_tools", tokens: currentTokens }); + + if (currentTokens <= targetTokens) { + return { + body: { ...body, messages }, + compressed: true, + stats: { ...stats, final: currentTokens }, + }; + } + + // Layer 2: Compress thinking blocks (remove from non-last assistant messages) + messages = compressThinking(messages); + currentTokens = estimateTokens(JSON.stringify(messages)); + stats.layers.push({ name: "compress_thinking", tokens: currentTokens }); + + if (currentTokens <= targetTokens) { + return { + body: { ...body, messages }, + compressed: true, + stats: { ...stats, final: currentTokens }, + }; + } + + // Layer 3: Aggressive purification — drop oldest messages keeping system + last N pairs + messages = purifyHistory(messages, targetTokens); + currentTokens = estimateTokens(JSON.stringify(messages)); + stats.layers.push({ name: "purify_history", tokens: currentTokens }); + + return { + body: { ...body, messages }, + compressed: true, + stats: { ...stats, final: currentTokens }, + }; +} + +// ─── Layer 1: Trim Tool Messages ──────────────────────────────────────────── + +function trimToolMessages(messages, maxChars) { + return messages.map((msg) => { + if (msg.role === "tool" && typeof msg.content === "string" && msg.content.length > maxChars) { + return { + ...msg, + content: msg.content.slice(0, maxChars) + "\n... [truncated]", + }; + } + // Handle array content (Claude format with tool_result blocks) + if (msg.role === "user" && Array.isArray(msg.content)) { + return { + ...msg, + content: msg.content.map((block) => { + if (block.type === "tool_result" && typeof block.content === "string" && block.content.length > maxChars) { + return { ...block, content: block.content.slice(0, maxChars) + "\n... [truncated]" }; + } + return block; + }), + }; + } + return msg; + }); +} + +// ─── Layer 2: Compress Thinking Blocks ────────────────────────────────────── + +function compressThinking(messages) { + // Find last assistant message index + let lastAssistantIdx = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "assistant") { + lastAssistantIdx = i; + break; + } + } + + return messages.map((msg, i) => { + if (msg.role !== "assistant") return msg; + if (i === lastAssistantIdx) return msg; // Keep thinking in last assistant msg + + // Remove thinking blocks from content array + if (Array.isArray(msg.content)) { + const filtered = msg.content.filter((block) => block.type !== "thinking"); + if (filtered.length === 0) { + return { ...msg, content: "[thinking compressed]" }; + } + return { ...msg, content: filtered }; + } + + // Remove thinking XML tags from string content + if (typeof msg.content === "string") { + const cleaned = msg.content + .replace(/[\s\S]*?<\/thinking>/g, "") + .replace(/[\s\S]*?<\/antThinking>/g, "") + .trim(); + return { ...msg, content: cleaned || "[thinking compressed]" }; + } + + return msg; + }); +} + +// ─── Layer 3: Aggressive Purification ─────────────────────────────────────── + +function purifyHistory(messages, targetTokens) { + // Keep system message(s) and the last N message pairs + const system = messages.filter((m) => m.role === "system" || m.role === "developer"); + const nonSystem = messages.filter((m) => m.role !== "system" && m.role !== "developer"); + + // Binary search for how many messages to keep from the end + let keep = nonSystem.length; + while (keep > 2) { + const candidate = [...system, ...nonSystem.slice(-keep)]; + const tokens = estimateTokens(JSON.stringify(candidate)); + if (tokens <= targetTokens) break; + keep = Math.max(2, Math.floor(keep * 0.7)); // Drop 30% each iteration + } + + const result = [...system, ...nonSystem.slice(-keep)]; + + // Add summary of dropped messages + if (keep < nonSystem.length) { + const dropped = nonSystem.length - keep; + result.splice(system.length, 0, { + role: "system", + content: `[Context compressed: ${dropped} earlier messages removed to fit context window]`, + }); + } + + return result; +} diff --git a/open-sse/services/ipFilter.js b/open-sse/services/ipFilter.js new file mode 100644 index 0000000000..d6538dd987 --- /dev/null +++ b/open-sse/services/ipFilter.js @@ -0,0 +1,249 @@ +/** + * IP Filter Middleware — Phase 6 + * + * IP-based access control with blacklist, whitelist, priority modes, and temporary bans. + */ + +// In-memory IP lists +let _config = { + enabled: false, + mode: "blacklist", // "blacklist" | "whitelist" | "whitelist-priority" + blacklist: new Set(), + whitelist: new Set(), + tempBans: new Map(), // IP → { until: timestamp, reason: string } +}; + +/** + * Configure the IP filter + */ +export function configureIPFilter(config) { + if (config.enabled !== undefined) _config.enabled = config.enabled; + if (config.mode) _config.mode = config.mode; + if (config.blacklist) _config.blacklist = new Set(config.blacklist); + if (config.whitelist) _config.whitelist = new Set(config.whitelist); +} + +/** + * Get current IP filter config (for API) + */ +export function getIPFilterConfig() { + return { + enabled: _config.enabled, + mode: _config.mode, + blacklist: Array.from(_config.blacklist), + whitelist: Array.from(_config.whitelist), + tempBans: Array.from(_config.tempBans.entries()).map(([ip, info]) => ({ + ip, + until: new Date(info.until).toISOString(), + reason: info.reason, + remainingMs: Math.max(0, info.until - Date.now()), + })), + }; +} + +/** + * Check if an IP is allowed + * @param {string} ip + * @returns {{ allowed: boolean, reason?: string }} + */ +export function checkIP(ip) { + if (!_config.enabled) return { allowed: true }; + if (!ip) return { allowed: true }; + + const normalizedIP = normalizeIP(ip); + + // Check temp bans first (highest priority) + const ban = _config.tempBans.get(normalizedIP); + if (ban) { + if (Date.now() < ban.until) { + return { allowed: false, reason: `Temporarily banned: ${ban.reason}` }; + } + _config.tempBans.delete(normalizedIP); // Expired + } + + switch (_config.mode) { + case "whitelist": + // Only whitelisted IPs allowed + if (!matchesAny(normalizedIP, _config.whitelist)) { + return { allowed: false, reason: "IP not in whitelist" }; + } + return { allowed: true }; + + case "whitelist-priority": + // Whitelist overrides blacklist + if (matchesAny(normalizedIP, _config.whitelist)) { + return { allowed: true }; + } + if (matchesAny(normalizedIP, _config.blacklist)) { + return { allowed: false, reason: "IP blacklisted" }; + } + return { allowed: true }; + + case "blacklist": + default: + // Blacklisted IPs blocked + if (matchesAny(normalizedIP, _config.blacklist)) { + return { allowed: false, reason: "IP blacklisted" }; + } + return { allowed: true }; + } +} + +/** + * Temporarily ban an IP + */ +export function tempBanIP(ip, durationMs, reason = "Automated ban") { + const normalizedIP = normalizeIP(ip); + _config.tempBans.set(normalizedIP, { + until: Date.now() + durationMs, + reason, + }); +} + +/** + * Remove a temporary ban + */ +export function removeTempBan(ip) { + _config.tempBans.delete(normalizeIP(ip)); +} + +/** + * Add IP to blacklist + */ +export function addToBlacklist(ip) { + _config.blacklist.add(normalizeIP(ip)); +} + +/** + * Remove IP from blacklist + */ +export function removeFromBlacklist(ip) { + _config.blacklist.delete(normalizeIP(ip)); +} + +/** + * Add IP to whitelist + */ +export function addToWhitelist(ip) { + _config.whitelist.add(normalizeIP(ip)); +} + +/** + * Remove IP from whitelist + */ +export function removeFromWhitelist(ip) { + _config.whitelist.delete(normalizeIP(ip)); +} + +/** + * Express/Next.js middleware factory + */ +export function createIPFilterMiddleware() { + return (req, res, next) => { + const ip = extractClientIP(req); + const { allowed, reason } = checkIP(ip); + if (!allowed) { + const statusCode = 403; + if (res.status) { + // Express + return res.status(statusCode).json({ error: reason || "Access denied" }); + } + // Raw response + res.writeHead(statusCode, { "Content-Type": "application/json" }); + return res.end(JSON.stringify({ error: reason || "Access denied" })); + } + if (next) next(); + }; +} + +/** + * For Next.js App Router — check IP from request object + */ +export function checkRequestIP(request) { + const ip = + request.headers?.get?.("x-forwarded-for")?.split(",")[0].trim() || + request.headers?.get?.("x-real-ip") || + request.headers?.get?.("cf-connecting-ip") || + request.ip || + "unknown"; + return checkIP(ip); +} + +// ─── Internal Helpers ─────────────────────────────────────────────────────── + +function normalizeIP(ip) { + if (!ip) return ""; + // Remove IPv6 prefix from IPv4-mapped addresses + return ip.replace(/^::ffff:/, "").trim(); +} + +function matchesAny(ip, ipSet) { + // Direct match + if (ipSet.has(ip)) return true; + + // CIDR match + for (const entry of ipSet) { + if (entry.includes("/") && matchesCIDR(ip, entry)) return true; + // Wildcard match (e.g., "192.168.*.*") + if (entry.includes("*") && matchesWildcard(ip, entry)) return true; + } + return false; +} + +function matchesCIDR(ip, cidr) { + try { + const [range, bits] = cidr.split("/"); + const mask = parseInt(bits, 10); + if (isNaN(mask) || mask < 0 || mask > 32) return false; + const ipNum = ipToNum(ip); + const rangeNum = ipToNum(range); + if (ipNum === null || rangeNum === null) return false; + const maskBits = (-1 << (32 - mask)) >>> 0; + return (ipNum & maskBits) === (rangeNum & maskBits); + } catch { + return false; + } +} + +function ipToNum(ip) { + const parts = ip.split("."); + if (parts.length !== 4) return null; + let num = 0; + for (const p of parts) { + const n = parseInt(p, 10); + if (isNaN(n) || n < 0 || n > 255) return null; + num = (num << 8) | n; + } + return num >>> 0; +} + +function matchesWildcard(ip, pattern) { + const ipParts = ip.split("."); + const patParts = pattern.split("."); + if (ipParts.length !== 4 || patParts.length !== 4) return false; + return ipParts.every((part, i) => patParts[i] === "*" || part === patParts[i]); +} + +function extractClientIP(req) { + return ( + req.headers?.["x-forwarded-for"]?.split(",")[0].trim() || + req.headers?.["x-real-ip"] || + req.headers?.["cf-connecting-ip"] || + req.socket?.remoteAddress || + req.ip || + "unknown" + ); +} + +/** + * Reset config (for testing) + */ +export function resetIPFilter() { + _config = { + enabled: false, + mode: "blacklist", + blacklist: new Set(), + whitelist: new Set(), + tempBans: new Map(), + }; +} diff --git a/open-sse/services/rateLimitManager.js b/open-sse/services/rateLimitManager.js index 16aac27ab6..14cc0804a0 100644 --- a/open-sse/services/rateLimitManager.js +++ b/open-sse/services/rateLimitManager.js @@ -8,6 +8,7 @@ */ import Bottleneck from "bottleneck"; +import { parseRetryAfterFromBody, lockModel } from "./accountFallback.js"; // Store limiters keyed by "provider:connectionId" (and optionally ":model") const limiters = new Map(); @@ -325,3 +326,39 @@ export function getAllRateLimitStatus() { } return result; } + +/** + * Update rate limiter based on API response body (JSON error responses). + * Providers embed retry info in JSON payloads in different formats. + * Should be called alongside updateFromHeaders for 4xx/5xx responses. + * + * @param {string} provider - Provider ID + * @param {string} connectionId - Connection ID + * @param {string|object} responseBody - Response body (string or parsed JSON) + * @param {number} status - HTTP status code + * @param {string} model - Model name (for per-model lockouts) + */ +export function updateFromResponseBody(provider, connectionId, responseBody, status, model = null) { + if (!enabledConnections.has(connectionId)) return; + + const { retryAfterMs, reason } = parseRetryAfterFromBody(responseBody); + + if (retryAfterMs && retryAfterMs > 0) { + const limiter = getLimiter(provider, connectionId, null); + console.log( + `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — body-parsed retry: ${Math.ceil(retryAfterMs / 1000)}s (${reason})` + ); + + limiter.updateSettings({ + reservoir: 0, + reservoirRefreshAmount: 60, + reservoirRefreshInterval: retryAfterMs, + }); + + // Also apply model-level lockout if model is known + if (model) { + lockModel(provider, connectionId, model, reason, retryAfterMs); + } + } +} + diff --git a/open-sse/services/sessionManager.js b/open-sse/services/sessionManager.js new file mode 100644 index 0000000000..11ffd11be3 --- /dev/null +++ b/open-sse/services/sessionManager.js @@ -0,0 +1,179 @@ +/** + * Session Fingerprinting — Phase 5 + * + * Generates stable session IDs for sticky routing, + * prompt caching, and per-session tracking. + */ + +import { createHash } from "node:crypto"; + +// In-memory session store with metadata +// key: sessionId → { createdAt, lastActive, requestCount, connectionId? } +const sessions = new Map(); + +// Auto-cleanup sessions older than 30 minutes +const SESSION_TTL_MS = 30 * 60 * 1000; +const _cleanupTimer = setInterval(() => { + const now = Date.now(); + for (const [key, entry] of sessions) { + if (now - entry.lastActive > SESSION_TTL_MS) sessions.delete(key); + } +}, 60_000); +_cleanupTimer.unref(); + +/** + * Generate a stable session fingerprint from request characteristics. + * Same client + same conversation → same session ID. + * + * Fingerprint factors: + * - System prompt hash (stable per conversation/tool) + * - First user message hash (stable per conversation) + * - Model name + * - Provider (optional) + * - Tools signature (sorted tool names) + * + * @param {object} body - Request body + * @param {object} [options] - Extra context + * @returns {string} Session ID (hex hash) + */ +export function generateSessionId(body, options = {}) { + const parts = []; + + // Model contributes to fingerprint + if (body.model) parts.push(`model:${body.model}`); + + // Provider binding + if (options.provider) parts.push(`provider:${options.provider}`); + + // System prompt hash (first 32 chars of system content) + const systemPrompt = extractSystemPrompt(body); + if (systemPrompt) { + parts.push(`sys:${hashShort(systemPrompt)}`); + } + + // First user message hash (identifies the conversation) + const firstUser = extractFirstUserMessage(body); + if (firstUser) { + parts.push(`user0:${hashShort(firstUser)}`); + } + + // Tools signature (sorted names) + if (body.tools && Array.isArray(body.tools) && body.tools.length > 0) { + const toolNames = body.tools + .map((t) => t.name || t.function?.name || "") + .filter(Boolean) + .sort() + .join(","); + if (toolNames) parts.push(`tools:${hashShort(toolNames)}`); + } + + // Connection ID for sticky routing + if (options.connectionId) parts.push(`conn:${options.connectionId}`); + + if (parts.length === 0) return null; + + const fingerprint = parts.join("|"); + return createHash("sha256").update(fingerprint).digest("hex").slice(0, 16); +} + +/** + * Touch or create a session + */ +export function touchSession(sessionId, connectionId = null) { + if (!sessionId) return; + const existing = sessions.get(sessionId); + if (existing) { + existing.lastActive = Date.now(); + existing.requestCount++; + if (connectionId) existing.connectionId = connectionId; + } else { + sessions.set(sessionId, { + createdAt: Date.now(), + lastActive: Date.now(), + requestCount: 1, + connectionId, + }); + } +} + +/** + * Get session info (for sticky routing decisions) + */ +export function getSessionInfo(sessionId) { + if (!sessionId) return null; + const entry = sessions.get(sessionId); + if (!entry) return null; + if (Date.now() - entry.lastActive > SESSION_TTL_MS) { + sessions.delete(sessionId); + return null; + } + return { ...entry }; +} + +/** + * Get the bound connection for a session (sticky routing) + */ +export function getSessionConnection(sessionId) { + const info = getSessionInfo(sessionId); + return info?.connectionId || null; +} + +/** + * Get session count (for dashboard) + */ +export function getActiveSessionCount() { + return sessions.size; +} + +/** + * Get all active sessions (for dashboard) + */ +export function getActiveSessions() { + const now = Date.now(); + const result = []; + for (const [id, entry] of sessions) { + if (now - entry.lastActive <= SESSION_TTL_MS) { + result.push({ sessionId: id, ...entry, ageMs: now - entry.createdAt }); + } + } + return result; +} + +/** + * Clear all sessions (for testing) + */ +export function clearSessions() { + sessions.clear(); +} + +// ─── Internal Helpers ─────────────────────────────────────────────────────── + +function hashShort(text) { + return createHash("sha256").update(text).digest("hex").slice(0, 8); +} + +function extractSystemPrompt(body) { + // Claude format: body.system + if (body.system) { + return typeof body.system === "string" ? body.system : JSON.stringify(body.system); + } + // OpenAI format: messages[0].role === "system" + if (body.messages && Array.isArray(body.messages)) { + const sys = body.messages.find((m) => m.role === "system" || m.role === "developer"); + if (sys) { + return typeof sys.content === "string" ? sys.content : JSON.stringify(sys.content); + } + } + return null; +} + +function extractFirstUserMessage(body) { + const messages = body.messages || body.input || []; + if (!Array.isArray(messages)) return null; + for (const msg of messages) { + if (msg.role === "user") { + return typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content); + } + } + return null; +} diff --git a/open-sse/services/signatureCache.js b/open-sse/services/signatureCache.js new file mode 100644 index 0000000000..82b810bb26 --- /dev/null +++ b/open-sse/services/signatureCache.js @@ -0,0 +1,171 @@ +/** + * Signature Cache — Phase 3 + * + * Dynamic 3-layer cache for thinking signatures (tool, model family, session). + * Replaces hardcoded thinking signature patterns with adaptive detection. + */ + +// 3-layer cache: tool → model family → session +// Each layer stores patterns detected from responses +const layers = { + tool: new Map(), // e.g. "cursor" → Set of signature patterns + family: new Map(), // e.g. "claude-sonnet" → Set of signature patterns + session: new Map(), // e.g. sessionId → Set of signature patterns +}; + +// Known default signatures (bootstrap — will be supplemented by learning) +const DEFAULT_SIGNATURES = [ + "", + "", + "", + "", + "", + "", +]; + +// Max entries per cache layer to prevent unbounded growth +const MAX_ENTRIES_PER_LAYER = 500; +const MAX_PATTERNS_PER_KEY = 50; + +/** + * Get all matching signatures for a given context. + * Checks all 3 layers and merges results with defaults. + * + * @param {object} context - { tool?, modelFamily?, sessionId? } + * @returns {string[]} Array of unique signature patterns + */ +export function getSignatures(context = {}) { + const patterns = new Set(DEFAULT_SIGNATURES); + + // Layer 1: Tool (e.g., "cursor", "cline", "antigravity") + if (context.tool && layers.tool.has(context.tool)) { + for (const p of layers.tool.get(context.tool)) patterns.add(p); + } + + // Layer 2: Model family (e.g., "claude-sonnet", "claude-opus") + if (context.modelFamily && layers.family.has(context.modelFamily)) { + for (const p of layers.family.get(context.modelFamily)) patterns.add(p); + } + + // Layer 3: Session-specific + if (context.sessionId && layers.session.has(context.sessionId)) { + for (const p of layers.session.get(context.sessionId)) patterns.add(p); + } + + return Array.from(patterns); +} + +/** + * Add a discovered signature pattern to the cache. + * + * @param {string} pattern - The signature pattern (e.g., "") + * @param {object} context - { tool?, modelFamily?, sessionId? } + */ +export function addSignature(pattern, context = {}) { + if (!pattern || typeof pattern !== "string") return; + + const addToLayer = (layer, key) => { + if (!key) return; + if (!layer.has(key)) { + if (layer.size >= MAX_ENTRIES_PER_LAYER) { + // Evict oldest entry + const firstKey = layer.keys().next().value; + layer.delete(firstKey); + } + layer.set(key, new Set()); + } + const set = layer.get(key); + if (set.size < MAX_PATTERNS_PER_KEY) { + set.add(pattern); + } + }; + + addToLayer(layers.tool, context.tool); + addToLayer(layers.family, context.modelFamily); + addToLayer(layers.session, context.sessionId); +} + +/** + * Detect signatures in a text chunk from streaming response. + * Auto-learns new patterns and adds them to cache. + * + * @param {string} text - Streaming text to scan + * @param {object} context - { tool?, modelFamily?, sessionId? } + * @returns {{ found: string[], cleaned: string }} Detected tags and cleaned text + */ +export function detectAndLearn(text, context = {}) { + if (!text || typeof text !== "string") return { found: [], cleaned: text }; + + const found = []; + let cleaned = text; + + // Check all known signatures + const known = getSignatures(context); + for (const sig of known) { + if (cleaned.includes(sig)) { + found.push(sig); + cleaned = cleaned.split(sig).join(""); + } + } + + // Auto-detect new XML-like thinking tags + const tagRegex = /<\/?([a-zA-Z_][a-zA-Z0-9_]*(?:Thinking|thinking|thought|Thought|internal_thought))>/g; + let match; + while ((match = tagRegex.exec(text)) !== null) { + const tag = match[0]; + if (!known.includes(tag)) { + found.push(tag); + addSignature(tag, context); + cleaned = cleaned.split(tag).join(""); + } + } + + return { found, cleaned: cleaned.trim() || cleaned }; +} + +/** + * Extract model family from model name + * "claude-sonnet-4-20250514" → "claude-sonnet" + * "gpt-4o-2024-08-06" → "gpt-4o" + */ +export function getModelFamily(model) { + if (!model) return null; + // Remove date suffixes and version numbers + const cleaned = model + .replace(/-\d{4}-\d{2}-\d{2}$/, "") // Remove YYYY-MM-DD suffix + .replace(/-\d{8,}$/, "") // Remove YYYYMMDD suffix + .replace(/-\d+(\.\d+)*$/, "") // Remove version suffix like -4 + .replace(/@.*$/, ""); // Remove @latest etc. + // Keep meaningful prefix + return cleaned || model; +} + +/** + * Get cache stats (for dashboard) + */ +export function getCacheStats() { + return { + tool: { + entries: layers.tool.size, + patterns: Array.from(layers.tool.values()).reduce((sum, s) => sum + s.size, 0), + }, + family: { + entries: layers.family.size, + patterns: Array.from(layers.family.values()).reduce((sum, s) => sum + s.size, 0), + }, + session: { + entries: layers.session.size, + patterns: Array.from(layers.session.values()).reduce((sum, s) => sum + s.size, 0), + }, + defaultCount: DEFAULT_SIGNATURES.length, + }; +} + +/** + * Clear all cache layers (for testing) + */ +export function clearCache() { + layers.tool.clear(); + layers.family.clear(); + layers.session.clear(); +} diff --git a/open-sse/services/systemPrompt.js b/open-sse/services/systemPrompt.js new file mode 100644 index 0000000000..f584a011aa --- /dev/null +++ b/open-sse/services/systemPrompt.js @@ -0,0 +1,66 @@ +/** + * System Prompt Injection — Phase 10 + * + * Injects a global system prompt into all requests at proxy level. + */ + +// In-memory config +let _config = { + enabled: false, + prompt: "", +}; + +/** + * Set system prompt config + */ +export function setSystemPromptConfig(config) { + _config = { ..._config, ...config }; +} + +/** + * Get system prompt config + */ +export function getSystemPromptConfig() { + return { ..._config }; +} + +/** + * Inject system prompt into request body. + * + * @param {object} body - Request body + * @param {string} [promptText] - Override prompt text + * @returns {object} Modified body + */ +export function injectSystemPrompt(body, promptText = null) { + const text = promptText || _config.prompt; + if (!text || !_config.enabled) return body; + if (!body || typeof body !== "object") return body; + if (body._skipSystemPrompt) return body; + + const result = { ...body }; + + // OpenAI/Claude format (messages[]) + if (result.messages && Array.isArray(result.messages)) { + const sysIdx = result.messages.findIndex((m) => m.role === "system" || m.role === "developer"); + result.messages = [...result.messages]; + if (sysIdx >= 0) { + // Prepend to existing system message + const msg = { ...result.messages[sysIdx] }; + msg.content = text + "\n\n" + (msg.content || ""); + result.messages[sysIdx] = msg; + } else { + result.messages = [{ role: "system", content: text }, ...result.messages]; + } + } + + // Claude format (system field) + if (result.system !== undefined) { + if (typeof result.system === "string") { + result.system = text + "\n\n" + result.system; + } else if (Array.isArray(result.system)) { + result.system = [{ type: "text", text }, ...result.system]; + } + } + + return result; +} diff --git a/open-sse/services/thinkingBudget.js b/open-sse/services/thinkingBudget.js new file mode 100644 index 0000000000..6ea4656a77 --- /dev/null +++ b/open-sse/services/thinkingBudget.js @@ -0,0 +1,186 @@ +/** + * Thinking Budget Control — Phase 2 + * + * Provides proxy-level control over AI thinking/reasoning budgets. + * Modes: auto, passthrough, custom, adaptive + */ + +// Thinking budget modes +export const ThinkingMode = { + AUTO: "auto", // Let provider decide (remove client's budget) + PASSTHROUGH: "passthrough", // No changes (current behavior) + CUSTOM: "custom", // Set fixed budget + ADAPTIVE: "adaptive", // Scale based on request complexity +}; + +// Effort → budget token mapping +export const EFFORT_BUDGETS = { + none: 0, + low: 1024, + medium: 10240, + high: 131072, +}; + +// Default config (passthrough = backward compatible) +export const DEFAULT_THINKING_CONFIG = { + mode: ThinkingMode.PASSTHROUGH, + customBudget: 10240, + effortLevel: "medium", +}; + +// In-memory config (loaded from DB on startup, or default) +let _config = { ...DEFAULT_THINKING_CONFIG }; + +/** + * Set the thinking budget config (called from settings API or startup) + */ +export function setThinkingBudgetConfig(config) { + _config = { ...DEFAULT_THINKING_CONFIG, ...config }; +} + +/** + * Get current thinking budget config + */ +export function getThinkingBudgetConfig() { + return { ..._config }; +} + +/** + * Apply thinking budget control to a request body. + * Called before format-specific translation. + * + * @param {object} body - Request body (any format) + * @param {object} [config] - Override config (defaults to stored config) + * @returns {object} Modified body + */ +export function applyThinkingBudget(body, config = null) { + const cfg = config || _config; + if (!body || typeof body !== "object") return body; + + switch (cfg.mode) { + case ThinkingMode.AUTO: + return stripThinkingConfig(body); + + case ThinkingMode.PASSTHROUGH: + return body; // No changes + + case ThinkingMode.CUSTOM: + return setCustomBudget(body, cfg.customBudget); + + case ThinkingMode.ADAPTIVE: + return applyAdaptiveBudget(body, cfg); + + default: + return body; + } +} + +/** + * AUTO mode: strip all thinking configuration, let provider decide + */ +function stripThinkingConfig(body) { + const result = { ...body }; + + // Claude format + delete result.thinking; + + // OpenAI format + delete result.reasoning_effort; + delete result.reasoning; + + // Gemini format + if (result.generationConfig) { + result.generationConfig = { ...result.generationConfig }; + delete result.generationConfig.thinking_config; + delete result.generationConfig.thinkingConfig; + } + + return result; +} + +/** + * CUSTOM mode: set exact budget tokens + */ +function setCustomBudget(body, budget) { + const result = { ...body }; + + // If body already has thinking config in Claude format, update it + if (result.thinking || hasThinkingCapableModel(result)) { + result.thinking = { + type: budget > 0 ? "enabled" : "disabled", + budget_tokens: budget, + }; + } + + // OpenAI reasoning_effort mapping + if (result.reasoning_effort !== undefined || result.reasoning !== undefined) { + if (budget <= 0) { + delete result.reasoning_effort; + delete result.reasoning; + } else if (budget <= 1024) { + result.reasoning_effort = "low"; + } else if (budget <= 10240) { + result.reasoning_effort = "medium"; + } else { + result.reasoning_effort = "high"; + } + } + + // Gemini thinking_config + if (result.generationConfig?.thinking_config || result.generationConfig?.thinkingConfig) { + result.generationConfig = { + ...result.generationConfig, + thinking_config: { thinking_budget: budget }, + }; + } + + return result; +} + +/** + * ADAPTIVE mode: scale budget based on request complexity + */ +function applyAdaptiveBudget(body, cfg) { + const messages = body.messages || body.input || []; + const messageCount = messages.length; + const tools = body.tools || []; + const toolCount = tools.length; + + // Get last user message length + let lastMsgLength = 0; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "user") { + lastMsgLength = typeof msg.content === "string" + ? msg.content.length + : JSON.stringify(msg.content || "").length; + break; + } + } + + // Calculate multiplier + let multiplier = 1.0; + if (messageCount > 10) multiplier += 0.5; + if (toolCount > 3) multiplier += 0.5; + if (lastMsgLength > 2000) multiplier += 0.3; + + const baseBudget = EFFORT_BUDGETS[cfg.effortLevel] || EFFORT_BUDGETS.medium; + const budget = Math.min(Math.ceil(baseBudget * multiplier), 131072); + + return setCustomBudget(body, budget); +} + +/** + * Check if model name suggests thinking capability + */ +function hasThinkingCapableModel(body) { + const model = body.model || ""; + return ( + model.includes("claude") || + model.includes("o1") || + model.includes("o3") || + model.includes("o4") || + model.includes("gemini") || + model.includes("thinking") + ); +} diff --git a/open-sse/services/wildcardRouter.js b/open-sse/services/wildcardRouter.js new file mode 100644 index 0000000000..9b09860a72 --- /dev/null +++ b/open-sse/services/wildcardRouter.js @@ -0,0 +1,117 @@ +/** + * Wildcard Model Routing — Phase 8 + * + * Glob-style wildcard matching for model aliases with specificity ranking. + * Integrates into the existing model resolution pipeline. + */ + +/** + * Match a model name against a pattern with glob wildcards. + * Supports * (any sequence) and ? (single char). + * + * @param {string} model - Model name to match + * @param {string} pattern - Pattern with wildcards + * @returns {boolean} + */ +export function wildcardMatch(model, pattern) { + if (!model || !pattern) return false; + if (pattern === "*") return true; + if (pattern === model) return true; + + // Convert glob pattern to regex + const regexStr = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*") + .replace(/\?/g, "."); + const regex = new RegExp(`^${regexStr}$`, "i"); + return regex.test(model); +} + +/** + * Calculate specificity score for a pattern. + * Higher score = more specific match. + * Used to rank multiple matching patterns. + * + * @param {string} pattern + * @returns {number} Specificity score (higher = more specific) + */ +export function getSpecificity(pattern) { + if (!pattern) return 0; + + let score = 0; + // Exact segments (no wildcards) contribute most + const segments = pattern.split(/[/*?]/); + for (const seg of segments) { + if (seg.length > 0) score += seg.length * 10; + } + // Wildcards reduce specificity + const wildcardCount = (pattern.match(/\*/g) || []).length; + const questionCount = (pattern.match(/\?/g) || []).length; + score -= wildcardCount * 50; + score -= questionCount * 5; + // Longer patterns tend to be more specific + score += pattern.length; + + return score; +} + +/** + * Find the best matching alias for a model name from a list of alias entries. + * Returns the most specific match. + * + * @param {string} model - Model name to resolve + * @param {Array<{ pattern: string, target: string, [key: string]: any }>} aliases - Alias entries + * @returns {{ pattern: string, target: string, specificity: number } | null} + */ +export function resolveWildcardAlias(model, aliases) { + if (!model || !aliases || !Array.isArray(aliases)) return null; + + const matches = []; + for (const alias of aliases) { + const pattern = alias.pattern || alias.alias || alias.from; + const target = alias.target || alias.model || alias.to; + if (!pattern || !target) continue; + + if (wildcardMatch(model, pattern)) { + matches.push({ + pattern, + target, + specificity: getSpecificity(pattern), + ...alias, + }); + } + } + + if (matches.length === 0) return null; + + // Sort by specificity (highest first) + matches.sort((a, b) => b.specificity - a.specificity); + return matches[0]; +} + +/** + * Resolve model through wildcard aliases, falling back to exact match. + * Can be integrated into existing model resolution. + * + * @param {string} model - Requested model name + * @param {Map|Object} exactAliases - Exact model alias map + * @param {Array} wildcardAliases - Wildcard pattern aliases + * @returns {string} Resolved model name + */ +export function resolveModel(model, exactAliases = {}, wildcardAliases = []) { + if (!model) return model; + + // 1. Check exact aliases first (fastest) + if (exactAliases instanceof Map) { + if (exactAliases.has(model)) return exactAliases.get(model); + } else if (exactAliases[model]) { + return exactAliases[model]; + } + + // 2. Check wildcard aliases + const match = resolveWildcardAlias(model, wildcardAliases); + if (match) return match.target; + + // 3. Return original + return model; +} diff --git a/open-sse/translator/index.js b/open-sse/translator/index.js index 6aa4bcd56a..3215da0ce9 100644 --- a/open-sse/translator/index.js +++ b/open-sse/translator/index.js @@ -3,6 +3,7 @@ import { ensureToolCallIds, fixMissingToolResponses } from "./helpers/toolCallHe import { prepareClaudeRequest } from "./helpers/claudeHelper.js"; import { filterToOpenAIFormat } from "./helpers/openaiHelper.js"; import { normalizeThinkingConfig } from "../services/provider.js"; +import { applyThinkingBudget } from "../services/thinkingBudget.js"; // Registry for translators. // NOTE: translator modules import this file and call register() at module-load time. @@ -119,6 +120,9 @@ export function translateRequest( ) { let result = body; + // Phase 2: Apply thinking budget control before normalization + result = applyThinkingBudget(result); + // Normalize thinking config: remove if lastMessage is not user normalizeThinkingConfig(result); diff --git a/package.json b/package.json index 5e94eacdca..880f55de8a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "0.1.0", + "version": "0.2.0", "description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": { diff --git a/src/app/(dashboard)/dashboard/settings/components/IPFilterSection.js b/src/app/(dashboard)/dashboard/settings/components/IPFilterSection.js new file mode 100644 index 0000000000..245a833cfe --- /dev/null +++ b/src/app/(dashboard)/dashboard/settings/components/IPFilterSection.js @@ -0,0 +1,217 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Card, Button, Input, Toggle } from "@/shared/components"; + +const MODES = [ + { value: "disabled", label: "Disabled", icon: "block" }, + { value: "blacklist", label: "Blacklist", icon: "do_not_disturb" }, + { value: "whitelist", label: "Whitelist", icon: "verified_user" }, + { value: "whitelist-priority", label: "WL Priority", icon: "priority_high" }, +]; + +export default function IPFilterSection() { + const [config, setConfig] = useState({ enabled: false, mode: "blacklist", blacklist: [], whitelist: [], tempBans: [] }); + const [loading, setLoading] = useState(true); + const [newIP, setNewIP] = useState(""); + const [listTarget, setListTarget] = useState("blacklist"); // "blacklist" | "whitelist" + + useEffect(() => { + loadConfig(); + }, []); + + const loadConfig = async () => { + try { + const res = await fetch("/api/settings/ip-filter"); + if (res.ok) setConfig(await res.json()); + } catch {} finally { + setLoading(false); + } + }; + + const updateConfig = async (updates) => { + try { + const res = await fetch("/api/settings/ip-filter", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(updates), + }); + if (res.ok) setConfig(await res.json()); + } catch {} + }; + + const toggleEnabled = () => updateConfig({ enabled: !config.enabled }); + + const setMode = (mode) => { + if (mode === "disabled") { + updateConfig({ enabled: false }); + } else { + updateConfig({ enabled: true, mode }); + } + }; + + const addIP = () => { + if (!newIP.trim()) return; + const key = listTarget === "blacklist" ? "addBlacklist" : "addWhitelist"; + updateConfig({ [key]: newIP.trim() }); + setNewIP(""); + }; + + const removeIP = (ip, list) => { + const key = list === "blacklist" ? "removeBlacklist" : "removeWhitelist"; + updateConfig({ [key]: ip }); + }; + + const removeBan = (ip) => updateConfig({ removeBan: ip }); + + const activeMode = !config.enabled ? "disabled" : config.mode; + + return ( + +
+
+ +
+
+

IP Access Control

+

Block or allow specific IP addresses

+
+
+ + {/* Mode selector */} +
+ {MODES.map((m) => ( + + ))} +
+ + {config.enabled && ( +
+ {/* Add IP */} +
+
+ setNewIP(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && addIP()} + /> +
+
+ + +
+
+ + {/* Blacklist */} + {config.blacklist.length > 0 && ( +
+

+ Blocked ({config.blacklist.length}) +

+
+ {config.blacklist.map((ip) => ( + + {ip} + + + ))} +
+
+ )} + + {/* Whitelist */} + {config.whitelist.length > 0 && ( +
+

+ Allowed ({config.whitelist.length}) +

+
+ {config.whitelist.map((ip) => ( + + {ip} + + + ))} +
+
+ )} + + {/* Temp Bans */} + {config.tempBans.length > 0 && ( +
+

+ Temporary Bans ({config.tempBans.length}) +

+
+ {config.tempBans.map((ban) => ( +
+
+ {ban.ip} + — {ban.reason} +
+
+ + {Math.ceil(ban.remainingMs / 60000)}m left + + +
+
+ ))} +
+
+ )} +
+ )} + + ); +} diff --git a/src/app/(dashboard)/dashboard/settings/components/PricingTab.js b/src/app/(dashboard)/dashboard/settings/components/PricingTab.js new file mode 100644 index 0000000000..8aea5d6afa --- /dev/null +++ b/src/app/(dashboard)/dashboard/settings/components/PricingTab.js @@ -0,0 +1,530 @@ +"use client"; + +import { useState, useEffect, useCallback, useMemo } from "react"; +import { Card } from "@/shared/components"; + +const PRICING_FIELDS = ["input", "output", "cached", "reasoning", "cache_creation"]; +const FIELD_LABELS = { + input: "Input", + output: "Output", + cached: "Cached", + reasoning: "Reasoning", + cache_creation: "Cache Write", +}; + +export default function PricingTab() { + const [catalog, setCatalog] = useState({}); + const [pricingData, setPricingData] = useState({}); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [saveStatus, setSaveStatus] = useState(""); + const [selectedProvider, setSelectedProvider] = useState(null); + const [expandedProviders, setExpandedProviders] = useState(new Set()); + const [searchQuery, setSearchQuery] = useState(""); + const [editedProviders, setEditedProviders] = useState(new Set()); + + // Load catalog + pricing + useEffect(() => { + loadData(); + }, []); + + const loadData = async () => { + setLoading(true); + try { + const [catalogRes, pricingRes] = await Promise.all([ + fetch("/api/pricing/models"), + fetch("/api/pricing"), + ]); + if (catalogRes.ok) setCatalog(await catalogRes.json()); + if (pricingRes.ok) setPricingData(await pricingRes.json()); + } catch (error) { + console.error("Failed to load pricing data:", error); + } finally { + setLoading(false); + } + }; + + // All providers sorted by model count (desc) + const allProviders = useMemo(() => { + const providers = Object.entries(catalog) + .map(([alias, info]) => ({ + alias, + ...info, + pricedModels: pricingData[alias] + ? Object.keys(pricingData[alias]).length + : 0, + })) + .sort((a, b) => b.modelCount - a.modelCount); + return providers; + }, [catalog, pricingData]); + + // Filter providers by search + const filteredProviders = useMemo(() => { + if (!searchQuery.trim()) return allProviders; + const q = searchQuery.toLowerCase(); + return allProviders.filter( + (p) => + p.alias.toLowerCase().includes(q) || + p.id.toLowerCase().includes(q) || + p.models.some( + (m) => + m.id.toLowerCase().includes(q) || + m.name.toLowerCase().includes(q) + ) + ); + }, [allProviders, searchQuery]); + + // Stats + const stats = useMemo(() => { + const totalModels = allProviders.reduce((s, p) => s + p.modelCount, 0); + const pricedCount = Object.values(pricingData).reduce( + (s, models) => s + Object.keys(models).length, + 0 + ); + return { + providers: allProviders.length, + totalModels, + pricedCount, + }; + }, [allProviders, pricingData]); + + const toggleProvider = useCallback((alias) => { + setExpandedProviders((prev) => { + const next = new Set(prev); + if (next.has(alias)) next.delete(alias); + else next.add(alias); + return next; + }); + }, []); + + const handlePricingChange = useCallback( + (provider, model, field, value) => { + const numValue = parseFloat(value); + if (isNaN(numValue) || numValue < 0) return; + + setPricingData((prev) => { + const next = { ...prev }; + if (!next[provider]) next[provider] = {}; + if (!next[provider][model]) + next[provider][model] = { input: 0, output: 0, cached: 0, reasoning: 0, cache_creation: 0 }; + next[provider][model] = { ...next[provider][model], [field]: numValue }; + return next; + }); + setEditedProviders((prev) => new Set(prev).add(provider)); + }, + [] + ); + + const saveProvider = useCallback( + async (providerAlias) => { + setSaving(true); + setSaveStatus(""); + try { + const providerPricing = pricingData[providerAlias] || {}; + const response = await fetch("/api/pricing", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ [providerAlias]: providerPricing }), + }); + + if (response.ok) { + setSaveStatus(`✅ ${providerAlias.toUpperCase()} saved`); + setEditedProviders((prev) => { + const next = new Set(prev); + next.delete(providerAlias); + return next; + }); + setTimeout(() => setSaveStatus(""), 3000); + } else { + const err = await response.json(); + setSaveStatus(`❌ Error: ${err.error}`); + } + } catch (error) { + setSaveStatus(`❌ Save failed: ${error.message}`); + } finally { + setSaving(false); + } + }, + [pricingData] + ); + + const resetProvider = useCallback( + async (providerAlias) => { + if ( + !confirm( + `Reset all pricing for ${providerAlias.toUpperCase()} to defaults?` + ) + ) + return; + try { + const response = await fetch( + `/api/pricing?provider=${providerAlias}`, + { method: "DELETE" } + ); + if (response.ok) { + const updated = await response.json(); + setPricingData(updated); + setSaveStatus(`🔄 ${providerAlias.toUpperCase()} reset to defaults`); + setEditedProviders((prev) => { + const next = new Set(prev); + next.delete(providerAlias); + return next; + }); + setTimeout(() => setSaveStatus(""), 3000); + } + } catch (error) { + setSaveStatus(`❌ Reset failed: ${error.message}`); + } + }, + [] + ); + + const selectProviderFilter = useCallback((alias) => { + setSelectedProvider((prev) => (prev === alias ? null : alias)); + }, []); + + // Which providers to display in the main area + const displayProviders = useMemo(() => { + if (selectedProvider) { + return filteredProviders.filter((p) => p.alias === selectedProvider); + } + return filteredProviders; + }, [filteredProviders, selectedProvider]); + + if (loading) { + return ( +
+
+ Loading pricing data... +
+
+ ); + } + + return ( +
+ {/* Header + Stats */} +
+
+

Model Pricing

+

+ Configure cost rates per model • All rates in{" "} + $/1M tokens +

+
+
+
+
+ Providers +
+
{stats.providers}
+
+
+
+ Registry +
+
{stats.totalModels}
+
+
+
Priced
+
+ {stats.pricedCount} +
+
+
+
+ + {/* Save Status */} + {saveStatus && ( +
+ {saveStatus} +
+ )} + + {/* Search + Provider Filter */} +
+
+ + search + + setSearchQuery(e.target.value)} + className="w-full pl-10 pr-3 py-2 bg-bg-base border border-border rounded-lg focus:outline-none focus:border-primary text-sm" + /> +
+ {selectedProvider && ( + + )} +
+ + {/* Provider Pills (quick filter) */} +
+ {allProviders.map((p) => ( + + ))} +
+ + {/* Provider Sections */} +
+ {displayProviders.map((provider) => ( + toggleProvider(provider.alias)} + onPricingChange={(model, field, value) => + handlePricingChange(provider.alias, model, field, value) + } + onSave={() => saveProvider(provider.alias)} + onReset={() => resetProvider(provider.alias)} + saving={saving} + /> + ))} + + {displayProviders.length === 0 && ( +
+ No providers match your search. +
+ )} +
+ + {/* Info Box */} + +

+ + info + + How Pricing Works +

+
+

+ Input: tokens sent to the model •{" "} + Output: tokens generated •{" "} + Cached: reused input (~50% of input rate) •{" "} + Reasoning: thinking tokens (falls back to Output) •{" "} + Cache Write: creating cache entries (falls back to + Input) +

+

+ Cost = (input × input_rate) + (output × output_rate) + (cached × + cached_rate) per million tokens. +

+
+ +
+ ); +} + +// ── Provider Section (collapsible) ────────────────────────────────────── + +function ProviderSection({ + provider, + pricingData, + isExpanded, + isEdited, + onToggle, + onPricingChange, + onSave, + onReset, + saving, +}) { + const pricedCount = Object.keys(pricingData).length; + const authBadge = + provider.authType === "oauth" + ? "OAuth" + : provider.authType === "apikey" + ? "API Key" + : provider.authType; + + return ( +
+ {/* Header (click to expand) */} + + + {/* Expanded: models table */} + {isExpanded && ( +
+ {/* Actions bar */} +
+ + {provider.modelCount} models •{" "} + {pricedCount} with pricing configured + +
+ + +
+
+ + {/* Table */} +
+ + + + + {PRICING_FIELDS.map((field) => ( + + ))} + + + + {provider.models.map((model) => ( + + onPricingChange(model.id, field, value) + } + /> + ))} + +
Model + {FIELD_LABELS[field]} +
+
+
+ )} +
+ ); +} + +// ── Model Row ──────────────────────────────────────────────────────────── + +function ModelRow({ model, pricing, onPricingChange }) { + const hasPricing = pricing && Object.values(pricing).some((v) => v > 0); + + return ( + + +
+ + {model.name} + {model.custom && ( + + custom + + )} + + {model.id} + +
+ + {PRICING_FIELDS.map((field) => ( + + onPricingChange(field, e.target.value)} + className="w-full px-2 py-1 text-right text-xs bg-transparent border border-transparent hover:border-border focus:border-primary focus:bg-bg-base rounded transition-colors outline-none tabular-nums" + /> + + ))} + + ); +} diff --git a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js index 00a626985f..dfc74a450c 100644 --- a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js +++ b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js @@ -1,105 +1,190 @@ "use client"; import { useState, useEffect } from "react"; -import { Card, Input, Toggle } from "@/shared/components"; +import { Card, Input, Toggle, Button } from "@/shared/components"; + +const STRATEGIES = [ + { value: "fill-first", label: "Fill First", desc: "Use accounts in priority order", icon: "vertical_align_top" }, + { value: "round-robin", label: "Round Robin", desc: "Cycle through all accounts", icon: "loop" }, + { value: "p2c", label: "P2C", desc: "Pick 2 random, use the healthier one", icon: "balance" }, +]; export default function RoutingTab() { const [settings, setSettings] = useState({ fallbackStrategy: "fill-first" }); const [loading, setLoading] = useState(true); + const [aliases, setAliases] = useState([]); + const [newPattern, setNewPattern] = useState(""); + const [newTarget, setNewTarget] = useState(""); useEffect(() => { fetch("/api/settings") .then((res) => res.json()) .then((data) => { setSettings(data); + setAliases(data.wildcardAliases || []); setLoading(false); }) .catch(() => setLoading(false)); }, []); - const updateFallbackStrategy = async (strategy) => { + const updateSetting = async (patch) => { try { const res = await fetch("/api/settings", { method: "PATCH", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ fallbackStrategy: strategy }), + body: JSON.stringify(patch), }); if (res.ok) { - setSettings((prev) => ({ ...prev, fallbackStrategy: strategy })); + setSettings((prev) => ({ ...prev, ...patch })); } } catch (err) { console.error("Failed to update settings:", err); } }; - const updateStickyLimit = async (limit) => { - const numLimit = parseInt(limit); - if (isNaN(numLimit) || numLimit < 1) return; - try { - const res = await fetch("/api/settings", { - method: "PATCH", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ stickyRoundRobinLimit: numLimit }), - }); - if (res.ok) { - setSettings((prev) => ({ ...prev, stickyRoundRobinLimit: numLimit })); - } - } catch (err) { - console.error("Failed to update sticky limit:", err); - } + const addAlias = async () => { + if (!newPattern.trim() || !newTarget.trim()) return; + const updated = [...aliases, { pattern: newPattern.trim(), target: newTarget.trim() }]; + await updateSetting({ wildcardAliases: updated }); + setAliases(updated); + setNewPattern(""); + setNewTarget(""); + }; + + const removeAlias = async (idx) => { + const updated = aliases.filter((_, i) => i !== idx); + await updateSetting({ wildcardAliases: updated }); + setAliases(updated); }; return ( - -
-
- -
-

Routing Strategy

-
-
-
-
-

Round Robin

-

Cycle through accounts to distribute load

+
+ {/* Strategy Selection */} + +
+
+
- - updateFallbackStrategy( - settings.fallbackStrategy === "round-robin" ? "fill-first" : "round-robin" - ) - } - disabled={loading} - /> +

Routing Strategy

+
+ +
+ {STRATEGIES.map((s) => ( + + ))}
{settings.fallbackStrategy === "round-robin" && ( -
+
-

Sticky Limit

-

Calls per account before switching

+

Sticky Limit

+

Calls per account before switching

updateStickyLimit(e.target.value)} + onChange={(e) => updateSetting({ stickyRoundRobinLimit: parseInt(e.target.value) })} disabled={loading} className="w-20 text-center" />
)} -

- {settings.fallbackStrategy === "round-robin" - ? `Currently distributing requests across all available accounts with ${settings.stickyRoundRobinLimit || 3} calls per account.` - : "Currently using accounts in priority order (Fill First)."} +

+ {settings.fallbackStrategy === "round-robin" && + `Distributing requests across accounts with ${settings.stickyRoundRobinLimit || 3} calls per account.`} + {settings.fallbackStrategy === "fill-first" && "Using accounts in priority order (Fill First)."} + {settings.fallbackStrategy === "p2c" && + "Power of Two Choices: picks 2 random accounts and routes to the healthier one."}

-
- + + + {/* Wildcard Aliases */} + +
+
+ +
+
+

Model Aliases

+

Wildcard patterns to remap model names • Use * and ?

+
+
+ + {aliases.length > 0 && ( +
+ {aliases.map((a, i) => ( +
+
+ {a.pattern} + arrow_forward + {a.target} +
+ +
+ ))} +
+ )} + +
+
+ setNewPattern(e.target.value)} + /> +
+
+ setNewTarget(e.target.value)} + /> +
+ +
+ +
); } diff --git a/src/app/(dashboard)/dashboard/settings/components/SecurityTab.js b/src/app/(dashboard)/dashboard/settings/components/SecurityTab.js index 1102ad0e86..112fb57ff6 100644 --- a/src/app/(dashboard)/dashboard/settings/components/SecurityTab.js +++ b/src/app/(dashboard)/dashboard/settings/components/SecurityTab.js @@ -2,6 +2,7 @@ import { useState, useEffect } from "react"; import { Card, Button, Input, Toggle } from "@/shared/components"; +import IPFilterSection from "./IPFilterSection"; export default function SecurityTab() { const [settings, setSettings] = useState({ requireLogin: false, hasPassword: false }); @@ -69,79 +70,82 @@ export default function SecurityTab() { }; return ( - -
-
- -
-

Security

-
-
-
-
-

Require login

-

- When ON, dashboard requires password. When OFF, access without login. -

+
+ +
+
+
- updateRequireLogin(!settings.requireLogin)} - disabled={loading} - /> +

Security

- {settings.requireLogin === true && ( -
- {settings.hasPassword && ( - setPasswords({ ...passwords, current: e.target.value })} - required - /> - )} -
- setPasswords({ ...passwords, new: e.target.value })} - required - /> - setPasswords({ ...passwords, confirm: e.target.value })} - required - /> -
- - {passStatus.message && ( -

- {passStatus.message} +

+
+
+

Require login

+

+ When ON, dashboard requires password. When OFF, access without login.

- )} - -
-
- - )} -
- + updateRequireLogin(!settings.requireLogin)} + disabled={loading} + /> +
+ {settings.requireLogin === true && ( +
+ {settings.hasPassword && ( + setPasswords({ ...passwords, current: e.target.value })} + required + /> + )} +
+ setPasswords({ ...passwords, new: e.target.value })} + required + /> + setPasswords({ ...passwords, confirm: e.target.value })} + required + /> +
+ + {passStatus.message && ( +

+ {passStatus.message} +

+ )} + +
+ +
+
+ )} +
+ + +
); } diff --git a/src/app/(dashboard)/dashboard/settings/components/SystemPromptTab.js b/src/app/(dashboard)/dashboard/settings/components/SystemPromptTab.js new file mode 100644 index 0000000000..0327490276 --- /dev/null +++ b/src/app/(dashboard)/dashboard/settings/components/SystemPromptTab.js @@ -0,0 +1,104 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Card, Toggle } from "@/shared/components"; + +export default function SystemPromptTab() { + const [config, setConfig] = useState({ enabled: false, prompt: "" }); + const [loading, setLoading] = useState(true); + const [status, setStatus] = useState(""); + const [debounceTimer, setDebounceTimer] = useState(null); + + useEffect(() => { + fetch("/api/settings/system-prompt") + .then((res) => res.json()) + .then((data) => { + setConfig(data); + setLoading(false); + }) + .catch(() => setLoading(false)); + }, []); + + const save = async (updates) => { + const newConfig = { ...config, ...updates }; + setConfig(newConfig); + setStatus(""); + try { + const res = await fetch("/api/settings/system-prompt", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(newConfig), + }); + if (res.ok) { + setStatus("saved"); + setTimeout(() => setStatus(""), 2000); + } + } catch { + setStatus("error"); + } + }; + + const handlePromptChange = (text) => { + setConfig((prev) => ({ ...prev, prompt: text })); + if (debounceTimer) clearTimeout(debounceTimer); + setDebounceTimer( + setTimeout(() => { + save({ prompt: text }); + }, 800) + ); + }; + + return ( + +
+
+ +
+
+

Global System Prompt

+

Injected into all requests at proxy level

+
+
+ {status === "saved" && ( + + check_circle Saved + + )} + save({ enabled: !config.enabled })} + disabled={loading} + /> +
+
+ + {config.enabled && ( +
+
+