diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4b57c85d7..e0d47c71c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [18, 22] + node-version: [20, 22] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -59,7 +59,7 @@ jobs: needs: build strategy: matrix: - node-version: [18, 22] + node-version: [20, 22] env: JWT_SECRET: ci-test-secret-with-sufficient-length-for-validation API_KEY_SECRET: ci-test-api-key-secret-long diff --git a/README.md b/README.md index c950b7311a..3ca6b718a8 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,72 @@ PORT=20128 NEXT_PUBLIC_BASE_URL=http://localhost:20128 npm run dev | ๐Ÿง  **Semantic Cache** | Two-tier cache reduces cost & latency | | โšก **Request Idempotency** | 5s dedup window for duplicate requests | | ๐Ÿ“ˆ **Progress Tracking** | Opt-in SSE progress events for streaming | +| ๐Ÿงช **LLM Evaluations** | Golden set testing with 4 match strategies | + +--- + +## ๐Ÿงช Evaluations (Evals) + +OmniRoute includes a built-in evaluation framework to test LLM response quality against a golden set. Access it via **Analytics โ†’ Evals** in the dashboard. + +### Built-in Golden Set + +The pre-loaded "OmniRoute Golden Set" contains 10 test cases covering: + +- Greetings, math, geography, code generation +- JSON format compliance, translation, markdown +- Safety refusal (harmful content), counting, boolean logic + +### How It Works + +1. Click **"Run Eval"** on a suite in the dashboard +2. Each test case is sent to your proxy endpoint (`/v1/chat/completions`) +3. Real LLM responses are collected and evaluated against expected criteria +4. Results show pass/fail status, latency per case, and overall pass rate + +### Evaluation Strategies + +| Strategy | Description | Example | +| ---------- | ------------------------------------------------ | -------------------------------- | +| `exact` | Output must match exactly | `"4"` | +| `contains` | Output must contain substring (case-insensitive) | `"Paris"` | +| `regex` | Output must match regex pattern | `"1.*2.*3"` | +| `custom` | Custom JS function returns true/false | `(output) => output.length > 10` | + +### API Usage + +```bash +# List all eval suites +curl http://localhost:20128/api/evals + +# Run a suite with pre-collected outputs +curl -X POST http://localhost:20128/api/evals \ + -H 'Content-Type: application/json' \ + -d '{"suiteId": "golden-set", "outputs": {"gs-01": "Hello there!", "gs-02": "4"}}' + +# Get suite details +curl http://localhost:20128/api/evals/golden-set +``` + +### Custom Suites + +Register custom suites programmatically via `registerSuite()` in `src/lib/evals/evalRunner.js`: + +```javascript +registerSuite({ + id: "my-suite", + name: "Custom Eval Suite", + cases: [ + { + id: "c-01", + name: "API response", + model: "gpt-4o", + input: { messages: [{ role: "user", content: "Say OK" }] }, + expected: { strategy: "contains", value: "OK" }, + }, + ], +}); +``` --- diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 13c500aa1e..10bb1ca67a 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -33,6 +33,12 @@ tags: description: Text embedding generation - name: Images description: Image generation + - name: Audio + description: Audio speech and transcription + - name: Moderations + description: Content moderation + - name: Rerank + description: Document reranking - name: Models description: Available model listing - name: Providers @@ -57,6 +63,12 @@ tags: description: System management (restart, shutdown, backup) - name: Pricing description: Model pricing configuration + - name: Cloud + description: Cloud worker authentication and sync + - name: Fallback + description: Fallback chain management + - name: Telemetry + description: Telemetry and token health monitoring paths: # โ”€โ”€โ”€ Proxy Endpoints โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -114,6 +126,23 @@ paths: "401": $ref: "#/components/responses/Unauthorized" + /api/v1/api/chat: + post: + tags: [Chat] + summary: Ollama-compatible chat endpoint + description: Provides compatibility with Ollama's /api/chat format. + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Chat response (JSON or streaming) + /api/v1/messages: post: tags: [Messages] @@ -266,6 +295,118 @@ paths: "200": description: Generated images + /api/v1/audio/speech: + post: + tags: [Audio] + summary: Generate speech audio + description: Text-to-speech endpoint. Routes to configured TTS providers. + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [input] + properties: + input: + type: string + model: + type: string + voice: + type: string + responses: + "200": + description: Audio data + + /api/v1/audio/transcriptions: + post: + tags: [Audio] + summary: Transcribe audio + description: Audio-to-text transcription endpoint. + security: + - BearerAuth: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: + type: string + format: binary + model: + type: string + responses: + "200": + description: Transcription result + + /api/v1/moderations: + post: + tags: [Moderations] + summary: Create moderation + description: Content moderation endpoint. Routes to configured moderation providers. + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [input] + properties: + input: + oneOf: + - type: string + - type: array + items: + type: string + responses: + "200": + description: Moderation result + + /api/v1/rerank: + post: + tags: [Rerank] + summary: Rerank documents + description: Document reranking endpoint. + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [query, documents] + properties: + query: + type: string + documents: + type: array + items: + type: string + model: + type: string + responses: + "200": + description: Reranked documents + + /api/v1: + get: + tags: [System] + summary: API v1 root endpoint + description: Returns basic API info and status. + security: + - BearerAuth: [] + responses: + "200": + description: API info + /api/v1/models: get: tags: [Models] @@ -319,6 +460,22 @@ paths: "200": description: Complete catalog with all providers + /api/models/availability: + get: + tags: [Models] + summary: Get model availability status + description: Returns availability data for all configured models across providers. + responses: + "200": + description: Model availability map + post: + tags: [Models] + summary: Refresh model availability + description: Triggers a re-check of model availability across all providers. + responses: + "200": + description: Availability refreshed + # โ”€โ”€โ”€ Management Endpoints โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/providers: @@ -631,6 +788,120 @@ paths: "200": description: Updated + /api/settings/ip-filter: + get: + tags: [Settings] + summary: Get IP filter configuration + description: Returns the current IP filter settings including blacklist, whitelist, and temp bans. + responses: + "200": + description: IP filter configuration + put: + tags: [Settings] + summary: Update IP filter configuration + description: | + Configure IP filtering with blacklist/whitelist modes, add/remove individual IPs, and manage temp bans. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + mode: + type: string + enum: [blacklist, whitelist] + blacklist: + type: array + items: + type: string + whitelist: + type: array + items: + type: string + addBlacklist: + type: string + removeBlacklist: + type: string + addWhitelist: + type: string + removeWhitelist: + type: string + tempBan: + type: object + properties: + ip: + type: string + durationMs: + type: integer + reason: + type: string + removeBan: + type: string + responses: + "200": + description: Updated IP filter configuration + + /api/settings/system-prompt: + get: + tags: [Settings] + summary: Get system prompt configuration + description: Returns the current system prompt injection settings. + responses: + "200": + description: System prompt configuration + put: + tags: [Settings] + summary: Update system prompt configuration + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + prompt: + type: string + enabled: + type: boolean + responses: + "200": + description: Updated system prompt configuration + + /api/settings/thinking-budget: + get: + tags: [Settings] + summary: Get thinking budget configuration + description: Returns the current thinking/reasoning budget settings for AI models. + responses: + "200": + description: Thinking budget configuration + put: + tags: [Settings] + summary: Update thinking budget configuration + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + mode: + type: string + description: Thinking mode (e.g., auto, manual, disabled) + customBudget: + type: integer + minimum: 0 + maximum: 131072 + effortLevel: + type: string + enum: [none, low, medium, high] + responses: + "200": + description: Updated thinking budget configuration + /api/rate-limit: get: tags: [Settings] @@ -638,6 +909,18 @@ paths: responses: "200": description: Rate limit settings + post: + tags: [Settings] + summary: Update rate limit configuration + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Updated rate limit settings /api/tags: get: @@ -740,6 +1023,28 @@ paths: "200": description: Request log entries + /api/usage/budget: + get: + tags: [Usage] + summary: Get usage budget status + description: Returns current budget limits and consumption. + responses: + "200": + description: Budget status + post: + tags: [Usage] + summary: Configure usage budget + description: Set or update budget limits for usage tracking. + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Updated budget configuration + # โ”€โ”€โ”€ Pricing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/pricing: @@ -764,6 +1069,15 @@ paths: "200": description: Default pricing data + /api/pricing/models: + get: + tags: [Pricing] + summary: Get pricing per model + description: Returns pricing information organized by model. + responses: + "200": + description: Per-model pricing data + # โ”€โ”€โ”€ Translator โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/translator/detect: @@ -901,6 +1215,246 @@ paths: "200": description: Guide settings + /api/cli-tools/antigravity-mitm: + get: + tags: [CLI Tools] + summary: Get Antigravity MITM proxy settings + responses: + "200": + description: MITM proxy configuration + post: + tags: [CLI Tools] + summary: Update Antigravity MITM proxy settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Updated MITM proxy configuration + delete: + tags: [CLI Tools] + summary: Reset Antigravity MITM proxy settings + responses: + "200": + description: MITM proxy settings reset + + /api/cli-tools/antigravity-mitm/alias: + get: + tags: [CLI Tools] + summary: Get Antigravity MITM alias configuration + responses: + "200": + description: Alias configuration + put: + tags: [CLI Tools] + summary: Update Antigravity MITM alias configuration + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Updated alias configuration + + /api/cli-tools/claude-settings: + get: + tags: [CLI Tools] + summary: Get Claude CLI settings + responses: + "200": + description: Claude CLI configuration + post: + tags: [CLI Tools] + summary: Apply Claude CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Claude CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset Claude CLI settings + responses: + "200": + description: Claude CLI settings reset + + /api/cli-tools/cline-settings: + get: + tags: [CLI Tools] + summary: Get Cline CLI settings + responses: + "200": + description: Cline CLI configuration + post: + tags: [CLI Tools] + summary: Apply Cline CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Cline CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset Cline CLI settings + responses: + "200": + description: Cline CLI settings reset + + /api/cli-tools/codex-profiles: + get: + tags: [CLI Tools] + summary: Get Codex profiles + responses: + "200": + description: Codex profile list + post: + tags: [CLI Tools] + summary: Create Codex profile + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Profile created + put: + tags: [CLI Tools] + summary: Update Codex profile + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Profile updated + delete: + tags: [CLI Tools] + summary: Delete Codex profile + responses: + "200": + description: Profile deleted + + /api/cli-tools/codex-settings: + get: + tags: [CLI Tools] + summary: Get Codex CLI settings + responses: + "200": + description: Codex CLI configuration + post: + tags: [CLI Tools] + summary: Apply Codex CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Codex CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset Codex CLI settings + responses: + "200": + description: Codex CLI settings reset + + /api/cli-tools/droid-settings: + get: + tags: [CLI Tools] + summary: Get Droid CLI settings + responses: + "200": + description: Droid CLI configuration + post: + tags: [CLI Tools] + summary: Apply Droid CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Droid CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset Droid CLI settings + responses: + "200": + description: Droid CLI settings reset + + /api/cli-tools/kilo-settings: + get: + tags: [CLI Tools] + summary: Get Kilo CLI settings + responses: + "200": + description: Kilo CLI configuration + post: + tags: [CLI Tools] + summary: Apply Kilo CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Kilo CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset Kilo CLI settings + responses: + "200": + description: Kilo CLI settings reset + + /api/cli-tools/openclaw-settings: + get: + tags: [CLI Tools] + summary: Get OpenClaw CLI settings + responses: + "200": + description: OpenClaw CLI configuration + post: + tags: [CLI Tools] + summary: Apply OpenClaw CLI settings + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: OpenClaw CLI settings applied + delete: + tags: [CLI Tools] + summary: Reset OpenClaw CLI settings + responses: + "200": + description: OpenClaw CLI settings reset + # โ”€โ”€โ”€ OAuth โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/oauth/{provider}/{action}: @@ -926,6 +1480,209 @@ paths: "302": description: Redirect to provider auth page + /api/oauth/cursor/auto-import: + get: + tags: [OAuth] + summary: Auto-import Cursor OAuth credentials + description: Automatically detects and imports Cursor credentials from local config. + responses: + "200": + description: Import result + + /api/oauth/cursor/import: + get: + tags: [OAuth] + summary: Get Cursor import status + responses: + "200": + description: Current import status + post: + tags: [OAuth] + summary: Import Cursor OAuth credentials + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Credentials imported + + /api/oauth/kiro/auto-import: + get: + tags: [OAuth] + summary: Auto-import Kiro OAuth credentials + description: Automatically detects and imports Kiro credentials from local config. + responses: + "200": + description: Import result + + /api/oauth/kiro/import: + get: + tags: [OAuth] + summary: Get Kiro import status + responses: + "200": + description: Current import status + post: + tags: [OAuth] + summary: Import Kiro OAuth credentials + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Credentials imported + + /api/oauth/kiro/social-authorize: + get: + tags: [OAuth] + summary: Initiate Kiro social OAuth authorization + description: Starts the social OAuth flow for Kiro. + responses: + "302": + description: Redirect to OAuth provider + + /api/oauth/kiro/social-exchange: + post: + tags: [OAuth] + summary: Exchange Kiro social OAuth token + description: Exchanges the authorization code for access tokens. + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Token exchange result + + # โ”€โ”€โ”€ Cloud โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + /api/cloud/auth: + post: + tags: [Cloud] + summary: Authenticate with cloud worker + description: Authenticates with the OmniRoute cloud worker for remote access. + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Authentication result + + /api/cloud/credentials/update: + put: + tags: [Cloud] + summary: Update cloud worker credentials + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Credentials updated + + /api/cloud/model/resolve: + post: + tags: [Cloud] + summary: Resolve model via cloud + description: Resolves a model request through the cloud worker. + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Resolved model info + + /api/cloud/models/alias: + get: + tags: [Cloud] + summary: Get cloud model aliases + responses: + "200": + description: Cloud model alias list + put: + tags: [Cloud] + summary: Update cloud model alias + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + "200": + description: Alias updated + + # โ”€โ”€โ”€ Fallback โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + /api/fallback/chains: + get: + tags: [Fallback] + summary: List fallback chains + description: Returns all registered fallback chains for model routing. + responses: + "200": + description: Fallback chain list + post: + tags: [Fallback] + summary: Create fallback chain + description: Registers a fallback routing chain for a model. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [model, chain] + properties: + model: + type: string + chain: + type: array + items: + type: object + properties: + provider: + type: string + priority: + type: integer + enabled: + type: boolean + responses: + "200": + description: Fallback chain created + delete: + tags: [Fallback] + summary: Delete fallback chain + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [model] + properties: + model: + type: string + responses: + "200": + description: Fallback chain deleted + # โ”€โ”€โ”€ System โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/auth/login: @@ -1087,6 +1844,41 @@ paths: "200": description: Caches cleared + /api/cache/stats: + get: + tags: [System] + summary: Get detailed cache statistics + description: Returns detailed statistics for all cache layers. + responses: + "200": + description: Detailed cache stats + delete: + tags: [System] + summary: Clear cache statistics + responses: + "200": + description: Cache stats cleared + + # โ”€โ”€โ”€ Telemetry & Token Health โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + /api/telemetry/summary: + get: + tags: [Telemetry] + summary: Get telemetry summary + description: Returns aggregated telemetry data including request metrics and performance stats. + responses: + "200": + description: Telemetry summary data + + /api/token-health: + get: + tags: [Telemetry] + summary: Get token health status + description: Returns health status of OAuth tokens across all providers. + responses: + "200": + description: Token health status + # โ”€โ”€โ”€ Evals & Policies โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /api/evals: @@ -1109,6 +1901,20 @@ paths: "200": description: Eval results + /api/evals/{suiteId}: + get: + tags: [System] + summary: Get eval suite details + parameters: + - name: suiteId + in: path + required: true + schema: + type: string + responses: + "200": + description: Eval suite details + /api/policies: get: tags: [System] @@ -1377,7 +2183,7 @@ components: type: string strategy: type: string - enum: [priority, weighted, round-robin] + enum: [priority, weighted, round-robin, random, least-used, cost-optimized] default: priority nodes: type: array diff --git a/open-sse/services/combo.js b/open-sse/services/combo.js index 60b1770363..1adcb9f3b3 100644 --- a/open-sse/services/combo.js +++ b/open-sse/services/combo.js @@ -1,11 +1,11 @@ /** * Shared combo (model combo) handling with fallback support - * Supports: priority (sequential), weighted (probabilistic), and round-robin (circular) strategies + * Supports: priority, weighted, round-robin, random, least-used, and cost-optimized strategies */ import { checkFallbackError, formatRetryAfter, getProviderProfile } from "./accountFallback.js"; import { unavailableResponse } from "../utils/error.js"; -import { recordComboRequest } from "./comboMetrics.js"; +import { recordComboRequest, getComboMetrics } from "./comboMetrics.js"; import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.js"; import * as semaphore from "./rateLimitSemaphore.js"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker.js"; @@ -150,9 +150,69 @@ function orderModelsForWeightedFallback(models, selectedModel) { return [selected, ...rest].filter(Boolean).map((e) => e.model); } +/** + * Fisher-Yates shuffle (in-place) + * @param {Array} arr + * @returns {Array} The shuffled array + */ +function shuffleArray(arr) { + for (let i = arr.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [arr[i], arr[j]] = [arr[j], arr[i]]; + } + return arr; +} + +/** + * Sort models by pricing (cheapest first) for cost-optimized strategy + * @param {Array} models - Model strings in "provider/model" format + * @returns {Promise>} Sorted model strings + */ +async function sortModelsByCost(models) { + try { + const { getPricingForModel } = await import("../../src/lib/localDb.js"); + const withCost = await Promise.all( + models.map(async (modelStr) => { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + try { + const pricing = await getPricingForModel(provider, model); + return { modelStr, cost: pricing?.input ?? Infinity }; + } catch { + return { modelStr, cost: Infinity }; + } + }) + ); + withCost.sort((a, b) => a.cost - b.cost); + return withCost.map((e) => e.modelStr); + } catch { + // If pricing lookup fails entirely, return original order + return models; + } +} + +/** + * Sort models by usage count (least-used first) for least-used strategy + * @param {Array} models - Model strings + * @param {string} comboName - Combo name for metrics lookup + * @returns {Array} Sorted model strings + */ +function sortModelsByUsage(models, comboName) { + const metrics = getComboMetrics(comboName); + if (!metrics || !metrics.byModel) return models; + + const withUsage = models.map((modelStr) => ({ + modelStr, + requests: metrics.byModel[modelStr]?.requests ?? 0, + })); + withUsage.sort((a, b) => a.requests - b.requests); + return withUsage.map((e) => e.modelStr); +} + /** * Handle combo chat with fallback - * Supports priority (sequential) and weighted (probabilistic) strategies + * Supports all 6 strategies: priority, weighted, round-robin, random, least-used, cost-optimized * @param {Object} options * @param {Object} options.body - Request body * @param {Object} options.combo - Full combo object { name, models, strategy, config } @@ -215,7 +275,7 @@ export async function handleComboChat({ ); } else { orderedModels = flatModels; - log.info("COMBO", `Priority with nested resolution: ${orderedModels.length} total models`); + log.info("COMBO", `${strategy} with nested resolution: ${orderedModels.length} total models`); } } else if (strategy === "weighted") { const selected = selectWeightedModel(models); @@ -225,6 +285,18 @@ export async function handleComboChat({ orderedModels = models.map((m) => normalizeModelEntry(m).model); } + // Apply strategy-specific ordering + if (strategy === "random") { + orderedModels = shuffleArray([...orderedModels]); + log.info("COMBO", `Random shuffle: ${orderedModels.length} models`); + } else if (strategy === "least-used") { + orderedModels = sortModelsByUsage(orderedModels, combo.name); + log.info("COMBO", `Least-used ordering: ${orderedModels[0]} has fewest requests`); + } else if (strategy === "cost-optimized") { + orderedModels = await sortModelsByCost(orderedModels); + log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`); + } + let lastError = null; let earliestRetryAfter = null; let lastStatus = null; diff --git a/src/app/(dashboard)/dashboard/HomePageClient.js b/src/app/(dashboard)/dashboard/HomePageClient.js new file mode 100644 index 0000000000..11d2827c78 --- /dev/null +++ b/src/app/(dashboard)/dashboard/HomePageClient.js @@ -0,0 +1,381 @@ +"use client"; + +import { useState, useEffect, useMemo, useCallback } from "react"; +import PropTypes from "prop-types"; +import Image from "next/image"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { Card, CardSkeleton, Button, Modal } from "@/shared/components"; +import { AI_PROVIDERS } from "@/shared/constants/providers"; +import { useNotificationStore } from "@/store/notificationStore"; + +export default function HomePageClient({ machineId }) { + const [providerConnections, setProviderConnections] = useState([]); + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [baseUrl, setBaseUrl] = useState("/v1"); + const [selectedProvider, setSelectedProvider] = useState(null); + + useEffect(() => { + if (typeof window !== "undefined") { + setBaseUrl(`${window.location.origin}/v1`); + } + }, []); + + const fetchData = useCallback(async () => { + try { + const [provRes, modelsRes] = await Promise.all([ + fetch("/api/providers"), + fetch("/api/models"), + ]); + if (provRes.ok) { + const provData = await provRes.json(); + setProviderConnections(provData.connections || []); + } + if (modelsRes.ok) { + const modelsData = await modelsRes.json(); + setModels(modelsData.models || []); + } + } catch (e) { + console.log("Error fetching data:", e); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + const providerStats = useMemo(() => { + return Object.entries(AI_PROVIDERS).map(([providerId, providerInfo]) => { + const connections = providerConnections.filter((conn) => conn.provider === providerId); + const connected = connections.filter( + (conn) => + conn.isActive !== false && + (conn.testStatus === "active" || + conn.testStatus === "success" || + conn.testStatus === "unknown") + ).length; + const errors = connections.filter( + (conn) => + conn.isActive !== false && + (conn.testStatus === "error" || + conn.testStatus === "expired" || + conn.testStatus === "unavailable") + ).length; + + const providerKeys = new Set([providerId, providerInfo.alias].filter(Boolean)); + const providerModels = models.filter((m) => providerKeys.has(m.provider)); + + return { + id: providerId, + provider: providerInfo, + total: connections.length, + connected, + errors, + modelCount: providerModels.length, + }; + }); + }, [providerConnections, models]); + + // Models for selected provider + const selectedProviderModels = useMemo(() => { + if (!selectedProvider) return []; + const providerKeys = new Set( + [selectedProvider.id, selectedProvider.provider?.alias].filter(Boolean) + ); + return models.filter((m) => providerKeys.has(m.provider)); + }, [selectedProvider, models]); + + const quickStartLinks = [ + { label: "Documentation", href: "/docs" }, + { label: "OpenAI API compatibility", href: "/docs#api-reference" }, + { label: "Cherry/Codex compatibility", href: "/docs#client-compatibility" }, + { label: "Report issue", href: "https://github.com/decolua/omniroute/issues", external: true }, + ]; + + if (loading) { + return ( +
+ + +
+ ); + } + + const currentEndpoint = baseUrl; + + return ( +
+ {/* Quick Start */} + +
+
+

Quick Start

+

+ First-time setup checklist for API clients and IDE tools. +

+
+ +
    +
  1. + 1. Create API key +

    + Generate one key per environment to isolate usage and revoke safely. +

    +
    2. +
  2. + 2. Connect provider account +

    + Configure providers in Dashboard and validate with Test Connection. +

    +
    3. +
  3. + 3. Use endpoint +

    + Point clients to {currentEndpoint} and send requests to{" "} + /chat/completions. +

    +
    4. +
  4. + 4. Monitor usage +

    + Track requests, tokens, errors, and cost in Usage and Request Logger. +

    +
    5. +
+ + +
+ + + {/* Providers Overview */} + +
+
+

Providers Overview

+

+ {providerStats.filter((item) => item.total > 0).length} configured of{" "} + {providerStats.length} available providers +

+
+ + settings + Manage Providers + +
+ +
+ {providerStats.map((item) => ( + setSelectedProvider(item)} + /> + ))} +
+ + + {/* Provider Models Modal */} + {selectedProvider && ( + setSelectedProvider(null)} + /> + )} +
+ ); +} + +HomePageClient.propTypes = { + machineId: PropTypes.string, +}; + +function ProviderOverviewCard({ item, onClick }) { + const [imgError, setImgError] = useState(false); + + const statusVariant = + item.errors > 0 ? "text-red-500" : item.connected > 0 ? "text-green-500" : "text-text-muted"; + + return ( + + ); +} + +ProviderOverviewCard.propTypes = { + item: PropTypes.shape({ + id: PropTypes.string.isRequired, + provider: PropTypes.shape({ + id: PropTypes.string.isRequired, + name: PropTypes.string.isRequired, + color: PropTypes.string, + textIcon: PropTypes.string, + }).isRequired, + total: PropTypes.number.isRequired, + connected: PropTypes.number.isRequired, + errors: PropTypes.number.isRequired, + modelCount: PropTypes.number.isRequired, + }).isRequired, + onClick: PropTypes.func.isRequired, +}; + +function ProviderModelsModal({ provider, models, onClose }) { + const [copiedModel, setCopiedModel] = useState(null); + const notify = useNotificationStore(); + const router = useRouter(); + + const navigateTo = (path) => { + onClose(); + router.push(path); + }; + + const handleCopy = (text) => { + navigator.clipboard.writeText(text); + setCopiedModel(text); + notify.success(`Copied: ${text}`); + setTimeout(() => setCopiedModel(null), 2000); + }; + + return ( + +
+ {/* Summary */} +
+ token + {models.length} model{models.length !== 1 ? "s" : ""} available + {provider.total > 0 && ( + + โ— {provider.connected} connection{provider.connected !== 1 ? "s" : ""} active + + )} +
+ + {models.length === 0 ? ( +
+ + search_off + +

No models available for this provider.

+

+ Configure a connection first in{" "} + +

+
+ ) : ( +
+ {models.map((m) => ( +
+
+

{m.fullModel}

+ {m.alias !== m.model && ( +

alias: {m.alias}

+ )} +
+ +
+ ))} +
+ )} + + {/* Actions */} +
+ + +
+
+ + ); +} + +ProviderModelsModal.propTypes = { + provider: PropTypes.object.isRequired, + models: PropTypes.array.isRequired, + onClose: PropTypes.func.isRequired, +}; diff --git a/src/app/(dashboard)/dashboard/analytics/page.js b/src/app/(dashboard)/dashboard/analytics/page.js new file mode 100644 index 0000000000..e262ae3eaa --- /dev/null +++ b/src/app/(dashboard)/dashboard/analytics/page.js @@ -0,0 +1,29 @@ +"use client"; + +import { useState, Suspense } from "react"; +import { UsageAnalytics, CardSkeleton, SegmentedControl } from "@/shared/components"; +import EvalsTab from "../usage/components/EvalsTab"; + +export default function AnalyticsPage() { + const [activeTab, setActiveTab] = useState("overview"); + + return ( +
+ + + {activeTab === "overview" && ( + }> + + + )} + {activeTab === "evals" && } +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/combos/page.js b/src/app/(dashboard)/dashboard/combos/page.js index 634ef15543..6bb7dce5a9 100644 --- a/src/app/(dashboard)/dashboard/combos/page.js +++ b/src/app/(dashboard)/dashboard/combos/page.js @@ -308,7 +308,13 @@ function ComboCard({ ? "bg-amber-500/15 text-amber-600 dark:text-amber-400" : strategy === "round-robin" ? "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400" - : "bg-blue-500/15 text-blue-600 dark:text-blue-400" + : strategy === "random" + ? "bg-purple-500/15 text-purple-600 dark:text-purple-400" + : strategy === "least-used" + ? "bg-cyan-500/15 text-cyan-600 dark:text-cyan-400" + : strategy === "cost-optimized" + ? "bg-teal-500/15 text-teal-600 dark:text-teal-400" + : "bg-blue-500/15 text-blue-600 dark:text-blue-400" }`} > {strategy} @@ -704,53 +710,43 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { {/* Strategy Toggle */}
-
- - - +
+ {[ + { value: "priority", label: "Priority", icon: "sort" }, + { value: "weighted", label: "Weighted", icon: "percent" }, + { value: "round-robin", label: "Round-Robin", icon: "autorenew" }, + { value: "random", label: "Random", icon: "shuffle" }, + { value: "least-used", label: "Least-Used", icon: "low_priority" }, + { value: "cost-optimized", label: "Cost-Opt", icon: "savings" }, + ].map((s) => ( + + ))}

- {strategy === "priority" - ? "Sequential fallback: tries model 1 first, then 2, etc." - : strategy === "weighted" - ? "Distributes traffic by weight percentage with fallback" - : "Circular distribution: each request goes to the next model in rotation"} + { + { + priority: "Sequential fallback: tries model 1 first, then 2, etc.", + weighted: "Distributes traffic by weight percentage with fallback", + "round-robin": + "Circular distribution: each request goes to the next model in rotation", + random: "Uniform random selection, then fallback to remaining models", + "least-used": "Picks the model with fewest requests, balancing load over time", + "cost-optimized": "Routes to the cheapest model first based on pricing", + }[strategy] + }

diff --git a/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js b/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js index 95de570cd1..825051f022 100644 --- a/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js +++ b/src/app/(dashboard)/dashboard/endpoint/EndpointPageClient.js @@ -367,141 +367,94 @@ export default function APIPageClient({ machineId }) { {copied === "endpoint_url" ? "Copied!" : "Copy"}
- - {/* Quick Start */} - -
-
-

Quick Start

-

- First-time setup checklist for API clients and IDE tools. -

-
- -
    -
  1. - 1. Create API key -

    - Generate one key per environment to isolate usage and revoke safely. -

    -
    2. -
  2. - 2. Connect provider account -

    - Configure providers in Dashboard and validate with Test Connection. -

    -
    3. -
  3. - 3. Use endpoint -

    - Point clients to {currentEndpoint} and send requests to{" "} - /chat/completions. -

    -
    4. -
  4. - 4. Monitor usage -

    - Track requests, tokens, errors, and cost in Usage and Request Logger. -

    -
    5. -
- - -
- - - {/* API Keys */} - -
-

API Keys

- -
- - {keys.length === 0 ? ( -
-
- vpn_key + {/* Registered Keys โ€” collapsible section inside API Endpoint card */} +
+ -
- ) : ( -
- {keys.map((key) => ( -
-
-

{key.name}

-
- {key.key} - -
-

- Created {new Date(key.createdAt).toLocaleDateString()} -

-
- +
+
+ Registered Keys + + {keys.length} {keys.length === 1 ? "key" : "keys"} +
- ))} -
- )} - +

+ Manage API keys used to authenticate requests to this endpoint +

+
+ + expand_more + + - {/* Providers Overview */} - -
-
-

Providers Overview

-

- {providerStats.filter((item) => item.total > 0).length} configured of{" "} - {providerStats.length} available providers -

-
-
+ {expandedEndpoint === "keys" && ( +
+
+

+ Each key isolates usage tracking and can be revoked independently. +

+ +
-
- {providerStats.map((item) => ( - setSelectedProvider(item)} - /> - ))} + {keys.length === 0 ? ( +
+
+ vpn_key +
+

No API keys yet

+

+ Create your first API key to get started +

+ +
+ ) : ( +
+ {keys.map((key) => ( +
+
+

{key.name}

+
+ {key.key} + +
+

+ Created {new Date(key.createdAt).toLocaleDateString()} +

+
+ +
+ ))} +
+ )} +
+ )}
diff --git a/src/app/(dashboard)/dashboard/health/page.js b/src/app/(dashboard)/dashboard/health/page.js index 6d82aa7523..4fc1ed5b85 100644 --- a/src/app/(dashboard)/dashboard/health/page.js +++ b/src/app/(dashboard)/dashboard/health/page.js @@ -8,6 +8,8 @@ * - Provider health (circuit breaker states) * - Rate limit status * - Active lockouts + * - Signature cache stats + * - Latency telemetry & prompt cache */ import { useState, useEffect, useCallback } from "react"; @@ -38,6 +40,9 @@ export default function HealthPage() { const [data, setData] = useState(null); const [error, setError] = useState(null); const [lastRefresh, setLastRefresh] = useState(null); + const [telemetry, setTelemetry] = useState(null); + const [cache, setCache] = useState(null); + const [signatureCache, setSignatureCache] = useState(null); const fetchHealth = useCallback(async () => { try { @@ -52,11 +57,31 @@ export default function HealthPage() { } }, []); + // Fetch telemetry, cache, and signature cache stats + const fetchExtras = useCallback(async () => { + const results = await Promise.allSettled([ + fetch("/api/telemetry/summary").then((r) => r.json()), + fetch("/api/cache/stats").then((r) => r.json()), + fetch("/api/rate-limits").then((r) => r.json()), + ]); + if (results[0].status === "fulfilled") setTelemetry(results[0].value); + if (results[1].status === "fulfilled") setCache(results[1].value); + if (results[2].status === "fulfilled" && results[2].value.cacheStats) { + setSignatureCache(results[2].value.cacheStats); + } + }, []); + useEffect(() => { fetchHealth(); - const interval = setInterval(fetchHealth, 15000); + fetchExtras(); + const interval = setInterval(() => { + fetchHealth(); + fetchExtras(); + }, 15000); return () => clearInterval(interval); - }, [fetchHealth]); + }, [fetchHealth, fetchExtras]); + + const fmtMs = (ms) => (ms != null ? `${Math.round(ms)}ms` : "โ€”"); if (!data && !error) { return ( @@ -107,7 +132,10 @@ export default function HealthPage() { )}
+ {/* Telemetry Cards โ€” Latency & Prompt Cache */} +
+ {/* Latency Card */} + +

+ speed + Latency +

+ {telemetry ? ( +
+
+ p50 + {fmtMs(telemetry.p50)} +
+
+ p95 + {fmtMs(telemetry.p95)} +
+
+ p99 + {fmtMs(telemetry.p99)} +
+
+ Total requests + {telemetry.totalRequests ?? 0} +
+
+ ) : ( +

No data yet

+ )} + + + {/* Prompt Cache Card */} + +

+ cached + Prompt Cache +

+ {cache ? ( +
+
+ Entries + + {cache.size}/{cache.maxSize} + +
+
+ Hit Rate + {cache.hitRate?.toFixed(1) ?? 0}% +
+
+ Hits / Misses + + {cache.hits ?? 0} / {cache.misses ?? 0} + +
+
+ ) : ( +

No data yet

+ )} + + + {/* Signature Cache Card */} + +

+ database + Signature Cache +

+ {signatureCache ? ( +
+ {[ + { label: "Defaults", value: signatureCache.defaultCount, color: "text-text-muted" }, + { + label: "Tool", + value: `${signatureCache.tool.entries}/${signatureCache.tool.patterns}`, + color: "text-blue-400", + }, + { + label: "Family", + value: `${signatureCache.family.entries}/${signatureCache.family.patterns}`, + color: "text-purple-400", + }, + { + label: "Session", + value: `${signatureCache.session.entries}/${signatureCache.session.patterns}`, + color: "text-cyan-400", + }, + ].map(({ label, value, color }) => ( +
+

{value}

+

{label}

+
+ ))} +
+ ) : ( +

No data yet

+ )} + +
+ {/* Provider Health */}

diff --git a/src/app/(dashboard)/dashboard/page.js b/src/app/(dashboard)/dashboard/page.js index 9818aa88fc..9b7ae9e971 100644 --- a/src/app/(dashboard)/dashboard/page.js +++ b/src/app/(dashboard)/dashboard/page.js @@ -1,7 +1,7 @@ import { redirect } from "next/navigation"; import { getMachineId } from "@/shared/utils/machine"; import { getSettings } from "@/lib/localDb"; -import EndpointPageClient from "./endpoint/EndpointPageClient"; +import HomePageClient from "./HomePageClient"; // Must be dynamic โ€” depends on DB state (setupComplete) that changes at runtime export const dynamic = "force-dynamic"; @@ -12,5 +12,5 @@ export default async function DashboardPage() { redirect("/dashboard/onboarding"); } const machineId = await getMachineId(); - return ; + return ; } diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.js b/src/app/(dashboard)/dashboard/providers/[id]/page.js index ce09e28a46..0c49fdd8e5 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.js +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.js @@ -370,8 +370,21 @@ export default function ProviderDetailPage() { if (!modelId) continue; const parts = modelId.split("/"); const baseAlias = parts[parts.length - 1]; - if (modelAliases[baseAlias]) continue; - await handleSetAlias(modelId, baseAlias, providerStorageAlias); + // Save as imported (default) model in the DB + await fetch("/api/provider-models", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: providerId, + modelId, + modelName: model.name || modelId, + source: "imported", + }), + }); + // Also create an alias for routing + if (!modelAliases[baseAlias]) { + await handleSetAlias(modelId, baseAlias, providerStorageAlias); + } importedCount += 1; } if (importedCount === 0) { diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityBadge.js b/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityBadge.js new file mode 100644 index 0000000000..7daa1f4f6c --- /dev/null +++ b/src/app/(dashboard)/dashboard/providers/components/ModelAvailabilityBadge.js @@ -0,0 +1,194 @@ +"use client"; + +/** + * ModelAvailabilityBadge โ€” compact inline status indicator + * + * Replaces the full ModelAvailabilityPanel card with a small badge + * that shows green when all models are operational, or amber/red + * when there are issues, with a hover popover for details. + */ + +import { useState, useEffect, useCallback, useRef } from "react"; +import { Button } from "@/shared/components"; +import { useNotificationStore } from "@/store/notificationStore"; + +const STATUS_CONFIG = { + available: { icon: "check_circle", color: "#22c55e", label: "Available" }, + cooldown: { icon: "schedule", color: "#f59e0b", label: "Cooldown" }, + unavailable: { icon: "error", color: "#ef4444", label: "Unavailable" }, + unknown: { icon: "help", color: "#6b7280", label: "Unknown" }, +}; + +export default function ModelAvailabilityBadge() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [expanded, setExpanded] = useState(false); + const [clearing, setClearing] = useState(null); + const ref = useRef(null); + const notify = useNotificationStore(); + + const fetchStatus = useCallback(async () => { + try { + const res = await fetch("/api/models/availability"); + if (res.ok) { + const json = await res.json(); + setData(json); + } + } catch { + // silent fail โ€” will retry + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchStatus(); + const interval = setInterval(fetchStatus, 30000); + return () => clearInterval(interval); + }, [fetchStatus]); + + // Close popover on outside click + useEffect(() => { + const handleClick = (e) => { + if (ref.current && !ref.current.contains(e.target)) { + setExpanded(false); + } + }; + if (expanded) document.addEventListener("mousedown", handleClick); + return () => document.removeEventListener("mousedown", handleClick); + }, [expanded]); + + const handleClearCooldown = async (provider, model) => { + setClearing(`${provider}:${model}`); + try { + const res = await fetch("/api/models/availability", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "clearCooldown", provider, model }), + }); + if (res.ok) { + notify.success(`Cooldown cleared for ${model}`); + await fetchStatus(); + } else { + notify.error("Failed to clear cooldown"); + } + } catch { + notify.error("Failed to clear cooldown"); + } finally { + setClearing(null); + } + }; + + if (loading) return null; + + const models = data?.models || []; + const unavailableCount = + data?.unavailableCount || models.filter((m) => m.status !== "available").length; + const isHealthy = unavailableCount === 0; + + // Group unhealthy models by provider + const byProvider = {}; + models.forEach((m) => { + if (m.status === "available") return; + const key = m.provider || "unknown"; + if (!byProvider[key]) byProvider[key] = []; + byProvider[key].push(m); + }); + + return ( +
+ + + {/* Expanded popover */} + {expanded && ( +
+
+
+ + {isHealthy ? "verified" : "warning"} + + Model Status +
+ +
+ +
+ {isHealthy ? ( +

+ All models are responding normally. +

+ ) : ( +
+ {Object.entries(byProvider).map(([provider, provModels]) => ( +
+

+ {provider} +

+
+ {provModels.map((m) => { + const status = STATUS_CONFIG[m.status] || STATUS_CONFIG.unknown; + const isClearing = clearing === `${m.provider}:${m.model}`; + return ( +
+
+ + {status.icon} + + + {m.model} + +
+ {m.status === "cooldown" && ( + + )} +
+ ); + })} +
+
+ ))} +
+ )} +
+
+ )} +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/providers/page.js b/src/app/(dashboard)/dashboard/providers/page.js index 478c63fcd8..bcf68e117a 100644 --- a/src/app/(dashboard)/dashboard/providers/page.js +++ b/src/app/(dashboard)/dashboard/providers/page.js @@ -13,7 +13,7 @@ import { import Link from "next/link"; import { getErrorCode, getRelativeTime } from "@/shared/utils"; import { useNotificationStore } from "@/store/notificationStore"; -import ModelAvailabilityPanel from "./components/ModelAvailabilityPanel"; +import ModelAvailabilityBadge from "./components/ModelAvailabilityBadge"; // Shared helper function to avoid code duplication between ProviderCard and ApiKeyProviderCard function getStatusDisplay(connected, error, errorCode) { @@ -203,22 +203,25 @@ export default function ProvidersPage() {

OAuth Providers

- +
+ + +
{Object.entries(OAUTH_PROVIDERS).map(([key, info]) => ( @@ -400,9 +403,6 @@ export default function ProvidersPage() {
)} - - {/* Model Availability */} -

); } diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js index 1f33188b9b..ccf3ee1d7e 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js @@ -82,22 +82,30 @@ export default function ComboDefaultsTab() {
- {["priority", "weighted", "round-robin"].map((s) => ( + {[ + { value: "priority", label: "Priority", icon: "sort" }, + { value: "weighted", label: "Weighted", icon: "percent" }, + { value: "round-robin", label: "Round-Robin", icon: "autorenew" }, + { value: "random", label: "Random", icon: "shuffle" }, + { value: "least-used", label: "Least-Used", icon: "low_priority" }, + { value: "cost-optimized", label: "Cost-Opt", icon: "savings" }, + ].map((s) => ( ))}
diff --git a/src/app/(dashboard)/dashboard/usage/components/EvalsTab.js b/src/app/(dashboard)/dashboard/usage/components/EvalsTab.js index bbc726e450..09fa58a425 100644 --- a/src/app/(dashboard)/dashboard/usage/components/EvalsTab.js +++ b/src/app/(dashboard)/dashboard/usage/components/EvalsTab.js @@ -3,7 +3,8 @@ /** * EvalsTab โ€” Batch F * - * Lists evaluation suites, runs evals, and shows results. + * Lists evaluation suites, runs evals against real LLM endpoints, + * and shows results. * API: GET/POST /api/evals, GET /api/evals/[suiteId] */ @@ -13,8 +14,10 @@ import { useNotificationStore } from "@/store/notificationStore"; export default function EvalsTab() { const [suites, setSuites] = useState([]); + const [apiKey, setApiKey] = useState(null); const [loading, setLoading] = useState(true); const [running, setRunning] = useState(null); + const [progress, setProgress] = useState({ current: 0, total: 0 }); const [results, setResults] = useState({}); const [search, setSearch] = useState(""); const [expanded, setExpanded] = useState(null); @@ -34,38 +37,103 @@ export default function EvalsTab() { } }, []); + const fetchApiKey = useCallback(async () => { + try { + const res = await fetch("/api/keys"); + if (!res.ok) return; + const data = await res.json(); + const firstKey = data?.keys?.[0]?.key || null; + setApiKey(firstKey); + } catch { + // silent + } + }, []); + useEffect(() => { fetchSuites(); - }, [fetchSuites]); + fetchApiKey(); + }, [fetchSuites, fetchApiKey]); - const handleRunEval = async (suite) => { - setRunning(suite.id); + /** + * Call the proxy LLM endpoint for a single eval case. + * Returns the assistant's response text. + */ + const callLLM = async (evalCase) => { try { + const headers = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + + const res = await fetch("/v1/chat/completions", { + method: "POST", + headers, + body: JSON.stringify({ + model: evalCase.model || "gpt-4o", + messages: evalCase.input?.messages || [], + max_tokens: 512, + stream: false, + }), + }); + if (!res.ok) { + return `[ERROR: HTTP ${res.status}]`; + } + const data = await res.json(); + return data.choices?.[0]?.message?.content || "[No content returned]"; + } catch (err) { + return `[ERROR: ${err.message}]`; + } + }; + + /** + * Run all cases: call LLM for each, then submit outputs for evaluation. + */ + const handleRunEval = async (suite) => { + const cases = suite.cases || []; + if (cases.length === 0) { + notify.warning("No test cases defined for this suite"); + return; + } + + setRunning(suite.id); + setProgress({ current: 0, total: cases.length }); + + try { + // Step 1: Call LLM for each case and collect outputs + const outputs = {}; + for (let i = 0; i < cases.length; i++) { + setProgress({ current: i + 1, total: cases.length }); + const response = await callLLM(cases[i]); + outputs[cases[i].id] = response; + } + + // Step 2: Submit outputs for evaluation const res = await fetch("/api/evals", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ suiteId: suite.id, - outputs: {}, + outputs, }), }); const data = await res.json(); setResults((prev) => ({ ...prev, [suite.id]: data })); - if (data.passed !== undefined) { - const total = (data.passed || 0) + (data.failed || 0); - if (data.failed === 0) { - notify.success(`All ${total} cases passed`, `Eval: ${suite.name}`); + + // Notify with results + if (data.summary) { + const { passed, failed, total } = data.summary; + if (failed === 0) { + notify.success(`All ${total} cases passed โœ…`, `Eval: ${suite.name}`); } else { - notify.warning( - `${data.passed}/${total} passed, ${data.failed} failed`, - `Eval: ${suite.name}` - ); + notify.warning(`${passed}/${total} passed, ${failed} failed`, `Eval: ${suite.name}`); } } + + // Auto-expand to show results + setExpanded(suite.id); } catch { notify.error("Eval run failed"); } finally { setRunning(null); + setProgress({ current: 0, total: 0 }); } }; @@ -97,10 +165,10 @@ export default function EvalsTab() { } const RESULT_COLUMNS = [ - { key: "caseId", label: "Case" }, + { key: "caseName", label: "Case" }, { key: "status", label: "Status" }, - { key: "expected", label: "Expected" }, - { key: "actual", label: "Actual" }, + { key: "durationMs", label: "Latency" }, + { key: "details", label: "Details" }, ]; return ( @@ -110,7 +178,12 @@ export default function EvalsTab() {
science
-

Evaluation Suites

+
+

Evaluation Suites

+

+ Run test cases against your LLM endpoints to validate response quality +

+
@@ -143,30 +216,62 @@ export default function EvalsTab() {

{suite.name || suite.id}

{caseCount} case{caseCount !== 1 ? "s" : ""} - {suiteResult && ( + {suite.description && โ€” {suite.description}} + {suiteResult?.summary && ( - โ€ข Last run: {suiteResult.passed || 0} โœ… {suiteResult.failed || 0} โŒ + โ€ข Last run: {suiteResult.summary.passed || 0} โœ…{" "} + {suiteResult.summary.failed || 0} โŒ ({suiteResult.summary.passRate}%) )}

- +
+ {isRunning && progress.total > 0 && ( + + {progress.current}/{progress.total} + + )} + +
{isExpanded && suiteResult?.results && (
+ {/* Summary bar */} + {suiteResult.summary && ( +
+
+ = 80 + ? "text-amber-400" + : "text-red-400" + }`} + > + {suiteResult.summary.passRate}% + + pass rate +
+
+ {suiteResult.summary.passed} passed ยท {suiteResult.summary.failed} failed + ยท {suiteResult.summary.total} total +
+
+ )} ({ @@ -181,15 +286,32 @@ export default function EvalsTab() { โŒ Failed ); } + if (col.key === "durationMs") { + return ( + + {row.durationMs != null ? `${row.durationMs}ms` : "โ€”"} + + ); + } + if (col.key === "details") { + const d = row.details || {}; + return ( + + {d.searchTerm + ? `Contains: "${d.searchTerm}"` + : d.pattern + ? `Regex: ${d.pattern}` + : d.expected + ? `Expected: "${String(d.expected).slice(0, 50)}"` + : row.error || "โ€”"} + + ); + } return ( - - {typeof row[col.key] === "object" - ? JSON.stringify(row[col.key]) - : row[col.key] || "โ€”"} - + {row[col.key] || "โ€”"} ); }} - maxHeight="300px" + maxHeight="400px" emptyMessage="No results yet" />
diff --git a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.js b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.js index 9f609a99df..7e7ff7b46e 100644 --- a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.js +++ b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.js @@ -350,10 +350,10 @@ export default function ProviderLimits() { className="inline-flex items-center gap-1.5 px-2.5 py-1 rounded-full text-xs font-semibold cursor-pointer" style={{ border: active - ? "1px solid var(--primary, #f97815)" + ? "1px solid var(--primary, #E54D5E)" : "1px solid rgba(255,255,255,0.12)", background: active ? "rgba(249,120,21,0.14)" : "transparent", - color: active ? "var(--primary, #f97815)" : "var(--text-muted)", + color: active ? "var(--primary, #E54D5E)" : "var(--text-muted)", }} > {tier.label} diff --git a/src/app/(dashboard)/dashboard/usage/components/RateLimitStatus.js b/src/app/(dashboard)/dashboard/usage/components/RateLimitStatus.js index a0d15ecde9..2cd955dcad 100644 --- a/src/app/(dashboard)/dashboard/usage/components/RateLimitStatus.js +++ b/src/app/(dashboard)/dashboard/usage/components/RateLimitStatus.js @@ -11,7 +11,8 @@ export default function RateLimitStatus() { try { const res = await fetch("/api/rate-limits"); if (res.ok) setData(await res.json()); - } catch {} finally { + } catch { + } finally { setLoading(false); } }, []); @@ -65,11 +66,14 @@ export default function RateLimitStatus() { bg-orange-500/5 border border-orange-500/15" >
- lock + + lock +

{lock.model}

- Account: {lock.accountId?.slice(0, 12) || "N/A"} + Account:{" "} + {lock.accountId?.slice(0, 12) || "N/A"} {lock.reason && <> โ€” {lock.reason}}

@@ -82,33 +86,6 @@ export default function RateLimitStatus() {
)} - - {/* Signature Cache Stats */} - {data.cacheStats && ( - -
-
- -
-

Signature Cache

-
-
- {[ - { label: "Defaults", value: data.cacheStats.defaultCount, color: "text-text-muted" }, - { label: "Tool", value: `${data.cacheStats.tool.entries}/${data.cacheStats.tool.patterns}`, color: "text-blue-400" }, - { label: "Family", value: `${data.cacheStats.family.entries}/${data.cacheStats.family.patterns}`, color: "text-purple-400" }, - { label: "Session", value: `${data.cacheStats.session.entries}/${data.cacheStats.session.patterns}`, color: "text-cyan-400" }, - ].map(({ label, value, color }) => ( -
-

{value}

-

{label}

-
- ))} -
- - )} ); } diff --git a/src/app/(dashboard)/dashboard/usage/page.js b/src/app/(dashboard)/dashboard/usage/page.js index ab5796c98e..68760f5887 100644 --- a/src/app/(dashboard)/dashboard/usage/page.js +++ b/src/app/(dashboard)/dashboard/usage/page.js @@ -1,59 +1,41 @@ "use client"; import { useState, Suspense } from "react"; -import { - UsageAnalytics, - RequestLoggerV2, - ProxyLogger, - CardSkeleton, - SegmentedControl, -} from "@/shared/components"; +import { RequestLoggerV2, ProxyLogger, CardSkeleton, SegmentedControl } from "@/shared/components"; import ProviderLimits from "./components/ProviderLimits"; import SessionsTab from "./components/SessionsTab"; import RateLimitStatus from "./components/RateLimitStatus"; -import BudgetTelemetryCards from "./components/BudgetTelemetryCards"; import BudgetTab from "./components/BudgetTab"; -import EvalsTab from "./components/EvalsTab"; export default function UsagePage() { - const [activeTab, setActiveTab] = useState("overview"); + const [activeTab, setActiveTab] = useState("limits"); return (
{/* Content */} - {activeTab === "overview" && ( - }> - - - - )} - {activeTab === "logs" && } - {activeTab === "proxy-logs" && } {activeTab === "limits" && (
}> +
)} - {activeTab === "sessions" && } + {activeTab === "logs" && } + {activeTab === "proxy-logs" && } {activeTab === "budget" && } - {activeTab === "evals" && }
); } diff --git a/src/app/api/models/catalog/route.js b/src/app/api/models/catalog/route.js index bb741403cb..741f1ef5af 100644 --- a/src/app/api/models/catalog/route.js +++ b/src/app/api/models/catalog/route.js @@ -74,7 +74,7 @@ export async function GET() { }); } - // Custom models + // Custom models (from DB) for (const [providerId, models] of Object.entries(customModelsMap)) { const alias = PROVIDER_ID_TO_ALIAS[providerId] || providerId; if (!catalog[alias]) { @@ -89,11 +89,13 @@ export async function GET() { const fullId = `${alias}/${model.id}`; // Skip duplicates if (catalog[alias].models.some((m) => m.id === fullId)) continue; + // Imported models are treated as default (not custom) + const isCustom = model.source !== "imported"; catalog[alias].models.push({ id: fullId, name: model.name || model.id, type: "chat", - custom: true, + custom: isCustom, }); } } diff --git a/src/app/api/provider-models/route.js b/src/app/api/provider-models/route.js index f287a9c900..d3ce7aa66a 100644 --- a/src/app/api/provider-models/route.js +++ b/src/app/api/provider-models/route.js @@ -32,7 +32,7 @@ export async function GET(request) { export async function POST(request) { try { const body = await request.json(); - const { provider, modelId, modelName } = body; + const { provider, modelId, modelName, source } = body; if (!provider || !modelId) { return Response.json( @@ -41,7 +41,7 @@ export async function POST(request) { ); } - const model = await addCustomModel(provider, modelId, modelName); + const model = await addCustomModel(provider, modelId, modelName, source || "manual"); return Response.json({ model }); } catch (error) { return Response.json( diff --git a/src/app/api/providers/[id]/models/route.js b/src/app/api/providers/[id]/models/route.js index 703f3fd218..e27e7e9858 100644 --- a/src/app/api/providers/[id]/models/route.js +++ b/src/app/api/providers/[id]/models/route.js @@ -5,6 +5,26 @@ import { isAnthropicCompatibleProvider, } from "@/shared/constants/providers"; +// Providers that return hardcoded models (no remote /models API) +const STATIC_MODEL_PROVIDERS = { + deepgram: () => [ + { id: "nova-3", name: "Nova 3 (Transcription)" }, + { id: "nova-2", name: "Nova 2 (Transcription)" }, + { id: "whisper-large", name: "Whisper Large (Transcription)" }, + { id: "aura-asteria-en", name: "Aura Asteria EN (TTS)" }, + { id: "aura-luna-en", name: "Aura Luna EN (TTS)" }, + { id: "aura-stella-en", name: "Aura Stella EN (TTS)" }, + ], + assemblyai: () => [ + { id: "universal-3-pro", name: "Universal 3 Pro (Transcription)" }, + { id: "universal-2", name: "Universal 2 (Transcription)" }, + ], + nanobanana: () => [ + { id: "nanobanana-flash", name: "NanoBanana Flash (Gemini 2.5 Flash)" }, + { id: "nanobanana-pro", name: "NanoBanana Pro (Gemini 3 Pro)" }, + ], +}; + // Provider models endpoints configuration const PROVIDER_MODELS_CONFIG = { claude: { @@ -272,6 +292,16 @@ export async function GET(request, { params }) { }); } + // Static model providers (no remote /models API) + const staticModelsFn = STATIC_MODEL_PROVIDERS[connection.provider]; + if (staticModelsFn) { + return NextResponse.json({ + provider: connection.provider, + connectionId: connection.id, + models: staticModelsFn(), + }); + } + const config = PROVIDER_MODELS_CONFIG[connection.provider]; if (!config) { return NextResponse.json( diff --git a/src/app/globals.css b/src/app/globals.css index ab0f4ee4b3..c10f260d95 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -3,40 +3,40 @@ @custom-variant dark (&:where(.dark, .dark *)); -/* macOS-inspired Color Palette with Terracotta Primary */ +/* OpenClaw ร— ClawHub Color Palette */ :root { - /* Primary - Warm Coral/Terracotta */ - --color-primary: #d97757; - --color-primary-hover: #c56243; + /* Primary - Coral Red (OpenClaw) */ + --color-primary: #e54d5e; + --color-primary-hover: #c93d4e; /* Light theme */ - --color-bg: #fbf9f6; - --color-bg-alt: #f5f1ed; + --color-bg: #f9f9fb; + --color-bg-alt: #f0f0f5; --color-surface: #ffffff; - --color-sidebar: rgba(246, 246, 246, 0.8); - --color-border: rgba(0, 0, 0, 0.1); - --color-text-main: #383733; - --color-text-muted: #75736e; + --color-sidebar: rgba(245, 245, 250, 0.8); + --color-border: rgba(0, 0, 0, 0.08); + --color-text-main: #1a1a2e; + --color-text-muted: #71717a; - /* Shadows - subtle macOS style */ + /* Shadows */ --shadow-soft: 0 1px 3px rgba(0, 0, 0, 0.02), 0 4px 12px rgba(0, 0, 0, 0.015); - --shadow-warm: 0 2px 12px -2px rgba(217, 119, 87, 0.12); - --shadow-elevated: 0 12px 28px -4px rgba(60, 50, 45, 0.06); + --shadow-warm: 0 2px 12px -2px rgba(229, 77, 94, 0.12); + --shadow-elevated: 0 12px 28px -4px rgba(20, 20, 40, 0.06); } .dark { - /* Dark theme */ - --color-bg: #191918; - --color-bg-alt: #1f1f1e; - --color-surface: #242423; - --color-sidebar: rgba(30, 30, 30, 0.8); - --color-border: rgba(255, 255, 255, 0.1); - --color-text-main: #ecebe8; - --color-text-muted: #9e9d99; + /* Dark theme (ClawHub deep) */ + --color-bg: #0b0e14; + --color-bg-alt: #111520; + --color-surface: #161b22; + --color-sidebar: rgba(16, 20, 30, 0.8); + --color-border: rgba(255, 255, 255, 0.08); + --color-text-main: #e6e6ef; + --color-text-muted: #a1a1aa; - /* Dark shadows - subtle macOS style */ + /* Dark shadows */ --shadow-soft: 0 1px 3px rgba(0, 0, 0, 0.15), 0 4px 12px rgba(0, 0, 0, 0.1); - --shadow-warm: 0 2px 12px -2px rgba(217, 119, 87, 0.15); + --shadow-warm: 0 2px 12px -2px rgba(229, 77, 94, 0.15); --shadow-elevated: 0 12px 28px -4px rgba(0, 0, 0, 0.3); } @@ -54,18 +54,18 @@ --color-text-muted: var(--color-text-muted); /* Static colors (for explicit light/dark usage) */ - --color-bg-light: #fbf9f6; - --color-bg-dark: #191918; + --color-bg-light: #f9f9fb; + --color-bg-dark: #0b0e14; --color-surface-light: #ffffff; - --color-surface-dark: #242423; - --color-sidebar-light: #f0efec; - --color-sidebar-dark: #1f1f1e; - --color-border-light: #e6e4dd; - --color-border-dark: #333331; - --color-text-main-light: #383733; - --color-text-main-dark: #ecebe8; - --color-text-muted-light: #75736e; - --color-text-muted-dark: #9e9d99; + --color-surface-dark: #161b22; + --color-sidebar-light: #ededf2; + --color-sidebar-dark: #111520; + --color-border-light: #e2e2ea; + --color-border-dark: #2d333b; + --color-text-main-light: #1a1a2e; + --color-text-main-dark: #e6e6ef; + --color-text-muted-light: #71717a; + --color-text-muted-dark: #a1a1aa; /* Shadows */ --shadow-soft: var(--shadow-soft); @@ -88,7 +88,7 @@ body { /* Selection */ ::selection { - background-color: rgba(217, 119, 87, 0.2); + background-color: rgba(229, 77, 94, 0.2); color: var(--color-primary); } @@ -142,11 +142,11 @@ body { /* Hero gradient */ .bg-hero-gradient { - background: linear-gradient(180deg, #f5f1ed 0%, #fefcfb 100%); + background: linear-gradient(180deg, #f0f0f5 0%, #f9f9fb 100%); } .dark .bg-hero-gradient { - background: linear-gradient(180deg, #1f1f1e 0%, #191918 100%); + background: linear-gradient(180deg, #111520 0%, #0b0e14 100%); } /* Material Symbols */ @@ -219,15 +219,15 @@ button .material-symbols-outlined, 0%, 100% { box-shadow: - 0 0 5px rgba(217, 119, 87, 0.3), - 0 0 10px rgba(217, 119, 87, 0.2); - border-color: rgba(217, 119, 87, 0.5); + 0 0 5px rgba(229, 77, 94, 0.3), + 0 0 10px rgba(229, 77, 94, 0.2); + border-color: rgba(229, 77, 94, 0.5); } 50% { box-shadow: - 0 0 10px rgba(217, 119, 87, 0.5), - 0 0 20px rgba(217, 119, 87, 0.3); - border-color: rgba(217, 119, 87, 0.8); + 0 0 10px rgba(229, 77, 94, 0.5), + 0 0 20px rgba(229, 77, 94, 0.3); + border-color: rgba(229, 77, 94, 0.8); } } @@ -243,7 +243,7 @@ button .material-symbols-outlined, } .dark .bg-vibrancy { - background: rgba(30, 30, 30, 0.72); + background: rgba(16, 20, 30, 0.72); } /* macOS Traffic Lights */ diff --git a/src/app/landing/components/AnimatedBackground.js b/src/app/landing/components/AnimatedBackground.js index 2f434cef64..f115df45b9 100644 --- a/src/app/landing/components/AnimatedBackground.js +++ b/src/app/landing/components/AnimatedBackground.js @@ -9,13 +9,13 @@ export default function AnimatedBackground() {
{/* Animated gradient orbs */} -
+
@@ -24,7 +24,7 @@ export default function AnimatedBackground() { className="absolute inset-0" style={{ background: - "radial-gradient(circle at center, transparent 0%, rgba(24, 20, 17, 0.4) 100%)", + "radial-gradient(circle at center, transparent 0%, rgba(11, 14, 20, 0.4) 100%)", }} />
diff --git a/src/app/landing/components/FlowAnimation.js b/src/app/landing/components/FlowAnimation.js index dc68f6a85d..a5b501c6b9 100644 --- a/src/app/landing/components/FlowAnimation.js +++ b/src/app/landing/components/FlowAnimation.js @@ -29,10 +29,10 @@ export default function FlowAnimation() { return (
{/* OmniRoute Hub - Center */} -
- hub +
+ hub OmniRoute -
+
{/* CLI Tools - Left side */} @@ -42,7 +42,7 @@ export default function FlowAnimation() { key={tool.id} className="flex items-center gap-3 opacity-70 hover:opacity-100 transition-opacity group" > -
+
{tool.name} @@ -132,7 +132,7 @@ export default function FlowAnimation() {
@@ -142,7 +142,7 @@ export default function FlowAnimation() {
{/* Mobile fallback */} -
+

Interactive diagram visible on desktop

diff --git a/src/app/landing/components/Footer.js b/src/app/landing/components/Footer.js index aacaa5f434..a42968b0dc 100644 --- a/src/app/landing/components/Footer.js +++ b/src/app/landing/components/Footer.js @@ -2,13 +2,13 @@ export default function Footer() { return ( -