diff --git a/README.md b/README.md index cc34c6e438..a6d9876fa1 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,8 @@ Developers pay $20–200/month for Claude Pro, Codex Pro, or GitHub Copilot. Eve - **Provider Limits Tracking** — Cached quota snapshots refresh on a server-side schedule (default `PROVIDER_LIMITS_SYNC_INTERVAL_MINUTES=70`) with manual refresh available in the UI - **Multi-Account Support** — Multiple accounts per provider with auto round-robin — when one runs out, switches to the next - **Custom Combos** — Customizable fallback chains with 13 balancing strategies (priority, weighted, fill-first, round-robin, P2C, random, least-used, cost-optimized, strict-random, auto, lkgp, context-optimized, **context-relay**) +- **Structured Combo Builder** — Build combos step-by-step with explicit provider + model + account selection, including repeated providers and fixed-account targets +- **Quota-Aware P2C** — Power-of-two account selection now factors quota headroom, backoff, recent errors, and consecutive use - **Codex Business Quotas** — Business/Team workspace quota monitoring directly in the dashboard @@ -2231,16 +2233,16 @@ Se não quiser criar credenciais próprias agora, ainda é possível usar o flux ## 🗺️ Roadmap -OmniRoute has **210+ features planned** across multiple development phases. Here are the key areas: +OmniRoute has **218+ features planned** across multiple development phases. Here are the key areas: -| Category | Planned Features | Highlights | -| ----------------------------- | ---------------- | -------------------------------------------------------------------------------------- | -| 🧠 **Routing & Intelligence** | 25+ | Lowest-latency routing, tag-based routing, quota preflight, P2C account selection | -| 🔒 **Security & Compliance** | 20+ | SSRF hardening, credential cloaking, rate-limit per endpoint, management key scoping | -| 📊 **Observability** | 15+ | OpenTelemetry integration, real-time quota monitoring, cost tracking per model | -| 🔄 **Provider Integrations** | 20+ | Dynamic model registry, provider cooldowns, multi-account Codex, Copilot quota parsing | -| ⚡ **Performance** | 15+ | Dual cache layer, prompt cache, response cache, streaming keepalive, batch API | -| 🌐 **Ecosystem** | 10+ | WebSocket API, config hot-reload, distributed config store, commercial mode | +| Category | Planned Features | Highlights | +| ----------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------- | +| 🧠 **Routing & Intelligence** | 25+ | Lowest-latency routing, tag-based routing, quota preflight, quota-aware P2C, step-based combo routing | +| 🔒 **Security & Compliance** | 20+ | SSRF hardening, credential cloaking, rate-limit per endpoint, management key scoping | +| 📊 **Observability** | 15+ | OpenTelemetry integration, real-time quota monitoring, combo target health, cost tracking per model | +| 🔄 **Provider Integrations** | 20+ | Dynamic model registry, provider cooldowns, multi-account Codex, Copilot quota parsing | +| ⚡ **Performance** | 15+ | Dual cache layer, prompt cache, response cache, streaming keepalive, batch API | +| 🌐 **Ecosystem** | 10+ | WebSocket API, config hot-reload, distributed config store, commercial mode | ### 🔜 Coming Soon diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9f9c481999..90dee6ce94 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ 🌐 **Languages:** 🇺🇸 [English](ARCHITECTURE.md) | 🇧🇷 [Português (Brasil)](i18n/pt-BR/ARCHITECTURE.md) | 🇪🇸 [Español](i18n/es/ARCHITECTURE.md) | 🇫🇷 [Français](i18n/fr/ARCHITECTURE.md) | 🇮🇹 [Italiano](i18n/it/ARCHITECTURE.md) | 🇷🇺 [Русский](i18n/ru/ARCHITECTURE.md) | 🇨🇳 [中文 (简体)](i18n/zh-CN/ARCHITECTURE.md) | 🇩🇪 [Deutsch](i18n/de/ARCHITECTURE.md) | 🇮🇳 [हिन्दी](i18n/in/ARCHITECTURE.md) | 🇹🇭 [ไทย](i18n/th/ARCHITECTURE.md) | 🇺🇦 [Українська](i18n/uk-UA/ARCHITECTURE.md) | 🇸🇦 [العربية](i18n/ar/ARCHITECTURE.md) | 🇯🇵 [日本語](i18n/ja/ARCHITECTURE.md) | 🇻🇳 [Tiếng Việt](i18n/vi/ARCHITECTURE.md) | 🇧🇬 [Български](i18n/bg/ARCHITECTURE.md) | 🇩🇰 [Dansk](i18n/da/ARCHITECTURE.md) | 🇫🇮 [Suomi](i18n/fi/ARCHITECTURE.md) | 🇮🇱 [עברית](i18n/he/ARCHITECTURE.md) | 🇭🇺 [Magyar](i18n/hu/ARCHITECTURE.md) | 🇮🇩 [Bahasa Indonesia](i18n/id/ARCHITECTURE.md) | 🇰🇷 [한국어](i18n/ko/ARCHITECTURE.md) | 🇲🇾 [Bahasa Melayu](i18n/ms/ARCHITECTURE.md) | 🇳🇱 [Nederlands](i18n/nl/ARCHITECTURE.md) | 🇳🇴 [Norsk](i18n/no/ARCHITECTURE.md) | 🇵🇹 [Português (Portugal)](i18n/pt/ARCHITECTURE.md) | 🇷🇴 [Română](i18n/ro/ARCHITECTURE.md) | 🇵🇱 [Polski](i18n/pl/ARCHITECTURE.md) | 🇸🇰 [Slovenčina](i18n/sk/ARCHITECTURE.md) | 🇸🇪 [Svenska](i18n/sv/ARCHITECTURE.md) | 🇵🇭 [Filipino](i18n/phi/ARCHITECTURE.md) | 🇨🇿 [Čeština](i18n/cs/ARCHITECTURE.md) -_Last updated: 2026-04-11_ +_Last updated: 2026-04-12_ ## Executive Summary @@ -14,7 +14,9 @@ Core capabilities: - OpenAI-compatible API surface for CLI/tools (100+ providers, 16 executors) - Request/response translation across provider formats - Model combo fallback (multi-model sequence) +- Structured combo steps (`provider + model + connection`) with runtime ordering by `compositeTiers` - Account-level fallback (multi-account per provider) +- Quota preflight and quota-aware P2C account selection in the main chat path - OAuth + API-key provider connection management (13 OAuth modules) - Embedding generation via `/v1/embeddings` (6 providers, 9 models) - Image generation via `/v1/images/generations` (10+ providers, 20+ models) @@ -45,6 +47,7 @@ Core capabilities: - Domain state persistence (SQLite write-through cache for fallbacks, budgets, lockouts, circuit breakers) - Policy engine for centralized request evaluation (lockout → budget → fallback) - Request telemetry with p50/p95/p99 latency aggregation +- Combo target telemetry and historical combo target health via `combo_execution_key` / `combo_step_id` - Correlation ID (X-Request-Id) for end-to-end tracing - Compliance audit logging with opt-out per API key - Eval framework for LLM quality assurance @@ -89,15 +92,15 @@ Main pages under `src/app/(dashboard)/dashboard/`: - `/dashboard` — quick start + provider overview - `/dashboard/endpoint` — endpoint proxy + MCP + A2A + API endpoint tabs - `/dashboard/providers` — provider connections and credentials -- `/dashboard/combos` — combo strategies, templates, model routing rules, manual persisted ordering +- `/dashboard/combos` — combo strategies, templates, step-based builder, model routing rules, manual persisted ordering - `/dashboard/costs` — cost aggregation and pricing visibility -- `/dashboard/analytics` — usage analytics and evaluations +- `/dashboard/analytics` — usage analytics, evaluations, combo target health - `/dashboard/limits` — quota/rate controls - `/dashboard/cli-tools` — CLI onboarding, runtime detection, config generation - `/dashboard/agents` — detected ACP agents + custom agent registration - `/dashboard/media` — image/video/music playground - `/dashboard/search-tools` — search provider testing and history -- `/dashboard/health` — uptime, circuit breakers, rate limits +- `/dashboard/health` — uptime, circuit breakers, rate limits, quota-monitored sessions - `/dashboard/logs` — request/proxy/audit/console logs - `/dashboard/settings` — system settings tabs (general, routing, combo defaults, etc.) - `/dashboard/api-manager` — API key lifecycle and model permissions diff --git a/docs/FEATURES.md b/docs/FEATURES.md index a6d4b4b3b7..c1e44fad5c 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -18,6 +18,13 @@ Manage AI provider connections: OAuth providers (Claude Code, Codex, Gemini CLI) Create model routing combos with 13 strategies: priority, weighted, round-robin, random, least-used, cost-optimized, strict-random, auto, fill-first, p2c, lkgp, context-optimized, and **context-relay**. Each combo chains multiple models with automatic fallback and includes quick templates and readiness checks. +Recent combo improvements: + +- **Structured combo builder** — create each step by selecting provider, model, and exact account/connection +- **Repeated provider support** — reuse the same provider many times in one combo as long as the `(provider, model, connection)` tuple is unique +- **Combo target health** — analytics and health surfaces now distinguish individual combo targets/steps instead of collapsing everything into model strings +- **Composite tier ordering** — `defaultTier -> fallbackTier` now influences runtime execution/fallback order for top-level combo steps + ![Combos Dashboard](screenshots/02-combos.png) --- @@ -32,7 +39,7 @@ Comprehensive usage analytics with token consumption, cost estimates, activity h ## 🏥 System Health -Real-time monitoring: uptime, memory, version, latency percentiles (p50/p95/p99), cache statistics, and provider circuit breaker states. +Real-time monitoring: uptime, memory, version, latency percentiles (p50/p95/p99), cache statistics, provider circuit breaker states, active quota-monitored sessions, and combo target health. ![Health Dashboard](screenshots/04-health.png) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 7dfd82b2cf..02ea42418f 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -81,6 +81,14 @@ type ComboRuntimeStep = label: string | null; }; +function isRecord(value): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function toTrimmedString(value): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + /** * Validate that a successful (HTTP 200) non-streaming response actually contains * meaningful content. Returns { valid: true } or { valid: false, reason }. @@ -240,9 +248,9 @@ function normalizeRuntimeStep(entry, comboName, index, allCombos, path = []) { } function getDirectComboTargets(combo) { - return (combo.models || []) - .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null)) - .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); + return getOrderedTopLevelRuntimeSteps(combo, null).filter( + (entry): entry is ResolvedComboTarget => entry?.kind === "model" + ); } function getTopLevelRuntimeSteps(combo, allCombos, path = []) { @@ -251,6 +259,84 @@ function getTopLevelRuntimeSteps(combo, allCombos, path = []) { .filter((entry): entry is ComboRuntimeStep => entry !== null); } +function getCompositeTierStepOrder(combo): string[] { + const compositeTiers = isRecord(combo?.config) ? combo.config.compositeTiers : null; + if (!isRecord(compositeTiers)) return []; + + const defaultTier = toTrimmedString(compositeTiers.defaultTier); + const tiers = isRecord(compositeTiers.tiers) ? compositeTiers.tiers : null; + if (!defaultTier || !tiers) return []; + + const orderedStepIds: string[] = []; + const visitedTiers = new Set(); + const seenStepIds = new Set(); + const tierEntries = new Map( + Object.entries(tiers) + .map(([tierName, rawTier]) => { + if (!isRecord(rawTier)) return null; + const normalizedTierName = toTrimmedString(tierName); + const stepId = toTrimmedString(rawTier.stepId); + const fallbackTier = toTrimmedString(rawTier.fallbackTier); + if (!normalizedTierName || !stepId) return null; + return [normalizedTierName, { stepId, fallbackTier }] as const; + }) + .filter(Boolean) + ); + + let currentTier = defaultTier; + while (currentTier && tierEntries.has(currentTier) && !visitedTiers.has(currentTier)) { + visitedTiers.add(currentTier); + const entry = tierEntries.get(currentTier); + if (!entry) break; + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + currentTier = entry.fallbackTier; + } + + for (const entry of tierEntries.values()) { + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + } + + return orderedStepIds; +} + +function hasCompositeTierRuntimeOrder(combo): boolean { + return getCompositeTierStepOrder(combo).length > 0; +} + +function orderRuntimeStepsByCompositeTiers(steps: ComboRuntimeStep[], combo): ComboRuntimeStep[] { + const orderedStepIds = getCompositeTierStepOrder(combo); + if (orderedStepIds.length === 0) return steps; + + const byStepId = new Map(steps.map((step) => [step.stepId, step])); + const seen = new Set(); + const ordered: ComboRuntimeStep[] = []; + + for (const stepId of orderedStepIds) { + const step = byStepId.get(stepId); + if (!step || seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + for (const step of steps) { + if (seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + return ordered; +} + +function getOrderedTopLevelRuntimeSteps(combo, allCombos, path = []) { + return orderRuntimeStepsByCompositeTiers(getTopLevelRuntimeSteps(combo, allCombos, path), combo); +} + function expandRuntimeStep(step, allCombos, visited = new Set(), depth = 0, path = []) { if (step.kind === "model") return [step]; if (depth > MAX_COMBO_DEPTH) return []; @@ -280,7 +366,7 @@ export function resolveNestedComboTargets( if (visited.has(combo.name)) return []; visited.add(combo.name); - const runtimeSteps = getTopLevelRuntimeSteps(combo, allCombos, path); + const runtimeSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos, path); const resolved: ResolvedComboTarget[] = []; for (const step of runtimeSteps) { @@ -401,12 +487,14 @@ function selectWeightedTarget(targets: Array<{ weight: number }>) { function orderTargetsForWeightedFallback( targets: Array<{ executionKey: string; weight: number }>, - selectedExecutionKey: string + selectedExecutionKey: string, + preserveExistingOrder = false ) { const selected = targets.find((target) => target.executionKey === selectedExecutionKey); - const rest = targets - .filter((target) => target.executionKey !== selectedExecutionKey) - .sort((a, b) => b.weight - a.weight); + const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey); + if (!preserveExistingOrder) { + rest.sort((a, b) => b.weight - a.weight); + } return [selected, ...rest].filter(Boolean); } @@ -742,9 +830,7 @@ export function resolveComboTargets(combo, allCombos) { } function resolveWeightedTargets(combo, allCombos) { - const topLevelSteps = allCombos - ? getTopLevelRuntimeSteps(combo, allCombos) - : getDirectComboTargets(combo); + const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos); if (topLevelSteps.length === 0) { return { orderedTargets: [], selectedStep: null }; } @@ -754,7 +840,11 @@ function resolveWeightedTargets(combo, allCombos) { return { orderedTargets: [], selectedStep: null }; } - const orderedSteps = orderTargetsForWeightedFallback(topLevelSteps, selectedStep.executionKey); + const orderedSteps = orderTargetsForWeightedFallback( + topLevelSteps, + selectedStep.executionKey, + hasCompositeTierRuntimeOrder(combo) + ); const expandedTargets = orderedSteps.flatMap((step) => { if (!allCombos) { return step.kind === "model" ? [step] : []; diff --git a/tests/unit/combo-routing-engine.test.mjs b/tests/unit/combo-routing-engine.test.mjs index 1e7f02ba3c..7fb30fc927 100644 --- a/tests/unit/combo-routing-engine.test.mjs +++ b/tests/unit/combo-routing-engine.test.mjs @@ -223,6 +223,68 @@ test("handleComboChat priority strategy defaults to first model and records succ assert.equal(metrics.strategy, "priority"); }); +test("handleComboChat priority strategy honors composite tier order before fallback", async () => { + const calls = []; + const combo = { + name: "priority-composite-tiers", + strategy: "priority", + models: [ + { + kind: "model", + id: "step-primary", + providerId: "openai", + model: "openai/gpt-4o-mini", + }, + { + kind: "model", + id: "step-backup", + providerId: "anthropic", + model: "claude/sonnet", + }, + { + kind: "model", + id: "step-last", + providerId: "google", + model: "gemini/gemini-2.5-flash", + }, + ], + config: { + maxRetries: 0, + compositeTiers: { + defaultTier: "backup", + tiers: { + backup: { + stepId: "step-backup", + fallbackTier: "primary", + }, + primary: { + stepId: "step-primary", + }, + }, + }, + }, + }; + + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === "claude/sonnet") { + return errorResponse(503, "backup failed"); + } + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, ["claude/sonnet", "openai/gpt-4o-mini"]); +}); + test("handleComboChat weighted strategy selects by weight and falls back in descending weight order", async () => { const originalRandom = Math.random; const calls = []; @@ -533,6 +595,64 @@ test("handleComboChat round-robin rotates sequentially across requests", async ( assert.deepEqual(calls, ["model-a", "model-b", "model-a"]); }); +test("handleComboChat round-robin starts from composite tier default ordering", async () => { + const calls = []; + const combo = { + name: "rr-composite-order", + strategy: "round-robin", + models: [ + { + kind: "model", + id: "step-primary", + providerId: "openai", + model: "openai/gpt-4o-mini", + }, + { + kind: "model", + id: "step-backup", + providerId: "anthropic", + model: "claude/sonnet", + }, + ], + config: { + maxRetries: 0, + concurrencyPerModel: 1, + queueTimeoutMs: 1000, + compositeTiers: { + defaultTier: "backup", + tiers: { + backup: { + stepId: "step-backup", + fallbackTier: "primary", + }, + primary: { + stepId: "step-primary", + }, + }, + }, + }, + }; + + for (let i = 0; i < 2; i++) { + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + } + + assert.deepEqual(calls, ["claude/sonnet", "openai/gpt-4o-mini"]); +}); + test("combo helpers short-circuit safely for missing combos, cycles, and excessive depth", () => { assert.equal(getComboFromData("missing", null), null); assert.equal(getComboModelsFromData("missing", { combos: [] }), null);