From 25815ae53de2ad71c6a04ae6911557ca06f0c41e Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 12 Apr 2026 11:02:13 -0300 Subject: [PATCH] fix(combo): honor composite tier order in runtime routing Apply composite tier ordering to top-level combo steps and direct targets so priority and round-robin strategies follow the configured defaultTier to fallbackTier chain before using remaining steps. Add defensive config parsing helpers, regression tests for composite tier ordering, and documentation updates for the structured combo builder, quota-aware P2C, and combo target health. --- README.md | 20 ++-- docs/ARCHITECTURE.md | 11 ++- docs/FEATURES.md | 9 +- open-sse/services/combo.ts | 114 ++++++++++++++++++--- tests/unit/combo-routing-engine.test.mjs | 120 +++++++++++++++++++++++ 5 files changed, 248 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index cc34c6e438..a6d9876fa1 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,8 @@ Developers pay $20–200/month for Claude Pro, Codex Pro, or GitHub Copilot. Eve - **Provider Limits Tracking** — Cached quota snapshots refresh on a server-side schedule (default `PROVIDER_LIMITS_SYNC_INTERVAL_MINUTES=70`) with manual refresh available in the UI - **Multi-Account Support** — Multiple accounts per provider with auto round-robin — when one runs out, switches to the next - **Custom Combos** — Customizable fallback chains with 13 balancing strategies (priority, weighted, fill-first, round-robin, P2C, random, least-used, cost-optimized, strict-random, auto, lkgp, context-optimized, **context-relay**) +- **Structured Combo Builder** — Build combos step-by-step with explicit provider + model + account selection, including repeated providers and fixed-account targets +- **Quota-Aware P2C** — Power-of-two account selection now factors quota headroom, backoff, recent errors, and consecutive use - **Codex Business Quotas** — Business/Team workspace quota monitoring directly in the dashboard @@ -2231,16 +2233,16 @@ Se não quiser criar credenciais próprias agora, ainda é possível usar o flux ## 🗺️ Roadmap -OmniRoute has **210+ features planned** across multiple development phases. Here are the key areas: +OmniRoute has **218+ features planned** across multiple development phases. Here are the key areas: -| Category | Planned Features | Highlights | -| ----------------------------- | ---------------- | -------------------------------------------------------------------------------------- | -| 🧠 **Routing & Intelligence** | 25+ | Lowest-latency routing, tag-based routing, quota preflight, P2C account selection | -| 🔒 **Security & Compliance** | 20+ | SSRF hardening, credential cloaking, rate-limit per endpoint, management key scoping | -| 📊 **Observability** | 15+ | OpenTelemetry integration, real-time quota monitoring, cost tracking per model | -| 🔄 **Provider Integrations** | 20+ | Dynamic model registry, provider cooldowns, multi-account Codex, Copilot quota parsing | -| ⚡ **Performance** | 15+ | Dual cache layer, prompt cache, response cache, streaming keepalive, batch API | -| 🌐 **Ecosystem** | 10+ | WebSocket API, config hot-reload, distributed config store, commercial mode | +| Category | Planned Features | Highlights | +| ----------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------- | +| 🧠 **Routing & Intelligence** | 25+ | Lowest-latency routing, tag-based routing, quota preflight, quota-aware P2C, step-based combo routing | +| 🔒 **Security & Compliance** | 20+ | SSRF hardening, credential cloaking, rate-limit per endpoint, management key scoping | +| 📊 **Observability** | 15+ | OpenTelemetry integration, real-time quota monitoring, combo target health, cost tracking per model | +| 🔄 **Provider Integrations** | 20+ | Dynamic model registry, provider cooldowns, multi-account Codex, Copilot quota parsing | +| ⚡ **Performance** | 15+ | Dual cache layer, prompt cache, response cache, streaming keepalive, batch API | +| 🌐 **Ecosystem** | 10+ | WebSocket API, config hot-reload, distributed config store, commercial mode | ### 🔜 Coming Soon diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9f9c481999..90dee6ce94 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ 🌐 **Languages:** 🇺🇸 [English](ARCHITECTURE.md) | 🇧🇷 [Português (Brasil)](i18n/pt-BR/ARCHITECTURE.md) | 🇪🇸 [Español](i18n/es/ARCHITECTURE.md) | 🇫🇷 [Français](i18n/fr/ARCHITECTURE.md) | 🇮🇹 [Italiano](i18n/it/ARCHITECTURE.md) | 🇷🇺 [Русский](i18n/ru/ARCHITECTURE.md) | 🇨🇳 [中文 (简体)](i18n/zh-CN/ARCHITECTURE.md) | 🇩🇪 [Deutsch](i18n/de/ARCHITECTURE.md) | 🇮🇳 [हिन्दी](i18n/in/ARCHITECTURE.md) | 🇹🇭 [ไทย](i18n/th/ARCHITECTURE.md) | 🇺🇦 [Українська](i18n/uk-UA/ARCHITECTURE.md) | 🇸🇦 [العربية](i18n/ar/ARCHITECTURE.md) | 🇯🇵 [日本語](i18n/ja/ARCHITECTURE.md) | 🇻🇳 [Tiếng Việt](i18n/vi/ARCHITECTURE.md) | 🇧🇬 [Български](i18n/bg/ARCHITECTURE.md) | 🇩🇰 [Dansk](i18n/da/ARCHITECTURE.md) | 🇫🇮 [Suomi](i18n/fi/ARCHITECTURE.md) | 🇮🇱 [עברית](i18n/he/ARCHITECTURE.md) | 🇭🇺 [Magyar](i18n/hu/ARCHITECTURE.md) | 🇮🇩 [Bahasa Indonesia](i18n/id/ARCHITECTURE.md) | 🇰🇷 [한국어](i18n/ko/ARCHITECTURE.md) | 🇲🇾 [Bahasa Melayu](i18n/ms/ARCHITECTURE.md) | 🇳🇱 [Nederlands](i18n/nl/ARCHITECTURE.md) | 🇳🇴 [Norsk](i18n/no/ARCHITECTURE.md) | 🇵🇹 [Português (Portugal)](i18n/pt/ARCHITECTURE.md) | 🇷🇴 [Română](i18n/ro/ARCHITECTURE.md) | 🇵🇱 [Polski](i18n/pl/ARCHITECTURE.md) | 🇸🇰 [Slovenčina](i18n/sk/ARCHITECTURE.md) | 🇸🇪 [Svenska](i18n/sv/ARCHITECTURE.md) | 🇵🇭 [Filipino](i18n/phi/ARCHITECTURE.md) | 🇨🇿 [Čeština](i18n/cs/ARCHITECTURE.md) -_Last updated: 2026-04-11_ +_Last updated: 2026-04-12_ ## Executive Summary @@ -14,7 +14,9 @@ Core capabilities: - OpenAI-compatible API surface for CLI/tools (100+ providers, 16 executors) - Request/response translation across provider formats - Model combo fallback (multi-model sequence) +- Structured combo steps (`provider + model + connection`) with runtime ordering by `compositeTiers` - Account-level fallback (multi-account per provider) +- Quota preflight and quota-aware P2C account selection in the main chat path - OAuth + API-key provider connection management (13 OAuth modules) - Embedding generation via `/v1/embeddings` (6 providers, 9 models) - Image generation via `/v1/images/generations` (10+ providers, 20+ models) @@ -45,6 +47,7 @@ Core capabilities: - Domain state persistence (SQLite write-through cache for fallbacks, budgets, lockouts, circuit breakers) - Policy engine for centralized request evaluation (lockout → budget → fallback) - Request telemetry with p50/p95/p99 latency aggregation +- Combo target telemetry and historical combo target health via `combo_execution_key` / `combo_step_id` - Correlation ID (X-Request-Id) for end-to-end tracing - Compliance audit logging with opt-out per API key - Eval framework for LLM quality assurance @@ -89,15 +92,15 @@ Main pages under `src/app/(dashboard)/dashboard/`: - `/dashboard` — quick start + provider overview - `/dashboard/endpoint` — endpoint proxy + MCP + A2A + API endpoint tabs - `/dashboard/providers` — provider connections and credentials -- `/dashboard/combos` — combo strategies, templates, model routing rules, manual persisted ordering +- `/dashboard/combos` — combo strategies, templates, step-based builder, model routing rules, manual persisted ordering - `/dashboard/costs` — cost aggregation and pricing visibility -- `/dashboard/analytics` — usage analytics and evaluations +- `/dashboard/analytics` — usage analytics, evaluations, combo target health - `/dashboard/limits` — quota/rate controls - `/dashboard/cli-tools` — CLI onboarding, runtime detection, config generation - `/dashboard/agents` — detected ACP agents + custom agent registration - `/dashboard/media` — image/video/music playground - `/dashboard/search-tools` — search provider testing and history -- `/dashboard/health` — uptime, circuit breakers, rate limits +- `/dashboard/health` — uptime, circuit breakers, rate limits, quota-monitored sessions - `/dashboard/logs` — request/proxy/audit/console logs - `/dashboard/settings` — system settings tabs (general, routing, combo defaults, etc.) - `/dashboard/api-manager` — API key lifecycle and model permissions diff --git a/docs/FEATURES.md b/docs/FEATURES.md index a6d4b4b3b7..c1e44fad5c 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -18,6 +18,13 @@ Manage AI provider connections: OAuth providers (Claude Code, Codex, Gemini CLI) Create model routing combos with 13 strategies: priority, weighted, round-robin, random, least-used, cost-optimized, strict-random, auto, fill-first, p2c, lkgp, context-optimized, and **context-relay**. Each combo chains multiple models with automatic fallback and includes quick templates and readiness checks. +Recent combo improvements: + +- **Structured combo builder** — create each step by selecting provider, model, and exact account/connection +- **Repeated provider support** — reuse the same provider many times in one combo as long as the `(provider, model, connection)` tuple is unique +- **Combo target health** — analytics and health surfaces now distinguish individual combo targets/steps instead of collapsing everything into model strings +- **Composite tier ordering** — `defaultTier -> fallbackTier` now influences runtime execution/fallback order for top-level combo steps + ![Combos Dashboard](screenshots/02-combos.png) --- @@ -32,7 +39,7 @@ Comprehensive usage analytics with token consumption, cost estimates, activity h ## 🏥 System Health -Real-time monitoring: uptime, memory, version, latency percentiles (p50/p95/p99), cache statistics, and provider circuit breaker states. +Real-time monitoring: uptime, memory, version, latency percentiles (p50/p95/p99), cache statistics, provider circuit breaker states, active quota-monitored sessions, and combo target health. ![Health Dashboard](screenshots/04-health.png) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 7dfd82b2cf..02ea42418f 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -81,6 +81,14 @@ type ComboRuntimeStep = label: string | null; }; +function isRecord(value): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function toTrimmedString(value): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + /** * Validate that a successful (HTTP 200) non-streaming response actually contains * meaningful content. Returns { valid: true } or { valid: false, reason }. @@ -240,9 +248,9 @@ function normalizeRuntimeStep(entry, comboName, index, allCombos, path = []) { } function getDirectComboTargets(combo) { - return (combo.models || []) - .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null)) - .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); + return getOrderedTopLevelRuntimeSteps(combo, null).filter( + (entry): entry is ResolvedComboTarget => entry?.kind === "model" + ); } function getTopLevelRuntimeSteps(combo, allCombos, path = []) { @@ -251,6 +259,84 @@ function getTopLevelRuntimeSteps(combo, allCombos, path = []) { .filter((entry): entry is ComboRuntimeStep => entry !== null); } +function getCompositeTierStepOrder(combo): string[] { + const compositeTiers = isRecord(combo?.config) ? combo.config.compositeTiers : null; + if (!isRecord(compositeTiers)) return []; + + const defaultTier = toTrimmedString(compositeTiers.defaultTier); + const tiers = isRecord(compositeTiers.tiers) ? compositeTiers.tiers : null; + if (!defaultTier || !tiers) return []; + + const orderedStepIds: string[] = []; + const visitedTiers = new Set(); + const seenStepIds = new Set(); + const tierEntries = new Map( + Object.entries(tiers) + .map(([tierName, rawTier]) => { + if (!isRecord(rawTier)) return null; + const normalizedTierName = toTrimmedString(tierName); + const stepId = toTrimmedString(rawTier.stepId); + const fallbackTier = toTrimmedString(rawTier.fallbackTier); + if (!normalizedTierName || !stepId) return null; + return [normalizedTierName, { stepId, fallbackTier }] as const; + }) + .filter(Boolean) + ); + + let currentTier = defaultTier; + while (currentTier && tierEntries.has(currentTier) && !visitedTiers.has(currentTier)) { + visitedTiers.add(currentTier); + const entry = tierEntries.get(currentTier); + if (!entry) break; + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + currentTier = entry.fallbackTier; + } + + for (const entry of tierEntries.values()) { + if (!seenStepIds.has(entry.stepId)) { + orderedStepIds.push(entry.stepId); + seenStepIds.add(entry.stepId); + } + } + + return orderedStepIds; +} + +function hasCompositeTierRuntimeOrder(combo): boolean { + return getCompositeTierStepOrder(combo).length > 0; +} + +function orderRuntimeStepsByCompositeTiers(steps: ComboRuntimeStep[], combo): ComboRuntimeStep[] { + const orderedStepIds = getCompositeTierStepOrder(combo); + if (orderedStepIds.length === 0) return steps; + + const byStepId = new Map(steps.map((step) => [step.stepId, step])); + const seen = new Set(); + const ordered: ComboRuntimeStep[] = []; + + for (const stepId of orderedStepIds) { + const step = byStepId.get(stepId); + if (!step || seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + for (const step of steps) { + if (seen.has(step.stepId)) continue; + ordered.push(step); + seen.add(step.stepId); + } + + return ordered; +} + +function getOrderedTopLevelRuntimeSteps(combo, allCombos, path = []) { + return orderRuntimeStepsByCompositeTiers(getTopLevelRuntimeSteps(combo, allCombos, path), combo); +} + function expandRuntimeStep(step, allCombos, visited = new Set(), depth = 0, path = []) { if (step.kind === "model") return [step]; if (depth > MAX_COMBO_DEPTH) return []; @@ -280,7 +366,7 @@ export function resolveNestedComboTargets( if (visited.has(combo.name)) return []; visited.add(combo.name); - const runtimeSteps = getTopLevelRuntimeSteps(combo, allCombos, path); + const runtimeSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos, path); const resolved: ResolvedComboTarget[] = []; for (const step of runtimeSteps) { @@ -401,12 +487,14 @@ function selectWeightedTarget(targets: Array<{ weight: number }>) { function orderTargetsForWeightedFallback( targets: Array<{ executionKey: string; weight: number }>, - selectedExecutionKey: string + selectedExecutionKey: string, + preserveExistingOrder = false ) { const selected = targets.find((target) => target.executionKey === selectedExecutionKey); - const rest = targets - .filter((target) => target.executionKey !== selectedExecutionKey) - .sort((a, b) => b.weight - a.weight); + const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey); + if (!preserveExistingOrder) { + rest.sort((a, b) => b.weight - a.weight); + } return [selected, ...rest].filter(Boolean); } @@ -742,9 +830,7 @@ export function resolveComboTargets(combo, allCombos) { } function resolveWeightedTargets(combo, allCombos) { - const topLevelSteps = allCombos - ? getTopLevelRuntimeSteps(combo, allCombos) - : getDirectComboTargets(combo); + const topLevelSteps = getOrderedTopLevelRuntimeSteps(combo, allCombos); if (topLevelSteps.length === 0) { return { orderedTargets: [], selectedStep: null }; } @@ -754,7 +840,11 @@ function resolveWeightedTargets(combo, allCombos) { return { orderedTargets: [], selectedStep: null }; } - const orderedSteps = orderTargetsForWeightedFallback(topLevelSteps, selectedStep.executionKey); + const orderedSteps = orderTargetsForWeightedFallback( + topLevelSteps, + selectedStep.executionKey, + hasCompositeTierRuntimeOrder(combo) + ); const expandedTargets = orderedSteps.flatMap((step) => { if (!allCombos) { return step.kind === "model" ? [step] : []; diff --git a/tests/unit/combo-routing-engine.test.mjs b/tests/unit/combo-routing-engine.test.mjs index 1e7f02ba3c..7fb30fc927 100644 --- a/tests/unit/combo-routing-engine.test.mjs +++ b/tests/unit/combo-routing-engine.test.mjs @@ -223,6 +223,68 @@ test("handleComboChat priority strategy defaults to first model and records succ assert.equal(metrics.strategy, "priority"); }); +test("handleComboChat priority strategy honors composite tier order before fallback", async () => { + const calls = []; + const combo = { + name: "priority-composite-tiers", + strategy: "priority", + models: [ + { + kind: "model", + id: "step-primary", + providerId: "openai", + model: "openai/gpt-4o-mini", + }, + { + kind: "model", + id: "step-backup", + providerId: "anthropic", + model: "claude/sonnet", + }, + { + kind: "model", + id: "step-last", + providerId: "google", + model: "gemini/gemini-2.5-flash", + }, + ], + config: { + maxRetries: 0, + compositeTiers: { + defaultTier: "backup", + tiers: { + backup: { + stepId: "step-backup", + fallbackTier: "primary", + }, + primary: { + stepId: "step-primary", + }, + }, + }, + }, + }; + + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + if (modelStr === "claude/sonnet") { + return errorResponse(503, "backup failed"); + } + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + assert.deepEqual(calls, ["claude/sonnet", "openai/gpt-4o-mini"]); +}); + test("handleComboChat weighted strategy selects by weight and falls back in descending weight order", async () => { const originalRandom = Math.random; const calls = []; @@ -533,6 +595,64 @@ test("handleComboChat round-robin rotates sequentially across requests", async ( assert.deepEqual(calls, ["model-a", "model-b", "model-a"]); }); +test("handleComboChat round-robin starts from composite tier default ordering", async () => { + const calls = []; + const combo = { + name: "rr-composite-order", + strategy: "round-robin", + models: [ + { + kind: "model", + id: "step-primary", + providerId: "openai", + model: "openai/gpt-4o-mini", + }, + { + kind: "model", + id: "step-backup", + providerId: "anthropic", + model: "claude/sonnet", + }, + ], + config: { + maxRetries: 0, + concurrencyPerModel: 1, + queueTimeoutMs: 1000, + compositeTiers: { + defaultTier: "backup", + tiers: { + backup: { + stepId: "step-backup", + fallbackTier: "primary", + }, + primary: { + stepId: "step-primary", + }, + }, + }, + }, + }; + + for (let i = 0; i < 2; i++) { + const result = await handleComboChat({ + body: {}, + combo, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + }); + + assert.equal(result.ok, true); + } + + assert.deepEqual(calls, ["claude/sonnet", "openai/gpt-4o-mini"]); +}); + test("combo helpers short-circuit safely for missing combos, cycles, and excessive depth", () => { assert.equal(getComboFromData("missing", null), null); assert.equal(getComboModelsFromData("missing", { combos: [] }), null);