From a43b1c921831961efb862a4bf070024b67ec9edc Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 15 Feb 2026 16:57:00 -0300 Subject: [PATCH] feat: add random, least-used, and cost-optimized combo strategies Extend combo routing with 3 new model selection strategies beyond the existing priority, weighted, and round-robin options: - random: Fisher-Yates shuffle for uniform distribution - least-used: sorts models by request count via comboMetrics - cost-optimized: sorts models by pricing (cheapest first) Also enhance the dashboard provider cards with model counts and clickable detail views for selected providers. --- open-sse/services/combo.js | 80 +++++++++- .../(dashboard)/dashboard/HomePageClient.js | 150 ++++++++++++++++-- src/app/(dashboard)/dashboard/combos/page.js | 88 +++++----- .../settings/components/ComboDefaultsTab.js | 24 ++- .../settings/components/RoutingTab.js | 5 +- src/domain/types.js | 2 +- src/shared/schemas/validation.js | 4 +- src/shared/validation/schemas.js | 5 +- 8 files changed, 282 insertions(+), 76 deletions(-) diff --git a/open-sse/services/combo.js b/open-sse/services/combo.js index 60b1770363..1adcb9f3b3 100644 --- a/open-sse/services/combo.js +++ b/open-sse/services/combo.js @@ -1,11 +1,11 @@ /** * Shared combo (model combo) handling with fallback support - * Supports: priority (sequential), weighted (probabilistic), and round-robin (circular) strategies + * Supports: priority, weighted, round-robin, random, least-used, and cost-optimized strategies */ import { checkFallbackError, formatRetryAfter, getProviderProfile } from "./accountFallback.js"; import { unavailableResponse } from "../utils/error.js"; -import { recordComboRequest } from "./comboMetrics.js"; +import { recordComboRequest, getComboMetrics } from "./comboMetrics.js"; import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.js"; import * as semaphore from "./rateLimitSemaphore.js"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker.js"; @@ -150,9 +150,69 @@ function orderModelsForWeightedFallback(models, selectedModel) { return [selected, ...rest].filter(Boolean).map((e) => e.model); } +/** + * Fisher-Yates shuffle (in-place) + * @param {Array} arr + * @returns {Array} The shuffled array + */ +function shuffleArray(arr) { + for (let i = arr.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [arr[i], arr[j]] = [arr[j], arr[i]]; + } + return arr; +} + +/** + * Sort models by pricing (cheapest first) for cost-optimized strategy + * @param {Array} models - Model strings in "provider/model" format + * @returns {Promise>} Sorted model strings + */ +async function sortModelsByCost(models) { + try { + const { getPricingForModel } = await import("../../src/lib/localDb.js"); + const withCost = await Promise.all( + models.map(async (modelStr) => { + const parsed = parseModel(modelStr); + const provider = parsed.provider || parsed.providerAlias || "unknown"; + const model = parsed.model || modelStr; + try { + const pricing = await getPricingForModel(provider, model); + return { modelStr, cost: pricing?.input ?? Infinity }; + } catch { + return { modelStr, cost: Infinity }; + } + }) + ); + withCost.sort((a, b) => a.cost - b.cost); + return withCost.map((e) => e.modelStr); + } catch { + // If pricing lookup fails entirely, return original order + return models; + } +} + +/** + * Sort models by usage count (least-used first) for least-used strategy + * @param {Array} models - Model strings + * @param {string} comboName - Combo name for metrics lookup + * @returns {Array} Sorted model strings + */ +function sortModelsByUsage(models, comboName) { + const metrics = getComboMetrics(comboName); + if (!metrics || !metrics.byModel) return models; + + const withUsage = models.map((modelStr) => ({ + modelStr, + requests: metrics.byModel[modelStr]?.requests ?? 0, + })); + withUsage.sort((a, b) => a.requests - b.requests); + return withUsage.map((e) => e.modelStr); +} + /** * Handle combo chat with fallback - * Supports priority (sequential) and weighted (probabilistic) strategies + * Supports all 6 strategies: priority, weighted, round-robin, random, least-used, cost-optimized * @param {Object} options * @param {Object} options.body - Request body * @param {Object} options.combo - Full combo object { name, models, strategy, config } @@ -215,7 +275,7 @@ export async function handleComboChat({ ); } else { orderedModels = flatModels; - log.info("COMBO", `Priority with nested resolution: ${orderedModels.length} total models`); + log.info("COMBO", `${strategy} with nested resolution: ${orderedModels.length} total models`); } } else if (strategy === "weighted") { const selected = selectWeightedModel(models); @@ -225,6 +285,18 @@ export async function handleComboChat({ orderedModels = models.map((m) => normalizeModelEntry(m).model); } + // Apply strategy-specific ordering + if (strategy === "random") { + orderedModels = shuffleArray([...orderedModels]); + log.info("COMBO", `Random shuffle: ${orderedModels.length} models`); + } else if (strategy === "least-used") { + orderedModels = sortModelsByUsage(orderedModels, combo.name); + log.info("COMBO", `Least-used ordering: ${orderedModels[0]} has fewest requests`); + } else if (strategy === "cost-optimized") { + orderedModels = await sortModelsByCost(orderedModels); + log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`); + } + let lastError = null; let earliestRetryAfter = null; let lastStatus = null; diff --git a/src/app/(dashboard)/dashboard/HomePageClient.js b/src/app/(dashboard)/dashboard/HomePageClient.js index 4348fc9fb6..384d9e2366 100644 --- a/src/app/(dashboard)/dashboard/HomePageClient.js +++ b/src/app/(dashboard)/dashboard/HomePageClient.js @@ -4,13 +4,16 @@ import { useState, useEffect, useMemo, useCallback } from "react"; import PropTypes from "prop-types"; import Image from "next/image"; import Link from "next/link"; -import { Card, CardSkeleton } from "@/shared/components"; +import { Card, CardSkeleton, Button, Modal } from "@/shared/components"; import { AI_PROVIDERS } from "@/shared/constants/providers"; +import { useNotificationStore } from "@/store/notificationStore"; export default function HomePageClient({ machineId }) { const [providerConnections, setProviderConnections] = useState([]); + const [models, setModels] = useState([]); const [loading, setLoading] = useState(true); const [baseUrl, setBaseUrl] = useState("/v1"); + const [selectedProvider, setSelectedProvider] = useState(null); useEffect(() => { if (typeof window !== "undefined") { @@ -20,10 +23,17 @@ export default function HomePageClient({ machineId }) { const fetchData = useCallback(async () => { try { - const [connRes] = await Promise.all([fetch("/api/connections")]); - if (connRes.ok) { - const connData = await connRes.json(); - setProviderConnections(connData); + const [provRes, modelsRes] = await Promise.all([ + fetch("/api/providers"), + fetch("/api/models"), + ]); + if (provRes.ok) { + const provData = await provRes.json(); + setProviderConnections(provData.connections || []); + } + if (modelsRes.ok) { + const modelsData = await modelsRes.json(); + setModels(modelsData.models || []); } } catch (e) { console.log("Error fetching data:", e); @@ -54,15 +64,24 @@ export default function HomePageClient({ machineId }) { conn.testStatus === "unavailable") ).length; + const providerModels = models.filter((m) => m.provider === providerId); + return { id: providerId, provider: providerInfo, total: connections.length, connected, errors, + modelCount: providerModels.length, }; }); - }, [providerConnections]); + }, [providerConnections, models]); + + // Models for selected provider + const selectedProviderModels = useMemo(() => { + if (!selectedProvider) return []; + return models.filter((m) => m.provider === selectedProvider.id); + }, [selectedProvider, models]); const quickStartLinks = [ { label: "Documentation", href: "/docs" }, @@ -162,10 +181,23 @@ export default function HomePageClient({ machineId }) {
{providerStats.map((item) => ( - + setSelectedProvider(item)} + /> ))}
+ + {/* Provider Models Modal */} + {selectedProvider && ( + setSelectedProvider(null)} + /> + )} ); } @@ -174,20 +206,20 @@ HomePageClient.propTypes = { machineId: PropTypes.string, }; -function ProviderOverviewCard({ item }) { +function ProviderOverviewCard({ item, onClick }) { const [imgError, setImgError] = useState(false); const statusVariant = item.errors > 0 ? "text-red-500" : item.connected > 0 ? "text-green-500" : "text-text-muted"; return ( -
{imgError ? ( @@ -219,9 +251,9 @@ function ProviderOverviewCard({ item }) {

- #{item.total} + #{item.modelCount}
- + ); } @@ -237,5 +269,95 @@ ProviderOverviewCard.propTypes = { total: PropTypes.number.isRequired, connected: PropTypes.number.isRequired, errors: PropTypes.number.isRequired, + modelCount: PropTypes.number.isRequired, }).isRequired, + onClick: PropTypes.func.isRequired, +}; + +function ProviderModelsModal({ provider, models, onClose }) { + const [copiedModel, setCopiedModel] = useState(null); + const notify = useNotificationStore(); + + const handleCopy = (text) => { + navigator.clipboard.writeText(text); + setCopiedModel(text); + notify.success(`Copied: ${text}`); + setTimeout(() => setCopiedModel(null), 2000); + }; + + return ( + +
+ {/* Summary */} +
+ token + {models.length} model{models.length !== 1 ? "s" : ""} available + {provider.total > 0 && ( + + ● {provider.connected} connection{provider.connected !== 1 ? "s" : ""} active + + )} +
+ + {models.length === 0 ? ( +
+ + search_off + +

No models available for this provider.

+

+ Configure a connection first in{" "} + + Providers + +

+
+ ) : ( +
+ {models.map((m) => ( +
+
+

{m.fullModel}

+ {m.alias !== m.model && ( +

alias: {m.alias}

+ )} +
+ +
+ ))} +
+ )} + + {/* Actions */} +
+ + + + +
+
+ + ); +} + +ProviderModelsModal.propTypes = { + provider: PropTypes.object.isRequired, + models: PropTypes.array.isRequired, + onClose: PropTypes.func.isRequired, }; diff --git a/src/app/(dashboard)/dashboard/combos/page.js b/src/app/(dashboard)/dashboard/combos/page.js index 634ef15543..6bb7dce5a9 100644 --- a/src/app/(dashboard)/dashboard/combos/page.js +++ b/src/app/(dashboard)/dashboard/combos/page.js @@ -308,7 +308,13 @@ function ComboCard({ ? "bg-amber-500/15 text-amber-600 dark:text-amber-400" : strategy === "round-robin" ? "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400" - : "bg-blue-500/15 text-blue-600 dark:text-blue-400" + : strategy === "random" + ? "bg-purple-500/15 text-purple-600 dark:text-purple-400" + : strategy === "least-used" + ? "bg-cyan-500/15 text-cyan-600 dark:text-cyan-400" + : strategy === "cost-optimized" + ? "bg-teal-500/15 text-teal-600 dark:text-teal-400" + : "bg-blue-500/15 text-blue-600 dark:text-blue-400" }`} > {strategy} @@ -704,53 +710,43 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { {/* Strategy Toggle */}
-
- - - +
+ {[ + { value: "priority", label: "Priority", icon: "sort" }, + { value: "weighted", label: "Weighted", icon: "percent" }, + { value: "round-robin", label: "Round-Robin", icon: "autorenew" }, + { value: "random", label: "Random", icon: "shuffle" }, + { value: "least-used", label: "Least-Used", icon: "low_priority" }, + { value: "cost-optimized", label: "Cost-Opt", icon: "savings" }, + ].map((s) => ( + + ))}

- {strategy === "priority" - ? "Sequential fallback: tries model 1 first, then 2, etc." - : strategy === "weighted" - ? "Distributes traffic by weight percentage with fallback" - : "Circular distribution: each request goes to the next model in rotation"} + { + { + priority: "Sequential fallback: tries model 1 first, then 2, etc.", + weighted: "Distributes traffic by weight percentage with fallback", + "round-robin": + "Circular distribution: each request goes to the next model in rotation", + random: "Uniform random selection, then fallback to remaining models", + "least-used": "Picks the model with fewest requests, balancing load over time", + "cost-optimized": "Routes to the cheapest model first based on pricing", + }[strategy] + }

diff --git a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js index 1f33188b9b..ccf3ee1d7e 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js +++ b/src/app/(dashboard)/dashboard/settings/components/ComboDefaultsTab.js @@ -82,22 +82,30 @@ export default function ComboDefaultsTab() {
- {["priority", "weighted", "round-robin"].map((s) => ( + {[ + { value: "priority", label: "Priority", icon: "sort" }, + { value: "weighted", label: "Weighted", icon: "percent" }, + { value: "round-robin", label: "Round-Robin", icon: "autorenew" }, + { value: "random", label: "Random", icon: "shuffle" }, + { value: "least-used", label: "Least-Used", icon: "low_priority" }, + { value: "cost-optimized", label: "Cost-Opt", icon: "savings" }, + ].map((s) => ( ))}
diff --git a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js index c6dd851ea2..23634d5774 100644 --- a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js +++ b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.js @@ -13,6 +13,7 @@ const STRATEGIES = [ }, { value: "round-robin", label: "Round Robin", desc: "Cycle through all accounts", icon: "loop" }, { value: "p2c", label: "P2C", desc: "Pick 2 random, use the healthier one", icon: "balance" }, + { value: "random", label: "Random", desc: "Pick a random account each request", icon: "shuffle" }, ]; export default function RoutingTab() { @@ -76,7 +77,7 @@ export default function RoutingTab() {

Routing Strategy

-
+
{STRATEGIES.map((s) => (