feat: add random, least-used, and cost-optimized combo strategies

Extend combo routing with 3 new model selection strategies beyond the
existing priority, weighted, and round-robin options:
- random: Fisher-Yates shuffle for uniform distribution
- least-used: sorts models by request count via comboMetrics
- cost-optimized: sorts models by pricing (cheapest first)

Also enhance the dashboard provider cards with model counts and
clickable detail views for selected providers.
This commit is contained in:
diegosouzapw
2026-02-15 16:57:00 -03:00
parent 413a9f2a69
commit a43b1c9218
8 changed files with 282 additions and 76 deletions

View File

@@ -1,11 +1,11 @@
/**
* Shared combo (model combo) handling with fallback support
* Supports: priority (sequential), weighted (probabilistic), and round-robin (circular) strategies
* Supports: priority, weighted, round-robin, random, least-used, and cost-optimized strategies
*/
import { checkFallbackError, formatRetryAfter, getProviderProfile } from "./accountFallback.js";
import { unavailableResponse } from "../utils/error.js";
import { recordComboRequest } from "./comboMetrics.js";
import { recordComboRequest, getComboMetrics } from "./comboMetrics.js";
import { resolveComboConfig, getDefaultComboConfig } from "./comboConfig.js";
import * as semaphore from "./rateLimitSemaphore.js";
import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker.js";
@@ -150,9 +150,69 @@ function orderModelsForWeightedFallback(models, selectedModel) {
return [selected, ...rest].filter(Boolean).map((e) => e.model);
}
/**
* Fisher-Yates shuffle (in-place)
* @param {Array} arr
* @returns {Array} The shuffled array
*/
function shuffleArray(arr) {
for (let i = arr.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr;
}
/**
* Sort models by pricing (cheapest first) for cost-optimized strategy
* @param {Array<string>} models - Model strings in "provider/model" format
* @returns {Promise<Array<string>>} Sorted model strings
*/
async function sortModelsByCost(models) {
try {
const { getPricingForModel } = await import("../../src/lib/localDb.js");
const withCost = await Promise.all(
models.map(async (modelStr) => {
const parsed = parseModel(modelStr);
const provider = parsed.provider || parsed.providerAlias || "unknown";
const model = parsed.model || modelStr;
try {
const pricing = await getPricingForModel(provider, model);
return { modelStr, cost: pricing?.input ?? Infinity };
} catch {
return { modelStr, cost: Infinity };
}
})
);
withCost.sort((a, b) => a.cost - b.cost);
return withCost.map((e) => e.modelStr);
} catch {
// If pricing lookup fails entirely, return original order
return models;
}
}
/**
* Sort models by usage count (least-used first) for least-used strategy
* @param {Array<string>} models - Model strings
* @param {string} comboName - Combo name for metrics lookup
* @returns {Array<string>} Sorted model strings
*/
function sortModelsByUsage(models, comboName) {
const metrics = getComboMetrics(comboName);
if (!metrics || !metrics.byModel) return models;
const withUsage = models.map((modelStr) => ({
modelStr,
requests: metrics.byModel[modelStr]?.requests ?? 0,
}));
withUsage.sort((a, b) => a.requests - b.requests);
return withUsage.map((e) => e.modelStr);
}
/**
* Handle combo chat with fallback
* Supports priority (sequential) and weighted (probabilistic) strategies
* Supports all 6 strategies: priority, weighted, round-robin, random, least-used, cost-optimized
* @param {Object} options
* @param {Object} options.body - Request body
* @param {Object} options.combo - Full combo object { name, models, strategy, config }
@@ -215,7 +275,7 @@ export async function handleComboChat({
);
} else {
orderedModels = flatModels;
log.info("COMBO", `Priority with nested resolution: ${orderedModels.length} total models`);
log.info("COMBO", `${strategy} with nested resolution: ${orderedModels.length} total models`);
}
} else if (strategy === "weighted") {
const selected = selectWeightedModel(models);
@@ -225,6 +285,18 @@ export async function handleComboChat({
orderedModels = models.map((m) => normalizeModelEntry(m).model);
}
// Apply strategy-specific ordering
if (strategy === "random") {
orderedModels = shuffleArray([...orderedModels]);
log.info("COMBO", `Random shuffle: ${orderedModels.length} models`);
} else if (strategy === "least-used") {
orderedModels = sortModelsByUsage(orderedModels, combo.name);
log.info("COMBO", `Least-used ordering: ${orderedModels[0]} has fewest requests`);
} else if (strategy === "cost-optimized") {
orderedModels = await sortModelsByCost(orderedModels);
log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`);
}
let lastError = null;
let earliestRetryAfter = null;
let lastStatus = null;

View File

@@ -4,13 +4,16 @@ import { useState, useEffect, useMemo, useCallback } from "react";
import PropTypes from "prop-types";
import Image from "next/image";
import Link from "next/link";
import { Card, CardSkeleton } from "@/shared/components";
import { Card, CardSkeleton, Button, Modal } from "@/shared/components";
import { AI_PROVIDERS } from "@/shared/constants/providers";
import { useNotificationStore } from "@/store/notificationStore";
export default function HomePageClient({ machineId }) {
const [providerConnections, setProviderConnections] = useState([]);
const [models, setModels] = useState([]);
const [loading, setLoading] = useState(true);
const [baseUrl, setBaseUrl] = useState("/v1");
const [selectedProvider, setSelectedProvider] = useState(null);
useEffect(() => {
if (typeof window !== "undefined") {
@@ -20,10 +23,17 @@ export default function HomePageClient({ machineId }) {
const fetchData = useCallback(async () => {
try {
const [connRes] = await Promise.all([fetch("/api/connections")]);
if (connRes.ok) {
const connData = await connRes.json();
setProviderConnections(connData);
const [provRes, modelsRes] = await Promise.all([
fetch("/api/providers"),
fetch("/api/models"),
]);
if (provRes.ok) {
const provData = await provRes.json();
setProviderConnections(provData.connections || []);
}
if (modelsRes.ok) {
const modelsData = await modelsRes.json();
setModels(modelsData.models || []);
}
} catch (e) {
console.log("Error fetching data:", e);
@@ -54,15 +64,24 @@ export default function HomePageClient({ machineId }) {
conn.testStatus === "unavailable")
).length;
const providerModels = models.filter((m) => m.provider === providerId);
return {
id: providerId,
provider: providerInfo,
total: connections.length,
connected,
errors,
modelCount: providerModels.length,
};
});
}, [providerConnections]);
}, [providerConnections, models]);
// Models for selected provider
const selectedProviderModels = useMemo(() => {
if (!selectedProvider) return [];
return models.filter((m) => m.provider === selectedProvider.id);
}, [selectedProvider, models]);
const quickStartLinks = [
{ label: "Documentation", href: "/docs" },
@@ -162,10 +181,23 @@ export default function HomePageClient({ machineId }) {
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-3">
{providerStats.map((item) => (
<ProviderOverviewCard key={item.id} item={item} />
<ProviderOverviewCard
key={item.id}
item={item}
onClick={() => setSelectedProvider(item)}
/>
))}
</div>
</Card>
{/* Provider Models Modal */}
{selectedProvider && (
<ProviderModelsModal
provider={selectedProvider}
models={selectedProviderModels}
onClose={() => setSelectedProvider(null)}
/>
)}
</div>
);
}
@@ -174,20 +206,20 @@ HomePageClient.propTypes = {
machineId: PropTypes.string,
};
function ProviderOverviewCard({ item }) {
function ProviderOverviewCard({ item, onClick }) {
const [imgError, setImgError] = useState(false);
const statusVariant =
item.errors > 0 ? "text-red-500" : item.connected > 0 ? "text-green-500" : "text-text-muted";
return (
<Link
href={`/dashboard/providers`}
className="border border-border rounded-lg p-3 hover:bg-surface/40 transition-colors"
<button
onClick={onClick}
className="border border-border rounded-lg p-3 hover:bg-surface/40 transition-colors text-left cursor-pointer w-full"
>
<div className="flex items-center gap-2.5">
<div
className="size-8 rounded-lg flex items-center justify-center"
className="size-8 rounded-lg flex items-center justify-center shrink-0"
style={{ backgroundColor: `${item.provider.color || "#888"}15` }}
>
{imgError ? (
@@ -219,9 +251,9 @@ function ProviderOverviewCard({ item }) {
</p>
</div>
<span className="text-xs text-text-muted">#{item.total}</span>
<span className="text-xs text-text-muted">#{item.modelCount}</span>
</div>
</Link>
</button>
);
}
@@ -237,5 +269,95 @@ ProviderOverviewCard.propTypes = {
total: PropTypes.number.isRequired,
connected: PropTypes.number.isRequired,
errors: PropTypes.number.isRequired,
modelCount: PropTypes.number.isRequired,
}).isRequired,
onClick: PropTypes.func.isRequired,
};
function ProviderModelsModal({ provider, models, onClose }) {
const [copiedModel, setCopiedModel] = useState(null);
const notify = useNotificationStore();
const handleCopy = (text) => {
navigator.clipboard.writeText(text);
setCopiedModel(text);
notify.success(`Copied: ${text}`);
setTimeout(() => setCopiedModel(null), 2000);
};
return (
<Modal isOpen={true} title={`${provider.provider.name} — Models`} onClose={onClose}>
<div className="flex flex-col gap-3">
{/* Summary */}
<div className="flex items-center gap-2 text-sm text-text-muted">
<span className="material-symbols-outlined text-[16px]">token</span>
{models.length} model{models.length !== 1 ? "s" : ""} available
{provider.total > 0 && (
<span className="ml-auto text-xs text-green-500">
{provider.connected} connection{provider.connected !== 1 ? "s" : ""} active
</span>
)}
</div>
{models.length === 0 ? (
<div className="text-center py-6">
<span className="material-symbols-outlined text-[32px] text-text-muted mb-2">
search_off
</span>
<p className="text-sm text-text-muted">No models available for this provider.</p>
<p className="text-xs text-text-muted mt-1">
Configure a connection first in{" "}
<Link href="/dashboard/providers" className="text-primary hover:underline">
Providers
</Link>
</p>
</div>
) : (
<div className="flex flex-col gap-1 max-h-[400px] overflow-y-auto">
{models.map((m) => (
<div
key={m.fullModel}
className="flex items-center justify-between px-3 py-2 rounded-lg hover:bg-surface/50 transition-colors group"
>
<div className="min-w-0 flex-1">
<p className="font-mono text-sm text-text-main truncate">{m.fullModel}</p>
{m.alias !== m.model && (
<p className="text-[10px] text-text-muted">alias: {m.alias}</p>
)}
</div>
<button
onClick={() => handleCopy(m.fullModel)}
className="shrink-0 ml-2 p-1.5 rounded-lg text-text-muted hover:text-text-main hover:bg-bg-subtle transition-colors opacity-0 group-hover:opacity-100"
title="Copy model name"
>
<span className="material-symbols-outlined text-[14px]">
{copiedModel === m.fullModel ? "check" : "content_copy"}
</span>
</button>
</div>
))}
</div>
)}
{/* Actions */}
<div className="flex gap-2 pt-2 border-t border-border">
<Link href={`/dashboard/providers/${provider.id}`} className="flex-1">
<Button variant="secondary" fullWidth size="sm">
<span className="material-symbols-outlined text-[14px] mr-1">settings</span>
Configure Provider
</Button>
</Link>
<Button variant="ghost" size="sm" onClick={onClose}>
Close
</Button>
</div>
</div>
</Modal>
);
}
ProviderModelsModal.propTypes = {
provider: PropTypes.object.isRequired,
models: PropTypes.array.isRequired,
onClose: PropTypes.func.isRequired,
};

View File

@@ -308,7 +308,13 @@ function ComboCard({
? "bg-amber-500/15 text-amber-600 dark:text-amber-400"
: strategy === "round-robin"
? "bg-emerald-500/15 text-emerald-600 dark:text-emerald-400"
: "bg-blue-500/15 text-blue-600 dark:text-blue-400"
: strategy === "random"
? "bg-purple-500/15 text-purple-600 dark:text-purple-400"
: strategy === "least-used"
? "bg-cyan-500/15 text-cyan-600 dark:text-cyan-400"
: strategy === "cost-optimized"
? "bg-teal-500/15 text-teal-600 dark:text-teal-400"
: "bg-blue-500/15 text-blue-600 dark:text-blue-400"
}`}
>
{strategy}
@@ -704,53 +710,43 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) {
{/* Strategy Toggle */}
<div>
<label className="text-sm font-medium mb-1.5 block">Routing Strategy</label>
<div className="flex gap-1 p-0.5 bg-black/5 dark:bg-white/5 rounded-lg">
<button
onClick={() => setStrategy("priority")}
className={`flex-1 py-1.5 px-3 rounded-md text-xs font-medium transition-all ${
strategy === "priority"
? "bg-white dark:bg-bg-main shadow-sm text-primary"
: "text-text-muted hover:text-text-main"
}`}
>
<span className="material-symbols-outlined text-[14px] align-middle mr-1">
sort
</span>
Priority
</button>
<button
onClick={() => setStrategy("weighted")}
className={`flex-1 py-1.5 px-3 rounded-md text-xs font-medium transition-all ${
strategy === "weighted"
? "bg-white dark:bg-bg-main shadow-sm text-primary"
: "text-text-muted hover:text-text-main"
}`}
>
<span className="material-symbols-outlined text-[14px] align-middle mr-1">
percent
</span>
Weighted
</button>
<button
onClick={() => setStrategy("round-robin")}
className={`flex-1 py-1.5 px-3 rounded-md text-xs font-medium transition-all ${
strategy === "round-robin"
? "bg-white dark:bg-bg-main shadow-sm text-primary"
: "text-text-muted hover:text-text-main"
}`}
>
<span className="material-symbols-outlined text-[14px] align-middle mr-1">
autorenew
</span>
Round-Robin
</button>
<div className="grid grid-cols-3 gap-1 p-0.5 bg-black/5 dark:bg-white/5 rounded-lg">
{[
{ value: "priority", label: "Priority", icon: "sort" },
{ value: "weighted", label: "Weighted", icon: "percent" },
{ value: "round-robin", label: "Round-Robin", icon: "autorenew" },
{ value: "random", label: "Random", icon: "shuffle" },
{ value: "least-used", label: "Least-Used", icon: "low_priority" },
{ value: "cost-optimized", label: "Cost-Opt", icon: "savings" },
].map((s) => (
<button
key={s.value}
onClick={() => setStrategy(s.value)}
className={`py-1.5 px-2 rounded-md text-xs font-medium transition-all ${
strategy === s.value
? "bg-white dark:bg-bg-main shadow-sm text-primary"
: "text-text-muted hover:text-text-main"
}`}
>
<span className="material-symbols-outlined text-[14px] align-middle mr-0.5">
{s.icon}
</span>
{s.label}
</button>
))}
</div>
<p className="text-[10px] text-text-muted mt-0.5">
{strategy === "priority"
? "Sequential fallback: tries model 1 first, then 2, etc."
: strategy === "weighted"
? "Distributes traffic by weight percentage with fallback"
: "Circular distribution: each request goes to the next model in rotation"}
{
{
priority: "Sequential fallback: tries model 1 first, then 2, etc.",
weighted: "Distributes traffic by weight percentage with fallback",
"round-robin":
"Circular distribution: each request goes to the next model in rotation",
random: "Uniform random selection, then fallback to remaining models",
"least-used": "Picks the model with fewest requests, balancing load over time",
"cost-optimized": "Routes to the cheapest model first based on pricing",
}[strategy]
}
</p>
</div>

View File

@@ -82,22 +82,30 @@ export default function ComboDefaultsTab() {
<div
role="tablist"
aria-label="Combo strategy"
className="inline-flex p-0.5 rounded-md bg-black/5 dark:bg-white/5"
className="grid grid-cols-3 gap-1 p-0.5 rounded-md bg-black/5 dark:bg-white/5"
>
{["priority", "weighted", "round-robin"].map((s) => (
{[
{ value: "priority", label: "Priority", icon: "sort" },
{ value: "weighted", label: "Weighted", icon: "percent" },
{ value: "round-robin", label: "Round-Robin", icon: "autorenew" },
{ value: "random", label: "Random", icon: "shuffle" },
{ value: "least-used", label: "Least-Used", icon: "low_priority" },
{ value: "cost-optimized", label: "Cost-Opt", icon: "savings" },
].map((s) => (
<button
key={s}
key={s.value}
role="tab"
aria-selected={comboDefaults.strategy === s}
onClick={() => setComboDefaults((prev) => ({ ...prev, strategy: s }))}
aria-selected={comboDefaults.strategy === s.value}
onClick={() => setComboDefaults((prev) => ({ ...prev, strategy: s.value }))}
className={cn(
"px-3 py-1 rounded text-xs font-medium transition-all capitalize",
comboDefaults.strategy === s
"px-2 py-1 rounded text-xs font-medium transition-all flex items-center justify-center gap-0.5",
comboDefaults.strategy === s.value
? "bg-white dark:bg-white/10 text-text-main shadow-sm"
: "text-text-muted hover:text-text-main"
)}
>
{s === "round-robin" ? "Round-Robin" : s}
<span className="material-symbols-outlined text-[14px]">{s.icon}</span>
{s.label}
</button>
))}
</div>

View File

@@ -13,6 +13,7 @@ const STRATEGIES = [
},
{ value: "round-robin", label: "Round Robin", desc: "Cycle through all accounts", icon: "loop" },
{ value: "p2c", label: "P2C", desc: "Pick 2 random, use the healthier one", icon: "balance" },
{ value: "random", label: "Random", desc: "Pick a random account each request", icon: "shuffle" },
];
export default function RoutingTab() {
@@ -76,7 +77,7 @@ export default function RoutingTab() {
<h3 className="text-lg font-semibold">Routing Strategy</h3>
</div>
<div className="grid grid-cols-3 gap-2 mb-4">
<div className="grid grid-cols-4 gap-2 mb-4">
{STRATEGIES.map((s) => (
<button
key={s.value}
@@ -132,6 +133,8 @@ export default function RoutingTab() {
"Using accounts in priority order (Fill First)."}
{settings.fallbackStrategy === "p2c" &&
"Power of Two Choices: picks 2 random accounts and routes to the healthier one."}
{settings.fallbackStrategy === "random" &&
"Picks a random account for each request — simple uniform distribution."}
</p>
</Card>

View File

@@ -36,7 +36,7 @@
* @typedef {Object} Combo
* @property {string} id - Combo unique ID
* @property {string} name - Display name
* @property {'priority'|'round-robin'|'random'|'least-used'} strategy - Selection strategy
* @property {'priority'|'weighted'|'round-robin'|'random'|'least-used'|'cost-optimized'} strategy - Selection strategy
* @property {Array<string|{model: string, weight?: number}>} models - Model entries
* @property {boolean} [isActive] - Whether the combo is active
*/

View File

@@ -38,7 +38,9 @@ export const comboNodeSchema = z.object({
export const comboSchema = z.object({
name: z.string().min(1, "Combo name is required").max(100),
model: z.string().min(1, "Model pattern is required"),
strategy: z.enum(["priority", "weighted", "round-robin", "cost-optimized"]).default("priority"),
strategy: z
.enum(["priority", "weighted", "round-robin", "random", "least-used", "cost-optimized"])
.default("priority"),
nodes: z.array(comboNodeSchema).min(1, "At least one node is required"),
isActive: z.boolean().default(true),
maxRetries: z.number().int().min(0).max(10).default(2),

View File

@@ -46,7 +46,10 @@ export const createComboSchema = z.object({
.max(100)
.regex(/^[a-zA-Z0-9_/.-]+$/, "Name can only contain letters, numbers, -, _, / and ."),
models: z.array(comboModelEntry).optional().default([]),
strategy: z.enum(["priority", "weighted"]).optional().default("priority"),
strategy: z
.enum(["priority", "weighted", "round-robin", "random", "least-used", "cost-optimized"])
.optional()
.default("priority"),
config: comboConfigSchema,
});