From ea61d00cf7509b34e2ee4608e72e09bd1210f67e Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 12 Apr 2026 10:34:10 -0300 Subject: [PATCH] =?UTF-8?q?feat:=20v3.6.4=20=E2=80=94=20Combo=20Builder=20?= =?UTF-8?q?v2,=20Composite=20Tiers,=20P2C=20Credentials,=20Observability?= =?UTF-8?q?=20Layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## New Features - Combo Builder v2 wizard UI (multi-stage: Basics → Steps → Strategy → Review) - Combo Step Architecture Schema v2 (ComboModelStep, ComboRefStep, pinned accounts) - Composite Tiers system for tiered model routing with fallback chains - Model Capabilities Registry (unified resolver merging specs + registry + synced data) - Observability module (buildHealthPayload, buildTelemetryPayload, buildSessionsSummary) - Session & Quota Monitor panels on Health dashboard - Combo Health per-target analytics via resolveNestedComboTargets() - Combo Builder Options API (GET /api/combos/builder/options) ## Performance - Middleware lazy loading (apiAuth, db/settings, modelSyncScheduler) - E2E auth bypass mode (NEXT_PUBLIC_OMNIROUTE_E2E_MODE) ## Bug Fixes - P2C credential selection with quota headroom awareness - Fixed-account combo steps bypass model cooldowns/circuit breakers - Combo metrics per-target tracking (byTarget with executionKey) - Call logs schema expansion (7 new columns + composite index) - Quota monitor lifecycle enrichment (status, snapshots, summary) - Codex quota fetcher hardening ## Maintenance - DB migration 021 (combo_call_log_targets) - Combo CRUD normalization on read - Playwright config + build script improvements - OpenAPI spec version sync to 3.6.4 ## Tests - 16 new test suites + 12 existing test updates - 86 files changed, +8318 -1378 lines --- .env.example | 5 +- CHANGELOG.md | 55 + docs/openapi.yaml | 2 +- open-sse/config/providerRegistry.ts | 2 + open-sse/handlers/chatCore.ts | 4 + open-sse/mcp-server/server.ts | 19 +- open-sse/mcp-server/tools/advancedTools.ts | 9 +- open-sse/services/codexQuotaFetcher.ts | 55 +- open-sse/services/combo.ts | 590 +++-- open-sse/services/comboMetrics.ts | 244 +- open-sse/services/contextManager.ts | 2 +- open-sse/services/model.ts | 19 + open-sse/services/modelCapabilities.ts | 97 +- open-sse/services/modelFamilyFallback.ts | 2 +- open-sse/services/quotaMonitor.ts | 259 +- open-sse/services/quotaPreflight.ts | 16 +- open-sse/services/thinkingBudget.ts | 11 +- .../translator/request/openai-to-gemini.ts | 2 +- package-lock.json | 4 +- package.json | 2 +- playwright.config.ts | 6 +- scripts/build-next-isolated.mjs | 9 +- scripts/run-next-playwright.mjs | 171 +- .../dashboard/analytics/ComboHealthTab.tsx | 72 + src/app/(dashboard)/dashboard/combos/page.tsx | 2242 +++++++++++------ src/app/(dashboard)/dashboard/health/page.tsx | 142 +- .../usage/components/BudgetTelemetryCards.tsx | 8 + .../cli-tools/openclaw/auto-order/route.ts | 14 +- src/app/api/combos/[id]/route.ts | 31 +- src/app/api/combos/builder/options/route.ts | 12 + src/app/api/combos/route.ts | 24 +- src/app/api/combos/test/route.ts | 57 +- src/app/api/monitoring/health/route.ts | 74 +- src/app/api/openapi/spec/route.ts | 15 +- src/app/api/telemetry/summary/route.ts | 10 +- src/app/api/usage/combo-health/route.ts | 273 +- src/app/api/v1beta/models/route.ts | 27 +- src/app/globals.css | 1 - src/domain/comboResolver.ts | 12 +- src/lib/combos/builderDraft.ts | 153 ++ src/lib/combos/builderOptions.ts | 544 ++++ src/lib/combos/compositeTiers.ts | 200 ++ src/lib/combos/steps.ts | 318 +++ src/lib/db/combos.ts | 123 +- src/lib/db/core.ts | 75 + src/lib/db/migrations/001_initial_schema.sql | 11 + .../migrations/021_combo_call_log_targets.sql | 5 + src/lib/db/settings.ts | 20 +- src/lib/modelCapabilities.ts | 280 ++ src/lib/modelsDevSync.ts | 106 +- src/lib/monitoring/observability.ts | 227 ++ src/lib/usage/callLogs.ts | 23 +- src/proxy.ts | 59 +- src/shared/types/utilization.ts | 17 + src/shared/utils/machineId.ts | 13 +- src/shared/validation/schemas.ts | 67 +- src/sse/handlers/chat.ts | 126 +- src/sse/handlers/chatHelpers.ts | 15 +- src/sse/services/auth.ts | 324 ++- src/sse/services/model.ts | 5 +- tests/e2e/combos-flow.spec.ts | 101 +- tests/integration/combo-routing-e2e.test.mjs | 98 + tests/unit/build-next-isolated.test.mjs | 15 +- tests/unit/call-log-cap.test.mjs | 37 + tests/unit/codex-quota-fetcher.test.mjs | 47 + tests/unit/combo-builder-draft.test.mjs | 186 ++ .../unit/combo-builder-options-route.test.mjs | 228 ++ tests/unit/combo-config.test.mjs | 121 +- tests/unit/combo-health-route.test.mjs | 266 ++ .../combo-routes-composite-tiers.test.mjs | 157 ++ tests/unit/combo-routing-engine.test.mjs | 132 +- tests/unit/combo-test-route.test.mjs | 58 + .../unit/composite-tiers-validation.test.mjs | 131 + tests/unit/db-combos-crud.test.mjs | 72 + tests/unit/db-core-init.test.mjs | 129 + tests/unit/machine-id.test.mjs | 11 +- .../unit/model-capabilities-registry.test.mjs | 105 + tests/unit/model-combo-mappings-db.test.mjs | 10 +- tests/unit/observability-payloads.test.mjs | 165 ++ tests/unit/openapi-spec-route.test.mjs | 15 + tests/unit/proxy-e2e-mode.test.mjs | 74 + tests/unit/quota-monitor.test.mjs | 65 + tests/unit/quota-preflight.test.mjs | 1 + tests/unit/run-next-playwright.test.mjs | 119 + tests/unit/services-branch-hardening.test.mjs | 25 +- tests/unit/sse-auth.test.mjs | 154 ++ tests/unit/telemetry-summary-route.test.mjs | 35 + 87 files changed, 8431 insertions(+), 1436 deletions(-) create mode 100644 src/app/api/combos/builder/options/route.ts create mode 100644 src/lib/combos/builderDraft.ts create mode 100644 src/lib/combos/builderOptions.ts create mode 100644 src/lib/combos/compositeTiers.ts create mode 100644 src/lib/combos/steps.ts create mode 100644 src/lib/db/migrations/021_combo_call_log_targets.sql create mode 100644 src/lib/modelCapabilities.ts create mode 100644 src/lib/monitoring/observability.ts create mode 100644 tests/unit/combo-builder-draft.test.mjs create mode 100644 tests/unit/combo-builder-options-route.test.mjs create mode 100644 tests/unit/combo-health-route.test.mjs create mode 100644 tests/unit/combo-routes-composite-tiers.test.mjs create mode 100644 tests/unit/composite-tiers-validation.test.mjs create mode 100644 tests/unit/model-capabilities-registry.test.mjs create mode 100644 tests/unit/observability-payloads.test.mjs create mode 100644 tests/unit/openapi-spec-route.test.mjs create mode 100644 tests/unit/proxy-e2e-mode.test.mjs create mode 100644 tests/unit/run-next-playwright.test.mjs create mode 100644 tests/unit/telemetry-summary-route.test.mjs diff --git a/.env.example b/.env.example index 4e12f640a3..844cb0659a 100644 --- a/.env.example +++ b/.env.example @@ -25,9 +25,8 @@ API_KEY_SECRET= # Initial admin login password — CHANGE THIS before first use! # Used by: bootstrap only — sets the initial dashboard password on first boot. # After first login you can change it from Dashboard → Settings → Security. -# Default: 123456 (insecure, for local dev only) -INITIAL_PASSWORD=123456 - +# Default: CHANGEME (insecure, for local dev only) +INITIAL_PASSWORD=CHANGEME # ═══════════════════════════════════════════════════════════════════════════════ # 2. STORAGE & DATABASE diff --git a/CHANGELOG.md b/CHANGELOG.md index bb7d57eee6..2f70fae9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,61 @@ --- +## [3.6.4] — 2026-04-12 + +### ✨ New Features + +- **Combo Builder v2 (Wizard UI):** Completely redesigned the combo creation/editing interface as a multi-stage wizard with stages: Basics → Steps → Strategy → Review. The builder fetches provider, model, and connection metadata via a new `GET /api/combos/builder/options` endpoint, enabling precise provider/model/account selection with duplicate detection and automatic next-connection suggestion. Heavy UI components (`ModelSelectModal`, `ProxyConfigModal`, `ModelRoutingSection`) are now lazily loaded via `next/dynamic` for faster initial page render +- **Combo Step Architecture (Schema v2):** Introduced a structured step model (`ComboModelStep`, `ComboRefStep`) replacing the legacy flat string/object combo entries. Steps carry explicit `id`, `kind`, `providerId`, `connectionId`, `weight`, and `label` fields, enabling pinned-account routing, cross-combo references, and per-step metrics. All combo CRUD operations normalize entries through the new `src/lib/combos/steps.ts` module. Zod schemas updated with `comboModelStepInputSchema` and `comboRefStepInputSchema` unions +- **Composite Tiers System:** Added tiered model routing via `config.compositeTiers` — each tier maps a named stage to a specific combo step with optional fallback chains. Includes comprehensive validation (`src/lib/combos/compositeTiers.ts`) ensuring step existence, preventing circular fallback, and validating default tier references. Zod schema enforcement blocks composite tiers on global defaults (concrete combos only) +- **Model Capabilities Registry:** Created `src/lib/modelCapabilities.ts` providing `getResolvedModelCapabilities()` — a unified resolver that merges static specs, provider registry data, and live-synced capabilities into a single `ResolvedModelCapabilities` object covering tool calling, reasoning, vision, context window, thinking budget, modalities, and model lifecycle metadata +- **Observability Module:** Extracted health and telemetry payload construction into `src/lib/monitoring/observability.ts` with `buildHealthPayload()`, `buildTelemetryPayload()`, and `buildSessionsSummary()` builders. The health endpoint now returns session activity, quota monitor status, and per-provider breakdowns alongside existing system metrics +- **Session & Quota Monitor Dashboard:** Added live Session Activity and Quota Monitors panels to the Health dashboard, showing active session counts, sticky-bound sessions, per-API-key breakdowns, and top session details alongside quota monitor alerting/exhausted/error status with per-provider drill-down +- **Combo Health Per-Target Analytics:** The combo-health API now resolves per-target metrics using the new `resolveNestedComboTargets()` function, providing step-level success rates, latency, and historical usage breakdowns per execution key — enabling per-account, per-connection health visibility + +### ⚡ Performance + +- **Middleware Lazy Loading:** Refactored `src/proxy.ts` to lazy-import `apiAuth`, `db/settings`, and `modelSyncScheduler` modules, reducing middleware cold-start overhead. Added inline `isPublicApiRoute()` to avoid loading the full auth module for public routes +- **E2E Auth Bypass:** Added `NEXT_PUBLIC_OMNIROUTE_E2E_MODE` environment flag to bypass authentication gates for dashboard and management API routes during Playwright E2E test runs + +### 🐛 Bug Fixes + +- **P2C Credential Selection:** Implemented Power-of-Two-Choices (P2C) connection scoring in `src/sse/services/auth.ts` with quota headroom awareness, error/recency penalties, and forced/excluded connection support. The new `getProviderCredentialsWithQuotaPreflight()` function integrates quota preflight checks directly into credential selection, eliminating the separate Codex-only preflight path +- **Fixed-Account Combo Steps:** Combo steps with explicit `connectionId` now correctly bypass provider-level model cooldowns and circuit breakers, preventing a single account failure from blocking pinned-connection routing for the same model +- **Combo Metrics Per-Target Tracking:** Extended `comboMetrics.ts` to track `byTarget` metrics keyed by execution path, recording per-step `provider`, `providerId`, `connectionId`, and `label` alongside existing per-model aggregates +- **Call Logs Schema Expansion:** Added `requested_model`, `request_type`, `tokens_cache_read`, `tokens_cache_creation`, `tokens_reasoning`, `combo_step_id`, and `combo_execution_key` columns to `call_logs` with auto-migration. Added composite index `idx_cl_combo_target` for efficient per-target historical queries +- **Quota Monitor Enrichment:** Expanded `quotaMonitor.ts` with full lifecycle state tracking (`status`, `startedAt`, `lastPolledAt`, `consecutiveFailures`, `totalPolls`, `totalAlerts`), ISO-formatted snapshots via `getQuotaMonitorSnapshots()`, and sorted summary via `getQuotaMonitorSummary()` +- **Codex Quota Fetcher Hardening:** Improved `codexQuotaFetcher.ts` with safer connection registration and quota fetch error handling + +### 🔧 Maintenance & Infrastructure + +- **DB Migration 021:** Added `combo_call_log_targets` migration for `combo_step_id` and `combo_execution_key` columns in call_logs +- **Combo CRUD Normalization:** `db/combos.ts` now normalizes all stored combo entries through the step normalization pipeline on read, ensuring consistent step IDs and kind annotations regardless of when the combo was created +- **Playwright Config:** Updated Playwright configuration and `run-next-playwright.mjs` script for improved E2E test orchestration +- **Build Script:** Updated `build-next-isolated.mjs` with additional reliability improvements + +### 🧪 Tests + +- **16 New Test Suites:** Added comprehensive test coverage including: + - `combo-builder-draft.test.mjs` (186 lines) — Builder draft step construction and validation + - `combo-builder-options-route.test.mjs` (228 lines) — Builder options API endpoint + - `combo-health-route.test.mjs` (266 lines) — Combo health analytics with per-target metrics + - `combo-routes-composite-tiers.test.mjs` (157 lines) — Composite tiers API integration + - `composite-tiers-validation.test.mjs` (131 lines) — Composite tier validation rules + - `db-combos-crud.test.mjs` — Combo CRUD with step normalization + - `db-core-init.test.mjs` (129 lines) — DB initialization and column migrations + - `model-capabilities-registry.test.mjs` (105 lines) — Model capabilities resolution + - `observability-payloads.test.mjs` (165 lines) — Health/telemetry payload construction + - `openapi-spec-route.test.mjs` — OpenAPI spec generation + - `proxy-e2e-mode.test.mjs` (74 lines) — E2E mode auth bypass + - `quota-monitor.test.mjs` — Quota monitor lifecycle state + - `run-next-playwright.test.mjs` (119 lines) — Playwright runner script + - `sse-auth.test.mjs` (154 lines) — P2C credential selection and quota preflight + - `telemetry-summary-route.test.mjs` (35 lines) — Telemetry summary endpoint + - Plus updates to 12 existing test files for compatibility with new step architecture + +--- + ## [3.6.3] — 2026-04-11 ### ✨ New Features diff --git a/docs/openapi.yaml b/docs/openapi.yaml index cd6d59b36f..9d6c58ea44 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OmniRoute API - version: 3.6.3 + version: 3.6.4 description: | OmniRoute is a local-first AI API proxy router. It provides an OpenAI-compatible endpoint that routes requests to multiple AI providers with load balancing, diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index 21aa1586be..38816c9559 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -14,6 +14,8 @@ export interface RegistryModel { id: string; name: string; toolCalling?: boolean; + supportsReasoning?: boolean; + supportsVision?: boolean; targetFormat?: string; unsupportedParams?: readonly string[]; /** Maximum context window in tokens */ diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 188b7aac30..cd73d3a122 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -479,6 +479,8 @@ export async function handleChatCore({ comboName, comboStrategy = null, isCombo = false, + comboStepId = null, + comboExecutionKey = null, disableEmergencyFallback = false, }) { let { provider, model, extendedContext } = modelInfo; @@ -746,6 +748,8 @@ export async function handleChatCore({ sourceFormat, targetFormat, comboName, + comboStepId, + comboExecutionKey, apiKeyId: apiKeyInfo?.id || null, apiKeyName: apiKeyInfo?.name || null, noLog: noLogEnabled, diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 7442d91e04..01fbfdaaff 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -12,6 +12,11 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { + getComboModelProvider, + getComboModelString, + getComboStepTarget, +} from "../../src/lib/combos/steps.ts"; import { MCP_TOOLS, @@ -105,11 +110,17 @@ function normalizeComboModels( rawModels: unknown ): Array<{ provider: string; model: string; priority: number }> { return toArray(rawModels).map((rawModel, index) => { - const model = toRecord(rawModel); + const modelRecord = toRecord(rawModel); + const modelString = getComboModelString(rawModel); + const target = getComboStepTarget(rawModel); + const provider = + getComboModelProvider(rawModel) || + (modelString ? "unknown" : target ? "combo" : toString(modelRecord.provider, "unknown")); + return { - provider: toString(model.provider, "unknown"), - model: toString(model.model, "unknown"), - priority: toNumber(model.priority, index + 1), + provider, + model: modelString || target || toString(modelRecord.model, "unknown"), + priority: toNumber(modelRecord.priority, index + 1), }; }); } diff --git a/open-sse/mcp-server/tools/advancedTools.ts b/open-sse/mcp-server/tools/advancedTools.ts index 30c356a468..b6d713c811 100644 --- a/open-sse/mcp-server/tools/advancedTools.ts +++ b/open-sse/mcp-server/tools/advancedTools.ts @@ -18,6 +18,11 @@ import { logToolCall } from "../audit.ts"; import { normalizeQuotaResponse } from "../../../src/shared/contracts/quota.ts"; import { resolveOmniRouteBaseUrl } from "../../../src/shared/utils/resolveOmniRouteBaseUrl.ts"; +import { + getComboModelProvider, + getComboModelString, + getComboStepTarget, +} from "../../../src/lib/combos/steps.ts"; const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl(); const OMNIROUTE_API_KEY = process.env.OMNIROUTE_API_KEY || ""; @@ -80,8 +85,8 @@ function getComboModels(combo: JsonRecord): ComboModel[] { const nestedModels = toArrayOfRecords(toRecord(combo.data).models); const sourceModels = directModels.length > 0 ? directModels : nestedModels; return sourceModels.map((model) => ({ - provider: toString(model.provider, "unknown"), - model: toString(model.model, ""), + provider: getComboModelProvider(model) || (getComboModelString(model) ? "unknown" : "combo"), + model: getComboModelString(model) || getComboStepTarget(model) || "", inputCostPer1M: toNumber(model.inputCostPer1M, 3.0), })); } diff --git a/open-sse/services/codexQuotaFetcher.ts b/open-sse/services/codexQuotaFetcher.ts index e28c0b1d61..ec31cc9215 100644 --- a/open-sse/services/codexQuotaFetcher.ts +++ b/open-sse/services/codexQuotaFetcher.ts @@ -76,6 +76,50 @@ export function registerCodexConnection(connectionId: string, meta: CodexConnect connectionRegistry.set(connectionId, meta); } +function getCodexConnectionMeta( + connectionId: string, + connection?: Record +): CodexConnectionMeta | null { + if (connection && typeof connection === "object") { + const providerSpecificData = + connection.providerSpecificData && + typeof connection.providerSpecificData === "object" && + !Array.isArray(connection.providerSpecificData) + ? (connection.providerSpecificData as Record) + : {}; + const accessToken = + typeof connection.accessToken === "string" && connection.accessToken.trim().length > 0 + ? connection.accessToken + : null; + const workspaceId = + typeof providerSpecificData.workspaceId === "string" && + providerSpecificData.workspaceId.trim().length > 0 + ? providerSpecificData.workspaceId + : undefined; + + if (accessToken) { + const meta = { accessToken, ...(workspaceId ? { workspaceId } : {}) }; + connectionRegistry.set(connectionId, meta); + return meta; + } + } + + return connectionRegistry.get(connectionId) || null; +} + +function getDominantResetAt(quota: { + window5h: { percentUsed: number; resetAt: string | null }; + window7d: { percentUsed: number; resetAt: string | null }; +}): string | null { + if (quota.window7d.percentUsed > quota.window5h.percentUsed) { + return quota.window7d.resetAt || quota.window5h.resetAt; + } + if (quota.window5h.percentUsed > quota.window7d.percentUsed) { + return quota.window5h.resetAt || quota.window7d.resetAt; + } + return quota.window7d.resetAt || quota.window5h.resetAt; +} + // ─── Core Fetcher ──────────────────────────────────────────────────────────── /** @@ -85,7 +129,10 @@ export function registerCodexConnection(connectionId: string, meta: CodexConnect * @param connectionId - Connection ID from the DB (used to look up credentials) * @returns QuotaInfo or null if fetch fails / no credentials */ -export async function fetchCodexQuota(connectionId: string): Promise { +export async function fetchCodexQuota( + connectionId: string, + connection?: Record +): Promise { // Check cache first const cached = quotaCache.get(connectionId); if (cached && Date.now() - cached.fetchedAt < CACHE_TTL_MS) { @@ -93,7 +140,7 @@ export async function fetchCodexQuota(connectionId: string): Promise"); +} + +function normalizeRuntimeStep(entry, comboName, index, allCombos, path = []) { + const step = normalizeComboStep(entry, { + comboName, + index, + allCombos, + }); + if (!step) return null; + + const executionKey = buildExecutionKey(path, step.id); + const label = typeof step.label === "string" ? step.label : null; + const weight = step.weight || 0; + + if (step.kind === "combo-ref") { + return { + kind: "combo-ref", + stepId: step.id, + executionKey, + comboName: step.comboName, + weight, + label, + }; + } + + const modelStr = getComboModelString(step); + if (!modelStr) return null; + + return { + kind: "model", + stepId: step.id, + executionKey, + modelStr, + provider: getTargetProvider(modelStr, step.providerId), + providerId: step.providerId || null, + connectionId: step.connectionId || null, + weight, + label, + } satisfies ResolvedComboTarget; +} + +function getDirectComboTargets(combo) { + return (combo.models || []) + .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null)) + .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); +} + +function getTopLevelRuntimeSteps(combo, allCombos, path = []) { + return (combo.models || []) + .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, allCombos, path)) + .filter((entry): entry is ComboRuntimeStep => entry !== null); +} + +function expandRuntimeStep(step, allCombos, visited = new Set(), depth = 0, path = []) { + if (step.kind === "model") return [step]; + if (depth > MAX_COMBO_DEPTH) return []; + + const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; + const nestedCombo = combos.find((combo) => combo.name === step.comboName); + if (!nestedCombo || visited.has(step.comboName)) return []; + + return resolveNestedComboTargets(nestedCombo, combos, new Set(visited), depth + 1, [ + ...path, + step.stepId, + ]); +} + +export function resolveNestedComboTargets( + combo, + allCombos, + visited = new Set(), + depth = 0, + path = [] +) { + const directTargets = (combo.models || []) + .map((entry, index) => normalizeRuntimeStep(entry, combo.name, index, null, path)) + .filter((entry): entry is ResolvedComboTarget => entry?.kind === "model"); + + if (depth > MAX_COMBO_DEPTH) return directTargets; + if (visited.has(combo.name)) return []; + visited.add(combo.name); + + const runtimeSteps = getTopLevelRuntimeSteps(combo, allCombos, path); + const resolved: ResolvedComboTarget[] = []; + + for (const step of runtimeSteps) { + if (step.kind === "combo-ref") { + resolved.push(...expandRuntimeStep(step, allCombos, new Set(visited), depth, path)); + continue; + } + resolved.push(step); + } + + return resolved; } /** @@ -232,37 +382,32 @@ export function resolveNestedComboModels(combo, allCombos, visited = new Set(), return resolved; } -/** - * Select a model using weighted random distribution - * @param {Array} models - Array of { model, weight } entries - * @returns {string} Selected model string - */ -function selectWeightedModel(models) { - const entries = models.map((m) => normalizeModelEntry(m)); - const totalWeight = entries.reduce((sum, m) => sum + m.weight, 0); +function selectWeightedTarget(targets: Array<{ weight: number }>) { + if (targets.length === 0) return null; + const totalWeight = targets.reduce((sum, target) => sum + (target.weight || 0), 0); if (totalWeight <= 0) { - // All weights are 0 → uniform random - return entries[Math.floor(Math.random() * entries.length)].model; + return targets[Math.floor(Math.random() * targets.length)]; } let random = Math.random() * totalWeight; - for (const entry of entries) { - random -= entry.weight; - if (random <= 0) return entry.model; + for (const target of targets) { + random -= target.weight || 0; + if (random <= 0) return target; } - return entries[entries.length - 1].model; // safety fallback + + return targets[targets.length - 1]; } -/** - * Order models for weighted fallback (selected first, then by descending weight) - */ -function orderModelsForWeightedFallback(models, selectedModel) { - const entries = models.map((m) => normalizeModelEntry(m)); - const selected = entries.find((e) => e.model === selectedModel); - const rest = entries.filter((e) => e.model !== selectedModel).sort((a, b) => b.weight - a.weight); // highest weight first for fallback - - return [selected, ...rest].filter(Boolean).map((e) => e.model); +function orderTargetsForWeightedFallback( + targets: Array<{ executionKey: string; weight: number }>, + selectedExecutionKey: string +) { + const selected = targets.find((target) => target.executionKey === selectedExecutionKey); + const rest = targets + .filter((target) => target.executionKey !== selectedExecutionKey) + .sort((a, b) => b.weight - a.weight); + return [selected, ...rest].filter(Boolean); } // shuffleArray and getNextModelFromDeck moved to src/shared/utils/shuffleDeck.ts @@ -297,6 +442,22 @@ async function sortModelsByCost(models) { } } +async function sortTargetsByCost(targets: ResolvedComboTarget[]) { + const orderedModels = await sortModelsByCost(targets.map((target) => target.modelStr)); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + /** * Sort models by usage count (least-used first) for least-used strategy * @param {Array} models - Model strings @@ -315,6 +476,25 @@ function sortModelsByUsage(models, comboName) { return withUsage.map((e) => e.modelStr); } +function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) { + const orderedModels = sortModelsByUsage( + targets.map((target) => target.modelStr), + comboName + ); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + /** * Sort models by context window size (largest first) for context-optimized strategy. * Uses models.dev synced capabilities to get context limits. @@ -333,6 +513,22 @@ function sortModelsByContextSize(models) { return withContext.map((e) => e.modelStr); } +function sortTargetsByContextSize(targets: ResolvedComboTarget[]) { + const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr)); + const byModel = new Map(); + for (const target of targets) { + const queue = byModel.get(target.modelStr) || []; + queue.push(target); + byModel.set(target.modelStr, queue); + } + return orderedModels + .map((modelStr) => { + const queue = byModel.get(modelStr); + return queue?.shift() || null; + }) + .filter((target): target is ResolvedComboTarget => target !== null); +} + function toTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; @@ -437,7 +633,7 @@ function getBootstrapLatencyMs(modelId) { return DEFAULT_MODEL_P95_MS[normalized] ?? 1500; } -async function buildAutoCandidates(modelStrings, comboName) { +async function buildAutoCandidates(targets, comboName) { const metrics = getComboMetrics(comboName); const { getPricingForModel } = await import("../../src/lib/localDb"); let historicalLatencyStats = {}; @@ -453,9 +649,10 @@ async function buildAutoCandidates(modelStrings, comboName) { } const candidates = await Promise.all( - modelStrings.map(async (modelStr) => { + targets.map(async (target) => { + const modelStr = target.modelStr; const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; + const provider = target.provider || parsed.provider || parsed.providerAlias || "unknown"; const model = parsed.model || modelStr; const historicalKey = `${provider}/${model}`; const historicalModelMetric = historicalLatencyStats[historicalKey] || null; @@ -503,11 +700,16 @@ async function buildAutoCandidates(modelStrings, comboName) { ? Math.max(10, historicalStdDev) : Math.max(10, p95LatencyMs * 0.1); - const breakerStateRaw = getCircuitBreaker(`combo:${modelStr}`)?.getStatus?.()?.state; + const breakerStateRaw = getCircuitBreaker( + getComboBreakerKey(comboName, target.executionKey) + )?.getStatus?.()?.state; const circuitBreakerState = breakerStateRaw === "OPEN" || breakerStateRaw === "HALF_OPEN" ? breakerStateRaw : "CLOSED"; return { + stepId: target.stepId, + executionKey: target.executionKey, + modelStr, provider, model, quotaRemaining: 100, @@ -526,6 +728,64 @@ async function buildAutoCandidates(modelStrings, comboName) { return candidates; } +function dedupeTargetsByExecutionKey(targets: ResolvedComboTarget[]) { + const seen = new Set(); + return targets.filter((target) => { + if (seen.has(target.executionKey)) return false; + seen.add(target.executionKey); + return true; + }); +} + +export function resolveComboTargets(combo, allCombos) { + return allCombos ? resolveNestedComboTargets(combo, allCombos) : getDirectComboTargets(combo); +} + +function resolveWeightedTargets(combo, allCombos) { + const topLevelSteps = allCombos + ? getTopLevelRuntimeSteps(combo, allCombos) + : getDirectComboTargets(combo); + if (topLevelSteps.length === 0) { + return { orderedTargets: [], selectedStep: null }; + } + + const selectedStep = selectWeightedTarget(topLevelSteps); + if (!selectedStep) { + return { orderedTargets: [], selectedStep: null }; + } + + const orderedSteps = orderTargetsForWeightedFallback(topLevelSteps, selectedStep.executionKey); + const expandedTargets = orderedSteps.flatMap((step) => { + if (!allCombos) { + return step.kind === "model" ? [step] : []; + } + return expandRuntimeStep(step, allCombos, new Set([combo.name])); + }); + + return { + orderedTargets: dedupeTargetsByExecutionKey(expandedTargets), + selectedStep, + }; +} + +function scoreAutoTargets(targets, candidates, taskType, weights) { + const candidateByExecutionKey = new Map( + candidates.map((candidate) => [candidate.executionKey, candidate]) + ); + return targets + .map((target) => { + const candidate = candidateByExecutionKey.get(target.executionKey); + if (!candidate) return null; + const factors = calculateFactors(candidate, candidates, taskType, getTaskFitness); + return { + target, + score: calculateScore(factors, weights), + }; + }) + .filter(Boolean) + .sort((a, b) => b.score - a.score); +} + /** * Handle combo chat with fallback. * @param {Object} options @@ -548,7 +808,6 @@ export async function handleComboChat({ relayOptions, }) { const strategy = combo.strategy || "priority"; - const models = combo.models || []; const relayConfig = strategy === "context-relay" ? resolveContextRelayConfig(relayOptions?.config || null) : null; @@ -566,8 +825,8 @@ export async function handleComboChat({ } // Wrap handleSingleModel to inject context caching tag on response (#401) const handleSingleModelWrapped = combo.context_cache_protection - ? async (b, modelStr) => { - const res = await handleSingleModel(b, modelStr); + ? async (b, modelStr, target) => { + const res = await handleSingleModel(b, modelStr, target); if (!res.ok) return res; // Non-streaming: inject tag into JSON response @@ -749,47 +1008,28 @@ export async function handleComboChat({ const maxRetries = config.maxRetries ?? 1; const retryDelayMs = config.retryDelayMs ?? 2000; - let orderedModels; + let orderedTargets = + strategy === "weighted" + ? resolveWeightedTargets(combo, allCombos)?.orderedTargets || [] + : resolveComboTargets(combo, allCombos); - // Resolve nested combos if allCombos provided - if (allCombos) { - const flatModels = resolveNestedComboModels(combo, allCombos); - if (strategy === "weighted") { - // For weighted + nested, select from original models then fallback sequentially - const selected = selectWeightedModel(models); - orderedModels = orderModelsForWeightedFallback(models, selected); - // If entries were nested, they are already resolved to flat - orderedModels = orderedModels.flatMap((m) => { - const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || []; - const nested = combos.find((c) => c.name === m); - if (nested) return resolveNestedComboModels(nested, allCombos); - return [m]; - }); - log.info( - "COMBO", - `Weighted selection with nested resolution: ${orderedModels.length} total models` - ); - } else { - orderedModels = flatModels; - log.info("COMBO", `${strategy} with nested resolution: ${orderedModels.length} total models`); - } - } else if (strategy === "weighted") { - const selected = selectWeightedModel(models); - orderedModels = orderModelsForWeightedFallback(models, selected); - log.info("COMBO", `Weighted selection: ${selected} (from ${models.length} models)`); - } else { - orderedModels = models.map((m) => normalizeModelEntry(m).model); + if (strategy === "weighted") { + log.info( + "COMBO", + `Weighted selection${allCombos ? " with nested resolution" : ""}: ${orderedTargets.length} total targets` + ); + } else if (allCombos) { + log.info("COMBO", `${strategy} with nested resolution: ${orderedTargets.length} total targets`); } - // Apply strategy-specific ordering if (strategy === "auto") { const requestHasTools = Array.isArray(body?.tools) && body.tools.length > 0; - let eligibleModels = [...orderedModels]; + let eligibleTargets = [...orderedTargets]; if (requestHasTools) { - const filtered = eligibleModels.filter((m) => supportsToolCalling(m)); + const filtered = eligibleTargets.filter((target) => supportsToolCalling(target.modelStr)); if (filtered.length > 0) { - eligibleModels = filtered; + eligibleTargets = filtered; } else { log.warn( "COMBO", @@ -816,14 +1056,7 @@ export async function handleComboChat({ const candidatePool = Array.isArray(autoConfigSource.candidatePool) ? autoConfigSource.candidatePool - : [ - ...new Set( - eligibleModels.map((m) => { - const parsed = parseModel(m); - return parsed.provider || parsed.providerAlias || "unknown"; - }) - ), - ]; + : [...new Set(eligibleTargets.map((target) => target.provider))]; const weights = autoConfigSource.weights && typeof autoConfigSource.weights === "object" @@ -838,7 +1071,6 @@ export async function handleComboChat({ const modePack = typeof autoConfigSource.modePack === "string" ? autoConfigSource.modePack : undefined; - // Retrieve last known good provider (LKGP) for this combo/model (#919) let lastKnownGoodProvider: string | undefined; try { const { getLKGP } = await import("../../src/lib/localDb"); @@ -848,7 +1080,7 @@ export async function handleComboChat({ log.warn("COMBO", "Failed to retrieve Last Known Good Provider. This is non-fatal.", { err }); } - const candidates = await buildAutoCandidates(eligibleModels, combo.name); + const candidates = await buildAutoCandidates(eligibleTargets, combo.name); if (candidates.length > 0) { let selectedProvider = null; let selectedModel = null; @@ -892,51 +1124,57 @@ export async function handleComboChat({ selectionReason = `score=${selection.score.toFixed(3)}${selection.isExploration ? " (exploration)" : ""}`; } - const modelLookup = new Map(); - for (const modelStr of eligibleModels) { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; - const modelId = parsed.model || modelStr; - modelLookup.set(`${provider}/${modelId}`, modelStr); - } + const scoredTargets = scoreAutoTargets(eligibleTargets, candidates, taskType, weights); + const rankedTargets = scoredTargets.map((entry) => entry.target); + const selectedTarget = + scoredTargets.find((entry) => { + const parsed = parseModel(entry.target.modelStr); + const modelId = parsed.model || entry.target.modelStr; + return entry.target.provider === selectedProvider && modelId === selectedModel; + })?.target || + rankedTargets[0] || + eligibleTargets[0]; - const ranked = scorePool(candidates, taskType, weights) - .map((r) => modelLookup.get(`${r.provider}/${r.model}`) || `${r.provider}/${r.model}`) - .filter(Boolean); - - const selectedModelStr = - modelLookup.get(`${selectedProvider}/${selectedModel}`) || - `${selectedProvider}/${selectedModel}`; - orderedModels = [...new Set([selectedModelStr, ...ranked, ...eligibleModels])]; + orderedTargets = dedupeTargetsByExecutionKey( + [selectedTarget, ...rankedTargets, ...eligibleTargets].filter(Boolean) + ); log.info( "COMBO", - `Auto selection: ${selectedModelStr} | intent=${intent} task=${taskType} | strategy=${routingStrategy} | ${selectionReason}` + `Auto selection: ${selectedTarget?.modelStr || `${selectedProvider}/${selectedModel}`} | intent=${intent} task=${taskType} | strategy=${routingStrategy} | ${selectionReason}` ); } else { log.warn("COMBO", "Auto strategy has no candidates, keeping default ordering"); } } else if (strategy === "strict-random") { - const selectedId = await getNextFromDeck(`combo:${combo.name}`, orderedModels); - // Put selected model first so the fallback loop tries it first - const rest = orderedModels.filter((m) => m !== selectedId); - orderedModels = [selectedId, ...rest]; + const selectedExecutionKey = await getNextFromDeck( + `combo:${combo.name}`, + orderedTargets.map((target) => target.executionKey) + ); + const selectedTarget = + orderedTargets.find((target) => target.executionKey === selectedExecutionKey) || null; + const rest = orderedTargets.filter((target) => target.executionKey !== selectedExecutionKey); + orderedTargets = [selectedTarget, ...rest].filter(Boolean); log.info( "COMBO", - `Strict-random deck: ${selectedId} selected (${orderedModels.length} models)` + `Strict-random deck: ${selectedExecutionKey} selected (${orderedTargets.length} targets)` ); } else if (strategy === "random") { - orderedModels = fisherYatesShuffle([...orderedModels]); - log.info("COMBO", `Random shuffle: ${orderedModels.length} models`); + orderedTargets = fisherYatesShuffle([...orderedTargets]); + log.info("COMBO", `Random shuffle: ${orderedTargets.length} targets`); } else if (strategy === "least-used") { - orderedModels = sortModelsByUsage(orderedModels, combo.name); - log.info("COMBO", `Least-used ordering: ${orderedModels[0]} has fewest requests`); + orderedTargets = sortTargetsByUsage(orderedTargets, combo.name); + log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`); } else if (strategy === "cost-optimized") { - orderedModels = await sortModelsByCost(orderedModels); - log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedModels[0]})`); + orderedTargets = await sortTargetsByCost(orderedTargets); + log.info("COMBO", `Cost-optimized ordering: cheapest first (${orderedTargets[0]?.modelStr})`); } else if (strategy === "context-optimized") { - orderedModels = sortModelsByContextSize(orderedModels); - log.info("COMBO", `Context-optimized ordering: largest first (${orderedModels[0]})`); + orderedTargets = sortTargetsByContextSize(orderedTargets); + log.info("COMBO", `Context-optimized ordering: largest first (${orderedTargets[0]?.modelStr})`); + } + + if (orderedTargets.length === 0) { + return unavailableResponse(503, "Combo has no executable targets"); } let lastError = null; @@ -944,13 +1182,14 @@ export async function handleComboChat({ let lastStatus = null; const startTime = Date.now(); let fallbackCount = 0; + let recordedAttempts = 0; - for (let i = 0; i < orderedModels.length; i++) { - const modelStr = orderedModels[i]; - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; + for (let i = 0; i < orderedTargets.length; i++) { + const target = orderedTargets[i]; + const modelStr = target.modelStr; + const provider = target.provider; const profile = getProviderProfile(provider); - const breakerKey = `combo:${modelStr}`; + const breakerKey = getComboBreakerKey(combo.name, target.executionKey); const breaker = getCircuitBreaker(breakerKey, { failureThreshold: profile.circuitBreakerThreshold, resetTimeout: profile.circuitBreakerReset, @@ -965,7 +1204,7 @@ export async function handleComboChat({ // Pre-check: skip models where all accounts are in cooldown if (isModelAvailable) { - const available = await isModelAvailable(modelStr); + const available = await isModelAvailable(modelStr, target); if (!available) { log.info("COMBO", `Skipping ${modelStr} (all accounts in cooldown)`); if (i > 0) fallbackCount++; @@ -985,10 +1224,10 @@ export async function handleComboChat({ log.info( "COMBO", - `Trying model ${i + 1}/${orderedModels.length}: ${modelStr}${retry > 0 ? ` (retry ${retry})` : ""}` + `Trying model ${i + 1}/${orderedTargets.length}: ${modelStr}${retry > 0 ? ` (retry ${retry})` : ""}` ); - const result = await handleSingleModelWrapped(body, modelStr); + const result = await handleSingleModelWrapped(body, modelStr, target); // Success — validate response quality before returning if (result.ok) { @@ -1004,7 +1243,9 @@ export async function handleComboChat({ latencyMs: Date.now() - startTime, fallbackCount, strategy, + target: toRecordedTarget(target), }); + recordedAttempts++; if (i > 0) fallbackCount++; break; // move to next model } @@ -1019,7 +1260,9 @@ export async function handleComboChat({ latencyMs, fallbackCount, strategy, + target: toRecordedTarget(target), }); + recordedAttempts++; // Context-relay intentionally splits responsibilities: // combo.ts decides whether a successful turn should generate a handoff, @@ -1063,7 +1306,12 @@ export async function handleComboChat({ // Record last known good provider (LKGP) for this combo/model (#919) if (provider) { import("../../src/lib/localDb") - .then(({ setLKGP }) => setLKGP(combo.name, combo.id || combo.name, provider)) + .then(({ setLKGP }) => + Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]) + ) .catch((err) => log.warn("COMBO", "Failed to record Last Known Good Provider. This is non-fatal.", { err, @@ -1129,6 +1377,14 @@ export async function handleComboChat({ if (!shouldFallback && !comboBadRequestFallback) { log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status }); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; return result; } @@ -1146,6 +1402,14 @@ export async function handleComboChat({ } // Done retrying this model + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy, + target: toRecordedTarget(target), + }); + recordedAttempts++; lastError = errorText || String(result.status); if (!lastStatus) lastStatus = result.status; if (i > 0) fallbackCount++; @@ -1161,13 +1425,15 @@ export async function handleComboChat({ } // Early exit: check if all models have breaker OPEN - const allBreakersOpen = orderedModels.every((m) => { - return !getCircuitBreaker(`combo:${m}`).canExecute(); + const allBreakersOpen = orderedTargets.every((target) => { + return !getCircuitBreaker(getComboBreakerKey(combo.name, target.executionKey)).canExecute(); }); // All models failed const latencyMs = Date.now() - startTime; - recordComboRequest(combo.name, null, { success: false, latencyMs, fallbackCount, strategy }); + if (recordedAttempts === 0) { + recordComboRequest(combo.name, null, { success: false, latencyMs, fallbackCount, strategy }); + } if (allBreakersOpen) { log.warn("COMBO", "All models have circuit breaker OPEN — aborting"); @@ -1226,7 +1492,6 @@ async function handleRoundRobinCombo({ settings, allCombos, }) { - const models = combo.models || []; const config = settings ? resolveComboConfig(combo, settings) : { ...getDefaultComboConfig(), ...(combo.config || {}) }; @@ -1235,17 +1500,10 @@ async function handleRoundRobinCombo({ const maxRetries = config.maxRetries ?? 1; const retryDelayMs = config.retryDelayMs ?? 2000; - // Resolve models (support nested combos) - let orderedModels; - if (allCombos) { - orderedModels = resolveNestedComboModels(combo, allCombos); - } else { - orderedModels = models.map((m) => normalizeModelEntry(m).model); - } - - const modelCount = orderedModels.length; + const orderedTargets = resolveComboTargets(combo, allCombos); + const modelCount = orderedTargets.length; if (modelCount === 0) { - return unavailableResponse(503, "Round-robin combo has no models"); + return unavailableResponse(503, "Round-robin combo has no executable targets"); } // Get and increment atomic counter @@ -1258,15 +1516,17 @@ async function handleRoundRobinCombo({ let lastStatus = null; let earliestRetryAfter = null; let fallbackCount = 0; + let recordedAttempts = 0; // Try each model starting from the round-robin target for (let offset = 0; offset < modelCount; offset++) { const modelIndex = (startIndex + offset) % modelCount; - const modelStr = orderedModels[modelIndex]; - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || "unknown"; + const target = orderedTargets[modelIndex]; + const modelStr = target.modelStr; + const provider = target.provider; const profile = getProviderProfile(provider); - const breakerKey = `combo:${modelStr}`; + const breakerKey = getComboBreakerKey(combo.name, target.executionKey); + const semaphoreKey = `combo:${combo.name}:${target.executionKey}`; const breaker = getCircuitBreaker(breakerKey, { failureThreshold: profile.circuitBreakerThreshold, resetTimeout: profile.circuitBreakerReset, @@ -1281,7 +1541,7 @@ async function handleRoundRobinCombo({ // Pre-check availability if (isModelAvailable) { - const available = await isModelAvailable(modelStr); + const available = await isModelAvailable(modelStr, target); if (!available) { log.info("COMBO-RR", `Skipping ${modelStr} (all accounts in cooldown)`); if (offset > 0) fallbackCount++; @@ -1292,7 +1552,7 @@ async function handleRoundRobinCombo({ // Acquire semaphore slot (may wait in queue) let release; try { - release = await semaphore.acquire(modelStr, { + release = await semaphore.acquire(semaphoreKey, { maxConcurrency: concurrency, timeoutMs: queueTimeout, }); @@ -1321,7 +1581,7 @@ async function handleRoundRobinCombo({ `[RR #${counter}] → ${modelStr}${offset > 0 ? ` (fallback +${offset})` : ""}${retry > 0 ? ` (retry ${retry})` : ""}` ); - const result = await handleSingleModel(body, modelStr); + const result = await handleSingleModel(body, modelStr, target); // Success — validate response quality before returning if (result.ok) { @@ -1337,7 +1597,9 @@ async function handleRoundRobinCombo({ latencyMs: Date.now() - startTime, fallbackCount, strategy: "round-robin", + target: toRecordedTarget(target), }); + recordedAttempts++; if (offset > 0) fallbackCount++; break; // move to next model } @@ -1352,7 +1614,27 @@ async function handleRoundRobinCombo({ latencyMs, fallbackCount, strategy: "round-robin", + target: toRecordedTarget(target), }); + recordedAttempts++; + if (provider) { + import("../../src/lib/localDb") + .then(({ setLKGP }) => + Promise.all([ + setLKGP(combo.name, target.executionKey, provider), + setLKGP(combo.name, combo.id || combo.name, provider), + ]) + ) + .catch((err) => + log.warn( + "COMBO-RR", + "Failed to record Last Known Good Provider. This is non-fatal.", + { + err, + } + ) + ); + } return result; } @@ -1404,7 +1686,7 @@ async function handleRoundRobinCombo({ // Transient errors → mark in semaphore AND record circuit breaker failure if (TRANSIENT_FOR_BREAKER.includes(result.status) && cooldownMs > 0) { - semaphore.markRateLimited(modelStr, cooldownMs); + semaphore.markRateLimited(semaphoreKey, cooldownMs); breaker._onFailure(); log.warn( "COMBO-RR", @@ -1414,6 +1696,14 @@ async function handleRoundRobinCombo({ if (!shouldFallback && !comboBadRequestFallback) { log.warn("COMBO-RR", `${modelStr} failed (no fallback)`, { status: result.status }); + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy: "round-robin", + target: toRecordedTarget(target), + }); + recordedAttempts++; return result; } @@ -1431,6 +1721,14 @@ async function handleRoundRobinCombo({ } // Done with this model + recordComboRequest(combo.name, modelStr, { + success: false, + latencyMs: Date.now() - startTime, + fallbackCount, + strategy: "round-robin", + target: toRecordedTarget(target), + }); + recordedAttempts++; lastError = errorText || String(result.status); if (!lastStatus) lastStatus = result.status; if (offset > 0) fallbackCount++; @@ -1451,16 +1749,18 @@ async function handleRoundRobinCombo({ // All models exhausted const latencyMs = Date.now() - startTime; - recordComboRequest(combo.name, null, { - success: false, - latencyMs, - fallbackCount, - strategy: "round-robin", - }); + if (recordedAttempts === 0) { + recordComboRequest(combo.name, null, { + success: false, + latencyMs, + fallbackCount, + strategy: "round-robin", + }); + } // Early exit: check if all models have breaker OPEN - const allBreakersOpen = orderedModels.every((m) => { - return !getCircuitBreaker(`combo:${m}`).canExecute(); + const allBreakersOpen = orderedTargets.every((target) => { + return !getCircuitBreaker(getComboBreakerKey(combo.name, target.executionKey)).canExecute(); }); if (allBreakersOpen) { diff --git a/open-sse/services/comboMetrics.ts b/open-sse/services/comboMetrics.ts index b6705204a5..931f8578f1 100644 --- a/open-sse/services/comboMetrics.ts +++ b/open-sse/services/comboMetrics.ts @@ -1,7 +1,7 @@ /** * In-memory combo metrics tracker - * Tracks per-combo and per-model request counts, latency, success/failure rates - * Provides API for reading metrics from the dashboard + * Tracks per-combo, per-model, and per-target request counts, latency, success/failure rates. + * Provides API for reading metrics from the dashboard. */ interface ModelMetrics { @@ -13,6 +13,16 @@ interface ModelMetrics { lastUsedAt: string | null; } +interface ComboTargetMetrics extends ModelMetrics { + executionKey: string; + stepId: string | null; + model: string; + provider: string | null; + providerId: string | null; + connectionId: string | null; + label: string | null; +} + interface ComboMetricsEntry { totalRequests: number; totalSuccesses: number; @@ -23,33 +33,140 @@ interface ComboMetricsEntry { lastUsedAt: string | null; intentCounts: Record; byModel: Record; + byTarget: Record; +} + +interface ModelMetricsView extends ModelMetrics { + avgLatencyMs: number; + successRate: number; +} + +interface ComboTargetMetricsView extends ComboTargetMetrics { + avgLatencyMs: number; + successRate: number; } interface ComboMetricsView extends ComboMetricsEntry { avgLatencyMs: number; successRate: number; fallbackRate: number; - byModel: Record< - string, - ModelMetrics & { - avgLatencyMs: number; - successRate: number; - } - >; + byModel: Record; + byTarget: Record; +} + +export interface ComboRequestTargetMeta { + executionKey?: string | null; + stepId?: string | null; + provider?: string | null; + providerId?: string | null; + connectionId?: string | null; + label?: string | null; +} + +function toNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function inferProvider(modelStr: string | null): string | null { + const model = toNonEmptyString(modelStr); + if (!model) return null; + const [provider] = model.split("/"); + return toNonEmptyString(provider); +} + +function createModelMetrics(): ModelMetrics { + return { + requests: 0, + successes: 0, + failures: 0, + totalLatencyMs: 0, + lastStatus: null, + lastUsedAt: null, + }; +} + +function createComboEntry(strategy: string): ComboMetricsEntry { + return { + totalRequests: 0, + totalSuccesses: 0, + totalFailures: 0, + totalFallbacks: 0, + totalLatencyMs: 0, + strategy, + lastUsedAt: null, + intentCounts: {}, + byModel: {}, + byTarget: {}, + }; +} + +function applyMetricOutcome( + metric: ModelMetrics, + success: boolean, + latencyMs: number, + usedAt: string +): void { + metric.requests++; + metric.totalLatencyMs += latencyMs; + metric.lastUsedAt = usedAt; + + if (success) { + metric.successes++; + metric.lastStatus = "ok"; + return; + } + + metric.failures++; + metric.lastStatus = "error"; +} + +function buildTargetMetric( + modelStr: string, + target: ComboRequestTargetMeta +): ComboTargetMetrics | null { + const executionKey = toNonEmptyString(target.executionKey) || toNonEmptyString(modelStr); + const model = toNonEmptyString(modelStr); + if (!executionKey || !model) return null; + + return { + executionKey, + stepId: toNonEmptyString(target.stepId), + model, + provider: toNonEmptyString(target.provider) || inferProvider(model), + providerId: toNonEmptyString(target.providerId), + connectionId: + target.connectionId === null ? null : (toNonEmptyString(target.connectionId) ?? null), + label: target.label === null ? null : (toNonEmptyString(target.label) ?? null), + ...createModelMetrics(), + }; +} + +function toMetricView( + metric: T +): T & { + avgLatencyMs: number; + successRate: number; +} { + return { + ...metric, + avgLatencyMs: metric.requests > 0 ? Math.round(metric.totalLatencyMs / metric.requests) : 0, + successRate: metric.requests > 0 ? Math.round((metric.successes / metric.requests) * 100) : 0, + }; } // In-memory store const metrics = new Map(); /** - * Record a combo request result + * Record a combo request result. * @param {string} comboName * @param {string} modelStr - The model that handled the request (or null if all failed) * @param {Object} options * @param {boolean} options.success * @param {number} options.latencyMs * @param {number} options.fallbackCount - How many fallbacks occurred - * @param {string} [options.strategy] - "priority" or "weighted" + * @param {string} [options.strategy] - Routing strategy name + * @param {Object} [options.target] - Step/execution metadata for structured combos */ export function recordComboRequest( comboName: string, @@ -59,28 +176,27 @@ export function recordComboRequest( latencyMs, fallbackCount = 0, strategy = "priority", - }: { success: boolean; latencyMs: number; fallbackCount?: number; strategy?: string } + target, + }: { + success: boolean; + latencyMs: number; + fallbackCount?: number; + strategy?: string; + target?: ComboRequestTargetMeta | null; + } ): void { if (!metrics.has(comboName)) { - metrics.set(comboName, { - totalRequests: 0, - totalSuccesses: 0, - totalFailures: 0, - totalFallbacks: 0, - totalLatencyMs: 0, - strategy, - lastUsedAt: null, - intentCounts: {}, - byModel: {}, - }); + metrics.set(comboName, createComboEntry(strategy)); } const combo = metrics.get(comboName); if (!combo) return; + + const usedAt = new Date().toISOString(); combo.totalRequests++; combo.totalLatencyMs += latencyMs; combo.totalFallbacks += fallbackCount; - combo.lastUsedAt = new Date().toISOString(); + combo.lastUsedAt = usedAt; combo.strategy = strategy; if (success) { @@ -89,35 +205,36 @@ export function recordComboRequest( combo.totalFailures++; } - // Per-model tracking - if (modelStr) { - if (!combo.byModel[modelStr]) { - combo.byModel[modelStr] = { - requests: 0, - successes: 0, - failures: 0, - totalLatencyMs: 0, - lastStatus: null, - lastUsedAt: null, - }; - } - const modelMetric = combo.byModel[modelStr]; - modelMetric.requests++; - modelMetric.totalLatencyMs += latencyMs; - modelMetric.lastUsedAt = new Date().toISOString(); + if (!modelStr) return; - if (success) { - modelMetric.successes++; - modelMetric.lastStatus = "ok"; - } else { - modelMetric.failures++; - modelMetric.lastStatus = "error"; - } + if (!combo.byModel[modelStr]) { + combo.byModel[modelStr] = createModelMetrics(); } + applyMetricOutcome(combo.byModel[modelStr], success, latencyMs, usedAt); + + const targetMetric = buildTargetMetric(modelStr, target || {}); + if (!targetMetric) return; + + if (!combo.byTarget[targetMetric.executionKey]) { + combo.byTarget[targetMetric.executionKey] = targetMetric; + } + + const existingTargetMetric = combo.byTarget[targetMetric.executionKey]; + existingTargetMetric.stepId = targetMetric.stepId || existingTargetMetric.stepId; + existingTargetMetric.provider = targetMetric.provider || existingTargetMetric.provider; + existingTargetMetric.providerId = targetMetric.providerId || existingTargetMetric.providerId; + existingTargetMetric.connectionId = + target?.connectionId === null + ? null + : (targetMetric.connectionId ?? existingTargetMetric.connectionId); + existingTargetMetric.label = + target?.label === null ? null : (targetMetric.label ?? existingTargetMetric.label); + + applyMetricOutcome(existingTargetMetric, success, latencyMs, usedAt); } /** - * Get metrics for a specific combo + * Get metrics for a specific combo. * @param {string} comboName * @returns {Object|null} */ @@ -135,20 +252,19 @@ export function getComboMetrics(comboName: string): ComboMetricsView | null { combo.totalRequests > 0 ? Math.round((combo.totalFallbacks / combo.totalRequests) * 100) : 0, intentCounts: { ...combo.intentCounts }, byModel: Object.fromEntries( - Object.entries(combo.byModel).map(([model, m]) => [ - model, - { - ...m, - avgLatencyMs: m.requests > 0 ? Math.round(m.totalLatencyMs / m.requests) : 0, - successRate: m.requests > 0 ? Math.round((m.successes / m.requests) * 100) : 0, - }, + Object.entries(combo.byModel).map(([model, metric]) => [model, toMetricView(metric)]) + ), + byTarget: Object.fromEntries( + Object.entries(combo.byTarget).map(([executionKey, metric]) => [ + executionKey, + toMetricView(metric), ]) ), }; } /** - * Get metrics for all combos + * Get metrics for all combos. * @returns {Object} Map of comboName → metrics */ export function getAllComboMetrics(): Record { @@ -164,17 +280,7 @@ export function getAllComboMetrics(): Record { */ export function recordComboIntent(comboName: string, intent: string): void { if (!metrics.has(comboName)) { - metrics.set(comboName, { - totalRequests: 0, - totalSuccesses: 0, - totalFailures: 0, - totalFallbacks: 0, - totalLatencyMs: 0, - strategy: "priority", - lastUsedAt: null, - intentCounts: {}, - byModel: {}, - }); + metrics.set(comboName, createComboEntry("priority")); } const combo = metrics.get(comboName); @@ -184,14 +290,14 @@ export function recordComboIntent(comboName: string, intent: string): void { } /** - * Reset metrics for a specific combo + * Reset metrics for a specific combo. */ export function resetComboMetrics(comboName: string): void { metrics.delete(comboName); } /** - * Reset all combo metrics + * Reset all combo metrics. */ export function resetAllComboMetrics(): void { metrics.clear(); diff --git a/open-sse/services/contextManager.ts b/open-sse/services/contextManager.ts index a899a141b5..190caa6c2a 100644 --- a/open-sse/services/contextManager.ts +++ b/open-sse/services/contextManager.ts @@ -6,7 +6,7 @@ */ import { REGISTRY } from "../config/providerRegistry.ts"; -import { getModelContextLimit } from "../../src/lib/modelsDevSync"; +import { getModelContextLimit } from "../../src/lib/modelCapabilities"; // Default token limits per provider (fallbacks when not in registry) const DEFAULT_LIMITS: Record = { diff --git a/open-sse/services/model.ts b/open-sse/services/model.ts index ec0f40d351..7c12bb3b73 100644 --- a/open-sse/services/model.ts +++ b/open-sse/services/model.ts @@ -71,6 +71,25 @@ function resolveProviderModelAlias(providerOrAlias, modelId) { return aliases?.[modelId] || modelId; } +/** + * Resolve a provider/model pair into canonical provider ID + provider-scoped model ID. + * Keeps provider-specific legacy aliases out of downstream capability and budget lookups. + */ +export function resolveCanonicalProviderModel(providerOrAlias, modelId) { + if (!modelId || typeof modelId !== "string") { + return { + provider: resolveProviderAlias(providerOrAlias), + model: modelId || null, + }; + } + + const provider = resolveProviderAlias(providerOrAlias); + return { + provider, + model: resolveProviderModelAlias(provider, modelId), + }; +} + /** * Parse model string: "alias/model" or "provider/model" or just alias * Supports [1m] suffix for extended 1M context window (e.g. "claude-sonnet-4-6[1m]") diff --git a/open-sse/services/modelCapabilities.ts b/open-sse/services/modelCapabilities.ts index 83472e0dac..9fe59098e7 100644 --- a/open-sse/services/modelCapabilities.ts +++ b/open-sse/services/modelCapabilities.ts @@ -1,92 +1,5 @@ -import { PROVIDER_ID_TO_ALIAS, PROVIDER_MODELS } from "../config/providerModels.ts"; -import { parseModel } from "./model.ts"; - -// Conservative denylist fallback used when registry metadata is absent. -// Keep small and explicit to avoid false negatives. -const TOOL_CALLING_UNSUPPORTED_PATTERNS = ["gpt-oss-120b", "deepseek-reasoner"]; - -function getRegistryToolCallingFlag(providerIdOrAlias: string, modelId: string): boolean | null { - const providerAlias = PROVIDER_ID_TO_ALIAS[providerIdOrAlias] || providerIdOrAlias; - const models = PROVIDER_MODELS[providerAlias]; - if (!Array.isArray(models)) return null; - const found = models.find((m) => m?.id === modelId); - if (!found) return null; - return typeof found.toolCalling === "boolean" ? found.toolCalling : null; -} - -/** - * Returns whether a model should be considered safe for structured function/tool calling. - * - * Decision order: - * 1) Provider registry metadata (toolCalling flag) when available. - * 2) Conservative denylist fallback for known problematic model families. - * 3) Default true. - */ -export function supportsToolCalling(modelStr: string): boolean { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || ""; - const model = parsed.model || modelStr; - - if (provider) { - const fromRegistry = getRegistryToolCallingFlag(provider, model); - if (fromRegistry !== null) return fromRegistry; - } - - const normalized = String(modelStr || "").toLowerCase(); - if (!normalized) return false; - - const blocked = TOOL_CALLING_UNSUPPORTED_PATTERNS.some((pattern) => { - if (normalized === pattern) return true; - if (normalized.endsWith(`/${pattern}`)) return true; - return normalized.includes(pattern); - }); - - return !blocked; -} - -// Models that do NOT support reasoning/thinking parameters. -// AG (Antigravity) claude-sonnet-4-6 routes through a Google internal API -// that returns 400 if thinking params are included. -const REASONING_UNSUPPORTED_PATTERNS = [ - "antigravity/claude-sonnet-4-6", - "antigravity/claude-sonnet-4-5", - "antigravity/claude-sonnet-4", -]; - -function getRegistryReasoningFlag(providerIdOrAlias: string, modelId: string): boolean | null { - const providerAlias = PROVIDER_ID_TO_ALIAS[providerIdOrAlias] || providerIdOrAlias; - const models = PROVIDER_MODELS[providerAlias]; - if (!Array.isArray(models)) return null; - const found = models.find((m) => m?.id === modelId); - if (!found) return null; - return typeof found.supportsReasoning === "boolean" ? found.supportsReasoning : null; -} - -/** - * Returns whether a model supports reasoning/thinking parameters. - * - * Decision order: - * 1) Provider registry metadata (supportsReasoning flag) when available. - * 2) Explicit denylist for known unsupported models (e.g. AG Claude Sonnet). - * 3) Default true (pass through — safe, provider will ignore if unsupported). - */ -export function supportsReasoning(modelStr: string): boolean { - const parsed = parseModel(modelStr); - const provider = parsed.provider || parsed.providerAlias || ""; - const model = parsed.model || modelStr; - - if (provider) { - const fromRegistry = getRegistryReasoningFlag(provider, model); - if (fromRegistry !== null) return fromRegistry; - } - - const normalized = String(modelStr || "").toLowerCase(); - if (!normalized) return true; - - const blocked = REASONING_UNSUPPORTED_PATTERNS.some( - (pattern) => - normalized === pattern || normalized.endsWith(`/${pattern}`) || normalized.includes(pattern) - ); - - return !blocked; -} +export { + getResolvedModelCapabilities, + supportsReasoning, + supportsToolCalling, +} from "../../src/lib/modelCapabilities.ts"; diff --git a/open-sse/services/modelFamilyFallback.ts b/open-sse/services/modelFamilyFallback.ts index 73e25f32db..c75d5bd00b 100644 --- a/open-sse/services/modelFamilyFallback.ts +++ b/open-sse/services/modelFamilyFallback.ts @@ -11,7 +11,7 @@ * (commit 6cea566, Mar 8 2026). */ -import { getModelContextLimit } from "../../src/lib/modelsDevSync"; +import { getModelContextLimit } from "../../src/lib/modelCapabilities"; import { parseModel } from "./model.ts"; import { CONTEXT_OVERFLOW_REGEX } from "./errorClassifier.ts"; diff --git a/open-sse/services/quotaMonitor.ts b/open-sse/services/quotaMonitor.ts index 9ca39ca711..c5e0e79631 100644 --- a/open-sse/services/quotaMonitor.ts +++ b/open-sse/services/quotaMonitor.ts @@ -9,6 +9,7 @@ */ import { registerQuotaFetcher, type QuotaFetcher } from "./quotaPreflight.ts"; +import { getSessionInfo } from "./sessionManager.ts"; export { registerQuotaFetcher }; export type { QuotaFetcher }; @@ -24,6 +25,55 @@ interface MonitorState { stopped: boolean; provider: string; accountId: string; + connectionSnapshot: Record | null; + sessionBound: boolean; + status: "starting" | "idle" | "healthy" | "warning" | "exhausted" | "error"; + startedAt: number; + lastPolledAt: number | null; + lastSuccessAt: number | null; + lastErrorAt: number | null; + lastError: string | null; + lastQuotaPercent: number | null; + lastQuotaUsed: number | null; + lastQuotaTotal: number | null; + lastResetAt: string | null; + lastAlertAt: number | null; + nextPollDelayMs: number | null; + nextPollAt: number | null; + totalPolls: number; + totalAlerts: number; + consecutiveFailures: number; +} + +export interface QuotaMonitorSnapshot { + sessionId: string; + provider: string; + accountId: string; + status: "starting" | "idle" | "healthy" | "warning" | "exhausted" | "error"; + startedAt: string; + lastPolledAt: string | null; + lastSuccessAt: string | null; + lastErrorAt: string | null; + lastError: string | null; + lastQuotaPercent: number | null; + lastQuotaUsed: number | null; + lastQuotaTotal: number | null; + lastResetAt: string | null; + lastAlertAt: string | null; + nextPollDelayMs: number | null; + nextPollAt: string | null; + totalPolls: number; + totalAlerts: number; + consecutiveFailures: number; +} + +export interface QuotaMonitorSummary { + active: number; + alerting: number; + exhausted: number; + errors: number; + statusCounts: Record; + byProvider: Record; } const activeMonitors = new Map(); @@ -45,47 +95,148 @@ function suppressedAlert( provider: string, accountId: string, percentUsed: number -): void { +): boolean { const key = `${sessionId}:${provider}:${accountId}`; const last = alertSuppression.get(key) ?? 0; - if (Date.now() - last < ALERT_SUPPRESS_WINDOW_MS) return; + if (Date.now() - last < ALERT_SUPPRESS_WINDOW_MS) return false; alertSuppression.set(key, Date.now()); console.warn( `[QuotaMonitor] session=${sessionId} ${provider}/${accountId}: ${(percentUsed * 100).toFixed(1)}% quota used` ); + return true; +} + +function toIsoTimestamp(value: number | null): string | null { + return typeof value === "number" && Number.isFinite(value) ? new Date(value).toISOString() : null; +} + +function getMonitorStatus(percentUsed: number | null): MonitorState["status"] { + if (!Number.isFinite(percentUsed)) return "idle"; + if ((percentUsed as number) >= EXHAUSTION_THRESHOLD) return "exhausted"; + if ((percentUsed as number) >= WARN_THRESHOLD) return "warning"; + return "healthy"; +} + +function toPublicSnapshot(sessionId: string, state: MonitorState): QuotaMonitorSnapshot { + return { + sessionId, + provider: state.provider, + accountId: state.accountId, + status: state.status, + startedAt: new Date(state.startedAt).toISOString(), + lastPolledAt: toIsoTimestamp(state.lastPolledAt), + lastSuccessAt: toIsoTimestamp(state.lastSuccessAt), + lastErrorAt: toIsoTimestamp(state.lastErrorAt), + lastError: state.lastError, + lastQuotaPercent: state.lastQuotaPercent, + lastQuotaUsed: state.lastQuotaUsed, + lastQuotaTotal: state.lastQuotaTotal, + lastResetAt: state.lastResetAt, + lastAlertAt: toIsoTimestamp(state.lastAlertAt), + nextPollDelayMs: state.nextPollDelayMs, + nextPollAt: toIsoTimestamp(state.nextPollAt), + totalPolls: state.totalPolls, + totalAlerts: state.totalAlerts, + consecutiveFailures: state.consecutiveFailures, + }; +} + +function sortSnapshots(snapshots: QuotaMonitorSnapshot[]): QuotaMonitorSnapshot[] { + const severityRank: Record = { + exhausted: 5, + warning: 4, + error: 3, + starting: 2, + idle: 1, + healthy: 0, + }; + + return [...snapshots].sort((left, right) => { + const severityDelta = severityRank[right.status] - severityRank[left.status]; + if (severityDelta !== 0) return severityDelta; + const quotaDelta = (right.lastQuotaPercent ?? -1) - (left.lastQuotaPercent ?? -1); + if (quotaDelta !== 0) return quotaDelta; + return ( + (right.lastPolledAt ? Date.parse(right.lastPolledAt) : 0) - + (left.lastPolledAt ? Date.parse(left.lastPolledAt) : 0) + ); + }); } function scheduleNextPoll(sessionId: string, intervalMs: number): void { const state = activeMonitors.get(sessionId); if (!state || state.stopped) return; + state.nextPollDelayMs = intervalMs; + state.nextPollAt = Date.now() + intervalMs; const { provider, accountId } = state; const timer = setTimeout(async () => { const current = activeMonitors.get(sessionId); if (!current || current.stopped) return; + if (current.sessionBound && !getSessionInfo(sessionId)) { + stopQuotaMonitor(sessionId); + return; + } try { const fetcher = quotaFetcherRegistry.get(provider); if (!fetcher) { + current.status = current.lastQuotaPercent === null ? "idle" : current.status; scheduleNextPoll(sessionId, NORMAL_INTERVAL_MS); return; } - const quota = await fetcher(accountId); - const percentUsed = quota?.percentUsed ?? 0; + current.lastPolledAt = Date.now(); + current.totalPolls += 1; + const previousStatus = current.status; + const quota = await fetcher(accountId, current.connectionSnapshot || undefined); + const percentUsed = + quota && typeof quota.percentUsed === "number" && Number.isFinite(quota.percentUsed) + ? quota.percentUsed + : null; + current.lastSuccessAt = Date.now(); + current.lastError = null; + current.lastErrorAt = null; + current.consecutiveFailures = 0; + current.lastQuotaPercent = percentUsed; + current.lastQuotaUsed = + quota && typeof quota.used === "number" && Number.isFinite(quota.used) ? quota.used : null; + current.lastQuotaTotal = + quota && typeof quota.total === "number" && Number.isFinite(quota.total) + ? quota.total + : null; + current.lastResetAt = + quota && typeof quota.resetAt === "string" && quota.resetAt.trim().length > 0 + ? quota.resetAt + : null; + current.status = getMonitorStatus(percentUsed); - if (percentUsed >= EXHAUSTION_THRESHOLD) { - suppressedAlert(sessionId, provider, accountId, percentUsed); - console.info( - `[QuotaMonitor] session=${sessionId}: marking ${accountId} for next-session cooldown` - ); + if (percentUsed !== null && percentUsed >= EXHAUSTION_THRESHOLD) { + const emittedAlert = suppressedAlert(sessionId, provider, accountId, percentUsed); + if (emittedAlert) { + current.lastAlertAt = Date.now(); + current.totalAlerts += 1; + } + if (emittedAlert || previousStatus !== "exhausted") { + console.info( + `[QuotaMonitor] session=${sessionId}: marking ${accountId} for next-session cooldown` + ); + } scheduleNextPoll(sessionId, CRITICAL_INTERVAL_MS); - } else if (percentUsed >= WARN_THRESHOLD) { - suppressedAlert(sessionId, provider, accountId, percentUsed); + } else if (percentUsed !== null && percentUsed >= WARN_THRESHOLD) { + const emittedAlert = suppressedAlert(sessionId, provider, accountId, percentUsed); + if (emittedAlert) { + current.lastAlertAt = Date.now(); + current.totalAlerts += 1; + } scheduleNextPoll(sessionId, CRITICAL_INTERVAL_MS); } else { scheduleNextPoll(sessionId, NORMAL_INTERVAL_MS); } - } catch { + } catch (error) { + current.lastErrorAt = Date.now(); + current.lastError = error instanceof Error ? error.message : String(error); + current.consecutiveFailures += 1; + current.status = "error"; scheduleNextPoll(sessionId, NORMAL_INTERVAL_MS); } }, intervalMs); @@ -101,9 +252,40 @@ export function startQuotaMonitor( connection: Record ): void { if (!isQuotaMonitorEnabled(connection)) return; - if (activeMonitors.has(sessionId)) return; + const current = activeMonitors.get(sessionId); + if (current && !current.stopped) { + if (current.provider === provider && current.accountId === accountId) { + current.connectionSnapshot = connection; + current.sessionBound = current.sessionBound || getSessionInfo(sessionId) !== null; + return; + } + stopQuotaMonitor(sessionId); + } - activeMonitors.set(sessionId, { timer: null, stopped: false, provider, accountId }); + activeMonitors.set(sessionId, { + timer: null, + stopped: false, + provider, + accountId, + connectionSnapshot: connection, + sessionBound: getSessionInfo(sessionId) !== null, + status: "starting", + startedAt: Date.now(), + lastPolledAt: null, + lastSuccessAt: null, + lastErrorAt: null, + lastError: null, + lastQuotaPercent: null, + lastQuotaUsed: null, + lastQuotaTotal: null, + lastResetAt: null, + lastAlertAt: null, + nextPollDelayMs: null, + nextPollAt: null, + totalPolls: 0, + totalAlerts: 0, + consecutiveFailures: 0, + }); scheduleNextPoll(sessionId, NORMAL_INTERVAL_MS); } @@ -124,3 +306,52 @@ export function stopQuotaMonitor(sessionId: string): void { export function getActiveMonitorCount(): number { return activeMonitors.size; } + +export function getQuotaMonitorSnapshot(sessionId: string): QuotaMonitorSnapshot | null { + const state = activeMonitors.get(sessionId); + if (!state || state.stopped) return null; + return toPublicSnapshot(sessionId, state); +} + +export function getQuotaMonitorSnapshots(): QuotaMonitorSnapshot[] { + const snapshots: QuotaMonitorSnapshot[] = []; + for (const [sessionId, state] of activeMonitors) { + if (state.stopped) continue; + snapshots.push(toPublicSnapshot(sessionId, state)); + } + return sortSnapshots(snapshots); +} + +export function getQuotaMonitorSummary(): QuotaMonitorSummary { + const snapshots = getQuotaMonitorSnapshots(); + const statusCounts: Record = { + starting: 0, + idle: 0, + healthy: 0, + warning: 0, + exhausted: 0, + error: 0, + }; + const byProvider: Record = {}; + + for (const snapshot of snapshots) { + statusCounts[snapshot.status] += 1; + byProvider[snapshot.provider] = (byProvider[snapshot.provider] || 0) + 1; + } + + return { + active: snapshots.length, + alerting: statusCounts.warning + statusCounts.exhausted, + exhausted: statusCounts.exhausted, + errors: statusCounts.error, + statusCounts, + byProvider, + }; +} + +export function clearQuotaMonitors(): void { + for (const sessionId of [...activeMonitors.keys()]) { + stopQuotaMonitor(sessionId); + } + alertSuppression.clear(); +} diff --git a/open-sse/services/quotaPreflight.ts b/open-sse/services/quotaPreflight.ts index a41f4482dc..1d1386f4bf 100644 --- a/open-sse/services/quotaPreflight.ts +++ b/open-sse/services/quotaPreflight.ts @@ -11,15 +11,20 @@ export interface PreflightQuotaResult { proceed: boolean; reason?: string; quotaPercent?: number; + resetAt?: string | null; } export interface QuotaInfo { used: number; total: number; percentUsed: number; + resetAt?: string | null; } -export type QuotaFetcher = (connectionId: string) => Promise; +export type QuotaFetcher = ( + connectionId: string, + connection?: Record +) => Promise; const EXHAUSTION_THRESHOLD = 0.95; const WARN_THRESHOLD = 0.8; @@ -51,7 +56,7 @@ export async function preflightQuota( let quota: QuotaInfo | null = null; try { - quota = await fetcher(connectionId); + quota = await fetcher(connectionId, connection); } catch { return { proceed: true }; } @@ -66,7 +71,12 @@ export async function preflightQuota( console.info( `[QuotaPreflight] ${provider}/${connectionId}: ${(percentUsed * 100).toFixed(1)}% used — switching` ); - return { proceed: false, reason: "quota_exhausted", quotaPercent: percentUsed }; + return { + proceed: false, + reason: "quota_exhausted", + quotaPercent: percentUsed, + resetAt: quota.resetAt ?? null, + }; } if (percentUsed >= WARN_THRESHOLD) { diff --git a/open-sse/services/thinkingBudget.ts b/open-sse/services/thinkingBudget.ts index e47cfbc9fb..ed000e0e13 100644 --- a/open-sse/services/thinkingBudget.ts +++ b/open-sse/services/thinkingBudget.ts @@ -13,8 +13,12 @@ export const ThinkingMode = { ADAPTIVE: "adaptive", // Scale based on request complexity }; -import { capThinkingBudget, getDefaultThinkingBudget } from "@/shared/constants/modelSpecs"; -import { supportsReasoning } from "./modelCapabilities.ts"; +import { + capThinkingBudget, + getDefaultThinkingBudget, + getResolvedModelCapabilities, + supportsReasoning, +} from "@/lib/modelCapabilities"; // Effort → budget token mapping export const EFFORT_BUDGETS = { @@ -289,6 +293,9 @@ function applyAdaptiveBudget(body, cfg) { */ export function hasThinkingCapableModel(body) { const model = body.model || ""; + const resolved = getResolvedModelCapabilities(model); + if (resolved.supportsThinking === true) return true; + if (resolved.supportsThinking === false) return false; return ( model.includes("claude") || model.includes("o1") || diff --git a/open-sse/translator/request/openai-to-gemini.ts b/open-sse/translator/request/openai-to-gemini.ts index 0e61231eb2..ba299fe2c3 100644 --- a/open-sse/translator/request/openai-to-gemini.ts +++ b/open-sse/translator/request/openai-to-gemini.ts @@ -8,7 +8,7 @@ import { capMaxOutputTokens, capThinkingBudget, getDefaultThinkingBudget, -} from "../../../src/shared/constants/modelSpecs.ts"; +} from "../../../src/lib/modelCapabilities.ts"; import * as crypto from "crypto"; diff --git a/package-lock.json b/package-lock.json index 6e2338bcff..223f050c5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "3.6.3", + "version": "3.6.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "3.6.3", + "version": "3.6.4", "hasInstallScript": true, "license": "MIT", "workspaces": [ diff --git a/package.json b/package.json index fa2112363b..4b121598d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "3.6.3", + "version": "3.6.4", "description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": { diff --git a/playwright.config.ts b/playwright.config.ts index def2d7cc9a..6340683b39 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,6 +4,10 @@ const dashboardPort = process.env.DASHBOARD_PORT || process.env.PORT || "20128"; const dashboardBaseUrl = `http://localhost:${dashboardPort}`; const webServerReadyUrl = `${dashboardBaseUrl}/api/monitoring/health`; const playwrightServerMode = process.env.OMNIROUTE_PLAYWRIGHT_SERVER_MODE || "start"; +const playwrightWebServerTimeout = Number.parseInt( + process.env.OMNIROUTE_PLAYWRIGHT_WEB_SERVER_TIMEOUT || "900000", + 10 +); export default defineConfig({ testDir: "./tests/e2e", @@ -33,6 +37,6 @@ export default defineConfig({ command: `node scripts/run-next-playwright.mjs ${playwrightServerMode}`, url: webServerReadyUrl, reuseExistingServer: !process.env.CI, - timeout: 300_000, + timeout: Number.isFinite(playwrightWebServerTimeout) ? playwrightWebServerTimeout : 900_000, }, }); diff --git a/scripts/build-next-isolated.mjs b/scripts/build-next-isolated.mjs index f062085ad6..2cc7194036 100644 --- a/scripts/build-next-isolated.mjs +++ b/scripts/build-next-isolated.mjs @@ -52,7 +52,7 @@ function runNextBuild() { const child = spawn(process.execPath, [nextBin, "build"], { cwd: projectRoot, stdio: "inherit", - env: process.env, + env: resolveNextBuildEnv(process.env), }); const forward = (signal) => { @@ -74,6 +74,13 @@ function runNextBuild() { }); } +export function resolveNextBuildEnv(baseEnv = process.env) { + return { + ...baseEnv, + NEXT_PRIVATE_BUILD_WORKER: baseEnv.NEXT_PRIVATE_BUILD_WORKER || "0", + }; +} + export async function main() { let moved = false; diff --git a/scripts/run-next-playwright.mjs b/scripts/run-next-playwright.mjs index 37cfb1bdf5..15b4acce03 100644 --- a/scripts/run-next-playwright.mjs +++ b/scripts/run-next-playwright.mjs @@ -1,8 +1,9 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; -import { existsSync, renameSync } from "node:fs"; -import { join } from "node:path"; +import { cpSync, existsSync, mkdirSync, readdirSync, renameSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; import { resolveRuntimePorts, sanitizeColorEnv, @@ -16,10 +17,19 @@ const cwd = process.cwd(); const appDir = join(cwd, "app"); const srcAppDir = join(cwd, "src", "app"); const appPage = join(appDir, "page.tsx"); -const backupDir = join(cwd, "app.__qa_backup"); +const defaultBackupDir = join(cwd, "app.__qa_backup"); +const backupDir = resolvePlaywrightAppBackupDir({ + cwd, + baseBackupExists: existsSync(defaultBackupDir), + appDirExists: existsSync(appDir), +}); +const usingAlternativeBackupDir = backupDir !== defaultBackupDir; const buildScript = join(cwd, "scripts", "build-next-isolated.mjs"); const standaloneServer = join(cwd, testDistDir(), "standalone", "server.js"); -const buildIdFile = join(cwd, testDistDir(), "BUILD_ID"); +const rootStaticDir = join(cwd, testDistDir(), "static"); +const rootPublicDir = join(cwd, "public"); +const standaloneStaticDir = join(cwd, testDistDir(), "standalone", ".next", "static"); +const standalonePublicDir = join(cwd, testDistDir(), "standalone", "public"); let appDirMoved = false; @@ -27,19 +37,90 @@ function testDistDir() { return process.env.NEXT_DIST_DIR || ".next"; } +export function resolvePlaywrightAppBackupDir({ + cwd, + baseBackupExists, + appDirExists, + pid = process.pid, + now = Date.now(), +}) { + const baseBackupDir = join(cwd, "app.__qa_backup"); + if (!baseBackupExists || !appDirExists) { + return baseBackupDir; + } + + return join(cwd, `app.__qa_backup.${pid}.${now}`); +} + function shouldMoveAppDir() { return existsSync(appDir) && !existsSync(appPage) && existsSync(srcAppDir); } +export function directoryHasEntries(dirPath) { + try { + return readdirSync(dirPath).length > 0; + } catch { + return false; + } +} + +export function standaloneAssetsNeedSync({ + standaloneServerPath, + rootStaticDirPath, + standaloneStaticDirPath, +}) { + return ( + existsSync(standaloneServerPath) && + existsSync(rootStaticDirPath) && + !directoryHasEntries(standaloneStaticDirPath) + ); +} + +export function syncStandaloneRuntimeAssets({ + standaloneServerPath, + rootStaticDirPath, + standaloneStaticDirPath, + rootPublicDirPath, + standalonePublicDirPath, + log = console, +}) { + if (!existsSync(standaloneServerPath)) return false; + + let changed = false; + + if (existsSync(rootPublicDirPath) && !directoryHasEntries(standalonePublicDirPath)) { + cpSync(rootPublicDirPath, standalonePublicDirPath, { + recursive: true, + force: true, + }); + changed = true; + } + + if (existsSync(rootStaticDirPath) && !directoryHasEntries(standaloneStaticDirPath)) { + mkdirSync(dirname(standaloneStaticDirPath), { + recursive: true, + }); + cpSync(rootStaticDirPath, standaloneStaticDirPath, { + recursive: true, + force: true, + }); + changed = true; + } + + if (changed) { + log.log("[Playwright WebServer] Rehydrated standalone static/public assets"); + } + + return changed; +} + function prepareAppDir() { if (!shouldMoveAppDir()) return; - if (existsSync(backupDir)) { + if (usingAlternativeBackupDir) { console.warn( - "[Playwright WebServer] app.__qa_backup already exists; leaving app/ in place. " + - "If tests hit 404 on every route, clear app/ artifacts before running e2e." + "[Playwright WebServer] Existing app.__qa_backup detected; using a per-run backup dir instead." ); - return; } renameSync(appDir, backupDir); @@ -55,14 +136,6 @@ function restoreAppDir() { console.log("[Playwright WebServer] Restored app/ directory"); } -process.on("exit", restoreAppDir); -process.on("uncaughtException", (error) => { - restoreAppDir(); - throw error; -}); - -prepareAppDir(); - const bootstrapEnvVars = bootstrapEnv({ quiet: true }); const runtimePorts = resolveRuntimePorts(bootstrapEnvVars); const testServerEnv = { @@ -73,8 +146,17 @@ const testServerEnv = { OMNIROUTE_DISABLE_TOKEN_HEALTHCHECK: process.env.OMNIROUTE_DISABLE_TOKEN_HEALTHCHECK || "1", OMNIROUTE_DISABLE_LOCAL_HEALTHCHECK: process.env.OMNIROUTE_DISABLE_LOCAL_HEALTHCHECK || "1", OMNIROUTE_HIDE_HEALTHCHECK_LOGS: process.env.OMNIROUTE_HIDE_HEALTHCHECK_LOGS || "1", + ...(process.env.OMNIROUTE_USE_TURBOPACK + ? { + OMNIROUTE_USE_TURBOPACK: process.env.OMNIROUTE_USE_TURBOPACK, + } + : {}), }; +export function shouldUseWebpackForPlaywrightDev({ mode, env }) { + return mode === "dev" && env.OMNIROUTE_USE_TURBOPACK !== "1"; +} + function runChild(command, args, env) { return new Promise((resolve) => { const child = spawn(command, args, { @@ -100,7 +182,7 @@ function runChild(command, args, env) { async function runBuildForStart() { if (mode !== "start") return; if (process.env.OMNIROUTE_PLAYWRIGHT_SKIP_BUILD === "1") return; - if (existsSync(buildIdFile)) return; + console.log("[Playwright WebServer] Building fresh standalone app for this run..."); const buildEnv = withRuntimePortEnv(testServerEnv, runtimePorts); const result = await runChild(process.execPath, [buildScript], buildEnv); @@ -115,18 +197,37 @@ async function runBuildForStart() { } } -await runBuildForStart(); -if (mode === "start") { - if (existsSync(standaloneServer)) { - spawnWithForwardedSignals(process.execPath, [standaloneServer], { - stdio: "inherit", - env: { - ...withRuntimePortEnv(testServerEnv, runtimePorts), - PORT: String(runtimePorts.dashboardPort), - HOSTNAME: process.env.HOSTNAME || "127.0.0.1", - }, - }); - } else { +export async function main() { + process.on("exit", restoreAppDir); + process.on("uncaughtException", (error) => { + restoreAppDir(); + throw error; + }); + + prepareAppDir(); + await runBuildForStart(); + + if (mode === "start") { + if (existsSync(standaloneServer)) { + syncStandaloneRuntimeAssets({ + standaloneServerPath: standaloneServer, + rootStaticDirPath: rootStaticDir, + standaloneStaticDirPath: standaloneStaticDir, + rootPublicDirPath: rootPublicDir, + standalonePublicDirPath: standalonePublicDir, + }); + + spawnWithForwardedSignals(process.execPath, [standaloneServer], { + stdio: "inherit", + env: { + ...withRuntimePortEnv(testServerEnv, runtimePorts), + PORT: String(runtimePorts.dashboardPort), + HOSTNAME: process.env.HOSTNAME || "127.0.0.1", + }, + }); + return; + } + const args = [ "./node_modules/next/dist/bin/next", "start", @@ -138,18 +239,26 @@ if (mode === "start") { stdio: "inherit", env: withRuntimePortEnv(testServerEnv, runtimePorts), }); + return; } -} else { + const args = [ "./node_modules/next/dist/bin/next", mode, - "--webpack", "--port", String(runtimePorts.dashboardPort), ]; + if (shouldUseWebpackForPlaywrightDev({ mode, env: testServerEnv })) { + args.splice(2, 0, "--webpack"); + } + spawnWithForwardedSignals(process.execPath, args, { stdio: "inherit", env: withRuntimePortEnv(testServerEnv, runtimePorts), }); } + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); +} diff --git a/src/app/(dashboard)/dashboard/analytics/ComboHealthTab.tsx b/src/app/(dashboard)/dashboard/analytics/ComboHealthTab.tsx index 3455c296c4..14d386e5e4 100644 --- a/src/app/(dashboard)/dashboard/analytics/ComboHealthTab.tsx +++ b/src/app/(dashboard)/dashboard/analytics/ComboHealthTab.tsx @@ -20,6 +20,10 @@ function formatShare(value: number) { return formatPercent(value * 100, 1); } +function formatPercentOrDash(value: number | null, digits = 1) { + return typeof value === "number" ? formatPercent(value, digits) : "n/a"; +} + function formatLatency(value: number) { return `${Math.round(value).toLocaleString()}ms`; } @@ -95,6 +99,7 @@ function ComboHealthCard({ combo }: { combo: ComboHealthMetrics }) { ), [combo.usageSkew.modelDistribution] ); + const targetHealth = combo.targetHealth || []; return ( @@ -251,6 +256,73 @@ function ComboHealthCard({ combo }: { combo: ComboHealthMetrics }) { + + {targetHealth.length > 0 ? ( +
+
+
Execution targets
+
+ Step-level runtime metrics and quota visibility for structured combo targets. +
+
+ +
+ {targetHealth.map((target) => ( +
+
+
+
+ {target.label || target.model} +
+
+ {target.provider} + {target.connectionId ? ` · ${target.connectionId.slice(0, 8)}` : ""} +
+
{target.stepId}
+
+
+ {target.lastStatus ? ( + + {target.lastStatus} + + ) : null} + + {target.requests} req + +
+
+ +
+ + 0 ? 1 : 0} + meta={formatLatency(target.avgLatencyMs)} + /> + +
+ +
+ Quota scope: {target.quotaScope} + {target.quotaTrend ? Trend: {target.quotaTrend} : null} + {target.quotaIsExhausted ? Exhausted : null} +
+
+ ))} +
+
+ ) : null} ); } diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index fbc7513896..34a529128d 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -1,24 +1,53 @@ "use client"; import { useState, useEffect, useCallback, useRef } from "react"; -import { - Card, - Button, - Modal, - Input, - Toggle, - CardSkeleton, - ModelSelectModal, - ProxyConfigModal, - EmptyState, -} from "@/shared/components"; +import dynamic from "next/dynamic"; +import Button from "@/shared/components/Button"; +import Card from "@/shared/components/Card"; +import { CardSkeleton } from "@/shared/components/Loading"; +import EmptyState from "@/shared/components/EmptyState"; +import Input from "@/shared/components/Input"; +import Modal from "@/shared/components/Modal"; +import Toggle from "@/shared/components/Toggle"; import Tooltip from "@/shared/components/Tooltip"; -import ModelRoutingSection from "@/shared/components/ModelRoutingSection"; import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard"; import { useNotificationStore } from "@/store/notificationStore"; import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies"; +import { + COMBO_BUILDER_AUTO_CONNECTION, + COMBO_BUILDER_STAGES, + buildPrecisionComboModelStep, + canAccessComboBuilderStage, + findNextSuggestedConnectionId, + getComboBuilderStageChecks, + getNextComboBuilderStage, + getPreviousComboBuilderStage, + hasExactModelStepDuplicate, + parseQualifiedModel, +} from "@/lib/combos/builderDraft"; import { useTranslations } from "next-intl"; +const ModelSelectModal = dynamic(() => import("@/shared/components/ModelSelectModal"), { + ssr: false, +}); +const ProxyConfigModal = dynamic(() => import("@/shared/components/ProxyConfigModal"), { + ssr: false, +}); +const ModelRoutingSection = dynamic(() => import("@/shared/components/ModelRoutingSection"), { + ssr: false, + loading: () => ( +
+
+ route +
+
+
+
+
+
+ ), +}); + // Validate combo name: letters, numbers, -, _, /, . const VALID_NAME_REGEX = /^[a-zA-Z0-9_/.-]+$/; @@ -197,6 +226,32 @@ const STRATEGY_RECOMMENDATIONS_FALLBACK = { }; const COMBO_USAGE_GUIDE_STORAGE_KEY = "omniroute:combos:hide-usage-guide"; +const COMBO_FORM_STAGE_META = [ + { + id: "basics", + fallbackLabel: "Basics", + fallbackDescription: "Name and starting template.", + icon: "looks_one", + }, + { + id: "steps", + fallbackLabel: "Steps", + fallbackDescription: "Provider, model and account selection.", + icon: "looks_two", + }, + { + id: "strategy", + fallbackLabel: "Strategy", + fallbackDescription: "Routing behavior and advanced settings.", + icon: "looks_3", + }, + { + id: "review", + fallbackLabel: "Review", + fallbackDescription: "Final verification before saving.", + icon: "fact_check", + }, +]; const COMBO_TEMPLATE_FALLBACK = { title: "Quick templates", @@ -367,11 +422,88 @@ function getStrategyRecommendationText(t, strategy, field) { // ───────────────────────────────────────────── function normalizeModelEntry(entry) { if (typeof entry === "string") return { model: entry, weight: 0 }; - return { model: entry.model, weight: entry.weight || 0 }; + if (entry?.kind === "combo-ref") { + return { + ...entry, + model: entry.comboName, + weight: entry.weight || 0, + }; + } + return { + ...entry, + model: entry.model, + weight: entry.weight || 0, + }; } function getModelString(entry) { - return typeof entry === "string" ? entry : entry.model; + if (typeof entry === "string") return entry; + if (entry?.kind === "combo-ref") return entry.comboName; + return entry.model; +} + +function findProviderNodeByIdentifier(providerNodes, providerIdentifier) { + return (providerNodes || []).find( + (node) => node.id === providerIdentifier || node.prefix === providerIdentifier + ); +} + +function findBuilderProviderByIdentifier(builderProviders, providerIdentifier) { + return (builderProviders || []).find( + (provider) => + provider.providerId === providerIdentifier || + provider.alias === providerIdentifier || + provider.prefix === providerIdentifier + ); +} + +function formatComboEntryDisplay( + entry, + { + providerNodes = [], + builderProviders = [], + includeConnection = false, + }: { + providerNodes?: any[]; + builderProviders?: any[]; + includeConnection?: boolean; + } = {} +) { + const normalizedEntry = normalizeModelEntry(entry); + if (normalizedEntry.kind === "combo-ref") { + return `Combo → ${normalizedEntry.comboName}`; + } + + const parsed = parseQualifiedModel(normalizedEntry.model); + if (!parsed) return normalizedEntry.model; + + const providerIdentifier = normalizedEntry.providerId || parsed.providerId; + const builderProvider = findBuilderProviderByIdentifier(builderProviders, providerIdentifier); + const providerNode = findProviderNodeByIdentifier(providerNodes, providerIdentifier); + const providerLabel = builderProvider?.displayName || providerNode?.name || providerIdentifier; + const modelLabel = + builderProvider?.models?.find((model) => model.id === parsed.modelId)?.name || parsed.modelId; + + if (!includeConnection) { + return `${providerLabel}/${modelLabel}`; + } + + const connectionId = normalizedEntry.connectionId || null; + const connectionLabel = + (connectionId && + builderProvider?.connections?.find((connection) => connection.id === connectionId)?.label) || + normalizedEntry.label || + null; + + if (connectionId) { + return `${providerLabel}/${modelLabel} · ${connectionLabel || `acct ${connectionId.slice(0, 8)}`}`; + } + + if (normalizedEntry.providerId || builderProvider) { + return `${providerLabel}/${modelLabel} · dynamic account`; + } + + return `${providerLabel}/${modelLabel}`; } // ───────────────────────────────────────────── @@ -398,6 +530,7 @@ export default function CombosPage() { const [comboDragIndex, setComboDragIndex] = useState(null); const [comboDragOverIndex, setComboDragOverIndex] = useState(null); const [savingComboOrder, setSavingComboOrder] = useState(false); + const comboDragIndexRef = useRef(null); useEffect(() => { fetchData(); @@ -571,6 +704,7 @@ export default function CombosPage() { }; const resetComboDragState = () => { + comboDragIndexRef.current = null; setComboDragIndex(null); setComboDragOverIndex(null); }; @@ -580,6 +714,7 @@ export default function CombosPage() { e.preventDefault(); return; } + comboDragIndexRef.current = index; setComboDragIndex(index); e.dataTransfer.effectAllowed = "move"; e.dataTransfer.setData("text/plain", combos[index]?.id || `${index}`); @@ -599,14 +734,15 @@ export default function CombosPage() { const handleComboDragOver = (e, index) => { e.preventDefault(); - if (comboDragIndex === null || comboDragIndex === index) return; + const activeDragIndex = comboDragIndexRef.current ?? comboDragIndex; + if (activeDragIndex === null || activeDragIndex === index) return; e.dataTransfer.dropEffect = "move"; setComboDragOverIndex(index); }; const handleComboDrop = async (e, dropIndex) => { e.preventDefault(); - const fromIndex = comboDragIndex; + const fromIndex = comboDragIndexRef.current ?? comboDragIndex; resetComboDragState(); if (fromIndex === null || fromIndex === dropIndex) return; @@ -1080,17 +1216,6 @@ function ComboCard({ const tc = useTranslations("common"); const strategyDescription = getStrategyDescription(t, strategy); - // Resolve provider UUID to user-defined name - const formatModelDisplay = (modelValue) => { - const parts = modelValue.split("/"); - if (parts.length !== 2) return modelValue; - const [providerIdentifier, modelId] = parts; - const matchedNode = (providerNodes || []).find( - (node) => node.id === providerIdentifier || node.prefix === providerIdentifier - ); - return matchedNode ? `${matchedNode.name}/${modelId}` : modelValue; - }; - return ( {t("noModels")} ) : ( models.slice(0, 3).map((entry, index) => { - const { model, weight } = normalizeModelEntry(entry); + const { weight } = normalizeModelEntry(entry); return ( - {formatModelDisplay(model)} + {formatComboEntryDisplay(entry, { + providerNodes, + includeConnection: true, + })} {strategy === "weighted" && weight > 0 ? ` (${weight}%)` : ""} ); @@ -1281,12 +1409,24 @@ function TestResultsView({ results }) { check_circle - - Resolved by:{" "} - - {results.resolvedBy} - - +
+
+ Resolved by:{" "} + + {results.resolvedBy} + +
+ {results.resolvedByTarget?.connectionId || results.resolvedByTarget?.stepId ? ( +
+ {results.resolvedByTarget?.connectionId + ? `account ${results.resolvedByTarget.connectionId.slice(0, 8)}` + : "dynamic account"} + {results.resolvedByTarget?.stepId + ? ` · step ${results.resolvedByTarget.stepId}` + : ""} +
+ ) : null} +
)} {results.results?.map((r, i) => ( @@ -1306,7 +1446,15 @@ function TestResultsView({ results }) { > {r.status === "ok" ? "check_circle" : r.status === "skipped" ? "skip_next" : "error"} - {r.model} +
+ {r.label || r.model} + {r.connectionId || r.stepId ? ( +
+ {r.connectionId ? `acct ${r.connectionId.slice(0, 8)}` : "dynamic account"} + {r.stepId ? ` · ${r.stepId}` : ""} +
+ ) : null} +
{r.latencyMs !== undefined && {r.latencyMs}ms} { - const parts = modelValue.split("/"); - if (parts.length !== 2) return false; + const parsed = parseQualifiedModel(modelValue); + if (!parsed) return false; - const [providerIdentifier, modelId] = parts; - const matchedNode = providerNodes.find( - (node) => node.id === providerIdentifier || node.prefix === providerIdentifier - ); + const { providerId: providerIdentifier, modelId } = parsed; + const matchedNode = findProviderNodeByIdentifier(providerNodes, providerIdentifier); const providerCandidates = [providerIdentifier]; if (matchedNode?.apiType) providerCandidates.push(matchedNode.apiType); @@ -1453,6 +1607,32 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { const [dragIndex, setDragIndex] = useState(null); const [dragOverIndex, setDragOverIndex] = useState(null); + const builderProviders = builderOptions.providers || []; + const builderComboRefs = (builderOptions.comboRefs || []).filter( + (comboRef) => comboRef.name !== combo?.name && comboRef.name !== name.trim() + ); + const selectedBuilderProvider = + builderProviders.find((provider) => provider.providerId === builderProviderId) || null; + const selectedBuilderModel = + selectedBuilderProvider?.models?.find((model) => model.id === builderModelId) || null; + const selectedBuilderConnections = selectedBuilderProvider?.connections || []; + const selectedBuilderConnection = + builderConnectionId && builderConnectionId !== COMBO_BUILDER_AUTO_CONNECTION + ? selectedBuilderConnections.find((connection) => connection.id === builderConnectionId) || + null + : null; + const builderCandidateStep = + selectedBuilderProvider && selectedBuilderModel + ? buildPrecisionComboModelStep({ + providerId: selectedBuilderProvider.providerId, + modelId: selectedBuilderModel.id, + connectionId: + builderConnectionId !== COMBO_BUILDER_AUTO_CONNECTION ? builderConnectionId : null, + connectionLabel: selectedBuilderConnection?.label || null, + }) + : null; + const builderHasDuplicate = + builderCandidateStep && hasExactModelStepDuplicate(models, builderCandidateStep); const weightTotal = models.reduce((sum, modelEntry) => sum + (modelEntry.weight || 0), 0); const pricedModelCount = models.reduce( (count, modelEntry) => count + (hasPricingForModel(modelEntry.model) ? 1 : 0), @@ -1471,6 +1651,33 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { pricedModelCount < models.length; const hasInvalidWeightedTotal = strategy === "weighted" && models.length > 0 && weightTotal !== 100; + const builderStageChecks = getComboBuilderStageChecks({ + name, + nameError, + modelsCount: models.length, + hasInvalidWeightedTotal, + hasCostOptimizedWithoutPricing, + }); + const canAdvanceFromCurrentStage = + builderStage === "basics" + ? builderStageChecks.basics + : builderStage === "steps" + ? builderStageChecks.steps + : true; + const currentStageIndex = COMBO_FORM_STAGE_META.findIndex( + (stageMeta) => stageMeta.id === builderStage + ); + const pinnedAccountCount = models.filter((entry) => Boolean(entry?.connectionId)).length; + const comboRefCount = models.filter((entry) => entry?.kind === "combo-ref").length; + const uniqueProviderCount = new Set( + models + .map((entry) => { + const target = getModelString(entry); + const parsed = parseQualifiedModel(target); + return entry?.providerId || parsed?.providerId || null; + }) + .filter(Boolean) + ).size; const saveBlocked = !name.trim() || !!nameError || @@ -1533,11 +1740,13 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { } const fetchModalData = async () => { + setBuilderLoading(true); try { - const [aliasesRes, nodesRes, pricingRes] = await Promise.all([ + const [aliasesRes, nodesRes, pricingRes, builderRes] = await Promise.all([ fetch("/api/models/alias"), fetch("/api/provider-nodes"), fetch("/api/pricing"), + fetch("/api/combos/builder/options"), ]); if (!aliasesRes.ok || !nodesRes.ok) { @@ -1546,6 +1755,7 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { ); } const pricingData = pricingRes.ok ? await pricingRes.json() : {}; + const builderData = builderRes.ok ? await builderRes.json() : {}; const [aliasesData, nodesData] = await Promise.all([aliasesRes.json(), nodesRes.json()]); setPricingByProvider( @@ -1555,8 +1765,15 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { ); setModelAliases(aliasesData.aliases || {}); setProviderNodes(nodesData.nodes || []); + setBuilderOptions({ + providers: builderData.providers || [], + comboRefs: builderData.comboRefs || [], + }); } catch (error) { console.error("Error fetching modal data:", error); + setBuilderOptions({ providers: [], comboRefs: [] }); + } finally { + setBuilderLoading(false); } }; @@ -1564,6 +1781,16 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { if (isOpen) fetchModalData(); }, [isOpen]); + useEffect(() => { + if (!isOpen) return; + setBuilderProviderId(""); + setBuilderModelId(""); + setBuilderConnectionId(COMBO_BUILDER_AUTO_CONNECTION); + setBuilderComboRefName(""); + setBuilderError(""); + setBuilderStage("basics"); + }, [combo?.id, isOpen]); + useEffect(() => { if (!isOpen) return; @@ -1610,6 +1837,14 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { }; }, [combo, getEmptyCreateDraftSnapshot, isOpen, resetFormForCombo]); + useEffect(() => { + if (!isOpen) return; + if (builderProviderId) return; + if (builderProviders.length === 1) { + setBuilderProviderId(builderProviders[0].providerId); + } + }, [builderProviderId, builderProviders, isOpen]); + useEffect(() => { if (!strategyChangeMountedRef.current) { strategyChangeMountedRef.current = true; @@ -1641,10 +1876,113 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { else setNameError(""); }; - const handleAddModel = (model) => { - if (!models.find((m) => m.model === model.value)) { - setModels([...models, { model: model.value, weight: 0 }]); + const handleBuilderProviderChange = (e) => { + const nextProviderId = e.target.value; + setBuilderProviderId(nextProviderId); + setBuilderModelId(""); + setBuilderConnectionId(COMBO_BUILDER_AUTO_CONNECTION); + setBuilderError(""); + }; + + const handleBuilderModelChange = (e) => { + const nextModelId = e.target.value; + setBuilderModelId(nextModelId); + setBuilderError(""); + + if (!nextModelId || !selectedBuilderProvider) { + setBuilderConnectionId(COMBO_BUILDER_AUTO_CONNECTION); + return; } + + setBuilderConnectionId( + findNextSuggestedConnectionId( + models, + selectedBuilderProvider.providerId, + nextModelId, + selectedBuilderProvider.connections || [] + ) + ); + }; + + const handleBuilderConnectionChange = (e) => { + setBuilderConnectionId(e.target.value || COMBO_BUILDER_AUTO_CONNECTION); + setBuilderError(""); + }; + + const handleGoToNextStage = () => { + setBuilderStage((currentStage) => getNextComboBuilderStage(currentStage)); + }; + + const handleGoToPreviousStage = () => { + setBuilderStage((currentStage) => getPreviousComboBuilderStage(currentStage)); + }; + + const handleAddBuilderStep = () => { + if (!selectedBuilderProvider || !selectedBuilderModel) { + return; + } + + const nextStep = buildPrecisionComboModelStep({ + providerId: selectedBuilderProvider.providerId, + modelId: selectedBuilderModel.id, + connectionId: + builderConnectionId !== COMBO_BUILDER_AUTO_CONNECTION ? builderConnectionId : null, + connectionLabel: selectedBuilderConnection?.label || null, + }); + + if (hasExactModelStepDuplicate(models, nextStep)) { + setBuilderError( + getI18nOrFallback( + t, + "builderDuplicateExact", + "This exact provider/model/account step is already in the combo." + ) + ); + return; + } + + const nextModels = [...models, nextStep]; + setModels(nextModels); + setBuilderError(""); + setBuilderConnectionId( + findNextSuggestedConnectionId( + nextModels, + selectedBuilderProvider.providerId, + selectedBuilderModel.id, + selectedBuilderConnections + ) + ); + }; + + const handleAddComboReference = () => { + if (!builderComboRefName) return; + + setModels([ + ...models, + { + kind: "combo-ref", + comboName: builderComboRefName, + weight: 0, + }, + ]); + setBuilderComboRefName(""); + setBuilderError(""); + }; + + const handleAddModel = (model) => { + const nextEntry = { model: model.value, weight: 0 }; + if (hasExactModelStepDuplicate(models, nextEntry)) { + setBuilderError( + getI18nOrFallback( + t, + "builderDuplicateExact", + "This exact provider/model/account step is already in the combo." + ) + ); + return; + } + setModels([...models, nextEntry]); + setBuilderError(""); }; const handleRemoveModel = (index) => { @@ -1751,23 +2089,14 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { // Format model display name with readable provider name const formatModelDisplay = useCallback( - (modelValue) => { - const parts = modelValue.split("/"); - if (parts.length !== 2) return modelValue; - - const [providerIdentifier, modelId] = parts; - // Match by node ID or prefix - const matchedNode = providerNodes.find( - (node) => node.id === providerIdentifier || node.prefix === providerIdentifier - ); - - if (matchedNode) { - return `${matchedNode.name}/${modelId}`; - } - - return modelValue; + (entry) => { + return formatComboEntryDisplay(entry, { + providerNodes, + builderProviders, + includeConnection: true, + }); }, - [providerNodes] + [builderProviders, providerNodes] ); const handleMoveUp = (index) => { @@ -1827,7 +2156,7 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { const saveData: any = { name: name.trim(), - models: strategy === "weighted" ? models : models.map((m) => m.model), + models, strategy, }; @@ -1866,683 +2195,1152 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders }) { size="full" >
- {/* Name */} -
- -

{t("nameHint")}

-
- - {!isEdit && ( -
-
-

- {getI18nOrFallback(t, "templatesTitle", COMBO_TEMPLATE_FALLBACK.title)} +

+
+
+

+ {getI18nOrFallback(t, "builderFlowTitle", "Combo Builder Flow")}

{getI18nOrFallback( t, - "templatesDescription", - COMBO_TEMPLATE_FALLBACK.description + "builderFlowDescription", + "Move through the stages in order to define the combo, build the steps, choose the routing strategy and review the result." )}

-
- {COMBO_TEMPLATES.map((template) => ( - - ))} -
-
- )} - - {/* Strategy Toggle */} -
-
- - - - help - - -
-
- {STRATEGY_OPTIONS.map((s) => ( - - ))} -
-

- {getStrategyDescription(t, strategy)} -

-
- -
-
- -
-
- - {/* Models */} -
-
- - {strategy === "weighted" && models.length > 1 && ( - - )} -
- - {models.length === 0 ? ( -
- - layers - -

{t("noModelsYet")}

-
- ) : ( -
- {models.map((entry, index) => ( -
handleDragStart(e, index)} - onDragEnd={handleDragEnd} - onDragOver={(e) => handleDragOver(e, index)} - onDrop={(e) => handleDrop(e, index)} - className={`group/item flex items-center gap-1.5 px-2 py-1.5 rounded-md transition-all cursor-grab active:cursor-grabbing ${ - dragOverIndex === index && dragIndex !== index - ? "bg-primary/10 border border-primary/30" - : "bg-black/[0.02] dark:bg-white/[0.02] hover:bg-black/[0.04] dark:hover:bg-white/[0.04] border border-transparent" - } ${dragIndex === index ? "opacity-50" : ""}`} - > - {/* Drag handle */} - - drag_indicator - - - {/* Index badge */} - - {index + 1} - - - {/* Model display */} -
- {formatModelDisplay(entry.model)} -
- - {strategy === "cost-optimized" && ( - - {hasPricingForModel(entry.model) - ? getI18nOrFallback(t, "pricingAvailableShort", "priced") - : getI18nOrFallback(t, "pricingMissingShort", "no-price")} - - )} - - {/* Weight input (weighted mode only) */} - {strategy === "weighted" && ( -
- handleWeightChange(index, e.target.value)} - className="w-10 text-[11px] text-center py-0.5 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> - % -
- )} - - {/* Priority arrows (priority mode) */} - {strategy === "priority" && ( -
- - -
- )} - - {/* Remove */} - -
- ))} -
- )} - - {/* Weight total indicator */} - {strategy === "weighted" && models.length > 0 && } - - {strategy === "cost-optimized" && models.length > 0 && ( -
-
- - {getI18nOrFallback(t, "pricingCoverage", "Pricing coverage")} - - - {pricedModelCount}/{models.length} ({pricingCoveragePercent}%) - -
-
-
0 - ? "bg-amber-500" - : "bg-red-500" - }`} - style={{ width: `${pricingCoveragePercent}%` }} - /> -
-

- {getI18nOrFallback( - t, - "pricingCoverageHint", - "Cost-optimized works best when all combo models have pricing." - )} -

-
- )} - - {hasNoModels && ( -
- warning - {t("noModelsYet")} -
- )} - - {hasInvalidWeightedTotal && ( -
- warning - - {t("weighted")} {weightTotal}% {"\u2260"} 100%. {t("autoBalance")} - -
- )} - - {hasRoundRobinSingleModel && ( -
- info - - {getI18nOrFallback( - t, - "warningRoundRobinSingleModel", - "Round-robin is most useful with at least 2 models." - )} - -
- )} - - {hasCostOptimizedPartialPricing && ( -
- warning - - {typeof t.has === "function" && t.has("warningCostOptimizedPartialPricing") - ? t("warningCostOptimizedPartialPricing", { - priced: pricedModelCount, - total: models.length, - }) - : `Only ${pricedModelCount} of ${models.length} models have pricing. Routing may be partially cost-aware.`} - -
- )} - - {hasCostOptimizedWithoutPricing && ( -
- warning - - {getI18nOrFallback( - t, - "warningCostOptimizedNoPricing", - "No pricing data found for this combo. Cost-optimized may route unexpectedly." - )} - -
- )} - -
- -
- - {/* Add Model button */} - -
- - {/* Advanced Config Toggle */} - - - {showAdvanced && ( -
-
-
- - - setConfig({ - ...config, - maxRetries: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - - setConfig({ - ...config, - retryDelayMs: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - - setConfig({ - ...config, - timeoutMs: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - setConfig({ ...config, healthCheckEnabled: e.target.checked })} - className="accent-primary" - /> -
-
- {strategy === "round-robin" && ( -
-
- - - setConfig({ - ...config, - concurrencyPerModel: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - - setConfig({ - ...config, - queueTimeoutMs: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- )} - {strategy === "context-relay" && ( -
-
- - - setConfig({ - ...config, - handoffThreshold: e.target.value ? Number(e.target.value) : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - - setConfig({ - ...config, - maxMessagesForSummary: e.target.value - ? Number(e.target.value) - : undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
- - - setConfig({ - ...config, - handoffModel: e.target.value || undefined, - }) - } - className="w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none" - /> -
-
-

- {getI18nOrFallback( - t, - "contextRelayProviderNote", - "Context Relay currently generates handoffs for Codex account rotation. Pair it with multiple accounts of the same provider for the best continuity." - )} -

-
-
- )} -

{t("advancedHint")}

-
- )} - - {/* Agent Features (#399 / #401 / #454) */} -
-
- smart_toy -

Agent Features

- - — optional, for agent/tool workflows + + {currentStageIndex + 1}/{COMBO_FORM_STAGE_META.length}
- {/* System Message Override */} -
- -