From b56a94ca3ee075326faa7cd53b878c6a14e04d63 Mon Sep 17 00:00:00 2001 From: KooshaPari <42529354+KooshaPari@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:54:50 -0700 Subject: [PATCH] feat(relay): gate bifrost auto routing by provider manifest (#5870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.44 — gates Bifrost auto-routing by the provider plugin manifest (only manifest-eligible providers reach the sidecar; ineligible/unknown fall back to the TS path with explicit reasons). Superset of #5869 (carries the full manifest + registry + docs). Resolved an integration-test conflict in favor of the release (which already subsumes this PR's readiness/removeDirWithRetry improvements). Validated locally: 4 provider-plugin-manifest + 11 relay-routing-backend tests green. Thanks @KooshaPari! Co-authored-by: diegosouzapw --- docs/README.md | 1 + docs/reference/PROVIDER_PLUGIN_MANIFEST.md | 73 +++++++ docs/reference/RELAY_BACKEND_STRATEGY.md | 7 + open-sse/config/providerPluginManifest.ts | 186 ++++++++++++++++++ .../config/providerPluginManifestRegistry.ts | 31 +++ .../api/v1/relay/chat/completions/route.ts | 14 +- .../relay/chat/completions/routingBackend.ts | 39 ++++ .../unit/api/v1/relay-routing-backend.test.ts | 54 +++++ tests/unit/provider-plugin-manifest.test.ts | 121 ++++++++++++ 9 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 docs/reference/PROVIDER_PLUGIN_MANIFEST.md create mode 100644 open-sse/config/providerPluginManifest.ts create mode 100644 open-sse/config/providerPluginManifestRegistry.ts create mode 100644 tests/unit/provider-plugin-manifest.test.ts diff --git a/docs/README.md b/docs/README.md index df0040172e..d43de85363 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,6 +71,7 @@ Lookup material — API surface, environment variables, CLI flags, provider cata - [API_REFERENCE.md](reference/API_REFERENCE.md) — REST API endpoints and shapes. - [PROVIDER_REFERENCE.md](reference/PROVIDER_REFERENCE.md) — auto-generated provider catalog (do not edit by hand). +- [PROVIDER_PLUGIN_MANIFEST.md](reference/PROVIDER_PLUGIN_MANIFEST.md) — sidecar-safe provider plugin contract for Bifrost and CLIProxyAPI migration. - [openapi.yaml](openapi.yaml) — OpenAPI spec for the public API. - [ENVIRONMENT.md](reference/ENVIRONMENT.md) — environment variables reference. - [FEATURE_FLAGS.md](reference/FEATURE_FLAGS.md) — feature flags and their defaults. diff --git a/docs/reference/PROVIDER_PLUGIN_MANIFEST.md b/docs/reference/PROVIDER_PLUGIN_MANIFEST.md new file mode 100644 index 0000000000..ca33e9e264 --- /dev/null +++ b/docs/reference/PROVIDER_PLUGIN_MANIFEST.md @@ -0,0 +1,73 @@ +--- +title: "Provider Plugin Manifest" +version: 3.8.42 +lastUpdated: 2026-07-01 +--- + +# Provider Plugin Manifest + +`open-sse/config/providerPluginManifest.ts` defines the JSON-safe provider +plugin contract. `open-sse/config/providerPluginManifestRegistry.ts` binds that +contract to the current provider registry for sidecars such as Bifrost, +CLIProxyAPI, or a future Go/Rust router. The TypeScript registry remains the +source of truth, but sidecars can consume the manifest without importing +executor code, OAuth defaults, headers, or process environment state. + +## Goal + +Move provider metadata toward a plugin contract so the hot request path can +eventually be owned by a lower-latency sidecar while OmniRoute keeps the +TypeScript route as the policy gate and fallback. The manifest is additive: it +does not change request routing by itself. + +## Contract + +The manifest contains: + +- provider id and alias +- upstream format and executor name +- auth type, auth header, and optional auth prefix +- static endpoint metadata +- sidecar eligibility and explicit reasons when a provider should stay on TS +- JSON-safe model metadata such as context length, vision/reasoning flags, and + unsupported params +- capability tags including `apikey`, `oauth`, `custom-executor`, + `passthrough-models`, `responses`, and `sidecar-candidate` + +The manifest intentionally excludes: + +- OAuth client secrets and default secret values +- runtime environment resolution +- request headers and public credential helpers +- dynamic URL builders +- executor functions +- session pool internals + +## Sidecar Use + +Sidecars should treat `sidecar.eligible` as a conservative candidate signal, not +as an unconditional routing decision. The first import target should be +API-key, static-endpoint providers using the default executor. Providers with +custom web executors, OAuth/session flows, dynamic URL builders, or pool config +stay on the TypeScript fallback path until a sidecar implements equivalent +behavior and telemetry proves parity. + +Suggested migration phases: + +1. Generate and validate the provider plugin manifest from the TS registry. +2. Teach Bifrost or CLIProxyAPI to import the manifest for API-key/static + providers. +3. Route eligible providers through the sidecar behind `OMNIROUTE_RELAY_BACKEND` + while keeping TS fallback enabled. +4. Promote providers only when success rate, p99 latency, streaming behavior, + and unsupported-param handling match the TS path. +5. Add sidecar-native plugins for custom executors one provider family at a + time. + +## Why Not Embed Providers Directly In Next + +The Next frontend should not own provider execution. It should call the API +boundary. The backend can then decide whether to use the TypeScript executor, +Bifrost, CLIProxyAPI, or a future native sidecar. This keeps request signing, +allowlist checks, DB policy, and fallback behavior centralized before any +sidecar handoff. diff --git a/docs/reference/RELAY_BACKEND_STRATEGY.md b/docs/reference/RELAY_BACKEND_STRATEGY.md index 6db4942e74..69e38f28a0 100644 --- a/docs/reference/RELAY_BACKEND_STRATEGY.md +++ b/docs/reference/RELAY_BACKEND_STRATEGY.md @@ -75,3 +75,10 @@ For sustained high RPM/RPS and strict success SLO: - `BIFROST_ENABLED=1` - Keep API keys, allowlist, sanitizer, and rate-limit checks enabled in route handlers (they always run before downstream forwarding). - Export fallback metrics from your reverse proxy and request logs so sidecar outages are visible within one minute. + +## Provider plugin contract + +Sidecars should import provider metadata through the JSON-safe provider plugin +manifest instead of depending on TypeScript executor internals. See +[Provider Plugin Manifest](./PROVIDER_PLUGIN_MANIFEST.md) for the sidecar +eligibility contract and migration phases. diff --git a/open-sse/config/providerPluginManifest.ts b/open-sse/config/providerPluginManifest.ts new file mode 100644 index 0000000000..b0a773e17c --- /dev/null +++ b/open-sse/config/providerPluginManifest.ts @@ -0,0 +1,186 @@ +import type { RegistryEntry, RegistryModel } from "./providers/shared.ts"; + +export type ProviderPluginCapability = + | "apikey" + | "custom-executor" + | "oauth" + | "passthrough-models" + | "responses" + | "sidecar-candidate"; + +export interface ProviderPluginModel { + id: string; + name: string; + contextLength?: number; + maxOutputTokens?: number; + toolCalling?: boolean; + supportsReasoning?: boolean; + supportsVision?: boolean; + unsupportedParams?: readonly string[]; + targetFormat?: string; +} + +export interface ProviderPluginManifestEntry { + id: string; + alias?: string; + format: string; + executor: string; + auth: { + type: string; + header: string; + prefix?: string; + }; + endpoints: { + baseUrl?: string; + baseUrls?: string[]; + responsesBaseUrl?: string; + chatPath?: string; + modelsUrl?: string; + }; + capabilities: ProviderPluginCapability[]; + passthroughModels: boolean; + defaultContextLength?: number; + timeoutMs?: number; + models: ProviderPluginModel[]; + sidecar: { + eligible: boolean; + reasons: string[]; + }; +} + +export interface ProviderPluginManifest { + schemaVersion: 1; + generatedFrom: "open-sse/config/providers"; + providers: ProviderPluginManifestEntry[]; +} + +const SIDECAR_COMPATIBLE_EXECUTORS = new Set(["default"]); + +function compactObject>(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter(([, entryValue]) => entryValue !== undefined), + ) as Partial; +} + +function mapModel(model: RegistryModel): ProviderPluginModel { + return compactObject({ + id: model.id, + name: model.name, + contextLength: model.contextLength, + maxOutputTokens: model.maxOutputTokens, + toolCalling: model.toolCalling, + supportsReasoning: model.supportsReasoning, + supportsVision: model.supportsVision, + unsupportedParams: model.unsupportedParams, + targetFormat: model.targetFormat, + }) as ProviderPluginModel; +} + +function sidecarEligibility(entry: RegistryEntry): { eligible: boolean; reasons: string[] } { + const reasons: string[] = []; + + if (!SIDECAR_COMPATIBLE_EXECUTORS.has(entry.executor)) { + reasons.push(`custom executor: ${entry.executor}`); + } + if (entry.authType !== "apikey" && entry.authType !== "optional" && entry.authType !== "none") { + reasons.push(`auth type requires TS handling: ${entry.authType}`); + } + if (!entry.baseUrl && !entry.baseUrls?.length && !entry.responsesBaseUrl) { + reasons.push("no static upstream endpoint"); + } + if (typeof entry.urlBuilder === "function") { + reasons.push("dynamic URL builder"); + } + if (entry.oauth) { + reasons.push("oauth metadata"); + } + if (entry.poolConfig) { + reasons.push("session pool config"); + } + + return { + eligible: reasons.length === 0, + reasons, + }; +} + +function capabilitiesFor(entry: RegistryEntry, eligible: boolean): ProviderPluginCapability[] { + const capabilities = new Set(); + + if (entry.authType === "apikey" || entry.authType === "optional") { + capabilities.add("apikey"); + } + if (entry.authType === "oauth" || entry.oauth) { + capabilities.add("oauth"); + } + if (entry.responsesBaseUrl) { + capabilities.add("responses"); + } + if (entry.passthroughModels) { + capabilities.add("passthrough-models"); + } + if (entry.executor !== "default") { + capabilities.add("custom-executor"); + } + if (eligible) { + capabilities.add("sidecar-candidate"); + } + + return [...capabilities].sort(); +} + +export function createProviderPluginManifestEntry( + entry: RegistryEntry, +): ProviderPluginManifestEntry { + const sidecar = sidecarEligibility(entry); + + return { + id: entry.id, + ...(entry.alias ? { alias: entry.alias } : {}), + format: entry.format, + executor: entry.executor, + auth: compactObject({ + type: entry.authType, + header: entry.authHeader, + prefix: entry.authPrefix, + }) as ProviderPluginManifestEntry["auth"], + endpoints: compactObject({ + baseUrl: entry.baseUrl, + baseUrls: entry.baseUrls, + responsesBaseUrl: entry.responsesBaseUrl, + chatPath: entry.chatPath, + modelsUrl: entry.modelsUrl, + }) as ProviderPluginManifestEntry["endpoints"], + capabilities: capabilitiesFor(entry, sidecar.eligible), + passthroughModels: entry.passthroughModels === true, + ...(typeof entry.defaultContextLength === "number" + ? { defaultContextLength: entry.defaultContextLength } + : {}), + ...(typeof entry.timeoutMs === "number" ? { timeoutMs: entry.timeoutMs } : {}), + models: (entry.models ?? []).map(mapModel), + sidecar, + }; +} + +export function generateProviderPluginManifestFromRegistry( + registry: Record, +): ProviderPluginManifest { + return { + schemaVersion: 1, + generatedFrom: "open-sse/config/providers", + providers: Object.values(registry) + .map(createProviderPluginManifestEntry) + .sort((a, b) => a.id.localeCompare(b.id)), + }; +} + +export function getProviderPluginManifestEntryFromRegistry( + registry: Record, + provider: string, +): ProviderPluginManifestEntry | null { + const entry = + registry[provider] || + Object.values(registry).find((candidate) => candidate.alias === provider); + + return entry ? createProviderPluginManifestEntry(entry) : null; +} diff --git a/open-sse/config/providerPluginManifestRegistry.ts b/open-sse/config/providerPluginManifestRegistry.ts new file mode 100644 index 0000000000..37b92ba4fb --- /dev/null +++ b/open-sse/config/providerPluginManifestRegistry.ts @@ -0,0 +1,31 @@ +import { REGISTRY } from "./providers/index.ts"; +import { + generateProviderPluginManifestFromRegistry, + getProviderPluginManifestEntryFromRegistry, + type ProviderPluginManifestEntry, +} from "./providerPluginManifest.ts"; + +export function generateProviderPluginManifest() { + return generateProviderPluginManifestFromRegistry(REGISTRY); +} + +export function getProviderPluginManifestEntry(provider: string) { + return getProviderPluginManifestEntryFromRegistry(REGISTRY, provider); +} + +export function getProviderPluginManifestEntryForModel( + model: string | undefined, +): ProviderPluginManifestEntry | null { + if (!model) return null; + + const providerPrefix = model.includes("/") ? model.split("/", 1)[0] : ""; + if (providerPrefix) { + const prefixed = getProviderPluginManifestEntry(providerPrefix); + if (prefixed) return prefixed; + } + + const manifest = generateProviderPluginManifest(); + return manifest.providers.find((provider) => + provider.models.some((candidate) => candidate.id === model), + ) ?? null; +} diff --git a/src/app/api/v1/relay/chat/completions/route.ts b/src/app/api/v1/relay/chat/completions/route.ts index d8d7afd693..9cfd7fe3de 100644 --- a/src/app/api/v1/relay/chat/completions/route.ts +++ b/src/app/api/v1/relay/chat/completions/route.ts @@ -22,9 +22,10 @@ import { getBifrostRoutingConfig, getRoutingFallbackHeader, resolveRelayRoutingBackend, - shouldTryBifrost, + shouldTryBifrostForRequest, type BifrostRoutingConfig, } from "./routingBackend"; +import { getProviderPluginManifestEntryForModel } from "@omniroute/open-sse/config/providerPluginManifestRegistry.ts"; import { finalizeReadableStream } from "./streamFinalizer"; import { clearBifrostFailure, @@ -290,7 +291,16 @@ export async function POST(request: Request) { const backend = resolveRelayRoutingBackend(); const bifrostConfig = getBifrostRoutingConfig(); let bifrostFallbackReason: string | null = null; - if (shouldTryBifrost(backend, bifrostConfig)) { + const bifrostDecision = shouldTryBifrostForRequest( + backend, + bifrostConfig, + parsedBody, + (model) => getProviderPluginManifestEntryForModel(model)?.sidecar ?? null + ); + if (bifrostDecision.fallbackReason) { + bifrostFallbackReason = bifrostDecision.fallbackReason; + } + if (bifrostDecision.tryBifrost) { const cooldown = backend === "auto" ? getActiveBifrostCooldown(bifrostConfig.baseUrl) : null; if (cooldown) { diff --git a/src/app/api/v1/relay/chat/completions/routingBackend.ts b/src/app/api/v1/relay/chat/completions/routingBackend.ts index 8604065c04..37b9e31d33 100644 --- a/src/app/api/v1/relay/chat/completions/routingBackend.ts +++ b/src/app/api/v1/relay/chat/completions/routingBackend.ts @@ -10,6 +10,18 @@ export interface BifrostRoutingConfig { enabled: boolean; } +export interface SidecarEligibility { + eligible: boolean; + reasons: readonly string[]; +} + +export type ProviderSidecarLookup = (model: string | undefined) => SidecarEligibility | null; + +export interface BifrostRoutingDecision { + tryBifrost: boolean; + fallbackReason?: string; +} + export function getBifrostRoutingConfig( env: NodeJS.ProcessEnv = process.env ): BifrostRoutingConfig | null { @@ -44,6 +56,33 @@ export function shouldTryBifrost( return Boolean(config?.enabled && backend !== "ts"); } +export function shouldTryBifrostForRequest( + backend: RelayRoutingBackend, + config: BifrostRoutingConfig | null, + body: unknown, + lookupProviderSidecar: ProviderSidecarLookup +): BifrostRoutingDecision { + if (!shouldTryBifrost(backend, config)) { + return { tryBifrost: false }; + } + if (backend === "bifrost") { + return { tryBifrost: true }; + } + + const model = typeof (body as { model?: unknown } | null)?.model === "string" + ? (body as { model: string }).model + : undefined; + const provider = lookupProviderSidecar(model); + if (provider?.eligible) { + return { tryBifrost: true }; + } + + return { + tryBifrost: false, + fallbackReason: provider ? "bifrost-ineligible" : "bifrost-provider-unknown", + }; +} + export function getRoutingFallbackHeader( backend: RelayRoutingBackend, config: BifrostRoutingConfig | null diff --git a/tests/unit/api/v1/relay-routing-backend.test.ts b/tests/unit/api/v1/relay-routing-backend.test.ts index c30af34b20..6e8e0e0aad 100644 --- a/tests/unit/api/v1/relay-routing-backend.test.ts +++ b/tests/unit/api/v1/relay-routing-backend.test.ts @@ -5,6 +5,7 @@ import { getRoutingFallbackHeader, resolveRelayRoutingBackend, shouldTryBifrost, + shouldTryBifrostForRequest, } from "../../../../src/app/api/v1/relay/chat/completions/routingBackend.ts"; test("relay routing backend defaults to TypeScript without bifrost", () => { @@ -98,3 +99,56 @@ test("relay routing backend keeps strict bifrost failures out of auto fallback a assert.equal(getRoutingFallbackHeader("bifrost", config), undefined); assert.equal(resolveRelayRoutingBackend({ OMNIROUTE_RELAY_BACKEND: "bifrost" }), "bifrost"); }); + +test("relay routing backend auto mode tries bifrost for manifest-eligible providers", () => { + const config = getBifrostRoutingConfig({ + BIFROST_BASE_URL: "http://127.0.0.1:8080", + }); + + assert.deepEqual( + shouldTryBifrostForRequest("auto", config, { model: "openai/gpt-4.1" }, () => ({ + eligible: true, + reasons: [], + })), + { tryBifrost: true } + ); +}); + +test("relay routing backend auto mode skips bifrost for manifest-ineligible providers", () => { + const config = getBifrostRoutingConfig({ + BIFROST_BASE_URL: "http://127.0.0.1:8080", + }); + + assert.deepEqual( + shouldTryBifrostForRequest("auto", config, { model: "cw/claude-sonnet-4.6" }, () => ({ + eligible: false, + reasons: ["custom executor: claude-web"], + })), + { tryBifrost: false, fallbackReason: "bifrost-ineligible" } + ); +}); + +test("relay routing backend auto mode keeps unknown providers on TS fallback", () => { + const config = getBifrostRoutingConfig({ + BIFROST_BASE_URL: "http://127.0.0.1:8080", + }); + + assert.deepEqual( + shouldTryBifrostForRequest("auto", config, { model: "unknown/model" }, () => null), + { tryBifrost: false, fallbackReason: "bifrost-provider-unknown" } + ); +}); + +test("relay routing backend strict bifrost bypasses manifest eligibility", () => { + const config = getBifrostRoutingConfig({ + BIFROST_BASE_URL: "http://127.0.0.1:8080", + }); + + assert.deepEqual( + shouldTryBifrostForRequest("bifrost", config, { model: "cw/claude-sonnet-4.6" }, () => ({ + eligible: false, + reasons: ["custom executor: claude-web"], + })), + { tryBifrost: true } + ); +}); diff --git a/tests/unit/provider-plugin-manifest.test.ts b/tests/unit/provider-plugin-manifest.test.ts new file mode 100644 index 0000000000..37ce937b1c --- /dev/null +++ b/tests/unit/provider-plugin-manifest.test.ts @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + generateProviderPluginManifestFromRegistry, + getProviderPluginManifestEntryFromRegistry, +} from "../../open-sse/config/providerPluginManifest.ts"; +import type { RegistryEntry } from "../../open-sse/config/providers/shared.ts"; + +const registryFixture: Record = { + openai: { + id: "openai", + alias: "openai", + format: "openai", + executor: "default", + baseUrl: "https://api.openai.com/v1/chat/completions", + authType: "apikey", + authHeader: "bearer", + defaultContextLength: 128000, + models: [ + { id: "gpt-4.1", name: "GPT-4.1", contextLength: 1047576 }, + { + id: "o3", + name: "O3", + contextLength: 200000, + unsupportedParams: ["temperature", "top_p"], + }, + ], + }, + anthropic: { + id: "anthropic", + alias: "anthropic", + format: "claude", + executor: "default", + baseUrl: "https://api.anthropic.com/v1/messages", + authType: "apikey", + authHeader: "x-api-key", + headers: { + "Anthropic-Version": "2023-06-01", + }, + models: [{ id: "claude-sonnet-4.6", name: "Claude Sonnet 4.6" }], + }, + "claude-web": { + id: "claude-web", + alias: "cw", + format: "openai", + executor: "claude-web", + baseUrl: "https://claude.ai/api/organizations", + authType: "apikey", + authHeader: "cookie", + models: [{ id: "claude-sonnet-4.6", name: "Claude 4.6 Sonnet (web)" }], + }, + claude: { + id: "claude", + alias: "claude", + format: "claude", + executor: "default", + baseUrl: "https://api.anthropic.com/v1/messages", + authType: "oauth", + authHeader: "x-api-key", + oauth: { + clientIdDefault: "public-client", + clientSecretDefault: "secret-that-must-not-export", + tokenUrl: "https://console.anthropic.com/oauth/token", + }, + models: [{ id: "claude-opus-4.7", name: "Claude Opus 4.7" }], + }, +}; + +test("provider plugin manifest is JSON-safe and stable enough for sidecars", () => { + const manifest = generateProviderPluginManifestFromRegistry(registryFixture); + const roundTripped = JSON.parse(JSON.stringify(manifest)); + + assert.equal(roundTripped.schemaVersion, 1); + assert.equal(roundTripped.generatedFrom, "open-sse/config/providers"); + assert.equal(roundTripped.providers.length, 4); + assert.deepEqual( + roundTripped.providers.map((provider: { id: string }) => provider.id), + [...roundTripped.providers.map((provider: { id: string }) => provider.id)].sort(), + ); +}); + +test("manifest exposes API-key default-executor providers as sidecar candidates", () => { + const openai = getProviderPluginManifestEntryFromRegistry(registryFixture, "openai"); + + assert.ok(openai); + assert.equal(openai.sidecar.eligible, true); + assert.deepEqual(openai.sidecar.reasons, []); + assert.ok(openai.capabilities.includes("apikey")); + assert.ok(openai.capabilities.includes("sidecar-candidate")); + assert.equal(openai.endpoints.baseUrl, "https://api.openai.com/v1/chat/completions"); + assert.ok(openai.models.some((model) => model.id === "gpt-4.1")); +}); + +test("manifest keeps custom web executors on the TypeScript fallback path", () => { + const claudeWeb = getProviderPluginManifestEntryFromRegistry(registryFixture, "cw"); + + assert.ok(claudeWeb); + assert.equal(claudeWeb.id, "claude-web"); + assert.equal(claudeWeb.sidecar.eligible, false); + assert.ok(claudeWeb.capabilities.includes("custom-executor")); + assert.ok(claudeWeb.sidecar.reasons.some((reason) => reason.includes("claude-web"))); +}); + +test("manifest does not export OAuth client secrets or dynamic functions", () => { + const manifest = generateProviderPluginManifestFromRegistry(registryFixture); + const serialized = JSON.stringify(manifest); + + assert.equal(serialized.includes("clientSecret"), false); + assert.equal(serialized.includes("clientSecretDefault"), false); + assert.equal(serialized.includes("clientSecretEnv"), false); + + const parsed = JSON.parse(serialized); + for (const provider of parsed.providers) { + assert.notEqual(typeof provider.endpoints?.urlBuilder, "function"); + assert.equal("oauth" in provider, false); + assert.equal("headers" in provider, false); + assert.equal("extraHeaders" in provider, false); + assert.equal("requestDefaults" in provider, false); + } +});