From ac0b7add272a65aa3dbde6a600bd8ca0d396d168 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Tue, 16 Jun 2026 08:12:28 -0400 Subject: [PATCH] Gate control-plane proxy direct fallback (#3963) Integrated into release/v3.8.27 --- .env.example | 5 ++ docs/reference/ENVIRONMENT.md | 25 ++++---- docs/reference/FEATURE_FLAGS.md | 32 +++++----- open-sse/utils/proxyFetch.ts | 20 ++++--- .../constants/featureFlagDefinitions.ts | 16 ++++- src/shared/utils/featureFlags.ts | 12 ++++ tests/unit/feature-flags-settings.test.ts | 33 +++++++++-- tests/unit/proxy-fetch.test.ts | 59 ++++++++++++------- 8 files changed, 139 insertions(+), 63 deletions(-) diff --git a/.env.example b/.env.example index 5481f1071c..bd6c563402 100644 --- a/.env.example +++ b/.env.example @@ -1212,6 +1212,11 @@ APP_LOG_TO_FILE=true # Health check result cache TTL (ms). Default: 30000 (30s) # PROXY_HEALTH_CACHE_TTL_MS=30000 +# Allow OAuth and provider validation flows to bypass a pinned proxy and connect +# directly when proxy reachability pre-checks fail. Default: false. +# Also configurable from Dashboard > Settings > Feature Flags. +# OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK=false + # Rate limit maximum wait time before failing a request (ms). Default: 120000 (2 min) # Used by: open-sse/services/rateLimitManager.ts # RATE_LIMIT_MAX_WAIT_MS=120000 diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index da8c73e059..c9d38755db 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -748,18 +748,19 @@ Anthropic-compatible provider instead. ## 21. Proxy Health -| Variable | Default | Source File | Description | -| ---------------------------- | ---------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PROXY_FAST_FAIL_TIMEOUT_MS` | `2000` | `src/lib/proxyHealth.ts` | Fast-fail health check timeout. | -| `PROXY_HEALTH_CACHE_TTL_MS` | `30000` | `src/lib/proxyHealth.ts` | Health check result cache TTL. | -| `RATE_LIMIT_MAX_WAIT_MS` | `120000` (2 min) | `open-sse/services/rateLimitManager.ts` | Max time to wait on a 429 before failing the request. | -| `RATE_LIMIT_AUTO_ENABLE` | _(unset)_ | `open-sse/services/rateLimitManager.ts` | Force the auto-enable rate limit safety net on/off regardless of the persisted Dashboard setting. Accepts `true`/`1`/`on` to force on, `false`/`0`/`off` to force off. | -| `PROVIDER_COOLDOWN_ENABLED` | _(unset → off)_ | `open-sse/services/providerCooldownTracker.ts` | Opt-in global cross-request provider/connection cooldown tracking. OFF by default (overlaps Connection Cooldown / Provider Circuit Breaker). Accepts `true`/`1`/`on` to enable. | -| `PROVIDER_COOLDOWN_MIN_MS` | `5000` | `open-sse/services/providerCooldownTracker.ts` | Minimum cooldown (ms) before a failed provider/connection is retried. Scaled exponentially with consecutive failures. Only used when `PROVIDER_COOLDOWN_ENABLED`. | -| `PROVIDER_COOLDOWN_MAX_MS` | `300000` (5 min) | `open-sse/services/providerCooldownTracker.ts` | Maximum cooldown (ms) cap before a failed provider/connection is retried regardless. Only used when `PROVIDER_COOLDOWN_ENABLED`. | -| `HEALTHCHECK_STAGGER_MS` | `3000` | `src/lib/tokenHealthCheck.ts` | Stagger interval (ms) between provider token healthchecks at startup. | -| `REQUEST_RETRY` | `2` | `src/sse/services/cooldownAwareRetry.ts` | Number of automatic retries on model-scoped cooldown responses before returning error to client. | -| `MAX_RETRY_INTERVAL_SEC` | `30` | `src/sse/services/cooldownAwareRetry.ts` | Max backoff interval (seconds) between cooldown retries. Capped by this value regardless of upstream `Retry-After`. | +| Variable | Default | Source File | Description | +| ----------------------------------------------- | ---------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PROXY_FAST_FAIL_TIMEOUT_MS` | `2000` | `src/lib/proxyHealth.ts` | Fast-fail health check timeout. | +| `PROXY_HEALTH_CACHE_TTL_MS` | `30000` | `src/lib/proxyHealth.ts` | Health check result cache TTL. | +| `OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK` | `false` | `src/shared/constants/featureFlagDefinitions.ts` | Allow OAuth and provider validation flows to bypass a pinned proxy and connect directly when proxy reachability pre-checks fail. Effective precedence is Feature Flags DB override > env var > default. | +| `RATE_LIMIT_MAX_WAIT_MS` | `120000` (2 min) | `open-sse/services/rateLimitManager.ts` | Max time to wait on a 429 before failing the request. | +| `RATE_LIMIT_AUTO_ENABLE` | _(unset)_ | `open-sse/services/rateLimitManager.ts` | Force the auto-enable rate limit safety net on/off regardless of the persisted Dashboard setting. Accepts `true`/`1`/`on` to force on, `false`/`0`/`off` to force off. | +| `PROVIDER_COOLDOWN_ENABLED` | _(unset → off)_ | `open-sse/services/providerCooldownTracker.ts` | Opt-in global cross-request provider/connection cooldown tracking. OFF by default (overlaps Connection Cooldown / Provider Circuit Breaker). Accepts `true`/`1`/`on` to enable. | +| `PROVIDER_COOLDOWN_MIN_MS` | `5000` | `open-sse/services/providerCooldownTracker.ts` | Minimum cooldown (ms) before a failed provider/connection is retried. Scaled exponentially with consecutive failures. Only used when `PROVIDER_COOLDOWN_ENABLED`. | +| `PROVIDER_COOLDOWN_MAX_MS` | `300000` (5 min) | `open-sse/services/providerCooldownTracker.ts` | Maximum cooldown (ms) cap before a failed provider/connection is retried regardless. Only used when `PROVIDER_COOLDOWN_ENABLED`. | +| `HEALTHCHECK_STAGGER_MS` | `3000` | `src/lib/tokenHealthCheck.ts` | Stagger interval (ms) between provider token healthchecks at startup. | +| `REQUEST_RETRY` | `2` | `src/sse/services/cooldownAwareRetry.ts` | Number of automatic retries on model-scoped cooldown responses before returning error to client. | +| `MAX_RETRY_INTERVAL_SEC` | `30` | `src/sse/services/cooldownAwareRetry.ts` | Max backoff interval (seconds) between cooldown retries. Capped by this value regardless of upstream `Retry-After`. | --- diff --git a/docs/reference/FEATURE_FLAGS.md b/docs/reference/FEATURE_FLAGS.md index d64bd539c0..5da6146a0a 100644 --- a/docs/reference/FEATURE_FLAGS.md +++ b/docs/reference/FEATURE_FLAGS.md @@ -1,7 +1,7 @@ --- title: "Feature Flags" -version: 3.8.24 -lastUpdated: 2026-06-13 +version: 3.8.26 +lastUpdated: 2026-06-16 --- # Feature Flags @@ -46,7 +46,7 @@ A boolean flag is considered **enabled** when its effective value is `"true"`, ## Flag Catalog -31 flags across 6 categories. **Default** is the definition default — the value +33 flags across 6 categories. **Default** is the definition default — the value used when neither a DB override nor an environment variable is present. ### Security (7) @@ -61,16 +61,17 @@ used when neither a DB override nor an environment variable is present. | `PII_RESPONSE_SANITIZATION_MODE` | enum | `redact` | Mode for PII response sanitization. Values: `redact`, `warn`, `block`, `off`. | | `OUTBOUND_SSRF_GUARD_ENABLED` | boolean | `true` | Block outbound requests to private/internal IP ranges. | -### Network (6) +### Network (7) -| Key | Type | Default | Restart | Description | -| --------------------------------------- | ------- | ------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ENABLE_TLS_FINGERPRINT` | boolean | `false` | ✓ | Enable TLS fingerprint stealth mode. | -| `ONEPROXY_ENABLED` | boolean | `true` | | Enable 1proxy request proxying. | -| `PROXY_AUTO_SELECT_ENABLED` | boolean | `false` | | When no proxy is assigned to a connection, auto-select the first working proxy from the registry. Off by default (otherwise any registry proxy becomes a global fallback — #3332). | -| `MITM_DISABLE_TLS_VERIFY` | boolean | `false` | ✓ | Disable TLS certificate verification for the MITM proxy. **Danger.** | -| `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` | boolean | `false` | | Allow provider URLs pointing to private/internal networks. | -| `ENABLE_CC_COMPATIBLE_PROVIDER` | boolean | `false` | ✓ | Enable Claude Code compatible provider mode. | +| Key | Type | Default | Restart | Description | +| ----------------------------------------------- | ------- | ------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ENABLE_TLS_FINGERPRINT` | boolean | `false` | ✓ | Enable TLS fingerprint stealth mode. | +| `ONEPROXY_ENABLED` | boolean | `true` | | Enable 1proxy request proxying. | +| `PROXY_AUTO_SELECT_ENABLED` | boolean | `false` | | When no proxy is assigned to a connection, auto-select the first working proxy from the registry. Off by default (otherwise any registry proxy becomes a global fallback — #3332). | +| `OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK` | boolean | `false` | | Allow OAuth and provider validation flows to bypass a pinned proxy and connect directly when proxy reachability pre-checks fail. Off by default because this can change egress IP. | +| `MITM_DISABLE_TLS_VERIFY` | boolean | `false` | ✓ | Disable TLS certificate verification for the MITM proxy. **Danger.** | +| `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` | boolean | `false` | | Allow provider URLs pointing to private/internal networks. | +| `ENABLE_CC_COMPATIBLE_PROVIDER` | boolean | `false` | ✓ | Enable Claude Code compatible provider mode. | ### Policies (3) @@ -80,7 +81,7 @@ used when neither a DB override nor an environment variable is present. | `RATE_LIMIT_AUTO_ENABLE` | boolean | `false` | | Automatically enable rate limiting based on usage patterns. | | `ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE` | boolean | `false` | ✓ | Allow multiple connections per compatibility node. | -### Runtime (9) +### Runtime (10) | Key | Type | Default | Restart | Description | | ------------------------------------------- | ------- | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -93,6 +94,7 @@ used when neither a DB override nor an environment variable is present. | `OMNIROUTE_CODEX_WS_ENABLED` | boolean | `true` | | Allow Codex to use the Responses-over-WebSocket transport. When off, Codex falls back to HTTP Responses. | | `OMNIROUTE_EMERGENCY_FALLBACK` | boolean | `true` | | Route budget-exhausted requests to the emergency free fallback provider/model. (See [Emergency Budget Fallback](#emergency-budget-fallback) below.) | | `MODEL_CATALOG_INCLUDE_NAMES` | boolean | `true` | | Include display-friendly name fields in `/v1/models` responses. Disable for clients that expect model IDs only. | +| `ARENA_ELO_SYNC_ENABLED` | boolean | `true` | | Enable periodic Arena AI leaderboard ELO sync for model intelligence rankings. | ### CLI (3) @@ -163,10 +165,10 @@ Returns every flag with its effective value, source, and a summary. "requiresRestart": false, "warningLevel": "caution", }, - // ... all 31 flags + // ... all 33 flags ], "summary": { - "total": 31, + "total": 33, "active": 0, "inactive": 0, "overriddenByDb": 0, diff --git a/open-sse/utils/proxyFetch.ts b/open-sse/utils/proxyFetch.ts index 42ff41394c..8c1e98dcb7 100644 --- a/open-sse/utils/proxyFetch.ts +++ b/open-sse/utils/proxyFetch.ts @@ -12,7 +12,10 @@ import { } from "./proxyDispatcher.ts"; import tlsClient from "./tlsClient.ts"; import { isProxyReachable } from "@/lib/proxyHealth"; -import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; +import { + isControlPlaneProxyDirectFallbackEnabled, + isFeatureFlagEnabled, +} from "@/shared/utils/featureFlags"; import { findWorkingProxy } from "./proxyFallback.ts"; function isTlsFingerprintEnabled() { @@ -216,12 +219,10 @@ export async function runWithProxyContext( const resolvedProxyUrl = effectiveProxyConfig ? proxyConfigToUrl(effectiveProxyConfig) : null; - // When set, a proxy that fails the reachability/family pre-checks degrades to a - // DIRECT connection instead of throwing. Use for control-plane operations (OAuth, - // connection tests, token refresh) where reaching the upstream matters more than - // egress-IP pinning — a dead pinned proxy must not surface as a generic 500. Data - // plane (chat) keeps the strict behaviour so per-account IP isolation is preserved. - const directFallbackOnUnreachable = opts?.directFallbackOnUnreachable === true; + // The caller must opt in, and the runtime feature flag must also be enabled. + // This fallback changes egress IP, so upgrades must not silently turn it on. + const directFallbackOnUnreachable = + opts?.directFallbackOnUnreachable === true && isControlPlaneProxyDirectFallbackEnabled(); // Run fn with the proxy context cleared so the request egresses directly. const runDirect = () => proxyContext.run(null, fn); @@ -288,12 +289,15 @@ export async function runWithProxyContext( /** * Like {@link runWithProxyContext}, but if the assigned proxy is unreachable or fails - * its pre-checks the request degrades to a DIRECT connection instead of throwing. + * its pre-checks the request can degrade to a DIRECT connection instead of throwing. * * For control-plane flows — OAuth code/token exchange, connection tests, token refresh — * where a dead pinned proxy must not block reaching the upstream (it otherwise surfaces * as a generic "Internal server error"). Data-plane chat keeps strict pinning via * runWithProxyContext so per-account egress-IP isolation is preserved. + * + * This remains disabled unless OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK is enabled + * from Feature Flags or the environment. */ export async function runWithProxyContextOrDirect(proxyConfig, fn) { return runWithProxyContext(proxyConfig, fn, { directFallbackOnUnreachable: true }); diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index 19924d3e05..3775ff3025 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -12,7 +12,7 @@ export interface FeatureFlagDefinition { } export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ - // ──────────────── Security (6) ──────────────── + // ──────────────── Security (7) ──────────────── { key: "REQUIRE_API_KEY", label: "Require API Key", @@ -94,7 +94,7 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ warningLevel: "info", }, - // ──────────────── Network (5) ──────────────── + // ──────────────── Network (7) ──────────────── { key: "ENABLE_TLS_FINGERPRINT", label: "TLS Fingerprint", @@ -129,6 +129,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: false, warningLevel: "caution", }, + { + key: "OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK", + label: "Control-Plane Proxy Direct Fallback", + description: + "Allow OAuth and provider validation flows to bypass a pinned proxy and connect directly when proxy reachability pre-checks fail. Off by default because this can change account egress IP.", + descriptionI18nKey: "featureFlagOmnirouteControlPlaneProxyDirectFallbackDescription", + category: "network", + defaultValue: "false", + type: "boolean", + requiresRestart: false, + warningLevel: "danger", + }, { key: "MITM_DISABLE_TLS_VERIFY", label: "Disable TLS Verify (MITM)", diff --git a/src/shared/utils/featureFlags.ts b/src/shared/utils/featureFlags.ts index 56ae586f4b..2a3db8454c 100644 --- a/src/shared/utils/featureFlags.ts +++ b/src/shared/utils/featureFlags.ts @@ -79,3 +79,15 @@ export function isModelCatalogNamesEnabled(): boolean { export function isArenaEloSyncEnabled(): boolean { return isFeatureFlagEnabled("ARENA_ELO_SYNC_ENABLED"); } + +export function isControlPlaneProxyDirectFallbackEnabled(): boolean { + try { + return isFeatureFlagEnabled("OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK"); + } catch (error) { + console.error( + "[featureFlags] Failed to resolve OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK, defaulting to disabled:", + error instanceof Error ? error.message : error + ); + return false; + } +} diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index 98268fad19..8e0bf77b0e 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -27,19 +27,20 @@ const { isCcCompatibleProviderEnabled, isModelCatalogNamesEnabled, isArenaEloSyncEnabled, + isControlPlaneProxyDirectFallbackEnabled, } = await import("../../src/shared/utils/featureFlags.ts"); // ────────────────────────────────────────────────────── // Test group 1 — Flag definitions registry // ────────────────────────────────────────────────────── describe("featureFlagDefinitions", () => { - it("has exactly 32 flag definitions", () => { - assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 32); + it("has exactly 33 flag definitions", () => { + assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 33); }); it("has unique keys for all flags", () => { const keys = FEATURE_FLAG_DEFINITIONS.map((d) => d.key); - assert.strictEqual(new Set(keys).size, 32); + assert.strictEqual(new Set(keys).size, 33); }); it("has valid categories for all flags", () => { @@ -114,6 +115,18 @@ describe("featureFlagDefinitions", () => { assert.strictEqual(def.defaultValue, "true"); assert.strictEqual(def.requiresRestart, false); }); + + it("defines control-plane proxy direct fallback as a network boolean flag disabled by default", () => { + const def = FEATURE_FLAG_DEFINITIONS.find( + (d) => d.key === "OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK" + ); + assert.ok(def, "OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK should exist"); + assert.strictEqual(def.category, "network"); + assert.strictEqual(def.type, "boolean"); + assert.strictEqual(def.defaultValue, "false"); + assert.strictEqual(def.requiresRestart, false); + assert.strictEqual(def.warningLevel, "danger"); + }); }); // ────────────────────────────────────────────────────── @@ -251,9 +264,9 @@ describe("resolveFeatureFlag", () => { }); describe("resolveAllFeatureFlags", () => { - it("returns all 32 flags", () => { + it("returns all 33 flags", () => { const all = resolveAllFeatureFlags(); - assert.strictEqual(all.length, 32); + assert.strictEqual(all.length, 33); }); it("marks DB-overridden flags with source 'db'", () => { @@ -326,6 +339,16 @@ describe("resolveFeatureFlag", () => { removeFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED"); } }); + + it("isControlPlaneProxyDirectFallbackEnabled defaults off and follows DB overrides", () => { + assert.strictEqual(isControlPlaneProxyDirectFallbackEnabled(), false); + try { + setFeatureFlagOverride("OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK", "true"); + assert.strictEqual(isControlPlaneProxyDirectFallbackEnabled(), true); + } finally { + removeFeatureFlagOverride("OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK"); + } + }); }); }); diff --git a/tests/unit/proxy-fetch.test.ts b/tests/unit/proxy-fetch.test.ts index cc9f1a3b84..04b7d47c9a 100644 --- a/tests/unit/proxy-fetch.test.ts +++ b/tests/unit/proxy-fetch.test.ts @@ -176,30 +176,47 @@ test("runWithProxyContext throws PROXY_UNREACHABLE for an unreachable proxy by d }); test("runWithProxyContext degrades to a direct connection when directFallbackOnUnreachable is set", async () => { - let ran = false; - const result = await runWithProxyContext( - { type: "http", host: "127.0.0.1", port: "9" }, - async () => { - ran = true; - return "direct-ok"; - }, - { directFallbackOnUnreachable: true } - ); + await withEnv({ OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK: "true" }, async () => { + let ran = false; + const result = await runWithProxyContext( + { type: "http", host: "127.0.0.1", port: "9" }, + async () => { + ran = true; + return "direct-ok"; + }, + { directFallbackOnUnreachable: true } + ); - assert.equal(ran, true, "callback must still run via a direct connection"); - assert.equal(result, "direct-ok"); + assert.equal(ran, true, "callback must still run via a direct connection"); + assert.equal(result, "direct-ok"); + }); +}); + +test("runWithProxyContext keeps strict pinning when the direct fallback feature flag is off", async () => { + await withEnv({ OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK: "false" }, async () => { + await assert.rejects( + runWithProxyContext( + { type: "http", host: "127.0.0.1", port: "9" }, + async () => "unreachable", + { directFallbackOnUnreachable: true } + ), + /Proxy unreachable/ + ); + }); }); test("runWithProxyContextOrDirect runs the callback directly when the proxy is unreachable", async () => { - let ran = false; - const result = await runWithProxyContextOrDirect( - { type: "http", host: "127.0.0.1", port: "9" }, - async () => { - ran = true; - return "ok"; - } - ); + await withEnv({ OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK: "true" }, async () => { + let ran = false; + const result = await runWithProxyContextOrDirect( + { type: "http", host: "127.0.0.1", port: "9" }, + async () => { + ran = true; + return "ok"; + } + ); - assert.equal(ran, true); - assert.equal(result, "ok"); + assert.equal(ran, true); + assert.equal(result, "ok"); + }); });