From de12d1c10942e414ef9e48d46745739fed1b9caa Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Sat, 13 Jun 2026 00:40:57 -0400 Subject: [PATCH] Expose emergency fallback in Feature Flags (#3752) Adds OMNIROUTE_EMERGENCY_FALLBACK to the Feature Flags page as a runtime boolean (default on, no restart); resolves through the feature-flag stack so DB overrides can enable/disable it while preserving the raw env fallback path. Integrated into release/v3.8.24. --- docs/architecture/RESILIENCE_GUIDE.md | 1 + docs/reference/ENVIRONMENT.md | 2 +- open-sse/services/emergencyFallback.ts | 47 ++++++- src/i18n/messages/en.json | 1 + .../constants/featureFlagDefinitions.ts | 13 +- tests/unit/emergency-fallback-service.test.ts | 51 +++++++- tests/unit/feature-flags-settings.test.ts | 19 ++- .../unit/services/emergency-fallback.test.ts | 117 ++++++++++++++++-- 8 files changed, 234 insertions(+), 17 deletions(-) diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md index 8424e50808..cd9f2c71df 100644 --- a/docs/architecture/RESILIENCE_GUIDE.md +++ b/docs/architecture/RESILIENCE_GUIDE.md @@ -124,6 +124,7 @@ Lists active lockouts with: provider, connection, model, reason, expiresAt. Oper - **Reset-aware routing** (v3.8.0) — prioritizes connections by quota reset time. - **Background mode degradation** — Responses API `background: true` degraded to sync with warning. - **Dynamic tool limit detection** — backs off providers when tool count limits hit. +- **Emergency fallback** — controlled by `OMNIROUTE_EMERGENCY_FALLBACK`; operators can override it from the Feature Flags page without a restart. --- diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index b51e645e6a..c2f98d23d9 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -874,7 +874,7 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `ALIBABA_CODING_PLAN_QUOTA_URL` | derived from host | `open-sse/services/bailianQuotaFetcher.ts` | Full quota URL override for Alibaba Bailian. | | `CONTEXT_RESERVE_TOKENS` | `1024` | `open-sse/services/contextManager.ts` | Tokens reserved for completion output when computing prompt budgets. | | `MODEL_ALIAS_COMPAT_ENABLED` | enabled | `open-sse/services/model.ts` | Toggle the legacy model-alias compatibility layer used by older clients. | -| `OMNIROUTE_EMERGENCY_FALLBACK` | enabled | `open-sse/services/emergencyFallback.ts` | Set `false` (or `0`) to disable the emergency budget-exhaustion fallback that reroutes failed requests to the free `nvidia`/`openai/gpt-oss-120b` model. | +| `OMNIROUTE_EMERGENCY_FALLBACK` | enabled | `open-sse/services/emergencyFallback.ts` | Set `false` (or `0`) to disable the emergency budget-exhaustion fallback that reroutes failed requests to the free `nvidia`/`openai/gpt-oss-120b` model. Effective precedence is Feature Flags DB override > env var > default; if unavailable, the service falls back to the raw env value. | | `COMMAND_CODE_CALLBACK_PORT` | _(unset)_ | `src/app/api/providers/command-code/auth/shared.ts` | Local port used for OAuth-style callbacks from the Command Code CLI helper. | | `COMMAND_CODE_VERSION` | `0.33.2` | `open-sse/executors/commandCode.ts` | Value sent as the `x-command-code-version` header to the Command Code upstream. Override to bump the CLI version. | | `MITM_LOCAL_PORT` | `443` | `src/mitm/server.cjs` | Local bind port for the MITM debug proxy. | diff --git a/open-sse/services/emergencyFallback.ts b/open-sse/services/emergencyFallback.ts index 3819848291..60e3914172 100644 --- a/open-sse/services/emergencyFallback.ts +++ b/open-sse/services/emergencyFallback.ts @@ -12,6 +12,16 @@ * `OMNIROUTE_EMERGENCY_FALLBACK=false` (or `0`). Default remains enabled. */ +import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; + +const EMERGENCY_FALLBACK_FLAG_KEY = "OMNIROUTE_EMERGENCY_FALLBACK"; +const EMERGENCY_FALLBACK_FLAG_CACHE_MS = 500; + +type FeatureFlagResolver = (key: string) => boolean; + +let emergencyFallbackFlagCache: { value: boolean; expiresAt: number } | null = null; +let emergencyFallbackFeatureFlagResolver: FeatureFlagResolver = isFeatureFlagEnabled; + export interface EmergencyFallbackConfig { enabled: boolean; provider: string; @@ -66,11 +76,46 @@ export interface NoFallbackDecision { export type FallbackResult = FallbackDecision | NoFallbackDecision; -export function isEmergencyFallbackEnvEnabled(): boolean { +function isEmergencyFallbackRawEnvEnabled(): boolean { const raw = process.env.OMNIROUTE_EMERGENCY_FALLBACK; return raw !== "false" && raw !== "0"; } +export function resetEmergencyFallbackEnvCache(): void { + emergencyFallbackFlagCache = null; +} + +export function setEmergencyFallbackFeatureFlagResolverForTest( + resolver: FeatureFlagResolver | null +): void { + emergencyFallbackFeatureFlagResolver = resolver ?? isFeatureFlagEnabled; + resetEmergencyFallbackEnvCache(); +} + +export function isEmergencyFallbackEnvEnabled(): boolean { + const now = Date.now(); + if (emergencyFallbackFlagCache && emergencyFallbackFlagCache.expiresAt > now) { + return emergencyFallbackFlagCache.value; + } + + let value: boolean; + try { + value = emergencyFallbackFeatureFlagResolver(EMERGENCY_FALLBACK_FLAG_KEY); + } catch (error) { + console.warn( + "[emergencyFallback] Feature flag resolution failed; falling back to raw env:", + error instanceof Error ? error.message : error + ); + value = isEmergencyFallbackRawEnvEnabled(); + } + + emergencyFallbackFlagCache = { + value, + expiresAt: now + EMERGENCY_FALLBACK_FLAG_CACHE_MS, + }; + return value; +} + export function shouldUseFallback( status: number, errorBody: string, diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 2de9742fc4..88400249c6 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -842,6 +842,7 @@ "batchActionRetryError": "Failed to retry failed requests. Try again.", "batchConceptRetentionNote": "Results and error files are retained for 30 days (Anthropic: 29 days)" }, + "featureFlagOmnirouteEmergencyFallbackDescription": "Route budget-exhausted requests to the emergency free fallback provider/model.", "sidebar": { "home": "Home", "dashboard": "Dashboard", diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index b404d7831c..06e0efe342 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -199,7 +199,7 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ warningLevel: "info", }, - // ──────────────── Runtime (8) ──────────────── + // ──────────────── Runtime (9) ──────────────── { key: "OMNIROUTE_MCP_ENFORCE_SCOPES", label: "MCP Enforce Scopes", @@ -279,6 +279,17 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: false, warningLevel: "info", }, + { + key: "OMNIROUTE_EMERGENCY_FALLBACK", + label: "Emergency Fallback", + description: "Route budget-exhausted requests to the emergency free fallback provider/model.", + descriptionI18nKey: "featureFlagOmnirouteEmergencyFallbackDescription", + category: "runtime", + defaultValue: "true", + type: "boolean", + requiresRestart: false, + warningLevel: "caution", + }, { key: "MODEL_CATALOG_INCLUDE_NAMES", label: "Model Catalog Names", diff --git a/tests/unit/emergency-fallback-service.test.ts b/tests/unit/emergency-fallback-service.test.ts index 45931c54f4..b229fb7012 100644 --- a/tests/unit/emergency-fallback-service.test.ts +++ b/tests/unit/emergency-fallback-service.test.ts @@ -1,8 +1,55 @@ import test from "node:test"; import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -const { EMERGENCY_FALLBACK_CONFIG, shouldUseFallback, isFallbackDecision } = - await import("../../open-sse/services/emergencyFallback.ts"); +const previousDataDir = process.env.DATA_DIR; +const previousDisableSqliteAutoBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-emergency-fallback-service-")); +process.env.DATA_DIR = tmpDir; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../src/lib/db/core.ts"); + +const { + EMERGENCY_FALLBACK_CONFIG, + shouldUseFallback, + isFallbackDecision, + resetEmergencyFallbackEnvCache, +} = await import("../../open-sse/services/emergencyFallback.ts"); + +function restoreEnv(name: string, value: string | undefined) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } +} + +function resetTestState() { + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.mkdirSync(tmpDir, { recursive: true }); + delete process.env.OMNIROUTE_EMERGENCY_FALLBACK; + resetEmergencyFallbackEnvCache(); +} + +test.beforeEach(() => { + resetTestState(); +}); + +test.afterEach(() => { + delete process.env.OMNIROUTE_EMERGENCY_FALLBACK; + resetEmergencyFallbackEnvCache(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + restoreEnv("DATA_DIR", previousDataDir); + restoreEnv("DISABLE_SQLITE_AUTO_BACKUP", previousDisableSqliteAutoBackup); +}); test("shouldUseFallback returns disabled when the feature flag is off", () => { const result = shouldUseFallback(402, "payment required", false, { diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index 9a3ccc424a..83ed227882 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -32,13 +32,13 @@ const { // Test group 1 — Flag definitions registry // ────────────────────────────────────────────────────── describe("featureFlagDefinitions", () => { - it("has exactly 30 flag definitions", () => { - assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 30); + it("has exactly 31 flag definitions", () => { + assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 31); }); it("has unique keys for all flags", () => { const keys = FEATURE_FLAG_DEFINITIONS.map((d) => d.key); - assert.strictEqual(new Set(keys).size, 30); + assert.strictEqual(new Set(keys).size, 31); }); it("has valid categories for all flags", () => { @@ -95,6 +95,15 @@ describe("featureFlagDefinitions", () => { assert.strictEqual(def.defaultValue, "true"); assert.strictEqual(def.requiresRestart, false); }); + + it("defines emergency fallback as a runtime boolean flag enabled by default", () => { + const def = FEATURE_FLAG_DEFINITIONS.find((d) => d.key === "OMNIROUTE_EMERGENCY_FALLBACK"); + assert.ok(def, "OMNIROUTE_EMERGENCY_FALLBACK should exist"); + assert.strictEqual(def.category, "runtime"); + assert.strictEqual(def.type, "boolean"); + assert.strictEqual(def.defaultValue, "true"); + assert.strictEqual(def.requiresRestart, false); + }); }); // ────────────────────────────────────────────────────── @@ -232,9 +241,9 @@ describe("resolveFeatureFlag", () => { }); describe("resolveAllFeatureFlags", () => { - it("returns all 30 flags", () => { + it("returns all 31 flags", () => { const all = resolveAllFeatureFlags(); - assert.strictEqual(all.length, 30); + assert.strictEqual(all.length, 31); }); it("marks DB-overridden flags with source 'db'", () => { diff --git a/tests/unit/services/emergency-fallback.test.ts b/tests/unit/services/emergency-fallback.test.ts index bf43dd5d59..bf6c39603b 100644 --- a/tests/unit/services/emergency-fallback.test.ts +++ b/tests/unit/services/emergency-fallback.test.ts @@ -1,10 +1,59 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const previousDataDir = process.env.DATA_DIR; +const previousDisableSqliteAutoBackup = process.env.DISABLE_SQLITE_AUTO_BACKUP; +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-emergency-fallback-")); +process.env.DATA_DIR = tmpDir; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../../src/lib/db/core.ts"); +const { setFeatureFlagOverride, removeFeatureFlagOverride } = + await import("../../../src/lib/db/featureFlags.ts"); +const { shouldUseFallback, isEmergencyFallbackEnvEnabled, EMERGENCY_FALLBACK_CONFIG, -} from "../../../open-sse/services/emergencyFallback.ts"; + resetEmergencyFallbackEnvCache, + setEmergencyFallbackFeatureFlagResolverForTest, +} = await import("../../../open-sse/services/emergencyFallback.ts"); + +function restoreEnv(name: string, value: string | undefined) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } +} + +function resetTestState() { + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.mkdirSync(tmpDir, { recursive: true }); + delete process.env.OMNIROUTE_EMERGENCY_FALLBACK; + resetEmergencyFallbackEnvCache(); + setEmergencyFallbackFeatureFlagResolverForTest(null); +} + +test.beforeEach(() => { + resetTestState(); +}); + +test.afterEach(() => { + resetEmergencyFallbackEnvCache(); + setEmergencyFallbackFeatureFlagResolverForTest(null); + delete process.env.OMNIROUTE_EMERGENCY_FALLBACK; +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + restoreEnv("DATA_DIR", previousDataDir); + restoreEnv("DISABLE_SQLITE_AUTO_BACKUP", previousDisableSqliteAutoBackup); +}); function withEnv(value: string | undefined, fn: () => void) { const previous = process.env.OMNIROUTE_EMERGENCY_FALLBACK; @@ -13,14 +62,23 @@ function withEnv(value: string | undefined, fn: () => void) { } else { process.env.OMNIROUTE_EMERGENCY_FALLBACK = value; } + resetEmergencyFallbackEnvCache(); try { fn(); } finally { - if (previous === undefined) { - delete process.env.OMNIROUTE_EMERGENCY_FALLBACK; - } else { - process.env.OMNIROUTE_EMERGENCY_FALLBACK = previous; - } + restoreEnv("OMNIROUTE_EMERGENCY_FALLBACK", previous); + resetEmergencyFallbackEnvCache(); + } +} + +function withFeatureFlagOverride(value: string, fn: () => void) { + try { + setFeatureFlagOverride("OMNIROUTE_EMERGENCY_FALLBACK", value); + resetEmergencyFallbackEnvCache(); + fn(); + } finally { + removeFeatureFlagOverride("OMNIROUTE_EMERGENCY_FALLBACK"); + resetEmergencyFallbackEnvCache(); } } @@ -60,6 +118,51 @@ test("OMNIROUTE_EMERGENCY_FALLBACK=0 disables the budget-keyword redirect", () = }); }); +test("DB feature flag override can disable an env-enabled fallback", () => { + withEnv("true", () => { + withFeatureFlagOverride("false", () => { + const decision = shouldUseFallback(402, "", false); + assert.equal(isEmergencyFallbackEnvEnabled(), false); + assert.equal(decision.shouldFallback, false); + assert.match(decision.reason, /OMNIROUTE_EMERGENCY_FALLBACK/); + }); + }); +}); + +test("DB feature flag override can enable an env-disabled fallback", () => { + withEnv("false", () => { + withFeatureFlagOverride("true", () => { + const decision = shouldUseFallback(402, "", false); + assert.equal(isEmergencyFallbackEnvEnabled(), true); + assert.equal(decision.shouldFallback, true); + }); + }); +}); + +test("raw env fallback is used when feature flag resolution throws", () => { + withEnv("0", () => { + const warnings: unknown[][] = []; + const previousWarn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args); + }; + setEmergencyFallbackFeatureFlagResolverForTest(() => { + throw new Error("feature flag store unavailable"); + }); + + try { + assert.equal(isEmergencyFallbackEnvEnabled(), false); + const decision = shouldUseFallback(402, "", false); + assert.equal(decision.shouldFallback, false); + assert.match(decision.reason, /OMNIROUTE_EMERGENCY_FALLBACK/); + assert.equal(warnings.length, 1); + assert.match(String(warnings[0]?.[0]), /Feature flag resolution failed/); + } finally { + console.warn = previousWarn; + } + }); +}); + test("explicit truthy values keep the fallback enabled", () => { withEnv("true", () => { assert.equal(isEmergencyFallbackEnvEnabled(), true);