From 2256e14e7847bf4f377aa5e1bc0bde429332662f Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Thu, 20 Aug 2026 22:03:26 -0300 Subject: [PATCH] feat(routing): add DISABLE_CONTEXT_WINDOW_CHECKS bypass for the direct-request input/context check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the default-off DISABLE_CONTEXT_WINDOW_CHECKS feature flag to let operators bypass OmniRoute's local context-window / max-input-token check (enforceOutputTokenBudget in chatCore.ts) for direct single-model requests, leaving upstream limits, prompt compression, and the model's own output-token cap active. Exposed via the dashboard feature-flag registry with localized labels/descriptions (de/en/pt-BR/vi). Rescoped from #10606: that PR also touched the combo-routing context filter (knownContextOverflow.ts / resolveAutoStrategy.ts / comboStructure.ts), but #10162 (merged earlier the same day, RFC #10141) already deleted that file and made combo-routing context checks advisory-only — never a hard reject. This PR keeps only the surviving chatCore.ts hard-reject bypass, which #10162 did not touch and which still has real value. Co-authored-by: JxnLexn <10897478+JxnLexn@users.noreply.github.com> --- .env.example | 9 ++++ .../features/disable-context-window-checks.md | 1 + docs/reference/ENVIRONMENT.md | 1 + docs/reference/FEATURE_FLAGS.md | 3 +- open-sse/handlers/chatCore.ts | 14 +++++-- src/i18n/messages/de.json | 5 +++ src/i18n/messages/en.json | 5 +++ src/i18n/messages/pt-BR.json | 5 +++ src/i18n/messages/vi.json | 5 +++ .../constants/featureFlagDefinitions.ts | 14 ++++++- src/shared/utils/featureFlags.ts | 16 ++++++++ .../chatcore-model-output-cap-wiring.test.ts | 30 ++++++++++++++ tests/unit/feature-flags-settings.test.ts | 41 ++++++++++++++++++- 13 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 changelog.d/features/disable-context-window-checks.md diff --git a/.env.example b/.env.example index 899ba5b0e7..698c09329b 100644 --- a/.env.example +++ b/.env.example @@ -417,6 +417,15 @@ ALLOW_API_KEY_REVEAL=false # by OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT and the heap-pressure shed instead. Set a positive # value only on memory-constrained deployments that need a hard ceiling. # OMNIROUTE_CHAT_HARD_MAX_MESSAGES=0 + +# Skip OmniRoute's local context-window and max-input-token check for direct +# single-model requests. Default: false (dangerous opt-in). +# The upstream provider still enforces its real limits, so enabling this can +# replace an early OmniRoute 400 with an upstream context-length error. +# Prompt compression and the model's own output-token cap remain active. +# Also configurable from Dashboard > Settings > Feature Flags; no restart is +# required. Used by: src/shared/utils/featureFlags.ts and open-sse/handlers/chatCore.ts. +# DISABLE_CONTEXT_WINDOW_CHECKS=false # How long a heavy request waits for heavyweight capacity before a retryable 503. # A short bounded wait serializes agent bursts instead of an instant 503; 0 = instant. # Default 2000 (2s). diff --git a/changelog.d/features/disable-context-window-checks.md b/changelog.d/features/disable-context-window-checks.md new file mode 100644 index 0000000000..1cdd3cc0a8 --- /dev/null +++ b/changelog.d/features/disable-context-window-checks.md @@ -0,0 +1 @@ +- feat(routing): add the default-off `DISABLE_CONTEXT_WINDOW_CHECKS` feature flag to let operators bypass OmniRoute's local context-window and max-input-token check for direct single-model requests, leaving upstream limits, prompt compression, and output-token caps intact. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 1104618012..c475f2cadf 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -282,6 +282,7 @@ OmniRoute provides a two-layer defense: request-side injection scanning and resp | `OMNIROUTE_PAYLOAD_RULES_RELOAD_MS` | `5000` | `open-sse/services/payloadRules.ts` | Reload interval (ms) for hot-reloading the payload rules file. Minimum `1000`. | | `OMNIROUTE_PREFER_CLAUDE_CODE_FOR_UNPREFIXED_CLAUDE_MODELS` | `false` | `open-sse/services/model.ts` | Opt-in: route bare `claude-*` model IDs from Claude Code clients through the Claude Code OAuth account instead of requiring a provider prefix. Explicit provider prefixes still win. Also configurable via a dashboard toggle on the Claude provider page. | | `COMBO_CONCURRENCY_PER_MODEL` | `3` | `open-sse/services/comboConfig.ts` | Per-model concurrency cap for round-robin combos (#9100). The round-robin combo semaphore was hard-capped at 3 concurrent requests per model with no override, serializing higher-concurrency traffic behind that cap. Validated to `>= 1`, clamped to `<= 32`. | +| `DISABLE_CONTEXT_WINDOW_CHECKS` | `false` | `open-sse/handlers/chatCore.ts` | Dangerous opt-in that skips OmniRoute's local context-window / max-input-token check for direct single-model requests. Upstream providers still enforce their actual limits; prompt compression and the model's own output-token cap remain active. Effective precedence is Feature Flags DB override > environment variable > default; no restart is required. | --- diff --git a/docs/reference/FEATURE_FLAGS.md b/docs/reference/FEATURE_FLAGS.md index 45eededb28..8b46db649b 100644 --- a/docs/reference/FEATURE_FLAGS.md +++ b/docs/reference/FEATURE_FLAGS.md @@ -76,13 +76,14 @@ used when neither a DB override nor an environment variable is present. | `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` | boolean | `true` | | Allow adding/validating providers on local/private addresses (127.0.0.1, localhost, LAN). On by default (local-first); disable for strict public-only blocking. Cloud-metadata stays blocked. | | `ENABLE_CC_COMPATIBLE_PROVIDER` | boolean | `false` | ✓ | Enable Claude Code compatible provider mode. | -### Policies (3) +### Policies (4) | Key | Type | Default | Restart | Description | | ----------------------------------------- | ------- | ---------- | ------- | ---------------------------------------------------------------------- | | `TOOL_POLICY_MODE` | enum | `disabled` | | Tool-use policy enforcement mode. Values: `disabled`, `warn`, `block`. | | `RATE_LIMIT_AUTO_ENABLE` | boolean | `false` | | Automatically enable rate limiting based on usage patterns. | | `ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE` | boolean | `false` | ✓ | Allow multiple connections per compatibility node. | +| `DISABLE_CONTEXT_WINDOW_CHECKS` | boolean | `false` | | Skip OmniRoute's local context-window / max-input-token check for direct single-model requests. Upstream limits still apply. | ### Runtime (11) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index c40bca8296..dcc281d3e3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -202,7 +202,10 @@ import { deriveRequestCapabilityRequirements, buildCapabilityMismatchMessage, } from "@/shared/constants/capabilities/capabilityFilter.ts"; -import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; +import { + areContextWindowChecksDisabled, + isFeatureFlagEnabled, +} from "@/shared/utils/featureFlags.ts"; import { resolveNoAuthEchoModel } from "./chatCore/noAuthEchoModel.ts"; import { REASONING_BUFFER_MIN_TRIGGER, @@ -2041,13 +2044,18 @@ export async function handleChatCore({ const modelOutputCap = toPositiveInteger( getExplicitModelOutputCap({ provider, model: effectiveModel }) ); + const contextWindowChecksDisabled = areContextWindowChecksDisabled(); const outputBudget = enforceOutputTokenBudget( body as Record, finalEstimatedInputTokens, - finalContextLimit, + contextWindowChecksDisabled ? Number.MAX_SAFE_INTEGER : finalContextLimit, targetFormat === FORMATS.CLAUDE && sourceFormat !== FORMATS.CLAUDE ? DEFAULT_MAX_TOKENS : 0, modelOutputCap, - toPositiveInteger(resolveInputTokenCapForGate({ provider, model: effectiveModel }, { isCombo })) + contextWindowChecksDisabled + ? null + : toPositiveInteger( + resolveInputTokenCapForGate({ provider, model: effectiveModel }, { isCombo }) + ) ); if (outputBudget.ok === false) { const exceededInputCap = outputBudget.maxInputTokens !== undefined; diff --git a/src/i18n/messages/de.json b/src/i18n/messages/de.json index fb4e14b485..c1ffc929ea 100644 --- a/src/i18n/messages/de.json +++ b/src/i18n/messages/de.json @@ -12716,6 +12716,10 @@ "ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE": { "description": "Mehrere Verbindungen für jeden Kompatibilitätsknoten zulassen." }, + "DISABLE_CONTEXT_WINDOW_CHECKS": { + "label": "Kontextfensterprüfungen deaktivieren", + "description": "Lokale Kontextfenster- und Maximaleingabetoken-Prüfung von OmniRoute für direkte Einzelmodell-Anfragen überspringen. Upstream-Anbieter erzwingen weiterhin ihre tatsächlichen Grenzen. Prompt-Komprimierung und Ausgabetoken-Grenzen bleiben aktiv." + }, "RESPONSES_PASSTHROUGH_DROP_COMMENTARY": { "description": "Interne Ausgabeelemente der Kommentarphase aus den Passthrough-Streams der Responses-API entfernen, bevor sie an Clients weitergeleitet werden. Deaktivieren Sie dieses Flag, um rohe Upstream-Kommentare zu erhalten." }, @@ -13356,6 +13360,7 @@ } }, "featureFlagCapabilityFilterEnabledDescription": "Lehnen Sie Anfragen ab, bevor sie versendet werden, wenn das Zielmodell über die erforderlichen Funktionen (Vision, Werkzeuge, strukturierte Ausgabe, Kontextfenster) nicht verfügt. Schützt direkte Einzelanbieteranfragen, die den Kombo-Schicht-Kompatibilitätsfilter umgehen.", + "featureFlagDisableContextWindowChecksDescription": "Lokale Kontextfenster- und Maximaleingabetoken-Prüfung von OmniRoute für direkte Einzelmodell-Anfragen überspringen. Upstream-Anbieter erzwingen weiterhin ihre tatsächlichen Grenzen. Prompt-Komprimierung und Ausgabetoken-Grenzen bleiben aktiv.", "publicSystem": { "notFound": { "title": "Seite nicht gefunden", diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index bdccf17e23..aab9e6eddc 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -12754,6 +12754,10 @@ "ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE": { "description": "Allow multiple connections for each compatibility node." }, + "DISABLE_CONTEXT_WINDOW_CHECKS": { + "label": "Disable Context Window Checks", + "description": "Skip OmniRoute's local context-window and max-input-token check for direct single-model requests. Upstream providers still enforce their actual limits. Prompt compression and output-token caps remain active." + }, "RESPONSES_PASSTHROUGH_DROP_COMMENTARY": { "description": "Remove internal commentary-phase output items from Responses API passthrough streams before forwarding them to clients. Disable this flag to receive raw upstream commentary." }, @@ -13394,6 +13398,7 @@ } }, "featureFlagCapabilityFilterEnabledDescription": "Reject requests before dispatch when the target model lacks required capabilities (vision, tools, structured output, context window). Protects direct single-provider requests that bypass the combo-layer compatibility filter.", + "featureFlagDisableContextWindowChecksDescription": "Skip OmniRoute's local context-window and max-input-token check for direct single-model requests. Upstream providers still enforce their actual limits. Prompt compression and output-token caps remain active.", "publicSystem": { "notFound": { "title": "Page not found", diff --git a/src/i18n/messages/pt-BR.json b/src/i18n/messages/pt-BR.json index a4f8a5f3cf..b3fe3125dd 100644 --- a/src/i18n/messages/pt-BR.json +++ b/src/i18n/messages/pt-BR.json @@ -12747,6 +12747,10 @@ "ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE": { "description": "Permite múltiplas conexões para cada nó de compatibilidade." }, + "DISABLE_CONTEXT_WINDOW_CHECKS": { + "label": "Desativar verificações da janela de contexto", + "description": "Ignora a verificação local do OmniRoute para janela de contexto e limite máximo de tokens de entrada em solicitações diretas a um único modelo. Os provedores upstream continuam aplicando seus limites reais. A compactação de prompts e os limites de tokens de saída permanecem ativos." + }, "RESPONSES_PASSTHROUGH_DROP_COMMENTARY": { "description": "Remove itens de saída da fase de comentário interno dos streams de passthrough da Responses API antes de encaminhá-los aos clientes. Desative esta flag para receber o comentário bruto do upstream." }, @@ -13387,6 +13391,7 @@ } }, "featureFlagCapabilityFilterEnabledDescription": "Rejeitar requisições antes do despacho quando o modelo alvo nao possui as capacidades necessarias (visao, ferramentas, saída estruturada, janela de contexto). Protege requisições diretas que ignoram o filtro de compatibilidade do combo.", + "featureFlagDisableContextWindowChecksDescription": "Ignora a verificação local do OmniRoute para janela de contexto e limite máximo de tokens de entrada em solicitações diretas a um único modelo. Os provedores upstream continuam aplicando seus limites reais. A compactação de prompts e os limites de tokens de saída permanecem ativos.", "publicSystem": { "notFound": { "title": "Página não encontrada", diff --git a/src/i18n/messages/vi.json b/src/i18n/messages/vi.json index eb1cb41b6e..b4ba071acd 100644 --- a/src/i18n/messages/vi.json +++ b/src/i18n/messages/vi.json @@ -12755,6 +12755,10 @@ "ALLOW_MULTI_CONNECTIONS_PER_COMPAT_NODE": { "description": "Cho phép nhiều kết nối trên mỗi node tương thích." }, + "DISABLE_CONTEXT_WINDOW_CHECKS": { + "label": "Tắt kiểm tra cửa sổ ngữ cảnh", + "description": "Bỏ qua kiểm tra cục bộ của OmniRoute về cửa sổ ngữ cảnh và giới hạn token đầu vào tối đa cho yêu cầu trực tiếp đến một mô hình đơn lẻ. Nhà cung cấp thượng nguồn vẫn áp dụng các giới hạn thực tế. Tính năng nén prompt và giới hạn token đầu ra vẫn hoạt động." + }, "RESPONSES_PASSTHROUGH_DROP_COMMENTARY": { "description": "Loại các mục đầu ra thuộc giai đoạn commentary nội bộ khỏi luồng chuyển tiếp Responses API trước khi gửi tới ứng dụng khách. Tắt cờ này để nhận nguyên dữ liệu commentary từ thượng nguồn." }, @@ -13395,6 +13399,7 @@ } }, "featureFlagCapabilityFilterEnabledDescription": "Từ chối yêu cầu trước khi gửi đi khi mô hình đích thiếu các khả năng bắt buộc (thị giác, công cụ, đầu ra có cấu trúc, cửa sổ ngữ cảnh). Bảo vệ các yêu cầu trực tiếp đến một nhà cung cấp khi chúng bỏ qua bộ lọc tương thích của combo.", + "featureFlagDisableContextWindowChecksDescription": "Bỏ qua kiểm tra cục bộ của OmniRoute về cửa sổ ngữ cảnh và giới hạn token đầu vào tối đa cho yêu cầu trực tiếp đến một mô hình đơn lẻ. Nhà cung cấp thượng nguồn vẫn áp dụng các giới hạn thực tế. Tính năng nén prompt và giới hạn token đầu ra vẫn hoạt động.", "publicSystem": { "notFound": { "title": "Không tìm thấy trang", diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index 98baeafbec..37861d2994 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -236,7 +236,7 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ warningLevel: "info", }, - // ──────────────── Policies (4) ──────────────── + // ──────────────── Policies (5) ──────────────── { key: "TOOL_POLICY_MODE", label: "Tool Policy Mode", @@ -271,6 +271,18 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: true, warningLevel: "info", }, + { + key: "DISABLE_CONTEXT_WINDOW_CHECKS", + label: "Disable Context Window Checks", + description: + "Skip OmniRoute's local context-window and max-input-token check for direct single-model requests. Upstream providers remain responsible for enforcing their actual limits. Off by default.", + descriptionI18nKey: "featureFlagDisableContextWindowChecksDescription", + category: "policies", + defaultValue: "false", + type: "boolean", + requiresRestart: false, + warningLevel: "danger", + }, { key: "CAPABILITY_FILTER_ENABLED", label: "Capability Filter", diff --git a/src/shared/utils/featureFlags.ts b/src/shared/utils/featureFlags.ts index 9a3582370d..54874fcbff 100644 --- a/src/shared/utils/featureFlags.ts +++ b/src/shared/utils/featureFlags.ts @@ -72,6 +72,22 @@ export function isCcCompatibleProviderEnabled(): boolean { return isFeatureFlagEnabled("ENABLE_CC_COMPATIBLE_PROVIDER"); } +/** + * Context-window checks are fail-safe: an unavailable flag store must never + * silently disable local request bounds. + */ +export function areContextWindowChecksDisabled(): boolean { + try { + return isFeatureFlagEnabled("DISABLE_CONTEXT_WINDOW_CHECKS"); + } catch (error) { + console.error( + "[featureFlags] Failed to resolve DISABLE_CONTEXT_WINDOW_CHECKS, keeping checks enabled:", + error instanceof Error ? error.message : error + ); + return false; + } +} + export function isApiKeyRevealEnabledFlag(): boolean { try { return isFeatureFlagEnabled("ALLOW_API_KEY_REVEAL"); diff --git a/tests/unit/chatcore-model-output-cap-wiring.test.ts b/tests/unit/chatcore-model-output-cap-wiring.test.ts index 0331aeda67..9cc0042a3e 100644 --- a/tests/unit/chatcore-model-output-cap-wiring.test.ts +++ b/tests/unit/chatcore-model-output-cap-wiring.test.ts @@ -12,6 +12,7 @@ process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const overridesDb = await import("../../src/lib/db/modelCapabilityOverrides.ts"); +const featureFlagsDb = await import("../../src/lib/db/featureFlags.ts"); const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); const PROVIDER = "capwire-testprov"; @@ -77,6 +78,7 @@ test.before(() => { test.after(() => { globalThis.fetch = originalFetch; + featureFlagsDb.removeFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS"); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); @@ -114,3 +116,31 @@ test("handleChatCore dispatches input within the model input cap", async () => { assert.equal(fetchCalls, 1); assert.ok(dispatchedBody, "input below the cap must reach the upstream"); }); + +test("DISABLE_CONTEXT_WINDOW_CHECKS lets direct-model input exceed the declared input cap", async () => { + dispatchedBody = null; + fetchCalls = 0; + featureFlagsDb.setFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS", "true"); + try { + const result = await handleChatCore(buildRequest(1, "x".repeat(200))); + assert.equal(result.success, true); + assert.equal(fetchCalls, 1, "disabled context checks must let the upstream decide"); + assert.ok(dispatchedBody, "oversized input must reach the upstream when the flag is enabled"); + } finally { + featureFlagsDb.removeFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS"); + } +}); + +test("DISABLE_CONTEXT_WINDOW_CHECKS keeps the direct model output cap active", async () => { + dispatchedBody = null; + fetchCalls = 0; + featureFlagsDb.setFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS", "true"); + try { + const result = await handleChatCore(buildRequest(REQUESTED_MAX_TOKENS, "x".repeat(200))); + assert.equal(result.success, true); + assert.equal(fetchCalls, 1); + assert.equal(dispatchedBody?.max_tokens, OUTPUT_CAP); + } finally { + featureFlagsDb.removeFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS"); + } +}); diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index ac75c52bfe..f26fe182de 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -28,9 +28,10 @@ const { isModelCatalogNamesEnabled, isArenaEloSyncEnabled, isControlPlaneProxyDirectFallbackEnabled, + areContextWindowChecksDisabled, } = await import("../../src/shared/utils/featureFlags.ts"); -const EXPECTED_FEATURE_FLAG_COUNT = 50; +const EXPECTED_FEATURE_FLAG_COUNT = 51; // ────────────────────────────────────────────────────── // Test group 1 — Flag definitions registry @@ -207,6 +208,16 @@ describe("featureFlagDefinitions", () => { assert.strictEqual(def.warningLevel, "caution"); } }); + + it("defines context-window check bypass as a dangerous opt-in policy flag", () => { + const def = FEATURE_FLAG_DEFINITIONS.find((d) => d.key === "DISABLE_CONTEXT_WINDOW_CHECKS"); + assert.ok(def, "DISABLE_CONTEXT_WINDOW_CHECKS should exist"); + assert.strictEqual(def.category, "policies"); + assert.strictEqual(def.type, "boolean"); + assert.strictEqual(def.defaultValue, "false"); + assert.strictEqual(def.requiresRestart, false); + assert.strictEqual(def.warningLevel, "danger"); + }); }); // ────────────────────────────────────────────────────── @@ -429,6 +440,34 @@ describe("resolveFeatureFlag", () => { removeFeatureFlagOverride("OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK"); } }); + + it("areContextWindowChecksDisabled defaults off and follows DB overrides", () => { + assert.strictEqual(areContextWindowChecksDisabled(), false); + try { + setFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS", "true"); + assert.strictEqual(areContextWindowChecksDisabled(), true); + } finally { + removeFeatureFlagOverride("DISABLE_CONTEXT_WINDOW_CHECKS"); + } + }); + + it("areContextWindowChecksDisabled keeps checks enabled when the flag store is unreadable", () => { + const originalError = console.error; + console.error = () => {}; + try { + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.mkdirSync(tmpDir, { recursive: true }); + const blockerPath = path.join(tmpDir, "storage.sqlite"); + fs.mkdirSync(blockerPath, { recursive: true }); + assert.strictEqual(areContextWindowChecksDisabled(), false); + } finally { + console.error = originalError; + core.resetDbInstance(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.mkdirSync(tmpDir, { recursive: true }); + } + }); }); });