From c43fbb1b3d7c8c81ff2d016e8eb919e906608575 Mon Sep 17 00:00:00 2001 From: Themedexperiencesusa Date: Fri, 18 Sep 2026 20:03:49 -0700 Subject: [PATCH] fix(security,resilience): block origin-IP header forwarding and treat 413 as retryable TPM (#13350) * fix(security): never forward origin-IP headers upstream Operator-set custom upstream headers could carry the client-origin IP (x-forwarded-for, x-real-ip, cf-connecting-ip, forwarded, via, ...) to the upstream provider, disclosing or spoofing it. Extend the FORBIDDEN denylist in upstreamHeaders.ts to cover the whole forwarding/IP set, mirroring the scrubbers already used by the Antigravity (antigravityHeaderScrub.ts) and Cursor CLI (cursorCliProxy.ts) paths, so the protection applies to every provider rather than those two. Covered by tests/unit/upstream-headers-sanitize.test.ts (6 passing). * fix(resilience): treat 413 payload-too-large as retryable TPM rate limit Providers with a tokens-per-minute cap (Groq among them) answer an oversized turn with 413 rather than 429. checkFallbackError() did not list 413 as retryable, so the request failed hard instead of falling back to another account or model. - Add PAYLOAD_TOO_LARGE (413) to HTTP_STATUS and to the retryable set - Return a MODEL_CAPACITY retryable fallback for 413 - Recognise "tokens per minute" / "tpm" as context-overflow patterns --------- Co-authored-by: Themedexperiencesusa <221764849+themedexperiencesusa@users.noreply.github.com> --- open-sse/config/constants.ts | 1 + open-sse/services/accountFallback.ts | 7 +++ src/shared/constants/upstreamHeaders.ts | 18 ++++++++ tests/unit/upstream-headers-sanitize.test.ts | 46 ++++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/open-sse/config/constants.ts b/open-sse/config/constants.ts index 82aeb0f3ba..d3d49d8eaa 100644 --- a/open-sse/config/constants.ts +++ b/open-sse/config/constants.ts @@ -187,6 +187,7 @@ export const HTTP_STATUS = { UNPROCESSABLE_ENTITY: 422, REQUEST_TIMEOUT: 408, GONE: 410, + PAYLOAD_TOO_LARGE: 413, RATE_LIMITED: 429, PLAN_LIMIT_EXCEEDED: 432, SERVER_ERROR: 500, diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index bedd556a52..db348e84bb 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -344,6 +344,8 @@ export const CONTEXT_OVERFLOW_PATTERNS = [ /\bmax.*token/i, /\btoken limit/i, /\brequest too large\b/i, + /\btokens per minute\b/i, + /\btpm\b/i, ]; // Structured error codes that reliably indicate model access denied @@ -1741,6 +1743,7 @@ export function checkFallbackError( const retryableStatuses = new Set([ HTTP_STATUS.REQUEST_TIMEOUT, HTTP_STATUS.RATE_LIMITED, + HTTP_STATUS.PAYLOAD_TOO_LARGE, HTTP_STATUS.SERVER_ERROR, HTTP_STATUS.BAD_GATEWAY, HTTP_STATUS.SERVICE_UNAVAILABLE, @@ -2208,6 +2211,10 @@ export function checkFallbackError( } if (status === HTTP_STATUS.NOT_ACCEPTABLE || retryableStatuses.has(status)) { + // 413 PAYLOAD_TOO_LARGE (TPM rate limits) should trigger fallback + if (status === HTTP_STATUS.PAYLOAD_TOO_LARGE) { + return buildRetryableFallback(RateLimitReason.MODEL_CAPACITY); + } return buildRetryableFallback(RateLimitReason.SERVER_ERROR); } diff --git a/src/shared/constants/upstreamHeaders.ts b/src/shared/constants/upstreamHeaders.ts index 5d9d7f7f08..fcc3f03d18 100644 --- a/src/shared/constants/upstreamHeaders.ts +++ b/src/shared/constants/upstreamHeaders.ts @@ -2,6 +2,12 @@ * User-supplied upstream extra headers: names we never forward (Host / hop-by-hop / framing). * Changing this list requires syncing: `sanitizeUpstreamHeadersMap` (models.ts), Zod * `upstreamHeaderNameSchema` / record refine (schemas.ts), and `upstream-headers-sanitize` tests. + * + * The forwarding/IP set (x-forwarded-for, x-real-ip, cf-connecting-ip, forwarded, via, …) + * is forbidden so the client-origin IP can never be disclosed (or spoofed) to the upstream + * provider through an operator-set custom upstream header. This mirrors the established + * scrubbers/denylists already used by the Antigravity (`antigravityHeaderScrub.ts`) and + * Cursor CLI (`cursorCliProxy.ts`) paths, extended here to cover every provider. */ const FORBIDDEN = new Set( [ @@ -24,6 +30,18 @@ const FORBIDDEN = new Set( "te", "trailer", "upgrade", + // Origin-IP disclosure: never send the client's forwarding headers upstream. + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-forwarded-port", + "x-forwarded-server", + "x-real-ip", + "cf-connecting-ip", + "true-client-ip", + "client-ip", + "forwarded", + "via", ].map((s) => s.toLowerCase()) ); diff --git a/tests/unit/upstream-headers-sanitize.test.ts b/tests/unit/upstream-headers-sanitize.test.ts index b9814e2d7d..935f4d5f7d 100644 --- a/tests/unit/upstream-headers-sanitize.test.ts +++ b/tests/unit/upstream-headers-sanitize.test.ts @@ -1,6 +1,10 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { sanitizeUpstreamHeadersMap } from "../../src/lib/db/models.ts"; +import { + isForbiddenUpstreamHeaderName, + isForbiddenCustomHeaderName, +} from "../../src/shared/constants/upstreamHeaders.ts"; test("sanitizeUpstreamHeadersMap: drops hop-by-hop / Host names", () => { const out = sanitizeUpstreamHeadersMap({ @@ -12,6 +16,48 @@ test("sanitizeUpstreamHeadersMap: drops hop-by-hop / Host names", () => { assert.deepEqual(out, { "X-Custom": "ok" }); }); +test("sanitizeUpstreamHeadersMap: drops origin-IP forwarding headers (no origin IP leak upstream)", () => { + const out = sanitizeUpstreamHeadersMap({ + "X-Custom": "kept", + "X-Forwarded-For": "203.0.113.9", + "X-Real-IP": "203.0.113.9", + "CF-Connecting-IP": "203.0.113.9", + Forwarded: "for=203.0.113.9", + Via: "1.1 proxy", + "True-Client-IP": "203.0.113.9", + "X-Forwarded-Host": "origin.example.com", + "X-Forwarded-Proto": "https", + }); + assert.deepEqual(out, { "X-Custom": "kept" }); +}); + +test("isForbiddenUpstreamHeaderName: blocks origin-IP forwarding headers", () => { + for (const name of [ + "x-forwarded-for", + "x-real-ip", + "cf-connecting-ip", + "forwarded", + "via", + "true-client-ip", + "client-ip", + "X-Forwarded-For", + "X-Real-IP", + "CF-Connecting-IP", + ]) { + assert.equal(isForbiddenUpstreamHeaderName(name), true, `${name} must be forbidden upstream`); + } + assert.equal(isForbiddenUpstreamHeaderName("x-custom-hdr"), false); +}); + +test("isForbiddenCustomHeaderName: blocks origin-IP forwarding headers for operator custom headers", () => { + assert.equal(isForbiddenCustomHeaderName("x-forwarded-for"), true); + assert.equal(isForbiddenCustomHeaderName("x-real-ip"), true); + assert.equal(isForbiddenCustomHeaderName("cf-connecting-ip"), true); + assert.equal(isForbiddenCustomHeaderName("forwarded"), true); + assert.equal(isForbiddenCustomHeaderName("via"), true); + assert.equal(isForbiddenCustomHeaderName("x-custom-hdr"), false); +}); + test("sanitizeUpstreamHeadersMap: drops values with CR/LF", () => { const out = sanitizeUpstreamHeadersMap({ Good: "a",