From 38494d9dffa6fa3157e7acf8d6d62d5cfc07b171 Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:04:18 -0300 Subject: [PATCH] fix(sse): classify a 2xx body as a disguised upstream failure (#13461) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pollinations and Perplexity-web can answer a genuine failure (expired session, exhausted free-tier credits) with HTTP 200 and a structurally normal completion whose assistant text is just the provider's own error prose. classifyProviderError() only inspects the body for 400/401/402/403/429, and detectMalformedNonStream() only checked structural emptiness, so the error text reached the client as a real answer and combo/auto-fallback never triggered. Adds classifyFakeSuccessBody() in errorClassifier.ts — allowlisted to pollinations/perplexity-web, reusing the existing CREDITS_EXHAUSTED_SIGNALS/ACCOUNT_DEACTIVATED_SIGNALS phrase lists, gated on short content with a dominant signal match — and wires it into detectMalformedNonStream() so the existing malformed-200 / combo-failover path picks it up with no other handler changes. Regression test: tests/unit/diagnostics-fake-success-13461.test.ts --- .../13461-fake-success-200-classifier.md | 1 + open-sse/handlers/chatCore.ts | 2 +- open-sse/services/errorClassifier.ts | 82 +++++++++++++ open-sse/utils/diagnostics.ts | 58 ++++++++- .../diagnostics-fake-success-13461.test.ts | 113 ++++++++++++++++++ 5 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixes/13461-fake-success-200-classifier.md create mode 100644 tests/unit/diagnostics-fake-success-13461.test.ts diff --git a/changelog.d/fixes/13461-fake-success-200-classifier.md b/changelog.d/fixes/13461-fake-success-200-classifier.md new file mode 100644 index 0000000000..6b57050794 --- /dev/null +++ b/changelog.d/fixes/13461-fake-success-200-classifier.md @@ -0,0 +1 @@ +- **fix(sse):** Pollinations and Perplexity-web requests now fail over instead of returning the provider's own "out of credits"/"account suspended" text as if it were a real answer — an HTTP 200 body whose short assistant message is dominated by a known credits-exhausted or account-deactivated phrase is now classified as a malformed response and triggers the existing combo/auto-fallback path (#13461) — thanks @arjav1181 diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 7ef9ef5541..ae35e30e6c 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -5390,7 +5390,7 @@ export async function handleChatCore({ // this check runs after translation + sanitization + tool-call execution to catch // cases where a provider returns a structurally valid raw body that translates into // choices:[] or output:[] with no usable content (Responses API shape included). - const malformedTranslatedReason = detectMalformedNonStream(translatedResponse); + const malformedTranslatedReason = detectMalformedNonStream(translatedResponse, provider); if (malformedTranslatedReason) { const totalLatency = Date.now() - startTime; const rawBytes = (() => { diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index c72d8f1ce8..2cefa33a34 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -1,4 +1,6 @@ import { + ACCOUNT_DEACTIVATED_SIGNALS, + CREDITS_EXHAUSTED_SIGNALS, isAccountDeactivated, isCreditsExhausted, isDailyQuotaExhausted, @@ -445,3 +447,83 @@ export function classifyProviderError( return null; } + +// ── "Fake success" 2xx body classifier (#13461) ───────────────────────────── +// +// Some free/web-session providers (reported: Pollinations, Perplexity web via +// cookie session) answer a genuine failure — expired session, exhausted +// free-tier credits — with HTTP 200 and a structurally normal completion +// whose assistant message is just the provider's own error prose. Neither +// classifyProviderError above (gated on 400/401/402/403/429 before it ever +// looks at the body — deliberately NOT changed by this fix, see below) nor +// detectMalformedNonStream (open-sse/utils/diagnostics.ts, structural +// emptiness only) catch this, so the error text is translated and forwarded +// to the client as if the model had genuinely answered with that sentence. +// +// Deliberately narrow, by owner decision (2026-09-15): +// - allowlist-only, starting with the two providers actually reported — +// never applied globally. classifyProviderError()'s status-code gate is +// intentionally left untouched; this lives in a separate sibling +// function instead of loosening that gate. +// - reuses the EXISTING, already-curated CREDITS_EXHAUSTED_SIGNALS / +// ACCOUNT_DEACTIVATED_SIGNALS phrase lists (open-sse/services/ +// accountFallback.ts) rather than inventing new fuzzy matching. +// - only trips on SHORT content whose matched signal covers a large +// fraction of it — a multi-paragraph answer that merely *mentions* the +// topic is long and/or the phrase is a small fraction of it, so it is +// never misclassified. +const FAKE_SUCCESS_BODY_ALLOWLIST = new Set(["pollinations", "perplexity-web"]); + +/** Exported for tests; not meant as a general-purpose provider predicate. */ +export function isFakeSuccessBodyAllowlistedProvider(provider?: string | null): boolean { + if (!provider) return false; + return FAKE_SUCCESS_BODY_ALLOWLIST.has(provider.toLowerCase()); +} + +// A real prose answer runs to paragraphs; a disguised upstream error is one +// short sentence. Generous headroom above every known signal phrase while +// still excluding genuine longer completions that merely mention the topic. +const FAKE_SUCCESS_MAX_CONTENT_LENGTH = 400; + +// The matched signal alone must make up a meaningful share of the message — +// keeps a legitimate answer that references the phrase in passing (as part +// of a much larger sentence/paragraph) from tripping this classifier. +const FAKE_SUCCESS_MIN_SIGNAL_COVERAGE = 0.12; + +function matchedSignalCoverage(lowerText: string, signals: readonly string[]): number { + let best = 0; + for (const signal of signals) { + if (lowerText.includes(signal) && signal.length > best) best = signal.length; + } + return lowerText.length > 0 ? best / lowerText.length : 0; +} + +/** + * Classify a *successful* (2xx) response's assistant-message text as a + * disguised upstream failure. Returns the matching ProviderErrorType, or + * null when the provider is not on the allowlist, the content is too long + * to be a bare error sentence, or no known signal phrase dominates it. + * + * Only ever meaningful for the narrow provider allowlist above — see + * isFakeSuccessBodyAllowlistedProvider and #13461. + */ +export function classifyFakeSuccessBody( + content: string, + provider?: string | null +): ProviderErrorType | null { + if (!isFakeSuccessBodyAllowlistedProvider(provider)) return null; + + const text = String(content || "").trim(); + if (!text || text.length > FAKE_SUCCESS_MAX_CONTENT_LENGTH) return null; + + const lower = text.toLowerCase(); + if (matchedSignalCoverage(lower, CREDITS_EXHAUSTED_SIGNALS) >= FAKE_SUCCESS_MIN_SIGNAL_COVERAGE) { + return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + } + if ( + matchedSignalCoverage(lower, ACCOUNT_DEACTIVATED_SIGNALS) >= FAKE_SUCCESS_MIN_SIGNAL_COVERAGE + ) { + return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED; + } + return null; +} diff --git a/open-sse/utils/diagnostics.ts b/open-sse/utils/diagnostics.ts index d7f56b54fa..bcfccee42b 100644 --- a/open-sse/utils/diagnostics.ts +++ b/open-sse/utils/diagnostics.ts @@ -10,6 +10,7 @@ */ import { sanitizeErrorMessage } from "./error.ts"; +import { classifyFakeSuccessBody } from "../services/errorClassifier.ts"; // ── Types ──────────────────────────────────────────────────────────────────── @@ -22,6 +23,10 @@ export type MalformedReason = | "parse_fail" | "empty_choices" | "empty_stream" + // #13461: a 2xx body whose assistant text is the provider's own error + // prose disguised as a successful completion (allowlisted providers only + // — see classifyFakeSuccessBody in open-sse/services/errorClassifier.ts). + | "content_is_upstream_error" | string; export interface ReportMalformed200Opts { @@ -52,6 +57,7 @@ const REASON_MESSAGES: Record = { parse_fail: "failed to parse upstream stream", empty_choices: "response had no usable choices/output", empty_stream: "upstream stream carried no content", + content_is_upstream_error: "upstream reported a failure disguised as a successful response", }; function describeReason(reason?: MalformedReason): string { @@ -172,8 +178,19 @@ export function synthResponsesFailure(reason?: MalformedReason): string { * - Claude Messages shape (type:"message" + content[]) is checked directly, * since a Claude client receives the body in that shape (no * `choices`/`object:"response"`). + * - #13461: for the narrow provider allowlist in classifyFakeSuccessBody + * (open-sse/services/errorClassifier.ts), a short Chat Completions + * assistant message that is dominated by a known credits-exhausted / + * account-deactivated phrase is treated as malformed too ("fake success") + * even though it carries non-empty content — see that function's doc + * comment for the false-positive guards. `provider` is optional and comes + * from the single call site in chatCore.ts; every other caller/shape is + * unaffected. */ -export function detectMalformedNonStream(resp: unknown): MalformedReason | null { +export function detectMalformedNonStream( + resp: unknown, + provider?: string | null +): MalformedReason | null { if (!resp || typeof resp !== "object") return "empty_choices"; const body = resp as Record; @@ -320,9 +337,41 @@ export function detectMalformedNonStream(resp: unknown): MalformedReason | null }); if (!anyHasOutput) return "empty_choices"; + + // #13461: only for the narrow provider allowlist — see classifyFakeSuccessBody's + // doc comment for the false-positive guards (short content + dominant signal). + if (provider && classifyFakeSuccessBody(extractChatCompletionText(choices), provider)) { + return "content_is_upstream_error"; + } + return null; } +// Joins every non-empty text-bearing field across all choices of a Chat +// Completions body into one string, for the #13461 fake-success check above. +// Mirrors the shapes `anyHasOutput` already recognizes as "real" content +// (plain string, Anthropic-style content-block array) — reasoning/tool_calls +// are intentionally excluded, since a disguised upstream error always +// surfaces as visible assistant text, never as a reasoning trace. +function extractChatCompletionText(choices: unknown[]): string { + const parts: string[] = []; + for (const choice of choices) { + const c = choice as Record; + const msg = c?.message as Record | undefined; + if (typeof msg?.content === "string") { + parts.push(msg.content as string); + } else if (Array.isArray(msg?.content)) { + for (const block of msg.content as unknown[]) { + const b = block as Record | null; + if (b && typeof b === "object" && b.type === "text" && typeof b.text === "string") { + parts.push(b.text as string); + } + } + } + } + return parts.join(" "); +} + export function describeMalformedNonStream( resp: unknown, reason: MalformedReason @@ -342,6 +391,13 @@ export function describeMalformedNonStream( type: "upstream_response_error", }; } + if (reason === "content_is_upstream_error") { + return { + message: "upstream reported a failure disguised as a successful response", + code: "upstream_fake_success", + type: "upstream_response_error", + }; + } return { message: reason === "no_terminal" diff --git a/tests/unit/diagnostics-fake-success-13461.test.ts b/tests/unit/diagnostics-fake-success-13461.test.ts new file mode 100644 index 0000000000..5c25403130 --- /dev/null +++ b/tests/unit/diagnostics-fake-success-13461.test.ts @@ -0,0 +1,113 @@ +/** + * Tests for the #13461 "fake success" 2xx-body classifier. + * + * A free/web-session provider (Pollinations, Perplexity web) can answer a + * genuine failure (expired session, exhausted free-tier credits) with HTTP + * 200 and a structurally normal completion whose message content is just + * the provider's own error prose. Neither classifyProviderError (gated on + * 400/401/402/403/429 before it ever looks at the body) nor + * detectMalformedNonStream (checks structural emptiness only) used to catch + * this — the text was forwarded to the client as if the model had genuinely + * answered with that sentence, and combo/auto-fallback never kicked in. + * + * Covers: + * (a) classifyFakeSuccessBody (open-sse/services/errorClassifier.ts) — new + * sibling classifier, allowlist-gated, reusing CREDITS_EXHAUSTED_SIGNALS + * / ACCOUNT_DEACTIVATED_SIGNALS with a short-content + signal-coverage + * guard. + * (b) detectMalformedNonStream (open-sse/utils/diagnostics.ts) — wired to + * consult the classifier for allowlisted providers so the existing + * malformed-200 / combo-failover path in chatCore.ts picks it up. + * (c) False-positive guards: non-allowlisted provider, long legitimate + * answer that merely mentions the topic, short reply without a + * recognized signal phrase. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { detectMalformedNonStream } from "../../open-sse/utils/diagnostics.ts"; +import { classifyFakeSuccessBody } from "../../open-sse/services/errorClassifier.ts"; + +function chatCompletion(content: string) { + return { + id: "chatcmpl-repro", + object: "chat.completion", + choices: [ + { + index: 0, + message: { role: "assistant", content }, + finish_reason: "stop", + }, + ], + }; +} + +// ── (a) classifyFakeSuccessBody ────────────────────────────────────────────── + +test("issue #13461: classifyFakeSuccessBody flags a short credits-exhausted body for an allowlisted provider", () => { + const content = + "You have run out of credits. Please sign up at https://enter.pollinations.ai to continue."; + assert.equal(classifyFakeSuccessBody(content, "pollinations"), "quota_exhausted"); +}); + +test("issue #13461: classifyFakeSuccessBody flags an account-deactivated body for an allowlisted provider", () => { + const content = "Sorry, your account has been suspended. Please contact support."; + assert.equal(classifyFakeSuccessBody(content, "perplexity-web"), "account_deactivated"); +}); + +test("issue #13461: classifyFakeSuccessBody ignores an unrecognized provider (allowlist guard)", () => { + const content = "Sorry, out of credits. Please sign up to continue."; + // Same identical phrase that IS recognized for an allowlisted provider — + // must stay untouched for a provider outside the initial allowlist so the + // blast radius of this fix stays controlled (#13461 owner decision). + assert.equal(classifyFakeSuccessBody(content, "openai"), null); + assert.equal(classifyFakeSuccessBody(content, "anthropic"), null); +}); + +test("issue #13461: classifyFakeSuccessBody ignores a long legitimate answer that merely mentions credits", () => { + const longAnswer = + "Managing your cloud spend well means watching a few things closely: set a monthly budget " + + "alert, review your invoice line items weekly, and make sure you never run out of credits " + + "mid-project by topping up before the low-balance warning fires. A lot of teams also sign up " + + "for a committed-use discount once their usage is predictable, which can meaningfully lower " + + "the effective per-unit cost over a full year of steady traffic."; + assert.ok( + longAnswer.length > 400, + "fixture must exceed the short-content guard to be meaningful" + ); + assert.equal(classifyFakeSuccessBody(longAnswer, "pollinations"), null); +}); + +test("issue #13461: classifyFakeSuccessBody ignores short content with no recognized signal phrase", () => { + const content = "Please go to perplexity.ai and sign up to continue using this feature."; + assert.equal(classifyFakeSuccessBody(content, "perplexity-web"), null); +}); + +// ── (b) detectMalformedNonStream wiring ────────────────────────────────────── + +test("issue #13461: detectMalformedNonStream flags HTTP 200 credits-exhausted text for an allowlisted provider", () => { + const translated = chatCompletion( + "You have run out of credits. Please sign up at https://enter.pollinations.ai to continue." + ); + assert.equal(detectMalformedNonStream(translated, "pollinations"), "content_is_upstream_error"); +}); + +test("issue #13461: detectMalformedNonStream leaves the identical body untouched without a provider", () => { + const translated = chatCompletion( + "You have run out of credits. Please sign up at https://enter.pollinations.ai to continue." + ); + assert.equal(detectMalformedNonStream(translated), null); +}); + +test("issue #13461: detectMalformedNonStream leaves the identical body untouched for a non-allowlisted provider", () => { + const translated = chatCompletion( + "You have run out of credits. Please sign up at https://enter.pollinations.ai to continue." + ); + assert.equal(detectMalformedNonStream(translated, "openai"), null); +}); + +test("issue #13461: detectMalformedNonStream never flags a normal completion for an allowlisted provider", () => { + const translated = chatCompletion("The capital of France is Paris."); + assert.equal(detectMalformedNonStream(translated, "pollinations"), null); +});