From 50b2c84ee1d0b840a3ee66d0b359d16a9aa82ce8 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Tue, 30 Jun 2026 13:27:17 -0300 Subject: [PATCH] =?UTF-8?q?feat(api):=20routing/4985=20=E2=80=94=20configu?= =?UTF-8?q?rable=20response-body=20validation=20+=20failover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A combo can declare a per-combo responseValidation predicate. When an upstream returns 200 OK but the parsed body fails the predicate, the combo fails over to the next target through the exact same path the built-in empty-content guard uses (executeTarget returns null + 502). No new failover wiring. - open-sse/services/combo/responseValidation.ts: pure declarative evaluator — forbiddenSubstrings / requiredSubstrings / minContentLength / jsonPathPredicates with a bounded dot-path resolver (no eval, no regex/ReDoS). - Zod responseValidationSchema in shared/validation/schemas/combo.ts (per-combo, in comboRuntimeConfigSchema); threaded to validateResponseQuality (new 4th param) at all three call-sites (combo.ts x2 + runtimeUnits.ts); default key in comboConfig.ts. - Evaluated on non-streaming responses; streaming keeps its existing empty-content guard. - UI: extracted ResponseValidationEditor.tsx (keeps the combos god-component lean) wired into the combo form; i18n keys + getI18nOrFallback fallbacks. TDD: evaluator (9), failover integration mirroring #5085 (2), editor UI (4). Existing quality + combo-config + combo UI suites green; typecheck:core 0, ESLint 0, cycles clean. --- open-sse/services/combo.ts | 14 +- open-sse/services/combo/responseValidation.ts | 206 ++++++++++++++++++ open-sse/services/combo/runtimeUnits.ts | 8 +- open-sse/services/combo/validateQuality.ts | 16 +- open-sse/services/comboConfig.ts | 4 + .../combos/ResponseValidationEditor.tsx | 199 +++++++++++++++++ src/app/(dashboard)/dashboard/combos/page.tsx | 18 ++ src/i18n/messages/en.json | 7 + src/shared/validation/schemas/combo.ts | 21 ++ ...combo-response-validation-failover.test.ts | 96 ++++++++ tests/unit/combo-response-validation.test.ts | 117 ++++++++++ .../combo-response-validation-editor.test.tsx | 84 +++++++ 12 files changed, 786 insertions(+), 4 deletions(-) create mode 100644 open-sse/services/combo/responseValidation.ts create mode 100644 src/app/(dashboard)/dashboard/combos/ResponseValidationEditor.tsx create mode 100644 tests/unit/combo-response-validation-failover.test.ts create mode 100644 tests/unit/combo-response-validation.test.ts create mode 100644 tests/unit/ui/combo-response-validation-editor.test.tsx diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 157018d553..c027f8213b 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -1971,7 +1971,12 @@ export async function handleComboChat({ undefined; const effectiveConnectionId = selectedConnectionId || target.connectionId || ""; - const quality = await validateResponseQuality(result, clientRequestedStream, log); + const quality = await validateResponseQuality( + result, + clientRequestedStream, + log, + config.responseValidation + ); if (!quality.valid) { log.warn( "COMBO", @@ -3016,7 +3021,12 @@ async function handleRoundRobinCombo({ // Success — validate response quality before returning if (result.ok) { - const quality = await validateResponseQuality(result, clientRequestedStream, log); + const quality = await validateResponseQuality( + result, + clientRequestedStream, + log, + config.responseValidation + ); if (!quality.valid) { log.warn( "COMBO-RR", diff --git a/open-sse/services/combo/responseValidation.ts b/open-sse/services/combo/responseValidation.ts new file mode 100644 index 0000000000..d54855ee4e --- /dev/null +++ b/open-sse/services/combo/responseValidation.ts @@ -0,0 +1,206 @@ +/** + * Feature 4985 — configurable response-body validation for combo routing. + * + * A combo can declare a `responseValidation` predicate. When an upstream returns 200 OK + * but the parsed body fails the predicate, `validateResponseQuality` reports it as + * invalid, which the combo orchestrator already treats exactly like an HTTP error + * (skip this target → fail over to the next). All checks are declarative and safe: + * substring matching (no regex / no ReDoS) and a bounded dot-path resolver (no eval). + */ + +export type JsonPathCondition = "exists" | "nonEmpty" | "equals" | "notEquals"; + +export interface JsonPathPredicate { + path: string; + condition: JsonPathCondition; + value?: string | number | boolean; +} + +export interface ResponseValidationConfig { + /** The assistant content must NOT contain any of these substrings. */ + forbiddenSubstrings?: string[]; + /** The assistant content must contain ALL of these substrings. */ + requiredSubstrings?: string[]; + /** The trimmed assistant content must be at least this many characters. */ + minContentLength?: number; + /** Structural/value checks against the parsed JSON body (shape validation). */ + jsonPathPredicates?: JsonPathPredicate[]; +} + +export interface ResponseValidationResult { + valid: boolean; + reason?: string; +} + +const MAX_REASON_SNIPPET = 60; + +function snippet(value: string): string { + return value.length > MAX_REASON_SNIPPET ? `${value.slice(0, MAX_REASON_SNIPPET)}…` : value; +} + +/** + * Parse a dot/bracket path (e.g. `choices[0].message.content`) into tokens with a + * single bounded left-to-right scan — no regex, no backtracking, no eval. + */ +export function parseJsonPath(path: string): Array { + const tokens: Array = []; + let buf = ""; + const flush = () => { + if (buf) { + tokens.push(buf); + buf = ""; + } + }; + for (let i = 0; i < path.length; i++) { + const ch = path[i]; + if (ch === ".") { + flush(); + } else if (ch === "[") { + flush(); + let inner = ""; + i++; + while (i < path.length && path[i] !== "]") { + inner += path[i]; + i++; + } + const trimmed = inner.trim(); + const n = Number(trimmed); + tokens.push(trimmed !== "" && Number.isInteger(n) ? n : trimmed); + } else { + buf += ch; + } + } + flush(); + return tokens; +} + +/** Resolve a dot-path against a parsed JSON value. Returns `undefined` if any hop misses. */ +export function resolveJsonPath(root: unknown, path: string): unknown { + let current: unknown = root; + for (const token of parseJsonPath(path)) { + if (current === null || current === undefined) return undefined; + if (typeof token === "number") { + if (!Array.isArray(current)) return undefined; + current = current[token]; + } else { + if (typeof current !== "object") return undefined; + current = (current as Record)[token]; + } + } + return current; +} + +function isNonEmpty(value: unknown): boolean { + if (value === null || value === undefined) return false; + if (typeof value === "string") return value.trim().length > 0; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === "object") return Object.keys(value as object).length > 0; + return Boolean(value); +} + +function checkCondition(value: unknown, condition: JsonPathCondition, expected: unknown): boolean { + switch (condition) { + case "exists": + return value !== undefined && value !== null; + case "nonEmpty": + return isNonEmpty(value); + case "equals": + return value === expected; + case "notEquals": + return value !== expected; + } +} + +/** Best-effort extraction of the assistant's text content from a chat/Responses body. */ +export function extractContentText(json: unknown): string { + if (!json || typeof json !== "object") return ""; + const obj = json as Record; + + // Chat Completions: choices[].message.content (string or array of parts). + const choices = obj.choices; + if (Array.isArray(choices)) { + const parts: string[] = []; + for (const choice of choices) { + const message = (choice as Record)?.message as + | Record + | undefined; + const content = message?.content; + if (typeof content === "string") parts.push(content); + else if (Array.isArray(content)) { + for (const part of content) { + const text = (part as Record)?.text; + if (typeof text === "string") parts.push(text); + } + } + } + if (parts.length) return parts.join(""); + } + + // Responses API: output[].content[].text + const output = obj.output; + if (Array.isArray(output)) { + const parts: string[] = []; + for (const item of output) { + const content = (item as Record)?.content; + if (Array.isArray(content)) { + for (const part of content) { + const text = (part as Record)?.text; + if (typeof text === "string") parts.push(text); + } + } + } + if (parts.length) return parts.join(""); + } + + return ""; +} + +/** + * Evaluate the configured predicate against a parsed JSON response body. + * Returns `{ valid: true }` when there is no config or all checks pass. + */ +export function evaluateResponseValidation( + json: unknown, + config: ResponseValidationConfig | undefined | null +): ResponseValidationResult { + if (!config || typeof config !== "object") return { valid: true }; + + const content = extractContentText(json); + + for (const sub of config.forbiddenSubstrings ?? []) { + if (typeof sub === "string" && sub.length > 0 && content.includes(sub)) { + return { valid: false, reason: `response contains forbidden substring "${snippet(sub)}"` }; + } + } + + for (const sub of config.requiredSubstrings ?? []) { + if (typeof sub === "string" && sub.length > 0 && !content.includes(sub)) { + return { valid: false, reason: `response missing required substring "${snippet(sub)}"` }; + } + } + + if ( + typeof config.minContentLength === "number" && + Number.isFinite(config.minContentLength) && + config.minContentLength > 0 && + content.trim().length < config.minContentLength + ) { + return { + valid: false, + reason: `response content shorter than ${config.minContentLength} chars`, + }; + } + + for (const predicate of config.jsonPathPredicates ?? []) { + if (!predicate || typeof predicate.path !== "string" || !predicate.path) continue; + const resolved = resolveJsonPath(json, predicate.path); + if (!checkCondition(resolved, predicate.condition, predicate.value)) { + return { + valid: false, + reason: `jsonpath check failed: "${snippet(predicate.path)}" ${predicate.condition}`, + }; + } + } + + return { valid: true }; +} diff --git a/open-sse/services/combo/runtimeUnits.ts b/open-sse/services/combo/runtimeUnits.ts index bceadb2fa4..96951b81a2 100644 --- a/open-sse/services/combo/runtimeUnits.ts +++ b/open-sse/services/combo/runtimeUnits.ts @@ -3,6 +3,7 @@ import { errorResponse } from "../../utils/error.ts"; import { recordComboRequest } from "../comboMetrics.ts"; import { resolveDelayMs } from "./comboPredicates.ts"; import { validateResponseQuality } from "./validateQuality.ts"; +import type { ResponseValidationConfig } from "./responseValidation.ts"; import type { ComboCollectionLike, ComboLike, @@ -227,7 +228,12 @@ export async function executeRuntimeUnitCombo(args: { }); return { response, unit }; } - const quality = await validateResponseQuality(response, clientRequestedStream, args.log); + const quality = await validateResponseQuality( + response, + clientRequestedStream, + args.log, + args.config.responseValidation as ResponseValidationConfig | undefined + ); if (quality.valid) { recordComboRequest(args.combo.name, unit.modelStr, { success: true, diff --git a/open-sse/services/combo/validateQuality.ts b/open-sse/services/combo/validateQuality.ts index e571c34200..fe7b00313e 100644 --- a/open-sse/services/combo/validateQuality.ts +++ b/open-sse/services/combo/validateQuality.ts @@ -10,6 +10,10 @@ import { createSSEDataLineNormalizer, isKnownNonClaudeStreamPayload, } from "../../utils/streamHelpers.ts"; +import { + evaluateResponseValidation, + type ResponseValidationConfig, +} from "./responseValidation.ts"; import { getReasoningTokens } from "../../../src/lib/usage/tokenAccounting.ts"; import type { ComboRetryAfter } from "./types.ts"; @@ -57,7 +61,8 @@ function responsesApiOutputHasContent(output: unknown): boolean { export async function validateResponseQuality( response: Response, isStreaming: boolean, - log: { warn?: (...args: unknown[]) => void } + log: { warn?: (...args: unknown[]) => void }, + responseValidation?: ResponseValidationConfig | null ): Promise<{ valid: boolean; reason?: string; clonedResponse?: Response }> { // Issue #3685: For Claude SSE streaming responses, use a BOUNDED PEEK to // detect the empty-content-block pattern (content_filter stop_reason with @@ -292,6 +297,15 @@ export async function validateResponseQuality( return { valid: false, reason: "response is not valid JSON" }; } + // Feature 4985: apply the combo's configured response-body predicate. A failure here + // fails over to the next target via the same path as the built-in empty-content checks. + if (responseValidation) { + const verdict = evaluateResponseValidation(json, responseValidation); + if (!verdict.valid) { + return { valid: false, reason: verdict.reason }; + } + } + const choices = json?.choices; if (json?.object === "response") { if (!responsesApiOutputHasContent(json.output)) return { valid: false, reason: "empty_choices" }; diff --git a/open-sse/services/comboConfig.ts b/open-sse/services/comboConfig.ts index 1378283796..aa26cd4703 100644 --- a/open-sse/services/comboConfig.ts +++ b/open-sse/services/comboConfig.ts @@ -6,6 +6,7 @@ */ import { MAX_TIMER_TIMEOUT_MS } from "../../src/shared/utils/runtimeTimeouts.ts"; +import type { ResponseValidationConfig } from "./combo/responseValidation.ts"; /** * Maximum number of concurrent pre-screen checks (provider profile + availability) @@ -63,6 +64,9 @@ const DEFAULT_COMBO_CONFIG = { resetAwareTieBandPercent: 5, resetAwareExhaustionGuardPercent: 10, failoverBeforeRetry: true, + // Feature 4985: configurable response-body validation predicate (per-combo). When set, + // a 200 OK whose body fails the predicate fails over to the next target. + responseValidation: undefined as ResponseValidationConfig | undefined, maxSetRetries: 0, setRetryDelayMs: 2000, // Zero-latency optimizations are opt-in because some modes can race targets or diff --git a/src/app/(dashboard)/dashboard/combos/ResponseValidationEditor.tsx b/src/app/(dashboard)/dashboard/combos/ResponseValidationEditor.tsx new file mode 100644 index 0000000000..f31d9fe979 --- /dev/null +++ b/src/app/(dashboard)/dashboard/combos/ResponseValidationEditor.tsx @@ -0,0 +1,199 @@ +"use client"; + +import React from "react"; + +// Feature 4985 — per-combo response-body validation editor. Extracted into its own file +// so the combos god-component (page.tsx) stays lean. Emits the same declarative shape the +// Zod `responseValidationSchema` validates; backend evaluates it in validateResponseQuality. + +export type JsonPathCondition = "exists" | "nonEmpty" | "equals" | "notEquals"; + +export interface ResponseValidationValue { + forbiddenSubstrings?: string[]; + requiredSubstrings?: string[]; + minContentLength?: number; + jsonPathPredicates?: Array<{ + path: string; + condition: JsonPathCondition; + value?: string | number | boolean; + }>; +} + +function tr(t: ((key: string) => string) | undefined, key: string, fallback: string): string { + if (!t) return fallback; + try { + const v = t(key); + return v && v !== key ? v : fallback; + } catch { + return fallback; + } +} + +const linesToArray = (text: string): string[] => + text + .split("\n") + .map((s) => s.trim()) + .filter(Boolean); + +const arrayToLines = (arr?: string[]): string => (arr ?? []).join("\n"); + +const INPUT_CLASS = + "w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none"; +const LABEL_CLASS = "text-[11px] font-medium text-text-muted block mb-0.5"; + +const CONDITIONS: JsonPathCondition[] = ["exists", "nonEmpty", "equals", "notEquals"]; + +export function ResponseValidationEditor({ + value, + onChange, + t, +}: { + value?: ResponseValidationValue | null; + onChange: (next: ResponseValidationValue | undefined) => void; + t?: (key: string) => string; +}) { + const v: ResponseValidationValue = value && typeof value === "object" ? value : {}; + + const emit = (draft: ResponseValidationValue) => { + const cleaned: ResponseValidationValue = {}; + if (draft.forbiddenSubstrings && draft.forbiddenSubstrings.length) + cleaned.forbiddenSubstrings = draft.forbiddenSubstrings; + if (draft.requiredSubstrings && draft.requiredSubstrings.length) + cleaned.requiredSubstrings = draft.requiredSubstrings; + if (typeof draft.minContentLength === "number" && draft.minContentLength > 0) + cleaned.minContentLength = draft.minContentLength; + if (draft.jsonPathPredicates && draft.jsonPathPredicates.length) + cleaned.jsonPathPredicates = draft.jsonPathPredicates; + onChange(Object.keys(cleaned).length ? cleaned : undefined); + }; + + const predicates = v.jsonPathPredicates ?? []; + + const updatePredicate = (index: number, patch: Partial<(typeof predicates)[number]>) => { + const next = predicates.map((p, i) => (i === index ? { ...p, ...patch } : p)); + emit({ ...v, jsonPathPredicates: next }); + }; + + return ( +
+

+ {tr( + t, + "responseValidationHelp", + "Fail over to the next target when a 200 OK body fails these checks (assistant content)." + )} +

+ +
+ +