feat(api): routing/4985 — configurable response-body validation + failover (#5684)

Integrated into release/v3.8.43.

open-sse/services/combo.ts

@@ -1971,7 +1971,12 @@ export async function handleComboChat({
               undefined;
             const effectiveConnectionId = selectedConnectionId || target.connectionId || "";
 
-            const quality = await validateResponseQuality(result, clientRequestedStream, log);
+            const quality = await validateResponseQuality(
+              result,
+              clientRequestedStream,
+              log,
+              config.responseValidation
+            );
             if (!quality.valid) {
               log.warn(
                 "COMBO",
@@ -3016,7 +3021,12 @@ async function handleRoundRobinCombo({
 
         // Success — validate response quality before returning
         if (result.ok) {
-          const quality = await validateResponseQuality(result, clientRequestedStream, log);
+          const quality = await validateResponseQuality(
+            result,
+            clientRequestedStream,
+            log,
+            config.responseValidation
+          );
           if (!quality.valid) {
             log.warn(
               "COMBO-RR",

open-sse/services/combo/responseValidation.ts

@@ -0,0 +1,206 @@
+/**
+ * Feature 4985 — configurable response-body validation for combo routing.
+ *
+ * A combo can declare a `responseValidation` predicate. When an upstream returns 200 OK
+ * but the parsed body fails the predicate, `validateResponseQuality` reports it as
+ * invalid, which the combo orchestrator already treats exactly like an HTTP error
+ * (skip this target → fail over to the next). All checks are declarative and safe:
+ * substring matching (no regex / no ReDoS) and a bounded dot-path resolver (no eval).
+ */
+
+export type JsonPathCondition = "exists" | "nonEmpty" | "equals" | "notEquals";
+
+export interface JsonPathPredicate {
+  path: string;
+  condition: JsonPathCondition;
+  value?: string | number | boolean;
+}
+
+export interface ResponseValidationConfig {
+  /** The assistant content must NOT contain any of these substrings. */
+  forbiddenSubstrings?: string[];
+  /** The assistant content must contain ALL of these substrings. */
+  requiredSubstrings?: string[];
+  /** The trimmed assistant content must be at least this many characters. */
+  minContentLength?: number;
+  /** Structural/value checks against the parsed JSON body (shape validation). */
+  jsonPathPredicates?: JsonPathPredicate[];
+}
+
+export interface ResponseValidationResult {
+  valid: boolean;
+  reason?: string;
+}
+
+const MAX_REASON_SNIPPET = 60;
+
+function snippet(value: string): string {
+  return value.length > MAX_REASON_SNIPPET ? `${value.slice(0, MAX_REASON_SNIPPET)}…` : value;
+}
+
+/**
+ * Parse a dot/bracket path (e.g. `choices[0].message.content`) into tokens with a
+ * single bounded left-to-right scan — no regex, no backtracking, no eval.
+ */
+export function parseJsonPath(path: string): Array<string | number> {
+  const tokens: Array<string | number> = [];
+  let buf = "";
+  const flush = () => {
+    if (buf) {
+      tokens.push(buf);
+      buf = "";
+    }
+  };
+  for (let i = 0; i < path.length; i++) {
+    const ch = path[i];
+    if (ch === ".") {
+      flush();
+    } else if (ch === "[") {
+      flush();
+      let inner = "";
+      i++;
+      while (i < path.length && path[i] !== "]") {
+        inner += path[i];
+        i++;
+      }
+      const trimmed = inner.trim();
+      const n = Number(trimmed);
+      tokens.push(trimmed !== "" && Number.isInteger(n) ? n : trimmed);
+    } else {
+      buf += ch;
+    }
+  }
+  flush();
+  return tokens;
+}
+
+/** Resolve a dot-path against a parsed JSON value. Returns `undefined` if any hop misses. */
+export function resolveJsonPath(root: unknown, path: string): unknown {
+  let current: unknown = root;
+  for (const token of parseJsonPath(path)) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof token === "number") {
+      if (!Array.isArray(current)) return undefined;
+      current = current[token];
+    } else {
+      if (typeof current !== "object") return undefined;
+      current = (current as Record<string, unknown>)[token];
+    }
+  }
+  return current;
+}
+
+function isNonEmpty(value: unknown): boolean {
+  if (value === null || value === undefined) return false;
+  if (typeof value === "string") return value.trim().length > 0;
+  if (Array.isArray(value)) return value.length > 0;
+  if (typeof value === "object") return Object.keys(value as object).length > 0;
+  return Boolean(value);
+}
+
+function checkCondition(value: unknown, condition: JsonPathCondition, expected: unknown): boolean {
+  switch (condition) {
+    case "exists":
+      return value !== undefined && value !== null;
+    case "nonEmpty":
+      return isNonEmpty(value);
+    case "equals":
+      return value === expected;
+    case "notEquals":
+      return value !== expected;
+  }
+}
+
+/** Best-effort extraction of the assistant's text content from a chat/Responses body. */
+export function extractContentText(json: unknown): string {
+  if (!json || typeof json !== "object") return "";
+  const obj = json as Record<string, unknown>;
+
+  // Chat Completions: choices[].message.content (string or array of parts).
+  const choices = obj.choices;
+  if (Array.isArray(choices)) {
+    const parts: string[] = [];
+    for (const choice of choices) {
+      const message = (choice as Record<string, unknown>)?.message as
+        | Record<string, unknown>
+        | undefined;
+      const content = message?.content;
+      if (typeof content === "string") parts.push(content);
+      else if (Array.isArray(content)) {
+        for (const part of content) {
+          const text = (part as Record<string, unknown>)?.text;
+          if (typeof text === "string") parts.push(text);
+        }
+      }
+    }
+    if (parts.length) return parts.join("");
+  }
+
+  // Responses API: output[].content[].text
+  const output = obj.output;
+  if (Array.isArray(output)) {
+    const parts: string[] = [];
+    for (const item of output) {
+      const content = (item as Record<string, unknown>)?.content;
+      if (Array.isArray(content)) {
+        for (const part of content) {
+          const text = (part as Record<string, unknown>)?.text;
+          if (typeof text === "string") parts.push(text);
+        }
+      }
+    }
+    if (parts.length) return parts.join("");
+  }
+
+  return "";
+}
+
+/**
+ * Evaluate the configured predicate against a parsed JSON response body.
+ * Returns `{ valid: true }` when there is no config or all checks pass.
+ */
+export function evaluateResponseValidation(
+  json: unknown,
+  config: ResponseValidationConfig | undefined | null
+): ResponseValidationResult {
+  if (!config || typeof config !== "object") return { valid: true };
+
+  const content = extractContentText(json);
+
+  for (const sub of config.forbiddenSubstrings ?? []) {
+    if (typeof sub === "string" && sub.length > 0 && content.includes(sub)) {
+      return { valid: false, reason: `response contains forbidden substring "${snippet(sub)}"` };
+    }
+  }
+
+  for (const sub of config.requiredSubstrings ?? []) {
+    if (typeof sub === "string" && sub.length > 0 && !content.includes(sub)) {
+      return { valid: false, reason: `response missing required substring "${snippet(sub)}"` };
+    }
+  }
+
+  if (
+    typeof config.minContentLength === "number" &&
+    Number.isFinite(config.minContentLength) &&
+    config.minContentLength > 0 &&
+    content.trim().length < config.minContentLength
+  ) {
+    return {
+      valid: false,
+      reason: `response content shorter than ${config.minContentLength} chars`,
+    };
+  }
+
+  for (const predicate of config.jsonPathPredicates ?? []) {
+    if (!predicate || typeof predicate.path !== "string" || !predicate.path) continue;
+    const resolved = resolveJsonPath(json, predicate.path);
+    if (!checkCondition(resolved, predicate.condition, predicate.value)) {
+      return {
+        valid: false,
+        reason: `jsonpath check failed: "${snippet(predicate.path)}" ${predicate.condition}`,
+      };
+    }
+  }
+
+  return { valid: true };
+}

open-sse/services/combo/runtimeUnits.ts

@@ -3,6 +3,7 @@ import { errorResponse } from "../../utils/error.ts";
 import { recordComboRequest } from "../comboMetrics.ts";
 import { resolveDelayMs } from "./comboPredicates.ts";
 import { validateResponseQuality } from "./validateQuality.ts";
+import type { ResponseValidationConfig } from "./responseValidation.ts";
 import type {
   ComboCollectionLike,
   ComboLike,
@@ -227,7 +228,12 @@ export async function executeRuntimeUnitCombo(args: {
           });
           return { response, unit };
         }
-        const quality = await validateResponseQuality(response, clientRequestedStream, args.log);
+        const quality = await validateResponseQuality(
+          response,
+          clientRequestedStream,
+          args.log,
+          args.config.responseValidation as ResponseValidationConfig | undefined
+        );
         if (quality.valid) {
           recordComboRequest(args.combo.name, unit.modelStr, {
             success: true,

open-sse/services/combo/validateQuality.ts

@@ -10,6 +10,10 @@ import {
   createSSEDataLineNormalizer,
   isKnownNonClaudeStreamPayload,
 } from "../../utils/streamHelpers.ts";
+import {
+  evaluateResponseValidation,
+  type ResponseValidationConfig,
+} from "./responseValidation.ts";
 import { getReasoningTokens } from "../../../src/lib/usage/tokenAccounting.ts";
 import type { ComboRetryAfter } from "./types.ts";
 
@@ -57,7 +61,8 @@ function responsesApiOutputHasContent(output: unknown): boolean {
 export async function validateResponseQuality(
   response: Response,
   isStreaming: boolean,
-  log: { warn?: (...args: unknown[]) => void }
+  log: { warn?: (...args: unknown[]) => void },
+  responseValidation?: ResponseValidationConfig | null
 ): Promise<{ valid: boolean; reason?: string; clonedResponse?: Response }> {
   // Issue #3685: For Claude SSE streaming responses, use a BOUNDED PEEK to
   // detect the empty-content-block pattern (content_filter stop_reason with
@@ -292,6 +297,15 @@ export async function validateResponseQuality(
     return { valid: false, reason: "response is not valid JSON" };
   }
 
+  // Feature 4985: apply the combo's configured response-body predicate. A failure here
+  // fails over to the next target via the same path as the built-in empty-content checks.
+  if (responseValidation) {
+    const verdict = evaluateResponseValidation(json, responseValidation);
+    if (!verdict.valid) {
+      return { valid: false, reason: verdict.reason };
+    }
+  }
+
   const choices = json?.choices;
   if (json?.object === "response") {
     if (!responsesApiOutputHasContent(json.output)) return { valid: false, reason: "empty_choices" };

open-sse/services/comboConfig.ts

@@ -6,6 +6,7 @@
  */
 
 import { MAX_TIMER_TIMEOUT_MS } from "../../src/shared/utils/runtimeTimeouts.ts";
+import type { ResponseValidationConfig } from "./combo/responseValidation.ts";
 
 /**
  * Maximum number of concurrent pre-screen checks (provider profile + availability)
@@ -63,6 +64,9 @@ const DEFAULT_COMBO_CONFIG = {
   resetAwareTieBandPercent: 5,
   resetAwareExhaustionGuardPercent: 10,
   failoverBeforeRetry: true,
+  // Feature 4985: configurable response-body validation predicate (per-combo). When set,
+  // a 200 OK whose body fails the predicate fails over to the next target.
+  responseValidation: undefined as ResponseValidationConfig | undefined,
   maxSetRetries: 0,
   setRetryDelayMs: 2000,
   // Zero-latency optimizations are opt-in because some modes can race targets or

src/app/(dashboard)/dashboard/combos/ResponseValidationEditor.tsx

@@ -0,0 +1,199 @@
+"use client";
+
+import React from "react";
+
+// Feature 4985 — per-combo response-body validation editor. Extracted into its own file
+// so the combos god-component (page.tsx) stays lean. Emits the same declarative shape the
+// Zod `responseValidationSchema` validates; backend evaluates it in validateResponseQuality.
+
+export type JsonPathCondition = "exists" | "nonEmpty" | "equals" | "notEquals";
+
+export interface ResponseValidationValue {
+  forbiddenSubstrings?: string[];
+  requiredSubstrings?: string[];
+  minContentLength?: number;
+  jsonPathPredicates?: Array<{
+    path: string;
+    condition: JsonPathCondition;
+    value?: string | number | boolean;
+  }>;
+}
+
+function tr(t: ((key: string) => string) | undefined, key: string, fallback: string): string {
+  if (!t) return fallback;
+  try {
+    const v = t(key);
+    return v && v !== key ? v : fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+const linesToArray = (text: string): string[] =>
+  text
+    .split("\n")
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+const arrayToLines = (arr?: string[]): string => (arr ?? []).join("\n");
+
+const INPUT_CLASS =
+  "w-full text-xs py-1.5 px-2 rounded border border-black/10 dark:border-white/10 bg-transparent focus:border-primary focus:outline-none";
+const LABEL_CLASS = "text-[11px] font-medium text-text-muted block mb-0.5";
+
+const CONDITIONS: JsonPathCondition[] = ["exists", "nonEmpty", "equals", "notEquals"];
+
+export function ResponseValidationEditor({
+  value,
+  onChange,
+  t,
+}: {
+  value?: ResponseValidationValue | null;
+  onChange: (next: ResponseValidationValue | undefined) => void;
+  t?: (key: string) => string;
+}) {
+  const v: ResponseValidationValue = value && typeof value === "object" ? value : {};
+
+  const emit = (draft: ResponseValidationValue) => {
+    const cleaned: ResponseValidationValue = {};
+    if (draft.forbiddenSubstrings && draft.forbiddenSubstrings.length)
+      cleaned.forbiddenSubstrings = draft.forbiddenSubstrings;
+    if (draft.requiredSubstrings && draft.requiredSubstrings.length)
+      cleaned.requiredSubstrings = draft.requiredSubstrings;
+    if (typeof draft.minContentLength === "number" && draft.minContentLength > 0)
+      cleaned.minContentLength = draft.minContentLength;
+    if (draft.jsonPathPredicates && draft.jsonPathPredicates.length)
+      cleaned.jsonPathPredicates = draft.jsonPathPredicates;
+    onChange(Object.keys(cleaned).length ? cleaned : undefined);
+  };
+
+  const predicates = v.jsonPathPredicates ?? [];
+
+  const updatePredicate = (index: number, patch: Partial<(typeof predicates)[number]>) => {
+    const next = predicates.map((p, i) => (i === index ? { ...p, ...patch } : p));
+    emit({ ...v, jsonPathPredicates: next });
+  };
+
+  return (
+    <div className="flex flex-col gap-2" data-testid="response-validation-editor">
+      <p className="text-[10px] text-text-muted">
+        {tr(
+          t,
+          "responseValidationHelp",
+          "Fail over to the next target when a 200 OK body fails these checks (assistant content)."
+        )}
+      </p>
+
+      <div>
+        <label className={LABEL_CLASS}>
+          {tr(t, "responseValidationForbidden", "Forbidden substrings (one per line)")}
+        </label>
+        <textarea
+          rows={2}
+          value={arrayToLines(v.forbiddenSubstrings)}
+          onChange={(e) => emit({ ...v, forbiddenSubstrings: linesToArray(e.target.value) })}
+          placeholder={"I cannot help\nas an AI"}
+          data-testid="rv-forbidden"
+          className={INPUT_CLASS + " font-mono"}
+        />
+      </div>
+
+      <div>
+        <label className={LABEL_CLASS}>
+          {tr(t, "responseValidationRequired", "Required substrings (one per line)")}
+        </label>
+        <textarea
+          rows={2}
+          value={arrayToLines(v.requiredSubstrings)}
+          onChange={(e) => emit({ ...v, requiredSubstrings: linesToArray(e.target.value) })}
+          data-testid="rv-required"
+          className={INPUT_CLASS + " font-mono"}
+        />
+      </div>
+
+      <div>
+        <label className={LABEL_CLASS}>
+          {tr(t, "responseValidationMinLength", "Minimum content length (chars)")}
+        </label>
+        <input
+          type="number"
+          min={0}
+          value={typeof v.minContentLength === "number" ? v.minContentLength : ""}
+          onChange={(e) => {
+            const n = parseInt(e.target.value, 10);
+            emit({ ...v, minContentLength: Number.isFinite(n) && n > 0 ? n : undefined });
+          }}
+          data-testid="rv-min-length"
+          className={INPUT_CLASS}
+        />
+      </div>
+
+      <div>
+        <label className={LABEL_CLASS}>
+          {tr(t, "responseValidationJsonPaths", "JSON-path checks")}
+        </label>
+        <div className="flex flex-col gap-1.5">
+          {predicates.map((predicate, index) => (
+            <div key={index} className="flex flex-wrap items-center gap-1.5" data-testid="rv-predicate-row">
+              <input
+                type="text"
+                value={predicate.path}
+                onChange={(e) => updatePredicate(index, { path: e.target.value })}
+                placeholder="choices[0].message.content"
+                data-testid="rv-predicate-path"
+                className={INPUT_CLASS + " font-mono flex-1 min-w-[140px]"}
+              />
+              <select
+                value={predicate.condition}
+                onChange={(e) =>
+                  updatePredicate(index, { condition: e.target.value as JsonPathCondition })
+                }
+                data-testid="rv-predicate-condition"
+                className={INPUT_CLASS + " w-auto"}
+              >
+                {CONDITIONS.map((c) => (
+                  <option key={c} value={c}>
+                    {c}
+                  </option>
+                ))}
+              </select>
+              {(predicate.condition === "equals" || predicate.condition === "notEquals") && (
+                <input
+                  type="text"
+                  value={predicate.value === undefined ? "" : String(predicate.value)}
+                  onChange={(e) => updatePredicate(index, { value: e.target.value })}
+                  placeholder="value"
+                  data-testid="rv-predicate-value"
+                  className={INPUT_CLASS + " w-auto"}
+                />
+              )}
+              <button
+                type="button"
+                onClick={() =>
+                  emit({ ...v, jsonPathPredicates: predicates.filter((_, i) => i !== index) })
+                }
+                data-testid="rv-predicate-remove"
+                className="text-[10px] px-1.5 py-1 rounded border border-black/10 dark:border-white/10 text-text-muted hover:text-red-500"
+              >
+                ✕
+              </button>
+            </div>
+          ))}
+          <button
+            type="button"
+            onClick={() =>
+              emit({
+                ...v,
+                jsonPathPredicates: [...predicates, { path: "", condition: "exists" }],
+              })
+            }
+            data-testid="rv-predicate-add"
+            className="self-start text-[10px] px-2 py-1 rounded border border-black/10 dark:border-white/10 text-text-muted hover:text-primary"
+          >
+            {tr(t, "responseValidationAddCheck", "+ Add check")}
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}

src/app/(dashboard)/dashboard/combos/page.tsx

@@ -14,6 +14,7 @@ import Toggle from "@/shared/components/Toggle";
 import Tooltip from "@/shared/components/Tooltip";
 import { useCopyToClipboard } from "@/shared/hooks/useCopyToClipboard";
 import { FieldLabelWithHelp, WeightTotalBar } from "./parts";
+import { ResponseValidationEditor, type ResponseValidationValue } from "./ResponseValidationEditor";
 import { pickDisplayValue } from "@/shared/utils/maskEmail";
 import useEmailPrivacyStore from "@/store/emailPrivacyStore";
 import { useNotificationStore } from "@/store/notificationStore";
@@ -4176,6 +4177,23 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
             </>
           )}
 
+          {/* Response Validation (4985) */}
+          {showStrategySection && (
+            <div className="flex flex-col gap-2 p-3 bg-black/[0.02] dark:bg-white/[0.02] rounded-lg border border-black/5 dark:border-white/5">
+              <div className="flex items-center gap-1.5 mb-1">
+                <span className="material-symbols-outlined text-[14px] text-primary">rule</span>
+                <p className="text-xs font-medium">
+                  {getI18nOrFallback(t, "responseValidationTitle", "Response validation")}
+                </p>
+              </div>
+              <ResponseValidationEditor
+                value={config.responseValidation as ResponseValidationValue | undefined}
+                onChange={(next) => setConfig({ ...config, responseValidation: next })}
+                t={t}
+              />
+            </div>
+          )}
+
           {/* Agent Features (#399 / #401 / #454) */}
           {showStrategySection && (
             <div className="flex flex-col gap-2 p-3 bg-black/[0.02] dark:bg-white/[0.02] rounded-lg border border-black/5 dark:border-white/5">

src/i18n/messages/en.json

@@ -2802,6 +2802,13 @@
     "browseLegacyCatalog": "Browse legacy combo catalog",
     "agentFeaturesTitle": "Agent Features",
     "agentFeaturesDescription": "Enable advanced features for agents using this combo",
+    "responseValidationTitle": "Response validation",
+    "responseValidationHelp": "Fail over to the next target when a 200 OK body fails these checks (assistant content).",
+    "responseValidationForbidden": "Forbidden substrings (one per line)",
+    "responseValidationRequired": "Required substrings (one per line)",
+    "responseValidationMinLength": "Minimum content length (chars)",
+    "responseValidationJsonPaths": "JSON-path checks",
+    "responseValidationAddCheck": "+ Add check",
     "agentFeaturesSystemMessageOverride": "Override system message",
     "agentFeaturesSystemMessagePlaceholder": "You are an expert assistant...",
     "agentFeaturesSystemMessageHint": "System message override for agents",

src/shared/validation/schemas/combo.ts

@@ -122,8 +122,29 @@ export const slaRoutingPolicySchema = z
   })
   .strict();
 
+// Feature 4985 — configurable response-body validation for combo routing. A 200 OK whose
+// body fails this predicate fails over to the next target (same path as an HTTP error).
+export const responseValidationSchema = z
+  .object({
+    forbiddenSubstrings: z.array(z.string().min(1).max(500)).max(50).optional(),
+    requiredSubstrings: z.array(z.string().min(1).max(500)).max(50).optional(),
+    minContentLength: z.coerce.number().int().min(0).max(1_000_000).optional(),
+    jsonPathPredicates: z
+      .array(
+        z.object({
+          path: z.string().trim().min(1).max(300),
+          condition: z.enum(["exists", "nonEmpty", "equals", "notEquals"]),
+          value: z.union([z.string().max(1000), z.number(), z.boolean()]).optional(),
+        })
+      )
+      .max(20)
+      .optional(),
+  })
+  .strict();
+
 export const comboRuntimeConfigSchema = z
   .object({
+    responseValidation: responseValidationSchema.optional(),
     strategy: comboStrategySchema.optional(),
     maxRetries: z.coerce.number().int().min(0).max(10).optional(),
     retryDelayMs: z.coerce.number().int().min(0).max(60000).optional(),

tests/unit/combo-response-validation-failover.test.ts

@@ -0,0 +1,96 @@
+/**
+ * Feature 4985 — a combo with a configured `responseValidation` predicate must fail over
+ * when a leg returns a 200 OK whose body fails the predicate, exactly like an HTTP error.
+ * Mirrors the #5085 empty-content failover harness (2 legs, different providers).
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-4985-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "combo-4985-test-secret";
+
+const { handleComboChat } = await import("../../open-sse/services/combo.ts");
+
+const noop = () => {};
+const log = { info: noop, warn: noop, debug: noop, error: noop };
+
+function content200(model: string, text: string) {
+  return new Response(
+    JSON.stringify({
+      id: "ok",
+      object: "chat.completion",
+      model,
+      choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }],
+    }),
+    { status: 200, headers: { "Content-Type": "application/json" } }
+  );
+}
+
+function makeCombo(models: string[], responseValidation: Record<string, unknown>) {
+  return {
+    name: "test-combo-4985",
+    strategy: "priority",
+    models: models.map((m) => ({ model: m })),
+    config: { responseValidation },
+  };
+}
+
+test("4985 fails over when leg 1's 200 body trips a forbidden substring", async () => {
+  const modelsCalled: string[] = [];
+  const handleSingleModel = async (_body: unknown, modelStr: string) => {
+    modelsCalled.push(modelStr);
+    if (modelsCalled.length === 1) return content200(modelStr, "Sorry, I cannot help with that.");
+    return content200(modelStr, "Here is the answer you asked for.");
+  };
+
+  const result = await handleComboChat({
+    body: { model: "test", messages: [{ role: "user", content: "hi" }] },
+    combo: makeCombo(["nvidia/minimaxai/minimax-m3", "openai/gpt-4o-mini"], {
+      forbiddenSubstrings: ["I cannot help"],
+    }),
+    handleSingleModel,
+    log,
+    settings: {},
+    allCombos: [],
+  });
+
+  assert.equal(
+    modelsCalled.length,
+    2,
+    `predicate failure on leg 1 must advance to leg 2, tried: ${modelsCalled.join(", ")}`
+  );
+  assert.equal(result.status, 200, "the combo must surface the healthy second leg's 200");
+  const body = JSON.parse(await result.text());
+  assert.match(
+    String(body?.choices?.[0]?.message?.content ?? ""),
+    /answer you asked for/,
+    "surfaced content must come from the leg that passed the predicate"
+  );
+});
+
+test("4985 does NOT fail over when the configured predicate passes", async () => {
+  const modelsCalled: string[] = [];
+  const handleSingleModel = async (_body: unknown, modelStr: string) => {
+    modelsCalled.push(modelStr);
+    return content200(modelStr, "A perfectly good answer.");
+  };
+
+  const result = await handleComboChat({
+    body: { model: "test", messages: [{ role: "user", content: "hi" }] },
+    combo: makeCombo(["nvidia/minimaxai/minimax-m3", "openai/gpt-4o-mini"], {
+      forbiddenSubstrings: ["I cannot help"],
+      minContentLength: 5,
+    }),
+    handleSingleModel,
+    log,
+    settings: {},
+    allCombos: [],
+  });
+
+  assert.equal(modelsCalled.length, 1, "a passing predicate must not trigger a needless failover");
+  assert.equal(result.status, 200);
+});

tests/unit/combo-response-validation.test.ts

@@ -0,0 +1,117 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  evaluateResponseValidation,
+  resolveJsonPath,
+  parseJsonPath,
+  extractContentText,
+  type ResponseValidationConfig,
+} from "../../open-sse/services/combo/responseValidation.ts";
+
+// Feature 4985 — configurable response-body validation predicate.
+
+const chat = (content: string | null, extra: Record<string, unknown> = {}) => ({
+  choices: [{ message: { content, ...extra } }],
+});
+
+test("no config → always valid", () => {
+  assert.deepEqual(evaluateResponseValidation(chat("anything"), undefined), { valid: true });
+  assert.deepEqual(evaluateResponseValidation(chat("anything"), null), { valid: true });
+  assert.deepEqual(evaluateResponseValidation(chat("anything"), {}), { valid: true });
+});
+
+test("forbiddenSubstrings: fails when the content contains one", () => {
+  const cfg: ResponseValidationConfig = { forbiddenSubstrings: ["I cannot help", "as an AI"] };
+  assert.equal(evaluateResponseValidation(chat("Sure, here you go"), cfg).valid, true);
+  const bad = evaluateResponseValidation(chat("Sorry, I cannot help with that"), cfg);
+  assert.equal(bad.valid, false);
+  assert.match(bad.reason ?? "", /forbidden substring/);
+});
+
+test("requiredSubstrings: fails when a required substring is missing", () => {
+  const cfg: ResponseValidationConfig = { requiredSubstrings: ["```"] };
+  assert.equal(evaluateResponseValidation(chat("```js\ncode\n```"), cfg).valid, true);
+  assert.equal(evaluateResponseValidation(chat("no fence here"), cfg).valid, false);
+});
+
+test("minContentLength: fails on near-empty content", () => {
+  const cfg: ResponseValidationConfig = { minContentLength: 10 };
+  assert.equal(evaluateResponseValidation(chat("plenty of characters here"), cfg).valid, true);
+  assert.equal(evaluateResponseValidation(chat("   hi   "), cfg).valid, false);
+});
+
+test("jsonPathPredicates: exists / nonEmpty / equals / notEquals", () => {
+  const body = chat("hello", { tool_calls: [] });
+  (body as Record<string, unknown>).usage = { total_tokens: 0 };
+  (body.choices[0] as Record<string, unknown>).finish_reason = "stop";
+
+  assert.equal(
+    evaluateResponseValidation(body, {
+      jsonPathPredicates: [{ path: "choices[0].message.content", condition: "nonEmpty" }],
+    }).valid,
+    true
+  );
+  assert.equal(
+    evaluateResponseValidation(body, {
+      jsonPathPredicates: [{ path: "choices[0].message.refusal", condition: "exists" }],
+    }).valid,
+    false
+  );
+  assert.equal(
+    evaluateResponseValidation(body, {
+      jsonPathPredicates: [{ path: "choices[0].finish_reason", condition: "equals", value: "stop" }],
+    }).valid,
+    true
+  );
+  assert.equal(
+    evaluateResponseValidation(body, {
+      jsonPathPredicates: [
+        { path: "choices[0].finish_reason", condition: "notEquals", value: "content_filter" },
+      ],
+    }).valid,
+    true
+  );
+  assert.equal(
+    evaluateResponseValidation(body, {
+      jsonPathPredicates: [{ path: "choices[0].finish_reason", condition: "equals", value: "length" }],
+    }).valid,
+    false
+  );
+});
+
+test("the first failing check wins; otherwise valid", () => {
+  const cfg: ResponseValidationConfig = {
+    forbiddenSubstrings: ["BAD"],
+    requiredSubstrings: ["GOOD"],
+    minContentLength: 3,
+  };
+  assert.equal(evaluateResponseValidation(chat("this is GOOD enough"), cfg).valid, true);
+  assert.equal(evaluateResponseValidation(chat("this is BAD and GOOD"), cfg).valid, false);
+});
+
+test("parseJsonPath tokenizes dot + bracket paths without regex", () => {
+  assert.deepEqual(parseJsonPath("choices[0].message.content"), ["choices", 0, "message", "content"]);
+  assert.deepEqual(parseJsonPath("a[1][2].b"), ["a", 1, 2, "b"]);
+  assert.deepEqual(parseJsonPath("plain"), ["plain"]);
+});
+
+test("resolveJsonPath returns undefined for missing hops (no throw)", () => {
+  const obj = { choices: [{ message: { content: "x" } }] };
+  assert.equal(resolveJsonPath(obj, "choices[0].message.content"), "x");
+  assert.equal(resolveJsonPath(obj, "choices[5].message.content"), undefined);
+  assert.equal(resolveJsonPath(obj, "a.b.c"), undefined);
+  assert.equal(resolveJsonPath(null, "a.b"), undefined);
+});
+
+test("extractContentText handles string, array parts, and Responses API output", () => {
+  assert.equal(extractContentText(chat("hello")), "hello");
+  assert.equal(
+    extractContentText({ choices: [{ message: { content: [{ text: "a" }, { text: "b" }] } }] }),
+    "ab"
+  );
+  assert.equal(
+    extractContentText({ output: [{ content: [{ text: "resp" }] }] }),
+    "resp"
+  );
+  assert.equal(extractContentText({}), "");
+});

tests/unit/ui/combo-response-validation-editor.test.tsx

@@ -0,0 +1,84 @@
+// @vitest-environment jsdom
+import React from "react";
+import { act } from "react";
+import { createRoot } from "react-dom/client";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  ResponseValidationEditor,
+  type ResponseValidationValue,
+} from "@/app/(dashboard)/dashboard/combos/ResponseValidationEditor";
+
+// Feature 4985 — the per-combo response-validation editor emits the declarative shape.
+
+let container: HTMLDivElement;
+let root: ReturnType<typeof createRoot> | undefined;
+
+beforeEach(() => {
+  container = document.createElement("div");
+  document.body.appendChild(container);
+});
+
+afterEach(() => {
+  act(() => root?.unmount());
+  root = undefined;
+  container.remove();
+  vi.restoreAllMocks();
+});
+
+function render(value: ResponseValidationValue | undefined, onChange: (v: unknown) => void) {
+  act(() => {
+    root = createRoot(container);
+    root.render(<ResponseValidationEditor value={value} onChange={onChange} />);
+  });
+}
+
+function setTextarea(testid: string, text: string) {
+  const el = container.querySelector<HTMLTextAreaElement>(`[data-testid="${testid}"]`);
+  const setter = Object.getOwnPropertyDescriptor(
+    window.HTMLTextAreaElement.prototype,
+    "value"
+  )!.set!;
+  act(() => {
+    setter.call(el, text);
+    el!.dispatchEvent(new Event("input", { bubbles: true }));
+  });
+}
+
+describe("ResponseValidationEditor (4985)", () => {
+  it("turns forbidden-substring lines into an array (trimmed, no blanks)", () => {
+    const onChange = vi.fn();
+    render(undefined, onChange);
+    setTextarea("rv-forbidden", "I cannot help\n\n  as an AI  \n");
+    expect(onChange).toHaveBeenLastCalledWith({
+      forbiddenSubstrings: ["I cannot help", "as an AI"],
+    });
+  });
+
+  it("clears the whole config back to undefined when every field is emptied", () => {
+    const onChange = vi.fn();
+    render({ forbiddenSubstrings: ["x"] }, onChange);
+    setTextarea("rv-forbidden", "");
+    expect(onChange).toHaveBeenLastCalledWith(undefined);
+  });
+
+  it("adds a json-path predicate row with sane defaults", () => {
+    const onChange = vi.fn();
+    render(undefined, onChange);
+    const addBtn = container.querySelector<HTMLButtonElement>('[data-testid="rv-predicate-add"]');
+    act(() => addBtn!.dispatchEvent(new MouseEvent("click", { bubbles: true })));
+    expect(onChange).toHaveBeenLastCalledWith({
+      jsonPathPredicates: [{ path: "", condition: "exists" }],
+    });
+  });
+
+  it("renders existing predicate rows from the value", () => {
+    render(
+      { jsonPathPredicates: [{ path: "choices[0].message.content", condition: "nonEmpty" }] },
+      vi.fn()
+    );
+    const rows = container.querySelectorAll('[data-testid="rv-predicate-row"]');
+    expect(rows.length).toBe(1);
+    const pathInput = container.querySelector<HTMLInputElement>('[data-testid="rv-predicate-path"]');
+    expect(pathInput?.value).toBe("choices[0].message.content");
+  });
+});