fix(evals): mark eval-runner requests as self-managed so cases measure the model (#13139) (#13206)

* fix(evals): mark eval-runner requests as self-managed so cases measure the model

executeEvalCase() built its request with only Content-Type and Authorization, so
every graded case picked up the chat path's contextual injections: a selected
output style was prepended as a system message (gated on
`x-omniroute-compression`) and, once the request carried an API key, retrieved
memory plus the built-in `memory_*` tools were appended (gated on
`x-omniroute-no-memory`). An evaluation therefore measured the operator's
injected context as much as the model, and passing an API key to a run made its
score worse, because the key is what gives the request a memory owner (Refs #13139).

Both are documented request-header opt-outs, so the runner now sets them on every
case. Request construction moves to an exported buildEvalCaseRequest() so the
header contract is testable without invoking the chat route.

* docs(changelog): add the eval-runner self-managed-context fragment (#13206)

---------

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

changelog.d/fixes/13206-eval-runner-self-managed-context.md

@@ -0,0 +1 @@
+- **fix(evals):** the eval runner now sends `x-omniroute-compression: off` and `x-omniroute-no-memory: true` on every case, so a graded case measures the model instead of the operator's injected output style, retrieved memory and `memory_*` tools ([#13139](https://github.com/diegosouzapw/OmniRoute/issues/13139), [#13206](https://github.com/diegosouzapw/OmniRoute/pull/13206)) — thanks @aaustinhuang

src/lib/evals/runtime.ts

@@ -183,11 +183,28 @@ function resolveCaseModel(evalCase: Record<string, unknown>, target: EvalTargetI
   return caseModel || "gpt-4o";
 }
 
-async function executeEvalCase(
+/**
+ * Build the chat-completions request for one eval case.
+ *
+ * The runner manages its own context: a graded case must measure the model
+ * answering the case, not the operator's injected context. Two injections
+ * otherwise apply on the ordinary chat path — a selected output style is
+ * prepended as a system message (gated on `x-omniroute-compression`), and
+ * retrieved memory plus the built-in `memory_*` tools are appended once the
+ * request carries an API key (gated on `x-omniroute-no-memory`). Both are
+ * request-header opt-outs, so the runner sets them on every case. Without them
+ * a graded case answers in the configured persona or spends its turn calling
+ * `memory_*`, and a run that passes an API key scores *worse* than one that
+ * does not, because the key is what gives the request a memory owner (#13139).
+ *
+ * Exported so the header contract can be asserted without invoking the chat
+ * route — see tests/unit/evals-runtime-self-managed-headers-13139.test.ts.
+ */
+export function buildEvalCaseRequest(
   evalCase: Record<string, unknown>,
   target: EvalTargetInput,
   apiKey: string | null
-): Promise<{ output: string; durationMs: number; error?: string }> {
+): Request {
   const input =
     evalCase.input && typeof evalCase.input === "object" && !Array.isArray(evalCase.input)
       ? (evalCase.input as Record<string, unknown>)
@@ -195,9 +212,7 @@ async function executeEvalCase(
   const model = resolveCaseModel(evalCase, target);
   const headers = new Headers({
     "Content-Type": "application/json",
-    // #13139 — Eval cases must measure the model, not injected context.
-    // Disable output-style injection (persona system messages) and memory
-    // injection (retrieved context + memory_* tools) so grading is clean.
+    // Self-managed context — see the docblock above.
     "x-omniroute-compression": "off",
     "x-omniroute-no-memory": "true",
   });
@@ -206,7 +221,7 @@ async function executeEvalCase(
     headers.set("Authorization", `Bearer ${apiKey}`);
   }
 
-  const request = new Request("http://localhost/api/v1/chat/completions", {
+  return new Request("http://localhost/api/v1/chat/completions", {
     method: "POST",
     headers,
     body: JSON.stringify({
@@ -219,6 +234,14 @@ async function executeEvalCase(
           : 512,
     }),
   });
+}
+
+async function executeEvalCase(
+  evalCase: Record<string, unknown>,
+  target: EvalTargetInput,
+  apiKey: string | null
+): Promise<{ output: string; durationMs: number; error?: string }> {
+  const request = buildEvalCaseRequest(evalCase, target, apiKey);
 
   const startedAt = Date.now();
   const response = await postChatCompletion(request);

tests/unit/evals-runtime-self-managed-headers-13139.test.ts

@@ -0,0 +1,98 @@
+/**
+ * Regression test for #13139 — the eval runner must mark its own requests as
+ * self-managed.
+ *
+ * The runner sends every graded case down the ordinary chat path, so a case
+ * picks up whatever that path injects: a selected output style is prepended as
+ * a system message (gated on `x-omniroute-compression !== "off"` —
+ * open-sse/handlers/chatCore.ts) and, when the run carries an API key,
+ * retrieved memory plus the built-in `memory_*` tools are added (gated on
+ * `x-omniroute-no-memory`). Both are request-header opt-outs and the runner set
+ * neither, so an evaluation measured the operator's injected context instead of
+ * the model.
+ *
+ * The case executor calls the chat route directly, so this pins the contract at
+ * the request it builds.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { buildEvalCaseRequest } from "../../src/lib/evals/runtime.ts";
+
+async function readBody(request: Request): Promise<Record<string, unknown>> {
+  return JSON.parse(await new Response(request.body).text()) as Record<string, unknown>;
+}
+
+test("eval case requests opt out of output-style and memory injection (#13139)", () => {
+  const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null);
+
+  assert.equal(request.headers.get("x-omniroute-compression"), "off");
+  assert.equal(request.headers.get("x-omniroute-no-memory"), "true");
+});
+
+test("the opt-out survives a run that carries an API key", () => {
+  // An API key is what gives the request a memory owner, so this is the
+  // configuration where the injection was worst: passing a key made a run score
+  // *worse*. Authorization must be added without dropping the opt-outs.
+  const request = buildEvalCaseRequest({ input: {} }, { type: "model", id: "gpt-4o" }, "sk-test");
+
+  assert.equal(request.headers.get("x-omniroute-compression"), "off");
+  assert.equal(request.headers.get("x-omniroute-no-memory"), "true");
+  assert.equal(request.headers.get("Authorization"), "Bearer sk-test");
+  assert.equal(request.headers.get("Content-Type"), "application/json");
+});
+
+test("no Authorization header is sent when the run has no API key", () => {
+  const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null);
+
+  assert.equal(request.headers.get("Authorization"), null);
+  assert.equal(request.headers.get("Content-Type"), "application/json");
+});
+
+test("the request stays a POST to the chat-completions route", () => {
+  const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null);
+
+  assert.equal(request.method, "POST");
+  assert.equal(request.url, "http://localhost/api/v1/chat/completions");
+});
+
+test("an explicit case max_tokens is honored and the 512 default still applies", async () => {
+  const explicit = buildEvalCaseRequest(
+    { input: { messages: [], max_tokens: 64 } },
+    { type: "suite-default" },
+    null
+  );
+  const explicitBody = await readBody(explicit);
+  assert.equal(explicitBody.max_tokens, 64);
+  assert.equal(explicitBody.stream, false);
+  assert.deepEqual(explicitBody.messages, []);
+
+  const implicit = buildEvalCaseRequest(
+    { input: { messages: [] } },
+    { type: "suite-default" },
+    null
+  );
+  assert.equal((await readBody(implicit)).max_tokens, 512);
+});
+
+test("case input fields, the resolved model and the non-streaming flag all reach the body", async () => {
+  const request = buildEvalCaseRequest(
+    { input: { messages: [{ role: "user", content: "hi" }], temperature: 0.2 } },
+    { type: "model", id: "claude-sonnet-4-20250514" },
+    null
+  );
+  const body = await readBody(request);
+
+  assert.equal(body.model, "claude-sonnet-4-20250514");
+  assert.equal(body.stream, false);
+  assert.deepEqual(body.messages, [{ role: "user", content: "hi" }]);
+  assert.equal(body.temperature, 0.2);
+});
+
+test("a non-object case input does not leak into the request body", async () => {
+  const request = buildEvalCaseRequest({ input: "not-an-object" }, { type: "suite-default" }, null);
+  const body = await readBody(request);
+
+  assert.equal(body.stream, false);
+  assert.equal(body.max_tokens, 512);
+  assert.equal(body.model, "gpt-4o");
+});