From 10bb627576523c8a3d3d141d7db4b49b51c8f50e Mon Sep 17 00:00:00 2001 From: Wu Shuwen Date: Fri, 18 Sep 2026 22:58:30 +0800 Subject: [PATCH] fix(evals): mark eval-runner requests as self-managed so cases measure the model (#13139) (#13206) * fix(evals): mark eval-runner requests as self-managed so cases measure the model executeEvalCase() built its request with only Content-Type and Authorization, so every graded case picked up the chat path's contextual injections: a selected output style was prepended as a system message (gated on `x-omniroute-compression`) and, once the request carried an API key, retrieved memory plus the built-in `memory_*` tools were appended (gated on `x-omniroute-no-memory`). An evaluation therefore measured the operator's injected context as much as the model, and passing an API key to a run made its score worse, because the key is what gives the request a memory owner (Refs #13139). Both are documented request-header opt-outs, so the runner now sets them on every case. Request construction moves to an exported buildEvalCaseRequest() so the header contract is testable without invoking the chat route. * docs(changelog): add the eval-runner self-managed-context fragment (#13206) --------- Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../13206-eval-runner-self-managed-context.md | 1 + src/lib/evals/runtime.ts | 35 +++++-- ...runtime-self-managed-headers-13139.test.ts | 98 +++++++++++++++++++ 3 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 changelog.d/fixes/13206-eval-runner-self-managed-context.md create mode 100644 tests/unit/evals-runtime-self-managed-headers-13139.test.ts diff --git a/changelog.d/fixes/13206-eval-runner-self-managed-context.md b/changelog.d/fixes/13206-eval-runner-self-managed-context.md new file mode 100644 index 0000000000..83852115bd --- /dev/null +++ b/changelog.d/fixes/13206-eval-runner-self-managed-context.md @@ -0,0 +1 @@ +- **fix(evals):** the eval runner now sends `x-omniroute-compression: off` and `x-omniroute-no-memory: true` on every case, so a graded case measures the model instead of the operator's injected output style, retrieved memory and `memory_*` tools ([#13139](https://github.com/diegosouzapw/OmniRoute/issues/13139), [#13206](https://github.com/diegosouzapw/OmniRoute/pull/13206)) — thanks @aaustinhuang diff --git a/src/lib/evals/runtime.ts b/src/lib/evals/runtime.ts index 75291ed1be..d46bae63c0 100644 --- a/src/lib/evals/runtime.ts +++ b/src/lib/evals/runtime.ts @@ -183,11 +183,28 @@ function resolveCaseModel(evalCase: Record, target: EvalTargetI return caseModel || "gpt-4o"; } -async function executeEvalCase( +/** + * Build the chat-completions request for one eval case. + * + * The runner manages its own context: a graded case must measure the model + * answering the case, not the operator's injected context. Two injections + * otherwise apply on the ordinary chat path — a selected output style is + * prepended as a system message (gated on `x-omniroute-compression`), and + * retrieved memory plus the built-in `memory_*` tools are appended once the + * request carries an API key (gated on `x-omniroute-no-memory`). Both are + * request-header opt-outs, so the runner sets them on every case. Without them + * a graded case answers in the configured persona or spends its turn calling + * `memory_*`, and a run that passes an API key scores *worse* than one that + * does not, because the key is what gives the request a memory owner (#13139). + * + * Exported so the header contract can be asserted without invoking the chat + * route — see tests/unit/evals-runtime-self-managed-headers-13139.test.ts. + */ +export function buildEvalCaseRequest( evalCase: Record, target: EvalTargetInput, apiKey: string | null -): Promise<{ output: string; durationMs: number; error?: string }> { +): Request { const input = evalCase.input && typeof evalCase.input === "object" && !Array.isArray(evalCase.input) ? (evalCase.input as Record) @@ -195,9 +212,7 @@ async function executeEvalCase( const model = resolveCaseModel(evalCase, target); const headers = new Headers({ "Content-Type": "application/json", - // #13139 — Eval cases must measure the model, not injected context. - // Disable output-style injection (persona system messages) and memory - // injection (retrieved context + memory_* tools) so grading is clean. + // Self-managed context — see the docblock above. "x-omniroute-compression": "off", "x-omniroute-no-memory": "true", }); @@ -206,7 +221,7 @@ async function executeEvalCase( headers.set("Authorization", `Bearer ${apiKey}`); } - const request = new Request("http://localhost/api/v1/chat/completions", { + return new Request("http://localhost/api/v1/chat/completions", { method: "POST", headers, body: JSON.stringify({ @@ -219,6 +234,14 @@ async function executeEvalCase( : 512, }), }); +} + +async function executeEvalCase( + evalCase: Record, + target: EvalTargetInput, + apiKey: string | null +): Promise<{ output: string; durationMs: number; error?: string }> { + const request = buildEvalCaseRequest(evalCase, target, apiKey); const startedAt = Date.now(); const response = await postChatCompletion(request); diff --git a/tests/unit/evals-runtime-self-managed-headers-13139.test.ts b/tests/unit/evals-runtime-self-managed-headers-13139.test.ts new file mode 100644 index 0000000000..71b33cff85 --- /dev/null +++ b/tests/unit/evals-runtime-self-managed-headers-13139.test.ts @@ -0,0 +1,98 @@ +/** + * Regression test for #13139 — the eval runner must mark its own requests as + * self-managed. + * + * The runner sends every graded case down the ordinary chat path, so a case + * picks up whatever that path injects: a selected output style is prepended as + * a system message (gated on `x-omniroute-compression !== "off"` — + * open-sse/handlers/chatCore.ts) and, when the run carries an API key, + * retrieved memory plus the built-in `memory_*` tools are added (gated on + * `x-omniroute-no-memory`). Both are request-header opt-outs and the runner set + * neither, so an evaluation measured the operator's injected context instead of + * the model. + * + * The case executor calls the chat route directly, so this pins the contract at + * the request it builds. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildEvalCaseRequest } from "../../src/lib/evals/runtime.ts"; + +async function readBody(request: Request): Promise> { + return JSON.parse(await new Response(request.body).text()) as Record; +} + +test("eval case requests opt out of output-style and memory injection (#13139)", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.headers.get("x-omniroute-compression"), "off"); + assert.equal(request.headers.get("x-omniroute-no-memory"), "true"); +}); + +test("the opt-out survives a run that carries an API key", () => { + // An API key is what gives the request a memory owner, so this is the + // configuration where the injection was worst: passing a key made a run score + // *worse*. Authorization must be added without dropping the opt-outs. + const request = buildEvalCaseRequest({ input: {} }, { type: "model", id: "gpt-4o" }, "sk-test"); + + assert.equal(request.headers.get("x-omniroute-compression"), "off"); + assert.equal(request.headers.get("x-omniroute-no-memory"), "true"); + assert.equal(request.headers.get("Authorization"), "Bearer sk-test"); + assert.equal(request.headers.get("Content-Type"), "application/json"); +}); + +test("no Authorization header is sent when the run has no API key", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.headers.get("Authorization"), null); + assert.equal(request.headers.get("Content-Type"), "application/json"); +}); + +test("the request stays a POST to the chat-completions route", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.method, "POST"); + assert.equal(request.url, "http://localhost/api/v1/chat/completions"); +}); + +test("an explicit case max_tokens is honored and the 512 default still applies", async () => { + const explicit = buildEvalCaseRequest( + { input: { messages: [], max_tokens: 64 } }, + { type: "suite-default" }, + null + ); + const explicitBody = await readBody(explicit); + assert.equal(explicitBody.max_tokens, 64); + assert.equal(explicitBody.stream, false); + assert.deepEqual(explicitBody.messages, []); + + const implicit = buildEvalCaseRequest( + { input: { messages: [] } }, + { type: "suite-default" }, + null + ); + assert.equal((await readBody(implicit)).max_tokens, 512); +}); + +test("case input fields, the resolved model and the non-streaming flag all reach the body", async () => { + const request = buildEvalCaseRequest( + { input: { messages: [{ role: "user", content: "hi" }], temperature: 0.2 } }, + { type: "model", id: "claude-sonnet-4-20250514" }, + null + ); + const body = await readBody(request); + + assert.equal(body.model, "claude-sonnet-4-20250514"); + assert.equal(body.stream, false); + assert.deepEqual(body.messages, [{ role: "user", content: "hi" }]); + assert.equal(body.temperature, 0.2); +}); + +test("a non-object case input does not leak into the request body", async () => { + const request = buildEvalCaseRequest({ input: "not-an-object" }, { type: "suite-default" }, null); + const body = await readBody(request); + + assert.equal(body.stream, false); + assert.equal(body.max_tokens, 512); + assert.equal(body.model, "gpt-4o"); +});