diff --git a/changelog.d/fixes/13206-eval-runner-self-managed-context.md b/changelog.d/fixes/13206-eval-runner-self-managed-context.md new file mode 100644 index 0000000000..83852115bd --- /dev/null +++ b/changelog.d/fixes/13206-eval-runner-self-managed-context.md @@ -0,0 +1 @@ +- **fix(evals):** the eval runner now sends `x-omniroute-compression: off` and `x-omniroute-no-memory: true` on every case, so a graded case measures the model instead of the operator's injected output style, retrieved memory and `memory_*` tools ([#13139](https://github.com/diegosouzapw/OmniRoute/issues/13139), [#13206](https://github.com/diegosouzapw/OmniRoute/pull/13206)) — thanks @aaustinhuang diff --git a/src/lib/evals/runtime.ts b/src/lib/evals/runtime.ts index 75291ed1be..d46bae63c0 100644 --- a/src/lib/evals/runtime.ts +++ b/src/lib/evals/runtime.ts @@ -183,11 +183,28 @@ function resolveCaseModel(evalCase: Record, target: EvalTargetI return caseModel || "gpt-4o"; } -async function executeEvalCase( +/** + * Build the chat-completions request for one eval case. + * + * The runner manages its own context: a graded case must measure the model + * answering the case, not the operator's injected context. Two injections + * otherwise apply on the ordinary chat path — a selected output style is + * prepended as a system message (gated on `x-omniroute-compression`), and + * retrieved memory plus the built-in `memory_*` tools are appended once the + * request carries an API key (gated on `x-omniroute-no-memory`). Both are + * request-header opt-outs, so the runner sets them on every case. Without them + * a graded case answers in the configured persona or spends its turn calling + * `memory_*`, and a run that passes an API key scores *worse* than one that + * does not, because the key is what gives the request a memory owner (#13139). + * + * Exported so the header contract can be asserted without invoking the chat + * route — see tests/unit/evals-runtime-self-managed-headers-13139.test.ts. + */ +export function buildEvalCaseRequest( evalCase: Record, target: EvalTargetInput, apiKey: string | null -): Promise<{ output: string; durationMs: number; error?: string }> { +): Request { const input = evalCase.input && typeof evalCase.input === "object" && !Array.isArray(evalCase.input) ? (evalCase.input as Record) @@ -195,9 +212,7 @@ async function executeEvalCase( const model = resolveCaseModel(evalCase, target); const headers = new Headers({ "Content-Type": "application/json", - // #13139 — Eval cases must measure the model, not injected context. - // Disable output-style injection (persona system messages) and memory - // injection (retrieved context + memory_* tools) so grading is clean. + // Self-managed context — see the docblock above. "x-omniroute-compression": "off", "x-omniroute-no-memory": "true", }); @@ -206,7 +221,7 @@ async function executeEvalCase( headers.set("Authorization", `Bearer ${apiKey}`); } - const request = new Request("http://localhost/api/v1/chat/completions", { + return new Request("http://localhost/api/v1/chat/completions", { method: "POST", headers, body: JSON.stringify({ @@ -219,6 +234,14 @@ async function executeEvalCase( : 512, }), }); +} + +async function executeEvalCase( + evalCase: Record, + target: EvalTargetInput, + apiKey: string | null +): Promise<{ output: string; durationMs: number; error?: string }> { + const request = buildEvalCaseRequest(evalCase, target, apiKey); const startedAt = Date.now(); const response = await postChatCompletion(request); diff --git a/tests/unit/evals-runtime-self-managed-headers-13139.test.ts b/tests/unit/evals-runtime-self-managed-headers-13139.test.ts new file mode 100644 index 0000000000..71b33cff85 --- /dev/null +++ b/tests/unit/evals-runtime-self-managed-headers-13139.test.ts @@ -0,0 +1,98 @@ +/** + * Regression test for #13139 — the eval runner must mark its own requests as + * self-managed. + * + * The runner sends every graded case down the ordinary chat path, so a case + * picks up whatever that path injects: a selected output style is prepended as + * a system message (gated on `x-omniroute-compression !== "off"` — + * open-sse/handlers/chatCore.ts) and, when the run carries an API key, + * retrieved memory plus the built-in `memory_*` tools are added (gated on + * `x-omniroute-no-memory`). Both are request-header opt-outs and the runner set + * neither, so an evaluation measured the operator's injected context instead of + * the model. + * + * The case executor calls the chat route directly, so this pins the contract at + * the request it builds. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildEvalCaseRequest } from "../../src/lib/evals/runtime.ts"; + +async function readBody(request: Request): Promise> { + return JSON.parse(await new Response(request.body).text()) as Record; +} + +test("eval case requests opt out of output-style and memory injection (#13139)", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.headers.get("x-omniroute-compression"), "off"); + assert.equal(request.headers.get("x-omniroute-no-memory"), "true"); +}); + +test("the opt-out survives a run that carries an API key", () => { + // An API key is what gives the request a memory owner, so this is the + // configuration where the injection was worst: passing a key made a run score + // *worse*. Authorization must be added without dropping the opt-outs. + const request = buildEvalCaseRequest({ input: {} }, { type: "model", id: "gpt-4o" }, "sk-test"); + + assert.equal(request.headers.get("x-omniroute-compression"), "off"); + assert.equal(request.headers.get("x-omniroute-no-memory"), "true"); + assert.equal(request.headers.get("Authorization"), "Bearer sk-test"); + assert.equal(request.headers.get("Content-Type"), "application/json"); +}); + +test("no Authorization header is sent when the run has no API key", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.headers.get("Authorization"), null); + assert.equal(request.headers.get("Content-Type"), "application/json"); +}); + +test("the request stays a POST to the chat-completions route", () => { + const request = buildEvalCaseRequest({ input: {} }, { type: "suite-default" }, null); + + assert.equal(request.method, "POST"); + assert.equal(request.url, "http://localhost/api/v1/chat/completions"); +}); + +test("an explicit case max_tokens is honored and the 512 default still applies", async () => { + const explicit = buildEvalCaseRequest( + { input: { messages: [], max_tokens: 64 } }, + { type: "suite-default" }, + null + ); + const explicitBody = await readBody(explicit); + assert.equal(explicitBody.max_tokens, 64); + assert.equal(explicitBody.stream, false); + assert.deepEqual(explicitBody.messages, []); + + const implicit = buildEvalCaseRequest( + { input: { messages: [] } }, + { type: "suite-default" }, + null + ); + assert.equal((await readBody(implicit)).max_tokens, 512); +}); + +test("case input fields, the resolved model and the non-streaming flag all reach the body", async () => { + const request = buildEvalCaseRequest( + { input: { messages: [{ role: "user", content: "hi" }], temperature: 0.2 } }, + { type: "model", id: "claude-sonnet-4-20250514" }, + null + ); + const body = await readBody(request); + + assert.equal(body.model, "claude-sonnet-4-20250514"); + assert.equal(body.stream, false); + assert.deepEqual(body.messages, [{ role: "user", content: "hi" }]); + assert.equal(body.temperature, 0.2); +}); + +test("a non-object case input does not leak into the request body", async () => { + const request = buildEvalCaseRequest({ input: "not-an-object" }, { type: "suite-default" }, null); + const body = await readBody(request); + + assert.equal(body.stream, false); + assert.equal(body.max_tokens, 512); + assert.equal(body.model, "gpt-4o"); +});