diff --git a/tests/unit/combo-10597-error-body-logging.test.ts b/tests/unit/combo-10597-error-body-logging.test.ts index 6df6a2cf49..a18f892716 100644 --- a/tests/unit/combo-10597-error-body-logging.test.ts +++ b/tests/unit/combo-10597-error-body-logging.test.ts @@ -48,14 +48,24 @@ function healthy200(model: string) { id: "ok", object: "chat.completion", model, - choices: [{ index: 0, message: { role: "assistant", content: "hello from " + model }, finish_reason: "stop" }], + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello from " + model }, + finish_reason: "stop", + }, + ], }), { status: 200, headers: { "Content-Type": "application/json" } } ); } function makeCombo(models: string[]) { - return { name: "test-combo-10597", strategy: "priority", models: models.map((m) => ({ model: m })) }; + return { + name: "test-combo-10597", + strategy: "priority", + models: models.map((m) => ({ model: m })), + }; } test("#10597 COMBO failure log must surface the upstream error body, not just the status code", async () => { @@ -79,7 +89,10 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th assert.equal(modelsCalled.length, 2); const failureLog = warnCalls.find( - (c) => typeof c.msg === "string" && c.msg.includes("claude/claude-opus-4-8") && c.msg.includes("failed") + (c) => + typeof c.msg === "string" && + c.msg.includes("claude/claude-opus-4-8") && + c.msg.includes("failed") ); assert.ok(failureLog, "expected a COMBO warn log for the failing leg"); @@ -89,3 +102,42 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th `expected the upstream error body to appear in the COMBO failure log, but got: ${serialized}` ); }); + +test("transcript-sensitive combo failures omit echoed transcript only from retained logs", async () => { + const transcriptSentinel = "PRIVATE_COMBO_ERROR_TRANSCRIPT_SENTINEL"; + const localWarnCalls: WarnCall[] = []; + const modelsCalled: string[] = []; + const result = await handleComboChat({ + body: { model: "test", messages: [{ role: "user", content: "processed video request" }] }, + combo: makeCombo(["claude/private-video", "openai/private-video-fallback"]), + handleSingleModel: async (_body: unknown, modelStr: string) => { + modelsCalled.push(modelStr); + if (modelsCalled.length === 1) { + return new Response(JSON.stringify({ error: { message: transcriptSentinel } }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + return healthy200(modelStr); + }, + log: { + info: () => {}, + debug: () => {}, + error: () => {}, + warn: (tag: string, msg: string, meta?: unknown) => { + localWarnCalls.push({ tag, msg, meta }); + }, + }, + settings: {}, + allCombos: [], + videoTranscriptSensitive: true, + }); + + assert.equal(result.status, 200); + assert.equal(modelsCalled.length, 2); + const retainedFailure = localWarnCalls.find((call) => call.msg.includes("failed, trying next")); + assert.ok(retainedFailure); + const serialized = JSON.stringify(retainedFailure); + assert.equal(serialized.includes(transcriptSentinel), false); + assert.match(serialized, /omitted: video transcript/); +}); diff --git a/tests/unit/combo-context-relay.test.ts b/tests/unit/combo-context-relay.test.ts index 5e83b460c3..57d3000e31 100644 --- a/tests/unit/combo-context-relay.test.ts +++ b/tests/unit/combo-context-relay.test.ts @@ -282,6 +282,52 @@ test("handleComboChat context-relay persists a handoff when codex quota reaches assert.equal(saved.fromAccount, connectionId); }); +test("handleComboChat does not persist a transcript-sensitive context-relay handoff", async () => { + const sessionId = "sess-private-video"; + const connectionId = "conn-private-video"; + touchSession(sessionId, connectionId); + registerCodexConnection(connectionId, { + accessToken: "token-private-video", + workspaceId: "ws-private-video", + }); + + let summaryCalls = 0; + globalThis.fetch = async (url) => { + if (String(url).includes("/backend-api/wham/usage")) return buildQuotaResponse(90); + throw new Error(`Unexpected fetch: ${String(url)}`); + }; + + const result = await handleComboChat({ + body: { + messages: [{ role: "user", content: "guardrail-produced video description" }], + }, + combo: { + name: "relay-private-video", + strategy: "context-relay", + models: ["codex/gpt-5.6-sol"], + config: { maxRetries: 0, handoffThreshold: 0.85, handoffProviders: ["codex"] }, + }, + handleSingleModel: async (body) => { + if (body._omnirouteInternalRequest === "context-handoff") summaryCalls += 1; + return okResponse(); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + allCombos: null, + relayOptions: { + sessionId, + config: { handoffThreshold: 0.85, handoffProviders: ["codex"] }, + }, + videoTranscriptSensitive: true, + }); + + await new Promise((resolve) => setTimeout(resolve, 75)); + assert.equal(result.ok, true); + assert.equal(summaryCalls, 0); + assert.equal(handoffDb.getHandoff(sessionId, "relay-private-video"), null); +}); + test("handleComboChat context-relay respects handoffProviders and skips generation when codex is disabled", async () => { const sessionId = "sess-disabled-provider"; const connectionId = "conn-disabled-provider"; diff --git a/tests/unit/context-handoff.test.ts b/tests/unit/context-handoff.test.ts index 845f118a5e..c3f09e6179 100644 --- a/tests/unit/context-handoff.test.ts +++ b/tests/unit/context-handoff.test.ts @@ -152,6 +152,40 @@ test("maybeGenerateHandoff skips below the warning threshold", async () => { assert.equal(handoffDb.getHandoff("sess-low", "relay-combo"), null); }); +test("maybeGenerateHandoff never summarizes a structurally transcript-sensitive history", async () => { + let called = false; + + contextHandoff.maybeGenerateHandoff({ + sessionId: "sess-private-video", + comboName: "relay-private-video", + connectionId: "conn-private-video", + percentUsed: 0.9, + messages: [ + { + role: "user", + content: [ + { + transcript: "PRIVATE_CONTEXT_HANDOFF_TRANSCRIPT_SENTINEL", + type: "input_video", + video_url: "data:video/mp4;base64,AA==", + }, + ], + }, + ], + model: "codex/gpt-5.6-sol", + expiresAt: null, + config: { handoffProviders: ["codex"] }, + handleSingleModel: async () => { + called = true; + return new Response("{}", { status: 200 }); + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(called, false); + assert.equal(handoffDb.getHandoff("sess-private-video", "relay-private-video"), null); +}); + test("maybeGenerateHandoff persists a structured handoff once the threshold is reached", async () => { const calls = []; diff --git a/tests/unit/rejected-request-usage.test.ts b/tests/unit/rejected-request-usage.test.ts index ed28f36fd6..2842d2531a 100644 --- a/tests/unit/rejected-request-usage.test.ts +++ b/tests/unit/rejected-request-usage.test.ts @@ -171,7 +171,6 @@ test("transcript-sensitive rejection omits retained error echoes and request cue apiKeyId: "key-transcript-sensitive-rejection", apiKeyName: "transcript-sensitive-rejection", requestBody, - videoTranscriptSensitive: true, }); let detail: Awaited> = null; diff --git a/tests/unit/request-log-payloads.test.ts b/tests/unit/request-log-payloads.test.ts index 788cdac557..a5273314c3 100644 --- a/tests/unit/request-log-payloads.test.ts +++ b/tests/unit/request-log-payloads.test.ts @@ -22,6 +22,7 @@ const { extractVideoTranscriptDescriptionFingerprints, fingerprintVideoTranscriptDescription, omitVideoTranscriptForLog, + resolveVideoTranscriptLogSensitivity, VIDEO_TRANSCRIPT_LOG_OMISSION_MARKER, } = await import("../../src/lib/guardrails/videoTranscriptLogRedaction.ts"); const { FORMATS } = await import("../../open-sse/translator/formats.ts"); @@ -596,6 +597,97 @@ test("fails closed when enumerable getters or proxies throw during transcript in } }); +test("fails closed when an enumerable function can replace the retained JSON representation", () => { + const privateTranscript = "PRIVATE_TOJSON_VIDEO_TRANSCRIPT_SENTINEL"; + const payload = { + safe: "retained diagnostic", + toJSON() { + return { + transcript: privateTranscript, + type: "input_video", + video_url: "data:video/mp4;base64,AA==", + }; + }, + }; + + assert.equal(containsVideoTranscriptForLog(payload), true); + const omitted = omitVideoTranscriptForLog(payload) as Record; + const serialized = JSON.stringify(omitted); + + assert.equal(serialized.includes(privateTranscript), false); + assert.equal(omitted.safe, "retained diagnostic"); + assert.notEqual(typeof omitted.toJSON, "function"); +}); + +test("treats explicit malformed and nested video carriers as transcript-sensitive", () => { + const privateTranscript = "PRIVATE_MALFORMED_VIDEO_TRANSCRIPT_SENTINEL"; + const payloads = [ + { + transcript: privateTranscript, + type: "input_video", + }, + { + transcript: privateTranscript, + type: "video_url", + video_url: "", + }, + { + input_video: { + transcript: privateTranscript, + url: "data:video/mp4;base64,AA==", + }, + type: "input_video", + }, + { + source: { transcript: privateTranscript }, + type: "video_source", + }, + { + source: { + media_type: "video/mp4", + transcript: privateTranscript, + }, + type: "video", + }, + ]; + + for (const payload of payloads) { + assert.equal(containsVideoTranscriptForLog(payload), true, JSON.stringify(payload)); + const serialized = JSON.stringify(omitVideoTranscriptForLog(payload)); + assert.equal(serialized.includes(privateTranscript), false, serialized); + assert.match(serialized, /omitted: video transcript/); + } +}); + +test("keeps raw request sensitivity after a guardrail removes the original carrier", () => { + const rawRequestBody = { + messages: [ + { + content: [ + { + transcript: "PRIVATE_RAW_REQUEST_TRANSCRIPT_SENTINEL", + type: "input_video", + video_url: "data:video/mp4;base64,AA==", + }, + ], + role: "user", + }, + ], + }; + const processedBody = { + messages: [{ content: "guardrail replaced the media", role: "user" }], + }; + + assert.equal(resolveVideoTranscriptLogSensitivity({ processedBody, rawRequestBody }), true); + assert.equal( + resolveVideoTranscriptLogSensitivity({ + processedBody: { metadata: { transcript: "ordinary audit label" } }, + rawRequestBody: { metadata: { transcript: "ordinary caller label" } }, + }), + false + ); +}); + test("fails closed at the aggregate traversal budget before a tail video transcript can leak", () => { const privateTranscript = "private over-budget video transcript sentinel"; const payload: unknown[] = Array.from( diff --git a/tests/unit/universal-handoff.test.ts b/tests/unit/universal-handoff.test.ts index 8ef12d4311..81f6b50ddf 100644 --- a/tests/unit/universal-handoff.test.ts +++ b/tests/unit/universal-handoff.test.ts @@ -160,6 +160,34 @@ test("providerAllowlist: empty allowlist allows all providers", async () => { assert.ok(calls.length > 0, "handleSingleModel MUST be called when allowlist is empty"); }); +test("transcript-sensitive model switches never generate a persistent universal handoff", async () => { + const calls: unknown[] = []; + maybeGenerateUniversalHandoff({ + sessionId: "ses_private_video", + comboName: "private-video-combo", + messages: [{ role: "user", content: "guardrail-produced video description" }], + prevModel: "openai/gpt-4o", + currModel: "anthropic/claude-3-5-sonnet", + videoTranscriptSensitive: true, + universalConfig: { + ...DEFAULT_UNIVERSAL_HANDOFF_CONFIG, + enabled: true, + providerAllowlist: [], + handoffModel: "anthropic/claude-3-5-sonnet", + }, + handleSingleModel: async (body, modelStr) => { + calls.push({ body, modelStr }); + return new Response(JSON.stringify({ choices: [{ message: { content: "{}" } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + }); + + await waitImmediate(); + assert.strictEqual(calls.length, 0); +}); + test("providerAllowlist: handoffModel takes precedence over currModel for allowlist check", async () => { const calls: unknown[] = []; await maybeGenerateUniversalHandoff({