test(video): expose residual transcript retention paths

This commit is contained in:
diegosouzapw
2026-08-26 20:47:09 -03:00
parent d92088c5dc
commit dabf8dd580
6 changed files with 255 additions and 4 deletions

View File

@@ -48,14 +48,24 @@ function healthy200(model: string) {
id: "ok",
object: "chat.completion",
model,
choices: [{ index: 0, message: { role: "assistant", content: "hello from " + model }, finish_reason: "stop" }],
choices: [
{
index: 0,
message: { role: "assistant", content: "hello from " + model },
finish_reason: "stop",
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
function makeCombo(models: string[]) {
return { name: "test-combo-10597", strategy: "priority", models: models.map((m) => ({ model: m })) };
return {
name: "test-combo-10597",
strategy: "priority",
models: models.map((m) => ({ model: m })),
};
}
test("#10597 COMBO failure log must surface the upstream error body, not just the status code", async () => {
@@ -79,7 +89,10 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th
assert.equal(modelsCalled.length, 2);
const failureLog = warnCalls.find(
(c) => typeof c.msg === "string" && c.msg.includes("claude/claude-opus-4-8") && c.msg.includes("failed")
(c) =>
typeof c.msg === "string" &&
c.msg.includes("claude/claude-opus-4-8") &&
c.msg.includes("failed")
);
assert.ok(failureLog, "expected a COMBO warn log for the failing leg");
@@ -89,3 +102,42 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th
`expected the upstream error body to appear in the COMBO failure log, but got: ${serialized}`
);
});
test("transcript-sensitive combo failures omit echoed transcript only from retained logs", async () => {
const transcriptSentinel = "PRIVATE_COMBO_ERROR_TRANSCRIPT_SENTINEL";
const localWarnCalls: WarnCall[] = [];
const modelsCalled: string[] = [];
const result = await handleComboChat({
body: { model: "test", messages: [{ role: "user", content: "processed video request" }] },
combo: makeCombo(["claude/private-video", "openai/private-video-fallback"]),
handleSingleModel: async (_body: unknown, modelStr: string) => {
modelsCalled.push(modelStr);
if (modelsCalled.length === 1) {
return new Response(JSON.stringify({ error: { message: transcriptSentinel } }), {
status: 400,
headers: { "Content-Type": "application/json" },
});
}
return healthy200(modelStr);
},
log: {
info: () => {},
debug: () => {},
error: () => {},
warn: (tag: string, msg: string, meta?: unknown) => {
localWarnCalls.push({ tag, msg, meta });
},
},
settings: {},
allCombos: [],
videoTranscriptSensitive: true,
});
assert.equal(result.status, 200);
assert.equal(modelsCalled.length, 2);
const retainedFailure = localWarnCalls.find((call) => call.msg.includes("failed, trying next"));
assert.ok(retainedFailure);
const serialized = JSON.stringify(retainedFailure);
assert.equal(serialized.includes(transcriptSentinel), false);
assert.match(serialized, /omitted: video transcript/);
});

View File

@@ -282,6 +282,52 @@ test("handleComboChat context-relay persists a handoff when codex quota reaches
assert.equal(saved.fromAccount, connectionId);
});
test("handleComboChat does not persist a transcript-sensitive context-relay handoff", async () => {
const sessionId = "sess-private-video";
const connectionId = "conn-private-video";
touchSession(sessionId, connectionId);
registerCodexConnection(connectionId, {
accessToken: "token-private-video",
workspaceId: "ws-private-video",
});
let summaryCalls = 0;
globalThis.fetch = async (url) => {
if (String(url).includes("/backend-api/wham/usage")) return buildQuotaResponse(90);
throw new Error(`Unexpected fetch: ${String(url)}`);
};
const result = await handleComboChat({
body: {
messages: [{ role: "user", content: "guardrail-produced video description" }],
},
combo: {
name: "relay-private-video",
strategy: "context-relay",
models: ["codex/gpt-5.6-sol"],
config: { maxRetries: 0, handoffThreshold: 0.85, handoffProviders: ["codex"] },
},
handleSingleModel: async (body) => {
if (body._omnirouteInternalRequest === "context-handoff") summaryCalls += 1;
return okResponse();
},
isModelAvailable: async () => true,
log: createLog(),
settings: null,
allCombos: null,
relayOptions: {
sessionId,
config: { handoffThreshold: 0.85, handoffProviders: ["codex"] },
},
videoTranscriptSensitive: true,
});
await new Promise((resolve) => setTimeout(resolve, 75));
assert.equal(result.ok, true);
assert.equal(summaryCalls, 0);
assert.equal(handoffDb.getHandoff(sessionId, "relay-private-video"), null);
});
test("handleComboChat context-relay respects handoffProviders and skips generation when codex is disabled", async () => {
const sessionId = "sess-disabled-provider";
const connectionId = "conn-disabled-provider";

View File

@@ -152,6 +152,40 @@ test("maybeGenerateHandoff skips below the warning threshold", async () => {
assert.equal(handoffDb.getHandoff("sess-low", "relay-combo"), null);
});
test("maybeGenerateHandoff never summarizes a structurally transcript-sensitive history", async () => {
let called = false;
contextHandoff.maybeGenerateHandoff({
sessionId: "sess-private-video",
comboName: "relay-private-video",
connectionId: "conn-private-video",
percentUsed: 0.9,
messages: [
{
role: "user",
content: [
{
transcript: "PRIVATE_CONTEXT_HANDOFF_TRANSCRIPT_SENTINEL",
type: "input_video",
video_url: "data:video/mp4;base64,AA==",
},
],
},
],
model: "codex/gpt-5.6-sol",
expiresAt: null,
config: { handoffProviders: ["codex"] },
handleSingleModel: async () => {
called = true;
return new Response("{}", { status: 200 });
},
});
await new Promise((resolve) => setTimeout(resolve, 50));
assert.equal(called, false);
assert.equal(handoffDb.getHandoff("sess-private-video", "relay-private-video"), null);
});
test("maybeGenerateHandoff persists a structured handoff once the threshold is reached", async () => {
const calls = [];

View File

@@ -171,7 +171,6 @@ test("transcript-sensitive rejection omits retained error echoes and request cue
apiKeyId: "key-transcript-sensitive-rejection",
apiKeyName: "transcript-sensitive-rejection",
requestBody,
videoTranscriptSensitive: true,
});
let detail: Awaited<ReturnType<typeof callLogs.getCallLogById>> = null;

View File

@@ -22,6 +22,7 @@ const {
extractVideoTranscriptDescriptionFingerprints,
fingerprintVideoTranscriptDescription,
omitVideoTranscriptForLog,
resolveVideoTranscriptLogSensitivity,
VIDEO_TRANSCRIPT_LOG_OMISSION_MARKER,
} = await import("../../src/lib/guardrails/videoTranscriptLogRedaction.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
@@ -596,6 +597,97 @@ test("fails closed when enumerable getters or proxies throw during transcript in
}
});
test("fails closed when an enumerable function can replace the retained JSON representation", () => {
const privateTranscript = "PRIVATE_TOJSON_VIDEO_TRANSCRIPT_SENTINEL";
const payload = {
safe: "retained diagnostic",
toJSON() {
return {
transcript: privateTranscript,
type: "input_video",
video_url: "data:video/mp4;base64,AA==",
};
},
};
assert.equal(containsVideoTranscriptForLog(payload), true);
const omitted = omitVideoTranscriptForLog(payload) as Record<string, unknown>;
const serialized = JSON.stringify(omitted);
assert.equal(serialized.includes(privateTranscript), false);
assert.equal(omitted.safe, "retained diagnostic");
assert.notEqual(typeof omitted.toJSON, "function");
});
test("treats explicit malformed and nested video carriers as transcript-sensitive", () => {
const privateTranscript = "PRIVATE_MALFORMED_VIDEO_TRANSCRIPT_SENTINEL";
const payloads = [
{
transcript: privateTranscript,
type: "input_video",
},
{
transcript: privateTranscript,
type: "video_url",
video_url: "",
},
{
input_video: {
transcript: privateTranscript,
url: "data:video/mp4;base64,AA==",
},
type: "input_video",
},
{
source: { transcript: privateTranscript },
type: "video_source",
},
{
source: {
media_type: "video/mp4",
transcript: privateTranscript,
},
type: "video",
},
];
for (const payload of payloads) {
assert.equal(containsVideoTranscriptForLog(payload), true, JSON.stringify(payload));
const serialized = JSON.stringify(omitVideoTranscriptForLog(payload));
assert.equal(serialized.includes(privateTranscript), false, serialized);
assert.match(serialized, /omitted: video transcript/);
}
});
test("keeps raw request sensitivity after a guardrail removes the original carrier", () => {
const rawRequestBody = {
messages: [
{
content: [
{
transcript: "PRIVATE_RAW_REQUEST_TRANSCRIPT_SENTINEL",
type: "input_video",
video_url: "data:video/mp4;base64,AA==",
},
],
role: "user",
},
],
};
const processedBody = {
messages: [{ content: "guardrail replaced the media", role: "user" }],
};
assert.equal(resolveVideoTranscriptLogSensitivity({ processedBody, rawRequestBody }), true);
assert.equal(
resolveVideoTranscriptLogSensitivity({
processedBody: { metadata: { transcript: "ordinary audit label" } },
rawRequestBody: { metadata: { transcript: "ordinary caller label" } },
}),
false
);
});
test("fails closed at the aggregate traversal budget before a tail video transcript can leak", () => {
const privateTranscript = "private over-budget video transcript sentinel";
const payload: unknown[] = Array.from(

View File

@@ -160,6 +160,34 @@ test("providerAllowlist: empty allowlist allows all providers", async () => {
assert.ok(calls.length > 0, "handleSingleModel MUST be called when allowlist is empty");
});
test("transcript-sensitive model switches never generate a persistent universal handoff", async () => {
const calls: unknown[] = [];
maybeGenerateUniversalHandoff({
sessionId: "ses_private_video",
comboName: "private-video-combo",
messages: [{ role: "user", content: "guardrail-produced video description" }],
prevModel: "openai/gpt-4o",
currModel: "anthropic/claude-3-5-sonnet",
videoTranscriptSensitive: true,
universalConfig: {
...DEFAULT_UNIVERSAL_HANDOFF_CONFIG,
enabled: true,
providerAllowlist: [],
handoffModel: "anthropic/claude-3-5-sonnet",
},
handleSingleModel: async (body, modelStr) => {
calls.push({ body, modelStr });
return new Response(JSON.stringify({ choices: [{ message: { content: "{}" } }] }), {
status: 200,
headers: { "content-type": "application/json" },
});
},
});
await waitImmediate();
assert.strictEqual(calls.length, 0);
});
test("providerAllowlist: handoffModel takes precedence over currModel for allowlist check", async () => {
const calls: unknown[] = [];
await maybeGenerateUniversalHandoff({