mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-21 14:22:14 +03:00
test(video): expose residual transcript retention paths
This commit is contained in:
@@ -48,14 +48,24 @@ function healthy200(model: string) {
|
||||
id: "ok",
|
||||
object: "chat.completion",
|
||||
model,
|
||||
choices: [{ index: 0, message: { role: "assistant", content: "hello from " + model }, finish_reason: "stop" }],
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: "assistant", content: "hello from " + model },
|
||||
finish_reason: "stop",
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
}
|
||||
|
||||
function makeCombo(models: string[]) {
|
||||
return { name: "test-combo-10597", strategy: "priority", models: models.map((m) => ({ model: m })) };
|
||||
return {
|
||||
name: "test-combo-10597",
|
||||
strategy: "priority",
|
||||
models: models.map((m) => ({ model: m })),
|
||||
};
|
||||
}
|
||||
|
||||
test("#10597 COMBO failure log must surface the upstream error body, not just the status code", async () => {
|
||||
@@ -79,7 +89,10 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th
|
||||
assert.equal(modelsCalled.length, 2);
|
||||
|
||||
const failureLog = warnCalls.find(
|
||||
(c) => typeof c.msg === "string" && c.msg.includes("claude/claude-opus-4-8") && c.msg.includes("failed")
|
||||
(c) =>
|
||||
typeof c.msg === "string" &&
|
||||
c.msg.includes("claude/claude-opus-4-8") &&
|
||||
c.msg.includes("failed")
|
||||
);
|
||||
assert.ok(failureLog, "expected a COMBO warn log for the failing leg");
|
||||
|
||||
@@ -89,3 +102,42 @@ test("#10597 COMBO failure log must surface the upstream error body, not just th
|
||||
`expected the upstream error body to appear in the COMBO failure log, but got: ${serialized}`
|
||||
);
|
||||
});
|
||||
|
||||
test("transcript-sensitive combo failures omit echoed transcript only from retained logs", async () => {
|
||||
const transcriptSentinel = "PRIVATE_COMBO_ERROR_TRANSCRIPT_SENTINEL";
|
||||
const localWarnCalls: WarnCall[] = [];
|
||||
const modelsCalled: string[] = [];
|
||||
const result = await handleComboChat({
|
||||
body: { model: "test", messages: [{ role: "user", content: "processed video request" }] },
|
||||
combo: makeCombo(["claude/private-video", "openai/private-video-fallback"]),
|
||||
handleSingleModel: async (_body: unknown, modelStr: string) => {
|
||||
modelsCalled.push(modelStr);
|
||||
if (modelsCalled.length === 1) {
|
||||
return new Response(JSON.stringify({ error: { message: transcriptSentinel } }), {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
return healthy200(modelStr);
|
||||
},
|
||||
log: {
|
||||
info: () => {},
|
||||
debug: () => {},
|
||||
error: () => {},
|
||||
warn: (tag: string, msg: string, meta?: unknown) => {
|
||||
localWarnCalls.push({ tag, msg, meta });
|
||||
},
|
||||
},
|
||||
settings: {},
|
||||
allCombos: [],
|
||||
videoTranscriptSensitive: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, 200);
|
||||
assert.equal(modelsCalled.length, 2);
|
||||
const retainedFailure = localWarnCalls.find((call) => call.msg.includes("failed, trying next"));
|
||||
assert.ok(retainedFailure);
|
||||
const serialized = JSON.stringify(retainedFailure);
|
||||
assert.equal(serialized.includes(transcriptSentinel), false);
|
||||
assert.match(serialized, /omitted: video transcript/);
|
||||
});
|
||||
|
||||
@@ -282,6 +282,52 @@ test("handleComboChat context-relay persists a handoff when codex quota reaches
|
||||
assert.equal(saved.fromAccount, connectionId);
|
||||
});
|
||||
|
||||
test("handleComboChat does not persist a transcript-sensitive context-relay handoff", async () => {
|
||||
const sessionId = "sess-private-video";
|
||||
const connectionId = "conn-private-video";
|
||||
touchSession(sessionId, connectionId);
|
||||
registerCodexConnection(connectionId, {
|
||||
accessToken: "token-private-video",
|
||||
workspaceId: "ws-private-video",
|
||||
});
|
||||
|
||||
let summaryCalls = 0;
|
||||
globalThis.fetch = async (url) => {
|
||||
if (String(url).includes("/backend-api/wham/usage")) return buildQuotaResponse(90);
|
||||
throw new Error(`Unexpected fetch: ${String(url)}`);
|
||||
};
|
||||
|
||||
const result = await handleComboChat({
|
||||
body: {
|
||||
messages: [{ role: "user", content: "guardrail-produced video description" }],
|
||||
},
|
||||
combo: {
|
||||
name: "relay-private-video",
|
||||
strategy: "context-relay",
|
||||
models: ["codex/gpt-5.6-sol"],
|
||||
config: { maxRetries: 0, handoffThreshold: 0.85, handoffProviders: ["codex"] },
|
||||
},
|
||||
handleSingleModel: async (body) => {
|
||||
if (body._omnirouteInternalRequest === "context-handoff") summaryCalls += 1;
|
||||
return okResponse();
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
allCombos: null,
|
||||
relayOptions: {
|
||||
sessionId,
|
||||
config: { handoffThreshold: 0.85, handoffProviders: ["codex"] },
|
||||
},
|
||||
videoTranscriptSensitive: true,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 75));
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(summaryCalls, 0);
|
||||
assert.equal(handoffDb.getHandoff(sessionId, "relay-private-video"), null);
|
||||
});
|
||||
|
||||
test("handleComboChat context-relay respects handoffProviders and skips generation when codex is disabled", async () => {
|
||||
const sessionId = "sess-disabled-provider";
|
||||
const connectionId = "conn-disabled-provider";
|
||||
|
||||
@@ -152,6 +152,40 @@ test("maybeGenerateHandoff skips below the warning threshold", async () => {
|
||||
assert.equal(handoffDb.getHandoff("sess-low", "relay-combo"), null);
|
||||
});
|
||||
|
||||
test("maybeGenerateHandoff never summarizes a structurally transcript-sensitive history", async () => {
|
||||
let called = false;
|
||||
|
||||
contextHandoff.maybeGenerateHandoff({
|
||||
sessionId: "sess-private-video",
|
||||
comboName: "relay-private-video",
|
||||
connectionId: "conn-private-video",
|
||||
percentUsed: 0.9,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
transcript: "PRIVATE_CONTEXT_HANDOFF_TRANSCRIPT_SENTINEL",
|
||||
type: "input_video",
|
||||
video_url: "data:video/mp4;base64,AA==",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
model: "codex/gpt-5.6-sol",
|
||||
expiresAt: null,
|
||||
config: { handoffProviders: ["codex"] },
|
||||
handleSingleModel: async () => {
|
||||
called = true;
|
||||
return new Response("{}", { status: 200 });
|
||||
},
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
assert.equal(called, false);
|
||||
assert.equal(handoffDb.getHandoff("sess-private-video", "relay-private-video"), null);
|
||||
});
|
||||
|
||||
test("maybeGenerateHandoff persists a structured handoff once the threshold is reached", async () => {
|
||||
const calls = [];
|
||||
|
||||
|
||||
@@ -171,7 +171,6 @@ test("transcript-sensitive rejection omits retained error echoes and request cue
|
||||
apiKeyId: "key-transcript-sensitive-rejection",
|
||||
apiKeyName: "transcript-sensitive-rejection",
|
||||
requestBody,
|
||||
videoTranscriptSensitive: true,
|
||||
});
|
||||
|
||||
let detail: Awaited<ReturnType<typeof callLogs.getCallLogById>> = null;
|
||||
|
||||
@@ -22,6 +22,7 @@ const {
|
||||
extractVideoTranscriptDescriptionFingerprints,
|
||||
fingerprintVideoTranscriptDescription,
|
||||
omitVideoTranscriptForLog,
|
||||
resolveVideoTranscriptLogSensitivity,
|
||||
VIDEO_TRANSCRIPT_LOG_OMISSION_MARKER,
|
||||
} = await import("../../src/lib/guardrails/videoTranscriptLogRedaction.ts");
|
||||
const { FORMATS } = await import("../../open-sse/translator/formats.ts");
|
||||
@@ -596,6 +597,97 @@ test("fails closed when enumerable getters or proxies throw during transcript in
|
||||
}
|
||||
});
|
||||
|
||||
test("fails closed when an enumerable function can replace the retained JSON representation", () => {
|
||||
const privateTranscript = "PRIVATE_TOJSON_VIDEO_TRANSCRIPT_SENTINEL";
|
||||
const payload = {
|
||||
safe: "retained diagnostic",
|
||||
toJSON() {
|
||||
return {
|
||||
transcript: privateTranscript,
|
||||
type: "input_video",
|
||||
video_url: "data:video/mp4;base64,AA==",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
assert.equal(containsVideoTranscriptForLog(payload), true);
|
||||
const omitted = omitVideoTranscriptForLog(payload) as Record<string, unknown>;
|
||||
const serialized = JSON.stringify(omitted);
|
||||
|
||||
assert.equal(serialized.includes(privateTranscript), false);
|
||||
assert.equal(omitted.safe, "retained diagnostic");
|
||||
assert.notEqual(typeof omitted.toJSON, "function");
|
||||
});
|
||||
|
||||
test("treats explicit malformed and nested video carriers as transcript-sensitive", () => {
|
||||
const privateTranscript = "PRIVATE_MALFORMED_VIDEO_TRANSCRIPT_SENTINEL";
|
||||
const payloads = [
|
||||
{
|
||||
transcript: privateTranscript,
|
||||
type: "input_video",
|
||||
},
|
||||
{
|
||||
transcript: privateTranscript,
|
||||
type: "video_url",
|
||||
video_url: "",
|
||||
},
|
||||
{
|
||||
input_video: {
|
||||
transcript: privateTranscript,
|
||||
url: "data:video/mp4;base64,AA==",
|
||||
},
|
||||
type: "input_video",
|
||||
},
|
||||
{
|
||||
source: { transcript: privateTranscript },
|
||||
type: "video_source",
|
||||
},
|
||||
{
|
||||
source: {
|
||||
media_type: "video/mp4",
|
||||
transcript: privateTranscript,
|
||||
},
|
||||
type: "video",
|
||||
},
|
||||
];
|
||||
|
||||
for (const payload of payloads) {
|
||||
assert.equal(containsVideoTranscriptForLog(payload), true, JSON.stringify(payload));
|
||||
const serialized = JSON.stringify(omitVideoTranscriptForLog(payload));
|
||||
assert.equal(serialized.includes(privateTranscript), false, serialized);
|
||||
assert.match(serialized, /omitted: video transcript/);
|
||||
}
|
||||
});
|
||||
|
||||
test("keeps raw request sensitivity after a guardrail removes the original carrier", () => {
|
||||
const rawRequestBody = {
|
||||
messages: [
|
||||
{
|
||||
content: [
|
||||
{
|
||||
transcript: "PRIVATE_RAW_REQUEST_TRANSCRIPT_SENTINEL",
|
||||
type: "input_video",
|
||||
video_url: "data:video/mp4;base64,AA==",
|
||||
},
|
||||
],
|
||||
role: "user",
|
||||
},
|
||||
],
|
||||
};
|
||||
const processedBody = {
|
||||
messages: [{ content: "guardrail replaced the media", role: "user" }],
|
||||
};
|
||||
|
||||
assert.equal(resolveVideoTranscriptLogSensitivity({ processedBody, rawRequestBody }), true);
|
||||
assert.equal(
|
||||
resolveVideoTranscriptLogSensitivity({
|
||||
processedBody: { metadata: { transcript: "ordinary audit label" } },
|
||||
rawRequestBody: { metadata: { transcript: "ordinary caller label" } },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("fails closed at the aggregate traversal budget before a tail video transcript can leak", () => {
|
||||
const privateTranscript = "private over-budget video transcript sentinel";
|
||||
const payload: unknown[] = Array.from(
|
||||
|
||||
@@ -160,6 +160,34 @@ test("providerAllowlist: empty allowlist allows all providers", async () => {
|
||||
assert.ok(calls.length > 0, "handleSingleModel MUST be called when allowlist is empty");
|
||||
});
|
||||
|
||||
test("transcript-sensitive model switches never generate a persistent universal handoff", async () => {
|
||||
const calls: unknown[] = [];
|
||||
maybeGenerateUniversalHandoff({
|
||||
sessionId: "ses_private_video",
|
||||
comboName: "private-video-combo",
|
||||
messages: [{ role: "user", content: "guardrail-produced video description" }],
|
||||
prevModel: "openai/gpt-4o",
|
||||
currModel: "anthropic/claude-3-5-sonnet",
|
||||
videoTranscriptSensitive: true,
|
||||
universalConfig: {
|
||||
...DEFAULT_UNIVERSAL_HANDOFF_CONFIG,
|
||||
enabled: true,
|
||||
providerAllowlist: [],
|
||||
handoffModel: "anthropic/claude-3-5-sonnet",
|
||||
},
|
||||
handleSingleModel: async (body, modelStr) => {
|
||||
calls.push({ body, modelStr });
|
||||
return new Response(JSON.stringify({ choices: [{ message: { content: "{}" } }] }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
await waitImmediate();
|
||||
assert.strictEqual(calls.length, 0);
|
||||
});
|
||||
|
||||
test("providerAllowlist: handoffModel takes precedence over currModel for allowlist check", async () => {
|
||||
const calls: unknown[] = [];
|
||||
await maybeGenerateUniversalHandoff({
|
||||
|
||||
Reference in New Issue
Block a user