From 4fee959ad204f40ffc6dbcbc346941df586e499c Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 27 Aug 2026 03:29:50 -0300 Subject: [PATCH] fix(video): preserve safe continuation and handoffs --- open-sse/handlers/chatCore.ts | 6 +- .../handlers/chatCore/memoryExtraction.ts | 77 +++++++++++++------ open-sse/services/combo.ts | 37 ++++----- open-sse/services/combo/dispatchPrelude.ts | 4 + open-sse/services/combo/types.ts | 2 + open-sse/services/contextHandoff.ts | 54 +++++++++---- src/lib/db/responsesContinuationStore.ts | 7 -- .../guardrails/videoTranscriptLogRedaction.ts | 28 +++++++ src/sse/handlers/chat.ts | 11 ++- 9 files changed, 156 insertions(+), 70 deletions(-) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 0c2ec5bb2b..faa63ab2ab 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -5123,7 +5123,8 @@ export async function handleChatCore({ if (memoryOwnerId && memorySettings?.enabled && memorySettings.maxTokens > 0) { const requestMemoryText = extractMemoryTextFromRequestBody( body as Record, - videoTranscriptSensitive + videoTranscriptSensitive, + { trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints } ); if (requestMemoryText) { extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId); @@ -5765,7 +5766,8 @@ export async function handleChatCore({ ) { const requestMemoryText = extractMemoryTextFromRequestBody( body as Record, - videoTranscriptSensitive + videoTranscriptSensitive, + { trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints } ); if (requestMemoryText) { extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId); diff --git a/open-sse/handlers/chatCore/memoryExtraction.ts b/open-sse/handlers/chatCore/memoryExtraction.ts index 3cbfababc8..f67ef4cda3 100644 --- a/open-sse/handlers/chatCore/memoryExtraction.ts +++ b/open-sse/handlers/chatCore/memoryExtraction.ts @@ -1,8 +1,37 @@ +import { + containsVideoTranscriptForLog, + omitVideoTranscriptDerivedTextForMemory, + type VideoTranscriptLogContext, +} from "../../../src/lib/guardrails/videoTranscriptLogRedaction.ts"; import { capMemoryExtractionText, MEMORY_EXTRACTION_TEXT_LIMIT } from "./logTruncation.ts"; -function normalizeMemoryInputText(value: unknown): string { +function normalizeMemoryInputText(value: unknown, context: VideoTranscriptLogContext = {}): string { if (typeof value !== "string") return ""; - return value.trim(); + return omitVideoTranscriptDerivedTextForMemory(value, context).trim(); +} + +function extractMemoryTextPart( + part: Record, + transcriptSensitive: boolean, + context: VideoTranscriptLogContext +): string { + try { + const rawText = typeof part?.text === "string" ? part.text : ""; + if (!rawText) return ""; + + const retainedText = normalizeMemoryInputText(rawText, context); + if (!retainedText) return ""; + if ( + transcriptSensitive && + containsVideoTranscriptForLog(part, context) && + retainedText === rawText.trim() + ) { + return ""; + } + return retainedText; + } catch { + return ""; + } } export function extractMemoryTextFromResponse( @@ -35,13 +64,15 @@ export function extractMemoryTextFromResponse( export function extractMemoryTextFromRequestBody( body: Record | null | undefined, - videoTranscriptSensitive = false + videoTranscriptSensitive = false, + context: VideoTranscriptLogContext = {} ): string { - // This bit is derived from the guardrail result. Caller-shaped lookalike text - // cannot suppress Memory extraction, while real media-derived cues cannot - // become durable facts (including through an adjacent response echo). - if (videoTranscriptSensitive) return ""; if (!body || typeof body !== "object") return ""; + // Re-check the structured body at the sink boundary. The explicit bit covers + // processed requests whose raw carrier was already replaced; trusted hashes + // identify only descriptions emitted by a modified Video Bridge guardrail. + const transcriptSensitive = + videoTranscriptSensitive || containsVideoTranscriptForLog(body, context); const messages = Array.isArray(body.messages) ? body.messages : null; if (messages && messages.length > 0) { @@ -49,18 +80,16 @@ export function extractMemoryTextFromRequestBody( const msg = messages[i] as Record; if (msg?.role !== "user") continue; - const messageText = normalizeMemoryInputText(msg.content); + const messageText = normalizeMemoryInputText(msg.content, context); if (messageText) { return capMemoryExtractionText(messageText); } if (Array.isArray(msg.content)) { const text = msg.content - .map((part: Record) => { - if (typeof part?.text === "string") return normalizeMemoryInputText(part.text); - if (part?.type === "input_text") return normalizeMemoryInputText(part.text); - return ""; - }) + .map((part: Record) => + extractMemoryTextPart(part, transcriptSensitive, context) + ) .filter(Boolean) .join("\n") .trim(); @@ -78,17 +107,15 @@ export function extractMemoryTextFromRequestBody( if (role && role !== "user") continue; if (itemType && itemType !== "message") continue; - const itemText = normalizeMemoryInputText(item?.content); + const itemText = normalizeMemoryInputText(item?.content, context); if (itemText) { return capMemoryExtractionText(itemText); } if (Array.isArray(item?.content)) { const text = item.content - .map((part: Record) => { - if (typeof part?.text === "string") return normalizeMemoryInputText(part.text); - if (part?.type === "input_text") return normalizeMemoryInputText(part.text); - return ""; - }) + .map((part: Record) => + extractMemoryTextPart(part, transcriptSensitive, context) + ) .filter(Boolean) .join("\n") .trim(); @@ -106,14 +133,14 @@ export function extractMemoryTextFromRequestBody( if (role && role !== "user") return ""; if (itemType && itemType !== "message") return ""; - if (typeof item?.content === "string") return normalizeMemoryInputText(item.content); + if (typeof item?.content === "string") { + return normalizeMemoryInputText(item.content, context); + } if (Array.isArray(item?.content)) { return item.content - .map((part: Record) => { - if (typeof part?.text === "string") return normalizeMemoryInputText(part.text); - if (part?.type === "input_text") return normalizeMemoryInputText(part.text); - return ""; - }) + .map((part: Record) => + extractMemoryTextPart(part, transcriptSensitive, context) + ) .filter(Boolean) .join("\n") .trim(); diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index c2de23f7ce..139b476629 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -745,6 +745,7 @@ async function handleComboChatInner({ endpointPath = null, requestHeaders = null, videoTranscriptSensitive = false, + videoTranscriptDescriptionFingerprints = [], invocationId, }: HandleComboChatOptions): Promise { const comboCtx = createComboContext({ body, combo, settings, relayOptions, log }); @@ -820,6 +821,7 @@ async function handleComboChatInner({ apiKeyAllowedConnections, hiddenModelsByProvider, videoTranscriptSensitive, + videoTranscriptDescriptionFingerprints, perTargetAdmission, deferContextOverflowWhenCompressible, compressionExclusions, @@ -877,6 +879,7 @@ async function handleComboChatInner({ apiKeyAllowedConnections, hiddenModelsByProvider, videoTranscriptSensitive, + videoTranscriptDescriptionFingerprints, perTargetAdmission, deferContextOverflowWhenCompressible, compressionExclusions, @@ -1891,6 +1894,7 @@ async function handleComboChatInner({ currModel: modelStr, universalConfig: universalHandoffConfig, videoTranscriptSensitive, + trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints, handleSingleModel: handleSingleModelWithTimeout, }); } @@ -1943,6 +1947,7 @@ async function handleComboChatInner({ expiresAt: resetCandidates[0] || null, config: relayConfig, videoTranscriptSensitive, + trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints, handleSingleModel: handleSingleModelWithTimeout, }); } @@ -3329,24 +3334,20 @@ async function handleRoundRobinCombo({ "COMBO-RR", `Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.` ); - return errorResponseWithComboDiagnostics( - 503, - "Maximum combo retry limit reached", - { - poolSize: modelCount, - attempted: globalAttempts, - excluded: [ - ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })), - ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))), - ], - attemptOrder: rrOutcomes.map((o) => ({ - provider: o.model.split("/")[0] || "unknown", - model: o.model, - })), - terminalReason: "max_attempts_exceeded", - recovery: buildRecoveryHint("max_attempts_exceeded"), - } - ); + return errorResponseWithComboDiagnostics(503, "Maximum combo retry limit reached", { + poolSize: modelCount, + attempted: globalAttempts, + excluded: [ + ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })), + ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))), + ], + attemptOrder: rrOutcomes.map((o) => ({ + provider: o.model.split("/")[0] || "unknown", + model: o.model, + })), + terminalReason: "max_attempts_exceeded", + recovery: buildRecoveryHint("max_attempts_exceeded"), + }); } if (retry > 0) { log.info( diff --git a/open-sse/services/combo/dispatchPrelude.ts b/open-sse/services/combo/dispatchPrelude.ts index eeeb3535ed..a55e7e98ba 100644 --- a/open-sse/services/combo/dispatchPrelude.ts +++ b/open-sse/services/combo/dispatchPrelude.ts @@ -86,6 +86,7 @@ type PreludeBaseOptionArgs = { apiKeyAllowedConnections?: string[] | null; hiddenModelsByProvider?: HiddenModelsByProvider; videoTranscriptSensitive?: boolean; + videoTranscriptDescriptionFingerprints?: readonly string[]; clientManagedResponsesContext?: boolean; /** #9654 Wave 2: per-target lane-aware admission probe (see HandleComboChatOptions). */ perTargetAdmission?: PerTargetAdmissionHook | null; @@ -114,6 +115,7 @@ function buildBaseOptions(a: PreludeBaseOptionArgs): HandleComboChatOptions { apiKeyAllowedConnections: a.apiKeyAllowedConnections, hiddenModelsByProvider: a.hiddenModelsByProvider, videoTranscriptSensitive: a.videoTranscriptSensitive, + videoTranscriptDescriptionFingerprints: a.videoTranscriptDescriptionFingerprints, invocationId: a.invocationId, clientManagedResponsesContext: a.clientManagedResponsesContext, perTargetAdmission: a.perTargetAdmission, @@ -418,6 +420,7 @@ export async function tryFusionDispatch(args: { apiKeyAllowedConnections?: string[] | null; hiddenModelsByProvider?: HiddenModelsByProvider; videoTranscriptSensitive?: boolean; + videoTranscriptDescriptionFingerprints?: readonly string[]; perTargetAdmission?: PerTargetAdmissionHook | null; deferContextOverflowWhenCompressible?: boolean; compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions; @@ -701,6 +704,7 @@ export async function tryRuntimeUnitDispatch(args: { apiKeyAllowedConnections?: string[] | null; hiddenModelsByProvider?: HiddenModelsByProvider; videoTranscriptSensitive?: boolean; + videoTranscriptDescriptionFingerprints?: readonly string[]; perTargetAdmission?: PerTargetAdmissionHook | null; deferContextOverflowWhenCompressible?: boolean; compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions; diff --git a/open-sse/services/combo/types.ts b/open-sse/services/combo/types.ts index f3154f29a9..956be4e06c 100644 --- a/open-sse/services/combo/types.ts +++ b/open-sse/services/combo/types.ts @@ -116,6 +116,8 @@ export type HandleComboChatOptions = { hiddenModelsByProvider?: HiddenModelsByProvider; /** Request-scoped retention bit derived before guardrails can replace a video carrier. */ videoTranscriptSensitive?: boolean; + /** Exact bounded identities emitted by a modified Video Bridge guardrail. */ + videoTranscriptDescriptionFingerprints?: readonly string[]; /** Native Responses clients (for example Codex CLI/Desktop) manage compaction themselves. */ clientManagedResponsesContext?: boolean; /** diff --git a/open-sse/services/contextHandoff.ts b/open-sse/services/contextHandoff.ts index 66aff36783..31a60b839a 100644 --- a/open-sse/services/contextHandoff.ts +++ b/open-sse/services/contextHandoff.ts @@ -5,7 +5,11 @@ import { type HandoffPayload, upsertHandoff, } from "../../src/lib/db/contextHandoffs.ts"; -import { containsVideoTranscriptForLog } from "../../src/lib/guardrails/videoTranscriptLogRedaction.ts"; +import { + containsVideoTranscriptForLog, + omitVideoTranscriptForLog, + type VideoTranscriptLogContext, +} from "../../src/lib/guardrails/videoTranscriptLogRedaction.ts"; import { estimateTokens } from "./contextManager.ts"; import { stripMarkdownCodeFence } from "../utils/aiSdkCompat.ts"; @@ -44,6 +48,23 @@ export type MessageLike = { content?: unknown; }; +type HandoffTranscriptContext = VideoTranscriptLogContext & { + videoTranscriptSensitive?: boolean; +}; + +function retainHandoffMessages( + messages: MessageLike[], + context: HandoffTranscriptContext +): MessageLike[] | null { + const source = Array.isArray(messages) ? messages : []; + const transcriptSensitive = + context.videoTranscriptSensitive === true || containsVideoTranscriptForLog(source, context); + if (!transcriptSensitive) return source; + + const retained = omitVideoTranscriptForLog(source, context); + return Array.isArray(retained) ? (retained as MessageLike[]) : null; +} + export interface ContextRelayConfig { handoffModel?: string; handoffThreshold?: number; @@ -448,21 +469,23 @@ export function maybeGenerateHandoff(options: { config?: ContextRelayConfig | null; /** Trusted request bit for carriers already replaced by the Video Bridge guardrail. */ videoTranscriptSensitive?: boolean; + /** Exact bounded identities emitted by a modified Video Bridge guardrail. */ + trustedDescriptionFingerprints?: readonly string[]; handleSingleModel: (body: Record, modelStr: string) => Promise; }): void { if (!options.sessionId || !options.connectionId) return; - if ( - options.videoTranscriptSensitive === true || - containsVideoTranscriptForLog(options.messages) - ) { - return; - } const relayConfig = resolveContextRelayConfig(options.config as Record); if (relayConfig.handoffProviders.length === 0) return; if (options.percentUsed < relayConfig.handoffThreshold) return; if (options.percentUsed >= HANDOFF_EXHAUSTION_THRESHOLD) return; + const retainedMessages = retainHandoffMessages(options.messages, { + videoTranscriptSensitive: options.videoTranscriptSensitive, + trustedDescriptionFingerprints: options.trustedDescriptionFingerprints, + }); + if (!retainedMessages) return; + cleanupExpiredHandoffs(); if (hasActiveHandoff(options.sessionId, options.comboName)) return; const inflightKey = getInflightKey(options.sessionId, options.comboName); @@ -472,6 +495,7 @@ export function maybeGenerateHandoff(options: { setImmediate(() => { generateHandoffAsync({ ...options, + messages: retainedMessages, sessionId: options.sessionId as string, connectionId: options.connectionId as string, config: relayConfig, @@ -701,14 +725,10 @@ export function maybeGenerateUniversalHandoff(options: { universalConfig: UniversalHandoffConfig; /** Trusted request bit for carriers already replaced by the Video Bridge guardrail. */ videoTranscriptSensitive?: boolean; + /** Exact bounded identities emitted by a modified Video Bridge guardrail. */ + trustedDescriptionFingerprints?: readonly string[]; handleSingleModel: (body: Record, modelStr: string) => Promise; }): void { - if ( - options.videoTranscriptSensitive === true || - containsVideoTranscriptForLog(options.messages) - ) { - return; - } const decision = shouldGenerateUniversalHandoff({ sessionId: options.sessionId, comboName: options.comboName, @@ -720,6 +740,12 @@ export function maybeGenerateUniversalHandoff(options: { if (decision !== "generate") return; if (!options.sessionId) return; + const retainedMessages = retainHandoffMessages(options.messages, { + videoTranscriptSensitive: options.videoTranscriptSensitive, + trustedDescriptionFingerprints: options.trustedDescriptionFingerprints, + }); + if (!retainedMessages) return; + const inflightKey = getInflightKey(options.sessionId, options.comboName); if (inflightHandoffGenerations.has(inflightKey)) return; inflightHandoffGenerations.add(inflightKey); @@ -730,7 +756,7 @@ export function maybeGenerateUniversalHandoff(options: { generateUniversalHandoffAsync({ sessionId: options.sessionId as string, comboName: options.comboName, - messages: options.messages, + messages: retainedMessages, prevModel: options.prevModel || "unknown", currModel: options.currModel, handoffModel: options.universalConfig.handoffModel || options.currModel, diff --git a/src/lib/db/responsesContinuationStore.ts b/src/lib/db/responsesContinuationStore.ts index 5153dc4ab4..ef7a6e3de3 100644 --- a/src/lib/db/responsesContinuationStore.ts +++ b/src/lib/db/responsesContinuationStore.ts @@ -19,7 +19,6 @@ */ import { getDbInstance } from "./core"; -import { VIDEO_TRANSCRIPT_REDACTION_PIPELINE_KEY } from "../guardrails/videoTranscriptLogRedaction"; import { readCallArtifact } from "../usage/callLogArtifacts"; export type ResponsesContinuationState = { @@ -81,12 +80,6 @@ export function resolvePreviousResponseState( const { artifact, state } = readCallArtifact(row.artifact_relpath); if (state !== "ready" || !artifact?.pipeline) return null; - // Video Bridge transcript descriptions are deliberately omitted from the - // persisted pipeline. Replaying that incomplete history would silently - // change the conversation, so require the client to resend full history. - // This marker is written by the server at the pipeline level; caller prose - // is never treated as authority for this privacy decision. - if (artifact.pipeline[VIDEO_TRANSCRIPT_REDACTION_PIPELINE_KEY] === true) return null; const clientRawRequest = artifact.pipeline.clientRawRequest as { body?: unknown } | undefined; const clientResponse = artifact.pipeline.clientResponse as diff --git a/src/lib/guardrails/videoTranscriptLogRedaction.ts b/src/lib/guardrails/videoTranscriptLogRedaction.ts index 1012546539..3ba3fdf05d 100644 --- a/src/lib/guardrails/videoTranscriptLogRedaction.ts +++ b/src/lib/guardrails/videoTranscriptLogRedaction.ts @@ -291,6 +291,34 @@ export function omitVideoTranscriptFromLogString( return result + value.slice(cursor); } +/** + * Remove complete trusted Video Bridge descriptions before Memory extraction. + * Logs may retain the non-transcript portion of a description, but durable + * user facts must come only from caller text, never from media-derived prose. + * Exact server-issued fingerprints let adjacent caller text survive even when + * both appear in the same string. Unknown/over-budget input fails closed. + */ +export function omitVideoTranscriptDerivedTextForMemory( + value: string, + context: VideoTranscriptLogContext = {} +): string { + try { + const ranges = findTrustedDescriptionRanges(value, context); + if (ranges === null) return ""; + if (ranges.length === 0) return value; + + let result = ""; + let cursor = 0; + for (const range of ranges) { + result += value.slice(cursor, range.start); + cursor = range.end; + } + return result + value.slice(cursor); + } catch { + return ""; + } +} + function fieldIsTranscript(key: string, carrier: boolean): boolean { return carrier && VIDEO_TRANSCRIPT_PAYLOAD_KEYS.has(key); } diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 1dcdcf349f..c6c29bc2df 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -1067,6 +1067,7 @@ async function handleChatImplementation( endpointPath: new URL(request.url).pathname, requestHeaders: request.headers, videoTranscriptSensitive, + videoTranscriptDescriptionFingerprints, clientManagedResponsesContext: sourceFormat === "openai-responses" && new URL(request.url).pathname.split("/").includes("responses") && @@ -1359,6 +1360,9 @@ async function handleSingleModelChat( comboStrategy: string | null = null, isCombo: boolean = false ) { + const videoTranscriptDescriptionFingerprints = + runtimeOptions.videoTranscriptDescriptionFingerprints ?? []; + // 1. Resolve model → provider/model const resolved = await resolveModelOrError( modelStr, @@ -1395,6 +1399,7 @@ async function handleSingleModelChat( endpointPath: clientRawRequest?.endpoint || "", requestHeaders: clientRawRequest?.headers, videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true, + videoTranscriptDescriptionFingerprints, clientManagedResponsesContext: sNetSourceFormat === "openai-responses" && String(clientRawRequest?.endpoint || "") @@ -1427,8 +1432,7 @@ async function handleSingleModelChat( conversationId: runtimeOptions?.conversationId ?? null, managedLease: runtimeOptions.managedLease ?? null, videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true, - videoTranscriptDescriptionFingerprints: - runtimeOptions.videoTranscriptDescriptionFingerprints ?? [], + videoTranscriptDescriptionFingerprints, // #7360 follow-up — see the primary handleSingleModel closure above. modelAbortSignal: target?.modelAbortSignal ?? null, }, @@ -1903,8 +1907,7 @@ async function handleSingleModelChat( reasoningTransportFallback: runtimeOptions.reasoningTransportFallback ?? "drop", managedLease: runtimeOptions.managedLease ?? null, videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true, - videoTranscriptDescriptionFingerprints: - runtimeOptions.videoTranscriptDescriptionFingerprints ?? [], + videoTranscriptDescriptionFingerprints, }, runtimeOptions );