fix(video): preserve safe continuation and handoffs

2026-09-14 02:42:24 +03:00 · 2026-08-27 03:29:50 -03:00
parent d62ecb5d98
commit 4fee959ad2
9 changed files with 156 additions and 70 deletions
--- a/open-sse/handlers/chatCore.ts
+++ b/open-sse/handlers/chatCore.ts
@@ -5123,7 +5123,8 @@ export async function handleChatCore({
    if (memoryOwnerId && memorySettings?.enabled && memorySettings.maxTokens > 0) {
      const requestMemoryText = extractMemoryTextFromRequestBody(
        body as Record<string, unknown>,
-        videoTranscriptSensitive
+        videoTranscriptSensitive,
+        { trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints }
      );
      if (requestMemoryText) {
        extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId);
@@ -5765,7 +5766,8 @@ export async function handleChatCore({
    ) {
      const requestMemoryText = extractMemoryTextFromRequestBody(
        body as Record<string, unknown>,
-        videoTranscriptSensitive
+        videoTranscriptSensitive,
+        { trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints }
      );
      if (requestMemoryText) {
        extractFacts(requestMemoryText, memoryOwnerId, pipelineSessionId);
--- a/open-sse/handlers/chatCore/memoryExtraction.ts
+++ b/open-sse/handlers/chatCore/memoryExtraction.ts
@@ -1,8 +1,37 @@
+import {
+  containsVideoTranscriptForLog,
+  omitVideoTranscriptDerivedTextForMemory,
+  type VideoTranscriptLogContext,
+} from "../../../src/lib/guardrails/videoTranscriptLogRedaction.ts";
 import { capMemoryExtractionText, MEMORY_EXTRACTION_TEXT_LIMIT } from "./logTruncation.ts";

-function normalizeMemoryInputText(value: unknown): string {
+function normalizeMemoryInputText(value: unknown, context: VideoTranscriptLogContext = {}): string {
  if (typeof value !== "string") return "";
-  return value.trim();
+  return omitVideoTranscriptDerivedTextForMemory(value, context).trim();
+}
+
+function extractMemoryTextPart(
+  part: Record<string, unknown>,
+  transcriptSensitive: boolean,
+  context: VideoTranscriptLogContext
+): string {
+  try {
+    const rawText = typeof part?.text === "string" ? part.text : "";
+    if (!rawText) return "";
+
+    const retainedText = normalizeMemoryInputText(rawText, context);
+    if (!retainedText) return "";
+    if (
+      transcriptSensitive &&
+      containsVideoTranscriptForLog(part, context) &&
+      retainedText === rawText.trim()
+    ) {
+      return "";
+    }
+    return retainedText;
+  } catch {
+    return "";
+  }
 }

 export function extractMemoryTextFromResponse(
@@ -35,13 +64,15 @@ export function extractMemoryTextFromResponse(

 export function extractMemoryTextFromRequestBody(
  body: Record<string, unknown> | null | undefined,
-  videoTranscriptSensitive = false
+  videoTranscriptSensitive = false,
+  context: VideoTranscriptLogContext = {}
 ): string {
-  // This bit is derived from the guardrail result. Caller-shaped lookalike text
-  // cannot suppress Memory extraction, while real media-derived cues cannot
-  // become durable facts (including through an adjacent response echo).
-  if (videoTranscriptSensitive) return "";
  if (!body || typeof body !== "object") return "";
+  // Re-check the structured body at the sink boundary. The explicit bit covers
+  // processed requests whose raw carrier was already replaced; trusted hashes
+  // identify only descriptions emitted by a modified Video Bridge guardrail.
+  const transcriptSensitive =
+    videoTranscriptSensitive || containsVideoTranscriptForLog(body, context);

  const messages = Array.isArray(body.messages) ? body.messages : null;
  if (messages && messages.length > 0) {
@@ -49,18 +80,16 @@ export function extractMemoryTextFromRequestBody(
      const msg = messages[i] as Record<string, unknown>;
      if (msg?.role !== "user") continue;

-      const messageText = normalizeMemoryInputText(msg.content);
+      const messageText = normalizeMemoryInputText(msg.content, context);
      if (messageText) {
        return capMemoryExtractionText(messageText);
      }

      if (Array.isArray(msg.content)) {
        const text = msg.content
-          .map((part: Record<string, unknown>) => {
-            if (typeof part?.text === "string") return normalizeMemoryInputText(part.text);
-            if (part?.type === "input_text") return normalizeMemoryInputText(part.text);
-            return "";
-          })
+          .map((part: Record<string, unknown>) =>
+            extractMemoryTextPart(part, transcriptSensitive, context)
+          )
          .filter(Boolean)
          .join("\n")
          .trim();
@@ -78,17 +107,15 @@ export function extractMemoryTextFromRequestBody(
      if (role && role !== "user") continue;
      if (itemType && itemType !== "message") continue;

-      const itemText = normalizeMemoryInputText(item?.content);
+      const itemText = normalizeMemoryInputText(item?.content, context);
      if (itemText) {
        return capMemoryExtractionText(itemText);
      }
      if (Array.isArray(item?.content)) {
        const text = item.content
-          .map((part: Record<string, unknown>) => {
-            if (typeof part?.text === "string") return normalizeMemoryInputText(part.text);
-            if (part?.type === "input_text") return normalizeMemoryInputText(part.text);
-            return "";
-          })
+          .map((part: Record<string, unknown>) =>
+            extractMemoryTextPart(part, transcriptSensitive, context)
+          )
          .filter(Boolean)
          .join("\n")
          .trim();
@@ -106,14 +133,14 @@ export function extractMemoryTextFromRequestBody(
        if (role && role !== "user") return "";
        if (itemType && itemType !== "message") return "";

-        if (typeof item?.content === "string") return normalizeMemoryInputText(item.content);
+        if (typeof item?.content === "string") {
+          return normalizeMemoryInputText(item.content, context);
+        }
        if (Array.isArray(item?.content)) {
          return item.content
-            .map((part: Record<string, unknown>) => {
-              if (typeof part?.text === "string") return normalizeMemoryInputText(part.text);
-              if (part?.type === "input_text") return normalizeMemoryInputText(part.text);
-              return "";
-            })
+            .map((part: Record<string, unknown>) =>
+              extractMemoryTextPart(part, transcriptSensitive, context)
+            )
            .filter(Boolean)
            .join("\n")
            .trim();
--- a/open-sse/services/combo.ts
+++ b/open-sse/services/combo.ts
@@ -745,6 +745,7 @@ async function handleComboChatInner({
  endpointPath = null,
  requestHeaders = null,
  videoTranscriptSensitive = false,
+  videoTranscriptDescriptionFingerprints = [],
  invocationId,
 }: HandleComboChatOptions): Promise<Response> {
  const comboCtx = createComboContext({ body, combo, settings, relayOptions, log });
@@ -820,6 +821,7 @@ async function handleComboChatInner({
    apiKeyAllowedConnections,
    hiddenModelsByProvider,
    videoTranscriptSensitive,
+    videoTranscriptDescriptionFingerprints,
    perTargetAdmission,
    deferContextOverflowWhenCompressible,
    compressionExclusions,
@@ -877,6 +879,7 @@ async function handleComboChatInner({
    apiKeyAllowedConnections,
    hiddenModelsByProvider,
    videoTranscriptSensitive,
+    videoTranscriptDescriptionFingerprints,
    perTargetAdmission,
    deferContextOverflowWhenCompressible,
    compressionExclusions,
@@ -1891,6 +1894,7 @@ async function handleComboChatInner({
                  currModel: modelStr,
                  universalConfig: universalHandoffConfig,
                  videoTranscriptSensitive,
+                  trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints,
                  handleSingleModel: handleSingleModelWithTimeout,
                });
              }
@@ -1943,6 +1947,7 @@ async function handleComboChatInner({
                    expiresAt: resetCandidates[0] || null,
                    config: relayConfig,
                    videoTranscriptSensitive,
+                    trustedDescriptionFingerprints: videoTranscriptDescriptionFingerprints,
                    handleSingleModel: handleSingleModelWithTimeout,
                  });
                }
@@ -3329,24 +3334,20 @@ async function handleRoundRobinCombo({
              "COMBO-RR",
              `Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.`
            );
-            return errorResponseWithComboDiagnostics(
-              503,
-              "Maximum combo retry limit reached",
-              {
-                poolSize: modelCount,
-                attempted: globalAttempts,
-                excluded: [
-                  ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
-                  ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
-                ],
-                attemptOrder: rrOutcomes.map((o) => ({
-                  provider: o.model.split("/")[0] || "unknown",
-                  model: o.model,
-                })),
-                terminalReason: "max_attempts_exceeded",
-                recovery: buildRecoveryHint("max_attempts_exceeded"),
-              }
-            );
+            return errorResponseWithComboDiagnostics(503, "Maximum combo retry limit reached", {
+              poolSize: modelCount,
+              attempted: globalAttempts,
+              excluded: [
+                ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
+                ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
+              ],
+              attemptOrder: rrOutcomes.map((o) => ({
+                provider: o.model.split("/")[0] || "unknown",
+                model: o.model,
+              })),
+              terminalReason: "max_attempts_exceeded",
+              recovery: buildRecoveryHint("max_attempts_exceeded"),
+            });
          }
          if (retry > 0) {
            log.info(
--- a/open-sse/services/combo/dispatchPrelude.ts
+++ b/open-sse/services/combo/dispatchPrelude.ts
@@ -86,6 +86,7 @@ type PreludeBaseOptionArgs = {
  apiKeyAllowedConnections?: string[] | null;
  hiddenModelsByProvider?: HiddenModelsByProvider;
  videoTranscriptSensitive?: boolean;
+  videoTranscriptDescriptionFingerprints?: readonly string[];
  clientManagedResponsesContext?: boolean;
  /** #9654 Wave 2: per-target lane-aware admission probe (see HandleComboChatOptions). */
  perTargetAdmission?: PerTargetAdmissionHook | null;
@@ -114,6 +115,7 @@ function buildBaseOptions(a: PreludeBaseOptionArgs): HandleComboChatOptions {
    apiKeyAllowedConnections: a.apiKeyAllowedConnections,
    hiddenModelsByProvider: a.hiddenModelsByProvider,
    videoTranscriptSensitive: a.videoTranscriptSensitive,
+    videoTranscriptDescriptionFingerprints: a.videoTranscriptDescriptionFingerprints,
    invocationId: a.invocationId,
    clientManagedResponsesContext: a.clientManagedResponsesContext,
    perTargetAdmission: a.perTargetAdmission,
@@ -418,6 +420,7 @@ export async function tryFusionDispatch(args: {
  apiKeyAllowedConnections?: string[] | null;
  hiddenModelsByProvider?: HiddenModelsByProvider;
  videoTranscriptSensitive?: boolean;
+  videoTranscriptDescriptionFingerprints?: readonly string[];
  perTargetAdmission?: PerTargetAdmissionHook | null;
  deferContextOverflowWhenCompressible?: boolean;
  compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
@@ -701,6 +704,7 @@ export async function tryRuntimeUnitDispatch(args: {
  apiKeyAllowedConnections?: string[] | null;
  hiddenModelsByProvider?: HiddenModelsByProvider;
  videoTranscriptSensitive?: boolean;
+  videoTranscriptDescriptionFingerprints?: readonly string[];
  perTargetAdmission?: PerTargetAdmissionHook | null;
  deferContextOverflowWhenCompressible?: boolean;
  compressionExclusions?: import("../compression/exclusions.ts").CompressionExclusions;
--- a/open-sse/services/combo/types.ts
+++ b/open-sse/services/combo/types.ts
@@ -116,6 +116,8 @@ export type HandleComboChatOptions = {
  hiddenModelsByProvider?: HiddenModelsByProvider;
  /** Request-scoped retention bit derived before guardrails can replace a video carrier. */
  videoTranscriptSensitive?: boolean;
+  /** Exact bounded identities emitted by a modified Video Bridge guardrail. */
+  videoTranscriptDescriptionFingerprints?: readonly string[];
  /** Native Responses clients (for example Codex CLI/Desktop) manage compaction themselves. */
  clientManagedResponsesContext?: boolean;
  /**
--- a/open-sse/services/contextHandoff.ts
+++ b/open-sse/services/contextHandoff.ts
@@ -5,7 +5,11 @@ import {
  type HandoffPayload,
  upsertHandoff,
 } from "../../src/lib/db/contextHandoffs.ts";
-import { containsVideoTranscriptForLog } from "../../src/lib/guardrails/videoTranscriptLogRedaction.ts";
+import {
+  containsVideoTranscriptForLog,
+  omitVideoTranscriptForLog,
+  type VideoTranscriptLogContext,
+} from "../../src/lib/guardrails/videoTranscriptLogRedaction.ts";
 import { estimateTokens } from "./contextManager.ts";
 import { stripMarkdownCodeFence } from "../utils/aiSdkCompat.ts";

@@ -44,6 +48,23 @@ export type MessageLike = {
  content?: unknown;
 };

+type HandoffTranscriptContext = VideoTranscriptLogContext & {
+  videoTranscriptSensitive?: boolean;
+};
+
+function retainHandoffMessages(
+  messages: MessageLike[],
+  context: HandoffTranscriptContext
+): MessageLike[] | null {
+  const source = Array.isArray(messages) ? messages : [];
+  const transcriptSensitive =
+    context.videoTranscriptSensitive === true || containsVideoTranscriptForLog(source, context);
+  if (!transcriptSensitive) return source;
+
+  const retained = omitVideoTranscriptForLog(source, context);
+  return Array.isArray(retained) ? (retained as MessageLike[]) : null;
+}
+
 export interface ContextRelayConfig {
  handoffModel?: string;
  handoffThreshold?: number;
@@ -448,21 +469,23 @@ export function maybeGenerateHandoff(options: {
  config?: ContextRelayConfig | null;
  /** Trusted request bit for carriers already replaced by the Video Bridge guardrail. */
  videoTranscriptSensitive?: boolean;
+  /** Exact bounded identities emitted by a modified Video Bridge guardrail. */
+  trustedDescriptionFingerprints?: readonly string[];
  handleSingleModel: (body: Record<string, unknown>, modelStr: string) => Promise<Response>;
 }): void {
  if (!options.sessionId || !options.connectionId) return;
-  if (
-    options.videoTranscriptSensitive === true ||
-    containsVideoTranscriptForLog(options.messages)
-  ) {
-    return;
-  }

  const relayConfig = resolveContextRelayConfig(options.config as Record<string, unknown>);
  if (relayConfig.handoffProviders.length === 0) return;
  if (options.percentUsed < relayConfig.handoffThreshold) return;
  if (options.percentUsed >= HANDOFF_EXHAUSTION_THRESHOLD) return;

+  const retainedMessages = retainHandoffMessages(options.messages, {
+    videoTranscriptSensitive: options.videoTranscriptSensitive,
+    trustedDescriptionFingerprints: options.trustedDescriptionFingerprints,
+  });
+  if (!retainedMessages) return;
+
  cleanupExpiredHandoffs();
  if (hasActiveHandoff(options.sessionId, options.comboName)) return;
  const inflightKey = getInflightKey(options.sessionId, options.comboName);
@@ -472,6 +495,7 @@ export function maybeGenerateHandoff(options: {
  setImmediate(() => {
    generateHandoffAsync({
      ...options,
+      messages: retainedMessages,
      sessionId: options.sessionId as string,
      connectionId: options.connectionId as string,
      config: relayConfig,
@@ -701,14 +725,10 @@ export function maybeGenerateUniversalHandoff(options: {
  universalConfig: UniversalHandoffConfig;
  /** Trusted request bit for carriers already replaced by the Video Bridge guardrail. */
  videoTranscriptSensitive?: boolean;
+  /** Exact bounded identities emitted by a modified Video Bridge guardrail. */
+  trustedDescriptionFingerprints?: readonly string[];
  handleSingleModel: (body: Record<string, unknown>, modelStr: string) => Promise<Response>;
 }): void {
-  if (
-    options.videoTranscriptSensitive === true ||
-    containsVideoTranscriptForLog(options.messages)
-  ) {
-    return;
-  }
  const decision = shouldGenerateUniversalHandoff({
    sessionId: options.sessionId,
    comboName: options.comboName,
@@ -720,6 +740,12 @@ export function maybeGenerateUniversalHandoff(options: {
  if (decision !== "generate") return;
  if (!options.sessionId) return;

+  const retainedMessages = retainHandoffMessages(options.messages, {
+    videoTranscriptSensitive: options.videoTranscriptSensitive,
+    trustedDescriptionFingerprints: options.trustedDescriptionFingerprints,
+  });
+  if (!retainedMessages) return;
+
  const inflightKey = getInflightKey(options.sessionId, options.comboName);
  if (inflightHandoffGenerations.has(inflightKey)) return;
  inflightHandoffGenerations.add(inflightKey);
@@ -730,7 +756,7 @@ export function maybeGenerateUniversalHandoff(options: {
    generateUniversalHandoffAsync({
      sessionId: options.sessionId as string,
      comboName: options.comboName,
-      messages: options.messages,
+      messages: retainedMessages,
      prevModel: options.prevModel || "unknown",
      currModel: options.currModel,
      handoffModel: options.universalConfig.handoffModel || options.currModel,
--- a/src/lib/db/responsesContinuationStore.ts
+++ b/src/lib/db/responsesContinuationStore.ts
@@ -19,7 +19,6 @@
 */

 import { getDbInstance } from "./core";
-import { VIDEO_TRANSCRIPT_REDACTION_PIPELINE_KEY } from "../guardrails/videoTranscriptLogRedaction";
 import { readCallArtifact } from "../usage/callLogArtifacts";

 export type ResponsesContinuationState = {
@@ -81,12 +80,6 @@ export function resolvePreviousResponseState(

  const { artifact, state } = readCallArtifact(row.artifact_relpath);
  if (state !== "ready" || !artifact?.pipeline) return null;
-  // Video Bridge transcript descriptions are deliberately omitted from the
-  // persisted pipeline. Replaying that incomplete history would silently
-  // change the conversation, so require the client to resend full history.
-  // This marker is written by the server at the pipeline level; caller prose
-  // is never treated as authority for this privacy decision.
-  if (artifact.pipeline[VIDEO_TRANSCRIPT_REDACTION_PIPELINE_KEY] === true) return null;

  const clientRawRequest = artifact.pipeline.clientRawRequest as { body?: unknown } | undefined;
  const clientResponse = artifact.pipeline.clientResponse as
--- a/src/lib/guardrails/videoTranscriptLogRedaction.ts
+++ b/src/lib/guardrails/videoTranscriptLogRedaction.ts
@@ -291,6 +291,34 @@ export function omitVideoTranscriptFromLogString(
  return result + value.slice(cursor);
 }

+/**
+ * Remove complete trusted Video Bridge descriptions before Memory extraction.
+ * Logs may retain the non-transcript portion of a description, but durable
+ * user facts must come only from caller text, never from media-derived prose.
+ * Exact server-issued fingerprints let adjacent caller text survive even when
+ * both appear in the same string. Unknown/over-budget input fails closed.
+ */
+export function omitVideoTranscriptDerivedTextForMemory(
+  value: string,
+  context: VideoTranscriptLogContext = {}
+): string {
+  try {
+    const ranges = findTrustedDescriptionRanges(value, context);
+    if (ranges === null) return "";
+    if (ranges.length === 0) return value;
+
+    let result = "";
+    let cursor = 0;
+    for (const range of ranges) {
+      result += value.slice(cursor, range.start);
+      cursor = range.end;
+    }
+    return result + value.slice(cursor);
+  } catch {
+    return "";
+  }
+}
+
 function fieldIsTranscript(key: string, carrier: boolean): boolean {
  return carrier && VIDEO_TRANSCRIPT_PAYLOAD_KEYS.has(key);
 }
--- a/src/sse/handlers/chat.ts
+++ b/src/sse/handlers/chat.ts
@@ -1067,6 +1067,7 @@ async function handleChatImplementation(
      endpointPath: new URL(request.url).pathname,
      requestHeaders: request.headers,
      videoTranscriptSensitive,
+      videoTranscriptDescriptionFingerprints,
      clientManagedResponsesContext:
        sourceFormat === "openai-responses" &&
        new URL(request.url).pathname.split("/").includes("responses") &&
@@ -1359,6 +1360,9 @@ async function handleSingleModelChat(
  comboStrategy: string | null = null,
  isCombo: boolean = false
 ) {
+  const videoTranscriptDescriptionFingerprints =
+    runtimeOptions.videoTranscriptDescriptionFingerprints ?? [];
+
  // 1. Resolve model → provider/model
  const resolved = await resolveModelOrError(
    modelStr,
@@ -1395,6 +1399,7 @@ async function handleSingleModelChat(
      endpointPath: clientRawRequest?.endpoint || "",
      requestHeaders: clientRawRequest?.headers,
      videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true,
+      videoTranscriptDescriptionFingerprints,
      clientManagedResponsesContext:
        sNetSourceFormat === "openai-responses" &&
        String(clientRawRequest?.endpoint || "")
@@ -1427,8 +1432,7 @@ async function handleSingleModelChat(
            conversationId: runtimeOptions?.conversationId ?? null,
            managedLease: runtimeOptions.managedLease ?? null,
            videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true,
-            videoTranscriptDescriptionFingerprints:
-              runtimeOptions.videoTranscriptDescriptionFingerprints ?? [],
+            videoTranscriptDescriptionFingerprints,
            // #7360 follow-up — see the primary handleSingleModel closure above.
            modelAbortSignal: target?.modelAbortSignal ?? null,
          },
@@ -1903,8 +1907,7 @@ async function handleSingleModelChat(
            reasoningTransportFallback: runtimeOptions.reasoningTransportFallback ?? "drop",
            managedLease: runtimeOptions.managedLease ?? null,
            videoTranscriptSensitive: runtimeOptions.videoTranscriptSensitive === true,
-            videoTranscriptDescriptionFingerprints:
-              runtimeOptions.videoTranscriptDescriptionFingerprints ?? [],
+            videoTranscriptDescriptionFingerprints,
          },
          runtimeOptions
        );