fix(compression): preserve upstream prompt cache for memory + compression (#3890) (#3936)

src/lib/memory/injection.ts

@@ -79,10 +79,24 @@ export function formatMemoryContext(memories: Memory[]): string {
  * @param provider - Provider identifier used to choose injection strategy
  * @returns A new request body with memories prepended to messages
  */
+export interface InjectMemoryOptions {
+  /**
+   * #3890: when the request uses prompt caching (cache_control breakpoints), prepending
+   * the memory message at index 0 shifts the entire cacheable prefix — and since the
+   * retrieved memories vary per user query, every cache breakpoint then misses on each
+   * turn (observed as a sustained cache-miss / cost spike). When `cacheSafe` is set, the
+   * memory message is inserted immediately before the LAST user message instead, so the
+   * cacheable prefix (system prompt + prior turns) stays byte-stable while memory still
+   * contextualizes the current turn.
+   */
+  cacheSafe?: boolean;
+}
+
 export function injectMemory(
   request: ChatRequest,
   memories: Memory[],
-  provider: string | null | undefined
+  provider: string | null | undefined,
+  options: InjectMemoryOptions = {}
 ): ChatRequest {
   if (!memories || memories.length === 0) {
     log.info("memory.injection.skipped", { reason: "no_memories", model: request.model });
@@ -97,26 +111,41 @@ export function injectMemory(
 
   const messages: ChatMessage[] = Array.isArray(request.messages) ? [...request.messages] : [];
 
+  // #3890: in a caching context, anchor the injection just before the LAST user message so
+  // the cacheable prefix (system prompt + prior turns) is preserved byte-for-byte. Falls
+  // back to a leading message when caching is off or there is no user turn to anchor on.
+  const cacheSafeIndex = options.cacheSafe ? messages.findLastIndex((m) => m.role === "user") : -1;
+
   if (providerSupportsSystemMessage(provider)) {
-    // Strategy 1: inject as a leading system message.
+    // Strategy 1: inject as a system message.
     // Prepending before any existing system messages keeps memory context
     // accessible without overriding the caller's own system instructions.
     const memorySystemMessage: ChatMessage = { role: "system", content: memoryText };
     log.info("memory.injection.injected", {
       count: memories.length,
-      strategy: "system",
+      strategy: cacheSafeIndex >= 0 ? "system-cache-safe" : "system",
       model: request.model,
     });
+    if (cacheSafeIndex >= 0) {
+      const next = [...messages];
+      next.splice(cacheSafeIndex, 0, memorySystemMessage);
+      return { ...request, messages: next };
+    }
     return { ...request, messages: [memorySystemMessage, ...messages] };
   } else {
-    // Strategy 2 (fallback): inject as the first user message.
+    // Strategy 2 (fallback): inject as a user message.
     // Used for providers like o1-mini that reject the system role.
     const memoryUserMessage: ChatMessage = { role: "user", content: memoryText };
     log.info("memory.injection.injected", {
       count: memories.length,
-      strategy: "user",
+      strategy: cacheSafeIndex >= 0 ? "user-cache-safe" : "user",
       model: request.model,
     });
+    if (cacheSafeIndex >= 0) {
+      const next = [...messages];
+      next.splice(cacheSafeIndex, 0, memoryUserMessage);
+      return { ...request, messages: next };
+    }
     return { ...request, messages: [memoryUserMessage, ...messages] };
   }
 }