Merge remote-tracking branch 'origin/release/v3.8.51' into fix/release-v3.8.51-basereds-orphans

src/app/api/telegram/update/route.ts

@@ -13,12 +13,18 @@
  *   3. Handles /start (returns the Mini App deep link) and everything else
  *      as a chat prompt proxied through the OmniRoute pipeline.
  */
+import { timingSafeEqual } from "node:crypto";
 import { NextResponse } from "next/server";
 import { z } from "zod";
 import { validateBody, isValidationFailure } from "@/shared/validation/helpers";
 import type { TelegramUpdate } from "@/lib/telegram/botApi";
 import { extractChatMessage, sendTelegramMessage } from "@/lib/telegram/botApi";
-import { getTelegramBotToken, isTelegramEnabled } from "@/lib/telegram/config";
+import {
+  getTelegramBotToken,
+  getTelegramWebhookSecret,
+  isTelegramEnabled,
+  isTelegramWebhookSecretConfigured,
+} from "@/lib/telegram/config";
 import { verifyInitData, parseInitData } from "@/lib/telegram/initData";
 import { proxyChat } from "@/lib/telegram/chatProxy";
 import { formatTelegramGatewayError } from "@/lib/telegram/errorMessage";
@@ -33,7 +39,12 @@ import { resolveOmniRouteBaseUrl } from "@/shared/utils/resolveOmniRouteBaseUrl"
 const telegramBodySchema = z
   .object({
     initData: z.string().optional(),
-    message: z.string().optional(),
+    // `message` is a STRING on the Mini App path ({ initData, message }) and an
+    // OBJECT on the webhook path (a Telegram update). Constraining it to a
+    // string rejected every real webhook delivery with 400 before any auth or
+    // routing ran, so accept either shape here and let each branch validate the
+    // shape it actually needs.
+    message: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(),
     update_id: z.number().optional(),
     // allow unknown update fields
   })
@@ -103,6 +114,21 @@ export async function POST(request: Request) {
   }
 
   // ── Bot webhook path: TelegramUpdate ─────────────────────────────────────
+  // Unlike the Mini App branch above (which verifies the initData HMAC), a
+  // webhook body carries no proof of origin: `chat.id` is attacker-chosen and
+  // reaches proxyChat(), which mints a real API key and spends upstream quota.
+  // Telegram's `secret_token` echo is the only authentication available here.
+  if (!isTelegramWebhookSecretConfigured()) {
+    return NextResponse.json(
+      { ok: false, error: "Telegram webhook secret not configured" },
+      { status: 503 }
+    );
+  }
+  const presentedSecret = request.headers.get("x-telegram-bot-api-secret-token") || "";
+  if (!webhookSecretMatches(presentedSecret, getTelegramWebhookSecret())) {
+    return NextResponse.json({ ok: false, error: "Unauthorized" }, { status: 401 });
+  }
+
   const update = body as unknown as TelegramUpdate;
   const chat = extractChatMessage(update);
   if (!chat) {
@@ -117,6 +143,22 @@ export async function POST(request: Request) {
   return NextResponse.json({ ok: true });
 }
 
+/**
+ * Constant-time comparison of the presented webhook secret against the
+ * configured one. A plain `===` short-circuits on the first differing byte and
+ * leaks the shared-prefix length through response timing; `timingSafeEqual`
+ * does not. It requires equal-length buffers, so a length mismatch is rejected
+ * up front (the length itself is not secret).
+ *
+ * Exported as a test seam only — not part of the route contract.
+ */
+export function webhookSecretMatches(presented: string, expected: string): boolean {
+  const a = Buffer.from(presented);
+  const b = Buffer.from(expected);
+  if (a.length !== b.length) return false;
+  return timingSafeEqual(a, b);
+}
+
 async function handleAndReply(chatId: number, text: string, messageId?: number): Promise<void> {
   try {
     const trimmed = text.trim();

src/app/api/tools/traffic-inspector/ws/route.ts

@@ -96,6 +96,14 @@ export async function GET(request: Request): Promise<Response> {
   }
 
   const acceptHeader = acceptKey(clientKey);
+
+  // The client can vanish during the upgrade round trip. `close` has then
+  // ALREADY fired, so the listeners below would never run and every resource
+  // acquired past this point would be held with no path to release it.
+  if (socket.destroyed) {
+    return new Response(null, { status: 101 });
+  }
+
   socket.write(
     [
       "HTTP/1.1 101 Switching Protocols",
@@ -106,21 +114,17 @@ export async function GET(request: Request): Promise<Response> {
     ].join("\r\n")
   );
 
-  const unsubscribe = globalTrafficBuffer.subscribe((ev) => {
-    sendText(socket, ev);
-  });
-
-  const pingTimer = setInterval(() => {
-    try {
-      socket.write(encodeWsFrame(0x09)); // ping
-    } catch {
-      cleanup();
-    }
-  }, PING_INTERVAL_MS);
+  let unsubscribe: (() => void) | null = null;
+  let pingTimer: ReturnType<typeof setInterval> | null = null;
+  let cleanedUp = false;
 
   function cleanup(): void {
-    clearInterval(pingTimer);
-    unsubscribe();
+    if (cleanedUp) return;
+    cleanedUp = true;
+    if (pingTimer) clearInterval(pingTimer);
+    pingTimer = null;
+    unsubscribe?.();
+    unsubscribe = null;
     try {
       socket.destroy();
     } catch {
@@ -128,14 +132,43 @@ export async function GET(request: Request): Promise<Response> {
     }
   }
 
-  socket.once("close", cleanup);
-  socket.once("error", cleanup);
-
-  // Never resolve — the socket is the response channel.
-  await new Promise<void>((resolve) => {
+  // Attached BEFORE any resource is acquired, so there is no window in which a
+  // subscriber or timer exists without a live path to cleanup().
+  const settled = new Promise<void>((resolve) => {
     socket.once("close", resolve);
     socket.once("error", resolve);
   });
+  socket.once("close", cleanup);
+  socket.once("error", cleanup);
+
+  // Re-check: `close` may have fired while we were writing the handshake, in
+  // which case the listeners above already ran and cleanup() is a no-op we
+  // still must not skip.
+  if (socket.destroyed) {
+    cleanup();
+    return new Response(null, { status: 101 });
+  }
+
+  unsubscribe = globalTrafficBuffer.subscribe((ev) => {
+    sendText(socket, ev);
+  });
+
+  pingTimer = setInterval(() => {
+    // `socket.write()` does NOT throw synchronously on a destroyed socket, so
+    // the destroyed check — not the catch — is what stops a dead interval.
+    if (socket.destroyed) {
+      cleanup();
+      return;
+    }
+    try {
+      socket.write(encodeWsFrame(0x09)); // ping
+    } catch {
+      cleanup();
+    }
+  }, PING_INTERVAL_MS);
+
+  // Never resolve — the socket is the response channel.
+  await settled;
 
   cleanup();
   return new Response(null, { status: 101 });

src/lib/acp/manager.ts

@@ -30,6 +30,34 @@ export interface AcpSession {
   createdAt: Date;
 }
 
+/**
+ * Upper bound for each per-session output buffer.
+ *
+ * Both buffers grow on every chunk a CLI agent writes and are only reset when
+ * the next prompt starts, so a chatty or looping agent can grow them without
+ * limit while the session stays alive. 1 MiB is far above a realistic agent
+ * response while keeping a stuck session's footprint bounded.
+ */
+const MAX_BUFFER_CHARS = 1_048_576;
+
+const TRUNCATION_NOTICE = "\n[...output truncated...]\n";
+
+/**
+ * Append to a buffer, keeping the most recent output when the cap is exceeded.
+ *
+ * The tail is what callers care about: `sendPrompt` resolves with the stdout
+ * collected since the prompt was written, and stderr is read for diagnostics
+ * after a failure. Dropping from the front keeps both useful.
+ */
+function appendCapped(buffer: string, chunk: string): string {
+  const combined = buffer + chunk;
+  if (combined.length <= MAX_BUFFER_CHARS) return combined;
+
+  const keep = MAX_BUFFER_CHARS - TRUNCATION_NOTICE.length;
+  if (keep <= 0) return combined.slice(-MAX_BUFFER_CHARS);
+  return TRUNCATION_NOTICE + combined.slice(-keep);
+}
+
 /**
  * ACP Session Manager
  *
@@ -79,17 +107,21 @@ export class AcpManager extends EventEmitter {
     };
 
     child.stdout?.on("data", (chunk: Buffer) => {
-      session.stdoutBuffer += chunk.toString();
+      session.stdoutBuffer = appendCapped(session.stdoutBuffer, chunk.toString());
       this.emit("stdout", { sessionId, data: chunk.toString() });
     });
 
     child.stderr?.on("data", (chunk: Buffer) => {
-      session.stderrBuffer += chunk.toString();
+      session.stderrBuffer = appendCapped(session.stderrBuffer, chunk.toString());
       this.emit("stderr", { sessionId, data: chunk.toString() });
     });
 
     child.on("exit", (code, signal) => {
       session.alive = false;
+      // Only kill() used to remove entries, so any agent that exited on its own
+      // stayed in the map forever. getActiveSessions() filters on `alive`, which
+      // hid the growth from callers.
+      this.sessions.delete(sessionId);
       this.emit("exit", { sessionId, code, signal });
     });
 
@@ -121,39 +153,46 @@ export class AcpManager extends EventEmitter {
     const session = this.sessions.get(sessionId);
     if (!session?.alive) throw new Error(`Session ${sessionId} is not alive`);
 
-    // Clear buffer before sending
+    // Clear buffers before sending. stderr is reset too: it was previously only
+    // ever appended to, so diagnostics for one prompt carried stale output from
+    // every earlier prompt in the session.
     session.stdoutBuffer = "";
+    session.stderrBuffer = "";
 
     // Send prompt
     this.sendInput(sessionId, prompt + "\n");
 
     // Wait for response (collect until process goes idle or timeout)
     return new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        reject(new Error(`ACP timeout after ${timeoutMs}ms`));
-      }, timeoutMs);
+      let idleTimer: ReturnType<typeof setTimeout> | undefined;
 
-      let idleTimer: ReturnType<typeof setTimeout>;
+      // Every outcome -- idle, exit, or timeout -- has to release the same
+      // resources. `acpManager` is a module-level singleton, so a branch that
+      // skips this leaks a listener per call for the lifetime of the process.
+      const settle = (finish: () => void) => {
+        clearTimeout(timer);
+        clearTimeout(idleTimer);
+        this.removeListener("stdout", onData);
+        this.removeListener("exit", onExit);
+        finish();
+      };
+
+      const timer = setTimeout(() => {
+        settle(() => reject(new Error(`ACP timeout after ${timeoutMs}ms`)));
+      }, timeoutMs);
 
       const onData = ({ sessionId: sid }: { sessionId: string }) => {
         if (sid !== sessionId) return;
         // Reset idle timer on new data
         clearTimeout(idleTimer);
         idleTimer = setTimeout(() => {
-          clearTimeout(timer);
-          this.removeListener("stdout", onData);
-          this.removeListener("exit", onExit);
-          resolve(session.stdoutBuffer);
+          settle(() => resolve(session.stdoutBuffer));
         }, 2000); // 2s idle = response complete
       };
 
       const onExit = ({ sessionId: sid }: { sessionId: string }) => {
         if (sid !== sessionId) return;
-        clearTimeout(timer);
-        clearTimeout(idleTimer);
-        this.removeListener("stdout", onData);
-        this.removeListener("exit", onExit);
-        resolve(session.stdoutBuffer);
+        settle(() => resolve(session.stdoutBuffer));
       };
 
       this.on("stdout", onData);

src/lib/cli-helper/log-streamer.ts

@@ -38,30 +38,37 @@ export function createLogStream(options: LogStreamOptions = {}): LogStream {
 
         if (!response.ok) {
           controller.error(new Error(`HTTP ${response.status}: ${response.statusText}`));
-          clearTimeout(timeoutId);
           return;
         }
 
         if (!response.body) {
           controller.error(new Error("Response body is null"));
-          clearTimeout(timeoutId);
           return;
         }
 
         const reader = response.body.getReader();
 
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) break;
-          if (signal.aborted) break;
-          controller.enqueue(value);
+        try {
+          while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            if (signal.aborted) break;
+            controller.enqueue(value);
+          }
+        } finally {
+          // Leaving the loop early (abort/throw) otherwise keeps the body locked
+          // and its socket held until GC.
+          await reader.cancel().catch(() => {});
         }
 
         controller.close();
-        clearTimeout(timeoutId);
       } catch (err) {
         if (signal.aborted) return; // Expected stop
         controller.error(err instanceof Error ? err : new Error(String(err)));
+      } finally {
+        // `stop()` aborts mid-fetch and returns through the `signal.aborted`
+        // branch above, so clearing the timer on the individual exit paths
+        // misses the one path stop() is built to take.
         clearTimeout(timeoutId);
       }
     },

src/lib/db/adapters/nodeSqliteAdapter.ts

@@ -35,26 +35,34 @@ export async function createNodeSqliteAdapter(filePath: string): Promise<SqliteA
   }, CHECKPOINT_INTERVAL_MS);
   (checkpointTimer as unknown as NodeJS.Timeout).unref?.();
 
+  // Declared before gracefulClose so the close path can detach them. Without
+  // this, every closed adapter leaves three closures pinned on `process` --
+  // each holding this adapter and its DatabaseSync handle alive -- and short-
+  // lived adapters (POST /api/db-backups/import opens one per request) trip
+  // Node's MaxListenersExceededWarning. #7494 fixed exactly this for sql.js.
+  const onBeforeExit = () => {
+    adapter.close();
+  };
+  const onSignal = () => {
+    adapter.close();
+    process.exit(0);
+  };
+
   function gracefulClose() {
     clearInterval(checkpointTimer as unknown as NodeJS.Timeout);
     try {
       db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
     } catch {}
+    process.removeListener("beforeExit", onBeforeExit);
+    process.removeListener("SIGINT", onSignal);
+    process.removeListener("SIGTERM", onSignal);
   }
 
   const adapter = createNodeSqliteAdapterFromDatabase(db, filePath, gracefulClose);
 
-  process.once("beforeExit", () => {
-    adapter.close();
-  });
-  process.once("SIGINT", () => {
-    adapter.close();
-    process.exit(0);
-  });
-  process.once("SIGTERM", () => {
-    adapter.close();
-    process.exit(0);
-  });
+  process.once("beforeExit", onBeforeExit);
+  process.once("SIGINT", onSignal);
+  process.once("SIGTERM", onSignal);
 
   return adapter;
 }

src/lib/gamification/notifications.ts

@@ -110,6 +110,13 @@ export function createBadgeNotificationStream(
         }
       };
 
+      // A client that disconnects while the route is still awaiting auth
+      // arrives here already aborted, and "abort" will never fire again --
+      // the timers above would then run for the lifetime of the process.
+      if (signal?.aborted) {
+        cleanup();
+        return;
+      }
       if (signal) {
         signal.addEventListener("abort", cleanup);
       }

src/lib/plugins/loader.ts

@@ -9,6 +9,7 @@
  */
 
 import { spawn } from "child_process";
+import type { ChildProcess } from "child_process";
 import { writeFile, readFile } from "fs/promises";
 import { rmSync } from "fs";
 import { join } from "path";
@@ -105,6 +106,37 @@ function forwardChildOutput(
  * against process exit — under `node --test --test-force-exit` the runner exits
  * before the promise settles, leaking one temp .mjs per plugin load.
  */
+/** Children already escalating to SIGKILL. Prevents re-arming a second timer + listener
+ *  for a child that is already being killed. */
+const escalating = new WeakSet<ChildProcess>();
+
+/**
+ * SIGTERM has already been sent; escalate to SIGKILL if the child ignores it.
+ *
+ * Must be idempotent per child. Every hook timeout hits this path, and a plugin that
+ * traps SIGTERM keeps taking calls, so re-arming would add one exit listener plus one
+ * killTimer closure per timeout — Node starts printing MaxListenersExceededWarning at 11.
+ * One pending kill per child is also all that is useful: SIGKILL cannot be ignored, so a
+ * second timer would only re-signal a corpse. (#12819)
+ */
+function escalateToSigkill(child: ChildProcess): void {
+  if (escalating.has(child)) return;
+  escalating.add(child);
+
+  const onExit = () => {
+    clearTimeout(killTimer);
+    escalating.delete(child);
+  };
+  const killTimer = setTimeout(() => {
+    child.removeListener("exit", onExit);
+    escalating.delete(child);
+    try {
+      child.kill("SIGKILL");
+    } catch {}
+  }, SIGKILL_GRACE_MS);
+  child.once("exit", onExit);
+}
+
 function removeHostScript(path: string): void {
   try {
     rmSync(path, { force: true });
@@ -293,12 +325,7 @@ export async function loadPlugin(
         }
         child.kill("SIGTERM");
         // Escalate to SIGKILL if plugin ignores SIGTERM
-        const killTimer = setTimeout(() => {
-          try {
-            child.kill("SIGKILL");
-          } catch {}
-        }, SIGKILL_GRACE_MS);
-        child.once("exit", () => clearTimeout(killTimer));
+        escalateToSigkill(child);
         reject(new Error(`Plugin hook '${hook}' timed out after ${timeout}ms`));
       }, timeout);
 
@@ -399,12 +426,7 @@ export async function loadPlugin(
   const cleanup = () => {
     child.kill("SIGTERM");
     // Escalate to SIGKILL after grace period
-    const killTimer = setTimeout(() => {
-      try {
-        child.kill("SIGKILL");
-      } catch {}
-    }, SIGKILL_GRACE_MS);
-    child.once("exit", () => clearTimeout(killTimer));
+    escalateToSigkill(child);
     removeHostScript(hostScriptPath);
     log.info("loader.cleanup", { name: manifest.name });
   };

src/lib/telegram/botApi.ts

@@ -5,7 +5,12 @@
  * replies and setWebhook for webhook registration. Streaming is emulated
  * by the caller via progressive edits (sendMessage / editMessageText).
  */
-import { getTelegramBotApiBase, getTelegramBotToken, getTelegramWebhookTimeoutMs } from "./config";
+import {
+  getTelegramBotApiBase,
+  getTelegramBotToken,
+  getTelegramWebhookTimeoutMs,
+  getTelegramWebhookSecret,
+} from "./config";
 
 export interface TelegramSendMessageParams {
   chat_id: number | string;
@@ -92,7 +97,15 @@ export async function setTelegramWebhook(
   opts: { dropPending?: boolean } = {}
 ): Promise<{ url: string; pending_update_count?: number }> {
   if (url) {
-    return botFetch("setWebhook", { url, drop_pending_updates: opts.dropPending ?? true });
+    // Register the shared secret so Telegram echoes it back as
+    // X-Telegram-Bot-Api-Secret-Token on every delivery; the webhook route
+    // rejects deliveries that do not carry it (#13172).
+    const secret = getTelegramWebhookSecret();
+    return botFetch("setWebhook", {
+      url,
+      drop_pending_updates: opts.dropPending ?? true,
+      ...(secret ? { secret_token: secret } : {}),
+    });
   }
   return botFetch("deleteWebhook", { drop_pending_updates: opts.dropPending ?? true });
 }

src/lib/telegram/chatProxy.ts

@@ -21,11 +21,31 @@ const DEFAULT_MODEL = process.env.TELEGRAM_DEFAULT_MODEL || "auto/chat";
  * Resolve (and lazily mint) an OmniRoute API key for a Telegram user.
  * Returns the plaintext key value, cached per user id.
  */
+// Bounded LRU. The webhook path passes a caller-supplied chat id, so the key
+// space is not limited to the real user population and an uncapped Map would
+// grow for the lifetime of the process. Insertion order is the recency order:
+// a hit re-inserts, and the oldest entry is dropped once the cap is reached.
+const KEY_CACHE_MAX_ENTRIES = 1000;
 const keyCache = new Map<number, string>();
 
+function rememberUserApiKey(telegramUserId: number, key: string): void {
+  // Re-insert so this id becomes the most recently used entry.
+  keyCache.delete(telegramUserId);
+  keyCache.set(telegramUserId, key);
+  while (keyCache.size > KEY_CACHE_MAX_ENTRIES) {
+    const oldest = keyCache.keys().next();
+    if (oldest.done) break;
+    keyCache.delete(oldest.value);
+  }
+}
+
 export async function resolveUserApiKey(telegramUserId: number): Promise<string> {
   const cached = keyCache.get(telegramUserId);
-  if (cached) return cached;
+  if (cached) {
+    // Refresh recency so an active user is not evicted by a burst of new ids.
+    rememberUserApiKey(telegramUserId, cached);
+    return cached;
+  }
 
   const machineId = (await getConsistentMachineId().catch(() => null)) || "0000000000000000";
 
@@ -39,12 +59,12 @@ export async function resolveUserApiKey(telegramUserId: number): Promise<string>
   );
   const matchKey = (match as { key?: string } | undefined)?.key;
   if (typeof matchKey === "string" && matchKey.length > 0) {
-    keyCache.set(telegramUserId, matchKey);
+    rememberUserApiKey(telegramUserId, matchKey);
     return matchKey;
   }
 
   const created = await createApiKey(`telegram:${telegramUserId}`, machineId);
-  keyCache.set(telegramUserId, created.key);
+  rememberUserApiKey(telegramUserId, created.key);
   return created.key;
 }
 

src/lib/telegram/config.ts

@@ -25,6 +25,30 @@ export function getTelegramWebhookTimeoutMs(): number {
   return Number.isInteger(parsed) && parsed > 0 ? parsed : DEFAULT_WEBHOOK_TIMEOUT_MS;
 }
 
+/**
+ * Shared secret for authenticating Telegram webhook deliveries.
+ *
+ * Telegram echoes the `secret_token` passed to `setWebhook` back on every
+ * delivery in the `X-Telegram-Bot-Api-Secret-Token` header, which is the only
+ * way to prove a webhook POST actually came from Telegram. Kept in the
+ * environment alongside the bot token so it is never stored in the DB.
+ */
+export function getTelegramWebhookSecret(): string {
+  return process.env.TELEGRAM_WEBHOOK_SECRET || "";
+}
+
+/**
+ * Whether webhook deliveries are authenticated.
+ *
+ * When no secret is configured the webhook path is rejected outright rather
+ * than served unauthenticated: an open path mints API keys and spends upstream
+ * quota for any caller (see #13172). The Mini App path is unaffected — it
+ * authenticates with the initData HMAC and does not use this secret.
+ */
+export function isTelegramWebhookSecretConfigured(): boolean {
+  return getTelegramWebhookSecret().length > 0;
+}
+
 export function getTelegramBotApiBase(): string {
   return process.env.TELEGRAM_BOT_API_BASE || "https://api.telegram.org";
 }

src/lib/usage/callLogArtifacts.ts

@@ -17,6 +17,17 @@ const OMITTED_FOR_SIZE_LIMIT = "[omitted: call log artifact size limit exceeded]
 const STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT =
   "[stream chunks omitted: call log artifact size limit exceeded]";
 
+/**
+ * True for a placeholder a size-limit fallback wrote in place of a real
+ * payload. Consumers that fall back from one artifact field to another
+ * (`maybeEnrichCompletedDetail`) must treat a marker as absent: it is a
+ * non-empty string, so a bare truthiness check happily "recovers" it and
+ * overwrites the real value it was meant to stand in for.
+ */
+export function isSizeLimitOmissionMarker(value: unknown): boolean {
+  return value === OMITTED_FOR_SIZE_LIMIT || value === STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT;
+}
+
 // The error is the only field that says *why* a request failed, and it is
 // typically ~90 bytes next to the multi-hundred-KB bodies that trip the cap.
 // Dropping it made a size-limited row undiagnosable: a provider outage, a local
@@ -182,33 +193,49 @@ function buildMinimalArtifactForSizeLimit(artifact: CallLogArtifact) {
   };
 }
 
-function serializeFinalSizeLimitFallback(artifact: CallLogArtifact, maxBytes: number): string {
-  const withSummary = JSON.stringify(buildMinimalArtifactForSizeLimit(artifact));
-  if (Buffer.byteLength(withSummary) <= maxBytes) {
-    return withSummary;
-  }
-
-  // The summary alone exceeded the cap (pathological). Keep the error so the
-  // row stays diagnosable, drop everything else including the summary body.
-  const errorOnly = JSON.stringify({
-    schemaVersion: artifact.schemaVersion,
-    _omniroute_truncated: true,
-    reason: SIZE_LIMIT_EXCEEDED_REASON,
+/**
+ * Fallback ladder for an artifact that does not fit its byte budget, ordered
+ * from "keeps the most" to "keeps the least": the first stage that fits wins.
+ *
+ * Ordering rule: drop the payload that most plausibly tripped the cap, and
+ * drop a payload that is *duplicated elsewhere in the artifact* before one
+ * that is unique. `pipeline` carries both sides of the exchange already
+ * translated (`clientRawRequest`/`providerRequest`/`providerResponse`/
+ * `clientResponse`), so evicting it to keep `requestBody` traded the whole
+ * upstream exchange -- including the only record of what the provider
+ * actually answered -- for a raw client prompt the pipeline already holds a
+ * translated copy of. Bodies go first now, and `pipeline` survives one stage
+ * longer; the previous order is still reached when dropping the bodies alone
+ * is not enough.
+ *
+ * Two consumers depend on that ordering, not just human diagnosis:
+ * `resolvePreviousResponseState` (db/responsesContinuationStore.ts) rebuilds
+ * `previous_response_id` history from `pipeline.clientRawRequest` /
+ * `pipeline.clientResponse` and returns null -- forcing the client to resend
+ * full history -- for any artifact whose pipeline was omitted; and
+ * `maybeEnrichCompletedDetail` (usage/completedRequestDetails.ts) reads
+ * `pipeline.providerResponse` in preference to `responseBody`.
+ */
+function buildSizeLimitStages(artifact: CallLogArtifact): Array<() => unknown> {
+  const omitBodies = <T extends object>(value: T) => ({
+    ...value,
+    requestBody: OMITTED_FOR_SIZE_LIMIT,
+    responseBody: OMITTED_FOR_SIZE_LIMIT,
     error: preserveErrorForSizeLimit(artifact.error),
   });
-  if (Buffer.byteLength(errorOnly) <= maxBytes) {
-    return errorOnly;
-  }
 
-  // Last resort: even the error-only payload did not fit. The error still
-  // rides along -- without it this row says only "something was too big",
-  // which is the state this change exists to remove.
-  return JSON.stringify({
-    schemaVersion: artifact.schemaVersion,
-    _omniroute_truncated: true,
-    reason: SIZE_LIMIT_EXCEEDED_REASON,
-    error: preserveErrorForSizeLimit(artifact.error),
-  });
+  return [
+    () => truncateArtifactForStorage(artifact),
+    // Bodies alone: worth a stage only when there is a pipeline to keep in
+    // exchange. Without one it produces the same bytes as the stage two lines
+    // below, so it is left out rather than costing a redundant stringify.
+    ...(artifact.pipeline ? [() => omitBodies(artifact)] : []),
+    () => omitOversizedPipeline(artifact),
+    () => omitBodies(omitOversizedPipeline(artifact)),
+    // The summary alone exceeded the cap (pathological). Keep the error so the
+    // row stays diagnosable, drop everything else including the summary body.
+    () => buildMinimalArtifactForSizeLimit(artifact),
+  ];
 }
 
 function serializeArtifactForStorage(artifact: CallLogArtifact): string {
@@ -227,27 +254,22 @@ function serializeArtifactForStorage(artifact: CallLogArtifact): string {
     return serialized;
   }
 
-  const truncated = JSON.stringify(truncateArtifactForStorage(artifact));
-  if (Buffer.byteLength(truncated) <= maxBytes) {
-    return truncated;
+  for (const buildStage of buildSizeLimitStages(artifact)) {
+    const candidate = JSON.stringify(buildStage());
+    if (Buffer.byteLength(candidate) <= maxBytes) {
+      return candidate;
+    }
   }
 
-  const withoutPipeline = JSON.stringify(omitOversizedPipeline(artifact));
-  if (Buffer.byteLength(withoutPipeline) <= maxBytes) {
-    return withoutPipeline;
-  }
-
-  const minimal = JSON.stringify({
-    ...omitOversizedPipeline(artifact),
-    requestBody: OMITTED_FOR_SIZE_LIMIT,
-    responseBody: OMITTED_FOR_SIZE_LIMIT,
+  // Last resort: not even the summary fit. The error still rides along --
+  // without it this row says only "something was too big", which is the state
+  // the size-limit fallbacks exist to remove.
+  return JSON.stringify({
+    schemaVersion: artifact.schemaVersion,
+    _omniroute_truncated: true,
+    reason: SIZE_LIMIT_EXCEEDED_REASON,
     error: preserveErrorForSizeLimit(artifact.error),
   });
-  if (Buffer.byteLength(minimal) <= maxBytes) {
-    return minimal;
-  }
-
-  return serializeFinalSizeLimitFallback(artifact, maxBytes);
 }
 
 export function writeCallArtifact(

src/lib/usage/completedRequestDetails.ts

@@ -50,13 +50,14 @@ export function clearCompletedDetails() {
   completedDetails.clear();
 }
 
+function isUnset(value: unknown): boolean {
+  return value === undefined || value === null;
+}
+
 export function maybeEnrichCompletedDetail(updated: PendingRequestDetail, connectionId: string) {
   void (async () => {
     try {
-      const missingProvider =
-        updated.providerResponse === undefined || updated.providerResponse === null;
-      const missingClient = updated.clientResponse === undefined || updated.clientResponse === null;
-      if (!missingProvider && !missingClient) return;
+      if (!isUnset(updated.providerResponse) && !isUnset(updated.clientResponse)) return;
 
       const db = getDbInstance();
       const sinceIso = new Date(Date.now() - 30_000).toISOString();
@@ -67,24 +68,32 @@ export function maybeEnrichCompletedDetail(updated: PendingRequestDetail, connec
         .all(connectionId, updated.model, sinceIso) as Array<{ artifact_relpath: string | null }>;
       for (const row of rows) {
         if (!row.artifact_relpath) continue;
-        const { readCallArtifact } = await import("./callLogArtifacts");
+        const { readCallArtifact, isSizeLimitOmissionMarker } = await import("./callLogArtifacts");
         const art = readCallArtifact(row.artifact_relpath);
         if (art.state !== "ready" || !art.artifact) continue;
         const pipeline = art.artifact.pipeline as
           | { providerResponse?: unknown; clientResponse?: unknown }
           | undefined;
-        if (missingProvider && pipeline?.providerResponse) {
+        // pipeline.* first: it is the translated payload of one specific side.
+        // `responseBody` is a single coarse value handed to both sides, so it
+        // may only fill a side still empty AFTER the pipeline had its turn --
+        // testing emptiness once before the loop let it overwrite the payload
+        // just recovered, showing a provider payload as the client response.
+        if (isUnset(updated.providerResponse) && pipeline?.providerResponse) {
           updated.providerResponse = pipeline.providerResponse;
         }
-        if (missingClient && pipeline?.clientResponse) {
+        if (isUnset(updated.clientResponse) && pipeline?.clientResponse) {
           updated.clientResponse = pipeline.clientResponse;
         }
-        if (
-          (missingProvider && art.artifact.responseBody) ||
-          (missingClient && art.artifact.responseBody)
-        ) {
-          if (missingProvider) updated.providerResponse = art.artifact.responseBody;
-          if (missingClient) updated.clientResponse = art.artifact.responseBody;
+        // A size-limited artifact stores an omission marker string in place of
+        // the body. It is truthy, so recovering it here overwrites a real
+        // payload with "[omitted: ...]".
+        const responseBody = isSizeLimitOmissionMarker(art.artifact.responseBody)
+          ? null
+          : art.artifact.responseBody;
+        if (responseBody) {
+          if (isUnset(updated.providerResponse)) updated.providerResponse = responseBody;
+          if (isUnset(updated.clientResponse)) updated.clientResponse = responseBody;
         }
         if (updated.providerResponse || updated.clientResponse) {
           if (completedDetails.has(updated.id)) storeCompletedDetail(updated);

src/sse/handlers/chat.ts

@@ -12,6 +12,7 @@ import { resolveRoutingModel, RoutingModelOps } from "./resolveRoutingModel";
 import {
   getProviderCredentialsWithQuotaPreflight,
   markAccountUnavailable,
+  buildExhaustionOptions,
   extractApiKey,
   isValidApiKey,
   extractSessionAffinityKey,
@@ -1781,7 +1782,8 @@ async function handleSingleModelChat(
           lastStatus,
           candidateAliases,
           isCombo,
-          shadowedNode
+          shadowedNode,
+          runtimeOptions?.correlationId ?? null
         );
         const lastFailedConnectionId =
           excludedConnectionIds.size > 0
@@ -2093,7 +2095,7 @@ async function handleSingleModelChat(
           provider,
           model,
           providerProfile,
-          { isCombo }
+          buildExhaustionOptions(runtimeOptions.correlationId ?? null, { isCombo })
         );
 
         if (shouldFallback && !hasForcedConnection) {
@@ -2142,7 +2144,7 @@ async function handleSingleModelChat(
           provider,
           model,
           providerProfile,
-          { isCombo }
+          buildExhaustionOptions(runtimeOptions.correlationId ?? null, { isCombo })
         );
 
         if (shouldFallback && !hasForcedConnection) {
@@ -2387,7 +2389,7 @@ async function handleSingleModelChat(
             provider,
             model,
             providerProfile,
-            {
+            buildExhaustionOptions(runtimeOptions.correlationId ?? null, {
               persistUnavailableState: !(
                 isCombo &&
                 result.status === 429 &&
@@ -2395,7 +2397,7 @@ async function handleSingleModelChat(
               ),
               isCombo,
               headers: result.response.headers,
-            }
+            })
           );
 
       // An explicit pin (combo step `connectionId` / `x-omniroute-connection`) is an

src/sse/handlers/chatHelpers.ts

@@ -3,7 +3,11 @@ import {
   getComboForModel,
   getModelInfoOrRetirementResponse,
 } from "../services/model";
-import { clearAccountError, markAccountUnavailable } from "../services/auth";
+import {
+  clearAccountError,
+  markAccountUnavailable,
+  buildExhaustionOptions,
+} from "../services/auth";
 import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts";
 import { createBuiltinAutoCombo } from "@omniroute/open-sse/services/autoCombo/builtinCatalog.ts";
 import * as log from "../utils/logger";
@@ -555,7 +559,7 @@ export async function executeChatWithBreaker({
                 provider,
                 model,
                 providerProfile,
-                { isCombo }
+                buildExhaustionOptions(correlationId ?? null, { isCombo })
               );
             },
           })
@@ -731,7 +735,8 @@ export function handleNoCredentials(
   lastStatus: number | null,
   candidateAliases?: readonly string[],
   isCombo: boolean = false,
-  shadowedNode: ShadowedProviderNode | null = null
+  shadowedNode: ShadowedProviderNode | null = null,
+  correlationId?: string | null
 ) {
   if (credentials?.allRateLimited) {
     const errorMsg = lastError || credentials.lastError || "Unavailable";
@@ -772,6 +777,7 @@ export function handleNoCredentials(
       provider,
       model,
       lastStatus,
+      ...(correlationId ? { correlationId } : {}),
     });
     return errorResponse(lastStatus, lastError);
   }

src/sse/services/auth.ts

@@ -2401,9 +2401,12 @@ export async function getProviderCredentialsWithQuotaPreflight(
 }
 
 /**
- * #10334 — Guard for the agentrouter-exclusive "connection scope" quota
- * cooldown branch in markAccountUnavailable. The "never terminal" invariant of
- * that branch is NOT structurally guaranteed by `ruleScope === "connection"`
+ * #10334 — Guard for the "connection scope" quota cooldown branch in
+ * markAccountUnavailable (agentrouter-exclusive in practice: no opencode-family
+ * rule matches 403 today, so only agentrouter's "额度不足" rule reaches this
+ * predicate via 403 — but opencode-family 429 header-quota hits also qualify
+ * via the 429 path). The "never terminal" invariant of that branch is NOT
+ * structurally guaranteed by `ruleScope === "connection"`
  * alone — it also depends on the provider rule table only ever pairing scope
  * "connection" with a genuinely transient reason. Today
  * (`buildAgentrouterRules()` in providerErrorRules.ts) that is true: the only
@@ -2547,6 +2550,26 @@ async function applyEgressIpLockout(
   }
 }
 
+/** Build the options for markAccountUnavailable on the chat exhaustion path.
+ * Single place that forwards the request id so no chat sender can forget it:
+ * every chat caller passes its in-scope id through here. */
+export function buildExhaustionOptions(
+  correlationId: string | null,
+  rest: {
+    persistUnavailableState?: boolean;
+    /** Caller is the combo engine — it records its own model-level lockouts. */
+    isCombo?: boolean;
+    headers?: Headers | Record<string, string> | null;
+  } = {}
+): {
+  persistUnavailableState?: boolean;
+  isCombo?: boolean;
+  headers?: Headers | Record<string, string> | null;
+  correlationId: string | null;
+} {
+  return { ...rest, correlationId };
+}
+
 /** Persist exponential-backoff state for an unavailable provider connection. */
 export async function markAccountUnavailable(
   connectionId: string,
@@ -2560,6 +2583,7 @@ export async function markAccountUnavailable(
     /** Caller is the combo engine — it records its own model-level lockouts. */
     isCombo?: boolean;
     headers?: Headers | Record<string, string> | null;
+    correlationId?: string | null;
   } = {}
 ) {
   const currentMutex = markMutexes.get(connectionId) || Promise.resolve();
@@ -2727,8 +2751,10 @@ export async function markAccountUnavailable(
 
     const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
 
-    // #10334 — agentrouter EXCLUSIVE: the matched provider rule declared scope
-    // "connection" for account-wide quota exhaustion ("额度不足"). agentrouter is
+    // #10334 — connection-scope branch: the matched provider rule declared scope
+    // "connection" for account-wide quota exhaustion (agentrouter "额度不足";
+    // exclusive in practice — no opencode-family rule matches 403 today).
+    // agentrouter is
     // a passthroughModels provider (isPerModelQuotaProvider === true), so without
     // this branch the next `if` would treat it like any other passthrough 429 and
     // lock a SINGLE model — leaving combo routing to burn one upstream call per
@@ -2754,6 +2780,15 @@ export async function markAccountUnavailable(
     // of cooldown" ends up producing a LONGER effective block for this one rule.
     // Not addressed here; flagged for a future #2997 follow-up if it proves to be
     // a real operator complaint.
+    //
+    // HONORS note: since the opencode family joined HONORS, an opencode-family
+    // 429 carrying upstream quota headers (x-ratelimit-remaining-*) also lands
+    // here with ruleScope "connection" — before the #10880 egress branch below,
+    // so sibling cooling is skipped on that path. Latent today: the only
+    // request-path caller forwarding headers is chat.ts:2383 (chat completions),
+    // and opencode upstreams rarely send those headers on 429 (the observed
+    // envelope is the headers-less "monthly usage limit" body, which keeps
+    // flowing to the egress block with ruleScope undefined).
     if (ruleScopeIsConnection && provider && !disableCooling) {
       const connectionCooldownMs =
         fallbackResult.cooldownMs > 0 ? fallbackResult.cooldownMs : COOLDOWN_MS.rateLimit;
@@ -2848,6 +2883,45 @@ export async function markAccountUnavailable(
 
     const isNvidiaModelGone = provider === "nvidia" && status === 410;
     const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs };
+    // Same persisted reason the agentrouter 403 model-scope branch hard-codes
+    // ("forbidden"): the lock key is the getModelLockKey tuple shared with the
+    // combo path, and the declared 1h (same order as that combo lock) is
+    // operator-clamped by recordModelLockoutFailure to mlSettings.maxCooldownMs
+    // (~30min default) — the verbatim 1h never escapes operator control.
+    // Narrow scope: status === 400 only (never a 403/429 rule), adjacent to
+    // :2843's per-model-quota status set (which excludes 400) — malformed 400s
+    // carry no ruleScope and fall through unchanged.
+    if (model && provider && status === 400 && fallbackResult.ruleScope === "model") {
+      // Single source of truth: the rule's own cooldownMs (surfaced on
+      // fallbackResult by the 400 pre-check in checkFallbackError). The literal
+      // is only the fallback for a rule that declares no cooldown — editing
+      // the rule's cooldownMs takes effect without touching this call site.
+      const ruleCooldownMs =
+        typeof fallbackResult.cooldownMs === "number" && fallbackResult.cooldownMs > 0
+          ? fallbackResult.cooldownMs
+          : 3_600_000;
+      const lockout = recordModelLockoutFailure(
+        provider,
+        connectionId,
+        model,
+        "model_capacity",
+        400,
+        ruleCooldownMs,
+        effectiveProviderProfile,
+        { exactCooldownMs: ruleCooldownMs, maxCooldownMs: mlSettings.maxCooldownMs }
+      );
+      updateProviderConnection(connectionId, {
+        lastErrorType: "model_capacity",
+        lastError: `Model ${model} model_capacity`,
+        lastErrorAt: new Date().toISOString(),
+        errorCode: status,
+      }).catch(() => {});
+      log.info(
+        "AUTH",
+        `Model-only lockout for ${provider}:${model} — ${status} model_capacity ${Math.ceil(lockout.cooldownMs / 1000)}s (rule scope=model, connection stays active)`
+      );
+      return { shouldFallback: true, cooldownMs: lockout.cooldownMs };
+    }
     if (
       isPerModelQuotaProvider &&
       provider &&
@@ -2878,7 +2952,10 @@ export async function markAccountUnavailable(
         }).catch(() => {});
         log.info(
           "AUTH",
-          `Server error for ${provider}:${model} — ${status} ${reason} (no model lockout, connection stays active for sibling models)`
+          `Server error for ${provider}:${model} — ${status} ${reason} (no model lockout, connection stays active for sibling models)`,
+          {
+            ...(options.correlationId ? { correlationId: options.correlationId } : {}),
+          }
         );
         return { shouldFallback: true, cooldownMs: 0 };
       }