feat(monitoring): expose structural chat admission snapshot + shed counters (#11244) (#11268)

The structural admission gate (src/shared/middleware/chatBodyAdmission.ts —
the bounded heavyweight lease + healthy-headroom path from #10110/#10437)
returns its 503 chat_admission_busy BEFORE request logging, so a shed left
no trace: no counter, no log line, and the process-wide snapshot was never
consumed by any route. This adds pure observability — admission behavior,
defaults, and thresholds are untouched.

- chatBodyAdmission.ts: in-memory shed history (shedTotal + shedsByReason)
  on ChatAdmissionController, recorded at the two capacity-driven give-up
  points in acquireHeavyWithin (queue_timeout when the bounded wait
  expires, queued_bytes_budget when the heap valve refuses to park). A
  client abort mid-wait is deliberately not counted — capacity was never
  denied. Each shed also emits exactly one structured pino warn (module
  chat-admission) with reason/activeHeavy/waiting/queuedBytes and the HMAC
  session fingerprint — never a raw credential. PerConnectionAdmission
  Controller.snapshot() now carries the counters plus activeHealthyHeadroom.
- /api/monitoring/health: the structural snapshot is exposed under a new
  additive chatAdmission key next to the existing adaptiveAdmission (the
  shadow-mode layer) — allowlisted projection in observability.ts, nothing
  removed from the current payload.
- Tests: tests/unit/chat-admission-visibility-11244.test.ts (RED→GREEN:
  shed counting per reason, abort exclusion, snapshot shape, and the warn
  log carrying the fingerprint but never the raw API key) + a chatAdmission
  allowlist-projection test in observability-payloads.test.ts.

Co-authored-by: Xiangzhe <bakryun0718@proton.me>

src/app/api/monitoring/health/route.ts

@@ -74,6 +74,7 @@ export async function GET(request: Request) {
       credentialHealthModule,
       localHealthModule,
       adaptiveAdmissionModule,
+      chatAdmissionModule,
       settingsResult,
       connectionsResult,
     ] = await Promise.allSettled([
@@ -86,6 +87,7 @@ export async function GET(request: Request) {
       import("@/lib/credentialHealth/cache"),
       import("@/lib/localHealthCheck"),
       import("@omniroute/open-sse/services/admission/runtime.ts"),
+      import("@/shared/middleware/chatBodyAdmission"),
       getCachedSettings(),
       getProviderConnections(),
     ]);
@@ -172,6 +174,17 @@ export async function GET(request: Request) {
             null
           )
         : null;
+    // #11244: the STRUCTURAL admission gate (chatBodyAdmission.ts — bounded
+    // heavyweight lease + shed counters), exposed next to but distinct from the
+    // adaptive shadow-mode snapshot above. Additive key — nothing existing moves.
+    const chatAdmission =
+      chatAdmissionModule.status === "fulfilled"
+        ? readHealthValue(
+            "chat admission",
+            () => chatAdmissionModule.value.perConnectionAdmissionController.snapshot(),
+            null
+          )
+        : null;
 
     const payload = buildHealthPayload({
       appVersion: APP_CONFIG.version,
@@ -200,6 +213,7 @@ export async function GET(request: Request) {
       activeSessionsByKey,
       credentialHealth,
       adaptiveAdmission,
+      chatAdmission,
     });
 
     healthPayloadCache = { payload, expiresAt: Date.now() + HEALTH_PAYLOAD_TTL_MS };
@@ -218,6 +232,7 @@ export async function GET(request: Request) {
       quotaMonitor: { ...fallbackQuotaMonitorSummary, monitors: [] },
       sessions: { activeCount: 0, stickyBoundCount: 0, byApiKey: {}, top: [] },
       adaptiveAdmission: null,
+      chatAdmission: null,
       dedup: { inflightRequests: 0 },
     });
   }

src/lib/monitoring/observability.ts

@@ -3,9 +3,48 @@ import {
   getCodexParentAccountDiagnostic,
 } from "@omniroute/open-sse/services/codexAccount/index.ts";
 import type { AdaptiveAdmissionPublicSnapshot } from "@omniroute/open-sse/services/admission/runtime.ts";
+import type { PerConnectionAdmissionController } from "@/shared/middleware/chatBodyAdmission";
 
 type JsonRecord = Record<string, unknown>;
 
+/** Process-wide structural chat-admission snapshot type (chatBodyAdmission.ts). */
+export type ChatAdmissionSnapshot = ReturnType<PerConnectionAdmissionController["snapshot"]>;
+
+/**
+ * Low-card structural chat-admission health summary (#11244) — the bounded
+ * heavyweight-lease gate from chatBodyAdmission.ts (#10110/#10437), NOT the
+ * adaptive shadow-mode layer above. Lane keys are opaque HMAC fairness
+ * fingerprints (resolveSessionId), never raw credentials.
+ */
+export type ChatAdmissionHealthSummary = {
+  activeHeavy: number;
+  activeHealthyHeadroom: number;
+  waiting: number;
+  queuedBytes: number;
+  shedTotal: number;
+  shedsByReason: Record<string, number>;
+  lanes: Array<{ key: string; waiting: number }>;
+};
+
+/**
+ * Explicit allowlisted projection of the structural admission snapshot.
+ * Never spreads the snapshot — only the documented low-cardinality fields pass.
+ */
+export function projectChatAdmissionSummary(
+  snapshot: ChatAdmissionSnapshot | null | undefined
+): ChatAdmissionHealthSummary | null {
+  if (!snapshot || typeof snapshot !== "object") return null;
+  return {
+    activeHeavy: snapshot.activeHeavy,
+    activeHealthyHeadroom: snapshot.activeHealthyHeadroom,
+    waiting: snapshot.waiting,
+    queuedBytes: snapshot.queuedBytes,
+    shedTotal: snapshot.shedTotal,
+    shedsByReason: { ...(snapshot.shedsByReason ?? {}) },
+    lanes: (snapshot.lanes ?? []).map((lane) => ({ key: lane.key, waiting: lane.waiting })),
+  };
+}
+
 /** Low-card adaptive-admission health summary — no tenant/request/body/queue details. */
 export type AdaptiveAdmissionHealthSummary = {
   mode: AdaptiveAdmissionPublicSnapshot["mode"];
@@ -160,6 +199,8 @@ interface BuildHealthPayloadOptions {
   };
   /** Optional injected public adaptive-admission snapshot; projected, never raw-spread. */
   adaptiveAdmission?: AdaptiveAdmissionPublicSnapshot | null;
+  /** #11244: optional structural chat-admission snapshot; projected, never raw-spread. */
+  chatAdmission?: ChatAdmissionSnapshot | null;
 }
 
 function limitMonitors(monitors: QuotaMonitorSnapshot[], maxItems = 8): QuotaMonitorSnapshot[] {
@@ -347,6 +388,7 @@ export function buildHealthPayload({
   activeSessionsByKey = {},
   credentialHealth,
   adaptiveAdmission = null,
+  chatAdmission = null,
   buildSha = null,
 }: BuildHealthPayloadOptions) {
   const timestamp = new Date().toISOString();
@@ -449,6 +491,9 @@ export function buildHealthPayload({
     sessions: buildSessionsSummary({ activeSessions, activeSessionsByKey }),
     credentialHealth, // may be undefined if credentialHealth module not loaded
     adaptiveAdmission: projectAdaptiveAdmissionSummary(adaptiveAdmission),
+    // #11244: the STRUCTURAL gate (chatBodyAdmission.ts) next to the adaptive one —
+    // distinct key so clients reading `adaptiveAdmission` are untouched.
+    chatAdmission: projectChatAdmissionSummary(chatAdmission),
     dedup: {
       inflightRequests,
     },

src/shared/middleware/chatBodyAdmission.ts

@@ -16,6 +16,7 @@
  */
 
 import { CORS_HEADERS } from "../utils/cors";
+import { createLogger } from "../utils/logger";
 import { createHmac } from "crypto";
 import v8 from "node:v8";
 import { trackRequest } from "../../lib/gracefulShutdown";
@@ -168,6 +169,47 @@ interface AdmissionWaiter {
   readonly resolve: () => void;
 }
 
+/**
+ * Why a structural shed (503 `chat_admission_busy`) happened (#11244):
+ * - `queue_timeout`: the bounded wait expired with no heavyweight capacity freed
+ *   (includes the `queueMs=0` legacy immediate-reject path — capacity was busy at
+ *   the instant the request arrived).
+ * - `queued_bytes_budget`: the queued-bytes heap valve (#9654 / U3) refused to
+ *   park the waiter because the buffered-body budget was already exhausted.
+ *
+ * A client abort mid-wait is deliberately NOT a shed: capacity was never denied,
+ * the caller simply left (its 503 is dropped on the dead connection).
+ */
+export type ChatAdmissionShedReason = "queue_timeout" | "queued_bytes_budget";
+
+/**
+ * One structural-shed observation, emitted to the shed sink at warn level.
+ * `lane` is the opaque fairness key — the HMAC fingerprint produced by
+ * `resolveSessionId` (or "anonymous"/"default"), never a raw credential.
+ */
+export interface ChatAdmissionShedEvent {
+  reason: ChatAdmissionShedReason;
+  activeHeavy: number;
+  waiting: number;
+  queuedBytes: number;
+  lane: string;
+}
+
+export type ChatAdmissionShedSink = (event: ChatAdmissionShedEvent) => void;
+
+const shedLog = createLogger("chat-admission");
+
+/**
+ * Default shed sink (#11244): exactly one structured warn per structural shed.
+ * The 503 returns BEFORE request logging, so without this line a shed left no
+ * trace anywhere. No raw credentials — `lane` is already the HMAC fingerprint,
+ * and the shared logger's redaction hook (logRedaction.ts) is the safety net.
+ * Nothing is logged for admitted requests (noise).
+ */
+function defaultChatAdmissionShedSink(event: ChatAdmissionShedEvent): void {
+  shedLog.warn(event, "structural chat admission shed (chat_admission_busy)");
+}
+
 /**
  * Process-local heavyweight reservation. The capacity check and increment execute in one
  * synchronous JavaScript turn, making acquisition atomic within an OmniRoute process.
@@ -190,6 +232,12 @@ export class ChatAdmissionController {
   /** Keys in creation order; #fairCursor scans them round-robin. */
   #fairKeys: string[] = [];
   #fairCursor = 0;
+  /** #11244: in-memory shed history (total + per reason). The 503 chat_admission_busy
+   * response returns before request logging, so without these counters a structural
+   * shed was invisible. Same in-memory lifetime as the rest of the snapshot state. */
+  #shedTotal = 0;
+  #shedsByReason = new Map<string, number>();
+  readonly #onShed: ChatAdmissionShedSink;
 
   constructor(
     readonly maxHeavyInFlight = 1,
@@ -197,7 +245,10 @@ export class ChatAdmissionController {
     /** #10437: bounded extra capacity for the healthy-heap fast path. `0` disables
      * the bypass entirely — every busy request then falls through to the same
      * bounded-wait/shed path used under real heap pressure. */
-    readonly healthyHeadroom = CHAT_ADMISSION_HEALTHY_HEADROOM
+    readonly healthyHeadroom = CHAT_ADMISSION_HEALTHY_HEADROOM,
+    /** #11244: sink notified once per structural shed. Defaults to the shared pino
+     * logger (warn); tests inject a capture/no-op sink. */
+    onShed: ChatAdmissionShedSink = defaultChatAdmissionShedSink
   ) {
     if (!Number.isSafeInteger(maxHeavyInFlight) || maxHeavyInFlight < 1) {
       throw new RangeError("maxHeavyInFlight must be a positive integer");
@@ -208,6 +259,7 @@ export class ChatAdmissionController {
     if (!Number.isSafeInteger(healthyHeadroom) || healthyHeadroom < 0) {
       throw new RangeError("healthyHeadroom must be a non-negative integer");
     }
+    this.#onShed = onShed;
   }
 
   get activeHeavy(): number {
@@ -264,6 +316,37 @@ export class ChatAdmissionController {
     return out;
   }
 
+  /** Total structural sheds since process start (#11244). */
+  get shedTotal(): number {
+    return this.#shedTotal;
+  }
+
+  /** Structural sheds by reason since process start (#11244). */
+  get shedsByReason(): Record<string, number> {
+    const out: Record<string, number> = {};
+    for (const [reason, count] of this.#shedsByReason) out[reason] = count;
+    return out;
+  }
+
+  /**
+   * Record one structural shed (503 chat_admission_busy) and notify the shed sink
+   * (#11244). Called internally at every capacity-driven give-up point in
+   * `acquireHeavyWithin`; public so the aggregate snapshot wiring and tests can
+   * exercise the same single path. `lane` is the opaque fairness key (HMAC
+   * fingerprint), never a raw credential.
+   */
+  recordShed(reason: ChatAdmissionShedReason, lane = "default"): void {
+    this.#shedTotal += 1;
+    this.#shedsByReason.set(reason, (this.#shedsByReason.get(reason) ?? 0) + 1);
+    this.#onShed({
+      reason,
+      activeHeavy: this.#activeHeavy,
+      waiting: this.waitingCount,
+      queuedBytes: this.#queuedBytes,
+      lane,
+    });
+  }
+
   tryAcquireHeavy(): ChatAdmissionLease | null {
     if (this.#activeHeavy >= this.maxHeavyInFlight) return null;
     this.#activeHeavy += 1;
@@ -318,9 +401,16 @@ export class ChatAdmissionController {
       const lease = this.tryAcquireHeavy();
       if (lease) return lease;
       const remaining = deadline - Date.now();
-      if (remaining <= 0) return null;
+      if (remaining <= 0) {
+        // Wait window exhausted (or queueMs=0 immediate reject) with capacity still
+        // busy — the caller answers the retryable 503. Count it (#11244).
+        this.recordShed("queue_timeout", sessionKey);
+        return null;
+      }
       // Heap valve: refuse to park when the queued-bytes budget is exhausted.
       if (queuedBytes > 0 && this.#queuedBytes + queuedBytes > this.maxQueuedBytes) {
+        // Same retryable 503, distinct cause: the wait itself would amplify the heap.
+        this.recordShed("queued_bytes_budget", sessionKey);
         return null;
       }
       this.#queuedBytes += queuedBytes;
@@ -367,7 +457,13 @@ export class ChatAdmissionController {
       // Cancel the deadline timer when abort/release wins; a fired timer is a no-op.
       if (deadlineTimer) clearTimeout(deadlineTimer);
       if (onAbort) signal?.removeEventListener("abort", onAbort);
-      if (timedOut) return null;
+      if (timedOut) {
+        // The deadline timer won the race: a genuine shed. When the client ABORT
+        // won instead (signal aborted while parked), capacity was never denied —
+        // the 503 is dropped on the dead connection, so it is not counted (#11244).
+        if (!signal?.aborted) this.recordShed("queue_timeout", sessionKey);
+        return null;
+      }
     }
   }
 
@@ -483,11 +579,18 @@ export class PerConnectionAdmissionController {
 
   constructor(
     readonly maxHeavyInFlight = 1,
-    // Deprecated pre-#10110 lane-eviction knobs: accepted for API
-    // compatibility and ignored — there are no per-session lanes to evict.
-    _opts?: { maxSessions?: number; sessionTtlMs?: number }
+    // `maxSessions`/`sessionTtlMs` are deprecated pre-#10110 lane-eviction knobs:
+    // accepted for API compatibility and ignored — there are no per-session lanes
+    // to evict. `onShed` (#11244) is live: it replaces the shed sink of the shared
+    // controller (tests inject a capture/no-op sink; production keeps the pino warn).
+    _opts?: { maxSessions?: number; sessionTtlMs?: number; onShed?: ChatAdmissionShedSink }
   ) {
-    this.#controller = new ChatAdmissionController(maxHeavyInFlight);
+    this.#controller = new ChatAdmissionController(
+      maxHeavyInFlight,
+      undefined,
+      undefined,
+      _opts?.onShed
+    );
   }
 
   /** Returns the process-global budget — the same instance for every session. */
@@ -497,20 +600,26 @@ export class PerConnectionAdmissionController {
 
   /**
    * Process-wide aggregate snapshot for observability: global totals plus
-   * per-key waiter depths. Keys are opaque scheduler keys, never raw
-   * credentials.
+   * per-key waiter depths and the #11244 shed history (total + per reason).
+   * Keys are opaque scheduler keys, never raw credentials.
    */
   snapshot(): {
     activeHeavy: number;
+    activeHealthyHeadroom: number;
     queuedBytes: number;
     waiting: number;
     lanes: ReadonlyArray<{ key: string; waiting: number }>;
+    shedTotal: number;
+    shedsByReason: Record<string, number>;
   } {
     return {
       activeHeavy: this.#controller.activeHeavy,
+      activeHealthyHeadroom: this.#controller.activeHealthyHeadroom,
       queuedBytes: this.#controller.queuedBytes,
       waiting: this.#controller.waitingCount,
       lanes: this.#controller.waitersByKey,
+      shedTotal: this.#controller.shedTotal,
+      shedsByReason: this.#controller.shedsByReason,
     };
   }