diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 7d2c31c111..028590a09e 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -67,7 +67,7 @@ import { resolveMemoryOwnerId, } from "./chatCore/memoryExtraction.ts"; import { CORS_HEADERS } from "../utils/cors.ts"; -import { checkHeapPressureGuard } from "../utils/heapPressure.ts"; +import { checkResourcePressureGuard } from "../utils/resourcePressure.ts"; import { normalizeHeaders } from "../utils/headers.ts"; import { resolveChatCoreRequestFormat } from "./chatCore/requestFormat.ts"; import { resolveChatCoreTargetFormat } from "./chatCore/targetFormat.ts"; @@ -359,13 +359,6 @@ import { isRpmExhausted, } from "../services/geminiRateLimitTracker.ts"; -// ── Global memory pressure guard ──────────────────────────────────────── -// Prevents OOM by rejecting new requests when V8 heap exceeds threshold. -// Self-healing: no counters to leak, no cleanup needed. The threshold -// auto-calibrates to 85% of the actual V8 heap ceiling (see heapPressure.ts) so -// it tracks --max-old-space-size across 1GB/2GB/large VPS instead of a fixed -// 200MB that sat below the app's own ~260MB baseline and rejected every request. - import { isSmallEnoughForSemanticCache } from "../utils/estimateSize.ts"; /** @@ -415,17 +408,16 @@ export async function handleChatCore({ createPiiTransform = null, correlationId = null, modelPinned = false, + skipResourcePressureGuard = false, }) { let { provider, model, extendedContext } = modelInfo; - // ── Memory pressure guard ──────────────────────────────────────────── - // Reject early if V8 heap is already near the 256MB limit. Prevents - // cascading OOM when many large-context requests arrive concurrently. - try { - const heapUsedMB = process.memoryUsage().heapUsed / (1024 * 1024); - const heapGuard = checkHeapPressureGuard(heapUsedMB); - if (heapGuard) return heapGuard; - } catch { - /* memoryUsage() never throws */ + if (!skipResourcePressureGuard) { + try { + const pressureGuard = checkResourcePressureGuard(); + if (pressureGuard) return pressureGuard; + } catch { + /* fail open */ + } } // Per-request model-routing metadata (first extracted slice of the request-setup phase). diff --git a/open-sse/services/admission/adaptation.ts b/open-sse/services/admission/adaptation.ts new file mode 100644 index 0000000000..c266c4b8f6 --- /dev/null +++ b/open-sse/services/admission/adaptation.ts @@ -0,0 +1,168 @@ +import type { AdmissionPressure, AdmissionReleaseOutcome } from "./types.ts"; + +export interface AdaptationParams { + minLimit: number; + maxLimit: number; + windowMs: number; + shortLatencyAlpha: number; + longLatencyAlpha: number; + increaseStep: number; + decreaseFactor: number; + criticalDecreaseFactor: number; + highUtilizationThreshold: number; + lowUtilizationThreshold: number; + latencyGradientThreshold: number; + maxIncreasePerWindow: number; +} + +export interface AdaptationState { + currentLimit: number; + shortLatencyEwma: number; + longLatencyEwma: number; + pressure: AdmissionPressure; + /** Sum of admitted cost * time contribution proxies in the open window. */ + windowActiveCostIntegral: number; + windowCompleted: number; + windowLatencySamples: number; + windowStartMs: number; + freezeGrowth: boolean; + /** + * When true, critical multiplicative decrease already applied for this window + * (e.g. via immediate observePressure). Window close must not re-apply it. + */ + criticalDecreaseConsumed: boolean; + utilization: number; +} + +export function clampLimit(value: number, minLimit: number, maxLimit: number): number { + if (!Number.isFinite(value)) return minLimit; + return Math.min(maxLimit, Math.max(minLimit, Math.floor(value))); +} + +export function createAdaptationState( + initialLimit: number, + minLimit: number, + maxLimit: number, + nowMs: number +): AdaptationState { + return { + currentLimit: clampLimit(initialLimit, minLimit, maxLimit), + shortLatencyEwma: 0, + longLatencyEwma: 0, + pressure: "normal", + windowActiveCostIntegral: 0, + windowCompleted: 0, + windowLatencySamples: 0, + windowStartMs: nowMs, + freezeGrowth: false, + criticalDecreaseConsumed: false, + utilization: 0, + }; +} + +export function noteLatency( + state: AdaptationState, + latencyMs: number, + params: AdaptationParams +): void { + const sample = Number.isFinite(latencyMs) && latencyMs >= 0 ? latencyMs : 0; + state.windowLatencySamples += 1; + const sa = params.shortLatencyAlpha; + const la = params.longLatencyAlpha; + if (state.shortLatencyEwma <= 0 && state.longLatencyEwma <= 0) { + state.shortLatencyEwma = sample; + state.longLatencyEwma = sample; + return; + } + state.shortLatencyEwma = sa * sample + (1 - sa) * state.shortLatencyEwma; + state.longLatencyEwma = la * sample + (1 - la) * state.longLatencyEwma; +} + +export function noteOutcome(state: AdaptationState, outcome: AdmissionReleaseOutcome): void { + // A single upstream business error freezes growth for the current window; it must not + // apply critical multiplicative collapse on its own. + if (outcome === "upstream_error") { + state.freezeGrowth = true; + return; + } + if (outcome === "timeout") { + state.freezeGrowth = true; + } +} + +export function setPressure(state: AdaptationState, pressure: AdmissionPressure): void { + const severity: Record = { normal: 0, high: 1, critical: 2 }; + if (severity[pressure] > severity[state.pressure]) state.pressure = pressure; +} + +/** + * Close the current feedback window and adjust the limit. + * Recovery (increase) is slower than decrease; idle/low utilization does not inflate. + */ +export function closeAdaptationWindow( + state: AdaptationState, + params: AdaptationParams, + nowMs: number +): void { + const elapsed = Math.max(1, Math.min(params.windowMs, nowMs - state.windowStartMs)); + // sampleActiveIntegral already accounts for every interval exactly once. + const avgActive = state.windowActiveCostIntegral / elapsed; + const util = state.currentLimit > 0 ? avgActive / state.currentLimit : 0; + state.utilization = Math.max(0, Math.min(1, util)); + + let next = state.currentLimit; + const gradient = + state.longLatencyEwma > 0 + ? (state.shortLatencyEwma - state.longLatencyEwma) / state.longLatencyEwma + : 0; + + if (state.pressure === "critical") { + // Immediate observePressure may already have applied the critical factor once. + if (!state.criticalDecreaseConsumed) { + next = Math.floor(next * params.criticalDecreaseFactor); + } + } else if ( + state.pressure === "high" || + (state.windowLatencySamples > 0 && gradient >= params.latencyGradientThreshold) + ) { + next = Math.floor(next * params.decreaseFactor); + } else if ( + !state.freezeGrowth && + state.pressure === "normal" && + state.utilization >= params.highUtilizationThreshold && + state.windowCompleted > 0 + ) { + const step = Math.min(params.increaseStep, params.maxIncreasePerWindow); + next = next + step; + } + // A genuinely low-utilization window recovers the latency baseline so stale gradients expire. + if (state.utilization <= params.lowUtilizationThreshold) { + state.shortLatencyEwma = state.longLatencyEwma; + } + + state.currentLimit = clampLimit(next, params.minLimit, params.maxLimit); + state.windowActiveCostIntegral = 0; + state.windowCompleted = 0; + state.windowLatencySamples = 0; + state.windowStartMs = nowMs; + state.freezeGrowth = false; + state.criticalDecreaseConsumed = false; + state.pressure = "normal"; +} + +export function sampleActiveIntegral( + state: AdaptationState, + activeCost: number, + dtMs: number +): void { + if (dtMs <= 0 || activeCost <= 0) return; + const boundedActiveCost = Math.min(activeCost, state.currentLimit); + const contribution = + dtMs > Math.floor(Number.MAX_SAFE_INTEGER / boundedActiveCost) + ? Number.MAX_SAFE_INTEGER + : boundedActiveCost * dtMs; + state.windowActiveCostIntegral = + contribution >= Number.MAX_SAFE_INTEGER - state.windowActiveCostIntegral + ? Number.MAX_SAFE_INTEGER + : state.windowActiveCostIntegral + contribution; +} diff --git a/open-sse/services/admission/config.ts b/open-sse/services/admission/config.ts new file mode 100644 index 0000000000..dfe9b07a01 --- /dev/null +++ b/open-sse/services/admission/config.ts @@ -0,0 +1,167 @@ +import { resolveCostConfig } from "./cost.ts"; +import { + MAX_ADMISSION_COST_OR_LIMIT, + MAX_ADMISSION_WINDOW_MS, + type AdaptiveAdmissionConfig, + type AdmissionMode, +} from "./types.ts"; +import type { AdaptationParams } from "./adaptation.ts"; + +export { MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_WINDOW_MS }; + +export interface ValidatedConfig { + mode: AdmissionMode; + minLimit: number; + maxLimit: number; + initialLimit: number; + maxQueueCount: number; + maxQueueCost: number; + defaultMaxWaitMs: number; + windowMs: number; + adaptation: AdaptationParams; + maxRequestCost: number; + costConfig: ReturnType; +} + +function requirePositiveInt( + name: string, + value: unknown, + max: number = MAX_ADMISSION_COST_OR_LIMIT +): number { + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value <= 0 || + !Number.isSafeInteger(value) + ) { + throw new RangeError(`${name} must be a positive safe integer`); + } + if (value > max) { + throw new RangeError(`${name} must be <= ${max}`); + } + return value; +} + +function requireUnitInterval(name: string, value: unknown, fallback: number): number { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1) { + throw new RangeError(`${name} must be in (0, 1]`); + } + return value; +} + +function requireDecreaseFactor(name: string, value: unknown, fallback: number): number { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value >= 1) { + throw new RangeError(`${name} must be in (0, 1)`); + } + return value; +} + +function resolveMode(mode: AdaptiveAdmissionConfig["mode"]): AdmissionMode { + if (mode === undefined) return "shadow"; + if (mode !== "off" && mode !== "shadow" && mode !== "enforce") { + throw new RangeError("mode must be off|shadow|enforce"); + } + return mode; +} + +function resolveAdaptationParams( + input: AdaptiveAdmissionConfig, + minLimit: number, + maxLimit: number, + windowMs: number +): AdaptationParams { + const decreaseFactor = requireDecreaseFactor("decreaseFactor", input.decreaseFactor, 0.8); + const criticalDecreaseFactor = requireDecreaseFactor( + "criticalDecreaseFactor", + input.criticalDecreaseFactor, + 0.5 + ); + const increaseStep = + input.increaseStep === undefined ? 1 : requirePositiveInt("increaseStep", input.increaseStep); + const maxIncreasePerWindow = + input.maxIncreasePerWindow === undefined + ? increaseStep + : requirePositiveInt("maxIncreasePerWindow", input.maxIncreasePerWindow); + + const shortLatencyAlpha = requireUnitInterval("shortLatencyAlpha", input.shortLatencyAlpha, 0.5); + const longLatencyAlpha = requireUnitInterval("longLatencyAlpha", input.longLatencyAlpha, 0.1); + const highUtilizationThreshold = requireUnitInterval( + "highUtilizationThreshold", + input.highUtilizationThreshold, + 0.7 + ); + const lowUtilizationThreshold = requireUnitInterval( + "lowUtilizationThreshold", + input.lowUtilizationThreshold, + 0.3 + ); + if (criticalDecreaseFactor > decreaseFactor) { + throw new RangeError("criticalDecreaseFactor must be <= decreaseFactor"); + } + if (lowUtilizationThreshold >= highUtilizationThreshold) { + throw new RangeError("lowUtilizationThreshold must be < highUtilizationThreshold"); + } + if (shortLatencyAlpha <= longLatencyAlpha) { + throw new RangeError("shortLatencyAlpha must be > longLatencyAlpha"); + } + + return { + minLimit, + maxLimit, + windowMs, + shortLatencyAlpha, + longLatencyAlpha, + increaseStep, + decreaseFactor, + criticalDecreaseFactor, + highUtilizationThreshold, + lowUtilizationThreshold, + latencyGradientThreshold: requireUnitInterval( + "latencyGradientThreshold", + input.latencyGradientThreshold, + 0.25 + ), + maxIncreasePerWindow, + }; +} + +export function validateConfig(input: AdaptiveAdmissionConfig): ValidatedConfig { + const minLimit = requirePositiveInt("minLimit", input.minLimit); + const maxLimit = requirePositiveInt("maxLimit", input.maxLimit); + if (minLimit > maxLimit) { + throw new RangeError("minLimit must be <= maxLimit"); + } + const initialLimit = requirePositiveInt("initialLimit", input.initialLimit); + // Queue count is not multiplied into cost×time products; keep the full safe-integer range. + const maxQueueCount = requirePositiveInt( + "maxQueueCount", + input.maxQueueCount, + Number.MAX_SAFE_INTEGER + ); + const maxQueueCost = requirePositiveInt("maxQueueCost", input.maxQueueCost); + const windowMs = + input.windowMs === undefined + ? 1000 + : requirePositiveInt("windowMs", input.windowMs, MAX_ADMISSION_WINDOW_MS); + const defaultMaxWaitMs = + input.defaultMaxWaitMs === undefined + ? 5_000 + : requirePositiveInt("defaultMaxWaitMs", input.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS); + const costConfig = resolveCostConfig(input.cost); + + return { + mode: resolveMode(input.mode), + minLimit, + maxLimit, + initialLimit, + maxQueueCount, + maxQueueCost, + defaultMaxWaitMs, + windowMs, + maxRequestCost: costConfig.maxRequestCost, + costConfig, + adaptation: resolveAdaptationParams(input, minLimit, maxLimit, windowMs), + }; +} diff --git a/open-sse/services/admission/controller.ts b/open-sse/services/admission/controller.ts new file mode 100644 index 0000000000..1051a64782 --- /dev/null +++ b/open-sse/services/admission/controller.ts @@ -0,0 +1,624 @@ +import { + closeAdaptationWindow, + createAdaptationState, + noteLatency, + noteOutcome, + sampleActiveIntegral, + setPressure, + type AdaptationState, +} from "./adaptation.ts"; +import { validateConfig, type ValidatedConfig } from "./config.ts"; +import { estimateAdmissionCost, normalizeRequestCost } from "./cost.ts"; +import { FairCostQueue, type QueueEntry } from "./queue.ts"; +import { + MAX_ADMISSION_WINDOW_MS, + createAdmissionRejectError, + type AdaptiveAdmissionConfig, + type AdmissionAcquireResult, + type AdmissionAdmitted, + type AdmissionClock, + type AdmissionLease, + type AdmissionPressure, + type AdmissionRejectCode, + type AdmissionReleaseMeta, + type AdmissionReleaseOutcome, + type AdmissionRequest, + type AdmissionSnapshot, + type ShadowDecision, +} from "./types.ts"; + +type VirtualDisposition = "active" | "queued" | "rejected" | "none"; + +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); + +/** Snapshot numbers are always finite safe integers; never emit rounded unsafe Number values. */ +function saturateSnapshotNumber(value: number): number { + if (!Number.isFinite(value) || value <= 0) return 0; + if (value >= Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER; + return Math.floor(value); +} + +function bigintToSnapshotNumber(value: bigint): number { + if (value <= 0n) return 0; + if (value >= MAX_SAFE_BIGINT) return Number.MAX_SAFE_INTEGER; + return Number(value); +} + +function addSaturated(total: number, delta: number): number { + if (delta <= 0) return saturateSnapshotNumber(total); + if (total >= Number.MAX_SAFE_INTEGER - delta) return Number.MAX_SAFE_INTEGER; + return total + delta; +} + +interface ActiveLeaseRecord { + id: string; + cost: number; + released: boolean; + admittedAtMs: number; + virtualDisposition: VirtualDisposition; +} + +interface QueuedPayload { + resolve: (value: AdmissionAdmitted) => void; + reject: (err: Error) => void; + signal?: AbortSignal; + onAbort?: () => void; +} + +let leaseSeq = 0; + +function nextId(prefix: string): string { + leaseSeq += 1; + return `${prefix}-${leaseSeq}`; +} + +function defaultClock(): AdmissionClock { + return { + now: () => Date.now(), + setTimer: (fn, delayMs) => { + const handle = setTimeout(fn, delayMs); + // Window/deadline timers must not pin the event loop open when idle. + if (typeof handle.unref === "function") handle.unref(); + return handle; + }, + clearTimer: (id) => clearTimeout(id as ReturnType), + }; +} + +/** + * Dependency-injected weighted adaptive admission controller. + * Pure in-process core: no env/settings/route wiring. + */ +export class AdaptiveAdmissionController { + private config: ValidatedConfig; + private readonly clock: AdmissionClock; + private adaptation: AdaptationState; + private queue: FairCostQueue; + private virtualQueue: FairCostQueue<{ recordId: string }>; + private readonly active = new Map(); + private activeCost = 0n; + private virtualActiveCost = 0; + private virtualActiveCount = 0; + private lastSampleMs: number; + private windowTimer: unknown = undefined; + private shutDown = false; + + private admittedCount = 0; + private rejectedCount = 0; + private wouldAdmitCount = 0; + private wouldQueueCount = 0; + private wouldRejectCount = 0; + + constructor(config: AdaptiveAdmissionConfig, clock?: Partial) { + this.config = validateConfig(config); + this.clock = { + now: clock?.now ?? defaultClock().now, + setTimer: clock?.setTimer ?? defaultClock().setTimer, + clearTimer: clock?.clearTimer ?? defaultClock().clearTimer, + }; + const now = this.clock.now(); + this.adaptation = createAdaptationState( + this.config.initialLimit, + this.config.minLimit, + this.config.maxLimit, + now + ); + this.queue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost); + this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost); + this.lastSampleMs = now; + this.armWindowTimer(); + } + + updateConfig(config: AdaptiveAdmissionConfig): void { + const next = validateConfig(config); + this.sampleIntegral(); + this.config = next; + this.adaptation.currentLimit = Math.min( + next.maxLimit, + Math.max(next.minLimit, this.adaptation.currentLimit) + ); + this.adaptation.windowStartMs = this.clock.now(); + this.adaptation.windowActiveCostIntegral = 0; + this.adaptation.windowCompleted = 0; + this.adaptation.windowLatencySamples = 0; + this.adaptation.freezeGrowth = false; + this.adaptation.criticalDecreaseConsumed = false; + this.adaptation.pressure = "normal"; + this.lastSampleMs = this.clock.now(); + + const drained = this.queue.drain(); + this.queue = new FairCostQueue(next.maxQueueCount, next.maxQueueCost); + for (const entry of drained) { + if (next.mode !== "enforce") { + this.clearEntryTimer(entry); + this.detachAbort(entry); + entry.payload.resolve(this.admit(entry.cost)); + continue; + } + // Cost above the new enforce limit must fail closed immediately, never strand until deadline. + if (entry.cost > this.adaptation.currentLimit) { + this.failQueued( + entry, + "ADMISSION_OVERSIZED", + "request cost exceeds max budget after config update" + ); + continue; + } + if (!this.queue.enqueue(entry)) { + this.failQueued(entry, "ADMISSION_QUEUE_FULL", "queue capacity reduced"); + } + } + + this.rebuildVirtualState(next.mode === "shadow"); + this.armWindowTimer(); + if (next.mode === "enforce") { + this.dispatch(); + } + } + + snapshot(): AdmissionSnapshot { + this.sampleIntegral(); + return { + mode: this.config.mode, + currentLimit: this.adaptation.currentLimit, + minLimit: this.config.minLimit, + maxLimit: this.config.maxLimit, + activeCost: bigintToSnapshotNumber(this.activeCost), + activeCount: saturateSnapshotNumber(this.active.size), + queuedCost: saturateSnapshotNumber(this.queue.totalCost), + queuedCount: saturateSnapshotNumber(this.queue.size), + virtualActiveCost: saturateSnapshotNumber(this.virtualActiveCost), + virtualActiveCount: saturateSnapshotNumber(this.virtualActiveCount), + virtualQueuedCost: saturateSnapshotNumber(this.virtualQueue.totalCost), + virtualQueuedCount: saturateSnapshotNumber(this.virtualQueue.size), + admittedCount: saturateSnapshotNumber(this.admittedCount), + rejectedCount: saturateSnapshotNumber(this.rejectedCount), + wouldAdmitCount: saturateSnapshotNumber(this.wouldAdmitCount), + wouldQueueCount: saturateSnapshotNumber(this.wouldQueueCount), + wouldRejectCount: saturateSnapshotNumber(this.wouldRejectCount), + shortLatencyEwma: this.adaptation.shortLatencyEwma, + longLatencyEwma: this.adaptation.longLatencyEwma, + utilization: this.adaptation.utilization, + pressure: this.adaptation.pressure, + shutdown: this.shutDown, + }; + } + + observePressure(pressure: AdmissionPressure): void { + setPressure(this.adaptation, pressure); + if (pressure === "critical") { + // Immediate fast decrease once per window; window close must not re-apply it. + if (!this.adaptation.criticalDecreaseConsumed) { + this.adaptation.currentLimit = Math.max( + this.config.minLimit, + Math.floor(this.adaptation.currentLimit * this.config.adaptation.criticalDecreaseFactor) + ); + this.adaptation.criticalDecreaseConsumed = true; + this.dispatch(); + this.dispatchVirtual(); + } + } + } + + /** Deterministic window tick for tests / injected clocks. */ + tick(): void { + this.sampleIntegral(); + closeAdaptationWindow(this.adaptation, this.config.adaptation, this.clock.now()); + // Real queue first, then virtual: raised limits must promote shadow-queued work + // before newer arrivals are classified against the updated budget. + this.dispatch(); + this.dispatchVirtual(); + } + + async acquire(request: AdmissionRequest): Promise { + if (this.shutDown) { + return this.reject("ADMISSION_SHUTDOWN", "admission controller is shut down"); + } + + if (request.signal?.aborted) { + return this.reject("ADMISSION_ABORTED", "request aborted before acquire"); + } + + if (request.pressure) setPressure(this.adaptation, request.pressure); + + const cost = this.resolveCost(request); + const mode = this.config.mode; + + if (mode === "off") { + return this.admitVirtual(cost); + } + + const limit = this.adaptation.currentLimit; + + if (mode === "shadow") { + return this.acquireShadow(request, cost, limit); + } + + // enforce + if (cost > limit) { + return this.reject("ADMISSION_OVERSIZED", "request cost exceeds max budget"); + } + + // Once work is queued, every newer request joins the same fair queue even if it + // currently fits. This makes bounded bypass accounting effective and prevents + // direct arrivals from indefinitely jumping an older reserved weighted request. + if (this.queue.size === 0 && this.activeCost + BigInt(cost) <= BigInt(limit)) { + return this.admit(cost); + } + + if (!this.queue.canAccept(cost)) { + return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full"); + } + + return this.enqueue(request, cost); + } + + shutdown(): void { + if (this.shutDown) return; + this.shutDown = true; + if (this.windowTimer !== undefined) { + this.clock.clearTimer(this.windowTimer); + this.windowTimer = undefined; + } + const drained = this.queue.drain(); + for (const entry of drained) { + this.clearEntryTimer(entry); + this.detachAbort(entry); + entry.payload.reject( + createAdmissionRejectError("ADMISSION_SHUTDOWN", "admission controller shut down") + ); + this.rejectedCount += 1; + } + } + + private resolveCost(request: AdmissionRequest): number { + if (request.cost !== undefined) { + return normalizeRequestCost(request.cost, this.config.maxRequestCost); + } + if (request.features) { + return estimateAdmissionCost(request.features, this.config.costConfig); + } + return 1; + } + + private acquireShadow(request: AdmissionRequest, cost: number, limit: number): AdmissionAdmitted { + let decision: ShadowDecision; + let disposition: VirtualDisposition; + if (cost > limit || !Number.isSafeInteger(cost)) { + decision = "would-reject"; + disposition = "rejected"; + this.wouldRejectCount += 1; + } else if (this.virtualActiveCost + cost <= limit) { + decision = "would-admit"; + disposition = "active"; + this.virtualActiveCost = addSaturated(this.virtualActiveCost, cost); + this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1); + this.wouldAdmitCount = addSaturated(this.wouldAdmitCount, 1); + } else if (this.virtualQueue.canAccept(cost)) { + decision = "would-queue"; + disposition = "queued"; + this.wouldQueueCount += 1; + } else { + decision = "would-reject"; + disposition = "rejected"; + this.wouldRejectCount += 1; + } + + const admitted = this.admit(cost, disposition); + if (disposition === "queued") { + this.virtualQueue.enqueue({ + id: admitted.lease.id, + tenantKey: request.tenantKey || "_default", + cost, + enqueuedAtMs: this.clock.now(), + deadlineMs: Number.MAX_SAFE_INTEGER, + payload: { recordId: admitted.lease.id }, + }); + } + return { ...admitted, shadowDecision: decision }; + } + + private admitVirtual(cost: number): AdmissionAdmitted { + // Mode off: no accounting. + const id = nextId("lease"); + const lease: AdmissionLease = { + id, + cost, + get released() { + return true; + }, + release: () => { + /* no-op */ + }, + }; + this.admittedCount += 1; + return { status: "admitted", lease }; + } + + private admit(cost: number, virtualDisposition: VirtualDisposition = "none"): AdmissionAdmitted { + this.sampleIntegral(); + const id = nextId("lease"); + const record: ActiveLeaseRecord = { + id, + cost, + released: false, + admittedAtMs: this.clock.now(), + virtualDisposition, + }; + this.active.set(id, record); + this.activeCost += BigInt(cost); + this.admittedCount += 1; + + const controller = this; + const lease: AdmissionLease = { + id, + cost, + get released() { + return record.released; + }, + release(outcome: AdmissionReleaseOutcome = "success", meta?: AdmissionReleaseMeta) { + controller.releaseLease(record, outcome, meta); + }, + }; + return { status: "admitted", lease }; + } + + private releaseLease( + record: ActiveLeaseRecord, + outcome: AdmissionReleaseOutcome, + meta?: AdmissionReleaseMeta + ): void { + if (record.released) return; + record.released = true; + // Sample while the lease still contributes to activeCost so utilization EWMA sees load. + this.sampleIntegral(); + if (this.active.has(record.id)) { + this.active.delete(record.id); + this.activeCost -= BigInt(record.cost); + } + + const latency = + meta?.latencyMs !== undefined + ? meta.latencyMs + : Math.max(0, this.clock.now() - record.admittedAtMs); + noteLatency(this.adaptation, latency, this.config.adaptation); + noteOutcome(this.adaptation, outcome); + this.adaptation.windowCompleted += 1; + if (meta?.pressure) setPressure(this.adaptation, meta.pressure); + this.releaseVirtual(record); + + this.dispatch(); + } + + private enqueue(request: AdmissionRequest, cost: number): AdmissionAcquireResult { + const id = nextId("q"); + const maxWait = normalizeRequestCost( + request.maxWaitMs ?? this.config.defaultMaxWaitMs, + MAX_ADMISSION_WINDOW_MS + ); + const now = this.clock.now(); + const deadlineMs = Math.min(Number.MAX_SAFE_INTEGER, now + maxWait); + + let settle: { + resolve: (v: AdmissionAdmitted) => void; + reject: (e: Error) => void; + }; + const promise = new Promise((resolve, reject) => { + settle = { resolve, reject }; + }); + + const entry: QueueEntry = { + id, + tenantKey: request.tenantKey && request.tenantKey.length > 0 ? request.tenantKey : "_default", + cost, + enqueuedAtMs: now, + deadlineMs, + payload: { + resolve: (v) => settle.resolve(v), + reject: (e) => settle.reject(e), + signal: request.signal, + }, + }; + + if (!this.queue.enqueue(entry)) { + return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full"); + } + + entry.timerId = this.clock.setTimer( + () => { + this.expireEntry(id, "ADMISSION_DEADLINE", "admission wait deadline exceeded"); + }, + Math.max(0, deadlineMs - now) + ); + + if (request.signal) { + const onAbort = () => { + this.expireEntry(id, "ADMISSION_ABORTED", "request aborted while queued"); + }; + entry.payload.onAbort = onAbort; + request.signal.addEventListener("abort", onAbort, { once: true }); + } + + // Capacity may have freed between check and enqueue in concurrent hosts; try dispatch. + this.dispatch(); + + return { status: "queued", promise }; + } + + private expireEntry(id: string, code: AdmissionRejectCode, message: string): void { + const entry = this.queue.removeById(id); + if (!entry) return; + this.clearEntryTimer(entry); + this.detachAbort(entry); + entry.payload.reject(createAdmissionRejectError(code, message)); + this.rejectedCount += 1; + // Resume enforce dispatch so a now-fitting successor is not stranded until + // unrelated activity. dispatch() is a no-op after shutdown / non-enforce. + this.dispatch(); + } + + private failQueued( + entry: QueueEntry, + code: AdmissionRejectCode, + message: string + ): void { + this.clearEntryTimer(entry); + this.detachAbort(entry); + entry.payload.reject(createAdmissionRejectError(code, message)); + this.rejectedCount += 1; + } + + private dispatch(): void { + if (this.shutDown || this.config.mode !== "enforce") return; + + while (this.queue.size > 0) { + const limit = this.adaptation.currentLimit; + const available = BigInt(limit) - this.activeCost; + if (available <= 0n) return; + const entry = this.queue.dequeue(Number(available)); + if (!entry) return; + this.clearEntryTimer(entry); + this.detachAbort(entry); + if (entry.payload.signal?.aborted) { + entry.payload.reject( + createAdmissionRejectError("ADMISSION_ABORTED", "request aborted while queued") + ); + this.rejectedCount += 1; + continue; + } + if (this.clock.now() >= entry.deadlineMs) { + entry.payload.reject( + createAdmissionRejectError("ADMISSION_DEADLINE", "admission wait deadline exceeded") + ); + this.rejectedCount += 1; + continue; + } + entry.payload.resolve(this.admit(entry.cost)); + } + } + + private releaseVirtual(record: ActiveLeaseRecord): void { + if (record.virtualDisposition === "active") { + this.virtualActiveCost -= record.cost; + this.virtualActiveCount -= 1; + } else if (record.virtualDisposition === "queued") { + this.virtualQueue.removeById(record.id); + } + record.virtualDisposition = "none"; + this.dispatchVirtual(); + } + + private dispatchVirtual(): void { + while (this.virtualQueue.size > 0) { + const available = this.adaptation.currentLimit - this.virtualActiveCost; + if (available <= 0) return; + const entry = this.virtualQueue.dequeue(available); + if (!entry) return; + const record = this.active.get(entry.payload.recordId); + if (!record || record.released) continue; + record.virtualDisposition = "active"; + this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost); + this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1); + } + } + + private rebuildVirtualState(enable: boolean): void { + this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost); + this.virtualActiveCost = 0; + this.virtualActiveCount = 0; + for (const record of this.active.values()) record.virtualDisposition = "none"; + if (!enable) return; + for (const record of this.active.values()) { + // Individually oversized work is virtual-rejected, never virtually queued. + if (record.cost > this.adaptation.currentLimit) { + record.virtualDisposition = "rejected"; + continue; + } + if (record.cost <= this.adaptation.currentLimit - this.virtualActiveCost) { + record.virtualDisposition = "active"; + this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost); + this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1); + } else if ( + this.virtualQueue.enqueue({ + id: record.id, + tenantKey: "_existing", + cost: record.cost, + enqueuedAtMs: record.admittedAtMs, + deadlineMs: Number.MAX_SAFE_INTEGER, + payload: { recordId: record.id }, + }) + ) { + record.virtualDisposition = "queued"; + } else { + record.virtualDisposition = "rejected"; + } + } + } + + private reject(code: AdmissionRejectCode, message: string): AdmissionAcquireResult { + this.rejectedCount += 1; + return { status: "rejected", code, message }; + } + + private clearEntryTimer(entry: QueueEntry): void { + if (entry.timerId !== undefined) { + this.clock.clearTimer(entry.timerId); + entry.timerId = undefined; + } + } + + private detachAbort(entry: QueueEntry): void { + if (entry.payload.signal && entry.payload.onAbort) { + entry.payload.signal.removeEventListener("abort", entry.payload.onAbort); + entry.payload.onAbort = undefined; + } + } + + private sampleIntegral(): void { + const now = this.clock.now(); + const dt = now - this.lastSampleMs; + if (dt > 0) { + // Cap at currentLimit before Number conversion so shadow oversubscription never + // feeds an unsafe rounded activeCost into the utilization integral. + const limit = this.adaptation.currentLimit; + const activeForIntegral = this.activeCost >= BigInt(limit) ? limit : Number(this.activeCost); + sampleActiveIntegral(this.adaptation, activeForIntegral, dt); + this.lastSampleMs = now; + } + } + + private armWindowTimer(): void { + if (this.windowTimer !== undefined) { + this.clock.clearTimer(this.windowTimer); + this.windowTimer = undefined; + } + if (this.shutDown || this.config.mode === "off") return; + const tick = () => { + this.tick(); + if (!this.shutDown && this.config.mode !== "off") { + this.windowTimer = this.clock.setTimer(tick, this.config.windowMs); + } + }; + this.windowTimer = this.clock.setTimer(tick, this.config.windowMs); + } +} diff --git a/open-sse/services/admission/cost.ts b/open-sse/services/admission/cost.ts new file mode 100644 index 0000000000..7d915aa919 --- /dev/null +++ b/open-sse/services/admission/cost.ts @@ -0,0 +1,107 @@ +import { + MAX_ADMISSION_COST_OR_LIMIT, + type AdmissionCostConfig, + type AdmissionCostFeatures, +} from "./types.ts"; + +export { MAX_ADMISSION_COST_OR_LIMIT }; + +export const DEFAULT_ADMISSION_COST_CONFIG: AdmissionCostConfig = Object.freeze({ + baseCost: 1, + bodyBytesPerUnit: 16_384, + tokensPerUnit: 1_024, + messagesPerUnit: 32, + toolsPerUnit: 8, + fanoutPerUnit: 1, + streamingClassCost: 1, + nonStreamingClassCost: 2, + maxRequestCost: 1_000, +}); + +function finiteNonNegative(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0; + return Math.min(value, Number.MAX_SAFE_INTEGER); +} + +function requirePositiveSafeInteger( + name: string, + value: unknown, + max: number = MAX_ADMISSION_COST_OR_LIMIT +): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) { + throw new RangeError(`${name} must be a positive safe integer`); + } + if (value > max) { + throw new RangeError(`${name} must be <= ${max}`); + } + return value; +} + +const COST_CONFIG_KEYS = [ + "baseCost", + "bodyBytesPerUnit", + "tokensPerUnit", + "messagesPerUnit", + "toolsPerUnit", + "fanoutPerUnit", + "streamingClassCost", + "nonStreamingClassCost", + "maxRequestCost", +] as const satisfies ReadonlyArray; + +/** Merge cost quanta after strictly validating every supplied value. */ +export function resolveCostConfig(partial?: Partial): AdmissionCostConfig { + const d = DEFAULT_ADMISSION_COST_CONFIG; + const resolved = {} as AdmissionCostConfig; + for (const key of COST_CONFIG_KEYS) { + resolved[key] = requirePositiveSafeInteger(key, partial?.[key] ?? d[key]); + } + return resolved; +} + +function unitsFrom(amount: number, quantum: number): number { + return amount <= 0 ? 0 : Math.ceil(amount / quantum); +} + +function addBounded(total: number, contribution: number, maximum: number): number { + if (contribution >= maximum - total) return maximum; + return total + contribution; +} + +/** Pure bounded cost estimator from transparent positive safe-integer quanta. */ +export function estimateAdmissionCost( + features: AdmissionCostFeatures, + config?: Partial +): number { + const cfg = resolveCostConfig(config); + const body = finiteNonNegative(features?.bodyBytes); + const tokens = finiteNonNegative(features?.estimatedInputTokens); + const messages = finiteNonNegative(features?.messageCount); + const tools = finiteNonNegative(features?.toolCount); + const fanout = Math.max(1, finiteNonNegative(features?.requestedFanout)); + const contributions = [ + unitsFrom(body, cfg.bodyBytesPerUnit), + unitsFrom(tokens, cfg.tokensPerUnit), + unitsFrom(messages, cfg.messagesPerUnit), + unitsFrom(tools, cfg.toolsPerUnit), + unitsFrom(fanout, cfg.fanoutPerUnit), + features?.streaming !== false ? cfg.streamingClassCost : cfg.nonStreamingClassCost, + ]; + + let total = Math.min(cfg.baseCost, cfg.maxRequestCost); + for (const contribution of contributions) { + total = addBounded(total, contribution, cfg.maxRequestCost); + if (total === cfg.maxRequestCost) break; + } + return total; +} + +/** Validate and bound a caller-supplied request cost. */ +export function normalizeRequestCost( + cost: unknown, + maxRequestCost: number = DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost +): number { + const max = requirePositiveSafeInteger("maxRequestCost", maxRequestCost); + const value = requirePositiveSafeInteger("request cost", cost); + return Math.min(value, max); +} diff --git a/open-sse/services/admission/index.ts b/open-sse/services/admission/index.ts new file mode 100644 index 0000000000..48c3a5ad47 --- /dev/null +++ b/open-sse/services/admission/index.ts @@ -0,0 +1,37 @@ +/** + * Pure weighted adaptive admission-control core. + * No route, settings, or environment wiring in this module surface. + */ + +export { + DEFAULT_ADMISSION_COST_CONFIG, + estimateAdmissionCost, + normalizeRequestCost, + resolveCostConfig, +} from "./cost.ts"; + +export { AdaptiveAdmissionController } from "./controller.ts"; + +export { + MAX_ADMISSION_COST_OR_LIMIT, + MAX_ADMISSION_WINDOW_MS, + createAdmissionRejectError, + type AdaptiveAdmissionConfig, + type AdmissionAcquireResult, + type AdmissionAdmitted, + type AdmissionClock, + type AdmissionCostConfig, + type AdmissionCostFeatures, + type AdmissionLease, + type AdmissionMode, + type AdmissionPressure, + type AdmissionQueued, + type AdmissionRejectCode, + type AdmissionRejectError, + type AdmissionRejected, + type AdmissionReleaseMeta, + type AdmissionReleaseOutcome, + type AdmissionRequest, + type AdmissionSnapshot, + type ShadowDecision, +} from "./types.ts"; diff --git a/open-sse/services/admission/queue.ts b/open-sse/services/admission/queue.ts new file mode 100644 index 0000000000..086a88d08c --- /dev/null +++ b/open-sse/services/admission/queue.ts @@ -0,0 +1,194 @@ +/** + * Bounded multi-tenant fair queue (round-robin across tenant buckets). + * Count + total cost caps; no unbounded arrays of timers beyond one per entry. + */ + +/** + * After this many pass-overs while unfittable, reserve capacity for the aged head + * instead of indefinitely admitting smaller work from other tenants. + */ +const MAX_UNFITTABLE_SKIPS = 2; + +export interface QueueEntry { + id: string; + tenantKey: string; + cost: number; + enqueuedAtMs: number; + deadlineMs: number; + payload: T; + timerId?: unknown; + /** Times this head was skipped because it did not fit available cost. */ + skipCount?: number; +} + +export interface FairQueueSnapshot { + count: number; + cost: number; +} + +export class FairCostQueue { + private readonly buckets = new Map[]>(); + private readonly order: string[] = []; + private cursor = 0; + private count = 0; + private cost = 0; + + constructor( + readonly maxCount: number, + readonly maxCost: number + ) {} + + get size(): number { + return this.count; + } + + get totalCost(): number { + return this.cost; + } + + snapshot(): FairQueueSnapshot { + return { count: this.count, cost: this.cost }; + } + + canAccept(entryCost: number): boolean { + if (!Number.isSafeInteger(entryCost) || entryCost <= 0) return false; + if (this.count >= this.maxCount) return false; + if (entryCost > this.maxCost - this.cost) return false; + return true; + } + + enqueue(entry: QueueEntry): boolean { + if (!this.canAccept(entry.cost)) return false; + let bucket = this.buckets.get(entry.tenantKey); + if (!bucket) { + bucket = []; + this.buckets.set(entry.tenantKey, bucket); + this.order.push(entry.tenantKey); + } + bucket.push(entry); + this.count += 1; + this.cost += entry.cost; + return true; + } + + /** + * Round-robin dequeue, optionally skipping tenant heads that do not fit available cost. + * After MAX_UNFITTABLE_SKIPS actual pass-overs, an unfittable head reserves capacity: + * smaller work is not admitted ahead of it until it fits, is removed, or capacity rises. + */ + dequeue(maxCost = Number.MAX_SAFE_INTEGER): QueueEntry | undefined { + if (this.count === 0) return undefined; + const n = this.order.length; + + // Bounded anti-starvation: prefer the oldest aged unfittable head once reserved. + let reserved: { idx: number; entry: QueueEntry } | undefined; + for (let i = 0; i < n; i++) { + const idx = (this.cursor + i) % n; + const tenant = this.order[idx]; + const entry = this.buckets.get(tenant)?.[0]; + if (!entry) continue; + if ((entry.skipCount ?? 0) >= MAX_UNFITTABLE_SKIPS) { + if (!reserved || entry.enqueuedAtMs < reserved.entry.enqueuedAtMs) { + reserved = { idx, entry }; + } + } + } + if (reserved) { + if (reserved.entry.cost > maxCost) return undefined; + return this.takeAt(reserved.idx); + } + + const bypassed: QueueEntry[] = []; + for (let i = 0; i < n; i++) { + const idx = (this.cursor + i) % n; + const tenant = this.order[idx]; + const bucket = this.buckets.get(tenant); + const entry = bucket?.[0]; + if (!entry) continue; + if (entry.cost > maxCost) { + bypassed.push(entry); + continue; + } + // Only an actual smaller admission counts as a pass-over. Merely polling + // with no available capacity must not age a head into reservation. + for (const skipped of bypassed) { + skipped.skipCount = (skipped.skipCount ?? 0) + 1; + } + return this.takeAt(idx); + } + return undefined; + } + + private takeAt(idx: number): QueueEntry | undefined { + const tenant = this.order[idx]; + const bucket = this.buckets.get(tenant); + const entry = bucket?.[0]; + if (!entry) return undefined; + bucket!.shift(); + this.count -= 1; + this.cost -= entry.cost; + entry.skipCount = 0; + if (bucket!.length === 0) { + this.buckets.delete(tenant); + this.order.splice(idx, 1); + this.cursor = this.order.length === 0 ? 0 : idx % this.order.length; + } else { + this.cursor = (idx + 1) % this.order.length; + } + return entry; + } + + /** Peek next without removing (for oversized-vs-limit checks). */ + peek(): QueueEntry | undefined { + if (this.count === 0) return undefined; + const n = this.order.length; + for (let i = 0; i < n; i++) { + const idx = (this.cursor + i) % n; + const tenant = this.order[idx]; + const bucket = this.buckets.get(tenant); + if (bucket && bucket.length > 0) return bucket[0]; + } + return undefined; + } + + removeById(id: string): QueueEntry | undefined { + for (let ti = 0; ti < this.order.length; ti++) { + const tenant = this.order[ti]; + const bucket = this.buckets.get(tenant); + if (!bucket) continue; + const idx = bucket.findIndex((e) => e.id === id); + if (idx < 0) continue; + const [entry] = bucket.splice(idx, 1); + this.count -= 1; + this.cost -= entry.cost; + if (bucket.length === 0) { + this.buckets.delete(tenant); + this.order.splice(ti, 1); + if (this.order.length === 0) { + this.cursor = 0; + } else if (ti < this.cursor) { + // Removing a prior bucket shifts the successor into cursor - 1. + this.cursor -= 1; + } else if (this.cursor >= this.order.length) { + // Removed the final bucket at the cursor; wrap to the head. + this.cursor = 0; + } + // ti === cursor: leave cursor so it now points at the logical successor. + // ti > cursor: cursor is unaffected. + } + return entry; + } + return undefined; + } + + drain(): QueueEntry[] { + const out: QueueEntry[] = []; + while (true) { + const e = this.dequeue(); + if (!e) break; + out.push(e); + } + this.cursor = 0; + return out; + } +} diff --git a/open-sse/services/admission/requestFeatures.ts b/open-sse/services/admission/requestFeatures.ts new file mode 100644 index 0000000000..0116a1e43b --- /dev/null +++ b/open-sse/services/admission/requestFeatures.ts @@ -0,0 +1,186 @@ +/** + * Cheap bounded admission cost features from an already-parsed request body. + * Never re-parses, stringifies, clones, or invokes toJSON. + */ + +import { estimateSizeFast } from "../../utils/estimateSize.ts"; +import type { AdmissionCostFeatures } from "./types.ts"; + +export type AdmissionFeatureExtractionContext = { + /** When set, wins over any body/wrapped stream field. */ + streaming?: boolean; +}; + +/** + * Max tools/functions array entries inspected. + * Uninspected tail is charged conservatively so truncation cannot undercharge cost. + */ +export const ADMISSION_TOOL_SCAN_BUDGET = 64; + +type FeatureDraft = { + messageCount: number; + toolCount: number; + requestedFanout: number | null; + streaming: boolean | null; +}; + +function isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function asArray(value: unknown): unknown[] | null { + return Array.isArray(value) ? value : null; +} + +function positiveInt(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null; + if (!Number.isSafeInteger(value)) { + return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(value)); + } + return value; +} + +function saturateCount(n: number): number { + if (!Number.isFinite(n) || n <= 0) return 0; + if (!Number.isSafeInteger(n)) { + return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(n)); + } + return n; +} + +/** + * Count all recognized tool aliases/layers under one shared entry budget. + * If their combined length cannot be inspected completely, saturate before indexed access + * so an unseen alias or wrapped tail cannot undercharge heavier declarations. + */ +function countTools(layers: Array>): number { + const sources: unknown[][] = []; + const seen = new Set(); + for (const layer of layers) { + for (const value of [layer.tools, layer.functions]) { + const source = asArray(value); + if (!source || seen.has(source)) continue; + seen.add(source); + sources.push(source); + } + } + + let entryCount = 0; + for (const source of sources) { + if (source.length > ADMISSION_TOOL_SCAN_BUDGET - entryCount) { + return Number.MAX_SAFE_INTEGER; + } + entryCount += source.length; + } + + let total = 0; + for (const source of sources) { + for (let i = 0; i < source.length; i++) { + const entry = source[i]; + if (isPlainObject(entry)) { + const declarations = asArray(entry.functionDeclarations); + if (declarations) { + total = Math.min(Number.MAX_SAFE_INTEGER, total + saturateCount(declarations.length)); + continue; + } + } + total = Math.min(Number.MAX_SAFE_INTEGER, total + 1); + } + } + return total; +} + +function countMessages(layer: Record): number { + const messages = asArray(layer.messages); + const contents = asArray(layer.contents); + const inputArr = asArray(layer.input); + let count = Math.max( + saturateCount(messages?.length ?? 0), + saturateCount(contents?.length ?? 0), + saturateCount(inputArr?.length ?? 0) + ); + // Responses API: non-empty string `input` is one input item. + if (count === 0 && typeof layer.input === "string" && layer.input.length > 0) { + count = 1; + } + return count; +} + +function readFanout(layer: Record): number | null { + const direct = + positiveInt(layer.n) ?? positiveInt(layer.candidateCount) ?? positiveInt(layer.candidate_count); + if (direct != null) return direct; + // Known nested Gemini/Antigravity shape only — no recursive walk. + if (isPlainObject(layer.generationConfig)) { + return ( + positiveInt(layer.generationConfig.candidateCount) ?? + positiveInt(layer.generationConfig.candidate_count) + ); + } + return null; +} + +function featureLayers(body: unknown): Array> { + const top = isPlainObject(body) ? body : null; + const wrapped = top && isPlainObject(top.request) ? top.request : null; + const layers: Array> = []; + if (top) layers.push(top); + if (wrapped) layers.push(wrapped); + return layers; +} + +function absorbLayer(draft: FeatureDraft, layer: Record): void { + if (draft.messageCount === 0) { + draft.messageCount = countMessages(layer); + } + if (draft.requestedFanout == null) { + draft.requestedFanout = readFanout(layer); + } + if (draft.streaming == null && "stream" in layer) { + draft.streaming = layer.stream === true; + } +} + +function resolveStreaming( + draftStreaming: boolean | null, + context?: AdmissionFeatureExtractionContext +): boolean { + if (context && "streaming" in context && context.streaming !== undefined) { + return context.streaming === true; + } + return draftStreaming ?? false; +} + +/** + * Inspect top-level fields and one known wrapper (`request`) only. + * Prefer the first non-empty match for each feature family. + */ +export function extractAdmissionCostFeatures( + body: unknown, + context?: AdmissionFeatureExtractionContext +): AdmissionCostFeatures { + const bodyBytes = estimateSizeFast(body); + const layers = featureLayers(body); + const draft: FeatureDraft = { + messageCount: 0, + toolCount: countTools(layers), + requestedFanout: null, + streaming: null, + }; + for (const layer of layers) { + absorbLayer(draft, layer); + } + + // Conservative token estimate from already-measured body size (no re-walk/stringify). + const estimatedInputTokens = + bodyBytes > 0 ? Math.min(Number.MAX_SAFE_INTEGER, Math.ceil(bodyBytes / 4)) : 0; + + return { + bodyBytes, + estimatedInputTokens, + messageCount: draft.messageCount, + toolCount: draft.toolCount, + requestedFanout: draft.requestedFanout ?? 1, + streaming: resolveStreaming(draft.streaming, context), + }; +} diff --git a/open-sse/services/admission/runtime.ts b/open-sse/services/admission/runtime.ts new file mode 100644 index 0000000000..ee0e10ec93 --- /dev/null +++ b/open-sse/services/admission/runtime.ts @@ -0,0 +1,614 @@ +/** + * Process-local adaptive admission runtime facade around the pure controller. + * No HTTP route wiring — suitable for later shared handleChat integration. + */ + +import { AdaptiveAdmissionController } from "./controller.ts"; +import { validateConfig } from "./config.ts"; +import { extractAdmissionCostFeatures } from "./requestFeatures.ts"; +import { + type AdaptiveAdmissionConfig, + type AdmissionAcquireResult, + type AdmissionClock, + type AdmissionLease, + type AdmissionMode, + type AdmissionPressure, + type AdmissionRejectCode, + type AdmissionReleaseOutcome, + type AdmissionSnapshot, + type ShadowDecision, +} from "./types.ts"; +import { buildErrorBody } from "../../utils/error.ts"; +import { CORS_HEADERS } from "../../utils/cors.ts"; +import { + checkResourcePressureGuard, + getResourcePressureObservation, + type ResourcePressureGuardResult, + type ResourcePressureObservation, +} from "../../utils/resourcePressure.ts"; +import type { PressureReason, PressureSeverity } from "../../utils/resourcePressurePolicy.ts"; + +export { extractAdmissionCostFeatures } from "./requestFeatures.ts"; + +export const DEFAULT_ADAPTIVE_ADMISSION_CONFIG: Readonly = Object.freeze({ + mode: "shadow", + minLimit: 8, + initialLimit: 64, + maxLimit: 1000, + maxQueueCount: 128, + maxQueueCost: 2000, + defaultMaxWaitMs: 5_000, + windowMs: 1_000, +}); + +const RUNTIME_STORE_KEY = Symbol.for("omniroute.adaptiveAdmission.runtime"); + +type RuntimeStore = { + runtime: AdaptiveAdmissionRuntime | null; +}; + +type GlobalWithRuntimeStore = typeof globalThis & { + [RUNTIME_STORE_KEY]?: RuntimeStore; +}; + +function getRuntimeStore(): RuntimeStore { + const globalWithStore = globalThis as GlobalWithRuntimeStore; + let store = globalWithStore[RUNTIME_STORE_KEY]; + if (!store) { + store = { runtime: null }; + globalWithStore[RUNTIME_STORE_KEY] = store; + } + return store; +} + +const ENV_KEYS = { + mode: "ADAPTIVE_ADMISSION_MODE", + minLimit: "ADAPTIVE_ADMISSION_MIN_LIMIT", + initialLimit: "ADAPTIVE_ADMISSION_INITIAL_LIMIT", + maxLimit: "ADAPTIVE_ADMISSION_MAX_LIMIT", + maxQueueCount: "ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT", + maxQueueCost: "ADAPTIVE_ADMISSION_MAX_QUEUE_COST", + defaultMaxWaitMs: "ADAPTIVE_ADMISSION_MAX_WAIT_MS", + windowMs: "ADAPTIVE_ADMISSION_WINDOW_MS", +} as const; + +function parsePositiveSafeInt(name: string, raw: string): number { + if (!/^[0-9]+$/.test(raw)) { + throw new RangeError(`${name} must be a positive safe integer`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new RangeError(`${name} must be a positive safe integer`); + } + return value; +} + +/** Strict env → config resolver. Throws clear config errors for direct callers. */ +export function resolveAdaptiveAdmissionConfigFromEnv( + env: NodeJS.ProcessEnv | Record = process.env +): AdaptiveAdmissionConfig { + const cfg: AdaptiveAdmissionConfig = { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG }; + + const modeRaw = env[ENV_KEYS.mode]; + if (modeRaw !== undefined && modeRaw !== "") { + if (modeRaw !== "off" && modeRaw !== "shadow" && modeRaw !== "enforce") { + throw new RangeError(`${ENV_KEYS.mode} must be off|shadow|enforce`); + } + cfg.mode = modeRaw; + } + + // Numeric env keys only — typed assignment without index-signature cast (TS2352). + type EnvIntField = Exclude; + const intFields = [ + "minLimit", + "initialLimit", + "maxLimit", + "maxQueueCount", + "maxQueueCost", + "defaultMaxWaitMs", + "windowMs", + ] as const satisfies ReadonlyArray; + for (const field of intFields) { + const envName = ENV_KEYS[field]; + const raw = env[envName]; + if (raw === undefined || raw === "") continue; + cfg[field] = parsePositiveSafeInt(envName, raw); + } + + // Shared pure validation — accept exact documented maxima, reject core-invalid configs. + validateConfig(cfg); + return cfg; +} + +export type AdaptiveAdmissionAcquireInput = { + /** Opaque fairness key; never exposed in snapshots or client errors. */ + tenantKey: string; + /** Already-parsed request body — must not be re-read or stringified for cost. */ + body: unknown; + signal?: AbortSignal; + maxWaitMs?: number; + /** Authoritative streaming class; wins body stream inference when set. */ + streaming?: boolean; +}; + +export type AdaptiveAdmissionAdmitted = { + status: "admitted"; + mode: AdmissionMode; + lease: AdmissionLease; + admittedAtMs: number; + shadowDecision?: ShadowDecision; +}; + +export type AdaptiveAdmissionRejected = { + status: "rejected"; + code: string; + response: Response; +}; + +export type AdaptiveAdmissionAcquireResult = AdaptiveAdmissionAdmitted | AdaptiveAdmissionRejected; + +export type AdaptiveAdmissionPublicSnapshot = AdmissionSnapshot & { + resourceSeverity: PressureSeverity; + resourceReason: PressureReason; + resourceObservedAtMs: number; + pressureGuardRejectCount: number; +}; + +export type AdaptiveAdmissionLifecycleOptions = { + admittedAtMs: number; + signal?: AbortSignal; + nowMs?: () => number; +}; + +export type AdaptiveAdmissionRuntimeOptions = { + config?: AdaptiveAdmissionConfig; + env?: NodeJS.ProcessEnv | Record; + clock?: Partial; + checkResourcePressure?: () => ResourcePressureGuardResult | null; + getResourcePressureObservation?: () => ResourcePressureObservation; + /** Test seam: observe pressure values fed into the controller after dedupe. */ + onPressureObserved?: (pressure: AdmissionPressure) => void; + warn?: (message: string) => void; + nowMs?: () => number; +}; + +/** Non-success release outcomes callers must choose explicitly for handler failures. */ +export type AdaptiveAdmissionFailureOutcome = Exclude; + +export type AdaptiveAdmissionRuntime = { + acquire(input: AdaptiveAdmissionAcquireInput): Promise; + snapshot(): AdaptiveAdmissionPublicSnapshot; + dispose(): void; + /** + * Release an admitted lease after a handler failure before any HTTP response exists. + * Callers must supply the concrete non-success outcome — never defaults to local_reject. + */ + releaseHandlerFailure( + lease: AdmissionLease, + outcome: AdaptiveAdmissionFailureOutcome, + options?: { admittedAtMs?: number; nowMs?: () => number } + ): void; + attachResponseLifecycle( + response: Response, + lease: AdmissionLease, + options: AdaptiveAdmissionLifecycleOptions + ): Response; +}; + +type RejectHttpMapping = { + status: number; + code: string; + message: string; + retryAfter?: string; +}; + +const REJECT_MAP: Record = { + ADMISSION_ABORTED: { + status: 499, + code: "admission_aborted", + message: "Request aborted", + }, + ADMISSION_OVERSIZED: { + status: 503, + code: "admission_oversized", + message: "Request too large for current capacity", + }, + ADMISSION_QUEUE_FULL: { + status: 503, + code: "admission_queue_full", + message: "Service temporarily unavailable", + retryAfter: "1", + }, + ADMISSION_DEADLINE: { + status: 503, + code: "admission_deadline", + message: "Service temporarily unavailable", + retryAfter: "1", + }, + ADMISSION_SHUTDOWN: { + status: 503, + code: "admission_shutdown", + message: "Service temporarily unavailable", + }, + ADMISSION_UNAVAILABLE: { + status: 503, + code: "admission_unavailable", + message: "Service temporarily unavailable", + retryAfter: "1", + }, +}; + +function isAdmissionRejectError( + err: unknown +): err is { code: AdmissionRejectCode; name: string; message: string } { + return ( + !!err && + typeof err === "object" && + (err as { name?: string }).name === "AdmissionRejectError" && + typeof (err as { code?: unknown }).code === "string" + ); +} + +function buildAdmissionRejectResponse(code: AdmissionRejectCode): AdaptiveAdmissionRejected { + const mapping = REJECT_MAP[code] ?? REJECT_MAP.ADMISSION_UNAVAILABLE; + const headers: Record = { + "Content-Type": "application/json", + ...CORS_HEADERS, + }; + if (mapping.retryAfter) headers["Retry-After"] = mapping.retryAfter; + const body = buildErrorBody(mapping.status, mapping.message, undefined, { + type: mapping.status === 499 ? "client_disconnected" : "server_error", + code: mapping.code, + }); + return { + status: "rejected", + code: mapping.code, + response: new Response(JSON.stringify(body), { + status: mapping.status, + headers, + }), + }; +} + +function observationIdentity(state: ResourcePressureObservation["state"]): string { + return `${state.observedAtMs}|${state.severity}|${state.reason}`; +} + +function toAdmissionPressure(severity: PressureSeverity): AdmissionPressure { + if (severity === "critical") return "critical"; + if (severity === "high") return "high"; + return "normal"; +} + +function isSseResponse(response: Response): boolean { + const contentType = response.headers.get("content-type") ?? ""; + return contentType.toLowerCase().includes("text/event-stream"); +} + +function releaseOnce( + lease: AdmissionLease, + outcome: AdmissionReleaseOutcome, + admittedAtMs: number | undefined, + nowMs: () => number +): void { + if (lease.released) return; + const latencyMs = admittedAtMs === undefined ? undefined : Math.max(0, nowMs() - admittedAtMs); + lease.release(outcome, latencyMs === undefined ? undefined : { latencyMs }); +} + +/** + * Map HTTP status (+ optional request signal) to admission release outcome. + * Cancellation always wins over status classification. + */ +function classifyHttpOutcome(status: number, signal?: AbortSignal): AdmissionReleaseOutcome { + if (signal?.aborted || status === 499) return "cancelled"; + if (status === 408 || status === 504) return "timeout"; + if (status >= 500) return "upstream_error"; + if (status >= 400) return "local_reject"; + // 2xx / 3xx (and rare 1xx) complete successfully from admission's perspective. + return "success"; +} + +class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime { + private readonly controller: AdaptiveAdmissionController; + private readonly checkResourcePressure: () => ResourcePressureGuardResult | null; + private readonly getResourcePressureObservation: () => ResourcePressureObservation; + private readonly onPressureObserved?: (pressure: AdmissionPressure) => void; + private readonly nowMs: () => number; + private lastObservationKey: string | null = null; + private lastResource: { + severity: PressureSeverity; + reason: PressureReason; + observedAtMs: number; + } = { severity: "normal", reason: "none", observedAtMs: 0 }; + private pressureGuardRejectCount = 0; + private disposed = false; + + constructor(options: AdaptiveAdmissionRuntimeOptions, config: AdaptiveAdmissionConfig) { + this.controller = new AdaptiveAdmissionController(config, options.clock); + this.checkResourcePressure = options.checkResourcePressure ?? checkResourcePressureGuard; + this.getResourcePressureObservation = + options.getResourcePressureObservation ?? getResourcePressureObservation; + this.onPressureObserved = options.onPressureObserved; + this.nowMs = options.nowMs ?? options.clock?.now ?? (() => Date.now()); + } + + async acquire(input: AdaptiveAdmissionAcquireInput): Promise { + if (this.disposed) { + return buildAdmissionRejectResponse("ADMISSION_SHUTDOWN"); + } + + // Independent safety fuse first — never acquire provider work on critical guard. + // Still feed pressure observations so the controller learns from critical samples. + let guard: ResourcePressureGuardResult | null = null; + try { + guard = this.checkResourcePressure(); + } catch { + // Fail open on sampling/check failures. + } + + this.feedFreshPressureObservation(); + + if (guard) { + this.pressureGuardRejectCount += 1; + return { + status: "rejected", + code: "resource_pressure", + response: guard.response, + }; + } + + const features = extractAdmissionCostFeatures( + input.body, + input.streaming === undefined ? undefined : { streaming: input.streaming } + ); + let result: AdmissionAcquireResult; + try { + result = await this.controller.acquire({ + tenantKey: input.tenantKey, + features, + signal: input.signal, + maxWaitMs: input.maxWaitMs, + }); + } catch (err) { + if (isAdmissionRejectError(err)) { + return buildAdmissionRejectResponse(err.code); + } + return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE"); + } + + if (result.status === "rejected") { + return buildAdmissionRejectResponse(result.code); + } + + if (result.status === "queued") { + try { + const admitted = await result.promise; + return { + status: "admitted", + mode: this.controller.snapshot().mode, + lease: admitted.lease, + admittedAtMs: this.nowMs(), + shadowDecision: admitted.shadowDecision, + }; + } catch (err) { + if (isAdmissionRejectError(err)) { + return buildAdmissionRejectResponse(err.code); + } + return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE"); + } + } + + return { + status: "admitted", + mode: this.controller.snapshot().mode, + lease: result.lease, + admittedAtMs: this.nowMs(), + shadowDecision: result.shadowDecision, + }; + } + + snapshot(): AdaptiveAdmissionPublicSnapshot { + const core = this.controller.snapshot(); + return { + ...core, + resourceSeverity: this.lastResource.severity, + resourceReason: this.lastResource.reason, + resourceObservedAtMs: this.lastResource.observedAtMs, + pressureGuardRejectCount: this.pressureGuardRejectCount, + }; + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + this.controller.shutdown(); + } + + releaseHandlerFailure( + lease: AdmissionLease, + outcome: AdaptiveAdmissionFailureOutcome, + options?: { admittedAtMs?: number; nowMs?: () => number } + ): void { + releaseOnce(lease, outcome, options?.admittedAtMs, options?.nowMs ?? this.nowMs); + } + + attachResponseLifecycle( + response: Response, + lease: AdmissionLease, + options: AdaptiveAdmissionLifecycleOptions + ): Response { + const nowMs = options.nowMs ?? this.nowMs; + const admittedAtMs = options.admittedAtMs; + + if (!response.body || !isSseResponse(response)) { + releaseOnce(lease, classifyHttpOutcome(response.status, options.signal), admittedAtMs, nowMs); + return response; + } + + const upstream = response.body; + const reader = upstream.getReader(); + let settled = false; + let readerCancelled = false; + + const settle = (outcome: AdmissionReleaseOutcome): void => { + if (settled) return; + settled = true; + releaseOnce(lease, outcome, admittedAtMs, nowMs); + }; + + const cancelReader = (reason?: unknown): void => { + if (readerCancelled) return; + readerCancelled = true; + void reader.cancel(reason).catch(() => { + /* ignore cancel races */ + }); + }; + + const onAbort = (): void => { + cancelReader(options.signal?.reason); + settle("cancelled"); + }; + + if (options.signal) { + if (options.signal.aborted) { + onAbort(); + } else { + options.signal.addEventListener("abort", onAbort, { once: true }); + } + } + + const detachAbort = (): void => { + options.signal?.removeEventListener("abort", onAbort); + }; + + const stream = new ReadableStream({ + async pull(controller) { + if (settled) { + controller.close(); + return; + } + try { + const { done, value } = await reader.read(); + if (done) { + detachAbort(); + settle(classifyHttpOutcome(response.status, options.signal)); + controller.close(); + return; + } + controller.enqueue(value); + } catch (err) { + detachAbort(); + settle(options.signal?.aborted ? "cancelled" : "upstream_error"); + controller.error(err); + } + }, + cancel(reason) { + detachAbort(); + cancelReader(reason); + settle("cancelled"); + }, + }); + + return new Response(stream, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + } + + private feedFreshPressureObservation(): void { + try { + const observation = this.getResourcePressureObservation(); + const state = observation.state; + this.lastResource = { + severity: state.severity, + reason: state.reason, + observedAtMs: state.observedAtMs, + }; + const key = observationIdentity(state); + if (state.observedAtMs <= 0) return; + if (key === this.lastObservationKey) return; + this.lastObservationKey = key; + const pressure = toAdmissionPressure(state.severity); + this.controller.observePressure(pressure); + this.onPressureObserved?.(pressure); + } catch { + // Fail open. + } + } +} + +function createRuntimeFromResolvedConfig( + options: AdaptiveAdmissionRuntimeOptions, + config: AdaptiveAdmissionConfig +): AdaptiveAdmissionRuntime { + return new AdaptiveAdmissionRuntimeImpl(options, config); +} + +/** + * Create an injected adaptive-admission runtime for tests or process use. + * Invalid explicit `config` still throws (direct callers want fail-fast). + */ +export function createAdaptiveAdmissionRuntime( + options: AdaptiveAdmissionRuntimeOptions = {} +): AdaptiveAdmissionRuntime { + const config = + options.config ?? + (options.env + ? resolveAdaptiveAdmissionConfigFromEnv(options.env) + : { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG }); + return createRuntimeFromResolvedConfig(options, config); +} + +function warnInvalidDefaultConfig(warn: ((message: string) => void) | undefined): void { + const message = + "[adaptiveAdmission] invalid environment configuration; using default shadow admission settings"; + if (warn) { + warn(message); + return; + } + console.warn(message); +} + +function createDefaultProcessRuntime( + options: AdaptiveAdmissionRuntimeOptions = {} +): AdaptiveAdmissionRuntime { + const warn = options.warn; + try { + const config = + options.config ?? resolveAdaptiveAdmissionConfigFromEnv(options.env ?? process.env); + return createRuntimeFromResolvedConfig(options, config); + } catch { + warnInvalidDefaultConfig(warn); + return createRuntimeFromResolvedConfig(options, { + ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, + }); + } +} + +/** Call-time process-global runtime (HMR-safe via globalThis symbol store). */ +export function getAdaptiveAdmissionRuntime(): AdaptiveAdmissionRuntime { + const store = getRuntimeStore(); + if (!store.runtime) { + store.runtime = createDefaultProcessRuntime(); + } + return store.runtime; +} + +/** Dispose previous controller and replace the process-global runtime. */ +export function reloadAdaptiveAdmissionRuntime( + options: AdaptiveAdmissionRuntimeOptions = {} +): AdaptiveAdmissionRuntime { + const store = getRuntimeStore(); + store.runtime?.dispose(); + store.runtime = createDefaultProcessRuntime(options); + return store.runtime; +} + +/** Test isolation: dispose and clear the process-global runtime slot. */ +export function resetAdaptiveAdmissionRuntimeForTests(): void { + const store = getRuntimeStore(); + store.runtime?.dispose(); + store.runtime = null; +} diff --git a/open-sse/services/admission/types.ts b/open-sse/services/admission/types.ts new file mode 100644 index 0000000000..2a8e537a40 --- /dev/null +++ b/open-sse/services/admission/types.ts @@ -0,0 +1,171 @@ +/** + * Pure weighted adaptive admission-control types. + * No route/settings wiring — dependency-injected controller seam only. + */ + +/** + * Upper bound for adaptation windows and wait deadlines that participate in + * cost×time products (utilization integrals, deadline offsets). + * 24h is far beyond practical control windows while keeping the product domain exact. + */ +export const MAX_ADMISSION_WINDOW_MS = 86_400_000; + +/** + * Upper bound for every validated cost, limit, and queue-cost quantum. + * Derived so `MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS` remains a + * safe integer: a full window at the maximum limit integrates to utilization 1.0 + * without saturating or rounding Number arithmetic. + */ +export const MAX_ADMISSION_COST_OR_LIMIT = Math.floor( + Number.MAX_SAFE_INTEGER / MAX_ADMISSION_WINDOW_MS +); + +export type AdmissionMode = "off" | "shadow" | "enforce"; + +export type AdmissionPressure = "normal" | "high" | "critical"; + +/** Local outcome categories. Upstream business errors must not collapse capacity. */ +export type AdmissionReleaseOutcome = + "success" | "upstream_error" | "timeout" | "local_reject" | "cancelled"; + +export type AdmissionRejectCode = + | "ADMISSION_OVERSIZED" + | "ADMISSION_QUEUE_FULL" + | "ADMISSION_DEADLINE" + | "ADMISSION_ABORTED" + | "ADMISSION_SHUTDOWN" + | "ADMISSION_UNAVAILABLE"; + +export type ShadowDecision = "would-admit" | "would-queue" | "would-reject"; + +export interface AdmissionCostFeatures { + bodyBytes?: number | null; + estimatedInputTokens?: number | null; + messageCount?: number | null; + toolCount?: number | null; + requestedFanout?: number | null; + streaming?: boolean | null; +} + +export interface AdmissionCostConfig { + baseCost: number; + bodyBytesPerUnit: number; + tokensPerUnit: number; + messagesPerUnit: number; + toolsPerUnit: number; + fanoutPerUnit: number; + streamingClassCost: number; + nonStreamingClassCost: number; + maxRequestCost: number; +} + +export interface AdaptiveAdmissionConfig { + mode?: AdmissionMode; + minLimit: number; + maxLimit: number; + initialLimit: number; + maxQueueCount: number; + maxQueueCost: number; + defaultMaxWaitMs?: number; + windowMs?: number; + shortLatencyAlpha?: number; + longLatencyAlpha?: number; + increaseStep?: number; + decreaseFactor?: number; + criticalDecreaseFactor?: number; + highUtilizationThreshold?: number; + lowUtilizationThreshold?: number; + latencyGradientThreshold?: number; + maxIncreasePerWindow?: number; + /** Optional cost quanta override used only when callers pass features instead of cost. */ + cost?: Partial; +} + +export interface AdmissionRequest { + /** Positive integer cost units. If omitted, `features` + cost config are used. */ + cost?: number; + features?: AdmissionCostFeatures; + /** Opaque fairness key; never exposed in snapshots. */ + tenantKey?: string; + maxWaitMs?: number; + signal?: AbortSignal; + pressure?: AdmissionPressure; +} + +export interface AdmissionReleaseMeta { + latencyMs?: number; + pressure?: AdmissionPressure; +} + +export interface AdmissionLease { + readonly id: string; + readonly cost: number; + readonly released: boolean; + release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta): void; +} + +export interface AdmissionAdmitted { + status: "admitted"; + lease: AdmissionLease; + shadowDecision?: ShadowDecision; +} + +export interface AdmissionQueued { + status: "queued"; + promise: Promise; +} + +export interface AdmissionRejected { + status: "rejected"; + code: AdmissionRejectCode; + message: string; + shadowDecision?: ShadowDecision; +} + +export type AdmissionAcquireResult = AdmissionAdmitted | AdmissionQueued | AdmissionRejected; + +export interface AdmissionSnapshot { + mode: AdmissionMode; + currentLimit: number; + minLimit: number; + maxLimit: number; + activeCost: number; + activeCount: number; + queuedCost: number; + queuedCount: number; + virtualActiveCost: number; + virtualActiveCount: number; + virtualQueuedCost: number; + virtualQueuedCount: number; + admittedCount: number; + rejectedCount: number; + wouldAdmitCount: number; + wouldQueueCount: number; + wouldRejectCount: number; + shortLatencyEwma: number; + longLatencyEwma: number; + utilization: number; + pressure: AdmissionPressure; + shutdown: boolean; +} + +export interface AdmissionClock { + now: () => number; + setTimer: (fn: () => void, delayMs: number) => unknown; + clearTimer: (id: unknown) => void; +} + +export interface AdmissionRejectError extends Error { + code: AdmissionRejectCode; + name: "AdmissionRejectError"; +} + +export function createAdmissionRejectError( + code: AdmissionRejectCode, + message: string +): AdmissionRejectError { + const err = new Error(message) as AdmissionRejectError; + err.name = "AdmissionRejectError"; + err.code = code; + return err; +} diff --git a/open-sse/utils/estimateSize.ts b/open-sse/utils/estimateSize.ts index ac320f9aad..8a6f5ef76d 100644 --- a/open-sse/utils/estimateSize.ts +++ b/open-sse/utils/estimateSize.ts @@ -1,32 +1,109 @@ /** - * Fast object-tree size estimator — walks without JSON.stringify. - * Safe for circular references (uses WeakSet). - * Early-exits at 256KB to avoid wasting CPU on huge payloads. + * Fast object-tree size estimator — walks without JSON.stringify / toJSON / clone. + * Safe for circular references (WeakSet). Iterative frames only (no recursive call stack). + * + * Budgets: + * - ESTIMATE_SIZE_BYTE_LIMIT (256 KiB): early-exit once counted bytes exceed the limit + * - ESTIMATE_SIZE_NODE_BUDGET: max value visits (containers + primitives/elements) + * + * Arrays are walked by index frame (never pre-push/copy every element reference). + * Plain objects yield own enumerable values incrementally (no Object.keys materialization). + * Node-budget exhaustion returns a value strictly above 256 KiB so callers fail closed. */ -export function estimateSizeFast(value: unknown): number { - let bytes = 0; - const stack: unknown[] = [value]; - const seen = new WeakSet(); - while (stack.length > 0) { - const v = stack.pop(); - if (v === null || v === undefined) continue; - if (typeof v === "string") { - bytes += v.length; - if (bytes > 262144) return bytes; - } else if (typeof v === "number") bytes += 8; - else if (typeof v === "boolean") bytes += 4; - else if (typeof v === "object") { - if (seen.has(v as object)) continue; - seen.add(v as object); - if (Array.isArray(v)) { - for (let i = 0; i < v.length; i++) stack.push(v[i]); - } else { - for (const key in v) { - if (Object.prototype.hasOwnProperty.call(v, key)) stack.push((v as Record)[key]); - } + +/** Byte early-exit threshold (256 KiB). */ +export const ESTIMATE_SIZE_BYTE_LIMIT = 262_144; + +/** + * Max value/element visits before fail-closed. + * Conservative cap keeps auxiliary stack/WeakSet growth bounded under adversarial input. + */ +export const ESTIMATE_SIZE_NODE_BUDGET = 16_384; + +type Frame = + | { t: "v"; v: unknown } + | { t: "a"; a: unknown[]; i: number } + | { t: "o"; o: object; it: Iterator }; + +function ownEnumerableKeyIterator(obj: object): Iterator { + return (function* ownEnumerableKeys() { + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + yield key; } } + })(); +} + +/** @returns next byte total, or a value > limit when the limit is exceeded. */ +function addPrimitiveBytes(bytes: number, v: string | number | boolean): number { + if (typeof v === "string") return bytes + v.length; + if (typeof v === "number") return bytes + 8; + return bytes + 4; +} + +function enqueueContainer(stack: Frame[], obj: object, seen: WeakSet): void { + if (seen.has(obj)) return; + seen.add(obj); + if (Array.isArray(obj)) { + if (obj.length > 0) stack.push({ t: "a", a: obj, i: 0 }); + return; } + stack.push({ t: "o", o: obj, it: ownEnumerableKeyIterator(obj) }); +} + +type ValueFrame = Extract; + +function isValueFrame(frame: Frame): frame is ValueFrame { + return frame.t === "v"; +} + +/** Expand a container frame into the next child value. */ +function expandContainerFrame(stack: Frame[], frame: Exclude): void { + if (frame.t === "a") { + if (frame.i >= frame.a.length) return; + if (frame.i + 1 < frame.a.length) { + stack.push({ t: "a", a: frame.a, i: frame.i + 1 }); + } + stack.push({ t: "v", v: frame.a[frame.i] }); + return; + } + const next = frame.it.next(); + if (next.done) return; + stack.push(frame); + stack.push({ t: "v", v: (frame.o as Record)[next.value] }); +} + +export function estimateSizeFast(value: unknown): number { + let bytes = 0; + let visitsLeft = ESTIMATE_SIZE_NODE_BUDGET; + const seen = new WeakSet(); + const stack: Frame[] = [{ t: "v", v: value }]; + + while (stack.length > 0) { + if (visitsLeft <= 0) return ESTIMATE_SIZE_BYTE_LIMIT + 1; + + const frame = stack.pop()!; + if (!isValueFrame(frame)) { + expandContainerFrame(stack, frame); + continue; + } + + visitsLeft -= 1; + const v = frame.v; + if (v === null || v === undefined) continue; + + const ty = typeof v; + if (ty === "string" || ty === "number" || ty === "boolean") { + bytes = addPrimitiveBytes(bytes, v as string | number | boolean); + if (bytes > ESTIMATE_SIZE_BYTE_LIMIT) return bytes; + continue; + } + if (ty === "object") { + enqueueContainer(stack, v as object, seen); + } + } + return bytes; } diff --git a/open-sse/utils/resourcePressure.ts b/open-sse/utils/resourcePressure.ts new file mode 100644 index 0000000000..acef067a1e --- /dev/null +++ b/open-sse/utils/resourcePressure.ts @@ -0,0 +1,249 @@ +import { checkHeapPressureGuard, HEAP_PRESSURE_THRESHOLD_MB } from "./heapPressure.ts"; +import { buildErrorBody } from "./error.ts"; +import { + createResourcePressureTracker, + resolveResourcePressureThresholds, + type PressureReason, + type ResourcePressureState, + type ResourcePressureThresholds, + type ResourceSignals, +} from "./resourcePressurePolicy.ts"; +import { + sampleResourceSignals, + type SampleResourceSignalsDeps, +} from "./resourcePressureSampler.ts"; + +const MB = 1024 * 1024; +const RETRY_AFTER_SECONDS = "5"; +const PRESSURE_MESSAGE = "Service temporarily unavailable due to resource pressure. Retry shortly."; + +export type ResourcePressureGuardResult = { + success: false; + status: 503; + error: string; + response: Response; +}; + +export type ResourcePressureObservation = { + signals: ResourceSignals | null; + state: ResourcePressureState; +}; + +export type ResourcePressureRuntimeOptions = { + thresholds?: Partial; + heapThresholdMb?: number | null; + immediateHeapUsedMb?: () => number; + sample?: () => Promise; + nowMs?: () => number; + schedule?: (refresh: () => void) => void; + staleAfterMs?: number; + maxStaleMs?: number; + retryAfterMs?: number; + samplerDeps?: SampleResourceSignalsDeps; +}; + +export type ResourcePressureRuntime = { + check: () => ResourcePressureGuardResult | null; + getObservation: () => ResourcePressureObservation; + whenRefreshSettled: () => Promise; + dispose: () => void; +}; + +function emptyState(): ResourcePressureState { + return { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + }; +} + +function requireDuration(name: string, value: number): number { + if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0 || value > 3_600_000) { + throw new RangeError(`${name} must be an integer between 0 and 3600000`); + } + return value; +} + +function buildCriticalGuard(reason: PressureReason): ResourcePressureGuardResult { + console.warn( + `[resourcePressure] critical pressure guard tripped (reason=${reason}); returning 503` + ); + return { + success: false, + status: 503, + error: PRESSURE_MESSAGE, + response: new Response( + JSON.stringify( + buildErrorBody(503, PRESSURE_MESSAGE, undefined, { + type: "server_error", + code: "resource_pressure", + }) + ), + { + status: 503, + headers: { "Content-Type": "application/json", "Retry-After": RETRY_AFTER_SECONDS }, + } + ), + }; +} + +function immediateHeapGuard( + heapUsedMb: number, + thresholdMb: number | null +): ResourcePressureGuardResult | null { + if (thresholdMb == null) return null; + const guard = checkHeapPressureGuard(heapUsedMb, thresholdMb); + if (!guard) return null; + return buildCriticalGuard("v8_heap_absolute"); +} + +export function createResourcePressureRuntime( + options: ResourcePressureRuntimeOptions = {} +): ResourcePressureRuntime { + const heapThresholdMb = + options.heapThresholdMb === undefined ? HEAP_PRESSURE_THRESHOLD_MB : options.heapThresholdMb; + if (heapThresholdMb !== null && (!Number.isFinite(heapThresholdMb) || heapThresholdMb <= 0)) { + throw new RangeError("heapThresholdMb must be positive and finite or null"); + } + const thresholds = resolveResourcePressureThresholds({ + ...options.thresholds, + heapAbsoluteThresholdMb: + options.thresholds?.heapAbsoluteThresholdMb === undefined + ? null + : options.thresholds.heapAbsoluteThresholdMb, + }); + const staleAfterMs = requireDuration("staleAfterMs", options.staleAfterMs ?? 1_000); + const maxStaleMs = requireDuration("maxStaleMs", options.maxStaleMs ?? 30_000); + const retryAfterMs = requireDuration("retryAfterMs", options.retryAfterMs ?? 1_000); + if (maxStaleMs < staleAfterMs) { + throw new RangeError("maxStaleMs must be greater than or equal to staleAfterMs"); + } + + const nowMs = options.nowMs ?? Date.now; + const immediateHeapUsedMb = + options.immediateHeapUsedMb ?? (() => process.memoryUsage().heapUsed / MB); + const sample = options.sample ?? (() => sampleResourceSignals(options.samplerDeps)); + const schedule = + options.schedule ?? + ((refresh) => { + const handle = setImmediate(refresh); + handle.unref(); + }); + const tracker = createResourcePressureTracker(thresholds); + + let lastSignals: ResourceSignals | null = null; + let state = emptyState(); + let lastRefreshAtMs = Number.NEGATIVE_INFINITY; + let nextRefreshAtMs = Number.NEGATIVE_INFINITY; + let scheduled = false; + let inFlight: Promise | null = null; + let disposed = false; + + const refresh = (): void => { + if (disposed || inFlight) return; + scheduled = false; + inFlight = Promise.resolve() + .then(sample) + .then((signals) => { + if (disposed) return; + const settledAtMs = nowMs(); + lastSignals = signals; + state = tracker.observe(signals); + lastRefreshAtMs = settledAtMs; + nextRefreshAtMs = settledAtMs + staleAfterMs; + }) + .catch(() => { + if (!disposed) nextRefreshAtMs = nowMs() + retryAfterMs; + }) + .finally(() => { + inFlight = null; + }); + }; + + const scheduleRefresh = (): void => { + if (disposed || scheduled || inFlight) return; + scheduled = true; + schedule(refresh); + }; + + return { + check() { + let heapUsedMb = 0; + try { + heapUsedMb = immediateHeapUsedMb(); + } catch { + heapUsedMb = 0; + } + const immediate = immediateHeapGuard(heapUsedMb, heapThresholdMb); + const now = nowMs(); + if (now >= nextRefreshAtMs) scheduleRefresh(); + if (immediate) { + state = { + severity: "critical", + reason: "v8_heap_absolute", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: now, + observedAtMs: now, + }; + return immediate; + } + const cacheAge = lastSignals ? Math.max(0, now - lastRefreshAtMs) : Number.POSITIVE_INFINITY; + return cacheAge <= maxStaleMs && state.severity === "critical" + ? buildCriticalGuard(state.reason) + : null; + }, + getObservation: () => ({ signals: lastSignals, state }), + whenRefreshSettled: async () => { + if (scheduled) await new Promise((resolve) => setImmediate(resolve)); + if (inFlight) await inFlight; + }, + dispose() { + disposed = true; + scheduled = false; + }, + }; +} + +let defaultRuntime = createResourcePressureRuntime(); + +export function checkResourcePressureGuard(): ResourcePressureGuardResult | null { + return defaultRuntime.check(); +} + +export function getResourcePressureObservation(): ResourcePressureObservation { + return defaultRuntime.getObservation(); +} + +/** Replaces and disposes the process singleton when configuration is reloaded. */ +export function reloadResourcePressureRuntime( + options: ResourcePressureRuntimeOptions = {} +): ResourcePressureRuntime { + defaultRuntime.dispose(); + defaultRuntime = createResourcePressureRuntime(options); + return defaultRuntime; +} + +export type { + PressureReason, + PressureSeverity, + ResourceMetricBytes, + ResourcePressureState, + ResourcePressureThresholds, + ResourcePressureTracker, + ResourceSignals, +} from "./resourcePressurePolicy.ts"; +export { + classifyAdaptiveResourcePressure as classifyResourcePressure, + createResourcePressureTracker, + resolveResourcePressureThresholds, +} from "./resourcePressurePolicy.ts"; +export { + sampleResourceSignals, + sanitizeMemoryBytes, + type ResourcePressureFs, + type SampleResourceSignalsDeps, +} from "./resourcePressureSampler.ts"; diff --git a/open-sse/utils/resourcePressurePolicy.ts b/open-sse/utils/resourcePressurePolicy.ts new file mode 100644 index 0000000000..49a535aca5 --- /dev/null +++ b/open-sse/utils/resourcePressurePolicy.ts @@ -0,0 +1,344 @@ +const MB = 1024 * 1024; +const MAX_SUSTAINED_SAMPLES = 10_000; + +export type PressureSeverity = "normal" | "high" | "critical"; + +export type PressureReason = + | "none" + | "v8_heap_ratio" + | "v8_heap_absolute" + | "cgroup_ratio" + | "cgroup_high" + | "psi_some" + | "psi_full" + | "oom_event"; + +export type ResourceMetricBytes = number | null; + +export type ResourceSignals = { + observedAtMs: number; + v8: { heapUsedBytes: number; heapLimitBytes: number }; + process: { + rssBytes: number; + externalBytes: number; + arrayBuffersBytes: number; + availableBytes: ResourceMetricBytes; + constrainedBytes: ResourceMetricBytes; + }; + cgroup: { + currentBytes: ResourceMetricBytes; + maxBytes: ResourceMetricBytes; + highBytes: ResourceMetricBytes; + events: { + low: ResourceMetricBytes; + high: ResourceMetricBytes; + max: ResourceMetricBytes; + oom: ResourceMetricBytes; + oom_kill: ResourceMetricBytes; + } | null; + }; + psi: { + someAvg10: number | null; + someAvg60: number | null; + someAvg300: number | null; + fullAvg10: number | null; + fullAvg60: number | null; + fullAvg300: number | null; + } | null; +}; + +export type ResourcePressureState = { + severity: PressureSeverity; + reason: PressureReason; + elevatedStreak: number; + recoveryStreak: number; + lastTransitionAtMs: number; + observedAtMs: number; +}; + +export type ResourcePressureThresholds = { + highRatio: number; + criticalRatio: number; + recoveryRatio: number; + highPsiAvg10: number; + criticalPsiAvg10: number; + recoveryPsiAvg10: number; + sustainedSamplesHigh: number; + sustainedSamplesCritical: number; + sustainedSamplesRecovery: number; + heapAbsoluteThresholdMb: number | null; +}; + +export const DEFAULT_RESOURCE_PRESSURE_THRESHOLDS: ResourcePressureThresholds = { + highRatio: 0.85, + criticalRatio: 0.92, + recoveryRatio: 0.75, + highPsiAvg10: 20, + criticalPsiAvg10: 40, + recoveryPsiAvg10: 10, + sustainedSamplesHigh: 2, + sustainedSamplesCritical: 2, + sustainedSamplesRecovery: 3, + heapAbsoluteThresholdMb: null, +}; + +type RawLevel = { severity: PressureSeverity; reason: PressureReason }; +type OomCounters = { oom: number | null; oomKill: number | null }; + +function requireFiniteRange(name: string, value: number, minimum: number, maximum: number): void { + if (!Number.isFinite(value) || value < minimum || value > maximum) { + throw new RangeError(`${name} must be finite and between ${minimum} and ${maximum}`); + } +} + +function requirePositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 1 || value > MAX_SUSTAINED_SAMPLES) { + throw new RangeError(`${name} must be an integer between 1 and ${MAX_SUSTAINED_SAMPLES}`); + } +} + +export function resolveResourcePressureThresholds( + partial: Partial = {} +): ResourcePressureThresholds { + const resolved = { ...DEFAULT_RESOURCE_PRESSURE_THRESHOLDS, ...partial }; + requireFiniteRange("recoveryRatio", resolved.recoveryRatio, 0, 1); + requireFiniteRange("highRatio", resolved.highRatio, 0, 1); + requireFiniteRange("criticalRatio", resolved.criticalRatio, 0, 1); + if (!( + resolved.recoveryRatio < resolved.highRatio && resolved.highRatio < resolved.criticalRatio + )) { + throw new RangeError("ratio thresholds must satisfy recovery < high < critical"); + } + + requireFiniteRange("recoveryPsiAvg10", resolved.recoveryPsiAvg10, 0, 100); + requireFiniteRange("highPsiAvg10", resolved.highPsiAvg10, 0, 100); + requireFiniteRange("criticalPsiAvg10", resolved.criticalPsiAvg10, 0, 100); + if (!( + resolved.recoveryPsiAvg10 < resolved.highPsiAvg10 && + resolved.highPsiAvg10 < resolved.criticalPsiAvg10 + )) { + throw new RangeError("PSI thresholds must satisfy recovery < high < critical"); + } + + requirePositiveInteger("sustainedSamplesHigh", resolved.sustainedSamplesHigh); + requirePositiveInteger("sustainedSamplesCritical", resolved.sustainedSamplesCritical); + requirePositiveInteger("sustainedSamplesRecovery", resolved.sustainedSamplesRecovery); + if ( + resolved.heapAbsoluteThresholdMb !== null && + (!Number.isFinite(resolved.heapAbsoluteThresholdMb) || resolved.heapAbsoluteThresholdMb <= 0) + ) { + throw new RangeError("heapAbsoluteThresholdMb must be positive and finite or null"); + } + return resolved; +} + +function severityRank(severity: PressureSeverity): number { + return severity === "critical" ? 2 : severity === "high" ? 1 : 0; +} + +function maxLevel(current: RawLevel, candidate: RawLevel | null): RawLevel { + if (!candidate || severityRank(candidate.severity) <= severityRank(current.severity)) { + return current; + } + return candidate; +} + +function ratioLevel( + used: number | null, + limit: number | null, + thresholds: ResourcePressureThresholds, + reason: PressureReason +): RawLevel | null { + if (used == null || limit == null || used < 0 || limit <= 0) return null; + const ratio = used / limit; + if (ratio >= thresholds.criticalRatio) return { severity: "critical", reason }; + if (ratio >= thresholds.highRatio) return { severity: "high", reason }; + return null; +} + +function psiLevel( + value: number | null, + thresholds: ResourcePressureThresholds, + reason: Extract +): RawLevel | null { + if (value == null || !Number.isFinite(value)) return null; + if (value >= thresholds.criticalPsiAvg10) return { severity: "critical", reason }; + if (value >= thresholds.highPsiAvg10) return { severity: "high", reason }; + return null; +} + +export function classifyAdaptiveResourcePressure( + signals: ResourceSignals, + thresholds: ResourcePressureThresholds +): RawLevel { + let best: RawLevel = { severity: "normal", reason: "none" }; + best = maxLevel( + best, + ratioLevel(signals.v8.heapUsedBytes, signals.v8.heapLimitBytes, thresholds, "v8_heap_ratio") + ); + best = maxLevel( + best, + ratioLevel(signals.cgroup.currentBytes, signals.cgroup.maxBytes, thresholds, "cgroup_ratio") + ); + best = maxLevel( + best, + ratioLevel(signals.cgroup.currentBytes, signals.cgroup.highBytes, thresholds, "cgroup_high") + ); + best = maxLevel(best, psiLevel(signals.psi?.someAvg10 ?? null, thresholds, "psi_some")); + return maxLevel(best, psiLevel(signals.psi?.fullAvg10 ?? null, thresholds, "psi_full")); +} + +function isRecovered(signals: ResourceSignals, thresholds: ResourcePressureThresholds): boolean { + const ratios: Array = [ + [signals.v8.heapUsedBytes, signals.v8.heapLimitBytes], + [signals.cgroup.currentBytes, signals.cgroup.maxBytes], + [signals.cgroup.currentBytes, signals.cgroup.highBytes], + ]; + if ( + ratios.some( + ([used, limit]) => + used != null && limit != null && limit > 0 && used / limit > thresholds.recoveryRatio + ) + ) { + return false; + } + if ( + thresholds.heapAbsoluteThresholdMb != null && + signals.v8.heapUsedBytes / MB > thresholds.heapAbsoluteThresholdMb * thresholds.recoveryRatio + ) { + return false; + } + return ![signals.psi?.someAvg10, signals.psi?.fullAvg10].some( + (value) => value != null && value > thresholds.recoveryPsiAvg10 + ); +} + +function hasCounterIncrease(previous: OomCounters, current: OomCounters): boolean { + return ( + (previous.oom != null && current.oom != null && current.oom > previous.oom) || + (previous.oomKill != null && current.oomKill != null && current.oomKill > previous.oomKill) + ); +} + +function countersReset(previous: OomCounters, current: OomCounters): boolean { + return ( + (previous.oom != null && current.oom != null && current.oom < previous.oom) || + (previous.oomKill != null && current.oomKill != null && current.oomKill < previous.oomKill) + ); +} + +function initialState(): ResourcePressureState { + return { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + }; +} + +export type ResourcePressureTracker = { + observe: (signals: ResourceSignals) => ResourcePressureState; + getState: () => ResourcePressureState; +}; + +export function createResourcePressureTracker( + partialThresholds: Partial = {} +): ResourcePressureTracker { + const thresholds = resolveResourcePressureThresholds(partialThresholds); + let state = initialState(); + let pending: RawLevel | null = null; + let previousOom: OomCounters | null = null; + + return { + observe(signals) { + const events = signals.cgroup.events; + const currentOom = events ? { oom: events.oom, oomKill: events.oom_kill } : null; + let oomEvent = false; + if (currentOom) { + if (previousOom && !countersReset(previousOom, currentOom)) { + oomEvent = hasCounterIncrease(previousOom, currentOom); + } + previousOom = currentOom; + } else { + previousOom = null; + } + + const raw = oomEvent + ? ({ severity: "critical", reason: "oom_event" } as const) + : classifyAdaptiveResourcePressure(signals, thresholds); + let { severity, reason, elevatedStreak, recoveryStreak } = state; + + if (oomEvent) { + severity = "critical"; + reason = "oom_event"; + elevatedStreak = 0; + recoveryStreak = 0; + pending = null; + } else if (severity === "normal") { + recoveryStreak = 0; + if (raw.severity === "normal") { + pending = null; + elevatedStreak = 0; + reason = "none"; + } else { + const samePending = pending?.severity === raw.severity && pending.reason === raw.reason; + pending = raw; + elevatedStreak = samePending ? elevatedStreak + 1 : 1; + const needed = + raw.severity === "critical" + ? thresholds.sustainedSamplesCritical + : thresholds.sustainedSamplesHigh; + if (elevatedStreak >= needed) { + severity = raw.severity; + reason = raw.reason; + elevatedStreak = 0; + pending = null; + } + } + } else if (severity === "high" && raw.severity === "critical") { + recoveryStreak = 0; + const samePending = pending?.severity === "critical" && pending.reason === raw.reason; + pending = raw; + elevatedStreak = samePending ? elevatedStreak + 1 : 1; + if (elevatedStreak >= thresholds.sustainedSamplesCritical) { + severity = "critical"; + reason = raw.reason; + elevatedStreak = 0; + pending = null; + } + } else if (raw.severity === severity) { + reason = raw.reason; + pending = null; + elevatedStreak = 0; + recoveryStreak = 0; + } else if (isRecovered(signals, thresholds)) { + pending = null; + elevatedStreak = 0; + recoveryStreak += 1; + if (recoveryStreak >= thresholds.sustainedSamplesRecovery) { + severity = "normal"; + reason = "none"; + recoveryStreak = 0; + } + } else { + pending = null; + elevatedStreak = 0; + recoveryStreak = 0; + } + + const transitioned = severity !== state.severity || reason !== state.reason; + state = { + severity, + reason, + elevatedStreak, + recoveryStreak, + lastTransitionAtMs: transitioned ? signals.observedAtMs : state.lastTransitionAtMs, + observedAtMs: signals.observedAtMs, + }; + return state; + }, + getState: () => state, + }; +} diff --git a/open-sse/utils/resourcePressureSampler.ts b/open-sse/utils/resourcePressureSampler.ts new file mode 100644 index 0000000000..994ebd712a --- /dev/null +++ b/open-sse/utils/resourcePressureSampler.ts @@ -0,0 +1,257 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import v8 from "node:v8"; +import type { ResourceSignals } from "./resourcePressurePolicy.ts"; + +const DEFAULT_CGROUP_ROOT = "/sys/fs/cgroup"; + +export type ResourcePressureFs = { + readText: (filePath: string) => Promise; +}; + +export type SampleResourceSignalsDeps = { + nowMs?: () => number; + memoryUsage?: () => NodeJS.MemoryUsage; + heapStatistics?: () => { heap_size_limit: number; used_heap_size?: number }; + availableMemory?: () => number | undefined; + constrainedMemory?: () => number | undefined; + fs?: ResourcePressureFs; +}; + +type Cgroup2Mount = { root: string; mountpoint: string }; + +async function defaultReadText(filePath: string): Promise { + try { + return await fs.readFile(filePath, "utf8"); + } catch { + return null; + } +} + +export function sanitizeMemoryBytes(value: unknown): number | null { + if (typeof value === "string") { + const trimmed = value.trim(); + if (!trimmed || trimmed === "max" || !/^\d+$/.test(trimmed) || trimmed.length > 15) { + return null; + } + value = Number(trimmed); + } + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null; + if (value >= Number.MAX_SAFE_INTEGER) return null; + return Math.floor(value); +} + +function safeNumber(call: (() => number | undefined) | undefined): number | null { + try { + return call ? sanitizeMemoryBytes(call()) : null; + } catch { + return null; + } +} + +export function decodeMountInfoPath(value: string): string | null { + if (value.includes("\0")) return null; + try { + return value.replace(/\\([0-7]{3})/g, (_match, octal: string) => + String.fromCharCode(Number.parseInt(octal, 8)) + ); + } catch { + return null; + } +} + +export function parseCgroupV2Path(contents: string | null): string | null { + if (!contents) return null; + for (const rawLine of contents.split("\n")) { + const line = rawLine.trim(); + if (!line.startsWith("0::")) continue; + const relativePath = line.slice(3); + if (!relativePath.startsWith("/") || relativePath.includes("\0")) return null; + return relativePath; + } + return null; +} + +export function parseCgroup2Mount(contents: string | null): Cgroup2Mount | null { + if (!contents) return null; + for (const rawLine of contents.split("\n")) { + const separator = rawLine.indexOf(" - "); + if (separator < 0) continue; + const left = rawLine.slice(0, separator).trim().split(/\s+/); + const right = rawLine + .slice(separator + 3) + .trim() + .split(/\s+/); + if (right[0] !== "cgroup2" || left.length < 5) continue; + const root = decodeMountInfoPath(left[3]); + const mountpoint = decodeMountInfoPath(left[4]); + if (!root?.startsWith("/") || !mountpoint?.startsWith("/")) return null; + return { root, mountpoint }; + } + return null; +} + +function isContained(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function hasTraversalSegment(value: string): boolean { + let decoded = value; + try { + decoded = decodeURIComponent(value); + } catch { + return true; + } + return decoded.split("/").some((segment) => segment === ".." || segment === "."); +} + +function resolveFromMount(cgroupPath: string, mount: Cgroup2Mount): string | null { + if ( + cgroupPath.includes("\0") || + mount.root.includes("\0") || + mount.mountpoint.includes("\0") || + hasTraversalSegment(cgroupPath) + ) { + return null; + } + const resolvedRoot = path.resolve(mount.root); + const resolvedCgroup = path.resolve(cgroupPath); + if (!isContained(resolvedRoot, resolvedCgroup)) return null; + const suffix = path.relative(resolvedRoot, resolvedCgroup); + const resolvedMountpoint = path.resolve(mount.mountpoint); + const candidate = path.resolve(resolvedMountpoint, suffix); + return isContained(resolvedMountpoint, candidate) ? candidate : null; +} + +export async function resolveCgroupDirectory( + readText: ResourcePressureFs["readText"], + options: { allowDefaultFallback?: boolean } = {} +): Promise { + try { + const [cgroupContents, mountInfo] = await Promise.all([ + readText("/proc/self/cgroup"), + readText("/proc/self/mountinfo"), + ]); + const cgroupPath = parseCgroupV2Path(cgroupContents); + const mount = parseCgroup2Mount(mountInfo); + if (cgroupPath && mount) { + const candidate = resolveFromMount(cgroupPath, mount); + if (candidate && (await readText(path.join(candidate, "memory.current"))) != null) { + return candidate; + } + if (!candidate) return null; + } + if (options.allowDefaultFallback === false) return null; + return (await readText(path.join(DEFAULT_CGROUP_ROOT, "memory.current"))) != null + ? DEFAULT_CGROUP_ROOT + : null; + } catch { + return null; + } +} + +function parseEventCounter(value: string): number | null { + const parsed = Number(value.trim()); + return Number.isFinite(parsed) && parsed >= 0 && parsed < Number.MAX_SAFE_INTEGER + ? Math.floor(parsed) + : null; +} + +function parseMemoryEvents(text: string | null): ResourceSignals["cgroup"]["events"] { + if (!text) return null; + const values = { low: null, high: null, max: null, oom: null, oom_kill: null } as Record< + "low" | "high" | "max" | "oom" | "oom_kill", + number | null + >; + let matched = false; + for (const line of text.split("\n")) { + const [key, rawValue] = line.trim().split(/\s+/, 2); + if (!(key in values) || rawValue == null) continue; + values[key as keyof typeof values] = parseEventCounter(rawValue); + matched = true; + } + return matched ? values : null; +} + +function parsePsiNumber(line: string, name: string): number | null { + const match = new RegExp(`(?:^|\\s)${name}=([0-9.]+)`).exec(line); + const parsed = match ? Number(match[1]) : Number.NaN; + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; +} + +function parsePsi(text: string | null): ResourceSignals["psi"] { + if (!text) return null; + const result: NonNullable = { + someAvg10: null, + someAvg60: null, + someAvg300: null, + fullAvg10: null, + fullAvg60: null, + fullAvg300: null, + }; + let matched = false; + for (const line of text.split("\n")) { + const kind = line.startsWith("some ") ? "some" : line.startsWith("full ") ? "full" : null; + if (!kind) continue; + result[`${kind}Avg10`] = parsePsiNumber(line, "avg10"); + result[`${kind}Avg60`] = parsePsiNumber(line, "avg60"); + result[`${kind}Avg300`] = parsePsiNumber(line, "avg300"); + matched = true; + } + return matched ? result : null; +} + +export async function sampleResourceSignals( + deps: SampleResourceSignalsDeps = {} +): Promise { + const readText = deps.fs?.readText ?? defaultReadText; + let memory: NodeJS.MemoryUsage; + try { + memory = (deps.memoryUsage ?? process.memoryUsage)(); + } catch { + memory = { rss: 0, heapTotal: 0, heapUsed: 0, external: 0, arrayBuffers: 0 }; + } + + let heapUsed = Math.max(0, Math.floor(memory.heapUsed || 0)); + let heapLimit = 0; + try { + const heap = (deps.heapStatistics ?? v8.getHeapStatistics)(); + heapLimit = sanitizeMemoryBytes(heap.heap_size_limit) ?? 0; + if (Number.isFinite(heap.used_heap_size)) { + heapUsed = Math.max(0, Math.floor(heap.used_heap_size ?? heapUsed)); + } + } catch { + /* retain process heap sample */ + } + + const cgroupDirectory = await resolveCgroupDirectory(readText); + const cgroupContents = cgroupDirectory + ? await Promise.all([ + readText(path.join(cgroupDirectory, "memory.current")), + readText(path.join(cgroupDirectory, "memory.max")), + readText(path.join(cgroupDirectory, "memory.high")), + readText(path.join(cgroupDirectory, "memory.events")), + ]) + : [null, null, null, null]; + const psi = await readText("/proc/pressure/memory").catch(() => null); + + return { + observedAtMs: (deps.nowMs ?? Date.now)(), + v8: { heapUsedBytes: heapUsed, heapLimitBytes: heapLimit }, + process: { + rssBytes: Math.max(0, Math.floor(memory.rss || 0)), + externalBytes: Math.max(0, Math.floor(memory.external || 0)), + arrayBuffersBytes: Math.max(0, Math.floor(memory.arrayBuffers || 0)), + availableBytes: safeNumber(deps.availableMemory ?? (() => process.availableMemory?.())), + constrainedBytes: safeNumber(deps.constrainedMemory ?? (() => process.constrainedMemory?.())), + }, + cgroup: { + currentBytes: sanitizeMemoryBytes(cgroupContents[0]), + maxBytes: sanitizeMemoryBytes(cgroupContents[1]), + highBytes: sanitizeMemoryBytes(cgroupContents[2]), + events: parseMemoryEvents(cgroupContents[3]), + }, + psi: parsePsi(psi), + }; +} diff --git a/tests/unit/adaptive-admission-controller.test.ts b/tests/unit/adaptive-admission-controller.test.ts new file mode 100644 index 0000000000..bd0785c0a3 --- /dev/null +++ b/tests/unit/adaptive-admission-controller.test.ts @@ -0,0 +1,985 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { + AdaptiveAdmissionController, + createAdmissionRejectError, + type AdaptiveAdmissionConfig, + type AdmissionLease, + type AdmissionPressure, + type AdmissionRequest, +} from "../../open-sse/services/admission/index.ts"; + +class FakeClock { + nowMs = 0; + private nextId = 1; + private timers = new Map void }>(); + + now = () => this.nowMs; + + setTimer = (fn: () => void, delayMs: number): number => { + const id = this.nextId++; + this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn }); + return id; + }; + + clearTimer = (id: number): void => { + this.timers.delete(id); + }; + + get pendingTimerCount(): number { + return this.timers.size; + } + + advance(ms: number): void { + const target = this.nowMs + ms; + while (true) { + let nextId: number | undefined; + let nextDue = Number.POSITIVE_INFINITY; + for (const [id, t] of this.timers) { + if (t.due <= target && t.due < nextDue) { + nextDue = t.due; + nextId = id; + } + } + if (nextId === undefined) { + this.nowMs = target; + return; + } + const timer = this.timers.get(nextId)!; + this.timers.delete(nextId); + this.nowMs = timer.due; + timer.fn(); + } + } +} + +function baseConfig(overrides: Partial = {}): AdaptiveAdmissionConfig { + return { + mode: "enforce", + minLimit: 10, + maxLimit: 100, + initialLimit: 20, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 1000, + windowMs: 100, + shortLatencyAlpha: 0.5, + longLatencyAlpha: 0.1, + increaseStep: 2, + decreaseFactor: 0.8, + criticalDecreaseFactor: 0.5, + highUtilizationThreshold: 0.7, + lowUtilizationThreshold: 0.3, + latencyGradientThreshold: 0.25, + maxIncreasePerWindow: 4, + ...overrides, + }; +} + +function req(partial: Partial & { cost: number }): AdmissionRequest { + return { + tenantKey: "t-default", + ...partial, + }; +} + +async function mustAdmit( + controller: AdaptiveAdmissionController, + request: AdmissionRequest +): Promise { + const result = await controller.acquire(request); + assert.equal(result.status, "admitted"); + if (result.status !== "admitted") throw new Error("expected admitted"); + return result.lease; +} + +describe("AdaptiveAdmissionController config and modes", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + function make(overrides: Partial = {}) { + return new AdaptiveAdmissionController(baseConfig(overrides), { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }); + } + + it("validates safe-integer bounds and ordered adaptation parameters", () => { + assert.throws(() => make({ minLimit: 50, maxLimit: 10 }), /minLimit/); + for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) { + assert.throws(() => make({ maxQueueCount: invalid }), /maxQueueCount/); + assert.throws(() => make({ initialLimit: invalid }), /initialLimit/); + } + assert.throws(() => make({ decreaseFactor: 1.2 }), /decreaseFactor/); + assert.throws( + () => make({ decreaseFactor: 0.5, criticalDecreaseFactor: 0.8 }), + /criticalDecreaseFactor/ + ); + assert.throws( + () => make({ lowUtilizationThreshold: 0.8, highUtilizationThreshold: 0.7 }), + /lowUtilizationThreshold/ + ); + assert.throws( + () => make({ shortLatencyAlpha: 0.1, longLatencyAlpha: 0.5 }), + /shortLatencyAlpha/ + ); + }); + + it("clamps initial limit into [minLimit, maxLimit]", () => { + const low = make({ initialLimit: 1, minLimit: 10 }); + assert.equal(low.snapshot().currentLimit, 10); + low.shutdown(); + const high = make({ initialLimit: 999, maxLimit: 100 }); + assert.equal(high.snapshot().currentLimit, 100); + high.shutdown(); + }); + + it("mode off never accounts cost or rejects", async () => { + const c = make({ mode: "off", initialLimit: 5 }); + const a = await c.acquire(req({ cost: 100 })); + const b = await c.acquire(req({ cost: 100 })); + assert.equal(a.status, "admitted"); + assert.equal(b.status, "admitted"); + const snap = c.snapshot(); + assert.equal(snap.activeCost, 0); + assert.equal(snap.activeCount, 0); + assert.equal(snap.rejectedCount, 0); + c.shutdown(); + }); + + it("defaults to shadow mode when mode omitted", () => { + const c = new AdaptiveAdmissionController( + { + minLimit: 10, + maxLimit: 100, + initialLimit: 20, + maxQueueCount: 2, + maxQueueCost: 20, + } as AdaptiveAdmissionConfig, + { now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer } + ); + assert.equal(c.snapshot().mode, "shadow"); + c.shutdown(); + }); +}); + +describe("shadow mode semantics", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + it("never rejects or delays while recording would-decisions and real active cost", async () => { + const c = new AdaptiveAdmissionController( + baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 10 }), + { now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer } + ); + + const first = await c.acquire(req({ cost: 8 })); + assert.equal(first.status, "admitted"); + if (first.status !== "admitted") return; + assert.equal(first.shadowDecision, "would-admit"); + assert.equal(c.snapshot().activeCost, 8); + + const second = await c.acquire(req({ cost: 8 })); + assert.equal(second.status, "admitted"); + if (second.status !== "admitted") return; + // Would have queued under enforce (active 8 + 8 > 10) but shadow admits immediately. + assert.equal(second.shadowDecision, "would-queue"); + assert.equal(c.snapshot().activeCost, 16); + assert.equal(c.snapshot().queuedCount, 0); + assert.ok((c.snapshot().wouldQueueCount ?? 0) >= 1); + + const oversized = await c.acquire(req({ cost: 50 })); + assert.equal(oversized.status, "admitted"); + if (oversized.status !== "admitted") return; + assert.equal(oversized.shadowDecision, "would-reject"); + assert.ok((c.snapshot().wouldRejectCount ?? 0) >= 1); + + first.lease.release("success"); + second.lease.release("success"); + oversized.lease.release("success"); + assert.equal(c.snapshot().activeCost, 0); + c.shutdown(); + }); + + it("simulates virtual queue saturation and promotes queued work on release", async () => { + const c = new AdaptiveAdmissionController( + baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 8 }), + { now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer } + ); + const active = await c.acquire(req({ cost: 8, tenantKey: "active" })); + const queued = await c.acquire(req({ cost: 8, tenantKey: "queued" })); + const saturated = await c.acquire(req({ cost: 8, tenantKey: "saturated" })); + assert.equal(active.status, "admitted"); + assert.equal(queued.status, "admitted"); + assert.equal(saturated.status, "admitted"); + if ( + active.status !== "admitted" || + queued.status !== "admitted" || + saturated.status !== "admitted" + ) { + return; + } + assert.equal(active.shadowDecision, "would-admit"); + assert.equal(queued.shadowDecision, "would-queue"); + assert.equal(saturated.shadowDecision, "would-reject"); + assert.deepEqual( + { + activeCost: c.snapshot().virtualActiveCost, + activeCount: c.snapshot().virtualActiveCount, + queuedCost: c.snapshot().virtualQueuedCost, + queuedCount: c.snapshot().virtualQueuedCount, + }, + { activeCost: 8, activeCount: 1, queuedCost: 8, queuedCount: 1 } + ); + + active.lease.release(); + assert.deepEqual( + { + activeCost: c.snapshot().virtualActiveCost, + activeCount: c.snapshot().virtualActiveCount, + queuedCost: c.snapshot().virtualQueuedCost, + queuedCount: c.snapshot().virtualQueuedCount, + }, + { activeCost: 8, activeCount: 1, queuedCost: 0, queuedCount: 0 } + ); + queued.lease.release(); + saturated.lease.release(); + c.shutdown(); + }); + + it("promotes shadow virtual queue after adaptation raises the limit", async () => { + const c = new AdaptiveAdmissionController( + baseConfig({ + mode: "shadow", + minLimit: 10, + maxLimit: 20, + initialLimit: 10, + maxQueueCount: 4, + maxQueueCost: 40, + windowMs: 100, + increaseStep: 5, + maxIncreasePerWindow: 5, + highUtilizationThreshold: 0.5, + }), + { now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer } + ); + + const active = await c.acquire(req({ cost: 10, tenantKey: "active" })); + const queued = await c.acquire(req({ cost: 5, tenantKey: "queued" })); + assert.equal(active.status, "admitted"); + assert.equal(queued.status, "admitted"); + if (active.status !== "admitted" || queued.status !== "admitted") return; + assert.equal(active.shadowDecision, "would-admit"); + assert.equal(queued.shadowDecision, "would-queue"); + assert.equal(c.snapshot().virtualActiveCost, 10); + assert.equal(c.snapshot().virtualQueuedCost, 5); + + // Raise the adaptive limit once while both leases remain open. Shadow admits a + // probe for completion evidence; active integral is capped at the current limit. + const probe = await c.acquire(req({ cost: 1, tenantKey: "probe" })); + assert.equal(probe.status, "admitted"); + if (probe.status === "admitted") { + clock.advance(80); + probe.lease.release("success", { latencyMs: 10 }); + clock.advance(20); + c.tick(); + } + + assert.equal(c.snapshot().currentLimit, 15); + // Queued virtual work must be promoted before newer arrivals are classified. + assert.equal(c.snapshot().virtualActiveCost, 15); + assert.equal(c.snapshot().virtualQueuedCost, 0); + + const later = await c.acquire(req({ cost: 5, tenantKey: "later" })); + assert.equal(later.status, "admitted"); + if (later.status !== "admitted") return; + // With virtual active already 15 at limit 15, a later cost-5 cannot would-admit. + assert.notEqual(later.shadowDecision, "would-admit"); + + active.lease.release(); + queued.lease.release(); + later.lease.release(); + c.shutdown(); + }); +}); + +describe("weighted enforce, queue, fairness, and races", () => { + let clock: FakeClock; + const live: AdaptiveAdmissionController[] = []; + beforeEach(() => { + clock = new FakeClock(); + live.length = 0; + }); + afterEach(() => { + for (const c of live) c.shutdown(); + live.length = 0; + }); + + function controller(overrides: Partial = {}) { + const c = new AdaptiveAdmissionController(baseConfig(overrides), { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }); + live.push(c); + return c; + } + + it("enforces weighted active-cost budget and rejects oversized requests immediately", async () => { + const c = controller({ initialLimit: 20 }); + const a = await mustAdmit(c, req({ cost: 12 })); + const b = await c.acquire(req({ cost: 12 })); + assert.equal(b.status, "queued"); + + const over = await c.acquire(req({ cost: 25 })); + assert.equal(over.status, "rejected"); + if (over.status === "rejected") { + assert.equal(over.code, "ADMISSION_OVERSIZED"); + } + + a.release("success"); + if (b.status === "queued") { + const admitted = await b.promise; + assert.equal(admitted.status, "admitted"); + admitted.lease.release("success"); + } + }); + + it("bounds queue by count and total queued cost", async () => { + const c = controller({ + minLimit: 10, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 2, + maxQueueCost: 15, + }); + const held = await mustAdmit(c, req({ cost: 10 })); + + const q1 = await c.acquire(req({ cost: 5, tenantKey: "a" })); + const q2 = await c.acquire(req({ cost: 5, tenantKey: "b" })); + assert.equal(q1.status, "queued"); + assert.equal(q2.status, "queued"); + assert.equal(c.snapshot().queuedCount, 2); + assert.equal(c.snapshot().queuedCost, 10); + + const byCount = await c.acquire(req({ cost: 1, tenantKey: "c" })); + assert.equal(byCount.status, "rejected"); + if (byCount.status === "rejected") assert.equal(byCount.code, "ADMISSION_QUEUE_FULL"); + + held.release("success"); + if (q1.status === "queued") (await q1.promise).lease.release("success"); + if (q2.status === "queued") (await q2.promise).lease.release("success"); + + const c2 = controller({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 10, + maxQueueCost: 7, + }); + const h = await mustAdmit(c2, req({ cost: 5 })); + // cost 3 fits limit but not active budget → queued (queuedCost=3). + // Another cost 5 fits the budget but 3+5 > maxQueueCost=7 → QUEUE_FULL. + const ok = await c2.acquire(req({ cost: 3 })); + assert.equal(ok.status, "queued"); + const costFull = await c2.acquire(req({ cost: 5 })); + assert.equal(costFull.status, "rejected"); + if (costFull.status === "rejected") assert.equal(costFull.code, "ADMISSION_QUEUE_FULL"); + h.release("success"); + if (ok.status === "queued") (await ok.promise).lease.release("success"); + }); + + it("expires deadline and abort without leaking queue slots", async () => { + const c = controller({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 50, + }); + const held = await mustAdmit(c, req({ cost: 5 })); + + const timed = await c.acquire(req({ cost: 3, maxWaitMs: 30 })); + assert.equal(timed.status, "queued"); + clock.advance(31); + if (timed.status === "queued") { + await assert.rejects(timed.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE"); + return true; + }); + } + assert.equal(c.snapshot().queuedCount, 0); + + const ac = new AbortController(); + const aborted = await c.acquire(req({ cost: 3, signal: ac.signal })); + assert.equal(aborted.status, "queued"); + ac.abort(); + if (aborted.status === "queued") { + await assert.rejects(aborted.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED"); + return true; + }); + } + assert.equal(c.snapshot().queuedCount, 0); + held.release("success"); + }); + + it("treats the exact deadline as expired and settles abort/release races once", async () => { + const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 5, defaultMaxWaitMs: 30 }); + const held = await mustAdmit(c, req({ cost: 5 })); + const ac = new AbortController(); + const queued = await c.acquire(req({ cost: 3, maxWaitMs: 30, signal: ac.signal })); + assert.equal(queued.status, "queued"); + + clock.advance(30); + ac.abort(); + held.release("success"); + + if (queued.status === "queued") { + await assert.rejects(queued.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE"); + return true; + }); + } + assert.equal(c.snapshot().queuedCount, 0); + assert.equal(c.snapshot().rejectedCount, 1); + }); + + it("shutdown rejects queued work and clears every fake-clock timer", async () => { + const c = controller({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 4, + maxQueueCost: 40, + }); + const held = await mustAdmit(c, req({ cost: 5 })); + const q = await c.acquire(req({ cost: 3, maxWaitMs: 5000 })); + assert.equal(q.status, "queued"); + assert.ok(clock.pendingTimerCount >= 2); + c.shutdown(); + if (q.status === "queued") { + await assert.rejects(q.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_SHUTDOWN"); + return true; + }); + } + assert.equal(c.snapshot().queuedCount, 0); + assert.equal(clock.pendingTimerCount, 0); + held.release("success"); + const after = await c.acquire(req({ cost: 1 })); + assert.equal(after.status, "rejected"); + if (after.status === "rejected") assert.equal(after.code, "ADMISSION_SHUTDOWN"); + }); + + it("updateConfig atomically settles queues, dispatches raised capacity, and respects decreases", async () => { + const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 20, windowMs: 100 }); + const held = await mustAdmit(c, req({ cost: 5 })); + const queued = await c.acquire(req({ cost: 5 })); + assert.equal(queued.status, "queued"); + + c.updateConfig(baseConfig({ minLimit: 10, initialLimit: 10, maxLimit: 20, windowMs: 50 })); + assert.equal(clock.pendingTimerCount, 1); + if (queued.status === "queued") { + const admitted = await queued.promise; + assert.equal(c.snapshot().activeCost, 10); + + c.updateConfig(baseConfig({ minLimit: 5, initialLimit: 5, maxLimit: 5, windowMs: 50 })); + const afterDecrease = await c.acquire(req({ cost: 1 })); + assert.equal(afterDecrease.status, "queued"); + + c.updateConfig(baseConfig({ mode: "shadow", minLimit: 5, initialLimit: 5, maxLimit: 5 })); + if (afterDecrease.status === "queued") { + const settled = await afterDecrease.promise; + assert.equal(settled.status, "admitted"); + settled.lease.release(); + } + admitted.lease.release(); + } + held.release(); + }); + + it("queue shrink rejects deterministic round-robin excess and preserves fitting entries", async () => { + const c = controller({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 4, + maxQueueCost: 20, + }); + const held = await mustAdmit(c, req({ cost: 5 })); + const first = await c.acquire(req({ cost: 2, tenantKey: "a" })); + const second = await c.acquire(req({ cost: 2, tenantKey: "b" })); + const third = await c.acquire(req({ cost: 2, tenantKey: "a" })); + assert.equal(first.status, "queued"); + assert.equal(second.status, "queued"); + assert.equal(third.status, "queued"); + + c.updateConfig( + baseConfig({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 2, + maxQueueCost: 4, + }) + ); + assert.equal(c.snapshot().queuedCount, 2); + if (third.status === "queued") { + await assert.rejects(third.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_QUEUE_FULL"); + return true; + }); + } + held.release(); + if (first.status === "queued") (await first.promise).lease.release(); + if (second.status === "queued") (await second.promise).lease.release(); + }); + + it("release is idempotent under race with abort", async () => { + const c = controller({ initialLimit: 10 }); + const lease = await mustAdmit(c, req({ cost: 4 })); + lease.release("success"); + lease.release("timeout"); + lease.release("success"); + assert.equal(c.snapshot().activeCost, 0); + assert.equal(c.snapshot().activeCount, 0); + }); + + it("fairly schedules across tenants under skew without exposing tenant ids", async () => { + const c = controller({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 10, + maxQueueCost: 100, + }); + const held = await mustAdmit(c, req({ cost: 5, tenantKey: "hold" })); + + const order: string[] = []; + const queued: Array> = []; + for (let i = 0; i < 4; i++) { + const r = await c.acquire(req({ cost: 5, tenantKey: "heavy" })); + assert.equal(r.status, "queued"); + if (r.status === "queued") { + queued.push( + r.promise.then((admitted) => { + order.push("heavy"); + admitted.lease.release("success"); + }) + ); + } + } + const light = await c.acquire(req({ cost: 5, tenantKey: "light" })); + assert.equal(light.status, "queued"); + if (light.status === "queued") { + queued.push( + light.promise.then((admitted) => { + order.push("light"); + admitted.lease.release("success"); + }) + ); + } + + // Free capacity one slot at a time. + held.release("success"); + await Promise.resolve(); + // After first release, one request should admit; keep draining by waiting microtasks between releases. + // Drain remaining by letting each admitted release free the next. + await Promise.all(queued); + + // Light must not be starved behind all four heavy requests. + const lightIndex = order.indexOf("light"); + assert.ok(lightIndex >= 0); + assert.ok(lightIndex < 4, `light scheduled too late: ${order.join(",")}`); + + const snap = c.snapshot(); + const json = JSON.stringify(snap); + assert.equal(json.includes("heavy"), false); + assert.equal(json.includes("light"), false); + assert.equal(json.includes("hold"), false); + }); + + it("dispatches a fitting tenant when another tenant's queue head cannot fit", async () => { + const c = controller({ + minLimit: 10, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 10, + maxQueueCost: 100, + }); + const heldSix = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" })); + const heldFour = await mustAdmit(c, req({ cost: 4, tenantKey: "holder" })); + const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" })); + const fitting = await c.acquire(req({ cost: 4, tenantKey: "fitting" })); + assert.equal(expensive.status, "queued"); + assert.equal(fitting.status, "queued"); + + heldFour.release("success"); + if (fitting.status === "queued") { + const admitted = await fitting.promise; + assert.equal(admitted.lease.cost, 4); + admitted.lease.release("success"); + } + assert.equal(c.snapshot().queuedCount, 1); + + heldSix.release("success"); + if (expensive.status === "queued") (await expensive.promise).lease.release("success"); + }); + + it("bounds starvation of an older unfittable cost-6 behind a stream of cost-2 work", async () => { + const c = controller({ + minLimit: 10, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 20, + maxQueueCost: 100, + defaultMaxWaitMs: 10_000, + }); + // Hold 6 so available=4: cost-2 can pass over cost-6 until reservation engages. + const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" })); + + const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" })); + assert.equal(expensive.status, "queued"); + + // Two actual smaller dequeues through queued promises (pass-overs that age the head). + const passOvers: AdmissionLease[] = []; + for (let i = 0; i < 2; i++) { + const r = await c.acquire(req({ cost: 2, tenantKey: `small-pass-${i}` })); + assert.equal(r.status, "queued", `pass-over ${i} should join the non-empty queue`); + if (r.status !== "queued") throw new Error("expected queued"); + const admitted = await r.promise; + assert.equal(admitted.lease.cost, 2); + passOvers.push(admitted.lease); + admitted.lease.release("success"); + assert.equal(c.snapshot().activeCost, 6); + } + assert.equal(passOvers.length, 2); + + // A subsequent fitting cost-2 must remain queued: capacity is reserved for cost-6. + // Without reservation accounting this would admit immediately and the assertion fails. + const blocked = await c.acquire(req({ cost: 2, tenantKey: "small-blocked" })); + assert.equal(blocked.status, "queued"); + await Promise.resolve(); + assert.equal(c.snapshot().activeCost, 6, "reserved head must block fitting smaller work"); + assert.equal(c.snapshot().queuedCount, 2); + + const order: number[] = []; + assert.equal(expensive.status, "queued"); + assert.equal(blocked.status, "queued"); + const expensiveDone = expensive.promise.then((admitted) => { + order.push(admitted.lease.cost); + return admitted; + }); + const blockedDone = blocked.promise.then((admitted) => { + order.push(admitted.lease.cost); + return admitted; + }); + + // Free enough capacity for cost-6; the older reserved request must admit first. + // With activeCost back at 0 both may fit in one dispatch turn, so only order is asserted. + held.release("success"); + const [expAdmitted, blockedAdmitted] = await Promise.all([expensiveDone, blockedDone]); + assert.equal(expAdmitted.lease.cost, 6); + assert.equal(blockedAdmitted.lease.cost, 2); + assert.deepEqual(order, [6, 2]); + expAdmitted.lease.release("success"); + blockedAdmitted.lease.release("success"); + assert.equal(c.snapshot().queuedCount, 0); + }); + + async function ageReservedCost6( + c: AdaptiveAdmissionController, + expensiveSignal?: AbortSignal, + expensiveMaxWaitMs?: number + ) { + const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" })); + const expensive = await c.acquire( + req({ + cost: 6, + tenantKey: "expensive", + signal: expensiveSignal, + maxWaitMs: expensiveMaxWaitMs, + }) + ); + assert.equal(expensive.status, "queued"); + for (let i = 0; i < 2; i++) { + const r = await c.acquire(req({ cost: 2, tenantKey: `age-pass-${i}` })); + assert.equal(r.status, "queued"); + if (r.status !== "queued") throw new Error("expected queued"); + (await r.promise).lease.release("success"); + } + const blocked = await c.acquire(req({ cost: 2, tenantKey: "age-blocked" })); + assert.equal(blocked.status, "queued"); + await Promise.resolve(); + assert.equal(c.snapshot().activeCost, 6); + assert.equal(c.snapshot().queuedCount, 2); + return { held, expensive, blocked }; + } + + it("aborting a reserved head immediately admits the next fitting request", async () => { + const c = controller({ + minLimit: 10, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 20, + maxQueueCost: 100, + defaultMaxWaitMs: 10_000, + }); + const ac = new AbortController(); + const { held, expensive, blocked } = await ageReservedCost6(c, ac.signal); + assert.equal(expensive.status, "queued"); + assert.equal(blocked.status, "queued"); + + ac.abort(); + // No tick / new arrival / release / config update — only the abort path. + if (expensive.status === "queued") { + await assert.rejects(expensive.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED"); + return true; + }); + } + if (blocked.status !== "queued") throw new Error("expected queued blocked request"); + const admitted = await blocked.promise; + assert.equal(admitted.lease.cost, 2); + assert.equal(c.snapshot().activeCost, 8); + assert.equal(c.snapshot().queuedCount, 0); + admitted.lease.release("success"); + held.release("success"); + }); + + it("deadline-expiring a reserved head immediately admits the next fitting request", async () => { + const c = controller({ + minLimit: 10, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 20, + maxQueueCost: 100, + defaultMaxWaitMs: 10_000, + }); + const { held, expensive, blocked } = await ageReservedCost6(c, undefined, 40); + assert.equal(expensive.status, "queued"); + assert.equal(blocked.status, "queued"); + + clock.advance(40); + // No tick / new arrival / release / config update — only the deadline timer. + if (expensive.status === "queued") { + await assert.rejects(expensive.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE"); + return true; + }); + } + if (blocked.status !== "queued") throw new Error("expected queued blocked request"); + const admitted = await blocked.promise; + assert.equal(admitted.lease.cost, 2); + assert.equal(c.snapshot().activeCost, 8); + assert.equal(c.snapshot().queuedCount, 0); + admitted.lease.release("success"); + held.release("success"); + }); +}); + +describe("adaptive algorithm", () => { + let clock: FakeClock; + const live: AdaptiveAdmissionController[] = []; + beforeEach(() => { + clock = new FakeClock(); + live.length = 0; + }); + afterEach(() => { + for (const c of live) c.shutdown(); + live.length = 0; + }); + + function controller(overrides: Partial = {}) { + const c = new AdaptiveAdmissionController(baseConfig(overrides), { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }); + live.push(c); + return c; + } + + async function complete( + c: AdaptiveAdmissionController, + cost: number, + latencyMs: number, + outcome: "success" | "upstream_error" | "timeout" = "success", + pressure: AdmissionPressure = "normal" + ) { + const lease = await mustAdmit(c, req({ cost, pressure })); + clock.advance(latencyMs); + lease.release(outcome, { latencyMs, pressure }); + } + + it("keeps currentLimit within validated bounds", async () => { + const c = controller({ initialLimit: 20, minLimit: 10, maxLimit: 30, increaseStep: 50 }); + for (let i = 0; i < 20; i++) { + await complete(c, 5, 5, "success", "normal"); + clock.advance(100); + } + assert.ok(c.snapshot().currentLimit <= 30); + assert.ok(c.snapshot().currentLimit >= 10); + + for (let i = 0; i < 10; i++) { + await complete(c, 5, 5, "success", "critical"); + clock.advance(100); + } + assert.ok(c.snapshot().currentLimit >= 10); + }); + + it("decreases rapidly under critical pressure", async () => { + const c = controller({ initialLimit: 80, minLimit: 10, maxLimit: 100 }); + const before = c.snapshot().currentLimit; + await complete(c, 10, 10, "success", "critical"); + clock.advance(100); + // Force a window tick with pressure observation. + c.observePressure("critical"); + clock.advance(100); + assert.ok(c.snapshot().currentLimit < before); + assert.ok(c.snapshot().currentLimit <= Math.ceil(before * 0.5) + 1); + }); + + it("applies criticalDecreaseFactor once for a single observePressure(critical)", () => { + const c = controller({ + initialLimit: 80, + minLimit: 10, + maxLimit: 100, + criticalDecreaseFactor: 0.5, + decreaseFactor: 0.8, + windowMs: 100, + }); + assert.equal(c.snapshot().currentLimit, 80); + + c.observePressure("critical"); + // Immediate fast decrease: 80 * 0.5 = 40. + assert.equal(c.snapshot().currentLimit, 40); + + // Closing the same window must not multiply again (would become 20). + clock.advance(100); + c.tick(); + assert.equal(c.snapshot().currentLimit, 40); + + // A fresh critical observation in a later window still decreases once. + c.observePressure("critical"); + assert.equal(c.snapshot().currentLimit, 20); + clock.advance(100); + c.tick(); + assert.equal(c.snapshot().currentLimit, 20); + }); + + it("decreases on high pressure or sustained latency gradient", async () => { + const c = controller({ + initialLimit: 50, + shortLatencyAlpha: 0.8, + longLatencyAlpha: 0.1, + latencyGradientThreshold: 0.2, + }); + // Seed long baseline with low latency. + for (let i = 0; i < 5; i++) { + await complete(c, 8, 10, "success", "normal"); + clock.advance(100); + } + const mid = c.snapshot().currentLimit; + // Spike short latency relative to long. + for (let i = 0; i < 5; i++) { + await complete(c, 8, 200, "success", "normal"); + clock.advance(100); + } + assert.ok(c.snapshot().currentLimit <= mid); + + const beforeHigh = c.snapshot().currentLimit; + c.observePressure("high"); + await complete(c, 8, 20, "success", "high"); + clock.advance(100); + assert.ok(c.snapshot().currentLimit <= beforeHigh); + }); + + it("increases slowly when healthy and highly utilized, and does not inflate when idle", async () => { + const c = controller({ + minLimit: 20, + maxLimit: 40, + initialLimit: 20, + increaseStep: 2, + maxIncreasePerWindow: 2, + highUtilizationThreshold: 0.5, + windowMs: 100, + }); + + // Idle windows should not inflate. + clock.advance(500); + c.tick(); + clock.advance(500); + c.tick(); + assert.equal(c.snapshot().currentLimit, 20); + + // Healthy high utilization: hold nearly full budget across most of each window. + for (let w = 0; w < 5; w++) { + const lease = await mustAdmit(c, req({ cost: 16, pressure: "normal" })); + clock.advance(80); + lease.release("success", { latencyMs: 10, pressure: "normal" }); + clock.advance(20); + c.tick(); + } + assert.ok(c.snapshot().currentLimit > 20); + assert.ok(c.snapshot().currentLimit <= 20 + 2 * 5); + }); + + it("does not collapse capacity on a single upstream business error", async () => { + const c = controller({ initialLimit: 40, decreaseFactor: 0.5, criticalDecreaseFactor: 0.5 }); + await complete(c, 10, 15, "upstream_error", "normal"); + clock.advance(100); + c.tick(); + // One business error may freeze growth but must not apply critical collapse. + assert.ok(c.snapshot().currentLimit >= 30); + }); + + it("integrates active utilization exactly once over a full window", async () => { + const c = controller({ + minLimit: 20, + initialLimit: 20, + maxLimit: 20, + highUtilizationThreshold: 0.9, + windowMs: 100, + }); + const lease = await mustAdmit(c, req({ cost: 8 })); + clock.advance(100); + assert.equal(c.snapshot().utilization, 0.4); + lease.release("success"); + }); + + it("consumes latency and pressure evidence only in the window where it was observed", async () => { + const c = controller({ + initialLimit: 80, + minLimit: 10, + maxLimit: 100, + decreaseFactor: 0.5, + criticalDecreaseFactor: 0.25, + windowMs: 100, + }); + + await complete(c, 8, 200, "success", "high"); + clock.advance(100); + const afterObservedWindow = c.snapshot().currentLimit; + assert.ok(afterObservedWindow < 80); + + clock.advance(500); + assert.equal(c.snapshot().currentLimit, afterObservedWindow); + }); +}); + +describe("createAdmissionRejectError", () => { + it("builds typed rejection errors", () => { + const err = createAdmissionRejectError("ADMISSION_QUEUE_FULL", "queue full"); + assert.equal(err.code, "ADMISSION_QUEUE_FULL"); + assert.equal(err.name, "AdmissionRejectError"); + assert.match(err.message, /queue full/); + }); +}); diff --git a/tests/unit/adaptive-admission-cost.test.ts b/tests/unit/adaptive-admission-cost.test.ts new file mode 100644 index 0000000000..620aa15f54 --- /dev/null +++ b/tests/unit/adaptive-admission-cost.test.ts @@ -0,0 +1,143 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + estimateAdmissionCost, + DEFAULT_ADMISSION_COST_CONFIG, + MAX_ADMISSION_COST_OR_LIMIT, + normalizeRequestCost, + resolveCostConfig, + type AdmissionCostConfig, + type AdmissionCostFeatures, +} from "../../open-sse/services/admission/index.ts"; + +function features(overrides: Partial = {}): AdmissionCostFeatures { + return { + bodyBytes: 0, + estimatedInputTokens: 0, + messageCount: 0, + toolCount: 0, + requestedFanout: 1, + streaming: true, + ...overrides, + }; +} + +describe("estimateAdmissionCost", () => { + it("returns a positive integer at least the base cost", () => { + const cost = estimateAdmissionCost(features()); + assert.equal(Number.isSafeInteger(cost), true); + assert.ok(cost >= DEFAULT_ADMISSION_COST_CONFIG.baseCost); + assert.ok(cost > 0); + }); + + it("is monotonic in body bytes, tokens, messages, tools, and fanout", () => { + const base = estimateAdmissionCost(features()); + assert.ok(estimateAdmissionCost(features({ bodyBytes: 50_000 })) >= base); + assert.ok(estimateAdmissionCost(features({ estimatedInputTokens: 8_000 })) >= base); + assert.ok(estimateAdmissionCost(features({ messageCount: 40 })) >= base); + assert.ok(estimateAdmissionCost(features({ toolCount: 20 })) >= base); + assert.ok(estimateAdmissionCost(features({ requestedFanout: 8 })) >= base); + }); + + it("rejects fractional, infinite, and unsafe cost configuration", () => { + for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) { + assert.throws( + () => resolveCostConfig({ bodyBytesPerUnit: invalid }), + /positive safe integer/ + ); + assert.throws(() => resolveCostConfig({ maxRequestCost: invalid }), /positive safe integer/); + assert.throws( + () => resolveCostConfig({ streamingClassCost: invalid }), + /positive safe integer/ + ); + } + }); + + it("normalizes caller costs only when they are positive safe integers", () => { + assert.equal(normalizeRequestCost(7, 10), 7); + for (const invalid of [0, -1, 0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) { + assert.throws(() => normalizeRequestCost(invalid, 10), /positive safe integer/); + } + assert.throws(() => normalizeRequestCost(1, Number.MAX_VALUE), /positive safe integer/); + assert.throws( + () => normalizeRequestCost(1, MAX_ADMISSION_COST_OR_LIMIT + 1), + /maxRequestCost|must be <=/ + ); + assert.equal( + normalizeRequestCost(MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_COST_OR_LIMIT), + MAX_ADMISSION_COST_OR_LIMIT + ); + }); + + it("clamps multiple overflowing feature contributions without unsafe arithmetic", () => { + const config: AdmissionCostConfig = { + ...DEFAULT_ADMISSION_COST_CONFIG, + maxRequestCost: 25, + }; + const cost = estimateAdmissionCost( + features({ + bodyBytes: Number.MAX_SAFE_INTEGER, + estimatedInputTokens: Number.MAX_SAFE_INTEGER, + messageCount: Number.MAX_SAFE_INTEGER, + toolCount: Number.MAX_SAFE_INTEGER, + requestedFanout: Number.MAX_SAFE_INTEGER, + }), + config + ); + assert.equal(cost, 25); + }); + + it("normalizes invalid, negative, and NaN inputs safely", () => { + const cost = estimateAdmissionCost({ + bodyBytes: Number.NaN, + estimatedInputTokens: -12, + messageCount: Number.POSITIVE_INFINITY, + toolCount: undefined, + requestedFanout: 0, + streaming: undefined, + } as AdmissionCostFeatures); + assert.equal(Number.isSafeInteger(cost), true); + assert.ok(cost >= 1); + assert.ok(cost <= DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost); + }); + + it("uses transparent configurable quanta without a fixed-MB claim", () => { + const config: AdmissionCostConfig = { + baseCost: 1, + bodyBytesPerUnit: 1000, + tokensPerUnit: 100, + messagesPerUnit: 10, + toolsPerUnit: 5, + fanoutPerUnit: 1, + streamingClassCost: 1, + nonStreamingClassCost: 3, + maxRequestCost: 1000, + }; + // 2500 bytes → 3 units (ceil), 250 tokens → 3 units, 1 message → 1, 0 tools, fanout 1 → 1, streaming class 1 + const cost = estimateAdmissionCost( + features({ + bodyBytes: 2500, + estimatedInputTokens: 250, + messageCount: 1, + toolCount: 0, + requestedFanout: 1, + streaming: true, + }), + config + ); + assert.equal(cost, 1 + 3 + 3 + 1 + 0 + 1 + 1); + + const nonStream = estimateAdmissionCost( + features({ + bodyBytes: 0, + estimatedInputTokens: 0, + messageCount: 0, + toolCount: 0, + requestedFanout: 1, + streaming: false, + }), + config + ); + assert.equal(nonStream, 1 + 0 + 0 + 0 + 0 + 1 + 3); + }); +}); diff --git a/tests/unit/adaptive-admission-domain.test.ts b/tests/unit/adaptive-admission-domain.test.ts new file mode 100644 index 0000000000..00c11abbac --- /dev/null +++ b/tests/unit/adaptive-admission-domain.test.ts @@ -0,0 +1,405 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { + AdaptiveAdmissionController, + MAX_ADMISSION_COST_OR_LIMIT, + MAX_ADMISSION_WINDOW_MS, + type AdaptiveAdmissionConfig, + type AdmissionLease, + type AdmissionRequest, +} from "../../open-sse/services/admission/index.ts"; + +class FakeClock { + nowMs = 0; + private nextId = 1; + private timers = new Map void }>(); + + now = () => this.nowMs; + + setTimer = (fn: () => void, delayMs: number): number => { + const id = this.nextId++; + this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn }); + return id; + }; + + clearTimer = (id: number): void => { + this.timers.delete(id); + }; + + get pendingTimerCount(): number { + return this.timers.size; + } + + advance(ms: number): void { + const target = this.nowMs + ms; + while (true) { + let nextId: number | undefined; + let nextDue = Number.POSITIVE_INFINITY; + for (const [id, t] of this.timers) { + if (t.due <= target && t.due < nextDue) { + nextDue = t.due; + nextId = id; + } + } + if (nextId === undefined) { + this.nowMs = target; + return; + } + const timer = this.timers.get(nextId)!; + this.timers.delete(nextId); + this.nowMs = timer.due; + timer.fn(); + } + } +} + +function baseConfig(overrides: Partial = {}): AdaptiveAdmissionConfig { + return { + mode: "enforce", + minLimit: 10, + maxLimit: 100, + initialLimit: 20, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 1000, + windowMs: 100, + shortLatencyAlpha: 0.5, + longLatencyAlpha: 0.1, + increaseStep: 2, + decreaseFactor: 0.8, + criticalDecreaseFactor: 0.5, + highUtilizationThreshold: 0.7, + lowUtilizationThreshold: 0.3, + latencyGradientThreshold: 0.25, + maxIncreasePerWindow: 4, + ...overrides, + }; +} + +function req(partial: Partial & { cost: number }): AdmissionRequest { + return { + tenantKey: "t-default", + ...partial, + }; +} + +async function mustAdmit( + controller: AdaptiveAdmissionController, + request: AdmissionRequest +): Promise { + const result = await controller.acquire(request); + assert.equal(result.status, "admitted"); + if (result.status !== "admitted") throw new Error("expected admitted"); + return result.lease; +} + +describe("admission operational domain", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + function make(overrides: Partial = {}) { + return new AdaptiveAdmissionController(baseConfig(overrides), { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }); + } + + it("exports an operational domain that rejects max+1 and accepts exact max", () => { + assert.ok(Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT)); + assert.ok(Number.isSafeInteger(MAX_ADMISSION_WINDOW_MS)); + assert.ok( + Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS), + "limit×window must remain a safe integer" + ); + assert.throws( + () => + make({ + minLimit: MAX_ADMISSION_COST_OR_LIMIT + 1, + maxLimit: MAX_ADMISSION_COST_OR_LIMIT + 1, + initialLimit: MAX_ADMISSION_COST_OR_LIMIT + 1, + }), + /minLimit|must be <=/ + ); + assert.throws( + () => make({ maxQueueCost: MAX_ADMISSION_COST_OR_LIMIT + 1 }), + /maxQueueCost|must be <=/ + ); + assert.throws(() => make({ windowMs: MAX_ADMISSION_WINDOW_MS + 1 }), /windowMs|must be <=/); + assert.throws( + () => make({ cost: { maxRequestCost: MAX_ADMISSION_COST_OR_LIMIT + 1 } }), + /maxRequestCost|must be <=/ + ); + assert.throws(() => make({ maxLimit: Number.MAX_SAFE_INTEGER }), /maxLimit|must be <=/); + + const max = MAX_ADMISSION_COST_OR_LIMIT; + const c = make({ + minLimit: max, + maxLimit: max, + initialLimit: max, + maxQueueCount: 2, + maxQueueCost: max, + windowMs: 1000, + cost: { maxRequestCost: max }, + }); + assert.equal(c.snapshot().currentLimit, max); + c.shutdown(); + }); + + it("keeps full utilization and multi-lease accounting exact at the domain max", async () => { + const max = MAX_ADMISSION_COST_OR_LIMIT; + const c = make({ + mode: "enforce", + minLimit: max, + maxLimit: max, + initialLimit: max, + maxQueueCount: 4, + maxQueueCost: max, + windowMs: 1000, + cost: { maxRequestCost: max }, + }); + + const full = await mustAdmit(c, req({ cost: max })); + assert.equal(c.snapshot().activeCost, max); + assert.equal(Number.isSafeInteger(c.snapshot().activeCost), true); + clock.advance(1000); + assert.equal(c.snapshot().utilization, 1); + full.release("success"); + assert.equal(c.snapshot().activeCost, 0); + assert.equal(c.snapshot().activeCount, 0); + + const left = Math.floor(max / 2); + const right = max - left; + const a = await mustAdmit(c, req({ cost: left })); + const b = await mustAdmit(c, req({ cost: right })); + assert.equal(c.snapshot().activeCost, max); + assert.equal(c.snapshot().activeCount, 2); + clock.advance(1000); + assert.equal(c.snapshot().utilization, 1); + a.release("success"); + assert.equal(c.snapshot().activeCost, right); + b.release("success"); + assert.equal(c.snapshot().activeCost, 0); + assert.equal(c.snapshot().activeCount, 0); + c.shutdown(); + }); +}); + +describe("updateConfig re-evaluation", () => { + let clock: FakeClock; + const live: AdaptiveAdmissionController[] = []; + beforeEach(() => { + clock = new FakeClock(); + live.length = 0; + }); + afterEach(() => { + for (const c of live) c.shutdown(); + live.length = 0; + }); + + function controller(overrides: Partial = {}) { + const c = new AdaptiveAdmissionController(baseConfig(overrides), { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }); + live.push(c); + return c; + } + + it("updateConfig rejects queued work above the new enforce limit immediately", async () => { + const c = controller({ + minLimit: 5, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 5_000, + }); + const held = await mustAdmit(c, req({ cost: 10 })); + const queued = await c.acquire(req({ cost: 8 })); + assert.equal(queued.status, "queued"); + + c.updateConfig( + baseConfig({ + minLimit: 5, + initialLimit: 5, + maxLimit: 5, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 5_000, + }) + ); + + if (queued.status === "queued") { + await assert.rejects(queued.promise, (err: unknown) => { + assert.equal((err as { code?: string }).code, "ADMISSION_OVERSIZED"); + return true; + }); + } + assert.equal(c.snapshot().queuedCount, 0); + held.release(); + }); + + it("updateConfig enforce→shadow classifies individually oversized active work as virtual rejected", async () => { + const c = controller({ + mode: "enforce", + minLimit: 5, + initialLimit: 20, + maxLimit: 20, + maxQueueCount: 4, + maxQueueCost: 40, + }); + const oversized = await mustAdmit(c, req({ cost: 15 })); + const fitting = await mustAdmit(c, req({ cost: 5 })); + + c.updateConfig( + baseConfig({ + mode: "shadow", + minLimit: 5, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 4, + maxQueueCost: 40, + }) + ); + + const snap = c.snapshot(); + // currentLimit clamps to 10; cost 15 is individually oversized → virtual rejected, not queued. + assert.equal(snap.currentLimit, 10); + assert.equal(snap.virtualActiveCost, 5); + assert.equal(snap.virtualActiveCount, 1); + assert.equal(snap.virtualQueuedCost, 0); + assert.equal(snap.virtualQueuedCount, 0); + // Real active leases remain until release. + assert.equal(snap.activeCost, 20); + assert.equal(snap.activeCount, 2); + + oversized.release(); + fitting.release(); + }); + + it("updateConfig rebuilds shadow virtual dispositions under new limits and queue bounds", async () => { + const c = controller({ + mode: "shadow", + minLimit: 5, + initialLimit: 20, + maxLimit: 20, + maxQueueCount: 2, + maxQueueCost: 12, + }); + const first = await c.acquire(req({ cost: 8 })); + const second = await c.acquire(req({ cost: 8 })); + const third = await c.acquire(req({ cost: 8 })); + assert.equal(first.status, "admitted"); + assert.equal(second.status, "admitted"); + assert.equal(third.status, "admitted"); + + c.updateConfig( + baseConfig({ + mode: "shadow", + minLimit: 5, + initialLimit: 10, + maxLimit: 10, + maxQueueCount: 1, + maxQueueCost: 8, + }) + ); + + const snap = c.snapshot(); + // One active (8), one queued (8), one rejected (queue full under new bounds). + assert.equal(snap.virtualActiveCost, 8); + assert.equal(snap.virtualActiveCount, 1); + assert.equal(snap.virtualQueuedCost, 8); + assert.equal(snap.virtualQueuedCount, 1); + assert.equal(snap.activeCount, 3); + + if (first.status === "admitted") first.lease.release(); + if (second.status === "admitted") second.lease.release(); + if (third.status === "admitted") third.lease.release(); + }); +}); + +describe("deterministic overload harness", () => { + async function runEqualServiceWindows(offeredPerWindow: number) { + const clock = new FakeClock(); + const c = new AdaptiveAdmissionController( + baseConfig({ + mode: "enforce", + initialLimit: 20, + minLimit: 20, + maxLimit: 20, + maxQueueCount: 1, + maxQueueCost: 1, + defaultMaxWaitMs: 20, + }), + { now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer } + ); + let completed = 0; + let fastRejected = 0; + let active: AdmissionLease[] = []; + + for (let window = 0; window < 20; window++) { + for (const lease of active) { + lease.release("success", { latencyMs: 10 }); + completed += 1; + } + active = []; + for (let i = 0; i < offeredPerWindow; i++) { + const result = await c.acquire(req({ cost: 5, tenantKey: `tenant-${i % 3}` })); + if (result.status === "admitted") active.push(result.lease); + else if (result.status === "rejected") fastRejected += 1; + else assert.fail("cost-5 excess must reject immediately when queue cost cap is 1"); + } + clock.advance(10); + const snapshot = c.snapshot(); + assert.ok(snapshot.activeCost <= 20); + assert.ok(snapshot.activeCount <= 4); + assert.equal(snapshot.queuedCost, 0); + assert.equal(snapshot.queuedCount, 0); + } + for (const lease of active) { + lease.release("success", { latencyMs: 10 }); + completed += 1; + } + c.shutdown(); + assert.equal(clock.pendingTimerCount, 0); + return { completed, fastRejected }; + } + + it("raises goodput to capacity then plateaus at 2× and 5× offered load", async () => { + // Capacity is 4 admits/window (limit 20, cost 5). Offered loads: 0.5×, 1×, 2×, 5×. + const low = await runEqualServiceWindows(2); + const atCapacity = await runEqualServiceWindows(4); + const doubleOver = await runEqualServiceWindows(8); + const fiveOver = await runEqualServiceWindows(20); + + assert.equal(low.completed, 40); + assert.equal(atCapacity.completed, 80); + assert.ok(atCapacity.completed >= low.completed * 1.9, "goodput must rise toward capacity"); + assert.equal( + doubleOver.completed, + atCapacity.completed, + "2× offered load must plateau at capacity" + ); + assert.equal( + fiveOver.completed, + atCapacity.completed, + "5× offered load must plateau at capacity" + ); + assert.equal(low.fastRejected, 0); + assert.equal(atCapacity.fastRejected, 0); + assert.equal( + doubleOver.fastRejected, + 4 * 20, + "2× excess rejects immediately with bounded queue" + ); + assert.equal( + fiveOver.fastRejected, + 16 * 20, + "5× excess rejects immediately with bounded queue" + ); + }); +}); diff --git a/tests/unit/adaptive-admission-features.test.ts b/tests/unit/adaptive-admission-features.test.ts new file mode 100644 index 0000000000..30ebaa8ff7 --- /dev/null +++ b/tests/unit/adaptive-admission-features.test.ts @@ -0,0 +1,255 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { estimateAdmissionCost } from "../../open-sse/services/admission/cost.ts"; +import { + ADMISSION_TOOL_SCAN_BUDGET, + extractAdmissionCostFeatures, +} from "../../open-sse/services/admission/requestFeatures.ts"; + +describe("bounded request feature extraction", () => { + it("does not call JSON.stringify or toJSON", () => { + let stringifyCalls = 0; + const original = JSON.stringify; + JSON.stringify = ((...args: Parameters) => { + stringifyCalls += 1; + return original.apply(JSON, args as [unknown]); + }) as typeof JSON.stringify; + try { + const body = { + toJSON() { + throw new Error("toJSON must not be invoked"); + }, + messages: [{ role: "user", content: "hello world" }], + tools: [{ type: "function", function: { name: "x" } }], + n: 3, + stream: true, + }; + const features = extractAdmissionCostFeatures(body); + assert.ok((features.bodyBytes ?? 0) > 0); + assert.ok((features.messageCount ?? 0) >= 1); + assert.ok((features.toolCount ?? 0) >= 1); + assert.equal(features.requestedFanout, 3); + assert.equal(features.streaming, true); + assert.ok((features.estimatedInputTokens ?? 0) > 0); + assert.equal(stringifyCalls, 0); + } finally { + JSON.stringify = original; + } + }); + + it("extracts production-realistic Chat, Responses, Gemini, and Antigravity shapes", () => { + // OpenAI Chat Completions — stream omitted defaults false (higher non-stream class). + const chat = extractAdmissionCostFeatures({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You are helpful." }, + { role: "user", content: "Summarize the logs" }, + ], + tools: [ + { + type: "function", + function: { name: "lookup", parameters: { type: "object" } }, + }, + { + type: "function", + function: { name: "write", parameters: { type: "object" } }, + }, + ], + n: 2, + }); + assert.equal(chat.messageCount, 2); + assert.equal(chat.toolCount, 2); + assert.equal(chat.requestedFanout, 2); + assert.equal(chat.streaming, false); + + // OpenAI Responses API — string input counts as one item; array counts length. + const responsesString = extractAdmissionCostFeatures({ + model: "gpt-4.1", + input: "What is the capital of France?", + tools: [{ type: "web_search_preview" }], + stream: true, + }); + assert.equal(responsesString.messageCount, 1); + assert.equal(responsesString.toolCount, 1); + assert.equal(responsesString.streaming, true); + + const responsesArray = extractAdmissionCostFeatures({ + model: "gpt-4.1", + input: [ + { role: "user", content: [{ type: "input_text", text: "q1" }] }, + { role: "user", content: [{ type: "input_text", text: "q2" }] }, + ], + stream: false, + n: 3, + }); + assert.equal(responsesArray.messageCount, 2); + assert.equal(responsesArray.requestedFanout, 3); + assert.equal(responsesArray.streaming, false); + + // Empty string input is not a content item. + const emptyInput = extractAdmissionCostFeatures({ input: "" }); + assert.equal(emptyInput.messageCount, 0); + + // Gemini generateContent — nested generationConfig.candidateCount + functionDeclarations. + const gemini = extractAdmissionCostFeatures({ + contents: [ + { role: "user", parts: [{ text: "hello" }] }, + { role: "model", parts: [{ text: "world" }] }, + ], + tools: [ + { + functionDeclarations: [ + { name: "get_weather", parameters: { type: "OBJECT" } }, + { name: "get_time", parameters: { type: "OBJECT" } }, + ], + }, + ], + generationConfig: { candidateCount: 4, temperature: 0.2 }, + }); + assert.equal(gemini.messageCount, 2); + assert.equal(gemini.toolCount, 2); + assert.equal(gemini.requestedFanout, 4); + assert.equal(gemini.streaming, false); + + // Antigravity-style wrapper under `request`. + const antigravity = extractAdmissionCostFeatures({ + request: { + contents: [{ role: "user", parts: [{ text: "hi" }] }], + tools: [ + { + functionDeclarations: [{ name: "a" }, { name: "b" }, { name: "c" }], + }, + ], + generationConfig: { candidateCount: 5 }, + stream: true, + }, + }); + assert.equal(antigravity.messageCount, 1); + assert.equal(antigravity.toolCount, 3); + assert.equal(antigravity.requestedFanout, 5); + assert.equal(antigravity.streaming, true); + + // Authoritative extraction context wins over body stream inference. + const overridden = extractAdmissionCostFeatures( + { messages: [{ role: "user", content: "x" }], stream: false }, + { streaming: true } + ); + assert.equal(overridden.streaming, true); + + const overriddenOff = extractAdmissionCostFeatures( + { messages: [{ role: "user", content: "x" }], stream: true }, + { streaming: false } + ); + assert.equal(overriddenOff.streaming, false); + }); + + it("bounds tool scans and never touches entries beyond the budget (conservative count)", () => { + // Huge leading string makes estimateSizeFast byte-exit before walking tools, + // so only countTools can touch the tools proxy — proving its scan bound alone. + const sizePad = "x".repeat(300_000); + + let accesses = 0; + const tools = new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 10_000; + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + const index = Number(prop); + accesses += 1; + if (index >= ADMISSION_TOOL_SCAN_BUDGET) { + throw new Error(`tool entry ${index} must not be touched`); + } + return { type: "function", function: { name: `t${index}` } }; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const features = extractAdmissionCostFeatures({ pad: sizePad, tools }); + // Any uninspected tail saturates the feature so heavier unseen entries cannot undercharge. + assert.equal(features.toolCount, Number.MAX_SAFE_INTEGER); + assert.equal(accesses, 0, "known oversized source should saturate before indexed access"); + + // functionDeclarations length is O(1); truncated tail still cannot undercharge. + let declAccesses = 0; + const geminiTools = new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 50; + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + const index = Number(prop); + declAccesses += 1; + if (index >= ADMISSION_TOOL_SCAN_BUDGET) { + throw new Error(`gemini tool entry ${index} must not be touched`); + } + return { + functionDeclarations: new Proxy([] as unknown[], { + get(t, p, r) { + if (p === "length") return 3; + if (typeof p === "string" && /^[0-9]+$/.test(p)) { + throw new Error("functionDeclarations elements need not be scanned"); + } + return Reflect.get(t, p, r); + }, + }), + }; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const geminiFeatures = extractAdmissionCostFeatures({ pad: sizePad, tools: geminiTools }); + assert.equal(geminiFeatures.toolCount, Number.MAX_SAFE_INTEGER); + assert.equal(declAccesses, 0); + + let aliasTouches = 0; + const sixtyFour = (label: string) => + new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET; + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + aliasTouches += 1; + return { name: `${label}-${prop}` }; + } + return Reflect.get(target, prop, receiver); + }, + }); + const aliases = extractAdmissionCostFeatures({ + pad: sizePad, + tools: sixtyFour("tool"), + functions: sixtyFour("function"), + }); + assert.equal(aliases.toolCount, Number.MAX_SAFE_INTEGER); + assert.ok(aliasTouches <= ADMISSION_TOOL_SCAN_BUDGET); + + let wrappedTouches = 0; + const wrappedTail = new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 1; + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + wrappedTouches += 1; + return { functionDeclarations: new Array(1_000).fill(null) }; + } + return Reflect.get(target, prop, receiver); + }, + }); + const layered = extractAdmissionCostFeatures({ + pad: sizePad, + tools: [{ type: "function" }], + request: { tools: wrappedTail }, + }); + assert.equal(layered.toolCount, Number.MAX_SAFE_INTEGER); + assert.equal(estimateAdmissionCost(layered), 1_000); + assert.equal(wrappedTouches, 0); + }); + + it("nested fanout under request wrapper is visible and stream defaults false", () => { + const features = extractAdmissionCostFeatures({ + request: { + messages: [{ role: "user", content: "x" }], + n: 7, + }, + }); + assert.equal(features.requestedFanout, 7); + assert.equal(features.streaming, false); + assert.equal(features.messageCount, 1); + }); +}); diff --git a/tests/unit/adaptive-admission-lifecycle.test.ts b/tests/unit/adaptive-admission-lifecycle.test.ts new file mode 100644 index 0000000000..835b96b093 --- /dev/null +++ b/tests/unit/adaptive-admission-lifecycle.test.ts @@ -0,0 +1,450 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + createAdaptiveAdmissionRuntime, + DEFAULT_ADAPTIVE_ADMISSION_CONFIG, + type AdaptiveAdmissionRuntime, +} from "../../open-sse/services/admission/runtime.ts"; +import { + type AdaptiveAdmissionConfig, + type AdmissionLease, + type AdmissionReleaseMeta, + type AdmissionReleaseOutcome, +} from "../../open-sse/services/admission/types.ts"; +import type { + ResourcePressureGuardResult, + ResourcePressureObservation, +} from "../../open-sse/utils/resourcePressure.ts"; + +/** Purpose-built lease spy: counts every release() while exposing released after first call. */ +function createSpyLease(id = "spy-lease", cost = 1) { + const calls: Array<{ outcome?: AdmissionReleaseOutcome; meta?: AdmissionReleaseMeta }> = []; + let released = false; + const lease: AdmissionLease = { + id, + cost, + get released() { + return released; + }, + release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta) { + calls.push({ outcome, meta }); + released = true; + }, + }; + return { + lease, + calls, + get releaseCount() { + return calls.length; + }, + }; +} + +class FakeClock { + nowMs = 0; + private nextId = 1; + private timers = new Map void }>(); + + now = () => this.nowMs; + + setTimer = (fn: () => void, delayMs: number): number => { + const id = this.nextId++; + this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn }); + return id; + }; + + clearTimer = (id: number): void => { + this.timers.delete(id); + }; + + get pendingTimerCount(): number { + return this.timers.size; + } + + advance(ms: number): void { + const target = this.nowMs + ms; + while (true) { + let nextId: number | undefined; + let nextDue = Number.POSITIVE_INFINITY; + for (const [id, t] of this.timers) { + if (t.due <= target && t.due < nextDue) { + nextDue = t.due; + nextId = id; + } + } + if (nextId === undefined) { + this.nowMs = target; + return; + } + const timer = this.timers.get(nextId)!; + this.timers.delete(nextId); + this.nowMs = timer.due; + timer.fn(); + } + } +} + +function emptyObservation( + overrides: Partial = {} +): ResourcePressureObservation { + return { + signals: null, + state: { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + ...overrides, + }, + }; +} + +function makeRuntime( + clock: FakeClock, + overrides: { + config?: AdaptiveAdmissionConfig; + check?: () => ResourcePressureGuardResult | null; + observe?: () => ResourcePressureObservation; + warn?: (message: string) => void; + } = {} +): AdaptiveAdmissionRuntime { + return createAdaptiveAdmissionRuntime({ + config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG }, + clock: { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }, + checkResourcePressure: overrides.check ?? (() => null), + getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()), + warn: overrides.warn, + }); +} + +describe("response lifecycle helpers", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + function attachJson( + runtime: AdaptiveAdmissionRuntime, + spy: ReturnType, + status: number, + options: { signal?: AbortSignal; admittedAtMs?: number } = {} + ) { + const admittedAtMs = options.admittedAtMs ?? clock.nowMs; + return runtime.attachResponseLifecycle( + new Response(JSON.stringify({ ok: status < 400 }), { + status, + headers: { "Content-Type": "application/json" }, + }), + spy.lease, + { admittedAtMs, signal: options.signal, nowMs: clock.now } + ); + } + + it("classifies non-SSE HTTP outcomes with cancellation winning", async () => { + const runtime = makeRuntime(clock); + const cases: Array<{ + status: number; + expected: AdmissionReleaseOutcome; + signal?: AbortSignal; + label: string; + }> = [ + { status: 200, expected: "success", label: "2xx" }, + { status: 302, expected: "success", label: "3xx" }, + { status: 400, expected: "local_reject", label: "ordinary 4xx" }, + { status: 429, expected: "local_reject", label: "429" }, + { status: 408, expected: "timeout", label: "408" }, + { status: 499, expected: "cancelled", label: "499" }, + { status: 504, expected: "timeout", label: "504" }, + { status: 502, expected: "upstream_error", label: "5xx" }, + { status: 500, expected: "upstream_error", label: "500" }, + ]; + + for (const c of cases) { + const spy = createSpyLease(`json-${c.label}`); + clock.nowMs = 100; + attachJson(runtime, spy, c.status, { admittedAtMs: 40 }); + assert.equal(spy.releaseCount, 1, c.label); + assert.equal(spy.calls[0]!.outcome, c.expected, c.label); + assert.equal(spy.calls[0]!.meta?.latencyMs, 60, c.label); + assert.equal(spy.lease.released, true, c.label); + } + + // Already-aborted signal wins over 2xx. + const ac = new AbortController(); + ac.abort(); + const abortedSpy = createSpyLease("aborted-2xx"); + clock.nowMs = 200; + attachJson(runtime, abortedSpy, 200, { signal: ac.signal, admittedAtMs: 150 }); + assert.equal(abortedSpy.releaseCount, 1); + assert.equal(abortedSpy.calls[0]!.outcome, "cancelled"); + assert.equal(abortedSpy.calls[0]!.meta?.latencyMs, 50); + runtime.dispose(); + }); + + it("classifies SSE completion outcomes using the request signal", async () => { + const runtime = makeRuntime(clock); + let spySuffix = 0; + + async function drainSse( + status: number, + signal?: AbortSignal, + expectImmediateRelease = false + ): Promise> { + const spy = createSpyLease(`sse-${status}-${spySuffix++}`); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: done\n\n")); + controller.close(); + }, + }); + clock.nowMs = 300; + const wrapped = runtime.attachResponseLifecycle( + new Response(body, { + status, + statusText: "OK", + headers: { "Content-Type": "text/event-stream" }, + }), + spy.lease, + { admittedAtMs: 250, signal, nowMs: clock.now } + ); + if (expectImmediateRelease) { + assert.equal(spy.releaseCount, 1); + return spy; + } + assert.equal(spy.releaseCount, 0); + await wrapped.text(); + return spy; + } + + const ok = await drainSse(200); + assert.equal(ok.releaseCount, 1); + assert.equal(ok.calls[0]!.outcome, "success"); + assert.equal(ok.calls[0]!.meta?.latencyMs, 50); + + const redirect = await drainSse(302); + assert.equal(redirect.calls[0]!.outcome, "success"); + + const ordinary4xx = await drainSse(404); + assert.equal(ordinary4xx.calls[0]!.outcome, "local_reject"); + + const tooMany = await drainSse(429); + assert.equal(tooMany.calls[0]!.outcome, "local_reject"); + + const requestTimeout = await drainSse(408); + assert.equal(requestTimeout.calls[0]!.outcome, "timeout"); + + const clientGone = await drainSse(499); + assert.equal(clientGone.calls[0]!.outcome, "cancelled"); + + const gatewayTimeout = await drainSse(504); + assert.equal(gatewayTimeout.calls[0]!.outcome, "timeout"); + + const upstream = await drainSse(503); + assert.equal(upstream.calls[0]!.outcome, "upstream_error"); + + // Already-aborted signal settles immediately as cancelled (wins over 2xx). + const ac = new AbortController(); + ac.abort(); + const abortedOk = await drainSse(200, ac.signal, true); + assert.equal(abortedOk.releaseCount, 1); + assert.equal(abortedOk.calls[0]!.outcome, "cancelled"); + + runtime.dispose(); + }); + + it("requires explicit non-success outcomes for handler failures", async () => { + const runtime = makeRuntime(clock); + const outcomes: Array> = [ + "local_reject", + "upstream_error", + "timeout", + "cancelled", + ]; + for (const outcome of outcomes) { + const spy = createSpyLease(`handler-${outcome}`); + clock.nowMs = 500; + runtime.releaseHandlerFailure(spy.lease, outcome, { + admittedAtMs: 400, + nowMs: clock.now, + }); + assert.equal(spy.releaseCount, 1, outcome); + assert.equal(spy.calls[0]!.outcome, outcome); + assert.equal(spy.calls[0]!.meta?.latencyMs, 100); + // Exactly-once: second call must not re-release. + runtime.releaseHandlerFailure(spy.lease, outcome, { + admittedAtMs: 400, + nowMs: clock.now, + }); + assert.equal(spy.releaseCount, 1, `${outcome} second`); + } + runtime.dispose(); + }); + + it("releases JSON/non-SSE responses immediately once", async () => { + const runtime = makeRuntime(clock); + const spy = createSpyLease("json-once"); + clock.nowMs = 80; + const wrapped = attachJson(runtime, spy, 200, { admittedAtMs: 20 }); + assert.equal(spy.releaseCount, 1); + assert.equal(spy.calls[0]!.outcome, "success"); + assert.equal(spy.calls[0]!.meta?.latencyMs, 60); + assert.equal(await wrapped.text(), JSON.stringify({ ok: true })); + runtime.attachResponseLifecycle( + new Response("{}", { status: 200, headers: { "Content-Type": "application/json" } }), + spy.lease, + { admittedAtMs: 20, nowMs: clock.now } + ); + assert.equal(spy.releaseCount, 1); + runtime.dispose(); + }); + + it("keeps SSE lease until stream drain and releases exactly once", async () => { + const runtime = makeRuntime(clock); + const spy = createSpyLease("sse-drain"); + let pullCount = 0; + const chunks = [ + new TextEncoder().encode("data: 1\n\n"), + new TextEncoder().encode("data: 2\n\n"), + ]; + const body = new ReadableStream({ + pull(controller) { + if (pullCount < chunks.length) { + controller.enqueue(chunks[pullCount++]); + return; + } + controller.close(); + }, + }); + clock.nowMs = 120; + const wrapped = runtime.attachResponseLifecycle( + new Response(body, { + status: 200, + statusText: "OK", + headers: { "Content-Type": "text/event-stream" }, + }), + spy.lease, + { admittedAtMs: 100, nowMs: clock.now } + ); + assert.equal(spy.releaseCount, 0); + assert.equal(wrapped.status, 200); + assert.equal(wrapped.statusText, "OK"); + assert.equal(wrapped.headers.get("Content-Type"), "text/event-stream"); + const text = await wrapped.text(); + assert.match(text, /data: 1/); + assert.match(text, /data: 2/); + assert.equal(spy.releaseCount, 1); + assert.equal(spy.calls[0]!.outcome, "success"); + assert.equal(spy.calls[0]!.meta?.latencyMs, 20); + // Drain again must not re-release (stream already consumed). + runtime.dispose(); + }); + + it("releases once on stream error", async () => { + const runtime = makeRuntime(clock); + const spy = createSpyLease("sse-error"); + const body = new ReadableStream({ + pull(controller) { + controller.error(new Error("upstream boom")); + }, + }); + clock.nowMs = 90; + const wrapped = runtime.attachResponseLifecycle( + new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }), + spy.lease, + { admittedAtMs: 70, nowMs: clock.now } + ); + await assert.rejects(async () => { + await wrapped.text(); + }); + assert.equal(spy.releaseCount, 1); + assert.equal(spy.calls[0]!.outcome, "upstream_error"); + assert.equal(spy.calls[0]!.meta?.latencyMs, 20); + runtime.dispose(); + }); + + it("releases once on consumer cancel without buffering", async () => { + const runtime = makeRuntime(clock); + const spy = createSpyLease("sse-cancel"); + let cancelCount = 0; + let pulled = 0; + const body = new ReadableStream({ + pull(controller) { + pulled += 1; + controller.enqueue(new TextEncoder().encode(`data: ${pulled}\n\n`)); + }, + cancel() { + cancelCount += 1; + }, + }); + clock.nowMs = 60; + const wrapped = runtime.attachResponseLifecycle( + new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }), + spy.lease, + { admittedAtMs: 10, nowMs: clock.now } + ); + const reader = wrapped.body!.getReader(); + await reader.read(); + assert.equal(spy.releaseCount, 0); + const pulledAfterFirst = pulled; + await reader.cancel("client gone"); + assert.equal(cancelCount, 1); + assert.equal(spy.releaseCount, 1); + assert.equal(spy.calls[0]!.outcome, "cancelled"); + assert.equal(spy.calls[0]!.meta?.latencyMs, 50); + // Laziness: no full buffering of the infinite producer. + assert.ok(pulledAfterFirst <= 2); + assert.ok(pulled < 20); + // Second cancel is a no-op for both reader cancel and lease release. + await reader.cancel("again"); + assert.equal(cancelCount, 1); + assert.equal(spy.releaseCount, 1); + runtime.dispose(); + }); + + it("request abort cancels the reader and releases once under races", async () => { + const runtime = makeRuntime(clock); + const spy = createSpyLease("sse-abort-race"); + const ac = new AbortController(); + let cancelCount = 0; + const body = new ReadableStream({ + async pull(controller) { + controller.enqueue(new TextEncoder().encode("data: ping\n\n")); + await new Promise(() => { + /* hang until cancel */ + }); + }, + cancel() { + cancelCount += 1; + }, + }); + clock.nowMs = 40; + const wrapped = runtime.attachResponseLifecycle( + new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }), + spy.lease, + { admittedAtMs: 10, signal: ac.signal, nowMs: clock.now } + ); + const reader = wrapped.body!.getReader(); + const first = reader.read(); + ac.abort(); + // Race: also cancel consumer. + void reader.cancel("race"); + await Promise.race([ + first.catch(() => undefined), + new Promise((resolve) => setImmediate(resolve)), + ]); + await Promise.resolve(); + await Promise.resolve(); + assert.equal(spy.releaseCount, 1); + assert.equal(spy.calls[0]!.outcome, "cancelled"); + assert.equal(typeof spy.calls[0]!.meta?.latencyMs, "number"); + assert.ok((spy.calls[0]!.meta?.latencyMs ?? -1) >= 0); + assert.equal(cancelCount, 1); + runtime.dispose(); + }); +}); diff --git a/tests/unit/adaptive-admission-queue.test.ts b/tests/unit/adaptive-admission-queue.test.ts new file mode 100644 index 0000000000..372d31e95d --- /dev/null +++ b/tests/unit/adaptive-admission-queue.test.ts @@ -0,0 +1,67 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { FairCostQueue, type QueueEntry } from "../../open-sse/services/admission/queue.ts"; + +describe("FairCostQueue removeById cursor preservation", () => { + function qEntry(id: string, tenantKey: string, cost = 1): QueueEntry<{ id: string }> { + return { + id, + tenantKey, + cost, + enqueuedAtMs: 0, + deadlineMs: Number.MAX_SAFE_INTEGER, + payload: { id }, + }; + } + + it("preserves the logical successor when removing a bucket before the cursor", () => { + const q = new FairCostQueue<{ id: string }>(10, 100); + assert.equal(q.enqueue(qEntry("a1", "a")), true); + assert.equal(q.enqueue(qEntry("a2", "a")), true); + assert.equal(q.enqueue(qEntry("b1", "b")), true); + assert.equal(q.enqueue(qEntry("c1", "c")), true); + + // Dequeue a1 leaves cursor at b while tenant a still has a2. + assert.equal(q.dequeue()?.id, "a1"); + assert.equal(q.removeById("a2")?.id, "a2"); + // Successor of the pre-removal cursor must remain b, not skip to c. + assert.equal(q.dequeue()?.id, "b1"); + assert.equal(q.dequeue()?.id, "c1"); + assert.equal(q.size, 0); + }); + + it("preserves the logical successor when removing the bucket at the cursor", () => { + const q = new FairCostQueue<{ id: string }>(10, 100); + assert.equal(q.enqueue(qEntry("a1", "a")), true); + assert.equal(q.enqueue(qEntry("b1", "b")), true); + assert.equal(q.enqueue(qEntry("c1", "c")), true); + + assert.equal(q.dequeue()?.id, "a1"); // cursor now at b + assert.equal(q.removeById("b1")?.id, "b1"); + assert.equal(q.dequeue()?.id, "c1"); + assert.equal(q.size, 0); + }); + + it("preserves the cursor when removing a bucket after the cursor", () => { + const q = new FairCostQueue<{ id: string }>(10, 100); + assert.equal(q.enqueue(qEntry("a1", "a")), true); + assert.equal(q.enqueue(qEntry("b1", "b")), true); + assert.equal(q.enqueue(qEntry("c1", "c")), true); + + assert.equal(q.dequeue()?.id, "a1"); // cursor now at b + assert.equal(q.removeById("c1")?.id, "c1"); + assert.equal(q.dequeue()?.id, "b1"); + assert.equal(q.size, 0); + }); + + it("resets the cursor when the final bucket is removed", () => { + const q = new FairCostQueue<{ id: string }>(10, 100); + assert.equal(q.enqueue(qEntry("a1", "a")), true); + assert.equal(q.enqueue(qEntry("b1", "b")), true); + assert.equal(q.dequeue()?.id, "a1"); // cursor at b + assert.equal(q.removeById("b1")?.id, "b1"); + assert.equal(q.size, 0); + assert.equal(q.enqueue(qEntry("d1", "d")), true); + assert.equal(q.dequeue()?.id, "d1"); + }); +}); diff --git a/tests/unit/adaptive-admission-runtime.test.ts b/tests/unit/adaptive-admission-runtime.test.ts new file mode 100644 index 0000000000..82417c7c32 --- /dev/null +++ b/tests/unit/adaptive-admission-runtime.test.ts @@ -0,0 +1,857 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { + createAdaptiveAdmissionRuntime, + getAdaptiveAdmissionRuntime, + reloadAdaptiveAdmissionRuntime, + resetAdaptiveAdmissionRuntimeForTests, + resolveAdaptiveAdmissionConfigFromEnv, + DEFAULT_ADAPTIVE_ADMISSION_CONFIG, + type AdaptiveAdmissionRuntime, +} from "../../open-sse/services/admission/runtime.ts"; +import { + MAX_ADMISSION_COST_OR_LIMIT, + MAX_ADMISSION_WINDOW_MS, + type AdaptiveAdmissionConfig, +} from "../../open-sse/services/admission/types.ts"; +import type { + ResourcePressureGuardResult, + ResourcePressureObservation, +} from "../../open-sse/utils/resourcePressure.ts"; +import { buildErrorBody } from "../../open-sse/utils/error.ts"; + +class FakeClock { + nowMs = 0; + private nextId = 1; + private timers = new Map void }>(); + + now = () => this.nowMs; + + setTimer = (fn: () => void, delayMs: number): number => { + const id = this.nextId++; + this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn }); + return id; + }; + + clearTimer = (id: number): void => { + this.timers.delete(id); + }; + + get pendingTimerCount(): number { + return this.timers.size; + } + + advance(ms: number): void { + const target = this.nowMs + ms; + while (true) { + let nextId: number | undefined; + let nextDue = Number.POSITIVE_INFINITY; + for (const [id, t] of this.timers) { + if (t.due <= target && t.due < nextDue) { + nextDue = t.due; + nextId = id; + } + } + if (nextId === undefined) { + this.nowMs = target; + return; + } + const timer = this.timers.get(nextId)!; + this.timers.delete(nextId); + this.nowMs = timer.due; + timer.fn(); + } + } +} + +function enforceConfig(overrides: Partial = {}): AdaptiveAdmissionConfig { + return { + mode: "enforce", + minLimit: 4, + maxLimit: 20, + initialLimit: 8, + maxQueueCount: 2, + maxQueueCost: 16, + defaultMaxWaitMs: 100, + windowMs: 50, + ...overrides, + }; +} + +function emptyObservation( + overrides: Partial = {} +): ResourcePressureObservation { + return { + signals: null, + state: { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + ...overrides, + }, + }; +} + +function criticalGuard(reason = "v8_heap_absolute"): ResourcePressureGuardResult { + const message = "Service temporarily unavailable due to resource pressure. Retry shortly."; + return { + success: false, + status: 503, + error: message, + response: new Response( + JSON.stringify( + buildErrorBody(503, message, undefined, { + type: "server_error", + code: "resource_pressure", + }) + ), + { + status: 503, + headers: { "Content-Type": "application/json", "Retry-After": "5" }, + } + ), + }; +} + +function makeRuntime( + clock: FakeClock, + overrides: { + config?: AdaptiveAdmissionConfig; + check?: () => ResourcePressureGuardResult | null; + observe?: () => ResourcePressureObservation; + warn?: (message: string) => void; + } = {} +): AdaptiveAdmissionRuntime { + return createAdaptiveAdmissionRuntime({ + config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG }, + clock: { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }, + checkResourcePressure: overrides.check ?? (() => null), + getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()), + warn: overrides.warn, + }); +} + +async function parseJson(response: Response): Promise> { + return JSON.parse(await response.text()) as Record; +} + +describe("adaptive admission runtime env + defaults", () => { + it("defaults to complete shadow config", () => { + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.mode, "shadow"); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.minLimit, 8); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.initialLimit, 64); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxLimit, 1000); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCount, 128); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCost, 2000); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.defaultMaxWaitMs, 5000); + assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.windowMs, 1000); + }); + + it("strictly resolves supported env names and rejects invalid values", () => { + const cfg = resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MODE: "enforce", + ADAPTIVE_ADMISSION_MIN_LIMIT: "10", + ADAPTIVE_ADMISSION_INITIAL_LIMIT: "20", + ADAPTIVE_ADMISSION_MAX_LIMIT: "30", + ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "40", + ADAPTIVE_ADMISSION_MAX_QUEUE_COST: "50", + ADAPTIVE_ADMISSION_MAX_WAIT_MS: "600", + ADAPTIVE_ADMISSION_WINDOW_MS: "700", + }); + assert.deepEqual( + { + mode: cfg.mode, + minLimit: cfg.minLimit, + initialLimit: cfg.initialLimit, + maxLimit: cfg.maxLimit, + maxQueueCount: cfg.maxQueueCount, + maxQueueCost: cfg.maxQueueCost, + defaultMaxWaitMs: cfg.defaultMaxWaitMs, + windowMs: cfg.windowMs, + }, + { + mode: "enforce", + minLimit: 10, + initialLimit: 20, + maxLimit: 30, + maxQueueCount: 40, + maxQueueCost: 50, + defaultMaxWaitMs: 600, + windowMs: 700, + } + ); + + assert.throws( + () => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MODE: "strict" }), + /ADAPTIVE_ADMISSION_MODE/ + ); + assert.throws( + () => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MIN_LIMIT: "0" }), + /ADAPTIVE_ADMISSION_MIN_LIMIT/ + ); + assert.throws( + () => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MAX_LIMIT: "1.5" }), + /ADAPTIVE_ADMISSION_MAX_LIMIT/ + ); + }); + + it("accepts exact documented maxima and rejects max+1 plus cross-field invalidity", () => { + const maxCost = String(MAX_ADMISSION_COST_OR_LIMIT); + const maxWindow = String(MAX_ADMISSION_WINDOW_MS); + const maxQueue = String(Number.MAX_SAFE_INTEGER); + + const atMaxima = resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MODE: "shadow", + ADAPTIVE_ADMISSION_MIN_LIMIT: "1", + ADAPTIVE_ADMISSION_INITIAL_LIMIT: maxCost, + ADAPTIVE_ADMISSION_MAX_LIMIT: maxCost, + ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: maxQueue, + ADAPTIVE_ADMISSION_MAX_QUEUE_COST: maxCost, + ADAPTIVE_ADMISSION_MAX_WAIT_MS: maxWindow, + ADAPTIVE_ADMISSION_WINDOW_MS: maxWindow, + }); + assert.equal(atMaxima.maxLimit, MAX_ADMISSION_COST_OR_LIMIT); + assert.equal(atMaxima.maxQueueCount, Number.MAX_SAFE_INTEGER); + assert.equal(atMaxima.windowMs, MAX_ADMISSION_WINDOW_MS); + assert.equal(atMaxima.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS); + + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MAX_LIMIT: String(MAX_ADMISSION_COST_OR_LIMIT + 1), + }), + /maxLimit|must be <=/ + ); + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MAX_QUEUE_COST: String(MAX_ADMISSION_COST_OR_LIMIT + 1), + }), + /maxQueueCost|must be <=/ + ); + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_WINDOW_MS: String(MAX_ADMISSION_WINDOW_MS + 1), + }), + /windowMs|must be <=/ + ); + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MAX_WAIT_MS: String(MAX_ADMISSION_WINDOW_MS + 1), + }), + /defaultMaxWaitMs|must be <=/ + ); + // Queue count uses full safe-integer range; beyond that fails lexical/safe-integer parsing. + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "9007199254740992", + }), + /ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT|safe integer/ + ); + assert.throws( + () => + resolveAdaptiveAdmissionConfigFromEnv({ + ADAPTIVE_ADMISSION_MIN_LIMIT: "20", + ADAPTIVE_ADMISSION_MAX_LIMIT: "10", + }), + /minLimit must be <= maxLimit/ + ); + }); + + it("default process runtime falls back to shadow on invalid env without crashing", () => { + resetAdaptiveAdmissionRuntimeForTests(); + const warnings: string[] = []; + const previous = process.env.ADAPTIVE_ADMISSION_MODE; + process.env.ADAPTIVE_ADMISSION_MODE = "not-a-mode"; + try { + const runtime = reloadAdaptiveAdmissionRuntime({ + warn: (message) => warnings.push(message), + checkResourcePressure: () => null, + getResourcePressureObservation: () => emptyObservation(), + }); + const snap = runtime.snapshot(); + assert.equal(snap.mode, "shadow"); + assert.equal(snap.minLimit, 8); + assert.equal(snap.initialLimit ?? snap.currentLimit >= 8, true); + assert.equal(warnings.length, 1); + assert.match( + warnings[0]!, + /invalid environment configuration; using default shadow admission settings/ + ); + assert.ok(!warnings.join("\n").includes("not-a-mode")); + assert.ok(!warnings.join("\n").toLowerCase().includes("secret")); + runtime.dispose(); + } finally { + if (previous === undefined) delete process.env.ADAPTIVE_ADMISSION_MODE; + else process.env.ADAPTIVE_ADMISSION_MODE = previous; + resetAdaptiveAdmissionRuntimeForTests(); + } + }); +}); + +describe("adaptive admission runtime modes", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + afterEach(() => { + resetAdaptiveAdmissionRuntimeForTests(); + }); + + it("default shadow always admits with a real lease and shadowDecision", async () => { + const runtime = makeRuntime(clock); + const result = await runtime.acquire({ + tenantKey: "tenant-secret-1", + body: { messages: [{ role: "user", content: "hi" }], stream: true }, + }); + assert.equal(result.status, "admitted"); + if (result.status !== "admitted") throw new Error("expected admitted"); + assert.equal(result.mode, "shadow"); + assert.ok(result.lease); + assert.equal(typeof result.lease.release, "function"); + assert.equal(result.lease.released, false); + assert.ok( + result.shadowDecision === "would-admit" || + result.shadowDecision === "would-queue" || + result.shadowDecision === "would-reject" + ); + result.lease.release("success"); + assert.equal(result.lease.released, true); + result.lease.release("success"); + runtime.dispose(); + }); + + it("explicit off admits without enforcing capacity", async () => { + const runtime = makeRuntime(clock, { + config: { + ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, + mode: "off", + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + }, + }); + const a = await runtime.acquire({ tenantKey: "t1", body: { messages: [] } }); + const b = await runtime.acquire({ tenantKey: "t2", body: { messages: [] } }); + assert.equal(a.status, "admitted"); + assert.equal(b.status, "admitted"); + if (a.status === "admitted") a.lease.release(); + if (b.status === "admitted") b.lease.release(); + runtime.dispose(); + }); + + it("explicit enforce can reject with sanitized HTTP response", async () => { + const runtime = makeRuntime(clock, { + config: enforceConfig({ + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + maxQueueCount: 1, + maxQueueCost: 1, + defaultMaxWaitMs: 50, + cost: { maxRequestCost: 1, baseCost: 1 }, + }), + }); + const first = await runtime.acquire({ + tenantKey: "t1", + body: { messages: [{ role: "user", content: "a" }], stream: true }, + }); + assert.equal(first.status, "admitted"); + + const secondPromise = runtime.acquire({ + tenantKey: "t2", + body: { messages: [{ role: "user", content: "b" }], stream: true }, + maxWaitMs: 50, + }); + // Drive injected deadline timer; no wall-clock sleeps. + clock.advance(50); + const second = await secondPromise; + assert.equal(second.status, "rejected"); + if (second.status !== "rejected") throw new Error("expected rejected"); + assert.equal(second.response.status, 503); + const body = await parseJson(second.response); + assert.equal(typeof body.error.message, "string"); + assert.match(second.code, /^admission_/); + assert.ok(!JSON.stringify(body).includes("t2")); + assert.ok(!JSON.stringify(body).includes("tenant")); + if (first.status === "admitted") first.lease.release(); + runtime.dispose(); + }); +}); + +describe("runtime streaming cost forwarding", () => { + it("acquire lease cost reflects input.streaming via feature extraction", async () => { + const clock = new FakeClock(); + // Sharply distinct streaming class costs; neutralize other feature contributions. + const runtime = makeRuntime(clock, { + config: { + ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, + mode: "shadow", + cost: { + baseCost: 1, + bodyBytesPerUnit: 1_000_000, + tokensPerUnit: 1_000_000, + messagesPerUnit: 1_000_000, + toolsPerUnit: 1_000_000, + fanoutPerUnit: 1_000_000, + streamingClassCost: 1, + nonStreamingClassCost: 50, + maxRequestCost: 1_000, + }, + }, + }); + + // Empty body keeps non-class contributions identical; stream omitted defaults false when not forwarded. + + const body = {}; + const streamed = await runtime.acquire({ + tenantKey: "stream-on", + body, + streaming: true, + }); + assert.equal(streamed.status, "admitted"); + if (streamed.status !== "admitted") throw new Error("expected admitted"); + const streamCost = streamed.lease.cost; + streamed.lease.release("success"); + + const nonStreamed = await runtime.acquire({ + tenantKey: "stream-off", + body, + streaming: false, + }); + assert.equal(nonStreamed.status, "admitted"); + if (nonStreamed.status !== "admitted") throw new Error("expected admitted"); + const nonStreamCost = nonStreamed.lease.cost; + nonStreamed.lease.release("success"); + + const defaulted = await runtime.acquire({ + tenantKey: "stream-default", + body, + }); + assert.equal(defaulted.status, "admitted"); + if (defaulted.status !== "admitted") throw new Error("expected admitted"); + const defaultCost = defaulted.lease.cost; + defaulted.lease.release("success"); + + runtime.dispose(); + + // base(1) + fanout unit(1) + class cost → streaming 3, non-streaming 52 + assert.equal(streamCost, 3); + assert.equal(nonStreamCost, 52); + assert.equal(defaultCost, 52); + assert.notEqual( + streamCost, + nonStreamCost, + "streaming true/false must produce different acquired lease costs" + ); + }); +}); + +describe("rejection mapping", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + it("maps ADMISSION_ABORTED to local 499 without Retry-After", async () => { + const runtime = makeRuntime(clock, { + config: enforceConfig({ + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 1000, + // Force unit cost so one admitted request fills the limit. + cost: { maxRequestCost: 1, baseCost: 1 }, + }), + }); + const holder = await runtime.acquire({ + tenantKey: "hold", + body: { messages: [{ role: "user", content: "hold" }], stream: true }, + }); + assert.equal(holder.status, "admitted"); + + const ac = new AbortController(); + const pending = runtime.acquire({ + tenantKey: "wait", + body: { messages: [{ role: "user", content: "wait" }], stream: true }, + signal: ac.signal, + maxWaitMs: 1000, + }); + ac.abort(); + const rejected = await pending; + assert.equal(rejected.status, "rejected"); + if (rejected.status !== "rejected") throw new Error("expected rejected"); + assert.equal(rejected.code, "admission_aborted"); + assert.equal(rejected.response.status, 499); + assert.equal(rejected.response.headers.get("Retry-After"), null); + const body = await parseJson(rejected.response); + assert.equal(body.error.code, "admission_aborted"); + assert.ok(!JSON.stringify(body).includes("wait")); + if (holder.status === "admitted") holder.lease.release(); + runtime.dispose(); + }); + + it("maps queue full / deadline / oversized to sanitized 503 codes", async () => { + const runtime = makeRuntime(clock, { + config: enforceConfig({ + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + maxQueueCount: 1, + maxQueueCost: 1, + defaultMaxWaitMs: 20, + cost: { maxRequestCost: 1, baseCost: 1, bodyBytesPerUnit: 1_000_000 }, + }), + }); + const hold = await runtime.acquire({ + tenantKey: "hold", + body: { stream: true }, + }); + assert.equal(hold.status, "admitted"); + + const deadlinePromise = runtime.acquire({ + tenantKey: "q1", + body: { stream: true }, + maxWaitMs: 20, + }); + clock.advance(20); + const deadlineRejected = await deadlinePromise; + assert.equal(deadlineRejected.status, "rejected"); + if (deadlineRejected.status === "rejected") { + assert.equal(deadlineRejected.response.status, 503); + assert.equal(deadlineRejected.code, "admission_deadline"); + const body = await parseJson(deadlineRejected.response); + assert.equal(body.error.code, "admission_deadline"); + assert.equal(deadlineRejected.response.headers.get("Retry-After"), "1"); + } + + // Fill the single queue slot then force queue_full on the next arrival. + const waiterPromise = runtime.acquire({ + tenantKey: "waiter", + body: { stream: true }, + maxWaitMs: 1_000, + }); + const full = await runtime.acquire({ + tenantKey: "full", + body: { stream: true }, + }); + assert.equal(full.status, "rejected"); + if (full.status === "rejected") { + assert.equal(full.code, "admission_queue_full"); + assert.equal(full.response.status, 503); + assert.equal(full.response.headers.get("Retry-After"), "1"); + const body = await parseJson(full.response); + assert.equal(body.error.code, "admission_queue_full"); + } + clock.advance(1_000); + await waiterPromise; + + // Oversized: cost features that exceed limit 1 with tiny max. + const oversizedRuntime = makeRuntime(clock, { + config: enforceConfig({ + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + maxQueueCount: 1, + maxQueueCost: 1, + cost: { + maxRequestCost: 100, + baseCost: 1, + bodyBytesPerUnit: 1, + tokensPerUnit: 1, + messagesPerUnit: 1, + toolsPerUnit: 1, + fanoutPerUnit: 1, + }, + }), + }); + const huge = await oversizedRuntime.acquire({ + tenantKey: "huge", + body: { + messages: Array.from({ length: 50 }, (_, i) => ({ + role: "user", + content: `m${i}-${"x".repeat(32)}`, + })), + stream: true, + }, + }); + assert.equal(huge.status, "rejected"); + if (huge.status === "rejected") { + assert.equal(huge.code, "admission_oversized"); + assert.equal(huge.response.status, 503); + const body = await parseJson(huge.response); + assert.equal(body.error.code, "admission_oversized"); + assert.ok(!JSON.stringify(body).toLowerCase().includes("cost")); + assert.ok(!JSON.stringify(body).includes("huge")); + } + if (hold.status === "admitted") hold.lease.release(); + runtime.dispose(); + oversizedRuntime.dispose(); + }); +}); + +describe("resource pressure integration", () => { + let clock: FakeClock; + beforeEach(() => { + clock = new FakeClock(); + }); + + it("returns the existing critical guard response without acquiring work", async () => { + let acquires = 0; + const guard = criticalGuard(); + const runtime = makeRuntime(clock, { + config: enforceConfig({ initialLimit: 10 }), + check: () => { + acquires += 1; + return guard; + }, + }); + const result = await runtime.acquire({ + tenantKey: "t-pressure", + body: { messages: [{ role: "user", content: "x" }] }, + }); + assert.equal(result.status, "rejected"); + if (result.status !== "rejected") throw new Error("expected rejected"); + assert.equal(result.response, guard.response); + assert.equal(result.code, "resource_pressure"); + assert.equal(runtime.snapshot().pressureGuardRejectCount, 1); + assert.equal(runtime.snapshot().activeCount, 0); + assert.equal(acquires, 1); + runtime.dispose(); + }); + + it("feeds fresh critical pressure observation even when the safety guard rejects", async () => { + const guard = criticalGuard(); + let observation = emptyObservation({ + severity: "critical", + reason: "v8_heap_absolute", + observedAtMs: 1_000, + }); + const pressures: string[] = []; + const runtime = createAdaptiveAdmissionRuntime({ + config: enforceConfig({ + initialLimit: 20, + minLimit: 4, + maxLimit: 20, + windowMs: 50, + criticalDecreaseFactor: 0.5, + }), + clock: { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }, + checkResourcePressure: () => guard, + getResourcePressureObservation: () => observation, + onPressureObserved: (pressure) => pressures.push(pressure), + }); + + const first = await runtime.acquire({ + tenantKey: "guarded", + body: { messages: [{ role: "user", content: "x" }] }, + }); + assert.equal(first.status, "rejected"); + if (first.status !== "rejected") throw new Error("expected rejected"); + // Exact same guard response identity; zero controller acquisition. + assert.equal(first.response, guard.response); + assert.equal(first.code, "resource_pressure"); + assert.equal(runtime.snapshot().activeCount, 0); + assert.deepEqual(pressures, ["critical"]); + // One critical reduction: floor(20 * 0.5) = 10. + assert.equal(runtime.snapshot().currentLimit, 10); + + // Replay same observation: no additional feed or reduction. + const second = await runtime.acquire({ + tenantKey: "guarded-2", + body: { messages: [{ role: "user", content: "y" }] }, + }); + assert.equal(second.response, guard.response); + assert.deepEqual(pressures, ["critical"]); + assert.equal(runtime.snapshot().currentLimit, 10); + + // New window resets criticalDecreaseConsumed; fresh observation may reduce again. + clock.advance(50); + observation = emptyObservation({ + severity: "critical", + reason: "v8_heap_absolute", + observedAtMs: 2_000, + }); + const third = await runtime.acquire({ + tenantKey: "guarded-3", + body: { messages: [{ role: "user", content: "z" }] }, + }); + assert.equal(third.response, guard.response); + assert.deepEqual(pressures, ["critical", "critical"]); + assert.equal(runtime.snapshot().currentLimit, 5); + runtime.dispose(); + }); + + it("dedupes unchanged observations and re-feeds genuinely fresh ones", async () => { + let observation = emptyObservation({ + severity: "high", + reason: "psi_some", + observedAtMs: 100, + }); + const pressures: string[] = []; + const runtime = createAdaptiveAdmissionRuntime({ + config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" }, + clock: { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }, + checkResourcePressure: () => null, + getResourcePressureObservation: () => observation, + onPressureObserved: (pressure) => pressures.push(pressure), + }); + + await runtime.acquire({ tenantKey: "a", body: {} }); + await runtime.acquire({ tenantKey: "b", body: {} }); + assert.deepEqual(pressures, ["high"]); + + observation = emptyObservation({ + severity: "high", + reason: "psi_some", + observedAtMs: 100, + }); + await runtime.acquire({ tenantKey: "c", body: {} }); + assert.deepEqual(pressures, ["high"]); + + observation = emptyObservation({ + severity: "critical", + reason: "psi_full", + observedAtMs: 200, + }); + await runtime.acquire({ tenantKey: "d", body: {} }); + assert.deepEqual(pressures, ["high", "critical"]); + runtime.dispose(); + }); + + it("fails open when pressure check or observation throws", async () => { + const runtime = makeRuntime(clock, { + config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" }, + check: () => { + throw new Error("check boom"); + }, + observe: () => { + throw new Error("observe boom"); + }, + }); + const result = await runtime.acquire({ tenantKey: "t", body: { messages: [] } }); + assert.equal(result.status, "admitted"); + if (result.status === "admitted") result.lease.release(); + runtime.dispose(); + }); +}); + +describe("public snapshot privacy", () => { + it("exposes only aggregate counters and low-cardinality resource fields", async () => { + const clock = new FakeClock(); + const runtime = makeRuntime(clock, { + observe: () => + emptyObservation({ + severity: "high", + reason: "cgroup_ratio", + observedAtMs: 42, + }), + }); + await runtime.acquire({ + tenantKey: "tenant-very-secret", + body: { + messages: [{ role: "user", content: "SECRET_PAYLOAD_XYZ" }], + api_key: "sk-live-secret", + }, + }); + const snap = runtime.snapshot(); + const text = JSON.stringify(snap); + assert.ok(!text.includes("tenant-very-secret")); + assert.ok(!text.includes("SECRET_PAYLOAD_XYZ")); + assert.ok(!text.includes("sk-live-secret")); + assert.ok(!text.includes("lease-")); + assert.equal(typeof snap.mode, "string"); + assert.equal(typeof snap.currentLimit, "number"); + assert.equal(typeof snap.activeCount, "number"); + assert.equal(snap.resourceSeverity, "high"); + assert.equal(snap.resourceReason, "cgroup_ratio"); + assert.equal(snap.resourceObservedAtMs, 42); + assert.equal(typeof snap.pressureGuardRejectCount, "number"); + const snapRecord = snap as unknown as Record; + assert.equal(snapRecord.tenants, undefined); + assert.equal(snapRecord.queue, undefined); + assert.equal(snapRecord.features, undefined); + runtime.dispose(); + }); +}); + +describe("process runtime reload isolation", () => { + afterEach(() => { + resetAdaptiveAdmissionRuntimeForTests(); + }); + + it("reload disposes previous queued work/timers and replaces the process runtime", async () => { + resetAdaptiveAdmissionRuntimeForTests(); + const clock = new FakeClock(); + const first = reloadAdaptiveAdmissionRuntime({ + config: enforceConfig({ + initialLimit: 1, + minLimit: 1, + maxLimit: 1, + maxQueueCount: 4, + maxQueueCost: 40, + defaultMaxWaitMs: 5_000, + windowMs: 1_000, + cost: { maxRequestCost: 1, baseCost: 1 }, + }), + clock: { + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }, + checkResourcePressure: () => null, + getResourcePressureObservation: () => emptyObservation(), + }); + + const hold = await first.acquire({ + tenantKey: "hold", + body: { messages: [{ role: "user", content: "h" }], stream: true }, + }); + assert.equal(hold.status, "admitted"); + assert.ok(clock.pendingTimerCount >= 1); + + const waiting = first.acquire({ + tenantKey: "waiter", + body: { messages: [{ role: "user", content: "w" }], stream: true }, + maxWaitMs: 5_000, + }); + + const second = reloadAdaptiveAdmissionRuntime({ + config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" }, + checkResourcePressure: () => null, + getResourcePressureObservation: () => emptyObservation(), + }); + assert.notEqual(second, first); + assert.equal(getAdaptiveAdmissionRuntime(), second); + + const rejected = await waiting; + assert.equal(rejected.status, "rejected"); + if (rejected.status === "rejected") { + assert.equal(rejected.code, "admission_shutdown"); + } + // Previous timers should be cleared by dispose/shutdown. + assert.equal(clock.pendingTimerCount, 0); + second.dispose(); + resetAdaptiveAdmissionRuntimeForTests(); + }); +}); diff --git a/tests/unit/estimateSizeFast.test.ts b/tests/unit/estimateSizeFast.test.ts index d84fc75086..d7e5a7b6a8 100644 --- a/tests/unit/estimateSizeFast.test.ts +++ b/tests/unit/estimateSizeFast.test.ts @@ -1,9 +1,12 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { estimateSizeFast, isSmallEnoughForSemanticCache } = await import( - "../../open-sse/utils/estimateSize.ts" -); +const { + estimateSizeFast, + isSmallEnoughForSemanticCache, + ESTIMATE_SIZE_BYTE_LIMIT, + ESTIMATE_SIZE_NODE_BUDGET, +} = await import("../../open-sse/utils/estimateSize.ts"); test("estimateSizeFast returns 0 for null/undefined", () => { assert.equal(estimateSizeFast(null), 0); @@ -65,6 +68,22 @@ test("estimateSizeFast early-exits at 262144 bytes (256KB)", () => { assert.ok(result >= 262144, `Should early-exit, got ${result}`); }); +test("estimateSizeFast checks byte limit after numbers and booleans", () => { + const almostForNumber = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 4); + const withNumber = estimateSizeFast([almostForNumber, 1]); + assert.ok( + withNumber > ESTIMATE_SIZE_BYTE_LIMIT, + `number contribution must trip byte limit, got ${withNumber}` + ); + // boolean is 4 bytes: start 3 under the limit so adding true exceeds (not merely equals). + const almostForBool = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 3); + const withBool = estimateSizeFast([almostForBool, true]); + assert.ok( + withBool > ESTIMATE_SIZE_BYTE_LIMIT, + `boolean contribution must trip byte limit, got ${withBool}` + ); +}); + test("estimateSizeFast handles mixed object/array nesting", () => { const data = { choices: [ @@ -109,3 +128,70 @@ test("estimateSizeFast handles Map-like objects (no infinite loop on iterables)" const result = estimateSizeFast(map); assert.ok(typeof result === "number"); }); + +/** + * Mutation-sensitive bound: a huge logical length with null/empty-object elements + * must not pre-touch every index or allocate all references. Node-budget exhaustion + * fails closed above 256 KiB so semantic-cache/admission never treat it as small. + */ +test("estimateSizeFast node budget fails closed on huge sparse null array without full traversal", () => { + let elementAccesses = 0; + const sparseNulls = new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return 5_000_000; + if (prop === Symbol.iterator) { + throw new Error("iterator must not be used"); + } + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + elementAccesses += 1; + return null; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const result = estimateSizeFast(sparseNulls); + assert.ok( + result > ESTIMATE_SIZE_BYTE_LIMIT, + `node-budget exhaustion must return >256KiB, got ${result}` + ); + assert.ok( + elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8, + `must not access far beyond node budget; accesses=${elementAccesses}` + ); + assert.ok(elementAccesses > 100, `expected many bounded visits, got ${elementAccesses}`); + assert.equal(isSmallEnoughForSemanticCache(sparseNulls), false); +}); + +test("estimateSizeFast node budget fails closed on empty-object / getter proxy array", () => { + let elementAccesses = 0; + let farGetterHits = 0; + const emptyObjectArray = new Proxy([] as unknown[], { + get(target, prop, receiver) { + if (prop === "length") return 2_000_000; + if (typeof prop === "string" && /^[0-9]+$/.test(prop)) { + const index = Number(prop); + elementAccesses += 1; + if (index >= ESTIMATE_SIZE_NODE_BUDGET) { + farGetterHits += 1; + } + // Fresh empty object per access — old impl would stack-push every reference. + return {}; + } + return Reflect.get(target, prop, receiver); + }, + }); + + const result = estimateSizeFast(emptyObjectArray); + assert.ok(result > ESTIMATE_SIZE_BYTE_LIMIT, `expected fail-closed, got ${result}`); + assert.ok( + elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8, + `accesses must stay near node budget; got ${elementAccesses}` + ); + assert.equal( + farGetterHits, + 0, + `entries beyond the node budget must not be touched; far hits=${farGetterHits}` + ); + assert.equal(isSmallEnoughForSemanticCache(emptyObjectArray), false); +}); diff --git a/tests/unit/resource-pressure-policy.test.ts b/tests/unit/resource-pressure-policy.test.ts new file mode 100644 index 0000000000..4db4986f52 --- /dev/null +++ b/tests/unit/resource-pressure-policy.test.ts @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + createResourcePressureTracker, + resolveResourcePressureThresholds, + type PressureReason, + type PressureSeverity, + type ResourcePressureState, + type ResourcePressureThresholds, + type ResourceSignals, +} from "../../open-sse/utils/resourcePressurePolicy.ts"; + +const MiB = 1024 ** 2; + +function baseSignals(overrides: Partial = {}): ResourceSignals { + return { + observedAtMs: 1_000, + v8: { heapUsedBytes: 100 * MiB, heapLimitBytes: 1_000 * MiB }, + process: { + rssBytes: 200 * MiB, + externalBytes: 10 * MiB, + arrayBuffersBytes: MiB, + availableBytes: null, + constrainedBytes: null, + }, + cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null }, + psi: null, + ...overrides, + }; +} + +const fastThresholds: Partial = { + highRatio: 0.8, + criticalRatio: 0.9, + recoveryRatio: 0.7, + highPsiAvg10: 20, + criticalPsiAvg10: 40, + recoveryPsiAvg10: 10, + sustainedSamplesHigh: 2, + sustainedSamplesCritical: 2, + sustainedSamplesRecovery: 2, + heapAbsoluteThresholdMb: null, +}; + +describe("resource pressure threshold validation", () => { + it("accepts every valid boundary", () => { + const thresholds = resolveResourcePressureThresholds({ + recoveryRatio: 0, + highRatio: 0.5, + criticalRatio: 1, + recoveryPsiAvg10: 0, + highPsiAvg10: 50, + criticalPsiAvg10: 100, + sustainedSamplesHigh: 1, + sustainedSamplesCritical: 1, + sustainedSamplesRecovery: 10_000, + heapAbsoluteThresholdMb: null, + }); + assert.equal(thresholds.recoveryRatio, 0); + assert.equal(thresholds.criticalRatio, 1); + assert.equal(thresholds.criticalPsiAvg10, 100); + assert.equal(thresholds.sustainedSamplesRecovery, 10_000); + assert.equal(thresholds.heapAbsoluteThresholdMb, null); + }); + + it("throws deterministically for invalid partial overrides", () => { + const invalid: Array> = [ + { recoveryRatio: -0.01 }, + { criticalRatio: 1.01 }, + { highRatio: Number.NaN }, + { recoveryRatio: 0.8, highRatio: 0.8 }, + { highRatio: 0.95, criticalRatio: 0.9 }, + { recoveryPsiAvg10: -1 }, + { criticalPsiAvg10: 101 }, + { highPsiAvg10: Number.POSITIVE_INFINITY }, + { recoveryPsiAvg10: 20, highPsiAvg10: 20 }, + { highPsiAvg10: 50, criticalPsiAvg10: 40 }, + { sustainedSamplesHigh: 0 }, + { sustainedSamplesCritical: 1.5 }, + { sustainedSamplesRecovery: 10_001 }, + { heapAbsoluteThresholdMb: 0 }, + { heapAbsoluteThresholdMb: Number.POSITIVE_INFINITY }, + ]; + for (const partial of invalid) { + assert.throws(() => resolveResourcePressureThresholds(partial), RangeError); + } + }); +}); + +describe("resource pressure policy", () => { + it("does not let high then critical count as two critical samples", () => { + const tracker = createResourcePressureTracker(fastThresholds); + const high = baseSignals({ + v8: { heapUsedBytes: 850 * MiB, heapLimitBytes: 1_000 * MiB }, + }); + const critical = baseSignals({ + v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB }, + }); + + assert.equal(tracker.observe(high).severity, "normal"); + assert.equal(tracker.observe(critical).severity, "normal"); + assert.equal(tracker.observe(critical).severity, "critical"); + }); + + it("resets pending streak when severity or reason alternates", () => { + const tracker = createResourcePressureTracker(fastThresholds); + const heapCritical = baseSignals({ + v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB }, + }); + const psiCritical = baseSignals({ + psi: { + someAvg10: 50, + someAvg60: null, + someAvg300: null, + fullAvg10: null, + fullAvg60: null, + fullAvg300: null, + }, + }); + + assert.equal(tracker.observe(heapCritical).severity, "normal"); + assert.equal(tracker.observe(psiCritical).severity, "normal"); + assert.equal(tracker.observe(psiCritical).severity, "critical"); + assert.equal(tracker.getState().reason, "psi_some"); + }); + + it("baselines cumulative OOM counters and only treats increases as events", () => { + const tracker = createResourcePressureTracker(fastThresholds); + const oomCounters = (oom: number, oom_kill: number, observedAtMs: number) => + baseSignals({ + observedAtMs, + cgroup: { + currentBytes: null, + maxBytes: null, + highBytes: null, + events: { low: 0, high: 0, max: 0, oom, oom_kill }, + }, + }); + + assert.equal(tracker.observe(oomCounters(7, 3, 1)).severity, "normal", "history baselines"); + assert.equal(tracker.observe(oomCounters(7, 3, 2)).severity, "normal", "unchanged history"); + + const event = tracker.observe(oomCounters(8, 3, 3)); + assert.equal(event.severity, "critical", "a new OOM event is immediately critical"); + assert.equal(event.reason, "oom_event"); + + assert.equal(tracker.observe(oomCounters(8, 3, 4)).severity, "critical"); + assert.equal( + tracker.observe(oomCounters(8, 3, 5)).severity, + "normal", + "unchanged allows recovery" + ); + }); + + it("re-baselines when OOM counters reset or the cgroup event source is replaced", () => { + const tracker = createResourcePressureTracker(fastThresholds); + const events = (oom: number, oom_kill: number) => + baseSignals({ + cgroup: { + currentBytes: null, + maxBytes: null, + highBytes: null, + events: { low: 0, high: 0, max: 0, oom, oom_kill }, + }, + }); + + assert.equal(tracker.observe(events(10, 4)).severity, "normal"); + assert.equal(tracker.observe(events(1, 0)).severity, "normal", "counter reset re-baselines"); + assert.equal( + tracker.observe({ ...events(1, 0), cgroup: { ...events(1, 0).cgroup, events: null } }) + .severity, + "normal" + ); + assert.equal(tracker.observe(events(9, 3)).severity, "normal", "replacement re-baselines"); + }); + + it("keeps snapshot state fields and bounded-cardinality values", () => { + const tracker = createResourcePressureTracker(fastThresholds); + const state: ResourcePressureState = tracker.observe(baseSignals()); + const severities = new Set(["normal", "high", "critical"]); + const reasons = new Set([ + "none", + "v8_heap_ratio", + "v8_heap_absolute", + "cgroup_ratio", + "cgroup_high", + "psi_some", + "psi_full", + "oom_event", + ]); + assert.ok(severities.has(state.severity)); + assert.ok(reasons.has(state.reason)); + assert.deepEqual(Object.keys(state).sort(), [ + "elevatedStreak", + "lastTransitionAtMs", + "observedAtMs", + "reason", + "recoveryStreak", + "severity", + ]); + }); +}); diff --git a/tests/unit/resource-pressure-runtime.test.ts b/tests/unit/resource-pressure-runtime.test.ts new file mode 100644 index 0000000000..a26e694f8b --- /dev/null +++ b/tests/unit/resource-pressure-runtime.test.ts @@ -0,0 +1,330 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + createResourcePressureRuntime, + type ResourcePressureRuntime, +} from "../../open-sse/utils/resourcePressure.ts"; +import type { ResourceSignals } from "../../open-sse/utils/resourcePressurePolicy.ts"; + +const MiB = 1024 ** 2; + +function signals(observedAtMs: number, heapUsedMb = 100): ResourceSignals { + return { + observedAtMs, + v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB }, + process: { + rssBytes: 200 * MiB, + externalBytes: 10 * MiB, + arrayBuffersBytes: MiB, + availableBytes: null, + constrainedBytes: null, + }, + cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null }, + psi: null, + }; +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +async function settleRefresh(runtime: ResourcePressureRuntime): Promise { + await runtime.whenRefreshSettled(); + await Promise.resolve(); +} + +describe("ResourcePressureRuntime stale-while-revalidate cache", () => { + it("does no proc/sys I/O in check(), while a cheap first-request heap breach sheds immediately", async () => { + let slowSamples = 0; + const runtime = createResourcePressureRuntime({ + heapThresholdMb: 200, + immediateHeapUsedMb: () => 201, + sample: async () => { + slowSamples += 1; + return signals(1); + }, + }); + + const guard = runtime.check(); + assert.ok(guard); + assert.equal(guard.status, 503); + assert.equal(slowSamples, 0, "request-path check must not invoke the async proc/sys sampler"); + assert.equal(runtime.getObservation().state.reason, "v8_heap_absolute"); + await settleRefresh(runtime); + assert.equal(slowSamples, 1, "refresh may run after the request-path decision"); + runtime.dispose(); + }); + + it("serves a fresh cached sample without scheduling another refresh", async () => { + let now = 0; + let calls = 0; + const runtime = createResourcePressureRuntime({ + nowMs: () => now, + staleAfterMs: 100, + immediateHeapUsedMb: () => 100, + sample: async () => { + calls += 1; + return signals(now); + }, + }); + + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 1); + now = 99; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 1); + runtime.dispose(); + }); + + it("schedules at most one refresh under concurrent stale checks", async () => { + let now = 0; + let calls = 0; + const pending = deferred(); + const runtime = createResourcePressureRuntime({ + nowMs: () => now, + staleAfterMs: 10, + immediateHeapUsedMb: () => 100, + sample: async () => { + calls += 1; + if (calls === 1) return signals(0); + return pending.promise; + }, + }); + + runtime.check(); + await settleRefresh(runtime); + now = 11; + for (let index = 0; index < 50; index += 1) runtime.check(); + assert.equal(calls, 1, "scheduled work must not run synchronously in check()"); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(calls, 2); + pending.resolve(signals(11)); + await settleRefresh(runtime); + assert.equal(calls, 2); + runtime.dispose(); + }); + + it("retains a bounded stale snapshot on refresh failure and retries only after backoff", async () => { + let now = 0; + let calls = 0; + const runtime = createResourcePressureRuntime({ + nowMs: () => now, + staleAfterMs: 10, + maxStaleMs: 100, + retryAfterMs: 20, + immediateHeapUsedMb: () => 100, + sample: async () => { + calls += 1; + if (calls === 1) return signals(0, 950); + throw new Error("proc unavailable"); + }, + thresholds: { + sustainedSamplesCritical: 1, + heapAbsoluteThresholdMb: null, + }, + }); + + runtime.check(); + await settleRefresh(runtime); + now = 11; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 2); + assert.equal(runtime.getObservation().signals?.observedAtMs, 0, "failure retains stale data"); + + now = 25; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 2, "failure backoff prevents a refresh storm"); + + now = 31; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 3); + + now = 101; + assert.equal(runtime.check(), null, "expired stale adaptive pressure fails open"); + runtime.dispose(); + }); + + it("measures failure backoff from settlement, not refresh start", async () => { + let now = 0; + let calls = 0; + const pending = deferred(); + const runtime = createResourcePressureRuntime({ + nowMs: () => now, + staleAfterMs: 10, + maxStaleMs: 100, + retryAfterMs: 20, + immediateHeapUsedMb: () => 100, + sample: async () => { + calls += 1; + if (calls === 1) return signals(0); + return pending.promise; + }, + }); + + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 1); + + now = 11; + runtime.check(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(calls, 2); + + // Slow failure: wall clock advances past retryAfter before the sample rejects. + now = 50; + pending.reject(new Error("proc unavailable")); + await settleRefresh(runtime); + assert.equal(calls, 2); + + // Retry must wait full retryAfterMs from settlement (50), not from start (11). + now = 69; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 2, "failure backoff starts at settlement, not refresh start"); + + now = 70; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 3); + runtime.dispose(); + }); + + it("measures success freshness from publication, not refresh start", async () => { + let now = 0; + let calls = 0; + const pending = deferred(); + const runtime = createResourcePressureRuntime({ + nowMs: () => now, + staleAfterMs: 20, + maxStaleMs: 100, + immediateHeapUsedMb: () => 100, + sample: async () => { + calls += 1; + if (calls === 1) return signals(0); + return pending.promise; + }, + }); + + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 1); + + now = 21; + runtime.check(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(calls, 2); + + // Slow success: wall clock advances past staleAfter before the sample resolves. + now = 100; + pending.resolve(signals(100)); + await settleRefresh(runtime); + assert.equal(calls, 2); + assert.equal(runtime.getObservation().signals?.observedAtMs, 100); + + // Freshness must run full staleAfterMs from publication (100), not start (21). + now = 119; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 2, "success freshness starts at publication, not refresh start"); + + now = 120; + runtime.check(); + await settleRefresh(runtime); + assert.equal(calls, 3); + runtime.dispose(); + }); + + it("default scheduler unrefs Immediate; injected schedulers stay caller-owned", async () => { + // Injected schedule is never wrapped: the runtime must not call unref on it. + let scheduled = 0; + let unrefCalled = 0; + const injected = (refresh: () => void) => { + scheduled += 1; + const handle = setImmediate(refresh); + const originalUnref = handle.unref.bind(handle); + handle.unref = () => { + unrefCalled += 1; + return originalUnref(); + }; + }; + + const withInjected = createResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => signals(1), + schedule: injected, + }); + withInjected.check(); + await settleRefresh(withInjected); + assert.equal(scheduled, 1); + assert.equal(unrefCalled, 0, "injected schedule handles remain caller-owned"); + withInjected.dispose(); + + // Default schedule path: capture the Immediate and prove it is unref'd so a + // pending refresh alone cannot keep the process alive. + const originalSetImmediate = globalThis.setImmediate; + let captured: NodeJS.Immediate | undefined; + globalThis.setImmediate = ((callback: (...args: unknown[]) => void, ...args: unknown[]) => { + const handle = originalSetImmediate(callback, ...args); + captured = handle; + return handle; + }) as typeof setImmediate; + try { + const runtime = createResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + // Never resolve: we only care about the scheduled Immediate ref state. + sample: () => new Promise(() => {}), + }); + runtime.check(); + assert.ok(captured, "default schedule must use setImmediate"); + assert.equal(captured.hasRef(), false, "default Immediate must be unref'd"); + runtime.dispose(); + if (captured) clearImmediate(captured); + } finally { + globalThis.setImmediate = originalSetImmediate; + } + }); + + it("dispose ignores late refresh results and independently owned runtimes do not share state", async () => { + const pending = deferred(); + let firstCalls = 0; + const first = createResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => { + firstCalls += 1; + return pending.promise; + }, + }); + first.check(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(firstCalls, 1); + first.dispose(); + pending.resolve(signals(1)); + await settleRefresh(first); + assert.equal( + first.getObservation().signals, + null, + "disposed runtime ignores late refresh results" + ); + + const second = createResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => signals(2), + }); + assert.notEqual(first, second); + second.check(); + await settleRefresh(second); + assert.equal(second.getObservation().signals?.observedAtMs, 2); + second.dispose(); + }); +}); diff --git a/tests/unit/resource-pressure-sampler.test.ts b/tests/unit/resource-pressure-sampler.test.ts new file mode 100644 index 0000000000..7410d40bd9 --- /dev/null +++ b/tests/unit/resource-pressure-sampler.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + decodeMountInfoPath, + parseCgroup2Mount, + parseCgroupV2Path, + resolveCgroupDirectory, + sampleResourceSignals, + sanitizeMemoryBytes, + type ResourcePressureFs, +} from "../../open-sse/utils/resourcePressureSampler.ts"; + +const MiB = 1024 ** 2; +const GiB = 1024 ** 3; + +function memoryUsage(heapUsed = 1): NodeJS.MemoryUsage { + return { + rss: 500 * MiB, + heapTotal: 300 * MiB, + heapUsed, + external: 12 * MiB, + arrayBuffers: 3 * MiB, + }; +} + +function mapFs(entries: ReadonlyArray): ResourcePressureFs { + const files = new Map(entries); + return { readText: async (filePath) => files.get(filePath) ?? null }; +} + +describe("resource pressure cgroup parsers", () => { + it("accepts only the exact unified cgroup entry", () => { + assert.equal(parseCgroupV2Path("2:cpu:/wrong\n0::/delegated/service\n"), "/delegated/service"); + assert.equal(parseCgroupV2Path("0:cpu:/wrong\n1::/also-wrong\n"), null); + }); + + it("decodes mountinfo octal escapes in root and mountpoint", () => { + assert.equal( + decodeMountInfoPath("/sys/fs/cgroup\\040space\\134unit"), + "/sys/fs/cgroup space\\unit" + ); + assert.deepEqual( + parseCgroup2Mount( + "43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n" + ), + { root: "/delegated root", mountpoint: "/sys/fs/cgroup space" } + ); + }); + + it("resolves delegated mount roots within the decoded mountpoint", async () => { + const fs = mapFs([ + ["/proc/self/cgroup", "0::/delegated root/team/service\n"], + [ + "/proc/self/mountinfo", + "43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n", + ], + ["/sys/fs/cgroup space/team/service/memory.current", "1\n"], + ]); + + assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup space/team/service"); + }); + + it("rejects NUL, traversal, malformed, and out-of-root cgroup paths", async () => { + for (const cgroupPath of [ + "/delegated/../escape", + "/delegated/%2e%2e/escape", + "/delegated/service\0escape", + "delegated/service", + "/other/service", + ]) { + const fs = mapFs([ + ["/proc/self/cgroup", `0::${cgroupPath}\n`], + ["/proc/self/mountinfo", "43 34 0:35 /delegated /safe/cgroup rw - cgroup2 cgroup2 rw\n"], + ["/safe/cgroup/memory.current", "1\n"], + ]); + assert.equal( + await resolveCgroupDirectory(fs.readText, { allowDefaultFallback: false }), + null, + cgroupPath + ); + } + }); + + it("falls back to the validated default cgroup root when proc metadata is malformed", async () => { + const fs = mapFs([ + ["/proc/self/cgroup", "malformed\n"], + ["/proc/self/mountinfo", "malformed\n"], + ["/sys/fs/cgroup/memory.current", "123\n"], + ]); + assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup"); + }); +}); + +describe("sampleResourceSignals", () => { + it("captures process, V8, cgroup, event, and PSI snapshot fields", async () => { + const fs = mapFs([ + ["/proc/self/cgroup", "0::/slice/service\n"], + ["/proc/self/mountinfo", "43 34 0:35 / /sys/fs/cgroup rw - cgroup2 cgroup2 rw\n"], + ["/sys/fs/cgroup/slice/service/memory.current", `${800 * MiB}\n`], + ["/sys/fs/cgroup/slice/service/memory.max", `${GiB}\n`], + ["/sys/fs/cgroup/slice/service/memory.high", "966367641\n"], + ["/sys/fs/cgroup/slice/service/memory.events", "low 1\nhigh 2\nmax 3\noom 4\noom_kill 5\n"], + [ + "/proc/pressure/memory", + "some avg10=1.50 avg60=2.00 avg300=3.25 total=9\nfull avg10=0.25 avg60=0.50 avg300=0.75 total=1\n", + ], + ]); + + const signals = await sampleResourceSignals({ + nowMs: () => 42, + memoryUsage: () => memoryUsage(250 * MiB), + heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 250 * MiB }), + availableMemory: () => 4 * GiB, + constrainedMemory: () => undefined, + fs, + }); + + assert.equal(signals.observedAtMs, 42); + assert.deepEqual(signals.v8, { heapUsedBytes: 250 * MiB, heapLimitBytes: GiB }); + assert.deepEqual(signals.process, { + rssBytes: 500 * MiB, + externalBytes: 12 * MiB, + arrayBuffersBytes: 3 * MiB, + availableBytes: 4 * GiB, + constrainedBytes: null, + }); + assert.deepEqual(signals.cgroup, { + currentBytes: 800 * MiB, + maxBytes: GiB, + highBytes: 966367641, + events: { low: 1, high: 2, max: 3, oom: 4, oom_kill: 5 }, + }); + assert.equal(signals.psi?.someAvg10, 1.5); + assert.equal(signals.psi?.fullAvg10, 0.25); + }); + + it("fails open when platform reads fail or return malformed values", async () => { + const signals = await sampleResourceSignals({ + memoryUsage: () => memoryUsage(), + heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 1 }), + availableMemory: () => { + throw new Error("unavailable"); + }, + constrainedMemory: () => Number.POSITIVE_INFINITY, + fs: { + readText: async (filePath) => { + if (filePath === "/proc/self/cgroup") throw new Error("unavailable"); + return "malformed"; + }, + }, + }); + assert.equal(signals.process.availableBytes, null); + assert.equal(signals.process.constrainedBytes, null); + assert.deepEqual(signals.cgroup, { + currentBytes: null, + maxBytes: null, + highBytes: null, + events: null, + }); + assert.equal(signals.psi, null); + }); + + it("treats missing, zero, max, and unsafe memory quantities as unavailable", () => { + for (const value of [ + undefined, + "", + "max", + 0, + -1, + Number.NaN, + Number.MAX_SAFE_INTEGER, + 2 ** 63, + ]) { + assert.equal(sanitizeMemoryBytes(value), null, String(value)); + } + assert.equal(sanitizeMemoryBytes("123"), 123); + }); +}); diff --git a/tests/unit/resource-pressure.test.ts b/tests/unit/resource-pressure.test.ts new file mode 100644 index 0000000000..14ea7d7457 --- /dev/null +++ b/tests/unit/resource-pressure.test.ts @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + createResourcePressureRuntime, + getResourcePressureObservation, + reloadResourcePressureRuntime, + type ResourceSignals, +} from "../../open-sse/utils/resourcePressure.ts"; + +const MiB = 1024 ** 2; + +function signals(observedAtMs: number, heapUsedMb: number): ResourceSignals { + return { + observedAtMs, + v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB }, + process: { + rssBytes: 200 * MiB, + externalBytes: 10 * MiB, + arrayBuffersBytes: MiB, + availableBytes: null, + constrainedBytes: null, + }, + cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null }, + psi: null, + }; +} + +describe("resource pressure HTTP guard facade", () => { + it("preserves strict immediate first-request heap shedding", async () => { + let samples = 0; + const runtime = createResourcePressureRuntime({ + heapThresholdMb: 200, + immediateHeapUsedMb: () => 201, + sample: async () => { + samples += 1; + return signals(1, 100); + }, + }); + + const guard = runtime.check(); + assert.ok(guard); + assert.equal(guard.status, 503); + assert.equal(samples, 0, "the asynchronous sampler cannot run in the request path"); + runtime.dispose(); + }); + + it("does not shed when heap usage equals the strict threshold", () => { + const runtime = createResourcePressureRuntime({ + heapThresholdMb: 200, + immediateHeapUsedMb: () => 200, + sample: async () => signals(1, 100), + }); + assert.equal(runtime.check(), null); + runtime.dispose(); + }); + + it("returns a sanitized standards-correct 503 with Retry-After", async () => { + const runtime = createResourcePressureRuntime({ + heapThresholdMb: 200, + immediateHeapUsedMb: () => 987, + sample: async () => signals(1, 100), + }); + + const guard = runtime.check(); + assert.ok(guard); + assert.equal(guard.success, false); + assert.equal(guard.status, 503); + assert.equal(guard.response.status, 503); + assert.equal(guard.response.headers.get("Retry-After"), "5"); + assert.equal(guard.response.headers.get("Content-Type"), "application/json"); + const payload = await guard.response.json(); + assert.deepEqual(payload.error, { + message: "Service temporarily unavailable due to resource pressure. Retry shortly.", + type: "server_error", + code: "resource_pressure", + }); + const clientText = JSON.stringify(payload) + guard.error; + assert.ok(!clientText.includes("987")); + assert.ok(!/\bMB\b/.test(clientText)); + runtime.dispose(); + }); + + it("reload atomically replaces and resets the thin default facade", async () => { + let firstCalls = 0; + reloadResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => { + firstCalls += 1; + return signals(1, 100); + }, + }); + const replacement = reloadResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => signals(2, 100), + }); + + assert.deepEqual(getResourcePressureObservation(), { + signals: null, + state: { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + }, + }); + assert.equal(firstCalls, 0, "replaced runtime must not retain or run scheduled work"); + replacement.dispose(); + }); + + it("exposes all observation snapshot fields", async () => { + const runtime = createResourcePressureRuntime({ + immediateHeapUsedMb: () => 100, + sample: async () => signals(42, 100), + }); + assert.deepEqual(runtime.getObservation(), { + signals: null, + state: { + severity: "normal", + reason: "none", + elevatedStreak: 0, + recoveryStreak: 0, + lastTransitionAtMs: 0, + observedAtMs: 0, + }, + }); + runtime.check(); + await runtime.whenRefreshSettled(); + assert.equal(runtime.getObservation().signals?.observedAtMs, 42); + assert.equal(runtime.getObservation().state.observedAtMs, 42); + runtime.dispose(); + }); +});