feat(admission): add adaptive overload and pressure controls

Add bounded weighted admission with fair queuing, deadline and cancellation handling, exact lease accounting, and a default-shadow runtime. Keep asynchronous resource-pressure shedding as an independent safety fuse and bound request feature estimation.
This commit is contained in:
Xiangzhe
2026-08-03 07:58:07 +08:00
committed by diegosouzapw
parent ce764bc6f3
commit a61020153c
26 changed files with 7323 additions and 44 deletions

View File

@@ -67,7 +67,7 @@ import {
resolveMemoryOwnerId,
} from "./chatCore/memoryExtraction.ts";
import { CORS_HEADERS } from "../utils/cors.ts";
import { checkHeapPressureGuard } from "../utils/heapPressure.ts";
import { checkResourcePressureGuard } from "../utils/resourcePressure.ts";
import { normalizeHeaders } from "../utils/headers.ts";
import { resolveChatCoreRequestFormat } from "./chatCore/requestFormat.ts";
import { resolveChatCoreTargetFormat } from "./chatCore/targetFormat.ts";
@@ -359,13 +359,6 @@ import {
isRpmExhausted,
} from "../services/geminiRateLimitTracker.ts";
// ── Global memory pressure guard ────────────────────────────────────────
// Prevents OOM by rejecting new requests when V8 heap exceeds threshold.
// Self-healing: no counters to leak, no cleanup needed. The threshold
// auto-calibrates to 85% of the actual V8 heap ceiling (see heapPressure.ts) so
// it tracks --max-old-space-size across 1GB/2GB/large VPS instead of a fixed
// 200MB that sat below the app's own ~260MB baseline and rejected every request.
import { isSmallEnoughForSemanticCache } from "../utils/estimateSize.ts";
/**
@@ -415,17 +408,16 @@ export async function handleChatCore({
createPiiTransform = null,
correlationId = null,
modelPinned = false,
skipResourcePressureGuard = false,
}) {
let { provider, model, extendedContext } = modelInfo;
// ── Memory pressure guard ────────────────────────────────────────────
// Reject early if V8 heap is already near the 256MB limit. Prevents
// cascading OOM when many large-context requests arrive concurrently.
try {
const heapUsedMB = process.memoryUsage().heapUsed / (1024 * 1024);
const heapGuard = checkHeapPressureGuard(heapUsedMB);
if (heapGuard) return heapGuard;
} catch {
/* memoryUsage() never throws */
if (!skipResourcePressureGuard) {
try {
const pressureGuard = checkResourcePressureGuard();
if (pressureGuard) return pressureGuard;
} catch {
/* fail open */
}
}
// Per-request model-routing metadata (first extracted slice of the request-setup phase).

View File

@@ -0,0 +1,168 @@
import type { AdmissionPressure, AdmissionReleaseOutcome } from "./types.ts";
export interface AdaptationParams {
minLimit: number;
maxLimit: number;
windowMs: number;
shortLatencyAlpha: number;
longLatencyAlpha: number;
increaseStep: number;
decreaseFactor: number;
criticalDecreaseFactor: number;
highUtilizationThreshold: number;
lowUtilizationThreshold: number;
latencyGradientThreshold: number;
maxIncreasePerWindow: number;
}
export interface AdaptationState {
currentLimit: number;
shortLatencyEwma: number;
longLatencyEwma: number;
pressure: AdmissionPressure;
/** Sum of admitted cost * time contribution proxies in the open window. */
windowActiveCostIntegral: number;
windowCompleted: number;
windowLatencySamples: number;
windowStartMs: number;
freezeGrowth: boolean;
/**
* When true, critical multiplicative decrease already applied for this window
* (e.g. via immediate observePressure). Window close must not re-apply it.
*/
criticalDecreaseConsumed: boolean;
utilization: number;
}
export function clampLimit(value: number, minLimit: number, maxLimit: number): number {
if (!Number.isFinite(value)) return minLimit;
return Math.min(maxLimit, Math.max(minLimit, Math.floor(value)));
}
export function createAdaptationState(
initialLimit: number,
minLimit: number,
maxLimit: number,
nowMs: number
): AdaptationState {
return {
currentLimit: clampLimit(initialLimit, minLimit, maxLimit),
shortLatencyEwma: 0,
longLatencyEwma: 0,
pressure: "normal",
windowActiveCostIntegral: 0,
windowCompleted: 0,
windowLatencySamples: 0,
windowStartMs: nowMs,
freezeGrowth: false,
criticalDecreaseConsumed: false,
utilization: 0,
};
}
export function noteLatency(
state: AdaptationState,
latencyMs: number,
params: AdaptationParams
): void {
const sample = Number.isFinite(latencyMs) && latencyMs >= 0 ? latencyMs : 0;
state.windowLatencySamples += 1;
const sa = params.shortLatencyAlpha;
const la = params.longLatencyAlpha;
if (state.shortLatencyEwma <= 0 && state.longLatencyEwma <= 0) {
state.shortLatencyEwma = sample;
state.longLatencyEwma = sample;
return;
}
state.shortLatencyEwma = sa * sample + (1 - sa) * state.shortLatencyEwma;
state.longLatencyEwma = la * sample + (1 - la) * state.longLatencyEwma;
}
export function noteOutcome(state: AdaptationState, outcome: AdmissionReleaseOutcome): void {
// A single upstream business error freezes growth for the current window; it must not
// apply critical multiplicative collapse on its own.
if (outcome === "upstream_error") {
state.freezeGrowth = true;
return;
}
if (outcome === "timeout") {
state.freezeGrowth = true;
}
}
export function setPressure(state: AdaptationState, pressure: AdmissionPressure): void {
const severity: Record<AdmissionPressure, number> = { normal: 0, high: 1, critical: 2 };
if (severity[pressure] > severity[state.pressure]) state.pressure = pressure;
}
/**
* Close the current feedback window and adjust the limit.
* Recovery (increase) is slower than decrease; idle/low utilization does not inflate.
*/
export function closeAdaptationWindow(
state: AdaptationState,
params: AdaptationParams,
nowMs: number
): void {
const elapsed = Math.max(1, Math.min(params.windowMs, nowMs - state.windowStartMs));
// sampleActiveIntegral already accounts for every interval exactly once.
const avgActive = state.windowActiveCostIntegral / elapsed;
const util = state.currentLimit > 0 ? avgActive / state.currentLimit : 0;
state.utilization = Math.max(0, Math.min(1, util));
let next = state.currentLimit;
const gradient =
state.longLatencyEwma > 0
? (state.shortLatencyEwma - state.longLatencyEwma) / state.longLatencyEwma
: 0;
if (state.pressure === "critical") {
// Immediate observePressure may already have applied the critical factor once.
if (!state.criticalDecreaseConsumed) {
next = Math.floor(next * params.criticalDecreaseFactor);
}
} else if (
state.pressure === "high" ||
(state.windowLatencySamples > 0 && gradient >= params.latencyGradientThreshold)
) {
next = Math.floor(next * params.decreaseFactor);
} else if (
!state.freezeGrowth &&
state.pressure === "normal" &&
state.utilization >= params.highUtilizationThreshold &&
state.windowCompleted > 0
) {
const step = Math.min(params.increaseStep, params.maxIncreasePerWindow);
next = next + step;
}
// A genuinely low-utilization window recovers the latency baseline so stale gradients expire.
if (state.utilization <= params.lowUtilizationThreshold) {
state.shortLatencyEwma = state.longLatencyEwma;
}
state.currentLimit = clampLimit(next, params.minLimit, params.maxLimit);
state.windowActiveCostIntegral = 0;
state.windowCompleted = 0;
state.windowLatencySamples = 0;
state.windowStartMs = nowMs;
state.freezeGrowth = false;
state.criticalDecreaseConsumed = false;
state.pressure = "normal";
}
export function sampleActiveIntegral(
state: AdaptationState,
activeCost: number,
dtMs: number
): void {
if (dtMs <= 0 || activeCost <= 0) return;
const boundedActiveCost = Math.min(activeCost, state.currentLimit);
const contribution =
dtMs > Math.floor(Number.MAX_SAFE_INTEGER / boundedActiveCost)
? Number.MAX_SAFE_INTEGER
: boundedActiveCost * dtMs;
state.windowActiveCostIntegral =
contribution >= Number.MAX_SAFE_INTEGER - state.windowActiveCostIntegral
? Number.MAX_SAFE_INTEGER
: state.windowActiveCostIntegral + contribution;
}

View File

@@ -0,0 +1,167 @@
import { resolveCostConfig } from "./cost.ts";
import {
MAX_ADMISSION_COST_OR_LIMIT,
MAX_ADMISSION_WINDOW_MS,
type AdaptiveAdmissionConfig,
type AdmissionMode,
} from "./types.ts";
import type { AdaptationParams } from "./adaptation.ts";
export { MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_WINDOW_MS };
export interface ValidatedConfig {
mode: AdmissionMode;
minLimit: number;
maxLimit: number;
initialLimit: number;
maxQueueCount: number;
maxQueueCost: number;
defaultMaxWaitMs: number;
windowMs: number;
adaptation: AdaptationParams;
maxRequestCost: number;
costConfig: ReturnType<typeof resolveCostConfig>;
}
function requirePositiveInt(
name: string,
value: unknown,
max: number = MAX_ADMISSION_COST_OR_LIMIT
): number {
if (
typeof value !== "number" ||
!Number.isFinite(value) ||
value <= 0 ||
!Number.isSafeInteger(value)
) {
throw new RangeError(`${name} must be a positive safe integer`);
}
if (value > max) {
throw new RangeError(`${name} must be <= ${max}`);
}
return value;
}
function requireUnitInterval(name: string, value: unknown, fallback: number): number {
if (value === undefined) return fallback;
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1) {
throw new RangeError(`${name} must be in (0, 1]`);
}
return value;
}
function requireDecreaseFactor(name: string, value: unknown, fallback: number): number {
if (value === undefined) return fallback;
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value >= 1) {
throw new RangeError(`${name} must be in (0, 1)`);
}
return value;
}
function resolveMode(mode: AdaptiveAdmissionConfig["mode"]): AdmissionMode {
if (mode === undefined) return "shadow";
if (mode !== "off" && mode !== "shadow" && mode !== "enforce") {
throw new RangeError("mode must be off|shadow|enforce");
}
return mode;
}
function resolveAdaptationParams(
input: AdaptiveAdmissionConfig,
minLimit: number,
maxLimit: number,
windowMs: number
): AdaptationParams {
const decreaseFactor = requireDecreaseFactor("decreaseFactor", input.decreaseFactor, 0.8);
const criticalDecreaseFactor = requireDecreaseFactor(
"criticalDecreaseFactor",
input.criticalDecreaseFactor,
0.5
);
const increaseStep =
input.increaseStep === undefined ? 1 : requirePositiveInt("increaseStep", input.increaseStep);
const maxIncreasePerWindow =
input.maxIncreasePerWindow === undefined
? increaseStep
: requirePositiveInt("maxIncreasePerWindow", input.maxIncreasePerWindow);
const shortLatencyAlpha = requireUnitInterval("shortLatencyAlpha", input.shortLatencyAlpha, 0.5);
const longLatencyAlpha = requireUnitInterval("longLatencyAlpha", input.longLatencyAlpha, 0.1);
const highUtilizationThreshold = requireUnitInterval(
"highUtilizationThreshold",
input.highUtilizationThreshold,
0.7
);
const lowUtilizationThreshold = requireUnitInterval(
"lowUtilizationThreshold",
input.lowUtilizationThreshold,
0.3
);
if (criticalDecreaseFactor > decreaseFactor) {
throw new RangeError("criticalDecreaseFactor must be <= decreaseFactor");
}
if (lowUtilizationThreshold >= highUtilizationThreshold) {
throw new RangeError("lowUtilizationThreshold must be < highUtilizationThreshold");
}
if (shortLatencyAlpha <= longLatencyAlpha) {
throw new RangeError("shortLatencyAlpha must be > longLatencyAlpha");
}
return {
minLimit,
maxLimit,
windowMs,
shortLatencyAlpha,
longLatencyAlpha,
increaseStep,
decreaseFactor,
criticalDecreaseFactor,
highUtilizationThreshold,
lowUtilizationThreshold,
latencyGradientThreshold: requireUnitInterval(
"latencyGradientThreshold",
input.latencyGradientThreshold,
0.25
),
maxIncreasePerWindow,
};
}
export function validateConfig(input: AdaptiveAdmissionConfig): ValidatedConfig {
const minLimit = requirePositiveInt("minLimit", input.minLimit);
const maxLimit = requirePositiveInt("maxLimit", input.maxLimit);
if (minLimit > maxLimit) {
throw new RangeError("minLimit must be <= maxLimit");
}
const initialLimit = requirePositiveInt("initialLimit", input.initialLimit);
// Queue count is not multiplied into cost×time products; keep the full safe-integer range.
const maxQueueCount = requirePositiveInt(
"maxQueueCount",
input.maxQueueCount,
Number.MAX_SAFE_INTEGER
);
const maxQueueCost = requirePositiveInt("maxQueueCost", input.maxQueueCost);
const windowMs =
input.windowMs === undefined
? 1000
: requirePositiveInt("windowMs", input.windowMs, MAX_ADMISSION_WINDOW_MS);
const defaultMaxWaitMs =
input.defaultMaxWaitMs === undefined
? 5_000
: requirePositiveInt("defaultMaxWaitMs", input.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS);
const costConfig = resolveCostConfig(input.cost);
return {
mode: resolveMode(input.mode),
minLimit,
maxLimit,
initialLimit,
maxQueueCount,
maxQueueCost,
defaultMaxWaitMs,
windowMs,
maxRequestCost: costConfig.maxRequestCost,
costConfig,
adaptation: resolveAdaptationParams(input, minLimit, maxLimit, windowMs),
};
}

View File

@@ -0,0 +1,624 @@
import {
closeAdaptationWindow,
createAdaptationState,
noteLatency,
noteOutcome,
sampleActiveIntegral,
setPressure,
type AdaptationState,
} from "./adaptation.ts";
import { validateConfig, type ValidatedConfig } from "./config.ts";
import { estimateAdmissionCost, normalizeRequestCost } from "./cost.ts";
import { FairCostQueue, type QueueEntry } from "./queue.ts";
import {
MAX_ADMISSION_WINDOW_MS,
createAdmissionRejectError,
type AdaptiveAdmissionConfig,
type AdmissionAcquireResult,
type AdmissionAdmitted,
type AdmissionClock,
type AdmissionLease,
type AdmissionPressure,
type AdmissionRejectCode,
type AdmissionReleaseMeta,
type AdmissionReleaseOutcome,
type AdmissionRequest,
type AdmissionSnapshot,
type ShadowDecision,
} from "./types.ts";
type VirtualDisposition = "active" | "queued" | "rejected" | "none";
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
/** Snapshot numbers are always finite safe integers; never emit rounded unsafe Number values. */
function saturateSnapshotNumber(value: number): number {
if (!Number.isFinite(value) || value <= 0) return 0;
if (value >= Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER;
return Math.floor(value);
}
function bigintToSnapshotNumber(value: bigint): number {
if (value <= 0n) return 0;
if (value >= MAX_SAFE_BIGINT) return Number.MAX_SAFE_INTEGER;
return Number(value);
}
function addSaturated(total: number, delta: number): number {
if (delta <= 0) return saturateSnapshotNumber(total);
if (total >= Number.MAX_SAFE_INTEGER - delta) return Number.MAX_SAFE_INTEGER;
return total + delta;
}
interface ActiveLeaseRecord {
id: string;
cost: number;
released: boolean;
admittedAtMs: number;
virtualDisposition: VirtualDisposition;
}
interface QueuedPayload {
resolve: (value: AdmissionAdmitted) => void;
reject: (err: Error) => void;
signal?: AbortSignal;
onAbort?: () => void;
}
let leaseSeq = 0;
function nextId(prefix: string): string {
leaseSeq += 1;
return `${prefix}-${leaseSeq}`;
}
function defaultClock(): AdmissionClock {
return {
now: () => Date.now(),
setTimer: (fn, delayMs) => {
const handle = setTimeout(fn, delayMs);
// Window/deadline timers must not pin the event loop open when idle.
if (typeof handle.unref === "function") handle.unref();
return handle;
},
clearTimer: (id) => clearTimeout(id as ReturnType<typeof setTimeout>),
};
}
/**
* Dependency-injected weighted adaptive admission controller.
* Pure in-process core: no env/settings/route wiring.
*/
export class AdaptiveAdmissionController {
private config: ValidatedConfig;
private readonly clock: AdmissionClock;
private adaptation: AdaptationState;
private queue: FairCostQueue<QueuedPayload>;
private virtualQueue: FairCostQueue<{ recordId: string }>;
private readonly active = new Map<string, ActiveLeaseRecord>();
private activeCost = 0n;
private virtualActiveCost = 0;
private virtualActiveCount = 0;
private lastSampleMs: number;
private windowTimer: unknown = undefined;
private shutDown = false;
private admittedCount = 0;
private rejectedCount = 0;
private wouldAdmitCount = 0;
private wouldQueueCount = 0;
private wouldRejectCount = 0;
constructor(config: AdaptiveAdmissionConfig, clock?: Partial<AdmissionClock>) {
this.config = validateConfig(config);
this.clock = {
now: clock?.now ?? defaultClock().now,
setTimer: clock?.setTimer ?? defaultClock().setTimer,
clearTimer: clock?.clearTimer ?? defaultClock().clearTimer,
};
const now = this.clock.now();
this.adaptation = createAdaptationState(
this.config.initialLimit,
this.config.minLimit,
this.config.maxLimit,
now
);
this.queue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
this.lastSampleMs = now;
this.armWindowTimer();
}
updateConfig(config: AdaptiveAdmissionConfig): void {
const next = validateConfig(config);
this.sampleIntegral();
this.config = next;
this.adaptation.currentLimit = Math.min(
next.maxLimit,
Math.max(next.minLimit, this.adaptation.currentLimit)
);
this.adaptation.windowStartMs = this.clock.now();
this.adaptation.windowActiveCostIntegral = 0;
this.adaptation.windowCompleted = 0;
this.adaptation.windowLatencySamples = 0;
this.adaptation.freezeGrowth = false;
this.adaptation.criticalDecreaseConsumed = false;
this.adaptation.pressure = "normal";
this.lastSampleMs = this.clock.now();
const drained = this.queue.drain();
this.queue = new FairCostQueue(next.maxQueueCount, next.maxQueueCost);
for (const entry of drained) {
if (next.mode !== "enforce") {
this.clearEntryTimer(entry);
this.detachAbort(entry);
entry.payload.resolve(this.admit(entry.cost));
continue;
}
// Cost above the new enforce limit must fail closed immediately, never strand until deadline.
if (entry.cost > this.adaptation.currentLimit) {
this.failQueued(
entry,
"ADMISSION_OVERSIZED",
"request cost exceeds max budget after config update"
);
continue;
}
if (!this.queue.enqueue(entry)) {
this.failQueued(entry, "ADMISSION_QUEUE_FULL", "queue capacity reduced");
}
}
this.rebuildVirtualState(next.mode === "shadow");
this.armWindowTimer();
if (next.mode === "enforce") {
this.dispatch();
}
}
snapshot(): AdmissionSnapshot {
this.sampleIntegral();
return {
mode: this.config.mode,
currentLimit: this.adaptation.currentLimit,
minLimit: this.config.minLimit,
maxLimit: this.config.maxLimit,
activeCost: bigintToSnapshotNumber(this.activeCost),
activeCount: saturateSnapshotNumber(this.active.size),
queuedCost: saturateSnapshotNumber(this.queue.totalCost),
queuedCount: saturateSnapshotNumber(this.queue.size),
virtualActiveCost: saturateSnapshotNumber(this.virtualActiveCost),
virtualActiveCount: saturateSnapshotNumber(this.virtualActiveCount),
virtualQueuedCost: saturateSnapshotNumber(this.virtualQueue.totalCost),
virtualQueuedCount: saturateSnapshotNumber(this.virtualQueue.size),
admittedCount: saturateSnapshotNumber(this.admittedCount),
rejectedCount: saturateSnapshotNumber(this.rejectedCount),
wouldAdmitCount: saturateSnapshotNumber(this.wouldAdmitCount),
wouldQueueCount: saturateSnapshotNumber(this.wouldQueueCount),
wouldRejectCount: saturateSnapshotNumber(this.wouldRejectCount),
shortLatencyEwma: this.adaptation.shortLatencyEwma,
longLatencyEwma: this.adaptation.longLatencyEwma,
utilization: this.adaptation.utilization,
pressure: this.adaptation.pressure,
shutdown: this.shutDown,
};
}
observePressure(pressure: AdmissionPressure): void {
setPressure(this.adaptation, pressure);
if (pressure === "critical") {
// Immediate fast decrease once per window; window close must not re-apply it.
if (!this.adaptation.criticalDecreaseConsumed) {
this.adaptation.currentLimit = Math.max(
this.config.minLimit,
Math.floor(this.adaptation.currentLimit * this.config.adaptation.criticalDecreaseFactor)
);
this.adaptation.criticalDecreaseConsumed = true;
this.dispatch();
this.dispatchVirtual();
}
}
}
/** Deterministic window tick for tests / injected clocks. */
tick(): void {
this.sampleIntegral();
closeAdaptationWindow(this.adaptation, this.config.adaptation, this.clock.now());
// Real queue first, then virtual: raised limits must promote shadow-queued work
// before newer arrivals are classified against the updated budget.
this.dispatch();
this.dispatchVirtual();
}
async acquire(request: AdmissionRequest): Promise<AdmissionAcquireResult> {
if (this.shutDown) {
return this.reject("ADMISSION_SHUTDOWN", "admission controller is shut down");
}
if (request.signal?.aborted) {
return this.reject("ADMISSION_ABORTED", "request aborted before acquire");
}
if (request.pressure) setPressure(this.adaptation, request.pressure);
const cost = this.resolveCost(request);
const mode = this.config.mode;
if (mode === "off") {
return this.admitVirtual(cost);
}
const limit = this.adaptation.currentLimit;
if (mode === "shadow") {
return this.acquireShadow(request, cost, limit);
}
// enforce
if (cost > limit) {
return this.reject("ADMISSION_OVERSIZED", "request cost exceeds max budget");
}
// Once work is queued, every newer request joins the same fair queue even if it
// currently fits. This makes bounded bypass accounting effective and prevents
// direct arrivals from indefinitely jumping an older reserved weighted request.
if (this.queue.size === 0 && this.activeCost + BigInt(cost) <= BigInt(limit)) {
return this.admit(cost);
}
if (!this.queue.canAccept(cost)) {
return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full");
}
return this.enqueue(request, cost);
}
shutdown(): void {
if (this.shutDown) return;
this.shutDown = true;
if (this.windowTimer !== undefined) {
this.clock.clearTimer(this.windowTimer);
this.windowTimer = undefined;
}
const drained = this.queue.drain();
for (const entry of drained) {
this.clearEntryTimer(entry);
this.detachAbort(entry);
entry.payload.reject(
createAdmissionRejectError("ADMISSION_SHUTDOWN", "admission controller shut down")
);
this.rejectedCount += 1;
}
}
private resolveCost(request: AdmissionRequest): number {
if (request.cost !== undefined) {
return normalizeRequestCost(request.cost, this.config.maxRequestCost);
}
if (request.features) {
return estimateAdmissionCost(request.features, this.config.costConfig);
}
return 1;
}
private acquireShadow(request: AdmissionRequest, cost: number, limit: number): AdmissionAdmitted {
let decision: ShadowDecision;
let disposition: VirtualDisposition;
if (cost > limit || !Number.isSafeInteger(cost)) {
decision = "would-reject";
disposition = "rejected";
this.wouldRejectCount += 1;
} else if (this.virtualActiveCost + cost <= limit) {
decision = "would-admit";
disposition = "active";
this.virtualActiveCost = addSaturated(this.virtualActiveCost, cost);
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
this.wouldAdmitCount = addSaturated(this.wouldAdmitCount, 1);
} else if (this.virtualQueue.canAccept(cost)) {
decision = "would-queue";
disposition = "queued";
this.wouldQueueCount += 1;
} else {
decision = "would-reject";
disposition = "rejected";
this.wouldRejectCount += 1;
}
const admitted = this.admit(cost, disposition);
if (disposition === "queued") {
this.virtualQueue.enqueue({
id: admitted.lease.id,
tenantKey: request.tenantKey || "_default",
cost,
enqueuedAtMs: this.clock.now(),
deadlineMs: Number.MAX_SAFE_INTEGER,
payload: { recordId: admitted.lease.id },
});
}
return { ...admitted, shadowDecision: decision };
}
private admitVirtual(cost: number): AdmissionAdmitted {
// Mode off: no accounting.
const id = nextId("lease");
const lease: AdmissionLease = {
id,
cost,
get released() {
return true;
},
release: () => {
/* no-op */
},
};
this.admittedCount += 1;
return { status: "admitted", lease };
}
private admit(cost: number, virtualDisposition: VirtualDisposition = "none"): AdmissionAdmitted {
this.sampleIntegral();
const id = nextId("lease");
const record: ActiveLeaseRecord = {
id,
cost,
released: false,
admittedAtMs: this.clock.now(),
virtualDisposition,
};
this.active.set(id, record);
this.activeCost += BigInt(cost);
this.admittedCount += 1;
const controller = this;
const lease: AdmissionLease = {
id,
cost,
get released() {
return record.released;
},
release(outcome: AdmissionReleaseOutcome = "success", meta?: AdmissionReleaseMeta) {
controller.releaseLease(record, outcome, meta);
},
};
return { status: "admitted", lease };
}
private releaseLease(
record: ActiveLeaseRecord,
outcome: AdmissionReleaseOutcome,
meta?: AdmissionReleaseMeta
): void {
if (record.released) return;
record.released = true;
// Sample while the lease still contributes to activeCost so utilization EWMA sees load.
this.sampleIntegral();
if (this.active.has(record.id)) {
this.active.delete(record.id);
this.activeCost -= BigInt(record.cost);
}
const latency =
meta?.latencyMs !== undefined
? meta.latencyMs
: Math.max(0, this.clock.now() - record.admittedAtMs);
noteLatency(this.adaptation, latency, this.config.adaptation);
noteOutcome(this.adaptation, outcome);
this.adaptation.windowCompleted += 1;
if (meta?.pressure) setPressure(this.adaptation, meta.pressure);
this.releaseVirtual(record);
this.dispatch();
}
private enqueue(request: AdmissionRequest, cost: number): AdmissionAcquireResult {
const id = nextId("q");
const maxWait = normalizeRequestCost(
request.maxWaitMs ?? this.config.defaultMaxWaitMs,
MAX_ADMISSION_WINDOW_MS
);
const now = this.clock.now();
const deadlineMs = Math.min(Number.MAX_SAFE_INTEGER, now + maxWait);
let settle: {
resolve: (v: AdmissionAdmitted) => void;
reject: (e: Error) => void;
};
const promise = new Promise<AdmissionAdmitted>((resolve, reject) => {
settle = { resolve, reject };
});
const entry: QueueEntry<QueuedPayload> = {
id,
tenantKey: request.tenantKey && request.tenantKey.length > 0 ? request.tenantKey : "_default",
cost,
enqueuedAtMs: now,
deadlineMs,
payload: {
resolve: (v) => settle.resolve(v),
reject: (e) => settle.reject(e),
signal: request.signal,
},
};
if (!this.queue.enqueue(entry)) {
return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full");
}
entry.timerId = this.clock.setTimer(
() => {
this.expireEntry(id, "ADMISSION_DEADLINE", "admission wait deadline exceeded");
},
Math.max(0, deadlineMs - now)
);
if (request.signal) {
const onAbort = () => {
this.expireEntry(id, "ADMISSION_ABORTED", "request aborted while queued");
};
entry.payload.onAbort = onAbort;
request.signal.addEventListener("abort", onAbort, { once: true });
}
// Capacity may have freed between check and enqueue in concurrent hosts; try dispatch.
this.dispatch();
return { status: "queued", promise };
}
private expireEntry(id: string, code: AdmissionRejectCode, message: string): void {
const entry = this.queue.removeById(id);
if (!entry) return;
this.clearEntryTimer(entry);
this.detachAbort(entry);
entry.payload.reject(createAdmissionRejectError(code, message));
this.rejectedCount += 1;
// Resume enforce dispatch so a now-fitting successor is not stranded until
// unrelated activity. dispatch() is a no-op after shutdown / non-enforce.
this.dispatch();
}
private failQueued(
entry: QueueEntry<QueuedPayload>,
code: AdmissionRejectCode,
message: string
): void {
this.clearEntryTimer(entry);
this.detachAbort(entry);
entry.payload.reject(createAdmissionRejectError(code, message));
this.rejectedCount += 1;
}
private dispatch(): void {
if (this.shutDown || this.config.mode !== "enforce") return;
while (this.queue.size > 0) {
const limit = this.adaptation.currentLimit;
const available = BigInt(limit) - this.activeCost;
if (available <= 0n) return;
const entry = this.queue.dequeue(Number(available));
if (!entry) return;
this.clearEntryTimer(entry);
this.detachAbort(entry);
if (entry.payload.signal?.aborted) {
entry.payload.reject(
createAdmissionRejectError("ADMISSION_ABORTED", "request aborted while queued")
);
this.rejectedCount += 1;
continue;
}
if (this.clock.now() >= entry.deadlineMs) {
entry.payload.reject(
createAdmissionRejectError("ADMISSION_DEADLINE", "admission wait deadline exceeded")
);
this.rejectedCount += 1;
continue;
}
entry.payload.resolve(this.admit(entry.cost));
}
}
private releaseVirtual(record: ActiveLeaseRecord): void {
if (record.virtualDisposition === "active") {
this.virtualActiveCost -= record.cost;
this.virtualActiveCount -= 1;
} else if (record.virtualDisposition === "queued") {
this.virtualQueue.removeById(record.id);
}
record.virtualDisposition = "none";
this.dispatchVirtual();
}
private dispatchVirtual(): void {
while (this.virtualQueue.size > 0) {
const available = this.adaptation.currentLimit - this.virtualActiveCost;
if (available <= 0) return;
const entry = this.virtualQueue.dequeue(available);
if (!entry) return;
const record = this.active.get(entry.payload.recordId);
if (!record || record.released) continue;
record.virtualDisposition = "active";
this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost);
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
}
}
private rebuildVirtualState(enable: boolean): void {
this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
this.virtualActiveCost = 0;
this.virtualActiveCount = 0;
for (const record of this.active.values()) record.virtualDisposition = "none";
if (!enable) return;
for (const record of this.active.values()) {
// Individually oversized work is virtual-rejected, never virtually queued.
if (record.cost > this.adaptation.currentLimit) {
record.virtualDisposition = "rejected";
continue;
}
if (record.cost <= this.adaptation.currentLimit - this.virtualActiveCost) {
record.virtualDisposition = "active";
this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost);
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
} else if (
this.virtualQueue.enqueue({
id: record.id,
tenantKey: "_existing",
cost: record.cost,
enqueuedAtMs: record.admittedAtMs,
deadlineMs: Number.MAX_SAFE_INTEGER,
payload: { recordId: record.id },
})
) {
record.virtualDisposition = "queued";
} else {
record.virtualDisposition = "rejected";
}
}
}
private reject(code: AdmissionRejectCode, message: string): AdmissionAcquireResult {
this.rejectedCount += 1;
return { status: "rejected", code, message };
}
private clearEntryTimer(entry: QueueEntry<QueuedPayload>): void {
if (entry.timerId !== undefined) {
this.clock.clearTimer(entry.timerId);
entry.timerId = undefined;
}
}
private detachAbort(entry: QueueEntry<QueuedPayload>): void {
if (entry.payload.signal && entry.payload.onAbort) {
entry.payload.signal.removeEventListener("abort", entry.payload.onAbort);
entry.payload.onAbort = undefined;
}
}
private sampleIntegral(): void {
const now = this.clock.now();
const dt = now - this.lastSampleMs;
if (dt > 0) {
// Cap at currentLimit before Number conversion so shadow oversubscription never
// feeds an unsafe rounded activeCost into the utilization integral.
const limit = this.adaptation.currentLimit;
const activeForIntegral = this.activeCost >= BigInt(limit) ? limit : Number(this.activeCost);
sampleActiveIntegral(this.adaptation, activeForIntegral, dt);
this.lastSampleMs = now;
}
}
private armWindowTimer(): void {
if (this.windowTimer !== undefined) {
this.clock.clearTimer(this.windowTimer);
this.windowTimer = undefined;
}
if (this.shutDown || this.config.mode === "off") return;
const tick = () => {
this.tick();
if (!this.shutDown && this.config.mode !== "off") {
this.windowTimer = this.clock.setTimer(tick, this.config.windowMs);
}
};
this.windowTimer = this.clock.setTimer(tick, this.config.windowMs);
}
}

View File

@@ -0,0 +1,107 @@
import {
MAX_ADMISSION_COST_OR_LIMIT,
type AdmissionCostConfig,
type AdmissionCostFeatures,
} from "./types.ts";
export { MAX_ADMISSION_COST_OR_LIMIT };
export const DEFAULT_ADMISSION_COST_CONFIG: AdmissionCostConfig = Object.freeze({
baseCost: 1,
bodyBytesPerUnit: 16_384,
tokensPerUnit: 1_024,
messagesPerUnit: 32,
toolsPerUnit: 8,
fanoutPerUnit: 1,
streamingClassCost: 1,
nonStreamingClassCost: 2,
maxRequestCost: 1_000,
});
function finiteNonNegative(value: unknown): number {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0;
return Math.min(value, Number.MAX_SAFE_INTEGER);
}
function requirePositiveSafeInteger(
name: string,
value: unknown,
max: number = MAX_ADMISSION_COST_OR_LIMIT
): number {
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
throw new RangeError(`${name} must be a positive safe integer`);
}
if (value > max) {
throw new RangeError(`${name} must be <= ${max}`);
}
return value;
}
const COST_CONFIG_KEYS = [
"baseCost",
"bodyBytesPerUnit",
"tokensPerUnit",
"messagesPerUnit",
"toolsPerUnit",
"fanoutPerUnit",
"streamingClassCost",
"nonStreamingClassCost",
"maxRequestCost",
] as const satisfies ReadonlyArray<keyof AdmissionCostConfig>;
/** Merge cost quanta after strictly validating every supplied value. */
export function resolveCostConfig(partial?: Partial<AdmissionCostConfig>): AdmissionCostConfig {
const d = DEFAULT_ADMISSION_COST_CONFIG;
const resolved = {} as AdmissionCostConfig;
for (const key of COST_CONFIG_KEYS) {
resolved[key] = requirePositiveSafeInteger(key, partial?.[key] ?? d[key]);
}
return resolved;
}
function unitsFrom(amount: number, quantum: number): number {
return amount <= 0 ? 0 : Math.ceil(amount / quantum);
}
function addBounded(total: number, contribution: number, maximum: number): number {
if (contribution >= maximum - total) return maximum;
return total + contribution;
}
/** Pure bounded cost estimator from transparent positive safe-integer quanta. */
export function estimateAdmissionCost(
features: AdmissionCostFeatures,
config?: Partial<AdmissionCostConfig>
): number {
const cfg = resolveCostConfig(config);
const body = finiteNonNegative(features?.bodyBytes);
const tokens = finiteNonNegative(features?.estimatedInputTokens);
const messages = finiteNonNegative(features?.messageCount);
const tools = finiteNonNegative(features?.toolCount);
const fanout = Math.max(1, finiteNonNegative(features?.requestedFanout));
const contributions = [
unitsFrom(body, cfg.bodyBytesPerUnit),
unitsFrom(tokens, cfg.tokensPerUnit),
unitsFrom(messages, cfg.messagesPerUnit),
unitsFrom(tools, cfg.toolsPerUnit),
unitsFrom(fanout, cfg.fanoutPerUnit),
features?.streaming !== false ? cfg.streamingClassCost : cfg.nonStreamingClassCost,
];
let total = Math.min(cfg.baseCost, cfg.maxRequestCost);
for (const contribution of contributions) {
total = addBounded(total, contribution, cfg.maxRequestCost);
if (total === cfg.maxRequestCost) break;
}
return total;
}
/** Validate and bound a caller-supplied request cost. */
export function normalizeRequestCost(
cost: unknown,
maxRequestCost: number = DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost
): number {
const max = requirePositiveSafeInteger("maxRequestCost", maxRequestCost);
const value = requirePositiveSafeInteger("request cost", cost);
return Math.min(value, max);
}

View File

@@ -0,0 +1,37 @@
/**
* Pure weighted adaptive admission-control core.
* No route, settings, or environment wiring in this module surface.
*/
export {
DEFAULT_ADMISSION_COST_CONFIG,
estimateAdmissionCost,
normalizeRequestCost,
resolveCostConfig,
} from "./cost.ts";
export { AdaptiveAdmissionController } from "./controller.ts";
export {
MAX_ADMISSION_COST_OR_LIMIT,
MAX_ADMISSION_WINDOW_MS,
createAdmissionRejectError,
type AdaptiveAdmissionConfig,
type AdmissionAcquireResult,
type AdmissionAdmitted,
type AdmissionClock,
type AdmissionCostConfig,
type AdmissionCostFeatures,
type AdmissionLease,
type AdmissionMode,
type AdmissionPressure,
type AdmissionQueued,
type AdmissionRejectCode,
type AdmissionRejectError,
type AdmissionRejected,
type AdmissionReleaseMeta,
type AdmissionReleaseOutcome,
type AdmissionRequest,
type AdmissionSnapshot,
type ShadowDecision,
} from "./types.ts";

View File

@@ -0,0 +1,194 @@
/**
* Bounded multi-tenant fair queue (round-robin across tenant buckets).
* Count + total cost caps; no unbounded arrays of timers beyond one per entry.
*/
/**
* After this many pass-overs while unfittable, reserve capacity for the aged head
* instead of indefinitely admitting smaller work from other tenants.
*/
const MAX_UNFITTABLE_SKIPS = 2;
export interface QueueEntry<T> {
id: string;
tenantKey: string;
cost: number;
enqueuedAtMs: number;
deadlineMs: number;
payload: T;
timerId?: unknown;
/** Times this head was skipped because it did not fit available cost. */
skipCount?: number;
}
export interface FairQueueSnapshot {
count: number;
cost: number;
}
export class FairCostQueue<T> {
private readonly buckets = new Map<string, QueueEntry<T>[]>();
private readonly order: string[] = [];
private cursor = 0;
private count = 0;
private cost = 0;
constructor(
readonly maxCount: number,
readonly maxCost: number
) {}
get size(): number {
return this.count;
}
get totalCost(): number {
return this.cost;
}
snapshot(): FairQueueSnapshot {
return { count: this.count, cost: this.cost };
}
canAccept(entryCost: number): boolean {
if (!Number.isSafeInteger(entryCost) || entryCost <= 0) return false;
if (this.count >= this.maxCount) return false;
if (entryCost > this.maxCost - this.cost) return false;
return true;
}
enqueue(entry: QueueEntry<T>): boolean {
if (!this.canAccept(entry.cost)) return false;
let bucket = this.buckets.get(entry.tenantKey);
if (!bucket) {
bucket = [];
this.buckets.set(entry.tenantKey, bucket);
this.order.push(entry.tenantKey);
}
bucket.push(entry);
this.count += 1;
this.cost += entry.cost;
return true;
}
/**
* Round-robin dequeue, optionally skipping tenant heads that do not fit available cost.
* After MAX_UNFITTABLE_SKIPS actual pass-overs, an unfittable head reserves capacity:
* smaller work is not admitted ahead of it until it fits, is removed, or capacity rises.
*/
dequeue(maxCost = Number.MAX_SAFE_INTEGER): QueueEntry<T> | undefined {
if (this.count === 0) return undefined;
const n = this.order.length;
// Bounded anti-starvation: prefer the oldest aged unfittable head once reserved.
let reserved: { idx: number; entry: QueueEntry<T> } | undefined;
for (let i = 0; i < n; i++) {
const idx = (this.cursor + i) % n;
const tenant = this.order[idx];
const entry = this.buckets.get(tenant)?.[0];
if (!entry) continue;
if ((entry.skipCount ?? 0) >= MAX_UNFITTABLE_SKIPS) {
if (!reserved || entry.enqueuedAtMs < reserved.entry.enqueuedAtMs) {
reserved = { idx, entry };
}
}
}
if (reserved) {
if (reserved.entry.cost > maxCost) return undefined;
return this.takeAt(reserved.idx);
}
const bypassed: QueueEntry<T>[] = [];
for (let i = 0; i < n; i++) {
const idx = (this.cursor + i) % n;
const tenant = this.order[idx];
const bucket = this.buckets.get(tenant);
const entry = bucket?.[0];
if (!entry) continue;
if (entry.cost > maxCost) {
bypassed.push(entry);
continue;
}
// Only an actual smaller admission counts as a pass-over. Merely polling
// with no available capacity must not age a head into reservation.
for (const skipped of bypassed) {
skipped.skipCount = (skipped.skipCount ?? 0) + 1;
}
return this.takeAt(idx);
}
return undefined;
}
private takeAt(idx: number): QueueEntry<T> | undefined {
const tenant = this.order[idx];
const bucket = this.buckets.get(tenant);
const entry = bucket?.[0];
if (!entry) return undefined;
bucket!.shift();
this.count -= 1;
this.cost -= entry.cost;
entry.skipCount = 0;
if (bucket!.length === 0) {
this.buckets.delete(tenant);
this.order.splice(idx, 1);
this.cursor = this.order.length === 0 ? 0 : idx % this.order.length;
} else {
this.cursor = (idx + 1) % this.order.length;
}
return entry;
}
/** Peek next without removing (for oversized-vs-limit checks). */
peek(): QueueEntry<T> | undefined {
if (this.count === 0) return undefined;
const n = this.order.length;
for (let i = 0; i < n; i++) {
const idx = (this.cursor + i) % n;
const tenant = this.order[idx];
const bucket = this.buckets.get(tenant);
if (bucket && bucket.length > 0) return bucket[0];
}
return undefined;
}
removeById(id: string): QueueEntry<T> | undefined {
for (let ti = 0; ti < this.order.length; ti++) {
const tenant = this.order[ti];
const bucket = this.buckets.get(tenant);
if (!bucket) continue;
const idx = bucket.findIndex((e) => e.id === id);
if (idx < 0) continue;
const [entry] = bucket.splice(idx, 1);
this.count -= 1;
this.cost -= entry.cost;
if (bucket.length === 0) {
this.buckets.delete(tenant);
this.order.splice(ti, 1);
if (this.order.length === 0) {
this.cursor = 0;
} else if (ti < this.cursor) {
// Removing a prior bucket shifts the successor into cursor - 1.
this.cursor -= 1;
} else if (this.cursor >= this.order.length) {
// Removed the final bucket at the cursor; wrap to the head.
this.cursor = 0;
}
// ti === cursor: leave cursor so it now points at the logical successor.
// ti > cursor: cursor is unaffected.
}
return entry;
}
return undefined;
}
drain(): QueueEntry<T>[] {
const out: QueueEntry<T>[] = [];
while (true) {
const e = this.dequeue();
if (!e) break;
out.push(e);
}
this.cursor = 0;
return out;
}
}

View File

@@ -0,0 +1,186 @@
/**
* Cheap bounded admission cost features from an already-parsed request body.
* Never re-parses, stringifies, clones, or invokes toJSON.
*/
import { estimateSizeFast } from "../../utils/estimateSize.ts";
import type { AdmissionCostFeatures } from "./types.ts";
export type AdmissionFeatureExtractionContext = {
/** When set, wins over any body/wrapped stream field. */
streaming?: boolean;
};
/**
* Max tools/functions array entries inspected.
* Uninspected tail is charged conservatively so truncation cannot undercharge cost.
*/
export const ADMISSION_TOOL_SCAN_BUDGET = 64;
type FeatureDraft = {
messageCount: number;
toolCount: number;
requestedFanout: number | null;
streaming: boolean | null;
};
function isPlainObject(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function asArray(value: unknown): unknown[] | null {
return Array.isArray(value) ? value : null;
}
function positiveInt(value: unknown): number | null {
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null;
if (!Number.isSafeInteger(value)) {
return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(value));
}
return value;
}
function saturateCount(n: number): number {
if (!Number.isFinite(n) || n <= 0) return 0;
if (!Number.isSafeInteger(n)) {
return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(n));
}
return n;
}
/**
* Count all recognized tool aliases/layers under one shared entry budget.
* If their combined length cannot be inspected completely, saturate before indexed access
* so an unseen alias or wrapped tail cannot undercharge heavier declarations.
*/
function countTools(layers: Array<Record<string, unknown>>): number {
const sources: unknown[][] = [];
const seen = new Set<unknown[]>();
for (const layer of layers) {
for (const value of [layer.tools, layer.functions]) {
const source = asArray(value);
if (!source || seen.has(source)) continue;
seen.add(source);
sources.push(source);
}
}
let entryCount = 0;
for (const source of sources) {
if (source.length > ADMISSION_TOOL_SCAN_BUDGET - entryCount) {
return Number.MAX_SAFE_INTEGER;
}
entryCount += source.length;
}
let total = 0;
for (const source of sources) {
for (let i = 0; i < source.length; i++) {
const entry = source[i];
if (isPlainObject(entry)) {
const declarations = asArray(entry.functionDeclarations);
if (declarations) {
total = Math.min(Number.MAX_SAFE_INTEGER, total + saturateCount(declarations.length));
continue;
}
}
total = Math.min(Number.MAX_SAFE_INTEGER, total + 1);
}
}
return total;
}
function countMessages(layer: Record<string, unknown>): number {
const messages = asArray(layer.messages);
const contents = asArray(layer.contents);
const inputArr = asArray(layer.input);
let count = Math.max(
saturateCount(messages?.length ?? 0),
saturateCount(contents?.length ?? 0),
saturateCount(inputArr?.length ?? 0)
);
// Responses API: non-empty string `input` is one input item.
if (count === 0 && typeof layer.input === "string" && layer.input.length > 0) {
count = 1;
}
return count;
}
function readFanout(layer: Record<string, unknown>): number | null {
const direct =
positiveInt(layer.n) ?? positiveInt(layer.candidateCount) ?? positiveInt(layer.candidate_count);
if (direct != null) return direct;
// Known nested Gemini/Antigravity shape only — no recursive walk.
if (isPlainObject(layer.generationConfig)) {
return (
positiveInt(layer.generationConfig.candidateCount) ??
positiveInt(layer.generationConfig.candidate_count)
);
}
return null;
}
function featureLayers(body: unknown): Array<Record<string, unknown>> {
const top = isPlainObject(body) ? body : null;
const wrapped = top && isPlainObject(top.request) ? top.request : null;
const layers: Array<Record<string, unknown>> = [];
if (top) layers.push(top);
if (wrapped) layers.push(wrapped);
return layers;
}
function absorbLayer(draft: FeatureDraft, layer: Record<string, unknown>): void {
if (draft.messageCount === 0) {
draft.messageCount = countMessages(layer);
}
if (draft.requestedFanout == null) {
draft.requestedFanout = readFanout(layer);
}
if (draft.streaming == null && "stream" in layer) {
draft.streaming = layer.stream === true;
}
}
function resolveStreaming(
draftStreaming: boolean | null,
context?: AdmissionFeatureExtractionContext
): boolean {
if (context && "streaming" in context && context.streaming !== undefined) {
return context.streaming === true;
}
return draftStreaming ?? false;
}
/**
* Inspect top-level fields and one known wrapper (`request`) only.
* Prefer the first non-empty match for each feature family.
*/
export function extractAdmissionCostFeatures(
body: unknown,
context?: AdmissionFeatureExtractionContext
): AdmissionCostFeatures {
const bodyBytes = estimateSizeFast(body);
const layers = featureLayers(body);
const draft: FeatureDraft = {
messageCount: 0,
toolCount: countTools(layers),
requestedFanout: null,
streaming: null,
};
for (const layer of layers) {
absorbLayer(draft, layer);
}
// Conservative token estimate from already-measured body size (no re-walk/stringify).
const estimatedInputTokens =
bodyBytes > 0 ? Math.min(Number.MAX_SAFE_INTEGER, Math.ceil(bodyBytes / 4)) : 0;
return {
bodyBytes,
estimatedInputTokens,
messageCount: draft.messageCount,
toolCount: draft.toolCount,
requestedFanout: draft.requestedFanout ?? 1,
streaming: resolveStreaming(draft.streaming, context),
};
}

View File

@@ -0,0 +1,614 @@
/**
* Process-local adaptive admission runtime facade around the pure controller.
* No HTTP route wiring — suitable for later shared handleChat integration.
*/
import { AdaptiveAdmissionController } from "./controller.ts";
import { validateConfig } from "./config.ts";
import { extractAdmissionCostFeatures } from "./requestFeatures.ts";
import {
type AdaptiveAdmissionConfig,
type AdmissionAcquireResult,
type AdmissionClock,
type AdmissionLease,
type AdmissionMode,
type AdmissionPressure,
type AdmissionRejectCode,
type AdmissionReleaseOutcome,
type AdmissionSnapshot,
type ShadowDecision,
} from "./types.ts";
import { buildErrorBody } from "../../utils/error.ts";
import { CORS_HEADERS } from "../../utils/cors.ts";
import {
checkResourcePressureGuard,
getResourcePressureObservation,
type ResourcePressureGuardResult,
type ResourcePressureObservation,
} from "../../utils/resourcePressure.ts";
import type { PressureReason, PressureSeverity } from "../../utils/resourcePressurePolicy.ts";
export { extractAdmissionCostFeatures } from "./requestFeatures.ts";
export const DEFAULT_ADAPTIVE_ADMISSION_CONFIG: Readonly<AdaptiveAdmissionConfig> = Object.freeze({
mode: "shadow",
minLimit: 8,
initialLimit: 64,
maxLimit: 1000,
maxQueueCount: 128,
maxQueueCost: 2000,
defaultMaxWaitMs: 5_000,
windowMs: 1_000,
});
const RUNTIME_STORE_KEY = Symbol.for("omniroute.adaptiveAdmission.runtime");
type RuntimeStore = {
runtime: AdaptiveAdmissionRuntime | null;
};
type GlobalWithRuntimeStore = typeof globalThis & {
[RUNTIME_STORE_KEY]?: RuntimeStore;
};
function getRuntimeStore(): RuntimeStore {
const globalWithStore = globalThis as GlobalWithRuntimeStore;
let store = globalWithStore[RUNTIME_STORE_KEY];
if (!store) {
store = { runtime: null };
globalWithStore[RUNTIME_STORE_KEY] = store;
}
return store;
}
const ENV_KEYS = {
mode: "ADAPTIVE_ADMISSION_MODE",
minLimit: "ADAPTIVE_ADMISSION_MIN_LIMIT",
initialLimit: "ADAPTIVE_ADMISSION_INITIAL_LIMIT",
maxLimit: "ADAPTIVE_ADMISSION_MAX_LIMIT",
maxQueueCount: "ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT",
maxQueueCost: "ADAPTIVE_ADMISSION_MAX_QUEUE_COST",
defaultMaxWaitMs: "ADAPTIVE_ADMISSION_MAX_WAIT_MS",
windowMs: "ADAPTIVE_ADMISSION_WINDOW_MS",
} as const;
function parsePositiveSafeInt(name: string, raw: string): number {
if (!/^[0-9]+$/.test(raw)) {
throw new RangeError(`${name} must be a positive safe integer`);
}
const value = Number(raw);
if (!Number.isSafeInteger(value) || value <= 0) {
throw new RangeError(`${name} must be a positive safe integer`);
}
return value;
}
/** Strict env → config resolver. Throws clear config errors for direct callers. */
export function resolveAdaptiveAdmissionConfigFromEnv(
env: NodeJS.ProcessEnv | Record<string, string | undefined> = process.env
): AdaptiveAdmissionConfig {
const cfg: AdaptiveAdmissionConfig = { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG };
const modeRaw = env[ENV_KEYS.mode];
if (modeRaw !== undefined && modeRaw !== "") {
if (modeRaw !== "off" && modeRaw !== "shadow" && modeRaw !== "enforce") {
throw new RangeError(`${ENV_KEYS.mode} must be off|shadow|enforce`);
}
cfg.mode = modeRaw;
}
// Numeric env keys only — typed assignment without index-signature cast (TS2352).
type EnvIntField = Exclude<keyof typeof ENV_KEYS, "mode">;
const intFields = [
"minLimit",
"initialLimit",
"maxLimit",
"maxQueueCount",
"maxQueueCost",
"defaultMaxWaitMs",
"windowMs",
] as const satisfies ReadonlyArray<EnvIntField>;
for (const field of intFields) {
const envName = ENV_KEYS[field];
const raw = env[envName];
if (raw === undefined || raw === "") continue;
cfg[field] = parsePositiveSafeInt(envName, raw);
}
// Shared pure validation — accept exact documented maxima, reject core-invalid configs.
validateConfig(cfg);
return cfg;
}
export type AdaptiveAdmissionAcquireInput = {
/** Opaque fairness key; never exposed in snapshots or client errors. */
tenantKey: string;
/** Already-parsed request body — must not be re-read or stringified for cost. */
body: unknown;
signal?: AbortSignal;
maxWaitMs?: number;
/** Authoritative streaming class; wins body stream inference when set. */
streaming?: boolean;
};
export type AdaptiveAdmissionAdmitted = {
status: "admitted";
mode: AdmissionMode;
lease: AdmissionLease;
admittedAtMs: number;
shadowDecision?: ShadowDecision;
};
export type AdaptiveAdmissionRejected = {
status: "rejected";
code: string;
response: Response;
};
export type AdaptiveAdmissionAcquireResult = AdaptiveAdmissionAdmitted | AdaptiveAdmissionRejected;
export type AdaptiveAdmissionPublicSnapshot = AdmissionSnapshot & {
resourceSeverity: PressureSeverity;
resourceReason: PressureReason;
resourceObservedAtMs: number;
pressureGuardRejectCount: number;
};
export type AdaptiveAdmissionLifecycleOptions = {
admittedAtMs: number;
signal?: AbortSignal;
nowMs?: () => number;
};
export type AdaptiveAdmissionRuntimeOptions = {
config?: AdaptiveAdmissionConfig;
env?: NodeJS.ProcessEnv | Record<string, string | undefined>;
clock?: Partial<AdmissionClock>;
checkResourcePressure?: () => ResourcePressureGuardResult | null;
getResourcePressureObservation?: () => ResourcePressureObservation;
/** Test seam: observe pressure values fed into the controller after dedupe. */
onPressureObserved?: (pressure: AdmissionPressure) => void;
warn?: (message: string) => void;
nowMs?: () => number;
};
/** Non-success release outcomes callers must choose explicitly for handler failures. */
export type AdaptiveAdmissionFailureOutcome = Exclude<AdmissionReleaseOutcome, "success">;
export type AdaptiveAdmissionRuntime = {
acquire(input: AdaptiveAdmissionAcquireInput): Promise<AdaptiveAdmissionAcquireResult>;
snapshot(): AdaptiveAdmissionPublicSnapshot;
dispose(): void;
/**
* Release an admitted lease after a handler failure before any HTTP response exists.
* Callers must supply the concrete non-success outcome — never defaults to local_reject.
*/
releaseHandlerFailure(
lease: AdmissionLease,
outcome: AdaptiveAdmissionFailureOutcome,
options?: { admittedAtMs?: number; nowMs?: () => number }
): void;
attachResponseLifecycle(
response: Response,
lease: AdmissionLease,
options: AdaptiveAdmissionLifecycleOptions
): Response;
};
type RejectHttpMapping = {
status: number;
code: string;
message: string;
retryAfter?: string;
};
const REJECT_MAP: Record<AdmissionRejectCode, RejectHttpMapping> = {
ADMISSION_ABORTED: {
status: 499,
code: "admission_aborted",
message: "Request aborted",
},
ADMISSION_OVERSIZED: {
status: 503,
code: "admission_oversized",
message: "Request too large for current capacity",
},
ADMISSION_QUEUE_FULL: {
status: 503,
code: "admission_queue_full",
message: "Service temporarily unavailable",
retryAfter: "1",
},
ADMISSION_DEADLINE: {
status: 503,
code: "admission_deadline",
message: "Service temporarily unavailable",
retryAfter: "1",
},
ADMISSION_SHUTDOWN: {
status: 503,
code: "admission_shutdown",
message: "Service temporarily unavailable",
},
ADMISSION_UNAVAILABLE: {
status: 503,
code: "admission_unavailable",
message: "Service temporarily unavailable",
retryAfter: "1",
},
};
function isAdmissionRejectError(
err: unknown
): err is { code: AdmissionRejectCode; name: string; message: string } {
return (
!!err &&
typeof err === "object" &&
(err as { name?: string }).name === "AdmissionRejectError" &&
typeof (err as { code?: unknown }).code === "string"
);
}
function buildAdmissionRejectResponse(code: AdmissionRejectCode): AdaptiveAdmissionRejected {
const mapping = REJECT_MAP[code] ?? REJECT_MAP.ADMISSION_UNAVAILABLE;
const headers: Record<string, string> = {
"Content-Type": "application/json",
...CORS_HEADERS,
};
if (mapping.retryAfter) headers["Retry-After"] = mapping.retryAfter;
const body = buildErrorBody(mapping.status, mapping.message, undefined, {
type: mapping.status === 499 ? "client_disconnected" : "server_error",
code: mapping.code,
});
return {
status: "rejected",
code: mapping.code,
response: new Response(JSON.stringify(body), {
status: mapping.status,
headers,
}),
};
}
function observationIdentity(state: ResourcePressureObservation["state"]): string {
return `${state.observedAtMs}|${state.severity}|${state.reason}`;
}
function toAdmissionPressure(severity: PressureSeverity): AdmissionPressure {
if (severity === "critical") return "critical";
if (severity === "high") return "high";
return "normal";
}
function isSseResponse(response: Response): boolean {
const contentType = response.headers.get("content-type") ?? "";
return contentType.toLowerCase().includes("text/event-stream");
}
function releaseOnce(
lease: AdmissionLease,
outcome: AdmissionReleaseOutcome,
admittedAtMs: number | undefined,
nowMs: () => number
): void {
if (lease.released) return;
const latencyMs = admittedAtMs === undefined ? undefined : Math.max(0, nowMs() - admittedAtMs);
lease.release(outcome, latencyMs === undefined ? undefined : { latencyMs });
}
/**
* Map HTTP status (+ optional request signal) to admission release outcome.
* Cancellation always wins over status classification.
*/
function classifyHttpOutcome(status: number, signal?: AbortSignal): AdmissionReleaseOutcome {
if (signal?.aborted || status === 499) return "cancelled";
if (status === 408 || status === 504) return "timeout";
if (status >= 500) return "upstream_error";
if (status >= 400) return "local_reject";
// 2xx / 3xx (and rare 1xx) complete successfully from admission's perspective.
return "success";
}
class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime {
private readonly controller: AdaptiveAdmissionController;
private readonly checkResourcePressure: () => ResourcePressureGuardResult | null;
private readonly getResourcePressureObservation: () => ResourcePressureObservation;
private readonly onPressureObserved?: (pressure: AdmissionPressure) => void;
private readonly nowMs: () => number;
private lastObservationKey: string | null = null;
private lastResource: {
severity: PressureSeverity;
reason: PressureReason;
observedAtMs: number;
} = { severity: "normal", reason: "none", observedAtMs: 0 };
private pressureGuardRejectCount = 0;
private disposed = false;
constructor(options: AdaptiveAdmissionRuntimeOptions, config: AdaptiveAdmissionConfig) {
this.controller = new AdaptiveAdmissionController(config, options.clock);
this.checkResourcePressure = options.checkResourcePressure ?? checkResourcePressureGuard;
this.getResourcePressureObservation =
options.getResourcePressureObservation ?? getResourcePressureObservation;
this.onPressureObserved = options.onPressureObserved;
this.nowMs = options.nowMs ?? options.clock?.now ?? (() => Date.now());
}
async acquire(input: AdaptiveAdmissionAcquireInput): Promise<AdaptiveAdmissionAcquireResult> {
if (this.disposed) {
return buildAdmissionRejectResponse("ADMISSION_SHUTDOWN");
}
// Independent safety fuse first — never acquire provider work on critical guard.
// Still feed pressure observations so the controller learns from critical samples.
let guard: ResourcePressureGuardResult | null = null;
try {
guard = this.checkResourcePressure();
} catch {
// Fail open on sampling/check failures.
}
this.feedFreshPressureObservation();
if (guard) {
this.pressureGuardRejectCount += 1;
return {
status: "rejected",
code: "resource_pressure",
response: guard.response,
};
}
const features = extractAdmissionCostFeatures(
input.body,
input.streaming === undefined ? undefined : { streaming: input.streaming }
);
let result: AdmissionAcquireResult;
try {
result = await this.controller.acquire({
tenantKey: input.tenantKey,
features,
signal: input.signal,
maxWaitMs: input.maxWaitMs,
});
} catch (err) {
if (isAdmissionRejectError(err)) {
return buildAdmissionRejectResponse(err.code);
}
return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE");
}
if (result.status === "rejected") {
return buildAdmissionRejectResponse(result.code);
}
if (result.status === "queued") {
try {
const admitted = await result.promise;
return {
status: "admitted",
mode: this.controller.snapshot().mode,
lease: admitted.lease,
admittedAtMs: this.nowMs(),
shadowDecision: admitted.shadowDecision,
};
} catch (err) {
if (isAdmissionRejectError(err)) {
return buildAdmissionRejectResponse(err.code);
}
return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE");
}
}
return {
status: "admitted",
mode: this.controller.snapshot().mode,
lease: result.lease,
admittedAtMs: this.nowMs(),
shadowDecision: result.shadowDecision,
};
}
snapshot(): AdaptiveAdmissionPublicSnapshot {
const core = this.controller.snapshot();
return {
...core,
resourceSeverity: this.lastResource.severity,
resourceReason: this.lastResource.reason,
resourceObservedAtMs: this.lastResource.observedAtMs,
pressureGuardRejectCount: this.pressureGuardRejectCount,
};
}
dispose(): void {
if (this.disposed) return;
this.disposed = true;
this.controller.shutdown();
}
releaseHandlerFailure(
lease: AdmissionLease,
outcome: AdaptiveAdmissionFailureOutcome,
options?: { admittedAtMs?: number; nowMs?: () => number }
): void {
releaseOnce(lease, outcome, options?.admittedAtMs, options?.nowMs ?? this.nowMs);
}
attachResponseLifecycle(
response: Response,
lease: AdmissionLease,
options: AdaptiveAdmissionLifecycleOptions
): Response {
const nowMs = options.nowMs ?? this.nowMs;
const admittedAtMs = options.admittedAtMs;
if (!response.body || !isSseResponse(response)) {
releaseOnce(lease, classifyHttpOutcome(response.status, options.signal), admittedAtMs, nowMs);
return response;
}
const upstream = response.body;
const reader = upstream.getReader();
let settled = false;
let readerCancelled = false;
const settle = (outcome: AdmissionReleaseOutcome): void => {
if (settled) return;
settled = true;
releaseOnce(lease, outcome, admittedAtMs, nowMs);
};
const cancelReader = (reason?: unknown): void => {
if (readerCancelled) return;
readerCancelled = true;
void reader.cancel(reason).catch(() => {
/* ignore cancel races */
});
};
const onAbort = (): void => {
cancelReader(options.signal?.reason);
settle("cancelled");
};
if (options.signal) {
if (options.signal.aborted) {
onAbort();
} else {
options.signal.addEventListener("abort", onAbort, { once: true });
}
}
const detachAbort = (): void => {
options.signal?.removeEventListener("abort", onAbort);
};
const stream = new ReadableStream<Uint8Array>({
async pull(controller) {
if (settled) {
controller.close();
return;
}
try {
const { done, value } = await reader.read();
if (done) {
detachAbort();
settle(classifyHttpOutcome(response.status, options.signal));
controller.close();
return;
}
controller.enqueue(value);
} catch (err) {
detachAbort();
settle(options.signal?.aborted ? "cancelled" : "upstream_error");
controller.error(err);
}
},
cancel(reason) {
detachAbort();
cancelReader(reason);
settle("cancelled");
},
});
return new Response(stream, {
status: response.status,
statusText: response.statusText,
headers: response.headers,
});
}
private feedFreshPressureObservation(): void {
try {
const observation = this.getResourcePressureObservation();
const state = observation.state;
this.lastResource = {
severity: state.severity,
reason: state.reason,
observedAtMs: state.observedAtMs,
};
const key = observationIdentity(state);
if (state.observedAtMs <= 0) return;
if (key === this.lastObservationKey) return;
this.lastObservationKey = key;
const pressure = toAdmissionPressure(state.severity);
this.controller.observePressure(pressure);
this.onPressureObserved?.(pressure);
} catch {
// Fail open.
}
}
}
function createRuntimeFromResolvedConfig(
options: AdaptiveAdmissionRuntimeOptions,
config: AdaptiveAdmissionConfig
): AdaptiveAdmissionRuntime {
return new AdaptiveAdmissionRuntimeImpl(options, config);
}
/**
* Create an injected adaptive-admission runtime for tests or process use.
* Invalid explicit `config` still throws (direct callers want fail-fast).
*/
export function createAdaptiveAdmissionRuntime(
options: AdaptiveAdmissionRuntimeOptions = {}
): AdaptiveAdmissionRuntime {
const config =
options.config ??
(options.env
? resolveAdaptiveAdmissionConfigFromEnv(options.env)
: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG });
return createRuntimeFromResolvedConfig(options, config);
}
function warnInvalidDefaultConfig(warn: ((message: string) => void) | undefined): void {
const message =
"[adaptiveAdmission] invalid environment configuration; using default shadow admission settings";
if (warn) {
warn(message);
return;
}
console.warn(message);
}
function createDefaultProcessRuntime(
options: AdaptiveAdmissionRuntimeOptions = {}
): AdaptiveAdmissionRuntime {
const warn = options.warn;
try {
const config =
options.config ?? resolveAdaptiveAdmissionConfigFromEnv(options.env ?? process.env);
return createRuntimeFromResolvedConfig(options, config);
} catch {
warnInvalidDefaultConfig(warn);
return createRuntimeFromResolvedConfig(options, {
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
});
}
}
/** Call-time process-global runtime (HMR-safe via globalThis symbol store). */
export function getAdaptiveAdmissionRuntime(): AdaptiveAdmissionRuntime {
const store = getRuntimeStore();
if (!store.runtime) {
store.runtime = createDefaultProcessRuntime();
}
return store.runtime;
}
/** Dispose previous controller and replace the process-global runtime. */
export function reloadAdaptiveAdmissionRuntime(
options: AdaptiveAdmissionRuntimeOptions = {}
): AdaptiveAdmissionRuntime {
const store = getRuntimeStore();
store.runtime?.dispose();
store.runtime = createDefaultProcessRuntime(options);
return store.runtime;
}
/** Test isolation: dispose and clear the process-global runtime slot. */
export function resetAdaptiveAdmissionRuntimeForTests(): void {
const store = getRuntimeStore();
store.runtime?.dispose();
store.runtime = null;
}

View File

@@ -0,0 +1,171 @@
/**
* Pure weighted adaptive admission-control types.
* No route/settings wiring — dependency-injected controller seam only.
*/
/**
* Upper bound for adaptation windows and wait deadlines that participate in
* cost×time products (utilization integrals, deadline offsets).
* 24h is far beyond practical control windows while keeping the product domain exact.
*/
export const MAX_ADMISSION_WINDOW_MS = 86_400_000;
/**
* Upper bound for every validated cost, limit, and queue-cost quantum.
* Derived so `MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS` remains a
* safe integer: a full window at the maximum limit integrates to utilization 1.0
* without saturating or rounding Number arithmetic.
*/
export const MAX_ADMISSION_COST_OR_LIMIT = Math.floor(
Number.MAX_SAFE_INTEGER / MAX_ADMISSION_WINDOW_MS
);
export type AdmissionMode = "off" | "shadow" | "enforce";
export type AdmissionPressure = "normal" | "high" | "critical";
/** Local outcome categories. Upstream business errors must not collapse capacity. */
export type AdmissionReleaseOutcome =
"success" | "upstream_error" | "timeout" | "local_reject" | "cancelled";
export type AdmissionRejectCode =
| "ADMISSION_OVERSIZED"
| "ADMISSION_QUEUE_FULL"
| "ADMISSION_DEADLINE"
| "ADMISSION_ABORTED"
| "ADMISSION_SHUTDOWN"
| "ADMISSION_UNAVAILABLE";
export type ShadowDecision = "would-admit" | "would-queue" | "would-reject";
export interface AdmissionCostFeatures {
bodyBytes?: number | null;
estimatedInputTokens?: number | null;
messageCount?: number | null;
toolCount?: number | null;
requestedFanout?: number | null;
streaming?: boolean | null;
}
export interface AdmissionCostConfig {
baseCost: number;
bodyBytesPerUnit: number;
tokensPerUnit: number;
messagesPerUnit: number;
toolsPerUnit: number;
fanoutPerUnit: number;
streamingClassCost: number;
nonStreamingClassCost: number;
maxRequestCost: number;
}
export interface AdaptiveAdmissionConfig {
mode?: AdmissionMode;
minLimit: number;
maxLimit: number;
initialLimit: number;
maxQueueCount: number;
maxQueueCost: number;
defaultMaxWaitMs?: number;
windowMs?: number;
shortLatencyAlpha?: number;
longLatencyAlpha?: number;
increaseStep?: number;
decreaseFactor?: number;
criticalDecreaseFactor?: number;
highUtilizationThreshold?: number;
lowUtilizationThreshold?: number;
latencyGradientThreshold?: number;
maxIncreasePerWindow?: number;
/** Optional cost quanta override used only when callers pass features instead of cost. */
cost?: Partial<AdmissionCostConfig>;
}
export interface AdmissionRequest {
/** Positive integer cost units. If omitted, `features` + cost config are used. */
cost?: number;
features?: AdmissionCostFeatures;
/** Opaque fairness key; never exposed in snapshots. */
tenantKey?: string;
maxWaitMs?: number;
signal?: AbortSignal;
pressure?: AdmissionPressure;
}
export interface AdmissionReleaseMeta {
latencyMs?: number;
pressure?: AdmissionPressure;
}
export interface AdmissionLease {
readonly id: string;
readonly cost: number;
readonly released: boolean;
release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta): void;
}
export interface AdmissionAdmitted {
status: "admitted";
lease: AdmissionLease;
shadowDecision?: ShadowDecision;
}
export interface AdmissionQueued {
status: "queued";
promise: Promise<AdmissionAdmitted>;
}
export interface AdmissionRejected {
status: "rejected";
code: AdmissionRejectCode;
message: string;
shadowDecision?: ShadowDecision;
}
export type AdmissionAcquireResult = AdmissionAdmitted | AdmissionQueued | AdmissionRejected;
export interface AdmissionSnapshot {
mode: AdmissionMode;
currentLimit: number;
minLimit: number;
maxLimit: number;
activeCost: number;
activeCount: number;
queuedCost: number;
queuedCount: number;
virtualActiveCost: number;
virtualActiveCount: number;
virtualQueuedCost: number;
virtualQueuedCount: number;
admittedCount: number;
rejectedCount: number;
wouldAdmitCount: number;
wouldQueueCount: number;
wouldRejectCount: number;
shortLatencyEwma: number;
longLatencyEwma: number;
utilization: number;
pressure: AdmissionPressure;
shutdown: boolean;
}
export interface AdmissionClock {
now: () => number;
setTimer: (fn: () => void, delayMs: number) => unknown;
clearTimer: (id: unknown) => void;
}
export interface AdmissionRejectError extends Error {
code: AdmissionRejectCode;
name: "AdmissionRejectError";
}
export function createAdmissionRejectError(
code: AdmissionRejectCode,
message: string
): AdmissionRejectError {
const err = new Error(message) as AdmissionRejectError;
err.name = "AdmissionRejectError";
err.code = code;
return err;
}

View File

@@ -1,32 +1,109 @@
/**
* Fast object-tree size estimator — walks without JSON.stringify.
* Safe for circular references (uses WeakSet).
* Early-exits at 256KB to avoid wasting CPU on huge payloads.
* Fast object-tree size estimator — walks without JSON.stringify / toJSON / clone.
* Safe for circular references (WeakSet). Iterative frames only (no recursive call stack).
*
* Budgets:
* - ESTIMATE_SIZE_BYTE_LIMIT (256 KiB): early-exit once counted bytes exceed the limit
* - ESTIMATE_SIZE_NODE_BUDGET: max value visits (containers + primitives/elements)
*
* Arrays are walked by index frame (never pre-push/copy every element reference).
* Plain objects yield own enumerable values incrementally (no Object.keys materialization).
* Node-budget exhaustion returns a value strictly above 256 KiB so callers fail closed.
*/
export function estimateSizeFast(value: unknown): number {
let bytes = 0;
const stack: unknown[] = [value];
const seen = new WeakSet();
while (stack.length > 0) {
const v = stack.pop();
if (v === null || v === undefined) continue;
if (typeof v === "string") {
bytes += v.length;
if (bytes > 262144) return bytes;
} else if (typeof v === "number") bytes += 8;
else if (typeof v === "boolean") bytes += 4;
else if (typeof v === "object") {
if (seen.has(v as object)) continue;
seen.add(v as object);
if (Array.isArray(v)) {
for (let i = 0; i < v.length; i++) stack.push(v[i]);
} else {
for (const key in v) {
if (Object.prototype.hasOwnProperty.call(v, key)) stack.push((v as Record<string, unknown>)[key]);
}
/** Byte early-exit threshold (256 KiB). */
export const ESTIMATE_SIZE_BYTE_LIMIT = 262_144;
/**
* Max value/element visits before fail-closed.
* Conservative cap keeps auxiliary stack/WeakSet growth bounded under adversarial input.
*/
export const ESTIMATE_SIZE_NODE_BUDGET = 16_384;
type Frame =
| { t: "v"; v: unknown }
| { t: "a"; a: unknown[]; i: number }
| { t: "o"; o: object; it: Iterator<string> };
function ownEnumerableKeyIterator(obj: object): Iterator<string> {
return (function* ownEnumerableKeys() {
for (const key in obj) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
yield key;
}
}
})();
}
/** @returns next byte total, or a value > limit when the limit is exceeded. */
function addPrimitiveBytes(bytes: number, v: string | number | boolean): number {
if (typeof v === "string") return bytes + v.length;
if (typeof v === "number") return bytes + 8;
return bytes + 4;
}
function enqueueContainer(stack: Frame[], obj: object, seen: WeakSet<object>): void {
if (seen.has(obj)) return;
seen.add(obj);
if (Array.isArray(obj)) {
if (obj.length > 0) stack.push({ t: "a", a: obj, i: 0 });
return;
}
stack.push({ t: "o", o: obj, it: ownEnumerableKeyIterator(obj) });
}
type ValueFrame = Extract<Frame, { t: "v" }>;
function isValueFrame(frame: Frame): frame is ValueFrame {
return frame.t === "v";
}
/** Expand a container frame into the next child value. */
function expandContainerFrame(stack: Frame[], frame: Exclude<Frame, ValueFrame>): void {
if (frame.t === "a") {
if (frame.i >= frame.a.length) return;
if (frame.i + 1 < frame.a.length) {
stack.push({ t: "a", a: frame.a, i: frame.i + 1 });
}
stack.push({ t: "v", v: frame.a[frame.i] });
return;
}
const next = frame.it.next();
if (next.done) return;
stack.push(frame);
stack.push({ t: "v", v: (frame.o as Record<string, unknown>)[next.value] });
}
export function estimateSizeFast(value: unknown): number {
let bytes = 0;
let visitsLeft = ESTIMATE_SIZE_NODE_BUDGET;
const seen = new WeakSet<object>();
const stack: Frame[] = [{ t: "v", v: value }];
while (stack.length > 0) {
if (visitsLeft <= 0) return ESTIMATE_SIZE_BYTE_LIMIT + 1;
const frame = stack.pop()!;
if (!isValueFrame(frame)) {
expandContainerFrame(stack, frame);
continue;
}
visitsLeft -= 1;
const v = frame.v;
if (v === null || v === undefined) continue;
const ty = typeof v;
if (ty === "string" || ty === "number" || ty === "boolean") {
bytes = addPrimitiveBytes(bytes, v as string | number | boolean);
if (bytes > ESTIMATE_SIZE_BYTE_LIMIT) return bytes;
continue;
}
if (ty === "object") {
enqueueContainer(stack, v as object, seen);
}
}
return bytes;
}

View File

@@ -0,0 +1,249 @@
import { checkHeapPressureGuard, HEAP_PRESSURE_THRESHOLD_MB } from "./heapPressure.ts";
import { buildErrorBody } from "./error.ts";
import {
createResourcePressureTracker,
resolveResourcePressureThresholds,
type PressureReason,
type ResourcePressureState,
type ResourcePressureThresholds,
type ResourceSignals,
} from "./resourcePressurePolicy.ts";
import {
sampleResourceSignals,
type SampleResourceSignalsDeps,
} from "./resourcePressureSampler.ts";
const MB = 1024 * 1024;
const RETRY_AFTER_SECONDS = "5";
const PRESSURE_MESSAGE = "Service temporarily unavailable due to resource pressure. Retry shortly.";
export type ResourcePressureGuardResult = {
success: false;
status: 503;
error: string;
response: Response;
};
export type ResourcePressureObservation = {
signals: ResourceSignals | null;
state: ResourcePressureState;
};
export type ResourcePressureRuntimeOptions = {
thresholds?: Partial<ResourcePressureThresholds>;
heapThresholdMb?: number | null;
immediateHeapUsedMb?: () => number;
sample?: () => Promise<ResourceSignals>;
nowMs?: () => number;
schedule?: (refresh: () => void) => void;
staleAfterMs?: number;
maxStaleMs?: number;
retryAfterMs?: number;
samplerDeps?: SampleResourceSignalsDeps;
};
export type ResourcePressureRuntime = {
check: () => ResourcePressureGuardResult | null;
getObservation: () => ResourcePressureObservation;
whenRefreshSettled: () => Promise<void>;
dispose: () => void;
};
function emptyState(): ResourcePressureState {
return {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
};
}
function requireDuration(name: string, value: number): number {
if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0 || value > 3_600_000) {
throw new RangeError(`${name} must be an integer between 0 and 3600000`);
}
return value;
}
function buildCriticalGuard(reason: PressureReason): ResourcePressureGuardResult {
console.warn(
`[resourcePressure] critical pressure guard tripped (reason=${reason}); returning 503`
);
return {
success: false,
status: 503,
error: PRESSURE_MESSAGE,
response: new Response(
JSON.stringify(
buildErrorBody(503, PRESSURE_MESSAGE, undefined, {
type: "server_error",
code: "resource_pressure",
})
),
{
status: 503,
headers: { "Content-Type": "application/json", "Retry-After": RETRY_AFTER_SECONDS },
}
),
};
}
function immediateHeapGuard(
heapUsedMb: number,
thresholdMb: number | null
): ResourcePressureGuardResult | null {
if (thresholdMb == null) return null;
const guard = checkHeapPressureGuard(heapUsedMb, thresholdMb);
if (!guard) return null;
return buildCriticalGuard("v8_heap_absolute");
}
export function createResourcePressureRuntime(
options: ResourcePressureRuntimeOptions = {}
): ResourcePressureRuntime {
const heapThresholdMb =
options.heapThresholdMb === undefined ? HEAP_PRESSURE_THRESHOLD_MB : options.heapThresholdMb;
if (heapThresholdMb !== null && (!Number.isFinite(heapThresholdMb) || heapThresholdMb <= 0)) {
throw new RangeError("heapThresholdMb must be positive and finite or null");
}
const thresholds = resolveResourcePressureThresholds({
...options.thresholds,
heapAbsoluteThresholdMb:
options.thresholds?.heapAbsoluteThresholdMb === undefined
? null
: options.thresholds.heapAbsoluteThresholdMb,
});
const staleAfterMs = requireDuration("staleAfterMs", options.staleAfterMs ?? 1_000);
const maxStaleMs = requireDuration("maxStaleMs", options.maxStaleMs ?? 30_000);
const retryAfterMs = requireDuration("retryAfterMs", options.retryAfterMs ?? 1_000);
if (maxStaleMs < staleAfterMs) {
throw new RangeError("maxStaleMs must be greater than or equal to staleAfterMs");
}
const nowMs = options.nowMs ?? Date.now;
const immediateHeapUsedMb =
options.immediateHeapUsedMb ?? (() => process.memoryUsage().heapUsed / MB);
const sample = options.sample ?? (() => sampleResourceSignals(options.samplerDeps));
const schedule =
options.schedule ??
((refresh) => {
const handle = setImmediate(refresh);
handle.unref();
});
const tracker = createResourcePressureTracker(thresholds);
let lastSignals: ResourceSignals | null = null;
let state = emptyState();
let lastRefreshAtMs = Number.NEGATIVE_INFINITY;
let nextRefreshAtMs = Number.NEGATIVE_INFINITY;
let scheduled = false;
let inFlight: Promise<void> | null = null;
let disposed = false;
const refresh = (): void => {
if (disposed || inFlight) return;
scheduled = false;
inFlight = Promise.resolve()
.then(sample)
.then((signals) => {
if (disposed) return;
const settledAtMs = nowMs();
lastSignals = signals;
state = tracker.observe(signals);
lastRefreshAtMs = settledAtMs;
nextRefreshAtMs = settledAtMs + staleAfterMs;
})
.catch(() => {
if (!disposed) nextRefreshAtMs = nowMs() + retryAfterMs;
})
.finally(() => {
inFlight = null;
});
};
const scheduleRefresh = (): void => {
if (disposed || scheduled || inFlight) return;
scheduled = true;
schedule(refresh);
};
return {
check() {
let heapUsedMb = 0;
try {
heapUsedMb = immediateHeapUsedMb();
} catch {
heapUsedMb = 0;
}
const immediate = immediateHeapGuard(heapUsedMb, heapThresholdMb);
const now = nowMs();
if (now >= nextRefreshAtMs) scheduleRefresh();
if (immediate) {
state = {
severity: "critical",
reason: "v8_heap_absolute",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: now,
observedAtMs: now,
};
return immediate;
}
const cacheAge = lastSignals ? Math.max(0, now - lastRefreshAtMs) : Number.POSITIVE_INFINITY;
return cacheAge <= maxStaleMs && state.severity === "critical"
? buildCriticalGuard(state.reason)
: null;
},
getObservation: () => ({ signals: lastSignals, state }),
whenRefreshSettled: async () => {
if (scheduled) await new Promise<void>((resolve) => setImmediate(resolve));
if (inFlight) await inFlight;
},
dispose() {
disposed = true;
scheduled = false;
},
};
}
let defaultRuntime = createResourcePressureRuntime();
export function checkResourcePressureGuard(): ResourcePressureGuardResult | null {
return defaultRuntime.check();
}
export function getResourcePressureObservation(): ResourcePressureObservation {
return defaultRuntime.getObservation();
}
/** Replaces and disposes the process singleton when configuration is reloaded. */
export function reloadResourcePressureRuntime(
options: ResourcePressureRuntimeOptions = {}
): ResourcePressureRuntime {
defaultRuntime.dispose();
defaultRuntime = createResourcePressureRuntime(options);
return defaultRuntime;
}
export type {
PressureReason,
PressureSeverity,
ResourceMetricBytes,
ResourcePressureState,
ResourcePressureThresholds,
ResourcePressureTracker,
ResourceSignals,
} from "./resourcePressurePolicy.ts";
export {
classifyAdaptiveResourcePressure as classifyResourcePressure,
createResourcePressureTracker,
resolveResourcePressureThresholds,
} from "./resourcePressurePolicy.ts";
export {
sampleResourceSignals,
sanitizeMemoryBytes,
type ResourcePressureFs,
type SampleResourceSignalsDeps,
} from "./resourcePressureSampler.ts";

View File

@@ -0,0 +1,344 @@
const MB = 1024 * 1024;
const MAX_SUSTAINED_SAMPLES = 10_000;
export type PressureSeverity = "normal" | "high" | "critical";
export type PressureReason =
| "none"
| "v8_heap_ratio"
| "v8_heap_absolute"
| "cgroup_ratio"
| "cgroup_high"
| "psi_some"
| "psi_full"
| "oom_event";
export type ResourceMetricBytes = number | null;
export type ResourceSignals = {
observedAtMs: number;
v8: { heapUsedBytes: number; heapLimitBytes: number };
process: {
rssBytes: number;
externalBytes: number;
arrayBuffersBytes: number;
availableBytes: ResourceMetricBytes;
constrainedBytes: ResourceMetricBytes;
};
cgroup: {
currentBytes: ResourceMetricBytes;
maxBytes: ResourceMetricBytes;
highBytes: ResourceMetricBytes;
events: {
low: ResourceMetricBytes;
high: ResourceMetricBytes;
max: ResourceMetricBytes;
oom: ResourceMetricBytes;
oom_kill: ResourceMetricBytes;
} | null;
};
psi: {
someAvg10: number | null;
someAvg60: number | null;
someAvg300: number | null;
fullAvg10: number | null;
fullAvg60: number | null;
fullAvg300: number | null;
} | null;
};
export type ResourcePressureState = {
severity: PressureSeverity;
reason: PressureReason;
elevatedStreak: number;
recoveryStreak: number;
lastTransitionAtMs: number;
observedAtMs: number;
};
export type ResourcePressureThresholds = {
highRatio: number;
criticalRatio: number;
recoveryRatio: number;
highPsiAvg10: number;
criticalPsiAvg10: number;
recoveryPsiAvg10: number;
sustainedSamplesHigh: number;
sustainedSamplesCritical: number;
sustainedSamplesRecovery: number;
heapAbsoluteThresholdMb: number | null;
};
export const DEFAULT_RESOURCE_PRESSURE_THRESHOLDS: ResourcePressureThresholds = {
highRatio: 0.85,
criticalRatio: 0.92,
recoveryRatio: 0.75,
highPsiAvg10: 20,
criticalPsiAvg10: 40,
recoveryPsiAvg10: 10,
sustainedSamplesHigh: 2,
sustainedSamplesCritical: 2,
sustainedSamplesRecovery: 3,
heapAbsoluteThresholdMb: null,
};
type RawLevel = { severity: PressureSeverity; reason: PressureReason };
type OomCounters = { oom: number | null; oomKill: number | null };
function requireFiniteRange(name: string, value: number, minimum: number, maximum: number): void {
if (!Number.isFinite(value) || value < minimum || value > maximum) {
throw new RangeError(`${name} must be finite and between ${minimum} and ${maximum}`);
}
}
function requirePositiveInteger(name: string, value: number): void {
if (!Number.isInteger(value) || value < 1 || value > MAX_SUSTAINED_SAMPLES) {
throw new RangeError(`${name} must be an integer between 1 and ${MAX_SUSTAINED_SAMPLES}`);
}
}
export function resolveResourcePressureThresholds(
partial: Partial<ResourcePressureThresholds> = {}
): ResourcePressureThresholds {
const resolved = { ...DEFAULT_RESOURCE_PRESSURE_THRESHOLDS, ...partial };
requireFiniteRange("recoveryRatio", resolved.recoveryRatio, 0, 1);
requireFiniteRange("highRatio", resolved.highRatio, 0, 1);
requireFiniteRange("criticalRatio", resolved.criticalRatio, 0, 1);
if (!(
resolved.recoveryRatio < resolved.highRatio && resolved.highRatio < resolved.criticalRatio
)) {
throw new RangeError("ratio thresholds must satisfy recovery < high < critical");
}
requireFiniteRange("recoveryPsiAvg10", resolved.recoveryPsiAvg10, 0, 100);
requireFiniteRange("highPsiAvg10", resolved.highPsiAvg10, 0, 100);
requireFiniteRange("criticalPsiAvg10", resolved.criticalPsiAvg10, 0, 100);
if (!(
resolved.recoveryPsiAvg10 < resolved.highPsiAvg10 &&
resolved.highPsiAvg10 < resolved.criticalPsiAvg10
)) {
throw new RangeError("PSI thresholds must satisfy recovery < high < critical");
}
requirePositiveInteger("sustainedSamplesHigh", resolved.sustainedSamplesHigh);
requirePositiveInteger("sustainedSamplesCritical", resolved.sustainedSamplesCritical);
requirePositiveInteger("sustainedSamplesRecovery", resolved.sustainedSamplesRecovery);
if (
resolved.heapAbsoluteThresholdMb !== null &&
(!Number.isFinite(resolved.heapAbsoluteThresholdMb) || resolved.heapAbsoluteThresholdMb <= 0)
) {
throw new RangeError("heapAbsoluteThresholdMb must be positive and finite or null");
}
return resolved;
}
function severityRank(severity: PressureSeverity): number {
return severity === "critical" ? 2 : severity === "high" ? 1 : 0;
}
function maxLevel(current: RawLevel, candidate: RawLevel | null): RawLevel {
if (!candidate || severityRank(candidate.severity) <= severityRank(current.severity)) {
return current;
}
return candidate;
}
function ratioLevel(
used: number | null,
limit: number | null,
thresholds: ResourcePressureThresholds,
reason: PressureReason
): RawLevel | null {
if (used == null || limit == null || used < 0 || limit <= 0) return null;
const ratio = used / limit;
if (ratio >= thresholds.criticalRatio) return { severity: "critical", reason };
if (ratio >= thresholds.highRatio) return { severity: "high", reason };
return null;
}
function psiLevel(
value: number | null,
thresholds: ResourcePressureThresholds,
reason: Extract<PressureReason, "psi_some" | "psi_full">
): RawLevel | null {
if (value == null || !Number.isFinite(value)) return null;
if (value >= thresholds.criticalPsiAvg10) return { severity: "critical", reason };
if (value >= thresholds.highPsiAvg10) return { severity: "high", reason };
return null;
}
export function classifyAdaptiveResourcePressure(
signals: ResourceSignals,
thresholds: ResourcePressureThresholds
): RawLevel {
let best: RawLevel = { severity: "normal", reason: "none" };
best = maxLevel(
best,
ratioLevel(signals.v8.heapUsedBytes, signals.v8.heapLimitBytes, thresholds, "v8_heap_ratio")
);
best = maxLevel(
best,
ratioLevel(signals.cgroup.currentBytes, signals.cgroup.maxBytes, thresholds, "cgroup_ratio")
);
best = maxLevel(
best,
ratioLevel(signals.cgroup.currentBytes, signals.cgroup.highBytes, thresholds, "cgroup_high")
);
best = maxLevel(best, psiLevel(signals.psi?.someAvg10 ?? null, thresholds, "psi_some"));
return maxLevel(best, psiLevel(signals.psi?.fullAvg10 ?? null, thresholds, "psi_full"));
}
function isRecovered(signals: ResourceSignals, thresholds: ResourcePressureThresholds): boolean {
const ratios: Array<readonly [number | null, number | null]> = [
[signals.v8.heapUsedBytes, signals.v8.heapLimitBytes],
[signals.cgroup.currentBytes, signals.cgroup.maxBytes],
[signals.cgroup.currentBytes, signals.cgroup.highBytes],
];
if (
ratios.some(
([used, limit]) =>
used != null && limit != null && limit > 0 && used / limit > thresholds.recoveryRatio
)
) {
return false;
}
if (
thresholds.heapAbsoluteThresholdMb != null &&
signals.v8.heapUsedBytes / MB > thresholds.heapAbsoluteThresholdMb * thresholds.recoveryRatio
) {
return false;
}
return ![signals.psi?.someAvg10, signals.psi?.fullAvg10].some(
(value) => value != null && value > thresholds.recoveryPsiAvg10
);
}
function hasCounterIncrease(previous: OomCounters, current: OomCounters): boolean {
return (
(previous.oom != null && current.oom != null && current.oom > previous.oom) ||
(previous.oomKill != null && current.oomKill != null && current.oomKill > previous.oomKill)
);
}
function countersReset(previous: OomCounters, current: OomCounters): boolean {
return (
(previous.oom != null && current.oom != null && current.oom < previous.oom) ||
(previous.oomKill != null && current.oomKill != null && current.oomKill < previous.oomKill)
);
}
function initialState(): ResourcePressureState {
return {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
};
}
export type ResourcePressureTracker = {
observe: (signals: ResourceSignals) => ResourcePressureState;
getState: () => ResourcePressureState;
};
export function createResourcePressureTracker(
partialThresholds: Partial<ResourcePressureThresholds> = {}
): ResourcePressureTracker {
const thresholds = resolveResourcePressureThresholds(partialThresholds);
let state = initialState();
let pending: RawLevel | null = null;
let previousOom: OomCounters | null = null;
return {
observe(signals) {
const events = signals.cgroup.events;
const currentOom = events ? { oom: events.oom, oomKill: events.oom_kill } : null;
let oomEvent = false;
if (currentOom) {
if (previousOom && !countersReset(previousOom, currentOom)) {
oomEvent = hasCounterIncrease(previousOom, currentOom);
}
previousOom = currentOom;
} else {
previousOom = null;
}
const raw = oomEvent
? ({ severity: "critical", reason: "oom_event" } as const)
: classifyAdaptiveResourcePressure(signals, thresholds);
let { severity, reason, elevatedStreak, recoveryStreak } = state;
if (oomEvent) {
severity = "critical";
reason = "oom_event";
elevatedStreak = 0;
recoveryStreak = 0;
pending = null;
} else if (severity === "normal") {
recoveryStreak = 0;
if (raw.severity === "normal") {
pending = null;
elevatedStreak = 0;
reason = "none";
} else {
const samePending = pending?.severity === raw.severity && pending.reason === raw.reason;
pending = raw;
elevatedStreak = samePending ? elevatedStreak + 1 : 1;
const needed =
raw.severity === "critical"
? thresholds.sustainedSamplesCritical
: thresholds.sustainedSamplesHigh;
if (elevatedStreak >= needed) {
severity = raw.severity;
reason = raw.reason;
elevatedStreak = 0;
pending = null;
}
}
} else if (severity === "high" && raw.severity === "critical") {
recoveryStreak = 0;
const samePending = pending?.severity === "critical" && pending.reason === raw.reason;
pending = raw;
elevatedStreak = samePending ? elevatedStreak + 1 : 1;
if (elevatedStreak >= thresholds.sustainedSamplesCritical) {
severity = "critical";
reason = raw.reason;
elevatedStreak = 0;
pending = null;
}
} else if (raw.severity === severity) {
reason = raw.reason;
pending = null;
elevatedStreak = 0;
recoveryStreak = 0;
} else if (isRecovered(signals, thresholds)) {
pending = null;
elevatedStreak = 0;
recoveryStreak += 1;
if (recoveryStreak >= thresholds.sustainedSamplesRecovery) {
severity = "normal";
reason = "none";
recoveryStreak = 0;
}
} else {
pending = null;
elevatedStreak = 0;
recoveryStreak = 0;
}
const transitioned = severity !== state.severity || reason !== state.reason;
state = {
severity,
reason,
elevatedStreak,
recoveryStreak,
lastTransitionAtMs: transitioned ? signals.observedAtMs : state.lastTransitionAtMs,
observedAtMs: signals.observedAtMs,
};
return state;
},
getState: () => state,
};
}

View File

@@ -0,0 +1,257 @@
import fs from "node:fs/promises";
import path from "node:path";
import v8 from "node:v8";
import type { ResourceSignals } from "./resourcePressurePolicy.ts";
const DEFAULT_CGROUP_ROOT = "/sys/fs/cgroup";
export type ResourcePressureFs = {
readText: (filePath: string) => Promise<string | null>;
};
export type SampleResourceSignalsDeps = {
nowMs?: () => number;
memoryUsage?: () => NodeJS.MemoryUsage;
heapStatistics?: () => { heap_size_limit: number; used_heap_size?: number };
availableMemory?: () => number | undefined;
constrainedMemory?: () => number | undefined;
fs?: ResourcePressureFs;
};
type Cgroup2Mount = { root: string; mountpoint: string };
async function defaultReadText(filePath: string): Promise<string | null> {
try {
return await fs.readFile(filePath, "utf8");
} catch {
return null;
}
}
export function sanitizeMemoryBytes(value: unknown): number | null {
if (typeof value === "string") {
const trimmed = value.trim();
if (!trimmed || trimmed === "max" || !/^\d+$/.test(trimmed) || trimmed.length > 15) {
return null;
}
value = Number(trimmed);
}
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null;
if (value >= Number.MAX_SAFE_INTEGER) return null;
return Math.floor(value);
}
function safeNumber(call: (() => number | undefined) | undefined): number | null {
try {
return call ? sanitizeMemoryBytes(call()) : null;
} catch {
return null;
}
}
export function decodeMountInfoPath(value: string): string | null {
if (value.includes("\0")) return null;
try {
return value.replace(/\\([0-7]{3})/g, (_match, octal: string) =>
String.fromCharCode(Number.parseInt(octal, 8))
);
} catch {
return null;
}
}
export function parseCgroupV2Path(contents: string | null): string | null {
if (!contents) return null;
for (const rawLine of contents.split("\n")) {
const line = rawLine.trim();
if (!line.startsWith("0::")) continue;
const relativePath = line.slice(3);
if (!relativePath.startsWith("/") || relativePath.includes("\0")) return null;
return relativePath;
}
return null;
}
export function parseCgroup2Mount(contents: string | null): Cgroup2Mount | null {
if (!contents) return null;
for (const rawLine of contents.split("\n")) {
const separator = rawLine.indexOf(" - ");
if (separator < 0) continue;
const left = rawLine.slice(0, separator).trim().split(/\s+/);
const right = rawLine
.slice(separator + 3)
.trim()
.split(/\s+/);
if (right[0] !== "cgroup2" || left.length < 5) continue;
const root = decodeMountInfoPath(left[3]);
const mountpoint = decodeMountInfoPath(left[4]);
if (!root?.startsWith("/") || !mountpoint?.startsWith("/")) return null;
return { root, mountpoint };
}
return null;
}
function isContained(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
}
function hasTraversalSegment(value: string): boolean {
let decoded = value;
try {
decoded = decodeURIComponent(value);
} catch {
return true;
}
return decoded.split("/").some((segment) => segment === ".." || segment === ".");
}
function resolveFromMount(cgroupPath: string, mount: Cgroup2Mount): string | null {
if (
cgroupPath.includes("\0") ||
mount.root.includes("\0") ||
mount.mountpoint.includes("\0") ||
hasTraversalSegment(cgroupPath)
) {
return null;
}
const resolvedRoot = path.resolve(mount.root);
const resolvedCgroup = path.resolve(cgroupPath);
if (!isContained(resolvedRoot, resolvedCgroup)) return null;
const suffix = path.relative(resolvedRoot, resolvedCgroup);
const resolvedMountpoint = path.resolve(mount.mountpoint);
const candidate = path.resolve(resolvedMountpoint, suffix);
return isContained(resolvedMountpoint, candidate) ? candidate : null;
}
export async function resolveCgroupDirectory(
readText: ResourcePressureFs["readText"],
options: { allowDefaultFallback?: boolean } = {}
): Promise<string | null> {
try {
const [cgroupContents, mountInfo] = await Promise.all([
readText("/proc/self/cgroup"),
readText("/proc/self/mountinfo"),
]);
const cgroupPath = parseCgroupV2Path(cgroupContents);
const mount = parseCgroup2Mount(mountInfo);
if (cgroupPath && mount) {
const candidate = resolveFromMount(cgroupPath, mount);
if (candidate && (await readText(path.join(candidate, "memory.current"))) != null) {
return candidate;
}
if (!candidate) return null;
}
if (options.allowDefaultFallback === false) return null;
return (await readText(path.join(DEFAULT_CGROUP_ROOT, "memory.current"))) != null
? DEFAULT_CGROUP_ROOT
: null;
} catch {
return null;
}
}
function parseEventCounter(value: string): number | null {
const parsed = Number(value.trim());
return Number.isFinite(parsed) && parsed >= 0 && parsed < Number.MAX_SAFE_INTEGER
? Math.floor(parsed)
: null;
}
function parseMemoryEvents(text: string | null): ResourceSignals["cgroup"]["events"] {
if (!text) return null;
const values = { low: null, high: null, max: null, oom: null, oom_kill: null } as Record<
"low" | "high" | "max" | "oom" | "oom_kill",
number | null
>;
let matched = false;
for (const line of text.split("\n")) {
const [key, rawValue] = line.trim().split(/\s+/, 2);
if (!(key in values) || rawValue == null) continue;
values[key as keyof typeof values] = parseEventCounter(rawValue);
matched = true;
}
return matched ? values : null;
}
function parsePsiNumber(line: string, name: string): number | null {
const match = new RegExp(`(?:^|\\s)${name}=([0-9.]+)`).exec(line);
const parsed = match ? Number(match[1]) : Number.NaN;
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
}
function parsePsi(text: string | null): ResourceSignals["psi"] {
if (!text) return null;
const result: NonNullable<ResourceSignals["psi"]> = {
someAvg10: null,
someAvg60: null,
someAvg300: null,
fullAvg10: null,
fullAvg60: null,
fullAvg300: null,
};
let matched = false;
for (const line of text.split("\n")) {
const kind = line.startsWith("some ") ? "some" : line.startsWith("full ") ? "full" : null;
if (!kind) continue;
result[`${kind}Avg10`] = parsePsiNumber(line, "avg10");
result[`${kind}Avg60`] = parsePsiNumber(line, "avg60");
result[`${kind}Avg300`] = parsePsiNumber(line, "avg300");
matched = true;
}
return matched ? result : null;
}
export async function sampleResourceSignals(
deps: SampleResourceSignalsDeps = {}
): Promise<ResourceSignals> {
const readText = deps.fs?.readText ?? defaultReadText;
let memory: NodeJS.MemoryUsage;
try {
memory = (deps.memoryUsage ?? process.memoryUsage)();
} catch {
memory = { rss: 0, heapTotal: 0, heapUsed: 0, external: 0, arrayBuffers: 0 };
}
let heapUsed = Math.max(0, Math.floor(memory.heapUsed || 0));
let heapLimit = 0;
try {
const heap = (deps.heapStatistics ?? v8.getHeapStatistics)();
heapLimit = sanitizeMemoryBytes(heap.heap_size_limit) ?? 0;
if (Number.isFinite(heap.used_heap_size)) {
heapUsed = Math.max(0, Math.floor(heap.used_heap_size ?? heapUsed));
}
} catch {
/* retain process heap sample */
}
const cgroupDirectory = await resolveCgroupDirectory(readText);
const cgroupContents = cgroupDirectory
? await Promise.all([
readText(path.join(cgroupDirectory, "memory.current")),
readText(path.join(cgroupDirectory, "memory.max")),
readText(path.join(cgroupDirectory, "memory.high")),
readText(path.join(cgroupDirectory, "memory.events")),
])
: [null, null, null, null];
const psi = await readText("/proc/pressure/memory").catch(() => null);
return {
observedAtMs: (deps.nowMs ?? Date.now)(),
v8: { heapUsedBytes: heapUsed, heapLimitBytes: heapLimit },
process: {
rssBytes: Math.max(0, Math.floor(memory.rss || 0)),
externalBytes: Math.max(0, Math.floor(memory.external || 0)),
arrayBuffersBytes: Math.max(0, Math.floor(memory.arrayBuffers || 0)),
availableBytes: safeNumber(deps.availableMemory ?? (() => process.availableMemory?.())),
constrainedBytes: safeNumber(deps.constrainedMemory ?? (() => process.constrainedMemory?.())),
},
cgroup: {
currentBytes: sanitizeMemoryBytes(cgroupContents[0]),
maxBytes: sanitizeMemoryBytes(cgroupContents[1]),
highBytes: sanitizeMemoryBytes(cgroupContents[2]),
events: parseMemoryEvents(cgroupContents[3]),
},
psi: parsePsi(psi),
};
}

View File

@@ -0,0 +1,985 @@
import { describe, it, beforeEach, afterEach } from "node:test";
import assert from "node:assert/strict";
import {
AdaptiveAdmissionController,
createAdmissionRejectError,
type AdaptiveAdmissionConfig,
type AdmissionLease,
type AdmissionPressure,
type AdmissionRequest,
} from "../../open-sse/services/admission/index.ts";
class FakeClock {
nowMs = 0;
private nextId = 1;
private timers = new Map<number, { due: number; fn: () => void }>();
now = () => this.nowMs;
setTimer = (fn: () => void, delayMs: number): number => {
const id = this.nextId++;
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
return id;
};
clearTimer = (id: number): void => {
this.timers.delete(id);
};
get pendingTimerCount(): number {
return this.timers.size;
}
advance(ms: number): void {
const target = this.nowMs + ms;
while (true) {
let nextId: number | undefined;
let nextDue = Number.POSITIVE_INFINITY;
for (const [id, t] of this.timers) {
if (t.due <= target && t.due < nextDue) {
nextDue = t.due;
nextId = id;
}
}
if (nextId === undefined) {
this.nowMs = target;
return;
}
const timer = this.timers.get(nextId)!;
this.timers.delete(nextId);
this.nowMs = timer.due;
timer.fn();
}
}
}
function baseConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
return {
mode: "enforce",
minLimit: 10,
maxLimit: 100,
initialLimit: 20,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 1000,
windowMs: 100,
shortLatencyAlpha: 0.5,
longLatencyAlpha: 0.1,
increaseStep: 2,
decreaseFactor: 0.8,
criticalDecreaseFactor: 0.5,
highUtilizationThreshold: 0.7,
lowUtilizationThreshold: 0.3,
latencyGradientThreshold: 0.25,
maxIncreasePerWindow: 4,
...overrides,
};
}
function req(partial: Partial<AdmissionRequest> & { cost: number }): AdmissionRequest {
return {
tenantKey: "t-default",
...partial,
};
}
async function mustAdmit(
controller: AdaptiveAdmissionController,
request: AdmissionRequest
): Promise<AdmissionLease> {
const result = await controller.acquire(request);
assert.equal(result.status, "admitted");
if (result.status !== "admitted") throw new Error("expected admitted");
return result.lease;
}
describe("AdaptiveAdmissionController config and modes", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
function make(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
return new AdaptiveAdmissionController(baseConfig(overrides), {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
});
}
it("validates safe-integer bounds and ordered adaptation parameters", () => {
assert.throws(() => make({ minLimit: 50, maxLimit: 10 }), /minLimit/);
for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
assert.throws(() => make({ maxQueueCount: invalid }), /maxQueueCount/);
assert.throws(() => make({ initialLimit: invalid }), /initialLimit/);
}
assert.throws(() => make({ decreaseFactor: 1.2 }), /decreaseFactor/);
assert.throws(
() => make({ decreaseFactor: 0.5, criticalDecreaseFactor: 0.8 }),
/criticalDecreaseFactor/
);
assert.throws(
() => make({ lowUtilizationThreshold: 0.8, highUtilizationThreshold: 0.7 }),
/lowUtilizationThreshold/
);
assert.throws(
() => make({ shortLatencyAlpha: 0.1, longLatencyAlpha: 0.5 }),
/shortLatencyAlpha/
);
});
it("clamps initial limit into [minLimit, maxLimit]", () => {
const low = make({ initialLimit: 1, minLimit: 10 });
assert.equal(low.snapshot().currentLimit, 10);
low.shutdown();
const high = make({ initialLimit: 999, maxLimit: 100 });
assert.equal(high.snapshot().currentLimit, 100);
high.shutdown();
});
it("mode off never accounts cost or rejects", async () => {
const c = make({ mode: "off", initialLimit: 5 });
const a = await c.acquire(req({ cost: 100 }));
const b = await c.acquire(req({ cost: 100 }));
assert.equal(a.status, "admitted");
assert.equal(b.status, "admitted");
const snap = c.snapshot();
assert.equal(snap.activeCost, 0);
assert.equal(snap.activeCount, 0);
assert.equal(snap.rejectedCount, 0);
c.shutdown();
});
it("defaults to shadow mode when mode omitted", () => {
const c = new AdaptiveAdmissionController(
{
minLimit: 10,
maxLimit: 100,
initialLimit: 20,
maxQueueCount: 2,
maxQueueCost: 20,
} as AdaptiveAdmissionConfig,
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
);
assert.equal(c.snapshot().mode, "shadow");
c.shutdown();
});
});
describe("shadow mode semantics", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
it("never rejects or delays while recording would-decisions and real active cost", async () => {
const c = new AdaptiveAdmissionController(
baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 10 }),
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
);
const first = await c.acquire(req({ cost: 8 }));
assert.equal(first.status, "admitted");
if (first.status !== "admitted") return;
assert.equal(first.shadowDecision, "would-admit");
assert.equal(c.snapshot().activeCost, 8);
const second = await c.acquire(req({ cost: 8 }));
assert.equal(second.status, "admitted");
if (second.status !== "admitted") return;
// Would have queued under enforce (active 8 + 8 > 10) but shadow admits immediately.
assert.equal(second.shadowDecision, "would-queue");
assert.equal(c.snapshot().activeCost, 16);
assert.equal(c.snapshot().queuedCount, 0);
assert.ok((c.snapshot().wouldQueueCount ?? 0) >= 1);
const oversized = await c.acquire(req({ cost: 50 }));
assert.equal(oversized.status, "admitted");
if (oversized.status !== "admitted") return;
assert.equal(oversized.shadowDecision, "would-reject");
assert.ok((c.snapshot().wouldRejectCount ?? 0) >= 1);
first.lease.release("success");
second.lease.release("success");
oversized.lease.release("success");
assert.equal(c.snapshot().activeCost, 0);
c.shutdown();
});
it("simulates virtual queue saturation and promotes queued work on release", async () => {
const c = new AdaptiveAdmissionController(
baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 8 }),
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
);
const active = await c.acquire(req({ cost: 8, tenantKey: "active" }));
const queued = await c.acquire(req({ cost: 8, tenantKey: "queued" }));
const saturated = await c.acquire(req({ cost: 8, tenantKey: "saturated" }));
assert.equal(active.status, "admitted");
assert.equal(queued.status, "admitted");
assert.equal(saturated.status, "admitted");
if (
active.status !== "admitted" ||
queued.status !== "admitted" ||
saturated.status !== "admitted"
) {
return;
}
assert.equal(active.shadowDecision, "would-admit");
assert.equal(queued.shadowDecision, "would-queue");
assert.equal(saturated.shadowDecision, "would-reject");
assert.deepEqual(
{
activeCost: c.snapshot().virtualActiveCost,
activeCount: c.snapshot().virtualActiveCount,
queuedCost: c.snapshot().virtualQueuedCost,
queuedCount: c.snapshot().virtualQueuedCount,
},
{ activeCost: 8, activeCount: 1, queuedCost: 8, queuedCount: 1 }
);
active.lease.release();
assert.deepEqual(
{
activeCost: c.snapshot().virtualActiveCost,
activeCount: c.snapshot().virtualActiveCount,
queuedCost: c.snapshot().virtualQueuedCost,
queuedCount: c.snapshot().virtualQueuedCount,
},
{ activeCost: 8, activeCount: 1, queuedCost: 0, queuedCount: 0 }
);
queued.lease.release();
saturated.lease.release();
c.shutdown();
});
it("promotes shadow virtual queue after adaptation raises the limit", async () => {
const c = new AdaptiveAdmissionController(
baseConfig({
mode: "shadow",
minLimit: 10,
maxLimit: 20,
initialLimit: 10,
maxQueueCount: 4,
maxQueueCost: 40,
windowMs: 100,
increaseStep: 5,
maxIncreasePerWindow: 5,
highUtilizationThreshold: 0.5,
}),
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
);
const active = await c.acquire(req({ cost: 10, tenantKey: "active" }));
const queued = await c.acquire(req({ cost: 5, tenantKey: "queued" }));
assert.equal(active.status, "admitted");
assert.equal(queued.status, "admitted");
if (active.status !== "admitted" || queued.status !== "admitted") return;
assert.equal(active.shadowDecision, "would-admit");
assert.equal(queued.shadowDecision, "would-queue");
assert.equal(c.snapshot().virtualActiveCost, 10);
assert.equal(c.snapshot().virtualQueuedCost, 5);
// Raise the adaptive limit once while both leases remain open. Shadow admits a
// probe for completion evidence; active integral is capped at the current limit.
const probe = await c.acquire(req({ cost: 1, tenantKey: "probe" }));
assert.equal(probe.status, "admitted");
if (probe.status === "admitted") {
clock.advance(80);
probe.lease.release("success", { latencyMs: 10 });
clock.advance(20);
c.tick();
}
assert.equal(c.snapshot().currentLimit, 15);
// Queued virtual work must be promoted before newer arrivals are classified.
assert.equal(c.snapshot().virtualActiveCost, 15);
assert.equal(c.snapshot().virtualQueuedCost, 0);
const later = await c.acquire(req({ cost: 5, tenantKey: "later" }));
assert.equal(later.status, "admitted");
if (later.status !== "admitted") return;
// With virtual active already 15 at limit 15, a later cost-5 cannot would-admit.
assert.notEqual(later.shadowDecision, "would-admit");
active.lease.release();
queued.lease.release();
later.lease.release();
c.shutdown();
});
});
describe("weighted enforce, queue, fairness, and races", () => {
let clock: FakeClock;
const live: AdaptiveAdmissionController[] = [];
beforeEach(() => {
clock = new FakeClock();
live.length = 0;
});
afterEach(() => {
for (const c of live) c.shutdown();
live.length = 0;
});
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
});
live.push(c);
return c;
}
it("enforces weighted active-cost budget and rejects oversized requests immediately", async () => {
const c = controller({ initialLimit: 20 });
const a = await mustAdmit(c, req({ cost: 12 }));
const b = await c.acquire(req({ cost: 12 }));
assert.equal(b.status, "queued");
const over = await c.acquire(req({ cost: 25 }));
assert.equal(over.status, "rejected");
if (over.status === "rejected") {
assert.equal(over.code, "ADMISSION_OVERSIZED");
}
a.release("success");
if (b.status === "queued") {
const admitted = await b.promise;
assert.equal(admitted.status, "admitted");
admitted.lease.release("success");
}
});
it("bounds queue by count and total queued cost", async () => {
const c = controller({
minLimit: 10,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 2,
maxQueueCost: 15,
});
const held = await mustAdmit(c, req({ cost: 10 }));
const q1 = await c.acquire(req({ cost: 5, tenantKey: "a" }));
const q2 = await c.acquire(req({ cost: 5, tenantKey: "b" }));
assert.equal(q1.status, "queued");
assert.equal(q2.status, "queued");
assert.equal(c.snapshot().queuedCount, 2);
assert.equal(c.snapshot().queuedCost, 10);
const byCount = await c.acquire(req({ cost: 1, tenantKey: "c" }));
assert.equal(byCount.status, "rejected");
if (byCount.status === "rejected") assert.equal(byCount.code, "ADMISSION_QUEUE_FULL");
held.release("success");
if (q1.status === "queued") (await q1.promise).lease.release("success");
if (q2.status === "queued") (await q2.promise).lease.release("success");
const c2 = controller({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 10,
maxQueueCost: 7,
});
const h = await mustAdmit(c2, req({ cost: 5 }));
// cost 3 fits limit but not active budget → queued (queuedCost=3).
// Another cost 5 fits the budget but 3+5 > maxQueueCost=7 → QUEUE_FULL.
const ok = await c2.acquire(req({ cost: 3 }));
assert.equal(ok.status, "queued");
const costFull = await c2.acquire(req({ cost: 5 }));
assert.equal(costFull.status, "rejected");
if (costFull.status === "rejected") assert.equal(costFull.code, "ADMISSION_QUEUE_FULL");
h.release("success");
if (ok.status === "queued") (await ok.promise).lease.release("success");
});
it("expires deadline and abort without leaking queue slots", async () => {
const c = controller({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 50,
});
const held = await mustAdmit(c, req({ cost: 5 }));
const timed = await c.acquire(req({ cost: 3, maxWaitMs: 30 }));
assert.equal(timed.status, "queued");
clock.advance(31);
if (timed.status === "queued") {
await assert.rejects(timed.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
return true;
});
}
assert.equal(c.snapshot().queuedCount, 0);
const ac = new AbortController();
const aborted = await c.acquire(req({ cost: 3, signal: ac.signal }));
assert.equal(aborted.status, "queued");
ac.abort();
if (aborted.status === "queued") {
await assert.rejects(aborted.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED");
return true;
});
}
assert.equal(c.snapshot().queuedCount, 0);
held.release("success");
});
it("treats the exact deadline as expired and settles abort/release races once", async () => {
const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 5, defaultMaxWaitMs: 30 });
const held = await mustAdmit(c, req({ cost: 5 }));
const ac = new AbortController();
const queued = await c.acquire(req({ cost: 3, maxWaitMs: 30, signal: ac.signal }));
assert.equal(queued.status, "queued");
clock.advance(30);
ac.abort();
held.release("success");
if (queued.status === "queued") {
await assert.rejects(queued.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
return true;
});
}
assert.equal(c.snapshot().queuedCount, 0);
assert.equal(c.snapshot().rejectedCount, 1);
});
it("shutdown rejects queued work and clears every fake-clock timer", async () => {
const c = controller({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 4,
maxQueueCost: 40,
});
const held = await mustAdmit(c, req({ cost: 5 }));
const q = await c.acquire(req({ cost: 3, maxWaitMs: 5000 }));
assert.equal(q.status, "queued");
assert.ok(clock.pendingTimerCount >= 2);
c.shutdown();
if (q.status === "queued") {
await assert.rejects(q.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_SHUTDOWN");
return true;
});
}
assert.equal(c.snapshot().queuedCount, 0);
assert.equal(clock.pendingTimerCount, 0);
held.release("success");
const after = await c.acquire(req({ cost: 1 }));
assert.equal(after.status, "rejected");
if (after.status === "rejected") assert.equal(after.code, "ADMISSION_SHUTDOWN");
});
it("updateConfig atomically settles queues, dispatches raised capacity, and respects decreases", async () => {
const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 20, windowMs: 100 });
const held = await mustAdmit(c, req({ cost: 5 }));
const queued = await c.acquire(req({ cost: 5 }));
assert.equal(queued.status, "queued");
c.updateConfig(baseConfig({ minLimit: 10, initialLimit: 10, maxLimit: 20, windowMs: 50 }));
assert.equal(clock.pendingTimerCount, 1);
if (queued.status === "queued") {
const admitted = await queued.promise;
assert.equal(c.snapshot().activeCost, 10);
c.updateConfig(baseConfig({ minLimit: 5, initialLimit: 5, maxLimit: 5, windowMs: 50 }));
const afterDecrease = await c.acquire(req({ cost: 1 }));
assert.equal(afterDecrease.status, "queued");
c.updateConfig(baseConfig({ mode: "shadow", minLimit: 5, initialLimit: 5, maxLimit: 5 }));
if (afterDecrease.status === "queued") {
const settled = await afterDecrease.promise;
assert.equal(settled.status, "admitted");
settled.lease.release();
}
admitted.lease.release();
}
held.release();
});
it("queue shrink rejects deterministic round-robin excess and preserves fitting entries", async () => {
const c = controller({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 4,
maxQueueCost: 20,
});
const held = await mustAdmit(c, req({ cost: 5 }));
const first = await c.acquire(req({ cost: 2, tenantKey: "a" }));
const second = await c.acquire(req({ cost: 2, tenantKey: "b" }));
const third = await c.acquire(req({ cost: 2, tenantKey: "a" }));
assert.equal(first.status, "queued");
assert.equal(second.status, "queued");
assert.equal(third.status, "queued");
c.updateConfig(
baseConfig({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 2,
maxQueueCost: 4,
})
);
assert.equal(c.snapshot().queuedCount, 2);
if (third.status === "queued") {
await assert.rejects(third.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_QUEUE_FULL");
return true;
});
}
held.release();
if (first.status === "queued") (await first.promise).lease.release();
if (second.status === "queued") (await second.promise).lease.release();
});
it("release is idempotent under race with abort", async () => {
const c = controller({ initialLimit: 10 });
const lease = await mustAdmit(c, req({ cost: 4 }));
lease.release("success");
lease.release("timeout");
lease.release("success");
assert.equal(c.snapshot().activeCost, 0);
assert.equal(c.snapshot().activeCount, 0);
});
it("fairly schedules across tenants under skew without exposing tenant ids", async () => {
const c = controller({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 10,
maxQueueCost: 100,
});
const held = await mustAdmit(c, req({ cost: 5, tenantKey: "hold" }));
const order: string[] = [];
const queued: Array<Promise<void>> = [];
for (let i = 0; i < 4; i++) {
const r = await c.acquire(req({ cost: 5, tenantKey: "heavy" }));
assert.equal(r.status, "queued");
if (r.status === "queued") {
queued.push(
r.promise.then((admitted) => {
order.push("heavy");
admitted.lease.release("success");
})
);
}
}
const light = await c.acquire(req({ cost: 5, tenantKey: "light" }));
assert.equal(light.status, "queued");
if (light.status === "queued") {
queued.push(
light.promise.then((admitted) => {
order.push("light");
admitted.lease.release("success");
})
);
}
// Free capacity one slot at a time.
held.release("success");
await Promise.resolve();
// After first release, one request should admit; keep draining by waiting microtasks between releases.
// Drain remaining by letting each admitted release free the next.
await Promise.all(queued);
// Light must not be starved behind all four heavy requests.
const lightIndex = order.indexOf("light");
assert.ok(lightIndex >= 0);
assert.ok(lightIndex < 4, `light scheduled too late: ${order.join(",")}`);
const snap = c.snapshot();
const json = JSON.stringify(snap);
assert.equal(json.includes("heavy"), false);
assert.equal(json.includes("light"), false);
assert.equal(json.includes("hold"), false);
});
it("dispatches a fitting tenant when another tenant's queue head cannot fit", async () => {
const c = controller({
minLimit: 10,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 10,
maxQueueCost: 100,
});
const heldSix = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
const heldFour = await mustAdmit(c, req({ cost: 4, tenantKey: "holder" }));
const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" }));
const fitting = await c.acquire(req({ cost: 4, tenantKey: "fitting" }));
assert.equal(expensive.status, "queued");
assert.equal(fitting.status, "queued");
heldFour.release("success");
if (fitting.status === "queued") {
const admitted = await fitting.promise;
assert.equal(admitted.lease.cost, 4);
admitted.lease.release("success");
}
assert.equal(c.snapshot().queuedCount, 1);
heldSix.release("success");
if (expensive.status === "queued") (await expensive.promise).lease.release("success");
});
it("bounds starvation of an older unfittable cost-6 behind a stream of cost-2 work", async () => {
const c = controller({
minLimit: 10,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 20,
maxQueueCost: 100,
defaultMaxWaitMs: 10_000,
});
// Hold 6 so available=4: cost-2 can pass over cost-6 until reservation engages.
const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" }));
assert.equal(expensive.status, "queued");
// Two actual smaller dequeues through queued promises (pass-overs that age the head).
const passOvers: AdmissionLease[] = [];
for (let i = 0; i < 2; i++) {
const r = await c.acquire(req({ cost: 2, tenantKey: `small-pass-${i}` }));
assert.equal(r.status, "queued", `pass-over ${i} should join the non-empty queue`);
if (r.status !== "queued") throw new Error("expected queued");
const admitted = await r.promise;
assert.equal(admitted.lease.cost, 2);
passOvers.push(admitted.lease);
admitted.lease.release("success");
assert.equal(c.snapshot().activeCost, 6);
}
assert.equal(passOvers.length, 2);
// A subsequent fitting cost-2 must remain queued: capacity is reserved for cost-6.
// Without reservation accounting this would admit immediately and the assertion fails.
const blocked = await c.acquire(req({ cost: 2, tenantKey: "small-blocked" }));
assert.equal(blocked.status, "queued");
await Promise.resolve();
assert.equal(c.snapshot().activeCost, 6, "reserved head must block fitting smaller work");
assert.equal(c.snapshot().queuedCount, 2);
const order: number[] = [];
assert.equal(expensive.status, "queued");
assert.equal(blocked.status, "queued");
const expensiveDone = expensive.promise.then((admitted) => {
order.push(admitted.lease.cost);
return admitted;
});
const blockedDone = blocked.promise.then((admitted) => {
order.push(admitted.lease.cost);
return admitted;
});
// Free enough capacity for cost-6; the older reserved request must admit first.
// With activeCost back at 0 both may fit in one dispatch turn, so only order is asserted.
held.release("success");
const [expAdmitted, blockedAdmitted] = await Promise.all([expensiveDone, blockedDone]);
assert.equal(expAdmitted.lease.cost, 6);
assert.equal(blockedAdmitted.lease.cost, 2);
assert.deepEqual(order, [6, 2]);
expAdmitted.lease.release("success");
blockedAdmitted.lease.release("success");
assert.equal(c.snapshot().queuedCount, 0);
});
async function ageReservedCost6(
c: AdaptiveAdmissionController,
expensiveSignal?: AbortSignal,
expensiveMaxWaitMs?: number
) {
const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
const expensive = await c.acquire(
req({
cost: 6,
tenantKey: "expensive",
signal: expensiveSignal,
maxWaitMs: expensiveMaxWaitMs,
})
);
assert.equal(expensive.status, "queued");
for (let i = 0; i < 2; i++) {
const r = await c.acquire(req({ cost: 2, tenantKey: `age-pass-${i}` }));
assert.equal(r.status, "queued");
if (r.status !== "queued") throw new Error("expected queued");
(await r.promise).lease.release("success");
}
const blocked = await c.acquire(req({ cost: 2, tenantKey: "age-blocked" }));
assert.equal(blocked.status, "queued");
await Promise.resolve();
assert.equal(c.snapshot().activeCost, 6);
assert.equal(c.snapshot().queuedCount, 2);
return { held, expensive, blocked };
}
it("aborting a reserved head immediately admits the next fitting request", async () => {
const c = controller({
minLimit: 10,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 20,
maxQueueCost: 100,
defaultMaxWaitMs: 10_000,
});
const ac = new AbortController();
const { held, expensive, blocked } = await ageReservedCost6(c, ac.signal);
assert.equal(expensive.status, "queued");
assert.equal(blocked.status, "queued");
ac.abort();
// No tick / new arrival / release / config update — only the abort path.
if (expensive.status === "queued") {
await assert.rejects(expensive.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED");
return true;
});
}
if (blocked.status !== "queued") throw new Error("expected queued blocked request");
const admitted = await blocked.promise;
assert.equal(admitted.lease.cost, 2);
assert.equal(c.snapshot().activeCost, 8);
assert.equal(c.snapshot().queuedCount, 0);
admitted.lease.release("success");
held.release("success");
});
it("deadline-expiring a reserved head immediately admits the next fitting request", async () => {
const c = controller({
minLimit: 10,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 20,
maxQueueCost: 100,
defaultMaxWaitMs: 10_000,
});
const { held, expensive, blocked } = await ageReservedCost6(c, undefined, 40);
assert.equal(expensive.status, "queued");
assert.equal(blocked.status, "queued");
clock.advance(40);
// No tick / new arrival / release / config update — only the deadline timer.
if (expensive.status === "queued") {
await assert.rejects(expensive.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
return true;
});
}
if (blocked.status !== "queued") throw new Error("expected queued blocked request");
const admitted = await blocked.promise;
assert.equal(admitted.lease.cost, 2);
assert.equal(c.snapshot().activeCost, 8);
assert.equal(c.snapshot().queuedCount, 0);
admitted.lease.release("success");
held.release("success");
});
});
describe("adaptive algorithm", () => {
let clock: FakeClock;
const live: AdaptiveAdmissionController[] = [];
beforeEach(() => {
clock = new FakeClock();
live.length = 0;
});
afterEach(() => {
for (const c of live) c.shutdown();
live.length = 0;
});
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
});
live.push(c);
return c;
}
async function complete(
c: AdaptiveAdmissionController,
cost: number,
latencyMs: number,
outcome: "success" | "upstream_error" | "timeout" = "success",
pressure: AdmissionPressure = "normal"
) {
const lease = await mustAdmit(c, req({ cost, pressure }));
clock.advance(latencyMs);
lease.release(outcome, { latencyMs, pressure });
}
it("keeps currentLimit within validated bounds", async () => {
const c = controller({ initialLimit: 20, minLimit: 10, maxLimit: 30, increaseStep: 50 });
for (let i = 0; i < 20; i++) {
await complete(c, 5, 5, "success", "normal");
clock.advance(100);
}
assert.ok(c.snapshot().currentLimit <= 30);
assert.ok(c.snapshot().currentLimit >= 10);
for (let i = 0; i < 10; i++) {
await complete(c, 5, 5, "success", "critical");
clock.advance(100);
}
assert.ok(c.snapshot().currentLimit >= 10);
});
it("decreases rapidly under critical pressure", async () => {
const c = controller({ initialLimit: 80, minLimit: 10, maxLimit: 100 });
const before = c.snapshot().currentLimit;
await complete(c, 10, 10, "success", "critical");
clock.advance(100);
// Force a window tick with pressure observation.
c.observePressure("critical");
clock.advance(100);
assert.ok(c.snapshot().currentLimit < before);
assert.ok(c.snapshot().currentLimit <= Math.ceil(before * 0.5) + 1);
});
it("applies criticalDecreaseFactor once for a single observePressure(critical)", () => {
const c = controller({
initialLimit: 80,
minLimit: 10,
maxLimit: 100,
criticalDecreaseFactor: 0.5,
decreaseFactor: 0.8,
windowMs: 100,
});
assert.equal(c.snapshot().currentLimit, 80);
c.observePressure("critical");
// Immediate fast decrease: 80 * 0.5 = 40.
assert.equal(c.snapshot().currentLimit, 40);
// Closing the same window must not multiply again (would become 20).
clock.advance(100);
c.tick();
assert.equal(c.snapshot().currentLimit, 40);
// A fresh critical observation in a later window still decreases once.
c.observePressure("critical");
assert.equal(c.snapshot().currentLimit, 20);
clock.advance(100);
c.tick();
assert.equal(c.snapshot().currentLimit, 20);
});
it("decreases on high pressure or sustained latency gradient", async () => {
const c = controller({
initialLimit: 50,
shortLatencyAlpha: 0.8,
longLatencyAlpha: 0.1,
latencyGradientThreshold: 0.2,
});
// Seed long baseline with low latency.
for (let i = 0; i < 5; i++) {
await complete(c, 8, 10, "success", "normal");
clock.advance(100);
}
const mid = c.snapshot().currentLimit;
// Spike short latency relative to long.
for (let i = 0; i < 5; i++) {
await complete(c, 8, 200, "success", "normal");
clock.advance(100);
}
assert.ok(c.snapshot().currentLimit <= mid);
const beforeHigh = c.snapshot().currentLimit;
c.observePressure("high");
await complete(c, 8, 20, "success", "high");
clock.advance(100);
assert.ok(c.snapshot().currentLimit <= beforeHigh);
});
it("increases slowly when healthy and highly utilized, and does not inflate when idle", async () => {
const c = controller({
minLimit: 20,
maxLimit: 40,
initialLimit: 20,
increaseStep: 2,
maxIncreasePerWindow: 2,
highUtilizationThreshold: 0.5,
windowMs: 100,
});
// Idle windows should not inflate.
clock.advance(500);
c.tick();
clock.advance(500);
c.tick();
assert.equal(c.snapshot().currentLimit, 20);
// Healthy high utilization: hold nearly full budget across most of each window.
for (let w = 0; w < 5; w++) {
const lease = await mustAdmit(c, req({ cost: 16, pressure: "normal" }));
clock.advance(80);
lease.release("success", { latencyMs: 10, pressure: "normal" });
clock.advance(20);
c.tick();
}
assert.ok(c.snapshot().currentLimit > 20);
assert.ok(c.snapshot().currentLimit <= 20 + 2 * 5);
});
it("does not collapse capacity on a single upstream business error", async () => {
const c = controller({ initialLimit: 40, decreaseFactor: 0.5, criticalDecreaseFactor: 0.5 });
await complete(c, 10, 15, "upstream_error", "normal");
clock.advance(100);
c.tick();
// One business error may freeze growth but must not apply critical collapse.
assert.ok(c.snapshot().currentLimit >= 30);
});
it("integrates active utilization exactly once over a full window", async () => {
const c = controller({
minLimit: 20,
initialLimit: 20,
maxLimit: 20,
highUtilizationThreshold: 0.9,
windowMs: 100,
});
const lease = await mustAdmit(c, req({ cost: 8 }));
clock.advance(100);
assert.equal(c.snapshot().utilization, 0.4);
lease.release("success");
});
it("consumes latency and pressure evidence only in the window where it was observed", async () => {
const c = controller({
initialLimit: 80,
minLimit: 10,
maxLimit: 100,
decreaseFactor: 0.5,
criticalDecreaseFactor: 0.25,
windowMs: 100,
});
await complete(c, 8, 200, "success", "high");
clock.advance(100);
const afterObservedWindow = c.snapshot().currentLimit;
assert.ok(afterObservedWindow < 80);
clock.advance(500);
assert.equal(c.snapshot().currentLimit, afterObservedWindow);
});
});
describe("createAdmissionRejectError", () => {
it("builds typed rejection errors", () => {
const err = createAdmissionRejectError("ADMISSION_QUEUE_FULL", "queue full");
assert.equal(err.code, "ADMISSION_QUEUE_FULL");
assert.equal(err.name, "AdmissionRejectError");
assert.match(err.message, /queue full/);
});
});

View File

@@ -0,0 +1,143 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import {
estimateAdmissionCost,
DEFAULT_ADMISSION_COST_CONFIG,
MAX_ADMISSION_COST_OR_LIMIT,
normalizeRequestCost,
resolveCostConfig,
type AdmissionCostConfig,
type AdmissionCostFeatures,
} from "../../open-sse/services/admission/index.ts";
function features(overrides: Partial<AdmissionCostFeatures> = {}): AdmissionCostFeatures {
return {
bodyBytes: 0,
estimatedInputTokens: 0,
messageCount: 0,
toolCount: 0,
requestedFanout: 1,
streaming: true,
...overrides,
};
}
describe("estimateAdmissionCost", () => {
it("returns a positive integer at least the base cost", () => {
const cost = estimateAdmissionCost(features());
assert.equal(Number.isSafeInteger(cost), true);
assert.ok(cost >= DEFAULT_ADMISSION_COST_CONFIG.baseCost);
assert.ok(cost > 0);
});
it("is monotonic in body bytes, tokens, messages, tools, and fanout", () => {
const base = estimateAdmissionCost(features());
assert.ok(estimateAdmissionCost(features({ bodyBytes: 50_000 })) >= base);
assert.ok(estimateAdmissionCost(features({ estimatedInputTokens: 8_000 })) >= base);
assert.ok(estimateAdmissionCost(features({ messageCount: 40 })) >= base);
assert.ok(estimateAdmissionCost(features({ toolCount: 20 })) >= base);
assert.ok(estimateAdmissionCost(features({ requestedFanout: 8 })) >= base);
});
it("rejects fractional, infinite, and unsafe cost configuration", () => {
for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
assert.throws(
() => resolveCostConfig({ bodyBytesPerUnit: invalid }),
/positive safe integer/
);
assert.throws(() => resolveCostConfig({ maxRequestCost: invalid }), /positive safe integer/);
assert.throws(
() => resolveCostConfig({ streamingClassCost: invalid }),
/positive safe integer/
);
}
});
it("normalizes caller costs only when they are positive safe integers", () => {
assert.equal(normalizeRequestCost(7, 10), 7);
for (const invalid of [0, -1, 0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
assert.throws(() => normalizeRequestCost(invalid, 10), /positive safe integer/);
}
assert.throws(() => normalizeRequestCost(1, Number.MAX_VALUE), /positive safe integer/);
assert.throws(
() => normalizeRequestCost(1, MAX_ADMISSION_COST_OR_LIMIT + 1),
/maxRequestCost|must be <=/
);
assert.equal(
normalizeRequestCost(MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_COST_OR_LIMIT),
MAX_ADMISSION_COST_OR_LIMIT
);
});
it("clamps multiple overflowing feature contributions without unsafe arithmetic", () => {
const config: AdmissionCostConfig = {
...DEFAULT_ADMISSION_COST_CONFIG,
maxRequestCost: 25,
};
const cost = estimateAdmissionCost(
features({
bodyBytes: Number.MAX_SAFE_INTEGER,
estimatedInputTokens: Number.MAX_SAFE_INTEGER,
messageCount: Number.MAX_SAFE_INTEGER,
toolCount: Number.MAX_SAFE_INTEGER,
requestedFanout: Number.MAX_SAFE_INTEGER,
}),
config
);
assert.equal(cost, 25);
});
it("normalizes invalid, negative, and NaN inputs safely", () => {
const cost = estimateAdmissionCost({
bodyBytes: Number.NaN,
estimatedInputTokens: -12,
messageCount: Number.POSITIVE_INFINITY,
toolCount: undefined,
requestedFanout: 0,
streaming: undefined,
} as AdmissionCostFeatures);
assert.equal(Number.isSafeInteger(cost), true);
assert.ok(cost >= 1);
assert.ok(cost <= DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost);
});
it("uses transparent configurable quanta without a fixed-MB claim", () => {
const config: AdmissionCostConfig = {
baseCost: 1,
bodyBytesPerUnit: 1000,
tokensPerUnit: 100,
messagesPerUnit: 10,
toolsPerUnit: 5,
fanoutPerUnit: 1,
streamingClassCost: 1,
nonStreamingClassCost: 3,
maxRequestCost: 1000,
};
// 2500 bytes → 3 units (ceil), 250 tokens → 3 units, 1 message → 1, 0 tools, fanout 1 → 1, streaming class 1
const cost = estimateAdmissionCost(
features({
bodyBytes: 2500,
estimatedInputTokens: 250,
messageCount: 1,
toolCount: 0,
requestedFanout: 1,
streaming: true,
}),
config
);
assert.equal(cost, 1 + 3 + 3 + 1 + 0 + 1 + 1);
const nonStream = estimateAdmissionCost(
features({
bodyBytes: 0,
estimatedInputTokens: 0,
messageCount: 0,
toolCount: 0,
requestedFanout: 1,
streaming: false,
}),
config
);
assert.equal(nonStream, 1 + 0 + 0 + 0 + 0 + 1 + 3);
});
});

View File

@@ -0,0 +1,405 @@
import { describe, it, beforeEach, afterEach } from "node:test";
import assert from "node:assert/strict";
import {
AdaptiveAdmissionController,
MAX_ADMISSION_COST_OR_LIMIT,
MAX_ADMISSION_WINDOW_MS,
type AdaptiveAdmissionConfig,
type AdmissionLease,
type AdmissionRequest,
} from "../../open-sse/services/admission/index.ts";
class FakeClock {
nowMs = 0;
private nextId = 1;
private timers = new Map<number, { due: number; fn: () => void }>();
now = () => this.nowMs;
setTimer = (fn: () => void, delayMs: number): number => {
const id = this.nextId++;
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
return id;
};
clearTimer = (id: number): void => {
this.timers.delete(id);
};
get pendingTimerCount(): number {
return this.timers.size;
}
advance(ms: number): void {
const target = this.nowMs + ms;
while (true) {
let nextId: number | undefined;
let nextDue = Number.POSITIVE_INFINITY;
for (const [id, t] of this.timers) {
if (t.due <= target && t.due < nextDue) {
nextDue = t.due;
nextId = id;
}
}
if (nextId === undefined) {
this.nowMs = target;
return;
}
const timer = this.timers.get(nextId)!;
this.timers.delete(nextId);
this.nowMs = timer.due;
timer.fn();
}
}
}
function baseConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
return {
mode: "enforce",
minLimit: 10,
maxLimit: 100,
initialLimit: 20,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 1000,
windowMs: 100,
shortLatencyAlpha: 0.5,
longLatencyAlpha: 0.1,
increaseStep: 2,
decreaseFactor: 0.8,
criticalDecreaseFactor: 0.5,
highUtilizationThreshold: 0.7,
lowUtilizationThreshold: 0.3,
latencyGradientThreshold: 0.25,
maxIncreasePerWindow: 4,
...overrides,
};
}
function req(partial: Partial<AdmissionRequest> & { cost: number }): AdmissionRequest {
return {
tenantKey: "t-default",
...partial,
};
}
async function mustAdmit(
controller: AdaptiveAdmissionController,
request: AdmissionRequest
): Promise<AdmissionLease> {
const result = await controller.acquire(request);
assert.equal(result.status, "admitted");
if (result.status !== "admitted") throw new Error("expected admitted");
return result.lease;
}
describe("admission operational domain", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
function make(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
return new AdaptiveAdmissionController(baseConfig(overrides), {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
});
}
it("exports an operational domain that rejects max+1 and accepts exact max", () => {
assert.ok(Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT));
assert.ok(Number.isSafeInteger(MAX_ADMISSION_WINDOW_MS));
assert.ok(
Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS),
"limit×window must remain a safe integer"
);
assert.throws(
() =>
make({
minLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
maxLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
initialLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
}),
/minLimit|must be <=/
);
assert.throws(
() => make({ maxQueueCost: MAX_ADMISSION_COST_OR_LIMIT + 1 }),
/maxQueueCost|must be <=/
);
assert.throws(() => make({ windowMs: MAX_ADMISSION_WINDOW_MS + 1 }), /windowMs|must be <=/);
assert.throws(
() => make({ cost: { maxRequestCost: MAX_ADMISSION_COST_OR_LIMIT + 1 } }),
/maxRequestCost|must be <=/
);
assert.throws(() => make({ maxLimit: Number.MAX_SAFE_INTEGER }), /maxLimit|must be <=/);
const max = MAX_ADMISSION_COST_OR_LIMIT;
const c = make({
minLimit: max,
maxLimit: max,
initialLimit: max,
maxQueueCount: 2,
maxQueueCost: max,
windowMs: 1000,
cost: { maxRequestCost: max },
});
assert.equal(c.snapshot().currentLimit, max);
c.shutdown();
});
it("keeps full utilization and multi-lease accounting exact at the domain max", async () => {
const max = MAX_ADMISSION_COST_OR_LIMIT;
const c = make({
mode: "enforce",
minLimit: max,
maxLimit: max,
initialLimit: max,
maxQueueCount: 4,
maxQueueCost: max,
windowMs: 1000,
cost: { maxRequestCost: max },
});
const full = await mustAdmit(c, req({ cost: max }));
assert.equal(c.snapshot().activeCost, max);
assert.equal(Number.isSafeInteger(c.snapshot().activeCost), true);
clock.advance(1000);
assert.equal(c.snapshot().utilization, 1);
full.release("success");
assert.equal(c.snapshot().activeCost, 0);
assert.equal(c.snapshot().activeCount, 0);
const left = Math.floor(max / 2);
const right = max - left;
const a = await mustAdmit(c, req({ cost: left }));
const b = await mustAdmit(c, req({ cost: right }));
assert.equal(c.snapshot().activeCost, max);
assert.equal(c.snapshot().activeCount, 2);
clock.advance(1000);
assert.equal(c.snapshot().utilization, 1);
a.release("success");
assert.equal(c.snapshot().activeCost, right);
b.release("success");
assert.equal(c.snapshot().activeCost, 0);
assert.equal(c.snapshot().activeCount, 0);
c.shutdown();
});
});
describe("updateConfig re-evaluation", () => {
let clock: FakeClock;
const live: AdaptiveAdmissionController[] = [];
beforeEach(() => {
clock = new FakeClock();
live.length = 0;
});
afterEach(() => {
for (const c of live) c.shutdown();
live.length = 0;
});
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
});
live.push(c);
return c;
}
it("updateConfig rejects queued work above the new enforce limit immediately", async () => {
const c = controller({
minLimit: 5,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 5_000,
});
const held = await mustAdmit(c, req({ cost: 10 }));
const queued = await c.acquire(req({ cost: 8 }));
assert.equal(queued.status, "queued");
c.updateConfig(
baseConfig({
minLimit: 5,
initialLimit: 5,
maxLimit: 5,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 5_000,
})
);
if (queued.status === "queued") {
await assert.rejects(queued.promise, (err: unknown) => {
assert.equal((err as { code?: string }).code, "ADMISSION_OVERSIZED");
return true;
});
}
assert.equal(c.snapshot().queuedCount, 0);
held.release();
});
it("updateConfig enforce→shadow classifies individually oversized active work as virtual rejected", async () => {
const c = controller({
mode: "enforce",
minLimit: 5,
initialLimit: 20,
maxLimit: 20,
maxQueueCount: 4,
maxQueueCost: 40,
});
const oversized = await mustAdmit(c, req({ cost: 15 }));
const fitting = await mustAdmit(c, req({ cost: 5 }));
c.updateConfig(
baseConfig({
mode: "shadow",
minLimit: 5,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 4,
maxQueueCost: 40,
})
);
const snap = c.snapshot();
// currentLimit clamps to 10; cost 15 is individually oversized → virtual rejected, not queued.
assert.equal(snap.currentLimit, 10);
assert.equal(snap.virtualActiveCost, 5);
assert.equal(snap.virtualActiveCount, 1);
assert.equal(snap.virtualQueuedCost, 0);
assert.equal(snap.virtualQueuedCount, 0);
// Real active leases remain until release.
assert.equal(snap.activeCost, 20);
assert.equal(snap.activeCount, 2);
oversized.release();
fitting.release();
});
it("updateConfig rebuilds shadow virtual dispositions under new limits and queue bounds", async () => {
const c = controller({
mode: "shadow",
minLimit: 5,
initialLimit: 20,
maxLimit: 20,
maxQueueCount: 2,
maxQueueCost: 12,
});
const first = await c.acquire(req({ cost: 8 }));
const second = await c.acquire(req({ cost: 8 }));
const third = await c.acquire(req({ cost: 8 }));
assert.equal(first.status, "admitted");
assert.equal(second.status, "admitted");
assert.equal(third.status, "admitted");
c.updateConfig(
baseConfig({
mode: "shadow",
minLimit: 5,
initialLimit: 10,
maxLimit: 10,
maxQueueCount: 1,
maxQueueCost: 8,
})
);
const snap = c.snapshot();
// One active (8), one queued (8), one rejected (queue full under new bounds).
assert.equal(snap.virtualActiveCost, 8);
assert.equal(snap.virtualActiveCount, 1);
assert.equal(snap.virtualQueuedCost, 8);
assert.equal(snap.virtualQueuedCount, 1);
assert.equal(snap.activeCount, 3);
if (first.status === "admitted") first.lease.release();
if (second.status === "admitted") second.lease.release();
if (third.status === "admitted") third.lease.release();
});
});
describe("deterministic overload harness", () => {
async function runEqualServiceWindows(offeredPerWindow: number) {
const clock = new FakeClock();
const c = new AdaptiveAdmissionController(
baseConfig({
mode: "enforce",
initialLimit: 20,
minLimit: 20,
maxLimit: 20,
maxQueueCount: 1,
maxQueueCost: 1,
defaultMaxWaitMs: 20,
}),
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
);
let completed = 0;
let fastRejected = 0;
let active: AdmissionLease[] = [];
for (let window = 0; window < 20; window++) {
for (const lease of active) {
lease.release("success", { latencyMs: 10 });
completed += 1;
}
active = [];
for (let i = 0; i < offeredPerWindow; i++) {
const result = await c.acquire(req({ cost: 5, tenantKey: `tenant-${i % 3}` }));
if (result.status === "admitted") active.push(result.lease);
else if (result.status === "rejected") fastRejected += 1;
else assert.fail("cost-5 excess must reject immediately when queue cost cap is 1");
}
clock.advance(10);
const snapshot = c.snapshot();
assert.ok(snapshot.activeCost <= 20);
assert.ok(snapshot.activeCount <= 4);
assert.equal(snapshot.queuedCost, 0);
assert.equal(snapshot.queuedCount, 0);
}
for (const lease of active) {
lease.release("success", { latencyMs: 10 });
completed += 1;
}
c.shutdown();
assert.equal(clock.pendingTimerCount, 0);
return { completed, fastRejected };
}
it("raises goodput to capacity then plateaus at 2× and 5× offered load", async () => {
// Capacity is 4 admits/window (limit 20, cost 5). Offered loads: 0.5×, 1×, 2×, 5×.
const low = await runEqualServiceWindows(2);
const atCapacity = await runEqualServiceWindows(4);
const doubleOver = await runEqualServiceWindows(8);
const fiveOver = await runEqualServiceWindows(20);
assert.equal(low.completed, 40);
assert.equal(atCapacity.completed, 80);
assert.ok(atCapacity.completed >= low.completed * 1.9, "goodput must rise toward capacity");
assert.equal(
doubleOver.completed,
atCapacity.completed,
"2× offered load must plateau at capacity"
);
assert.equal(
fiveOver.completed,
atCapacity.completed,
"5× offered load must plateau at capacity"
);
assert.equal(low.fastRejected, 0);
assert.equal(atCapacity.fastRejected, 0);
assert.equal(
doubleOver.fastRejected,
4 * 20,
"2× excess rejects immediately with bounded queue"
);
assert.equal(
fiveOver.fastRejected,
16 * 20,
"5× excess rejects immediately with bounded queue"
);
});
});

View File

@@ -0,0 +1,255 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { estimateAdmissionCost } from "../../open-sse/services/admission/cost.ts";
import {
ADMISSION_TOOL_SCAN_BUDGET,
extractAdmissionCostFeatures,
} from "../../open-sse/services/admission/requestFeatures.ts";
describe("bounded request feature extraction", () => {
it("does not call JSON.stringify or toJSON", () => {
let stringifyCalls = 0;
const original = JSON.stringify;
JSON.stringify = ((...args: Parameters<typeof JSON.stringify>) => {
stringifyCalls += 1;
return original.apply(JSON, args as [unknown]);
}) as typeof JSON.stringify;
try {
const body = {
toJSON() {
throw new Error("toJSON must not be invoked");
},
messages: [{ role: "user", content: "hello world" }],
tools: [{ type: "function", function: { name: "x" } }],
n: 3,
stream: true,
};
const features = extractAdmissionCostFeatures(body);
assert.ok((features.bodyBytes ?? 0) > 0);
assert.ok((features.messageCount ?? 0) >= 1);
assert.ok((features.toolCount ?? 0) >= 1);
assert.equal(features.requestedFanout, 3);
assert.equal(features.streaming, true);
assert.ok((features.estimatedInputTokens ?? 0) > 0);
assert.equal(stringifyCalls, 0);
} finally {
JSON.stringify = original;
}
});
it("extracts production-realistic Chat, Responses, Gemini, and Antigravity shapes", () => {
// OpenAI Chat Completions — stream omitted defaults false (higher non-stream class).
const chat = extractAdmissionCostFeatures({
model: "gpt-4o",
messages: [
{ role: "system", content: "You are helpful." },
{ role: "user", content: "Summarize the logs" },
],
tools: [
{
type: "function",
function: { name: "lookup", parameters: { type: "object" } },
},
{
type: "function",
function: { name: "write", parameters: { type: "object" } },
},
],
n: 2,
});
assert.equal(chat.messageCount, 2);
assert.equal(chat.toolCount, 2);
assert.equal(chat.requestedFanout, 2);
assert.equal(chat.streaming, false);
// OpenAI Responses API — string input counts as one item; array counts length.
const responsesString = extractAdmissionCostFeatures({
model: "gpt-4.1",
input: "What is the capital of France?",
tools: [{ type: "web_search_preview" }],
stream: true,
});
assert.equal(responsesString.messageCount, 1);
assert.equal(responsesString.toolCount, 1);
assert.equal(responsesString.streaming, true);
const responsesArray = extractAdmissionCostFeatures({
model: "gpt-4.1",
input: [
{ role: "user", content: [{ type: "input_text", text: "q1" }] },
{ role: "user", content: [{ type: "input_text", text: "q2" }] },
],
stream: false,
n: 3,
});
assert.equal(responsesArray.messageCount, 2);
assert.equal(responsesArray.requestedFanout, 3);
assert.equal(responsesArray.streaming, false);
// Empty string input is not a content item.
const emptyInput = extractAdmissionCostFeatures({ input: "" });
assert.equal(emptyInput.messageCount, 0);
// Gemini generateContent — nested generationConfig.candidateCount + functionDeclarations.
const gemini = extractAdmissionCostFeatures({
contents: [
{ role: "user", parts: [{ text: "hello" }] },
{ role: "model", parts: [{ text: "world" }] },
],
tools: [
{
functionDeclarations: [
{ name: "get_weather", parameters: { type: "OBJECT" } },
{ name: "get_time", parameters: { type: "OBJECT" } },
],
},
],
generationConfig: { candidateCount: 4, temperature: 0.2 },
});
assert.equal(gemini.messageCount, 2);
assert.equal(gemini.toolCount, 2);
assert.equal(gemini.requestedFanout, 4);
assert.equal(gemini.streaming, false);
// Antigravity-style wrapper under `request`.
const antigravity = extractAdmissionCostFeatures({
request: {
contents: [{ role: "user", parts: [{ text: "hi" }] }],
tools: [
{
functionDeclarations: [{ name: "a" }, { name: "b" }, { name: "c" }],
},
],
generationConfig: { candidateCount: 5 },
stream: true,
},
});
assert.equal(antigravity.messageCount, 1);
assert.equal(antigravity.toolCount, 3);
assert.equal(antigravity.requestedFanout, 5);
assert.equal(antigravity.streaming, true);
// Authoritative extraction context wins over body stream inference.
const overridden = extractAdmissionCostFeatures(
{ messages: [{ role: "user", content: "x" }], stream: false },
{ streaming: true }
);
assert.equal(overridden.streaming, true);
const overriddenOff = extractAdmissionCostFeatures(
{ messages: [{ role: "user", content: "x" }], stream: true },
{ streaming: false }
);
assert.equal(overriddenOff.streaming, false);
});
it("bounds tool scans and never touches entries beyond the budget (conservative count)", () => {
// Huge leading string makes estimateSizeFast byte-exit before walking tools,
// so only countTools can touch the tools proxy — proving its scan bound alone.
const sizePad = "x".repeat(300_000);
let accesses = 0;
const tools = new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 10_000;
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
const index = Number(prop);
accesses += 1;
if (index >= ADMISSION_TOOL_SCAN_BUDGET) {
throw new Error(`tool entry ${index} must not be touched`);
}
return { type: "function", function: { name: `t${index}` } };
}
return Reflect.get(target, prop, receiver);
},
});
const features = extractAdmissionCostFeatures({ pad: sizePad, tools });
// Any uninspected tail saturates the feature so heavier unseen entries cannot undercharge.
assert.equal(features.toolCount, Number.MAX_SAFE_INTEGER);
assert.equal(accesses, 0, "known oversized source should saturate before indexed access");
// functionDeclarations length is O(1); truncated tail still cannot undercharge.
let declAccesses = 0;
const geminiTools = new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 50;
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
const index = Number(prop);
declAccesses += 1;
if (index >= ADMISSION_TOOL_SCAN_BUDGET) {
throw new Error(`gemini tool entry ${index} must not be touched`);
}
return {
functionDeclarations: new Proxy([] as unknown[], {
get(t, p, r) {
if (p === "length") return 3;
if (typeof p === "string" && /^[0-9]+$/.test(p)) {
throw new Error("functionDeclarations elements need not be scanned");
}
return Reflect.get(t, p, r);
},
}),
};
}
return Reflect.get(target, prop, receiver);
},
});
const geminiFeatures = extractAdmissionCostFeatures({ pad: sizePad, tools: geminiTools });
assert.equal(geminiFeatures.toolCount, Number.MAX_SAFE_INTEGER);
assert.equal(declAccesses, 0);
let aliasTouches = 0;
const sixtyFour = (label: string) =>
new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET;
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
aliasTouches += 1;
return { name: `${label}-${prop}` };
}
return Reflect.get(target, prop, receiver);
},
});
const aliases = extractAdmissionCostFeatures({
pad: sizePad,
tools: sixtyFour("tool"),
functions: sixtyFour("function"),
});
assert.equal(aliases.toolCount, Number.MAX_SAFE_INTEGER);
assert.ok(aliasTouches <= ADMISSION_TOOL_SCAN_BUDGET);
let wrappedTouches = 0;
const wrappedTail = new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 1;
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
wrappedTouches += 1;
return { functionDeclarations: new Array(1_000).fill(null) };
}
return Reflect.get(target, prop, receiver);
},
});
const layered = extractAdmissionCostFeatures({
pad: sizePad,
tools: [{ type: "function" }],
request: { tools: wrappedTail },
});
assert.equal(layered.toolCount, Number.MAX_SAFE_INTEGER);
assert.equal(estimateAdmissionCost(layered), 1_000);
assert.equal(wrappedTouches, 0);
});
it("nested fanout under request wrapper is visible and stream defaults false", () => {
const features = extractAdmissionCostFeatures({
request: {
messages: [{ role: "user", content: "x" }],
n: 7,
},
});
assert.equal(features.requestedFanout, 7);
assert.equal(features.streaming, false);
assert.equal(features.messageCount, 1);
});
});

View File

@@ -0,0 +1,450 @@
import { describe, it, beforeEach } from "node:test";
import assert from "node:assert/strict";
import {
createAdaptiveAdmissionRuntime,
DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
type AdaptiveAdmissionRuntime,
} from "../../open-sse/services/admission/runtime.ts";
import {
type AdaptiveAdmissionConfig,
type AdmissionLease,
type AdmissionReleaseMeta,
type AdmissionReleaseOutcome,
} from "../../open-sse/services/admission/types.ts";
import type {
ResourcePressureGuardResult,
ResourcePressureObservation,
} from "../../open-sse/utils/resourcePressure.ts";
/** Purpose-built lease spy: counts every release() while exposing released after first call. */
function createSpyLease(id = "spy-lease", cost = 1) {
const calls: Array<{ outcome?: AdmissionReleaseOutcome; meta?: AdmissionReleaseMeta }> = [];
let released = false;
const lease: AdmissionLease = {
id,
cost,
get released() {
return released;
},
release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta) {
calls.push({ outcome, meta });
released = true;
},
};
return {
lease,
calls,
get releaseCount() {
return calls.length;
},
};
}
class FakeClock {
nowMs = 0;
private nextId = 1;
private timers = new Map<number, { due: number; fn: () => void }>();
now = () => this.nowMs;
setTimer = (fn: () => void, delayMs: number): number => {
const id = this.nextId++;
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
return id;
};
clearTimer = (id: number): void => {
this.timers.delete(id);
};
get pendingTimerCount(): number {
return this.timers.size;
}
advance(ms: number): void {
const target = this.nowMs + ms;
while (true) {
let nextId: number | undefined;
let nextDue = Number.POSITIVE_INFINITY;
for (const [id, t] of this.timers) {
if (t.due <= target && t.due < nextDue) {
nextDue = t.due;
nextId = id;
}
}
if (nextId === undefined) {
this.nowMs = target;
return;
}
const timer = this.timers.get(nextId)!;
this.timers.delete(nextId);
this.nowMs = timer.due;
timer.fn();
}
}
}
function emptyObservation(
overrides: Partial<ResourcePressureObservation["state"]> = {}
): ResourcePressureObservation {
return {
signals: null,
state: {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
...overrides,
},
};
}
function makeRuntime(
clock: FakeClock,
overrides: {
config?: AdaptiveAdmissionConfig;
check?: () => ResourcePressureGuardResult | null;
observe?: () => ResourcePressureObservation;
warn?: (message: string) => void;
} = {}
): AdaptiveAdmissionRuntime {
return createAdaptiveAdmissionRuntime({
config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG },
clock: {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
},
checkResourcePressure: overrides.check ?? (() => null),
getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()),
warn: overrides.warn,
});
}
describe("response lifecycle helpers", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
function attachJson(
runtime: AdaptiveAdmissionRuntime,
spy: ReturnType<typeof createSpyLease>,
status: number,
options: { signal?: AbortSignal; admittedAtMs?: number } = {}
) {
const admittedAtMs = options.admittedAtMs ?? clock.nowMs;
return runtime.attachResponseLifecycle(
new Response(JSON.stringify({ ok: status < 400 }), {
status,
headers: { "Content-Type": "application/json" },
}),
spy.lease,
{ admittedAtMs, signal: options.signal, nowMs: clock.now }
);
}
it("classifies non-SSE HTTP outcomes with cancellation winning", async () => {
const runtime = makeRuntime(clock);
const cases: Array<{
status: number;
expected: AdmissionReleaseOutcome;
signal?: AbortSignal;
label: string;
}> = [
{ status: 200, expected: "success", label: "2xx" },
{ status: 302, expected: "success", label: "3xx" },
{ status: 400, expected: "local_reject", label: "ordinary 4xx" },
{ status: 429, expected: "local_reject", label: "429" },
{ status: 408, expected: "timeout", label: "408" },
{ status: 499, expected: "cancelled", label: "499" },
{ status: 504, expected: "timeout", label: "504" },
{ status: 502, expected: "upstream_error", label: "5xx" },
{ status: 500, expected: "upstream_error", label: "500" },
];
for (const c of cases) {
const spy = createSpyLease(`json-${c.label}`);
clock.nowMs = 100;
attachJson(runtime, spy, c.status, { admittedAtMs: 40 });
assert.equal(spy.releaseCount, 1, c.label);
assert.equal(spy.calls[0]!.outcome, c.expected, c.label);
assert.equal(spy.calls[0]!.meta?.latencyMs, 60, c.label);
assert.equal(spy.lease.released, true, c.label);
}
// Already-aborted signal wins over 2xx.
const ac = new AbortController();
ac.abort();
const abortedSpy = createSpyLease("aborted-2xx");
clock.nowMs = 200;
attachJson(runtime, abortedSpy, 200, { signal: ac.signal, admittedAtMs: 150 });
assert.equal(abortedSpy.releaseCount, 1);
assert.equal(abortedSpy.calls[0]!.outcome, "cancelled");
assert.equal(abortedSpy.calls[0]!.meta?.latencyMs, 50);
runtime.dispose();
});
it("classifies SSE completion outcomes using the request signal", async () => {
const runtime = makeRuntime(clock);
let spySuffix = 0;
async function drainSse(
status: number,
signal?: AbortSignal,
expectImmediateRelease = false
): Promise<ReturnType<typeof createSpyLease>> {
const spy = createSpyLease(`sse-${status}-${spySuffix++}`);
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode("data: done\n\n"));
controller.close();
},
});
clock.nowMs = 300;
const wrapped = runtime.attachResponseLifecycle(
new Response(body, {
status,
statusText: "OK",
headers: { "Content-Type": "text/event-stream" },
}),
spy.lease,
{ admittedAtMs: 250, signal, nowMs: clock.now }
);
if (expectImmediateRelease) {
assert.equal(spy.releaseCount, 1);
return spy;
}
assert.equal(spy.releaseCount, 0);
await wrapped.text();
return spy;
}
const ok = await drainSse(200);
assert.equal(ok.releaseCount, 1);
assert.equal(ok.calls[0]!.outcome, "success");
assert.equal(ok.calls[0]!.meta?.latencyMs, 50);
const redirect = await drainSse(302);
assert.equal(redirect.calls[0]!.outcome, "success");
const ordinary4xx = await drainSse(404);
assert.equal(ordinary4xx.calls[0]!.outcome, "local_reject");
const tooMany = await drainSse(429);
assert.equal(tooMany.calls[0]!.outcome, "local_reject");
const requestTimeout = await drainSse(408);
assert.equal(requestTimeout.calls[0]!.outcome, "timeout");
const clientGone = await drainSse(499);
assert.equal(clientGone.calls[0]!.outcome, "cancelled");
const gatewayTimeout = await drainSse(504);
assert.equal(gatewayTimeout.calls[0]!.outcome, "timeout");
const upstream = await drainSse(503);
assert.equal(upstream.calls[0]!.outcome, "upstream_error");
// Already-aborted signal settles immediately as cancelled (wins over 2xx).
const ac = new AbortController();
ac.abort();
const abortedOk = await drainSse(200, ac.signal, true);
assert.equal(abortedOk.releaseCount, 1);
assert.equal(abortedOk.calls[0]!.outcome, "cancelled");
runtime.dispose();
});
it("requires explicit non-success outcomes for handler failures", async () => {
const runtime = makeRuntime(clock);
const outcomes: Array<Exclude<AdmissionReleaseOutcome, "success">> = [
"local_reject",
"upstream_error",
"timeout",
"cancelled",
];
for (const outcome of outcomes) {
const spy = createSpyLease(`handler-${outcome}`);
clock.nowMs = 500;
runtime.releaseHandlerFailure(spy.lease, outcome, {
admittedAtMs: 400,
nowMs: clock.now,
});
assert.equal(spy.releaseCount, 1, outcome);
assert.equal(spy.calls[0]!.outcome, outcome);
assert.equal(spy.calls[0]!.meta?.latencyMs, 100);
// Exactly-once: second call must not re-release.
runtime.releaseHandlerFailure(spy.lease, outcome, {
admittedAtMs: 400,
nowMs: clock.now,
});
assert.equal(spy.releaseCount, 1, `${outcome} second`);
}
runtime.dispose();
});
it("releases JSON/non-SSE responses immediately once", async () => {
const runtime = makeRuntime(clock);
const spy = createSpyLease("json-once");
clock.nowMs = 80;
const wrapped = attachJson(runtime, spy, 200, { admittedAtMs: 20 });
assert.equal(spy.releaseCount, 1);
assert.equal(spy.calls[0]!.outcome, "success");
assert.equal(spy.calls[0]!.meta?.latencyMs, 60);
assert.equal(await wrapped.text(), JSON.stringify({ ok: true }));
runtime.attachResponseLifecycle(
new Response("{}", { status: 200, headers: { "Content-Type": "application/json" } }),
spy.lease,
{ admittedAtMs: 20, nowMs: clock.now }
);
assert.equal(spy.releaseCount, 1);
runtime.dispose();
});
it("keeps SSE lease until stream drain and releases exactly once", async () => {
const runtime = makeRuntime(clock);
const spy = createSpyLease("sse-drain");
let pullCount = 0;
const chunks = [
new TextEncoder().encode("data: 1\n\n"),
new TextEncoder().encode("data: 2\n\n"),
];
const body = new ReadableStream<Uint8Array>({
pull(controller) {
if (pullCount < chunks.length) {
controller.enqueue(chunks[pullCount++]);
return;
}
controller.close();
},
});
clock.nowMs = 120;
const wrapped = runtime.attachResponseLifecycle(
new Response(body, {
status: 200,
statusText: "OK",
headers: { "Content-Type": "text/event-stream" },
}),
spy.lease,
{ admittedAtMs: 100, nowMs: clock.now }
);
assert.equal(spy.releaseCount, 0);
assert.equal(wrapped.status, 200);
assert.equal(wrapped.statusText, "OK");
assert.equal(wrapped.headers.get("Content-Type"), "text/event-stream");
const text = await wrapped.text();
assert.match(text, /data: 1/);
assert.match(text, /data: 2/);
assert.equal(spy.releaseCount, 1);
assert.equal(spy.calls[0]!.outcome, "success");
assert.equal(spy.calls[0]!.meta?.latencyMs, 20);
// Drain again must not re-release (stream already consumed).
runtime.dispose();
});
it("releases once on stream error", async () => {
const runtime = makeRuntime(clock);
const spy = createSpyLease("sse-error");
const body = new ReadableStream<Uint8Array>({
pull(controller) {
controller.error(new Error("upstream boom"));
},
});
clock.nowMs = 90;
const wrapped = runtime.attachResponseLifecycle(
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
spy.lease,
{ admittedAtMs: 70, nowMs: clock.now }
);
await assert.rejects(async () => {
await wrapped.text();
});
assert.equal(spy.releaseCount, 1);
assert.equal(spy.calls[0]!.outcome, "upstream_error");
assert.equal(spy.calls[0]!.meta?.latencyMs, 20);
runtime.dispose();
});
it("releases once on consumer cancel without buffering", async () => {
const runtime = makeRuntime(clock);
const spy = createSpyLease("sse-cancel");
let cancelCount = 0;
let pulled = 0;
const body = new ReadableStream<Uint8Array>({
pull(controller) {
pulled += 1;
controller.enqueue(new TextEncoder().encode(`data: ${pulled}\n\n`));
},
cancel() {
cancelCount += 1;
},
});
clock.nowMs = 60;
const wrapped = runtime.attachResponseLifecycle(
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
spy.lease,
{ admittedAtMs: 10, nowMs: clock.now }
);
const reader = wrapped.body!.getReader();
await reader.read();
assert.equal(spy.releaseCount, 0);
const pulledAfterFirst = pulled;
await reader.cancel("client gone");
assert.equal(cancelCount, 1);
assert.equal(spy.releaseCount, 1);
assert.equal(spy.calls[0]!.outcome, "cancelled");
assert.equal(spy.calls[0]!.meta?.latencyMs, 50);
// Laziness: no full buffering of the infinite producer.
assert.ok(pulledAfterFirst <= 2);
assert.ok(pulled < 20);
// Second cancel is a no-op for both reader cancel and lease release.
await reader.cancel("again");
assert.equal(cancelCount, 1);
assert.equal(spy.releaseCount, 1);
runtime.dispose();
});
it("request abort cancels the reader and releases once under races", async () => {
const runtime = makeRuntime(clock);
const spy = createSpyLease("sse-abort-race");
const ac = new AbortController();
let cancelCount = 0;
const body = new ReadableStream<Uint8Array>({
async pull(controller) {
controller.enqueue(new TextEncoder().encode("data: ping\n\n"));
await new Promise<void>(() => {
/* hang until cancel */
});
},
cancel() {
cancelCount += 1;
},
});
clock.nowMs = 40;
const wrapped = runtime.attachResponseLifecycle(
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
spy.lease,
{ admittedAtMs: 10, signal: ac.signal, nowMs: clock.now }
);
const reader = wrapped.body!.getReader();
const first = reader.read();
ac.abort();
// Race: also cancel consumer.
void reader.cancel("race");
await Promise.race([
first.catch(() => undefined),
new Promise((resolve) => setImmediate(resolve)),
]);
await Promise.resolve();
await Promise.resolve();
assert.equal(spy.releaseCount, 1);
assert.equal(spy.calls[0]!.outcome, "cancelled");
assert.equal(typeof spy.calls[0]!.meta?.latencyMs, "number");
assert.ok((spy.calls[0]!.meta?.latencyMs ?? -1) >= 0);
assert.equal(cancelCount, 1);
runtime.dispose();
});
});

View File

@@ -0,0 +1,67 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { FairCostQueue, type QueueEntry } from "../../open-sse/services/admission/queue.ts";
describe("FairCostQueue removeById cursor preservation", () => {
function qEntry(id: string, tenantKey: string, cost = 1): QueueEntry<{ id: string }> {
return {
id,
tenantKey,
cost,
enqueuedAtMs: 0,
deadlineMs: Number.MAX_SAFE_INTEGER,
payload: { id },
};
}
it("preserves the logical successor when removing a bucket before the cursor", () => {
const q = new FairCostQueue<{ id: string }>(10, 100);
assert.equal(q.enqueue(qEntry("a1", "a")), true);
assert.equal(q.enqueue(qEntry("a2", "a")), true);
assert.equal(q.enqueue(qEntry("b1", "b")), true);
assert.equal(q.enqueue(qEntry("c1", "c")), true);
// Dequeue a1 leaves cursor at b while tenant a still has a2.
assert.equal(q.dequeue()?.id, "a1");
assert.equal(q.removeById("a2")?.id, "a2");
// Successor of the pre-removal cursor must remain b, not skip to c.
assert.equal(q.dequeue()?.id, "b1");
assert.equal(q.dequeue()?.id, "c1");
assert.equal(q.size, 0);
});
it("preserves the logical successor when removing the bucket at the cursor", () => {
const q = new FairCostQueue<{ id: string }>(10, 100);
assert.equal(q.enqueue(qEntry("a1", "a")), true);
assert.equal(q.enqueue(qEntry("b1", "b")), true);
assert.equal(q.enqueue(qEntry("c1", "c")), true);
assert.equal(q.dequeue()?.id, "a1"); // cursor now at b
assert.equal(q.removeById("b1")?.id, "b1");
assert.equal(q.dequeue()?.id, "c1");
assert.equal(q.size, 0);
});
it("preserves the cursor when removing a bucket after the cursor", () => {
const q = new FairCostQueue<{ id: string }>(10, 100);
assert.equal(q.enqueue(qEntry("a1", "a")), true);
assert.equal(q.enqueue(qEntry("b1", "b")), true);
assert.equal(q.enqueue(qEntry("c1", "c")), true);
assert.equal(q.dequeue()?.id, "a1"); // cursor now at b
assert.equal(q.removeById("c1")?.id, "c1");
assert.equal(q.dequeue()?.id, "b1");
assert.equal(q.size, 0);
});
it("resets the cursor when the final bucket is removed", () => {
const q = new FairCostQueue<{ id: string }>(10, 100);
assert.equal(q.enqueue(qEntry("a1", "a")), true);
assert.equal(q.enqueue(qEntry("b1", "b")), true);
assert.equal(q.dequeue()?.id, "a1"); // cursor at b
assert.equal(q.removeById("b1")?.id, "b1");
assert.equal(q.size, 0);
assert.equal(q.enqueue(qEntry("d1", "d")), true);
assert.equal(q.dequeue()?.id, "d1");
});
});

View File

@@ -0,0 +1,857 @@
import { describe, it, beforeEach, afterEach } from "node:test";
import assert from "node:assert/strict";
import {
createAdaptiveAdmissionRuntime,
getAdaptiveAdmissionRuntime,
reloadAdaptiveAdmissionRuntime,
resetAdaptiveAdmissionRuntimeForTests,
resolveAdaptiveAdmissionConfigFromEnv,
DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
type AdaptiveAdmissionRuntime,
} from "../../open-sse/services/admission/runtime.ts";
import {
MAX_ADMISSION_COST_OR_LIMIT,
MAX_ADMISSION_WINDOW_MS,
type AdaptiveAdmissionConfig,
} from "../../open-sse/services/admission/types.ts";
import type {
ResourcePressureGuardResult,
ResourcePressureObservation,
} from "../../open-sse/utils/resourcePressure.ts";
import { buildErrorBody } from "../../open-sse/utils/error.ts";
class FakeClock {
nowMs = 0;
private nextId = 1;
private timers = new Map<number, { due: number; fn: () => void }>();
now = () => this.nowMs;
setTimer = (fn: () => void, delayMs: number): number => {
const id = this.nextId++;
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
return id;
};
clearTimer = (id: number): void => {
this.timers.delete(id);
};
get pendingTimerCount(): number {
return this.timers.size;
}
advance(ms: number): void {
const target = this.nowMs + ms;
while (true) {
let nextId: number | undefined;
let nextDue = Number.POSITIVE_INFINITY;
for (const [id, t] of this.timers) {
if (t.due <= target && t.due < nextDue) {
nextDue = t.due;
nextId = id;
}
}
if (nextId === undefined) {
this.nowMs = target;
return;
}
const timer = this.timers.get(nextId)!;
this.timers.delete(nextId);
this.nowMs = timer.due;
timer.fn();
}
}
}
function enforceConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
return {
mode: "enforce",
minLimit: 4,
maxLimit: 20,
initialLimit: 8,
maxQueueCount: 2,
maxQueueCost: 16,
defaultMaxWaitMs: 100,
windowMs: 50,
...overrides,
};
}
function emptyObservation(
overrides: Partial<ResourcePressureObservation["state"]> = {}
): ResourcePressureObservation {
return {
signals: null,
state: {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
...overrides,
},
};
}
function criticalGuard(reason = "v8_heap_absolute"): ResourcePressureGuardResult {
const message = "Service temporarily unavailable due to resource pressure. Retry shortly.";
return {
success: false,
status: 503,
error: message,
response: new Response(
JSON.stringify(
buildErrorBody(503, message, undefined, {
type: "server_error",
code: "resource_pressure",
})
),
{
status: 503,
headers: { "Content-Type": "application/json", "Retry-After": "5" },
}
),
};
}
function makeRuntime(
clock: FakeClock,
overrides: {
config?: AdaptiveAdmissionConfig;
check?: () => ResourcePressureGuardResult | null;
observe?: () => ResourcePressureObservation;
warn?: (message: string) => void;
} = {}
): AdaptiveAdmissionRuntime {
return createAdaptiveAdmissionRuntime({
config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG },
clock: {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
},
checkResourcePressure: overrides.check ?? (() => null),
getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()),
warn: overrides.warn,
});
}
async function parseJson(response: Response): Promise<Record<string, unknown>> {
return JSON.parse(await response.text()) as Record<string, unknown>;
}
describe("adaptive admission runtime env + defaults", () => {
it("defaults to complete shadow config", () => {
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.mode, "shadow");
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.minLimit, 8);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.initialLimit, 64);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxLimit, 1000);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCount, 128);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCost, 2000);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.defaultMaxWaitMs, 5000);
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.windowMs, 1000);
});
it("strictly resolves supported env names and rejects invalid values", () => {
const cfg = resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MODE: "enforce",
ADAPTIVE_ADMISSION_MIN_LIMIT: "10",
ADAPTIVE_ADMISSION_INITIAL_LIMIT: "20",
ADAPTIVE_ADMISSION_MAX_LIMIT: "30",
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "40",
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: "50",
ADAPTIVE_ADMISSION_MAX_WAIT_MS: "600",
ADAPTIVE_ADMISSION_WINDOW_MS: "700",
});
assert.deepEqual(
{
mode: cfg.mode,
minLimit: cfg.minLimit,
initialLimit: cfg.initialLimit,
maxLimit: cfg.maxLimit,
maxQueueCount: cfg.maxQueueCount,
maxQueueCost: cfg.maxQueueCost,
defaultMaxWaitMs: cfg.defaultMaxWaitMs,
windowMs: cfg.windowMs,
},
{
mode: "enforce",
minLimit: 10,
initialLimit: 20,
maxLimit: 30,
maxQueueCount: 40,
maxQueueCost: 50,
defaultMaxWaitMs: 600,
windowMs: 700,
}
);
assert.throws(
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MODE: "strict" }),
/ADAPTIVE_ADMISSION_MODE/
);
assert.throws(
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MIN_LIMIT: "0" }),
/ADAPTIVE_ADMISSION_MIN_LIMIT/
);
assert.throws(
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MAX_LIMIT: "1.5" }),
/ADAPTIVE_ADMISSION_MAX_LIMIT/
);
});
it("accepts exact documented maxima and rejects max+1 plus cross-field invalidity", () => {
const maxCost = String(MAX_ADMISSION_COST_OR_LIMIT);
const maxWindow = String(MAX_ADMISSION_WINDOW_MS);
const maxQueue = String(Number.MAX_SAFE_INTEGER);
const atMaxima = resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MODE: "shadow",
ADAPTIVE_ADMISSION_MIN_LIMIT: "1",
ADAPTIVE_ADMISSION_INITIAL_LIMIT: maxCost,
ADAPTIVE_ADMISSION_MAX_LIMIT: maxCost,
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: maxQueue,
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: maxCost,
ADAPTIVE_ADMISSION_MAX_WAIT_MS: maxWindow,
ADAPTIVE_ADMISSION_WINDOW_MS: maxWindow,
});
assert.equal(atMaxima.maxLimit, MAX_ADMISSION_COST_OR_LIMIT);
assert.equal(atMaxima.maxQueueCount, Number.MAX_SAFE_INTEGER);
assert.equal(atMaxima.windowMs, MAX_ADMISSION_WINDOW_MS);
assert.equal(atMaxima.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS);
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MAX_LIMIT: String(MAX_ADMISSION_COST_OR_LIMIT + 1),
}),
/maxLimit|must be <=/
);
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: String(MAX_ADMISSION_COST_OR_LIMIT + 1),
}),
/maxQueueCost|must be <=/
);
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_WINDOW_MS: String(MAX_ADMISSION_WINDOW_MS + 1),
}),
/windowMs|must be <=/
);
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MAX_WAIT_MS: String(MAX_ADMISSION_WINDOW_MS + 1),
}),
/defaultMaxWaitMs|must be <=/
);
// Queue count uses full safe-integer range; beyond that fails lexical/safe-integer parsing.
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "9007199254740992",
}),
/ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT|safe integer/
);
assert.throws(
() =>
resolveAdaptiveAdmissionConfigFromEnv({
ADAPTIVE_ADMISSION_MIN_LIMIT: "20",
ADAPTIVE_ADMISSION_MAX_LIMIT: "10",
}),
/minLimit must be <= maxLimit/
);
});
it("default process runtime falls back to shadow on invalid env without crashing", () => {
resetAdaptiveAdmissionRuntimeForTests();
const warnings: string[] = [];
const previous = process.env.ADAPTIVE_ADMISSION_MODE;
process.env.ADAPTIVE_ADMISSION_MODE = "not-a-mode";
try {
const runtime = reloadAdaptiveAdmissionRuntime({
warn: (message) => warnings.push(message),
checkResourcePressure: () => null,
getResourcePressureObservation: () => emptyObservation(),
});
const snap = runtime.snapshot();
assert.equal(snap.mode, "shadow");
assert.equal(snap.minLimit, 8);
assert.equal(snap.initialLimit ?? snap.currentLimit >= 8, true);
assert.equal(warnings.length, 1);
assert.match(
warnings[0]!,
/invalid environment configuration; using default shadow admission settings/
);
assert.ok(!warnings.join("\n").includes("not-a-mode"));
assert.ok(!warnings.join("\n").toLowerCase().includes("secret"));
runtime.dispose();
} finally {
if (previous === undefined) delete process.env.ADAPTIVE_ADMISSION_MODE;
else process.env.ADAPTIVE_ADMISSION_MODE = previous;
resetAdaptiveAdmissionRuntimeForTests();
}
});
});
describe("adaptive admission runtime modes", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
afterEach(() => {
resetAdaptiveAdmissionRuntimeForTests();
});
it("default shadow always admits with a real lease and shadowDecision", async () => {
const runtime = makeRuntime(clock);
const result = await runtime.acquire({
tenantKey: "tenant-secret-1",
body: { messages: [{ role: "user", content: "hi" }], stream: true },
});
assert.equal(result.status, "admitted");
if (result.status !== "admitted") throw new Error("expected admitted");
assert.equal(result.mode, "shadow");
assert.ok(result.lease);
assert.equal(typeof result.lease.release, "function");
assert.equal(result.lease.released, false);
assert.ok(
result.shadowDecision === "would-admit" ||
result.shadowDecision === "would-queue" ||
result.shadowDecision === "would-reject"
);
result.lease.release("success");
assert.equal(result.lease.released, true);
result.lease.release("success");
runtime.dispose();
});
it("explicit off admits without enforcing capacity", async () => {
const runtime = makeRuntime(clock, {
config: {
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
mode: "off",
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
},
});
const a = await runtime.acquire({ tenantKey: "t1", body: { messages: [] } });
const b = await runtime.acquire({ tenantKey: "t2", body: { messages: [] } });
assert.equal(a.status, "admitted");
assert.equal(b.status, "admitted");
if (a.status === "admitted") a.lease.release();
if (b.status === "admitted") b.lease.release();
runtime.dispose();
});
it("explicit enforce can reject with sanitized HTTP response", async () => {
const runtime = makeRuntime(clock, {
config: enforceConfig({
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
maxQueueCount: 1,
maxQueueCost: 1,
defaultMaxWaitMs: 50,
cost: { maxRequestCost: 1, baseCost: 1 },
}),
});
const first = await runtime.acquire({
tenantKey: "t1",
body: { messages: [{ role: "user", content: "a" }], stream: true },
});
assert.equal(first.status, "admitted");
const secondPromise = runtime.acquire({
tenantKey: "t2",
body: { messages: [{ role: "user", content: "b" }], stream: true },
maxWaitMs: 50,
});
// Drive injected deadline timer; no wall-clock sleeps.
clock.advance(50);
const second = await secondPromise;
assert.equal(second.status, "rejected");
if (second.status !== "rejected") throw new Error("expected rejected");
assert.equal(second.response.status, 503);
const body = await parseJson(second.response);
assert.equal(typeof body.error.message, "string");
assert.match(second.code, /^admission_/);
assert.ok(!JSON.stringify(body).includes("t2"));
assert.ok(!JSON.stringify(body).includes("tenant"));
if (first.status === "admitted") first.lease.release();
runtime.dispose();
});
});
describe("runtime streaming cost forwarding", () => {
it("acquire lease cost reflects input.streaming via feature extraction", async () => {
const clock = new FakeClock();
// Sharply distinct streaming class costs; neutralize other feature contributions.
const runtime = makeRuntime(clock, {
config: {
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
mode: "shadow",
cost: {
baseCost: 1,
bodyBytesPerUnit: 1_000_000,
tokensPerUnit: 1_000_000,
messagesPerUnit: 1_000_000,
toolsPerUnit: 1_000_000,
fanoutPerUnit: 1_000_000,
streamingClassCost: 1,
nonStreamingClassCost: 50,
maxRequestCost: 1_000,
},
},
});
// Empty body keeps non-class contributions identical; stream omitted defaults false when not forwarded.
const body = {};
const streamed = await runtime.acquire({
tenantKey: "stream-on",
body,
streaming: true,
});
assert.equal(streamed.status, "admitted");
if (streamed.status !== "admitted") throw new Error("expected admitted");
const streamCost = streamed.lease.cost;
streamed.lease.release("success");
const nonStreamed = await runtime.acquire({
tenantKey: "stream-off",
body,
streaming: false,
});
assert.equal(nonStreamed.status, "admitted");
if (nonStreamed.status !== "admitted") throw new Error("expected admitted");
const nonStreamCost = nonStreamed.lease.cost;
nonStreamed.lease.release("success");
const defaulted = await runtime.acquire({
tenantKey: "stream-default",
body,
});
assert.equal(defaulted.status, "admitted");
if (defaulted.status !== "admitted") throw new Error("expected admitted");
const defaultCost = defaulted.lease.cost;
defaulted.lease.release("success");
runtime.dispose();
// base(1) + fanout unit(1) + class cost → streaming 3, non-streaming 52
assert.equal(streamCost, 3);
assert.equal(nonStreamCost, 52);
assert.equal(defaultCost, 52);
assert.notEqual(
streamCost,
nonStreamCost,
"streaming true/false must produce different acquired lease costs"
);
});
});
describe("rejection mapping", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
it("maps ADMISSION_ABORTED to local 499 without Retry-After", async () => {
const runtime = makeRuntime(clock, {
config: enforceConfig({
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 1000,
// Force unit cost so one admitted request fills the limit.
cost: { maxRequestCost: 1, baseCost: 1 },
}),
});
const holder = await runtime.acquire({
tenantKey: "hold",
body: { messages: [{ role: "user", content: "hold" }], stream: true },
});
assert.equal(holder.status, "admitted");
const ac = new AbortController();
const pending = runtime.acquire({
tenantKey: "wait",
body: { messages: [{ role: "user", content: "wait" }], stream: true },
signal: ac.signal,
maxWaitMs: 1000,
});
ac.abort();
const rejected = await pending;
assert.equal(rejected.status, "rejected");
if (rejected.status !== "rejected") throw new Error("expected rejected");
assert.equal(rejected.code, "admission_aborted");
assert.equal(rejected.response.status, 499);
assert.equal(rejected.response.headers.get("Retry-After"), null);
const body = await parseJson(rejected.response);
assert.equal(body.error.code, "admission_aborted");
assert.ok(!JSON.stringify(body).includes("wait"));
if (holder.status === "admitted") holder.lease.release();
runtime.dispose();
});
it("maps queue full / deadline / oversized to sanitized 503 codes", async () => {
const runtime = makeRuntime(clock, {
config: enforceConfig({
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
maxQueueCount: 1,
maxQueueCost: 1,
defaultMaxWaitMs: 20,
cost: { maxRequestCost: 1, baseCost: 1, bodyBytesPerUnit: 1_000_000 },
}),
});
const hold = await runtime.acquire({
tenantKey: "hold",
body: { stream: true },
});
assert.equal(hold.status, "admitted");
const deadlinePromise = runtime.acquire({
tenantKey: "q1",
body: { stream: true },
maxWaitMs: 20,
});
clock.advance(20);
const deadlineRejected = await deadlinePromise;
assert.equal(deadlineRejected.status, "rejected");
if (deadlineRejected.status === "rejected") {
assert.equal(deadlineRejected.response.status, 503);
assert.equal(deadlineRejected.code, "admission_deadline");
const body = await parseJson(deadlineRejected.response);
assert.equal(body.error.code, "admission_deadline");
assert.equal(deadlineRejected.response.headers.get("Retry-After"), "1");
}
// Fill the single queue slot then force queue_full on the next arrival.
const waiterPromise = runtime.acquire({
tenantKey: "waiter",
body: { stream: true },
maxWaitMs: 1_000,
});
const full = await runtime.acquire({
tenantKey: "full",
body: { stream: true },
});
assert.equal(full.status, "rejected");
if (full.status === "rejected") {
assert.equal(full.code, "admission_queue_full");
assert.equal(full.response.status, 503);
assert.equal(full.response.headers.get("Retry-After"), "1");
const body = await parseJson(full.response);
assert.equal(body.error.code, "admission_queue_full");
}
clock.advance(1_000);
await waiterPromise;
// Oversized: cost features that exceed limit 1 with tiny max.
const oversizedRuntime = makeRuntime(clock, {
config: enforceConfig({
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
maxQueueCount: 1,
maxQueueCost: 1,
cost: {
maxRequestCost: 100,
baseCost: 1,
bodyBytesPerUnit: 1,
tokensPerUnit: 1,
messagesPerUnit: 1,
toolsPerUnit: 1,
fanoutPerUnit: 1,
},
}),
});
const huge = await oversizedRuntime.acquire({
tenantKey: "huge",
body: {
messages: Array.from({ length: 50 }, (_, i) => ({
role: "user",
content: `m${i}-${"x".repeat(32)}`,
})),
stream: true,
},
});
assert.equal(huge.status, "rejected");
if (huge.status === "rejected") {
assert.equal(huge.code, "admission_oversized");
assert.equal(huge.response.status, 503);
const body = await parseJson(huge.response);
assert.equal(body.error.code, "admission_oversized");
assert.ok(!JSON.stringify(body).toLowerCase().includes("cost"));
assert.ok(!JSON.stringify(body).includes("huge"));
}
if (hold.status === "admitted") hold.lease.release();
runtime.dispose();
oversizedRuntime.dispose();
});
});
describe("resource pressure integration", () => {
let clock: FakeClock;
beforeEach(() => {
clock = new FakeClock();
});
it("returns the existing critical guard response without acquiring work", async () => {
let acquires = 0;
const guard = criticalGuard();
const runtime = makeRuntime(clock, {
config: enforceConfig({ initialLimit: 10 }),
check: () => {
acquires += 1;
return guard;
},
});
const result = await runtime.acquire({
tenantKey: "t-pressure",
body: { messages: [{ role: "user", content: "x" }] },
});
assert.equal(result.status, "rejected");
if (result.status !== "rejected") throw new Error("expected rejected");
assert.equal(result.response, guard.response);
assert.equal(result.code, "resource_pressure");
assert.equal(runtime.snapshot().pressureGuardRejectCount, 1);
assert.equal(runtime.snapshot().activeCount, 0);
assert.equal(acquires, 1);
runtime.dispose();
});
it("feeds fresh critical pressure observation even when the safety guard rejects", async () => {
const guard = criticalGuard();
let observation = emptyObservation({
severity: "critical",
reason: "v8_heap_absolute",
observedAtMs: 1_000,
});
const pressures: string[] = [];
const runtime = createAdaptiveAdmissionRuntime({
config: enforceConfig({
initialLimit: 20,
minLimit: 4,
maxLimit: 20,
windowMs: 50,
criticalDecreaseFactor: 0.5,
}),
clock: {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
},
checkResourcePressure: () => guard,
getResourcePressureObservation: () => observation,
onPressureObserved: (pressure) => pressures.push(pressure),
});
const first = await runtime.acquire({
tenantKey: "guarded",
body: { messages: [{ role: "user", content: "x" }] },
});
assert.equal(first.status, "rejected");
if (first.status !== "rejected") throw new Error("expected rejected");
// Exact same guard response identity; zero controller acquisition.
assert.equal(first.response, guard.response);
assert.equal(first.code, "resource_pressure");
assert.equal(runtime.snapshot().activeCount, 0);
assert.deepEqual(pressures, ["critical"]);
// One critical reduction: floor(20 * 0.5) = 10.
assert.equal(runtime.snapshot().currentLimit, 10);
// Replay same observation: no additional feed or reduction.
const second = await runtime.acquire({
tenantKey: "guarded-2",
body: { messages: [{ role: "user", content: "y" }] },
});
assert.equal(second.response, guard.response);
assert.deepEqual(pressures, ["critical"]);
assert.equal(runtime.snapshot().currentLimit, 10);
// New window resets criticalDecreaseConsumed; fresh observation may reduce again.
clock.advance(50);
observation = emptyObservation({
severity: "critical",
reason: "v8_heap_absolute",
observedAtMs: 2_000,
});
const third = await runtime.acquire({
tenantKey: "guarded-3",
body: { messages: [{ role: "user", content: "z" }] },
});
assert.equal(third.response, guard.response);
assert.deepEqual(pressures, ["critical", "critical"]);
assert.equal(runtime.snapshot().currentLimit, 5);
runtime.dispose();
});
it("dedupes unchanged observations and re-feeds genuinely fresh ones", async () => {
let observation = emptyObservation({
severity: "high",
reason: "psi_some",
observedAtMs: 100,
});
const pressures: string[] = [];
const runtime = createAdaptiveAdmissionRuntime({
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
clock: {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
},
checkResourcePressure: () => null,
getResourcePressureObservation: () => observation,
onPressureObserved: (pressure) => pressures.push(pressure),
});
await runtime.acquire({ tenantKey: "a", body: {} });
await runtime.acquire({ tenantKey: "b", body: {} });
assert.deepEqual(pressures, ["high"]);
observation = emptyObservation({
severity: "high",
reason: "psi_some",
observedAtMs: 100,
});
await runtime.acquire({ tenantKey: "c", body: {} });
assert.deepEqual(pressures, ["high"]);
observation = emptyObservation({
severity: "critical",
reason: "psi_full",
observedAtMs: 200,
});
await runtime.acquire({ tenantKey: "d", body: {} });
assert.deepEqual(pressures, ["high", "critical"]);
runtime.dispose();
});
it("fails open when pressure check or observation throws", async () => {
const runtime = makeRuntime(clock, {
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
check: () => {
throw new Error("check boom");
},
observe: () => {
throw new Error("observe boom");
},
});
const result = await runtime.acquire({ tenantKey: "t", body: { messages: [] } });
assert.equal(result.status, "admitted");
if (result.status === "admitted") result.lease.release();
runtime.dispose();
});
});
describe("public snapshot privacy", () => {
it("exposes only aggregate counters and low-cardinality resource fields", async () => {
const clock = new FakeClock();
const runtime = makeRuntime(clock, {
observe: () =>
emptyObservation({
severity: "high",
reason: "cgroup_ratio",
observedAtMs: 42,
}),
});
await runtime.acquire({
tenantKey: "tenant-very-secret",
body: {
messages: [{ role: "user", content: "SECRET_PAYLOAD_XYZ" }],
api_key: "sk-live-secret",
},
});
const snap = runtime.snapshot();
const text = JSON.stringify(snap);
assert.ok(!text.includes("tenant-very-secret"));
assert.ok(!text.includes("SECRET_PAYLOAD_XYZ"));
assert.ok(!text.includes("sk-live-secret"));
assert.ok(!text.includes("lease-"));
assert.equal(typeof snap.mode, "string");
assert.equal(typeof snap.currentLimit, "number");
assert.equal(typeof snap.activeCount, "number");
assert.equal(snap.resourceSeverity, "high");
assert.equal(snap.resourceReason, "cgroup_ratio");
assert.equal(snap.resourceObservedAtMs, 42);
assert.equal(typeof snap.pressureGuardRejectCount, "number");
const snapRecord = snap as unknown as Record<string, unknown>;
assert.equal(snapRecord.tenants, undefined);
assert.equal(snapRecord.queue, undefined);
assert.equal(snapRecord.features, undefined);
runtime.dispose();
});
});
describe("process runtime reload isolation", () => {
afterEach(() => {
resetAdaptiveAdmissionRuntimeForTests();
});
it("reload disposes previous queued work/timers and replaces the process runtime", async () => {
resetAdaptiveAdmissionRuntimeForTests();
const clock = new FakeClock();
const first = reloadAdaptiveAdmissionRuntime({
config: enforceConfig({
initialLimit: 1,
minLimit: 1,
maxLimit: 1,
maxQueueCount: 4,
maxQueueCost: 40,
defaultMaxWaitMs: 5_000,
windowMs: 1_000,
cost: { maxRequestCost: 1, baseCost: 1 },
}),
clock: {
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
},
checkResourcePressure: () => null,
getResourcePressureObservation: () => emptyObservation(),
});
const hold = await first.acquire({
tenantKey: "hold",
body: { messages: [{ role: "user", content: "h" }], stream: true },
});
assert.equal(hold.status, "admitted");
assert.ok(clock.pendingTimerCount >= 1);
const waiting = first.acquire({
tenantKey: "waiter",
body: { messages: [{ role: "user", content: "w" }], stream: true },
maxWaitMs: 5_000,
});
const second = reloadAdaptiveAdmissionRuntime({
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
checkResourcePressure: () => null,
getResourcePressureObservation: () => emptyObservation(),
});
assert.notEqual(second, first);
assert.equal(getAdaptiveAdmissionRuntime(), second);
const rejected = await waiting;
assert.equal(rejected.status, "rejected");
if (rejected.status === "rejected") {
assert.equal(rejected.code, "admission_shutdown");
}
// Previous timers should be cleared by dispose/shutdown.
assert.equal(clock.pendingTimerCount, 0);
second.dispose();
resetAdaptiveAdmissionRuntimeForTests();
});
});

View File

@@ -1,9 +1,12 @@
import test from "node:test";
import assert from "node:assert/strict";
const { estimateSizeFast, isSmallEnoughForSemanticCache } = await import(
"../../open-sse/utils/estimateSize.ts"
);
const {
estimateSizeFast,
isSmallEnoughForSemanticCache,
ESTIMATE_SIZE_BYTE_LIMIT,
ESTIMATE_SIZE_NODE_BUDGET,
} = await import("../../open-sse/utils/estimateSize.ts");
test("estimateSizeFast returns 0 for null/undefined", () => {
assert.equal(estimateSizeFast(null), 0);
@@ -65,6 +68,22 @@ test("estimateSizeFast early-exits at 262144 bytes (256KB)", () => {
assert.ok(result >= 262144, `Should early-exit, got ${result}`);
});
test("estimateSizeFast checks byte limit after numbers and booleans", () => {
const almostForNumber = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 4);
const withNumber = estimateSizeFast([almostForNumber, 1]);
assert.ok(
withNumber > ESTIMATE_SIZE_BYTE_LIMIT,
`number contribution must trip byte limit, got ${withNumber}`
);
// boolean is 4 bytes: start 3 under the limit so adding true exceeds (not merely equals).
const almostForBool = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 3);
const withBool = estimateSizeFast([almostForBool, true]);
assert.ok(
withBool > ESTIMATE_SIZE_BYTE_LIMIT,
`boolean contribution must trip byte limit, got ${withBool}`
);
});
test("estimateSizeFast handles mixed object/array nesting", () => {
const data = {
choices: [
@@ -109,3 +128,70 @@ test("estimateSizeFast handles Map-like objects (no infinite loop on iterables)"
const result = estimateSizeFast(map);
assert.ok(typeof result === "number");
});
/**
* Mutation-sensitive bound: a huge logical length with null/empty-object elements
* must not pre-touch every index or allocate all references. Node-budget exhaustion
* fails closed above 256 KiB so semantic-cache/admission never treat it as small.
*/
test("estimateSizeFast node budget fails closed on huge sparse null array without full traversal", () => {
let elementAccesses = 0;
const sparseNulls = new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return 5_000_000;
if (prop === Symbol.iterator) {
throw new Error("iterator must not be used");
}
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
elementAccesses += 1;
return null;
}
return Reflect.get(target, prop, receiver);
},
});
const result = estimateSizeFast(sparseNulls);
assert.ok(
result > ESTIMATE_SIZE_BYTE_LIMIT,
`node-budget exhaustion must return >256KiB, got ${result}`
);
assert.ok(
elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8,
`must not access far beyond node budget; accesses=${elementAccesses}`
);
assert.ok(elementAccesses > 100, `expected many bounded visits, got ${elementAccesses}`);
assert.equal(isSmallEnoughForSemanticCache(sparseNulls), false);
});
test("estimateSizeFast node budget fails closed on empty-object / getter proxy array", () => {
let elementAccesses = 0;
let farGetterHits = 0;
const emptyObjectArray = new Proxy([] as unknown[], {
get(target, prop, receiver) {
if (prop === "length") return 2_000_000;
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
const index = Number(prop);
elementAccesses += 1;
if (index >= ESTIMATE_SIZE_NODE_BUDGET) {
farGetterHits += 1;
}
// Fresh empty object per access — old impl would stack-push every reference.
return {};
}
return Reflect.get(target, prop, receiver);
},
});
const result = estimateSizeFast(emptyObjectArray);
assert.ok(result > ESTIMATE_SIZE_BYTE_LIMIT, `expected fail-closed, got ${result}`);
assert.ok(
elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8,
`accesses must stay near node budget; got ${elementAccesses}`
);
assert.equal(
farGetterHits,
0,
`entries beyond the node budget must not be touched; far hits=${farGetterHits}`
);
assert.equal(isSmallEnoughForSemanticCache(emptyObjectArray), false);
});

View File

@@ -0,0 +1,202 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
createResourcePressureTracker,
resolveResourcePressureThresholds,
type PressureReason,
type PressureSeverity,
type ResourcePressureState,
type ResourcePressureThresholds,
type ResourceSignals,
} from "../../open-sse/utils/resourcePressurePolicy.ts";
const MiB = 1024 ** 2;
function baseSignals(overrides: Partial<ResourceSignals> = {}): ResourceSignals {
return {
observedAtMs: 1_000,
v8: { heapUsedBytes: 100 * MiB, heapLimitBytes: 1_000 * MiB },
process: {
rssBytes: 200 * MiB,
externalBytes: 10 * MiB,
arrayBuffersBytes: MiB,
availableBytes: null,
constrainedBytes: null,
},
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
psi: null,
...overrides,
};
}
const fastThresholds: Partial<ResourcePressureThresholds> = {
highRatio: 0.8,
criticalRatio: 0.9,
recoveryRatio: 0.7,
highPsiAvg10: 20,
criticalPsiAvg10: 40,
recoveryPsiAvg10: 10,
sustainedSamplesHigh: 2,
sustainedSamplesCritical: 2,
sustainedSamplesRecovery: 2,
heapAbsoluteThresholdMb: null,
};
describe("resource pressure threshold validation", () => {
it("accepts every valid boundary", () => {
const thresholds = resolveResourcePressureThresholds({
recoveryRatio: 0,
highRatio: 0.5,
criticalRatio: 1,
recoveryPsiAvg10: 0,
highPsiAvg10: 50,
criticalPsiAvg10: 100,
sustainedSamplesHigh: 1,
sustainedSamplesCritical: 1,
sustainedSamplesRecovery: 10_000,
heapAbsoluteThresholdMb: null,
});
assert.equal(thresholds.recoveryRatio, 0);
assert.equal(thresholds.criticalRatio, 1);
assert.equal(thresholds.criticalPsiAvg10, 100);
assert.equal(thresholds.sustainedSamplesRecovery, 10_000);
assert.equal(thresholds.heapAbsoluteThresholdMb, null);
});
it("throws deterministically for invalid partial overrides", () => {
const invalid: Array<Partial<ResourcePressureThresholds>> = [
{ recoveryRatio: -0.01 },
{ criticalRatio: 1.01 },
{ highRatio: Number.NaN },
{ recoveryRatio: 0.8, highRatio: 0.8 },
{ highRatio: 0.95, criticalRatio: 0.9 },
{ recoveryPsiAvg10: -1 },
{ criticalPsiAvg10: 101 },
{ highPsiAvg10: Number.POSITIVE_INFINITY },
{ recoveryPsiAvg10: 20, highPsiAvg10: 20 },
{ highPsiAvg10: 50, criticalPsiAvg10: 40 },
{ sustainedSamplesHigh: 0 },
{ sustainedSamplesCritical: 1.5 },
{ sustainedSamplesRecovery: 10_001 },
{ heapAbsoluteThresholdMb: 0 },
{ heapAbsoluteThresholdMb: Number.POSITIVE_INFINITY },
];
for (const partial of invalid) {
assert.throws(() => resolveResourcePressureThresholds(partial), RangeError);
}
});
});
describe("resource pressure policy", () => {
it("does not let high then critical count as two critical samples", () => {
const tracker = createResourcePressureTracker(fastThresholds);
const high = baseSignals({
v8: { heapUsedBytes: 850 * MiB, heapLimitBytes: 1_000 * MiB },
});
const critical = baseSignals({
v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB },
});
assert.equal(tracker.observe(high).severity, "normal");
assert.equal(tracker.observe(critical).severity, "normal");
assert.equal(tracker.observe(critical).severity, "critical");
});
it("resets pending streak when severity or reason alternates", () => {
const tracker = createResourcePressureTracker(fastThresholds);
const heapCritical = baseSignals({
v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB },
});
const psiCritical = baseSignals({
psi: {
someAvg10: 50,
someAvg60: null,
someAvg300: null,
fullAvg10: null,
fullAvg60: null,
fullAvg300: null,
},
});
assert.equal(tracker.observe(heapCritical).severity, "normal");
assert.equal(tracker.observe(psiCritical).severity, "normal");
assert.equal(tracker.observe(psiCritical).severity, "critical");
assert.equal(tracker.getState().reason, "psi_some");
});
it("baselines cumulative OOM counters and only treats increases as events", () => {
const tracker = createResourcePressureTracker(fastThresholds);
const oomCounters = (oom: number, oom_kill: number, observedAtMs: number) =>
baseSignals({
observedAtMs,
cgroup: {
currentBytes: null,
maxBytes: null,
highBytes: null,
events: { low: 0, high: 0, max: 0, oom, oom_kill },
},
});
assert.equal(tracker.observe(oomCounters(7, 3, 1)).severity, "normal", "history baselines");
assert.equal(tracker.observe(oomCounters(7, 3, 2)).severity, "normal", "unchanged history");
const event = tracker.observe(oomCounters(8, 3, 3));
assert.equal(event.severity, "critical", "a new OOM event is immediately critical");
assert.equal(event.reason, "oom_event");
assert.equal(tracker.observe(oomCounters(8, 3, 4)).severity, "critical");
assert.equal(
tracker.observe(oomCounters(8, 3, 5)).severity,
"normal",
"unchanged allows recovery"
);
});
it("re-baselines when OOM counters reset or the cgroup event source is replaced", () => {
const tracker = createResourcePressureTracker(fastThresholds);
const events = (oom: number, oom_kill: number) =>
baseSignals({
cgroup: {
currentBytes: null,
maxBytes: null,
highBytes: null,
events: { low: 0, high: 0, max: 0, oom, oom_kill },
},
});
assert.equal(tracker.observe(events(10, 4)).severity, "normal");
assert.equal(tracker.observe(events(1, 0)).severity, "normal", "counter reset re-baselines");
assert.equal(
tracker.observe({ ...events(1, 0), cgroup: { ...events(1, 0).cgroup, events: null } })
.severity,
"normal"
);
assert.equal(tracker.observe(events(9, 3)).severity, "normal", "replacement re-baselines");
});
it("keeps snapshot state fields and bounded-cardinality values", () => {
const tracker = createResourcePressureTracker(fastThresholds);
const state: ResourcePressureState = tracker.observe(baseSignals());
const severities = new Set<PressureSeverity>(["normal", "high", "critical"]);
const reasons = new Set<PressureReason>([
"none",
"v8_heap_ratio",
"v8_heap_absolute",
"cgroup_ratio",
"cgroup_high",
"psi_some",
"psi_full",
"oom_event",
]);
assert.ok(severities.has(state.severity));
assert.ok(reasons.has(state.reason));
assert.deepEqual(Object.keys(state).sort(), [
"elevatedStreak",
"lastTransitionAtMs",
"observedAtMs",
"reason",
"recoveryStreak",
"severity",
]);
});
});

View File

@@ -0,0 +1,330 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
createResourcePressureRuntime,
type ResourcePressureRuntime,
} from "../../open-sse/utils/resourcePressure.ts";
import type { ResourceSignals } from "../../open-sse/utils/resourcePressurePolicy.ts";
const MiB = 1024 ** 2;
function signals(observedAtMs: number, heapUsedMb = 100): ResourceSignals {
return {
observedAtMs,
v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB },
process: {
rssBytes: 200 * MiB,
externalBytes: 10 * MiB,
arrayBuffersBytes: MiB,
availableBytes: null,
constrainedBytes: null,
},
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
psi: null,
};
}
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
async function settleRefresh(runtime: ResourcePressureRuntime): Promise<void> {
await runtime.whenRefreshSettled();
await Promise.resolve();
}
describe("ResourcePressureRuntime stale-while-revalidate cache", () => {
it("does no proc/sys I/O in check(), while a cheap first-request heap breach sheds immediately", async () => {
let slowSamples = 0;
const runtime = createResourcePressureRuntime({
heapThresholdMb: 200,
immediateHeapUsedMb: () => 201,
sample: async () => {
slowSamples += 1;
return signals(1);
},
});
const guard = runtime.check();
assert.ok(guard);
assert.equal(guard.status, 503);
assert.equal(slowSamples, 0, "request-path check must not invoke the async proc/sys sampler");
assert.equal(runtime.getObservation().state.reason, "v8_heap_absolute");
await settleRefresh(runtime);
assert.equal(slowSamples, 1, "refresh may run after the request-path decision");
runtime.dispose();
});
it("serves a fresh cached sample without scheduling another refresh", async () => {
let now = 0;
let calls = 0;
const runtime = createResourcePressureRuntime({
nowMs: () => now,
staleAfterMs: 100,
immediateHeapUsedMb: () => 100,
sample: async () => {
calls += 1;
return signals(now);
},
});
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 1);
now = 99;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 1);
runtime.dispose();
});
it("schedules at most one refresh under concurrent stale checks", async () => {
let now = 0;
let calls = 0;
const pending = deferred<ResourceSignals>();
const runtime = createResourcePressureRuntime({
nowMs: () => now,
staleAfterMs: 10,
immediateHeapUsedMb: () => 100,
sample: async () => {
calls += 1;
if (calls === 1) return signals(0);
return pending.promise;
},
});
runtime.check();
await settleRefresh(runtime);
now = 11;
for (let index = 0; index < 50; index += 1) runtime.check();
assert.equal(calls, 1, "scheduled work must not run synchronously in check()");
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(calls, 2);
pending.resolve(signals(11));
await settleRefresh(runtime);
assert.equal(calls, 2);
runtime.dispose();
});
it("retains a bounded stale snapshot on refresh failure and retries only after backoff", async () => {
let now = 0;
let calls = 0;
const runtime = createResourcePressureRuntime({
nowMs: () => now,
staleAfterMs: 10,
maxStaleMs: 100,
retryAfterMs: 20,
immediateHeapUsedMb: () => 100,
sample: async () => {
calls += 1;
if (calls === 1) return signals(0, 950);
throw new Error("proc unavailable");
},
thresholds: {
sustainedSamplesCritical: 1,
heapAbsoluteThresholdMb: null,
},
});
runtime.check();
await settleRefresh(runtime);
now = 11;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 2);
assert.equal(runtime.getObservation().signals?.observedAtMs, 0, "failure retains stale data");
now = 25;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 2, "failure backoff prevents a refresh storm");
now = 31;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 3);
now = 101;
assert.equal(runtime.check(), null, "expired stale adaptive pressure fails open");
runtime.dispose();
});
it("measures failure backoff from settlement, not refresh start", async () => {
let now = 0;
let calls = 0;
const pending = deferred<ResourceSignals>();
const runtime = createResourcePressureRuntime({
nowMs: () => now,
staleAfterMs: 10,
maxStaleMs: 100,
retryAfterMs: 20,
immediateHeapUsedMb: () => 100,
sample: async () => {
calls += 1;
if (calls === 1) return signals(0);
return pending.promise;
},
});
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 1);
now = 11;
runtime.check();
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(calls, 2);
// Slow failure: wall clock advances past retryAfter before the sample rejects.
now = 50;
pending.reject(new Error("proc unavailable"));
await settleRefresh(runtime);
assert.equal(calls, 2);
// Retry must wait full retryAfterMs from settlement (50), not from start (11).
now = 69;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 2, "failure backoff starts at settlement, not refresh start");
now = 70;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 3);
runtime.dispose();
});
it("measures success freshness from publication, not refresh start", async () => {
let now = 0;
let calls = 0;
const pending = deferred<ResourceSignals>();
const runtime = createResourcePressureRuntime({
nowMs: () => now,
staleAfterMs: 20,
maxStaleMs: 100,
immediateHeapUsedMb: () => 100,
sample: async () => {
calls += 1;
if (calls === 1) return signals(0);
return pending.promise;
},
});
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 1);
now = 21;
runtime.check();
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(calls, 2);
// Slow success: wall clock advances past staleAfter before the sample resolves.
now = 100;
pending.resolve(signals(100));
await settleRefresh(runtime);
assert.equal(calls, 2);
assert.equal(runtime.getObservation().signals?.observedAtMs, 100);
// Freshness must run full staleAfterMs from publication (100), not start (21).
now = 119;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 2, "success freshness starts at publication, not refresh start");
now = 120;
runtime.check();
await settleRefresh(runtime);
assert.equal(calls, 3);
runtime.dispose();
});
it("default scheduler unrefs Immediate; injected schedulers stay caller-owned", async () => {
// Injected schedule is never wrapped: the runtime must not call unref on it.
let scheduled = 0;
let unrefCalled = 0;
const injected = (refresh: () => void) => {
scheduled += 1;
const handle = setImmediate(refresh);
const originalUnref = handle.unref.bind(handle);
handle.unref = () => {
unrefCalled += 1;
return originalUnref();
};
};
const withInjected = createResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => signals(1),
schedule: injected,
});
withInjected.check();
await settleRefresh(withInjected);
assert.equal(scheduled, 1);
assert.equal(unrefCalled, 0, "injected schedule handles remain caller-owned");
withInjected.dispose();
// Default schedule path: capture the Immediate and prove it is unref'd so a
// pending refresh alone cannot keep the process alive.
const originalSetImmediate = globalThis.setImmediate;
let captured: NodeJS.Immediate | undefined;
globalThis.setImmediate = ((callback: (...args: unknown[]) => void, ...args: unknown[]) => {
const handle = originalSetImmediate(callback, ...args);
captured = handle;
return handle;
}) as typeof setImmediate;
try {
const runtime = createResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
// Never resolve: we only care about the scheduled Immediate ref state.
sample: () => new Promise(() => {}),
});
runtime.check();
assert.ok(captured, "default schedule must use setImmediate");
assert.equal(captured.hasRef(), false, "default Immediate must be unref'd");
runtime.dispose();
if (captured) clearImmediate(captured);
} finally {
globalThis.setImmediate = originalSetImmediate;
}
});
it("dispose ignores late refresh results and independently owned runtimes do not share state", async () => {
const pending = deferred<ResourceSignals>();
let firstCalls = 0;
const first = createResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => {
firstCalls += 1;
return pending.promise;
},
});
first.check();
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(firstCalls, 1);
first.dispose();
pending.resolve(signals(1));
await settleRefresh(first);
assert.equal(
first.getObservation().signals,
null,
"disposed runtime ignores late refresh results"
);
const second = createResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => signals(2),
});
assert.notEqual(first, second);
second.check();
await settleRefresh(second);
assert.equal(second.getObservation().signals?.observedAtMs, 2);
second.dispose();
});
});

View File

@@ -0,0 +1,178 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
decodeMountInfoPath,
parseCgroup2Mount,
parseCgroupV2Path,
resolveCgroupDirectory,
sampleResourceSignals,
sanitizeMemoryBytes,
type ResourcePressureFs,
} from "../../open-sse/utils/resourcePressureSampler.ts";
const MiB = 1024 ** 2;
const GiB = 1024 ** 3;
function memoryUsage(heapUsed = 1): NodeJS.MemoryUsage {
return {
rss: 500 * MiB,
heapTotal: 300 * MiB,
heapUsed,
external: 12 * MiB,
arrayBuffers: 3 * MiB,
};
}
function mapFs(entries: ReadonlyArray<readonly [string, string]>): ResourcePressureFs {
const files = new Map(entries);
return { readText: async (filePath) => files.get(filePath) ?? null };
}
describe("resource pressure cgroup parsers", () => {
it("accepts only the exact unified cgroup entry", () => {
assert.equal(parseCgroupV2Path("2:cpu:/wrong\n0::/delegated/service\n"), "/delegated/service");
assert.equal(parseCgroupV2Path("0:cpu:/wrong\n1::/also-wrong\n"), null);
});
it("decodes mountinfo octal escapes in root and mountpoint", () => {
assert.equal(
decodeMountInfoPath("/sys/fs/cgroup\\040space\\134unit"),
"/sys/fs/cgroup space\\unit"
);
assert.deepEqual(
parseCgroup2Mount(
"43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n"
),
{ root: "/delegated root", mountpoint: "/sys/fs/cgroup space" }
);
});
it("resolves delegated mount roots within the decoded mountpoint", async () => {
const fs = mapFs([
["/proc/self/cgroup", "0::/delegated root/team/service\n"],
[
"/proc/self/mountinfo",
"43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n",
],
["/sys/fs/cgroup space/team/service/memory.current", "1\n"],
]);
assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup space/team/service");
});
it("rejects NUL, traversal, malformed, and out-of-root cgroup paths", async () => {
for (const cgroupPath of [
"/delegated/../escape",
"/delegated/%2e%2e/escape",
"/delegated/service\0escape",
"delegated/service",
"/other/service",
]) {
const fs = mapFs([
["/proc/self/cgroup", `0::${cgroupPath}\n`],
["/proc/self/mountinfo", "43 34 0:35 /delegated /safe/cgroup rw - cgroup2 cgroup2 rw\n"],
["/safe/cgroup/memory.current", "1\n"],
]);
assert.equal(
await resolveCgroupDirectory(fs.readText, { allowDefaultFallback: false }),
null,
cgroupPath
);
}
});
it("falls back to the validated default cgroup root when proc metadata is malformed", async () => {
const fs = mapFs([
["/proc/self/cgroup", "malformed\n"],
["/proc/self/mountinfo", "malformed\n"],
["/sys/fs/cgroup/memory.current", "123\n"],
]);
assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup");
});
});
describe("sampleResourceSignals", () => {
it("captures process, V8, cgroup, event, and PSI snapshot fields", async () => {
const fs = mapFs([
["/proc/self/cgroup", "0::/slice/service\n"],
["/proc/self/mountinfo", "43 34 0:35 / /sys/fs/cgroup rw - cgroup2 cgroup2 rw\n"],
["/sys/fs/cgroup/slice/service/memory.current", `${800 * MiB}\n`],
["/sys/fs/cgroup/slice/service/memory.max", `${GiB}\n`],
["/sys/fs/cgroup/slice/service/memory.high", "966367641\n"],
["/sys/fs/cgroup/slice/service/memory.events", "low 1\nhigh 2\nmax 3\noom 4\noom_kill 5\n"],
[
"/proc/pressure/memory",
"some avg10=1.50 avg60=2.00 avg300=3.25 total=9\nfull avg10=0.25 avg60=0.50 avg300=0.75 total=1\n",
],
]);
const signals = await sampleResourceSignals({
nowMs: () => 42,
memoryUsage: () => memoryUsage(250 * MiB),
heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 250 * MiB }),
availableMemory: () => 4 * GiB,
constrainedMemory: () => undefined,
fs,
});
assert.equal(signals.observedAtMs, 42);
assert.deepEqual(signals.v8, { heapUsedBytes: 250 * MiB, heapLimitBytes: GiB });
assert.deepEqual(signals.process, {
rssBytes: 500 * MiB,
externalBytes: 12 * MiB,
arrayBuffersBytes: 3 * MiB,
availableBytes: 4 * GiB,
constrainedBytes: null,
});
assert.deepEqual(signals.cgroup, {
currentBytes: 800 * MiB,
maxBytes: GiB,
highBytes: 966367641,
events: { low: 1, high: 2, max: 3, oom: 4, oom_kill: 5 },
});
assert.equal(signals.psi?.someAvg10, 1.5);
assert.equal(signals.psi?.fullAvg10, 0.25);
});
it("fails open when platform reads fail or return malformed values", async () => {
const signals = await sampleResourceSignals({
memoryUsage: () => memoryUsage(),
heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 1 }),
availableMemory: () => {
throw new Error("unavailable");
},
constrainedMemory: () => Number.POSITIVE_INFINITY,
fs: {
readText: async (filePath) => {
if (filePath === "/proc/self/cgroup") throw new Error("unavailable");
return "malformed";
},
},
});
assert.equal(signals.process.availableBytes, null);
assert.equal(signals.process.constrainedBytes, null);
assert.deepEqual(signals.cgroup, {
currentBytes: null,
maxBytes: null,
highBytes: null,
events: null,
});
assert.equal(signals.psi, null);
});
it("treats missing, zero, max, and unsafe memory quantities as unavailable", () => {
for (const value of [
undefined,
"",
"max",
0,
-1,
Number.NaN,
Number.MAX_SAFE_INTEGER,
2 ** 63,
]) {
assert.equal(sanitizeMemoryBytes(value), null, String(value));
}
assert.equal(sanitizeMemoryBytes("123"), 123);
});
});

View File

@@ -0,0 +1,134 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
createResourcePressureRuntime,
getResourcePressureObservation,
reloadResourcePressureRuntime,
type ResourceSignals,
} from "../../open-sse/utils/resourcePressure.ts";
const MiB = 1024 ** 2;
function signals(observedAtMs: number, heapUsedMb: number): ResourceSignals {
return {
observedAtMs,
v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB },
process: {
rssBytes: 200 * MiB,
externalBytes: 10 * MiB,
arrayBuffersBytes: MiB,
availableBytes: null,
constrainedBytes: null,
},
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
psi: null,
};
}
describe("resource pressure HTTP guard facade", () => {
it("preserves strict immediate first-request heap shedding", async () => {
let samples = 0;
const runtime = createResourcePressureRuntime({
heapThresholdMb: 200,
immediateHeapUsedMb: () => 201,
sample: async () => {
samples += 1;
return signals(1, 100);
},
});
const guard = runtime.check();
assert.ok(guard);
assert.equal(guard.status, 503);
assert.equal(samples, 0, "the asynchronous sampler cannot run in the request path");
runtime.dispose();
});
it("does not shed when heap usage equals the strict threshold", () => {
const runtime = createResourcePressureRuntime({
heapThresholdMb: 200,
immediateHeapUsedMb: () => 200,
sample: async () => signals(1, 100),
});
assert.equal(runtime.check(), null);
runtime.dispose();
});
it("returns a sanitized standards-correct 503 with Retry-After", async () => {
const runtime = createResourcePressureRuntime({
heapThresholdMb: 200,
immediateHeapUsedMb: () => 987,
sample: async () => signals(1, 100),
});
const guard = runtime.check();
assert.ok(guard);
assert.equal(guard.success, false);
assert.equal(guard.status, 503);
assert.equal(guard.response.status, 503);
assert.equal(guard.response.headers.get("Retry-After"), "5");
assert.equal(guard.response.headers.get("Content-Type"), "application/json");
const payload = await guard.response.json();
assert.deepEqual(payload.error, {
message: "Service temporarily unavailable due to resource pressure. Retry shortly.",
type: "server_error",
code: "resource_pressure",
});
const clientText = JSON.stringify(payload) + guard.error;
assert.ok(!clientText.includes("987"));
assert.ok(!/\bMB\b/.test(clientText));
runtime.dispose();
});
it("reload atomically replaces and resets the thin default facade", async () => {
let firstCalls = 0;
reloadResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => {
firstCalls += 1;
return signals(1, 100);
},
});
const replacement = reloadResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => signals(2, 100),
});
assert.deepEqual(getResourcePressureObservation(), {
signals: null,
state: {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
},
});
assert.equal(firstCalls, 0, "replaced runtime must not retain or run scheduled work");
replacement.dispose();
});
it("exposes all observation snapshot fields", async () => {
const runtime = createResourcePressureRuntime({
immediateHeapUsedMb: () => 100,
sample: async () => signals(42, 100),
});
assert.deepEqual(runtime.getObservation(), {
signals: null,
state: {
severity: "normal",
reason: "none",
elevatedStreak: 0,
recoveryStreak: 0,
lastTransitionAtMs: 0,
observedAtMs: 0,
},
});
runtime.check();
await runtime.whenRefreshSettled();
assert.equal(runtime.getObservation().signals?.observedAtMs, 42);
assert.equal(runtime.getObservation().state.observedAtMs, 42);
runtime.dispose();
});
});