mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-19 05:32:19 +03:00
feat(admission): add adaptive overload and pressure controls
Add bounded weighted admission with fair queuing, deadline and cancellation handling, exact lease accounting, and a default-shadow runtime. Keep asynchronous resource-pressure shedding as an independent safety fuse and bound request feature estimation.
This commit is contained in:
@@ -67,7 +67,7 @@ import {
|
||||
resolveMemoryOwnerId,
|
||||
} from "./chatCore/memoryExtraction.ts";
|
||||
import { CORS_HEADERS } from "../utils/cors.ts";
|
||||
import { checkHeapPressureGuard } from "../utils/heapPressure.ts";
|
||||
import { checkResourcePressureGuard } from "../utils/resourcePressure.ts";
|
||||
import { normalizeHeaders } from "../utils/headers.ts";
|
||||
import { resolveChatCoreRequestFormat } from "./chatCore/requestFormat.ts";
|
||||
import { resolveChatCoreTargetFormat } from "./chatCore/targetFormat.ts";
|
||||
@@ -359,13 +359,6 @@ import {
|
||||
isRpmExhausted,
|
||||
} from "../services/geminiRateLimitTracker.ts";
|
||||
|
||||
// ── Global memory pressure guard ────────────────────────────────────────
|
||||
// Prevents OOM by rejecting new requests when V8 heap exceeds threshold.
|
||||
// Self-healing: no counters to leak, no cleanup needed. The threshold
|
||||
// auto-calibrates to 85% of the actual V8 heap ceiling (see heapPressure.ts) so
|
||||
// it tracks --max-old-space-size across 1GB/2GB/large VPS instead of a fixed
|
||||
// 200MB that sat below the app's own ~260MB baseline and rejected every request.
|
||||
|
||||
import { isSmallEnoughForSemanticCache } from "../utils/estimateSize.ts";
|
||||
|
||||
/**
|
||||
@@ -415,17 +408,16 @@ export async function handleChatCore({
|
||||
createPiiTransform = null,
|
||||
correlationId = null,
|
||||
modelPinned = false,
|
||||
skipResourcePressureGuard = false,
|
||||
}) {
|
||||
let { provider, model, extendedContext } = modelInfo;
|
||||
// ── Memory pressure guard ────────────────────────────────────────────
|
||||
// Reject early if V8 heap is already near the 256MB limit. Prevents
|
||||
// cascading OOM when many large-context requests arrive concurrently.
|
||||
try {
|
||||
const heapUsedMB = process.memoryUsage().heapUsed / (1024 * 1024);
|
||||
const heapGuard = checkHeapPressureGuard(heapUsedMB);
|
||||
if (heapGuard) return heapGuard;
|
||||
} catch {
|
||||
/* memoryUsage() never throws */
|
||||
if (!skipResourcePressureGuard) {
|
||||
try {
|
||||
const pressureGuard = checkResourcePressureGuard();
|
||||
if (pressureGuard) return pressureGuard;
|
||||
} catch {
|
||||
/* fail open */
|
||||
}
|
||||
}
|
||||
|
||||
// Per-request model-routing metadata (first extracted slice of the request-setup phase).
|
||||
|
||||
168
open-sse/services/admission/adaptation.ts
Normal file
168
open-sse/services/admission/adaptation.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import type { AdmissionPressure, AdmissionReleaseOutcome } from "./types.ts";
|
||||
|
||||
export interface AdaptationParams {
|
||||
minLimit: number;
|
||||
maxLimit: number;
|
||||
windowMs: number;
|
||||
shortLatencyAlpha: number;
|
||||
longLatencyAlpha: number;
|
||||
increaseStep: number;
|
||||
decreaseFactor: number;
|
||||
criticalDecreaseFactor: number;
|
||||
highUtilizationThreshold: number;
|
||||
lowUtilizationThreshold: number;
|
||||
latencyGradientThreshold: number;
|
||||
maxIncreasePerWindow: number;
|
||||
}
|
||||
|
||||
export interface AdaptationState {
|
||||
currentLimit: number;
|
||||
shortLatencyEwma: number;
|
||||
longLatencyEwma: number;
|
||||
pressure: AdmissionPressure;
|
||||
/** Sum of admitted cost * time contribution proxies in the open window. */
|
||||
windowActiveCostIntegral: number;
|
||||
windowCompleted: number;
|
||||
windowLatencySamples: number;
|
||||
windowStartMs: number;
|
||||
freezeGrowth: boolean;
|
||||
/**
|
||||
* When true, critical multiplicative decrease already applied for this window
|
||||
* (e.g. via immediate observePressure). Window close must not re-apply it.
|
||||
*/
|
||||
criticalDecreaseConsumed: boolean;
|
||||
utilization: number;
|
||||
}
|
||||
|
||||
export function clampLimit(value: number, minLimit: number, maxLimit: number): number {
|
||||
if (!Number.isFinite(value)) return minLimit;
|
||||
return Math.min(maxLimit, Math.max(minLimit, Math.floor(value)));
|
||||
}
|
||||
|
||||
export function createAdaptationState(
|
||||
initialLimit: number,
|
||||
minLimit: number,
|
||||
maxLimit: number,
|
||||
nowMs: number
|
||||
): AdaptationState {
|
||||
return {
|
||||
currentLimit: clampLimit(initialLimit, minLimit, maxLimit),
|
||||
shortLatencyEwma: 0,
|
||||
longLatencyEwma: 0,
|
||||
pressure: "normal",
|
||||
windowActiveCostIntegral: 0,
|
||||
windowCompleted: 0,
|
||||
windowLatencySamples: 0,
|
||||
windowStartMs: nowMs,
|
||||
freezeGrowth: false,
|
||||
criticalDecreaseConsumed: false,
|
||||
utilization: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function noteLatency(
|
||||
state: AdaptationState,
|
||||
latencyMs: number,
|
||||
params: AdaptationParams
|
||||
): void {
|
||||
const sample = Number.isFinite(latencyMs) && latencyMs >= 0 ? latencyMs : 0;
|
||||
state.windowLatencySamples += 1;
|
||||
const sa = params.shortLatencyAlpha;
|
||||
const la = params.longLatencyAlpha;
|
||||
if (state.shortLatencyEwma <= 0 && state.longLatencyEwma <= 0) {
|
||||
state.shortLatencyEwma = sample;
|
||||
state.longLatencyEwma = sample;
|
||||
return;
|
||||
}
|
||||
state.shortLatencyEwma = sa * sample + (1 - sa) * state.shortLatencyEwma;
|
||||
state.longLatencyEwma = la * sample + (1 - la) * state.longLatencyEwma;
|
||||
}
|
||||
|
||||
export function noteOutcome(state: AdaptationState, outcome: AdmissionReleaseOutcome): void {
|
||||
// A single upstream business error freezes growth for the current window; it must not
|
||||
// apply critical multiplicative collapse on its own.
|
||||
if (outcome === "upstream_error") {
|
||||
state.freezeGrowth = true;
|
||||
return;
|
||||
}
|
||||
if (outcome === "timeout") {
|
||||
state.freezeGrowth = true;
|
||||
}
|
||||
}
|
||||
|
||||
export function setPressure(state: AdaptationState, pressure: AdmissionPressure): void {
|
||||
const severity: Record<AdmissionPressure, number> = { normal: 0, high: 1, critical: 2 };
|
||||
if (severity[pressure] > severity[state.pressure]) state.pressure = pressure;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the current feedback window and adjust the limit.
|
||||
* Recovery (increase) is slower than decrease; idle/low utilization does not inflate.
|
||||
*/
|
||||
export function closeAdaptationWindow(
|
||||
state: AdaptationState,
|
||||
params: AdaptationParams,
|
||||
nowMs: number
|
||||
): void {
|
||||
const elapsed = Math.max(1, Math.min(params.windowMs, nowMs - state.windowStartMs));
|
||||
// sampleActiveIntegral already accounts for every interval exactly once.
|
||||
const avgActive = state.windowActiveCostIntegral / elapsed;
|
||||
const util = state.currentLimit > 0 ? avgActive / state.currentLimit : 0;
|
||||
state.utilization = Math.max(0, Math.min(1, util));
|
||||
|
||||
let next = state.currentLimit;
|
||||
const gradient =
|
||||
state.longLatencyEwma > 0
|
||||
? (state.shortLatencyEwma - state.longLatencyEwma) / state.longLatencyEwma
|
||||
: 0;
|
||||
|
||||
if (state.pressure === "critical") {
|
||||
// Immediate observePressure may already have applied the critical factor once.
|
||||
if (!state.criticalDecreaseConsumed) {
|
||||
next = Math.floor(next * params.criticalDecreaseFactor);
|
||||
}
|
||||
} else if (
|
||||
state.pressure === "high" ||
|
||||
(state.windowLatencySamples > 0 && gradient >= params.latencyGradientThreshold)
|
||||
) {
|
||||
next = Math.floor(next * params.decreaseFactor);
|
||||
} else if (
|
||||
!state.freezeGrowth &&
|
||||
state.pressure === "normal" &&
|
||||
state.utilization >= params.highUtilizationThreshold &&
|
||||
state.windowCompleted > 0
|
||||
) {
|
||||
const step = Math.min(params.increaseStep, params.maxIncreasePerWindow);
|
||||
next = next + step;
|
||||
}
|
||||
// A genuinely low-utilization window recovers the latency baseline so stale gradients expire.
|
||||
if (state.utilization <= params.lowUtilizationThreshold) {
|
||||
state.shortLatencyEwma = state.longLatencyEwma;
|
||||
}
|
||||
|
||||
state.currentLimit = clampLimit(next, params.minLimit, params.maxLimit);
|
||||
state.windowActiveCostIntegral = 0;
|
||||
state.windowCompleted = 0;
|
||||
state.windowLatencySamples = 0;
|
||||
state.windowStartMs = nowMs;
|
||||
state.freezeGrowth = false;
|
||||
state.criticalDecreaseConsumed = false;
|
||||
state.pressure = "normal";
|
||||
}
|
||||
|
||||
export function sampleActiveIntegral(
|
||||
state: AdaptationState,
|
||||
activeCost: number,
|
||||
dtMs: number
|
||||
): void {
|
||||
if (dtMs <= 0 || activeCost <= 0) return;
|
||||
const boundedActiveCost = Math.min(activeCost, state.currentLimit);
|
||||
const contribution =
|
||||
dtMs > Math.floor(Number.MAX_SAFE_INTEGER / boundedActiveCost)
|
||||
? Number.MAX_SAFE_INTEGER
|
||||
: boundedActiveCost * dtMs;
|
||||
state.windowActiveCostIntegral =
|
||||
contribution >= Number.MAX_SAFE_INTEGER - state.windowActiveCostIntegral
|
||||
? Number.MAX_SAFE_INTEGER
|
||||
: state.windowActiveCostIntegral + contribution;
|
||||
}
|
||||
167
open-sse/services/admission/config.ts
Normal file
167
open-sse/services/admission/config.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import { resolveCostConfig } from "./cost.ts";
|
||||
import {
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
MAX_ADMISSION_WINDOW_MS,
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionMode,
|
||||
} from "./types.ts";
|
||||
import type { AdaptationParams } from "./adaptation.ts";
|
||||
|
||||
export { MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_WINDOW_MS };
|
||||
|
||||
export interface ValidatedConfig {
|
||||
mode: AdmissionMode;
|
||||
minLimit: number;
|
||||
maxLimit: number;
|
||||
initialLimit: number;
|
||||
maxQueueCount: number;
|
||||
maxQueueCost: number;
|
||||
defaultMaxWaitMs: number;
|
||||
windowMs: number;
|
||||
adaptation: AdaptationParams;
|
||||
maxRequestCost: number;
|
||||
costConfig: ReturnType<typeof resolveCostConfig>;
|
||||
}
|
||||
|
||||
function requirePositiveInt(
|
||||
name: string,
|
||||
value: unknown,
|
||||
max: number = MAX_ADMISSION_COST_OR_LIMIT
|
||||
): number {
|
||||
if (
|
||||
typeof value !== "number" ||
|
||||
!Number.isFinite(value) ||
|
||||
value <= 0 ||
|
||||
!Number.isSafeInteger(value)
|
||||
) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
if (value > max) {
|
||||
throw new RangeError(`${name} must be <= ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireUnitInterval(name: string, value: unknown, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1) {
|
||||
throw new RangeError(`${name} must be in (0, 1]`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireDecreaseFactor(name: string, value: unknown, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value >= 1) {
|
||||
throw new RangeError(`${name} must be in (0, 1)`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveMode(mode: AdaptiveAdmissionConfig["mode"]): AdmissionMode {
|
||||
if (mode === undefined) return "shadow";
|
||||
if (mode !== "off" && mode !== "shadow" && mode !== "enforce") {
|
||||
throw new RangeError("mode must be off|shadow|enforce");
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
function resolveAdaptationParams(
|
||||
input: AdaptiveAdmissionConfig,
|
||||
minLimit: number,
|
||||
maxLimit: number,
|
||||
windowMs: number
|
||||
): AdaptationParams {
|
||||
const decreaseFactor = requireDecreaseFactor("decreaseFactor", input.decreaseFactor, 0.8);
|
||||
const criticalDecreaseFactor = requireDecreaseFactor(
|
||||
"criticalDecreaseFactor",
|
||||
input.criticalDecreaseFactor,
|
||||
0.5
|
||||
);
|
||||
const increaseStep =
|
||||
input.increaseStep === undefined ? 1 : requirePositiveInt("increaseStep", input.increaseStep);
|
||||
const maxIncreasePerWindow =
|
||||
input.maxIncreasePerWindow === undefined
|
||||
? increaseStep
|
||||
: requirePositiveInt("maxIncreasePerWindow", input.maxIncreasePerWindow);
|
||||
|
||||
const shortLatencyAlpha = requireUnitInterval("shortLatencyAlpha", input.shortLatencyAlpha, 0.5);
|
||||
const longLatencyAlpha = requireUnitInterval("longLatencyAlpha", input.longLatencyAlpha, 0.1);
|
||||
const highUtilizationThreshold = requireUnitInterval(
|
||||
"highUtilizationThreshold",
|
||||
input.highUtilizationThreshold,
|
||||
0.7
|
||||
);
|
||||
const lowUtilizationThreshold = requireUnitInterval(
|
||||
"lowUtilizationThreshold",
|
||||
input.lowUtilizationThreshold,
|
||||
0.3
|
||||
);
|
||||
if (criticalDecreaseFactor > decreaseFactor) {
|
||||
throw new RangeError("criticalDecreaseFactor must be <= decreaseFactor");
|
||||
}
|
||||
if (lowUtilizationThreshold >= highUtilizationThreshold) {
|
||||
throw new RangeError("lowUtilizationThreshold must be < highUtilizationThreshold");
|
||||
}
|
||||
if (shortLatencyAlpha <= longLatencyAlpha) {
|
||||
throw new RangeError("shortLatencyAlpha must be > longLatencyAlpha");
|
||||
}
|
||||
|
||||
return {
|
||||
minLimit,
|
||||
maxLimit,
|
||||
windowMs,
|
||||
shortLatencyAlpha,
|
||||
longLatencyAlpha,
|
||||
increaseStep,
|
||||
decreaseFactor,
|
||||
criticalDecreaseFactor,
|
||||
highUtilizationThreshold,
|
||||
lowUtilizationThreshold,
|
||||
latencyGradientThreshold: requireUnitInterval(
|
||||
"latencyGradientThreshold",
|
||||
input.latencyGradientThreshold,
|
||||
0.25
|
||||
),
|
||||
maxIncreasePerWindow,
|
||||
};
|
||||
}
|
||||
|
||||
export function validateConfig(input: AdaptiveAdmissionConfig): ValidatedConfig {
|
||||
const minLimit = requirePositiveInt("minLimit", input.minLimit);
|
||||
const maxLimit = requirePositiveInt("maxLimit", input.maxLimit);
|
||||
if (minLimit > maxLimit) {
|
||||
throw new RangeError("minLimit must be <= maxLimit");
|
||||
}
|
||||
const initialLimit = requirePositiveInt("initialLimit", input.initialLimit);
|
||||
// Queue count is not multiplied into cost×time products; keep the full safe-integer range.
|
||||
const maxQueueCount = requirePositiveInt(
|
||||
"maxQueueCount",
|
||||
input.maxQueueCount,
|
||||
Number.MAX_SAFE_INTEGER
|
||||
);
|
||||
const maxQueueCost = requirePositiveInt("maxQueueCost", input.maxQueueCost);
|
||||
const windowMs =
|
||||
input.windowMs === undefined
|
||||
? 1000
|
||||
: requirePositiveInt("windowMs", input.windowMs, MAX_ADMISSION_WINDOW_MS);
|
||||
const defaultMaxWaitMs =
|
||||
input.defaultMaxWaitMs === undefined
|
||||
? 5_000
|
||||
: requirePositiveInt("defaultMaxWaitMs", input.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS);
|
||||
const costConfig = resolveCostConfig(input.cost);
|
||||
|
||||
return {
|
||||
mode: resolveMode(input.mode),
|
||||
minLimit,
|
||||
maxLimit,
|
||||
initialLimit,
|
||||
maxQueueCount,
|
||||
maxQueueCost,
|
||||
defaultMaxWaitMs,
|
||||
windowMs,
|
||||
maxRequestCost: costConfig.maxRequestCost,
|
||||
costConfig,
|
||||
adaptation: resolveAdaptationParams(input, minLimit, maxLimit, windowMs),
|
||||
};
|
||||
}
|
||||
624
open-sse/services/admission/controller.ts
Normal file
624
open-sse/services/admission/controller.ts
Normal file
@@ -0,0 +1,624 @@
|
||||
import {
|
||||
closeAdaptationWindow,
|
||||
createAdaptationState,
|
||||
noteLatency,
|
||||
noteOutcome,
|
||||
sampleActiveIntegral,
|
||||
setPressure,
|
||||
type AdaptationState,
|
||||
} from "./adaptation.ts";
|
||||
import { validateConfig, type ValidatedConfig } from "./config.ts";
|
||||
import { estimateAdmissionCost, normalizeRequestCost } from "./cost.ts";
|
||||
import { FairCostQueue, type QueueEntry } from "./queue.ts";
|
||||
import {
|
||||
MAX_ADMISSION_WINDOW_MS,
|
||||
createAdmissionRejectError,
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionAcquireResult,
|
||||
type AdmissionAdmitted,
|
||||
type AdmissionClock,
|
||||
type AdmissionLease,
|
||||
type AdmissionPressure,
|
||||
type AdmissionRejectCode,
|
||||
type AdmissionReleaseMeta,
|
||||
type AdmissionReleaseOutcome,
|
||||
type AdmissionRequest,
|
||||
type AdmissionSnapshot,
|
||||
type ShadowDecision,
|
||||
} from "./types.ts";
|
||||
|
||||
type VirtualDisposition = "active" | "queued" | "rejected" | "none";
|
||||
|
||||
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
||||
|
||||
/** Snapshot numbers are always finite safe integers; never emit rounded unsafe Number values. */
|
||||
function saturateSnapshotNumber(value: number): number {
|
||||
if (!Number.isFinite(value) || value <= 0) return 0;
|
||||
if (value >= Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER;
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
function bigintToSnapshotNumber(value: bigint): number {
|
||||
if (value <= 0n) return 0;
|
||||
if (value >= MAX_SAFE_BIGINT) return Number.MAX_SAFE_INTEGER;
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
function addSaturated(total: number, delta: number): number {
|
||||
if (delta <= 0) return saturateSnapshotNumber(total);
|
||||
if (total >= Number.MAX_SAFE_INTEGER - delta) return Number.MAX_SAFE_INTEGER;
|
||||
return total + delta;
|
||||
}
|
||||
|
||||
interface ActiveLeaseRecord {
|
||||
id: string;
|
||||
cost: number;
|
||||
released: boolean;
|
||||
admittedAtMs: number;
|
||||
virtualDisposition: VirtualDisposition;
|
||||
}
|
||||
|
||||
interface QueuedPayload {
|
||||
resolve: (value: AdmissionAdmitted) => void;
|
||||
reject: (err: Error) => void;
|
||||
signal?: AbortSignal;
|
||||
onAbort?: () => void;
|
||||
}
|
||||
|
||||
let leaseSeq = 0;
|
||||
|
||||
function nextId(prefix: string): string {
|
||||
leaseSeq += 1;
|
||||
return `${prefix}-${leaseSeq}`;
|
||||
}
|
||||
|
||||
function defaultClock(): AdmissionClock {
|
||||
return {
|
||||
now: () => Date.now(),
|
||||
setTimer: (fn, delayMs) => {
|
||||
const handle = setTimeout(fn, delayMs);
|
||||
// Window/deadline timers must not pin the event loop open when idle.
|
||||
if (typeof handle.unref === "function") handle.unref();
|
||||
return handle;
|
||||
},
|
||||
clearTimer: (id) => clearTimeout(id as ReturnType<typeof setTimeout>),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dependency-injected weighted adaptive admission controller.
|
||||
* Pure in-process core: no env/settings/route wiring.
|
||||
*/
|
||||
export class AdaptiveAdmissionController {
|
||||
private config: ValidatedConfig;
|
||||
private readonly clock: AdmissionClock;
|
||||
private adaptation: AdaptationState;
|
||||
private queue: FairCostQueue<QueuedPayload>;
|
||||
private virtualQueue: FairCostQueue<{ recordId: string }>;
|
||||
private readonly active = new Map<string, ActiveLeaseRecord>();
|
||||
private activeCost = 0n;
|
||||
private virtualActiveCost = 0;
|
||||
private virtualActiveCount = 0;
|
||||
private lastSampleMs: number;
|
||||
private windowTimer: unknown = undefined;
|
||||
private shutDown = false;
|
||||
|
||||
private admittedCount = 0;
|
||||
private rejectedCount = 0;
|
||||
private wouldAdmitCount = 0;
|
||||
private wouldQueueCount = 0;
|
||||
private wouldRejectCount = 0;
|
||||
|
||||
constructor(config: AdaptiveAdmissionConfig, clock?: Partial<AdmissionClock>) {
|
||||
this.config = validateConfig(config);
|
||||
this.clock = {
|
||||
now: clock?.now ?? defaultClock().now,
|
||||
setTimer: clock?.setTimer ?? defaultClock().setTimer,
|
||||
clearTimer: clock?.clearTimer ?? defaultClock().clearTimer,
|
||||
};
|
||||
const now = this.clock.now();
|
||||
this.adaptation = createAdaptationState(
|
||||
this.config.initialLimit,
|
||||
this.config.minLimit,
|
||||
this.config.maxLimit,
|
||||
now
|
||||
);
|
||||
this.queue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
|
||||
this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
|
||||
this.lastSampleMs = now;
|
||||
this.armWindowTimer();
|
||||
}
|
||||
|
||||
updateConfig(config: AdaptiveAdmissionConfig): void {
|
||||
const next = validateConfig(config);
|
||||
this.sampleIntegral();
|
||||
this.config = next;
|
||||
this.adaptation.currentLimit = Math.min(
|
||||
next.maxLimit,
|
||||
Math.max(next.minLimit, this.adaptation.currentLimit)
|
||||
);
|
||||
this.adaptation.windowStartMs = this.clock.now();
|
||||
this.adaptation.windowActiveCostIntegral = 0;
|
||||
this.adaptation.windowCompleted = 0;
|
||||
this.adaptation.windowLatencySamples = 0;
|
||||
this.adaptation.freezeGrowth = false;
|
||||
this.adaptation.criticalDecreaseConsumed = false;
|
||||
this.adaptation.pressure = "normal";
|
||||
this.lastSampleMs = this.clock.now();
|
||||
|
||||
const drained = this.queue.drain();
|
||||
this.queue = new FairCostQueue(next.maxQueueCount, next.maxQueueCost);
|
||||
for (const entry of drained) {
|
||||
if (next.mode !== "enforce") {
|
||||
this.clearEntryTimer(entry);
|
||||
this.detachAbort(entry);
|
||||
entry.payload.resolve(this.admit(entry.cost));
|
||||
continue;
|
||||
}
|
||||
// Cost above the new enforce limit must fail closed immediately, never strand until deadline.
|
||||
if (entry.cost > this.adaptation.currentLimit) {
|
||||
this.failQueued(
|
||||
entry,
|
||||
"ADMISSION_OVERSIZED",
|
||||
"request cost exceeds max budget after config update"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!this.queue.enqueue(entry)) {
|
||||
this.failQueued(entry, "ADMISSION_QUEUE_FULL", "queue capacity reduced");
|
||||
}
|
||||
}
|
||||
|
||||
this.rebuildVirtualState(next.mode === "shadow");
|
||||
this.armWindowTimer();
|
||||
if (next.mode === "enforce") {
|
||||
this.dispatch();
|
||||
}
|
||||
}
|
||||
|
||||
snapshot(): AdmissionSnapshot {
|
||||
this.sampleIntegral();
|
||||
return {
|
||||
mode: this.config.mode,
|
||||
currentLimit: this.adaptation.currentLimit,
|
||||
minLimit: this.config.minLimit,
|
||||
maxLimit: this.config.maxLimit,
|
||||
activeCost: bigintToSnapshotNumber(this.activeCost),
|
||||
activeCount: saturateSnapshotNumber(this.active.size),
|
||||
queuedCost: saturateSnapshotNumber(this.queue.totalCost),
|
||||
queuedCount: saturateSnapshotNumber(this.queue.size),
|
||||
virtualActiveCost: saturateSnapshotNumber(this.virtualActiveCost),
|
||||
virtualActiveCount: saturateSnapshotNumber(this.virtualActiveCount),
|
||||
virtualQueuedCost: saturateSnapshotNumber(this.virtualQueue.totalCost),
|
||||
virtualQueuedCount: saturateSnapshotNumber(this.virtualQueue.size),
|
||||
admittedCount: saturateSnapshotNumber(this.admittedCount),
|
||||
rejectedCount: saturateSnapshotNumber(this.rejectedCount),
|
||||
wouldAdmitCount: saturateSnapshotNumber(this.wouldAdmitCount),
|
||||
wouldQueueCount: saturateSnapshotNumber(this.wouldQueueCount),
|
||||
wouldRejectCount: saturateSnapshotNumber(this.wouldRejectCount),
|
||||
shortLatencyEwma: this.adaptation.shortLatencyEwma,
|
||||
longLatencyEwma: this.adaptation.longLatencyEwma,
|
||||
utilization: this.adaptation.utilization,
|
||||
pressure: this.adaptation.pressure,
|
||||
shutdown: this.shutDown,
|
||||
};
|
||||
}
|
||||
|
||||
observePressure(pressure: AdmissionPressure): void {
|
||||
setPressure(this.adaptation, pressure);
|
||||
if (pressure === "critical") {
|
||||
// Immediate fast decrease once per window; window close must not re-apply it.
|
||||
if (!this.adaptation.criticalDecreaseConsumed) {
|
||||
this.adaptation.currentLimit = Math.max(
|
||||
this.config.minLimit,
|
||||
Math.floor(this.adaptation.currentLimit * this.config.adaptation.criticalDecreaseFactor)
|
||||
);
|
||||
this.adaptation.criticalDecreaseConsumed = true;
|
||||
this.dispatch();
|
||||
this.dispatchVirtual();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Deterministic window tick for tests / injected clocks. */
|
||||
tick(): void {
|
||||
this.sampleIntegral();
|
||||
closeAdaptationWindow(this.adaptation, this.config.adaptation, this.clock.now());
|
||||
// Real queue first, then virtual: raised limits must promote shadow-queued work
|
||||
// before newer arrivals are classified against the updated budget.
|
||||
this.dispatch();
|
||||
this.dispatchVirtual();
|
||||
}
|
||||
|
||||
async acquire(request: AdmissionRequest): Promise<AdmissionAcquireResult> {
|
||||
if (this.shutDown) {
|
||||
return this.reject("ADMISSION_SHUTDOWN", "admission controller is shut down");
|
||||
}
|
||||
|
||||
if (request.signal?.aborted) {
|
||||
return this.reject("ADMISSION_ABORTED", "request aborted before acquire");
|
||||
}
|
||||
|
||||
if (request.pressure) setPressure(this.adaptation, request.pressure);
|
||||
|
||||
const cost = this.resolveCost(request);
|
||||
const mode = this.config.mode;
|
||||
|
||||
if (mode === "off") {
|
||||
return this.admitVirtual(cost);
|
||||
}
|
||||
|
||||
const limit = this.adaptation.currentLimit;
|
||||
|
||||
if (mode === "shadow") {
|
||||
return this.acquireShadow(request, cost, limit);
|
||||
}
|
||||
|
||||
// enforce
|
||||
if (cost > limit) {
|
||||
return this.reject("ADMISSION_OVERSIZED", "request cost exceeds max budget");
|
||||
}
|
||||
|
||||
// Once work is queued, every newer request joins the same fair queue even if it
|
||||
// currently fits. This makes bounded bypass accounting effective and prevents
|
||||
// direct arrivals from indefinitely jumping an older reserved weighted request.
|
||||
if (this.queue.size === 0 && this.activeCost + BigInt(cost) <= BigInt(limit)) {
|
||||
return this.admit(cost);
|
||||
}
|
||||
|
||||
if (!this.queue.canAccept(cost)) {
|
||||
return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full");
|
||||
}
|
||||
|
||||
return this.enqueue(request, cost);
|
||||
}
|
||||
|
||||
shutdown(): void {
|
||||
if (this.shutDown) return;
|
||||
this.shutDown = true;
|
||||
if (this.windowTimer !== undefined) {
|
||||
this.clock.clearTimer(this.windowTimer);
|
||||
this.windowTimer = undefined;
|
||||
}
|
||||
const drained = this.queue.drain();
|
||||
for (const entry of drained) {
|
||||
this.clearEntryTimer(entry);
|
||||
this.detachAbort(entry);
|
||||
entry.payload.reject(
|
||||
createAdmissionRejectError("ADMISSION_SHUTDOWN", "admission controller shut down")
|
||||
);
|
||||
this.rejectedCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private resolveCost(request: AdmissionRequest): number {
|
||||
if (request.cost !== undefined) {
|
||||
return normalizeRequestCost(request.cost, this.config.maxRequestCost);
|
||||
}
|
||||
if (request.features) {
|
||||
return estimateAdmissionCost(request.features, this.config.costConfig);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
private acquireShadow(request: AdmissionRequest, cost: number, limit: number): AdmissionAdmitted {
|
||||
let decision: ShadowDecision;
|
||||
let disposition: VirtualDisposition;
|
||||
if (cost > limit || !Number.isSafeInteger(cost)) {
|
||||
decision = "would-reject";
|
||||
disposition = "rejected";
|
||||
this.wouldRejectCount += 1;
|
||||
} else if (this.virtualActiveCost + cost <= limit) {
|
||||
decision = "would-admit";
|
||||
disposition = "active";
|
||||
this.virtualActiveCost = addSaturated(this.virtualActiveCost, cost);
|
||||
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
|
||||
this.wouldAdmitCount = addSaturated(this.wouldAdmitCount, 1);
|
||||
} else if (this.virtualQueue.canAccept(cost)) {
|
||||
decision = "would-queue";
|
||||
disposition = "queued";
|
||||
this.wouldQueueCount += 1;
|
||||
} else {
|
||||
decision = "would-reject";
|
||||
disposition = "rejected";
|
||||
this.wouldRejectCount += 1;
|
||||
}
|
||||
|
||||
const admitted = this.admit(cost, disposition);
|
||||
if (disposition === "queued") {
|
||||
this.virtualQueue.enqueue({
|
||||
id: admitted.lease.id,
|
||||
tenantKey: request.tenantKey || "_default",
|
||||
cost,
|
||||
enqueuedAtMs: this.clock.now(),
|
||||
deadlineMs: Number.MAX_SAFE_INTEGER,
|
||||
payload: { recordId: admitted.lease.id },
|
||||
});
|
||||
}
|
||||
return { ...admitted, shadowDecision: decision };
|
||||
}
|
||||
|
||||
private admitVirtual(cost: number): AdmissionAdmitted {
|
||||
// Mode off: no accounting.
|
||||
const id = nextId("lease");
|
||||
const lease: AdmissionLease = {
|
||||
id,
|
||||
cost,
|
||||
get released() {
|
||||
return true;
|
||||
},
|
||||
release: () => {
|
||||
/* no-op */
|
||||
},
|
||||
};
|
||||
this.admittedCount += 1;
|
||||
return { status: "admitted", lease };
|
||||
}
|
||||
|
||||
private admit(cost: number, virtualDisposition: VirtualDisposition = "none"): AdmissionAdmitted {
|
||||
this.sampleIntegral();
|
||||
const id = nextId("lease");
|
||||
const record: ActiveLeaseRecord = {
|
||||
id,
|
||||
cost,
|
||||
released: false,
|
||||
admittedAtMs: this.clock.now(),
|
||||
virtualDisposition,
|
||||
};
|
||||
this.active.set(id, record);
|
||||
this.activeCost += BigInt(cost);
|
||||
this.admittedCount += 1;
|
||||
|
||||
const controller = this;
|
||||
const lease: AdmissionLease = {
|
||||
id,
|
||||
cost,
|
||||
get released() {
|
||||
return record.released;
|
||||
},
|
||||
release(outcome: AdmissionReleaseOutcome = "success", meta?: AdmissionReleaseMeta) {
|
||||
controller.releaseLease(record, outcome, meta);
|
||||
},
|
||||
};
|
||||
return { status: "admitted", lease };
|
||||
}
|
||||
|
||||
private releaseLease(
|
||||
record: ActiveLeaseRecord,
|
||||
outcome: AdmissionReleaseOutcome,
|
||||
meta?: AdmissionReleaseMeta
|
||||
): void {
|
||||
if (record.released) return;
|
||||
record.released = true;
|
||||
// Sample while the lease still contributes to activeCost so utilization EWMA sees load.
|
||||
this.sampleIntegral();
|
||||
if (this.active.has(record.id)) {
|
||||
this.active.delete(record.id);
|
||||
this.activeCost -= BigInt(record.cost);
|
||||
}
|
||||
|
||||
const latency =
|
||||
meta?.latencyMs !== undefined
|
||||
? meta.latencyMs
|
||||
: Math.max(0, this.clock.now() - record.admittedAtMs);
|
||||
noteLatency(this.adaptation, latency, this.config.adaptation);
|
||||
noteOutcome(this.adaptation, outcome);
|
||||
this.adaptation.windowCompleted += 1;
|
||||
if (meta?.pressure) setPressure(this.adaptation, meta.pressure);
|
||||
this.releaseVirtual(record);
|
||||
|
||||
this.dispatch();
|
||||
}
|
||||
|
||||
private enqueue(request: AdmissionRequest, cost: number): AdmissionAcquireResult {
|
||||
const id = nextId("q");
|
||||
const maxWait = normalizeRequestCost(
|
||||
request.maxWaitMs ?? this.config.defaultMaxWaitMs,
|
||||
MAX_ADMISSION_WINDOW_MS
|
||||
);
|
||||
const now = this.clock.now();
|
||||
const deadlineMs = Math.min(Number.MAX_SAFE_INTEGER, now + maxWait);
|
||||
|
||||
let settle: {
|
||||
resolve: (v: AdmissionAdmitted) => void;
|
||||
reject: (e: Error) => void;
|
||||
};
|
||||
const promise = new Promise<AdmissionAdmitted>((resolve, reject) => {
|
||||
settle = { resolve, reject };
|
||||
});
|
||||
|
||||
const entry: QueueEntry<QueuedPayload> = {
|
||||
id,
|
||||
tenantKey: request.tenantKey && request.tenantKey.length > 0 ? request.tenantKey : "_default",
|
||||
cost,
|
||||
enqueuedAtMs: now,
|
||||
deadlineMs,
|
||||
payload: {
|
||||
resolve: (v) => settle.resolve(v),
|
||||
reject: (e) => settle.reject(e),
|
||||
signal: request.signal,
|
||||
},
|
||||
};
|
||||
|
||||
if (!this.queue.enqueue(entry)) {
|
||||
return this.reject("ADMISSION_QUEUE_FULL", "admission queue is full");
|
||||
}
|
||||
|
||||
entry.timerId = this.clock.setTimer(
|
||||
() => {
|
||||
this.expireEntry(id, "ADMISSION_DEADLINE", "admission wait deadline exceeded");
|
||||
},
|
||||
Math.max(0, deadlineMs - now)
|
||||
);
|
||||
|
||||
if (request.signal) {
|
||||
const onAbort = () => {
|
||||
this.expireEntry(id, "ADMISSION_ABORTED", "request aborted while queued");
|
||||
};
|
||||
entry.payload.onAbort = onAbort;
|
||||
request.signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
|
||||
// Capacity may have freed between check and enqueue in concurrent hosts; try dispatch.
|
||||
this.dispatch();
|
||||
|
||||
return { status: "queued", promise };
|
||||
}
|
||||
|
||||
private expireEntry(id: string, code: AdmissionRejectCode, message: string): void {
|
||||
const entry = this.queue.removeById(id);
|
||||
if (!entry) return;
|
||||
this.clearEntryTimer(entry);
|
||||
this.detachAbort(entry);
|
||||
entry.payload.reject(createAdmissionRejectError(code, message));
|
||||
this.rejectedCount += 1;
|
||||
// Resume enforce dispatch so a now-fitting successor is not stranded until
|
||||
// unrelated activity. dispatch() is a no-op after shutdown / non-enforce.
|
||||
this.dispatch();
|
||||
}
|
||||
|
||||
private failQueued(
|
||||
entry: QueueEntry<QueuedPayload>,
|
||||
code: AdmissionRejectCode,
|
||||
message: string
|
||||
): void {
|
||||
this.clearEntryTimer(entry);
|
||||
this.detachAbort(entry);
|
||||
entry.payload.reject(createAdmissionRejectError(code, message));
|
||||
this.rejectedCount += 1;
|
||||
}
|
||||
|
||||
private dispatch(): void {
|
||||
if (this.shutDown || this.config.mode !== "enforce") return;
|
||||
|
||||
while (this.queue.size > 0) {
|
||||
const limit = this.adaptation.currentLimit;
|
||||
const available = BigInt(limit) - this.activeCost;
|
||||
if (available <= 0n) return;
|
||||
const entry = this.queue.dequeue(Number(available));
|
||||
if (!entry) return;
|
||||
this.clearEntryTimer(entry);
|
||||
this.detachAbort(entry);
|
||||
if (entry.payload.signal?.aborted) {
|
||||
entry.payload.reject(
|
||||
createAdmissionRejectError("ADMISSION_ABORTED", "request aborted while queued")
|
||||
);
|
||||
this.rejectedCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (this.clock.now() >= entry.deadlineMs) {
|
||||
entry.payload.reject(
|
||||
createAdmissionRejectError("ADMISSION_DEADLINE", "admission wait deadline exceeded")
|
||||
);
|
||||
this.rejectedCount += 1;
|
||||
continue;
|
||||
}
|
||||
entry.payload.resolve(this.admit(entry.cost));
|
||||
}
|
||||
}
|
||||
|
||||
private releaseVirtual(record: ActiveLeaseRecord): void {
|
||||
if (record.virtualDisposition === "active") {
|
||||
this.virtualActiveCost -= record.cost;
|
||||
this.virtualActiveCount -= 1;
|
||||
} else if (record.virtualDisposition === "queued") {
|
||||
this.virtualQueue.removeById(record.id);
|
||||
}
|
||||
record.virtualDisposition = "none";
|
||||
this.dispatchVirtual();
|
||||
}
|
||||
|
||||
private dispatchVirtual(): void {
|
||||
while (this.virtualQueue.size > 0) {
|
||||
const available = this.adaptation.currentLimit - this.virtualActiveCost;
|
||||
if (available <= 0) return;
|
||||
const entry = this.virtualQueue.dequeue(available);
|
||||
if (!entry) return;
|
||||
const record = this.active.get(entry.payload.recordId);
|
||||
if (!record || record.released) continue;
|
||||
record.virtualDisposition = "active";
|
||||
this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost);
|
||||
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
|
||||
}
|
||||
}
|
||||
|
||||
private rebuildVirtualState(enable: boolean): void {
|
||||
this.virtualQueue = new FairCostQueue(this.config.maxQueueCount, this.config.maxQueueCost);
|
||||
this.virtualActiveCost = 0;
|
||||
this.virtualActiveCount = 0;
|
||||
for (const record of this.active.values()) record.virtualDisposition = "none";
|
||||
if (!enable) return;
|
||||
for (const record of this.active.values()) {
|
||||
// Individually oversized work is virtual-rejected, never virtually queued.
|
||||
if (record.cost > this.adaptation.currentLimit) {
|
||||
record.virtualDisposition = "rejected";
|
||||
continue;
|
||||
}
|
||||
if (record.cost <= this.adaptation.currentLimit - this.virtualActiveCost) {
|
||||
record.virtualDisposition = "active";
|
||||
this.virtualActiveCost = addSaturated(this.virtualActiveCost, record.cost);
|
||||
this.virtualActiveCount = addSaturated(this.virtualActiveCount, 1);
|
||||
} else if (
|
||||
this.virtualQueue.enqueue({
|
||||
id: record.id,
|
||||
tenantKey: "_existing",
|
||||
cost: record.cost,
|
||||
enqueuedAtMs: record.admittedAtMs,
|
||||
deadlineMs: Number.MAX_SAFE_INTEGER,
|
||||
payload: { recordId: record.id },
|
||||
})
|
||||
) {
|
||||
record.virtualDisposition = "queued";
|
||||
} else {
|
||||
record.virtualDisposition = "rejected";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private reject(code: AdmissionRejectCode, message: string): AdmissionAcquireResult {
|
||||
this.rejectedCount += 1;
|
||||
return { status: "rejected", code, message };
|
||||
}
|
||||
|
||||
private clearEntryTimer(entry: QueueEntry<QueuedPayload>): void {
|
||||
if (entry.timerId !== undefined) {
|
||||
this.clock.clearTimer(entry.timerId);
|
||||
entry.timerId = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private detachAbort(entry: QueueEntry<QueuedPayload>): void {
|
||||
if (entry.payload.signal && entry.payload.onAbort) {
|
||||
entry.payload.signal.removeEventListener("abort", entry.payload.onAbort);
|
||||
entry.payload.onAbort = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private sampleIntegral(): void {
|
||||
const now = this.clock.now();
|
||||
const dt = now - this.lastSampleMs;
|
||||
if (dt > 0) {
|
||||
// Cap at currentLimit before Number conversion so shadow oversubscription never
|
||||
// feeds an unsafe rounded activeCost into the utilization integral.
|
||||
const limit = this.adaptation.currentLimit;
|
||||
const activeForIntegral = this.activeCost >= BigInt(limit) ? limit : Number(this.activeCost);
|
||||
sampleActiveIntegral(this.adaptation, activeForIntegral, dt);
|
||||
this.lastSampleMs = now;
|
||||
}
|
||||
}
|
||||
|
||||
private armWindowTimer(): void {
|
||||
if (this.windowTimer !== undefined) {
|
||||
this.clock.clearTimer(this.windowTimer);
|
||||
this.windowTimer = undefined;
|
||||
}
|
||||
if (this.shutDown || this.config.mode === "off") return;
|
||||
const tick = () => {
|
||||
this.tick();
|
||||
if (!this.shutDown && this.config.mode !== "off") {
|
||||
this.windowTimer = this.clock.setTimer(tick, this.config.windowMs);
|
||||
}
|
||||
};
|
||||
this.windowTimer = this.clock.setTimer(tick, this.config.windowMs);
|
||||
}
|
||||
}
|
||||
107
open-sse/services/admission/cost.ts
Normal file
107
open-sse/services/admission/cost.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import {
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
type AdmissionCostConfig,
|
||||
type AdmissionCostFeatures,
|
||||
} from "./types.ts";
|
||||
|
||||
export { MAX_ADMISSION_COST_OR_LIMIT };
|
||||
|
||||
export const DEFAULT_ADMISSION_COST_CONFIG: AdmissionCostConfig = Object.freeze({
|
||||
baseCost: 1,
|
||||
bodyBytesPerUnit: 16_384,
|
||||
tokensPerUnit: 1_024,
|
||||
messagesPerUnit: 32,
|
||||
toolsPerUnit: 8,
|
||||
fanoutPerUnit: 1,
|
||||
streamingClassCost: 1,
|
||||
nonStreamingClassCost: 2,
|
||||
maxRequestCost: 1_000,
|
||||
});
|
||||
|
||||
function finiteNonNegative(value: unknown): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0;
|
||||
return Math.min(value, Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
|
||||
function requirePositiveSafeInteger(
|
||||
name: string,
|
||||
value: unknown,
|
||||
max: number = MAX_ADMISSION_COST_OR_LIMIT
|
||||
): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
if (value > max) {
|
||||
throw new RangeError(`${name} must be <= ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const COST_CONFIG_KEYS = [
|
||||
"baseCost",
|
||||
"bodyBytesPerUnit",
|
||||
"tokensPerUnit",
|
||||
"messagesPerUnit",
|
||||
"toolsPerUnit",
|
||||
"fanoutPerUnit",
|
||||
"streamingClassCost",
|
||||
"nonStreamingClassCost",
|
||||
"maxRequestCost",
|
||||
] as const satisfies ReadonlyArray<keyof AdmissionCostConfig>;
|
||||
|
||||
/** Merge cost quanta after strictly validating every supplied value. */
|
||||
export function resolveCostConfig(partial?: Partial<AdmissionCostConfig>): AdmissionCostConfig {
|
||||
const d = DEFAULT_ADMISSION_COST_CONFIG;
|
||||
const resolved = {} as AdmissionCostConfig;
|
||||
for (const key of COST_CONFIG_KEYS) {
|
||||
resolved[key] = requirePositiveSafeInteger(key, partial?.[key] ?? d[key]);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function unitsFrom(amount: number, quantum: number): number {
|
||||
return amount <= 0 ? 0 : Math.ceil(amount / quantum);
|
||||
}
|
||||
|
||||
function addBounded(total: number, contribution: number, maximum: number): number {
|
||||
if (contribution >= maximum - total) return maximum;
|
||||
return total + contribution;
|
||||
}
|
||||
|
||||
/** Pure bounded cost estimator from transparent positive safe-integer quanta. */
|
||||
export function estimateAdmissionCost(
|
||||
features: AdmissionCostFeatures,
|
||||
config?: Partial<AdmissionCostConfig>
|
||||
): number {
|
||||
const cfg = resolveCostConfig(config);
|
||||
const body = finiteNonNegative(features?.bodyBytes);
|
||||
const tokens = finiteNonNegative(features?.estimatedInputTokens);
|
||||
const messages = finiteNonNegative(features?.messageCount);
|
||||
const tools = finiteNonNegative(features?.toolCount);
|
||||
const fanout = Math.max(1, finiteNonNegative(features?.requestedFanout));
|
||||
const contributions = [
|
||||
unitsFrom(body, cfg.bodyBytesPerUnit),
|
||||
unitsFrom(tokens, cfg.tokensPerUnit),
|
||||
unitsFrom(messages, cfg.messagesPerUnit),
|
||||
unitsFrom(tools, cfg.toolsPerUnit),
|
||||
unitsFrom(fanout, cfg.fanoutPerUnit),
|
||||
features?.streaming !== false ? cfg.streamingClassCost : cfg.nonStreamingClassCost,
|
||||
];
|
||||
|
||||
let total = Math.min(cfg.baseCost, cfg.maxRequestCost);
|
||||
for (const contribution of contributions) {
|
||||
total = addBounded(total, contribution, cfg.maxRequestCost);
|
||||
if (total === cfg.maxRequestCost) break;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Validate and bound a caller-supplied request cost. */
|
||||
export function normalizeRequestCost(
|
||||
cost: unknown,
|
||||
maxRequestCost: number = DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost
|
||||
): number {
|
||||
const max = requirePositiveSafeInteger("maxRequestCost", maxRequestCost);
|
||||
const value = requirePositiveSafeInteger("request cost", cost);
|
||||
return Math.min(value, max);
|
||||
}
|
||||
37
open-sse/services/admission/index.ts
Normal file
37
open-sse/services/admission/index.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Pure weighted adaptive admission-control core.
|
||||
* No route, settings, or environment wiring in this module surface.
|
||||
*/
|
||||
|
||||
export {
|
||||
DEFAULT_ADMISSION_COST_CONFIG,
|
||||
estimateAdmissionCost,
|
||||
normalizeRequestCost,
|
||||
resolveCostConfig,
|
||||
} from "./cost.ts";
|
||||
|
||||
export { AdaptiveAdmissionController } from "./controller.ts";
|
||||
|
||||
export {
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
MAX_ADMISSION_WINDOW_MS,
|
||||
createAdmissionRejectError,
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionAcquireResult,
|
||||
type AdmissionAdmitted,
|
||||
type AdmissionClock,
|
||||
type AdmissionCostConfig,
|
||||
type AdmissionCostFeatures,
|
||||
type AdmissionLease,
|
||||
type AdmissionMode,
|
||||
type AdmissionPressure,
|
||||
type AdmissionQueued,
|
||||
type AdmissionRejectCode,
|
||||
type AdmissionRejectError,
|
||||
type AdmissionRejected,
|
||||
type AdmissionReleaseMeta,
|
||||
type AdmissionReleaseOutcome,
|
||||
type AdmissionRequest,
|
||||
type AdmissionSnapshot,
|
||||
type ShadowDecision,
|
||||
} from "./types.ts";
|
||||
194
open-sse/services/admission/queue.ts
Normal file
194
open-sse/services/admission/queue.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Bounded multi-tenant fair queue (round-robin across tenant buckets).
|
||||
* Count + total cost caps; no unbounded arrays of timers beyond one per entry.
|
||||
*/
|
||||
|
||||
/**
|
||||
* After this many pass-overs while unfittable, reserve capacity for the aged head
|
||||
* instead of indefinitely admitting smaller work from other tenants.
|
||||
*/
|
||||
const MAX_UNFITTABLE_SKIPS = 2;
|
||||
|
||||
export interface QueueEntry<T> {
|
||||
id: string;
|
||||
tenantKey: string;
|
||||
cost: number;
|
||||
enqueuedAtMs: number;
|
||||
deadlineMs: number;
|
||||
payload: T;
|
||||
timerId?: unknown;
|
||||
/** Times this head was skipped because it did not fit available cost. */
|
||||
skipCount?: number;
|
||||
}
|
||||
|
||||
export interface FairQueueSnapshot {
|
||||
count: number;
|
||||
cost: number;
|
||||
}
|
||||
|
||||
export class FairCostQueue<T> {
|
||||
private readonly buckets = new Map<string, QueueEntry<T>[]>();
|
||||
private readonly order: string[] = [];
|
||||
private cursor = 0;
|
||||
private count = 0;
|
||||
private cost = 0;
|
||||
|
||||
constructor(
|
||||
readonly maxCount: number,
|
||||
readonly maxCost: number
|
||||
) {}
|
||||
|
||||
get size(): number {
|
||||
return this.count;
|
||||
}
|
||||
|
||||
get totalCost(): number {
|
||||
return this.cost;
|
||||
}
|
||||
|
||||
snapshot(): FairQueueSnapshot {
|
||||
return { count: this.count, cost: this.cost };
|
||||
}
|
||||
|
||||
canAccept(entryCost: number): boolean {
|
||||
if (!Number.isSafeInteger(entryCost) || entryCost <= 0) return false;
|
||||
if (this.count >= this.maxCount) return false;
|
||||
if (entryCost > this.maxCost - this.cost) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
enqueue(entry: QueueEntry<T>): boolean {
|
||||
if (!this.canAccept(entry.cost)) return false;
|
||||
let bucket = this.buckets.get(entry.tenantKey);
|
||||
if (!bucket) {
|
||||
bucket = [];
|
||||
this.buckets.set(entry.tenantKey, bucket);
|
||||
this.order.push(entry.tenantKey);
|
||||
}
|
||||
bucket.push(entry);
|
||||
this.count += 1;
|
||||
this.cost += entry.cost;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Round-robin dequeue, optionally skipping tenant heads that do not fit available cost.
|
||||
* After MAX_UNFITTABLE_SKIPS actual pass-overs, an unfittable head reserves capacity:
|
||||
* smaller work is not admitted ahead of it until it fits, is removed, or capacity rises.
|
||||
*/
|
||||
dequeue(maxCost = Number.MAX_SAFE_INTEGER): QueueEntry<T> | undefined {
|
||||
if (this.count === 0) return undefined;
|
||||
const n = this.order.length;
|
||||
|
||||
// Bounded anti-starvation: prefer the oldest aged unfittable head once reserved.
|
||||
let reserved: { idx: number; entry: QueueEntry<T> } | undefined;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const idx = (this.cursor + i) % n;
|
||||
const tenant = this.order[idx];
|
||||
const entry = this.buckets.get(tenant)?.[0];
|
||||
if (!entry) continue;
|
||||
if ((entry.skipCount ?? 0) >= MAX_UNFITTABLE_SKIPS) {
|
||||
if (!reserved || entry.enqueuedAtMs < reserved.entry.enqueuedAtMs) {
|
||||
reserved = { idx, entry };
|
||||
}
|
||||
}
|
||||
}
|
||||
if (reserved) {
|
||||
if (reserved.entry.cost > maxCost) return undefined;
|
||||
return this.takeAt(reserved.idx);
|
||||
}
|
||||
|
||||
const bypassed: QueueEntry<T>[] = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const idx = (this.cursor + i) % n;
|
||||
const tenant = this.order[idx];
|
||||
const bucket = this.buckets.get(tenant);
|
||||
const entry = bucket?.[0];
|
||||
if (!entry) continue;
|
||||
if (entry.cost > maxCost) {
|
||||
bypassed.push(entry);
|
||||
continue;
|
||||
}
|
||||
// Only an actual smaller admission counts as a pass-over. Merely polling
|
||||
// with no available capacity must not age a head into reservation.
|
||||
for (const skipped of bypassed) {
|
||||
skipped.skipCount = (skipped.skipCount ?? 0) + 1;
|
||||
}
|
||||
return this.takeAt(idx);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private takeAt(idx: number): QueueEntry<T> | undefined {
|
||||
const tenant = this.order[idx];
|
||||
const bucket = this.buckets.get(tenant);
|
||||
const entry = bucket?.[0];
|
||||
if (!entry) return undefined;
|
||||
bucket!.shift();
|
||||
this.count -= 1;
|
||||
this.cost -= entry.cost;
|
||||
entry.skipCount = 0;
|
||||
if (bucket!.length === 0) {
|
||||
this.buckets.delete(tenant);
|
||||
this.order.splice(idx, 1);
|
||||
this.cursor = this.order.length === 0 ? 0 : idx % this.order.length;
|
||||
} else {
|
||||
this.cursor = (idx + 1) % this.order.length;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Peek next without removing (for oversized-vs-limit checks). */
|
||||
peek(): QueueEntry<T> | undefined {
|
||||
if (this.count === 0) return undefined;
|
||||
const n = this.order.length;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const idx = (this.cursor + i) % n;
|
||||
const tenant = this.order[idx];
|
||||
const bucket = this.buckets.get(tenant);
|
||||
if (bucket && bucket.length > 0) return bucket[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
removeById(id: string): QueueEntry<T> | undefined {
|
||||
for (let ti = 0; ti < this.order.length; ti++) {
|
||||
const tenant = this.order[ti];
|
||||
const bucket = this.buckets.get(tenant);
|
||||
if (!bucket) continue;
|
||||
const idx = bucket.findIndex((e) => e.id === id);
|
||||
if (idx < 0) continue;
|
||||
const [entry] = bucket.splice(idx, 1);
|
||||
this.count -= 1;
|
||||
this.cost -= entry.cost;
|
||||
if (bucket.length === 0) {
|
||||
this.buckets.delete(tenant);
|
||||
this.order.splice(ti, 1);
|
||||
if (this.order.length === 0) {
|
||||
this.cursor = 0;
|
||||
} else if (ti < this.cursor) {
|
||||
// Removing a prior bucket shifts the successor into cursor - 1.
|
||||
this.cursor -= 1;
|
||||
} else if (this.cursor >= this.order.length) {
|
||||
// Removed the final bucket at the cursor; wrap to the head.
|
||||
this.cursor = 0;
|
||||
}
|
||||
// ti === cursor: leave cursor so it now points at the logical successor.
|
||||
// ti > cursor: cursor is unaffected.
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
drain(): QueueEntry<T>[] {
|
||||
const out: QueueEntry<T>[] = [];
|
||||
while (true) {
|
||||
const e = this.dequeue();
|
||||
if (!e) break;
|
||||
out.push(e);
|
||||
}
|
||||
this.cursor = 0;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
186
open-sse/services/admission/requestFeatures.ts
Normal file
186
open-sse/services/admission/requestFeatures.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Cheap bounded admission cost features from an already-parsed request body.
|
||||
* Never re-parses, stringifies, clones, or invokes toJSON.
|
||||
*/
|
||||
|
||||
import { estimateSizeFast } from "../../utils/estimateSize.ts";
|
||||
import type { AdmissionCostFeatures } from "./types.ts";
|
||||
|
||||
export type AdmissionFeatureExtractionContext = {
|
||||
/** When set, wins over any body/wrapped stream field. */
|
||||
streaming?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Max tools/functions array entries inspected.
|
||||
* Uninspected tail is charged conservatively so truncation cannot undercharge cost.
|
||||
*/
|
||||
export const ADMISSION_TOOL_SCAN_BUDGET = 64;
|
||||
|
||||
type FeatureDraft = {
|
||||
messageCount: number;
|
||||
toolCount: number;
|
||||
requestedFanout: number | null;
|
||||
streaming: boolean | null;
|
||||
};
|
||||
|
||||
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] | null {
|
||||
return Array.isArray(value) ? value : null;
|
||||
}
|
||||
|
||||
function positiveInt(value: unknown): number | null {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null;
|
||||
if (!Number.isSafeInteger(value)) {
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(value));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function saturateCount(n: number): number {
|
||||
if (!Number.isFinite(n) || n <= 0) return 0;
|
||||
if (!Number.isSafeInteger(n)) {
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, Math.floor(n));
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count all recognized tool aliases/layers under one shared entry budget.
|
||||
* If their combined length cannot be inspected completely, saturate before indexed access
|
||||
* so an unseen alias or wrapped tail cannot undercharge heavier declarations.
|
||||
*/
|
||||
function countTools(layers: Array<Record<string, unknown>>): number {
|
||||
const sources: unknown[][] = [];
|
||||
const seen = new Set<unknown[]>();
|
||||
for (const layer of layers) {
|
||||
for (const value of [layer.tools, layer.functions]) {
|
||||
const source = asArray(value);
|
||||
if (!source || seen.has(source)) continue;
|
||||
seen.add(source);
|
||||
sources.push(source);
|
||||
}
|
||||
}
|
||||
|
||||
let entryCount = 0;
|
||||
for (const source of sources) {
|
||||
if (source.length > ADMISSION_TOOL_SCAN_BUDGET - entryCount) {
|
||||
return Number.MAX_SAFE_INTEGER;
|
||||
}
|
||||
entryCount += source.length;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
for (const source of sources) {
|
||||
for (let i = 0; i < source.length; i++) {
|
||||
const entry = source[i];
|
||||
if (isPlainObject(entry)) {
|
||||
const declarations = asArray(entry.functionDeclarations);
|
||||
if (declarations) {
|
||||
total = Math.min(Number.MAX_SAFE_INTEGER, total + saturateCount(declarations.length));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
total = Math.min(Number.MAX_SAFE_INTEGER, total + 1);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function countMessages(layer: Record<string, unknown>): number {
|
||||
const messages = asArray(layer.messages);
|
||||
const contents = asArray(layer.contents);
|
||||
const inputArr = asArray(layer.input);
|
||||
let count = Math.max(
|
||||
saturateCount(messages?.length ?? 0),
|
||||
saturateCount(contents?.length ?? 0),
|
||||
saturateCount(inputArr?.length ?? 0)
|
||||
);
|
||||
// Responses API: non-empty string `input` is one input item.
|
||||
if (count === 0 && typeof layer.input === "string" && layer.input.length > 0) {
|
||||
count = 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function readFanout(layer: Record<string, unknown>): number | null {
|
||||
const direct =
|
||||
positiveInt(layer.n) ?? positiveInt(layer.candidateCount) ?? positiveInt(layer.candidate_count);
|
||||
if (direct != null) return direct;
|
||||
// Known nested Gemini/Antigravity shape only — no recursive walk.
|
||||
if (isPlainObject(layer.generationConfig)) {
|
||||
return (
|
||||
positiveInt(layer.generationConfig.candidateCount) ??
|
||||
positiveInt(layer.generationConfig.candidate_count)
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function featureLayers(body: unknown): Array<Record<string, unknown>> {
|
||||
const top = isPlainObject(body) ? body : null;
|
||||
const wrapped = top && isPlainObject(top.request) ? top.request : null;
|
||||
const layers: Array<Record<string, unknown>> = [];
|
||||
if (top) layers.push(top);
|
||||
if (wrapped) layers.push(wrapped);
|
||||
return layers;
|
||||
}
|
||||
|
||||
function absorbLayer(draft: FeatureDraft, layer: Record<string, unknown>): void {
|
||||
if (draft.messageCount === 0) {
|
||||
draft.messageCount = countMessages(layer);
|
||||
}
|
||||
if (draft.requestedFanout == null) {
|
||||
draft.requestedFanout = readFanout(layer);
|
||||
}
|
||||
if (draft.streaming == null && "stream" in layer) {
|
||||
draft.streaming = layer.stream === true;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveStreaming(
|
||||
draftStreaming: boolean | null,
|
||||
context?: AdmissionFeatureExtractionContext
|
||||
): boolean {
|
||||
if (context && "streaming" in context && context.streaming !== undefined) {
|
||||
return context.streaming === true;
|
||||
}
|
||||
return draftStreaming ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect top-level fields and one known wrapper (`request`) only.
|
||||
* Prefer the first non-empty match for each feature family.
|
||||
*/
|
||||
export function extractAdmissionCostFeatures(
|
||||
body: unknown,
|
||||
context?: AdmissionFeatureExtractionContext
|
||||
): AdmissionCostFeatures {
|
||||
const bodyBytes = estimateSizeFast(body);
|
||||
const layers = featureLayers(body);
|
||||
const draft: FeatureDraft = {
|
||||
messageCount: 0,
|
||||
toolCount: countTools(layers),
|
||||
requestedFanout: null,
|
||||
streaming: null,
|
||||
};
|
||||
for (const layer of layers) {
|
||||
absorbLayer(draft, layer);
|
||||
}
|
||||
|
||||
// Conservative token estimate from already-measured body size (no re-walk/stringify).
|
||||
const estimatedInputTokens =
|
||||
bodyBytes > 0 ? Math.min(Number.MAX_SAFE_INTEGER, Math.ceil(bodyBytes / 4)) : 0;
|
||||
|
||||
return {
|
||||
bodyBytes,
|
||||
estimatedInputTokens,
|
||||
messageCount: draft.messageCount,
|
||||
toolCount: draft.toolCount,
|
||||
requestedFanout: draft.requestedFanout ?? 1,
|
||||
streaming: resolveStreaming(draft.streaming, context),
|
||||
};
|
||||
}
|
||||
614
open-sse/services/admission/runtime.ts
Normal file
614
open-sse/services/admission/runtime.ts
Normal file
@@ -0,0 +1,614 @@
|
||||
/**
|
||||
* Process-local adaptive admission runtime facade around the pure controller.
|
||||
* No HTTP route wiring — suitable for later shared handleChat integration.
|
||||
*/
|
||||
|
||||
import { AdaptiveAdmissionController } from "./controller.ts";
|
||||
import { validateConfig } from "./config.ts";
|
||||
import { extractAdmissionCostFeatures } from "./requestFeatures.ts";
|
||||
import {
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionAcquireResult,
|
||||
type AdmissionClock,
|
||||
type AdmissionLease,
|
||||
type AdmissionMode,
|
||||
type AdmissionPressure,
|
||||
type AdmissionRejectCode,
|
||||
type AdmissionReleaseOutcome,
|
||||
type AdmissionSnapshot,
|
||||
type ShadowDecision,
|
||||
} from "./types.ts";
|
||||
import { buildErrorBody } from "../../utils/error.ts";
|
||||
import { CORS_HEADERS } from "../../utils/cors.ts";
|
||||
import {
|
||||
checkResourcePressureGuard,
|
||||
getResourcePressureObservation,
|
||||
type ResourcePressureGuardResult,
|
||||
type ResourcePressureObservation,
|
||||
} from "../../utils/resourcePressure.ts";
|
||||
import type { PressureReason, PressureSeverity } from "../../utils/resourcePressurePolicy.ts";
|
||||
|
||||
export { extractAdmissionCostFeatures } from "./requestFeatures.ts";
|
||||
|
||||
export const DEFAULT_ADAPTIVE_ADMISSION_CONFIG: Readonly<AdaptiveAdmissionConfig> = Object.freeze({
|
||||
mode: "shadow",
|
||||
minLimit: 8,
|
||||
initialLimit: 64,
|
||||
maxLimit: 1000,
|
||||
maxQueueCount: 128,
|
||||
maxQueueCost: 2000,
|
||||
defaultMaxWaitMs: 5_000,
|
||||
windowMs: 1_000,
|
||||
});
|
||||
|
||||
const RUNTIME_STORE_KEY = Symbol.for("omniroute.adaptiveAdmission.runtime");
|
||||
|
||||
type RuntimeStore = {
|
||||
runtime: AdaptiveAdmissionRuntime | null;
|
||||
};
|
||||
|
||||
type GlobalWithRuntimeStore = typeof globalThis & {
|
||||
[RUNTIME_STORE_KEY]?: RuntimeStore;
|
||||
};
|
||||
|
||||
function getRuntimeStore(): RuntimeStore {
|
||||
const globalWithStore = globalThis as GlobalWithRuntimeStore;
|
||||
let store = globalWithStore[RUNTIME_STORE_KEY];
|
||||
if (!store) {
|
||||
store = { runtime: null };
|
||||
globalWithStore[RUNTIME_STORE_KEY] = store;
|
||||
}
|
||||
return store;
|
||||
}
|
||||
|
||||
const ENV_KEYS = {
|
||||
mode: "ADAPTIVE_ADMISSION_MODE",
|
||||
minLimit: "ADAPTIVE_ADMISSION_MIN_LIMIT",
|
||||
initialLimit: "ADAPTIVE_ADMISSION_INITIAL_LIMIT",
|
||||
maxLimit: "ADAPTIVE_ADMISSION_MAX_LIMIT",
|
||||
maxQueueCount: "ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT",
|
||||
maxQueueCost: "ADAPTIVE_ADMISSION_MAX_QUEUE_COST",
|
||||
defaultMaxWaitMs: "ADAPTIVE_ADMISSION_MAX_WAIT_MS",
|
||||
windowMs: "ADAPTIVE_ADMISSION_WINDOW_MS",
|
||||
} as const;
|
||||
|
||||
function parsePositiveSafeInt(name: string, raw: string): number {
|
||||
if (!/^[0-9]+$/.test(raw)) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
const value = Number(raw);
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Strict env → config resolver. Throws clear config errors for direct callers. */
|
||||
export function resolveAdaptiveAdmissionConfigFromEnv(
|
||||
env: NodeJS.ProcessEnv | Record<string, string | undefined> = process.env
|
||||
): AdaptiveAdmissionConfig {
|
||||
const cfg: AdaptiveAdmissionConfig = { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG };
|
||||
|
||||
const modeRaw = env[ENV_KEYS.mode];
|
||||
if (modeRaw !== undefined && modeRaw !== "") {
|
||||
if (modeRaw !== "off" && modeRaw !== "shadow" && modeRaw !== "enforce") {
|
||||
throw new RangeError(`${ENV_KEYS.mode} must be off|shadow|enforce`);
|
||||
}
|
||||
cfg.mode = modeRaw;
|
||||
}
|
||||
|
||||
// Numeric env keys only — typed assignment without index-signature cast (TS2352).
|
||||
type EnvIntField = Exclude<keyof typeof ENV_KEYS, "mode">;
|
||||
const intFields = [
|
||||
"minLimit",
|
||||
"initialLimit",
|
||||
"maxLimit",
|
||||
"maxQueueCount",
|
||||
"maxQueueCost",
|
||||
"defaultMaxWaitMs",
|
||||
"windowMs",
|
||||
] as const satisfies ReadonlyArray<EnvIntField>;
|
||||
for (const field of intFields) {
|
||||
const envName = ENV_KEYS[field];
|
||||
const raw = env[envName];
|
||||
if (raw === undefined || raw === "") continue;
|
||||
cfg[field] = parsePositiveSafeInt(envName, raw);
|
||||
}
|
||||
|
||||
// Shared pure validation — accept exact documented maxima, reject core-invalid configs.
|
||||
validateConfig(cfg);
|
||||
return cfg;
|
||||
}
|
||||
|
||||
export type AdaptiveAdmissionAcquireInput = {
|
||||
/** Opaque fairness key; never exposed in snapshots or client errors. */
|
||||
tenantKey: string;
|
||||
/** Already-parsed request body — must not be re-read or stringified for cost. */
|
||||
body: unknown;
|
||||
signal?: AbortSignal;
|
||||
maxWaitMs?: number;
|
||||
/** Authoritative streaming class; wins body stream inference when set. */
|
||||
streaming?: boolean;
|
||||
};
|
||||
|
||||
export type AdaptiveAdmissionAdmitted = {
|
||||
status: "admitted";
|
||||
mode: AdmissionMode;
|
||||
lease: AdmissionLease;
|
||||
admittedAtMs: number;
|
||||
shadowDecision?: ShadowDecision;
|
||||
};
|
||||
|
||||
export type AdaptiveAdmissionRejected = {
|
||||
status: "rejected";
|
||||
code: string;
|
||||
response: Response;
|
||||
};
|
||||
|
||||
export type AdaptiveAdmissionAcquireResult = AdaptiveAdmissionAdmitted | AdaptiveAdmissionRejected;
|
||||
|
||||
export type AdaptiveAdmissionPublicSnapshot = AdmissionSnapshot & {
|
||||
resourceSeverity: PressureSeverity;
|
||||
resourceReason: PressureReason;
|
||||
resourceObservedAtMs: number;
|
||||
pressureGuardRejectCount: number;
|
||||
};
|
||||
|
||||
export type AdaptiveAdmissionLifecycleOptions = {
|
||||
admittedAtMs: number;
|
||||
signal?: AbortSignal;
|
||||
nowMs?: () => number;
|
||||
};
|
||||
|
||||
export type AdaptiveAdmissionRuntimeOptions = {
|
||||
config?: AdaptiveAdmissionConfig;
|
||||
env?: NodeJS.ProcessEnv | Record<string, string | undefined>;
|
||||
clock?: Partial<AdmissionClock>;
|
||||
checkResourcePressure?: () => ResourcePressureGuardResult | null;
|
||||
getResourcePressureObservation?: () => ResourcePressureObservation;
|
||||
/** Test seam: observe pressure values fed into the controller after dedupe. */
|
||||
onPressureObserved?: (pressure: AdmissionPressure) => void;
|
||||
warn?: (message: string) => void;
|
||||
nowMs?: () => number;
|
||||
};
|
||||
|
||||
/** Non-success release outcomes callers must choose explicitly for handler failures. */
|
||||
export type AdaptiveAdmissionFailureOutcome = Exclude<AdmissionReleaseOutcome, "success">;
|
||||
|
||||
export type AdaptiveAdmissionRuntime = {
|
||||
acquire(input: AdaptiveAdmissionAcquireInput): Promise<AdaptiveAdmissionAcquireResult>;
|
||||
snapshot(): AdaptiveAdmissionPublicSnapshot;
|
||||
dispose(): void;
|
||||
/**
|
||||
* Release an admitted lease after a handler failure before any HTTP response exists.
|
||||
* Callers must supply the concrete non-success outcome — never defaults to local_reject.
|
||||
*/
|
||||
releaseHandlerFailure(
|
||||
lease: AdmissionLease,
|
||||
outcome: AdaptiveAdmissionFailureOutcome,
|
||||
options?: { admittedAtMs?: number; nowMs?: () => number }
|
||||
): void;
|
||||
attachResponseLifecycle(
|
||||
response: Response,
|
||||
lease: AdmissionLease,
|
||||
options: AdaptiveAdmissionLifecycleOptions
|
||||
): Response;
|
||||
};
|
||||
|
||||
type RejectHttpMapping = {
|
||||
status: number;
|
||||
code: string;
|
||||
message: string;
|
||||
retryAfter?: string;
|
||||
};
|
||||
|
||||
const REJECT_MAP: Record<AdmissionRejectCode, RejectHttpMapping> = {
|
||||
ADMISSION_ABORTED: {
|
||||
status: 499,
|
||||
code: "admission_aborted",
|
||||
message: "Request aborted",
|
||||
},
|
||||
ADMISSION_OVERSIZED: {
|
||||
status: 503,
|
||||
code: "admission_oversized",
|
||||
message: "Request too large for current capacity",
|
||||
},
|
||||
ADMISSION_QUEUE_FULL: {
|
||||
status: 503,
|
||||
code: "admission_queue_full",
|
||||
message: "Service temporarily unavailable",
|
||||
retryAfter: "1",
|
||||
},
|
||||
ADMISSION_DEADLINE: {
|
||||
status: 503,
|
||||
code: "admission_deadline",
|
||||
message: "Service temporarily unavailable",
|
||||
retryAfter: "1",
|
||||
},
|
||||
ADMISSION_SHUTDOWN: {
|
||||
status: 503,
|
||||
code: "admission_shutdown",
|
||||
message: "Service temporarily unavailable",
|
||||
},
|
||||
ADMISSION_UNAVAILABLE: {
|
||||
status: 503,
|
||||
code: "admission_unavailable",
|
||||
message: "Service temporarily unavailable",
|
||||
retryAfter: "1",
|
||||
},
|
||||
};
|
||||
|
||||
function isAdmissionRejectError(
|
||||
err: unknown
|
||||
): err is { code: AdmissionRejectCode; name: string; message: string } {
|
||||
return (
|
||||
!!err &&
|
||||
typeof err === "object" &&
|
||||
(err as { name?: string }).name === "AdmissionRejectError" &&
|
||||
typeof (err as { code?: unknown }).code === "string"
|
||||
);
|
||||
}
|
||||
|
||||
function buildAdmissionRejectResponse(code: AdmissionRejectCode): AdaptiveAdmissionRejected {
|
||||
const mapping = REJECT_MAP[code] ?? REJECT_MAP.ADMISSION_UNAVAILABLE;
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
...CORS_HEADERS,
|
||||
};
|
||||
if (mapping.retryAfter) headers["Retry-After"] = mapping.retryAfter;
|
||||
const body = buildErrorBody(mapping.status, mapping.message, undefined, {
|
||||
type: mapping.status === 499 ? "client_disconnected" : "server_error",
|
||||
code: mapping.code,
|
||||
});
|
||||
return {
|
||||
status: "rejected",
|
||||
code: mapping.code,
|
||||
response: new Response(JSON.stringify(body), {
|
||||
status: mapping.status,
|
||||
headers,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function observationIdentity(state: ResourcePressureObservation["state"]): string {
|
||||
return `${state.observedAtMs}|${state.severity}|${state.reason}`;
|
||||
}
|
||||
|
||||
function toAdmissionPressure(severity: PressureSeverity): AdmissionPressure {
|
||||
if (severity === "critical") return "critical";
|
||||
if (severity === "high") return "high";
|
||||
return "normal";
|
||||
}
|
||||
|
||||
function isSseResponse(response: Response): boolean {
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
return contentType.toLowerCase().includes("text/event-stream");
|
||||
}
|
||||
|
||||
function releaseOnce(
|
||||
lease: AdmissionLease,
|
||||
outcome: AdmissionReleaseOutcome,
|
||||
admittedAtMs: number | undefined,
|
||||
nowMs: () => number
|
||||
): void {
|
||||
if (lease.released) return;
|
||||
const latencyMs = admittedAtMs === undefined ? undefined : Math.max(0, nowMs() - admittedAtMs);
|
||||
lease.release(outcome, latencyMs === undefined ? undefined : { latencyMs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Map HTTP status (+ optional request signal) to admission release outcome.
|
||||
* Cancellation always wins over status classification.
|
||||
*/
|
||||
function classifyHttpOutcome(status: number, signal?: AbortSignal): AdmissionReleaseOutcome {
|
||||
if (signal?.aborted || status === 499) return "cancelled";
|
||||
if (status === 408 || status === 504) return "timeout";
|
||||
if (status >= 500) return "upstream_error";
|
||||
if (status >= 400) return "local_reject";
|
||||
// 2xx / 3xx (and rare 1xx) complete successfully from admission's perspective.
|
||||
return "success";
|
||||
}
|
||||
|
||||
class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime {
|
||||
private readonly controller: AdaptiveAdmissionController;
|
||||
private readonly checkResourcePressure: () => ResourcePressureGuardResult | null;
|
||||
private readonly getResourcePressureObservation: () => ResourcePressureObservation;
|
||||
private readonly onPressureObserved?: (pressure: AdmissionPressure) => void;
|
||||
private readonly nowMs: () => number;
|
||||
private lastObservationKey: string | null = null;
|
||||
private lastResource: {
|
||||
severity: PressureSeverity;
|
||||
reason: PressureReason;
|
||||
observedAtMs: number;
|
||||
} = { severity: "normal", reason: "none", observedAtMs: 0 };
|
||||
private pressureGuardRejectCount = 0;
|
||||
private disposed = false;
|
||||
|
||||
constructor(options: AdaptiveAdmissionRuntimeOptions, config: AdaptiveAdmissionConfig) {
|
||||
this.controller = new AdaptiveAdmissionController(config, options.clock);
|
||||
this.checkResourcePressure = options.checkResourcePressure ?? checkResourcePressureGuard;
|
||||
this.getResourcePressureObservation =
|
||||
options.getResourcePressureObservation ?? getResourcePressureObservation;
|
||||
this.onPressureObserved = options.onPressureObserved;
|
||||
this.nowMs = options.nowMs ?? options.clock?.now ?? (() => Date.now());
|
||||
}
|
||||
|
||||
async acquire(input: AdaptiveAdmissionAcquireInput): Promise<AdaptiveAdmissionAcquireResult> {
|
||||
if (this.disposed) {
|
||||
return buildAdmissionRejectResponse("ADMISSION_SHUTDOWN");
|
||||
}
|
||||
|
||||
// Independent safety fuse first — never acquire provider work on critical guard.
|
||||
// Still feed pressure observations so the controller learns from critical samples.
|
||||
let guard: ResourcePressureGuardResult | null = null;
|
||||
try {
|
||||
guard = this.checkResourcePressure();
|
||||
} catch {
|
||||
// Fail open on sampling/check failures.
|
||||
}
|
||||
|
||||
this.feedFreshPressureObservation();
|
||||
|
||||
if (guard) {
|
||||
this.pressureGuardRejectCount += 1;
|
||||
return {
|
||||
status: "rejected",
|
||||
code: "resource_pressure",
|
||||
response: guard.response,
|
||||
};
|
||||
}
|
||||
|
||||
const features = extractAdmissionCostFeatures(
|
||||
input.body,
|
||||
input.streaming === undefined ? undefined : { streaming: input.streaming }
|
||||
);
|
||||
let result: AdmissionAcquireResult;
|
||||
try {
|
||||
result = await this.controller.acquire({
|
||||
tenantKey: input.tenantKey,
|
||||
features,
|
||||
signal: input.signal,
|
||||
maxWaitMs: input.maxWaitMs,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAdmissionRejectError(err)) {
|
||||
return buildAdmissionRejectResponse(err.code);
|
||||
}
|
||||
return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE");
|
||||
}
|
||||
|
||||
if (result.status === "rejected") {
|
||||
return buildAdmissionRejectResponse(result.code);
|
||||
}
|
||||
|
||||
if (result.status === "queued") {
|
||||
try {
|
||||
const admitted = await result.promise;
|
||||
return {
|
||||
status: "admitted",
|
||||
mode: this.controller.snapshot().mode,
|
||||
lease: admitted.lease,
|
||||
admittedAtMs: this.nowMs(),
|
||||
shadowDecision: admitted.shadowDecision,
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAdmissionRejectError(err)) {
|
||||
return buildAdmissionRejectResponse(err.code);
|
||||
}
|
||||
return buildAdmissionRejectResponse("ADMISSION_UNAVAILABLE");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: "admitted",
|
||||
mode: this.controller.snapshot().mode,
|
||||
lease: result.lease,
|
||||
admittedAtMs: this.nowMs(),
|
||||
shadowDecision: result.shadowDecision,
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): AdaptiveAdmissionPublicSnapshot {
|
||||
const core = this.controller.snapshot();
|
||||
return {
|
||||
...core,
|
||||
resourceSeverity: this.lastResource.severity,
|
||||
resourceReason: this.lastResource.reason,
|
||||
resourceObservedAtMs: this.lastResource.observedAtMs,
|
||||
pressureGuardRejectCount: this.pressureGuardRejectCount,
|
||||
};
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) return;
|
||||
this.disposed = true;
|
||||
this.controller.shutdown();
|
||||
}
|
||||
|
||||
releaseHandlerFailure(
|
||||
lease: AdmissionLease,
|
||||
outcome: AdaptiveAdmissionFailureOutcome,
|
||||
options?: { admittedAtMs?: number; nowMs?: () => number }
|
||||
): void {
|
||||
releaseOnce(lease, outcome, options?.admittedAtMs, options?.nowMs ?? this.nowMs);
|
||||
}
|
||||
|
||||
attachResponseLifecycle(
|
||||
response: Response,
|
||||
lease: AdmissionLease,
|
||||
options: AdaptiveAdmissionLifecycleOptions
|
||||
): Response {
|
||||
const nowMs = options.nowMs ?? this.nowMs;
|
||||
const admittedAtMs = options.admittedAtMs;
|
||||
|
||||
if (!response.body || !isSseResponse(response)) {
|
||||
releaseOnce(lease, classifyHttpOutcome(response.status, options.signal), admittedAtMs, nowMs);
|
||||
return response;
|
||||
}
|
||||
|
||||
const upstream = response.body;
|
||||
const reader = upstream.getReader();
|
||||
let settled = false;
|
||||
let readerCancelled = false;
|
||||
|
||||
const settle = (outcome: AdmissionReleaseOutcome): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
releaseOnce(lease, outcome, admittedAtMs, nowMs);
|
||||
};
|
||||
|
||||
const cancelReader = (reason?: unknown): void => {
|
||||
if (readerCancelled) return;
|
||||
readerCancelled = true;
|
||||
void reader.cancel(reason).catch(() => {
|
||||
/* ignore cancel races */
|
||||
});
|
||||
};
|
||||
|
||||
const onAbort = (): void => {
|
||||
cancelReader(options.signal?.reason);
|
||||
settle("cancelled");
|
||||
};
|
||||
|
||||
if (options.signal) {
|
||||
if (options.signal.aborted) {
|
||||
onAbort();
|
||||
} else {
|
||||
options.signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
}
|
||||
|
||||
const detachAbort = (): void => {
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
if (settled) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
detachAbort();
|
||||
settle(classifyHttpOutcome(response.status, options.signal));
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(value);
|
||||
} catch (err) {
|
||||
detachAbort();
|
||||
settle(options.signal?.aborted ? "cancelled" : "upstream_error");
|
||||
controller.error(err);
|
||||
}
|
||||
},
|
||||
cancel(reason) {
|
||||
detachAbort();
|
||||
cancelReader(reason);
|
||||
settle("cancelled");
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
}
|
||||
|
||||
private feedFreshPressureObservation(): void {
|
||||
try {
|
||||
const observation = this.getResourcePressureObservation();
|
||||
const state = observation.state;
|
||||
this.lastResource = {
|
||||
severity: state.severity,
|
||||
reason: state.reason,
|
||||
observedAtMs: state.observedAtMs,
|
||||
};
|
||||
const key = observationIdentity(state);
|
||||
if (state.observedAtMs <= 0) return;
|
||||
if (key === this.lastObservationKey) return;
|
||||
this.lastObservationKey = key;
|
||||
const pressure = toAdmissionPressure(state.severity);
|
||||
this.controller.observePressure(pressure);
|
||||
this.onPressureObserved?.(pressure);
|
||||
} catch {
|
||||
// Fail open.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function createRuntimeFromResolvedConfig(
|
||||
options: AdaptiveAdmissionRuntimeOptions,
|
||||
config: AdaptiveAdmissionConfig
|
||||
): AdaptiveAdmissionRuntime {
|
||||
return new AdaptiveAdmissionRuntimeImpl(options, config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an injected adaptive-admission runtime for tests or process use.
|
||||
* Invalid explicit `config` still throws (direct callers want fail-fast).
|
||||
*/
|
||||
export function createAdaptiveAdmissionRuntime(
|
||||
options: AdaptiveAdmissionRuntimeOptions = {}
|
||||
): AdaptiveAdmissionRuntime {
|
||||
const config =
|
||||
options.config ??
|
||||
(options.env
|
||||
? resolveAdaptiveAdmissionConfigFromEnv(options.env)
|
||||
: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG });
|
||||
return createRuntimeFromResolvedConfig(options, config);
|
||||
}
|
||||
|
||||
function warnInvalidDefaultConfig(warn: ((message: string) => void) | undefined): void {
|
||||
const message =
|
||||
"[adaptiveAdmission] invalid environment configuration; using default shadow admission settings";
|
||||
if (warn) {
|
||||
warn(message);
|
||||
return;
|
||||
}
|
||||
console.warn(message);
|
||||
}
|
||||
|
||||
function createDefaultProcessRuntime(
|
||||
options: AdaptiveAdmissionRuntimeOptions = {}
|
||||
): AdaptiveAdmissionRuntime {
|
||||
const warn = options.warn;
|
||||
try {
|
||||
const config =
|
||||
options.config ?? resolveAdaptiveAdmissionConfigFromEnv(options.env ?? process.env);
|
||||
return createRuntimeFromResolvedConfig(options, config);
|
||||
} catch {
|
||||
warnInvalidDefaultConfig(warn);
|
||||
return createRuntimeFromResolvedConfig(options, {
|
||||
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Call-time process-global runtime (HMR-safe via globalThis symbol store). */
|
||||
export function getAdaptiveAdmissionRuntime(): AdaptiveAdmissionRuntime {
|
||||
const store = getRuntimeStore();
|
||||
if (!store.runtime) {
|
||||
store.runtime = createDefaultProcessRuntime();
|
||||
}
|
||||
return store.runtime;
|
||||
}
|
||||
|
||||
/** Dispose previous controller and replace the process-global runtime. */
|
||||
export function reloadAdaptiveAdmissionRuntime(
|
||||
options: AdaptiveAdmissionRuntimeOptions = {}
|
||||
): AdaptiveAdmissionRuntime {
|
||||
const store = getRuntimeStore();
|
||||
store.runtime?.dispose();
|
||||
store.runtime = createDefaultProcessRuntime(options);
|
||||
return store.runtime;
|
||||
}
|
||||
|
||||
/** Test isolation: dispose and clear the process-global runtime slot. */
|
||||
export function resetAdaptiveAdmissionRuntimeForTests(): void {
|
||||
const store = getRuntimeStore();
|
||||
store.runtime?.dispose();
|
||||
store.runtime = null;
|
||||
}
|
||||
171
open-sse/services/admission/types.ts
Normal file
171
open-sse/services/admission/types.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Pure weighted adaptive admission-control types.
|
||||
* No route/settings wiring — dependency-injected controller seam only.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Upper bound for adaptation windows and wait deadlines that participate in
|
||||
* cost×time products (utilization integrals, deadline offsets).
|
||||
* 24h is far beyond practical control windows while keeping the product domain exact.
|
||||
*/
|
||||
export const MAX_ADMISSION_WINDOW_MS = 86_400_000;
|
||||
|
||||
/**
|
||||
* Upper bound for every validated cost, limit, and queue-cost quantum.
|
||||
* Derived so `MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS` remains a
|
||||
* safe integer: a full window at the maximum limit integrates to utilization 1.0
|
||||
* without saturating or rounding Number arithmetic.
|
||||
*/
|
||||
export const MAX_ADMISSION_COST_OR_LIMIT = Math.floor(
|
||||
Number.MAX_SAFE_INTEGER / MAX_ADMISSION_WINDOW_MS
|
||||
);
|
||||
|
||||
export type AdmissionMode = "off" | "shadow" | "enforce";
|
||||
|
||||
export type AdmissionPressure = "normal" | "high" | "critical";
|
||||
|
||||
/** Local outcome categories. Upstream business errors must not collapse capacity. */
|
||||
export type AdmissionReleaseOutcome =
|
||||
"success" | "upstream_error" | "timeout" | "local_reject" | "cancelled";
|
||||
|
||||
export type AdmissionRejectCode =
|
||||
| "ADMISSION_OVERSIZED"
|
||||
| "ADMISSION_QUEUE_FULL"
|
||||
| "ADMISSION_DEADLINE"
|
||||
| "ADMISSION_ABORTED"
|
||||
| "ADMISSION_SHUTDOWN"
|
||||
| "ADMISSION_UNAVAILABLE";
|
||||
|
||||
export type ShadowDecision = "would-admit" | "would-queue" | "would-reject";
|
||||
|
||||
export interface AdmissionCostFeatures {
|
||||
bodyBytes?: number | null;
|
||||
estimatedInputTokens?: number | null;
|
||||
messageCount?: number | null;
|
||||
toolCount?: number | null;
|
||||
requestedFanout?: number | null;
|
||||
streaming?: boolean | null;
|
||||
}
|
||||
|
||||
export interface AdmissionCostConfig {
|
||||
baseCost: number;
|
||||
bodyBytesPerUnit: number;
|
||||
tokensPerUnit: number;
|
||||
messagesPerUnit: number;
|
||||
toolsPerUnit: number;
|
||||
fanoutPerUnit: number;
|
||||
streamingClassCost: number;
|
||||
nonStreamingClassCost: number;
|
||||
maxRequestCost: number;
|
||||
}
|
||||
|
||||
export interface AdaptiveAdmissionConfig {
|
||||
mode?: AdmissionMode;
|
||||
minLimit: number;
|
||||
maxLimit: number;
|
||||
initialLimit: number;
|
||||
maxQueueCount: number;
|
||||
maxQueueCost: number;
|
||||
defaultMaxWaitMs?: number;
|
||||
windowMs?: number;
|
||||
shortLatencyAlpha?: number;
|
||||
longLatencyAlpha?: number;
|
||||
increaseStep?: number;
|
||||
decreaseFactor?: number;
|
||||
criticalDecreaseFactor?: number;
|
||||
highUtilizationThreshold?: number;
|
||||
lowUtilizationThreshold?: number;
|
||||
latencyGradientThreshold?: number;
|
||||
maxIncreasePerWindow?: number;
|
||||
/** Optional cost quanta override used only when callers pass features instead of cost. */
|
||||
cost?: Partial<AdmissionCostConfig>;
|
||||
}
|
||||
|
||||
export interface AdmissionRequest {
|
||||
/** Positive integer cost units. If omitted, `features` + cost config are used. */
|
||||
cost?: number;
|
||||
features?: AdmissionCostFeatures;
|
||||
/** Opaque fairness key; never exposed in snapshots. */
|
||||
tenantKey?: string;
|
||||
maxWaitMs?: number;
|
||||
signal?: AbortSignal;
|
||||
pressure?: AdmissionPressure;
|
||||
}
|
||||
|
||||
export interface AdmissionReleaseMeta {
|
||||
latencyMs?: number;
|
||||
pressure?: AdmissionPressure;
|
||||
}
|
||||
|
||||
export interface AdmissionLease {
|
||||
readonly id: string;
|
||||
readonly cost: number;
|
||||
readonly released: boolean;
|
||||
release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta): void;
|
||||
}
|
||||
|
||||
export interface AdmissionAdmitted {
|
||||
status: "admitted";
|
||||
lease: AdmissionLease;
|
||||
shadowDecision?: ShadowDecision;
|
||||
}
|
||||
|
||||
export interface AdmissionQueued {
|
||||
status: "queued";
|
||||
promise: Promise<AdmissionAdmitted>;
|
||||
}
|
||||
|
||||
export interface AdmissionRejected {
|
||||
status: "rejected";
|
||||
code: AdmissionRejectCode;
|
||||
message: string;
|
||||
shadowDecision?: ShadowDecision;
|
||||
}
|
||||
|
||||
export type AdmissionAcquireResult = AdmissionAdmitted | AdmissionQueued | AdmissionRejected;
|
||||
|
||||
export interface AdmissionSnapshot {
|
||||
mode: AdmissionMode;
|
||||
currentLimit: number;
|
||||
minLimit: number;
|
||||
maxLimit: number;
|
||||
activeCost: number;
|
||||
activeCount: number;
|
||||
queuedCost: number;
|
||||
queuedCount: number;
|
||||
virtualActiveCost: number;
|
||||
virtualActiveCount: number;
|
||||
virtualQueuedCost: number;
|
||||
virtualQueuedCount: number;
|
||||
admittedCount: number;
|
||||
rejectedCount: number;
|
||||
wouldAdmitCount: number;
|
||||
wouldQueueCount: number;
|
||||
wouldRejectCount: number;
|
||||
shortLatencyEwma: number;
|
||||
longLatencyEwma: number;
|
||||
utilization: number;
|
||||
pressure: AdmissionPressure;
|
||||
shutdown: boolean;
|
||||
}
|
||||
|
||||
export interface AdmissionClock {
|
||||
now: () => number;
|
||||
setTimer: (fn: () => void, delayMs: number) => unknown;
|
||||
clearTimer: (id: unknown) => void;
|
||||
}
|
||||
|
||||
export interface AdmissionRejectError extends Error {
|
||||
code: AdmissionRejectCode;
|
||||
name: "AdmissionRejectError";
|
||||
}
|
||||
|
||||
export function createAdmissionRejectError(
|
||||
code: AdmissionRejectCode,
|
||||
message: string
|
||||
): AdmissionRejectError {
|
||||
const err = new Error(message) as AdmissionRejectError;
|
||||
err.name = "AdmissionRejectError";
|
||||
err.code = code;
|
||||
return err;
|
||||
}
|
||||
@@ -1,32 +1,109 @@
|
||||
/**
|
||||
* Fast object-tree size estimator — walks without JSON.stringify.
|
||||
* Safe for circular references (uses WeakSet).
|
||||
* Early-exits at 256KB to avoid wasting CPU on huge payloads.
|
||||
* Fast object-tree size estimator — walks without JSON.stringify / toJSON / clone.
|
||||
* Safe for circular references (WeakSet). Iterative frames only (no recursive call stack).
|
||||
*
|
||||
* Budgets:
|
||||
* - ESTIMATE_SIZE_BYTE_LIMIT (256 KiB): early-exit once counted bytes exceed the limit
|
||||
* - ESTIMATE_SIZE_NODE_BUDGET: max value visits (containers + primitives/elements)
|
||||
*
|
||||
* Arrays are walked by index frame (never pre-push/copy every element reference).
|
||||
* Plain objects yield own enumerable values incrementally (no Object.keys materialization).
|
||||
* Node-budget exhaustion returns a value strictly above 256 KiB so callers fail closed.
|
||||
*/
|
||||
export function estimateSizeFast(value: unknown): number {
|
||||
let bytes = 0;
|
||||
const stack: unknown[] = [value];
|
||||
const seen = new WeakSet();
|
||||
while (stack.length > 0) {
|
||||
const v = stack.pop();
|
||||
if (v === null || v === undefined) continue;
|
||||
if (typeof v === "string") {
|
||||
bytes += v.length;
|
||||
if (bytes > 262144) return bytes;
|
||||
} else if (typeof v === "number") bytes += 8;
|
||||
else if (typeof v === "boolean") bytes += 4;
|
||||
else if (typeof v === "object") {
|
||||
if (seen.has(v as object)) continue;
|
||||
seen.add(v as object);
|
||||
if (Array.isArray(v)) {
|
||||
for (let i = 0; i < v.length; i++) stack.push(v[i]);
|
||||
} else {
|
||||
for (const key in v) {
|
||||
if (Object.prototype.hasOwnProperty.call(v, key)) stack.push((v as Record<string, unknown>)[key]);
|
||||
}
|
||||
|
||||
/** Byte early-exit threshold (256 KiB). */
|
||||
export const ESTIMATE_SIZE_BYTE_LIMIT = 262_144;
|
||||
|
||||
/**
|
||||
* Max value/element visits before fail-closed.
|
||||
* Conservative cap keeps auxiliary stack/WeakSet growth bounded under adversarial input.
|
||||
*/
|
||||
export const ESTIMATE_SIZE_NODE_BUDGET = 16_384;
|
||||
|
||||
type Frame =
|
||||
| { t: "v"; v: unknown }
|
||||
| { t: "a"; a: unknown[]; i: number }
|
||||
| { t: "o"; o: object; it: Iterator<string> };
|
||||
|
||||
function ownEnumerableKeyIterator(obj: object): Iterator<string> {
|
||||
return (function* ownEnumerableKeys() {
|
||||
for (const key in obj) {
|
||||
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
||||
yield key;
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
/** @returns next byte total, or a value > limit when the limit is exceeded. */
|
||||
function addPrimitiveBytes(bytes: number, v: string | number | boolean): number {
|
||||
if (typeof v === "string") return bytes + v.length;
|
||||
if (typeof v === "number") return bytes + 8;
|
||||
return bytes + 4;
|
||||
}
|
||||
|
||||
function enqueueContainer(stack: Frame[], obj: object, seen: WeakSet<object>): void {
|
||||
if (seen.has(obj)) return;
|
||||
seen.add(obj);
|
||||
if (Array.isArray(obj)) {
|
||||
if (obj.length > 0) stack.push({ t: "a", a: obj, i: 0 });
|
||||
return;
|
||||
}
|
||||
stack.push({ t: "o", o: obj, it: ownEnumerableKeyIterator(obj) });
|
||||
}
|
||||
|
||||
type ValueFrame = Extract<Frame, { t: "v" }>;
|
||||
|
||||
function isValueFrame(frame: Frame): frame is ValueFrame {
|
||||
return frame.t === "v";
|
||||
}
|
||||
|
||||
/** Expand a container frame into the next child value. */
|
||||
function expandContainerFrame(stack: Frame[], frame: Exclude<Frame, ValueFrame>): void {
|
||||
if (frame.t === "a") {
|
||||
if (frame.i >= frame.a.length) return;
|
||||
if (frame.i + 1 < frame.a.length) {
|
||||
stack.push({ t: "a", a: frame.a, i: frame.i + 1 });
|
||||
}
|
||||
stack.push({ t: "v", v: frame.a[frame.i] });
|
||||
return;
|
||||
}
|
||||
const next = frame.it.next();
|
||||
if (next.done) return;
|
||||
stack.push(frame);
|
||||
stack.push({ t: "v", v: (frame.o as Record<string, unknown>)[next.value] });
|
||||
}
|
||||
|
||||
export function estimateSizeFast(value: unknown): number {
|
||||
let bytes = 0;
|
||||
let visitsLeft = ESTIMATE_SIZE_NODE_BUDGET;
|
||||
const seen = new WeakSet<object>();
|
||||
const stack: Frame[] = [{ t: "v", v: value }];
|
||||
|
||||
while (stack.length > 0) {
|
||||
if (visitsLeft <= 0) return ESTIMATE_SIZE_BYTE_LIMIT + 1;
|
||||
|
||||
const frame = stack.pop()!;
|
||||
if (!isValueFrame(frame)) {
|
||||
expandContainerFrame(stack, frame);
|
||||
continue;
|
||||
}
|
||||
|
||||
visitsLeft -= 1;
|
||||
const v = frame.v;
|
||||
if (v === null || v === undefined) continue;
|
||||
|
||||
const ty = typeof v;
|
||||
if (ty === "string" || ty === "number" || ty === "boolean") {
|
||||
bytes = addPrimitiveBytes(bytes, v as string | number | boolean);
|
||||
if (bytes > ESTIMATE_SIZE_BYTE_LIMIT) return bytes;
|
||||
continue;
|
||||
}
|
||||
if (ty === "object") {
|
||||
enqueueContainer(stack, v as object, seen);
|
||||
}
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
|
||||
249
open-sse/utils/resourcePressure.ts
Normal file
249
open-sse/utils/resourcePressure.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
import { checkHeapPressureGuard, HEAP_PRESSURE_THRESHOLD_MB } from "./heapPressure.ts";
|
||||
import { buildErrorBody } from "./error.ts";
|
||||
import {
|
||||
createResourcePressureTracker,
|
||||
resolveResourcePressureThresholds,
|
||||
type PressureReason,
|
||||
type ResourcePressureState,
|
||||
type ResourcePressureThresholds,
|
||||
type ResourceSignals,
|
||||
} from "./resourcePressurePolicy.ts";
|
||||
import {
|
||||
sampleResourceSignals,
|
||||
type SampleResourceSignalsDeps,
|
||||
} from "./resourcePressureSampler.ts";
|
||||
|
||||
const MB = 1024 * 1024;
|
||||
const RETRY_AFTER_SECONDS = "5";
|
||||
const PRESSURE_MESSAGE = "Service temporarily unavailable due to resource pressure. Retry shortly.";
|
||||
|
||||
export type ResourcePressureGuardResult = {
|
||||
success: false;
|
||||
status: 503;
|
||||
error: string;
|
||||
response: Response;
|
||||
};
|
||||
|
||||
export type ResourcePressureObservation = {
|
||||
signals: ResourceSignals | null;
|
||||
state: ResourcePressureState;
|
||||
};
|
||||
|
||||
export type ResourcePressureRuntimeOptions = {
|
||||
thresholds?: Partial<ResourcePressureThresholds>;
|
||||
heapThresholdMb?: number | null;
|
||||
immediateHeapUsedMb?: () => number;
|
||||
sample?: () => Promise<ResourceSignals>;
|
||||
nowMs?: () => number;
|
||||
schedule?: (refresh: () => void) => void;
|
||||
staleAfterMs?: number;
|
||||
maxStaleMs?: number;
|
||||
retryAfterMs?: number;
|
||||
samplerDeps?: SampleResourceSignalsDeps;
|
||||
};
|
||||
|
||||
export type ResourcePressureRuntime = {
|
||||
check: () => ResourcePressureGuardResult | null;
|
||||
getObservation: () => ResourcePressureObservation;
|
||||
whenRefreshSettled: () => Promise<void>;
|
||||
dispose: () => void;
|
||||
};
|
||||
|
||||
function emptyState(): ResourcePressureState {
|
||||
return {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function requireDuration(name: string, value: number): number {
|
||||
if (!Number.isFinite(value) || !Number.isInteger(value) || value < 0 || value > 3_600_000) {
|
||||
throw new RangeError(`${name} must be an integer between 0 and 3600000`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function buildCriticalGuard(reason: PressureReason): ResourcePressureGuardResult {
|
||||
console.warn(
|
||||
`[resourcePressure] critical pressure guard tripped (reason=${reason}); returning 503`
|
||||
);
|
||||
return {
|
||||
success: false,
|
||||
status: 503,
|
||||
error: PRESSURE_MESSAGE,
|
||||
response: new Response(
|
||||
JSON.stringify(
|
||||
buildErrorBody(503, PRESSURE_MESSAGE, undefined, {
|
||||
type: "server_error",
|
||||
code: "resource_pressure",
|
||||
})
|
||||
),
|
||||
{
|
||||
status: 503,
|
||||
headers: { "Content-Type": "application/json", "Retry-After": RETRY_AFTER_SECONDS },
|
||||
}
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function immediateHeapGuard(
|
||||
heapUsedMb: number,
|
||||
thresholdMb: number | null
|
||||
): ResourcePressureGuardResult | null {
|
||||
if (thresholdMb == null) return null;
|
||||
const guard = checkHeapPressureGuard(heapUsedMb, thresholdMb);
|
||||
if (!guard) return null;
|
||||
return buildCriticalGuard("v8_heap_absolute");
|
||||
}
|
||||
|
||||
export function createResourcePressureRuntime(
|
||||
options: ResourcePressureRuntimeOptions = {}
|
||||
): ResourcePressureRuntime {
|
||||
const heapThresholdMb =
|
||||
options.heapThresholdMb === undefined ? HEAP_PRESSURE_THRESHOLD_MB : options.heapThresholdMb;
|
||||
if (heapThresholdMb !== null && (!Number.isFinite(heapThresholdMb) || heapThresholdMb <= 0)) {
|
||||
throw new RangeError("heapThresholdMb must be positive and finite or null");
|
||||
}
|
||||
const thresholds = resolveResourcePressureThresholds({
|
||||
...options.thresholds,
|
||||
heapAbsoluteThresholdMb:
|
||||
options.thresholds?.heapAbsoluteThresholdMb === undefined
|
||||
? null
|
||||
: options.thresholds.heapAbsoluteThresholdMb,
|
||||
});
|
||||
const staleAfterMs = requireDuration("staleAfterMs", options.staleAfterMs ?? 1_000);
|
||||
const maxStaleMs = requireDuration("maxStaleMs", options.maxStaleMs ?? 30_000);
|
||||
const retryAfterMs = requireDuration("retryAfterMs", options.retryAfterMs ?? 1_000);
|
||||
if (maxStaleMs < staleAfterMs) {
|
||||
throw new RangeError("maxStaleMs must be greater than or equal to staleAfterMs");
|
||||
}
|
||||
|
||||
const nowMs = options.nowMs ?? Date.now;
|
||||
const immediateHeapUsedMb =
|
||||
options.immediateHeapUsedMb ?? (() => process.memoryUsage().heapUsed / MB);
|
||||
const sample = options.sample ?? (() => sampleResourceSignals(options.samplerDeps));
|
||||
const schedule =
|
||||
options.schedule ??
|
||||
((refresh) => {
|
||||
const handle = setImmediate(refresh);
|
||||
handle.unref();
|
||||
});
|
||||
const tracker = createResourcePressureTracker(thresholds);
|
||||
|
||||
let lastSignals: ResourceSignals | null = null;
|
||||
let state = emptyState();
|
||||
let lastRefreshAtMs = Number.NEGATIVE_INFINITY;
|
||||
let nextRefreshAtMs = Number.NEGATIVE_INFINITY;
|
||||
let scheduled = false;
|
||||
let inFlight: Promise<void> | null = null;
|
||||
let disposed = false;
|
||||
|
||||
const refresh = (): void => {
|
||||
if (disposed || inFlight) return;
|
||||
scheduled = false;
|
||||
inFlight = Promise.resolve()
|
||||
.then(sample)
|
||||
.then((signals) => {
|
||||
if (disposed) return;
|
||||
const settledAtMs = nowMs();
|
||||
lastSignals = signals;
|
||||
state = tracker.observe(signals);
|
||||
lastRefreshAtMs = settledAtMs;
|
||||
nextRefreshAtMs = settledAtMs + staleAfterMs;
|
||||
})
|
||||
.catch(() => {
|
||||
if (!disposed) nextRefreshAtMs = nowMs() + retryAfterMs;
|
||||
})
|
||||
.finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
};
|
||||
|
||||
const scheduleRefresh = (): void => {
|
||||
if (disposed || scheduled || inFlight) return;
|
||||
scheduled = true;
|
||||
schedule(refresh);
|
||||
};
|
||||
|
||||
return {
|
||||
check() {
|
||||
let heapUsedMb = 0;
|
||||
try {
|
||||
heapUsedMb = immediateHeapUsedMb();
|
||||
} catch {
|
||||
heapUsedMb = 0;
|
||||
}
|
||||
const immediate = immediateHeapGuard(heapUsedMb, heapThresholdMb);
|
||||
const now = nowMs();
|
||||
if (now >= nextRefreshAtMs) scheduleRefresh();
|
||||
if (immediate) {
|
||||
state = {
|
||||
severity: "critical",
|
||||
reason: "v8_heap_absolute",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: now,
|
||||
observedAtMs: now,
|
||||
};
|
||||
return immediate;
|
||||
}
|
||||
const cacheAge = lastSignals ? Math.max(0, now - lastRefreshAtMs) : Number.POSITIVE_INFINITY;
|
||||
return cacheAge <= maxStaleMs && state.severity === "critical"
|
||||
? buildCriticalGuard(state.reason)
|
||||
: null;
|
||||
},
|
||||
getObservation: () => ({ signals: lastSignals, state }),
|
||||
whenRefreshSettled: async () => {
|
||||
if (scheduled) await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
if (inFlight) await inFlight;
|
||||
},
|
||||
dispose() {
|
||||
disposed = true;
|
||||
scheduled = false;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let defaultRuntime = createResourcePressureRuntime();
|
||||
|
||||
export function checkResourcePressureGuard(): ResourcePressureGuardResult | null {
|
||||
return defaultRuntime.check();
|
||||
}
|
||||
|
||||
export function getResourcePressureObservation(): ResourcePressureObservation {
|
||||
return defaultRuntime.getObservation();
|
||||
}
|
||||
|
||||
/** Replaces and disposes the process singleton when configuration is reloaded. */
|
||||
export function reloadResourcePressureRuntime(
|
||||
options: ResourcePressureRuntimeOptions = {}
|
||||
): ResourcePressureRuntime {
|
||||
defaultRuntime.dispose();
|
||||
defaultRuntime = createResourcePressureRuntime(options);
|
||||
return defaultRuntime;
|
||||
}
|
||||
|
||||
export type {
|
||||
PressureReason,
|
||||
PressureSeverity,
|
||||
ResourceMetricBytes,
|
||||
ResourcePressureState,
|
||||
ResourcePressureThresholds,
|
||||
ResourcePressureTracker,
|
||||
ResourceSignals,
|
||||
} from "./resourcePressurePolicy.ts";
|
||||
export {
|
||||
classifyAdaptiveResourcePressure as classifyResourcePressure,
|
||||
createResourcePressureTracker,
|
||||
resolveResourcePressureThresholds,
|
||||
} from "./resourcePressurePolicy.ts";
|
||||
export {
|
||||
sampleResourceSignals,
|
||||
sanitizeMemoryBytes,
|
||||
type ResourcePressureFs,
|
||||
type SampleResourceSignalsDeps,
|
||||
} from "./resourcePressureSampler.ts";
|
||||
344
open-sse/utils/resourcePressurePolicy.ts
Normal file
344
open-sse/utils/resourcePressurePolicy.ts
Normal file
@@ -0,0 +1,344 @@
|
||||
const MB = 1024 * 1024;
|
||||
const MAX_SUSTAINED_SAMPLES = 10_000;
|
||||
|
||||
export type PressureSeverity = "normal" | "high" | "critical";
|
||||
|
||||
export type PressureReason =
|
||||
| "none"
|
||||
| "v8_heap_ratio"
|
||||
| "v8_heap_absolute"
|
||||
| "cgroup_ratio"
|
||||
| "cgroup_high"
|
||||
| "psi_some"
|
||||
| "psi_full"
|
||||
| "oom_event";
|
||||
|
||||
export type ResourceMetricBytes = number | null;
|
||||
|
||||
export type ResourceSignals = {
|
||||
observedAtMs: number;
|
||||
v8: { heapUsedBytes: number; heapLimitBytes: number };
|
||||
process: {
|
||||
rssBytes: number;
|
||||
externalBytes: number;
|
||||
arrayBuffersBytes: number;
|
||||
availableBytes: ResourceMetricBytes;
|
||||
constrainedBytes: ResourceMetricBytes;
|
||||
};
|
||||
cgroup: {
|
||||
currentBytes: ResourceMetricBytes;
|
||||
maxBytes: ResourceMetricBytes;
|
||||
highBytes: ResourceMetricBytes;
|
||||
events: {
|
||||
low: ResourceMetricBytes;
|
||||
high: ResourceMetricBytes;
|
||||
max: ResourceMetricBytes;
|
||||
oom: ResourceMetricBytes;
|
||||
oom_kill: ResourceMetricBytes;
|
||||
} | null;
|
||||
};
|
||||
psi: {
|
||||
someAvg10: number | null;
|
||||
someAvg60: number | null;
|
||||
someAvg300: number | null;
|
||||
fullAvg10: number | null;
|
||||
fullAvg60: number | null;
|
||||
fullAvg300: number | null;
|
||||
} | null;
|
||||
};
|
||||
|
||||
export type ResourcePressureState = {
|
||||
severity: PressureSeverity;
|
||||
reason: PressureReason;
|
||||
elevatedStreak: number;
|
||||
recoveryStreak: number;
|
||||
lastTransitionAtMs: number;
|
||||
observedAtMs: number;
|
||||
};
|
||||
|
||||
export type ResourcePressureThresholds = {
|
||||
highRatio: number;
|
||||
criticalRatio: number;
|
||||
recoveryRatio: number;
|
||||
highPsiAvg10: number;
|
||||
criticalPsiAvg10: number;
|
||||
recoveryPsiAvg10: number;
|
||||
sustainedSamplesHigh: number;
|
||||
sustainedSamplesCritical: number;
|
||||
sustainedSamplesRecovery: number;
|
||||
heapAbsoluteThresholdMb: number | null;
|
||||
};
|
||||
|
||||
export const DEFAULT_RESOURCE_PRESSURE_THRESHOLDS: ResourcePressureThresholds = {
|
||||
highRatio: 0.85,
|
||||
criticalRatio: 0.92,
|
||||
recoveryRatio: 0.75,
|
||||
highPsiAvg10: 20,
|
||||
criticalPsiAvg10: 40,
|
||||
recoveryPsiAvg10: 10,
|
||||
sustainedSamplesHigh: 2,
|
||||
sustainedSamplesCritical: 2,
|
||||
sustainedSamplesRecovery: 3,
|
||||
heapAbsoluteThresholdMb: null,
|
||||
};
|
||||
|
||||
type RawLevel = { severity: PressureSeverity; reason: PressureReason };
|
||||
type OomCounters = { oom: number | null; oomKill: number | null };
|
||||
|
||||
function requireFiniteRange(name: string, value: number, minimum: number, maximum: number): void {
|
||||
if (!Number.isFinite(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be finite and between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function requirePositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isInteger(value) || value < 1 || value > MAX_SUSTAINED_SAMPLES) {
|
||||
throw new RangeError(`${name} must be an integer between 1 and ${MAX_SUSTAINED_SAMPLES}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveResourcePressureThresholds(
|
||||
partial: Partial<ResourcePressureThresholds> = {}
|
||||
): ResourcePressureThresholds {
|
||||
const resolved = { ...DEFAULT_RESOURCE_PRESSURE_THRESHOLDS, ...partial };
|
||||
requireFiniteRange("recoveryRatio", resolved.recoveryRatio, 0, 1);
|
||||
requireFiniteRange("highRatio", resolved.highRatio, 0, 1);
|
||||
requireFiniteRange("criticalRatio", resolved.criticalRatio, 0, 1);
|
||||
if (!(
|
||||
resolved.recoveryRatio < resolved.highRatio && resolved.highRatio < resolved.criticalRatio
|
||||
)) {
|
||||
throw new RangeError("ratio thresholds must satisfy recovery < high < critical");
|
||||
}
|
||||
|
||||
requireFiniteRange("recoveryPsiAvg10", resolved.recoveryPsiAvg10, 0, 100);
|
||||
requireFiniteRange("highPsiAvg10", resolved.highPsiAvg10, 0, 100);
|
||||
requireFiniteRange("criticalPsiAvg10", resolved.criticalPsiAvg10, 0, 100);
|
||||
if (!(
|
||||
resolved.recoveryPsiAvg10 < resolved.highPsiAvg10 &&
|
||||
resolved.highPsiAvg10 < resolved.criticalPsiAvg10
|
||||
)) {
|
||||
throw new RangeError("PSI thresholds must satisfy recovery < high < critical");
|
||||
}
|
||||
|
||||
requirePositiveInteger("sustainedSamplesHigh", resolved.sustainedSamplesHigh);
|
||||
requirePositiveInteger("sustainedSamplesCritical", resolved.sustainedSamplesCritical);
|
||||
requirePositiveInteger("sustainedSamplesRecovery", resolved.sustainedSamplesRecovery);
|
||||
if (
|
||||
resolved.heapAbsoluteThresholdMb !== null &&
|
||||
(!Number.isFinite(resolved.heapAbsoluteThresholdMb) || resolved.heapAbsoluteThresholdMb <= 0)
|
||||
) {
|
||||
throw new RangeError("heapAbsoluteThresholdMb must be positive and finite or null");
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function severityRank(severity: PressureSeverity): number {
|
||||
return severity === "critical" ? 2 : severity === "high" ? 1 : 0;
|
||||
}
|
||||
|
||||
function maxLevel(current: RawLevel, candidate: RawLevel | null): RawLevel {
|
||||
if (!candidate || severityRank(candidate.severity) <= severityRank(current.severity)) {
|
||||
return current;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function ratioLevel(
|
||||
used: number | null,
|
||||
limit: number | null,
|
||||
thresholds: ResourcePressureThresholds,
|
||||
reason: PressureReason
|
||||
): RawLevel | null {
|
||||
if (used == null || limit == null || used < 0 || limit <= 0) return null;
|
||||
const ratio = used / limit;
|
||||
if (ratio >= thresholds.criticalRatio) return { severity: "critical", reason };
|
||||
if (ratio >= thresholds.highRatio) return { severity: "high", reason };
|
||||
return null;
|
||||
}
|
||||
|
||||
function psiLevel(
|
||||
value: number | null,
|
||||
thresholds: ResourcePressureThresholds,
|
||||
reason: Extract<PressureReason, "psi_some" | "psi_full">
|
||||
): RawLevel | null {
|
||||
if (value == null || !Number.isFinite(value)) return null;
|
||||
if (value >= thresholds.criticalPsiAvg10) return { severity: "critical", reason };
|
||||
if (value >= thresholds.highPsiAvg10) return { severity: "high", reason };
|
||||
return null;
|
||||
}
|
||||
|
||||
export function classifyAdaptiveResourcePressure(
|
||||
signals: ResourceSignals,
|
||||
thresholds: ResourcePressureThresholds
|
||||
): RawLevel {
|
||||
let best: RawLevel = { severity: "normal", reason: "none" };
|
||||
best = maxLevel(
|
||||
best,
|
||||
ratioLevel(signals.v8.heapUsedBytes, signals.v8.heapLimitBytes, thresholds, "v8_heap_ratio")
|
||||
);
|
||||
best = maxLevel(
|
||||
best,
|
||||
ratioLevel(signals.cgroup.currentBytes, signals.cgroup.maxBytes, thresholds, "cgroup_ratio")
|
||||
);
|
||||
best = maxLevel(
|
||||
best,
|
||||
ratioLevel(signals.cgroup.currentBytes, signals.cgroup.highBytes, thresholds, "cgroup_high")
|
||||
);
|
||||
best = maxLevel(best, psiLevel(signals.psi?.someAvg10 ?? null, thresholds, "psi_some"));
|
||||
return maxLevel(best, psiLevel(signals.psi?.fullAvg10 ?? null, thresholds, "psi_full"));
|
||||
}
|
||||
|
||||
function isRecovered(signals: ResourceSignals, thresholds: ResourcePressureThresholds): boolean {
|
||||
const ratios: Array<readonly [number | null, number | null]> = [
|
||||
[signals.v8.heapUsedBytes, signals.v8.heapLimitBytes],
|
||||
[signals.cgroup.currentBytes, signals.cgroup.maxBytes],
|
||||
[signals.cgroup.currentBytes, signals.cgroup.highBytes],
|
||||
];
|
||||
if (
|
||||
ratios.some(
|
||||
([used, limit]) =>
|
||||
used != null && limit != null && limit > 0 && used / limit > thresholds.recoveryRatio
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
thresholds.heapAbsoluteThresholdMb != null &&
|
||||
signals.v8.heapUsedBytes / MB > thresholds.heapAbsoluteThresholdMb * thresholds.recoveryRatio
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return ![signals.psi?.someAvg10, signals.psi?.fullAvg10].some(
|
||||
(value) => value != null && value > thresholds.recoveryPsiAvg10
|
||||
);
|
||||
}
|
||||
|
||||
function hasCounterIncrease(previous: OomCounters, current: OomCounters): boolean {
|
||||
return (
|
||||
(previous.oom != null && current.oom != null && current.oom > previous.oom) ||
|
||||
(previous.oomKill != null && current.oomKill != null && current.oomKill > previous.oomKill)
|
||||
);
|
||||
}
|
||||
|
||||
function countersReset(previous: OomCounters, current: OomCounters): boolean {
|
||||
return (
|
||||
(previous.oom != null && current.oom != null && current.oom < previous.oom) ||
|
||||
(previous.oomKill != null && current.oomKill != null && current.oomKill < previous.oomKill)
|
||||
);
|
||||
}
|
||||
|
||||
function initialState(): ResourcePressureState {
|
||||
return {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export type ResourcePressureTracker = {
|
||||
observe: (signals: ResourceSignals) => ResourcePressureState;
|
||||
getState: () => ResourcePressureState;
|
||||
};
|
||||
|
||||
export function createResourcePressureTracker(
|
||||
partialThresholds: Partial<ResourcePressureThresholds> = {}
|
||||
): ResourcePressureTracker {
|
||||
const thresholds = resolveResourcePressureThresholds(partialThresholds);
|
||||
let state = initialState();
|
||||
let pending: RawLevel | null = null;
|
||||
let previousOom: OomCounters | null = null;
|
||||
|
||||
return {
|
||||
observe(signals) {
|
||||
const events = signals.cgroup.events;
|
||||
const currentOom = events ? { oom: events.oom, oomKill: events.oom_kill } : null;
|
||||
let oomEvent = false;
|
||||
if (currentOom) {
|
||||
if (previousOom && !countersReset(previousOom, currentOom)) {
|
||||
oomEvent = hasCounterIncrease(previousOom, currentOom);
|
||||
}
|
||||
previousOom = currentOom;
|
||||
} else {
|
||||
previousOom = null;
|
||||
}
|
||||
|
||||
const raw = oomEvent
|
||||
? ({ severity: "critical", reason: "oom_event" } as const)
|
||||
: classifyAdaptiveResourcePressure(signals, thresholds);
|
||||
let { severity, reason, elevatedStreak, recoveryStreak } = state;
|
||||
|
||||
if (oomEvent) {
|
||||
severity = "critical";
|
||||
reason = "oom_event";
|
||||
elevatedStreak = 0;
|
||||
recoveryStreak = 0;
|
||||
pending = null;
|
||||
} else if (severity === "normal") {
|
||||
recoveryStreak = 0;
|
||||
if (raw.severity === "normal") {
|
||||
pending = null;
|
||||
elevatedStreak = 0;
|
||||
reason = "none";
|
||||
} else {
|
||||
const samePending = pending?.severity === raw.severity && pending.reason === raw.reason;
|
||||
pending = raw;
|
||||
elevatedStreak = samePending ? elevatedStreak + 1 : 1;
|
||||
const needed =
|
||||
raw.severity === "critical"
|
||||
? thresholds.sustainedSamplesCritical
|
||||
: thresholds.sustainedSamplesHigh;
|
||||
if (elevatedStreak >= needed) {
|
||||
severity = raw.severity;
|
||||
reason = raw.reason;
|
||||
elevatedStreak = 0;
|
||||
pending = null;
|
||||
}
|
||||
}
|
||||
} else if (severity === "high" && raw.severity === "critical") {
|
||||
recoveryStreak = 0;
|
||||
const samePending = pending?.severity === "critical" && pending.reason === raw.reason;
|
||||
pending = raw;
|
||||
elevatedStreak = samePending ? elevatedStreak + 1 : 1;
|
||||
if (elevatedStreak >= thresholds.sustainedSamplesCritical) {
|
||||
severity = "critical";
|
||||
reason = raw.reason;
|
||||
elevatedStreak = 0;
|
||||
pending = null;
|
||||
}
|
||||
} else if (raw.severity === severity) {
|
||||
reason = raw.reason;
|
||||
pending = null;
|
||||
elevatedStreak = 0;
|
||||
recoveryStreak = 0;
|
||||
} else if (isRecovered(signals, thresholds)) {
|
||||
pending = null;
|
||||
elevatedStreak = 0;
|
||||
recoveryStreak += 1;
|
||||
if (recoveryStreak >= thresholds.sustainedSamplesRecovery) {
|
||||
severity = "normal";
|
||||
reason = "none";
|
||||
recoveryStreak = 0;
|
||||
}
|
||||
} else {
|
||||
pending = null;
|
||||
elevatedStreak = 0;
|
||||
recoveryStreak = 0;
|
||||
}
|
||||
|
||||
const transitioned = severity !== state.severity || reason !== state.reason;
|
||||
state = {
|
||||
severity,
|
||||
reason,
|
||||
elevatedStreak,
|
||||
recoveryStreak,
|
||||
lastTransitionAtMs: transitioned ? signals.observedAtMs : state.lastTransitionAtMs,
|
||||
observedAtMs: signals.observedAtMs,
|
||||
};
|
||||
return state;
|
||||
},
|
||||
getState: () => state,
|
||||
};
|
||||
}
|
||||
257
open-sse/utils/resourcePressureSampler.ts
Normal file
257
open-sse/utils/resourcePressureSampler.ts
Normal file
@@ -0,0 +1,257 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import v8 from "node:v8";
|
||||
import type { ResourceSignals } from "./resourcePressurePolicy.ts";
|
||||
|
||||
const DEFAULT_CGROUP_ROOT = "/sys/fs/cgroup";
|
||||
|
||||
export type ResourcePressureFs = {
|
||||
readText: (filePath: string) => Promise<string | null>;
|
||||
};
|
||||
|
||||
export type SampleResourceSignalsDeps = {
|
||||
nowMs?: () => number;
|
||||
memoryUsage?: () => NodeJS.MemoryUsage;
|
||||
heapStatistics?: () => { heap_size_limit: number; used_heap_size?: number };
|
||||
availableMemory?: () => number | undefined;
|
||||
constrainedMemory?: () => number | undefined;
|
||||
fs?: ResourcePressureFs;
|
||||
};
|
||||
|
||||
type Cgroup2Mount = { root: string; mountpoint: string };
|
||||
|
||||
async function defaultReadText(filePath: string): Promise<string | null> {
|
||||
try {
|
||||
return await fs.readFile(filePath, "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function sanitizeMemoryBytes(value: unknown): number | null {
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed || trimmed === "max" || !/^\d+$/.test(trimmed) || trimmed.length > 15) {
|
||||
return null;
|
||||
}
|
||||
value = Number(trimmed);
|
||||
}
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return null;
|
||||
if (value >= Number.MAX_SAFE_INTEGER) return null;
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
function safeNumber(call: (() => number | undefined) | undefined): number | null {
|
||||
try {
|
||||
return call ? sanitizeMemoryBytes(call()) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeMountInfoPath(value: string): string | null {
|
||||
if (value.includes("\0")) return null;
|
||||
try {
|
||||
return value.replace(/\\([0-7]{3})/g, (_match, octal: string) =>
|
||||
String.fromCharCode(Number.parseInt(octal, 8))
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseCgroupV2Path(contents: string | null): string | null {
|
||||
if (!contents) return null;
|
||||
for (const rawLine of contents.split("\n")) {
|
||||
const line = rawLine.trim();
|
||||
if (!line.startsWith("0::")) continue;
|
||||
const relativePath = line.slice(3);
|
||||
if (!relativePath.startsWith("/") || relativePath.includes("\0")) return null;
|
||||
return relativePath;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseCgroup2Mount(contents: string | null): Cgroup2Mount | null {
|
||||
if (!contents) return null;
|
||||
for (const rawLine of contents.split("\n")) {
|
||||
const separator = rawLine.indexOf(" - ");
|
||||
if (separator < 0) continue;
|
||||
const left = rawLine.slice(0, separator).trim().split(/\s+/);
|
||||
const right = rawLine
|
||||
.slice(separator + 3)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
if (right[0] !== "cgroup2" || left.length < 5) continue;
|
||||
const root = decodeMountInfoPath(left[3]);
|
||||
const mountpoint = decodeMountInfoPath(left[4]);
|
||||
if (!root?.startsWith("/") || !mountpoint?.startsWith("/")) return null;
|
||||
return { root, mountpoint };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isContained(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
||||
}
|
||||
|
||||
function hasTraversalSegment(value: string): boolean {
|
||||
let decoded = value;
|
||||
try {
|
||||
decoded = decodeURIComponent(value);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
return decoded.split("/").some((segment) => segment === ".." || segment === ".");
|
||||
}
|
||||
|
||||
function resolveFromMount(cgroupPath: string, mount: Cgroup2Mount): string | null {
|
||||
if (
|
||||
cgroupPath.includes("\0") ||
|
||||
mount.root.includes("\0") ||
|
||||
mount.mountpoint.includes("\0") ||
|
||||
hasTraversalSegment(cgroupPath)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const resolvedRoot = path.resolve(mount.root);
|
||||
const resolvedCgroup = path.resolve(cgroupPath);
|
||||
if (!isContained(resolvedRoot, resolvedCgroup)) return null;
|
||||
const suffix = path.relative(resolvedRoot, resolvedCgroup);
|
||||
const resolvedMountpoint = path.resolve(mount.mountpoint);
|
||||
const candidate = path.resolve(resolvedMountpoint, suffix);
|
||||
return isContained(resolvedMountpoint, candidate) ? candidate : null;
|
||||
}
|
||||
|
||||
export async function resolveCgroupDirectory(
|
||||
readText: ResourcePressureFs["readText"],
|
||||
options: { allowDefaultFallback?: boolean } = {}
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const [cgroupContents, mountInfo] = await Promise.all([
|
||||
readText("/proc/self/cgroup"),
|
||||
readText("/proc/self/mountinfo"),
|
||||
]);
|
||||
const cgroupPath = parseCgroupV2Path(cgroupContents);
|
||||
const mount = parseCgroup2Mount(mountInfo);
|
||||
if (cgroupPath && mount) {
|
||||
const candidate = resolveFromMount(cgroupPath, mount);
|
||||
if (candidate && (await readText(path.join(candidate, "memory.current"))) != null) {
|
||||
return candidate;
|
||||
}
|
||||
if (!candidate) return null;
|
||||
}
|
||||
if (options.allowDefaultFallback === false) return null;
|
||||
return (await readText(path.join(DEFAULT_CGROUP_ROOT, "memory.current"))) != null
|
||||
? DEFAULT_CGROUP_ROOT
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseEventCounter(value: string): number | null {
|
||||
const parsed = Number(value.trim());
|
||||
return Number.isFinite(parsed) && parsed >= 0 && parsed < Number.MAX_SAFE_INTEGER
|
||||
? Math.floor(parsed)
|
||||
: null;
|
||||
}
|
||||
|
||||
function parseMemoryEvents(text: string | null): ResourceSignals["cgroup"]["events"] {
|
||||
if (!text) return null;
|
||||
const values = { low: null, high: null, max: null, oom: null, oom_kill: null } as Record<
|
||||
"low" | "high" | "max" | "oom" | "oom_kill",
|
||||
number | null
|
||||
>;
|
||||
let matched = false;
|
||||
for (const line of text.split("\n")) {
|
||||
const [key, rawValue] = line.trim().split(/\s+/, 2);
|
||||
if (!(key in values) || rawValue == null) continue;
|
||||
values[key as keyof typeof values] = parseEventCounter(rawValue);
|
||||
matched = true;
|
||||
}
|
||||
return matched ? values : null;
|
||||
}
|
||||
|
||||
function parsePsiNumber(line: string, name: string): number | null {
|
||||
const match = new RegExp(`(?:^|\\s)${name}=([0-9.]+)`).exec(line);
|
||||
const parsed = match ? Number(match[1]) : Number.NaN;
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function parsePsi(text: string | null): ResourceSignals["psi"] {
|
||||
if (!text) return null;
|
||||
const result: NonNullable<ResourceSignals["psi"]> = {
|
||||
someAvg10: null,
|
||||
someAvg60: null,
|
||||
someAvg300: null,
|
||||
fullAvg10: null,
|
||||
fullAvg60: null,
|
||||
fullAvg300: null,
|
||||
};
|
||||
let matched = false;
|
||||
for (const line of text.split("\n")) {
|
||||
const kind = line.startsWith("some ") ? "some" : line.startsWith("full ") ? "full" : null;
|
||||
if (!kind) continue;
|
||||
result[`${kind}Avg10`] = parsePsiNumber(line, "avg10");
|
||||
result[`${kind}Avg60`] = parsePsiNumber(line, "avg60");
|
||||
result[`${kind}Avg300`] = parsePsiNumber(line, "avg300");
|
||||
matched = true;
|
||||
}
|
||||
return matched ? result : null;
|
||||
}
|
||||
|
||||
export async function sampleResourceSignals(
|
||||
deps: SampleResourceSignalsDeps = {}
|
||||
): Promise<ResourceSignals> {
|
||||
const readText = deps.fs?.readText ?? defaultReadText;
|
||||
let memory: NodeJS.MemoryUsage;
|
||||
try {
|
||||
memory = (deps.memoryUsage ?? process.memoryUsage)();
|
||||
} catch {
|
||||
memory = { rss: 0, heapTotal: 0, heapUsed: 0, external: 0, arrayBuffers: 0 };
|
||||
}
|
||||
|
||||
let heapUsed = Math.max(0, Math.floor(memory.heapUsed || 0));
|
||||
let heapLimit = 0;
|
||||
try {
|
||||
const heap = (deps.heapStatistics ?? v8.getHeapStatistics)();
|
||||
heapLimit = sanitizeMemoryBytes(heap.heap_size_limit) ?? 0;
|
||||
if (Number.isFinite(heap.used_heap_size)) {
|
||||
heapUsed = Math.max(0, Math.floor(heap.used_heap_size ?? heapUsed));
|
||||
}
|
||||
} catch {
|
||||
/* retain process heap sample */
|
||||
}
|
||||
|
||||
const cgroupDirectory = await resolveCgroupDirectory(readText);
|
||||
const cgroupContents = cgroupDirectory
|
||||
? await Promise.all([
|
||||
readText(path.join(cgroupDirectory, "memory.current")),
|
||||
readText(path.join(cgroupDirectory, "memory.max")),
|
||||
readText(path.join(cgroupDirectory, "memory.high")),
|
||||
readText(path.join(cgroupDirectory, "memory.events")),
|
||||
])
|
||||
: [null, null, null, null];
|
||||
const psi = await readText("/proc/pressure/memory").catch(() => null);
|
||||
|
||||
return {
|
||||
observedAtMs: (deps.nowMs ?? Date.now)(),
|
||||
v8: { heapUsedBytes: heapUsed, heapLimitBytes: heapLimit },
|
||||
process: {
|
||||
rssBytes: Math.max(0, Math.floor(memory.rss || 0)),
|
||||
externalBytes: Math.max(0, Math.floor(memory.external || 0)),
|
||||
arrayBuffersBytes: Math.max(0, Math.floor(memory.arrayBuffers || 0)),
|
||||
availableBytes: safeNumber(deps.availableMemory ?? (() => process.availableMemory?.())),
|
||||
constrainedBytes: safeNumber(deps.constrainedMemory ?? (() => process.constrainedMemory?.())),
|
||||
},
|
||||
cgroup: {
|
||||
currentBytes: sanitizeMemoryBytes(cgroupContents[0]),
|
||||
maxBytes: sanitizeMemoryBytes(cgroupContents[1]),
|
||||
highBytes: sanitizeMemoryBytes(cgroupContents[2]),
|
||||
events: parseMemoryEvents(cgroupContents[3]),
|
||||
},
|
||||
psi: parsePsi(psi),
|
||||
};
|
||||
}
|
||||
985
tests/unit/adaptive-admission-controller.test.ts
Normal file
985
tests/unit/adaptive-admission-controller.test.ts
Normal file
@@ -0,0 +1,985 @@
|
||||
import { describe, it, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
AdaptiveAdmissionController,
|
||||
createAdmissionRejectError,
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionLease,
|
||||
type AdmissionPressure,
|
||||
type AdmissionRequest,
|
||||
} from "../../open-sse/services/admission/index.ts";
|
||||
|
||||
class FakeClock {
|
||||
nowMs = 0;
|
||||
private nextId = 1;
|
||||
private timers = new Map<number, { due: number; fn: () => void }>();
|
||||
|
||||
now = () => this.nowMs;
|
||||
|
||||
setTimer = (fn: () => void, delayMs: number): number => {
|
||||
const id = this.nextId++;
|
||||
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
|
||||
return id;
|
||||
};
|
||||
|
||||
clearTimer = (id: number): void => {
|
||||
this.timers.delete(id);
|
||||
};
|
||||
|
||||
get pendingTimerCount(): number {
|
||||
return this.timers.size;
|
||||
}
|
||||
|
||||
advance(ms: number): void {
|
||||
const target = this.nowMs + ms;
|
||||
while (true) {
|
||||
let nextId: number | undefined;
|
||||
let nextDue = Number.POSITIVE_INFINITY;
|
||||
for (const [id, t] of this.timers) {
|
||||
if (t.due <= target && t.due < nextDue) {
|
||||
nextDue = t.due;
|
||||
nextId = id;
|
||||
}
|
||||
}
|
||||
if (nextId === undefined) {
|
||||
this.nowMs = target;
|
||||
return;
|
||||
}
|
||||
const timer = this.timers.get(nextId)!;
|
||||
this.timers.delete(nextId);
|
||||
this.nowMs = timer.due;
|
||||
timer.fn();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function baseConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
|
||||
return {
|
||||
mode: "enforce",
|
||||
minLimit: 10,
|
||||
maxLimit: 100,
|
||||
initialLimit: 20,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 1000,
|
||||
windowMs: 100,
|
||||
shortLatencyAlpha: 0.5,
|
||||
longLatencyAlpha: 0.1,
|
||||
increaseStep: 2,
|
||||
decreaseFactor: 0.8,
|
||||
criticalDecreaseFactor: 0.5,
|
||||
highUtilizationThreshold: 0.7,
|
||||
lowUtilizationThreshold: 0.3,
|
||||
latencyGradientThreshold: 0.25,
|
||||
maxIncreasePerWindow: 4,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function req(partial: Partial<AdmissionRequest> & { cost: number }): AdmissionRequest {
|
||||
return {
|
||||
tenantKey: "t-default",
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
async function mustAdmit(
|
||||
controller: AdaptiveAdmissionController,
|
||||
request: AdmissionRequest
|
||||
): Promise<AdmissionLease> {
|
||||
const result = await controller.acquire(request);
|
||||
assert.equal(result.status, "admitted");
|
||||
if (result.status !== "admitted") throw new Error("expected admitted");
|
||||
return result.lease;
|
||||
}
|
||||
|
||||
describe("AdaptiveAdmissionController config and modes", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
function make(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
|
||||
return new AdaptiveAdmissionController(baseConfig(overrides), {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
});
|
||||
}
|
||||
|
||||
it("validates safe-integer bounds and ordered adaptation parameters", () => {
|
||||
assert.throws(() => make({ minLimit: 50, maxLimit: 10 }), /minLimit/);
|
||||
for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
|
||||
assert.throws(() => make({ maxQueueCount: invalid }), /maxQueueCount/);
|
||||
assert.throws(() => make({ initialLimit: invalid }), /initialLimit/);
|
||||
}
|
||||
assert.throws(() => make({ decreaseFactor: 1.2 }), /decreaseFactor/);
|
||||
assert.throws(
|
||||
() => make({ decreaseFactor: 0.5, criticalDecreaseFactor: 0.8 }),
|
||||
/criticalDecreaseFactor/
|
||||
);
|
||||
assert.throws(
|
||||
() => make({ lowUtilizationThreshold: 0.8, highUtilizationThreshold: 0.7 }),
|
||||
/lowUtilizationThreshold/
|
||||
);
|
||||
assert.throws(
|
||||
() => make({ shortLatencyAlpha: 0.1, longLatencyAlpha: 0.5 }),
|
||||
/shortLatencyAlpha/
|
||||
);
|
||||
});
|
||||
|
||||
it("clamps initial limit into [minLimit, maxLimit]", () => {
|
||||
const low = make({ initialLimit: 1, minLimit: 10 });
|
||||
assert.equal(low.snapshot().currentLimit, 10);
|
||||
low.shutdown();
|
||||
const high = make({ initialLimit: 999, maxLimit: 100 });
|
||||
assert.equal(high.snapshot().currentLimit, 100);
|
||||
high.shutdown();
|
||||
});
|
||||
|
||||
it("mode off never accounts cost or rejects", async () => {
|
||||
const c = make({ mode: "off", initialLimit: 5 });
|
||||
const a = await c.acquire(req({ cost: 100 }));
|
||||
const b = await c.acquire(req({ cost: 100 }));
|
||||
assert.equal(a.status, "admitted");
|
||||
assert.equal(b.status, "admitted");
|
||||
const snap = c.snapshot();
|
||||
assert.equal(snap.activeCost, 0);
|
||||
assert.equal(snap.activeCount, 0);
|
||||
assert.equal(snap.rejectedCount, 0);
|
||||
c.shutdown();
|
||||
});
|
||||
|
||||
it("defaults to shadow mode when mode omitted", () => {
|
||||
const c = new AdaptiveAdmissionController(
|
||||
{
|
||||
minLimit: 10,
|
||||
maxLimit: 100,
|
||||
initialLimit: 20,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: 20,
|
||||
} as AdaptiveAdmissionConfig,
|
||||
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
|
||||
);
|
||||
assert.equal(c.snapshot().mode, "shadow");
|
||||
c.shutdown();
|
||||
});
|
||||
});
|
||||
|
||||
describe("shadow mode semantics", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
it("never rejects or delays while recording would-decisions and real active cost", async () => {
|
||||
const c = new AdaptiveAdmissionController(
|
||||
baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 10 }),
|
||||
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
|
||||
);
|
||||
|
||||
const first = await c.acquire(req({ cost: 8 }));
|
||||
assert.equal(first.status, "admitted");
|
||||
if (first.status !== "admitted") return;
|
||||
assert.equal(first.shadowDecision, "would-admit");
|
||||
assert.equal(c.snapshot().activeCost, 8);
|
||||
|
||||
const second = await c.acquire(req({ cost: 8 }));
|
||||
assert.equal(second.status, "admitted");
|
||||
if (second.status !== "admitted") return;
|
||||
// Would have queued under enforce (active 8 + 8 > 10) but shadow admits immediately.
|
||||
assert.equal(second.shadowDecision, "would-queue");
|
||||
assert.equal(c.snapshot().activeCost, 16);
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
assert.ok((c.snapshot().wouldQueueCount ?? 0) >= 1);
|
||||
|
||||
const oversized = await c.acquire(req({ cost: 50 }));
|
||||
assert.equal(oversized.status, "admitted");
|
||||
if (oversized.status !== "admitted") return;
|
||||
assert.equal(oversized.shadowDecision, "would-reject");
|
||||
assert.ok((c.snapshot().wouldRejectCount ?? 0) >= 1);
|
||||
|
||||
first.lease.release("success");
|
||||
second.lease.release("success");
|
||||
oversized.lease.release("success");
|
||||
assert.equal(c.snapshot().activeCost, 0);
|
||||
c.shutdown();
|
||||
});
|
||||
|
||||
it("simulates virtual queue saturation and promotes queued work on release", async () => {
|
||||
const c = new AdaptiveAdmissionController(
|
||||
baseConfig({ mode: "shadow", initialLimit: 10, maxQueueCount: 1, maxQueueCost: 8 }),
|
||||
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
|
||||
);
|
||||
const active = await c.acquire(req({ cost: 8, tenantKey: "active" }));
|
||||
const queued = await c.acquire(req({ cost: 8, tenantKey: "queued" }));
|
||||
const saturated = await c.acquire(req({ cost: 8, tenantKey: "saturated" }));
|
||||
assert.equal(active.status, "admitted");
|
||||
assert.equal(queued.status, "admitted");
|
||||
assert.equal(saturated.status, "admitted");
|
||||
if (
|
||||
active.status !== "admitted" ||
|
||||
queued.status !== "admitted" ||
|
||||
saturated.status !== "admitted"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
assert.equal(active.shadowDecision, "would-admit");
|
||||
assert.equal(queued.shadowDecision, "would-queue");
|
||||
assert.equal(saturated.shadowDecision, "would-reject");
|
||||
assert.deepEqual(
|
||||
{
|
||||
activeCost: c.snapshot().virtualActiveCost,
|
||||
activeCount: c.snapshot().virtualActiveCount,
|
||||
queuedCost: c.snapshot().virtualQueuedCost,
|
||||
queuedCount: c.snapshot().virtualQueuedCount,
|
||||
},
|
||||
{ activeCost: 8, activeCount: 1, queuedCost: 8, queuedCount: 1 }
|
||||
);
|
||||
|
||||
active.lease.release();
|
||||
assert.deepEqual(
|
||||
{
|
||||
activeCost: c.snapshot().virtualActiveCost,
|
||||
activeCount: c.snapshot().virtualActiveCount,
|
||||
queuedCost: c.snapshot().virtualQueuedCost,
|
||||
queuedCount: c.snapshot().virtualQueuedCount,
|
||||
},
|
||||
{ activeCost: 8, activeCount: 1, queuedCost: 0, queuedCount: 0 }
|
||||
);
|
||||
queued.lease.release();
|
||||
saturated.lease.release();
|
||||
c.shutdown();
|
||||
});
|
||||
|
||||
it("promotes shadow virtual queue after adaptation raises the limit", async () => {
|
||||
const c = new AdaptiveAdmissionController(
|
||||
baseConfig({
|
||||
mode: "shadow",
|
||||
minLimit: 10,
|
||||
maxLimit: 20,
|
||||
initialLimit: 10,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
windowMs: 100,
|
||||
increaseStep: 5,
|
||||
maxIncreasePerWindow: 5,
|
||||
highUtilizationThreshold: 0.5,
|
||||
}),
|
||||
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
|
||||
);
|
||||
|
||||
const active = await c.acquire(req({ cost: 10, tenantKey: "active" }));
|
||||
const queued = await c.acquire(req({ cost: 5, tenantKey: "queued" }));
|
||||
assert.equal(active.status, "admitted");
|
||||
assert.equal(queued.status, "admitted");
|
||||
if (active.status !== "admitted" || queued.status !== "admitted") return;
|
||||
assert.equal(active.shadowDecision, "would-admit");
|
||||
assert.equal(queued.shadowDecision, "would-queue");
|
||||
assert.equal(c.snapshot().virtualActiveCost, 10);
|
||||
assert.equal(c.snapshot().virtualQueuedCost, 5);
|
||||
|
||||
// Raise the adaptive limit once while both leases remain open. Shadow admits a
|
||||
// probe for completion evidence; active integral is capped at the current limit.
|
||||
const probe = await c.acquire(req({ cost: 1, tenantKey: "probe" }));
|
||||
assert.equal(probe.status, "admitted");
|
||||
if (probe.status === "admitted") {
|
||||
clock.advance(80);
|
||||
probe.lease.release("success", { latencyMs: 10 });
|
||||
clock.advance(20);
|
||||
c.tick();
|
||||
}
|
||||
|
||||
assert.equal(c.snapshot().currentLimit, 15);
|
||||
// Queued virtual work must be promoted before newer arrivals are classified.
|
||||
assert.equal(c.snapshot().virtualActiveCost, 15);
|
||||
assert.equal(c.snapshot().virtualQueuedCost, 0);
|
||||
|
||||
const later = await c.acquire(req({ cost: 5, tenantKey: "later" }));
|
||||
assert.equal(later.status, "admitted");
|
||||
if (later.status !== "admitted") return;
|
||||
// With virtual active already 15 at limit 15, a later cost-5 cannot would-admit.
|
||||
assert.notEqual(later.shadowDecision, "would-admit");
|
||||
|
||||
active.lease.release();
|
||||
queued.lease.release();
|
||||
later.lease.release();
|
||||
c.shutdown();
|
||||
});
|
||||
});
|
||||
|
||||
describe("weighted enforce, queue, fairness, and races", () => {
|
||||
let clock: FakeClock;
|
||||
const live: AdaptiveAdmissionController[] = [];
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
live.length = 0;
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const c of live) c.shutdown();
|
||||
live.length = 0;
|
||||
});
|
||||
|
||||
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
|
||||
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
});
|
||||
live.push(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
it("enforces weighted active-cost budget and rejects oversized requests immediately", async () => {
|
||||
const c = controller({ initialLimit: 20 });
|
||||
const a = await mustAdmit(c, req({ cost: 12 }));
|
||||
const b = await c.acquire(req({ cost: 12 }));
|
||||
assert.equal(b.status, "queued");
|
||||
|
||||
const over = await c.acquire(req({ cost: 25 }));
|
||||
assert.equal(over.status, "rejected");
|
||||
if (over.status === "rejected") {
|
||||
assert.equal(over.code, "ADMISSION_OVERSIZED");
|
||||
}
|
||||
|
||||
a.release("success");
|
||||
if (b.status === "queued") {
|
||||
const admitted = await b.promise;
|
||||
assert.equal(admitted.status, "admitted");
|
||||
admitted.lease.release("success");
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds queue by count and total queued cost", async () => {
|
||||
const c = controller({
|
||||
minLimit: 10,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: 15,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 10 }));
|
||||
|
||||
const q1 = await c.acquire(req({ cost: 5, tenantKey: "a" }));
|
||||
const q2 = await c.acquire(req({ cost: 5, tenantKey: "b" }));
|
||||
assert.equal(q1.status, "queued");
|
||||
assert.equal(q2.status, "queued");
|
||||
assert.equal(c.snapshot().queuedCount, 2);
|
||||
assert.equal(c.snapshot().queuedCost, 10);
|
||||
|
||||
const byCount = await c.acquire(req({ cost: 1, tenantKey: "c" }));
|
||||
assert.equal(byCount.status, "rejected");
|
||||
if (byCount.status === "rejected") assert.equal(byCount.code, "ADMISSION_QUEUE_FULL");
|
||||
|
||||
held.release("success");
|
||||
if (q1.status === "queued") (await q1.promise).lease.release("success");
|
||||
if (q2.status === "queued") (await q2.promise).lease.release("success");
|
||||
|
||||
const c2 = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 10,
|
||||
maxQueueCost: 7,
|
||||
});
|
||||
const h = await mustAdmit(c2, req({ cost: 5 }));
|
||||
// cost 3 fits limit but not active budget → queued (queuedCost=3).
|
||||
// Another cost 5 fits the budget but 3+5 > maxQueueCost=7 → QUEUE_FULL.
|
||||
const ok = await c2.acquire(req({ cost: 3 }));
|
||||
assert.equal(ok.status, "queued");
|
||||
const costFull = await c2.acquire(req({ cost: 5 }));
|
||||
assert.equal(costFull.status, "rejected");
|
||||
if (costFull.status === "rejected") assert.equal(costFull.code, "ADMISSION_QUEUE_FULL");
|
||||
h.release("success");
|
||||
if (ok.status === "queued") (await ok.promise).lease.release("success");
|
||||
});
|
||||
|
||||
it("expires deadline and abort without leaking queue slots", async () => {
|
||||
const c = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 50,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 5 }));
|
||||
|
||||
const timed = await c.acquire(req({ cost: 3, maxWaitMs: 30 }));
|
||||
assert.equal(timed.status, "queued");
|
||||
clock.advance(31);
|
||||
if (timed.status === "queued") {
|
||||
await assert.rejects(timed.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
|
||||
const ac = new AbortController();
|
||||
const aborted = await c.acquire(req({ cost: 3, signal: ac.signal }));
|
||||
assert.equal(aborted.status, "queued");
|
||||
ac.abort();
|
||||
if (aborted.status === "queued") {
|
||||
await assert.rejects(aborted.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
held.release("success");
|
||||
});
|
||||
|
||||
it("treats the exact deadline as expired and settles abort/release races once", async () => {
|
||||
const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 5, defaultMaxWaitMs: 30 });
|
||||
const held = await mustAdmit(c, req({ cost: 5 }));
|
||||
const ac = new AbortController();
|
||||
const queued = await c.acquire(req({ cost: 3, maxWaitMs: 30, signal: ac.signal }));
|
||||
assert.equal(queued.status, "queued");
|
||||
|
||||
clock.advance(30);
|
||||
ac.abort();
|
||||
held.release("success");
|
||||
|
||||
if (queued.status === "queued") {
|
||||
await assert.rejects(queued.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
assert.equal(c.snapshot().rejectedCount, 1);
|
||||
});
|
||||
|
||||
it("shutdown rejects queued work and clears every fake-clock timer", async () => {
|
||||
const c = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 5 }));
|
||||
const q = await c.acquire(req({ cost: 3, maxWaitMs: 5000 }));
|
||||
assert.equal(q.status, "queued");
|
||||
assert.ok(clock.pendingTimerCount >= 2);
|
||||
c.shutdown();
|
||||
if (q.status === "queued") {
|
||||
await assert.rejects(q.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_SHUTDOWN");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
assert.equal(clock.pendingTimerCount, 0);
|
||||
held.release("success");
|
||||
const after = await c.acquire(req({ cost: 1 }));
|
||||
assert.equal(after.status, "rejected");
|
||||
if (after.status === "rejected") assert.equal(after.code, "ADMISSION_SHUTDOWN");
|
||||
});
|
||||
|
||||
it("updateConfig atomically settles queues, dispatches raised capacity, and respects decreases", async () => {
|
||||
const c = controller({ minLimit: 5, initialLimit: 5, maxLimit: 20, windowMs: 100 });
|
||||
const held = await mustAdmit(c, req({ cost: 5 }));
|
||||
const queued = await c.acquire(req({ cost: 5 }));
|
||||
assert.equal(queued.status, "queued");
|
||||
|
||||
c.updateConfig(baseConfig({ minLimit: 10, initialLimit: 10, maxLimit: 20, windowMs: 50 }));
|
||||
assert.equal(clock.pendingTimerCount, 1);
|
||||
if (queued.status === "queued") {
|
||||
const admitted = await queued.promise;
|
||||
assert.equal(c.snapshot().activeCost, 10);
|
||||
|
||||
c.updateConfig(baseConfig({ minLimit: 5, initialLimit: 5, maxLimit: 5, windowMs: 50 }));
|
||||
const afterDecrease = await c.acquire(req({ cost: 1 }));
|
||||
assert.equal(afterDecrease.status, "queued");
|
||||
|
||||
c.updateConfig(baseConfig({ mode: "shadow", minLimit: 5, initialLimit: 5, maxLimit: 5 }));
|
||||
if (afterDecrease.status === "queued") {
|
||||
const settled = await afterDecrease.promise;
|
||||
assert.equal(settled.status, "admitted");
|
||||
settled.lease.release();
|
||||
}
|
||||
admitted.lease.release();
|
||||
}
|
||||
held.release();
|
||||
});
|
||||
|
||||
it("queue shrink rejects deterministic round-robin excess and preserves fitting entries", async () => {
|
||||
const c = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 20,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 5 }));
|
||||
const first = await c.acquire(req({ cost: 2, tenantKey: "a" }));
|
||||
const second = await c.acquire(req({ cost: 2, tenantKey: "b" }));
|
||||
const third = await c.acquire(req({ cost: 2, tenantKey: "a" }));
|
||||
assert.equal(first.status, "queued");
|
||||
assert.equal(second.status, "queued");
|
||||
assert.equal(third.status, "queued");
|
||||
|
||||
c.updateConfig(
|
||||
baseConfig({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: 4,
|
||||
})
|
||||
);
|
||||
assert.equal(c.snapshot().queuedCount, 2);
|
||||
if (third.status === "queued") {
|
||||
await assert.rejects(third.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_QUEUE_FULL");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
held.release();
|
||||
if (first.status === "queued") (await first.promise).lease.release();
|
||||
if (second.status === "queued") (await second.promise).lease.release();
|
||||
});
|
||||
|
||||
it("release is idempotent under race with abort", async () => {
|
||||
const c = controller({ initialLimit: 10 });
|
||||
const lease = await mustAdmit(c, req({ cost: 4 }));
|
||||
lease.release("success");
|
||||
lease.release("timeout");
|
||||
lease.release("success");
|
||||
assert.equal(c.snapshot().activeCost, 0);
|
||||
assert.equal(c.snapshot().activeCount, 0);
|
||||
});
|
||||
|
||||
it("fairly schedules across tenants under skew without exposing tenant ids", async () => {
|
||||
const c = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 10,
|
||||
maxQueueCost: 100,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 5, tenantKey: "hold" }));
|
||||
|
||||
const order: string[] = [];
|
||||
const queued: Array<Promise<void>> = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const r = await c.acquire(req({ cost: 5, tenantKey: "heavy" }));
|
||||
assert.equal(r.status, "queued");
|
||||
if (r.status === "queued") {
|
||||
queued.push(
|
||||
r.promise.then((admitted) => {
|
||||
order.push("heavy");
|
||||
admitted.lease.release("success");
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
const light = await c.acquire(req({ cost: 5, tenantKey: "light" }));
|
||||
assert.equal(light.status, "queued");
|
||||
if (light.status === "queued") {
|
||||
queued.push(
|
||||
light.promise.then((admitted) => {
|
||||
order.push("light");
|
||||
admitted.lease.release("success");
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// Free capacity one slot at a time.
|
||||
held.release("success");
|
||||
await Promise.resolve();
|
||||
// After first release, one request should admit; keep draining by waiting microtasks between releases.
|
||||
// Drain remaining by letting each admitted release free the next.
|
||||
await Promise.all(queued);
|
||||
|
||||
// Light must not be starved behind all four heavy requests.
|
||||
const lightIndex = order.indexOf("light");
|
||||
assert.ok(lightIndex >= 0);
|
||||
assert.ok(lightIndex < 4, `light scheduled too late: ${order.join(",")}`);
|
||||
|
||||
const snap = c.snapshot();
|
||||
const json = JSON.stringify(snap);
|
||||
assert.equal(json.includes("heavy"), false);
|
||||
assert.equal(json.includes("light"), false);
|
||||
assert.equal(json.includes("hold"), false);
|
||||
});
|
||||
|
||||
it("dispatches a fitting tenant when another tenant's queue head cannot fit", async () => {
|
||||
const c = controller({
|
||||
minLimit: 10,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 10,
|
||||
maxQueueCost: 100,
|
||||
});
|
||||
const heldSix = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
|
||||
const heldFour = await mustAdmit(c, req({ cost: 4, tenantKey: "holder" }));
|
||||
const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" }));
|
||||
const fitting = await c.acquire(req({ cost: 4, tenantKey: "fitting" }));
|
||||
assert.equal(expensive.status, "queued");
|
||||
assert.equal(fitting.status, "queued");
|
||||
|
||||
heldFour.release("success");
|
||||
if (fitting.status === "queued") {
|
||||
const admitted = await fitting.promise;
|
||||
assert.equal(admitted.lease.cost, 4);
|
||||
admitted.lease.release("success");
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 1);
|
||||
|
||||
heldSix.release("success");
|
||||
if (expensive.status === "queued") (await expensive.promise).lease.release("success");
|
||||
});
|
||||
|
||||
it("bounds starvation of an older unfittable cost-6 behind a stream of cost-2 work", async () => {
|
||||
const c = controller({
|
||||
minLimit: 10,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 20,
|
||||
maxQueueCost: 100,
|
||||
defaultMaxWaitMs: 10_000,
|
||||
});
|
||||
// Hold 6 so available=4: cost-2 can pass over cost-6 until reservation engages.
|
||||
const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
|
||||
|
||||
const expensive = await c.acquire(req({ cost: 6, tenantKey: "expensive" }));
|
||||
assert.equal(expensive.status, "queued");
|
||||
|
||||
// Two actual smaller dequeues through queued promises (pass-overs that age the head).
|
||||
const passOvers: AdmissionLease[] = [];
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const r = await c.acquire(req({ cost: 2, tenantKey: `small-pass-${i}` }));
|
||||
assert.equal(r.status, "queued", `pass-over ${i} should join the non-empty queue`);
|
||||
if (r.status !== "queued") throw new Error("expected queued");
|
||||
const admitted = await r.promise;
|
||||
assert.equal(admitted.lease.cost, 2);
|
||||
passOvers.push(admitted.lease);
|
||||
admitted.lease.release("success");
|
||||
assert.equal(c.snapshot().activeCost, 6);
|
||||
}
|
||||
assert.equal(passOvers.length, 2);
|
||||
|
||||
// A subsequent fitting cost-2 must remain queued: capacity is reserved for cost-6.
|
||||
// Without reservation accounting this would admit immediately and the assertion fails.
|
||||
const blocked = await c.acquire(req({ cost: 2, tenantKey: "small-blocked" }));
|
||||
assert.equal(blocked.status, "queued");
|
||||
await Promise.resolve();
|
||||
assert.equal(c.snapshot().activeCost, 6, "reserved head must block fitting smaller work");
|
||||
assert.equal(c.snapshot().queuedCount, 2);
|
||||
|
||||
const order: number[] = [];
|
||||
assert.equal(expensive.status, "queued");
|
||||
assert.equal(blocked.status, "queued");
|
||||
const expensiveDone = expensive.promise.then((admitted) => {
|
||||
order.push(admitted.lease.cost);
|
||||
return admitted;
|
||||
});
|
||||
const blockedDone = blocked.promise.then((admitted) => {
|
||||
order.push(admitted.lease.cost);
|
||||
return admitted;
|
||||
});
|
||||
|
||||
// Free enough capacity for cost-6; the older reserved request must admit first.
|
||||
// With activeCost back at 0 both may fit in one dispatch turn, so only order is asserted.
|
||||
held.release("success");
|
||||
const [expAdmitted, blockedAdmitted] = await Promise.all([expensiveDone, blockedDone]);
|
||||
assert.equal(expAdmitted.lease.cost, 6);
|
||||
assert.equal(blockedAdmitted.lease.cost, 2);
|
||||
assert.deepEqual(order, [6, 2]);
|
||||
expAdmitted.lease.release("success");
|
||||
blockedAdmitted.lease.release("success");
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
});
|
||||
|
||||
async function ageReservedCost6(
|
||||
c: AdaptiveAdmissionController,
|
||||
expensiveSignal?: AbortSignal,
|
||||
expensiveMaxWaitMs?: number
|
||||
) {
|
||||
const held = await mustAdmit(c, req({ cost: 6, tenantKey: "holder" }));
|
||||
const expensive = await c.acquire(
|
||||
req({
|
||||
cost: 6,
|
||||
tenantKey: "expensive",
|
||||
signal: expensiveSignal,
|
||||
maxWaitMs: expensiveMaxWaitMs,
|
||||
})
|
||||
);
|
||||
assert.equal(expensive.status, "queued");
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const r = await c.acquire(req({ cost: 2, tenantKey: `age-pass-${i}` }));
|
||||
assert.equal(r.status, "queued");
|
||||
if (r.status !== "queued") throw new Error("expected queued");
|
||||
(await r.promise).lease.release("success");
|
||||
}
|
||||
const blocked = await c.acquire(req({ cost: 2, tenantKey: "age-blocked" }));
|
||||
assert.equal(blocked.status, "queued");
|
||||
await Promise.resolve();
|
||||
assert.equal(c.snapshot().activeCost, 6);
|
||||
assert.equal(c.snapshot().queuedCount, 2);
|
||||
return { held, expensive, blocked };
|
||||
}
|
||||
|
||||
it("aborting a reserved head immediately admits the next fitting request", async () => {
|
||||
const c = controller({
|
||||
minLimit: 10,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 20,
|
||||
maxQueueCost: 100,
|
||||
defaultMaxWaitMs: 10_000,
|
||||
});
|
||||
const ac = new AbortController();
|
||||
const { held, expensive, blocked } = await ageReservedCost6(c, ac.signal);
|
||||
assert.equal(expensive.status, "queued");
|
||||
assert.equal(blocked.status, "queued");
|
||||
|
||||
ac.abort();
|
||||
// No tick / new arrival / release / config update — only the abort path.
|
||||
if (expensive.status === "queued") {
|
||||
await assert.rejects(expensive.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_ABORTED");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
if (blocked.status !== "queued") throw new Error("expected queued blocked request");
|
||||
const admitted = await blocked.promise;
|
||||
assert.equal(admitted.lease.cost, 2);
|
||||
assert.equal(c.snapshot().activeCost, 8);
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
admitted.lease.release("success");
|
||||
held.release("success");
|
||||
});
|
||||
|
||||
it("deadline-expiring a reserved head immediately admits the next fitting request", async () => {
|
||||
const c = controller({
|
||||
minLimit: 10,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 20,
|
||||
maxQueueCost: 100,
|
||||
defaultMaxWaitMs: 10_000,
|
||||
});
|
||||
const { held, expensive, blocked } = await ageReservedCost6(c, undefined, 40);
|
||||
assert.equal(expensive.status, "queued");
|
||||
assert.equal(blocked.status, "queued");
|
||||
|
||||
clock.advance(40);
|
||||
// No tick / new arrival / release / config update — only the deadline timer.
|
||||
if (expensive.status === "queued") {
|
||||
await assert.rejects(expensive.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_DEADLINE");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
if (blocked.status !== "queued") throw new Error("expected queued blocked request");
|
||||
const admitted = await blocked.promise;
|
||||
assert.equal(admitted.lease.cost, 2);
|
||||
assert.equal(c.snapshot().activeCost, 8);
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
admitted.lease.release("success");
|
||||
held.release("success");
|
||||
});
|
||||
});
|
||||
|
||||
describe("adaptive algorithm", () => {
|
||||
let clock: FakeClock;
|
||||
const live: AdaptiveAdmissionController[] = [];
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
live.length = 0;
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const c of live) c.shutdown();
|
||||
live.length = 0;
|
||||
});
|
||||
|
||||
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
|
||||
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
});
|
||||
live.push(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
async function complete(
|
||||
c: AdaptiveAdmissionController,
|
||||
cost: number,
|
||||
latencyMs: number,
|
||||
outcome: "success" | "upstream_error" | "timeout" = "success",
|
||||
pressure: AdmissionPressure = "normal"
|
||||
) {
|
||||
const lease = await mustAdmit(c, req({ cost, pressure }));
|
||||
clock.advance(latencyMs);
|
||||
lease.release(outcome, { latencyMs, pressure });
|
||||
}
|
||||
|
||||
it("keeps currentLimit within validated bounds", async () => {
|
||||
const c = controller({ initialLimit: 20, minLimit: 10, maxLimit: 30, increaseStep: 50 });
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await complete(c, 5, 5, "success", "normal");
|
||||
clock.advance(100);
|
||||
}
|
||||
assert.ok(c.snapshot().currentLimit <= 30);
|
||||
assert.ok(c.snapshot().currentLimit >= 10);
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await complete(c, 5, 5, "success", "critical");
|
||||
clock.advance(100);
|
||||
}
|
||||
assert.ok(c.snapshot().currentLimit >= 10);
|
||||
});
|
||||
|
||||
it("decreases rapidly under critical pressure", async () => {
|
||||
const c = controller({ initialLimit: 80, minLimit: 10, maxLimit: 100 });
|
||||
const before = c.snapshot().currentLimit;
|
||||
await complete(c, 10, 10, "success", "critical");
|
||||
clock.advance(100);
|
||||
// Force a window tick with pressure observation.
|
||||
c.observePressure("critical");
|
||||
clock.advance(100);
|
||||
assert.ok(c.snapshot().currentLimit < before);
|
||||
assert.ok(c.snapshot().currentLimit <= Math.ceil(before * 0.5) + 1);
|
||||
});
|
||||
|
||||
it("applies criticalDecreaseFactor once for a single observePressure(critical)", () => {
|
||||
const c = controller({
|
||||
initialLimit: 80,
|
||||
minLimit: 10,
|
||||
maxLimit: 100,
|
||||
criticalDecreaseFactor: 0.5,
|
||||
decreaseFactor: 0.8,
|
||||
windowMs: 100,
|
||||
});
|
||||
assert.equal(c.snapshot().currentLimit, 80);
|
||||
|
||||
c.observePressure("critical");
|
||||
// Immediate fast decrease: 80 * 0.5 = 40.
|
||||
assert.equal(c.snapshot().currentLimit, 40);
|
||||
|
||||
// Closing the same window must not multiply again (would become 20).
|
||||
clock.advance(100);
|
||||
c.tick();
|
||||
assert.equal(c.snapshot().currentLimit, 40);
|
||||
|
||||
// A fresh critical observation in a later window still decreases once.
|
||||
c.observePressure("critical");
|
||||
assert.equal(c.snapshot().currentLimit, 20);
|
||||
clock.advance(100);
|
||||
c.tick();
|
||||
assert.equal(c.snapshot().currentLimit, 20);
|
||||
});
|
||||
|
||||
it("decreases on high pressure or sustained latency gradient", async () => {
|
||||
const c = controller({
|
||||
initialLimit: 50,
|
||||
shortLatencyAlpha: 0.8,
|
||||
longLatencyAlpha: 0.1,
|
||||
latencyGradientThreshold: 0.2,
|
||||
});
|
||||
// Seed long baseline with low latency.
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await complete(c, 8, 10, "success", "normal");
|
||||
clock.advance(100);
|
||||
}
|
||||
const mid = c.snapshot().currentLimit;
|
||||
// Spike short latency relative to long.
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await complete(c, 8, 200, "success", "normal");
|
||||
clock.advance(100);
|
||||
}
|
||||
assert.ok(c.snapshot().currentLimit <= mid);
|
||||
|
||||
const beforeHigh = c.snapshot().currentLimit;
|
||||
c.observePressure("high");
|
||||
await complete(c, 8, 20, "success", "high");
|
||||
clock.advance(100);
|
||||
assert.ok(c.snapshot().currentLimit <= beforeHigh);
|
||||
});
|
||||
|
||||
it("increases slowly when healthy and highly utilized, and does not inflate when idle", async () => {
|
||||
const c = controller({
|
||||
minLimit: 20,
|
||||
maxLimit: 40,
|
||||
initialLimit: 20,
|
||||
increaseStep: 2,
|
||||
maxIncreasePerWindow: 2,
|
||||
highUtilizationThreshold: 0.5,
|
||||
windowMs: 100,
|
||||
});
|
||||
|
||||
// Idle windows should not inflate.
|
||||
clock.advance(500);
|
||||
c.tick();
|
||||
clock.advance(500);
|
||||
c.tick();
|
||||
assert.equal(c.snapshot().currentLimit, 20);
|
||||
|
||||
// Healthy high utilization: hold nearly full budget across most of each window.
|
||||
for (let w = 0; w < 5; w++) {
|
||||
const lease = await mustAdmit(c, req({ cost: 16, pressure: "normal" }));
|
||||
clock.advance(80);
|
||||
lease.release("success", { latencyMs: 10, pressure: "normal" });
|
||||
clock.advance(20);
|
||||
c.tick();
|
||||
}
|
||||
assert.ok(c.snapshot().currentLimit > 20);
|
||||
assert.ok(c.snapshot().currentLimit <= 20 + 2 * 5);
|
||||
});
|
||||
|
||||
it("does not collapse capacity on a single upstream business error", async () => {
|
||||
const c = controller({ initialLimit: 40, decreaseFactor: 0.5, criticalDecreaseFactor: 0.5 });
|
||||
await complete(c, 10, 15, "upstream_error", "normal");
|
||||
clock.advance(100);
|
||||
c.tick();
|
||||
// One business error may freeze growth but must not apply critical collapse.
|
||||
assert.ok(c.snapshot().currentLimit >= 30);
|
||||
});
|
||||
|
||||
it("integrates active utilization exactly once over a full window", async () => {
|
||||
const c = controller({
|
||||
minLimit: 20,
|
||||
initialLimit: 20,
|
||||
maxLimit: 20,
|
||||
highUtilizationThreshold: 0.9,
|
||||
windowMs: 100,
|
||||
});
|
||||
const lease = await mustAdmit(c, req({ cost: 8 }));
|
||||
clock.advance(100);
|
||||
assert.equal(c.snapshot().utilization, 0.4);
|
||||
lease.release("success");
|
||||
});
|
||||
|
||||
it("consumes latency and pressure evidence only in the window where it was observed", async () => {
|
||||
const c = controller({
|
||||
initialLimit: 80,
|
||||
minLimit: 10,
|
||||
maxLimit: 100,
|
||||
decreaseFactor: 0.5,
|
||||
criticalDecreaseFactor: 0.25,
|
||||
windowMs: 100,
|
||||
});
|
||||
|
||||
await complete(c, 8, 200, "success", "high");
|
||||
clock.advance(100);
|
||||
const afterObservedWindow = c.snapshot().currentLimit;
|
||||
assert.ok(afterObservedWindow < 80);
|
||||
|
||||
clock.advance(500);
|
||||
assert.equal(c.snapshot().currentLimit, afterObservedWindow);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createAdmissionRejectError", () => {
|
||||
it("builds typed rejection errors", () => {
|
||||
const err = createAdmissionRejectError("ADMISSION_QUEUE_FULL", "queue full");
|
||||
assert.equal(err.code, "ADMISSION_QUEUE_FULL");
|
||||
assert.equal(err.name, "AdmissionRejectError");
|
||||
assert.match(err.message, /queue full/);
|
||||
});
|
||||
});
|
||||
143
tests/unit/adaptive-admission-cost.test.ts
Normal file
143
tests/unit/adaptive-admission-cost.test.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
estimateAdmissionCost,
|
||||
DEFAULT_ADMISSION_COST_CONFIG,
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
normalizeRequestCost,
|
||||
resolveCostConfig,
|
||||
type AdmissionCostConfig,
|
||||
type AdmissionCostFeatures,
|
||||
} from "../../open-sse/services/admission/index.ts";
|
||||
|
||||
function features(overrides: Partial<AdmissionCostFeatures> = {}): AdmissionCostFeatures {
|
||||
return {
|
||||
bodyBytes: 0,
|
||||
estimatedInputTokens: 0,
|
||||
messageCount: 0,
|
||||
toolCount: 0,
|
||||
requestedFanout: 1,
|
||||
streaming: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("estimateAdmissionCost", () => {
|
||||
it("returns a positive integer at least the base cost", () => {
|
||||
const cost = estimateAdmissionCost(features());
|
||||
assert.equal(Number.isSafeInteger(cost), true);
|
||||
assert.ok(cost >= DEFAULT_ADMISSION_COST_CONFIG.baseCost);
|
||||
assert.ok(cost > 0);
|
||||
});
|
||||
|
||||
it("is monotonic in body bytes, tokens, messages, tools, and fanout", () => {
|
||||
const base = estimateAdmissionCost(features());
|
||||
assert.ok(estimateAdmissionCost(features({ bodyBytes: 50_000 })) >= base);
|
||||
assert.ok(estimateAdmissionCost(features({ estimatedInputTokens: 8_000 })) >= base);
|
||||
assert.ok(estimateAdmissionCost(features({ messageCount: 40 })) >= base);
|
||||
assert.ok(estimateAdmissionCost(features({ toolCount: 20 })) >= base);
|
||||
assert.ok(estimateAdmissionCost(features({ requestedFanout: 8 })) >= base);
|
||||
});
|
||||
|
||||
it("rejects fractional, infinite, and unsafe cost configuration", () => {
|
||||
for (const invalid of [0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
|
||||
assert.throws(
|
||||
() => resolveCostConfig({ bodyBytesPerUnit: invalid }),
|
||||
/positive safe integer/
|
||||
);
|
||||
assert.throws(() => resolveCostConfig({ maxRequestCost: invalid }), /positive safe integer/);
|
||||
assert.throws(
|
||||
() => resolveCostConfig({ streamingClassCost: invalid }),
|
||||
/positive safe integer/
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("normalizes caller costs only when they are positive safe integers", () => {
|
||||
assert.equal(normalizeRequestCost(7, 10), 7);
|
||||
for (const invalid of [0, -1, 0.5, Number.POSITIVE_INFINITY, Number.MAX_VALUE]) {
|
||||
assert.throws(() => normalizeRequestCost(invalid, 10), /positive safe integer/);
|
||||
}
|
||||
assert.throws(() => normalizeRequestCost(1, Number.MAX_VALUE), /positive safe integer/);
|
||||
assert.throws(
|
||||
() => normalizeRequestCost(1, MAX_ADMISSION_COST_OR_LIMIT + 1),
|
||||
/maxRequestCost|must be <=/
|
||||
);
|
||||
assert.equal(
|
||||
normalizeRequestCost(MAX_ADMISSION_COST_OR_LIMIT, MAX_ADMISSION_COST_OR_LIMIT),
|
||||
MAX_ADMISSION_COST_OR_LIMIT
|
||||
);
|
||||
});
|
||||
|
||||
it("clamps multiple overflowing feature contributions without unsafe arithmetic", () => {
|
||||
const config: AdmissionCostConfig = {
|
||||
...DEFAULT_ADMISSION_COST_CONFIG,
|
||||
maxRequestCost: 25,
|
||||
};
|
||||
const cost = estimateAdmissionCost(
|
||||
features({
|
||||
bodyBytes: Number.MAX_SAFE_INTEGER,
|
||||
estimatedInputTokens: Number.MAX_SAFE_INTEGER,
|
||||
messageCount: Number.MAX_SAFE_INTEGER,
|
||||
toolCount: Number.MAX_SAFE_INTEGER,
|
||||
requestedFanout: Number.MAX_SAFE_INTEGER,
|
||||
}),
|
||||
config
|
||||
);
|
||||
assert.equal(cost, 25);
|
||||
});
|
||||
|
||||
it("normalizes invalid, negative, and NaN inputs safely", () => {
|
||||
const cost = estimateAdmissionCost({
|
||||
bodyBytes: Number.NaN,
|
||||
estimatedInputTokens: -12,
|
||||
messageCount: Number.POSITIVE_INFINITY,
|
||||
toolCount: undefined,
|
||||
requestedFanout: 0,
|
||||
streaming: undefined,
|
||||
} as AdmissionCostFeatures);
|
||||
assert.equal(Number.isSafeInteger(cost), true);
|
||||
assert.ok(cost >= 1);
|
||||
assert.ok(cost <= DEFAULT_ADMISSION_COST_CONFIG.maxRequestCost);
|
||||
});
|
||||
|
||||
it("uses transparent configurable quanta without a fixed-MB claim", () => {
|
||||
const config: AdmissionCostConfig = {
|
||||
baseCost: 1,
|
||||
bodyBytesPerUnit: 1000,
|
||||
tokensPerUnit: 100,
|
||||
messagesPerUnit: 10,
|
||||
toolsPerUnit: 5,
|
||||
fanoutPerUnit: 1,
|
||||
streamingClassCost: 1,
|
||||
nonStreamingClassCost: 3,
|
||||
maxRequestCost: 1000,
|
||||
};
|
||||
// 2500 bytes → 3 units (ceil), 250 tokens → 3 units, 1 message → 1, 0 tools, fanout 1 → 1, streaming class 1
|
||||
const cost = estimateAdmissionCost(
|
||||
features({
|
||||
bodyBytes: 2500,
|
||||
estimatedInputTokens: 250,
|
||||
messageCount: 1,
|
||||
toolCount: 0,
|
||||
requestedFanout: 1,
|
||||
streaming: true,
|
||||
}),
|
||||
config
|
||||
);
|
||||
assert.equal(cost, 1 + 3 + 3 + 1 + 0 + 1 + 1);
|
||||
|
||||
const nonStream = estimateAdmissionCost(
|
||||
features({
|
||||
bodyBytes: 0,
|
||||
estimatedInputTokens: 0,
|
||||
messageCount: 0,
|
||||
toolCount: 0,
|
||||
requestedFanout: 1,
|
||||
streaming: false,
|
||||
}),
|
||||
config
|
||||
);
|
||||
assert.equal(nonStream, 1 + 0 + 0 + 0 + 0 + 1 + 3);
|
||||
});
|
||||
});
|
||||
405
tests/unit/adaptive-admission-domain.test.ts
Normal file
405
tests/unit/adaptive-admission-domain.test.ts
Normal file
@@ -0,0 +1,405 @@
|
||||
import { describe, it, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
AdaptiveAdmissionController,
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
MAX_ADMISSION_WINDOW_MS,
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionLease,
|
||||
type AdmissionRequest,
|
||||
} from "../../open-sse/services/admission/index.ts";
|
||||
|
||||
class FakeClock {
|
||||
nowMs = 0;
|
||||
private nextId = 1;
|
||||
private timers = new Map<number, { due: number; fn: () => void }>();
|
||||
|
||||
now = () => this.nowMs;
|
||||
|
||||
setTimer = (fn: () => void, delayMs: number): number => {
|
||||
const id = this.nextId++;
|
||||
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
|
||||
return id;
|
||||
};
|
||||
|
||||
clearTimer = (id: number): void => {
|
||||
this.timers.delete(id);
|
||||
};
|
||||
|
||||
get pendingTimerCount(): number {
|
||||
return this.timers.size;
|
||||
}
|
||||
|
||||
advance(ms: number): void {
|
||||
const target = this.nowMs + ms;
|
||||
while (true) {
|
||||
let nextId: number | undefined;
|
||||
let nextDue = Number.POSITIVE_INFINITY;
|
||||
for (const [id, t] of this.timers) {
|
||||
if (t.due <= target && t.due < nextDue) {
|
||||
nextDue = t.due;
|
||||
nextId = id;
|
||||
}
|
||||
}
|
||||
if (nextId === undefined) {
|
||||
this.nowMs = target;
|
||||
return;
|
||||
}
|
||||
const timer = this.timers.get(nextId)!;
|
||||
this.timers.delete(nextId);
|
||||
this.nowMs = timer.due;
|
||||
timer.fn();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function baseConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
|
||||
return {
|
||||
mode: "enforce",
|
||||
minLimit: 10,
|
||||
maxLimit: 100,
|
||||
initialLimit: 20,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 1000,
|
||||
windowMs: 100,
|
||||
shortLatencyAlpha: 0.5,
|
||||
longLatencyAlpha: 0.1,
|
||||
increaseStep: 2,
|
||||
decreaseFactor: 0.8,
|
||||
criticalDecreaseFactor: 0.5,
|
||||
highUtilizationThreshold: 0.7,
|
||||
lowUtilizationThreshold: 0.3,
|
||||
latencyGradientThreshold: 0.25,
|
||||
maxIncreasePerWindow: 4,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function req(partial: Partial<AdmissionRequest> & { cost: number }): AdmissionRequest {
|
||||
return {
|
||||
tenantKey: "t-default",
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
async function mustAdmit(
|
||||
controller: AdaptiveAdmissionController,
|
||||
request: AdmissionRequest
|
||||
): Promise<AdmissionLease> {
|
||||
const result = await controller.acquire(request);
|
||||
assert.equal(result.status, "admitted");
|
||||
if (result.status !== "admitted") throw new Error("expected admitted");
|
||||
return result.lease;
|
||||
}
|
||||
|
||||
describe("admission operational domain", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
function make(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
|
||||
return new AdaptiveAdmissionController(baseConfig(overrides), {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
});
|
||||
}
|
||||
|
||||
it("exports an operational domain that rejects max+1 and accepts exact max", () => {
|
||||
assert.ok(Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT));
|
||||
assert.ok(Number.isSafeInteger(MAX_ADMISSION_WINDOW_MS));
|
||||
assert.ok(
|
||||
Number.isSafeInteger(MAX_ADMISSION_COST_OR_LIMIT * MAX_ADMISSION_WINDOW_MS),
|
||||
"limit×window must remain a safe integer"
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
make({
|
||||
minLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
|
||||
maxLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
|
||||
initialLimit: MAX_ADMISSION_COST_OR_LIMIT + 1,
|
||||
}),
|
||||
/minLimit|must be <=/
|
||||
);
|
||||
assert.throws(
|
||||
() => make({ maxQueueCost: MAX_ADMISSION_COST_OR_LIMIT + 1 }),
|
||||
/maxQueueCost|must be <=/
|
||||
);
|
||||
assert.throws(() => make({ windowMs: MAX_ADMISSION_WINDOW_MS + 1 }), /windowMs|must be <=/);
|
||||
assert.throws(
|
||||
() => make({ cost: { maxRequestCost: MAX_ADMISSION_COST_OR_LIMIT + 1 } }),
|
||||
/maxRequestCost|must be <=/
|
||||
);
|
||||
assert.throws(() => make({ maxLimit: Number.MAX_SAFE_INTEGER }), /maxLimit|must be <=/);
|
||||
|
||||
const max = MAX_ADMISSION_COST_OR_LIMIT;
|
||||
const c = make({
|
||||
minLimit: max,
|
||||
maxLimit: max,
|
||||
initialLimit: max,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: max,
|
||||
windowMs: 1000,
|
||||
cost: { maxRequestCost: max },
|
||||
});
|
||||
assert.equal(c.snapshot().currentLimit, max);
|
||||
c.shutdown();
|
||||
});
|
||||
|
||||
it("keeps full utilization and multi-lease accounting exact at the domain max", async () => {
|
||||
const max = MAX_ADMISSION_COST_OR_LIMIT;
|
||||
const c = make({
|
||||
mode: "enforce",
|
||||
minLimit: max,
|
||||
maxLimit: max,
|
||||
initialLimit: max,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: max,
|
||||
windowMs: 1000,
|
||||
cost: { maxRequestCost: max },
|
||||
});
|
||||
|
||||
const full = await mustAdmit(c, req({ cost: max }));
|
||||
assert.equal(c.snapshot().activeCost, max);
|
||||
assert.equal(Number.isSafeInteger(c.snapshot().activeCost), true);
|
||||
clock.advance(1000);
|
||||
assert.equal(c.snapshot().utilization, 1);
|
||||
full.release("success");
|
||||
assert.equal(c.snapshot().activeCost, 0);
|
||||
assert.equal(c.snapshot().activeCount, 0);
|
||||
|
||||
const left = Math.floor(max / 2);
|
||||
const right = max - left;
|
||||
const a = await mustAdmit(c, req({ cost: left }));
|
||||
const b = await mustAdmit(c, req({ cost: right }));
|
||||
assert.equal(c.snapshot().activeCost, max);
|
||||
assert.equal(c.snapshot().activeCount, 2);
|
||||
clock.advance(1000);
|
||||
assert.equal(c.snapshot().utilization, 1);
|
||||
a.release("success");
|
||||
assert.equal(c.snapshot().activeCost, right);
|
||||
b.release("success");
|
||||
assert.equal(c.snapshot().activeCost, 0);
|
||||
assert.equal(c.snapshot().activeCount, 0);
|
||||
c.shutdown();
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateConfig re-evaluation", () => {
|
||||
let clock: FakeClock;
|
||||
const live: AdaptiveAdmissionController[] = [];
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
live.length = 0;
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const c of live) c.shutdown();
|
||||
live.length = 0;
|
||||
});
|
||||
|
||||
function controller(overrides: Partial<AdaptiveAdmissionConfig> = {}) {
|
||||
const c = new AdaptiveAdmissionController(baseConfig(overrides), {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
});
|
||||
live.push(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
it("updateConfig rejects queued work above the new enforce limit immediately", async () => {
|
||||
const c = controller({
|
||||
minLimit: 5,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 5_000,
|
||||
});
|
||||
const held = await mustAdmit(c, req({ cost: 10 }));
|
||||
const queued = await c.acquire(req({ cost: 8 }));
|
||||
assert.equal(queued.status, "queued");
|
||||
|
||||
c.updateConfig(
|
||||
baseConfig({
|
||||
minLimit: 5,
|
||||
initialLimit: 5,
|
||||
maxLimit: 5,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 5_000,
|
||||
})
|
||||
);
|
||||
|
||||
if (queued.status === "queued") {
|
||||
await assert.rejects(queued.promise, (err: unknown) => {
|
||||
assert.equal((err as { code?: string }).code, "ADMISSION_OVERSIZED");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(c.snapshot().queuedCount, 0);
|
||||
held.release();
|
||||
});
|
||||
|
||||
it("updateConfig enforce→shadow classifies individually oversized active work as virtual rejected", async () => {
|
||||
const c = controller({
|
||||
mode: "enforce",
|
||||
minLimit: 5,
|
||||
initialLimit: 20,
|
||||
maxLimit: 20,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
});
|
||||
const oversized = await mustAdmit(c, req({ cost: 15 }));
|
||||
const fitting = await mustAdmit(c, req({ cost: 5 }));
|
||||
|
||||
c.updateConfig(
|
||||
baseConfig({
|
||||
mode: "shadow",
|
||||
minLimit: 5,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
})
|
||||
);
|
||||
|
||||
const snap = c.snapshot();
|
||||
// currentLimit clamps to 10; cost 15 is individually oversized → virtual rejected, not queued.
|
||||
assert.equal(snap.currentLimit, 10);
|
||||
assert.equal(snap.virtualActiveCost, 5);
|
||||
assert.equal(snap.virtualActiveCount, 1);
|
||||
assert.equal(snap.virtualQueuedCost, 0);
|
||||
assert.equal(snap.virtualQueuedCount, 0);
|
||||
// Real active leases remain until release.
|
||||
assert.equal(snap.activeCost, 20);
|
||||
assert.equal(snap.activeCount, 2);
|
||||
|
||||
oversized.release();
|
||||
fitting.release();
|
||||
});
|
||||
|
||||
it("updateConfig rebuilds shadow virtual dispositions under new limits and queue bounds", async () => {
|
||||
const c = controller({
|
||||
mode: "shadow",
|
||||
minLimit: 5,
|
||||
initialLimit: 20,
|
||||
maxLimit: 20,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: 12,
|
||||
});
|
||||
const first = await c.acquire(req({ cost: 8 }));
|
||||
const second = await c.acquire(req({ cost: 8 }));
|
||||
const third = await c.acquire(req({ cost: 8 }));
|
||||
assert.equal(first.status, "admitted");
|
||||
assert.equal(second.status, "admitted");
|
||||
assert.equal(third.status, "admitted");
|
||||
|
||||
c.updateConfig(
|
||||
baseConfig({
|
||||
mode: "shadow",
|
||||
minLimit: 5,
|
||||
initialLimit: 10,
|
||||
maxLimit: 10,
|
||||
maxQueueCount: 1,
|
||||
maxQueueCost: 8,
|
||||
})
|
||||
);
|
||||
|
||||
const snap = c.snapshot();
|
||||
// One active (8), one queued (8), one rejected (queue full under new bounds).
|
||||
assert.equal(snap.virtualActiveCost, 8);
|
||||
assert.equal(snap.virtualActiveCount, 1);
|
||||
assert.equal(snap.virtualQueuedCost, 8);
|
||||
assert.equal(snap.virtualQueuedCount, 1);
|
||||
assert.equal(snap.activeCount, 3);
|
||||
|
||||
if (first.status === "admitted") first.lease.release();
|
||||
if (second.status === "admitted") second.lease.release();
|
||||
if (third.status === "admitted") third.lease.release();
|
||||
});
|
||||
});
|
||||
|
||||
describe("deterministic overload harness", () => {
|
||||
async function runEqualServiceWindows(offeredPerWindow: number) {
|
||||
const clock = new FakeClock();
|
||||
const c = new AdaptiveAdmissionController(
|
||||
baseConfig({
|
||||
mode: "enforce",
|
||||
initialLimit: 20,
|
||||
minLimit: 20,
|
||||
maxLimit: 20,
|
||||
maxQueueCount: 1,
|
||||
maxQueueCost: 1,
|
||||
defaultMaxWaitMs: 20,
|
||||
}),
|
||||
{ now: clock.now, setTimer: clock.setTimer, clearTimer: clock.clearTimer }
|
||||
);
|
||||
let completed = 0;
|
||||
let fastRejected = 0;
|
||||
let active: AdmissionLease[] = [];
|
||||
|
||||
for (let window = 0; window < 20; window++) {
|
||||
for (const lease of active) {
|
||||
lease.release("success", { latencyMs: 10 });
|
||||
completed += 1;
|
||||
}
|
||||
active = [];
|
||||
for (let i = 0; i < offeredPerWindow; i++) {
|
||||
const result = await c.acquire(req({ cost: 5, tenantKey: `tenant-${i % 3}` }));
|
||||
if (result.status === "admitted") active.push(result.lease);
|
||||
else if (result.status === "rejected") fastRejected += 1;
|
||||
else assert.fail("cost-5 excess must reject immediately when queue cost cap is 1");
|
||||
}
|
||||
clock.advance(10);
|
||||
const snapshot = c.snapshot();
|
||||
assert.ok(snapshot.activeCost <= 20);
|
||||
assert.ok(snapshot.activeCount <= 4);
|
||||
assert.equal(snapshot.queuedCost, 0);
|
||||
assert.equal(snapshot.queuedCount, 0);
|
||||
}
|
||||
for (const lease of active) {
|
||||
lease.release("success", { latencyMs: 10 });
|
||||
completed += 1;
|
||||
}
|
||||
c.shutdown();
|
||||
assert.equal(clock.pendingTimerCount, 0);
|
||||
return { completed, fastRejected };
|
||||
}
|
||||
|
||||
it("raises goodput to capacity then plateaus at 2× and 5× offered load", async () => {
|
||||
// Capacity is 4 admits/window (limit 20, cost 5). Offered loads: 0.5×, 1×, 2×, 5×.
|
||||
const low = await runEqualServiceWindows(2);
|
||||
const atCapacity = await runEqualServiceWindows(4);
|
||||
const doubleOver = await runEqualServiceWindows(8);
|
||||
const fiveOver = await runEqualServiceWindows(20);
|
||||
|
||||
assert.equal(low.completed, 40);
|
||||
assert.equal(atCapacity.completed, 80);
|
||||
assert.ok(atCapacity.completed >= low.completed * 1.9, "goodput must rise toward capacity");
|
||||
assert.equal(
|
||||
doubleOver.completed,
|
||||
atCapacity.completed,
|
||||
"2× offered load must plateau at capacity"
|
||||
);
|
||||
assert.equal(
|
||||
fiveOver.completed,
|
||||
atCapacity.completed,
|
||||
"5× offered load must plateau at capacity"
|
||||
);
|
||||
assert.equal(low.fastRejected, 0);
|
||||
assert.equal(atCapacity.fastRejected, 0);
|
||||
assert.equal(
|
||||
doubleOver.fastRejected,
|
||||
4 * 20,
|
||||
"2× excess rejects immediately with bounded queue"
|
||||
);
|
||||
assert.equal(
|
||||
fiveOver.fastRejected,
|
||||
16 * 20,
|
||||
"5× excess rejects immediately with bounded queue"
|
||||
);
|
||||
});
|
||||
});
|
||||
255
tests/unit/adaptive-admission-features.test.ts
Normal file
255
tests/unit/adaptive-admission-features.test.ts
Normal file
@@ -0,0 +1,255 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { estimateAdmissionCost } from "../../open-sse/services/admission/cost.ts";
|
||||
import {
|
||||
ADMISSION_TOOL_SCAN_BUDGET,
|
||||
extractAdmissionCostFeatures,
|
||||
} from "../../open-sse/services/admission/requestFeatures.ts";
|
||||
|
||||
describe("bounded request feature extraction", () => {
|
||||
it("does not call JSON.stringify or toJSON", () => {
|
||||
let stringifyCalls = 0;
|
||||
const original = JSON.stringify;
|
||||
JSON.stringify = ((...args: Parameters<typeof JSON.stringify>) => {
|
||||
stringifyCalls += 1;
|
||||
return original.apply(JSON, args as [unknown]);
|
||||
}) as typeof JSON.stringify;
|
||||
try {
|
||||
const body = {
|
||||
toJSON() {
|
||||
throw new Error("toJSON must not be invoked");
|
||||
},
|
||||
messages: [{ role: "user", content: "hello world" }],
|
||||
tools: [{ type: "function", function: { name: "x" } }],
|
||||
n: 3,
|
||||
stream: true,
|
||||
};
|
||||
const features = extractAdmissionCostFeatures(body);
|
||||
assert.ok((features.bodyBytes ?? 0) > 0);
|
||||
assert.ok((features.messageCount ?? 0) >= 1);
|
||||
assert.ok((features.toolCount ?? 0) >= 1);
|
||||
assert.equal(features.requestedFanout, 3);
|
||||
assert.equal(features.streaming, true);
|
||||
assert.ok((features.estimatedInputTokens ?? 0) > 0);
|
||||
assert.equal(stringifyCalls, 0);
|
||||
} finally {
|
||||
JSON.stringify = original;
|
||||
}
|
||||
});
|
||||
|
||||
it("extracts production-realistic Chat, Responses, Gemini, and Antigravity shapes", () => {
|
||||
// OpenAI Chat Completions — stream omitted defaults false (higher non-stream class).
|
||||
const chat = extractAdmissionCostFeatures({
|
||||
model: "gpt-4o",
|
||||
messages: [
|
||||
{ role: "system", content: "You are helpful." },
|
||||
{ role: "user", content: "Summarize the logs" },
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "lookup", parameters: { type: "object" } },
|
||||
},
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "write", parameters: { type: "object" } },
|
||||
},
|
||||
],
|
||||
n: 2,
|
||||
});
|
||||
assert.equal(chat.messageCount, 2);
|
||||
assert.equal(chat.toolCount, 2);
|
||||
assert.equal(chat.requestedFanout, 2);
|
||||
assert.equal(chat.streaming, false);
|
||||
|
||||
// OpenAI Responses API — string input counts as one item; array counts length.
|
||||
const responsesString = extractAdmissionCostFeatures({
|
||||
model: "gpt-4.1",
|
||||
input: "What is the capital of France?",
|
||||
tools: [{ type: "web_search_preview" }],
|
||||
stream: true,
|
||||
});
|
||||
assert.equal(responsesString.messageCount, 1);
|
||||
assert.equal(responsesString.toolCount, 1);
|
||||
assert.equal(responsesString.streaming, true);
|
||||
|
||||
const responsesArray = extractAdmissionCostFeatures({
|
||||
model: "gpt-4.1",
|
||||
input: [
|
||||
{ role: "user", content: [{ type: "input_text", text: "q1" }] },
|
||||
{ role: "user", content: [{ type: "input_text", text: "q2" }] },
|
||||
],
|
||||
stream: false,
|
||||
n: 3,
|
||||
});
|
||||
assert.equal(responsesArray.messageCount, 2);
|
||||
assert.equal(responsesArray.requestedFanout, 3);
|
||||
assert.equal(responsesArray.streaming, false);
|
||||
|
||||
// Empty string input is not a content item.
|
||||
const emptyInput = extractAdmissionCostFeatures({ input: "" });
|
||||
assert.equal(emptyInput.messageCount, 0);
|
||||
|
||||
// Gemini generateContent — nested generationConfig.candidateCount + functionDeclarations.
|
||||
const gemini = extractAdmissionCostFeatures({
|
||||
contents: [
|
||||
{ role: "user", parts: [{ text: "hello" }] },
|
||||
{ role: "model", parts: [{ text: "world" }] },
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: [
|
||||
{ name: "get_weather", parameters: { type: "OBJECT" } },
|
||||
{ name: "get_time", parameters: { type: "OBJECT" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: { candidateCount: 4, temperature: 0.2 },
|
||||
});
|
||||
assert.equal(gemini.messageCount, 2);
|
||||
assert.equal(gemini.toolCount, 2);
|
||||
assert.equal(gemini.requestedFanout, 4);
|
||||
assert.equal(gemini.streaming, false);
|
||||
|
||||
// Antigravity-style wrapper under `request`.
|
||||
const antigravity = extractAdmissionCostFeatures({
|
||||
request: {
|
||||
contents: [{ role: "user", parts: [{ text: "hi" }] }],
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: [{ name: "a" }, { name: "b" }, { name: "c" }],
|
||||
},
|
||||
],
|
||||
generationConfig: { candidateCount: 5 },
|
||||
stream: true,
|
||||
},
|
||||
});
|
||||
assert.equal(antigravity.messageCount, 1);
|
||||
assert.equal(antigravity.toolCount, 3);
|
||||
assert.equal(antigravity.requestedFanout, 5);
|
||||
assert.equal(antigravity.streaming, true);
|
||||
|
||||
// Authoritative extraction context wins over body stream inference.
|
||||
const overridden = extractAdmissionCostFeatures(
|
||||
{ messages: [{ role: "user", content: "x" }], stream: false },
|
||||
{ streaming: true }
|
||||
);
|
||||
assert.equal(overridden.streaming, true);
|
||||
|
||||
const overriddenOff = extractAdmissionCostFeatures(
|
||||
{ messages: [{ role: "user", content: "x" }], stream: true },
|
||||
{ streaming: false }
|
||||
);
|
||||
assert.equal(overriddenOff.streaming, false);
|
||||
});
|
||||
|
||||
it("bounds tool scans and never touches entries beyond the budget (conservative count)", () => {
|
||||
// Huge leading string makes estimateSizeFast byte-exit before walking tools,
|
||||
// so only countTools can touch the tools proxy — proving its scan bound alone.
|
||||
const sizePad = "x".repeat(300_000);
|
||||
|
||||
let accesses = 0;
|
||||
const tools = new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 10_000;
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
const index = Number(prop);
|
||||
accesses += 1;
|
||||
if (index >= ADMISSION_TOOL_SCAN_BUDGET) {
|
||||
throw new Error(`tool entry ${index} must not be touched`);
|
||||
}
|
||||
return { type: "function", function: { name: `t${index}` } };
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
const features = extractAdmissionCostFeatures({ pad: sizePad, tools });
|
||||
// Any uninspected tail saturates the feature so heavier unseen entries cannot undercharge.
|
||||
assert.equal(features.toolCount, Number.MAX_SAFE_INTEGER);
|
||||
assert.equal(accesses, 0, "known oversized source should saturate before indexed access");
|
||||
|
||||
// functionDeclarations length is O(1); truncated tail still cannot undercharge.
|
||||
let declAccesses = 0;
|
||||
const geminiTools = new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 50;
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
const index = Number(prop);
|
||||
declAccesses += 1;
|
||||
if (index >= ADMISSION_TOOL_SCAN_BUDGET) {
|
||||
throw new Error(`gemini tool entry ${index} must not be touched`);
|
||||
}
|
||||
return {
|
||||
functionDeclarations: new Proxy([] as unknown[], {
|
||||
get(t, p, r) {
|
||||
if (p === "length") return 3;
|
||||
if (typeof p === "string" && /^[0-9]+$/.test(p)) {
|
||||
throw new Error("functionDeclarations elements need not be scanned");
|
||||
}
|
||||
return Reflect.get(t, p, r);
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
const geminiFeatures = extractAdmissionCostFeatures({ pad: sizePad, tools: geminiTools });
|
||||
assert.equal(geminiFeatures.toolCount, Number.MAX_SAFE_INTEGER);
|
||||
assert.equal(declAccesses, 0);
|
||||
|
||||
let aliasTouches = 0;
|
||||
const sixtyFour = (label: string) =>
|
||||
new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET;
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
aliasTouches += 1;
|
||||
return { name: `${label}-${prop}` };
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
const aliases = extractAdmissionCostFeatures({
|
||||
pad: sizePad,
|
||||
tools: sixtyFour("tool"),
|
||||
functions: sixtyFour("function"),
|
||||
});
|
||||
assert.equal(aliases.toolCount, Number.MAX_SAFE_INTEGER);
|
||||
assert.ok(aliasTouches <= ADMISSION_TOOL_SCAN_BUDGET);
|
||||
|
||||
let wrappedTouches = 0;
|
||||
const wrappedTail = new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return ADMISSION_TOOL_SCAN_BUDGET + 1;
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
wrappedTouches += 1;
|
||||
return { functionDeclarations: new Array(1_000).fill(null) };
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
const layered = extractAdmissionCostFeatures({
|
||||
pad: sizePad,
|
||||
tools: [{ type: "function" }],
|
||||
request: { tools: wrappedTail },
|
||||
});
|
||||
assert.equal(layered.toolCount, Number.MAX_SAFE_INTEGER);
|
||||
assert.equal(estimateAdmissionCost(layered), 1_000);
|
||||
assert.equal(wrappedTouches, 0);
|
||||
});
|
||||
|
||||
it("nested fanout under request wrapper is visible and stream defaults false", () => {
|
||||
const features = extractAdmissionCostFeatures({
|
||||
request: {
|
||||
messages: [{ role: "user", content: "x" }],
|
||||
n: 7,
|
||||
},
|
||||
});
|
||||
assert.equal(features.requestedFanout, 7);
|
||||
assert.equal(features.streaming, false);
|
||||
assert.equal(features.messageCount, 1);
|
||||
});
|
||||
});
|
||||
450
tests/unit/adaptive-admission-lifecycle.test.ts
Normal file
450
tests/unit/adaptive-admission-lifecycle.test.ts
Normal file
@@ -0,0 +1,450 @@
|
||||
import { describe, it, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
createAdaptiveAdmissionRuntime,
|
||||
DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
|
||||
type AdaptiveAdmissionRuntime,
|
||||
} from "../../open-sse/services/admission/runtime.ts";
|
||||
import {
|
||||
type AdaptiveAdmissionConfig,
|
||||
type AdmissionLease,
|
||||
type AdmissionReleaseMeta,
|
||||
type AdmissionReleaseOutcome,
|
||||
} from "../../open-sse/services/admission/types.ts";
|
||||
import type {
|
||||
ResourcePressureGuardResult,
|
||||
ResourcePressureObservation,
|
||||
} from "../../open-sse/utils/resourcePressure.ts";
|
||||
|
||||
/** Purpose-built lease spy: counts every release() while exposing released after first call. */
|
||||
function createSpyLease(id = "spy-lease", cost = 1) {
|
||||
const calls: Array<{ outcome?: AdmissionReleaseOutcome; meta?: AdmissionReleaseMeta }> = [];
|
||||
let released = false;
|
||||
const lease: AdmissionLease = {
|
||||
id,
|
||||
cost,
|
||||
get released() {
|
||||
return released;
|
||||
},
|
||||
release(outcome?: AdmissionReleaseOutcome, meta?: AdmissionReleaseMeta) {
|
||||
calls.push({ outcome, meta });
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
return {
|
||||
lease,
|
||||
calls,
|
||||
get releaseCount() {
|
||||
return calls.length;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
class FakeClock {
|
||||
nowMs = 0;
|
||||
private nextId = 1;
|
||||
private timers = new Map<number, { due: number; fn: () => void }>();
|
||||
|
||||
now = () => this.nowMs;
|
||||
|
||||
setTimer = (fn: () => void, delayMs: number): number => {
|
||||
const id = this.nextId++;
|
||||
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
|
||||
return id;
|
||||
};
|
||||
|
||||
clearTimer = (id: number): void => {
|
||||
this.timers.delete(id);
|
||||
};
|
||||
|
||||
get pendingTimerCount(): number {
|
||||
return this.timers.size;
|
||||
}
|
||||
|
||||
advance(ms: number): void {
|
||||
const target = this.nowMs + ms;
|
||||
while (true) {
|
||||
let nextId: number | undefined;
|
||||
let nextDue = Number.POSITIVE_INFINITY;
|
||||
for (const [id, t] of this.timers) {
|
||||
if (t.due <= target && t.due < nextDue) {
|
||||
nextDue = t.due;
|
||||
nextId = id;
|
||||
}
|
||||
}
|
||||
if (nextId === undefined) {
|
||||
this.nowMs = target;
|
||||
return;
|
||||
}
|
||||
const timer = this.timers.get(nextId)!;
|
||||
this.timers.delete(nextId);
|
||||
this.nowMs = timer.due;
|
||||
timer.fn();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function emptyObservation(
|
||||
overrides: Partial<ResourcePressureObservation["state"]> = {}
|
||||
): ResourcePressureObservation {
|
||||
return {
|
||||
signals: null,
|
||||
state: {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeRuntime(
|
||||
clock: FakeClock,
|
||||
overrides: {
|
||||
config?: AdaptiveAdmissionConfig;
|
||||
check?: () => ResourcePressureGuardResult | null;
|
||||
observe?: () => ResourcePressureObservation;
|
||||
warn?: (message: string) => void;
|
||||
} = {}
|
||||
): AdaptiveAdmissionRuntime {
|
||||
return createAdaptiveAdmissionRuntime({
|
||||
config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG },
|
||||
clock: {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
},
|
||||
checkResourcePressure: overrides.check ?? (() => null),
|
||||
getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()),
|
||||
warn: overrides.warn,
|
||||
});
|
||||
}
|
||||
|
||||
describe("response lifecycle helpers", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
function attachJson(
|
||||
runtime: AdaptiveAdmissionRuntime,
|
||||
spy: ReturnType<typeof createSpyLease>,
|
||||
status: number,
|
||||
options: { signal?: AbortSignal; admittedAtMs?: number } = {}
|
||||
) {
|
||||
const admittedAtMs = options.admittedAtMs ?? clock.nowMs;
|
||||
return runtime.attachResponseLifecycle(
|
||||
new Response(JSON.stringify({ ok: status < 400 }), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
spy.lease,
|
||||
{ admittedAtMs, signal: options.signal, nowMs: clock.now }
|
||||
);
|
||||
}
|
||||
|
||||
it("classifies non-SSE HTTP outcomes with cancellation winning", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const cases: Array<{
|
||||
status: number;
|
||||
expected: AdmissionReleaseOutcome;
|
||||
signal?: AbortSignal;
|
||||
label: string;
|
||||
}> = [
|
||||
{ status: 200, expected: "success", label: "2xx" },
|
||||
{ status: 302, expected: "success", label: "3xx" },
|
||||
{ status: 400, expected: "local_reject", label: "ordinary 4xx" },
|
||||
{ status: 429, expected: "local_reject", label: "429" },
|
||||
{ status: 408, expected: "timeout", label: "408" },
|
||||
{ status: 499, expected: "cancelled", label: "499" },
|
||||
{ status: 504, expected: "timeout", label: "504" },
|
||||
{ status: 502, expected: "upstream_error", label: "5xx" },
|
||||
{ status: 500, expected: "upstream_error", label: "500" },
|
||||
];
|
||||
|
||||
for (const c of cases) {
|
||||
const spy = createSpyLease(`json-${c.label}`);
|
||||
clock.nowMs = 100;
|
||||
attachJson(runtime, spy, c.status, { admittedAtMs: 40 });
|
||||
assert.equal(spy.releaseCount, 1, c.label);
|
||||
assert.equal(spy.calls[0]!.outcome, c.expected, c.label);
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 60, c.label);
|
||||
assert.equal(spy.lease.released, true, c.label);
|
||||
}
|
||||
|
||||
// Already-aborted signal wins over 2xx.
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const abortedSpy = createSpyLease("aborted-2xx");
|
||||
clock.nowMs = 200;
|
||||
attachJson(runtime, abortedSpy, 200, { signal: ac.signal, admittedAtMs: 150 });
|
||||
assert.equal(abortedSpy.releaseCount, 1);
|
||||
assert.equal(abortedSpy.calls[0]!.outcome, "cancelled");
|
||||
assert.equal(abortedSpy.calls[0]!.meta?.latencyMs, 50);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("classifies SSE completion outcomes using the request signal", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
let spySuffix = 0;
|
||||
|
||||
async function drainSse(
|
||||
status: number,
|
||||
signal?: AbortSignal,
|
||||
expectImmediateRelease = false
|
||||
): Promise<ReturnType<typeof createSpyLease>> {
|
||||
const spy = createSpyLease(`sse-${status}-${spySuffix++}`);
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("data: done\n\n"));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
clock.nowMs = 300;
|
||||
const wrapped = runtime.attachResponseLifecycle(
|
||||
new Response(body, {
|
||||
status,
|
||||
statusText: "OK",
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 250, signal, nowMs: clock.now }
|
||||
);
|
||||
if (expectImmediateRelease) {
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
return spy;
|
||||
}
|
||||
assert.equal(spy.releaseCount, 0);
|
||||
await wrapped.text();
|
||||
return spy;
|
||||
}
|
||||
|
||||
const ok = await drainSse(200);
|
||||
assert.equal(ok.releaseCount, 1);
|
||||
assert.equal(ok.calls[0]!.outcome, "success");
|
||||
assert.equal(ok.calls[0]!.meta?.latencyMs, 50);
|
||||
|
||||
const redirect = await drainSse(302);
|
||||
assert.equal(redirect.calls[0]!.outcome, "success");
|
||||
|
||||
const ordinary4xx = await drainSse(404);
|
||||
assert.equal(ordinary4xx.calls[0]!.outcome, "local_reject");
|
||||
|
||||
const tooMany = await drainSse(429);
|
||||
assert.equal(tooMany.calls[0]!.outcome, "local_reject");
|
||||
|
||||
const requestTimeout = await drainSse(408);
|
||||
assert.equal(requestTimeout.calls[0]!.outcome, "timeout");
|
||||
|
||||
const clientGone = await drainSse(499);
|
||||
assert.equal(clientGone.calls[0]!.outcome, "cancelled");
|
||||
|
||||
const gatewayTimeout = await drainSse(504);
|
||||
assert.equal(gatewayTimeout.calls[0]!.outcome, "timeout");
|
||||
|
||||
const upstream = await drainSse(503);
|
||||
assert.equal(upstream.calls[0]!.outcome, "upstream_error");
|
||||
|
||||
// Already-aborted signal settles immediately as cancelled (wins over 2xx).
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const abortedOk = await drainSse(200, ac.signal, true);
|
||||
assert.equal(abortedOk.releaseCount, 1);
|
||||
assert.equal(abortedOk.calls[0]!.outcome, "cancelled");
|
||||
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("requires explicit non-success outcomes for handler failures", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const outcomes: Array<Exclude<AdmissionReleaseOutcome, "success">> = [
|
||||
"local_reject",
|
||||
"upstream_error",
|
||||
"timeout",
|
||||
"cancelled",
|
||||
];
|
||||
for (const outcome of outcomes) {
|
||||
const spy = createSpyLease(`handler-${outcome}`);
|
||||
clock.nowMs = 500;
|
||||
runtime.releaseHandlerFailure(spy.lease, outcome, {
|
||||
admittedAtMs: 400,
|
||||
nowMs: clock.now,
|
||||
});
|
||||
assert.equal(spy.releaseCount, 1, outcome);
|
||||
assert.equal(spy.calls[0]!.outcome, outcome);
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 100);
|
||||
// Exactly-once: second call must not re-release.
|
||||
runtime.releaseHandlerFailure(spy.lease, outcome, {
|
||||
admittedAtMs: 400,
|
||||
nowMs: clock.now,
|
||||
});
|
||||
assert.equal(spy.releaseCount, 1, `${outcome} second`);
|
||||
}
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("releases JSON/non-SSE responses immediately once", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const spy = createSpyLease("json-once");
|
||||
clock.nowMs = 80;
|
||||
const wrapped = attachJson(runtime, spy, 200, { admittedAtMs: 20 });
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
assert.equal(spy.calls[0]!.outcome, "success");
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 60);
|
||||
assert.equal(await wrapped.text(), JSON.stringify({ ok: true }));
|
||||
runtime.attachResponseLifecycle(
|
||||
new Response("{}", { status: 200, headers: { "Content-Type": "application/json" } }),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 20, nowMs: clock.now }
|
||||
);
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("keeps SSE lease until stream drain and releases exactly once", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const spy = createSpyLease("sse-drain");
|
||||
let pullCount = 0;
|
||||
const chunks = [
|
||||
new TextEncoder().encode("data: 1\n\n"),
|
||||
new TextEncoder().encode("data: 2\n\n"),
|
||||
];
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (pullCount < chunks.length) {
|
||||
controller.enqueue(chunks[pullCount++]);
|
||||
return;
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
clock.nowMs = 120;
|
||||
const wrapped = runtime.attachResponseLifecycle(
|
||||
new Response(body, {
|
||||
status: 200,
|
||||
statusText: "OK",
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 100, nowMs: clock.now }
|
||||
);
|
||||
assert.equal(spy.releaseCount, 0);
|
||||
assert.equal(wrapped.status, 200);
|
||||
assert.equal(wrapped.statusText, "OK");
|
||||
assert.equal(wrapped.headers.get("Content-Type"), "text/event-stream");
|
||||
const text = await wrapped.text();
|
||||
assert.match(text, /data: 1/);
|
||||
assert.match(text, /data: 2/);
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
assert.equal(spy.calls[0]!.outcome, "success");
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 20);
|
||||
// Drain again must not re-release (stream already consumed).
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("releases once on stream error", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const spy = createSpyLease("sse-error");
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
controller.error(new Error("upstream boom"));
|
||||
},
|
||||
});
|
||||
clock.nowMs = 90;
|
||||
const wrapped = runtime.attachResponseLifecycle(
|
||||
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 70, nowMs: clock.now }
|
||||
);
|
||||
await assert.rejects(async () => {
|
||||
await wrapped.text();
|
||||
});
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
assert.equal(spy.calls[0]!.outcome, "upstream_error");
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 20);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("releases once on consumer cancel without buffering", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const spy = createSpyLease("sse-cancel");
|
||||
let cancelCount = 0;
|
||||
let pulled = 0;
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
pulled += 1;
|
||||
controller.enqueue(new TextEncoder().encode(`data: ${pulled}\n\n`));
|
||||
},
|
||||
cancel() {
|
||||
cancelCount += 1;
|
||||
},
|
||||
});
|
||||
clock.nowMs = 60;
|
||||
const wrapped = runtime.attachResponseLifecycle(
|
||||
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 10, nowMs: clock.now }
|
||||
);
|
||||
const reader = wrapped.body!.getReader();
|
||||
await reader.read();
|
||||
assert.equal(spy.releaseCount, 0);
|
||||
const pulledAfterFirst = pulled;
|
||||
await reader.cancel("client gone");
|
||||
assert.equal(cancelCount, 1);
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
assert.equal(spy.calls[0]!.outcome, "cancelled");
|
||||
assert.equal(spy.calls[0]!.meta?.latencyMs, 50);
|
||||
// Laziness: no full buffering of the infinite producer.
|
||||
assert.ok(pulledAfterFirst <= 2);
|
||||
assert.ok(pulled < 20);
|
||||
// Second cancel is a no-op for both reader cancel and lease release.
|
||||
await reader.cancel("again");
|
||||
assert.equal(cancelCount, 1);
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("request abort cancels the reader and releases once under races", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const spy = createSpyLease("sse-abort-race");
|
||||
const ac = new AbortController();
|
||||
let cancelCount = 0;
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("data: ping\n\n"));
|
||||
await new Promise<void>(() => {
|
||||
/* hang until cancel */
|
||||
});
|
||||
},
|
||||
cancel() {
|
||||
cancelCount += 1;
|
||||
},
|
||||
});
|
||||
clock.nowMs = 40;
|
||||
const wrapped = runtime.attachResponseLifecycle(
|
||||
new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }),
|
||||
spy.lease,
|
||||
{ admittedAtMs: 10, signal: ac.signal, nowMs: clock.now }
|
||||
);
|
||||
const reader = wrapped.body!.getReader();
|
||||
const first = reader.read();
|
||||
ac.abort();
|
||||
// Race: also cancel consumer.
|
||||
void reader.cancel("race");
|
||||
await Promise.race([
|
||||
first.catch(() => undefined),
|
||||
new Promise((resolve) => setImmediate(resolve)),
|
||||
]);
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
assert.equal(spy.releaseCount, 1);
|
||||
assert.equal(spy.calls[0]!.outcome, "cancelled");
|
||||
assert.equal(typeof spy.calls[0]!.meta?.latencyMs, "number");
|
||||
assert.ok((spy.calls[0]!.meta?.latencyMs ?? -1) >= 0);
|
||||
assert.equal(cancelCount, 1);
|
||||
runtime.dispose();
|
||||
});
|
||||
});
|
||||
67
tests/unit/adaptive-admission-queue.test.ts
Normal file
67
tests/unit/adaptive-admission-queue.test.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { FairCostQueue, type QueueEntry } from "../../open-sse/services/admission/queue.ts";
|
||||
|
||||
describe("FairCostQueue removeById cursor preservation", () => {
|
||||
function qEntry(id: string, tenantKey: string, cost = 1): QueueEntry<{ id: string }> {
|
||||
return {
|
||||
id,
|
||||
tenantKey,
|
||||
cost,
|
||||
enqueuedAtMs: 0,
|
||||
deadlineMs: Number.MAX_SAFE_INTEGER,
|
||||
payload: { id },
|
||||
};
|
||||
}
|
||||
|
||||
it("preserves the logical successor when removing a bucket before the cursor", () => {
|
||||
const q = new FairCostQueue<{ id: string }>(10, 100);
|
||||
assert.equal(q.enqueue(qEntry("a1", "a")), true);
|
||||
assert.equal(q.enqueue(qEntry("a2", "a")), true);
|
||||
assert.equal(q.enqueue(qEntry("b1", "b")), true);
|
||||
assert.equal(q.enqueue(qEntry("c1", "c")), true);
|
||||
|
||||
// Dequeue a1 leaves cursor at b while tenant a still has a2.
|
||||
assert.equal(q.dequeue()?.id, "a1");
|
||||
assert.equal(q.removeById("a2")?.id, "a2");
|
||||
// Successor of the pre-removal cursor must remain b, not skip to c.
|
||||
assert.equal(q.dequeue()?.id, "b1");
|
||||
assert.equal(q.dequeue()?.id, "c1");
|
||||
assert.equal(q.size, 0);
|
||||
});
|
||||
|
||||
it("preserves the logical successor when removing the bucket at the cursor", () => {
|
||||
const q = new FairCostQueue<{ id: string }>(10, 100);
|
||||
assert.equal(q.enqueue(qEntry("a1", "a")), true);
|
||||
assert.equal(q.enqueue(qEntry("b1", "b")), true);
|
||||
assert.equal(q.enqueue(qEntry("c1", "c")), true);
|
||||
|
||||
assert.equal(q.dequeue()?.id, "a1"); // cursor now at b
|
||||
assert.equal(q.removeById("b1")?.id, "b1");
|
||||
assert.equal(q.dequeue()?.id, "c1");
|
||||
assert.equal(q.size, 0);
|
||||
});
|
||||
|
||||
it("preserves the cursor when removing a bucket after the cursor", () => {
|
||||
const q = new FairCostQueue<{ id: string }>(10, 100);
|
||||
assert.equal(q.enqueue(qEntry("a1", "a")), true);
|
||||
assert.equal(q.enqueue(qEntry("b1", "b")), true);
|
||||
assert.equal(q.enqueue(qEntry("c1", "c")), true);
|
||||
|
||||
assert.equal(q.dequeue()?.id, "a1"); // cursor now at b
|
||||
assert.equal(q.removeById("c1")?.id, "c1");
|
||||
assert.equal(q.dequeue()?.id, "b1");
|
||||
assert.equal(q.size, 0);
|
||||
});
|
||||
|
||||
it("resets the cursor when the final bucket is removed", () => {
|
||||
const q = new FairCostQueue<{ id: string }>(10, 100);
|
||||
assert.equal(q.enqueue(qEntry("a1", "a")), true);
|
||||
assert.equal(q.enqueue(qEntry("b1", "b")), true);
|
||||
assert.equal(q.dequeue()?.id, "a1"); // cursor at b
|
||||
assert.equal(q.removeById("b1")?.id, "b1");
|
||||
assert.equal(q.size, 0);
|
||||
assert.equal(q.enqueue(qEntry("d1", "d")), true);
|
||||
assert.equal(q.dequeue()?.id, "d1");
|
||||
});
|
||||
});
|
||||
857
tests/unit/adaptive-admission-runtime.test.ts
Normal file
857
tests/unit/adaptive-admission-runtime.test.ts
Normal file
@@ -0,0 +1,857 @@
|
||||
import { describe, it, beforeEach, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
createAdaptiveAdmissionRuntime,
|
||||
getAdaptiveAdmissionRuntime,
|
||||
reloadAdaptiveAdmissionRuntime,
|
||||
resetAdaptiveAdmissionRuntimeForTests,
|
||||
resolveAdaptiveAdmissionConfigFromEnv,
|
||||
DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
|
||||
type AdaptiveAdmissionRuntime,
|
||||
} from "../../open-sse/services/admission/runtime.ts";
|
||||
import {
|
||||
MAX_ADMISSION_COST_OR_LIMIT,
|
||||
MAX_ADMISSION_WINDOW_MS,
|
||||
type AdaptiveAdmissionConfig,
|
||||
} from "../../open-sse/services/admission/types.ts";
|
||||
import type {
|
||||
ResourcePressureGuardResult,
|
||||
ResourcePressureObservation,
|
||||
} from "../../open-sse/utils/resourcePressure.ts";
|
||||
import { buildErrorBody } from "../../open-sse/utils/error.ts";
|
||||
|
||||
class FakeClock {
|
||||
nowMs = 0;
|
||||
private nextId = 1;
|
||||
private timers = new Map<number, { due: number; fn: () => void }>();
|
||||
|
||||
now = () => this.nowMs;
|
||||
|
||||
setTimer = (fn: () => void, delayMs: number): number => {
|
||||
const id = this.nextId++;
|
||||
this.timers.set(id, { due: this.nowMs + Math.max(0, delayMs), fn });
|
||||
return id;
|
||||
};
|
||||
|
||||
clearTimer = (id: number): void => {
|
||||
this.timers.delete(id);
|
||||
};
|
||||
|
||||
get pendingTimerCount(): number {
|
||||
return this.timers.size;
|
||||
}
|
||||
|
||||
advance(ms: number): void {
|
||||
const target = this.nowMs + ms;
|
||||
while (true) {
|
||||
let nextId: number | undefined;
|
||||
let nextDue = Number.POSITIVE_INFINITY;
|
||||
for (const [id, t] of this.timers) {
|
||||
if (t.due <= target && t.due < nextDue) {
|
||||
nextDue = t.due;
|
||||
nextId = id;
|
||||
}
|
||||
}
|
||||
if (nextId === undefined) {
|
||||
this.nowMs = target;
|
||||
return;
|
||||
}
|
||||
const timer = this.timers.get(nextId)!;
|
||||
this.timers.delete(nextId);
|
||||
this.nowMs = timer.due;
|
||||
timer.fn();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function enforceConfig(overrides: Partial<AdaptiveAdmissionConfig> = {}): AdaptiveAdmissionConfig {
|
||||
return {
|
||||
mode: "enforce",
|
||||
minLimit: 4,
|
||||
maxLimit: 20,
|
||||
initialLimit: 8,
|
||||
maxQueueCount: 2,
|
||||
maxQueueCost: 16,
|
||||
defaultMaxWaitMs: 100,
|
||||
windowMs: 50,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyObservation(
|
||||
overrides: Partial<ResourcePressureObservation["state"]> = {}
|
||||
): ResourcePressureObservation {
|
||||
return {
|
||||
signals: null,
|
||||
state: {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function criticalGuard(reason = "v8_heap_absolute"): ResourcePressureGuardResult {
|
||||
const message = "Service temporarily unavailable due to resource pressure. Retry shortly.";
|
||||
return {
|
||||
success: false,
|
||||
status: 503,
|
||||
error: message,
|
||||
response: new Response(
|
||||
JSON.stringify(
|
||||
buildErrorBody(503, message, undefined, {
|
||||
type: "server_error",
|
||||
code: "resource_pressure",
|
||||
})
|
||||
),
|
||||
{
|
||||
status: 503,
|
||||
headers: { "Content-Type": "application/json", "Retry-After": "5" },
|
||||
}
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function makeRuntime(
|
||||
clock: FakeClock,
|
||||
overrides: {
|
||||
config?: AdaptiveAdmissionConfig;
|
||||
check?: () => ResourcePressureGuardResult | null;
|
||||
observe?: () => ResourcePressureObservation;
|
||||
warn?: (message: string) => void;
|
||||
} = {}
|
||||
): AdaptiveAdmissionRuntime {
|
||||
return createAdaptiveAdmissionRuntime({
|
||||
config: overrides.config ?? { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG },
|
||||
clock: {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
},
|
||||
checkResourcePressure: overrides.check ?? (() => null),
|
||||
getResourcePressureObservation: overrides.observe ?? (() => emptyObservation()),
|
||||
warn: overrides.warn,
|
||||
});
|
||||
}
|
||||
|
||||
async function parseJson(response: Response): Promise<Record<string, unknown>> {
|
||||
return JSON.parse(await response.text()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("adaptive admission runtime env + defaults", () => {
|
||||
it("defaults to complete shadow config", () => {
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.mode, "shadow");
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.minLimit, 8);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.initialLimit, 64);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxLimit, 1000);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCount, 128);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.maxQueueCost, 2000);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.defaultMaxWaitMs, 5000);
|
||||
assert.equal(DEFAULT_ADAPTIVE_ADMISSION_CONFIG.windowMs, 1000);
|
||||
});
|
||||
|
||||
it("strictly resolves supported env names and rejects invalid values", () => {
|
||||
const cfg = resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MODE: "enforce",
|
||||
ADAPTIVE_ADMISSION_MIN_LIMIT: "10",
|
||||
ADAPTIVE_ADMISSION_INITIAL_LIMIT: "20",
|
||||
ADAPTIVE_ADMISSION_MAX_LIMIT: "30",
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "40",
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: "50",
|
||||
ADAPTIVE_ADMISSION_MAX_WAIT_MS: "600",
|
||||
ADAPTIVE_ADMISSION_WINDOW_MS: "700",
|
||||
});
|
||||
assert.deepEqual(
|
||||
{
|
||||
mode: cfg.mode,
|
||||
minLimit: cfg.minLimit,
|
||||
initialLimit: cfg.initialLimit,
|
||||
maxLimit: cfg.maxLimit,
|
||||
maxQueueCount: cfg.maxQueueCount,
|
||||
maxQueueCost: cfg.maxQueueCost,
|
||||
defaultMaxWaitMs: cfg.defaultMaxWaitMs,
|
||||
windowMs: cfg.windowMs,
|
||||
},
|
||||
{
|
||||
mode: "enforce",
|
||||
minLimit: 10,
|
||||
initialLimit: 20,
|
||||
maxLimit: 30,
|
||||
maxQueueCount: 40,
|
||||
maxQueueCost: 50,
|
||||
defaultMaxWaitMs: 600,
|
||||
windowMs: 700,
|
||||
}
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MODE: "strict" }),
|
||||
/ADAPTIVE_ADMISSION_MODE/
|
||||
);
|
||||
assert.throws(
|
||||
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MIN_LIMIT: "0" }),
|
||||
/ADAPTIVE_ADMISSION_MIN_LIMIT/
|
||||
);
|
||||
assert.throws(
|
||||
() => resolveAdaptiveAdmissionConfigFromEnv({ ADAPTIVE_ADMISSION_MAX_LIMIT: "1.5" }),
|
||||
/ADAPTIVE_ADMISSION_MAX_LIMIT/
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts exact documented maxima and rejects max+1 plus cross-field invalidity", () => {
|
||||
const maxCost = String(MAX_ADMISSION_COST_OR_LIMIT);
|
||||
const maxWindow = String(MAX_ADMISSION_WINDOW_MS);
|
||||
const maxQueue = String(Number.MAX_SAFE_INTEGER);
|
||||
|
||||
const atMaxima = resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MODE: "shadow",
|
||||
ADAPTIVE_ADMISSION_MIN_LIMIT: "1",
|
||||
ADAPTIVE_ADMISSION_INITIAL_LIMIT: maxCost,
|
||||
ADAPTIVE_ADMISSION_MAX_LIMIT: maxCost,
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: maxQueue,
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: maxCost,
|
||||
ADAPTIVE_ADMISSION_MAX_WAIT_MS: maxWindow,
|
||||
ADAPTIVE_ADMISSION_WINDOW_MS: maxWindow,
|
||||
});
|
||||
assert.equal(atMaxima.maxLimit, MAX_ADMISSION_COST_OR_LIMIT);
|
||||
assert.equal(atMaxima.maxQueueCount, Number.MAX_SAFE_INTEGER);
|
||||
assert.equal(atMaxima.windowMs, MAX_ADMISSION_WINDOW_MS);
|
||||
assert.equal(atMaxima.defaultMaxWaitMs, MAX_ADMISSION_WINDOW_MS);
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MAX_LIMIT: String(MAX_ADMISSION_COST_OR_LIMIT + 1),
|
||||
}),
|
||||
/maxLimit|must be <=/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COST: String(MAX_ADMISSION_COST_OR_LIMIT + 1),
|
||||
}),
|
||||
/maxQueueCost|must be <=/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_WINDOW_MS: String(MAX_ADMISSION_WINDOW_MS + 1),
|
||||
}),
|
||||
/windowMs|must be <=/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MAX_WAIT_MS: String(MAX_ADMISSION_WINDOW_MS + 1),
|
||||
}),
|
||||
/defaultMaxWaitMs|must be <=/
|
||||
);
|
||||
// Queue count uses full safe-integer range; beyond that fails lexical/safe-integer parsing.
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT: "9007199254740992",
|
||||
}),
|
||||
/ADAPTIVE_ADMISSION_MAX_QUEUE_COUNT|safe integer/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
resolveAdaptiveAdmissionConfigFromEnv({
|
||||
ADAPTIVE_ADMISSION_MIN_LIMIT: "20",
|
||||
ADAPTIVE_ADMISSION_MAX_LIMIT: "10",
|
||||
}),
|
||||
/minLimit must be <= maxLimit/
|
||||
);
|
||||
});
|
||||
|
||||
it("default process runtime falls back to shadow on invalid env without crashing", () => {
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
const warnings: string[] = [];
|
||||
const previous = process.env.ADAPTIVE_ADMISSION_MODE;
|
||||
process.env.ADAPTIVE_ADMISSION_MODE = "not-a-mode";
|
||||
try {
|
||||
const runtime = reloadAdaptiveAdmissionRuntime({
|
||||
warn: (message) => warnings.push(message),
|
||||
checkResourcePressure: () => null,
|
||||
getResourcePressureObservation: () => emptyObservation(),
|
||||
});
|
||||
const snap = runtime.snapshot();
|
||||
assert.equal(snap.mode, "shadow");
|
||||
assert.equal(snap.minLimit, 8);
|
||||
assert.equal(snap.initialLimit ?? snap.currentLimit >= 8, true);
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(
|
||||
warnings[0]!,
|
||||
/invalid environment configuration; using default shadow admission settings/
|
||||
);
|
||||
assert.ok(!warnings.join("\n").includes("not-a-mode"));
|
||||
assert.ok(!warnings.join("\n").toLowerCase().includes("secret"));
|
||||
runtime.dispose();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.ADAPTIVE_ADMISSION_MODE;
|
||||
else process.env.ADAPTIVE_ADMISSION_MODE = previous;
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("adaptive admission runtime modes", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
afterEach(() => {
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
});
|
||||
|
||||
it("default shadow always admits with a real lease and shadowDecision", async () => {
|
||||
const runtime = makeRuntime(clock);
|
||||
const result = await runtime.acquire({
|
||||
tenantKey: "tenant-secret-1",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: true },
|
||||
});
|
||||
assert.equal(result.status, "admitted");
|
||||
if (result.status !== "admitted") throw new Error("expected admitted");
|
||||
assert.equal(result.mode, "shadow");
|
||||
assert.ok(result.lease);
|
||||
assert.equal(typeof result.lease.release, "function");
|
||||
assert.equal(result.lease.released, false);
|
||||
assert.ok(
|
||||
result.shadowDecision === "would-admit" ||
|
||||
result.shadowDecision === "would-queue" ||
|
||||
result.shadowDecision === "would-reject"
|
||||
);
|
||||
result.lease.release("success");
|
||||
assert.equal(result.lease.released, true);
|
||||
result.lease.release("success");
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("explicit off admits without enforcing capacity", async () => {
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: {
|
||||
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
|
||||
mode: "off",
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
},
|
||||
});
|
||||
const a = await runtime.acquire({ tenantKey: "t1", body: { messages: [] } });
|
||||
const b = await runtime.acquire({ tenantKey: "t2", body: { messages: [] } });
|
||||
assert.equal(a.status, "admitted");
|
||||
assert.equal(b.status, "admitted");
|
||||
if (a.status === "admitted") a.lease.release();
|
||||
if (b.status === "admitted") b.lease.release();
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("explicit enforce can reject with sanitized HTTP response", async () => {
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: enforceConfig({
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
maxQueueCount: 1,
|
||||
maxQueueCost: 1,
|
||||
defaultMaxWaitMs: 50,
|
||||
cost: { maxRequestCost: 1, baseCost: 1 },
|
||||
}),
|
||||
});
|
||||
const first = await runtime.acquire({
|
||||
tenantKey: "t1",
|
||||
body: { messages: [{ role: "user", content: "a" }], stream: true },
|
||||
});
|
||||
assert.equal(first.status, "admitted");
|
||||
|
||||
const secondPromise = runtime.acquire({
|
||||
tenantKey: "t2",
|
||||
body: { messages: [{ role: "user", content: "b" }], stream: true },
|
||||
maxWaitMs: 50,
|
||||
});
|
||||
// Drive injected deadline timer; no wall-clock sleeps.
|
||||
clock.advance(50);
|
||||
const second = await secondPromise;
|
||||
assert.equal(second.status, "rejected");
|
||||
if (second.status !== "rejected") throw new Error("expected rejected");
|
||||
assert.equal(second.response.status, 503);
|
||||
const body = await parseJson(second.response);
|
||||
assert.equal(typeof body.error.message, "string");
|
||||
assert.match(second.code, /^admission_/);
|
||||
assert.ok(!JSON.stringify(body).includes("t2"));
|
||||
assert.ok(!JSON.stringify(body).includes("tenant"));
|
||||
if (first.status === "admitted") first.lease.release();
|
||||
runtime.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime streaming cost forwarding", () => {
|
||||
it("acquire lease cost reflects input.streaming via feature extraction", async () => {
|
||||
const clock = new FakeClock();
|
||||
// Sharply distinct streaming class costs; neutralize other feature contributions.
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: {
|
||||
...DEFAULT_ADAPTIVE_ADMISSION_CONFIG,
|
||||
mode: "shadow",
|
||||
cost: {
|
||||
baseCost: 1,
|
||||
bodyBytesPerUnit: 1_000_000,
|
||||
tokensPerUnit: 1_000_000,
|
||||
messagesPerUnit: 1_000_000,
|
||||
toolsPerUnit: 1_000_000,
|
||||
fanoutPerUnit: 1_000_000,
|
||||
streamingClassCost: 1,
|
||||
nonStreamingClassCost: 50,
|
||||
maxRequestCost: 1_000,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Empty body keeps non-class contributions identical; stream omitted defaults false when not forwarded.
|
||||
|
||||
const body = {};
|
||||
const streamed = await runtime.acquire({
|
||||
tenantKey: "stream-on",
|
||||
body,
|
||||
streaming: true,
|
||||
});
|
||||
assert.equal(streamed.status, "admitted");
|
||||
if (streamed.status !== "admitted") throw new Error("expected admitted");
|
||||
const streamCost = streamed.lease.cost;
|
||||
streamed.lease.release("success");
|
||||
|
||||
const nonStreamed = await runtime.acquire({
|
||||
tenantKey: "stream-off",
|
||||
body,
|
||||
streaming: false,
|
||||
});
|
||||
assert.equal(nonStreamed.status, "admitted");
|
||||
if (nonStreamed.status !== "admitted") throw new Error("expected admitted");
|
||||
const nonStreamCost = nonStreamed.lease.cost;
|
||||
nonStreamed.lease.release("success");
|
||||
|
||||
const defaulted = await runtime.acquire({
|
||||
tenantKey: "stream-default",
|
||||
body,
|
||||
});
|
||||
assert.equal(defaulted.status, "admitted");
|
||||
if (defaulted.status !== "admitted") throw new Error("expected admitted");
|
||||
const defaultCost = defaulted.lease.cost;
|
||||
defaulted.lease.release("success");
|
||||
|
||||
runtime.dispose();
|
||||
|
||||
// base(1) + fanout unit(1) + class cost → streaming 3, non-streaming 52
|
||||
assert.equal(streamCost, 3);
|
||||
assert.equal(nonStreamCost, 52);
|
||||
assert.equal(defaultCost, 52);
|
||||
assert.notEqual(
|
||||
streamCost,
|
||||
nonStreamCost,
|
||||
"streaming true/false must produce different acquired lease costs"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rejection mapping", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
it("maps ADMISSION_ABORTED to local 499 without Retry-After", async () => {
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: enforceConfig({
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 1000,
|
||||
// Force unit cost so one admitted request fills the limit.
|
||||
cost: { maxRequestCost: 1, baseCost: 1 },
|
||||
}),
|
||||
});
|
||||
const holder = await runtime.acquire({
|
||||
tenantKey: "hold",
|
||||
body: { messages: [{ role: "user", content: "hold" }], stream: true },
|
||||
});
|
||||
assert.equal(holder.status, "admitted");
|
||||
|
||||
const ac = new AbortController();
|
||||
const pending = runtime.acquire({
|
||||
tenantKey: "wait",
|
||||
body: { messages: [{ role: "user", content: "wait" }], stream: true },
|
||||
signal: ac.signal,
|
||||
maxWaitMs: 1000,
|
||||
});
|
||||
ac.abort();
|
||||
const rejected = await pending;
|
||||
assert.equal(rejected.status, "rejected");
|
||||
if (rejected.status !== "rejected") throw new Error("expected rejected");
|
||||
assert.equal(rejected.code, "admission_aborted");
|
||||
assert.equal(rejected.response.status, 499);
|
||||
assert.equal(rejected.response.headers.get("Retry-After"), null);
|
||||
const body = await parseJson(rejected.response);
|
||||
assert.equal(body.error.code, "admission_aborted");
|
||||
assert.ok(!JSON.stringify(body).includes("wait"));
|
||||
if (holder.status === "admitted") holder.lease.release();
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("maps queue full / deadline / oversized to sanitized 503 codes", async () => {
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: enforceConfig({
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
maxQueueCount: 1,
|
||||
maxQueueCost: 1,
|
||||
defaultMaxWaitMs: 20,
|
||||
cost: { maxRequestCost: 1, baseCost: 1, bodyBytesPerUnit: 1_000_000 },
|
||||
}),
|
||||
});
|
||||
const hold = await runtime.acquire({
|
||||
tenantKey: "hold",
|
||||
body: { stream: true },
|
||||
});
|
||||
assert.equal(hold.status, "admitted");
|
||||
|
||||
const deadlinePromise = runtime.acquire({
|
||||
tenantKey: "q1",
|
||||
body: { stream: true },
|
||||
maxWaitMs: 20,
|
||||
});
|
||||
clock.advance(20);
|
||||
const deadlineRejected = await deadlinePromise;
|
||||
assert.equal(deadlineRejected.status, "rejected");
|
||||
if (deadlineRejected.status === "rejected") {
|
||||
assert.equal(deadlineRejected.response.status, 503);
|
||||
assert.equal(deadlineRejected.code, "admission_deadline");
|
||||
const body = await parseJson(deadlineRejected.response);
|
||||
assert.equal(body.error.code, "admission_deadline");
|
||||
assert.equal(deadlineRejected.response.headers.get("Retry-After"), "1");
|
||||
}
|
||||
|
||||
// Fill the single queue slot then force queue_full on the next arrival.
|
||||
const waiterPromise = runtime.acquire({
|
||||
tenantKey: "waiter",
|
||||
body: { stream: true },
|
||||
maxWaitMs: 1_000,
|
||||
});
|
||||
const full = await runtime.acquire({
|
||||
tenantKey: "full",
|
||||
body: { stream: true },
|
||||
});
|
||||
assert.equal(full.status, "rejected");
|
||||
if (full.status === "rejected") {
|
||||
assert.equal(full.code, "admission_queue_full");
|
||||
assert.equal(full.response.status, 503);
|
||||
assert.equal(full.response.headers.get("Retry-After"), "1");
|
||||
const body = await parseJson(full.response);
|
||||
assert.equal(body.error.code, "admission_queue_full");
|
||||
}
|
||||
clock.advance(1_000);
|
||||
await waiterPromise;
|
||||
|
||||
// Oversized: cost features that exceed limit 1 with tiny max.
|
||||
const oversizedRuntime = makeRuntime(clock, {
|
||||
config: enforceConfig({
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
maxQueueCount: 1,
|
||||
maxQueueCost: 1,
|
||||
cost: {
|
||||
maxRequestCost: 100,
|
||||
baseCost: 1,
|
||||
bodyBytesPerUnit: 1,
|
||||
tokensPerUnit: 1,
|
||||
messagesPerUnit: 1,
|
||||
toolsPerUnit: 1,
|
||||
fanoutPerUnit: 1,
|
||||
},
|
||||
}),
|
||||
});
|
||||
const huge = await oversizedRuntime.acquire({
|
||||
tenantKey: "huge",
|
||||
body: {
|
||||
messages: Array.from({ length: 50 }, (_, i) => ({
|
||||
role: "user",
|
||||
content: `m${i}-${"x".repeat(32)}`,
|
||||
})),
|
||||
stream: true,
|
||||
},
|
||||
});
|
||||
assert.equal(huge.status, "rejected");
|
||||
if (huge.status === "rejected") {
|
||||
assert.equal(huge.code, "admission_oversized");
|
||||
assert.equal(huge.response.status, 503);
|
||||
const body = await parseJson(huge.response);
|
||||
assert.equal(body.error.code, "admission_oversized");
|
||||
assert.ok(!JSON.stringify(body).toLowerCase().includes("cost"));
|
||||
assert.ok(!JSON.stringify(body).includes("huge"));
|
||||
}
|
||||
if (hold.status === "admitted") hold.lease.release();
|
||||
runtime.dispose();
|
||||
oversizedRuntime.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("resource pressure integration", () => {
|
||||
let clock: FakeClock;
|
||||
beforeEach(() => {
|
||||
clock = new FakeClock();
|
||||
});
|
||||
|
||||
it("returns the existing critical guard response without acquiring work", async () => {
|
||||
let acquires = 0;
|
||||
const guard = criticalGuard();
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: enforceConfig({ initialLimit: 10 }),
|
||||
check: () => {
|
||||
acquires += 1;
|
||||
return guard;
|
||||
},
|
||||
});
|
||||
const result = await runtime.acquire({
|
||||
tenantKey: "t-pressure",
|
||||
body: { messages: [{ role: "user", content: "x" }] },
|
||||
});
|
||||
assert.equal(result.status, "rejected");
|
||||
if (result.status !== "rejected") throw new Error("expected rejected");
|
||||
assert.equal(result.response, guard.response);
|
||||
assert.equal(result.code, "resource_pressure");
|
||||
assert.equal(runtime.snapshot().pressureGuardRejectCount, 1);
|
||||
assert.equal(runtime.snapshot().activeCount, 0);
|
||||
assert.equal(acquires, 1);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("feeds fresh critical pressure observation even when the safety guard rejects", async () => {
|
||||
const guard = criticalGuard();
|
||||
let observation = emptyObservation({
|
||||
severity: "critical",
|
||||
reason: "v8_heap_absolute",
|
||||
observedAtMs: 1_000,
|
||||
});
|
||||
const pressures: string[] = [];
|
||||
const runtime = createAdaptiveAdmissionRuntime({
|
||||
config: enforceConfig({
|
||||
initialLimit: 20,
|
||||
minLimit: 4,
|
||||
maxLimit: 20,
|
||||
windowMs: 50,
|
||||
criticalDecreaseFactor: 0.5,
|
||||
}),
|
||||
clock: {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
},
|
||||
checkResourcePressure: () => guard,
|
||||
getResourcePressureObservation: () => observation,
|
||||
onPressureObserved: (pressure) => pressures.push(pressure),
|
||||
});
|
||||
|
||||
const first = await runtime.acquire({
|
||||
tenantKey: "guarded",
|
||||
body: { messages: [{ role: "user", content: "x" }] },
|
||||
});
|
||||
assert.equal(first.status, "rejected");
|
||||
if (first.status !== "rejected") throw new Error("expected rejected");
|
||||
// Exact same guard response identity; zero controller acquisition.
|
||||
assert.equal(first.response, guard.response);
|
||||
assert.equal(first.code, "resource_pressure");
|
||||
assert.equal(runtime.snapshot().activeCount, 0);
|
||||
assert.deepEqual(pressures, ["critical"]);
|
||||
// One critical reduction: floor(20 * 0.5) = 10.
|
||||
assert.equal(runtime.snapshot().currentLimit, 10);
|
||||
|
||||
// Replay same observation: no additional feed or reduction.
|
||||
const second = await runtime.acquire({
|
||||
tenantKey: "guarded-2",
|
||||
body: { messages: [{ role: "user", content: "y" }] },
|
||||
});
|
||||
assert.equal(second.response, guard.response);
|
||||
assert.deepEqual(pressures, ["critical"]);
|
||||
assert.equal(runtime.snapshot().currentLimit, 10);
|
||||
|
||||
// New window resets criticalDecreaseConsumed; fresh observation may reduce again.
|
||||
clock.advance(50);
|
||||
observation = emptyObservation({
|
||||
severity: "critical",
|
||||
reason: "v8_heap_absolute",
|
||||
observedAtMs: 2_000,
|
||||
});
|
||||
const third = await runtime.acquire({
|
||||
tenantKey: "guarded-3",
|
||||
body: { messages: [{ role: "user", content: "z" }] },
|
||||
});
|
||||
assert.equal(third.response, guard.response);
|
||||
assert.deepEqual(pressures, ["critical", "critical"]);
|
||||
assert.equal(runtime.snapshot().currentLimit, 5);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("dedupes unchanged observations and re-feeds genuinely fresh ones", async () => {
|
||||
let observation = emptyObservation({
|
||||
severity: "high",
|
||||
reason: "psi_some",
|
||||
observedAtMs: 100,
|
||||
});
|
||||
const pressures: string[] = [];
|
||||
const runtime = createAdaptiveAdmissionRuntime({
|
||||
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
|
||||
clock: {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
},
|
||||
checkResourcePressure: () => null,
|
||||
getResourcePressureObservation: () => observation,
|
||||
onPressureObserved: (pressure) => pressures.push(pressure),
|
||||
});
|
||||
|
||||
await runtime.acquire({ tenantKey: "a", body: {} });
|
||||
await runtime.acquire({ tenantKey: "b", body: {} });
|
||||
assert.deepEqual(pressures, ["high"]);
|
||||
|
||||
observation = emptyObservation({
|
||||
severity: "high",
|
||||
reason: "psi_some",
|
||||
observedAtMs: 100,
|
||||
});
|
||||
await runtime.acquire({ tenantKey: "c", body: {} });
|
||||
assert.deepEqual(pressures, ["high"]);
|
||||
|
||||
observation = emptyObservation({
|
||||
severity: "critical",
|
||||
reason: "psi_full",
|
||||
observedAtMs: 200,
|
||||
});
|
||||
await runtime.acquire({ tenantKey: "d", body: {} });
|
||||
assert.deepEqual(pressures, ["high", "critical"]);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("fails open when pressure check or observation throws", async () => {
|
||||
const runtime = makeRuntime(clock, {
|
||||
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
|
||||
check: () => {
|
||||
throw new Error("check boom");
|
||||
},
|
||||
observe: () => {
|
||||
throw new Error("observe boom");
|
||||
},
|
||||
});
|
||||
const result = await runtime.acquire({ tenantKey: "t", body: { messages: [] } });
|
||||
assert.equal(result.status, "admitted");
|
||||
if (result.status === "admitted") result.lease.release();
|
||||
runtime.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("public snapshot privacy", () => {
|
||||
it("exposes only aggregate counters and low-cardinality resource fields", async () => {
|
||||
const clock = new FakeClock();
|
||||
const runtime = makeRuntime(clock, {
|
||||
observe: () =>
|
||||
emptyObservation({
|
||||
severity: "high",
|
||||
reason: "cgroup_ratio",
|
||||
observedAtMs: 42,
|
||||
}),
|
||||
});
|
||||
await runtime.acquire({
|
||||
tenantKey: "tenant-very-secret",
|
||||
body: {
|
||||
messages: [{ role: "user", content: "SECRET_PAYLOAD_XYZ" }],
|
||||
api_key: "sk-live-secret",
|
||||
},
|
||||
});
|
||||
const snap = runtime.snapshot();
|
||||
const text = JSON.stringify(snap);
|
||||
assert.ok(!text.includes("tenant-very-secret"));
|
||||
assert.ok(!text.includes("SECRET_PAYLOAD_XYZ"));
|
||||
assert.ok(!text.includes("sk-live-secret"));
|
||||
assert.ok(!text.includes("lease-"));
|
||||
assert.equal(typeof snap.mode, "string");
|
||||
assert.equal(typeof snap.currentLimit, "number");
|
||||
assert.equal(typeof snap.activeCount, "number");
|
||||
assert.equal(snap.resourceSeverity, "high");
|
||||
assert.equal(snap.resourceReason, "cgroup_ratio");
|
||||
assert.equal(snap.resourceObservedAtMs, 42);
|
||||
assert.equal(typeof snap.pressureGuardRejectCount, "number");
|
||||
const snapRecord = snap as unknown as Record<string, unknown>;
|
||||
assert.equal(snapRecord.tenants, undefined);
|
||||
assert.equal(snapRecord.queue, undefined);
|
||||
assert.equal(snapRecord.features, undefined);
|
||||
runtime.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("process runtime reload isolation", () => {
|
||||
afterEach(() => {
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
});
|
||||
|
||||
it("reload disposes previous queued work/timers and replaces the process runtime", async () => {
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
const clock = new FakeClock();
|
||||
const first = reloadAdaptiveAdmissionRuntime({
|
||||
config: enforceConfig({
|
||||
initialLimit: 1,
|
||||
minLimit: 1,
|
||||
maxLimit: 1,
|
||||
maxQueueCount: 4,
|
||||
maxQueueCost: 40,
|
||||
defaultMaxWaitMs: 5_000,
|
||||
windowMs: 1_000,
|
||||
cost: { maxRequestCost: 1, baseCost: 1 },
|
||||
}),
|
||||
clock: {
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
},
|
||||
checkResourcePressure: () => null,
|
||||
getResourcePressureObservation: () => emptyObservation(),
|
||||
});
|
||||
|
||||
const hold = await first.acquire({
|
||||
tenantKey: "hold",
|
||||
body: { messages: [{ role: "user", content: "h" }], stream: true },
|
||||
});
|
||||
assert.equal(hold.status, "admitted");
|
||||
assert.ok(clock.pendingTimerCount >= 1);
|
||||
|
||||
const waiting = first.acquire({
|
||||
tenantKey: "waiter",
|
||||
body: { messages: [{ role: "user", content: "w" }], stream: true },
|
||||
maxWaitMs: 5_000,
|
||||
});
|
||||
|
||||
const second = reloadAdaptiveAdmissionRuntime({
|
||||
config: { ...DEFAULT_ADAPTIVE_ADMISSION_CONFIG, mode: "shadow" },
|
||||
checkResourcePressure: () => null,
|
||||
getResourcePressureObservation: () => emptyObservation(),
|
||||
});
|
||||
assert.notEqual(second, first);
|
||||
assert.equal(getAdaptiveAdmissionRuntime(), second);
|
||||
|
||||
const rejected = await waiting;
|
||||
assert.equal(rejected.status, "rejected");
|
||||
if (rejected.status === "rejected") {
|
||||
assert.equal(rejected.code, "admission_shutdown");
|
||||
}
|
||||
// Previous timers should be cleared by dispose/shutdown.
|
||||
assert.equal(clock.pendingTimerCount, 0);
|
||||
second.dispose();
|
||||
resetAdaptiveAdmissionRuntimeForTests();
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,12 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { estimateSizeFast, isSmallEnoughForSemanticCache } = await import(
|
||||
"../../open-sse/utils/estimateSize.ts"
|
||||
);
|
||||
const {
|
||||
estimateSizeFast,
|
||||
isSmallEnoughForSemanticCache,
|
||||
ESTIMATE_SIZE_BYTE_LIMIT,
|
||||
ESTIMATE_SIZE_NODE_BUDGET,
|
||||
} = await import("../../open-sse/utils/estimateSize.ts");
|
||||
|
||||
test("estimateSizeFast returns 0 for null/undefined", () => {
|
||||
assert.equal(estimateSizeFast(null), 0);
|
||||
@@ -65,6 +68,22 @@ test("estimateSizeFast early-exits at 262144 bytes (256KB)", () => {
|
||||
assert.ok(result >= 262144, `Should early-exit, got ${result}`);
|
||||
});
|
||||
|
||||
test("estimateSizeFast checks byte limit after numbers and booleans", () => {
|
||||
const almostForNumber = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 4);
|
||||
const withNumber = estimateSizeFast([almostForNumber, 1]);
|
||||
assert.ok(
|
||||
withNumber > ESTIMATE_SIZE_BYTE_LIMIT,
|
||||
`number contribution must trip byte limit, got ${withNumber}`
|
||||
);
|
||||
// boolean is 4 bytes: start 3 under the limit so adding true exceeds (not merely equals).
|
||||
const almostForBool = "x".repeat(ESTIMATE_SIZE_BYTE_LIMIT - 3);
|
||||
const withBool = estimateSizeFast([almostForBool, true]);
|
||||
assert.ok(
|
||||
withBool > ESTIMATE_SIZE_BYTE_LIMIT,
|
||||
`boolean contribution must trip byte limit, got ${withBool}`
|
||||
);
|
||||
});
|
||||
|
||||
test("estimateSizeFast handles mixed object/array nesting", () => {
|
||||
const data = {
|
||||
choices: [
|
||||
@@ -109,3 +128,70 @@ test("estimateSizeFast handles Map-like objects (no infinite loop on iterables)"
|
||||
const result = estimateSizeFast(map);
|
||||
assert.ok(typeof result === "number");
|
||||
});
|
||||
|
||||
/**
|
||||
* Mutation-sensitive bound: a huge logical length with null/empty-object elements
|
||||
* must not pre-touch every index or allocate all references. Node-budget exhaustion
|
||||
* fails closed above 256 KiB so semantic-cache/admission never treat it as small.
|
||||
*/
|
||||
test("estimateSizeFast node budget fails closed on huge sparse null array without full traversal", () => {
|
||||
let elementAccesses = 0;
|
||||
const sparseNulls = new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return 5_000_000;
|
||||
if (prop === Symbol.iterator) {
|
||||
throw new Error("iterator must not be used");
|
||||
}
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
elementAccesses += 1;
|
||||
return null;
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
const result = estimateSizeFast(sparseNulls);
|
||||
assert.ok(
|
||||
result > ESTIMATE_SIZE_BYTE_LIMIT,
|
||||
`node-budget exhaustion must return >256KiB, got ${result}`
|
||||
);
|
||||
assert.ok(
|
||||
elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8,
|
||||
`must not access far beyond node budget; accesses=${elementAccesses}`
|
||||
);
|
||||
assert.ok(elementAccesses > 100, `expected many bounded visits, got ${elementAccesses}`);
|
||||
assert.equal(isSmallEnoughForSemanticCache(sparseNulls), false);
|
||||
});
|
||||
|
||||
test("estimateSizeFast node budget fails closed on empty-object / getter proxy array", () => {
|
||||
let elementAccesses = 0;
|
||||
let farGetterHits = 0;
|
||||
const emptyObjectArray = new Proxy([] as unknown[], {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "length") return 2_000_000;
|
||||
if (typeof prop === "string" && /^[0-9]+$/.test(prop)) {
|
||||
const index = Number(prop);
|
||||
elementAccesses += 1;
|
||||
if (index >= ESTIMATE_SIZE_NODE_BUDGET) {
|
||||
farGetterHits += 1;
|
||||
}
|
||||
// Fresh empty object per access — old impl would stack-push every reference.
|
||||
return {};
|
||||
}
|
||||
return Reflect.get(target, prop, receiver);
|
||||
},
|
||||
});
|
||||
|
||||
const result = estimateSizeFast(emptyObjectArray);
|
||||
assert.ok(result > ESTIMATE_SIZE_BYTE_LIMIT, `expected fail-closed, got ${result}`);
|
||||
assert.ok(
|
||||
elementAccesses <= ESTIMATE_SIZE_NODE_BUDGET + 8,
|
||||
`accesses must stay near node budget; got ${elementAccesses}`
|
||||
);
|
||||
assert.equal(
|
||||
farGetterHits,
|
||||
0,
|
||||
`entries beyond the node budget must not be touched; far hits=${farGetterHits}`
|
||||
);
|
||||
assert.equal(isSmallEnoughForSemanticCache(emptyObjectArray), false);
|
||||
});
|
||||
|
||||
202
tests/unit/resource-pressure-policy.test.ts
Normal file
202
tests/unit/resource-pressure-policy.test.ts
Normal file
@@ -0,0 +1,202 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
createResourcePressureTracker,
|
||||
resolveResourcePressureThresholds,
|
||||
type PressureReason,
|
||||
type PressureSeverity,
|
||||
type ResourcePressureState,
|
||||
type ResourcePressureThresholds,
|
||||
type ResourceSignals,
|
||||
} from "../../open-sse/utils/resourcePressurePolicy.ts";
|
||||
|
||||
const MiB = 1024 ** 2;
|
||||
|
||||
function baseSignals(overrides: Partial<ResourceSignals> = {}): ResourceSignals {
|
||||
return {
|
||||
observedAtMs: 1_000,
|
||||
v8: { heapUsedBytes: 100 * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
process: {
|
||||
rssBytes: 200 * MiB,
|
||||
externalBytes: 10 * MiB,
|
||||
arrayBuffersBytes: MiB,
|
||||
availableBytes: null,
|
||||
constrainedBytes: null,
|
||||
},
|
||||
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
|
||||
psi: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const fastThresholds: Partial<ResourcePressureThresholds> = {
|
||||
highRatio: 0.8,
|
||||
criticalRatio: 0.9,
|
||||
recoveryRatio: 0.7,
|
||||
highPsiAvg10: 20,
|
||||
criticalPsiAvg10: 40,
|
||||
recoveryPsiAvg10: 10,
|
||||
sustainedSamplesHigh: 2,
|
||||
sustainedSamplesCritical: 2,
|
||||
sustainedSamplesRecovery: 2,
|
||||
heapAbsoluteThresholdMb: null,
|
||||
};
|
||||
|
||||
describe("resource pressure threshold validation", () => {
|
||||
it("accepts every valid boundary", () => {
|
||||
const thresholds = resolveResourcePressureThresholds({
|
||||
recoveryRatio: 0,
|
||||
highRatio: 0.5,
|
||||
criticalRatio: 1,
|
||||
recoveryPsiAvg10: 0,
|
||||
highPsiAvg10: 50,
|
||||
criticalPsiAvg10: 100,
|
||||
sustainedSamplesHigh: 1,
|
||||
sustainedSamplesCritical: 1,
|
||||
sustainedSamplesRecovery: 10_000,
|
||||
heapAbsoluteThresholdMb: null,
|
||||
});
|
||||
assert.equal(thresholds.recoveryRatio, 0);
|
||||
assert.equal(thresholds.criticalRatio, 1);
|
||||
assert.equal(thresholds.criticalPsiAvg10, 100);
|
||||
assert.equal(thresholds.sustainedSamplesRecovery, 10_000);
|
||||
assert.equal(thresholds.heapAbsoluteThresholdMb, null);
|
||||
});
|
||||
|
||||
it("throws deterministically for invalid partial overrides", () => {
|
||||
const invalid: Array<Partial<ResourcePressureThresholds>> = [
|
||||
{ recoveryRatio: -0.01 },
|
||||
{ criticalRatio: 1.01 },
|
||||
{ highRatio: Number.NaN },
|
||||
{ recoveryRatio: 0.8, highRatio: 0.8 },
|
||||
{ highRatio: 0.95, criticalRatio: 0.9 },
|
||||
{ recoveryPsiAvg10: -1 },
|
||||
{ criticalPsiAvg10: 101 },
|
||||
{ highPsiAvg10: Number.POSITIVE_INFINITY },
|
||||
{ recoveryPsiAvg10: 20, highPsiAvg10: 20 },
|
||||
{ highPsiAvg10: 50, criticalPsiAvg10: 40 },
|
||||
{ sustainedSamplesHigh: 0 },
|
||||
{ sustainedSamplesCritical: 1.5 },
|
||||
{ sustainedSamplesRecovery: 10_001 },
|
||||
{ heapAbsoluteThresholdMb: 0 },
|
||||
{ heapAbsoluteThresholdMb: Number.POSITIVE_INFINITY },
|
||||
];
|
||||
for (const partial of invalid) {
|
||||
assert.throws(() => resolveResourcePressureThresholds(partial), RangeError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("resource pressure policy", () => {
|
||||
it("does not let high then critical count as two critical samples", () => {
|
||||
const tracker = createResourcePressureTracker(fastThresholds);
|
||||
const high = baseSignals({
|
||||
v8: { heapUsedBytes: 850 * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
});
|
||||
const critical = baseSignals({
|
||||
v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
});
|
||||
|
||||
assert.equal(tracker.observe(high).severity, "normal");
|
||||
assert.equal(tracker.observe(critical).severity, "normal");
|
||||
assert.equal(tracker.observe(critical).severity, "critical");
|
||||
});
|
||||
|
||||
it("resets pending streak when severity or reason alternates", () => {
|
||||
const tracker = createResourcePressureTracker(fastThresholds);
|
||||
const heapCritical = baseSignals({
|
||||
v8: { heapUsedBytes: 950 * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
});
|
||||
const psiCritical = baseSignals({
|
||||
psi: {
|
||||
someAvg10: 50,
|
||||
someAvg60: null,
|
||||
someAvg300: null,
|
||||
fullAvg10: null,
|
||||
fullAvg60: null,
|
||||
fullAvg300: null,
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(tracker.observe(heapCritical).severity, "normal");
|
||||
assert.equal(tracker.observe(psiCritical).severity, "normal");
|
||||
assert.equal(tracker.observe(psiCritical).severity, "critical");
|
||||
assert.equal(tracker.getState().reason, "psi_some");
|
||||
});
|
||||
|
||||
it("baselines cumulative OOM counters and only treats increases as events", () => {
|
||||
const tracker = createResourcePressureTracker(fastThresholds);
|
||||
const oomCounters = (oom: number, oom_kill: number, observedAtMs: number) =>
|
||||
baseSignals({
|
||||
observedAtMs,
|
||||
cgroup: {
|
||||
currentBytes: null,
|
||||
maxBytes: null,
|
||||
highBytes: null,
|
||||
events: { low: 0, high: 0, max: 0, oom, oom_kill },
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(tracker.observe(oomCounters(7, 3, 1)).severity, "normal", "history baselines");
|
||||
assert.equal(tracker.observe(oomCounters(7, 3, 2)).severity, "normal", "unchanged history");
|
||||
|
||||
const event = tracker.observe(oomCounters(8, 3, 3));
|
||||
assert.equal(event.severity, "critical", "a new OOM event is immediately critical");
|
||||
assert.equal(event.reason, "oom_event");
|
||||
|
||||
assert.equal(tracker.observe(oomCounters(8, 3, 4)).severity, "critical");
|
||||
assert.equal(
|
||||
tracker.observe(oomCounters(8, 3, 5)).severity,
|
||||
"normal",
|
||||
"unchanged allows recovery"
|
||||
);
|
||||
});
|
||||
|
||||
it("re-baselines when OOM counters reset or the cgroup event source is replaced", () => {
|
||||
const tracker = createResourcePressureTracker(fastThresholds);
|
||||
const events = (oom: number, oom_kill: number) =>
|
||||
baseSignals({
|
||||
cgroup: {
|
||||
currentBytes: null,
|
||||
maxBytes: null,
|
||||
highBytes: null,
|
||||
events: { low: 0, high: 0, max: 0, oom, oom_kill },
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(tracker.observe(events(10, 4)).severity, "normal");
|
||||
assert.equal(tracker.observe(events(1, 0)).severity, "normal", "counter reset re-baselines");
|
||||
assert.equal(
|
||||
tracker.observe({ ...events(1, 0), cgroup: { ...events(1, 0).cgroup, events: null } })
|
||||
.severity,
|
||||
"normal"
|
||||
);
|
||||
assert.equal(tracker.observe(events(9, 3)).severity, "normal", "replacement re-baselines");
|
||||
});
|
||||
|
||||
it("keeps snapshot state fields and bounded-cardinality values", () => {
|
||||
const tracker = createResourcePressureTracker(fastThresholds);
|
||||
const state: ResourcePressureState = tracker.observe(baseSignals());
|
||||
const severities = new Set<PressureSeverity>(["normal", "high", "critical"]);
|
||||
const reasons = new Set<PressureReason>([
|
||||
"none",
|
||||
"v8_heap_ratio",
|
||||
"v8_heap_absolute",
|
||||
"cgroup_ratio",
|
||||
"cgroup_high",
|
||||
"psi_some",
|
||||
"psi_full",
|
||||
"oom_event",
|
||||
]);
|
||||
assert.ok(severities.has(state.severity));
|
||||
assert.ok(reasons.has(state.reason));
|
||||
assert.deepEqual(Object.keys(state).sort(), [
|
||||
"elevatedStreak",
|
||||
"lastTransitionAtMs",
|
||||
"observedAtMs",
|
||||
"reason",
|
||||
"recoveryStreak",
|
||||
"severity",
|
||||
]);
|
||||
});
|
||||
});
|
||||
330
tests/unit/resource-pressure-runtime.test.ts
Normal file
330
tests/unit/resource-pressure-runtime.test.ts
Normal file
@@ -0,0 +1,330 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
createResourcePressureRuntime,
|
||||
type ResourcePressureRuntime,
|
||||
} from "../../open-sse/utils/resourcePressure.ts";
|
||||
import type { ResourceSignals } from "../../open-sse/utils/resourcePressurePolicy.ts";
|
||||
|
||||
const MiB = 1024 ** 2;
|
||||
|
||||
function signals(observedAtMs: number, heapUsedMb = 100): ResourceSignals {
|
||||
return {
|
||||
observedAtMs,
|
||||
v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
process: {
|
||||
rssBytes: 200 * MiB,
|
||||
externalBytes: 10 * MiB,
|
||||
arrayBuffersBytes: MiB,
|
||||
availableBytes: null,
|
||||
constrainedBytes: null,
|
||||
},
|
||||
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
|
||||
psi: null,
|
||||
};
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
async function settleRefresh(runtime: ResourcePressureRuntime): Promise<void> {
|
||||
await runtime.whenRefreshSettled();
|
||||
await Promise.resolve();
|
||||
}
|
||||
|
||||
describe("ResourcePressureRuntime stale-while-revalidate cache", () => {
|
||||
it("does no proc/sys I/O in check(), while a cheap first-request heap breach sheds immediately", async () => {
|
||||
let slowSamples = 0;
|
||||
const runtime = createResourcePressureRuntime({
|
||||
heapThresholdMb: 200,
|
||||
immediateHeapUsedMb: () => 201,
|
||||
sample: async () => {
|
||||
slowSamples += 1;
|
||||
return signals(1);
|
||||
},
|
||||
});
|
||||
|
||||
const guard = runtime.check();
|
||||
assert.ok(guard);
|
||||
assert.equal(guard.status, 503);
|
||||
assert.equal(slowSamples, 0, "request-path check must not invoke the async proc/sys sampler");
|
||||
assert.equal(runtime.getObservation().state.reason, "v8_heap_absolute");
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(slowSamples, 1, "refresh may run after the request-path decision");
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("serves a fresh cached sample without scheduling another refresh", async () => {
|
||||
let now = 0;
|
||||
let calls = 0;
|
||||
const runtime = createResourcePressureRuntime({
|
||||
nowMs: () => now,
|
||||
staleAfterMs: 100,
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
calls += 1;
|
||||
return signals(now);
|
||||
},
|
||||
});
|
||||
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 1);
|
||||
now = 99;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 1);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("schedules at most one refresh under concurrent stale checks", async () => {
|
||||
let now = 0;
|
||||
let calls = 0;
|
||||
const pending = deferred<ResourceSignals>();
|
||||
const runtime = createResourcePressureRuntime({
|
||||
nowMs: () => now,
|
||||
staleAfterMs: 10,
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) return signals(0);
|
||||
return pending.promise;
|
||||
},
|
||||
});
|
||||
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
now = 11;
|
||||
for (let index = 0; index < 50; index += 1) runtime.check();
|
||||
assert.equal(calls, 1, "scheduled work must not run synchronously in check()");
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(calls, 2);
|
||||
pending.resolve(signals(11));
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("retains a bounded stale snapshot on refresh failure and retries only after backoff", async () => {
|
||||
let now = 0;
|
||||
let calls = 0;
|
||||
const runtime = createResourcePressureRuntime({
|
||||
nowMs: () => now,
|
||||
staleAfterMs: 10,
|
||||
maxStaleMs: 100,
|
||||
retryAfterMs: 20,
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) return signals(0, 950);
|
||||
throw new Error("proc unavailable");
|
||||
},
|
||||
thresholds: {
|
||||
sustainedSamplesCritical: 1,
|
||||
heapAbsoluteThresholdMb: null,
|
||||
},
|
||||
});
|
||||
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
now = 11;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(runtime.getObservation().signals?.observedAtMs, 0, "failure retains stale data");
|
||||
|
||||
now = 25;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2, "failure backoff prevents a refresh storm");
|
||||
|
||||
now = 31;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 3);
|
||||
|
||||
now = 101;
|
||||
assert.equal(runtime.check(), null, "expired stale adaptive pressure fails open");
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("measures failure backoff from settlement, not refresh start", async () => {
|
||||
let now = 0;
|
||||
let calls = 0;
|
||||
const pending = deferred<ResourceSignals>();
|
||||
const runtime = createResourcePressureRuntime({
|
||||
nowMs: () => now,
|
||||
staleAfterMs: 10,
|
||||
maxStaleMs: 100,
|
||||
retryAfterMs: 20,
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) return signals(0);
|
||||
return pending.promise;
|
||||
},
|
||||
});
|
||||
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 1);
|
||||
|
||||
now = 11;
|
||||
runtime.check();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(calls, 2);
|
||||
|
||||
// Slow failure: wall clock advances past retryAfter before the sample rejects.
|
||||
now = 50;
|
||||
pending.reject(new Error("proc unavailable"));
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2);
|
||||
|
||||
// Retry must wait full retryAfterMs from settlement (50), not from start (11).
|
||||
now = 69;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2, "failure backoff starts at settlement, not refresh start");
|
||||
|
||||
now = 70;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 3);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("measures success freshness from publication, not refresh start", async () => {
|
||||
let now = 0;
|
||||
let calls = 0;
|
||||
const pending = deferred<ResourceSignals>();
|
||||
const runtime = createResourcePressureRuntime({
|
||||
nowMs: () => now,
|
||||
staleAfterMs: 20,
|
||||
maxStaleMs: 100,
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) return signals(0);
|
||||
return pending.promise;
|
||||
},
|
||||
});
|
||||
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 1);
|
||||
|
||||
now = 21;
|
||||
runtime.check();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(calls, 2);
|
||||
|
||||
// Slow success: wall clock advances past staleAfter before the sample resolves.
|
||||
now = 100;
|
||||
pending.resolve(signals(100));
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(runtime.getObservation().signals?.observedAtMs, 100);
|
||||
|
||||
// Freshness must run full staleAfterMs from publication (100), not start (21).
|
||||
now = 119;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 2, "success freshness starts at publication, not refresh start");
|
||||
|
||||
now = 120;
|
||||
runtime.check();
|
||||
await settleRefresh(runtime);
|
||||
assert.equal(calls, 3);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("default scheduler unrefs Immediate; injected schedulers stay caller-owned", async () => {
|
||||
// Injected schedule is never wrapped: the runtime must not call unref on it.
|
||||
let scheduled = 0;
|
||||
let unrefCalled = 0;
|
||||
const injected = (refresh: () => void) => {
|
||||
scheduled += 1;
|
||||
const handle = setImmediate(refresh);
|
||||
const originalUnref = handle.unref.bind(handle);
|
||||
handle.unref = () => {
|
||||
unrefCalled += 1;
|
||||
return originalUnref();
|
||||
};
|
||||
};
|
||||
|
||||
const withInjected = createResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => signals(1),
|
||||
schedule: injected,
|
||||
});
|
||||
withInjected.check();
|
||||
await settleRefresh(withInjected);
|
||||
assert.equal(scheduled, 1);
|
||||
assert.equal(unrefCalled, 0, "injected schedule handles remain caller-owned");
|
||||
withInjected.dispose();
|
||||
|
||||
// Default schedule path: capture the Immediate and prove it is unref'd so a
|
||||
// pending refresh alone cannot keep the process alive.
|
||||
const originalSetImmediate = globalThis.setImmediate;
|
||||
let captured: NodeJS.Immediate | undefined;
|
||||
globalThis.setImmediate = ((callback: (...args: unknown[]) => void, ...args: unknown[]) => {
|
||||
const handle = originalSetImmediate(callback, ...args);
|
||||
captured = handle;
|
||||
return handle;
|
||||
}) as typeof setImmediate;
|
||||
try {
|
||||
const runtime = createResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
// Never resolve: we only care about the scheduled Immediate ref state.
|
||||
sample: () => new Promise(() => {}),
|
||||
});
|
||||
runtime.check();
|
||||
assert.ok(captured, "default schedule must use setImmediate");
|
||||
assert.equal(captured.hasRef(), false, "default Immediate must be unref'd");
|
||||
runtime.dispose();
|
||||
if (captured) clearImmediate(captured);
|
||||
} finally {
|
||||
globalThis.setImmediate = originalSetImmediate;
|
||||
}
|
||||
});
|
||||
|
||||
it("dispose ignores late refresh results and independently owned runtimes do not share state", async () => {
|
||||
const pending = deferred<ResourceSignals>();
|
||||
let firstCalls = 0;
|
||||
const first = createResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
firstCalls += 1;
|
||||
return pending.promise;
|
||||
},
|
||||
});
|
||||
first.check();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(firstCalls, 1);
|
||||
first.dispose();
|
||||
pending.resolve(signals(1));
|
||||
await settleRefresh(first);
|
||||
assert.equal(
|
||||
first.getObservation().signals,
|
||||
null,
|
||||
"disposed runtime ignores late refresh results"
|
||||
);
|
||||
|
||||
const second = createResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => signals(2),
|
||||
});
|
||||
assert.notEqual(first, second);
|
||||
second.check();
|
||||
await settleRefresh(second);
|
||||
assert.equal(second.getObservation().signals?.observedAtMs, 2);
|
||||
second.dispose();
|
||||
});
|
||||
});
|
||||
178
tests/unit/resource-pressure-sampler.test.ts
Normal file
178
tests/unit/resource-pressure-sampler.test.ts
Normal file
@@ -0,0 +1,178 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
decodeMountInfoPath,
|
||||
parseCgroup2Mount,
|
||||
parseCgroupV2Path,
|
||||
resolveCgroupDirectory,
|
||||
sampleResourceSignals,
|
||||
sanitizeMemoryBytes,
|
||||
type ResourcePressureFs,
|
||||
} from "../../open-sse/utils/resourcePressureSampler.ts";
|
||||
|
||||
const MiB = 1024 ** 2;
|
||||
const GiB = 1024 ** 3;
|
||||
|
||||
function memoryUsage(heapUsed = 1): NodeJS.MemoryUsage {
|
||||
return {
|
||||
rss: 500 * MiB,
|
||||
heapTotal: 300 * MiB,
|
||||
heapUsed,
|
||||
external: 12 * MiB,
|
||||
arrayBuffers: 3 * MiB,
|
||||
};
|
||||
}
|
||||
|
||||
function mapFs(entries: ReadonlyArray<readonly [string, string]>): ResourcePressureFs {
|
||||
const files = new Map(entries);
|
||||
return { readText: async (filePath) => files.get(filePath) ?? null };
|
||||
}
|
||||
|
||||
describe("resource pressure cgroup parsers", () => {
|
||||
it("accepts only the exact unified cgroup entry", () => {
|
||||
assert.equal(parseCgroupV2Path("2:cpu:/wrong\n0::/delegated/service\n"), "/delegated/service");
|
||||
assert.equal(parseCgroupV2Path("0:cpu:/wrong\n1::/also-wrong\n"), null);
|
||||
});
|
||||
|
||||
it("decodes mountinfo octal escapes in root and mountpoint", () => {
|
||||
assert.equal(
|
||||
decodeMountInfoPath("/sys/fs/cgroup\\040space\\134unit"),
|
||||
"/sys/fs/cgroup space\\unit"
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseCgroup2Mount(
|
||||
"43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n"
|
||||
),
|
||||
{ root: "/delegated root", mountpoint: "/sys/fs/cgroup space" }
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves delegated mount roots within the decoded mountpoint", async () => {
|
||||
const fs = mapFs([
|
||||
["/proc/self/cgroup", "0::/delegated root/team/service\n"],
|
||||
[
|
||||
"/proc/self/mountinfo",
|
||||
"43 34 0:35 /delegated\\040root /sys/fs/cgroup\\040space rw - cgroup2 cgroup2 rw\n",
|
||||
],
|
||||
["/sys/fs/cgroup space/team/service/memory.current", "1\n"],
|
||||
]);
|
||||
|
||||
assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup space/team/service");
|
||||
});
|
||||
|
||||
it("rejects NUL, traversal, malformed, and out-of-root cgroup paths", async () => {
|
||||
for (const cgroupPath of [
|
||||
"/delegated/../escape",
|
||||
"/delegated/%2e%2e/escape",
|
||||
"/delegated/service\0escape",
|
||||
"delegated/service",
|
||||
"/other/service",
|
||||
]) {
|
||||
const fs = mapFs([
|
||||
["/proc/self/cgroup", `0::${cgroupPath}\n`],
|
||||
["/proc/self/mountinfo", "43 34 0:35 /delegated /safe/cgroup rw - cgroup2 cgroup2 rw\n"],
|
||||
["/safe/cgroup/memory.current", "1\n"],
|
||||
]);
|
||||
assert.equal(
|
||||
await resolveCgroupDirectory(fs.readText, { allowDefaultFallback: false }),
|
||||
null,
|
||||
cgroupPath
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to the validated default cgroup root when proc metadata is malformed", async () => {
|
||||
const fs = mapFs([
|
||||
["/proc/self/cgroup", "malformed\n"],
|
||||
["/proc/self/mountinfo", "malformed\n"],
|
||||
["/sys/fs/cgroup/memory.current", "123\n"],
|
||||
]);
|
||||
assert.equal(await resolveCgroupDirectory(fs.readText), "/sys/fs/cgroup");
|
||||
});
|
||||
});
|
||||
|
||||
describe("sampleResourceSignals", () => {
|
||||
it("captures process, V8, cgroup, event, and PSI snapshot fields", async () => {
|
||||
const fs = mapFs([
|
||||
["/proc/self/cgroup", "0::/slice/service\n"],
|
||||
["/proc/self/mountinfo", "43 34 0:35 / /sys/fs/cgroup rw - cgroup2 cgroup2 rw\n"],
|
||||
["/sys/fs/cgroup/slice/service/memory.current", `${800 * MiB}\n`],
|
||||
["/sys/fs/cgroup/slice/service/memory.max", `${GiB}\n`],
|
||||
["/sys/fs/cgroup/slice/service/memory.high", "966367641\n"],
|
||||
["/sys/fs/cgroup/slice/service/memory.events", "low 1\nhigh 2\nmax 3\noom 4\noom_kill 5\n"],
|
||||
[
|
||||
"/proc/pressure/memory",
|
||||
"some avg10=1.50 avg60=2.00 avg300=3.25 total=9\nfull avg10=0.25 avg60=0.50 avg300=0.75 total=1\n",
|
||||
],
|
||||
]);
|
||||
|
||||
const signals = await sampleResourceSignals({
|
||||
nowMs: () => 42,
|
||||
memoryUsage: () => memoryUsage(250 * MiB),
|
||||
heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 250 * MiB }),
|
||||
availableMemory: () => 4 * GiB,
|
||||
constrainedMemory: () => undefined,
|
||||
fs,
|
||||
});
|
||||
|
||||
assert.equal(signals.observedAtMs, 42);
|
||||
assert.deepEqual(signals.v8, { heapUsedBytes: 250 * MiB, heapLimitBytes: GiB });
|
||||
assert.deepEqual(signals.process, {
|
||||
rssBytes: 500 * MiB,
|
||||
externalBytes: 12 * MiB,
|
||||
arrayBuffersBytes: 3 * MiB,
|
||||
availableBytes: 4 * GiB,
|
||||
constrainedBytes: null,
|
||||
});
|
||||
assert.deepEqual(signals.cgroup, {
|
||||
currentBytes: 800 * MiB,
|
||||
maxBytes: GiB,
|
||||
highBytes: 966367641,
|
||||
events: { low: 1, high: 2, max: 3, oom: 4, oom_kill: 5 },
|
||||
});
|
||||
assert.equal(signals.psi?.someAvg10, 1.5);
|
||||
assert.equal(signals.psi?.fullAvg10, 0.25);
|
||||
});
|
||||
|
||||
it("fails open when platform reads fail or return malformed values", async () => {
|
||||
const signals = await sampleResourceSignals({
|
||||
memoryUsage: () => memoryUsage(),
|
||||
heapStatistics: () => ({ heap_size_limit: GiB, used_heap_size: 1 }),
|
||||
availableMemory: () => {
|
||||
throw new Error("unavailable");
|
||||
},
|
||||
constrainedMemory: () => Number.POSITIVE_INFINITY,
|
||||
fs: {
|
||||
readText: async (filePath) => {
|
||||
if (filePath === "/proc/self/cgroup") throw new Error("unavailable");
|
||||
return "malformed";
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(signals.process.availableBytes, null);
|
||||
assert.equal(signals.process.constrainedBytes, null);
|
||||
assert.deepEqual(signals.cgroup, {
|
||||
currentBytes: null,
|
||||
maxBytes: null,
|
||||
highBytes: null,
|
||||
events: null,
|
||||
});
|
||||
assert.equal(signals.psi, null);
|
||||
});
|
||||
|
||||
it("treats missing, zero, max, and unsafe memory quantities as unavailable", () => {
|
||||
for (const value of [
|
||||
undefined,
|
||||
"",
|
||||
"max",
|
||||
0,
|
||||
-1,
|
||||
Number.NaN,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
2 ** 63,
|
||||
]) {
|
||||
assert.equal(sanitizeMemoryBytes(value), null, String(value));
|
||||
}
|
||||
assert.equal(sanitizeMemoryBytes("123"), 123);
|
||||
});
|
||||
});
|
||||
134
tests/unit/resource-pressure.test.ts
Normal file
134
tests/unit/resource-pressure.test.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
createResourcePressureRuntime,
|
||||
getResourcePressureObservation,
|
||||
reloadResourcePressureRuntime,
|
||||
type ResourceSignals,
|
||||
} from "../../open-sse/utils/resourcePressure.ts";
|
||||
|
||||
const MiB = 1024 ** 2;
|
||||
|
||||
function signals(observedAtMs: number, heapUsedMb: number): ResourceSignals {
|
||||
return {
|
||||
observedAtMs,
|
||||
v8: { heapUsedBytes: heapUsedMb * MiB, heapLimitBytes: 1_000 * MiB },
|
||||
process: {
|
||||
rssBytes: 200 * MiB,
|
||||
externalBytes: 10 * MiB,
|
||||
arrayBuffersBytes: MiB,
|
||||
availableBytes: null,
|
||||
constrainedBytes: null,
|
||||
},
|
||||
cgroup: { currentBytes: null, maxBytes: null, highBytes: null, events: null },
|
||||
psi: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("resource pressure HTTP guard facade", () => {
|
||||
it("preserves strict immediate first-request heap shedding", async () => {
|
||||
let samples = 0;
|
||||
const runtime = createResourcePressureRuntime({
|
||||
heapThresholdMb: 200,
|
||||
immediateHeapUsedMb: () => 201,
|
||||
sample: async () => {
|
||||
samples += 1;
|
||||
return signals(1, 100);
|
||||
},
|
||||
});
|
||||
|
||||
const guard = runtime.check();
|
||||
assert.ok(guard);
|
||||
assert.equal(guard.status, 503);
|
||||
assert.equal(samples, 0, "the asynchronous sampler cannot run in the request path");
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("does not shed when heap usage equals the strict threshold", () => {
|
||||
const runtime = createResourcePressureRuntime({
|
||||
heapThresholdMb: 200,
|
||||
immediateHeapUsedMb: () => 200,
|
||||
sample: async () => signals(1, 100),
|
||||
});
|
||||
assert.equal(runtime.check(), null);
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("returns a sanitized standards-correct 503 with Retry-After", async () => {
|
||||
const runtime = createResourcePressureRuntime({
|
||||
heapThresholdMb: 200,
|
||||
immediateHeapUsedMb: () => 987,
|
||||
sample: async () => signals(1, 100),
|
||||
});
|
||||
|
||||
const guard = runtime.check();
|
||||
assert.ok(guard);
|
||||
assert.equal(guard.success, false);
|
||||
assert.equal(guard.status, 503);
|
||||
assert.equal(guard.response.status, 503);
|
||||
assert.equal(guard.response.headers.get("Retry-After"), "5");
|
||||
assert.equal(guard.response.headers.get("Content-Type"), "application/json");
|
||||
const payload = await guard.response.json();
|
||||
assert.deepEqual(payload.error, {
|
||||
message: "Service temporarily unavailable due to resource pressure. Retry shortly.",
|
||||
type: "server_error",
|
||||
code: "resource_pressure",
|
||||
});
|
||||
const clientText = JSON.stringify(payload) + guard.error;
|
||||
assert.ok(!clientText.includes("987"));
|
||||
assert.ok(!/\bMB\b/.test(clientText));
|
||||
runtime.dispose();
|
||||
});
|
||||
|
||||
it("reload atomically replaces and resets the thin default facade", async () => {
|
||||
let firstCalls = 0;
|
||||
reloadResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => {
|
||||
firstCalls += 1;
|
||||
return signals(1, 100);
|
||||
},
|
||||
});
|
||||
const replacement = reloadResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => signals(2, 100),
|
||||
});
|
||||
|
||||
assert.deepEqual(getResourcePressureObservation(), {
|
||||
signals: null,
|
||||
state: {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
},
|
||||
});
|
||||
assert.equal(firstCalls, 0, "replaced runtime must not retain or run scheduled work");
|
||||
replacement.dispose();
|
||||
});
|
||||
|
||||
it("exposes all observation snapshot fields", async () => {
|
||||
const runtime = createResourcePressureRuntime({
|
||||
immediateHeapUsedMb: () => 100,
|
||||
sample: async () => signals(42, 100),
|
||||
});
|
||||
assert.deepEqual(runtime.getObservation(), {
|
||||
signals: null,
|
||||
state: {
|
||||
severity: "normal",
|
||||
reason: "none",
|
||||
elevatedStreak: 0,
|
||||
recoveryStreak: 0,
|
||||
lastTransitionAtMs: 0,
|
||||
observedAtMs: 0,
|
||||
},
|
||||
});
|
||||
runtime.check();
|
||||
await runtime.whenRefreshSettled();
|
||||
assert.equal(runtime.getObservation().signals?.observedAtMs, 42);
|
||||
assert.equal(runtime.getObservation().state.observedAtMs, 42);
|
||||
runtime.dispose();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user