mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-16 20:02:45 +03:00
fix(admission): measure request bodies with the active cost budget (#13762)
Merged. Good catch: the feature extractor was silently capped by the size estimator's 256 KiB default, so every request above it reported the same flattened body/token estimate and admission decided on a number that was not the request. Deriving the measurement limit from the active cost budget — runtime overrides included — is the right place to fix it. Validated as a combined board first (this PR merged with the 21 siblings of the same wave on the release tip): eslint with the frozen suppressions, typecheck:core, check:open-sse-typecheck, complexity, cognitive-complexity, changelog-integrity, i18n new-key coverage, docs-counts, docs-sync, migration-numbering, provider-consistency and a duplicate-identifier audit all green, plus 176 passing / 0 failing focused node:test cases across the 25 test files the wave touches and the dashboard test under Vitest (2/0). Then re-validated alone on the fresh tip before this merge: conflicts re-resolved, file sizes rebaselined for this PR's own growth, eslint and this PR's focused tests re-run. Thank you.
This commit is contained in:
@@ -4,11 +4,13 @@
|
||||
*/
|
||||
|
||||
import { estimateSizeFast } from "../../utils/estimateSize.ts";
|
||||
import type { AdmissionCostFeatures } from "./types.ts";
|
||||
import { resolveCostConfig } from "./cost.ts";
|
||||
import type { AdmissionCostConfig, AdmissionCostFeatures } from "./types.ts";
|
||||
|
||||
export type AdmissionFeatureExtractionContext = {
|
||||
/** When set, wins over any body/wrapped stream field. */
|
||||
streaming?: boolean;
|
||||
cost?: Partial<AdmissionCostConfig>;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -159,7 +161,8 @@ export function extractAdmissionCostFeatures(
|
||||
body: unknown,
|
||||
context?: AdmissionFeatureExtractionContext
|
||||
): AdmissionCostFeatures {
|
||||
const bodyBytes = estimateSizeFast(body);
|
||||
const cost = resolveCostConfig(context?.cost);
|
||||
const bodyBytes = estimateSizeFast(body, cost.bodyBytesPerUnit * cost.maxRequestCost);
|
||||
const layers = featureLayers(body);
|
||||
const draft: FeatureDraft = {
|
||||
messageCount: 0,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
import { AdaptiveAdmissionController } from "./controller.ts";
|
||||
import { validateConfig } from "./config.ts";
|
||||
import { resolveCostConfig } from "./cost.ts";
|
||||
import { extractAdmissionCostFeatures } from "./requestFeatures.ts";
|
||||
import {
|
||||
type AdaptiveAdmissionConfig,
|
||||
@@ -323,6 +324,7 @@ function classifyHttpOutcome(status: number, signal?: AbortSignal): AdmissionRel
|
||||
|
||||
class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime {
|
||||
private readonly controller: AdaptiveAdmissionController;
|
||||
private readonly costConfig: ReturnType<typeof resolveCostConfig>;
|
||||
private readonly checkResourcePressure: () => ResourcePressureGuardResult | null;
|
||||
private readonly getResourcePressureObservation: () => ResourcePressureObservation;
|
||||
private readonly onPressureObserved?: (pressure: AdmissionPressure) => void;
|
||||
@@ -338,6 +340,7 @@ class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime {
|
||||
|
||||
constructor(options: AdaptiveAdmissionRuntimeOptions, config: AdaptiveAdmissionConfig) {
|
||||
this.controller = new AdaptiveAdmissionController(config, options.clock);
|
||||
this.costConfig = resolveCostConfig(config.cost);
|
||||
this.checkResourcePressure = options.checkResourcePressure ?? checkResourcePressureGuard;
|
||||
this.getResourcePressureObservation =
|
||||
options.getResourcePressureObservation ?? getResourcePressureObservation;
|
||||
@@ -372,7 +375,10 @@ class AdaptiveAdmissionRuntimeImpl implements AdaptiveAdmissionRuntime {
|
||||
|
||||
const features = extractAdmissionCostFeatures(
|
||||
input.body,
|
||||
input.streaming === undefined ? undefined : { streaming: input.streaming }
|
||||
{
|
||||
streaming: input.streaming,
|
||||
cost: this.costConfig,
|
||||
}
|
||||
);
|
||||
let result: AdmissionAcquireResult;
|
||||
try {
|
||||
|
||||
@@ -37,6 +37,25 @@ describe("bounded request feature extraction", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("measures admission bodies beyond the default size-estimator limit", () => {
|
||||
const body = {
|
||||
messages: Array.from({ length: 200 }, () => ({ role: "user", content: "x".repeat(10_000) })),
|
||||
};
|
||||
const features = extractAdmissionCostFeatures(body);
|
||||
|
||||
assert.ok((features.bodyBytes ?? 0) > 1_000_000);
|
||||
assert.ok((features.estimatedInputTokens ?? 0) > 250_000);
|
||||
});
|
||||
|
||||
it("uses the active admission cost budget when measuring bodies", () => {
|
||||
const body = { payload: "x".repeat(17_000_000) };
|
||||
const features = extractAdmissionCostFeatures(body, {
|
||||
cost: { bodyBytesPerUnit: 1_000_000, maxRequestCost: 20 },
|
||||
});
|
||||
|
||||
assert.ok((features.bodyBytes ?? 0) > 16_384_000);
|
||||
});
|
||||
|
||||
it("extracts production-realistic Chat, Responses, Gemini, and Antigravity shapes", () => {
|
||||
// OpenAI Chat Completions — stream omitted defaults false (higher non-stream class).
|
||||
const chat = extractAdmissionCostFeatures({
|
||||
@@ -146,7 +165,7 @@ describe("bounded request feature extraction", () => {
|
||||
it("bounds tool scans and never touches entries beyond the budget (conservative count)", () => {
|
||||
// Huge leading string makes estimateSizeFast byte-exit before walking tools,
|
||||
// so only countTools can touch the tools proxy — proving its scan bound alone.
|
||||
const sizePad = "x".repeat(300_000);
|
||||
const sizePad = "x".repeat(17_000_000);
|
||||
|
||||
let accesses = 0;
|
||||
const tools = new Proxy([] as unknown[], {
|
||||
|
||||
Reference in New Issue
Block a user