diff --git a/src/app/api/resilience/route.ts b/src/app/api/resilience/route.ts index 8a623a871e..2881041748 100644 --- a/src/app/api/resilience/route.ts +++ b/src/app/api/resilience/route.ts @@ -111,9 +111,15 @@ function normalizeLegacyPatch(body: JsonRecord): ResilienceSettingsPatch { } async function syncRuntimeSettings(resilienceSettings: ResilienceSettings) { - const { applyRequestQueueSettings } = - await import("@omniroute/open-sse/services/rateLimitManager"); + const [{ applyRequestQueueSettings }, { setProviderQuotaOverrides }] = await Promise.all([ + import("@omniroute/open-sse/services/rateLimitManager"), + import("@omniroute/open-sse/services/providerDefaultRateLimit"), + ]); await applyRequestQueueSettings(resilienceSettings.requestQueue); + // #6846 Phase 2: re-apply per-provider RPM/concurrency overrides on the hot + // path so a PATCH takes effect without a process restart. Mirrors the call in + // rateLimitManager.ts::initializeRateLimits() (startup). + setProviderQuotaOverrides(resilienceSettings.providerQuotaOverrides); } /** @@ -136,6 +142,7 @@ export async function GET() { comboCooldownWait: resilience.comboCooldownWait, quotaShareConcurrencyLimit: resilience.quotaShareConcurrencyLimit, providerCooldown: resilience.providerCooldown, + providerQuotaOverrides: resilience.providerQuotaOverrides, legacy: buildLegacyResilienceCompat(resilience), }); } catch (err: unknown) { @@ -208,6 +215,12 @@ export async function PATCH(request) { providerCooldown: body.providerCooldown as ResilienceSettingsPatch["providerCooldown"], } : {}), + ...(body.providerQuotaOverrides + ? { + providerQuotaOverrides: + body.providerQuotaOverrides as ResilienceSettingsPatch["providerQuotaOverrides"], + } + : {}), ...normalizeLegacyPatch(body), }); @@ -245,6 +258,7 @@ export async function PATCH(request) { comboCooldownWait: nextResilience.comboCooldownWait, quotaShareConcurrencyLimit: nextResilience.quotaShareConcurrencyLimit, providerCooldown: nextResilience.providerCooldown, + providerQuotaOverrides: nextResilience.providerQuotaOverrides, legacy: buildLegacyResilienceCompat(nextResilience), }); } catch (err: unknown) { diff --git a/src/shared/validation/schemas/settings.ts b/src/shared/validation/schemas/settings.ts index 8f115d7f28..a8d158cf73 100644 --- a/src/shared/validation/schemas/settings.ts +++ b/src/shared/validation/schemas/settings.ts @@ -158,6 +158,21 @@ export const updateResilienceSchema = z .strict() .optional(), defaults: legacyResilienceDefaultsSchema.optional(), + // #6846 Phase 2: per-provider operator overrides for the header-less + // "provider default" static budget (open-sse/services/providerDefaultRateLimit.ts) + // and its companion per-connection concurrency cap. Mirrors + // ProviderQuotaOverrideSettings in src/lib/resilience/settings/types.ts. + providerQuotaOverrides: z + .record( + z.string().min(1), + z + .object({ + rpm: z.number().int().min(1).optional(), + concurrency: z.number().int().min(1).optional(), + }) + .strict() + ) + .optional(), }) .strict() .superRefine((value, ctx) => { @@ -170,7 +185,8 @@ export const updateResilienceSchema = z !value.quotaShareConcurrencyLimit && !value.providerCooldown && !value.profiles && - !value.defaults + !value.defaults && + !value.providerQuotaOverrides ) { ctx.addIssue({ code: z.ZodIssueCode.custom, diff --git a/tests/unit/resilience-settings-provider-quota-overrides.test.ts b/tests/unit/resilience-settings-provider-quota-overrides.test.ts new file mode 100644 index 0000000000..374cdfd484 --- /dev/null +++ b/tests/unit/resilience-settings-provider-quota-overrides.test.ts @@ -0,0 +1,142 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { + DEFAULT_RESILIENCE_SETTINGS, + mergeResilienceSettings, + resolveResilienceSettings, + type ResilienceSettings, +} from "../../src/lib/resilience/settings.ts"; +import { updateResilienceSchema } from "../../src/shared/validation/schemas.ts"; + +function cloneDefaults(): ResilienceSettings { + return structuredClone(DEFAULT_RESILIENCE_SETTINGS); +} + +// --- Schema (PATCH /api/resilience body) --- + +test("updateResilienceSchema accepts providerQuotaOverrides entries", () => { + const parsed = updateResilienceSchema.safeParse({ + providerQuotaOverrides: { + minimax: { rpm: 30, concurrency: 4 }, + nvidia: { rpm: 60 }, + }, + }); + assert.equal(parsed.success, true, "valid override map should parse"); +}); + +test("updateResilienceSchema allows a body containing only providerQuotaOverrides", () => { + // The superRefine requires at least one field; a lone override map is a + // legitimate update and must not trigger "Must provide resilience settings". + const parsed = updateResilienceSchema.safeParse({ + providerQuotaOverrides: { minimax: { rpm: 30 } }, + }); + assert.equal(parsed.success, true); +}); + +test("updateResilienceSchema rejects non-positive rpm/concurrency", () => { + assert.equal( + updateResilienceSchema.safeParse({ + providerQuotaOverrides: { minimax: { rpm: 0 } }, + }).success, + false, + "rpm must be >= 1" + ); + assert.equal( + updateResilienceSchema.safeParse({ + providerQuotaOverrides: { minimax: { concurrency: -1 } }, + }).success, + false, + "concurrency must be >= 1" + ); +}); + +test("updateResilienceSchema rejects unknown keys inside an override entry", () => { + assert.equal( + updateResilienceSchema.safeParse({ + providerQuotaOverrides: { minimax: { rpm: 30, windowMs: 60_000 } }, + }).success, + false, + "override entries are .strict() — unknown keys rejected" + ); +}); + +// --- mergeResilienceSettings --- + +test("mergeResilienceSettings stores provider quota overrides", () => { + const next = mergeResilienceSettings(cloneDefaults(), { + providerQuotaOverrides: { + minimax: { rpm: 30, concurrency: 4 }, + }, + }); + assert.deepEqual(next.providerQuotaOverrides, { minimax: { rpm: 30, concurrency: 4 } }); +}); + +test("mergeResilienceSettings preserves existing overrides when patch omits the map", () => { + const current = mergeResilienceSettings(cloneDefaults(), { + providerQuotaOverrides: { nvidia: { rpm: 40 } }, + }); + const next = mergeResilienceSettings(current, { + requestQueue: { maxWaitMs: 60_000 }, + }); + assert.deepEqual(next.providerQuotaOverrides, { nvidia: { rpm: 40 } }); +}); + +test("mergeResilienceSettings drops invalid override entries via normalization", () => { + const next = mergeResilienceSettings(cloneDefaults(), { + providerQuotaOverrides: { + minimax: { rpm: 0 }, // non-positive → dropped by normalizeProviderQuotaOverrideEntry + } as never, + }); + assert.deepEqual(next.providerQuotaOverrides, {}); +}); + +// --- resolveResilienceSettings round-trip --- + +test("resolveResilienceSettings round-trips stored provider quota overrides", () => { + const resolved = resolveResilienceSettings({ + resilienceSettings: { + providerQuotaOverrides: { + minimax: { rpm: 30, concurrency: 4 }, + }, + }, + }); + assert.deepEqual(resolved.providerQuotaOverrides, { minimax: { rpm: 30, concurrency: 4 } }); +}); + +// --- Route wiring (GET/PATCH response + hot reload) --- + +const RESILIENCE_ROUTE_PATH = path.resolve(process.cwd(), "src/app/api/resilience/route.ts"); + +test("GET /api/resilience returns providerQuotaOverrides", () => { + const source = fs.readFileSync(RESILIENCE_ROUTE_PATH, "utf8"); + assert.match( + source, + /providerQuotaOverrides:\s*resilience\.providerQuotaOverrides\b/, + "GET response should expose the resolved override map" + ); +}); + +test("PATCH /api/resilience returns providerQuotaOverrides and propagates it into the merge", () => { + const source = fs.readFileSync(RESILIENCE_ROUTE_PATH, "utf8"); + assert.match( + source, + /providerQuotaOverrides:\s*nextResilience\.providerQuotaOverrides\b/, + "PATCH response should expose the merged override map" + ); + assert.match( + source, + /body\.providerQuotaOverrides/, + "PATCH should read providerQuotaOverrides from the validated body" + ); +}); + +test("syncRuntimeSettings re-applies provider quota overrides on the hot path", () => { + const source = fs.readFileSync(RESILIENCE_ROUTE_PATH, "utf8"); + assert.match( + source, + /setProviderQuotaOverrides\(resilienceSettings\.providerQuotaOverrides\)/, + "PATCH should hot-reload overrides without a process restart" + ); +});