mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-18 21:22:28 +03:00
feat(api): add provider quota telemetry, adaptive routing, and status inventory (#10148)
* feat(api): add provider quota telemetry, adaptive routing, and status inventory Adds a read-only OmniRoute status/inventory surface plus supporting resilience and usage-tracking infrastructure: - src/lib/quota/providerQuotaTelemetry.ts, providerCapabilities.ts: provider quota state and capability signals, sourced from configured metadata rather than invented values; unknown stays unknown. - src/lib/resilience/adaptiveCircuit.ts, failureClassification.ts: circuit state with lazy recovery and explicit failure classification. - src/lib/usage/usageLedger.ts, budgetGuard.ts, modelPricingRegistry.ts: internal usage tracking and budget allow/warn/deny decisions, kept separate from upstream-reported quota (never conflated). - src/lib/routing/adaptiveRouting.ts: excludes exhausted-quota and open-circuit candidates from routing, penalizes approaching-limit. - src/lib/omnirouteStatus.ts + src/app/api/omniroute/status, route/preview: read-only status endpoint; never issues a live upstream model request (asserted via liveRequestExecuted: false). - src/lib/db/quotaPools.ts: adds ensurePool() for idempotent pool management by automation/CLI callers, following the existing group-demo default-group convention. - scripts/omniroute-verify.mjs (+ omniroute:verify script): local verification against the running gateway. 9 new unit tests, all passing. typecheck:core clean relative to base (release/v3.8.50) -- the 2 pre-existing gateways.ts errors are tracked separately in #9985 and untouched by this change. * test(cli): align cli-machine-token assertions with HMAC-SHA256 64-char format The quota-telemetry feature hardens cliToken to HMAC-SHA256(machineId, SALT) (64-char hex, pristine machine id). Update the regression test to the new format and mirror the production derivation in the different-machine-id check. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: adevwithpurpose <adevwithpurpose@users.noreply.github.com> Co-authored-by: desamours-hub <desamours-hub@users.noreply.github.com> Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
@@ -2,7 +2,7 @@ import { apiFetch, isServerUp } from "../api.mjs";
|
||||
import { t } from "../i18n.mjs";
|
||||
|
||||
export function registerQuota(program) {
|
||||
program
|
||||
const quota = program
|
||||
.command("quota")
|
||||
.description(t("quota.description"))
|
||||
.option("--provider <id>", "Filter by provider")
|
||||
@@ -12,6 +12,60 @@ export function registerQuota(program) {
|
||||
const exitCode = await runQuotaCommand({ ...opts, output: globalOpts.output });
|
||||
if (exitCode !== 0) process.exit(exitCode);
|
||||
});
|
||||
|
||||
quota
|
||||
.command("status")
|
||||
.description("Show truthful OmniRoute gateway, quota, pool, and circuit state")
|
||||
.action(async (opts, cmd) => runBoundedJson("/api/omniroute/status", cmd.optsWithGlobals()));
|
||||
|
||||
quota
|
||||
.command("preview")
|
||||
.description("Preview allocation enforcement without an upstream request")
|
||||
.requiredOption("--api-key-id <id>", "API key id")
|
||||
.requiredOption("--pool-id <id>", "quota pool id")
|
||||
.option("--tokens <n>", "estimated token usage")
|
||||
.action(async (opts, cmd) => {
|
||||
const params = new URLSearchParams({ apiKeyId: opts.apiKeyId, poolId: opts.poolId });
|
||||
if (opts.tokens != null) params.set("estimatedTokens", opts.tokens);
|
||||
await runBoundedJson(`/api/quota/preview?${params}`, cmd.optsWithGlobals());
|
||||
});
|
||||
|
||||
quota
|
||||
.command("ensure <json>")
|
||||
.description("Idempotently create or update a quota pool from a JSON object")
|
||||
.action(async (json, opts, cmd) => {
|
||||
let body;
|
||||
try {
|
||||
body = JSON.parse(json);
|
||||
} catch {
|
||||
console.error("Invalid pool JSON");
|
||||
process.exit(2);
|
||||
}
|
||||
await runBoundedJson("/api/quota/pools?ensure=true", cmd.optsWithGlobals(), {
|
||||
method: "POST",
|
||||
body,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runBoundedJson(path, opts, request = {}) {
|
||||
const started = performance.now();
|
||||
const res = await apiFetch(path, {
|
||||
...request,
|
||||
retry: false,
|
||||
timeout: Math.min(opts.timeout ?? 5000, 5000),
|
||||
acceptNotOk: true,
|
||||
});
|
||||
const elapsed = Math.round(performance.now() - started);
|
||||
if (process.env.OMNIROUTE_DEBUG === "1") {
|
||||
console.error(`[omniroute] ${request.method ?? "GET"} ${path} completed in ${elapsed}ms`);
|
||||
}
|
||||
const payload = await res.json().catch(() => ({ error: `HTTP ${res.status}` }));
|
||||
if (!res.ok) {
|
||||
console.error(JSON.stringify(payload));
|
||||
process.exit(res.exitCode ?? 1);
|
||||
}
|
||||
console.log(JSON.stringify(payload, null, 2));
|
||||
}
|
||||
|
||||
export async function runQuotaCommand(opts = {}) {
|
||||
|
||||
@@ -23,12 +23,11 @@ export async function getCliToken() {
|
||||
// Same resolution order as src/lib/machineToken.ts.
|
||||
const mod = await import("node-machine-id");
|
||||
const machineIdSync = mod.machineIdSync ?? mod.default?.machineIdSync;
|
||||
const mid = machineIdSync();
|
||||
_cached = crypto
|
||||
.createHash("sha256")
|
||||
.update(mid + salt)
|
||||
.digest("hex")
|
||||
.substring(0, 32);
|
||||
if (typeof machineIdSync !== "function") throw new Error("machine-id API unavailable");
|
||||
// machineIdSync(true) returns the original unhashed hardware ID — mirrors
|
||||
// getMachineTokenSync() in src/lib/machineToken.ts (#10148 cliToken hardening).
|
||||
const mid = machineIdSync(true);
|
||||
_cached = crypto.createHmac("sha256", mid).update(salt).digest("hex");
|
||||
} catch (e) {
|
||||
// Swallowing here changes control flow (every management call goes out
|
||||
// unauthenticated and 401s), so leave a breadcrumb rather than failing mute.
|
||||
|
||||
9
docs/OMNIROUTE_ALLOCATION_HANDOFF.md
Normal file
9
docs/OMNIROUTE_ALLOCATION_HANDOFF.md
Normal file
@@ -0,0 +1,9 @@
|
||||
# OmniRoute Allocation Handoff
|
||||
|
||||
Allocation is not provider quota.
|
||||
|
||||
Quota pools define which API keys may consume a provider pool and how hard, soft, or burst policies apply. Provider quota is external capacity reported by a provider or an explicitly configured source. Ghostlight internal budgets are governance limits defined by the administrator.
|
||||
|
||||
The `ensurePool` operation is idempotent: an identical pool is unchanged, a changed allocation is updated, and a missing pool is created. This is intended for automation and bounded API callers.
|
||||
|
||||
The read-only status endpoint is `GET /api/omniroute/status`. The verification command is `npm run omniroute:verify`; it makes no live model request.
|
||||
9
docs/OMNIROUTE_PROVIDER_FAILOVER.md
Normal file
9
docs/OMNIROUTE_PROVIDER_FAILOVER.md
Normal file
@@ -0,0 +1,9 @@
|
||||
# OmniRoute Provider Failover
|
||||
|
||||
Failures are classified before retry decisions are made.
|
||||
|
||||
Transient failures such as timeouts, network errors, rate limits, and provider 5xx responses may fail over. Authentication errors, permission errors, invalid requests, unavailable models, and unknown failures are not retried blindly.
|
||||
|
||||
The default cross-provider policy allows up to three provider attempts, retries rate limits and timeouts, and keeps administrative disablement separate from temporary circuit state.
|
||||
|
||||
Circuit states are `closed`, `open`, and `half_open`. A cooldown schedules a bounded probe; a successful probe closes the circuit and a failed probe reopens it.
|
||||
17
docs/OMNIROUTE_QUOTA_TELEMETRY.md
Normal file
17
docs/OMNIROUTE_QUOTA_TELEMETRY.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# OmniRoute Quota Telemetry
|
||||
|
||||
OmniRoute separates provider quota telemetry from Ghostlight accounting.
|
||||
|
||||
## Truthful states
|
||||
|
||||
- `healthy` means a source reported usable remaining capacity.
|
||||
- `approaching_limit` means a source reported remaining capacity at or below the configured threshold.
|
||||
- `exhausted` is emitted only when a source reports zero capacity or usage at its limit.
|
||||
- `unavailable` means a supported source failed to return data.
|
||||
- `unknown` means no supported source exists or no provider limit is known.
|
||||
|
||||
Unknown is not exhausted and does not disable a provider.
|
||||
|
||||
Sources are preferred in this order: official provider API, authenticated usage API, explicitly mapped response headers, administrator configuration, local estimates, unknown. Local estimates are never presented as provider billing data.
|
||||
|
||||
Response headers are parsed only through an explicit provider mapping. Generic header names are not assumed globally.
|
||||
11
docs/OMNIROUTE_ROUTING_POLICY.md
Normal file
11
docs/OMNIROUTE_ROUTING_POLICY.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# OmniRoute Routing Policy
|
||||
|
||||
Routing preserves the existing capability and combo selection logic, then applies allocation, health, circuit, quota, latency, reliability, model preference, and cost preference factors.
|
||||
|
||||
The adaptive score is explainable and returns both the selected candidate and all ranked candidates. Exhausted quota, denied allocation, and open circuits are ineligible. Unknown quota remains eligible with a neutral quota factor.
|
||||
|
||||
Route preview is deterministic and performs zero upstream model requests:
|
||||
|
||||
`POST /api/omniroute/route/preview`
|
||||
|
||||
The response includes candidate scores, factors, reasons, the selected provider, and `liveRequestExecuted: false`.
|
||||
@@ -100,6 +100,7 @@
|
||||
"build:secure": "OMNIROUTE_BUILD_PROFILE=minimal node scripts/build/build-next-isolated.mjs",
|
||||
"build:backend": "cross-env OMNIROUTE_BUILD_BACKEND_ONLY=1 node scripts/build/build-next-isolated.mjs",
|
||||
"build:cli": "node --import tsx scripts/build/prepublish.ts",
|
||||
"omniroute:verify": "node scripts/check/omniroute-verify.mjs",
|
||||
"build:release": "rm -rf .build dist && OMNIROUTE_BUILD_SHA=$(git rev-parse --short HEAD) npm run build && npm run build:cli && node scripts/build/write-build-sha.mjs",
|
||||
"build:native:tproxy": "cd src/mitm/tproxy/native && npx --yes node-gyp rebuild",
|
||||
"start": "node scripts/dev/run-next.mjs start",
|
||||
|
||||
69
scripts/check/omniroute-verify.mjs
Normal file
69
scripts/check/omniroute-verify.mjs
Normal file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { CLI_TOKEN_HEADER, getCliToken } from "../../bin/cli/utils/cliToken.mjs";
|
||||
|
||||
const baseUrl = (process.env.OMNIROUTE_BASE_URL || "http://127.0.0.1:20128").replace(/\/$/, "");
|
||||
const apiKey = process.env.OMNIROUTE_API_KEY || "";
|
||||
const timeoutMs = 5000;
|
||||
|
||||
async function get(path) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let hardTimer;
|
||||
const hardTimeout = new Promise((_, reject) => {
|
||||
hardTimer = setTimeout(
|
||||
() => reject(new Error(`request timeout after ${timeoutMs}ms`)),
|
||||
timeoutMs + 100
|
||||
);
|
||||
});
|
||||
try {
|
||||
const response = await Promise.race([
|
||||
fetch(`${baseUrl}${path}`, {
|
||||
headers: {
|
||||
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
||||
[CLI_TOKEN_HEADER]: await getCliToken(),
|
||||
},
|
||||
signal: controller.signal,
|
||||
}),
|
||||
hardTimeout,
|
||||
]);
|
||||
const body = await response.json().catch(() => null);
|
||||
return { ok: response.ok, status: response.status, body };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
clearTimeout(hardTimer);
|
||||
}
|
||||
}
|
||||
|
||||
function check(label, passed, detail = "") {
|
||||
console.log(`${label}: ${passed ? "PASS" : "FAIL"}${detail ? ` (${detail})` : ""}`);
|
||||
return passed;
|
||||
}
|
||||
|
||||
console.log("OmniRoute Verification");
|
||||
console.log(`Gateway: ${baseUrl}`);
|
||||
const results = [];
|
||||
|
||||
try {
|
||||
const models = await get("/v1/models");
|
||||
results.push(check("Gateway", models.ok, `HTTP ${models.status}`));
|
||||
const modelCount = Array.isArray(models.body?.data) ? models.body.data.length : 0;
|
||||
results.push(check("Catalog", modelCount > 0, `${modelCount} models`));
|
||||
|
||||
const pools = await get("/api/quota/pools");
|
||||
const poolRows = Array.isArray(pools.body?.pools) ? pools.body.pools : [];
|
||||
const allocations = poolRows.reduce((sum, pool) => sum + (pool.allocations?.length || 0), 0);
|
||||
results.push(check("Pools", pools.ok, `${poolRows.length}`));
|
||||
results.push(check("Allocations", pools.ok && allocations >= poolRows.length, `${allocations}`));
|
||||
|
||||
const status = await get("/api/omniroute/status");
|
||||
results.push(check("Status API", status.ok, `HTTP ${status.status}`));
|
||||
results.push(check("No live request", status.body?.liveRequestExecuted === false));
|
||||
} catch (error) {
|
||||
results.push(
|
||||
check("Verification", false, error instanceof Error ? error.message : String(error))
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Live upstream requests: 0`);
|
||||
if (results.some((passed) => !passed)) process.exitCode = 1;
|
||||
36
src/app/api/omniroute/route/preview/route.ts
Normal file
36
src/app/api/omniroute/route/preview/route.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { rankCandidates } from "@/lib/routing/adaptiveRouting";
|
||||
|
||||
const candidateSchema = z.object({
|
||||
providerId: z.string().min(1),
|
||||
modelId: z.string().min(1),
|
||||
capabilityScore: z.number().min(0).max(1),
|
||||
allocation: z.enum(["allow", "warn", "deny"]),
|
||||
healthScore: z.number().min(0).max(1),
|
||||
circuit: z.enum(["closed", "open", "half_open"]),
|
||||
quota: z.enum(["healthy", "approaching_limit", "exhausted", "unavailable", "unknown"]),
|
||||
latencyMs: z.number().nonnegative().optional(),
|
||||
errorRate: z.number().min(0).max(1).optional(),
|
||||
modelPreference: z.number().min(0).max(1).optional(),
|
||||
costPreference: z.number().min(0).max(1).optional(),
|
||||
});
|
||||
|
||||
const requestSchema = z.object({ candidates: z.array(candidateSchema).min(1).max(100) });
|
||||
|
||||
/** Deterministic routing preview. It never calls an upstream provider. */
|
||||
export async function POST(request: Request): Promise<Response> {
|
||||
const authError = await requireManagementAuth(request);
|
||||
if (authError) return authError;
|
||||
const parsed = requestSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return NextResponse.json({ error: parsed.error.message }, { status: 400 });
|
||||
|
||||
const result = rankCandidates(parsed.data);
|
||||
return NextResponse.json({
|
||||
request: { candidateCount: parsed.data.candidates.length },
|
||||
...result,
|
||||
selected: result.selected?.providerId ?? null,
|
||||
liveRequestExecuted: false,
|
||||
});
|
||||
}
|
||||
21
src/app/api/omniroute/status/route.ts
Normal file
21
src/app/api/omniroute/status/route.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { buildOmniRouteStatus } from "@/lib/omnirouteStatus";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/** Read-only operational status; never performs an upstream model request. */
|
||||
export async function GET(request: Request): Promise<Response> {
|
||||
const authError = await requireManagementAuth(request);
|
||||
if (authError) return authError;
|
||||
|
||||
try {
|
||||
return NextResponse.json({
|
||||
generatedAt: new Date().toISOString(),
|
||||
liveRequestExecuted: false,
|
||||
...(await buildOmniRouteStatus()),
|
||||
});
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Failed to build OmniRoute status" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { NextResponse } from "next/server";
|
||||
import { buildErrorBody } from "@omniroute/open-sse/utils/error";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { PoolCreateSchema } from "@/shared/schemas/quota";
|
||||
import { listPools, createPool } from "@/lib/localDb";
|
||||
import { listPools, createPool, ensurePool } from "@/lib/localDb";
|
||||
import { logAuditEvent, getAuditRequestContext } from "@/lib/compliance/index";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
@@ -48,17 +48,25 @@ export async function POST(request: Request): Promise<Response> {
|
||||
return NextResponse.json(buildErrorBody(400, parsed.error.message), { status: 400 });
|
||||
}
|
||||
|
||||
const pool = createPool(parsed.data);
|
||||
const ensure = new URL(request.url).searchParams.get("ensure") === "true";
|
||||
const ensured = ensure ? ensurePool(parsed.data) : null;
|
||||
const pool = ensured?.pool ?? createPool(parsed.data);
|
||||
const ctx = getAuditRequestContext(request);
|
||||
logAuditEvent({
|
||||
action: "quota.pool.created",
|
||||
action: ensured?.updated ? "quota.pool.updated" : "quota.pool.created",
|
||||
target: pool.id,
|
||||
metadata: { connectionId: pool.connectionId, name: pool.name },
|
||||
metadata: {
|
||||
connectionId: pool.connectionId,
|
||||
name: pool.name,
|
||||
ensure,
|
||||
created: ensured?.created ?? true,
|
||||
updated: ensured?.updated ?? false,
|
||||
},
|
||||
ipAddress: ctx.ipAddress ?? undefined,
|
||||
requestId: ctx.requestId,
|
||||
});
|
||||
|
||||
return NextResponse.json({ pool }, { status: 201 });
|
||||
return NextResponse.json({ pool, ...(ensured ? { created: ensured.created, updated: ensured.updated } : {}) }, { status: ensured?.created === false ? 200 : 201 });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Failed to create pool";
|
||||
return NextResponse.json(buildErrorBody(500, message), { status: 500 });
|
||||
|
||||
@@ -109,6 +109,12 @@ export interface PoolUpdate {
|
||||
connectionIds?: string[];
|
||||
}
|
||||
|
||||
export interface EnsurePoolResult {
|
||||
pool: QuotaPool;
|
||||
created: boolean;
|
||||
updated: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -427,6 +433,50 @@ export function createPool(input: PoolCreate): QuotaPool {
|
||||
return result;
|
||||
}
|
||||
|
||||
function allocationFingerprint(allocations: PoolAllocation[] = []): string {
|
||||
return JSON.stringify(
|
||||
allocations
|
||||
.map((allocation) => ({
|
||||
apiKeyId: allocation.apiKeyId,
|
||||
weight: allocation.weight,
|
||||
capValue: allocation.capValue ?? null,
|
||||
capUnit: allocation.capUnit ?? null,
|
||||
policy: allocation.policy,
|
||||
}))
|
||||
.sort((left, right) => left.apiKeyId.localeCompare(right.apiKeyId))
|
||||
);
|
||||
}
|
||||
|
||||
/** Idempotent pool management for automation and bounded CLI callers. */
|
||||
export function ensurePool(input: PoolCreate): EnsurePoolResult {
|
||||
const members = input.connectionIds && input.connectionIds.length > 0
|
||||
? input.connectionIds
|
||||
: [input.connectionId];
|
||||
const groupId = input.groupId || "group-demo";
|
||||
const existing = listPools().items.find((pool) => {
|
||||
return pool.name === input.name && pool.groupId === groupId;
|
||||
});
|
||||
|
||||
if (!existing) return { pool: createPool(input), created: true, updated: false };
|
||||
|
||||
const allocationsChanged =
|
||||
input.allocations !== undefined &&
|
||||
allocationFingerprint(existing.allocations) !== allocationFingerprint(input.allocations);
|
||||
const membersChanged =
|
||||
existing.connectionIds.length !== members.length ||
|
||||
existing.connectionIds.some((id) => !members.includes(id));
|
||||
if (!allocationsChanged && !membersChanged) {
|
||||
return { pool: existing, created: false, updated: false };
|
||||
}
|
||||
|
||||
const update: PoolUpdate = { connectionIds: members };
|
||||
if (input.allocations !== undefined) update.allocations = input.allocations;
|
||||
if (input.groupId !== undefined) update.groupId = input.groupId;
|
||||
const updated = updatePool(existing.id, update);
|
||||
if (!updated) throw new Error(`Quota pool disappeared during ensure: ${existing.id}`);
|
||||
return { pool: updated, created: false, updated: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing pool's name, allocations, and/or member connections.
|
||||
* Returns updated pool, or null if pool not found.
|
||||
|
||||
@@ -618,6 +618,7 @@ export {
|
||||
listPools,
|
||||
getPool,
|
||||
getPoolsByGroup,
|
||||
ensurePool,
|
||||
createPool,
|
||||
updatePool,
|
||||
deletePool,
|
||||
|
||||
87
src/lib/omnirouteStatus.ts
Normal file
87
src/lib/omnirouteStatus.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import { getDbInstance, pingDb } from "@/lib/db/core";
|
||||
import { listPools } from "@/lib/db/quotaPools";
|
||||
|
||||
interface ProviderStatusRow {
|
||||
id: string;
|
||||
provider: string;
|
||||
is_active: number | boolean | null;
|
||||
test_status: string | null;
|
||||
last_error: string | null;
|
||||
}
|
||||
|
||||
function readProviderStatusRows(): ProviderStatusRow[] {
|
||||
const db = getDbInstance();
|
||||
return db
|
||||
.prepare<ProviderStatusRow>(
|
||||
"SELECT id, provider, is_active, test_status, last_error FROM provider_connections"
|
||||
)
|
||||
.all();
|
||||
}
|
||||
|
||||
export async function buildOmniRouteStatus() {
|
||||
const [connections, circuitModule, quotaMonitorModule] = await Promise.all([
|
||||
Promise.resolve(readProviderStatusRows()),
|
||||
import("@/shared/utils/circuitBreaker").catch(() => null),
|
||||
import("../../open-sse/services/quotaMonitor").catch(() => null),
|
||||
]);
|
||||
const pools = listPools().items;
|
||||
const circuitStatuses = circuitModule?.getAllCircuitBreakerStatuses() ?? null;
|
||||
const quotaSummary = quotaMonitorModule?.getQuotaMonitorSummary() ?? null;
|
||||
const active = connections.filter(
|
||||
(connection) => connection.is_active !== 0 && connection.is_active !== false
|
||||
);
|
||||
const disabled = connections.filter(
|
||||
(connection) => connection.is_active === 0 || connection.is_active === false
|
||||
);
|
||||
const healthy = active.filter((connection) => connection.test_status === "active");
|
||||
|
||||
return {
|
||||
gateway: pingDb() ? "healthy" : "degraded",
|
||||
catalog: { available: true },
|
||||
providers: {
|
||||
configured: connections.length,
|
||||
active: active.length,
|
||||
healthy: healthy.length,
|
||||
disabled: disabled.length,
|
||||
connections: connections.map((connection) => ({
|
||||
id: connection.id,
|
||||
provider: connection.provider,
|
||||
active: connection.is_active !== 0 && connection.is_active !== false,
|
||||
health:
|
||||
connection.is_active === 0 || connection.is_active === false
|
||||
? "disabled"
|
||||
: connection.test_status === "active"
|
||||
? "healthy"
|
||||
: "unknown",
|
||||
failureState: connection.last_error ? "recent_error" : "none",
|
||||
})),
|
||||
},
|
||||
pools: {
|
||||
count: pools.length,
|
||||
allocations: pools.reduce((count, pool) => count + pool.allocations.length, 0),
|
||||
items: pools.map((pool) => ({
|
||||
id: pool.id,
|
||||
name: pool.name,
|
||||
connectionIds: pool.connectionIds,
|
||||
allocationCount: pool.allocations.length,
|
||||
})),
|
||||
},
|
||||
quotaMonitoring: {
|
||||
authoritative: quotaSummary?.active ?? 0,
|
||||
headerBased: 0,
|
||||
configured: 0,
|
||||
unsupported: quotaSummary ? Math.max(0, active.length - quotaSummary.active) : null,
|
||||
status: quotaSummary?.active ? "partial" : "unknown",
|
||||
},
|
||||
circuits: circuitStatuses
|
||||
? {
|
||||
open: circuitStatuses.filter((c) => c.state === "OPEN").length,
|
||||
halfOpen: circuitStatuses.filter((c) => c.state === "HALF_OPEN").length,
|
||||
closed: circuitStatuses.filter((c) => c.state === "CLOSED").length,
|
||||
source: "persisted circuit breaker registry",
|
||||
}
|
||||
: { status: "unknown", source: "resilience subsystem unavailable" },
|
||||
usage: { source: "usage_history and call_logs", liveRequestsExecuted: false },
|
||||
budgets: { source: "internal governance limits", upstreamQuotaClaims: false },
|
||||
};
|
||||
}
|
||||
37
src/lib/quota/providerCapabilities.ts
Normal file
37
src/lib/quota/providerCapabilities.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
export interface ProviderCapabilities {
|
||||
providerId: string;
|
||||
quotaApi: boolean;
|
||||
usageApi: boolean;
|
||||
rateLimitHeaders: boolean;
|
||||
streaming: boolean;
|
||||
toolUse: boolean;
|
||||
coding: boolean;
|
||||
vision: boolean;
|
||||
longContext: boolean;
|
||||
}
|
||||
|
||||
const registry = new Map<string, ProviderCapabilities>();
|
||||
|
||||
export function registerProviderCapabilities(capabilities: ProviderCapabilities): void {
|
||||
registry.set(capabilities.providerId, { ...capabilities });
|
||||
}
|
||||
|
||||
export function getProviderCapabilities(providerId: string): ProviderCapabilities {
|
||||
return (
|
||||
registry.get(providerId) ?? {
|
||||
providerId,
|
||||
quotaApi: false,
|
||||
usageApi: false,
|
||||
rateLimitHeaders: false,
|
||||
streaming: false,
|
||||
toolUse: false,
|
||||
coding: false,
|
||||
vision: false,
|
||||
longContext: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export function listProviderCapabilities(): ProviderCapabilities[] {
|
||||
return [...registry.values()].map((capabilities) => ({ ...capabilities }));
|
||||
}
|
||||
248
src/lib/quota/providerQuotaTelemetry.ts
Normal file
248
src/lib/quota/providerQuotaTelemetry.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
/** Provider-neutral quota telemetry contracts and header normalization. */
|
||||
|
||||
export type QuotaDimensionName =
|
||||
| "requests"
|
||||
| "tokens"
|
||||
| "input_tokens"
|
||||
| "output_tokens"
|
||||
| "credits"
|
||||
| "currency"
|
||||
| "daily_requests"
|
||||
| "weekly_requests"
|
||||
| "monthly_requests"
|
||||
| "rate_limit"
|
||||
| "unknown";
|
||||
|
||||
export type QuotaValueSource =
|
||||
"provider_api" | "response_headers" | "configured" | "estimated" | "unknown";
|
||||
|
||||
export type QuotaConfidence = "authoritative" | "high" | "medium" | "low" | "unknown";
|
||||
|
||||
export interface QuotaValue {
|
||||
dimension: QuotaDimensionName;
|
||||
limit?: number;
|
||||
used?: number;
|
||||
remaining?: number;
|
||||
resetAt?: string;
|
||||
unit?: string;
|
||||
source: QuotaValueSource;
|
||||
confidence: QuotaConfidence;
|
||||
}
|
||||
|
||||
export type ProviderQuotaStatus =
|
||||
"healthy" | "approaching_limit" | "exhausted" | "unavailable" | "unknown";
|
||||
|
||||
export interface ProviderQuotaState {
|
||||
providerId: string;
|
||||
connectionId: string;
|
||||
supported: boolean;
|
||||
fetchedAt: string;
|
||||
values: QuotaValue[];
|
||||
status: ProviderQuotaStatus;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ProviderConnectionForQuota {
|
||||
id: string;
|
||||
provider: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface ProviderQuotaMonitor {
|
||||
providerId: string;
|
||||
supportedDimensions(): Promise<QuotaDimensionName[]>;
|
||||
fetchQuotaState(connection: ProviderConnectionForQuota): Promise<ProviderQuotaState>;
|
||||
}
|
||||
|
||||
export type QuotaSourceKind =
|
||||
"provider_api" | "response_headers" | "configured" | "estimated" | "unknown";
|
||||
|
||||
export interface QuotaSourceAdapter {
|
||||
kind: QuotaSourceKind;
|
||||
supports(providerId: string): boolean;
|
||||
read(connection: ProviderConnectionForQuota): Promise<QuotaValue[]>;
|
||||
}
|
||||
|
||||
const SOURCE_PRIORITY: QuotaSourceKind[] = [
|
||||
"provider_api",
|
||||
"response_headers",
|
||||
"configured",
|
||||
"estimated",
|
||||
"unknown",
|
||||
];
|
||||
|
||||
function sourceRank(source: QuotaSourceKind): number {
|
||||
return SOURCE_PRIORITY.indexOf(source);
|
||||
}
|
||||
|
||||
function finite(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function statusForValues(values: QuotaValue[], approachingThreshold: number): ProviderQuotaStatus {
|
||||
if (values.length === 0) return "unknown";
|
||||
|
||||
const exhausted = values.some(
|
||||
(value) =>
|
||||
(finite(value.remaining) && value.remaining <= 0) ||
|
||||
(finite(value.used) && finite(value.limit) && value.used >= value.limit)
|
||||
);
|
||||
if (exhausted) return "exhausted";
|
||||
|
||||
const approaching = values.some((value) => {
|
||||
if (!finite(value.remaining) || !finite(value.limit) || value.limit <= 0) return false;
|
||||
return value.remaining / value.limit <= approachingThreshold;
|
||||
});
|
||||
return approaching ? "approaching_limit" : "healthy";
|
||||
}
|
||||
|
||||
export function unknownQuotaState(
|
||||
providerId: string,
|
||||
connectionId: string,
|
||||
fetchedAt = new Date().toISOString(),
|
||||
error?: string
|
||||
): ProviderQuotaState {
|
||||
return {
|
||||
providerId,
|
||||
connectionId,
|
||||
supported: false,
|
||||
fetchedAt,
|
||||
values: [],
|
||||
status: "unknown",
|
||||
...(error ? { error } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the best available source without treating missing provider data as exhaustion.
|
||||
* Sources are selected per dimension, so a provider API can coexist with header data.
|
||||
*/
|
||||
export async function collectQuotaState(
|
||||
connection: ProviderConnectionForQuota,
|
||||
adapters: QuotaSourceAdapter[],
|
||||
options: { approachingThreshold?: number; fetchedAt?: string } = {}
|
||||
): Promise<ProviderQuotaState> {
|
||||
const fetchedAt = options.fetchedAt ?? new Date().toISOString();
|
||||
const byDimension = new Map<QuotaDimensionName, QuotaValue>();
|
||||
let supported = false;
|
||||
let lastError: string | undefined;
|
||||
|
||||
for (const adapter of adapters) {
|
||||
if (!adapter.supports(connection.provider)) continue;
|
||||
supported = true;
|
||||
try {
|
||||
const values = await adapter.read(connection);
|
||||
for (const value of values) {
|
||||
const current = byDimension.get(value.dimension);
|
||||
if (!current || sourceRank(value.source) < sourceRank(current.source)) {
|
||||
byDimension.set(value.dimension, value);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
|
||||
const values = [...byDimension.values()];
|
||||
return {
|
||||
providerId: connection.provider,
|
||||
connectionId: connection.id,
|
||||
supported,
|
||||
fetchedAt,
|
||||
values,
|
||||
status:
|
||||
values.length > 0
|
||||
? statusForValues(values, options.approachingThreshold ?? 0.2)
|
||||
: supported
|
||||
? "unavailable"
|
||||
: "unknown",
|
||||
...(lastError ? { error: lastError } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface RateLimitHeaderMapping {
|
||||
limit?: string;
|
||||
remaining?: string;
|
||||
reset?: string;
|
||||
retryAfter?: string;
|
||||
dimension?: QuotaDimensionName;
|
||||
unit?: string;
|
||||
}
|
||||
|
||||
export interface RateLimitSnapshot {
|
||||
providerId: string;
|
||||
connectionId: string;
|
||||
capturedAt: string;
|
||||
requestLimit?: number;
|
||||
requestsRemaining?: number;
|
||||
resetAt?: string;
|
||||
retryAfterSeconds?: number;
|
||||
source: "response_headers";
|
||||
}
|
||||
|
||||
function headerValue(headers: Headers | Record<string, string | undefined>, name?: string) {
|
||||
if (!name) return undefined;
|
||||
if (headers instanceof Headers)
|
||||
return headers.get(name) ?? headers.get(name.toLowerCase()) ?? undefined;
|
||||
const lower = name.toLowerCase();
|
||||
const key = Object.keys(headers).find((candidate) => candidate.toLowerCase() === lower);
|
||||
return key ? headers[key] : undefined;
|
||||
}
|
||||
|
||||
function numberHeader(value: string | undefined): number | undefined {
|
||||
if (value === undefined || value.trim() === "") return undefined;
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function resetHeaderToIso(value: string | undefined): string | undefined {
|
||||
const parsed = numberHeader(value);
|
||||
if (parsed === undefined)
|
||||
return value && !Number.isNaN(Date.parse(value)) ? new Date(value).toISOString() : undefined;
|
||||
const milliseconds = parsed > 10_000_000_000 ? parsed : parsed * 1000;
|
||||
return new Date(milliseconds).toISOString();
|
||||
}
|
||||
|
||||
export function parseRateLimitHeaders(
|
||||
headers: Headers | Record<string, string | undefined>,
|
||||
providerId: string,
|
||||
connectionId: string,
|
||||
mapping: RateLimitHeaderMapping,
|
||||
capturedAt = new Date().toISOString()
|
||||
): { value: QuotaValue; snapshot: RateLimitSnapshot } | null {
|
||||
const limit = numberHeader(headerValue(headers, mapping.limit));
|
||||
const remaining = numberHeader(headerValue(headers, mapping.remaining));
|
||||
const resetAt = resetHeaderToIso(headerValue(headers, mapping.reset));
|
||||
const retryAfterSeconds = numberHeader(headerValue(headers, mapping.retryAfter));
|
||||
if (
|
||||
limit === undefined &&
|
||||
remaining === undefined &&
|
||||
!resetAt &&
|
||||
retryAfterSeconds === undefined
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const dimension = mapping.dimension ?? "rate_limit";
|
||||
return {
|
||||
value: {
|
||||
dimension,
|
||||
...(limit !== undefined ? { limit } : {}),
|
||||
...(remaining !== undefined ? { remaining } : {}),
|
||||
...(resetAt ? { resetAt } : {}),
|
||||
...(mapping.unit ? { unit: mapping.unit } : {}),
|
||||
source: "response_headers",
|
||||
confidence: "high",
|
||||
},
|
||||
snapshot: {
|
||||
providerId,
|
||||
connectionId,
|
||||
capturedAt,
|
||||
...(limit !== undefined ? { requestLimit: limit } : {}),
|
||||
...(remaining !== undefined ? { requestsRemaining: remaining } : {}),
|
||||
...(resetAt ? { resetAt } : {}),
|
||||
...(retryAfterSeconds !== undefined ? { retryAfterSeconds } : {}),
|
||||
source: "response_headers",
|
||||
},
|
||||
};
|
||||
}
|
||||
60
src/lib/resilience/adaptiveCircuit.ts
Normal file
60
src/lib/resilience/adaptiveCircuit.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
export type AdaptiveCircuitState = "closed" | "open" | "half_open";
|
||||
|
||||
export interface AdaptiveCircuit {
|
||||
state: AdaptiveCircuitState;
|
||||
failureCount: number;
|
||||
successCount: number;
|
||||
lastFailureAt?: string;
|
||||
openedAt?: string;
|
||||
halfOpenAt?: string;
|
||||
nextProbeAt?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export function createAdaptiveCircuit(): AdaptiveCircuit {
|
||||
return { state: "closed", failureCount: 0, successCount: 0 };
|
||||
}
|
||||
|
||||
export function observeCircuit(
|
||||
current: AdaptiveCircuit,
|
||||
event: "failure" | "success" | "probe",
|
||||
options: { now?: Date; failureThreshold?: number; cooldownMs?: number; reason?: string } = {}
|
||||
): AdaptiveCircuit {
|
||||
const now = options.now ?? new Date();
|
||||
const failureThreshold = options.failureThreshold ?? 3;
|
||||
const cooldownMs = options.cooldownMs ?? 60_000;
|
||||
const next = { ...current };
|
||||
|
||||
if (event === "failure") {
|
||||
next.failureCount += 1;
|
||||
next.successCount = 0;
|
||||
next.lastFailureAt = now.toISOString();
|
||||
next.reason = options.reason;
|
||||
if (next.state === "half_open" || next.failureCount >= failureThreshold) {
|
||||
next.state = "open";
|
||||
next.openedAt = now.toISOString();
|
||||
next.nextProbeAt = new Date(now.getTime() + cooldownMs).toISOString();
|
||||
next.halfOpenAt = undefined;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
if (event === "probe") {
|
||||
if (next.state === "open" && (!next.nextProbeAt || now >= new Date(next.nextProbeAt))) {
|
||||
next.state = "half_open";
|
||||
next.halfOpenAt = now.toISOString();
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
next.successCount += 1;
|
||||
if (next.state === "half_open" || next.successCount >= 1) {
|
||||
next.state = "closed";
|
||||
next.failureCount = 0;
|
||||
next.openedAt = undefined;
|
||||
next.halfOpenAt = undefined;
|
||||
next.nextProbeAt = undefined;
|
||||
next.reason = undefined;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
75
src/lib/resilience/failureClassification.ts
Normal file
75
src/lib/resilience/failureClassification.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
export type ProviderFailureType =
|
||||
| "authentication_error"
|
||||
| "rate_limit"
|
||||
| "quota_exhausted"
|
||||
| "timeout"
|
||||
| "network_error"
|
||||
| "provider_5xx"
|
||||
| "invalid_request"
|
||||
| "model_unavailable"
|
||||
| "permission_error"
|
||||
| "unknown";
|
||||
|
||||
export interface ProviderFailure {
|
||||
type: ProviderFailureType;
|
||||
retryable: boolean;
|
||||
providerId: string;
|
||||
connectionId?: string;
|
||||
statusCode?: number;
|
||||
retryAfter?: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export function classifyProviderFailure(input: {
|
||||
providerId: string;
|
||||
connectionId?: string;
|
||||
statusCode?: number;
|
||||
code?: string;
|
||||
message?: string;
|
||||
retryAfter?: number;
|
||||
}): ProviderFailure {
|
||||
const message = input.message?.trim() || "Provider request failed";
|
||||
const normalized = `${input.code ?? ""} ${message}`.toLowerCase();
|
||||
const status = input.statusCode;
|
||||
let type: ProviderFailureType = "unknown";
|
||||
let retryable = false;
|
||||
|
||||
if (
|
||||
status === 401 ||
|
||||
status === 403 ||
|
||||
/invalid.*(key|token)|unauthori[sz]ed|forbidden/.test(normalized)
|
||||
) {
|
||||
type =
|
||||
status === 403 || normalized.includes("permission")
|
||||
? "permission_error"
|
||||
: "authentication_error";
|
||||
} else if (status === 408 || status === 504 || /timeout|timed out|etimedout/.test(normalized)) {
|
||||
type = "timeout";
|
||||
retryable = true;
|
||||
} else if (status === 429 || /rate.?limit|too many requests|retry.?after/.test(normalized)) {
|
||||
type = /quota|insufficient balance|credits exhausted|balance is \$0/.test(normalized)
|
||||
? "quota_exhausted"
|
||||
: "rate_limit";
|
||||
retryable = type === "rate_limit";
|
||||
} else if (status !== undefined && status >= 500) {
|
||||
type = "provider_5xx";
|
||||
retryable = true;
|
||||
} else if (status === 400 || /invalid request|malformed|unsupported parameter/.test(normalized)) {
|
||||
type = "invalid_request";
|
||||
} else if (status === 404 || /model unavailable|model not found|unknown model/.test(normalized)) {
|
||||
type = "model_unavailable";
|
||||
} else if (/network|econnreset|econnrefused|enotfound|fetch failed|socket/.test(normalized)) {
|
||||
type = "network_error";
|
||||
retryable = true;
|
||||
}
|
||||
|
||||
return {
|
||||
type,
|
||||
retryable,
|
||||
providerId: input.providerId,
|
||||
...(input.connectionId ? { connectionId: input.connectionId } : {}),
|
||||
...(status !== undefined ? { statusCode: status } : {}),
|
||||
...(input.retryAfter !== undefined ? { retryAfter: input.retryAfter } : {}),
|
||||
message,
|
||||
};
|
||||
}
|
||||
153
src/lib/routing/adaptiveRouting.ts
Normal file
153
src/lib/routing/adaptiveRouting.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
import type { ProviderFailure } from "@/lib/resilience/failureClassification";
|
||||
import type { ProviderQuotaStatus } from "@/lib/quota/providerQuotaTelemetry";
|
||||
|
||||
export type AllocationDecision = "allow" | "warn" | "deny";
|
||||
export type CircuitState = "closed" | "open" | "half_open";
|
||||
|
||||
export interface RoutingCandidate {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
capabilityScore: number;
|
||||
allocation: AllocationDecision;
|
||||
healthScore: number;
|
||||
circuit: CircuitState;
|
||||
quota: ProviderQuotaStatus;
|
||||
latencyMs?: number;
|
||||
errorRate?: number;
|
||||
modelPreference?: number;
|
||||
costPreference?: number;
|
||||
}
|
||||
|
||||
export interface RoutingExplanation {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
reasons: string[];
|
||||
factors: Record<string, number | string>;
|
||||
}
|
||||
|
||||
export interface RankedRoutingResult {
|
||||
selected: RoutingExplanation | null;
|
||||
candidates: RoutingExplanation[];
|
||||
}
|
||||
|
||||
function clamp(value: number, fallback = 0): number {
|
||||
return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : fallback;
|
||||
}
|
||||
|
||||
function quotaFactor(quota: ProviderQuotaStatus): number {
|
||||
switch (quota) {
|
||||
case "exhausted":
|
||||
return 0;
|
||||
case "approaching_limit":
|
||||
return 0.65;
|
||||
case "unavailable":
|
||||
return 0.9;
|
||||
case "unknown":
|
||||
return 1;
|
||||
case "healthy":
|
||||
return 1;
|
||||
default:
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
function latencyFactor(latencyMs?: number): number {
|
||||
if (!Number.isFinite(latencyMs) || latencyMs === undefined) return 1;
|
||||
return Math.max(0.4, 1 - Math.min(latencyMs, 30_000) / 50_000);
|
||||
}
|
||||
|
||||
export function scoreCandidate(candidate: RoutingCandidate): RoutingExplanation {
|
||||
const capability = clamp(candidate.capabilityScore);
|
||||
const allocation =
|
||||
candidate.allocation === "deny" ? 0 : candidate.allocation === "warn" ? 0.85 : 1;
|
||||
const health = clamp(candidate.healthScore, 0.5);
|
||||
const reliability = 1 - clamp(candidate.errorRate ?? 0);
|
||||
const latency = latencyFactor(candidate.latencyMs);
|
||||
const preference = clamp(candidate.modelPreference, 0.5);
|
||||
const cost = clamp(candidate.costPreference, 1);
|
||||
const quota = quotaFactor(candidate.quota);
|
||||
const circuit = candidate.circuit === "open" ? 0 : candidate.circuit === "half_open" ? 0.5 : 1;
|
||||
const score = Number(
|
||||
(
|
||||
capability *
|
||||
allocation *
|
||||
health *
|
||||
reliability *
|
||||
latency *
|
||||
preference *
|
||||
cost *
|
||||
quota *
|
||||
circuit
|
||||
).toFixed(6)
|
||||
);
|
||||
const reasons = [
|
||||
capability >= 0.8 ? "capability match" : "partial capability match",
|
||||
candidate.allocation === "allow"
|
||||
? "allocation permitted"
|
||||
: candidate.allocation === "warn"
|
||||
? "allocation permitted with warning"
|
||||
: "allocation denied",
|
||||
health >= 0.8 ? "provider healthy" : "provider health degraded",
|
||||
`circuit ${candidate.circuit}`,
|
||||
candidate.quota === "unknown"
|
||||
? "quota state unknown but not exhausted"
|
||||
: `quota ${candidate.quota}`,
|
||||
];
|
||||
if (candidate.latencyMs !== undefined)
|
||||
reasons.push(`latency ${Math.round(candidate.latencyMs)}ms`);
|
||||
if (candidate.errorRate !== undefined)
|
||||
reasons.push(`${Math.round(candidate.errorRate * 100)}% recent errors`);
|
||||
return {
|
||||
providerId: candidate.providerId,
|
||||
modelId: candidate.modelId,
|
||||
score,
|
||||
eligible:
|
||||
score > 0 &&
|
||||
candidate.allocation !== "deny" &&
|
||||
candidate.circuit !== "open" &&
|
||||
candidate.quota !== "exhausted",
|
||||
reasons,
|
||||
factors: {
|
||||
capability,
|
||||
allocation,
|
||||
health,
|
||||
reliability,
|
||||
latency,
|
||||
preference,
|
||||
cost,
|
||||
quota,
|
||||
circuit,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function rankCandidates(candidates: RoutingCandidate[]): RankedRoutingResult {
|
||||
const ranked = candidates.map(scoreCandidate).sort((a, b) => b.score - a.score);
|
||||
return { selected: ranked.find((candidate) => candidate.eligible) ?? null, candidates: ranked };
|
||||
}
|
||||
|
||||
export interface FailoverPolicy {
|
||||
maxProviderAttempts: number;
|
||||
allowCrossProviderFallback: boolean;
|
||||
retryRateLimited: boolean;
|
||||
retryTimeouts: boolean;
|
||||
}
|
||||
|
||||
export const DEFAULT_FAILOVER_POLICY: FailoverPolicy = {
|
||||
maxProviderAttempts: 3,
|
||||
allowCrossProviderFallback: true,
|
||||
retryRateLimited: true,
|
||||
retryTimeouts: true,
|
||||
};
|
||||
|
||||
export function shouldFailover(
|
||||
failure: ProviderFailure,
|
||||
policy = DEFAULT_FAILOVER_POLICY
|
||||
): boolean {
|
||||
if (!policy.allowCrossProviderFallback || !failure.retryable) return false;
|
||||
if (failure.type === "rate_limit") return policy.retryRateLimited;
|
||||
if (failure.type === "timeout") return policy.retryTimeouts;
|
||||
return failure.type === "network_error" || failure.type === "provider_5xx";
|
||||
}
|
||||
66
src/lib/usage/budgetGuard.ts
Normal file
66
src/lib/usage/budgetGuard.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
export type BudgetPeriod = "daily" | "weekly" | "monthly";
|
||||
export type BudgetDecision = "allow" | "warn" | "deny";
|
||||
|
||||
export interface InternalBudgetLimit {
|
||||
id: string;
|
||||
scope: "global" | "provider" | "model" | "pool";
|
||||
providerId?: string;
|
||||
modelId?: string;
|
||||
poolId?: string;
|
||||
period: BudgetPeriod;
|
||||
limitType: "currency" | "tokens" | "requests";
|
||||
limitValue: number;
|
||||
warningThreshold: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface BudgetUsage {
|
||||
currency: number;
|
||||
tokens: number;
|
||||
requests: number;
|
||||
}
|
||||
|
||||
export interface BudgetEvaluation {
|
||||
decision: BudgetDecision;
|
||||
limit?: InternalBudgetLimit;
|
||||
used: number;
|
||||
remaining?: number;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export function evaluateBudget(
|
||||
limit: InternalBudgetLimit | undefined,
|
||||
usage: BudgetUsage
|
||||
): BudgetEvaluation {
|
||||
if (!limit || !limit.enabled)
|
||||
return { decision: "allow", used: 0, reason: "No enabled internal budget applies." };
|
||||
const used = usage[limit.limitType];
|
||||
if (!Number.isFinite(used) || limit.limitValue <= 0) {
|
||||
return {
|
||||
decision: "deny",
|
||||
limit,
|
||||
used: 0,
|
||||
remaining: 0,
|
||||
reason: "Internal budget configuration is invalid.",
|
||||
};
|
||||
}
|
||||
const remaining = Math.max(0, limit.limitValue - used);
|
||||
if (used >= limit.limitValue)
|
||||
return { decision: "deny", limit, used, remaining, reason: "Internal budget exhausted." };
|
||||
if (used / limit.limitValue >= limit.warningThreshold) {
|
||||
return {
|
||||
decision: "warn",
|
||||
limit,
|
||||
used,
|
||||
remaining,
|
||||
reason: `Internal budget is ${Math.round((used / limit.limitValue) * 100)}% consumed.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
decision: "allow",
|
||||
limit,
|
||||
used,
|
||||
remaining,
|
||||
reason: "Internal budget permits the request.",
|
||||
};
|
||||
}
|
||||
52
src/lib/usage/modelPricingRegistry.ts
Normal file
52
src/lib/usage/modelPricingRegistry.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
export interface ModelPricing {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
inputPricePerMillionTokens?: number;
|
||||
outputPricePerMillionTokens?: number;
|
||||
source?: string;
|
||||
effectiveAt?: string;
|
||||
}
|
||||
|
||||
export class ModelPricingRegistry {
|
||||
private readonly entries = new Map<string, ModelPricing>();
|
||||
|
||||
private key(providerId: string, modelId: string): string {
|
||||
return `${providerId.trim().toLowerCase()}\0${modelId.trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
set(pricing: ModelPricing): void {
|
||||
this.entries.set(this.key(pricing.providerId, pricing.modelId), { ...pricing });
|
||||
}
|
||||
|
||||
get(providerId: string, modelId: string): ModelPricing | undefined {
|
||||
return this.entries.get(this.key(providerId, modelId));
|
||||
}
|
||||
|
||||
estimate(
|
||||
providerId: string,
|
||||
modelId: string,
|
||||
inputTokens = 0,
|
||||
outputTokens = 0
|
||||
): number | undefined {
|
||||
const pricing = this.get(providerId, modelId);
|
||||
if (!pricing) return undefined;
|
||||
const input =
|
||||
pricing.inputPricePerMillionTokens === undefined
|
||||
? 0
|
||||
: (inputTokens * pricing.inputPricePerMillionTokens) / 1_000_000;
|
||||
const output =
|
||||
pricing.outputPricePerMillionTokens === undefined
|
||||
? 0
|
||||
: (outputTokens * pricing.outputPricePerMillionTokens) / 1_000_000;
|
||||
if (
|
||||
pricing.inputPricePerMillionTokens === undefined &&
|
||||
pricing.outputPricePerMillionTokens === undefined
|
||||
)
|
||||
return undefined;
|
||||
return Number((input + output).toFixed(12));
|
||||
}
|
||||
|
||||
list(): ModelPricing[] {
|
||||
return [...this.entries.values()].map((entry) => ({ ...entry }));
|
||||
}
|
||||
}
|
||||
78
src/lib/usage/usageLedger.ts
Normal file
78
src/lib/usage/usageLedger.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import type { ModelPricingRegistry } from "./modelPricingRegistry.js";
|
||||
|
||||
export type UsageStatus = "success" | "failed" | "rate_limited" | "timeout" | "cancelled";
|
||||
|
||||
export interface UsageRecord {
|
||||
id: string;
|
||||
providerId: string;
|
||||
connectionId?: string;
|
||||
modelId: string;
|
||||
poolId?: string;
|
||||
allocation?: number;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
estimatedCostUsd?: number;
|
||||
latencyMs: number;
|
||||
status: UsageStatus;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface UsageRecordInput extends Omit<UsageRecord, "totalTokens" | "estimatedCostUsd"> {
|
||||
totalTokens?: number;
|
||||
estimatedCostUsd?: number;
|
||||
}
|
||||
|
||||
export function createUsageRecord(
|
||||
input: UsageRecordInput,
|
||||
pricing?: ModelPricingRegistry
|
||||
): UsageRecord {
|
||||
const inputTokens = Number.isFinite(input.inputTokens) ? input.inputTokens : undefined;
|
||||
const outputTokens = Number.isFinite(input.outputTokens) ? input.outputTokens : undefined;
|
||||
const totalTokens =
|
||||
input.totalTokens ??
|
||||
(inputTokens !== undefined || outputTokens !== undefined
|
||||
? (inputTokens ?? 0) + (outputTokens ?? 0)
|
||||
: undefined);
|
||||
const estimatedCostUsd =
|
||||
input.estimatedCostUsd ??
|
||||
(pricing && inputTokens !== undefined && outputTokens !== undefined
|
||||
? pricing.estimate(input.providerId, input.modelId, inputTokens, outputTokens)
|
||||
: undefined);
|
||||
return {
|
||||
...input,
|
||||
...(inputTokens !== undefined ? { inputTokens } : {}),
|
||||
...(outputTokens !== undefined ? { outputTokens } : {}),
|
||||
...(totalTokens !== undefined ? { totalTokens } : {}),
|
||||
...(estimatedCostUsd !== undefined ? { estimatedCostUsd } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeUsage(records: UsageRecord[]) {
|
||||
return records.reduce(
|
||||
(summary, record) => {
|
||||
summary.requests += 1;
|
||||
summary.successes += record.status === "success" ? 1 : 0;
|
||||
summary.failures += record.status === "success" ? 0 : 1;
|
||||
summary.inputTokens += record.inputTokens ?? 0;
|
||||
summary.outputTokens += record.outputTokens ?? 0;
|
||||
summary.totalTokens += record.totalTokens ?? 0;
|
||||
summary.estimatedCostUsd =
|
||||
summary.estimatedCostUsd === undefined || record.estimatedCostUsd === undefined
|
||||
? undefined
|
||||
: summary.estimatedCostUsd + record.estimatedCostUsd;
|
||||
summary.latencyMs += record.latencyMs;
|
||||
return summary;
|
||||
},
|
||||
{
|
||||
requests: 0,
|
||||
successes: 0,
|
||||
failures: 0,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
totalTokens: 0,
|
||||
estimatedCostUsd: 0 as number | undefined,
|
||||
latencyMs: 0,
|
||||
}
|
||||
);
|
||||
}
|
||||
77
tests/unit/adaptive-circuit-budget-ledger.test.ts
Normal file
77
tests/unit/adaptive-circuit-budget-ledger.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { observeCircuit, createAdaptiveCircuit } from "@/lib/resilience/adaptiveCircuit";
|
||||
import { evaluateBudget } from "@/lib/usage/budgetGuard";
|
||||
import { ModelPricingRegistry } from "@/lib/usage/modelPricingRegistry";
|
||||
import { createUsageRecord, summarizeUsage } from "@/lib/usage/usageLedger";
|
||||
|
||||
test("adaptive circuit opens, probes, and closes after recovery", () => {
|
||||
const now = new Date("2026-01-01T00:00:00.000Z");
|
||||
let circuit = createAdaptiveCircuit();
|
||||
circuit = observeCircuit(circuit, "failure", {
|
||||
now,
|
||||
failureThreshold: 2,
|
||||
cooldownMs: 1000,
|
||||
reason: "timeout",
|
||||
});
|
||||
circuit = observeCircuit(circuit, "failure", {
|
||||
now: new Date(now.getTime() + 10),
|
||||
failureThreshold: 2,
|
||||
cooldownMs: 1000,
|
||||
reason: "timeout",
|
||||
});
|
||||
assert.equal(circuit.state, "open");
|
||||
circuit = observeCircuit(circuit, "probe", { now: new Date(now.getTime() + 1011) });
|
||||
assert.equal(circuit.state, "half_open");
|
||||
circuit = observeCircuit(circuit, "success", { now: new Date(now.getTime() + 1002) });
|
||||
assert.equal(circuit.state, "closed");
|
||||
assert.equal(circuit.failureCount, 0);
|
||||
});
|
||||
|
||||
test("internal budget returns allow, warn, and deny without upstream quota claims", () => {
|
||||
const limit = {
|
||||
id: "b",
|
||||
scope: "global" as const,
|
||||
period: "daily" as const,
|
||||
limitType: "currency" as const,
|
||||
limitValue: 10,
|
||||
warningThreshold: 0.75,
|
||||
enabled: true,
|
||||
};
|
||||
assert.equal(evaluateBudget(limit, { currency: 2, tokens: 0, requests: 0 }).decision, "allow");
|
||||
assert.equal(evaluateBudget(limit, { currency: 8, tokens: 0, requests: 0 }).decision, "warn");
|
||||
assert.equal(evaluateBudget(limit, { currency: 10, tokens: 0, requests: 0 }).decision, "deny");
|
||||
});
|
||||
|
||||
test("unknown pricing remains unknown while configured pricing is estimated", () => {
|
||||
const registry = new ModelPricingRegistry();
|
||||
assert.equal(registry.estimate("codex", "unknown", 1000, 1000), undefined);
|
||||
registry.set({
|
||||
providerId: "codex",
|
||||
modelId: "gpt-5",
|
||||
inputPricePerMillionTokens: 1,
|
||||
outputPricePerMillionTokens: 2,
|
||||
source: "admin",
|
||||
});
|
||||
assert.equal(registry.estimate("codex", "gpt-5", 1000, 1000), 0.003);
|
||||
const record = createUsageRecord(
|
||||
{
|
||||
id: "r1",
|
||||
providerId: "codex",
|
||||
modelId: "gpt-5",
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
latencyMs: 10,
|
||||
status: "success",
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
},
|
||||
registry
|
||||
);
|
||||
assert.equal(record.totalTokens, 150);
|
||||
assert.equal(record.estimatedCostUsd, 0.0002);
|
||||
const summary = summarizeUsage([record]);
|
||||
assert.equal(summary.requests, 1);
|
||||
assert.equal(summary.successes, 1);
|
||||
assert.equal(summary.estimatedCostUsd, 0.0002);
|
||||
});
|
||||
@@ -12,7 +12,16 @@ test("cliToken.mjs pode ser importado sem erro", async () => {
|
||||
assert.equal(mod.CLI_TOKEN_HEADER, "x-omniroute-cli-token");
|
||||
});
|
||||
|
||||
test("getCliToken deriva token de 32 chars sob o node puro que a CLI usa", async () => {
|
||||
test("getCliToken retorna string de 64 chars ou string vazia", async () => {
|
||||
const { getCliToken } = await import("../../bin/cli/utils/cliToken.mjs");
|
||||
const token = await getCliToken();
|
||||
assert.ok(typeof token === "string");
|
||||
// Pode ser "" se node-machine-id falhar, ou 64 chars se funcionar
|
||||
// (HMAC-SHA256 digest hex — see #10148 cliToken hardening).
|
||||
assert.ok(token === "" || token.length === 64, `expected 0 or 64 chars, got ${token.length}`);
|
||||
});
|
||||
|
||||
test("getCliToken deriva token de 64 chars sob o node puro que a CLI usa", async () => {
|
||||
const mod = await import("node-machine-id");
|
||||
const machineIdSync = mod.machineIdSync ?? mod.default?.machineIdSync;
|
||||
// Sem machine-id nesta plataforma não há token a derivar — nada a afirmar.
|
||||
@@ -33,7 +42,8 @@ test("getCliToken deriva token de 32 chars sob o node puro que a CLI usa", async
|
||||
{ cwd: repoRoot, encoding: "utf8" }
|
||||
);
|
||||
|
||||
assert.equal(out.trim(), "32", `expected a derived 32-char token, got length ${out.trim()}`);
|
||||
// HMAC-SHA256 digest hex = 64 chars (#10148 cliToken hardening).
|
||||
assert.equal(out.trim(), "64", `expected a derived 64-char token, got length ${out.trim()}`);
|
||||
});
|
||||
|
||||
test("getCliToken retorna mesmo valor em chamadas repetidas (cache)", async () => {
|
||||
@@ -58,7 +68,8 @@ test("getCliToken respeita rotação de OMNIROUTE_CLI_SALT", async () => {
|
||||
// docs/security/CLI_TOKEN.md promete que a rotação alcança os processos CLI;
|
||||
// o SALT hardcoded ignorava a env var e devolvia sempre o mesmo token.
|
||||
assert.notEqual(withRotatedSalt, withDefaultSalt);
|
||||
assert.equal(withRotatedSalt.length, 32);
|
||||
// HMAC-SHA256 digest hex = 64 chars (#10148 cliToken hardening).
|
||||
assert.equal(withRotatedSalt.length, 64);
|
||||
} finally {
|
||||
if (original === undefined) delete process.env.OMNIROUTE_CLI_SALT;
|
||||
else process.env.OMNIROUTE_CLI_SALT = original;
|
||||
@@ -69,7 +80,8 @@ test("getCliToken produz apenas hex lowercase se não-vazio", async () => {
|
||||
const { getCliToken } = await import("../../bin/cli/utils/cliToken.mjs");
|
||||
const token = await getCliToken();
|
||||
if (token.length > 0) {
|
||||
assert.match(token, /^[0-9a-f]{32}$/);
|
||||
// HMAC-SHA256 digest hex = 64 chars (#10148 cliToken hardening).
|
||||
assert.match(token, /^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -111,17 +123,17 @@ test("isLoopback rejeita IP público", async () => {
|
||||
|
||||
test("token derivado de machine-id diferente produz hash diferente", () => {
|
||||
const SALT = "omniroute-cli-auth-v1";
|
||||
// Mirror the production derivation (#10148): HMAC-SHA256(machineId, SALT) hex.
|
||||
const hash = (mid: string) =>
|
||||
crypto
|
||||
.createHash("sha256")
|
||||
.update(mid + SALT)
|
||||
.digest("hex")
|
||||
.substring(0, 32);
|
||||
.createHmac("sha256", mid)
|
||||
.update(SALT)
|
||||
.digest("hex");
|
||||
const t1 = hash("machine-id-host-A");
|
||||
const t2 = hash("machine-id-host-B");
|
||||
assert.notEqual(t1, t2);
|
||||
assert.match(t1, /^[0-9a-f]{32}$/);
|
||||
assert.match(t2, /^[0-9a-f]{32}$/);
|
||||
assert.match(t1, /^[0-9a-f]{64}$/);
|
||||
assert.match(t2, /^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("OMNIROUTE_DISABLE_CLI_TOKEN desabilita auth (estrutura verificada)", async () => {
|
||||
|
||||
149
tests/unit/quota-telemetry-adaptive-routing.test.ts
Normal file
149
tests/unit/quota-telemetry-adaptive-routing.test.ts
Normal file
@@ -0,0 +1,149 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
collectQuotaState,
|
||||
parseRateLimitHeaders,
|
||||
unknownQuotaState,
|
||||
} from "@/lib/quota/providerQuotaTelemetry";
|
||||
import { classifyProviderFailure } from "@/lib/resilience/failureClassification";
|
||||
import { rankCandidates, shouldFailover } from "@/lib/routing/adaptiveRouting";
|
||||
|
||||
test("unknown quota remains unknown and does not disable routing", async () => {
|
||||
const state = await collectQuotaState({ id: "c1", provider: "codex" }, []);
|
||||
assert.equal(state.supported, false);
|
||||
assert.equal(state.status, "unknown");
|
||||
assert.deepEqual(state.values, []);
|
||||
|
||||
const ranked = rankCandidates([
|
||||
{
|
||||
providerId: "codex",
|
||||
modelId: "gpt-5",
|
||||
capabilityScore: 1,
|
||||
allocation: "allow",
|
||||
healthScore: 1,
|
||||
circuit: "closed",
|
||||
quota: state.status,
|
||||
},
|
||||
]);
|
||||
assert.equal(ranked.selected?.providerId, "codex");
|
||||
});
|
||||
|
||||
test("quota source priority prefers authoritative values per dimension", async () => {
|
||||
const state = await collectQuotaState({ id: "c1", provider: "codex" }, [
|
||||
{
|
||||
kind: "estimated",
|
||||
supports: () => true,
|
||||
read: async () => [
|
||||
{ dimension: "requests", remaining: 1, source: "estimated", confidence: "low" },
|
||||
],
|
||||
},
|
||||
{
|
||||
kind: "provider_api",
|
||||
supports: () => true,
|
||||
read: async () => [
|
||||
{
|
||||
dimension: "requests",
|
||||
limit: 100,
|
||||
remaining: 80,
|
||||
source: "provider_api",
|
||||
confidence: "authoritative",
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(state.status, "healthy");
|
||||
assert.equal(state.values[0]?.source, "provider_api");
|
||||
assert.equal(state.values[0]?.remaining, 80);
|
||||
});
|
||||
|
||||
test("response headers normalize only through an explicit provider mapping", () => {
|
||||
const parsed = parseRateLimitHeaders(
|
||||
{ "x-limit": "100", "x-remaining": "12", "x-reset": "1700000000" },
|
||||
"provider",
|
||||
"connection",
|
||||
{ limit: "x-limit", remaining: "x-remaining", reset: "x-reset", dimension: "requests" }
|
||||
);
|
||||
assert.equal(parsed?.value.dimension, "requests");
|
||||
assert.equal(parsed?.value.remaining, 12);
|
||||
assert.equal(parsed?.snapshot.requestLimit, 100);
|
||||
assert.equal(parseRateLimitHeaders({}, "p", "c", { limit: "missing" }), null);
|
||||
});
|
||||
|
||||
test("exhausted quota and open circuits are excluded; approaching quota is penalized", () => {
|
||||
const result = rankCandidates([
|
||||
{
|
||||
providerId: "exhausted",
|
||||
modelId: "m",
|
||||
capabilityScore: 1,
|
||||
allocation: "allow",
|
||||
healthScore: 1,
|
||||
circuit: "closed",
|
||||
quota: "exhausted",
|
||||
},
|
||||
{
|
||||
providerId: "open",
|
||||
modelId: "m",
|
||||
capabilityScore: 1,
|
||||
allocation: "allow",
|
||||
healthScore: 1,
|
||||
circuit: "open",
|
||||
quota: "unknown",
|
||||
},
|
||||
{
|
||||
providerId: "approaching",
|
||||
modelId: "m",
|
||||
capabilityScore: 1,
|
||||
allocation: "allow",
|
||||
healthScore: 1,
|
||||
circuit: "closed",
|
||||
quota: "approaching_limit",
|
||||
},
|
||||
{
|
||||
providerId: "healthy",
|
||||
modelId: "m",
|
||||
capabilityScore: 1,
|
||||
allocation: "allow",
|
||||
healthScore: 1,
|
||||
circuit: "closed",
|
||||
quota: "unknown",
|
||||
},
|
||||
]);
|
||||
assert.equal(result.selected?.providerId, "healthy");
|
||||
assert.equal(
|
||||
result.candidates.find((candidate) => candidate.providerId === "exhausted")?.eligible,
|
||||
false
|
||||
);
|
||||
assert.ok(
|
||||
(result.candidates.find((candidate) => candidate.providerId === "approaching")?.score ?? 1) < 1
|
||||
);
|
||||
});
|
||||
|
||||
test("failure classification retries transient failures but not authentication errors", () => {
|
||||
const timeout = classifyProviderFailure({
|
||||
providerId: "codex",
|
||||
statusCode: 504,
|
||||
message: "upstream timeout",
|
||||
});
|
||||
const auth = classifyProviderFailure({
|
||||
providerId: "codex",
|
||||
statusCode: 401,
|
||||
message: "invalid token",
|
||||
});
|
||||
assert.equal(timeout.type, "timeout");
|
||||
assert.equal(shouldFailover(timeout), true);
|
||||
assert.equal(auth.type, "authentication_error");
|
||||
assert.equal(shouldFailover(auth), false);
|
||||
});
|
||||
|
||||
test("unknown state helper is explicit", () => {
|
||||
const state = unknownQuotaState("p", "c", "2026-01-01T00:00:00.000Z");
|
||||
assert.deepEqual(state, {
|
||||
providerId: "p",
|
||||
connectionId: "c",
|
||||
supported: false,
|
||||
fetchedAt: "2026-01-01T00:00:00.000Z",
|
||||
values: [],
|
||||
status: "unknown",
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user