mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix: resolve MCP endpoint auth bypass (#1970) and finalize DB settings
This commit is contained in:
@@ -27,6 +27,7 @@
|
||||
|
||||
- **fix(db):** resolve legacy encryption fallback causing re-encryption loops (#1941)
|
||||
- **fix(auth):** fix Codex assistant final_answer response sanitization (#1965)
|
||||
- **fix(mcp):** reclassify MCP endpoints to ensure API key authentication works even when dashboard auth is enabled (#1970)
|
||||
- **fix(providers):** allow local OpenAI-compatible endpoints (like Ollama) to be added without an API key (fixes #1893)
|
||||
- **fix(providers):** bypass AgentRouter unauthorized_client_error by spoofing Claude CLI headers via Anthropic endpoints (fixes #1921)
|
||||
- **fix(copilot):** emit compatible reasoning text deltas (#1919 — thanks @ivan-mezentsev)
|
||||
|
||||
@@ -121,10 +121,11 @@ export default function SystemStorageTab() {
|
||||
if (!dbSettings) return;
|
||||
setDbSettingsSaving(true);
|
||||
try {
|
||||
const { logs, backup, cache, retention, aggregation, optimization } = dbSettings;
|
||||
const res = await fetch("/api/settings/database", {
|
||||
method: "PUT",
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(dbSettings),
|
||||
body: JSON.stringify({ logs, backup, cache, retention, aggregation, optimization }),
|
||||
});
|
||||
if (res.ok) {
|
||||
await loadDatabaseSettings();
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getSettings, updateSettings } from "@/lib/localDb";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
import { z } from "zod";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { DatabaseSettings, DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
|
||||
import { getDatabaseStats } from "@/lib/db/stats";
|
||||
import { databaseSettingsSchema } from "@/shared/validation/settingsSchemas";
|
||||
import { getDatabaseSettings, updateDatabaseSettings } from "@/lib/db/databaseSettings";
|
||||
|
||||
type UserSettableKeys = keyof Omit<DatabaseSettings, "location" | "stats">;
|
||||
const databaseSettingsPatchSchema = databaseSettingsSchema.partial().strict();
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
@@ -14,33 +12,7 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = await getSettings();
|
||||
const dbStats = await getDatabaseStats();
|
||||
|
||||
// Get current settings from key_value table (user settings)
|
||||
const userSettings: Partial<DatabaseSettings> = {};
|
||||
const allKeys = await getAllUserSettableKeys();
|
||||
|
||||
for (const key of allKeys) {
|
||||
const value = settings[key as keyof typeof settings];
|
||||
if (value !== undefined) {
|
||||
(userSettings as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// Merge with defaults and stats
|
||||
const merged: DatabaseSettings = {
|
||||
location: dbStats.location,
|
||||
logs: { ...DEFAULT_DATABASE_SETTINGS.logs, ...userSettings.logs },
|
||||
backup: { ...DEFAULT_DATABASE_SETTINGS.backup, ...userSettings.backup },
|
||||
cache: { ...DEFAULT_DATABASE_SETTINGS.cache, ...userSettings.cache },
|
||||
retention: { ...DEFAULT_DATABASE_SETTINGS.retention, ...userSettings.retention },
|
||||
aggregation: { ...DEFAULT_DATABASE_SETTINGS.aggregation, ...userSettings.aggregation },
|
||||
optimization: { ...DEFAULT_DATABASE_SETTINGS.optimization, ...userSettings.optimization },
|
||||
stats: dbStats.stats,
|
||||
};
|
||||
|
||||
return NextResponse.json(merged);
|
||||
return NextResponse.json(getDatabaseSettings());
|
||||
} catch (error) {
|
||||
console.error("Error getting database settings:", error);
|
||||
return NextResponse.json({ error: "Failed to load database settings" }, { status: 500 });
|
||||
@@ -53,89 +25,23 @@ export async function PATCH(request: NextRequest) {
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = await getSettings();
|
||||
const userSettableKeys = await getUserSettableKeys();
|
||||
|
||||
const validation = await validateBody(
|
||||
request,
|
||||
z
|
||||
.record(z.string(), z.unknown())
|
||||
.partial()
|
||||
.refine(
|
||||
(data) => {
|
||||
// Ensure only user-settable fields are present
|
||||
return Object.keys(data).every((key) => userSettableKeys.includes(key));
|
||||
},
|
||||
{ message: "Attempting to set restricted fields" }
|
||||
)
|
||||
);
|
||||
const rawBody = await request.json();
|
||||
const validation = validateBody(databaseSettingsPatchSchema, rawBody);
|
||||
|
||||
if (isValidationFailure(validation)) {
|
||||
return validation;
|
||||
return NextResponse.json({ error: validation.error }, { status: 400 });
|
||||
}
|
||||
|
||||
const updates = validation.data;
|
||||
const updatedSettings = { ...settings };
|
||||
|
||||
// Update only user-settable fields
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
if (userSettableKeys.includes(key)) {
|
||||
(updatedSettings as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
await updateSettings(updatedSettings);
|
||||
updateDatabaseSettings(validation.data);
|
||||
|
||||
// Return merged settings (GET response pattern)
|
||||
return await GET(request);
|
||||
return NextResponse.json(getDatabaseSettings());
|
||||
} catch (error) {
|
||||
console.error("Error updating database settings:", error);
|
||||
return NextResponse.json({ error: "Failed to update database settings" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
async function getUserSettableKeys(): Promise<UserSettableKeys[]> {
|
||||
// These are the fields that users are allowed to modify
|
||||
const allowedKeys: UserSettableKeys[] = [
|
||||
// Logs
|
||||
"logs",
|
||||
|
||||
// Backup
|
||||
"backup",
|
||||
|
||||
// Cache
|
||||
"cache",
|
||||
|
||||
// Retention
|
||||
"retention",
|
||||
|
||||
// Aggregation
|
||||
"aggregation",
|
||||
|
||||
// Optimization
|
||||
"optimization",
|
||||
];
|
||||
|
||||
return allowedKeys;
|
||||
}
|
||||
|
||||
async function getAllUserSettableKeys(): Promise<string[]> {
|
||||
const settableKeys: string[] = [];
|
||||
const userSettableSections = await getUserSettableKeys();
|
||||
|
||||
// Get all nested keys under each user-settable section
|
||||
const allDefaultKeys = Object.keys(DEFAULT_DATABASE_SETTINGS) as (keyof DatabaseSettings)[];
|
||||
|
||||
for (const section of userSettableSections) {
|
||||
if (section in DEFAULT_DATABASE_SETTINGS) {
|
||||
const sectionKeys = Object.keys(
|
||||
DEFAULT_DATABASE_SETTINGS[section as keyof typeof DEFAULT_DATABASE_SETTINGS]
|
||||
);
|
||||
sectionKeys.forEach((key) => {
|
||||
settableKeys.push(`${section}.${key}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return settableKeys;
|
||||
export async function PUT(request: NextRequest) {
|
||||
return PATCH(request);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getSettings, updateSettings } from "@/lib/localDb";
|
||||
import { getRuntimePorts } from "@/lib/runtime/ports";
|
||||
import { databaseSettingsSchema } from "@/shared/validation/settingsSchemas";
|
||||
import { updateSettingsSchema } from "@/shared/validation/settingsSchemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { z } from "zod";
|
||||
import { getConsistentMachineId } from "@/shared/utils/machineId";
|
||||
import { validateProxyUrl, upsertUpstreamProxyConfig } from "@/lib/db/upstreamProxy";
|
||||
import {
|
||||
|
||||
@@ -5,35 +5,15 @@
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "./core";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
import type { DatabaseSettings } from "@/types/databaseSettings";
|
||||
import { getUserDatabaseSettings } from "./databaseSettings";
|
||||
|
||||
interface CleanupResult {
|
||||
deleted: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract database settings from the full settings object with proper typing.
|
||||
*/
|
||||
function getDatabaseSettings(): DatabaseSettings["retention"] {
|
||||
const settings = getSettings();
|
||||
// Database settings are stored under the 'databaseSettings' key in the main settings
|
||||
const dbSettings = (settings as Record<string, unknown>).databaseSettings as
|
||||
| DatabaseSettings
|
||||
| undefined;
|
||||
return (
|
||||
dbSettings?.retention ?? {
|
||||
quotaSnapshots: 30,
|
||||
compressionAnalytics: 30,
|
||||
mcpAudit: 30,
|
||||
a2aEvents: 30,
|
||||
callLogs: 7,
|
||||
usageHistory: 90,
|
||||
memoryEntries: 90,
|
||||
autoCleanupEnabled: false,
|
||||
}
|
||||
);
|
||||
function getRetentionSettings() {
|
||||
return getUserDatabaseSettings().retention;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,7 +21,7 @@ function getDatabaseSettings(): DatabaseSettings["retention"] {
|
||||
*/
|
||||
export async function cleanupQuotaSnapshots(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.quotaSnapshots;
|
||||
const cutoffDate = new Date();
|
||||
@@ -71,7 +51,7 @@ export async function cleanupQuotaSnapshots(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupCallLogs(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.callLogs;
|
||||
const cutoffDate = new Date();
|
||||
@@ -99,7 +79,7 @@ export async function cleanupCallLogs(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupUsageHistory(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.usageHistory;
|
||||
const cutoffDate = new Date();
|
||||
@@ -129,7 +109,7 @@ export async function cleanupUsageHistory(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupCompressionAnalytics(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.compressionAnalytics;
|
||||
const cutoffDate = new Date();
|
||||
@@ -159,7 +139,7 @@ export async function cleanupCompressionAnalytics(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupMcpAudit(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.mcpAudit;
|
||||
const cutoffDate = new Date();
|
||||
@@ -189,7 +169,7 @@ export async function cleanupMcpAudit(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupA2aEvents(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.a2aEvents;
|
||||
const cutoffDate = new Date();
|
||||
@@ -217,7 +197,7 @@ export async function cleanupA2aEvents(): Promise<CleanupResult> {
|
||||
*/
|
||||
export async function cleanupMemoryEntries(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
|
||||
const retentionDays = retention.memoryEntries;
|
||||
const cutoffDate = new Date();
|
||||
@@ -250,7 +230,7 @@ export async function runAutoCleanup(): Promise<{
|
||||
totalErrors: number;
|
||||
results: Record<string, CleanupResult>;
|
||||
}> {
|
||||
const retention = getDatabaseSettings();
|
||||
const retention = getRetentionSettings();
|
||||
const autoCleanupEnabled = retention.autoCleanupEnabled;
|
||||
|
||||
if (!autoCleanupEnabled) {
|
||||
@@ -321,20 +301,20 @@ export async function purgeCallLogs(): Promise<CleanupResult> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge ALL detailed_logs immediately (no retention check).
|
||||
* Purge ALL request_detail_logs immediately (no retention check).
|
||||
*/
|
||||
export async function purgeDetailedLogs(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM detailed_logs");
|
||||
const stmt = db.prepare("DELETE FROM request_detail_logs");
|
||||
const runResult = stmt.run();
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Purged ${result.deleted} detailed_logs`);
|
||||
console.log(`[Cleanup] Purged ${result.deleted} request_detail_logs`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error purging detailed_logs:", err);
|
||||
console.error("[Cleanup] Error purging request_detail_logs:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
|
||||
254
src/lib/db/databaseSettings.ts
Normal file
254
src/lib/db/databaseSettings.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
import fs from "node:fs";
|
||||
|
||||
import { DEFAULT_DATABASE_SETTINGS, type DatabaseSettings } from "@/types/databaseSettings";
|
||||
|
||||
import { backupDbFile } from "./backup";
|
||||
import { DATA_DIR, SQLITE_FILE, getDbInstance } from "./core";
|
||||
import { invalidateDbCache } from "./readCache";
|
||||
import { getDatabaseStats } from "./stats";
|
||||
|
||||
const DATABASE_SETTINGS_NAMESPACE = "databaseSettings";
|
||||
|
||||
export type UserDatabaseSettings = Omit<DatabaseSettings, "location" | "stats">;
|
||||
type DatabaseSettingsSection = keyof UserDatabaseSettings;
|
||||
|
||||
const DATABASE_SETTINGS_SECTIONS = Object.keys(
|
||||
DEFAULT_DATABASE_SETTINGS
|
||||
) as DatabaseSettingsSection[];
|
||||
|
||||
const LEGACY_FLAT_KEYS: {
|
||||
[TSection in DatabaseSettingsSection]: Partial<
|
||||
Record<keyof UserDatabaseSettings[TSection] & string, string[]>
|
||||
>;
|
||||
} = {
|
||||
logs: {
|
||||
detailedLogsEnabled: ["detailedLogsEnabled"],
|
||||
callLogPipelineEnabled: ["callLogPipelineEnabled"],
|
||||
maxDetailSizeKb: ["maxDetailSizeKb"],
|
||||
ringBufferSize: ["ringBufferSize"],
|
||||
},
|
||||
backup: {
|
||||
autoBackupEnabled: ["autoBackupEnabled"],
|
||||
autoBackupFrequency: ["autoBackupFrequency"],
|
||||
keepLastNBackups: ["keepLastNBackups"],
|
||||
},
|
||||
cache: {
|
||||
semanticCacheEnabled: ["semanticCacheEnabled"],
|
||||
semanticCacheMaxSize: ["semanticCacheMaxSize"],
|
||||
semanticCacheTTL: ["semanticCacheTTL"],
|
||||
promptCacheEnabled: ["promptCacheEnabled"],
|
||||
promptCacheStrategy: ["promptCacheStrategy"],
|
||||
alwaysPreserveClientCache: ["alwaysPreserveClientCache"],
|
||||
},
|
||||
retention: {
|
||||
quotaSnapshots: ["quotaSnapshots"],
|
||||
compressionAnalytics: ["compressionAnalytics"],
|
||||
mcpAudit: ["mcpAudit"],
|
||||
a2aEvents: ["a2aEvents"],
|
||||
callLogs: ["callLogs"],
|
||||
usageHistory: ["usageHistory"],
|
||||
memoryEntries: ["memoryEntries"],
|
||||
autoCleanupEnabled: ["autoCleanupEnabled"],
|
||||
},
|
||||
aggregation: {
|
||||
enabled: ["aggregationEnabled", "enabled"],
|
||||
rawDataRetentionDays: ["rawDataRetentionDays"],
|
||||
granularity: ["granularity"],
|
||||
},
|
||||
optimization: {
|
||||
autoVacuumMode: ["autoVacuumMode"],
|
||||
scheduledVacuum: ["scheduledVacuum"],
|
||||
vacuumHour: ["vacuumHour"],
|
||||
pageSize: ["pageSize"],
|
||||
cacheSize: ["cacheSize"],
|
||||
optimizeOnStartup: ["optimizeOnStartup"],
|
||||
},
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function cloneDefaultSettings(): UserDatabaseSettings {
|
||||
return structuredClone(DEFAULT_DATABASE_SETTINGS) as UserDatabaseSettings;
|
||||
}
|
||||
|
||||
function parseStoredValue(rawValue: unknown): unknown {
|
||||
if (typeof rawValue !== "string") return rawValue;
|
||||
|
||||
try {
|
||||
return JSON.parse(rawValue);
|
||||
} catch {
|
||||
return rawValue;
|
||||
}
|
||||
}
|
||||
|
||||
function readNamespace(namespace: string): Record<string, unknown> {
|
||||
const db = getDbInstance();
|
||||
const rows = db
|
||||
.prepare("SELECT key, value FROM key_value WHERE namespace = ?")
|
||||
.all(namespace) as Array<{ key: string; value: string }>;
|
||||
|
||||
const values: Record<string, unknown> = {};
|
||||
for (const row of rows) {
|
||||
values[row.key] = parseStoredValue(row.value);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function mergeSectionObject(
|
||||
target: UserDatabaseSettings,
|
||||
section: DatabaseSettingsSection,
|
||||
value: unknown
|
||||
) {
|
||||
if (!isRecord(value)) return;
|
||||
|
||||
const sectionTarget = target[section] as Record<string, unknown>;
|
||||
const defaultSection = DEFAULT_DATABASE_SETTINGS[section] as Record<string, unknown>;
|
||||
|
||||
for (const key of Object.keys(defaultSection)) {
|
||||
if (value[key] !== undefined) {
|
||||
sectionTarget[key] = value[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function mergeTopLevelSections(target: UserDatabaseSettings, values: Record<string, unknown>) {
|
||||
for (const section of DATABASE_SETTINGS_SECTIONS) {
|
||||
mergeSectionObject(target, section, values[section]);
|
||||
}
|
||||
}
|
||||
|
||||
function mergeDatabaseSettingsNamespace(
|
||||
target: UserDatabaseSettings,
|
||||
values: Record<string, unknown>
|
||||
) {
|
||||
for (const section of DATABASE_SETTINGS_SECTIONS) {
|
||||
const defaultSection = DEFAULT_DATABASE_SETTINGS[section] as Record<string, unknown>;
|
||||
const sectionTarget = target[section] as Record<string, unknown>;
|
||||
const flatAliases = LEGACY_FLAT_KEYS[section];
|
||||
|
||||
for (const key of Object.keys(defaultSection)) {
|
||||
for (const alias of flatAliases[key] ?? []) {
|
||||
if (values[alias] !== undefined) {
|
||||
sectionTarget[key] = values[alias];
|
||||
}
|
||||
}
|
||||
|
||||
const nestedKey = `${section}.${key}`;
|
||||
if (values[nestedKey] !== undefined) {
|
||||
sectionTarget[key] = values[nestedKey];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getWalSizeBytes(): number {
|
||||
if (!SQLITE_FILE) return 0;
|
||||
|
||||
try {
|
||||
const walPath = `${SQLITE_FILE}-wal`;
|
||||
return fs.existsSync(walPath) ? fs.statSync(walPath).size : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function getSchemaVersion(): number {
|
||||
const db = getDbInstance();
|
||||
|
||||
try {
|
||||
const row = db
|
||||
.prepare("SELECT MAX(CAST(version AS INTEGER)) AS version FROM _omniroute_migrations")
|
||||
.get() as { version: number | null } | undefined;
|
||||
return row?.version ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function getFreelistCount(): number {
|
||||
try {
|
||||
return getDbInstance().pragma("freelist_count", { simple: true }) as number;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function getIntegrityCheck(): "ok" | "error" | null {
|
||||
try {
|
||||
const result = getDbInstance().pragma("quick_check", { simple: true }) as string;
|
||||
return result === "ok" ? "ok" : "error";
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getUserDatabaseSettings(): UserDatabaseSettings {
|
||||
const settings = cloneDefaultSettings();
|
||||
const mainSettings = readNamespace("settings");
|
||||
const databaseSettingsValue = mainSettings[DATABASE_SETTINGS_NAMESPACE];
|
||||
|
||||
if (isRecord(databaseSettingsValue)) {
|
||||
mergeTopLevelSections(settings, databaseSettingsValue);
|
||||
}
|
||||
|
||||
mergeTopLevelSections(settings, mainSettings);
|
||||
mergeDatabaseSettingsNamespace(settings, readNamespace(DATABASE_SETTINGS_NAMESPACE));
|
||||
|
||||
return settings;
|
||||
}
|
||||
|
||||
export function getDatabaseSettings(): DatabaseSettings {
|
||||
const dbStats = getDatabaseStats();
|
||||
|
||||
return {
|
||||
...getUserDatabaseSettings(),
|
||||
location: {
|
||||
databasePath: SQLITE_FILE ?? ":memory:",
|
||||
dataDir: DATA_DIR,
|
||||
walSizeBytes: getWalSizeBytes(),
|
||||
schemaVersion: getSchemaVersion(),
|
||||
},
|
||||
stats: {
|
||||
databaseSizeBytes: dbStats.totalSize,
|
||||
pageCount: dbStats.pageCount,
|
||||
freelistCount: getFreelistCount(),
|
||||
lastVacuumAt: null,
|
||||
lastOptimizationAt: null,
|
||||
integrityCheck: getIntegrityCheck(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function updateDatabaseSettings(
|
||||
updates: Partial<UserDatabaseSettings>
|
||||
): UserDatabaseSettings {
|
||||
const nextSettings = getUserDatabaseSettings();
|
||||
|
||||
for (const section of DATABASE_SETTINGS_SECTIONS) {
|
||||
if (updates[section] !== undefined) {
|
||||
mergeSectionObject(nextSettings, section, updates[section]);
|
||||
}
|
||||
}
|
||||
|
||||
const db = getDbInstance();
|
||||
const insert = db.prepare(
|
||||
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)"
|
||||
);
|
||||
const tx = db.transaction(() => {
|
||||
for (const section of DATABASE_SETTINGS_SECTIONS) {
|
||||
const sectionValues = nextSettings[section] as Record<string, unknown>;
|
||||
|
||||
for (const [key, value] of Object.entries(sectionValues)) {
|
||||
insert.run(DATABASE_SETTINGS_NAMESPACE, `${section}.${key}`, JSON.stringify(value));
|
||||
}
|
||||
}
|
||||
});
|
||||
tx();
|
||||
|
||||
backupDbFile("pre-write");
|
||||
invalidateDbCache("settings");
|
||||
|
||||
return nextSettings;
|
||||
}
|
||||
@@ -288,7 +288,7 @@ export function migrateLegacyEncryptedString(ciphertext: string | null | undefin
|
||||
const parts = rawPayload.split(":");
|
||||
if (parts.length !== 3) return { updated: false, value: ciphertext };
|
||||
|
||||
const [ivHex, authTagHex, encryptedHex] = parts;
|
||||
const [ivHex, encryptedHex, authTagHex] = parts;
|
||||
const iv = Buffer.from(ivHex, "hex");
|
||||
const authTag = Buffer.from(authTagHex, "hex");
|
||||
const encrypted = Buffer.from(encryptedHex, "hex");
|
||||
|
||||
@@ -146,6 +146,14 @@ export {
|
||||
|
||||
export type { PricingSource, PricingSourceMap } from "./db/settings";
|
||||
|
||||
export {
|
||||
getDatabaseSettings,
|
||||
getUserDatabaseSettings,
|
||||
updateDatabaseSettings,
|
||||
} from "./db/databaseSettings";
|
||||
|
||||
export type { UserDatabaseSettings } from "./db/databaseSettings";
|
||||
|
||||
export {
|
||||
// Proxy Registry
|
||||
listProxies,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "../db/core";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
import { getUserDatabaseSettings } from "../db/databaseSettings";
|
||||
|
||||
interface AggregationResult {
|
||||
processed: number;
|
||||
@@ -27,10 +27,6 @@ export async function rollupDailyUsage(
|
||||
toDate: string
|
||||
): Promise<AggregationResult> {
|
||||
const db = getDbInstance();
|
||||
const settings = await getSettings();
|
||||
|
||||
// Get retention settings
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
|
||||
const result: AggregationResult = {
|
||||
processed: 0,
|
||||
@@ -54,10 +50,10 @@ export async function rollupDailyUsage(
|
||||
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
|
||||
GROUP BY provider, model, DATE(created_at)
|
||||
ON CONFLICT(provider, model, date) DO UPDATE SET
|
||||
total_requests = total_requests + excluded.total_requests,
|
||||
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
|
||||
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
|
||||
total_cost = total_cost + excluded.total_cost
|
||||
total_requests = excluded.total_requests,
|
||||
total_input_tokens = excluded.total_input_tokens,
|
||||
total_output_tokens = excluded.total_output_tokens,
|
||||
total_cost = excluded.total_cost
|
||||
`;
|
||||
|
||||
const stmt = db.prepare(aggregateQuery);
|
||||
@@ -88,10 +84,6 @@ export async function rollupHourlyQuota(
|
||||
toDate: string
|
||||
): Promise<AggregationResult> {
|
||||
const db = getDbInstance();
|
||||
const settings = await getSettings();
|
||||
|
||||
// Get retention settings
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
|
||||
const result: AggregationResult = {
|
||||
processed: 0,
|
||||
@@ -115,10 +107,10 @@ export async function rollupHourlyQuota(
|
||||
WHERE created_at >= ? AND created_at <= ?
|
||||
GROUP BY provider, model, datetime(strftime('%Y-%m-%d %H:00:00', created_at))
|
||||
ON CONFLICT(provider, model, date_hour) DO UPDATE SET
|
||||
total_requests = total_requests + excluded.total_requests,
|
||||
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
|
||||
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
|
||||
total_cost = total_cost + excluded.total_cost
|
||||
total_requests = excluded.total_requests,
|
||||
total_input_tokens = excluded.total_input_tokens,
|
||||
total_output_tokens = excluded.total_output_tokens,
|
||||
total_cost = excluded.total_cost
|
||||
`;
|
||||
|
||||
const stmt = db.prepare(aggregateQuery);
|
||||
@@ -145,8 +137,7 @@ export async function rollupHourlyQuota(
|
||||
* @returns ISO date string (YYYY-MM-DD)
|
||||
*/
|
||||
export async function getRawDataCutoffDate(): Promise<string> {
|
||||
const settings = await getSettings();
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
const rawDataRetentionDays = getUserDatabaseSettings().aggregation.rawDataRetentionDays;
|
||||
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - rawDataRetentionDays);
|
||||
@@ -158,6 +149,5 @@ export async function getRawDataCutoffDate(): Promise<string> {
|
||||
* Check if aggregation is enabled in settings.
|
||||
*/
|
||||
export async function isAggregationEnabled(): Promise<boolean> {
|
||||
const settings = await getSettings();
|
||||
return (settings.aggregation as any)?.enabled ?? false;
|
||||
return getUserDatabaseSettings().aggregation.enabled;
|
||||
}
|
||||
|
||||
@@ -61,10 +61,16 @@ export function classifyRoute(rawPath: string, method: string = "GET"): RouteCla
|
||||
};
|
||||
}
|
||||
|
||||
if (normalizedPath === "/api/v1" || normalizedPath.startsWith("/api/v1/")) {
|
||||
if (
|
||||
normalizedPath === "/api/v1" ||
|
||||
normalizedPath.startsWith("/api/v1/") ||
|
||||
normalizedPath.startsWith("/api/mcp/")
|
||||
) {
|
||||
return {
|
||||
routeClass: "CLIENT_API",
|
||||
reason: aliasReason ?? "client_api_v1",
|
||||
reason:
|
||||
aliasReason ??
|
||||
(normalizedPath.startsWith("/api/mcp/") ? "client_api_mcp" : "client_api_v1"),
|
||||
normalizedPath,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -163,9 +163,10 @@ export const databaseSettingsSchema = z.object(
|
||||
.or(z.literal("daily"))
|
||||
.or(z.literal("weekly"))
|
||||
.or(z.literal("monthly")),
|
||||
pageSize: z.number().multipleOf(512).min(512).max(16384),
|
||||
cacheSize: z.number().int().min(1000).max(1000000),
|
||||
mmapSize: z.number().int().min(0),
|
||||
vacuumHour: z.number().int().min(0).max(23),
|
||||
pageSize: z.number().multipleOf(512).min(512).max(65536),
|
||||
cacheSize: z.number().int().min(-1000000).max(1000000),
|
||||
optimizeOnStartup: z.boolean(),
|
||||
}),
|
||||
|
||||
// Skip location and stats as they're read-only
|
||||
|
||||
170
tests/unit/database-settings-maintenance.test.ts
Normal file
170
tests/unit/database-settings-maintenance.test.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-db-settings-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const databaseSettings = await import("../../src/lib/db/databaseSettings.ts");
|
||||
const databaseSettingsRoute = await import("../../src/app/api/settings/database/route.ts");
|
||||
const cleanup = await import("../../src/lib/db/cleanup.ts");
|
||||
const aggregateHistory = await import("../../src/lib/usage/aggregateHistory.ts");
|
||||
|
||||
type CountRow = {
|
||||
count: number;
|
||||
};
|
||||
|
||||
type UsageSummaryRow = {
|
||||
total_requests: number;
|
||||
total_input_tokens: number;
|
||||
total_output_tokens: number;
|
||||
total_cost: number;
|
||||
};
|
||||
|
||||
function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
function makeJsonRequest(method: string, body?: unknown): Request {
|
||||
return new Request("http://localhost/api/settings/database", {
|
||||
method,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
resetStorage();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
resetStorage();
|
||||
});
|
||||
|
||||
test("database settings route returns mapped stats and persists editable sections", async () => {
|
||||
const current = databaseSettings.getUserDatabaseSettings();
|
||||
const response = await databaseSettingsRoute.PATCH(
|
||||
makeJsonRequest("PATCH", {
|
||||
retention: { ...current.retention, callLogs: 12, autoCleanupEnabled: false },
|
||||
aggregation: { ...current.aggregation, enabled: false, rawDataRetentionDays: 8 },
|
||||
}) as never
|
||||
);
|
||||
const body = await response.json();
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(body.retention.callLogs, 12);
|
||||
assert.equal(body.aggregation.rawDataRetentionDays, 8);
|
||||
assert.equal(typeof body.location.databasePath, "string");
|
||||
assert.equal(typeof body.location.dataDir, "string");
|
||||
assert.equal(typeof body.location.walSizeBytes, "number");
|
||||
assert.equal(typeof body.stats.databaseSizeBytes, "number");
|
||||
assert.equal(typeof body.stats.pageCount, "number");
|
||||
assert.equal(typeof body.stats.freelistCount, "number");
|
||||
|
||||
const db = core.getDbInstance();
|
||||
const stored = db
|
||||
.prepare(
|
||||
"SELECT value FROM key_value WHERE namespace = 'databaseSettings' AND key = 'retention.callLogs'"
|
||||
)
|
||||
.get() as { value: string } | undefined;
|
||||
assert.equal(JSON.parse(stored?.value ?? "null"), 12);
|
||||
|
||||
const getResponse = await databaseSettingsRoute.GET(makeJsonRequest("GET") as never);
|
||||
const getBody = await getResponse.json();
|
||||
|
||||
assert.equal(getResponse.status, 200);
|
||||
assert.equal(getBody.retention.callLogs, 12);
|
||||
assert.equal(getBody.aggregation.rawDataRetentionDays, 8);
|
||||
});
|
||||
|
||||
test("database settings reader supports legacy flat keys and lets nested saves win", () => {
|
||||
const db = core.getDbInstance();
|
||||
db.prepare(
|
||||
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('databaseSettings', ?, ?)"
|
||||
).run("callLogs", JSON.stringify(99));
|
||||
|
||||
assert.equal(databaseSettings.getUserDatabaseSettings().retention.callLogs, 99);
|
||||
|
||||
databaseSettings.updateDatabaseSettings({
|
||||
retention: {
|
||||
...databaseSettings.getUserDatabaseSettings().retention,
|
||||
callLogs: 7,
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(databaseSettings.getUserDatabaseSettings().retention.callLogs, 7);
|
||||
});
|
||||
|
||||
test("purgeDetailedLogs deletes request_detail_logs", async () => {
|
||||
const db = core.getDbInstance();
|
||||
db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run(
|
||||
"detail-1",
|
||||
new Date().toISOString(),
|
||||
10
|
||||
);
|
||||
db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run(
|
||||
"detail-2",
|
||||
new Date().toISOString(),
|
||||
20
|
||||
);
|
||||
|
||||
const result = await cleanup.purgeDetailedLogs();
|
||||
|
||||
assert.equal(result.errors, 0);
|
||||
assert.equal(result.deleted, 2);
|
||||
assert.equal(
|
||||
(db.prepare("SELECT COUNT(*) AS count FROM request_detail_logs").get() as CountRow).count,
|
||||
0
|
||||
);
|
||||
});
|
||||
|
||||
test("usage aggregation upserts replace recomputed totals instead of adding them twice", async () => {
|
||||
const db = core.getDbInstance();
|
||||
const insertSnapshot = db.prepare(
|
||||
`INSERT INTO quota_snapshots
|
||||
(provider, connection_id, window_key, remaining_percentage, is_exhausted, raw_data, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
);
|
||||
|
||||
insertSnapshot.run(
|
||||
"openai",
|
||||
"conn-1",
|
||||
"daily",
|
||||
90,
|
||||
0,
|
||||
JSON.stringify({ model: "gpt-test", input_tokens: 10, output_tokens: 4, cost: 0.25 }),
|
||||
"2026-05-01 10:15:00"
|
||||
);
|
||||
insertSnapshot.run(
|
||||
"openai",
|
||||
"conn-1",
|
||||
"daily",
|
||||
80,
|
||||
0,
|
||||
JSON.stringify({ model: "gpt-test", input_tokens: 20, output_tokens: 6, cost: 0.5 }),
|
||||
"2026-05-01 10:45:00"
|
||||
);
|
||||
|
||||
await aggregateHistory.rollupDailyUsage("2026-05-01", "2026-05-01");
|
||||
await aggregateHistory.rollupDailyUsage("2026-05-01", "2026-05-01");
|
||||
await aggregateHistory.rollupHourlyQuota("2026-05-01 10:00:00", "2026-05-01 10:59:59");
|
||||
await aggregateHistory.rollupHourlyQuota("2026-05-01 10:00:00", "2026-05-01 10:59:59");
|
||||
|
||||
const daily = db.prepare("SELECT * FROM daily_usage_summary").get() as UsageSummaryRow;
|
||||
const hourly = db.prepare("SELECT * FROM hourly_usage_summary").get() as UsageSummaryRow;
|
||||
|
||||
assert.equal(daily.total_requests, 2);
|
||||
assert.equal(daily.total_input_tokens, 30);
|
||||
assert.equal(daily.total_output_tokens, 10);
|
||||
assert.equal(daily.total_cost, 0.75);
|
||||
assert.equal(hourly.total_requests, 2);
|
||||
assert.equal(hourly.total_input_tokens, 30);
|
||||
assert.equal(hourly.total_output_tokens, 10);
|
||||
assert.equal(hourly.total_cost, 0.75);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createCipheriv, createHash, randomBytes, scryptSync } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
@@ -10,6 +11,16 @@ async function importFresh(modulePath) {
|
||||
return import(`${url}?test=${Date.now()}-${Math.random().toString(16).slice(2)}`);
|
||||
}
|
||||
|
||||
function encryptWithLegacyDynamicSalt(secret: string, plaintext: string): string {
|
||||
const key = scryptSync(secret, createHash("sha256").update(secret).digest().slice(0, 16), 32);
|
||||
const iv = randomBytes(16);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
let encrypted = cipher.update(plaintext, "utf8", "hex");
|
||||
encrypted += cipher.final("hex");
|
||||
const authTag = cipher.getAuthTag().toString("hex");
|
||||
return `enc:v1:${iv.toString("hex")}:${encrypted}:${authTag}`;
|
||||
}
|
||||
|
||||
test.after(() => {
|
||||
if (ORIGINAL_STORAGE_KEY === undefined) {
|
||||
delete process.env.STORAGE_ENCRYPTION_KEY;
|
||||
@@ -79,3 +90,20 @@ test("decrypt returns null when the value is malformed or the key is wrong", asy
|
||||
assert.equal(secondModule.decrypt(encrypted), null);
|
||||
assert.equal(secondModule.decrypt("enc:v1:not-valid"), null);
|
||||
});
|
||||
|
||||
test("legacy encryption migration parses ciphertext in canonical payload order", async () => {
|
||||
process.env.STORAGE_ENCRYPTION_KEY = "task-304-legacy-secret";
|
||||
const encryption = await importFresh("src/lib/db/encryption.ts");
|
||||
const legacyCiphertext = encryptWithLegacyDynamicSalt(
|
||||
process.env.STORAGE_ENCRYPTION_KEY,
|
||||
"legacy-provider-token"
|
||||
);
|
||||
|
||||
assert.equal(encryption.decrypt(legacyCiphertext), null);
|
||||
|
||||
const migrated = encryption.migrateLegacyEncryptedString(legacyCiphertext);
|
||||
|
||||
assert.equal(migrated.updated, true);
|
||||
assert.match(migrated.value, /^enc:v1:/);
|
||||
assert.equal(encryption.decrypt(migrated.value), "legacy-provider-token");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user