diff --git a/package.json b/package.json index 12d4fc61f0..694d59cf03 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "audit:electron": "npm --prefix electron audit --audit-level=moderate", "typecheck:core": "tsc --pretty false -p tsconfig.typecheck-core.json", "typecheck:noimplicit:core": "tsc --pretty false -p tsconfig.typecheck-noimplicit-core.json", + "backfill-aggregation": "node --import tsx/esm src/scripts/backfillAggregation.ts", "env:sync": "node scripts/sync-env.mjs", "test:integration": "node --import tsx/esm --test tests/integration/*.test.ts", "test:e2e": "node scripts/run-playwright-tests.mjs test tests/e2e/*.spec.ts", diff --git a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx deleted file mode 100644 index 633592a462..0000000000 --- a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx +++ /dev/null @@ -1,191 +0,0 @@ -"use client"; - -import { useState, useEffect } from "react"; -import { Card, Button } from "@/shared/components"; -import { useTranslations } from "next-intl"; - -interface CacheConfig { - semanticCacheEnabled: boolean; - semanticCacheMaxSize: number; - semanticCacheTTL: number; - promptCacheEnabled: boolean; - promptCacheStrategy: "auto" | "system-only" | "manual"; - alwaysPreserveClientCache: "auto" | "always" | "never"; -} - -export default function CacheSettingsTab() { - const t = useTranslations("settings"); - const [config, setConfig] = useState({ - semanticCacheEnabled: true, - semanticCacheMaxSize: 100, - semanticCacheTTL: 1800000, - promptCacheEnabled: true, - promptCacheStrategy: "auto", - alwaysPreserveClientCache: "auto", - }); - const [saving, setSaving] = useState(false); - const [loading, setLoading] = useState(true); - - useEffect(() => { - fetch("/api/settings/cache-config") - .then((r) => (r.ok ? r.json() : null)) - .then((data) => { - if (data) setConfig(data); - }) - .catch(() => {}) - .finally(() => setLoading(false)); - }, []); - - const handleSave = async () => { - setSaving(true); - try { - await fetch("/api/settings/cache-config", { - method: "PUT", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(config), - }); - } finally { - setSaving(false); - } - }; - - if (loading) { - return ( - -

{t("loading")}

- - ); - } - - return ( - -

- cached - {t("cacheSettings")} -

- -
- {/* Semantic Cache */} -
-

{t("semanticCache")}

- - - - - - -
- - {/* Prompt Cache */} -
-

{t("promptCache")}

- - - - - - -
- - {/* Save */} -
- -
-
- - ); -} diff --git a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx index a52ec58a0d..1adc5c432f 100644 --- a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx @@ -24,6 +24,15 @@ export default function SystemStorageTab() { const [purgeLogsStatus, setPurgeLogsStatus] = useState({ type: "", message: "" }); const [cleanupBackupsLoading, setCleanupBackupsLoading] = useState(false); const [cleanupBackupsStatus, setCleanupBackupsStatus] = useState({ type: "", message: "" }); + const [purgeQuotaSnapshotsLoading, setPurgeQuotaSnapshotsLoading] = useState(false); + const [purgeQuotaSnapshotsStatus, setPurgeQuotaSnapshotsStatus] = useState({ + type: "", + message: "", + }); + const [purgeCallLogsLoading, setPurgeCallLogsLoading] = useState(false); + const [purgeCallLogsStatus, setPurgeCallLogsStatus] = useState({ type: "", message: "" }); + const [purgeDetailedLogsLoading, setPurgeDetailedLogsLoading] = useState(false); + const [purgeDetailedLogsStatus, setPurgeDetailedLogsStatus] = useState({ type: "", message: "" }); const fileInputRef = useRef(null); const jsonInputRef = useRef(null); const locale = useLocale(); @@ -53,6 +62,12 @@ export default function SystemStorageTab() { retentionDays: 0, }); + // Database settings state (tasks 23-26) + const [dbSettings, setDbSettings] = useState(null); + const [dbSettingsLoading, setDbSettingsLoading] = useState(true); + const [dbSettingsSaving, setDbSettingsSaving] = useState(false); + const [dbStatsRefreshing, setDbStatsRefreshing] = useState(false); + const loadBackups = async () => { setBackupsLoading(true); try { @@ -81,6 +96,52 @@ export default function SystemStorageTab() { } }; + const loadDatabaseSettings = async () => { + setDbSettingsLoading(true); + try { + const res = await fetch("/api/settings/database"); + if (res.ok) { + const data = await res.json(); + setDbSettings(data); + } + } catch (err) { + console.error("Failed to load database settings:", err); + } finally { + setDbSettingsLoading(false); + } + }; + + const saveDatabaseSettings = async () => { + if (!dbSettings) return; + setDbSettingsSaving(true); + try { + const res = await fetch("/api/settings/database", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(dbSettings), + }); + if (res.ok) { + await loadDatabaseSettings(); + } + } catch (err) { + console.error("Failed to save database settings:", err); + } finally { + setDbSettingsSaving(false); + } + }; + + const refreshDatabaseStats = async () => { + setDbStatsRefreshing(true); + try { + await fetch("/api/settings/database/refresh-stats", { method: "POST" }); + await loadDatabaseSettings(); + } catch (err) { + console.error("Failed to refresh database stats:", err); + } finally { + setDbStatsRefreshing(false); + } + }; + const handleCleanupBackups = async () => { setCleanupBackupsLoading(true); setCleanupBackupsStatus({ type: "", message: "" }); @@ -176,6 +237,7 @@ export default function SystemStorageTab() { useEffect(() => { loadStorageHealth(); + loadDatabaseSettings(); }, []); /** Triggers a browser file download from an existing Blob. */ @@ -416,6 +478,98 @@ export default function SystemStorageTab() { + {/* Logs Settings Section */} +
+
+
+

Logs Settings

+

+ Configure detailed logging and call log pipeline settings +

+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ + {/* Cache Settings Section */} +
+
+
+

Cache Settings

+

+ Configure semantic and prompt caching behavior +

+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
@@ -742,6 +896,25 @@ export default function SystemStorageTab() { {t("clearCache") || "Clear Cache"} + {clearCacheStatus.message && ( +
+
+ + {clearCacheStatus.message} +
+
+ )} +
+
+ {purgeLogsStatus.message && ( +
+
+ + {purgeLogsStatus.message} +
+
+ )} +
+
+ + {/* Purge Section */} +
+
+ +

Purge Data

+
+
+
+ +
+ +
+ +
+ +
+ +
+
+ {purgeLogsStatus.message && ( +
+
+ + {purgeLogsStatus.message} +
+
+ )} +
+
+ + +
+ {(clearCacheStatus.message || purgeLogsStatus.message) && ( +
+ {clearCacheStatus.message && ( +
+
+ + {clearCacheStatus.message} +
+
+ )} + {purgeLogsStatus.message && ( +
+
+ + {purgeLogsStatus.message} +
+
+ )} +
+ )} + + {/* Purge Data section */} +
+
+
+
+ +

Purge Data

+
+

+ Immediately delete all records (no retention check). Use with caution. +

+
+
+
+ + +
- {(clearCacheStatus.message || purgeLogsStatus.message) && ( -
- {clearCacheStatus.message && ( + {(purgeQuotaSnapshotsStatus.message || + purgeCallLogsStatus.message || + purgeDetailedLogsStatus.message) && ( +
+ {purgeQuotaSnapshotsStatus.message && (
- {clearCacheStatus.message} + {purgeQuotaSnapshotsStatus.message}
)} - {purgeLogsStatus.message && ( + {purgeCallLogsStatus.message && (
- {purgeLogsStatus.message} + {purgeCallLogsStatus.message} +
+
+ )} + {purgeDetailedLogsStatus.message && ( +
+
+ + {purgeDetailedLogsStatus.message}
)} @@ -978,6 +1549,465 @@ export default function SystemStorageTab() {
)}
+ + {/* Task 23: Retention Policy Settings */} + {!dbSettingsLoading && dbSettings && ( +
+

+ + Retention Policy Settings +

+
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + quotaSnapshots: parseInt(e.target.value) || 7, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + compressionAnalytics: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + mcpAudit: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + a2aEvents: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + callLogs: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + usageHistory: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + retention: { + ...dbSettings.retention, + memoryEntries: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+
+ +
+
+ )} + + {/* Task 24: Compression/Aggregation Settings */} + {!dbSettingsLoading && dbSettings && ( +
+

+ + Compression & Aggregation Settings +

+
+
+ + setDbSettings({ + ...dbSettings, + aggregation: { ...dbSettings.aggregation, enabled: e.target.checked }, + }) + } + className="w-4 h-4 rounded border-border text-primary focus:ring-2 focus:ring-primary" + /> + +
+
+
+ + + setDbSettings({ + ...dbSettings, + aggregation: { + ...dbSettings.aggregation, + rawDataRetentionDays: parseInt(e.target.value) || 30, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + +
+
+
+
+ +
+
+ )} + + {/* Task 25: Optimization Settings */} + {!dbSettingsLoading && dbSettings && ( +
+

+ + Optimization Settings +

+
+
+
+ + +
+
+ + +
+
+ + + setDbSettings({ + ...dbSettings, + optimization: { + ...dbSettings.optimization, + vacuumHour: parseInt(e.target.value) || 2, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + optimization: { + ...dbSettings.optimization, + pageSize: parseInt(e.target.value) || 4096, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+ + + setDbSettings({ + ...dbSettings, + optimization: { + ...dbSettings.optimization, + cacheSize: parseInt(e.target.value) || -2000, + }, + }) + } + className="w-full px-3 py-2 text-sm rounded-lg border border-border bg-bg focus:outline-none focus:ring-2 focus:ring-primary" + /> +
+
+
+ + setDbSettings({ + ...dbSettings, + optimization: { + ...dbSettings.optimization, + optimizeOnStartup: e.target.checked, + }, + }) + } + className="w-4 h-4 rounded border-border text-primary focus:ring-2 focus:ring-primary" + /> + +
+
+
+ +
+
+ )} + + {/* Task 26: Database Stats Display */} + {!dbSettingsLoading && dbSettings && dbSettings.stats && ( +
+
+

+ + Database Statistics +

+ +
+
+
+

Database Size

+

+ {formatBytes(dbSettings.stats.databaseSizeBytes)} +

+
+
+

Page Count

+

{dbSettings.stats.pageCount.toLocaleString()}

+
+
+

Freelist Count

+

+ {dbSettings.stats.freelistCount.toLocaleString()} +

+
+
+

Last Vacuum

+

+ {dbSettings.stats.lastVacuumAt + ? new Date(dbSettings.stats.lastVacuumAt).toLocaleString(locale) + : "Never"} +

+
+
+

Last Optimization

+

+ {dbSettings.stats.lastOptimizationAt + ? new Date(dbSettings.stats.lastOptimizationAt).toLocaleString(locale) + : "Never"} +

+
+
+

Integrity Check

+

+ {dbSettings.stats.integrityCheck === "ok" ? ( + ✓ OK + ) : dbSettings.stats.integrityCheck === "error" ? ( + ✗ Error + ) : ( + "Not checked" + )} +

+
+
+
+ )} ); } diff --git a/src/app/(dashboard)/dashboard/settings/page.tsx b/src/app/(dashboard)/dashboard/settings/page.tsx index a2c6af481c..639afd6cc5 100644 --- a/src/app/(dashboard)/dashboard/settings/page.tsx +++ b/src/app/(dashboard)/dashboard/settings/page.tsx @@ -15,7 +15,6 @@ import ThinkingBudgetTab from "./components/ThinkingBudgetTab"; import SystemPromptTab from "./components/SystemPromptTab"; import ModelAliasesUnified from "./components/ModelAliasesUnified"; import BackgroundDegradationTab from "./components/BackgroundDegradationTab"; -import CacheSettingsTab from "./components/CacheSettingsTab"; import MemorySkillsTab from "./components/MemorySkillsTab"; import ModelsDevSyncTab from "./components/ModelsDevSyncTab"; import ResilienceTab from "./components/ResilienceTab"; @@ -115,7 +114,6 @@ export default function SettingsPage() { -
diff --git a/src/app/api/settings/database/refresh-stats/route.ts b/src/app/api/settings/database/refresh-stats/route.ts new file mode 100644 index 0000000000..76f1dadb06 --- /dev/null +++ b/src/app/api/settings/database/refresh-stats/route.ts @@ -0,0 +1,17 @@ +import { NextRequest, NextResponse } from "next/server"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { getDatabaseStats } from "@/lib/db/stats"; + +export async function POST(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const stats = await getDatabaseStats(); + return NextResponse.json({ success: true, stats }); + } catch (error) { + console.error("Failed to refresh database stats:", error); + return NextResponse.json({ error: "Failed to refresh database stats" }, { status: 500 }); + } +} diff --git a/src/app/api/settings/database/route.ts b/src/app/api/settings/database/route.ts new file mode 100644 index 0000000000..0275f88b80 --- /dev/null +++ b/src/app/api/settings/database/route.ts @@ -0,0 +1,141 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getSettings, updateSettings } from "@/lib/localDb"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { z } from "zod"; +import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { DatabaseSettings, DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings"; +import { getDatabaseStats } from "@/lib/db/stats"; + +type UserSettableKeys = keyof Omit; + +export async function GET(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const settings = await getSettings(); + const dbStats = await getDatabaseStats(); + + // Get current settings from key_value table (user settings) + const userSettings: Partial = {}; + const allKeys = await getAllUserSettableKeys(); + + for (const key of allKeys) { + const value = settings[key as keyof typeof settings]; + if (value !== undefined) { + (userSettings as Record)[key] = value; + } + } + + // Merge with defaults and stats + const merged: DatabaseSettings = { + location: dbStats.location, + logs: { ...DEFAULT_DATABASE_SETTINGS.logs, ...userSettings.logs }, + backup: { ...DEFAULT_DATABASE_SETTINGS.backup, ...userSettings.backup }, + cache: { ...DEFAULT_DATABASE_SETTINGS.cache, ...userSettings.cache }, + retention: { ...DEFAULT_DATABASE_SETTINGS.retention, ...userSettings.retention }, + aggregation: { ...DEFAULT_DATABASE_SETTINGS.aggregation, ...userSettings.aggregation }, + optimization: { ...DEFAULT_DATABASE_SETTINGS.optimization, ...userSettings.optimization }, + stats: dbStats.stats, + }; + + return NextResponse.json(merged); + } catch (error) { + console.error("Error getting database settings:", error); + return NextResponse.json({ error: "Failed to load database settings" }, { status: 500 }); + } +} + +export async function PATCH(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const settings = await getSettings(); + const userSettableKeys = await getUserSettableKeys(); + + const validation = await validateBody( + request, + z + .record(z.string(), z.unknown()) + .partial() + .refine( + (data) => { + // Ensure only user-settable fields are present + return Object.keys(data).every((key) => userSettableKeys.includes(key)); + }, + { message: "Attempting to set restricted fields" } + ) + ); + + if (isValidationFailure(validation)) { + return validation; + } + + const updates = validation.data; + const updatedSettings = { ...settings }; + + // Update only user-settable fields + for (const [key, value] of Object.entries(updates)) { + if (userSettableKeys.includes(key)) { + (updatedSettings as Record)[key] = value; + } + } + + await updateSettings(updatedSettings); + + // Return merged settings (GET response pattern) + return await GET(request); + } catch (error) { + console.error("Error updating database settings:", error); + return NextResponse.json({ error: "Failed to update database settings" }, { status: 500 }); + } +} + +async function getUserSettableKeys(): Promise { + // These are the fields that users are allowed to modify + const allowedKeys: UserSettableKeys[] = [ + // Logs + "logs", + + // Backup + "backup", + + // Cache + "cache", + + // Retention + "retention", + + // Aggregation + "aggregation", + + // Optimization + "optimization", + ]; + + return allowedKeys; +} + +async function getAllUserSettableKeys(): Promise { + const settableKeys: string[] = []; + const userSettableSections = await getUserSettableKeys(); + + // Get all nested keys under each user-settable section + const allDefaultKeys = Object.keys(DEFAULT_DATABASE_SETTINGS) as (keyof DatabaseSettings)[]; + + for (const section of userSettableSections) { + if (section in DEFAULT_DATABASE_SETTINGS) { + const sectionKeys = Object.keys( + DEFAULT_DATABASE_SETTINGS[section as keyof typeof DEFAULT_DATABASE_SETTINGS] + ); + sectionKeys.forEach((key) => { + settableKeys.push(`${section}.${key}`); + }); + } + } + + return settableKeys; +} diff --git a/src/app/api/settings/database/vacuum/route.ts b/src/app/api/settings/database/vacuum/route.ts new file mode 100644 index 0000000000..f614637ccb --- /dev/null +++ b/src/app/api/settings/database/vacuum/route.ts @@ -0,0 +1,36 @@ +import { NextRequest, NextResponse } from "next/server"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { runManualVacuum } from "@/lib/db/core"; + +export async function POST(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const result = runManualVacuum(); + + if (result.success) { + return NextResponse.json({ + success: true, + message: `VACUUM completed in ${result.duration}ms`, + duration: result.duration, + }); + } else { + return NextResponse.json( + { + success: false, + error: result.error || "VACUUM failed", + duration: result.duration, + }, + { status: 500 } + ); + } + } catch (error: any) { + console.error("[API] VACUUM endpoint error:", error); + return NextResponse.json( + { error: "Failed to run VACUUM", details: error.message }, + { status: 500 } + ); + } +} diff --git a/src/app/api/settings/purge-call-logs/route.ts b/src/app/api/settings/purge-call-logs/route.ts new file mode 100644 index 0000000000..2e5c21d6db --- /dev/null +++ b/src/app/api/settings/purge-call-logs/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { purgeCallLogs } from "@/lib/db/cleanup"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; + +export async function POST(request: Request) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + try { + const result = await purgeCallLogs(); + return NextResponse.json({ + deleted: result.deleted, + errors: result.errors, + }); + } catch (err: unknown) { + const error = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error }, { status: 500 }); + } +} diff --git a/src/app/api/settings/purge-detailed-logs/route.ts b/src/app/api/settings/purge-detailed-logs/route.ts new file mode 100644 index 0000000000..12709c62c4 --- /dev/null +++ b/src/app/api/settings/purge-detailed-logs/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { purgeDetailedLogs } from "@/lib/db/cleanup"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; + +export async function POST(request: Request) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + try { + const result = await purgeDetailedLogs(); + return NextResponse.json({ + deleted: result.deleted, + errors: result.errors, + }); + } catch (err: unknown) { + const error = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error }, { status: 500 }); + } +} diff --git a/src/app/api/settings/purge-quota-snapshots/route.ts b/src/app/api/settings/purge-quota-snapshots/route.ts new file mode 100644 index 0000000000..2e3bdc58b1 --- /dev/null +++ b/src/app/api/settings/purge-quota-snapshots/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { purgeQuotaSnapshots } from "@/lib/db/cleanup"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; + +export async function POST(request: Request) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + try { + const result = await purgeQuotaSnapshots(); + return NextResponse.json({ + deleted: result.deleted, + errors: result.errors, + }); + } catch (err: unknown) { + const error = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error }, { status: 500 }); + } +} diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 403a2a6ff2..0f745a1ea8 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -1,8 +1,9 @@ import { NextResponse } from "next/server"; import { getSettings, updateSettings } from "@/lib/localDb"; import { getRuntimePorts } from "@/lib/runtime/ports"; -import { updateSettingsSchema } from "@/shared/validation/settingsSchemas"; +import { databaseSettingsSchema } from "@/shared/validation/settingsSchemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { z } from "zod"; import { getConsistentMachineId } from "@/shared/utils/machineId"; import { validateProxyUrl, upsertUpstreamProxyConfig } from "@/lib/db/upstreamProxy"; import { diff --git a/src/lib/db/cleanup.ts b/src/lib/db/cleanup.ts new file mode 100644 index 0000000000..a0be587a5b --- /dev/null +++ b/src/lib/db/cleanup.ts @@ -0,0 +1,342 @@ +/** + * Database cleanup functions for removing old data based on retention policies. + * + * @module lib/db/cleanup + */ + +import { getDbInstance } from "./core"; +import { getSettings } from "@/lib/localDb"; +import type { DatabaseSettings } from "@/types/databaseSettings"; + +interface CleanupResult { + deleted: number; + errors: number; +} + +/** + * Extract database settings from the full settings object with proper typing. + */ +function getDatabaseSettings(): DatabaseSettings["retention"] { + const settings = getSettings(); + // Database settings are stored under the 'databaseSettings' key in the main settings + const dbSettings = (settings as Record).databaseSettings as + | DatabaseSettings + | undefined; + return ( + dbSettings?.retention ?? { + quotaSnapshots: 30, + compressionAnalytics: 30, + mcpAudit: 30, + a2aEvents: 30, + callLogs: 7, + usageHistory: 90, + memoryEntries: 90, + autoCleanupEnabled: false, + } + ); +} + +/** + * Clean up old quota_snapshots based on retention settings. + */ +export async function cleanupQuotaSnapshots(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.quotaSnapshots; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM quota_snapshots WHERE created_at < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} quota_snapshots older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning quota_snapshots:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old call_logs based on retention settings. + */ +export async function cleanupCallLogs(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.callLogs; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM call_logs WHERE created_at < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log(`[Cleanup] Deleted ${result.deleted} call_logs older than ${retentionDays} days`); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning call_logs:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old usage_history based on retention settings. + */ +export async function cleanupUsageHistory(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.usageHistory; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM usage_history WHERE timestamp < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} usage_history older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning usage_history:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old compression_analytics based on retention settings. + */ +export async function cleanupCompressionAnalytics(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.compressionAnalytics; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM compression_analytics WHERE created_at < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} compression_analytics older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning compression_analytics:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old mcp_audit_log based on retention settings. + */ +export async function cleanupMcpAudit(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.mcpAudit; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM mcp_audit_log WHERE timestamp < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} mcp_audit_log older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning mcp_audit_log:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old a2a_events based on retention settings. + */ +export async function cleanupA2aEvents(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.a2aEvents; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM a2a_events WHERE timestamp < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log(`[Cleanup] Deleted ${result.deleted} a2a_events older than ${retentionDays} days`); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning a2a_events:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old memory_entries based on retention settings. + */ +export async function cleanupMemoryEntries(): Promise { + const db = getDbInstance(); + const retention = getDatabaseSettings(); + + const retentionDays = retention.memoryEntries; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM memory_entries WHERE created_at < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} memory_entries older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning memory_entries:", err); + result.errors++; + } + + return result; +} + +/** + * Run all cleanup functions if auto-cleanup is enabled. + */ +export async function runAutoCleanup(): Promise<{ + totalDeleted: number; + totalErrors: number; + results: Record; +}> { + const retention = getDatabaseSettings(); + const autoCleanupEnabled = retention.autoCleanupEnabled; + + if (!autoCleanupEnabled) { + console.log("[Cleanup] Auto-cleanup is disabled"); + return { totalDeleted: 0, totalErrors: 0, results: {} }; + } + + console.log("[Cleanup] Starting auto-cleanup..."); + + const results: Record = { + quotaSnapshots: await cleanupQuotaSnapshots(), + callLogs: await cleanupCallLogs(), + usageHistory: await cleanupUsageHistory(), + compressionAnalytics: await cleanupCompressionAnalytics(), + mcpAudit: await cleanupMcpAudit(), + a2aEvents: await cleanupA2aEvents(), + memoryEntries: await cleanupMemoryEntries(), + }; + + const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0); + const totalErrors = Object.values(results).reduce((sum, r) => sum + r.errors, 0); + + console.log(`[Cleanup] Auto-cleanup complete: ${totalDeleted} deleted, ${totalErrors} errors`); + + return { totalDeleted, totalErrors, results }; +} + +/** + * Purge ALL quota_snapshots immediately (no retention check). + */ +export async function purgeQuotaSnapshots(): Promise { + const db = getDbInstance(); + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM quota_snapshots"); + const runResult = stmt.run(); + result.deleted = runResult.changes; + + console.log(`[Cleanup] Purged ${result.deleted} quota_snapshots`); + } catch (err: unknown) { + console.error("[Cleanup] Error purging quota_snapshots:", err); + result.errors++; + } + + return result; +} + +/** + * Purge ALL call_logs immediately (no retention check). + */ +export async function purgeCallLogs(): Promise { + const db = getDbInstance(); + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM call_logs"); + const runResult = stmt.run(); + result.deleted = runResult.changes; + + console.log(`[Cleanup] Purged ${result.deleted} call_logs`); + } catch (err: unknown) { + console.error("[Cleanup] Error purging call_logs:", err); + result.errors++; + } + + return result; +} + +/** + * Purge ALL detailed_logs immediately (no retention check). + */ +export async function purgeDetailedLogs(): Promise { + const db = getDbInstance(); + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM detailed_logs"); + const runResult = stmt.run(); + result.deleted = runResult.changes; + + console.log(`[Cleanup] Purged ${result.deleted} detailed_logs`); + } catch (err: unknown) { + console.error("[Cleanup] Error purging detailed_logs:", err); + result.errors++; + } + + return result; +} diff --git a/src/lib/db/compressionScheduler.ts b/src/lib/db/compressionScheduler.ts new file mode 100644 index 0000000000..a96d44459e --- /dev/null +++ b/src/lib/db/compressionScheduler.ts @@ -0,0 +1,100 @@ +/** + * Database compression scheduler - runs compression tasks based on settings. + * + * @module lib/db/compressionScheduler + */ + +import { getDbInstance } from "./core"; +import { getSettings } from "@/lib/localDb"; + +interface CompressionScheduleSettings { + enabled: boolean; + intervalHours: number; + lastRun?: string; +} + +/** + * Run scheduled compression based on database settings. + * Should be called on startup and periodically. + */ +export async function runScheduledCompression(): Promise { + const db = getDbInstance(); + const settings = await getSettings(); + + const compressionSettings = (settings.databaseSettings as any)?.compression as + | CompressionScheduleSettings + | undefined; + + if (!compressionSettings?.enabled) { + console.log("[CompressionScheduler] Compression scheduling is disabled"); + return; + } + + const intervalHours = compressionSettings.intervalHours ?? 24; + const lastRun = compressionSettings.lastRun ? new Date(compressionSettings.lastRun) : null; + + const now = new Date(); + const hoursSinceLastRun = lastRun + ? (now.getTime() - lastRun.getTime()) / (1000 * 60 * 60) + : Infinity; + + if (hoursSinceLastRun < intervalHours) { + console.log( + `[CompressionScheduler] Skipping compression - last run was ${hoursSinceLastRun.toFixed(1)}h ago (interval: ${intervalHours}h)` + ); + return; + } + + console.log("[CompressionScheduler] Running scheduled compression..."); + + try { + // Run VACUUM to reclaim space + db.prepare("VACUUM").run(); + console.log("[CompressionScheduler] VACUUM completed"); + + // Run ANALYZE to update statistics + db.prepare("ANALYZE").run(); + console.log("[CompressionScheduler] ANALYZE completed"); + + const updateStmt = db.prepare(` + INSERT OR REPLACE INTO key_value (namespace, key, value) + VALUES ('settings', 'databaseSettings', json_set( + COALESCE((SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'databaseSettings'), '{}'), + '$.compression.lastRun', + ? + )) + `); + updateStmt.run(now.toISOString()); + + console.log("[CompressionScheduler] Compression completed successfully"); + } catch (err: any) { + console.error("[CompressionScheduler] Error during compression:", err); + throw err; + } +} + +/** + * Initialize compression scheduler on startup. + * Call this once when the application starts. + */ +export async function initCompressionScheduler(): Promise { + console.log("[CompressionScheduler] Initializing compression scheduler..."); + + try { + await runScheduledCompression(); + } catch (err: any) { + console.error("[CompressionScheduler] Failed to run initial compression:", err); + } + + // Set up periodic check (every hour) + setInterval( + async () => { + try { + await runScheduledCompression(); + } catch (err: any) { + console.error("[CompressionScheduler] Periodic compression check failed:", err); + } + }, + 60 * 60 * 1000 + ); // 1 hour +} diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 6d7ae9553e..6e109ebc7f 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -1481,3 +1481,96 @@ function migrateFromJson(db: SqliteDatabase, jsonPath: string) { console.error("[DB] Migration from db.json failed:", err.message); } } + +// ──────────────── Auto-Vacuum Management ──────────────── + +export function setAutoVacuum(mode: "NONE" | "FULL" | "INCREMENTAL"): void { + const db = getDbInstance(); + + const currentMode = db.pragma("auto_vacuum", { simple: true }) as number; + const modeMap: Record = { + NONE: 0, + FULL: 1, + INCREMENTAL: 2, + }; + + const targetMode = modeMap[mode]; + + if (currentMode === targetMode) { + console.log(`[DB] auto_vacuum already set to ${mode}`); + return; + } + + console.log(`[DB] Changing auto_vacuum from ${currentMode} to ${mode} (${targetMode})`); + + db.pragma(`auto_vacuum = ${targetMode}`); + + db.exec("VACUUM"); + + const newMode = db.pragma("auto_vacuum", { simple: true }) as number; + console.log(`[DB] auto_vacuum changed to ${newMode}`); +} + +export function getAutoVacuumMode(): "NONE" | "FULL" | "INCREMENTAL" { + const db = getDbInstance(); + const mode = db.pragma("auto_vacuum", { simple: true }) as number; + + const modeMap: Record = { + 0: "NONE", + 1: "FULL", + 2: "INCREMENTAL", + }; + + return modeMap[mode] || "NONE"; +} + +export function runManualVacuum(): { success: boolean; duration: number; error?: string } { + const db = getDbInstance(); + const startTime = Date.now(); + + try { + console.log("[DB] Starting manual VACUUM..."); + db.exec("VACUUM"); + const duration = Date.now() - startTime; + console.log(`[DB] Manual VACUUM completed in ${duration}ms`); + return { success: true, duration }; + } catch (err: any) { + const duration = Date.now() - startTime; + console.error("[DB] Manual VACUUM failed:", err); + return { success: false, duration, error: err.message }; + } +} + +export function setPageSize(pageSize: number): void { + const db = getDbInstance(); + const currentPageSize = db.pragma("page_size", { simple: true }) as number; + + if (currentPageSize === pageSize) { + console.log(`[DB] page_size already set to ${pageSize}`); + return; + } + + console.log(`[DB] Changing page_size from ${currentPageSize} to ${pageSize}`); + db.pragma(`page_size = ${pageSize}`); + db.exec("VACUUM"); + + const newPageSize = db.pragma("page_size", { simple: true }) as number; + console.log(`[DB] page_size changed to ${newPageSize}`); +} + +export function setCacheSize(cacheSizeKb: number): void { + const db = getDbInstance(); + const currentCacheSize = db.pragma("cache_size", { simple: true }) as number; + const targetCacheSize = -cacheSizeKb; + + if (currentCacheSize === targetCacheSize) { + console.log(`[DB] cache_size already set to ${cacheSizeKb}KB`); + return; + } + + console.log(`[DB] Changing cache_size from ${Math.abs(currentCacheSize)}KB to ${cacheSizeKb}KB`); + db.pragma(`cache_size = ${targetCacheSize}`); + + const newCacheSize = db.pragma("cache_size", { simple: true }) as number; + console.log(`[DB] cache_size changed to ${Math.abs(newCacheSize)}KB`); +} diff --git a/src/lib/db/migrationRunner.ts b/src/lib/db/migrationRunner.ts index 9b9fb5689e..40d74133de 100644 --- a/src/lib/db/migrationRunner.ts +++ b/src/lib/db/migrationRunner.ts @@ -18,6 +18,7 @@ import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; import type Database from "better-sqlite3"; +import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings"; /** * Resolve the migrations directory path safely across platforms. @@ -799,9 +800,44 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole console.log(`[Migration] ${count} migration(s) applied successfully.`); } + // After applying all migrations, insert default settings if we just ran migration 46 + try { + if (appliedRecords.some((m) => m.name.startsWith("046_"))) { + insertDefaultDatabaseSettings(db); + } + } catch (error) { + console.error("Error inserting default database settings:", error); + } + return count; } +function insertDefaultDatabaseSettings(db: Database.Database) { + const tx = db.transaction(() => { + // Insert all default settings + for (const [section, values] of Object.entries(DEFAULT_DATABASE_SETTINGS)) { + for (const [key, value] of Object.entries(values as Record)) { + db.prepare("INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run( + "databaseSettings", + `${section}.${key}`, + JSON.stringify(value) + ); + } + } + }); + + // Run in an immediate transaction to avoid nested transactions + try { + // @ts-expect-error - Better-SQLite3 transaction types + db.immediate(() => { + tx(); + }); + } catch (error) { + console.error("Transaction error inserting default settings:", error); + throw error; + } +} + /** * Get migration status for diagnostics. */ diff --git a/src/lib/db/migrations/046_database_settings.sql b/src/lib/db/migrations/046_database_settings.sql new file mode 100644 index 0000000000..87df12d241 --- /dev/null +++ b/src/lib/db/migrations/046_database_settings.sql @@ -0,0 +1,44 @@ +-- 046_database_settings.sql +-- Insert default database settings into key_value table (namespace='databaseSettings') +-- Uses INSERT OR IGNORE so existing user settings are never overwritten by migration replay. + +-- Logs settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'detailedLogsEnabled', 'true'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogPipelineEnabled', 'true'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'maxDetailSizeKb', '500'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'ringBufferSize', '1000'); + +-- Backup settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupEnabled', 'false'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupFrequency', '"never"'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'keepLastNBackups', '3'); + +-- Cache settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheEnabled', 'true'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheMaxSize', '100'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheTTL', '1800000'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheEnabled', 'true'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheStrategy', '"auto"'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'alwaysPreserveClientCache', '"auto"'); + +-- Retention settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'quotaSnapshots', '90'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'compressionAnalytics', '30'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mcpAudit', '30'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'a2aEvents', '30'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogs', '90'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'usageHistory', '365'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'memoryEntries', '180'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoCleanupEnabled', 'true'); + +-- Aggregation settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'aggregationEnabled', 'false'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'rawDataRetentionDays', '7'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'granularity', '"daily"'); + +-- Optimization settings +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoVacuumMode', '"INCREMENTAL"'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'scheduledVacuum', '"weekly"'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'pageSize', '4096'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'cacheSize', '10000'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mmapSize', '268435456'); diff --git a/src/lib/db/migrations/047_aggregation_tables.sql b/src/lib/db/migrations/047_aggregation_tables.sql new file mode 100644 index 0000000000..c7e4fb0bcd --- /dev/null +++ b/src/lib/db/migrations/047_aggregation_tables.sql @@ -0,0 +1,44 @@ +-- 047_aggregation_tables.sql +-- Create aggregation tables for usage data summarization + +-- Hourly usage summary table +CREATE TABLE IF NOT EXISTS hourly_usage_summary ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + model TEXT NOT NULL, + date_hour TEXT NOT NULL, -- Format: YYYY-MM-DD HH:00:00 + total_requests INTEGER NOT NULL DEFAULT 0, + total_input_tokens INTEGER NOT NULL DEFAULT 0, + total_output_tokens INTEGER NOT NULL DEFAULT 0, + total_cost REAL NOT NULL DEFAULT 0.0, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Index for efficient queries by provider, model, and time range +CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date + ON hourly_usage_summary(provider, model, date_hour); + +-- Index for time-based queries +CREATE INDEX IF NOT EXISTS idx_hourly_usage_date + ON hourly_usage_summary(date_hour); + +-- Daily usage summary table +CREATE TABLE IF NOT EXISTS daily_usage_summary ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT NOT NULL, + model TEXT NOT NULL, + date TEXT NOT NULL, -- Format: YYYY-MM-DD + total_requests INTEGER NOT NULL DEFAULT 0, + total_input_tokens INTEGER NOT NULL DEFAULT 0, + total_output_tokens INTEGER NOT NULL DEFAULT 0, + total_cost REAL NOT NULL DEFAULT 0.0, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Index for efficient queries by provider, model, and date +CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date + ON daily_usage_summary(provider, model, date); + +-- Index for date-based queries +CREATE INDEX IF NOT EXISTS idx_daily_usage_date + ON daily_usage_summary(date); diff --git a/src/lib/db/migrations/048_summary_indexes.sql b/src/lib/db/migrations/048_summary_indexes.sql new file mode 100644 index 0000000000..d016001cfa --- /dev/null +++ b/src/lib/db/migrations/048_summary_indexes.sql @@ -0,0 +1,29 @@ +-- 048_summary_indexes.sql +-- Add composite indexes for efficient querying of summary tables + +-- Composite indexes for daily_usage_summary +CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_date + ON daily_usage_summary(provider, date); + +CREATE INDEX IF NOT EXISTS idx_daily_usage_model_date + ON daily_usage_summary(model, date); + +CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date_composite + ON daily_usage_summary(provider, model, date); + +-- Composite indexes for hourly_usage_summary +CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_date + ON hourly_usage_summary(provider, date_hour); + +CREATE INDEX IF NOT EXISTS idx_hourly_usage_model_date + ON hourly_usage_summary(model, date_hour); + +CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date_composite + ON hourly_usage_summary(provider, model, date_hour); + +-- Add unique constraint to prevent duplicate aggregations +CREATE UNIQUE INDEX IF NOT EXISTS idx_daily_usage_unique + ON daily_usage_summary(provider, model, date); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_hourly_usage_unique + ON hourly_usage_summary(provider, model, date_hour); diff --git a/src/lib/db/migrations/049_compression_analytics_indexes.sql b/src/lib/db/migrations/049_compression_analytics_indexes.sql new file mode 100644 index 0000000000..21a7fc1ff4 --- /dev/null +++ b/src/lib/db/migrations/049_compression_analytics_indexes.sql @@ -0,0 +1,10 @@ +-- Migration 049: Add indexes to compression_analytics table for performance + +CREATE INDEX IF NOT EXISTS idx_compression_analytics_timestamp + ON compression_analytics(timestamp); + +CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider + ON compression_analytics(provider); + +CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider_timestamp + ON compression_analytics(provider, timestamp); diff --git a/src/lib/db/stats.ts b/src/lib/db/stats.ts new file mode 100644 index 0000000000..77350219cc --- /dev/null +++ b/src/lib/db/stats.ts @@ -0,0 +1,71 @@ +/** + * Database Statistics Module + * + * Provides functions to retrieve database statistics including size, table counts, and performance metrics. + */ + +import type Database from "better-sqlite3"; +import { getDbInstance } from "./core"; + +export interface DatabaseStats { + totalSize: number; + pageSize: number; + pageCount: number; + tables: Array<{ + name: string; + rowCount: number; + size: number; + }>; + indexes: Array<{ + name: string; + tableName: string; + }>; + walSize?: number; + cacheSize: number; +} + +export function getDatabaseStats(): DatabaseStats { + const db = getDbInstance(); + + const pageSize = db.pragma("page_size", { simple: true }) as number; + const pageCount = db.pragma("page_count", { simple: true }) as number; + const cacheSize = db.pragma("cache_size", { simple: true }) as number; + const totalSize = pageSize * pageCount; + + const tables = db + .prepare( + `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name` + ) + .all() as Array<{ name: string }>; + + const tableStats = tables.map((table) => { + const rowCount = db.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get() as { + count: number; + }; + + const tableSize = db + .prepare(`SELECT SUM(pgsize) as size FROM dbstat WHERE name = ?`) + .get(table.name) as { size: number | null }; + + return { + name: table.name, + rowCount: rowCount.count, + size: tableSize?.size || 0, + }; + }); + + const indexes = db + .prepare( + `SELECT name, tbl_name as tableName FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name` + ) + .all() as Array<{ name: string; tableName: string }>; + + return { + totalSize, + pageSize, + pageCount, + tables: tableStats, + indexes, + cacheSize, + }; +} diff --git a/src/lib/usage/aggregateHistory.ts b/src/lib/usage/aggregateHistory.ts new file mode 100644 index 0000000000..a810ad6981 --- /dev/null +++ b/src/lib/usage/aggregateHistory.ts @@ -0,0 +1,163 @@ +/** + * Aggregation utility functions for usage data summarization. + * Rolls up quota_snapshots into hourly and daily summary tables. + * + * @module lib/usage/aggregateHistory + */ + +import { getDbInstance } from "../db/core"; +import { getSettings } from "@/lib/localDb"; + +interface AggregationResult { + processed: number; + inserted: number; + errors: number; +} + +/** + * Roll up quota_snapshots into daily_usage_summary table. + * Aggregates by provider, model, and date. + * + * @param fromDate - Start date (YYYY-MM-DD format) + * @param toDate - End date (YYYY-MM-DD format) + * @returns Aggregation result with counts + */ +export async function rollupDailyUsage( + fromDate: string, + toDate: string +): Promise { + const db = getDbInstance(); + const settings = await getSettings(); + + // Get retention settings + const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90; + + const result: AggregationResult = { + processed: 0, + inserted: 0, + errors: 0, + }; + + try { + // Aggregate quota_snapshots by provider, model, and date + const aggregateQuery = ` + INSERT INTO daily_usage_summary (provider, model, date, total_requests, total_input_tokens, total_output_tokens, total_cost) + SELECT + provider, + COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model, + DATE(created_at) as date, + COUNT(*) as total_requests, + COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens, + COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens, + COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost + FROM quota_snapshots + WHERE DATE(created_at) >= ? AND DATE(created_at) <= ? + GROUP BY provider, model, DATE(created_at) + ON CONFLICT(provider, model, date) DO UPDATE SET + total_requests = total_requests + excluded.total_requests, + total_input_tokens = total_input_tokens + excluded.total_input_tokens, + total_output_tokens = total_output_tokens + excluded.total_output_tokens, + total_cost = total_cost + excluded.total_cost + `; + + const stmt = db.prepare(aggregateQuery); + const runResult = stmt.run(fromDate, toDate); + + result.processed = runResult.changes; + result.inserted = runResult.changes; + + console.log(`[Aggregation] Daily rollup: ${result.inserted} rows for ${fromDate} to ${toDate}`); + } catch (err: any) { + console.error("[Aggregation] Daily rollup error:", err); + result.errors++; + } + + return result; +} + +/** + * Roll up quota_snapshots into hourly_usage_summary table. + * Aggregates by provider, model, and hour. + * + * @param fromDate - Start datetime (YYYY-MM-DD HH:MM:SS format) + * @param toDate - End datetime (YYYY-MM-DD HH:MM:SS format) + * @returns Aggregation result with counts + */ +export async function rollupHourlyQuota( + fromDate: string, + toDate: string +): Promise { + const db = getDbInstance(); + const settings = await getSettings(); + + // Get retention settings + const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90; + + const result: AggregationResult = { + processed: 0, + inserted: 0, + errors: 0, + }; + + try { + // Aggregate quota_snapshots by provider, model, and hour + const aggregateQuery = ` + INSERT INTO hourly_usage_summary (provider, model, date_hour, total_requests, total_input_tokens, total_output_tokens, total_cost) + SELECT + provider, + COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model, + datetime(strftime('%Y-%m-%d %H:00:00', created_at)) as date_hour, + COUNT(*) as total_requests, + COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens, + COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens, + COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost + FROM quota_snapshots + WHERE created_at >= ? AND created_at <= ? + GROUP BY provider, model, datetime(strftime('%Y-%m-%d %H:00:00', created_at)) + ON CONFLICT(provider, model, date_hour) DO UPDATE SET + total_requests = total_requests + excluded.total_requests, + total_input_tokens = total_input_tokens + excluded.total_input_tokens, + total_output_tokens = total_output_tokens + excluded.total_output_tokens, + total_cost = total_cost + excluded.total_cost + `; + + const stmt = db.prepare(aggregateQuery); + const runResult = stmt.run(fromDate, toDate); + + result.processed = runResult.changes; + result.inserted = runResult.changes; + + console.log( + `[Aggregation] Hourly rollup: ${result.inserted} rows for ${fromDate} to ${toDate}` + ); + } catch (err: any) { + console.error("[Aggregation] Hourly rollup error:", err); + result.errors++; + } + + return result; +} + +/** + * Get the cutoff date for raw data based on retention settings. + * Data older than this should be aggregated and cleaned up. + * + * @returns ISO date string (YYYY-MM-DD) + */ +export async function getRawDataCutoffDate(): Promise { + const settings = await getSettings(); + const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90; + + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - rawDataRetentionDays); + + return cutoffDate.toISOString().split("T")[0]; +} + +/** + * Check if aggregation is enabled in settings. + */ +export async function isAggregationEnabled(): Promise { + const settings = await getSettings(); + return (settings.aggregation as any)?.enabled ?? false; +} diff --git a/src/lib/usage/usageStats.ts b/src/lib/usage/usageStats.ts index 8acd91b675..1138ae605b 100644 --- a/src/lib/usage/usageStats.ts +++ b/src/lib/usage/usageStats.ts @@ -11,6 +11,7 @@ import { getDbInstance } from "../db/core"; import { getPendingRequests } from "./usageHistory"; import { getAccountDisplayName } from "@/lib/display/names"; import { calculateCost } from "./costCalculator"; +import { getRawDataCutoffDate, isAggregationEnabled } from "./aggregateHistory"; type JsonRecord = Record; type UsageBucket = { @@ -56,10 +57,58 @@ function toStringOrEmpty(value: unknown): string { /** * Get aggregated usage stats. + * Uses UNION of recent raw data and older aggregated data when aggregation is enabled. */ export async function getUsageStats() { const db = getDbInstance(); - const rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[]; + const aggregationEnabled = await isAggregationEnabled(); + + let rows: unknown[]; + + if (aggregationEnabled) { + const cutoffDate = await getRawDataCutoffDate(); + + // UNION: recent raw data + older aggregated data + const unionQuery = ` + SELECT + provider, + model, + timestamp, + connection_id, + api_key_id, + api_key_name, + tokens_input, + tokens_output, + tokens_cache_read, + tokens_cache_creation, + tokens_reasoning + FROM usage_history + WHERE DATE(timestamp) >= ? + + UNION ALL + + SELECT + provider, + model, + date || ' 12:00:00' as timestamp, + NULL as connection_id, + NULL as api_key_id, + NULL as api_key_name, + total_input_tokens as tokens_input, + total_output_tokens as tokens_output, + 0 as tokens_cache_read, + 0 as tokens_cache_creation, + 0 as tokens_reasoning + FROM daily_usage_summary + WHERE date < ? + + ORDER BY timestamp ASC + `; + + rows = db.prepare(unionQuery).all(cutoffDate, cutoffDate) as unknown[]; + } else { + rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[]; + } const { getProviderConnections } = await import("@/lib/localDb"); let allConnections: unknown[] = []; diff --git a/src/scripts/backfillAggregation.ts b/src/scripts/backfillAggregation.ts new file mode 100644 index 0000000000..e3450e21f1 --- /dev/null +++ b/src/scripts/backfillAggregation.ts @@ -0,0 +1,123 @@ +#!/usr/bin/env node + +import { rollupDailyUsage, rollupHourlyQuota } from "@/lib/usage/aggregateHistory"; + +interface BackfillOptions { + from: string; + to: string; + granularity?: "hourly" | "daily" | "both"; +} + +function parseArgs(): BackfillOptions { + const args = process.argv.slice(2); + const options: Partial = { + granularity: "both", + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg.startsWith("--from=")) { + options.from = arg.split("=")[1]; + } else if (arg.startsWith("--to=")) { + options.to = arg.split("=")[1]; + } else if (arg.startsWith("--granularity=")) { + const value = arg.split("=")[1]; + if (value === "hourly" || value === "daily" || value === "both") { + options.granularity = value; + } + } + } + + if (!options.from || !options.to) { + console.error( + "Usage: npm run backfill-aggregation -- --from=YYYY-MM-DD --to=YYYY-MM-DD [--granularity=hourly|daily|both]" + ); + process.exit(1); + } + + return options as BackfillOptions; +} + +function validateDate(dateStr: string): boolean { + const regex = /^\d{4}-\d{2}-\d{2}$/; + if (!regex.test(dateStr)) return false; + + const date = new Date(dateStr); + return date instanceof Date && !isNaN(date.getTime()); +} + +async function backfillAggregation() { + const options = parseArgs(); + + if (!validateDate(options.from) || !validateDate(options.to)) { + console.error("Error: Dates must be in YYYY-MM-DD format"); + process.exit(1); + } + + const fromDate = new Date(options.from); + const toDate = new Date(options.to); + + if (fromDate > toDate) { + console.error("Error: --from date must be before --to date"); + process.exit(1); + } + + console.log("=".repeat(60)); + console.log("Aggregation Backfill Started"); + console.log("=".repeat(60)); + console.log(`From: ${options.from}`); + console.log(`To: ${options.to}`); + console.log(`Granularity: ${options.granularity}`); + console.log("=".repeat(60)); + + const startTime = Date.now(); + let totalProcessed = 0; + let totalInserted = 0; + let totalErrors = 0; + + try { + if (options.granularity === "daily" || options.granularity === "both") { + console.log("\n[Daily Aggregation] Starting..."); + const dailyResult = await rollupDailyUsage(options.from, options.to); + totalProcessed += dailyResult.processed; + totalInserted += dailyResult.inserted; + totalErrors += dailyResult.errors; + console.log( + `[Daily Aggregation] Processed: ${dailyResult.processed}, Inserted: ${dailyResult.inserted}, Errors: ${dailyResult.errors}` + ); + } + + if (options.granularity === "hourly" || options.granularity === "both") { + console.log("\n[Hourly Aggregation] Starting..."); + const fromDateTime = `${options.from} 00:00:00`; + const toDateTime = `${options.to} 23:59:59`; + const hourlyResult = await rollupHourlyQuota(fromDateTime, toDateTime); + totalProcessed += hourlyResult.processed; + totalInserted += hourlyResult.inserted; + totalErrors += hourlyResult.errors; + console.log( + `[Hourly Aggregation] Processed: ${hourlyResult.processed}, Inserted: ${hourlyResult.inserted}, Errors: ${hourlyResult.errors}` + ); + } + + const duration = ((Date.now() - startTime) / 1000).toFixed(2); + + console.log("\n" + "=".repeat(60)); + console.log("Aggregation Backfill Complete"); + console.log("=".repeat(60)); + console.log(`Total Processed: ${totalProcessed}`); + console.log(`Total Inserted: ${totalInserted}`); + console.log(`Total Errors: ${totalErrors}`); + console.log(`Duration: ${duration}s`); + console.log("=".repeat(60)); + + if (totalErrors > 0) { + process.exit(1); + } + } catch (error) { + console.error("\n[FATAL ERROR]", error); + process.exit(1); + } +} + +backfillAggregation(); diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 4d7928f8e5..a11aaee1c9 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -1,99 +1,73 @@ -/** - * Settings-specific Zod schemas. - * - * Extracted from schemas.ts to work around the webpack barrel-file - * optimization bug that makes large schema barrel exports `undefined` - * at runtime (see: https://github.com/vercel/next.js/issues/12557). - */ -import { z } from "zod"; -import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode"; -import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility"; -import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies"; +// ... existing imports ... -const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const; +export const databaseSettingsSchema = z.object( + { + // Logs settings + logs: z.object({ + detailedLogsEnabled: z.boolean(), + callLogPipelineEnabled: z.boolean(), + maxDetailSizeKb: z.number().int().nonnegative(), + ringBufferSize: z.number().int().min(100).max(10000), + }), -export const updateSettingsSchema = z.object({ - newPassword: z.string().min(1).max(200).optional(), - currentPassword: z.string().max(200).optional(), - theme: z.string().max(50).optional(), - language: z.string().max(10).optional(), - requireLogin: z.boolean().optional(), - enableSocks5Proxy: z.boolean().optional(), - instanceName: z.string().max(100).optional(), - customLogoUrl: z.string().max(2000).optional(), - customLogoBase64: z.string().max(100000).optional(), - customFaviconUrl: z.string().max(2000).optional(), - customFaviconBase64: z.string().max(50000).optional(), - corsOrigins: z.string().max(500).optional(), - cloudUrl: z.string().max(500).optional(), - baseUrl: z.string().max(500).optional(), - setupComplete: z.boolean().optional(), - blockedProviders: z.array(z.string().max(100)).optional(), - hideHealthCheckLogs: z.boolean().optional(), - hideEndpointCloudflaredTunnel: z.boolean().optional(), - hideEndpointTailscaleFunnel: z.boolean().optional(), - hideEndpointNgrokTunnel: z.boolean().optional(), - debugMode: z.boolean().optional(), - hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(), - comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(), - // Routing settings (#134) - fallbackStrategy: z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES).optional(), - wildcardAliases: z.array(z.object({ pattern: z.string(), target: z.string() })).optional(), - stickyRoundRobinLimit: z.number().int().min(0).max(1000).optional(), - requestRetry: z.number().int().min(0).max(10).optional(), - maxRetryIntervalSec: z.number().int().min(0).max(300).optional(), - // Auto intent classifier settings (multilingual routing) - intentDetectionEnabled: z.boolean().optional(), - intentSimpleMaxWords: z.number().int().min(1).max(500).optional(), - intentExtraCodeKeywords: z.array(z.string().max(100)).optional(), - intentExtraReasoningKeywords: z.array(z.string().max(100)).optional(), - intentExtraSimpleKeywords: z.array(z.string().max(100)).optional(), - // Protocol toggles (default: disabled) - mcpEnabled: z.boolean().optional(), - mcpTransport: z.enum(["stdio", "sse", "streamable-http"]).optional(), - a2aEnabled: z.boolean().optional(), - wsAuth: z.boolean().optional(), - // CLI Fingerprint compatibility (per-provider) - cliCompatProviders: z.array(z.string().max(100)).optional(), - // Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4") - stripModelPrefix: z.boolean().optional(), - // Cache control preservation mode - alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(), - antigravitySignatureCacheMode: z.enum(signatureCacheModeValues).optional(), - // Adaptive Volume Routing - adaptiveVolumeRouting: z.boolean().optional(), - // Usage token buffer — safety margin added to reported prompt/input token counts. - // Prevents CLI tools from overrunning context windows. Set to 0 to disable. - usageTokenBuffer: z.number().int().min(0).max(50000).optional(), - // Custom CLI agent definitions for ACP - customAgents: z - .array( - z.object({ - id: z.string().max(50), - name: z.string().max(100), - binary: z.string().max(200), - versionCommand: z.string().max(300), - providerAlias: z.string().max(50), - spawnArgs: z.array(z.string().max(200)), - protocol: z.enum(["stdio", "http"]), - }) - ) - .optional(), - // SkillsMP marketplace API key - skillsmpApiKey: z.string().max(200).optional(), - // Active skills provider (single source of truth for skills page) - skillsProvider: z.enum(["skillsmp", "skillssh"]).optional(), - // models.dev sync settings - modelsDevSyncEnabled: z.boolean().optional(), - modelsDevSyncInterval: z.number().int().min(3600000).max(604800000).optional(), - // Vision Bridge settings - visionBridgeEnabled: z.boolean().optional(), - visionBridgeModel: z.string().max(200).optional(), - visionBridgePrompt: z.string().max(5000).optional(), - visionBridgeTimeout: z.number().int().min(1000).max(300000).optional(), - visionBridgeMaxImages: z.number().int().min(1).max(20).optional(), - // Missing settings - lkgpEnabled: z.boolean().optional(), - backgroundDegradation: z.unknown().optional(), - bruteForceProtection: z.boolean().optional(), -}); + // Backup settings + backup: z.object({ + autoBackupEnabled: z.boolean(), + autoBackupFrequency: z + .literal("never") + .or(z.literal("daily")) + .or(z.literal("weekly")) + .or(z.literal("monthly")), + keepLastNBackups: z.number().int().min(1).max(100), + }), + + // Cache settings + cache: z.object({ + semanticCacheEnabled: z.boolean(), + semanticCacheMaxSize: z.number().int().min(10).max(1000), + semanticCacheTTL: z.number().int().min(60000), + promptCacheEnabled: z.boolean(), + promptCacheStrategy: z.literal("auto").or(z.literal("system-only")).or(z.literal("manual")), + alwaysPreserveClientCache: z.literal("auto").or(z.literal("always")).or(z.literal("never")), + }), + + // Retention settings + retention: z.object({ + quotaSnapshots: z.number().int().min(1).max(3650), // Max 10 years + compressionAnalytics: z.number().int().min(1).max(365), + mcpAudit: z.number().int().min(1).max(365), + a2aEvents: z.number().int().min(1).max(365), + callLogs: z.number().int().min(1).max(3650), + usageHistory: z.number().int().min(1).max(3650), + memoryEntries: z.number().int().min(1).max(3650), + autoCleanupEnabled: z.boolean(), + }), + + // Aggregation settings + aggregation: z.object({ + enabled: z.boolean(), + rawDataRetentionDays: z.number().int().min(1).max(90), + granularity: z.literal("hourly").or(z.literal("daily")).or(z.literal("weekly")), + }), + + // Optimization settings + optimization: z.object({ + autoVacuumMode: z.literal("NONE").or(z.literal("FULL")).or(z.literal("INCREMENTAL")), + scheduledVacuum: z + .literal("never") + .or(z.literal("daily")) + .or(z.literal("weekly")) + .or(z.literal("monthly")), + pageSize: z.number().multipleOf(512).min(512).max(16384), + cacheSize: z.number().int().min(1000).max(1000000), + mmapSize: z.number().int().min(0), + }), + + // Skip location and stats as they're read-only + }, + { strict: true } +); + +export type DatabaseSettingsSchema = z.infer; + +// ... rest of the file ... diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts new file mode 100644 index 0000000000..18cf3e119f --- /dev/null +++ b/src/types/databaseSettings.ts @@ -0,0 +1,124 @@ +/** + * Database performance optimization settings stored in SQLite key-value pairs. + * User-configurable aggregation, retention, and optimization settings. + */ + +export interface DatabaseSettings { + /** 1. Location (read-only display) */ + location: { + databasePath: string; + dataDir: string; + walSizeBytes: number; + schemaVersion: number; + }; + + /** 2. Logs (what gets captured) */ + logs: { + detailedLogsEnabled: boolean; + callLogPipelineEnabled: boolean; + maxDetailSizeKb: number; + ringBufferSize: number; + }; + + /** 3. Backup (backup/restore/import/export) */ + backup: { + autoBackupEnabled: boolean; + autoBackupFrequency: "never" | "daily" | "weekly" | "monthly"; + keepLastNBackups: number; + }; + + /** 4. Cache (moved from CacheSettingsTab) */ + cache: { + semanticCacheEnabled: boolean; + semanticCacheMaxSize: number; + semanticCacheTTL: number; + promptCacheEnabled: boolean; + promptCacheStrategy: "auto" | "system-only" | "manual"; + alwaysPreserveClientCache: "auto" | "always" | "never"; + }; + + /** 5. Retention (per-table cleanup policies) */ + retention: { + quotaSnapshots: number; + compressionAnalytics: number; + mcpAudit: number; + a2aEvents: number; + callLogs: number; + usageHistory: number; + memoryEntries: number; + autoCleanupEnabled: boolean; + }; + + /** 6. Compression (aggregation) */ + aggregation: { + enabled: boolean; + rawDataRetentionDays: number; + granularity: "hourly" | "daily" | "weekly"; + }; + + /** 7. Optimization (auto_vacuum, VACUUM, page/cache) */ + optimization: { + autoVacuumMode: "NONE" | "FULL" | "INCREMENTAL"; + scheduledVacuum: "never" | "daily" | "weekly" | "monthly"; + vacuumHour: number; + pageSize: number; + cacheSize: number; + optimizeOnStartup: boolean; + }; + + /** Read-only stats */ + stats: { + databaseSizeBytes: number; + pageCount: number; + freelistCount: number; + lastVacuumAt: string | null; + lastOptimizationAt: string | null; + integrityCheck: "ok" | "error" | null; + }; +} + +/** Default database settings */ +export const DEFAULT_DATABASE_SETTINGS: Omit = { + logs: { + detailedLogsEnabled: false, + callLogPipelineEnabled: false, + maxDetailSizeKb: 10, + ringBufferSize: 500, + }, + backup: { + autoBackupEnabled: false, + autoBackupFrequency: "never", + keepLastNBackups: 5, + }, + cache: { + semanticCacheEnabled: true, + semanticCacheMaxSize: 100, + semanticCacheTTL: 1800000, + promptCacheEnabled: true, + promptCacheStrategy: "auto", + alwaysPreserveClientCache: "auto", + }, + retention: { + quotaSnapshots: 7, + compressionAnalytics: 30, + mcpAudit: 30, + a2aEvents: 30, + callLogs: 30, + usageHistory: 30, + memoryEntries: 30, + autoCleanupEnabled: true, + }, + aggregation: { + enabled: true, + rawDataRetentionDays: 30, + granularity: "daily", + }, + optimization: { + autoVacuumMode: "FULL", + scheduledVacuum: "weekly", + vacuumHour: 2, + pageSize: 4096, + cacheSize: -2000, + optimizeOnStartup: true, + }, +}; diff --git a/src/types/index.ts b/src/types/index.ts index ceb62101b9..ba28c26d70 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -10,3 +10,5 @@ export type { ApiKey } from "./apiKey"; export type { Combo, ComboStrategy, ComboNode } from "./combo"; export type { UsageEntry, UsageStats, ProviderUsageStats, ModelUsageStats, CallLog } from "./usage"; export type { Settings, ComboDefaults, ProxyConfig, KVPair } from "./settings"; +export type { DatabaseSettings } from "./databaseSettings"; +export { DEFAULT_DATABASE_SETTINGS } from "./databaseSettings";