feat(db): consolidate all database settings into SystemStorageTab (closes #1935) (#1947)

Integrated into release/v3.7.9
This commit is contained in:
Paijo
2026-05-05 05:25:41 +07:00
committed by GitHub
parent 3f900a833e
commit 6a45ea2c97
26 changed files with 2601 additions and 307 deletions

View File

@@ -87,6 +87,7 @@
"audit:electron": "npm --prefix electron audit --audit-level=moderate",
"typecheck:core": "tsc --pretty false -p tsconfig.typecheck-core.json",
"typecheck:noimplicit:core": "tsc --pretty false -p tsconfig.typecheck-noimplicit-core.json",
"backfill-aggregation": "node --import tsx/esm src/scripts/backfillAggregation.ts",
"env:sync": "node scripts/sync-env.mjs",
"test:integration": "node --import tsx/esm --test tests/integration/*.test.ts",
"test:e2e": "node scripts/run-playwright-tests.mjs test tests/e2e/*.spec.ts",

View File

@@ -1,191 +0,0 @@
"use client";
import { useState, useEffect } from "react";
import { Card, Button } from "@/shared/components";
import { useTranslations } from "next-intl";
interface CacheConfig {
semanticCacheEnabled: boolean;
semanticCacheMaxSize: number;
semanticCacheTTL: number;
promptCacheEnabled: boolean;
promptCacheStrategy: "auto" | "system-only" | "manual";
alwaysPreserveClientCache: "auto" | "always" | "never";
}
export default function CacheSettingsTab() {
const t = useTranslations("settings");
const [config, setConfig] = useState<CacheConfig>({
semanticCacheEnabled: true,
semanticCacheMaxSize: 100,
semanticCacheTTL: 1800000,
promptCacheEnabled: true,
promptCacheStrategy: "auto",
alwaysPreserveClientCache: "auto",
});
const [saving, setSaving] = useState(false);
const [loading, setLoading] = useState(true);
useEffect(() => {
fetch("/api/settings/cache-config")
.then((r) => (r.ok ? r.json() : null))
.then((data) => {
if (data) setConfig(data);
})
.catch(() => {})
.finally(() => setLoading(false));
}, []);
const handleSave = async () => {
setSaving(true);
try {
await fetch("/api/settings/cache-config", {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(config),
});
} finally {
setSaving(false);
}
};
if (loading) {
return (
<Card className="p-6">
<p className="text-sm text-text-muted">{t("loading")}</p>
</Card>
);
}
return (
<Card className="p-6">
<h3 className="text-lg font-semibold text-text-main flex items-center gap-2 mb-4">
<span className="material-symbols-outlined text-[20px]">cached</span>
{t("cacheSettings")}
</h3>
<div className="space-y-6">
{/* Semantic Cache */}
<div className="space-y-3">
<h4 className="text-sm font-medium text-text-main">{t("semanticCache")}</h4>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("enabled")}</span>
<button
onClick={() =>
setConfig((c) => ({ ...c, semanticCacheEnabled: !c.semanticCacheEnabled }))
}
className={`relative w-10 h-5 rounded-full transition-colors ${
config.semanticCacheEnabled ? "bg-green-500" : "bg-border"
}`}
>
<span
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
config.semanticCacheEnabled ? "left-5" : "left-0.5"
}`}
/>
</button>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("maxEntries")}</span>
<input
type="number"
min={1}
max={1000}
value={config.semanticCacheMaxSize}
onChange={(e) =>
setConfig((c) => ({ ...c, semanticCacheMaxSize: parseInt(e.target.value) || 100 }))
}
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
/>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("ttlMinutes")}</span>
<input
type="number"
min={1}
max={1440}
value={Math.round(config.semanticCacheTTL / 60000)}
onChange={(e) =>
setConfig((c) => ({
...c,
semanticCacheTTL: (parseInt(e.target.value) || 30) * 60000,
}))
}
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
/>
</label>
</div>
{/* Prompt Cache */}
<div className="space-y-3 pt-4 border-t border-border/30">
<h4 className="text-sm font-medium text-text-main">{t("promptCache")}</h4>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("enabled")}</span>
<button
onClick={() =>
setConfig((c) => ({ ...c, promptCacheEnabled: !c.promptCacheEnabled }))
}
className={`relative w-10 h-5 rounded-full transition-colors ${
config.promptCacheEnabled ? "bg-green-500" : "bg-border"
}`}
>
<span
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
config.promptCacheEnabled ? "left-5" : "left-0.5"
}`}
/>
</button>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("strategy")}</span>
<select
value={config.promptCacheStrategy}
onChange={(e) =>
setConfig((c) => ({
...c,
promptCacheStrategy: e.target.value as CacheConfig["promptCacheStrategy"],
}))
}
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
>
<option value="auto">Auto</option>
<option value="system-only">System Only</option>
<option value="manual">Manual</option>
</select>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("preserveClientCache")}</span>
<select
value={config.alwaysPreserveClientCache}
onChange={(e) =>
setConfig((c) => ({
...c,
alwaysPreserveClientCache: e.target
.value as CacheConfig["alwaysPreserveClientCache"],
}))
}
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
>
<option value="auto">Auto</option>
<option value="always">Always</option>
<option value="never">Never</option>
</select>
</label>
</div>
{/* Save */}
<div className="pt-4 border-t border-border/30">
<Button onClick={handleSave} disabled={saving} size="sm">
{saving ? t("saving") : t("save")}
</Button>
</div>
</div>
</Card>
);
}

View File

@@ -15,7 +15,6 @@ import ThinkingBudgetTab from "./components/ThinkingBudgetTab";
import SystemPromptTab from "./components/SystemPromptTab";
import ModelAliasesUnified from "./components/ModelAliasesUnified";
import BackgroundDegradationTab from "./components/BackgroundDegradationTab";
import CacheSettingsTab from "./components/CacheSettingsTab";
import MemorySkillsTab from "./components/MemorySkillsTab";
import ModelsDevSyncTab from "./components/ModelsDevSyncTab";
import ResilienceTab from "./components/ResilienceTab";
@@ -115,7 +114,6 @@ export default function SettingsPage() {
</Link>
<VisionBridgeSettingsTab />
<SystemPromptTab />
<CacheSettingsTab />
<MemorySkillsTab />
<ModelsDevSyncTab />
</div>

View File

@@ -0,0 +1,17 @@
import { NextRequest, NextResponse } from "next/server";
import { isAuthenticated } from "@/shared/utils/apiAuth";
import { getDatabaseStats } from "@/lib/db/stats";
export async function POST(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const stats = await getDatabaseStats();
return NextResponse.json({ success: true, stats });
} catch (error) {
console.error("Failed to refresh database stats:", error);
return NextResponse.json({ error: "Failed to refresh database stats" }, { status: 500 });
}
}

View File

@@ -0,0 +1,141 @@
import { NextRequest, NextResponse } from "next/server";
import { getSettings, updateSettings } from "@/lib/localDb";
import { isAuthenticated } from "@/shared/utils/apiAuth";
import { z } from "zod";
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
import { DatabaseSettings, DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
import { getDatabaseStats } from "@/lib/db/stats";
type UserSettableKeys = keyof Omit<DatabaseSettings, "location" | "stats">;
export async function GET(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const settings = await getSettings();
const dbStats = await getDatabaseStats();
// Get current settings from key_value table (user settings)
const userSettings: Partial<DatabaseSettings> = {};
const allKeys = await getAllUserSettableKeys();
for (const key of allKeys) {
const value = settings[key as keyof typeof settings];
if (value !== undefined) {
(userSettings as Record<string, unknown>)[key] = value;
}
}
// Merge with defaults and stats
const merged: DatabaseSettings = {
location: dbStats.location,
logs: { ...DEFAULT_DATABASE_SETTINGS.logs, ...userSettings.logs },
backup: { ...DEFAULT_DATABASE_SETTINGS.backup, ...userSettings.backup },
cache: { ...DEFAULT_DATABASE_SETTINGS.cache, ...userSettings.cache },
retention: { ...DEFAULT_DATABASE_SETTINGS.retention, ...userSettings.retention },
aggregation: { ...DEFAULT_DATABASE_SETTINGS.aggregation, ...userSettings.aggregation },
optimization: { ...DEFAULT_DATABASE_SETTINGS.optimization, ...userSettings.optimization },
stats: dbStats.stats,
};
return NextResponse.json(merged);
} catch (error) {
console.error("Error getting database settings:", error);
return NextResponse.json({ error: "Failed to load database settings" }, { status: 500 });
}
}
export async function PATCH(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const settings = await getSettings();
const userSettableKeys = await getUserSettableKeys();
const validation = await validateBody(
request,
z
.record(z.string(), z.unknown())
.partial()
.refine(
(data) => {
// Ensure only user-settable fields are present
return Object.keys(data).every((key) => userSettableKeys.includes(key));
},
{ message: "Attempting to set restricted fields" }
)
);
if (isValidationFailure(validation)) {
return validation;
}
const updates = validation.data;
const updatedSettings = { ...settings };
// Update only user-settable fields
for (const [key, value] of Object.entries(updates)) {
if (userSettableKeys.includes(key)) {
(updatedSettings as Record<string, unknown>)[key] = value;
}
}
await updateSettings(updatedSettings);
// Return merged settings (GET response pattern)
return await GET(request);
} catch (error) {
console.error("Error updating database settings:", error);
return NextResponse.json({ error: "Failed to update database settings" }, { status: 500 });
}
}
async function getUserSettableKeys(): Promise<UserSettableKeys[]> {
// These are the fields that users are allowed to modify
const allowedKeys: UserSettableKeys[] = [
// Logs
"logs",
// Backup
"backup",
// Cache
"cache",
// Retention
"retention",
// Aggregation
"aggregation",
// Optimization
"optimization",
];
return allowedKeys;
}
async function getAllUserSettableKeys(): Promise<string[]> {
const settableKeys: string[] = [];
const userSettableSections = await getUserSettableKeys();
// Get all nested keys under each user-settable section
const allDefaultKeys = Object.keys(DEFAULT_DATABASE_SETTINGS) as (keyof DatabaseSettings)[];
for (const section of userSettableSections) {
if (section in DEFAULT_DATABASE_SETTINGS) {
const sectionKeys = Object.keys(
DEFAULT_DATABASE_SETTINGS[section as keyof typeof DEFAULT_DATABASE_SETTINGS]
);
sectionKeys.forEach((key) => {
settableKeys.push(`${section}.${key}`);
});
}
}
return settableKeys;
}

View File

@@ -0,0 +1,36 @@
import { NextRequest, NextResponse } from "next/server";
import { isAuthenticated } from "@/shared/utils/apiAuth";
import { runManualVacuum } from "@/lib/db/core";
export async function POST(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const result = runManualVacuum();
if (result.success) {
return NextResponse.json({
success: true,
message: `VACUUM completed in ${result.duration}ms`,
duration: result.duration,
});
} else {
return NextResponse.json(
{
success: false,
error: result.error || "VACUUM failed",
duration: result.duration,
},
{ status: 500 }
);
}
} catch (error: any) {
console.error("[API] VACUUM endpoint error:", error);
return NextResponse.json(
{ error: "Failed to run VACUUM", details: error.message },
{ status: 500 }
);
}
}

View File

@@ -0,0 +1,19 @@
import { NextResponse } from "next/server";
import { purgeCallLogs } from "@/lib/db/cleanup";
import { isAuthenticated } from "@/shared/utils/apiAuth";
export async function POST(request: Request) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const result = await purgeCallLogs();
return NextResponse.json({
deleted: result.deleted,
errors: result.errors,
});
} catch (err: unknown) {
const error = err instanceof Error ? err.message : String(err);
return NextResponse.json({ error }, { status: 500 });
}
}

View File

@@ -0,0 +1,19 @@
import { NextResponse } from "next/server";
import { purgeDetailedLogs } from "@/lib/db/cleanup";
import { isAuthenticated } from "@/shared/utils/apiAuth";
export async function POST(request: Request) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const result = await purgeDetailedLogs();
return NextResponse.json({
deleted: result.deleted,
errors: result.errors,
});
} catch (err: unknown) {
const error = err instanceof Error ? err.message : String(err);
return NextResponse.json({ error }, { status: 500 });
}
}

View File

@@ -0,0 +1,19 @@
import { NextResponse } from "next/server";
import { purgeQuotaSnapshots } from "@/lib/db/cleanup";
import { isAuthenticated } from "@/shared/utils/apiAuth";
export async function POST(request: Request) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const result = await purgeQuotaSnapshots();
return NextResponse.json({
deleted: result.deleted,
errors: result.errors,
});
} catch (err: unknown) {
const error = err instanceof Error ? err.message : String(err);
return NextResponse.json({ error }, { status: 500 });
}
}

View File

@@ -1,8 +1,9 @@
import { NextResponse } from "next/server";
import { getSettings, updateSettings } from "@/lib/localDb";
import { getRuntimePorts } from "@/lib/runtime/ports";
import { updateSettingsSchema } from "@/shared/validation/settingsSchemas";
import { databaseSettingsSchema } from "@/shared/validation/settingsSchemas";
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
import { z } from "zod";
import { getConsistentMachineId } from "@/shared/utils/machineId";
import { validateProxyUrl, upsertUpstreamProxyConfig } from "@/lib/db/upstreamProxy";
import {

342
src/lib/db/cleanup.ts Normal file
View File

@@ -0,0 +1,342 @@
/**
* Database cleanup functions for removing old data based on retention policies.
*
* @module lib/db/cleanup
*/
import { getDbInstance } from "./core";
import { getSettings } from "@/lib/localDb";
import type { DatabaseSettings } from "@/types/databaseSettings";
interface CleanupResult {
deleted: number;
errors: number;
}
/**
* Extract database settings from the full settings object with proper typing.
*/
function getDatabaseSettings(): DatabaseSettings["retention"] {
const settings = getSettings();
// Database settings are stored under the 'databaseSettings' key in the main settings
const dbSettings = (settings as Record<string, unknown>).databaseSettings as
| DatabaseSettings
| undefined;
return (
dbSettings?.retention ?? {
quotaSnapshots: 30,
compressionAnalytics: 30,
mcpAudit: 30,
a2aEvents: 30,
callLogs: 7,
usageHistory: 90,
memoryEntries: 90,
autoCleanupEnabled: false,
}
);
}
/**
* Clean up old quota_snapshots based on retention settings.
*/
export async function cleanupQuotaSnapshots(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.quotaSnapshots;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM quota_snapshots WHERE created_at < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(
`[Cleanup] Deleted ${result.deleted} quota_snapshots older than ${retentionDays} days`
);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning quota_snapshots:", err);
result.errors++;
}
return result;
}
/**
* Clean up old call_logs based on retention settings.
*/
export async function cleanupCallLogs(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.callLogs;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM call_logs WHERE created_at < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(`[Cleanup] Deleted ${result.deleted} call_logs older than ${retentionDays} days`);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning call_logs:", err);
result.errors++;
}
return result;
}
/**
* Clean up old usage_history based on retention settings.
*/
export async function cleanupUsageHistory(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.usageHistory;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM usage_history WHERE timestamp < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(
`[Cleanup] Deleted ${result.deleted} usage_history older than ${retentionDays} days`
);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning usage_history:", err);
result.errors++;
}
return result;
}
/**
* Clean up old compression_analytics based on retention settings.
*/
export async function cleanupCompressionAnalytics(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.compressionAnalytics;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM compression_analytics WHERE created_at < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(
`[Cleanup] Deleted ${result.deleted} compression_analytics older than ${retentionDays} days`
);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning compression_analytics:", err);
result.errors++;
}
return result;
}
/**
* Clean up old mcp_audit_log based on retention settings.
*/
export async function cleanupMcpAudit(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.mcpAudit;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM mcp_audit_log WHERE timestamp < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(
`[Cleanup] Deleted ${result.deleted} mcp_audit_log older than ${retentionDays} days`
);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning mcp_audit_log:", err);
result.errors++;
}
return result;
}
/**
* Clean up old a2a_events based on retention settings.
*/
export async function cleanupA2aEvents(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.a2aEvents;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM a2a_events WHERE timestamp < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(`[Cleanup] Deleted ${result.deleted} a2a_events older than ${retentionDays} days`);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning a2a_events:", err);
result.errors++;
}
return result;
}
/**
* Clean up old memory_entries based on retention settings.
*/
export async function cleanupMemoryEntries(): Promise<CleanupResult> {
const db = getDbInstance();
const retention = getDatabaseSettings();
const retentionDays = retention.memoryEntries;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM memory_entries WHERE created_at < ?");
const runResult = stmt.run(cutoffISO);
result.deleted = runResult.changes;
console.log(
`[Cleanup] Deleted ${result.deleted} memory_entries older than ${retentionDays} days`
);
} catch (err: unknown) {
console.error("[Cleanup] Error cleaning memory_entries:", err);
result.errors++;
}
return result;
}
/**
* Run all cleanup functions if auto-cleanup is enabled.
*/
export async function runAutoCleanup(): Promise<{
totalDeleted: number;
totalErrors: number;
results: Record<string, CleanupResult>;
}> {
const retention = getDatabaseSettings();
const autoCleanupEnabled = retention.autoCleanupEnabled;
if (!autoCleanupEnabled) {
console.log("[Cleanup] Auto-cleanup is disabled");
return { totalDeleted: 0, totalErrors: 0, results: {} };
}
console.log("[Cleanup] Starting auto-cleanup...");
const results: Record<string, CleanupResult> = {
quotaSnapshots: await cleanupQuotaSnapshots(),
callLogs: await cleanupCallLogs(),
usageHistory: await cleanupUsageHistory(),
compressionAnalytics: await cleanupCompressionAnalytics(),
mcpAudit: await cleanupMcpAudit(),
a2aEvents: await cleanupA2aEvents(),
memoryEntries: await cleanupMemoryEntries(),
};
const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0);
const totalErrors = Object.values(results).reduce((sum, r) => sum + r.errors, 0);
console.log(`[Cleanup] Auto-cleanup complete: ${totalDeleted} deleted, ${totalErrors} errors`);
return { totalDeleted, totalErrors, results };
}
/**
* Purge ALL quota_snapshots immediately (no retention check).
*/
export async function purgeQuotaSnapshots(): Promise<CleanupResult> {
const db = getDbInstance();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM quota_snapshots");
const runResult = stmt.run();
result.deleted = runResult.changes;
console.log(`[Cleanup] Purged ${result.deleted} quota_snapshots`);
} catch (err: unknown) {
console.error("[Cleanup] Error purging quota_snapshots:", err);
result.errors++;
}
return result;
}
/**
* Purge ALL call_logs immediately (no retention check).
*/
export async function purgeCallLogs(): Promise<CleanupResult> {
const db = getDbInstance();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM call_logs");
const runResult = stmt.run();
result.deleted = runResult.changes;
console.log(`[Cleanup] Purged ${result.deleted} call_logs`);
} catch (err: unknown) {
console.error("[Cleanup] Error purging call_logs:", err);
result.errors++;
}
return result;
}
/**
* Purge ALL detailed_logs immediately (no retention check).
*/
export async function purgeDetailedLogs(): Promise<CleanupResult> {
const db = getDbInstance();
const result: CleanupResult = { deleted: 0, errors: 0 };
try {
const stmt = db.prepare("DELETE FROM detailed_logs");
const runResult = stmt.run();
result.deleted = runResult.changes;
console.log(`[Cleanup] Purged ${result.deleted} detailed_logs`);
} catch (err: unknown) {
console.error("[Cleanup] Error purging detailed_logs:", err);
result.errors++;
}
return result;
}

View File

@@ -0,0 +1,100 @@
/**
* Database compression scheduler - runs compression tasks based on settings.
*
* @module lib/db/compressionScheduler
*/
import { getDbInstance } from "./core";
import { getSettings } from "@/lib/localDb";
interface CompressionScheduleSettings {
enabled: boolean;
intervalHours: number;
lastRun?: string;
}
/**
* Run scheduled compression based on database settings.
* Should be called on startup and periodically.
*/
export async function runScheduledCompression(): Promise<void> {
const db = getDbInstance();
const settings = await getSettings();
const compressionSettings = (settings.databaseSettings as any)?.compression as
| CompressionScheduleSettings
| undefined;
if (!compressionSettings?.enabled) {
console.log("[CompressionScheduler] Compression scheduling is disabled");
return;
}
const intervalHours = compressionSettings.intervalHours ?? 24;
const lastRun = compressionSettings.lastRun ? new Date(compressionSettings.lastRun) : null;
const now = new Date();
const hoursSinceLastRun = lastRun
? (now.getTime() - lastRun.getTime()) / (1000 * 60 * 60)
: Infinity;
if (hoursSinceLastRun < intervalHours) {
console.log(
`[CompressionScheduler] Skipping compression - last run was ${hoursSinceLastRun.toFixed(1)}h ago (interval: ${intervalHours}h)`
);
return;
}
console.log("[CompressionScheduler] Running scheduled compression...");
try {
// Run VACUUM to reclaim space
db.prepare("VACUUM").run();
console.log("[CompressionScheduler] VACUUM completed");
// Run ANALYZE to update statistics
db.prepare("ANALYZE").run();
console.log("[CompressionScheduler] ANALYZE completed");
const updateStmt = db.prepare(`
INSERT OR REPLACE INTO key_value (namespace, key, value)
VALUES ('settings', 'databaseSettings', json_set(
COALESCE((SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'databaseSettings'), '{}'),
'$.compression.lastRun',
?
))
`);
updateStmt.run(now.toISOString());
console.log("[CompressionScheduler] Compression completed successfully");
} catch (err: any) {
console.error("[CompressionScheduler] Error during compression:", err);
throw err;
}
}
/**
* Initialize compression scheduler on startup.
* Call this once when the application starts.
*/
export async function initCompressionScheduler(): Promise<void> {
console.log("[CompressionScheduler] Initializing compression scheduler...");
try {
await runScheduledCompression();
} catch (err: any) {
console.error("[CompressionScheduler] Failed to run initial compression:", err);
}
// Set up periodic check (every hour)
setInterval(
async () => {
try {
await runScheduledCompression();
} catch (err: any) {
console.error("[CompressionScheduler] Periodic compression check failed:", err);
}
},
60 * 60 * 1000
); // 1 hour
}

View File

@@ -1481,3 +1481,96 @@ function migrateFromJson(db: SqliteDatabase, jsonPath: string) {
console.error("[DB] Migration from db.json failed:", err.message);
}
}
// ──────────────── Auto-Vacuum Management ────────────────
export function setAutoVacuum(mode: "NONE" | "FULL" | "INCREMENTAL"): void {
const db = getDbInstance();
const currentMode = db.pragma("auto_vacuum", { simple: true }) as number;
const modeMap: Record<string, number> = {
NONE: 0,
FULL: 1,
INCREMENTAL: 2,
};
const targetMode = modeMap[mode];
if (currentMode === targetMode) {
console.log(`[DB] auto_vacuum already set to ${mode}`);
return;
}
console.log(`[DB] Changing auto_vacuum from ${currentMode} to ${mode} (${targetMode})`);
db.pragma(`auto_vacuum = ${targetMode}`);
db.exec("VACUUM");
const newMode = db.pragma("auto_vacuum", { simple: true }) as number;
console.log(`[DB] auto_vacuum changed to ${newMode}`);
}
export function getAutoVacuumMode(): "NONE" | "FULL" | "INCREMENTAL" {
const db = getDbInstance();
const mode = db.pragma("auto_vacuum", { simple: true }) as number;
const modeMap: Record<number, "NONE" | "FULL" | "INCREMENTAL"> = {
0: "NONE",
1: "FULL",
2: "INCREMENTAL",
};
return modeMap[mode] || "NONE";
}
export function runManualVacuum(): { success: boolean; duration: number; error?: string } {
const db = getDbInstance();
const startTime = Date.now();
try {
console.log("[DB] Starting manual VACUUM...");
db.exec("VACUUM");
const duration = Date.now() - startTime;
console.log(`[DB] Manual VACUUM completed in ${duration}ms`);
return { success: true, duration };
} catch (err: any) {
const duration = Date.now() - startTime;
console.error("[DB] Manual VACUUM failed:", err);
return { success: false, duration, error: err.message };
}
}
export function setPageSize(pageSize: number): void {
const db = getDbInstance();
const currentPageSize = db.pragma("page_size", { simple: true }) as number;
if (currentPageSize === pageSize) {
console.log(`[DB] page_size already set to ${pageSize}`);
return;
}
console.log(`[DB] Changing page_size from ${currentPageSize} to ${pageSize}`);
db.pragma(`page_size = ${pageSize}`);
db.exec("VACUUM");
const newPageSize = db.pragma("page_size", { simple: true }) as number;
console.log(`[DB] page_size changed to ${newPageSize}`);
}
export function setCacheSize(cacheSizeKb: number): void {
const db = getDbInstance();
const currentCacheSize = db.pragma("cache_size", { simple: true }) as number;
const targetCacheSize = -cacheSizeKb;
if (currentCacheSize === targetCacheSize) {
console.log(`[DB] cache_size already set to ${cacheSizeKb}KB`);
return;
}
console.log(`[DB] Changing cache_size from ${Math.abs(currentCacheSize)}KB to ${cacheSizeKb}KB`);
db.pragma(`cache_size = ${targetCacheSize}`);
const newCacheSize = db.pragma("cache_size", { simple: true }) as number;
console.log(`[DB] cache_size changed to ${Math.abs(newCacheSize)}KB`);
}

View File

@@ -18,6 +18,7 @@ import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import type Database from "better-sqlite3";
import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
/**
* Resolve the migrations directory path safely across platforms.
@@ -799,9 +800,44 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole
console.log(`[Migration] ${count} migration(s) applied successfully.`);
}
// After applying all migrations, insert default settings if we just ran migration 46
try {
if (appliedRecords.some((m) => m.name.startsWith("046_"))) {
insertDefaultDatabaseSettings(db);
}
} catch (error) {
console.error("Error inserting default database settings:", error);
}
return count;
}
function insertDefaultDatabaseSettings(db: Database.Database) {
const tx = db.transaction(() => {
// Insert all default settings
for (const [section, values] of Object.entries(DEFAULT_DATABASE_SETTINGS)) {
for (const [key, value] of Object.entries(values as Record<string, unknown>)) {
db.prepare("INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
"databaseSettings",
`${section}.${key}`,
JSON.stringify(value)
);
}
}
});
// Run in an immediate transaction to avoid nested transactions
try {
// @ts-expect-error - Better-SQLite3 transaction types
db.immediate(() => {
tx();
});
} catch (error) {
console.error("Transaction error inserting default settings:", error);
throw error;
}
}
/**
* Get migration status for diagnostics.
*/

View File

@@ -0,0 +1,44 @@
-- 046_database_settings.sql
-- Insert default database settings into key_value table (namespace='databaseSettings')
-- Uses INSERT OR IGNORE so existing user settings are never overwritten by migration replay.
-- Logs settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'detailedLogsEnabled', 'true');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogPipelineEnabled', 'true');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'maxDetailSizeKb', '500');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'ringBufferSize', '1000');
-- Backup settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupEnabled', 'false');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupFrequency', '"never"');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'keepLastNBackups', '3');
-- Cache settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheEnabled', 'true');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheMaxSize', '100');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheTTL', '1800000');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheEnabled', 'true');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheStrategy', '"auto"');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'alwaysPreserveClientCache', '"auto"');
-- Retention settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'quotaSnapshots', '90');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'compressionAnalytics', '30');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mcpAudit', '30');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'a2aEvents', '30');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogs', '90');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'usageHistory', '365');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'memoryEntries', '180');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoCleanupEnabled', 'true');
-- Aggregation settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'aggregationEnabled', 'false');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'rawDataRetentionDays', '7');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'granularity', '"daily"');
-- Optimization settings
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoVacuumMode', '"INCREMENTAL"');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'scheduledVacuum', '"weekly"');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'pageSize', '4096');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'cacheSize', '10000');
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mmapSize', '268435456');

View File

@@ -0,0 +1,44 @@
-- 047_aggregation_tables.sql
-- Create aggregation tables for usage data summarization
-- Hourly usage summary table
CREATE TABLE IF NOT EXISTS hourly_usage_summary (
id INTEGER PRIMARY KEY AUTOINCREMENT,
provider TEXT NOT NULL,
model TEXT NOT NULL,
date_hour TEXT NOT NULL, -- Format: YYYY-MM-DD HH:00:00
total_requests INTEGER NOT NULL DEFAULT 0,
total_input_tokens INTEGER NOT NULL DEFAULT 0,
total_output_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0.0,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
-- Index for efficient queries by provider, model, and time range
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date
ON hourly_usage_summary(provider, model, date_hour);
-- Index for time-based queries
CREATE INDEX IF NOT EXISTS idx_hourly_usage_date
ON hourly_usage_summary(date_hour);
-- Daily usage summary table
CREATE TABLE IF NOT EXISTS daily_usage_summary (
id INTEGER PRIMARY KEY AUTOINCREMENT,
provider TEXT NOT NULL,
model TEXT NOT NULL,
date TEXT NOT NULL, -- Format: YYYY-MM-DD
total_requests INTEGER NOT NULL DEFAULT 0,
total_input_tokens INTEGER NOT NULL DEFAULT 0,
total_output_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0.0,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
-- Index for efficient queries by provider, model, and date
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date
ON daily_usage_summary(provider, model, date);
-- Index for date-based queries
CREATE INDEX IF NOT EXISTS idx_daily_usage_date
ON daily_usage_summary(date);

View File

@@ -0,0 +1,29 @@
-- 048_summary_indexes.sql
-- Add composite indexes for efficient querying of summary tables
-- Composite indexes for daily_usage_summary
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_date
ON daily_usage_summary(provider, date);
CREATE INDEX IF NOT EXISTS idx_daily_usage_model_date
ON daily_usage_summary(model, date);
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date_composite
ON daily_usage_summary(provider, model, date);
-- Composite indexes for hourly_usage_summary
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_date
ON hourly_usage_summary(provider, date_hour);
CREATE INDEX IF NOT EXISTS idx_hourly_usage_model_date
ON hourly_usage_summary(model, date_hour);
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date_composite
ON hourly_usage_summary(provider, model, date_hour);
-- Add unique constraint to prevent duplicate aggregations
CREATE UNIQUE INDEX IF NOT EXISTS idx_daily_usage_unique
ON daily_usage_summary(provider, model, date);
CREATE UNIQUE INDEX IF NOT EXISTS idx_hourly_usage_unique
ON hourly_usage_summary(provider, model, date_hour);

View File

@@ -0,0 +1,10 @@
-- Migration 049: Add indexes to compression_analytics table for performance
CREATE INDEX IF NOT EXISTS idx_compression_analytics_timestamp
ON compression_analytics(timestamp);
CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider
ON compression_analytics(provider);
CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider_timestamp
ON compression_analytics(provider, timestamp);

71
src/lib/db/stats.ts Normal file
View File

@@ -0,0 +1,71 @@
/**
* Database Statistics Module
*
* Provides functions to retrieve database statistics including size, table counts, and performance metrics.
*/
import type Database from "better-sqlite3";
import { getDbInstance } from "./core";
export interface DatabaseStats {
totalSize: number;
pageSize: number;
pageCount: number;
tables: Array<{
name: string;
rowCount: number;
size: number;
}>;
indexes: Array<{
name: string;
tableName: string;
}>;
walSize?: number;
cacheSize: number;
}
export function getDatabaseStats(): DatabaseStats {
const db = getDbInstance();
const pageSize = db.pragma("page_size", { simple: true }) as number;
const pageCount = db.pragma("page_count", { simple: true }) as number;
const cacheSize = db.pragma("cache_size", { simple: true }) as number;
const totalSize = pageSize * pageCount;
const tables = db
.prepare(
`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`
)
.all() as Array<{ name: string }>;
const tableStats = tables.map((table) => {
const rowCount = db.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get() as {
count: number;
};
const tableSize = db
.prepare(`SELECT SUM(pgsize) as size FROM dbstat WHERE name = ?`)
.get(table.name) as { size: number | null };
return {
name: table.name,
rowCount: rowCount.count,
size: tableSize?.size || 0,
};
});
const indexes = db
.prepare(
`SELECT name, tbl_name as tableName FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name`
)
.all() as Array<{ name: string; tableName: string }>;
return {
totalSize,
pageSize,
pageCount,
tables: tableStats,
indexes,
cacheSize,
};
}

View File

@@ -0,0 +1,163 @@
/**
* Aggregation utility functions for usage data summarization.
* Rolls up quota_snapshots into hourly and daily summary tables.
*
* @module lib/usage/aggregateHistory
*/
import { getDbInstance } from "../db/core";
import { getSettings } from "@/lib/localDb";
interface AggregationResult {
processed: number;
inserted: number;
errors: number;
}
/**
* Roll up quota_snapshots into daily_usage_summary table.
* Aggregates by provider, model, and date.
*
* @param fromDate - Start date (YYYY-MM-DD format)
* @param toDate - End date (YYYY-MM-DD format)
* @returns Aggregation result with counts
*/
export async function rollupDailyUsage(
fromDate: string,
toDate: string
): Promise<AggregationResult> {
const db = getDbInstance();
const settings = await getSettings();
// Get retention settings
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
const result: AggregationResult = {
processed: 0,
inserted: 0,
errors: 0,
};
try {
// Aggregate quota_snapshots by provider, model, and date
const aggregateQuery = `
INSERT INTO daily_usage_summary (provider, model, date, total_requests, total_input_tokens, total_output_tokens, total_cost)
SELECT
provider,
COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model,
DATE(created_at) as date,
COUNT(*) as total_requests,
COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens,
COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens,
COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost
FROM quota_snapshots
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
GROUP BY provider, model, DATE(created_at)
ON CONFLICT(provider, model, date) DO UPDATE SET
total_requests = total_requests + excluded.total_requests,
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
total_cost = total_cost + excluded.total_cost
`;
const stmt = db.prepare(aggregateQuery);
const runResult = stmt.run(fromDate, toDate);
result.processed = runResult.changes;
result.inserted = runResult.changes;
console.log(`[Aggregation] Daily rollup: ${result.inserted} rows for ${fromDate} to ${toDate}`);
} catch (err: any) {
console.error("[Aggregation] Daily rollup error:", err);
result.errors++;
}
return result;
}
/**
* Roll up quota_snapshots into hourly_usage_summary table.
* Aggregates by provider, model, and hour.
*
* @param fromDate - Start datetime (YYYY-MM-DD HH:MM:SS format)
* @param toDate - End datetime (YYYY-MM-DD HH:MM:SS format)
* @returns Aggregation result with counts
*/
export async function rollupHourlyQuota(
fromDate: string,
toDate: string
): Promise<AggregationResult> {
const db = getDbInstance();
const settings = await getSettings();
// Get retention settings
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
const result: AggregationResult = {
processed: 0,
inserted: 0,
errors: 0,
};
try {
// Aggregate quota_snapshots by provider, model, and hour
const aggregateQuery = `
INSERT INTO hourly_usage_summary (provider, model, date_hour, total_requests, total_input_tokens, total_output_tokens, total_cost)
SELECT
provider,
COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model,
datetime(strftime('%Y-%m-%d %H:00:00', created_at)) as date_hour,
COUNT(*) as total_requests,
COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens,
COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens,
COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost
FROM quota_snapshots
WHERE created_at >= ? AND created_at <= ?
GROUP BY provider, model, datetime(strftime('%Y-%m-%d %H:00:00', created_at))
ON CONFLICT(provider, model, date_hour) DO UPDATE SET
total_requests = total_requests + excluded.total_requests,
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
total_cost = total_cost + excluded.total_cost
`;
const stmt = db.prepare(aggregateQuery);
const runResult = stmt.run(fromDate, toDate);
result.processed = runResult.changes;
result.inserted = runResult.changes;
console.log(
`[Aggregation] Hourly rollup: ${result.inserted} rows for ${fromDate} to ${toDate}`
);
} catch (err: any) {
console.error("[Aggregation] Hourly rollup error:", err);
result.errors++;
}
return result;
}
/**
* Get the cutoff date for raw data based on retention settings.
* Data older than this should be aggregated and cleaned up.
*
* @returns ISO date string (YYYY-MM-DD)
*/
export async function getRawDataCutoffDate(): Promise<string> {
const settings = await getSettings();
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - rawDataRetentionDays);
return cutoffDate.toISOString().split("T")[0];
}
/**
* Check if aggregation is enabled in settings.
*/
export async function isAggregationEnabled(): Promise<boolean> {
const settings = await getSettings();
return (settings.aggregation as any)?.enabled ?? false;
}

View File

@@ -11,6 +11,7 @@ import { getDbInstance } from "../db/core";
import { getPendingRequests } from "./usageHistory";
import { getAccountDisplayName } from "@/lib/display/names";
import { calculateCost } from "./costCalculator";
import { getRawDataCutoffDate, isAggregationEnabled } from "./aggregateHistory";
type JsonRecord = Record<string, unknown>;
type UsageBucket = {
@@ -56,10 +57,58 @@ function toStringOrEmpty(value: unknown): string {
/**
* Get aggregated usage stats.
* Uses UNION of recent raw data and older aggregated data when aggregation is enabled.
*/
export async function getUsageStats() {
const db = getDbInstance();
const rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[];
const aggregationEnabled = await isAggregationEnabled();
let rows: unknown[];
if (aggregationEnabled) {
const cutoffDate = await getRawDataCutoffDate();
// UNION: recent raw data + older aggregated data
const unionQuery = `
SELECT
provider,
model,
timestamp,
connection_id,
api_key_id,
api_key_name,
tokens_input,
tokens_output,
tokens_cache_read,
tokens_cache_creation,
tokens_reasoning
FROM usage_history
WHERE DATE(timestamp) >= ?
UNION ALL
SELECT
provider,
model,
date || ' 12:00:00' as timestamp,
NULL as connection_id,
NULL as api_key_id,
NULL as api_key_name,
total_input_tokens as tokens_input,
total_output_tokens as tokens_output,
0 as tokens_cache_read,
0 as tokens_cache_creation,
0 as tokens_reasoning
FROM daily_usage_summary
WHERE date < ?
ORDER BY timestamp ASC
`;
rows = db.prepare(unionQuery).all(cutoffDate, cutoffDate) as unknown[];
} else {
rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[];
}
const { getProviderConnections } = await import("@/lib/localDb");
let allConnections: unknown[] = [];

View File

@@ -0,0 +1,123 @@
#!/usr/bin/env node
import { rollupDailyUsage, rollupHourlyQuota } from "@/lib/usage/aggregateHistory";
interface BackfillOptions {
from: string;
to: string;
granularity?: "hourly" | "daily" | "both";
}
function parseArgs(): BackfillOptions {
const args = process.argv.slice(2);
const options: Partial<BackfillOptions> = {
granularity: "both",
};
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg.startsWith("--from=")) {
options.from = arg.split("=")[1];
} else if (arg.startsWith("--to=")) {
options.to = arg.split("=")[1];
} else if (arg.startsWith("--granularity=")) {
const value = arg.split("=")[1];
if (value === "hourly" || value === "daily" || value === "both") {
options.granularity = value;
}
}
}
if (!options.from || !options.to) {
console.error(
"Usage: npm run backfill-aggregation -- --from=YYYY-MM-DD --to=YYYY-MM-DD [--granularity=hourly|daily|both]"
);
process.exit(1);
}
return options as BackfillOptions;
}
function validateDate(dateStr: string): boolean {
const regex = /^\d{4}-\d{2}-\d{2}$/;
if (!regex.test(dateStr)) return false;
const date = new Date(dateStr);
return date instanceof Date && !isNaN(date.getTime());
}
async function backfillAggregation() {
const options = parseArgs();
if (!validateDate(options.from) || !validateDate(options.to)) {
console.error("Error: Dates must be in YYYY-MM-DD format");
process.exit(1);
}
const fromDate = new Date(options.from);
const toDate = new Date(options.to);
if (fromDate > toDate) {
console.error("Error: --from date must be before --to date");
process.exit(1);
}
console.log("=".repeat(60));
console.log("Aggregation Backfill Started");
console.log("=".repeat(60));
console.log(`From: ${options.from}`);
console.log(`To: ${options.to}`);
console.log(`Granularity: ${options.granularity}`);
console.log("=".repeat(60));
const startTime = Date.now();
let totalProcessed = 0;
let totalInserted = 0;
let totalErrors = 0;
try {
if (options.granularity === "daily" || options.granularity === "both") {
console.log("\n[Daily Aggregation] Starting...");
const dailyResult = await rollupDailyUsage(options.from, options.to);
totalProcessed += dailyResult.processed;
totalInserted += dailyResult.inserted;
totalErrors += dailyResult.errors;
console.log(
`[Daily Aggregation] Processed: ${dailyResult.processed}, Inserted: ${dailyResult.inserted}, Errors: ${dailyResult.errors}`
);
}
if (options.granularity === "hourly" || options.granularity === "both") {
console.log("\n[Hourly Aggregation] Starting...");
const fromDateTime = `${options.from} 00:00:00`;
const toDateTime = `${options.to} 23:59:59`;
const hourlyResult = await rollupHourlyQuota(fromDateTime, toDateTime);
totalProcessed += hourlyResult.processed;
totalInserted += hourlyResult.inserted;
totalErrors += hourlyResult.errors;
console.log(
`[Hourly Aggregation] Processed: ${hourlyResult.processed}, Inserted: ${hourlyResult.inserted}, Errors: ${hourlyResult.errors}`
);
}
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
console.log("\n" + "=".repeat(60));
console.log("Aggregation Backfill Complete");
console.log("=".repeat(60));
console.log(`Total Processed: ${totalProcessed}`);
console.log(`Total Inserted: ${totalInserted}`);
console.log(`Total Errors: ${totalErrors}`);
console.log(`Duration: ${duration}s`);
console.log("=".repeat(60));
if (totalErrors > 0) {
process.exit(1);
}
} catch (error) {
console.error("\n[FATAL ERROR]", error);
process.exit(1);
}
}
backfillAggregation();

View File

@@ -1,99 +1,73 @@
/**
* Settings-specific Zod schemas.
*
* Extracted from schemas.ts to work around the webpack barrel-file
* optimization bug that makes large schema barrel exports `undefined`
* at runtime (see: https://github.com/vercel/next.js/issues/12557).
*/
import { z } from "zod";
import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility";
import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
// ... existing imports ...
const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const;
export const databaseSettingsSchema = z.object(
{
// Logs settings
logs: z.object({
detailedLogsEnabled: z.boolean(),
callLogPipelineEnabled: z.boolean(),
maxDetailSizeKb: z.number().int().nonnegative(),
ringBufferSize: z.number().int().min(100).max(10000),
}),
export const updateSettingsSchema = z.object({
newPassword: z.string().min(1).max(200).optional(),
currentPassword: z.string().max(200).optional(),
theme: z.string().max(50).optional(),
language: z.string().max(10).optional(),
requireLogin: z.boolean().optional(),
enableSocks5Proxy: z.boolean().optional(),
instanceName: z.string().max(100).optional(),
customLogoUrl: z.string().max(2000).optional(),
customLogoBase64: z.string().max(100000).optional(),
customFaviconUrl: z.string().max(2000).optional(),
customFaviconBase64: z.string().max(50000).optional(),
corsOrigins: z.string().max(500).optional(),
cloudUrl: z.string().max(500).optional(),
baseUrl: z.string().max(500).optional(),
setupComplete: z.boolean().optional(),
blockedProviders: z.array(z.string().max(100)).optional(),
hideHealthCheckLogs: z.boolean().optional(),
hideEndpointCloudflaredTunnel: z.boolean().optional(),
hideEndpointTailscaleFunnel: z.boolean().optional(),
hideEndpointNgrokTunnel: z.boolean().optional(),
debugMode: z.boolean().optional(),
hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(),
comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(),
// Routing settings (#134)
fallbackStrategy: z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES).optional(),
wildcardAliases: z.array(z.object({ pattern: z.string(), target: z.string() })).optional(),
stickyRoundRobinLimit: z.number().int().min(0).max(1000).optional(),
requestRetry: z.number().int().min(0).max(10).optional(),
maxRetryIntervalSec: z.number().int().min(0).max(300).optional(),
// Auto intent classifier settings (multilingual routing)
intentDetectionEnabled: z.boolean().optional(),
intentSimpleMaxWords: z.number().int().min(1).max(500).optional(),
intentExtraCodeKeywords: z.array(z.string().max(100)).optional(),
intentExtraReasoningKeywords: z.array(z.string().max(100)).optional(),
intentExtraSimpleKeywords: z.array(z.string().max(100)).optional(),
// Protocol toggles (default: disabled)
mcpEnabled: z.boolean().optional(),
mcpTransport: z.enum(["stdio", "sse", "streamable-http"]).optional(),
a2aEnabled: z.boolean().optional(),
wsAuth: z.boolean().optional(),
// CLI Fingerprint compatibility (per-provider)
cliCompatProviders: z.array(z.string().max(100)).optional(),
// Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4")
stripModelPrefix: z.boolean().optional(),
// Cache control preservation mode
alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(),
antigravitySignatureCacheMode: z.enum(signatureCacheModeValues).optional(),
// Adaptive Volume Routing
adaptiveVolumeRouting: z.boolean().optional(),
// Usage token buffer — safety margin added to reported prompt/input token counts.
// Prevents CLI tools from overrunning context windows. Set to 0 to disable.
usageTokenBuffer: z.number().int().min(0).max(50000).optional(),
// Custom CLI agent definitions for ACP
customAgents: z
.array(
z.object({
id: z.string().max(50),
name: z.string().max(100),
binary: z.string().max(200),
versionCommand: z.string().max(300),
providerAlias: z.string().max(50),
spawnArgs: z.array(z.string().max(200)),
protocol: z.enum(["stdio", "http"]),
})
)
.optional(),
// SkillsMP marketplace API key
skillsmpApiKey: z.string().max(200).optional(),
// Active skills provider (single source of truth for skills page)
skillsProvider: z.enum(["skillsmp", "skillssh"]).optional(),
// models.dev sync settings
modelsDevSyncEnabled: z.boolean().optional(),
modelsDevSyncInterval: z.number().int().min(3600000).max(604800000).optional(),
// Vision Bridge settings
visionBridgeEnabled: z.boolean().optional(),
visionBridgeModel: z.string().max(200).optional(),
visionBridgePrompt: z.string().max(5000).optional(),
visionBridgeTimeout: z.number().int().min(1000).max(300000).optional(),
visionBridgeMaxImages: z.number().int().min(1).max(20).optional(),
// Missing settings
lkgpEnabled: z.boolean().optional(),
backgroundDegradation: z.unknown().optional(),
bruteForceProtection: z.boolean().optional(),
});
// Backup settings
backup: z.object({
autoBackupEnabled: z.boolean(),
autoBackupFrequency: z
.literal("never")
.or(z.literal("daily"))
.or(z.literal("weekly"))
.or(z.literal("monthly")),
keepLastNBackups: z.number().int().min(1).max(100),
}),
// Cache settings
cache: z.object({
semanticCacheEnabled: z.boolean(),
semanticCacheMaxSize: z.number().int().min(10).max(1000),
semanticCacheTTL: z.number().int().min(60000),
promptCacheEnabled: z.boolean(),
promptCacheStrategy: z.literal("auto").or(z.literal("system-only")).or(z.literal("manual")),
alwaysPreserveClientCache: z.literal("auto").or(z.literal("always")).or(z.literal("never")),
}),
// Retention settings
retention: z.object({
quotaSnapshots: z.number().int().min(1).max(3650), // Max 10 years
compressionAnalytics: z.number().int().min(1).max(365),
mcpAudit: z.number().int().min(1).max(365),
a2aEvents: z.number().int().min(1).max(365),
callLogs: z.number().int().min(1).max(3650),
usageHistory: z.number().int().min(1).max(3650),
memoryEntries: z.number().int().min(1).max(3650),
autoCleanupEnabled: z.boolean(),
}),
// Aggregation settings
aggregation: z.object({
enabled: z.boolean(),
rawDataRetentionDays: z.number().int().min(1).max(90),
granularity: z.literal("hourly").or(z.literal("daily")).or(z.literal("weekly")),
}),
// Optimization settings
optimization: z.object({
autoVacuumMode: z.literal("NONE").or(z.literal("FULL")).or(z.literal("INCREMENTAL")),
scheduledVacuum: z
.literal("never")
.or(z.literal("daily"))
.or(z.literal("weekly"))
.or(z.literal("monthly")),
pageSize: z.number().multipleOf(512).min(512).max(16384),
cacheSize: z.number().int().min(1000).max(1000000),
mmapSize: z.number().int().min(0),
}),
// Skip location and stats as they're read-only
},
{ strict: true }
);
export type DatabaseSettingsSchema = z.infer<typeof databaseSettingsSchema>;
// ... rest of the file ...

View File

@@ -0,0 +1,124 @@
/**
* Database performance optimization settings stored in SQLite key-value pairs.
* User-configurable aggregation, retention, and optimization settings.
*/
export interface DatabaseSettings {
/** 1. Location (read-only display) */
location: {
databasePath: string;
dataDir: string;
walSizeBytes: number;
schemaVersion: number;
};
/** 2. Logs (what gets captured) */
logs: {
detailedLogsEnabled: boolean;
callLogPipelineEnabled: boolean;
maxDetailSizeKb: number;
ringBufferSize: number;
};
/** 3. Backup (backup/restore/import/export) */
backup: {
autoBackupEnabled: boolean;
autoBackupFrequency: "never" | "daily" | "weekly" | "monthly";
keepLastNBackups: number;
};
/** 4. Cache (moved from CacheSettingsTab) */
cache: {
semanticCacheEnabled: boolean;
semanticCacheMaxSize: number;
semanticCacheTTL: number;
promptCacheEnabled: boolean;
promptCacheStrategy: "auto" | "system-only" | "manual";
alwaysPreserveClientCache: "auto" | "always" | "never";
};
/** 5. Retention (per-table cleanup policies) */
retention: {
quotaSnapshots: number;
compressionAnalytics: number;
mcpAudit: number;
a2aEvents: number;
callLogs: number;
usageHistory: number;
memoryEntries: number;
autoCleanupEnabled: boolean;
};
/** 6. Compression (aggregation) */
aggregation: {
enabled: boolean;
rawDataRetentionDays: number;
granularity: "hourly" | "daily" | "weekly";
};
/** 7. Optimization (auto_vacuum, VACUUM, page/cache) */
optimization: {
autoVacuumMode: "NONE" | "FULL" | "INCREMENTAL";
scheduledVacuum: "never" | "daily" | "weekly" | "monthly";
vacuumHour: number;
pageSize: number;
cacheSize: number;
optimizeOnStartup: boolean;
};
/** Read-only stats */
stats: {
databaseSizeBytes: number;
pageCount: number;
freelistCount: number;
lastVacuumAt: string | null;
lastOptimizationAt: string | null;
integrityCheck: "ok" | "error" | null;
};
}
/** Default database settings */
export const DEFAULT_DATABASE_SETTINGS: Omit<DatabaseSettings, "location" | "stats"> = {
logs: {
detailedLogsEnabled: false,
callLogPipelineEnabled: false,
maxDetailSizeKb: 10,
ringBufferSize: 500,
},
backup: {
autoBackupEnabled: false,
autoBackupFrequency: "never",
keepLastNBackups: 5,
},
cache: {
semanticCacheEnabled: true,
semanticCacheMaxSize: 100,
semanticCacheTTL: 1800000,
promptCacheEnabled: true,
promptCacheStrategy: "auto",
alwaysPreserveClientCache: "auto",
},
retention: {
quotaSnapshots: 7,
compressionAnalytics: 30,
mcpAudit: 30,
a2aEvents: 30,
callLogs: 30,
usageHistory: 30,
memoryEntries: 30,
autoCleanupEnabled: true,
},
aggregation: {
enabled: true,
rawDataRetentionDays: 30,
granularity: "daily",
},
optimization: {
autoVacuumMode: "FULL",
scheduledVacuum: "weekly",
vacuumHour: 2,
pageSize: 4096,
cacheSize: -2000,
optimizeOnStartup: true,
},
};

View File

@@ -10,3 +10,5 @@ export type { ApiKey } from "./apiKey";
export type { Combo, ComboStrategy, ComboNode } from "./combo";
export type { UsageEntry, UsageStats, ProviderUsageStats, ModelUsageStats, CallLog } from "./usage";
export type { Settings, ComboDefaults, ProxyConfig, KVPair } from "./settings";
export type { DatabaseSettings } from "./databaseSettings";
export { DEFAULT_DATABASE_SETTINGS } from "./databaseSettings";