mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-04 06:12:10 +03:00
Integrated into release/v3.7.9
This commit is contained in:
@@ -87,6 +87,7 @@
|
||||
"audit:electron": "npm --prefix electron audit --audit-level=moderate",
|
||||
"typecheck:core": "tsc --pretty false -p tsconfig.typecheck-core.json",
|
||||
"typecheck:noimplicit:core": "tsc --pretty false -p tsconfig.typecheck-noimplicit-core.json",
|
||||
"backfill-aggregation": "node --import tsx/esm src/scripts/backfillAggregation.ts",
|
||||
"env:sync": "node scripts/sync-env.mjs",
|
||||
"test:integration": "node --import tsx/esm --test tests/integration/*.test.ts",
|
||||
"test:e2e": "node scripts/run-playwright-tests.mjs test tests/e2e/*.spec.ts",
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { Card, Button } from "@/shared/components";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
interface CacheConfig {
|
||||
semanticCacheEnabled: boolean;
|
||||
semanticCacheMaxSize: number;
|
||||
semanticCacheTTL: number;
|
||||
promptCacheEnabled: boolean;
|
||||
promptCacheStrategy: "auto" | "system-only" | "manual";
|
||||
alwaysPreserveClientCache: "auto" | "always" | "never";
|
||||
}
|
||||
|
||||
export default function CacheSettingsTab() {
|
||||
const t = useTranslations("settings");
|
||||
const [config, setConfig] = useState<CacheConfig>({
|
||||
semanticCacheEnabled: true,
|
||||
semanticCacheMaxSize: 100,
|
||||
semanticCacheTTL: 1800000,
|
||||
promptCacheEnabled: true,
|
||||
promptCacheStrategy: "auto",
|
||||
alwaysPreserveClientCache: "auto",
|
||||
});
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
fetch("/api/settings/cache-config")
|
||||
.then((r) => (r.ok ? r.json() : null))
|
||||
.then((data) => {
|
||||
if (data) setConfig(data);
|
||||
})
|
||||
.catch(() => {})
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
await fetch("/api/settings/cache-config", {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(config),
|
||||
});
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Card className="p-6">
|
||||
<p className="text-sm text-text-muted">{t("loading")}</p>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Card className="p-6">
|
||||
<h3 className="text-lg font-semibold text-text-main flex items-center gap-2 mb-4">
|
||||
<span className="material-symbols-outlined text-[20px]">cached</span>
|
||||
{t("cacheSettings")}
|
||||
</h3>
|
||||
|
||||
<div className="space-y-6">
|
||||
{/* Semantic Cache */}
|
||||
<div className="space-y-3">
|
||||
<h4 className="text-sm font-medium text-text-main">{t("semanticCache")}</h4>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("enabled")}</span>
|
||||
<button
|
||||
onClick={() =>
|
||||
setConfig((c) => ({ ...c, semanticCacheEnabled: !c.semanticCacheEnabled }))
|
||||
}
|
||||
className={`relative w-10 h-5 rounded-full transition-colors ${
|
||||
config.semanticCacheEnabled ? "bg-green-500" : "bg-border"
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
|
||||
config.semanticCacheEnabled ? "left-5" : "left-0.5"
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("maxEntries")}</span>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={1000}
|
||||
value={config.semanticCacheMaxSize}
|
||||
onChange={(e) =>
|
||||
setConfig((c) => ({ ...c, semanticCacheMaxSize: parseInt(e.target.value) || 100 }))
|
||||
}
|
||||
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("ttlMinutes")}</span>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={1440}
|
||||
value={Math.round(config.semanticCacheTTL / 60000)}
|
||||
onChange={(e) =>
|
||||
setConfig((c) => ({
|
||||
...c,
|
||||
semanticCacheTTL: (parseInt(e.target.value) || 30) * 60000,
|
||||
}))
|
||||
}
|
||||
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Prompt Cache */}
|
||||
<div className="space-y-3 pt-4 border-t border-border/30">
|
||||
<h4 className="text-sm font-medium text-text-main">{t("promptCache")}</h4>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("enabled")}</span>
|
||||
<button
|
||||
onClick={() =>
|
||||
setConfig((c) => ({ ...c, promptCacheEnabled: !c.promptCacheEnabled }))
|
||||
}
|
||||
className={`relative w-10 h-5 rounded-full transition-colors ${
|
||||
config.promptCacheEnabled ? "bg-green-500" : "bg-border"
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
|
||||
config.promptCacheEnabled ? "left-5" : "left-0.5"
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("strategy")}</span>
|
||||
<select
|
||||
value={config.promptCacheStrategy}
|
||||
onChange={(e) =>
|
||||
setConfig((c) => ({
|
||||
...c,
|
||||
promptCacheStrategy: e.target.value as CacheConfig["promptCacheStrategy"],
|
||||
}))
|
||||
}
|
||||
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
|
||||
>
|
||||
<option value="auto">Auto</option>
|
||||
<option value="system-only">System Only</option>
|
||||
<option value="manual">Manual</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center justify-between">
|
||||
<span className="text-sm text-text-muted">{t("preserveClientCache")}</span>
|
||||
<select
|
||||
value={config.alwaysPreserveClientCache}
|
||||
onChange={(e) =>
|
||||
setConfig((c) => ({
|
||||
...c,
|
||||
alwaysPreserveClientCache: e.target
|
||||
.value as CacheConfig["alwaysPreserveClientCache"],
|
||||
}))
|
||||
}
|
||||
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
|
||||
>
|
||||
<option value="auto">Auto</option>
|
||||
<option value="always">Always</option>
|
||||
<option value="never">Never</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Save */}
|
||||
<div className="pt-4 border-t border-border/30">
|
||||
<Button onClick={handleSave} disabled={saving} size="sm">
|
||||
{saving ? t("saving") : t("save")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,6 @@ import ThinkingBudgetTab from "./components/ThinkingBudgetTab";
|
||||
import SystemPromptTab from "./components/SystemPromptTab";
|
||||
import ModelAliasesUnified from "./components/ModelAliasesUnified";
|
||||
import BackgroundDegradationTab from "./components/BackgroundDegradationTab";
|
||||
import CacheSettingsTab from "./components/CacheSettingsTab";
|
||||
import MemorySkillsTab from "./components/MemorySkillsTab";
|
||||
import ModelsDevSyncTab from "./components/ModelsDevSyncTab";
|
||||
import ResilienceTab from "./components/ResilienceTab";
|
||||
@@ -115,7 +114,6 @@ export default function SettingsPage() {
|
||||
</Link>
|
||||
<VisionBridgeSettingsTab />
|
||||
<SystemPromptTab />
|
||||
<CacheSettingsTab />
|
||||
<MemorySkillsTab />
|
||||
<ModelsDevSyncTab />
|
||||
</div>
|
||||
|
||||
17
src/app/api/settings/database/refresh-stats/route.ts
Normal file
17
src/app/api/settings/database/refresh-stats/route.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
import { getDatabaseStats } from "@/lib/db/stats";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
try {
|
||||
const stats = await getDatabaseStats();
|
||||
return NextResponse.json({ success: true, stats });
|
||||
} catch (error) {
|
||||
console.error("Failed to refresh database stats:", error);
|
||||
return NextResponse.json({ error: "Failed to refresh database stats" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
141
src/app/api/settings/database/route.ts
Normal file
141
src/app/api/settings/database/route.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getSettings, updateSettings } from "@/lib/localDb";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
import { z } from "zod";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { DatabaseSettings, DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
|
||||
import { getDatabaseStats } from "@/lib/db/stats";
|
||||
|
||||
type UserSettableKeys = keyof Omit<DatabaseSettings, "location" | "stats">;
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = await getSettings();
|
||||
const dbStats = await getDatabaseStats();
|
||||
|
||||
// Get current settings from key_value table (user settings)
|
||||
const userSettings: Partial<DatabaseSettings> = {};
|
||||
const allKeys = await getAllUserSettableKeys();
|
||||
|
||||
for (const key of allKeys) {
|
||||
const value = settings[key as keyof typeof settings];
|
||||
if (value !== undefined) {
|
||||
(userSettings as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// Merge with defaults and stats
|
||||
const merged: DatabaseSettings = {
|
||||
location: dbStats.location,
|
||||
logs: { ...DEFAULT_DATABASE_SETTINGS.logs, ...userSettings.logs },
|
||||
backup: { ...DEFAULT_DATABASE_SETTINGS.backup, ...userSettings.backup },
|
||||
cache: { ...DEFAULT_DATABASE_SETTINGS.cache, ...userSettings.cache },
|
||||
retention: { ...DEFAULT_DATABASE_SETTINGS.retention, ...userSettings.retention },
|
||||
aggregation: { ...DEFAULT_DATABASE_SETTINGS.aggregation, ...userSettings.aggregation },
|
||||
optimization: { ...DEFAULT_DATABASE_SETTINGS.optimization, ...userSettings.optimization },
|
||||
stats: dbStats.stats,
|
||||
};
|
||||
|
||||
return NextResponse.json(merged);
|
||||
} catch (error) {
|
||||
console.error("Error getting database settings:", error);
|
||||
return NextResponse.json({ error: "Failed to load database settings" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function PATCH(request: NextRequest) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = await getSettings();
|
||||
const userSettableKeys = await getUserSettableKeys();
|
||||
|
||||
const validation = await validateBody(
|
||||
request,
|
||||
z
|
||||
.record(z.string(), z.unknown())
|
||||
.partial()
|
||||
.refine(
|
||||
(data) => {
|
||||
// Ensure only user-settable fields are present
|
||||
return Object.keys(data).every((key) => userSettableKeys.includes(key));
|
||||
},
|
||||
{ message: "Attempting to set restricted fields" }
|
||||
)
|
||||
);
|
||||
|
||||
if (isValidationFailure(validation)) {
|
||||
return validation;
|
||||
}
|
||||
|
||||
const updates = validation.data;
|
||||
const updatedSettings = { ...settings };
|
||||
|
||||
// Update only user-settable fields
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
if (userSettableKeys.includes(key)) {
|
||||
(updatedSettings as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
await updateSettings(updatedSettings);
|
||||
|
||||
// Return merged settings (GET response pattern)
|
||||
return await GET(request);
|
||||
} catch (error) {
|
||||
console.error("Error updating database settings:", error);
|
||||
return NextResponse.json({ error: "Failed to update database settings" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
async function getUserSettableKeys(): Promise<UserSettableKeys[]> {
|
||||
// These are the fields that users are allowed to modify
|
||||
const allowedKeys: UserSettableKeys[] = [
|
||||
// Logs
|
||||
"logs",
|
||||
|
||||
// Backup
|
||||
"backup",
|
||||
|
||||
// Cache
|
||||
"cache",
|
||||
|
||||
// Retention
|
||||
"retention",
|
||||
|
||||
// Aggregation
|
||||
"aggregation",
|
||||
|
||||
// Optimization
|
||||
"optimization",
|
||||
];
|
||||
|
||||
return allowedKeys;
|
||||
}
|
||||
|
||||
async function getAllUserSettableKeys(): Promise<string[]> {
|
||||
const settableKeys: string[] = [];
|
||||
const userSettableSections = await getUserSettableKeys();
|
||||
|
||||
// Get all nested keys under each user-settable section
|
||||
const allDefaultKeys = Object.keys(DEFAULT_DATABASE_SETTINGS) as (keyof DatabaseSettings)[];
|
||||
|
||||
for (const section of userSettableSections) {
|
||||
if (section in DEFAULT_DATABASE_SETTINGS) {
|
||||
const sectionKeys = Object.keys(
|
||||
DEFAULT_DATABASE_SETTINGS[section as keyof typeof DEFAULT_DATABASE_SETTINGS]
|
||||
);
|
||||
sectionKeys.forEach((key) => {
|
||||
settableKeys.push(`${section}.${key}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return settableKeys;
|
||||
}
|
||||
36
src/app/api/settings/database/vacuum/route.ts
Normal file
36
src/app/api/settings/database/vacuum/route.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
import { runManualVacuum } from "@/lib/db/core";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = runManualVacuum();
|
||||
|
||||
if (result.success) {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: `VACUUM completed in ${result.duration}ms`,
|
||||
duration: result.duration,
|
||||
});
|
||||
} else {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: result.error || "VACUUM failed",
|
||||
duration: result.duration,
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[API] VACUUM endpoint error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Failed to run VACUUM", details: error.message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
19
src/app/api/settings/purge-call-logs/route.ts
Normal file
19
src/app/api/settings/purge-call-logs/route.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { purgeCallLogs } from "@/lib/db/cleanup";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
try {
|
||||
const result = await purgeCallLogs();
|
||||
return NextResponse.json({
|
||||
deleted: result.deleted,
|
||||
errors: result.errors,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
return NextResponse.json({ error }, { status: 500 });
|
||||
}
|
||||
}
|
||||
19
src/app/api/settings/purge-detailed-logs/route.ts
Normal file
19
src/app/api/settings/purge-detailed-logs/route.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { purgeDetailedLogs } from "@/lib/db/cleanup";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
try {
|
||||
const result = await purgeDetailedLogs();
|
||||
return NextResponse.json({
|
||||
deleted: result.deleted,
|
||||
errors: result.errors,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
return NextResponse.json({ error }, { status: 500 });
|
||||
}
|
||||
}
|
||||
19
src/app/api/settings/purge-quota-snapshots/route.ts
Normal file
19
src/app/api/settings/purge-quota-snapshots/route.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { purgeQuotaSnapshots } from "@/lib/db/cleanup";
|
||||
import { isAuthenticated } from "@/shared/utils/apiAuth";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
if (!(await isAuthenticated(request))) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
try {
|
||||
const result = await purgeQuotaSnapshots();
|
||||
return NextResponse.json({
|
||||
deleted: result.deleted,
|
||||
errors: result.errors,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
return NextResponse.json({ error }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getSettings, updateSettings } from "@/lib/localDb";
|
||||
import { getRuntimePorts } from "@/lib/runtime/ports";
|
||||
import { updateSettingsSchema } from "@/shared/validation/settingsSchemas";
|
||||
import { databaseSettingsSchema } from "@/shared/validation/settingsSchemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { z } from "zod";
|
||||
import { getConsistentMachineId } from "@/shared/utils/machineId";
|
||||
import { validateProxyUrl, upsertUpstreamProxyConfig } from "@/lib/db/upstreamProxy";
|
||||
import {
|
||||
|
||||
342
src/lib/db/cleanup.ts
Normal file
342
src/lib/db/cleanup.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* Database cleanup functions for removing old data based on retention policies.
|
||||
*
|
||||
* @module lib/db/cleanup
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "./core";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
import type { DatabaseSettings } from "@/types/databaseSettings";
|
||||
|
||||
interface CleanupResult {
|
||||
deleted: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract database settings from the full settings object with proper typing.
|
||||
*/
|
||||
function getDatabaseSettings(): DatabaseSettings["retention"] {
|
||||
const settings = getSettings();
|
||||
// Database settings are stored under the 'databaseSettings' key in the main settings
|
||||
const dbSettings = (settings as Record<string, unknown>).databaseSettings as
|
||||
| DatabaseSettings
|
||||
| undefined;
|
||||
return (
|
||||
dbSettings?.retention ?? {
|
||||
quotaSnapshots: 30,
|
||||
compressionAnalytics: 30,
|
||||
mcpAudit: 30,
|
||||
a2aEvents: 30,
|
||||
callLogs: 7,
|
||||
usageHistory: 90,
|
||||
memoryEntries: 90,
|
||||
autoCleanupEnabled: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old quota_snapshots based on retention settings.
|
||||
*/
|
||||
export async function cleanupQuotaSnapshots(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.quotaSnapshots;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM quota_snapshots WHERE created_at < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Cleanup] Deleted ${result.deleted} quota_snapshots older than ${retentionDays} days`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning quota_snapshots:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old call_logs based on retention settings.
|
||||
*/
|
||||
export async function cleanupCallLogs(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.callLogs;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM call_logs WHERE created_at < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Deleted ${result.deleted} call_logs older than ${retentionDays} days`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning call_logs:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old usage_history based on retention settings.
|
||||
*/
|
||||
export async function cleanupUsageHistory(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.usageHistory;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM usage_history WHERE timestamp < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Cleanup] Deleted ${result.deleted} usage_history older than ${retentionDays} days`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning usage_history:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old compression_analytics based on retention settings.
|
||||
*/
|
||||
export async function cleanupCompressionAnalytics(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.compressionAnalytics;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM compression_analytics WHERE created_at < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Cleanup] Deleted ${result.deleted} compression_analytics older than ${retentionDays} days`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning compression_analytics:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old mcp_audit_log based on retention settings.
|
||||
*/
|
||||
export async function cleanupMcpAudit(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.mcpAudit;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM mcp_audit_log WHERE timestamp < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Cleanup] Deleted ${result.deleted} mcp_audit_log older than ${retentionDays} days`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning mcp_audit_log:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old a2a_events based on retention settings.
|
||||
*/
|
||||
export async function cleanupA2aEvents(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.a2aEvents;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM a2a_events WHERE timestamp < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Deleted ${result.deleted} a2a_events older than ${retentionDays} days`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning a2a_events:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old memory_entries based on retention settings.
|
||||
*/
|
||||
export async function cleanupMemoryEntries(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const retention = getDatabaseSettings();
|
||||
|
||||
const retentionDays = retention.memoryEntries;
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
const cutoffISO = cutoffDate.toISOString();
|
||||
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM memory_entries WHERE created_at < ?");
|
||||
const runResult = stmt.run(cutoffISO);
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Cleanup] Deleted ${result.deleted} memory_entries older than ${retentionDays} days`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error cleaning memory_entries:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all cleanup functions if auto-cleanup is enabled.
|
||||
*/
|
||||
export async function runAutoCleanup(): Promise<{
|
||||
totalDeleted: number;
|
||||
totalErrors: number;
|
||||
results: Record<string, CleanupResult>;
|
||||
}> {
|
||||
const retention = getDatabaseSettings();
|
||||
const autoCleanupEnabled = retention.autoCleanupEnabled;
|
||||
|
||||
if (!autoCleanupEnabled) {
|
||||
console.log("[Cleanup] Auto-cleanup is disabled");
|
||||
return { totalDeleted: 0, totalErrors: 0, results: {} };
|
||||
}
|
||||
|
||||
console.log("[Cleanup] Starting auto-cleanup...");
|
||||
|
||||
const results: Record<string, CleanupResult> = {
|
||||
quotaSnapshots: await cleanupQuotaSnapshots(),
|
||||
callLogs: await cleanupCallLogs(),
|
||||
usageHistory: await cleanupUsageHistory(),
|
||||
compressionAnalytics: await cleanupCompressionAnalytics(),
|
||||
mcpAudit: await cleanupMcpAudit(),
|
||||
a2aEvents: await cleanupA2aEvents(),
|
||||
memoryEntries: await cleanupMemoryEntries(),
|
||||
};
|
||||
|
||||
const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0);
|
||||
const totalErrors = Object.values(results).reduce((sum, r) => sum + r.errors, 0);
|
||||
|
||||
console.log(`[Cleanup] Auto-cleanup complete: ${totalDeleted} deleted, ${totalErrors} errors`);
|
||||
|
||||
return { totalDeleted, totalErrors, results };
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge ALL quota_snapshots immediately (no retention check).
|
||||
*/
|
||||
export async function purgeQuotaSnapshots(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM quota_snapshots");
|
||||
const runResult = stmt.run();
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Purged ${result.deleted} quota_snapshots`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error purging quota_snapshots:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge ALL call_logs immediately (no retention check).
|
||||
*/
|
||||
export async function purgeCallLogs(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM call_logs");
|
||||
const runResult = stmt.run();
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Purged ${result.deleted} call_logs`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error purging call_logs:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge ALL detailed_logs immediately (no retention check).
|
||||
*/
|
||||
export async function purgeDetailedLogs(): Promise<CleanupResult> {
|
||||
const db = getDbInstance();
|
||||
const result: CleanupResult = { deleted: 0, errors: 0 };
|
||||
|
||||
try {
|
||||
const stmt = db.prepare("DELETE FROM detailed_logs");
|
||||
const runResult = stmt.run();
|
||||
result.deleted = runResult.changes;
|
||||
|
||||
console.log(`[Cleanup] Purged ${result.deleted} detailed_logs`);
|
||||
} catch (err: unknown) {
|
||||
console.error("[Cleanup] Error purging detailed_logs:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
100
src/lib/db/compressionScheduler.ts
Normal file
100
src/lib/db/compressionScheduler.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* Database compression scheduler - runs compression tasks based on settings.
|
||||
*
|
||||
* @module lib/db/compressionScheduler
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "./core";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
|
||||
interface CompressionScheduleSettings {
|
||||
enabled: boolean;
|
||||
intervalHours: number;
|
||||
lastRun?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run scheduled compression based on database settings.
|
||||
* Should be called on startup and periodically.
|
||||
*/
|
||||
export async function runScheduledCompression(): Promise<void> {
|
||||
const db = getDbInstance();
|
||||
const settings = await getSettings();
|
||||
|
||||
const compressionSettings = (settings.databaseSettings as any)?.compression as
|
||||
| CompressionScheduleSettings
|
||||
| undefined;
|
||||
|
||||
if (!compressionSettings?.enabled) {
|
||||
console.log("[CompressionScheduler] Compression scheduling is disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalHours = compressionSettings.intervalHours ?? 24;
|
||||
const lastRun = compressionSettings.lastRun ? new Date(compressionSettings.lastRun) : null;
|
||||
|
||||
const now = new Date();
|
||||
const hoursSinceLastRun = lastRun
|
||||
? (now.getTime() - lastRun.getTime()) / (1000 * 60 * 60)
|
||||
: Infinity;
|
||||
|
||||
if (hoursSinceLastRun < intervalHours) {
|
||||
console.log(
|
||||
`[CompressionScheduler] Skipping compression - last run was ${hoursSinceLastRun.toFixed(1)}h ago (interval: ${intervalHours}h)`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[CompressionScheduler] Running scheduled compression...");
|
||||
|
||||
try {
|
||||
// Run VACUUM to reclaim space
|
||||
db.prepare("VACUUM").run();
|
||||
console.log("[CompressionScheduler] VACUUM completed");
|
||||
|
||||
// Run ANALYZE to update statistics
|
||||
db.prepare("ANALYZE").run();
|
||||
console.log("[CompressionScheduler] ANALYZE completed");
|
||||
|
||||
const updateStmt = db.prepare(`
|
||||
INSERT OR REPLACE INTO key_value (namespace, key, value)
|
||||
VALUES ('settings', 'databaseSettings', json_set(
|
||||
COALESCE((SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'databaseSettings'), '{}'),
|
||||
'$.compression.lastRun',
|
||||
?
|
||||
))
|
||||
`);
|
||||
updateStmt.run(now.toISOString());
|
||||
|
||||
console.log("[CompressionScheduler] Compression completed successfully");
|
||||
} catch (err: any) {
|
||||
console.error("[CompressionScheduler] Error during compression:", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize compression scheduler on startup.
|
||||
* Call this once when the application starts.
|
||||
*/
|
||||
export async function initCompressionScheduler(): Promise<void> {
|
||||
console.log("[CompressionScheduler] Initializing compression scheduler...");
|
||||
|
||||
try {
|
||||
await runScheduledCompression();
|
||||
} catch (err: any) {
|
||||
console.error("[CompressionScheduler] Failed to run initial compression:", err);
|
||||
}
|
||||
|
||||
// Set up periodic check (every hour)
|
||||
setInterval(
|
||||
async () => {
|
||||
try {
|
||||
await runScheduledCompression();
|
||||
} catch (err: any) {
|
||||
console.error("[CompressionScheduler] Periodic compression check failed:", err);
|
||||
}
|
||||
},
|
||||
60 * 60 * 1000
|
||||
); // 1 hour
|
||||
}
|
||||
@@ -1481,3 +1481,96 @@ function migrateFromJson(db: SqliteDatabase, jsonPath: string) {
|
||||
console.error("[DB] Migration from db.json failed:", err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────── Auto-Vacuum Management ────────────────
|
||||
|
||||
export function setAutoVacuum(mode: "NONE" | "FULL" | "INCREMENTAL"): void {
|
||||
const db = getDbInstance();
|
||||
|
||||
const currentMode = db.pragma("auto_vacuum", { simple: true }) as number;
|
||||
const modeMap: Record<string, number> = {
|
||||
NONE: 0,
|
||||
FULL: 1,
|
||||
INCREMENTAL: 2,
|
||||
};
|
||||
|
||||
const targetMode = modeMap[mode];
|
||||
|
||||
if (currentMode === targetMode) {
|
||||
console.log(`[DB] auto_vacuum already set to ${mode}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[DB] Changing auto_vacuum from ${currentMode} to ${mode} (${targetMode})`);
|
||||
|
||||
db.pragma(`auto_vacuum = ${targetMode}`);
|
||||
|
||||
db.exec("VACUUM");
|
||||
|
||||
const newMode = db.pragma("auto_vacuum", { simple: true }) as number;
|
||||
console.log(`[DB] auto_vacuum changed to ${newMode}`);
|
||||
}
|
||||
|
||||
export function getAutoVacuumMode(): "NONE" | "FULL" | "INCREMENTAL" {
|
||||
const db = getDbInstance();
|
||||
const mode = db.pragma("auto_vacuum", { simple: true }) as number;
|
||||
|
||||
const modeMap: Record<number, "NONE" | "FULL" | "INCREMENTAL"> = {
|
||||
0: "NONE",
|
||||
1: "FULL",
|
||||
2: "INCREMENTAL",
|
||||
};
|
||||
|
||||
return modeMap[mode] || "NONE";
|
||||
}
|
||||
|
||||
export function runManualVacuum(): { success: boolean; duration: number; error?: string } {
|
||||
const db = getDbInstance();
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log("[DB] Starting manual VACUUM...");
|
||||
db.exec("VACUUM");
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`[DB] Manual VACUUM completed in ${duration}ms`);
|
||||
return { success: true, duration };
|
||||
} catch (err: any) {
|
||||
const duration = Date.now() - startTime;
|
||||
console.error("[DB] Manual VACUUM failed:", err);
|
||||
return { success: false, duration, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
export function setPageSize(pageSize: number): void {
|
||||
const db = getDbInstance();
|
||||
const currentPageSize = db.pragma("page_size", { simple: true }) as number;
|
||||
|
||||
if (currentPageSize === pageSize) {
|
||||
console.log(`[DB] page_size already set to ${pageSize}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[DB] Changing page_size from ${currentPageSize} to ${pageSize}`);
|
||||
db.pragma(`page_size = ${pageSize}`);
|
||||
db.exec("VACUUM");
|
||||
|
||||
const newPageSize = db.pragma("page_size", { simple: true }) as number;
|
||||
console.log(`[DB] page_size changed to ${newPageSize}`);
|
||||
}
|
||||
|
||||
export function setCacheSize(cacheSizeKb: number): void {
|
||||
const db = getDbInstance();
|
||||
const currentCacheSize = db.pragma("cache_size", { simple: true }) as number;
|
||||
const targetCacheSize = -cacheSizeKb;
|
||||
|
||||
if (currentCacheSize === targetCacheSize) {
|
||||
console.log(`[DB] cache_size already set to ${cacheSizeKb}KB`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[DB] Changing cache_size from ${Math.abs(currentCacheSize)}KB to ${cacheSizeKb}KB`);
|
||||
db.pragma(`cache_size = ${targetCacheSize}`);
|
||||
|
||||
const newCacheSize = db.pragma("cache_size", { simple: true }) as number;
|
||||
console.log(`[DB] cache_size changed to ${Math.abs(newCacheSize)}KB`);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import fs from "fs";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import type Database from "better-sqlite3";
|
||||
import { DEFAULT_DATABASE_SETTINGS } from "@/types/databaseSettings";
|
||||
|
||||
/**
|
||||
* Resolve the migrations directory path safely across platforms.
|
||||
@@ -799,9 +800,44 @@ export function runMigrations(db: Database.Database, options?: { isNewDb?: boole
|
||||
console.log(`[Migration] ${count} migration(s) applied successfully.`);
|
||||
}
|
||||
|
||||
// After applying all migrations, insert default settings if we just ran migration 46
|
||||
try {
|
||||
if (appliedRecords.some((m) => m.name.startsWith("046_"))) {
|
||||
insertDefaultDatabaseSettings(db);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error inserting default database settings:", error);
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
function insertDefaultDatabaseSettings(db: Database.Database) {
|
||||
const tx = db.transaction(() => {
|
||||
// Insert all default settings
|
||||
for (const [section, values] of Object.entries(DEFAULT_DATABASE_SETTINGS)) {
|
||||
for (const [key, value] of Object.entries(values as Record<string, unknown>)) {
|
||||
db.prepare("INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
|
||||
"databaseSettings",
|
||||
`${section}.${key}`,
|
||||
JSON.stringify(value)
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Run in an immediate transaction to avoid nested transactions
|
||||
try {
|
||||
// @ts-expect-error - Better-SQLite3 transaction types
|
||||
db.immediate(() => {
|
||||
tx();
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Transaction error inserting default settings:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get migration status for diagnostics.
|
||||
*/
|
||||
|
||||
44
src/lib/db/migrations/046_database_settings.sql
Normal file
44
src/lib/db/migrations/046_database_settings.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- 046_database_settings.sql
|
||||
-- Insert default database settings into key_value table (namespace='databaseSettings')
|
||||
-- Uses INSERT OR IGNORE so existing user settings are never overwritten by migration replay.
|
||||
|
||||
-- Logs settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'detailedLogsEnabled', 'true');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogPipelineEnabled', 'true');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'maxDetailSizeKb', '500');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'ringBufferSize', '1000');
|
||||
|
||||
-- Backup settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupEnabled', 'false');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoBackupFrequency', '"never"');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'keepLastNBackups', '3');
|
||||
|
||||
-- Cache settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheEnabled', 'true');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheMaxSize', '100');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'semanticCacheTTL', '1800000');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheEnabled', 'true');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'promptCacheStrategy', '"auto"');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'alwaysPreserveClientCache', '"auto"');
|
||||
|
||||
-- Retention settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'quotaSnapshots', '90');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'compressionAnalytics', '30');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mcpAudit', '30');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'a2aEvents', '30');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogs', '90');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'usageHistory', '365');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'memoryEntries', '180');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoCleanupEnabled', 'true');
|
||||
|
||||
-- Aggregation settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'aggregationEnabled', 'false');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'rawDataRetentionDays', '7');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'granularity', '"daily"');
|
||||
|
||||
-- Optimization settings
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'autoVacuumMode', '"INCREMENTAL"');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'scheduledVacuum', '"weekly"');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'pageSize', '4096');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'cacheSize', '10000');
|
||||
INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mmapSize', '268435456');
|
||||
44
src/lib/db/migrations/047_aggregation_tables.sql
Normal file
44
src/lib/db/migrations/047_aggregation_tables.sql
Normal file
@@ -0,0 +1,44 @@
|
||||
-- 047_aggregation_tables.sql
|
||||
-- Create aggregation tables for usage data summarization
|
||||
|
||||
-- Hourly usage summary table
|
||||
CREATE TABLE IF NOT EXISTS hourly_usage_summary (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
date_hour TEXT NOT NULL, -- Format: YYYY-MM-DD HH:00:00
|
||||
total_requests INTEGER NOT NULL DEFAULT 0,
|
||||
total_input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost REAL NOT NULL DEFAULT 0.0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Index for efficient queries by provider, model, and time range
|
||||
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date
|
||||
ON hourly_usage_summary(provider, model, date_hour);
|
||||
|
||||
-- Index for time-based queries
|
||||
CREATE INDEX IF NOT EXISTS idx_hourly_usage_date
|
||||
ON hourly_usage_summary(date_hour);
|
||||
|
||||
-- Daily usage summary table
|
||||
CREATE TABLE IF NOT EXISTS daily_usage_summary (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
date TEXT NOT NULL, -- Format: YYYY-MM-DD
|
||||
total_requests INTEGER NOT NULL DEFAULT 0,
|
||||
total_input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost REAL NOT NULL DEFAULT 0.0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Index for efficient queries by provider, model, and date
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date
|
||||
ON daily_usage_summary(provider, model, date);
|
||||
|
||||
-- Index for date-based queries
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_usage_date
|
||||
ON daily_usage_summary(date);
|
||||
29
src/lib/db/migrations/048_summary_indexes.sql
Normal file
29
src/lib/db/migrations/048_summary_indexes.sql
Normal file
@@ -0,0 +1,29 @@
|
||||
-- 048_summary_indexes.sql
|
||||
-- Add composite indexes for efficient querying of summary tables
|
||||
|
||||
-- Composite indexes for daily_usage_summary
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_date
|
||||
ON daily_usage_summary(provider, date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_usage_model_date
|
||||
ON daily_usage_summary(model, date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_usage_provider_model_date_composite
|
||||
ON daily_usage_summary(provider, model, date);
|
||||
|
||||
-- Composite indexes for hourly_usage_summary
|
||||
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_date
|
||||
ON hourly_usage_summary(provider, date_hour);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_hourly_usage_model_date
|
||||
ON hourly_usage_summary(model, date_hour);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_hourly_usage_provider_model_date_composite
|
||||
ON hourly_usage_summary(provider, model, date_hour);
|
||||
|
||||
-- Add unique constraint to prevent duplicate aggregations
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_daily_usage_unique
|
||||
ON daily_usage_summary(provider, model, date);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_hourly_usage_unique
|
||||
ON hourly_usage_summary(provider, model, date_hour);
|
||||
10
src/lib/db/migrations/049_compression_analytics_indexes.sql
Normal file
10
src/lib/db/migrations/049_compression_analytics_indexes.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
-- Migration 049: Add indexes to compression_analytics table for performance
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_compression_analytics_timestamp
|
||||
ON compression_analytics(timestamp);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider
|
||||
ON compression_analytics(provider);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_compression_analytics_provider_timestamp
|
||||
ON compression_analytics(provider, timestamp);
|
||||
71
src/lib/db/stats.ts
Normal file
71
src/lib/db/stats.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Database Statistics Module
|
||||
*
|
||||
* Provides functions to retrieve database statistics including size, table counts, and performance metrics.
|
||||
*/
|
||||
|
||||
import type Database from "better-sqlite3";
|
||||
import { getDbInstance } from "./core";
|
||||
|
||||
export interface DatabaseStats {
|
||||
totalSize: number;
|
||||
pageSize: number;
|
||||
pageCount: number;
|
||||
tables: Array<{
|
||||
name: string;
|
||||
rowCount: number;
|
||||
size: number;
|
||||
}>;
|
||||
indexes: Array<{
|
||||
name: string;
|
||||
tableName: string;
|
||||
}>;
|
||||
walSize?: number;
|
||||
cacheSize: number;
|
||||
}
|
||||
|
||||
export function getDatabaseStats(): DatabaseStats {
|
||||
const db = getDbInstance();
|
||||
|
||||
const pageSize = db.pragma("page_size", { simple: true }) as number;
|
||||
const pageCount = db.pragma("page_count", { simple: true }) as number;
|
||||
const cacheSize = db.pragma("cache_size", { simple: true }) as number;
|
||||
const totalSize = pageSize * pageCount;
|
||||
|
||||
const tables = db
|
||||
.prepare(
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`
|
||||
)
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
const tableStats = tables.map((table) => {
|
||||
const rowCount = db.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get() as {
|
||||
count: number;
|
||||
};
|
||||
|
||||
const tableSize = db
|
||||
.prepare(`SELECT SUM(pgsize) as size FROM dbstat WHERE name = ?`)
|
||||
.get(table.name) as { size: number | null };
|
||||
|
||||
return {
|
||||
name: table.name,
|
||||
rowCount: rowCount.count,
|
||||
size: tableSize?.size || 0,
|
||||
};
|
||||
});
|
||||
|
||||
const indexes = db
|
||||
.prepare(
|
||||
`SELECT name, tbl_name as tableName FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name`
|
||||
)
|
||||
.all() as Array<{ name: string; tableName: string }>;
|
||||
|
||||
return {
|
||||
totalSize,
|
||||
pageSize,
|
||||
pageCount,
|
||||
tables: tableStats,
|
||||
indexes,
|
||||
cacheSize,
|
||||
};
|
||||
}
|
||||
163
src/lib/usage/aggregateHistory.ts
Normal file
163
src/lib/usage/aggregateHistory.ts
Normal file
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Aggregation utility functions for usage data summarization.
|
||||
* Rolls up quota_snapshots into hourly and daily summary tables.
|
||||
*
|
||||
* @module lib/usage/aggregateHistory
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "../db/core";
|
||||
import { getSettings } from "@/lib/localDb";
|
||||
|
||||
interface AggregationResult {
|
||||
processed: number;
|
||||
inserted: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll up quota_snapshots into daily_usage_summary table.
|
||||
* Aggregates by provider, model, and date.
|
||||
*
|
||||
* @param fromDate - Start date (YYYY-MM-DD format)
|
||||
* @param toDate - End date (YYYY-MM-DD format)
|
||||
* @returns Aggregation result with counts
|
||||
*/
|
||||
export async function rollupDailyUsage(
|
||||
fromDate: string,
|
||||
toDate: string
|
||||
): Promise<AggregationResult> {
|
||||
const db = getDbInstance();
|
||||
const settings = await getSettings();
|
||||
|
||||
// Get retention settings
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
|
||||
const result: AggregationResult = {
|
||||
processed: 0,
|
||||
inserted: 0,
|
||||
errors: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// Aggregate quota_snapshots by provider, model, and date
|
||||
const aggregateQuery = `
|
||||
INSERT INTO daily_usage_summary (provider, model, date, total_requests, total_input_tokens, total_output_tokens, total_cost)
|
||||
SELECT
|
||||
provider,
|
||||
COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model,
|
||||
DATE(created_at) as date,
|
||||
COUNT(*) as total_requests,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost
|
||||
FROM quota_snapshots
|
||||
WHERE DATE(created_at) >= ? AND DATE(created_at) <= ?
|
||||
GROUP BY provider, model, DATE(created_at)
|
||||
ON CONFLICT(provider, model, date) DO UPDATE SET
|
||||
total_requests = total_requests + excluded.total_requests,
|
||||
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
|
||||
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
|
||||
total_cost = total_cost + excluded.total_cost
|
||||
`;
|
||||
|
||||
const stmt = db.prepare(aggregateQuery);
|
||||
const runResult = stmt.run(fromDate, toDate);
|
||||
|
||||
result.processed = runResult.changes;
|
||||
result.inserted = runResult.changes;
|
||||
|
||||
console.log(`[Aggregation] Daily rollup: ${result.inserted} rows for ${fromDate} to ${toDate}`);
|
||||
} catch (err: any) {
|
||||
console.error("[Aggregation] Daily rollup error:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll up quota_snapshots into hourly_usage_summary table.
|
||||
* Aggregates by provider, model, and hour.
|
||||
*
|
||||
* @param fromDate - Start datetime (YYYY-MM-DD HH:MM:SS format)
|
||||
* @param toDate - End datetime (YYYY-MM-DD HH:MM:SS format)
|
||||
* @returns Aggregation result with counts
|
||||
*/
|
||||
export async function rollupHourlyQuota(
|
||||
fromDate: string,
|
||||
toDate: string
|
||||
): Promise<AggregationResult> {
|
||||
const db = getDbInstance();
|
||||
const settings = await getSettings();
|
||||
|
||||
// Get retention settings
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
|
||||
const result: AggregationResult = {
|
||||
processed: 0,
|
||||
inserted: 0,
|
||||
errors: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// Aggregate quota_snapshots by provider, model, and hour
|
||||
const aggregateQuery = `
|
||||
INSERT INTO hourly_usage_summary (provider, model, date_hour, total_requests, total_input_tokens, total_output_tokens, total_cost)
|
||||
SELECT
|
||||
provider,
|
||||
COALESCE(json_extract(raw_data, '$.model'), 'unknown') as model,
|
||||
datetime(strftime('%Y-%m-%d %H:00:00', created_at)) as date_hour,
|
||||
COUNT(*) as total_requests,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.input_tokens') AS INTEGER)), 0) as total_input_tokens,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.output_tokens') AS INTEGER)), 0) as total_output_tokens,
|
||||
COALESCE(SUM(CAST(json_extract(raw_data, '$.cost') AS REAL)), 0.0) as total_cost
|
||||
FROM quota_snapshots
|
||||
WHERE created_at >= ? AND created_at <= ?
|
||||
GROUP BY provider, model, datetime(strftime('%Y-%m-%d %H:00:00', created_at))
|
||||
ON CONFLICT(provider, model, date_hour) DO UPDATE SET
|
||||
total_requests = total_requests + excluded.total_requests,
|
||||
total_input_tokens = total_input_tokens + excluded.total_input_tokens,
|
||||
total_output_tokens = total_output_tokens + excluded.total_output_tokens,
|
||||
total_cost = total_cost + excluded.total_cost
|
||||
`;
|
||||
|
||||
const stmt = db.prepare(aggregateQuery);
|
||||
const runResult = stmt.run(fromDate, toDate);
|
||||
|
||||
result.processed = runResult.changes;
|
||||
result.inserted = runResult.changes;
|
||||
|
||||
console.log(
|
||||
`[Aggregation] Hourly rollup: ${result.inserted} rows for ${fromDate} to ${toDate}`
|
||||
);
|
||||
} catch (err: any) {
|
||||
console.error("[Aggregation] Hourly rollup error:", err);
|
||||
result.errors++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cutoff date for raw data based on retention settings.
|
||||
* Data older than this should be aggregated and cleaned up.
|
||||
*
|
||||
* @returns ISO date string (YYYY-MM-DD)
|
||||
*/
|
||||
export async function getRawDataCutoffDate(): Promise<string> {
|
||||
const settings = await getSettings();
|
||||
const rawDataRetentionDays = (settings.aggregation as any)?.rawDataRetentionDays ?? 90;
|
||||
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - rawDataRetentionDays);
|
||||
|
||||
return cutoffDate.toISOString().split("T")[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if aggregation is enabled in settings.
|
||||
*/
|
||||
export async function isAggregationEnabled(): Promise<boolean> {
|
||||
const settings = await getSettings();
|
||||
return (settings.aggregation as any)?.enabled ?? false;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { getDbInstance } from "../db/core";
|
||||
import { getPendingRequests } from "./usageHistory";
|
||||
import { getAccountDisplayName } from "@/lib/display/names";
|
||||
import { calculateCost } from "./costCalculator";
|
||||
import { getRawDataCutoffDate, isAggregationEnabled } from "./aggregateHistory";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
type UsageBucket = {
|
||||
@@ -56,10 +57,58 @@ function toStringOrEmpty(value: unknown): string {
|
||||
|
||||
/**
|
||||
* Get aggregated usage stats.
|
||||
* Uses UNION of recent raw data and older aggregated data when aggregation is enabled.
|
||||
*/
|
||||
export async function getUsageStats() {
|
||||
const db = getDbInstance();
|
||||
const rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[];
|
||||
const aggregationEnabled = await isAggregationEnabled();
|
||||
|
||||
let rows: unknown[];
|
||||
|
||||
if (aggregationEnabled) {
|
||||
const cutoffDate = await getRawDataCutoffDate();
|
||||
|
||||
// UNION: recent raw data + older aggregated data
|
||||
const unionQuery = `
|
||||
SELECT
|
||||
provider,
|
||||
model,
|
||||
timestamp,
|
||||
connection_id,
|
||||
api_key_id,
|
||||
api_key_name,
|
||||
tokens_input,
|
||||
tokens_output,
|
||||
tokens_cache_read,
|
||||
tokens_cache_creation,
|
||||
tokens_reasoning
|
||||
FROM usage_history
|
||||
WHERE DATE(timestamp) >= ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
provider,
|
||||
model,
|
||||
date || ' 12:00:00' as timestamp,
|
||||
NULL as connection_id,
|
||||
NULL as api_key_id,
|
||||
NULL as api_key_name,
|
||||
total_input_tokens as tokens_input,
|
||||
total_output_tokens as tokens_output,
|
||||
0 as tokens_cache_read,
|
||||
0 as tokens_cache_creation,
|
||||
0 as tokens_reasoning
|
||||
FROM daily_usage_summary
|
||||
WHERE date < ?
|
||||
|
||||
ORDER BY timestamp ASC
|
||||
`;
|
||||
|
||||
rows = db.prepare(unionQuery).all(cutoffDate, cutoffDate) as unknown[];
|
||||
} else {
|
||||
rows = db.prepare("SELECT * FROM usage_history ORDER BY timestamp ASC").all() as unknown[];
|
||||
}
|
||||
|
||||
const { getProviderConnections } = await import("@/lib/localDb");
|
||||
let allConnections: unknown[] = [];
|
||||
|
||||
123
src/scripts/backfillAggregation.ts
Normal file
123
src/scripts/backfillAggregation.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { rollupDailyUsage, rollupHourlyQuota } from "@/lib/usage/aggregateHistory";
|
||||
|
||||
interface BackfillOptions {
|
||||
from: string;
|
||||
to: string;
|
||||
granularity?: "hourly" | "daily" | "both";
|
||||
}
|
||||
|
||||
function parseArgs(): BackfillOptions {
|
||||
const args = process.argv.slice(2);
|
||||
const options: Partial<BackfillOptions> = {
|
||||
granularity: "both",
|
||||
};
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg.startsWith("--from=")) {
|
||||
options.from = arg.split("=")[1];
|
||||
} else if (arg.startsWith("--to=")) {
|
||||
options.to = arg.split("=")[1];
|
||||
} else if (arg.startsWith("--granularity=")) {
|
||||
const value = arg.split("=")[1];
|
||||
if (value === "hourly" || value === "daily" || value === "both") {
|
||||
options.granularity = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.from || !options.to) {
|
||||
console.error(
|
||||
"Usage: npm run backfill-aggregation -- --from=YYYY-MM-DD --to=YYYY-MM-DD [--granularity=hourly|daily|both]"
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return options as BackfillOptions;
|
||||
}
|
||||
|
||||
function validateDate(dateStr: string): boolean {
|
||||
const regex = /^\d{4}-\d{2}-\d{2}$/;
|
||||
if (!regex.test(dateStr)) return false;
|
||||
|
||||
const date = new Date(dateStr);
|
||||
return date instanceof Date && !isNaN(date.getTime());
|
||||
}
|
||||
|
||||
async function backfillAggregation() {
|
||||
const options = parseArgs();
|
||||
|
||||
if (!validateDate(options.from) || !validateDate(options.to)) {
|
||||
console.error("Error: Dates must be in YYYY-MM-DD format");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const fromDate = new Date(options.from);
|
||||
const toDate = new Date(options.to);
|
||||
|
||||
if (fromDate > toDate) {
|
||||
console.error("Error: --from date must be before --to date");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("=".repeat(60));
|
||||
console.log("Aggregation Backfill Started");
|
||||
console.log("=".repeat(60));
|
||||
console.log(`From: ${options.from}`);
|
||||
console.log(`To: ${options.to}`);
|
||||
console.log(`Granularity: ${options.granularity}`);
|
||||
console.log("=".repeat(60));
|
||||
|
||||
const startTime = Date.now();
|
||||
let totalProcessed = 0;
|
||||
let totalInserted = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
try {
|
||||
if (options.granularity === "daily" || options.granularity === "both") {
|
||||
console.log("\n[Daily Aggregation] Starting...");
|
||||
const dailyResult = await rollupDailyUsage(options.from, options.to);
|
||||
totalProcessed += dailyResult.processed;
|
||||
totalInserted += dailyResult.inserted;
|
||||
totalErrors += dailyResult.errors;
|
||||
console.log(
|
||||
`[Daily Aggregation] Processed: ${dailyResult.processed}, Inserted: ${dailyResult.inserted}, Errors: ${dailyResult.errors}`
|
||||
);
|
||||
}
|
||||
|
||||
if (options.granularity === "hourly" || options.granularity === "both") {
|
||||
console.log("\n[Hourly Aggregation] Starting...");
|
||||
const fromDateTime = `${options.from} 00:00:00`;
|
||||
const toDateTime = `${options.to} 23:59:59`;
|
||||
const hourlyResult = await rollupHourlyQuota(fromDateTime, toDateTime);
|
||||
totalProcessed += hourlyResult.processed;
|
||||
totalInserted += hourlyResult.inserted;
|
||||
totalErrors += hourlyResult.errors;
|
||||
console.log(
|
||||
`[Hourly Aggregation] Processed: ${hourlyResult.processed}, Inserted: ${hourlyResult.inserted}, Errors: ${hourlyResult.errors}`
|
||||
);
|
||||
}
|
||||
|
||||
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
|
||||
console.log("\n" + "=".repeat(60));
|
||||
console.log("Aggregation Backfill Complete");
|
||||
console.log("=".repeat(60));
|
||||
console.log(`Total Processed: ${totalProcessed}`);
|
||||
console.log(`Total Inserted: ${totalInserted}`);
|
||||
console.log(`Total Errors: ${totalErrors}`);
|
||||
console.log(`Duration: ${duration}s`);
|
||||
console.log("=".repeat(60));
|
||||
|
||||
if (totalErrors > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("\n[FATAL ERROR]", error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
backfillAggregation();
|
||||
@@ -1,99 +1,73 @@
|
||||
/**
|
||||
* Settings-specific Zod schemas.
|
||||
*
|
||||
* Extracted from schemas.ts to work around the webpack barrel-file
|
||||
* optimization bug that makes large schema barrel exports `undefined`
|
||||
* at runtime (see: https://github.com/vercel/next.js/issues/12557).
|
||||
*/
|
||||
import { z } from "zod";
|
||||
import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
|
||||
import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility";
|
||||
import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
|
||||
// ... existing imports ...
|
||||
|
||||
const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const;
|
||||
export const databaseSettingsSchema = z.object(
|
||||
{
|
||||
// Logs settings
|
||||
logs: z.object({
|
||||
detailedLogsEnabled: z.boolean(),
|
||||
callLogPipelineEnabled: z.boolean(),
|
||||
maxDetailSizeKb: z.number().int().nonnegative(),
|
||||
ringBufferSize: z.number().int().min(100).max(10000),
|
||||
}),
|
||||
|
||||
export const updateSettingsSchema = z.object({
|
||||
newPassword: z.string().min(1).max(200).optional(),
|
||||
currentPassword: z.string().max(200).optional(),
|
||||
theme: z.string().max(50).optional(),
|
||||
language: z.string().max(10).optional(),
|
||||
requireLogin: z.boolean().optional(),
|
||||
enableSocks5Proxy: z.boolean().optional(),
|
||||
instanceName: z.string().max(100).optional(),
|
||||
customLogoUrl: z.string().max(2000).optional(),
|
||||
customLogoBase64: z.string().max(100000).optional(),
|
||||
customFaviconUrl: z.string().max(2000).optional(),
|
||||
customFaviconBase64: z.string().max(50000).optional(),
|
||||
corsOrigins: z.string().max(500).optional(),
|
||||
cloudUrl: z.string().max(500).optional(),
|
||||
baseUrl: z.string().max(500).optional(),
|
||||
setupComplete: z.boolean().optional(),
|
||||
blockedProviders: z.array(z.string().max(100)).optional(),
|
||||
hideHealthCheckLogs: z.boolean().optional(),
|
||||
hideEndpointCloudflaredTunnel: z.boolean().optional(),
|
||||
hideEndpointTailscaleFunnel: z.boolean().optional(),
|
||||
hideEndpointNgrokTunnel: z.boolean().optional(),
|
||||
debugMode: z.boolean().optional(),
|
||||
hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(),
|
||||
comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(),
|
||||
// Routing settings (#134)
|
||||
fallbackStrategy: z.enum(ACCOUNT_FALLBACK_STRATEGY_VALUES).optional(),
|
||||
wildcardAliases: z.array(z.object({ pattern: z.string(), target: z.string() })).optional(),
|
||||
stickyRoundRobinLimit: z.number().int().min(0).max(1000).optional(),
|
||||
requestRetry: z.number().int().min(0).max(10).optional(),
|
||||
maxRetryIntervalSec: z.number().int().min(0).max(300).optional(),
|
||||
// Auto intent classifier settings (multilingual routing)
|
||||
intentDetectionEnabled: z.boolean().optional(),
|
||||
intentSimpleMaxWords: z.number().int().min(1).max(500).optional(),
|
||||
intentExtraCodeKeywords: z.array(z.string().max(100)).optional(),
|
||||
intentExtraReasoningKeywords: z.array(z.string().max(100)).optional(),
|
||||
intentExtraSimpleKeywords: z.array(z.string().max(100)).optional(),
|
||||
// Protocol toggles (default: disabled)
|
||||
mcpEnabled: z.boolean().optional(),
|
||||
mcpTransport: z.enum(["stdio", "sse", "streamable-http"]).optional(),
|
||||
a2aEnabled: z.boolean().optional(),
|
||||
wsAuth: z.boolean().optional(),
|
||||
// CLI Fingerprint compatibility (per-provider)
|
||||
cliCompatProviders: z.array(z.string().max(100)).optional(),
|
||||
// Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4")
|
||||
stripModelPrefix: z.boolean().optional(),
|
||||
// Cache control preservation mode
|
||||
alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(),
|
||||
antigravitySignatureCacheMode: z.enum(signatureCacheModeValues).optional(),
|
||||
// Adaptive Volume Routing
|
||||
adaptiveVolumeRouting: z.boolean().optional(),
|
||||
// Usage token buffer — safety margin added to reported prompt/input token counts.
|
||||
// Prevents CLI tools from overrunning context windows. Set to 0 to disable.
|
||||
usageTokenBuffer: z.number().int().min(0).max(50000).optional(),
|
||||
// Custom CLI agent definitions for ACP
|
||||
customAgents: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.string().max(50),
|
||||
name: z.string().max(100),
|
||||
binary: z.string().max(200),
|
||||
versionCommand: z.string().max(300),
|
||||
providerAlias: z.string().max(50),
|
||||
spawnArgs: z.array(z.string().max(200)),
|
||||
protocol: z.enum(["stdio", "http"]),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
// SkillsMP marketplace API key
|
||||
skillsmpApiKey: z.string().max(200).optional(),
|
||||
// Active skills provider (single source of truth for skills page)
|
||||
skillsProvider: z.enum(["skillsmp", "skillssh"]).optional(),
|
||||
// models.dev sync settings
|
||||
modelsDevSyncEnabled: z.boolean().optional(),
|
||||
modelsDevSyncInterval: z.number().int().min(3600000).max(604800000).optional(),
|
||||
// Vision Bridge settings
|
||||
visionBridgeEnabled: z.boolean().optional(),
|
||||
visionBridgeModel: z.string().max(200).optional(),
|
||||
visionBridgePrompt: z.string().max(5000).optional(),
|
||||
visionBridgeTimeout: z.number().int().min(1000).max(300000).optional(),
|
||||
visionBridgeMaxImages: z.number().int().min(1).max(20).optional(),
|
||||
// Missing settings
|
||||
lkgpEnabled: z.boolean().optional(),
|
||||
backgroundDegradation: z.unknown().optional(),
|
||||
bruteForceProtection: z.boolean().optional(),
|
||||
});
|
||||
// Backup settings
|
||||
backup: z.object({
|
||||
autoBackupEnabled: z.boolean(),
|
||||
autoBackupFrequency: z
|
||||
.literal("never")
|
||||
.or(z.literal("daily"))
|
||||
.or(z.literal("weekly"))
|
||||
.or(z.literal("monthly")),
|
||||
keepLastNBackups: z.number().int().min(1).max(100),
|
||||
}),
|
||||
|
||||
// Cache settings
|
||||
cache: z.object({
|
||||
semanticCacheEnabled: z.boolean(),
|
||||
semanticCacheMaxSize: z.number().int().min(10).max(1000),
|
||||
semanticCacheTTL: z.number().int().min(60000),
|
||||
promptCacheEnabled: z.boolean(),
|
||||
promptCacheStrategy: z.literal("auto").or(z.literal("system-only")).or(z.literal("manual")),
|
||||
alwaysPreserveClientCache: z.literal("auto").or(z.literal("always")).or(z.literal("never")),
|
||||
}),
|
||||
|
||||
// Retention settings
|
||||
retention: z.object({
|
||||
quotaSnapshots: z.number().int().min(1).max(3650), // Max 10 years
|
||||
compressionAnalytics: z.number().int().min(1).max(365),
|
||||
mcpAudit: z.number().int().min(1).max(365),
|
||||
a2aEvents: z.number().int().min(1).max(365),
|
||||
callLogs: z.number().int().min(1).max(3650),
|
||||
usageHistory: z.number().int().min(1).max(3650),
|
||||
memoryEntries: z.number().int().min(1).max(3650),
|
||||
autoCleanupEnabled: z.boolean(),
|
||||
}),
|
||||
|
||||
// Aggregation settings
|
||||
aggregation: z.object({
|
||||
enabled: z.boolean(),
|
||||
rawDataRetentionDays: z.number().int().min(1).max(90),
|
||||
granularity: z.literal("hourly").or(z.literal("daily")).or(z.literal("weekly")),
|
||||
}),
|
||||
|
||||
// Optimization settings
|
||||
optimization: z.object({
|
||||
autoVacuumMode: z.literal("NONE").or(z.literal("FULL")).or(z.literal("INCREMENTAL")),
|
||||
scheduledVacuum: z
|
||||
.literal("never")
|
||||
.or(z.literal("daily"))
|
||||
.or(z.literal("weekly"))
|
||||
.or(z.literal("monthly")),
|
||||
pageSize: z.number().multipleOf(512).min(512).max(16384),
|
||||
cacheSize: z.number().int().min(1000).max(1000000),
|
||||
mmapSize: z.number().int().min(0),
|
||||
}),
|
||||
|
||||
// Skip location and stats as they're read-only
|
||||
},
|
||||
{ strict: true }
|
||||
);
|
||||
|
||||
export type DatabaseSettingsSchema = z.infer<typeof databaseSettingsSchema>;
|
||||
|
||||
// ... rest of the file ...
|
||||
|
||||
124
src/types/databaseSettings.ts
Normal file
124
src/types/databaseSettings.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* Database performance optimization settings stored in SQLite key-value pairs.
|
||||
* User-configurable aggregation, retention, and optimization settings.
|
||||
*/
|
||||
|
||||
export interface DatabaseSettings {
|
||||
/** 1. Location (read-only display) */
|
||||
location: {
|
||||
databasePath: string;
|
||||
dataDir: string;
|
||||
walSizeBytes: number;
|
||||
schemaVersion: number;
|
||||
};
|
||||
|
||||
/** 2. Logs (what gets captured) */
|
||||
logs: {
|
||||
detailedLogsEnabled: boolean;
|
||||
callLogPipelineEnabled: boolean;
|
||||
maxDetailSizeKb: number;
|
||||
ringBufferSize: number;
|
||||
};
|
||||
|
||||
/** 3. Backup (backup/restore/import/export) */
|
||||
backup: {
|
||||
autoBackupEnabled: boolean;
|
||||
autoBackupFrequency: "never" | "daily" | "weekly" | "monthly";
|
||||
keepLastNBackups: number;
|
||||
};
|
||||
|
||||
/** 4. Cache (moved from CacheSettingsTab) */
|
||||
cache: {
|
||||
semanticCacheEnabled: boolean;
|
||||
semanticCacheMaxSize: number;
|
||||
semanticCacheTTL: number;
|
||||
promptCacheEnabled: boolean;
|
||||
promptCacheStrategy: "auto" | "system-only" | "manual";
|
||||
alwaysPreserveClientCache: "auto" | "always" | "never";
|
||||
};
|
||||
|
||||
/** 5. Retention (per-table cleanup policies) */
|
||||
retention: {
|
||||
quotaSnapshots: number;
|
||||
compressionAnalytics: number;
|
||||
mcpAudit: number;
|
||||
a2aEvents: number;
|
||||
callLogs: number;
|
||||
usageHistory: number;
|
||||
memoryEntries: number;
|
||||
autoCleanupEnabled: boolean;
|
||||
};
|
||||
|
||||
/** 6. Compression (aggregation) */
|
||||
aggregation: {
|
||||
enabled: boolean;
|
||||
rawDataRetentionDays: number;
|
||||
granularity: "hourly" | "daily" | "weekly";
|
||||
};
|
||||
|
||||
/** 7. Optimization (auto_vacuum, VACUUM, page/cache) */
|
||||
optimization: {
|
||||
autoVacuumMode: "NONE" | "FULL" | "INCREMENTAL";
|
||||
scheduledVacuum: "never" | "daily" | "weekly" | "monthly";
|
||||
vacuumHour: number;
|
||||
pageSize: number;
|
||||
cacheSize: number;
|
||||
optimizeOnStartup: boolean;
|
||||
};
|
||||
|
||||
/** Read-only stats */
|
||||
stats: {
|
||||
databaseSizeBytes: number;
|
||||
pageCount: number;
|
||||
freelistCount: number;
|
||||
lastVacuumAt: string | null;
|
||||
lastOptimizationAt: string | null;
|
||||
integrityCheck: "ok" | "error" | null;
|
||||
};
|
||||
}
|
||||
|
||||
/** Default database settings */
|
||||
export const DEFAULT_DATABASE_SETTINGS: Omit<DatabaseSettings, "location" | "stats"> = {
|
||||
logs: {
|
||||
detailedLogsEnabled: false,
|
||||
callLogPipelineEnabled: false,
|
||||
maxDetailSizeKb: 10,
|
||||
ringBufferSize: 500,
|
||||
},
|
||||
backup: {
|
||||
autoBackupEnabled: false,
|
||||
autoBackupFrequency: "never",
|
||||
keepLastNBackups: 5,
|
||||
},
|
||||
cache: {
|
||||
semanticCacheEnabled: true,
|
||||
semanticCacheMaxSize: 100,
|
||||
semanticCacheTTL: 1800000,
|
||||
promptCacheEnabled: true,
|
||||
promptCacheStrategy: "auto",
|
||||
alwaysPreserveClientCache: "auto",
|
||||
},
|
||||
retention: {
|
||||
quotaSnapshots: 7,
|
||||
compressionAnalytics: 30,
|
||||
mcpAudit: 30,
|
||||
a2aEvents: 30,
|
||||
callLogs: 30,
|
||||
usageHistory: 30,
|
||||
memoryEntries: 30,
|
||||
autoCleanupEnabled: true,
|
||||
},
|
||||
aggregation: {
|
||||
enabled: true,
|
||||
rawDataRetentionDays: 30,
|
||||
granularity: "daily",
|
||||
},
|
||||
optimization: {
|
||||
autoVacuumMode: "FULL",
|
||||
scheduledVacuum: "weekly",
|
||||
vacuumHour: 2,
|
||||
pageSize: 4096,
|
||||
cacheSize: -2000,
|
||||
optimizeOnStartup: true,
|
||||
},
|
||||
};
|
||||
@@ -10,3 +10,5 @@ export type { ApiKey } from "./apiKey";
|
||||
export type { Combo, ComboStrategy, ComboNode } from "./combo";
|
||||
export type { UsageEntry, UsageStats, ProviderUsageStats, ModelUsageStats, CallLog } from "./usage";
|
||||
export type { Settings, ComboDefaults, ProxyConfig, KVPair } from "./settings";
|
||||
export type { DatabaseSettings } from "./databaseSettings";
|
||||
export { DEFAULT_DATABASE_SETTINGS } from "./databaseSettings";
|
||||
|
||||
Reference in New Issue
Block a user