diff --git a/CHANGELOG.md b/CHANGELOG.md index 05f0b51a11..de4b42b393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## [Unreleased] +> [!WARNING] +> **BREAKING CHANGE: request logging, retention, and logging environment variables have been redesigned.** +> On the first startup after upgrading, OmniRoute archives legacy request logs from `DATA_DIR/logs/`, legacy `DATA_DIR/call_logs/`, and `DATA_DIR/log.txt` into `DATA_DIR/log_archives/*.zip`, then removes the deprecated layout and switches to the new unified artifact format under `DATA_DIR/call_logs/`. + +### ✨ New Features + +- **Unified Request Log Artifacts:** Request logging now stores one SQLite index row plus one JSON artifact per request under `DATA_DIR/call_logs/`, with optional pipeline capture embedded in the same file. + +### ⚠️ Breaking Changes + +- **Request Log Layout:** Removed the old multi-file `DATA_DIR/logs/` request log sessions and the `DATA_DIR/log.txt` summary file. New requests are written as single JSON artifacts in `DATA_DIR/call_logs/YYYY-MM-DD/`. +- **Logging Environment Variables:** Replaced `LOG_*`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` with the new `APP_LOG_*` and `CALL_LOG_RETENTION_DAYS` configuration model. +- **Pipeline Toggle Setting:** Replaced the legacy `detailed_logs_enabled` setting with `call_log_pipeline_enabled`. New pipeline details are embedded inside the request artifact instead of being stored as separate `request_detail_logs` records. + +### 🛠️ Maintenance + +- **Legacy Request Log Upgrade Backup:** Upgrades now archive old `data/logs/`, legacy `data/call_logs/`, and `data/log.txt` layouts into `DATA_DIR/log_archives/*.zip` before removing the deprecated structure. +- **Streaming Usage Persistence:** Streaming requests now write a single `usage_history` row on completion instead of emitting a duplicate in-progress usage row with empty status metadata. + --- ## [3.3.11] - 2026-03-31 diff --git a/README.md b/README.md index 1a5d4b79de..f3ae4c9bae 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,30 @@ _Your universal API proxy — one endpoint, 67+ providers, zero downtime. Now wi --- +## Breaking Change: Unified Logging Upgrade + +> [!WARNING] +> **This release changes both the on-disk request log layout and the logging environment variables.** +> +> If you are upgrading an existing instance: +> +> - Request logs now live in `DATA_DIR/call_logs/YYYY-MM-DD/` as **one JSON artifact per request**. +> - The old `DATA_DIR/logs/` session folders and `DATA_DIR/log.txt` summary file are removed. +> - On the first startup after upgrading, OmniRoute creates a safety backup at `DATA_DIR/log_archives/*.zip` before removing the deprecated request log layout. +> - Legacy logging env vars such as `LOG_TO_FILE`, `LOG_FILE_PATH`, `LOG_MAX_FILE_SIZE`, `LOG_RETENTION_DAYS`, `LOG_LEVEL`, `LOG_FORMAT`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` are no longer supported. +> - Use the new env model instead: +> - `APP_LOG_TO_FILE` +> - `APP_LOG_FILE_PATH` +> - `APP_LOG_MAX_FILE_SIZE` +> - `APP_LOG_RETENTION_DAYS` +> - `APP_LOG_LEVEL` +> - `APP_LOG_FORMAT` +> - `CALL_LOG_RETENTION_DAYS` +> +> For release details and upgrade notes, see the [CHANGELOG](CHANGELOG.md). + +--- + ## 🆕 What's New in v3.0.0 > **Upgrading from v2.9.5?** — See the [full CHANGELOG](CHANGELOG.md#300--2026-03-22-release-candidate--not-yet-merged-to-main) for all changes. @@ -1811,7 +1835,9 @@ opencode **No request logs** -- Set `ENABLE_REQUEST_LOGS=true` in `.env` +- Request artifacts are written to `DATA_DIR/call_logs/` as one JSON file per request +- Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads +- Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log` **Connection test shows "Invalid" for OpenAI-compatible providers** diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 26df8a5791..d8a46133d3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -23,7 +23,7 @@ import { import { HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts"; import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection } from "@/lib/db/providers"; -import { isDetailedLoggingEnabled, saveRequestDetailLog } from "@/lib/db/detailedLogs"; +import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; import { logAuditEvent } from "@/lib/compliance"; import { handleBypassRequest } from "../utils/bypassHandler.ts"; import { @@ -536,6 +536,24 @@ export async function handleChatCore({ claudeCacheUsageMeta?: Record; }) => { const callLogId = generateRequestId(); + const pipelinePayloads = detailedLoggingEnabled ? reqLogger?.getPipelinePayloads?.() : null; + + if (pipelinePayloads) { + if (providerResponse !== undefined) { + pipelinePayloads.providerResponse = providerResponse as Record; + } + if (clientResponse !== undefined) { + pipelinePayloads.clientResponse = clientResponse as Record; + } + if (error) { + pipelinePayloads.error = { + ...(typeof pipelinePayloads.error === "object" && pipelinePayloads.error + ? (pipelinePayloads.error as Record) + : {}), + message: error, + }; + } + } saveCallLog({ id: callLogId, @@ -568,31 +586,8 @@ export async function handleChatCore({ apiKeyId: apiKeyInfo?.id || null, apiKeyName: apiKeyInfo?.name || null, noLog: noLogEnabled, + pipelinePayloads, }).catch(() => {}); - - if (!detailedLoggingEnabled) { - return; - } - - try { - saveRequestDetailLog({ - call_log_id: callLogId, - client_request: clientRawRequest?.body ?? body, - translated_request: providerRequest ?? null, - provider_response: providerResponse ?? null, - client_response: clientResponse ?? null, - provider, - model, - source_format: sourceFormat, - target_format: targetFormat, - duration_ms: Date.now() - startTime, - api_key_id: apiKeyInfo?.id || null, - no_log: noLogEnabled, - }); - } catch (err) { - const errMessage = err instanceof Error ? err.message : String(err); - log?.debug?.("DETAIL_LOG", `Failed to save detailed log: ${errMessage}`); - } }; // Primary path: merge client model id + alias target so config on either key applies; resolved diff --git a/open-sse/utils/logger.ts b/open-sse/utils/logger.ts index 0620c752e8..b52f0b4128 100644 --- a/open-sse/utils/logger.ts +++ b/open-sse/utils/logger.ts @@ -2,7 +2,7 @@ * Structured console logger utility for omniroute. * * Provides consistent, machine-parseable log output across the codebase. - * Supports two output formats controlled by LOG_FORMAT env var: + * Supports two output formats controlled by APP_LOG_FORMAT env var: * - "text" (default): [LEVEL] [TAG] message {metadata} * - "json": Single-line JSON objects for log aggregators * @@ -18,17 +18,16 @@ * reqLog.info("AUTH", "Token refreshed", { provider: "claude" }); * * Environment variables: - * LOG_LEVEL — minimum level: debug | info | warn | error (default: info) - * LOG_FORMAT — output format: text | json (default: text) + * APP_LOG_LEVEL — minimum level: debug | info | warn | error (default: info) + * APP_LOG_FORMAT — output format: text | json (default: text) */ +import { getAppLogFormat, getAppLogLevel } from "../../src/lib/logEnv"; const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 }; -const currentLevel = - LEVELS[((typeof process !== "undefined" && process.env?.LOG_LEVEL) || "info").toLowerCase()] ?? - LEVELS.info; +const currentLevel = LEVELS[getAppLogLevel("info").toLowerCase()] ?? LEVELS.info; -const jsonFormat = typeof process !== "undefined" && process.env?.LOG_FORMAT === "json"; +const jsonFormat = getAppLogFormat("text") === "json"; let requestCounter = 0; diff --git a/open-sse/utils/requestLogger.ts b/open-sse/utils/requestLogger.ts index be4f7e91ce..c99549f5b0 100644 --- a/open-sse/utils/requestLogger.ts +++ b/open-sse/utils/requestLogger.ts @@ -1,92 +1,117 @@ -// Check if running in Node.js environment (has fs module) -const isNode = - typeof process !== "undefined" && process.versions?.node && typeof window === "undefined"; +type JsonRecord = Record; -// Check if logging is enabled via environment variable (default: false) -const LOGGING_ENABLED = - typeof process !== "undefined" && process.env?.ENABLE_REQUEST_LOGS === "true"; +type HeaderInput = + | Headers + | Record + | { entries?: () => IterableIterator<[string, string]> } + | null + | undefined; -let fs = null; -let path = null; -let LOGS_DIR = null; +export type RequestPipelinePayloads = { + clientRawRequest?: JsonRecord; + sourceRequest?: JsonRecord; + openaiRequest?: JsonRecord; + providerRequest?: JsonRecord; + providerResponse?: JsonRecord; + clientResponse?: JsonRecord; + error?: JsonRecord; + streamChunks?: { + provider?: string[]; + openai?: string[]; + client?: string[]; + }; +}; -// Lazy load Node.js modules (avoid top-level await) -async function ensureNodeModules() { - if (!isNode || !LOGGING_ENABLED || fs) return; - try { - fs = await import("fs"); - path = await import("path"); - const { resolveDataDir } = await import("../../src/lib/dataPaths"); - LOGS_DIR = path.join(resolveDataDir(), "logs"); - } catch { - // Running in non-Node environment (Worker, Browser, etc.) - } -} +type RequestLogger = { + sessionPath: null; + logClientRawRequest: (endpoint: unknown, body: unknown, headers?: HeaderInput) => void; + logRawRequest: (body: unknown, headers?: HeaderInput) => void; + logOpenAIRequest: (body: unknown) => void; + logTargetRequest: (url: unknown, headers: HeaderInput, body: unknown) => void; + logProviderResponse: ( + status: unknown, + statusText: unknown, + headers: HeaderInput, + body: unknown + ) => void; + appendProviderChunk: (chunk: string) => void; + appendOpenAIChunk: (chunk: string) => void; + logConvertedResponse: (body: unknown) => void; + appendConvertedChunk: (chunk: string) => void; + logError: (error: unknown, requestBody?: unknown) => void; + getPipelinePayloads: () => RequestPipelinePayloads | null; +}; -// Format timestamp for folder name: 20251228_143045 -function formatTimestamp(date = new Date()) { - const pad = (n) => String(n).padStart(2, "0"); - const y = date.getFullYear(); - const m = pad(date.getMonth() + 1); - const d = pad(date.getDate()); - const h = pad(date.getHours()); - const min = pad(date.getMinutes()); - const s = pad(date.getSeconds()); - return `${y}${m}${d}_${h}${min}${s}`; -} - -// Create log session folder: {sourceFormat}_{targetFormat}_{model}_{timestamp} -async function createLogSession(sourceFormat, targetFormat, model) { - await ensureNodeModules(); - if (!fs || !LOGS_DIR) return null; - - try { - await fs.promises.mkdir(LOGS_DIR, { recursive: true }); - - const timestamp = formatTimestamp(); - const safeModel = (model || "unknown").replace(/[/:]/g, "-"); - const folderName = `${sourceFormat}_${targetFormat}_${safeModel}_${timestamp}`; - const sessionPath = path.join(LOGS_DIR, folderName); - - await fs.promises.mkdir(sessionPath, { recursive: true }); - - return sessionPath; - } catch (err) { - console.log("[LOG] Failed to create log session:", err.message); - return null; - } -} - -// Write JSON file (async, fire-and-forget) -function writeJsonFile(sessionPath, filename, data) { - if (!fs || !sessionPath) return; - - const filePath = path.join(sessionPath, filename); - fs.promises - .writeFile(filePath, JSON.stringify(data, null, 2)) - .catch((err) => console.log(`[LOG] Failed to write ${filename}:`, err.message)); -} - -// Mask sensitive data in headers before writing to log files -function maskSensitiveHeaders(headers) { +function maskSensitiveHeaders(headers: HeaderInput): Record { if (!headers) return {}; - const masked = { ...headers }; + + const headerEntries = + typeof (headers as Headers).entries === "function" + ? Object.fromEntries((headers as Headers).entries()) + : { ...(headers as Record) }; + + const masked = { ...headerEntries }; const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token"]; for (const key of Object.keys(masked)) { const lowerKey = key.toLowerCase(); - if (sensitiveKeys.some((sk) => lowerKey.includes(sk))) { - const value = masked[key]; - if (value && value.length > 20) { - masked[key] = value.slice(0, 10) + "..." + value.slice(-5); - } + if (!sensitiveKeys.some((candidate) => lowerKey.includes(candidate))) { + continue; + } + + const value = masked[key]; + if (typeof value === "string" && value.length > 20) { + masked[key] = `${value.slice(0, 10)}...${value.slice(-5)}`; + } else if (value) { + masked[key] = "[REDACTED]"; } } + return masked; } -// No-op logger when logging is disabled -function createNoOpLogger() { +function createEmptyStreamChunks() { + return { + provider: [] as string[], + openai: [] as string[], + client: [] as string[], + }; +} + +function hasOwnValues(value: unknown): boolean { + return Boolean(value && typeof value === "object" && Object.keys(value as JsonRecord).length > 0); +} + +function compactPipelinePayloads( + payloads: RequestPipelinePayloads +): RequestPipelinePayloads | null { + const result: RequestPipelinePayloads = {}; + + for (const [key, value] of Object.entries(payloads)) { + if (value === null || value === undefined) { + continue; + } + + if (key === "streamChunks" && value && typeof value === "object") { + const chunkRecord = value as Record; + const compactedChunks = Object.fromEntries( + Object.entries(chunkRecord).filter( + ([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0 + ) + ); + if (Object.keys(compactedChunks).length > 0) { + result.streamChunks = compactedChunks as RequestPipelinePayloads["streamChunks"]; + } + continue; + } + + result[key as keyof RequestPipelinePayloads] = value as never; + } + + return hasOwnValues(result) ? result : null; +} + +function createNoOpLogger(): RequestLogger { return { sessionPath: null, logClientRawRequest() {}, @@ -99,151 +124,106 @@ function createNoOpLogger() { logConvertedResponse() {}, appendConvertedChunk() {}, logError() {}, + getPipelinePayloads() { + return null; + }, }; } -/** - * Create a new log session and return logger functions - * @param {string} sourceFormat - Source format from client (claude, openai, etc.) - * @param {string} targetFormat - Target format to provider (antigravity, gemini-cli, etc.) - * @param {string} model - Model name - * @returns {Promise} Promise that resolves to logger object with methods to log each stage - */ -export async function createRequestLogger(sourceFormat, targetFormat, model) { - // Return no-op logger if logging is disabled - if (!LOGGING_ENABLED) { - return createNoOpLogger(); - } - - // Wait for session to be created before returning logger - const sessionPath = await createLogSession(sourceFormat, targetFormat, model); +export async function createRequestLogger( + _sourceFormat?: string, + _targetFormat?: string, + _model?: string +): Promise { + const streamChunks = createEmptyStreamChunks(); + const payloads: RequestPipelinePayloads = { + streamChunks, + }; return { - get sessionPath() { - return sessionPath; - }, + sessionPath: null, - // 1. Log client raw request (before all conversion steps) logClientRawRequest(endpoint, body, headers = {}) { - writeJsonFile(sessionPath, "1_req_client.json", { + payloads.clientRawRequest = { timestamp: new Date().toISOString(), endpoint, headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 2. Log raw request from client (after initial conversion like responsesApi) logRawRequest(body, headers = {}) { - writeJsonFile(sessionPath, "2_req_source.json", { + payloads.sourceRequest = { timestamp: new Date().toISOString(), headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 3. Log OpenAI intermediate format (source → openai) logOpenAIRequest(body) { - writeJsonFile(sessionPath, "3_req_openai.json", { + payloads.openaiRequest = { timestamp: new Date().toISOString(), body, - }); + }; }, - // 4. Log target format request (openai → target) logTargetRequest(url, headers, body) { - writeJsonFile(sessionPath, "4_req_target.json", { + payloads.providerRequest = { timestamp: new Date().toISOString(), url, headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 5. Log provider response (for non-streaming or error) logProviderResponse(status, statusText, headers, body) { - const filename = "5_res_provider.json"; - writeJsonFile(sessionPath, filename, { + payloads.providerResponse = { timestamp: new Date().toISOString(), status, statusText, - headers: headers - ? typeof headers.entries === "function" - ? Object.fromEntries(headers.entries()) - : headers - : {}, + headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 5. Append streaming chunk to provider response (async) appendProviderChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "5_res_provider.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.provider.push(chunk); + } }, - // 6. Append OpenAI intermediate chunks (async) appendOpenAIChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "6_res_openai.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.openai.push(chunk); + } }, - // 7. Log converted response to client (for non-streaming) logConvertedResponse(body) { - writeJsonFile(sessionPath, "7_res_client.json", { + payloads.clientResponse = { timestamp: new Date().toISOString(), body, - }); + }; }, - // 7b. Append streaming chunk to converted response (async) appendConvertedChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "7_res_client.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.client.push(chunk); + } }, - // 8. Log error logError(error, requestBody = null) { - writeJsonFile(sessionPath, "6_error.json", { + payloads.error = { timestamp: new Date().toISOString(), - error: error?.message || String(error), - stack: error?.stack, - requestBody, - }); - }, - }; -} - -// Legacy logError (kept for backward compatibility, converted to async) -export function logError(provider, { error, url, model, requestBody }) { - if (!fs || !LOGS_DIR) return; - - const writeLog = async () => { - try { - await fs.promises.mkdir(LOGS_DIR, { recursive: true }); - - const date = new Date().toISOString().split("T")[0]; - const logPath = path.join(LOGS_DIR, `${provider}-${date}.log`); - - const logEntry = { - timestamp: new Date().toISOString(), - type: "error", - provider, - model, - url, - error: error?.message || String(error), - stack: error?.stack, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, requestBody, }; + }, - await fs.promises.appendFile(logPath, JSON.stringify(logEntry) + "\n"); - } catch (err) { - console.log("[LOG] Failed to write error log:", err.message); - } + getPipelinePayloads() { + return compactPipelinePayloads(payloads); + }, }; - - writeLog(); } + +export function logError(_provider: string, _entry: unknown) {} diff --git a/open-sse/utils/usageTracking.ts b/open-sse/utils/usageTracking.ts index 8c42a62026..ccf08709ce 100644 --- a/open-sse/utils/usageTracking.ts +++ b/open-sse/utils/usageTracking.ts @@ -2,7 +2,7 @@ * Token Usage Tracking - Extract, normalize, estimate and log token usage */ -import { saveRequestUsage, appendRequestLog } from "@/lib/usageDb"; +import { appendRequestLog } from "@/lib/usageDb"; import { getLoggedInputTokens, getLoggedOutputTokens, @@ -444,7 +444,8 @@ export function logUsage(provider, usage, model = null, connectionId = null, api console.log(msg); - // Save to usage DB with cache-read tracked separately from the main input counter. + // Streaming requests persist usage once in chatCore's completion callback. + // Keep this helper side-effect free apart from console visibility. const tokens = { input: inTokens, output: outTokens, @@ -452,13 +453,5 @@ export function logUsage(provider, usage, model = null, connectionId = null, api cacheCreation: cacheCreation || 0, reasoning: reasoning || 0, }; - saveRequestUsage({ - model, - provider, - connectionId, - apiKeyId: apiKeyInfo?.id || undefined, - apiKeyName: apiKeyInfo?.name || undefined, - tokens, - }).catch(() => {}); appendRequestLog({ model, provider, connectionId, tokens, status: "200 OK" }).catch(() => {}); } diff --git a/package-lock.json b/package-lock.json index 587052306f..c884d26fcb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "undici": "^7.19.2", "uuid": "^13.0.0", "wreq-js": "^2.0.1", + "yazl": "^3.3.1", "zod": "^4.3.6", "zustand": "^5.0.10" }, @@ -7654,6 +7655,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -20226,6 +20236,15 @@ "node": ">=8" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index fe3706a8ce..a67370006b 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,7 @@ "undici": "^7.19.2", "uuid": "^13.0.0", "wreq-js": "^2.0.1", + "yazl": "^3.3.1", "zod": "^4.3.6", "zustand": "^5.0.10" }, diff --git a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx index 342cb619e1..96c821dbcc 100644 --- a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx @@ -18,11 +18,6 @@ export default function SystemStorageTab() { const [importStatus, setImportStatus] = useState({ type: "", message: "" }); const [confirmImport, setConfirmImport] = useState(false); const [pendingImportFile, setPendingImportFile] = useState(null); - const [maxCallLogs, setMaxCallLogs] = useState(10000); - const [maxCallLogsDraft, setMaxCallLogsDraft] = useState("10000"); - const [settingsLoading, setSettingsLoading] = useState(true); - const [maxCallLogsSaving, setMaxCallLogsSaving] = useState(false); - const [maxCallLogsStatus, setMaxCallLogsStatus] = useState({ type: "", message: "" }); const fileInputRef = useRef(null); const locale = useLocale(); const t = useTranslations("settings"); @@ -31,7 +26,10 @@ export default function SystemStorageTab() { driver: "sqlite", dbPath: "~/.omniroute/storage.sqlite", sizeBytes: 0, - retentionDays: 90, + retentionDays: { + app: 7, + call: 7, + }, lastBackupAt: null, }); @@ -59,27 +57,6 @@ export default function SystemStorageTab() { } }; - const loadSettings = async () => { - setSettingsLoading(true); - try { - const res = await fetch("/api/settings"); - if (!res.ok) return; - const data = await res.json(); - const value = - typeof data.maxCallLogs === "number" && - Number.isInteger(data.maxCallLogs) && - data.maxCallLogs > 0 - ? data.maxCallLogs - : 10000; - setMaxCallLogs(value); - setMaxCallLogsDraft(String(value)); - } catch (err) { - console.error("Failed to fetch settings:", err); - } finally { - setSettingsLoading(false); - } - }; - const handleManualBackup = async () => { setManualBackupLoading(true); setManualBackupStatus({ type: "", message: "" }); @@ -145,47 +122,8 @@ export default function SystemStorageTab() { useEffect(() => { loadStorageHealth(); - loadSettings(); }, []); - const handleSaveMaxCallLogs = async () => { - const parsed = Number.parseInt(maxCallLogsDraft, 10); - if (!Number.isInteger(parsed) || parsed <= 0) { - setMaxCallLogsStatus({ - type: "error", - message: "Enter a positive integer for the call log limit.", - }); - return; - } - - setMaxCallLogsSaving(true); - setMaxCallLogsStatus({ type: "", message: "" }); - try { - const res = await fetch("/api/settings", { - method: "PATCH", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ maxCallLogs: parsed }), - }); - const data = await res.json(); - if (!res.ok) { - throw new Error(data.error || "Failed to save call log limit"); - } - setMaxCallLogs(parsed); - setMaxCallLogsDraft(String(parsed)); - setMaxCallLogsStatus({ - type: "success", - message: "Call log retention limit saved.", - }); - } catch (err) { - setMaxCallLogsStatus({ - type: "error", - message: (err as Error).message || "Failed to save call log limit", - }); - } finally { - setMaxCallLogsSaving(false); - } - }; - const handleExport = async () => { setExportLoading(true); try { @@ -345,53 +283,23 @@ export default function SystemStorageTab() {
-
+
-

Call log retention limit

+

Log retention policy

- Keep only the most recent call log entries in SQLite. Older entries are pruned - automatically after each new request log is saved. + Request logs follow CALL_LOG_RETENTION_DAYS. Application and audit logs + follow APP_LOG_RETENTION_DAYS.

- - {maxCallLogs.toLocaleString()} - -
- -
- setMaxCallLogsDraft(e.target.value)} - disabled={settingsLoading || maxCallLogsSaving} - className="w-40 rounded-lg border border-border bg-bg-secondary px-3 py-2 text-sm text-text-main focus:outline-none focus:ring-1 focus:ring-primary/40" - aria-label="Call log retention limit" - /> - -
- - {maxCallLogsStatus.message && ( -
- {maxCallLogsStatus.message} +
+ + Call {storageHealth.retentionDays.call}d + + + App {storageHealth.retentionDays.app}d +
- )} +
{/* Export / Import */} diff --git a/src/app/api/logs/console/route.ts b/src/app/api/logs/console/route.ts index 3c7f5a4dbd..96fb706825 100644 --- a/src/app/api/logs/console/route.ts +++ b/src/app/api/logs/console/route.ts @@ -12,7 +12,7 @@ import { NextRequest, NextResponse } from "next/server"; import { readFileSync, existsSync } from "fs"; -import { join } from "path"; +import { getAppLogFilePath } from "@/lib/logEnv"; const LEVEL_ORDER: Record = { trace: 5, @@ -34,7 +34,7 @@ const NUMERIC_LEVEL_MAP: Record = { }; function getLogFilePath(): string { - return process.env.LOG_FILE_PATH || join(process.cwd(), "logs", "application", "app.log"); + return getAppLogFilePath(); } function parseLevel(raw: string | number): string { diff --git a/src/app/api/logs/detail/route.ts b/src/app/api/logs/detail/route.ts index e2734b5cd3..4a85321e42 100644 --- a/src/app/api/logs/detail/route.ts +++ b/src/app/api/logs/detail/route.ts @@ -1,6 +1,6 @@ /** - * GET /api/logs/detail — List detailed request logs + current enabled flag - * POST /api/logs/detail — Enable/disable detailed logging + * GET /api/logs/detail — List legacy detailed request logs + current enabled flag + * POST /api/logs/detail — Enable/disable pipeline capture for unified call log artifacts */ import { NextRequest, NextResponse } from "next/server"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; @@ -35,13 +35,13 @@ export async function POST(req: NextRequest) { const body = await req.json(); const enabled = body.enabled === true || body.enabled === "1"; - await updateSettings({ detailed_logs_enabled: enabled }); + await updateSettings({ call_log_pipeline_enabled: enabled }); return NextResponse.json({ success: true, enabled, message: enabled - ? "Detailed logging enabled. Pipeline bodies will be captured for new requests." - : "Detailed logging disabled.", + ? "Pipeline capture enabled. New request artifacts will include per-stage payloads." + : "Pipeline capture disabled.", }); } diff --git a/src/app/api/logs/export/route.ts b/src/app/api/logs/export/route.ts index afb0a24c3c..3ee0390ff9 100644 --- a/src/app/api/logs/export/route.ts +++ b/src/app/api/logs/export/route.ts @@ -17,18 +17,12 @@ export async function GET(request: Request) { let rows: unknown[] = []; let tableName = ""; - if (logType === "call-logs") { + if (logType === "call-logs" || logType === "request-logs") { tableName = "call_logs"; const stmt = db.prepare( "SELECT * FROM call_logs WHERE timestamp >= @since ORDER BY timestamp DESC" ); rows = stmt.all({ since }); - } else if (logType === "request-logs") { - tableName = "request_logs"; - const stmt = db.prepare( - "SELECT * FROM request_logs WHERE timestamp >= @since ORDER BY timestamp DESC" - ); - rows = stmt.all({ since }); } else if (logType === "proxy-logs") { tableName = "proxy_logs"; const stmt = db.prepare( diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 7bf98a878e..575090df5c 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -19,14 +19,12 @@ export async function GET() { setCliCompatProviders(settings.cliCompatProviders as string[]); } - const enableRequestLogs = process.env.ENABLE_REQUEST_LOGS === "true"; const runtimePorts = getRuntimePorts(); const cloudUrl = process.env.CLOUD_URL || process.env.NEXT_PUBLIC_CLOUD_URL || null; const machineId = await getConsistentMachineId(); return NextResponse.json({ ...safeSettings, - enableRequestLogs, hasPassword: !!password || !!process.env.INITIAL_PASSWORD, runtimePorts, apiPort: runtimePorts.apiPort, @@ -114,11 +112,6 @@ export async function PATCH(request) { setCliCompatProviders(body.cliCompatProviders || []); } - if ("maxCallLogs" in body) { - const { invalidateCallLogsMaxCache } = await import("@/lib/usage/callLogs"); - invalidateCallLogsMaxCache(); - } - // Sync cache control settings to runtime cache if ("alwaysPreserveClientCache" in body) { const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings"); diff --git a/src/app/api/storage/health/route.ts b/src/app/api/storage/health/route.ts index ad8aaa56fd..42a2c5f3e5 100644 --- a/src/app/api/storage/health/route.ts +++ b/src/app/api/storage/health/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import path from "path"; import fs from "fs"; import { resolveDataDir } from "@/lib/dataPaths"; +import { getAppLogRetentionDays, getCallLogRetentionDays } from "@/lib/logEnv"; /** * GET /api/storage/health — Return database storage information. @@ -56,7 +57,10 @@ export async function GET() { sizeBytes, lastBackupAt, backupCount, - retentionDays: 90, + retentionDays: { + app: getAppLogRetentionDays(), + call: getCallLogRetentionDays(), + }, dataDir: dataDir.startsWith(homeDir) ? "~" + dataDir.slice(homeDir.length) : dataDir, }); } catch (error) { diff --git a/src/domain/types.ts b/src/domain/types.ts index ef516fa406..a2dc1b3ade 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -96,11 +96,10 @@ * @property {boolean} hasPassword - Whether a password has been set * @property {string} [theme] - UI theme * @property {string} [language] - UI language - * @property {boolean} [enableRequestLogs] - Whether request logging is enabled * @property {boolean} [enableSocks5Proxy] - Whether SOCKS5 proxy is allowed * @property {string} [instanceName] - Instance display name * @property {string} [corsOrigins] - Allowed CORS origins - * @property {number} [logRetentionDays] - Log retention in days + * @property {boolean} [call_log_pipeline_enabled] - Whether per-request pipeline capture is enabled * @property {string[]} [hiddenSidebarItems] - Sidebar entry ids hidden for visual decluttering */ diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 529b3aa451..8ecf14c4a3 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -65,6 +65,9 @@ async function ensureSecrets(): Promise { export async function registerNodejs(): Promise { await ensureSecrets(); + // Trigger request-log layout migration during startup, before any request hits usageDb. + await import("@/lib/usage/migrations"); + const { initConsoleInterceptor } = await import("@/lib/consoleInterceptor"); initConsoleInterceptor(); @@ -127,7 +130,14 @@ export async function registerNodejs(): Promise { console.log("[COMPLIANCE] Audit log table initialized"); const cleanup = cleanupExpiredLogs(); - if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) { + if ( + cleanup.deletedUsage || + cleanup.deletedCallLogs || + cleanup.deletedProxyLogs || + cleanup.deletedRequestDetailLogs || + cleanup.deletedAuditLogs || + cleanup.deletedMcpAuditLogs + ) { console.log("[COMPLIANCE] Expired log cleanup:", cleanup); } } catch (err: unknown) { diff --git a/src/lib/compliance/index.ts b/src/lib/compliance/index.ts index f5577497a2..df7ae54407 100644 --- a/src/lib/compliance/index.ts +++ b/src/lib/compliance/index.ts @@ -2,7 +2,7 @@ * Compliance Controls — T-43 * * Implements compliance features: - * - LOG_RETENTION_DAYS: automatic log cleanup + * - APP_LOG_RETENTION_DAYS / CALL_LOG_RETENTION_DAYS: automatic log cleanup * - noLog opt-out per API key * - audit_log table for administrative actions * @@ -10,6 +10,7 @@ */ import { getDbInstance } from "../db/core"; +import { getAppLogRetentionDays, getCallLogRetentionDays } from "../logEnv"; /** @returns {import("better-sqlite3").Database | null} */ function getDb() { @@ -20,8 +21,6 @@ function getDb() { } } -const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "90", 10); - /** * Initialize the audit_log table. */ @@ -212,60 +211,122 @@ export function isNoLog(apiKeyId: string) { // ─── Log Retention / Cleanup ──────────────── /** - * Get the configured retention period. - * @returns {number} Days + * Get the configured retention periods. */ export function getRetentionDays() { - return LOG_RETENTION_DAYS; + return { + app: getAppLogRetentionDays(), + call: getCallLogRetentionDays(), + }; } /** - * Clean up logs older than LOG_RETENTION_DAYS. + * Clean up logs using split APP/CALL retention windows. * Should be called periodically (e.g. daily cron or on startup). * - * @returns {{ deletedUsage: number, deletedCallLogs: number, deletedAuditLogs: number }} + * @returns {{ + * deletedUsage: number, + * deletedCallLogs: number, + * deletedProxyLogs: number, + * deletedRequestDetailLogs: number, + * deletedAuditLogs: number, + * deletedMcpAuditLogs: number, + * appRetentionDays: number, + * callRetentionDays: number + * }} */ export function cleanupExpiredLogs() { const db = getDb(); - if (!db) return { deletedUsage: 0, deletedCallLogs: 0, deletedAuditLogs: 0 }; + const appRetentionDays = getAppLogRetentionDays(); + const callRetentionDays = getCallLogRetentionDays(); - const cutoff = new Date(Date.now() - LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000).toISOString(); + if (!db) { + return { + deletedUsage: 0, + deletedCallLogs: 0, + deletedProxyLogs: 0, + deletedRequestDetailLogs: 0, + deletedAuditLogs: 0, + deletedMcpAuditLogs: 0, + appRetentionDays, + callRetentionDays, + }; + } + + const callCutoff = new Date(Date.now() - callRetentionDays * 24 * 60 * 60 * 1000).toISOString(); + const appCutoff = new Date(Date.now() - appRetentionDays * 24 * 60 * 60 * 1000).toISOString(); let deletedUsage = 0; let deletedCallLogs = 0; + let deletedProxyLogs = 0; + let deletedRequestDetailLogs = 0; let deletedAuditLogs = 0; + let deletedMcpAuditLogs = 0; try { - // Clean usage_history - const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(cutoff); + const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(callCutoff); deletedUsage = r1.changes; } catch { /* table may not exist */ } try { - // Clean call_logs - const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(cutoff); + const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(callCutoff); deletedCallLogs = r2.changes; } catch { /* table may not exist */ } try { - // Clean audit_log (keep longer, 2x retention) - const auditCutoff = new Date( - Date.now() - LOG_RETENTION_DAYS * 2 * 24 * 60 * 60 * 1000 - ).toISOString(); - const r3 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(auditCutoff); - deletedAuditLogs = r3.changes; + const r3 = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?").run(callCutoff); + deletedProxyLogs = r3.changes; + } catch { + /* table may not exist */ + } + + try { + const r4 = db.prepare("DELETE FROM request_detail_logs WHERE timestamp < ?").run(callCutoff); + deletedRequestDetailLogs = r4.changes; + } catch { + /* legacy table may not exist */ + } + + try { + const r5 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(appCutoff); + deletedAuditLogs = r5.changes; + } catch { + /* table may not exist */ + } + + try { + const r6 = db.prepare("DELETE FROM mcp_tool_audit WHERE created_at < ?").run(appCutoff); + deletedMcpAuditLogs = r6.changes; } catch { /* table may not exist */ } logAuditEvent({ action: "compliance.cleanup", - details: { deletedUsage, deletedCallLogs, deletedAuditLogs, retentionDays: LOG_RETENTION_DAYS }, + details: { + deletedUsage, + deletedCallLogs, + deletedProxyLogs, + deletedRequestDetailLogs, + deletedAuditLogs, + deletedMcpAuditLogs, + appRetentionDays, + callRetentionDays, + }, }); - return { deletedUsage, deletedCallLogs, deletedAuditLogs }; + return { + deletedUsage, + deletedCallLogs, + deletedProxyLogs, + deletedRequestDetailLogs, + deletedAuditLogs, + deletedMcpAuditLogs, + appRetentionDays, + callRetentionDays, + }; } diff --git a/src/lib/consoleInterceptor.ts b/src/lib/consoleInterceptor.ts index 21d90a2101..5e8f146ffc 100644 --- a/src/lib/consoleInterceptor.ts +++ b/src/lib/consoleInterceptor.ts @@ -12,9 +12,10 @@ import { appendFileSync, existsSync, mkdirSync } from "fs"; import { dirname, resolve } from "path"; +import { getAppLogFilePath, getAppLogToFile } from "./logEnv"; -const logToFile = process.env.LOG_TO_FILE !== "false"; -const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log"); +const logToFile = getAppLogToFile(); +const logFilePath = resolve(getAppLogFilePath()); declare global { var __omnirouteConsoleInterceptorInit: boolean | undefined; diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 399cf9c09c..14d9ee8448 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -173,7 +173,9 @@ const SCHEMA_SQL = ` combo_name TEXT, request_body TEXT, response_body TEXT, - error TEXT + error TEXT, + artifact_relpath TEXT, + has_pipeline_details INTEGER DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); @@ -376,6 +378,27 @@ function ensureUsageHistoryColumns(db: SqliteDatabase) { } } +function ensureCallLogsColumns(db: SqliteDatabase) { + try { + const columns = db.prepare("PRAGMA table_info(call_logs)").all() as Array<{ + name?: string; + }>; + const columnNames = new Set(columns.map((column) => String(column.name ?? ""))); + + if (!columnNames.has("artifact_relpath")) { + db.exec("ALTER TABLE call_logs ADD COLUMN artifact_relpath TEXT"); + console.log("[DB] Added call_logs.artifact_relpath column"); + } + if (!columnNames.has("has_pipeline_details")) { + db.exec("ALTER TABLE call_logs ADD COLUMN has_pipeline_details INTEGER DEFAULT 0"); + console.log("[DB] Added call_logs.has_pipeline_details column"); + } + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn("[DB] Failed to verify call_logs schema:", message); + } +} + export function getDbInstance(): SqliteDatabase { const existing = getDb(); if (existing) return existing; @@ -468,6 +491,7 @@ export function getDbInstance(): SqliteDatabase { db.exec(SCHEMA_SQL); ensureProviderConnectionsColumns(db); ensureUsageHistoryColumns(db); + ensureCallLogsColumns(db); // ── Versioned Migrations ── // Auto-seed 001 as applied (the inline SCHEMA_SQL already created these tables) diff --git a/src/lib/db/detailedLogs.ts b/src/lib/db/detailedLogs.ts index 9de0131e16..d879cf96b1 100644 --- a/src/lib/db/detailedLogs.ts +++ b/src/lib/db/detailedLogs.ts @@ -1,9 +1,9 @@ /** * Detailed Request Logs DB Layer (#378) * - * Saves full request/response bodies at each pipeline stage. - * Ring-buffer of 500 entries enforced by SQL trigger in migration 006. - * Only active when settings.detailed_logs_enabled = "1". + * Legacy compatibility layer for detailed request logs. + * New requests now store pipeline details inside unified call log artifacts. + * This module remains available for reading historical request_detail_logs rows. */ import { v4 as uuidv4 } from "uuid"; import { getDbInstance } from "./core"; @@ -37,7 +37,7 @@ export interface RequestDetailLog { export async function isDetailedLoggingEnabled(): Promise { try { const settings = await getSettings(); - const val = settings.detailed_logs_enabled; + const val = settings.call_log_pipeline_enabled; return val === true || val === "1" || val === "true"; } catch { return false; diff --git a/src/lib/db/migrations/001_initial_schema.sql b/src/lib/db/migrations/001_initial_schema.sql index 4f128b8ab7..49c33a09c3 100644 --- a/src/lib/db/migrations/001_initial_schema.sql +++ b/src/lib/db/migrations/001_initial_schema.sql @@ -109,8 +109,8 @@ CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); CREATE TABLE IF NOT EXISTS call_logs ( - id TEXT PRIMARY KEY, - timestamp TEXT NOT NULL, + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, method TEXT, path TEXT, status INTEGER, @@ -124,11 +124,13 @@ CREATE TABLE IF NOT EXISTS call_logs ( source_format TEXT, target_format TEXT, api_key_id TEXT, - api_key_name TEXT, - combo_name TEXT, - request_body TEXT, - response_body TEXT, - error TEXT + api_key_name TEXT, + combo_name TEXT, + request_body TEXT, + response_body TEXT, + error TEXT, + artifact_relpath TEXT, + has_pipeline_details INTEGER DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); diff --git a/src/lib/db/migrations/013_unified_log_artifacts.sql b/src/lib/db/migrations/013_unified_log_artifacts.sql new file mode 100644 index 0000000000..9f98ba7220 --- /dev/null +++ b/src/lib/db/migrations/013_unified_log_artifacts.sql @@ -0,0 +1,15 @@ +-- 013_unified_log_artifacts.sql +-- Switch request logging to unified single-file artifacts and prefixed settings. + +INSERT OR REPLACE INTO key_value (namespace, key, value) +VALUES ( + 'settings', + 'call_log_pipeline_enabled', + COALESCE( + (SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'detailed_logs_enabled'), + 'false' + ) +); + +DELETE FROM key_value +WHERE namespace = 'settings' AND key IN ('detailed_logs_enabled', 'maxCallLogs', 'MAX_CALL_LOGS'); diff --git a/src/lib/logEnv.ts b/src/lib/logEnv.ts new file mode 100644 index 0000000000..381a329408 --- /dev/null +++ b/src/lib/logEnv.ts @@ -0,0 +1,61 @@ +import path from "path"; + +const DEFAULT_APP_LOG_RETENTION_DAYS = 7; +const DEFAULT_CALL_LOG_RETENTION_DAYS = 7; +const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024; +const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log"); + +function parsePositiveInt(value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +export function parseFileSize(raw: string | undefined): number { + if (!raw) return DEFAULT_APP_LOG_MAX_SIZE; + const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i); + if (!match) return DEFAULT_APP_LOG_MAX_SIZE; + const num = parseInt(match[1], 10); + const unit = (match[2] || "").toLowerCase(); + switch (unit) { + case "k": + case "kb": + return num * 1024; + case "m": + case "mb": + return num * 1024 * 1024; + case "g": + case "gb": + return num * 1024 * 1024 * 1024; + default: + return num; + } +} + +export function getAppLogToFile(): boolean { + return process.env.APP_LOG_TO_FILE !== "false"; +} + +export function getAppLogFilePath(): string { + return process.env.APP_LOG_FILE_PATH || DEFAULT_APP_LOG_PATH; +} + +export function getAppLogMaxFileSize(): number { + return parseFileSize(process.env.APP_LOG_MAX_FILE_SIZE); +} + +export function getAppLogRetentionDays(): number { + return parsePositiveInt(process.env.APP_LOG_RETENTION_DAYS, DEFAULT_APP_LOG_RETENTION_DAYS); +} + +export function getCallLogRetentionDays(): number { + return parsePositiveInt(process.env.CALL_LOG_RETENTION_DAYS, DEFAULT_CALL_LOG_RETENTION_DAYS); +} + +export function getAppLogLevel(defaultLevel: string): string { + return process.env.APP_LOG_LEVEL || defaultLevel; +} + +export function getAppLogFormat(defaultFormat: string): string { + return process.env.APP_LOG_FORMAT || defaultFormat; +} diff --git a/src/lib/logRotation.ts b/src/lib/logRotation.ts index 80035199de..c4644135ef 100644 --- a/src/lib/logRotation.ts +++ b/src/lib/logRotation.ts @@ -7,48 +7,26 @@ * - Creating the log directory on startup * * Configuration via env vars: - * - LOG_TO_FILE: enable file logging (default: true) - * - LOG_FILE_PATH: path to log file (default: logs/application/app.log) - * - LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB) - * - LOG_RETENTION_DAYS: days to keep old logs (default: 7) + * - APP_LOG_TO_FILE: enable file logging (default: true) + * - APP_LOG_FILE_PATH: path to log file (default: logs/application/app.log) + * - APP_LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB) + * - APP_LOG_RETENTION_DAYS: days to keep old logs (default: 7) */ import { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync } from "fs"; import { dirname, join, basename, extname } from "path"; - -const DEFAULT_LOG_PATH = "logs/application/app.log"; -const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50MB -const DEFAULT_RETENTION_DAYS = 7; - -function parseFileSize(raw: string | undefined): number { - if (!raw) return DEFAULT_MAX_SIZE; - const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i); - if (!match) return DEFAULT_MAX_SIZE; - const num = parseInt(match[1], 10); - const unit = (match[2] || "").toLowerCase(); - switch (unit) { - case "k": - case "kb": - return num * 1024; - case "m": - case "mb": - return num * 1024 * 1024; - case "g": - case "gb": - return num * 1024 * 1024 * 1024; - default: - return num; - } -} +import { + getAppLogFilePath, + getAppLogMaxFileSize, + getAppLogRetentionDays, + getAppLogToFile, +} from "./logEnv"; export function getLogConfig() { - const logToFile = process.env.LOG_TO_FILE !== "false"; - const logFilePath = process.env.LOG_FILE_PATH || join(process.cwd(), DEFAULT_LOG_PATH); - const maxFileSize = parseFileSize(process.env.LOG_MAX_FILE_SIZE); - const retentionDays = parseInt( - process.env.LOG_RETENTION_DAYS || String(DEFAULT_RETENTION_DAYS), - 10 - ); + const logToFile = getAppLogToFile(); + const logFilePath = getAppLogFilePath() || join(process.cwd(), "logs/application/app.log"); + const maxFileSize = getAppLogMaxFileSize(); + const retentionDays = getAppLogRetentionDays(); return { logToFile, logFilePath, maxFileSize, retentionDays }; } diff --git a/src/lib/proxyLogger.ts b/src/lib/proxyLogger.ts index ff8f8caf3b..d3fb2140d4 100644 --- a/src/lib/proxyLogger.ts +++ b/src/lib/proxyLogger.ts @@ -11,7 +11,7 @@ import { getDbInstance, isCloud, isBuildPhase } from "./db/core"; const shouldPersistToDisk = !isCloud && !isBuildPhase; -const MAX_ENTRIES = parseInt(process.env.PROXY_LOG_MAX_ENTRIES || "200", 10); +const MAX_IN_MEMORY_ENTRIES = 200; interface ProxyInfo { type: string; @@ -56,7 +56,7 @@ function loadFromDb() { const db = getDbInstance(); const rows = db .prepare("SELECT * FROM proxy_logs ORDER BY timestamp DESC LIMIT ?") - .all(MAX_ENTRIES) as any[]; + .all(MAX_IN_MEMORY_ENTRIES) as any[]; for (const row of rows) { proxyLogs.push({ @@ -113,8 +113,8 @@ export function logProxyEvent(entry: Partial) { // 1. In-memory ring buffer (newest first) proxyLogs.unshift(log); - if (proxyLogs.length > MAX_ENTRIES) { - proxyLogs.length = MAX_ENTRIES; + if (proxyLogs.length > MAX_IN_MEMORY_ENTRIES) { + proxyLogs.length = MAX_IN_MEMORY_ENTRIES; } // 2. Persist to SQLite @@ -147,16 +147,6 @@ export function logProxyEvent(entry: Partial) { account: log.account, tlsFingerprint: log.tlsFingerprint ? 1 : 0, }); - - // Trim old entries - const count = (db.prepare("SELECT COUNT(*) as cnt FROM proxy_logs").get() as any)?.cnt || 0; - if (count > MAX_ENTRIES) { - db.prepare( - `DELETE FROM proxy_logs WHERE id IN ( - SELECT id FROM proxy_logs ORDER BY timestamp ASC LIMIT ? - )` - ).run(count - MAX_ENTRIES); - } } catch (err: any) { console.warn("[proxyLogger] Failed to persist:", err.message); } diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index 7b1d6aa870..cca04ee932 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -2,24 +2,57 @@ * Call Logs — extracted from usageDb.js (T-15) * * Structured call log management: save, query, rotate, and - * full-payload disk storage for the Logger UI. + * unified single-artifact disk storage for the Logger UI. * * @module lib/usage/callLogs */ -import path from "path"; import fs from "fs"; +import path from "path"; +import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { getDbInstance } from "../db/core"; -import { getSettings } from "../db/settings"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; import { shouldPersistToDisk, CALL_LOGS_DIR } from "./migrations"; import { getLoggedInputTokens, getLoggedOutputTokens } from "./tokenAccounting"; import { isNoLog } from "../compliance"; import { sanitizePII } from "../piiSanitizer"; -import { protectPayloadForLog, parseStoredPayload } from "../logPayloads"; +import { + protectPayloadForLog, + parseStoredPayload, + serializePayloadForStorage, +} from "../logPayloads"; +import { getCallLogRetentionDays } from "../logEnv"; type JsonRecord = Record; +type CallLogArtifact = { + schemaVersion: 2; + summary: { + id: string; + timestamp: string; + method: string; + path: string; + status: number; + model: string; + requestedModel: string | null; + provider: string; + account: string; + connectionId: string | null; + duration: number; + tokens: { in: number; out: number }; + requestType: string | null; + sourceFormat: string | null; + targetFormat: string | null; + apiKeyId: string | null; + apiKeyName: string | null; + comboName: string | null; + }; + requestBody: unknown; + responseBody: unknown; + error: unknown; + pipeline?: RequestPipelinePayloads; +}; + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } @@ -34,7 +67,7 @@ function toNumber(value: unknown): number { } function toStringOrNull(value: unknown): string | null { - return typeof value === "string" ? value : null; + return typeof value === "string" && value.trim().length > 0 ? value : null; } function hasTruncatedFlag(value: unknown): boolean { @@ -42,74 +75,224 @@ function hasTruncatedFlag(value: unknown): boolean { return (value as Record)._truncated === true; } -const DEFAULT_MAX_CALL_LOGS = 10000; -const CALL_LOGS_MAX_CACHE_TTL_MS = 30_000; - -const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "7", 10); -const CALL_LOG_PAYLOAD_MODE = (() => { - const value = (process.env.CALL_LOG_PAYLOAD_MODE || "full").toLowerCase(); - return value === "full" || value === "metadata" || value === "none" ? value : "full"; -})(); -const shouldLogPayloadInDb = CALL_LOG_PAYLOAD_MODE !== "none"; -const shouldLogPayloadOnDisk = CALL_LOG_PAYLOAD_MODE === "full"; - -let callLogsMaxCache = { - value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS, - expiresAt: 0, -}; - -function resolveCallLogsMaxValue(value: unknown): number | null { - if (typeof value === "number" && Number.isInteger(value) && value > 0) return value; - if (typeof value === "string" && value.trim().length > 0) { - const parsed = Number.parseInt(value, 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +function sanitizeErrorForLog(error: unknown): unknown { + if (error === null || error === undefined) return null; + if (typeof error === "string") return sanitizePII(error).text; + if (error instanceof Error) { + return { + message: sanitizePII(error.message).text, + stack: sanitizePII(error.stack || "").text || undefined, + name: error.name, + }; } - return null; + return protectPayloadForLog(error); } -async function getMaxCallLogs(): Promise { - const now = Date.now(); - if (callLogsMaxCache.expiresAt > now) { - return callLogsMaxCache.value; - } - - let value = resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS; - +function toStoredErrorString(error: unknown): string | null { + const sanitized = sanitizeErrorForLog(error); + if (sanitized === null || sanitized === undefined) return null; + if (typeof sanitized === "string") return sanitized; try { - const { getSettings } = await import("@/lib/localDb"); - const settings = await getSettings(); - const configured = - resolveCallLogsMaxValue(settings.maxCallLogs) ?? - resolveCallLogsMaxValue(settings.MAX_CALL_LOGS); - if (configured !== null) { - value = configured; - } + return JSON.stringify(sanitized); } catch { - // Fall back to env/default cap when settings are unavailable. + return String(sanitized); + } +} + +function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | null { + if (!payloads || typeof payloads !== "object") return null; + + const protectedPayloads: RequestPipelinePayloads = {}; + for (const [key, value] of Object.entries(payloads as JsonRecord)) { + if (value === null || value === undefined) { + continue; + } + + if (key === "streamChunks" && value && typeof value === "object") { + const chunks = value as Record; + const compacted = Object.fromEntries( + Object.entries(chunks).filter( + ([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0 + ) + ); + if (Object.keys(compacted).length > 0) { + protectedPayloads.streamChunks = protectPayloadForLog( + compacted + ) as RequestPipelinePayloads["streamChunks"]; + } + continue; + } + + protectedPayloads[key as keyof RequestPipelinePayloads] = protectPayloadForLog(value) as never; } - callLogsMaxCache = { - value, - expiresAt: now + CALL_LOGS_MAX_CACHE_TTL_MS, - }; - return value; + return Object.keys(protectedPayloads).length > 0 ? protectedPayloads : null; } -export function invalidateCallLogsMaxCache(): void { - callLogsMaxCache = { - value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS, - expiresAt: 0, - }; -} let logIdCounter = 0; function generateLogId() { logIdCounter++; return `${Date.now()}-${logIdCounter}`; } -/** - * Save a structured call log entry. - */ +async function resolveAccountName(connectionId: string | null | undefined) { + let account = connectionId ? connectionId.slice(0, 8) : "-"; + + if (!connectionId) { + return account; + } + + try { + const { getProviderConnections } = await import("@/lib/localDb"); + const connections = await getProviderConnections(); + const conn = connections.find((item) => item.id === connectionId); + if (conn) { + account = conn.name || conn.email || account; + } + } catch { + // Best-effort lookup only. + } + + return account; +} + +function buildArtifactRelativePath(timestamp: string, id: string) { + const parsed = new Date(timestamp); + const safeTimestamp = ( + Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString() + ).replace(/[:]/g, "-"); + const dateFolder = safeTimestamp.slice(0, 10); + return path.posix.join(dateFolder, `${safeTimestamp}_${id}.json`); +} + +function buildArtifact( + logEntry: { + id: string; + timestamp: string; + method: string; + path: string; + status: number; + model: string; + requestedModel: string | null; + provider: string; + account: string; + connectionId: string | null; + duration: number; + tokensIn: number; + tokensOut: number; + requestType: string | null; + sourceFormat: string | null; + targetFormat: string | null; + apiKeyId: string | null; + apiKeyName: string | null; + comboName: string | null; + }, + requestBody: unknown, + responseBody: unknown, + error: unknown, + pipelinePayloads: RequestPipelinePayloads | null +): CallLogArtifact { + return { + schemaVersion: 2, + summary: { + id: logEntry.id, + timestamp: logEntry.timestamp, + method: logEntry.method, + path: logEntry.path, + status: logEntry.status, + model: logEntry.model, + requestedModel: logEntry.requestedModel, + provider: logEntry.provider, + account: logEntry.account, + connectionId: logEntry.connectionId, + duration: logEntry.duration, + tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut }, + requestType: logEntry.requestType, + sourceFormat: logEntry.sourceFormat, + targetFormat: logEntry.targetFormat, + apiKeyId: logEntry.apiKeyId, + apiKeyName: logEntry.apiKeyName, + comboName: logEntry.comboName, + }, + requestBody: requestBody ?? null, + responseBody: responseBody ?? null, + error: error ?? null, + ...(pipelinePayloads ? { pipeline: pipelinePayloads } : {}), + }; +} + +function writeCallArtifact(artifact: CallLogArtifact): string | null { + if (!CALL_LOGS_DIR) return null; + + const relPath = buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id); + const absPath = path.join(CALL_LOGS_DIR, relPath); + + try { + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2)); + return relPath; + } catch (error) { + console.error("[callLogs] Failed to write request artifact:", (error as Error).message); + return null; + } +} + +function readArtifactFromDisk(relativePath: string | null) { + if (!CALL_LOGS_DIR || !relativePath) return null; + + try { + const absPath = path.join(CALL_LOGS_DIR, relativePath); + if (!fs.existsSync(absPath)) return null; + return JSON.parse(fs.readFileSync(absPath, "utf8")) as CallLogArtifact; + } catch (error) { + console.error("[callLogs] Failed to read request artifact:", (error as Error).message); + return null; + } +} + +function readLegacyLogFromDisk(entry: { + timestamp: string | null; + model: string | null; + status: number; +}) { + if (!CALL_LOGS_DIR || !entry.timestamp) return null; + + try { + const date = new Date(entry.timestamp); + if (Number.isNaN(date.getTime())) return null; + + const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart( + 2, + "0" + )}-${String(date.getDate()).padStart(2, "0")}`; + const dir = path.join(CALL_LOGS_DIR, dateFolder); + if (!fs.existsSync(dir)) return null; + + const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart( + 2, + "0" + )}${String(date.getSeconds()).padStart(2, "0")}`; + const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-"); + const expectedName = `${time}_${safeModel}_${entry.status}.json`; + + const exactPath = path.join(dir, expectedName); + if (fs.existsSync(exactPath)) { + return JSON.parse(fs.readFileSync(exactPath, "utf8")); + } + + const files = fs + .readdirSync(dir) + .filter((file) => file.startsWith(time) && file.endsWith(`_${entry.status}.json`)); + if (files.length > 0) { + return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8")); + } + } catch (error) { + console.error("[callLogs] Failed to read legacy disk log:", (error as Error).message); + } + + return null; +} + export async function saveCallLog(entry: any) { if (!shouldPersistToDisk) return; @@ -117,44 +300,23 @@ export async function saveCallLog(entry: any) { const apiKeyId = entry.apiKeyId || null; const noLogEnabled = Boolean(entry.noLog) || (apiKeyId ? isNoLog(apiKeyId) : false); - const protectedRequestBody = - noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.requestBody); - const protectedResponseBody = - noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.responseBody); + const protectedRequestBody = noLogEnabled ? null : protectPayloadForLog(entry.requestBody); + const protectedResponseBody = noLogEnabled ? null : protectPayloadForLog(entry.responseBody); + const protectedPipelinePayloads = noLogEnabled + ? null + : protectPipelinePayloads(entry.pipelinePayloads ?? entry.pipeline ?? null); + const protectedError = sanitizeErrorForLog(entry.error); - // Resolve account name - let account = entry.connectionId ? entry.connectionId.slice(0, 8) : "-"; - try { - const { getProviderConnections } = await import("@/lib/localDb"); - const connections = await getProviderConnections(); - const conn = connections.find((c) => c.id === entry.connectionId); - if (conn) account = conn.name || conn.email || account; - } catch {} - - // Truncate large payloads for DB storage (keep under 8KB each) - const truncatePayload = (obj: any) => { - if (!obj) return null; - const str = JSON.stringify(obj); - if (str.length <= 8192) return str; - try { - return JSON.stringify({ - _truncated: true, - _originalSize: str.length, - _preview: str.slice(0, 8192) + "...", - }); - } catch { - return JSON.stringify({ _truncated: true }); - } - }; + const account = await resolveAccountName(entry.connectionId || null); const logEntry = { id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(), - timestamp: new Date().toISOString(), + timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(), method: entry.method || "POST", path: entry.path || "/v1/chat/completions", status: entry.status || 0, model: entry.model || "-", - requestedModel: entry.requestedModel || null, // T01: model the client asked for + requestedModel: entry.requestedModel || null, provider: entry.provider || "-", account, connectionId: entry.connectionId || null, @@ -167,119 +329,82 @@ export async function saveCallLog(entry: any) { apiKeyId, apiKeyName: entry.apiKeyName || null, comboName: entry.comboName || null, - requestBody: truncatePayload(protectedRequestBody), - responseBody: truncatePayload(protectedResponseBody), - error: typeof entry.error === "string" ? sanitizePII(entry.error).text : entry.error || null, + requestBody: serializePayloadForStorage(protectedRequestBody, 8192), + responseBody: serializePayloadForStorage(protectedResponseBody, 8192), + error: toStoredErrorString(protectedError), }; - // 1. Insert into SQLite const db = getDbInstance(); db.prepare( ` - INSERT INTO call_logs (id, timestamp, method, path, status, model, requested_model, provider, - account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error) - VALUES (@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, - @account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, @targetFormat, - @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error) + INSERT INTO call_logs ( + id, timestamp, method, path, status, model, requested_model, provider, + account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, + target_format, api_key_id, api_key_name, combo_name, request_body, response_body, error, + artifact_relpath, has_pipeline_details + ) + VALUES ( + @id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, + @account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, + @targetFormat, @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, + NULL, 0 + ) ` ).run(logEntry); - // 2. Trim old entries beyond max - const maxLogs = await getMaxCallLogs(); - const countRow = asRecord(db.prepare("SELECT COUNT(*) as cnt FROM call_logs").get()); - const count = toNumber(countRow.cnt); - if (count > maxLogs) { - db.prepare( - ` - DELETE FROM call_logs WHERE id IN ( - SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ? - ) - ` - ).run(count - maxLogs); - } - - // 3. Write full payload to disk file (untruncated) - // Disabled when no-log is active or payload mode is metadata/none. - if ( - shouldLogPayloadOnDisk && - !noLogEnabled && - (protectedRequestBody !== null || protectedResponseBody !== null) - ) { - writeCallLogToDisk( - { ...logEntry, tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut } }, + if (!noLogEnabled) { + const artifact = buildArtifact( + logEntry, protectedRequestBody, - protectedResponseBody + protectedResponseBody, + protectedError, + protectedPipelinePayloads ); + const artifactRelPath = writeCallArtifact(artifact); + + if (artifactRelPath) { + db.prepare( + ` + UPDATE call_logs + SET artifact_relpath = ?, has_pipeline_details = ? + WHERE id = ? + ` + ).run(artifactRelPath, protectedPipelinePayloads ? 1 : 0, logEntry.id); + } } - } catch (error: any) { - console.error("[callLogs] Failed to save call log:", error.message); + } catch (error) { + console.error("[callLogs] Failed to save call log:", (error as Error).message); } } -/** - * Write call log as JSON file to disk (full payloads, not truncated). - */ -function writeCallLogToDisk(logEntry: any, requestBody: any, responseBody: any) { - if (!CALL_LOGS_DIR) return; - - try { - const now = new Date(); - const dateFolder = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}`; - const dir = path.join(CALL_LOGS_DIR, dateFolder); - - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); - - const safeModel = (logEntry.model || "unknown").replace(/[/:]/g, "-"); - const time = `${String(now.getHours()).padStart(2, "0")}${String(now.getMinutes()).padStart(2, "0")}${String(now.getSeconds()).padStart(2, "0")}`; - const filename = `${time}_${safeModel}_${logEntry.status}.json`; - - const fullEntry = { - ...logEntry, - requestBody: requestBody || null, - responseBody: responseBody || null, - }; - - fs.writeFileSync(path.join(dir, filename), JSON.stringify(fullEntry, null, 2)); - } catch (err: any) { - console.error("[callLogs] Failed to write disk log:", err.message); - } -} - -/** - * Rotate old call log directories (keep last 7 days). - */ export function rotateCallLogs() { if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return; try { const entries = fs.readdirSync(CALL_LOGS_DIR); const now = Date.now(); - const retentionMs = LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000; + const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000; for (const entry of entries) { const entryPath = path.join(CALL_LOGS_DIR, entry); const stat = fs.statSync(entryPath); if (stat.isDirectory() && now - stat.mtimeMs > retentionMs) { fs.rmSync(entryPath, { recursive: true, force: true }); - console.log(`[callLogs] Rotated old logs: ${entry}`); } } - } catch (err: any) { - console.error("[callLogs] Failed to rotate logs:", err.message); + } catch (error) { + console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message); } } -// Run rotation on startup if (shouldPersistToDisk) { try { rotateCallLogs(); - } catch {} + } catch { + // Best-effort startup cleanup. + } } -/** - * Get call logs with optional filtering. - */ export async function getCallLogs(filter: any = {}) { const db = getDbInstance(); let sql = "SELECT * FROM call_logs"; @@ -292,8 +417,8 @@ export async function getCallLogs(filter: any = {}) { } else if (filter.status === "ok") { conditions.push("status >= 200 AND status < 300"); } else { - const statusCode = parseInt(filter.status); - if (!isNaN(statusCode)) { + const statusCode = parseInt(filter.status, 10); + if (!Number.isNaN(statusCode)) { conditions.push("status = @statusCode"); params.statusCode = statusCode; } @@ -347,7 +472,7 @@ export async function getCallLogs(filter: any = {}) { path: toStringOrNull(l.path), status: toNumber(l.status), model: toStringOrNull(l.model), - requestedModel: toStringOrNull(l.requested_model), // T01: original model from client + requestedModel: toStringOrNull(l.requested_model), provider: toStringOrNull(l.provider), account: toStringOrNull(l.account), duration: toNumber(l.duration), @@ -360,19 +485,30 @@ export async function getCallLogs(filter: any = {}) { apiKeyName: toStringOrNull(l.api_key_name), hasRequestBody: typeof l.request_body === "string" && l.request_body.length > 0, hasResponseBody: typeof l.response_body === "string" && l.response_body.length > 0, + hasPipelineDetails: toNumber(l.has_pipeline_details) === 1, }; }); } -/** - * Get a single call log by ID (with full payloads from disk when available). - */ +function buildLegacyPipelinePayloads(id: string) { + const detailed = getRequestDetailLogByCallLogId(id); + if (!detailed) return null; + + return { + clientRequest: detailed.client_request ?? null, + providerRequest: detailed.translated_request ?? null, + providerResponse: detailed.provider_response ?? null, + clientResponse: detailed.client_response ?? null, + }; +} + export async function getCallLogById(id: string) { const db = getDbInstance(); const row = db.prepare("SELECT * FROM call_logs WHERE id = ?").get(id); if (!row) return null; - const entryRow = asRecord(row); + const entryRow = asRecord(row); + const artifactRelPath = toStringOrNull(entryRow.artifact_relpath); const entry = { id: toStringOrNull(entryRow.id), timestamp: toStringOrNull(entryRow.timestamp), @@ -394,72 +530,47 @@ export async function getCallLogById(id: string) { requestBody: parseStoredPayload(entryRow.request_body), responseBody: parseStoredPayload(entryRow.response_body), error: toStringOrNull(entryRow.error), + artifactRelPath, + hasPipelineDetails: toNumber(entryRow.has_pipeline_details) === 1, }; - // If payloads were truncated, try to read full version from disk - const needsDisk = hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody); - if (needsDisk && CALL_LOGS_DIR) { - try { - const diskEntry = readFullLogFromDisk(entry); - if (diskEntry) { - return { - ...entry, - requestBody: diskEntry.requestBody ?? entry.requestBody, - responseBody: diskEntry.responseBody ?? entry.responseBody, - }; - } - } catch (err: any) { - console.error("[callLogs] Failed to read full log from disk:", err.message); + const artifact = readArtifactFromDisk(artifactRelPath); + if (artifact) { + return { + ...entry, + requestBody: artifact.requestBody ?? entry.requestBody, + responseBody: artifact.responseBody ?? entry.responseBody, + error: artifact.error ?? entry.error, + pipelinePayloads: artifact.pipeline ?? null, + hasPipelineDetails: Boolean(artifact.pipeline) || entry.hasPipelineDetails, + }; + } + + const needsLegacyDisk = + hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody) || !artifactRelPath; + if (needsLegacyDisk) { + const legacyEntry = readLegacyLogFromDisk(entry); + if (legacyEntry) { + const legacyPipeline = buildLegacyPipelinePayloads(id); + return { + ...entry, + requestBody: legacyEntry.requestBody ?? entry.requestBody, + responseBody: legacyEntry.responseBody ?? entry.responseBody, + error: legacyEntry.error ?? entry.error, + pipelinePayloads: legacyPipeline, + hasPipelineDetails: Boolean(legacyPipeline), + }; } } - const detailed = getRequestDetailLogByCallLogId(id); - if (!detailed) { - return entry; + const legacyPipeline = buildLegacyPipelinePayloads(id); + if (legacyPipeline) { + return { + ...entry, + pipelinePayloads: legacyPipeline, + hasPipelineDetails: true, + }; } - return { - ...entry, - pipelinePayloads: { - clientRequest: detailed.client_request ?? null, - providerRequest: detailed.translated_request ?? null, - providerResponse: detailed.provider_response ?? null, - clientResponse: detailed.client_response ?? null, - }, - }; -} - -/** - * Read the full (untruncated) log entry from disk. - */ -function readFullLogFromDisk(entry: any) { - if (!CALL_LOGS_DIR || !entry.timestamp) return null; - - try { - const date = new Date(entry.timestamp); - const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; - const dir = path.join(CALL_LOGS_DIR, dateFolder); - - if (!fs.existsSync(dir)) return null; - - const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart(2, "0")}${String(date.getSeconds()).padStart(2, "0")}`; - const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-"); - const expectedName = `${time}_${safeModel}_${entry.status}.json`; - - const exactPath = path.join(dir, expectedName); - if (fs.existsSync(exactPath)) { - return JSON.parse(fs.readFileSync(exactPath, "utf8")); - } - - const files = fs - .readdirSync(dir) - .filter((f) => f.startsWith(time) && f.endsWith(`_${entry.status}.json`)); - if (files.length > 0) { - return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8")); - } - } catch (err: any) { - console.error("[callLogs] Disk log read error:", err.message); - } - - return null; + return entry; } diff --git a/src/lib/usage/migrations.ts b/src/lib/usage/migrations.ts index 83e5fc9d62..2cd0275aef 100644 --- a/src/lib/usage/migrations.ts +++ b/src/lib/usage/migrations.ts @@ -1,42 +1,47 @@ /** * Usage Migrations — extracted from usageDb.js (T-15) * - * Handles legacy file migration (.data → data/) and JSON → SQLite migration. - * Runs automatically on module load when shouldPersistToDisk is true. + * Handles legacy file migration (.data → data/), JSON → SQLite migration, + * and one-time archival of legacy request log layouts into a zip backup. * * @module lib/usage/migrations */ -import path from "path"; import fs from "fs"; +import path from "path"; +import { ZipFile } from "yazl"; import { getDbInstance, isCloud, isBuildPhase, DATA_DIR } from "../db/core"; import { getLegacyDotDataDir, isSamePath } from "../dataPaths"; export const shouldPersistToDisk = !isCloud && !isBuildPhase; -// ──────────────── File Paths ──────────────── - const LEGACY_DATA_DIR = isCloud ? null : getLegacyDotDataDir(); -export const LOG_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt"); export const CALL_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "call_logs"); +export const LOG_ARCHIVES_DIR = isCloud ? null : path.join(DATA_DIR, "log_archives"); + +const LEGACY_LAYOUT_MARKER = + isCloud || !LOG_ARCHIVES_DIR ? null : path.join(LOG_ARCHIVES_DIR, "legacy-request-logs.json"); + +const CURRENT_REQUEST_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "logs"); +const CURRENT_REQUEST_SUMMARY_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt"); // Legacy paths const LEGACY_DB_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "usage.json"); -const LEGACY_LOG_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "log.txt"); const LEGACY_CALL_LOGS_DB_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs.json"); -const LEGACY_CALL_LOGS_DIR = - isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs"); - // Current-location JSON files (for migration into SQLite) const USAGE_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "usage.json"); const CALL_LOGS_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "call_logs.json"); -// ──────────────── Legacy File Migration ──────────────── +type ArchiveTarget = { + sourcePath: string; + archiveRoot: string; + deleteAfterArchive: boolean; +}; -function copyIfMissing(fromPath, toPath, label) { +function copyIfMissing(fromPath: string | null, toPath: string | null, label: string) { if (!fromPath || !toPath) return; if (!fs.existsSync(fromPath) || fs.existsSync(toPath)) return; @@ -48,27 +53,185 @@ function copyIfMissing(fromPath, toPath, label) { console.log(`[usageDb] Migrated ${label}: ${fromPath} -> ${toPath}`); } +function containsLegacyCallLogLayout(dirPath: string | null): boolean { + if (!dirPath || !fs.existsSync(dirPath)) return false; + + try { + const topLevelEntries = fs.readdirSync(dirPath); + for (const topLevelEntry of topLevelEntries) { + const topLevelPath = path.join(dirPath, topLevelEntry); + const stat = fs.statSync(topLevelPath); + if (stat.isFile() && /^\d{6}_.+_\d{3}\.json$/i.test(topLevelEntry)) { + return true; + } + if (!stat.isDirectory()) { + continue; + } + + const nestedEntries = fs.readdirSync(topLevelPath); + for (const nestedEntry of nestedEntries) { + if (/^\d{6}_.+_\d{3}\.json$/i.test(nestedEntry)) { + return true; + } + } + } + } catch { + return false; + } + + return false; +} + +function ensureArchiveDir() { + if (!LOG_ARCHIVES_DIR) return; + fs.mkdirSync(LOG_ARCHIVES_DIR, { recursive: true }); +} + +function listArchiveTargets(): ArchiveTarget[] { + const targets: ArchiveTarget[] = []; + + if (CURRENT_REQUEST_LOGS_DIR && fs.existsSync(CURRENT_REQUEST_LOGS_DIR)) { + targets.push({ + sourcePath: CURRENT_REQUEST_LOGS_DIR, + archiveRoot: "data/logs", + deleteAfterArchive: true, + }); + } + + if (CURRENT_REQUEST_SUMMARY_FILE && fs.existsSync(CURRENT_REQUEST_SUMMARY_FILE)) { + targets.push({ + sourcePath: CURRENT_REQUEST_SUMMARY_FILE, + archiveRoot: "data/log.txt", + deleteAfterArchive: true, + }); + } + + if (CALL_LOGS_DIR && containsLegacyCallLogLayout(CALL_LOGS_DIR)) { + targets.push({ + sourcePath: CALL_LOGS_DIR, + archiveRoot: "data/call_logs", + deleteAfterArchive: true, + }); + } + + return targets; +} + +function addPathToZip(zipFile: ZipFile, sourcePath: string, archivePath: string) { + const stat = fs.statSync(sourcePath); + if (stat.isDirectory()) { + const entries = fs.readdirSync(sourcePath); + if (entries.length === 0) { + zipFile.addEmptyDirectory(archivePath); + return; + } + + for (const entry of entries) { + addPathToZip(zipFile, path.join(sourcePath, entry), path.posix.join(archivePath, entry)); + } + return; + } + + zipFile.addFile(sourcePath, archivePath); +} + +function createLegacyArchive(targets: ArchiveTarget[]): Promise { + return new Promise((resolve, reject) => { + if (!LOG_ARCHIVES_DIR) { + reject(new Error("LOG_ARCHIVES_DIR is not configured")); + return; + } + + ensureArchiveDir(); + + const timestamp = new Date().toISOString().replace(/[:]/g, "-"); + const archiveFilename = `${timestamp}_legacy-request-logs.zip`; + const archivePath = path.join(LOG_ARCHIVES_DIR, archiveFilename); + const zipFile = new ZipFile(); + const output = fs.createWriteStream(archivePath); + + output.on("close", () => resolve(archiveFilename)); + output.on("error", (error) => { + fs.rmSync(archivePath, { force: true }); + reject(error); + }); + zipFile.outputStream.pipe(output); + + try { + for (const target of targets) { + addPathToZip(zipFile, target.sourcePath, target.archiveRoot); + } + zipFile.end(); + } catch (error) { + fs.rmSync(archivePath, { force: true }); + zipFile.end(); + reject(error); + } + }); +} + +function writeLegacyLayoutMarker(archiveFilename: string) { + if (!LEGACY_LAYOUT_MARKER) return; + ensureArchiveDir(); + fs.writeFileSync( + LEGACY_LAYOUT_MARKER, + JSON.stringify( + { + migratedAt: new Date().toISOString(), + archiveFilename, + }, + null, + 2 + ) + ); +} + +function deleteArchivedTargets(targets: ArchiveTarget[]) { + for (const target of targets) { + if (!target.deleteAfterArchive || !fs.existsSync(target.sourcePath)) { + continue; + } + + const stat = fs.statSync(target.sourcePath); + if (stat.isDirectory()) { + fs.rmSync(target.sourcePath, { recursive: true, force: true }); + } else { + fs.rmSync(target.sourcePath, { force: true }); + } + } +} + export function migrateLegacyUsageFiles() { if (!shouldPersistToDisk || !LEGACY_DATA_DIR) return; if (isSamePath(DATA_DIR, LEGACY_DATA_DIR)) return; try { copyIfMissing(LEGACY_DB_FILE, USAGE_JSON_FILE, "usage history"); - copyIfMissing(LEGACY_LOG_FILE, LOG_FILE, "request log"); copyIfMissing(LEGACY_CALL_LOGS_DB_FILE, CALL_LOGS_JSON_FILE, "call log index"); - copyIfMissing(LEGACY_CALL_LOGS_DIR, CALL_LOGS_DIR, "call log files"); } catch (error) { - console.error("[usageDb] Legacy migration failed:", error.message); + console.error("[usageDb] Legacy migration failed:", (error as Error).message); } } -// ──────────────── JSON → SQLite Migration ──────────────── +export async function archiveLegacyRequestLogs() { + if (!shouldPersistToDisk) return null; + if (LEGACY_LAYOUT_MARKER && fs.existsSync(LEGACY_LAYOUT_MARKER)) return null; + + const targets = listArchiveTargets(); + if (targets.length === 0) return null; + + const archiveFilename = await createLegacyArchive(targets); + deleteArchivedTargets(targets); + writeLegacyLayoutMarker(archiveFilename); + + console.log(`[usageDb] Archived legacy request logs to ${archiveFilename}`); + return archiveFilename; +} export function migrateUsageJsonToSqlite() { if (!shouldPersistToDisk) return; const db = getDbInstance(); - // 1. Migrate usage.json if (USAGE_JSON_FILE && fs.existsSync(USAGE_JSON_FILE)) { try { const raw = fs.readFileSync(USAGE_JSON_FILE, "utf-8"); @@ -118,13 +281,12 @@ export function migrateUsageJsonToSqlite() { console.log(`[usageDb] ✓ Migrated ${history.length} usage entries`); } - fs.renameSync(USAGE_JSON_FILE, USAGE_JSON_FILE + ".migrated"); - } catch (err) { - console.error("[usageDb] Failed to migrate usage.json:", err.message); + fs.renameSync(USAGE_JSON_FILE, `${USAGE_JSON_FILE}.migrated`); + } catch (error) { + console.error("[usageDb] Failed to migrate usage.json:", (error as Error).message); } } - // 2. Migrate call_logs.json if (CALL_LOGS_JSON_FILE && fs.existsSync(CALL_LOGS_JSON_FILE)) { try { const raw = fs.readFileSync(CALL_LOGS_JSON_FILE, "utf-8"); @@ -137,10 +299,12 @@ export function migrateUsageJsonToSqlite() { const insert = db.prepare(` INSERT OR IGNORE INTO call_logs (id, timestamp, method, path, status, model, provider, account, connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error) + api_key_id, api_key_name, combo_name, request_body, response_body, error, + artifact_relpath, has_pipeline_details) VALUES (@id, @timestamp, @method, @path, @status, @model, @provider, @account, @connectionId, @duration, @tokensIn, @tokensOut, @sourceFormat, @targetFormat, - @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error) + @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, + NULL, 0) `); const tx = db.transaction(() => { @@ -173,21 +337,25 @@ export function migrateUsageJsonToSqlite() { console.log(`[usageDb] ✓ Migrated ${logs.length} call log entries`); } - fs.renameSync(CALL_LOGS_JSON_FILE, CALL_LOGS_JSON_FILE + ".migrated"); - } catch (err) { - console.error("[usageDb] Failed to migrate call_logs.json:", err.message); + fs.renameSync(CALL_LOGS_JSON_FILE, `${CALL_LOGS_JSON_FILE}.migrated`); + } catch (error) { + console.error("[usageDb] Failed to migrate call_logs.json:", (error as Error).message); } } } -// ──────────────── Run on load ──────────────── - migrateLegacyUsageFiles(); if (shouldPersistToDisk) { + try { + await archiveLegacyRequestLogs(); + } catch (error) { + console.error("[usageDb] Failed to archive legacy request logs:", (error as Error).message); + } + try { migrateUsageJsonToSqlite(); } catch { - /* ok */ + // Best-effort startup migration. } } diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index d8e1630862..8a8ee52f3f 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -378,31 +378,18 @@ export async function getModelLatencyStats( return stats; } -// ──────────────── Request Log (log.txt) ──────────────── - -import fs from "fs"; -import { LOG_FILE } from "./migrations"; - -function formatLogDate(date = new Date()) { - const pad = (n: number) => String(n).padStart(2, "0"); - const d = pad(date.getDate()); - const m = pad(date.getMonth() + 1); - const y = date.getFullYear(); - const h = pad(date.getHours()); - const min = pad(date.getMinutes()); - const s = pad(date.getSeconds()); - return `${d}-${m}-${y} ${h}:${min}:${s}`; -} +// ──────────────── Request Log Compatibility Shim ──────────────── /** - * Append to log.txt. + * Legacy compatibility shim. + * Request summary lines are no longer written to data/log.txt. */ export async function appendRequestLog({ - model, - provider, - connectionId, - tokens, - status, + model: _model, + provider: _provider, + connectionId: _connectionId, + tokens: _tokens, + status: _status, }: { model?: string; provider?: string; @@ -410,55 +397,39 @@ export async function appendRequestLog({ tokens?: any; status?: string | number; }) { - if (!shouldPersistToDisk) return; - - try { - const timestamp = formatLogDate(); - const p = provider?.toUpperCase() || "-"; - const m = model || "-"; - - let account = connectionId ? connectionId.slice(0, 8) : "-"; - try { - const { getProviderConnections } = await import("@/lib/localDb"); - const connections = await getProviderConnections(); - const connRaw = connections.find((c) => asRecord(c).id === connectionId); - if (connRaw) { - const conn = asRecord(connRaw); - account = toStringOrNull(conn.name) || toStringOrNull(conn.email) || account; - } - } catch {} - - const sent = tokens ? getLoggedInputTokens(tokens) : "-"; - const received = tokens ? getLoggedOutputTokens(tokens) : "-"; - - const line = `${timestamp} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${status}\n`; - fs.appendFileSync(LOG_FILE, line); - - const content = fs.readFileSync(LOG_FILE, "utf-8"); - const lines = content.trim().split("\n"); - if (lines.length > 200) { - fs.writeFileSync(LOG_FILE, lines.slice(-200).join("\n") + "\n"); - } - } catch (error: any) { - console.error("Failed to append to log.txt:", error.message); - } + // Deprecated: request summaries now come from SQLite call_logs. } /** - * Get last N lines of log.txt. + * Return recent request summaries generated from SQLite call_logs rows. */ export async function getRecentLogs(limit = 200) { - if (!shouldPersistToDisk) return []; - if (!fs || typeof fs.existsSync !== "function") return []; - if (!LOG_FILE) return []; - if (!fs.existsSync(LOG_FILE)) return []; - try { - const content = fs.readFileSync(LOG_FILE, "utf-8"); - const lines = content.trim().split("\n"); - return lines.slice(-limit).reverse(); + const db = getDbInstance(); + const rows = db + .prepare( + ` + SELECT timestamp, model, provider, account, tokens_in, tokens_out, status + FROM call_logs + ORDER BY timestamp DESC + LIMIT ? + ` + ) + .all(limit) as Array>; + + return rows.map((row) => { + const timestamp = + typeof row.timestamp === "string" ? row.timestamp : new Date().toISOString(); + const provider = typeof row.provider === "string" ? row.provider.toUpperCase() : "-"; + const model = typeof row.model === "string" ? row.model : "-"; + const account = typeof row.account === "string" ? row.account : "-"; + const tokensIn = toNumber(row.tokens_in); + const tokensOut = toNumber(row.tokens_out); + const status = typeof row.status === "number" ? row.status : String(row.status || "-"); + return `${timestamp} | ${model} | ${provider} | ${account} | ${tokensIn} | ${tokensOut} | ${status}`; + }); } catch (error: any) { - console.error("[usageDb] Failed to read log.txt:", error.message); + console.error("[usageDb] Failed to read recent call logs:", error.message); return []; } } diff --git a/src/server-init.ts b/src/server-init.ts index f254547c98..bf83efd0eb 100644 --- a/src/server-init.ts +++ b/src/server-init.ts @@ -5,6 +5,9 @@ import { initAuditLog, cleanupExpiredLogs, logAuditEvent } from "./lib/complianc import { initConsoleInterceptor } from "./lib/consoleInterceptor"; async function startServer() { + // Trigger request-log layout migration during startup, before serving requests. + await import("./lib/usage/migrations"); + // Console interceptor: capture all console output to log file (must be first) initConsoleInterceptor(); @@ -22,7 +25,14 @@ async function startServer() { // Compliance: One-time cleanup of expired logs try { const cleanup = cleanupExpiredLogs(); - if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) { + if ( + cleanup.deletedUsage || + cleanup.deletedCallLogs || + cleanup.deletedProxyLogs || + cleanup.deletedRequestDetailLogs || + cleanup.deletedAuditLogs || + cleanup.deletedMcpAuditLogs + ) { console.log("[COMPLIANCE] Expired log cleanup:", cleanup); } } catch (err) { diff --git a/src/shared/components/ConsoleLogViewer.tsx b/src/shared/components/ConsoleLogViewer.tsx index eb53c00300..97680ffc0f 100644 --- a/src/shared/components/ConsoleLogViewer.tsx +++ b/src/shared/components/ConsoleLogViewer.tsx @@ -206,7 +206,7 @@ export default function ConsoleLogViewer() { error {error} - — Make sure the application is writing logs to file (LOG_TO_FILE=true) + — Make sure the application is writing logs to file (APP_LOG_TO_FILE=true)
)} @@ -237,7 +237,7 @@ export default function ConsoleLogViewer() {

{t("noLogEntries")}

- Ensure LOG_TO_FILE=true is set in your .env file + Ensure APP_LOG_TO_FILE=true is set in your .env file

) : ( diff --git a/src/shared/components/RequestLoggerDetail.tsx b/src/shared/components/RequestLoggerDetail.tsx index e4cf1fbfa0..19a45caad3 100644 --- a/src/shared/components/RequestLoggerDetail.tsx +++ b/src/shared/components/RequestLoggerDetail.tsx @@ -92,27 +92,21 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC const pipelinePayloads = detail?.pipelinePayloads || null; const payloadSections = pipelinePayloads ? [ - { - key: "client-request", - title: "Client Request", - json: toPrettyJson(pipelinePayloads.clientRequest), - }, - { - key: "provider-request", - title: "Provider Request", - json: toPrettyJson(pipelinePayloads.providerRequest), - }, - { - key: "provider-response", - title: "Provider Response", - json: toPrettyJson(pipelinePayloads.providerResponse), - }, - { - key: "client-response", - title: "Client Response", - json: toPrettyJson(pipelinePayloads.clientResponse), - }, - ].filter((section) => section.json) + ["clientRawRequest", "Client Raw Request"], + ["clientRequest", "Client Request"], + ["sourceRequest", "Source Request"], + ["openaiRequest", "OpenAI Request"], + ["providerRequest", "Provider Request"], + ["providerResponse", "Provider Response"], + ["clientResponse", "Client Response"], + ["error", "Pipeline Error"], + ] + .map(([key, title]) => ({ + key, + title, + json: toPrettyJson(pipelinePayloads[key]), + })) + .filter((section) => section.json) : []; const requestJson = detail?.requestBody ? toPrettyJson(detail.requestBody) : null; const responseJson = detail?.responseBody ? toPrettyJson(detail.responseBody) : null; diff --git a/src/shared/components/RequestLoggerV2.tsx b/src/shared/components/RequestLoggerV2.tsx index e8a1ca366d..5b5fba9fc1 100644 --- a/src/shared/components/RequestLoggerV2.tsx +++ b/src/shared/components/RequestLoggerV2.tsx @@ -315,7 +315,7 @@ export default function RequestLoggerV2() { ? "bg-amber-500/10 border-amber-500/30 text-amber-700 dark:text-amber-300" : "bg-bg-subtle border-border text-text-muted" }`} - title="Capture four-stage pipeline payloads for new requests" + title="Capture per-request pipeline payloads inside the unified call log artifact" >
- Call logs are also saved as JSON files to {`{DATA_DIR}/call_logs/`} with 7-day - rotation. + Each request is also saved as a single JSON artifact in{" "} + {`{DATA_DIR}/call_logs/`}.
{/* Detail Modal */} diff --git a/src/shared/schemas/validation.ts b/src/shared/schemas/validation.ts index bcf0bda487..e73a657a38 100644 --- a/src/shared/schemas/validation.ts +++ b/src/shared/schemas/validation.ts @@ -60,8 +60,6 @@ export const settingsSchema = z requireLogin: z.boolean().optional(), password: z.string().min(6, "Password must be at least 6 characters").optional(), defaultModel: z.string().optional(), - enableRequestLogs: z.boolean().optional(), - maxLogRetention: z.number().int().min(1).max(365).optional(), rateLimitEnabled: z.boolean().optional(), rateLimitPerMinute: z.number().int().min(0).optional(), }) diff --git a/src/shared/utils/logger.ts b/src/shared/utils/logger.ts index a5f8f414d9..edb2874200 100644 --- a/src/shared/utils/logger.ts +++ b/src/shared/utils/logger.ts @@ -10,17 +10,18 @@ * In development, output is pretty-printed via pino-pretty. * In production, output is structured JSON for log aggregation. * - * When LOG_TO_FILE is enabled (default: true), logs are also written - * as JSON lines to the file specified by LOG_FILE_PATH. + * When APP_LOG_TO_FILE is enabled (default: true), logs are also written + * as JSON lines to the file specified by APP_LOG_FILE_PATH. */ import pino from "pino"; import { resolve } from "path"; import { getLogConfig, initLogRotation } from "@/lib/logRotation"; +import { getAppLogLevel } from "@/lib/logEnv"; const isDev = process.env.NODE_ENV !== "production"; const baseConfig: pino.LoggerOptions = { - level: process.env.LOG_LEVEL || (isDev ? "debug" : "info"), + level: getAppLogLevel(isDev ? "debug" : "info"), base: { service: "omniroute" }, timestamp: pino.stdTimeFunctions.isoTime, formatters: { diff --git a/src/shared/utils/structuredLogger.ts b/src/shared/utils/structuredLogger.ts index 03b4e98e54..c6d2bd03ba 100644 --- a/src/shared/utils/structuredLogger.ts +++ b/src/shared/utils/structuredLogger.ts @@ -5,7 +5,7 @@ * and human-readable output for development. Replaces scattered console.log * calls with consistent, parseable log entries. * - * When LOG_TO_FILE is enabled, log entries are also appended as JSON lines + * When APP_LOG_TO_FILE is enabled, log entries are also appended as JSON lines * to the application log file for the Console Log Viewer. * * @module shared/utils/structuredLogger @@ -14,6 +14,7 @@ import { getCorrelationId } from "../middleware/correlationId"; import { appendFileSync, existsSync, mkdirSync } from "fs"; import { dirname, resolve } from "path"; +import { getAppLogFilePath, getAppLogLevel, getAppLogToFile } from "@/lib/logEnv"; const LOG_LEVELS: Record = { debug: 10, @@ -23,12 +24,12 @@ const LOG_LEVELS: Record = { fatal: 50, }; -const currentLevel = LOG_LEVELS[process.env.LOG_LEVEL?.toLowerCase() || ""] || LOG_LEVELS.info; +const currentLevel = LOG_LEVELS[getAppLogLevel("info").toLowerCase() || ""] || LOG_LEVELS.info; const isProduction = process.env.NODE_ENV === "production"; // File logging configuration -const logToFile = process.env.LOG_TO_FILE !== "false"; -const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log"); +const logToFile = getAppLogToFile(); +const logFilePath = resolve(getAppLogFilePath()); // Ensure log directory exists once at module load if (logToFile) { diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index d4e12d4181..fbab5451aa 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -148,12 +148,9 @@ export const updateSettingsSchema = z.object({ theme: z.string().max(50).optional(), language: z.string().max(10).optional(), requireLogin: z.boolean().optional(), - enableRequestLogs: z.boolean().optional(), enableSocks5Proxy: z.boolean().optional(), instanceName: z.string().max(100).optional(), corsOrigins: z.string().max(500).optional(), - logRetentionDays: z.number().int().min(1).max(365).optional(), - maxCallLogs: z.number().int().min(1).max(1_000_000).optional(), cloudUrl: z.string().max(500).optional(), baseUrl: z.string().max(500).optional(), setupComplete: z.boolean().optional(), diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 9d56f2b394..ac174837a6 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -14,12 +14,9 @@ export const updateSettingsSchema = z.object({ theme: z.string().max(50).optional(), language: z.string().max(10).optional(), requireLogin: z.boolean().optional(), - enableRequestLogs: z.boolean().optional(), enableSocks5Proxy: z.boolean().optional(), instanceName: z.string().max(100).optional(), corsOrigins: z.string().max(500).optional(), - logRetentionDays: z.number().int().min(1).max(365).optional(), - maxCallLogs: z.number().int().min(1).max(1_000_000).optional(), cloudUrl: z.string().max(500).optional(), baseUrl: z.string().max(500).optional(), setupComplete: z.boolean().optional(), diff --git a/tests/unit/batch-b-final.test.mjs b/tests/unit/batch-b-final.test.mjs index e11b2e87e1..d1843605ca 100644 --- a/tests/unit/batch-b-final.test.mjs +++ b/tests/unit/batch-b-final.test.mjs @@ -40,7 +40,13 @@ describe("evalRunner", () => { it("should evaluate exact match", () => { const result = evaluateCase( - { id: "t1", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } }, + { + id: "t1", + name: "test", + model: "test", + input: {}, + expected: { strategy: "exact", value: "hello" }, + }, "hello" ); assert.equal(result.passed, true); @@ -48,7 +54,13 @@ describe("evalRunner", () => { it("should fail exact match on mismatch", () => { const result = evaluateCase( - { id: "t2", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } }, + { + id: "t2", + name: "test", + model: "test", + input: {}, + expected: { strategy: "exact", value: "hello" }, + }, "world" ); assert.equal(result.passed, false); @@ -56,7 +68,13 @@ describe("evalRunner", () => { it("should evaluate contains (case-insensitive)", () => { const result = evaluateCase( - { id: "t3", name: "test", model: "test", input: {}, expected: { strategy: "contains", value: "paris" } }, + { + id: "t3", + name: "test", + model: "test", + input: {}, + expected: { strategy: "contains", value: "paris" }, + }, "The capital is Paris." ); assert.equal(result.passed, true); @@ -64,7 +82,13 @@ describe("evalRunner", () => { it("should evaluate regex", () => { const result = evaluateCase( - { id: "t4", name: "test", model: "test", input: {}, expected: { strategy: "regex", value: "\\d+" } }, + { + id: "t4", + name: "test", + model: "test", + input: {}, + expected: { strategy: "regex", value: "\\d+" }, + }, "The answer is 42." ); assert.equal(result.passed, true); @@ -73,7 +97,10 @@ describe("evalRunner", () => { it("should evaluate custom function", () => { const result = evaluateCase( { - id: "t5", name: "test", model: "test", input: {}, + id: "t5", + name: "test", + model: "test", + input: {}, expected: { strategy: "custom", fn: (output) => output.length > 5 }, }, "this is long enough" @@ -95,8 +122,20 @@ describe("evalRunner", () => { id: "test-suite", name: "Test Suite", cases: [ - { id: "c1", name: "pass", model: "m", input: {}, expected: { strategy: "contains", value: "yes" } }, - { id: "c2", name: "fail", model: "m", input: {}, expected: { strategy: "contains", value: "no" } }, + { + id: "c1", + name: "pass", + model: "m", + input: {}, + expected: { strategy: "contains", value: "yes" }, + }, + { + id: "c2", + name: "fail", + model: "m", + input: {}, + expected: { strategy: "contains", value: "no" }, + }, ], }); @@ -225,7 +264,7 @@ describe("compliance", () => { assert.equal(isNoLog("key-1"), false); }); - it("should have default retention of 90 days", () => { - assert.equal(getRetentionDays(), 90); + it("should expose split default retention windows", () => { + assert.deepEqual(getRetentionDays(), { app: 7, call: 7 }); }); }); diff --git a/tests/unit/call-log-cap.test.mjs b/tests/unit/call-log-cap.test.mjs index 314fe13bdd..a2fe775d49 100644 --- a/tests/unit/call-log-cap.test.mjs +++ b/tests/unit/call-log-cap.test.mjs @@ -4,19 +4,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-cap-")); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-artifacts-")); process.env.DATA_DIR = TEST_DATA_DIR; -const ORIGINAL_CALL_LOGS_MAX = process.env.CALL_LOGS_MAX; +process.env.CALL_LOG_RETENTION_DAYS = "1"; const core = await import("../../src/lib/db/core.ts"); -const localDb = await import("../../src/lib/localDb.ts"); const callLogs = await import("../../src/lib/usage/callLogs.ts"); async function resetStorage() { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); - callLogs.invalidateCallLogsMaxCache(); } test.beforeEach(async () => { @@ -26,61 +24,69 @@ test.beforeEach(async () => { test.after(() => { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); - if (ORIGINAL_CALL_LOGS_MAX === undefined) { - delete process.env.CALL_LOGS_MAX; - } else { - process.env.CALL_LOGS_MAX = ORIGINAL_CALL_LOGS_MAX; - } }); -test("call logs respect the configurable maxCallLogs setting", async () => { - await localDb.updateSettings({ maxCallLogs: 3 }); - callLogs.invalidateCallLogsMaxCache(); +test("call logs store a single per-request artifact with pipeline details", async () => { + const timestamp = "2026-03-30T12:34:56.789Z"; + const logId = "req_artifact_1"; - for (let i = 1; i <= 5; i++) { - await callLogs.saveCallLog({ - method: "POST", - path: "/v1/chat/completions", - status: 200, - model: `model-${i}`, - provider: "openai", - duration: i, - requestBody: { index: i }, - responseBody: { ok: true, index: i }, - }); - } + await callLogs.saveCallLog({ + id: logId, + timestamp, + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + requestedModel: "openai/gpt-5", + provider: "openai", + duration: 42, + requestBody: { messages: [{ role: "user", content: "hello" }] }, + responseBody: { id: "resp_1", choices: [{ message: { content: "world" } }] }, + pipelinePayloads: { + clientRawRequest: { body: { raw: true } }, + providerRequest: { body: { translated: true } }, + providerResponse: { body: { upstream: true } }, + clientResponse: { body: { final: true } }, + }, + }); - const logs = await callLogs.getCallLogs({ limit: 10 }); + const logs = await callLogs.getCallLogs({ limit: 5 }); + assert.equal(logs.length, 1); + assert.equal(logs[0].hasPipelineDetails, true); - assert.equal(logs.length, 3); - assert.deepEqual( - logs.map((entry) => entry.model), - ["model-5", "model-4", "model-3"] + const detail = await callLogs.getCallLogById(logId); + assert.equal(detail?.requestedModel, "openai/gpt-5"); + assert.equal(detail?.pipelinePayloads?.clientRawRequest?.body?.raw, true); + assert.equal(detail?.pipelinePayloads?.providerRequest?.body?.translated, true); + assert.equal(detail?.pipelinePayloads?.providerResponse?.body?.upstream, true); + assert.equal(detail?.pipelinePayloads?.clientResponse?.body?.final, true); + assert.match( + detail?.artifactRelPath || "", + /^2026-03-30\/2026-03-30T12-34-56\.789Z_req_artifact_1\.json$/ ); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", detail.artifactRelPath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.equal(artifact.summary.id, logId); + assert.equal(artifact.summary.requestedModel, "openai/gpt-5"); + assert.equal(artifact.pipeline.clientRawRequest.body.raw, true); }); -test("call logs keep honoring CALL_LOGS_MAX when maxCallLogs was never saved", async () => { - process.env.CALL_LOGS_MAX = "2"; - callLogs.invalidateCallLogsMaxCache(); +test("call log artifact rotation removes directories older than CALL_LOG_RETENTION_DAYS", async () => { + const oldDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-10"); + const freshDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-30"); + fs.mkdirSync(oldDir, { recursive: true }); + fs.mkdirSync(freshDir, { recursive: true }); + fs.writeFileSync(path.join(oldDir, "old.json"), "{}"); + fs.writeFileSync(path.join(freshDir, "fresh.json"), "{}"); - for (let i = 1; i <= 4; i++) { - await callLogs.saveCallLog({ - method: "POST", - path: "/v1/chat/completions", - status: 200, - model: `env-model-${i}`, - provider: "openai", - duration: i, - requestBody: { index: i }, - responseBody: { ok: true, index: i }, - }); - } + const oldTime = new Date("2026-03-10T00:00:00.000Z"); + const freshTime = new Date(); + fs.utimesSync(oldDir, oldTime, oldTime); + fs.utimesSync(freshDir, freshTime, freshTime); - const logs = await callLogs.getCallLogs({ limit: 10 }); + callLogs.rotateCallLogs(); - assert.equal(logs.length, 2); - assert.deepEqual( - logs.map((entry) => entry.model), - ["env-model-4", "env-model-3"] - ); + assert.equal(fs.existsSync(oldDir), false); + assert.equal(fs.existsSync(freshDir), true); }); diff --git a/tests/unit/console-log-levels.test.mjs b/tests/unit/console-log-levels.test.mjs index d268f7aba0..ab90ab1278 100644 --- a/tests/unit/console-log-levels.test.mjs +++ b/tests/unit/console-log-levels.test.mjs @@ -7,16 +7,16 @@ import path from "node:path"; const TEST_LOG_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-console-log-levels-")); const TEST_LOG_PATH = path.join(TEST_LOG_DIR, "app.log"); -const originalLogFilePath = process.env.LOG_FILE_PATH; -process.env.LOG_FILE_PATH = TEST_LOG_PATH; +const originalLogFilePath = process.env.APP_LOG_FILE_PATH; +process.env.APP_LOG_FILE_PATH = TEST_LOG_PATH; const route = await import("../../src/app/api/logs/console/route.ts"); test.after(() => { if (originalLogFilePath === undefined) { - delete process.env.LOG_FILE_PATH; + delete process.env.APP_LOG_FILE_PATH; } else { - process.env.LOG_FILE_PATH = originalLogFilePath; + process.env.APP_LOG_FILE_PATH = originalLogFilePath; } fs.rmSync(TEST_LOG_DIR, { recursive: true, force: true }); }); diff --git a/tests/unit/log-retention.test.mjs b/tests/unit/log-retention.test.mjs new file mode 100644 index 0000000000..7f4bc2b931 --- /dev/null +++ b/tests/unit/log-retention.test.mjs @@ -0,0 +1,134 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-retention-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.APP_LOG_RETENTION_DAYS = "2"; +process.env.CALL_LOG_RETENTION_DAYS = "1"; + +const core = await import("../../src/lib/db/core.ts"); +const compliance = await import("../../src/lib/compliance/index.ts"); + +function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => { + resetStorage(); +}); + +test.after(() => { + resetStorage(); +}); + +test("cleanupExpiredLogs uses separate APP and CALL retention windows", () => { + compliance.initAuditLog(); + const db = core.getDbInstance(); + + const oldCallTs = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString(); + const freshCallTs = new Date().toISOString(); + const oldAppTs = new Date(Date.now() - 5 * 24 * 60 * 60 * 1000).toISOString(); + const freshAppTs = new Date().toISOString(); + + db.prepare( + "INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ).run("openai", "old-usage", 1, 1, 1, 1, 1, oldCallTs); + db.prepare( + "INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ).run("openai", "fresh-usage", 1, 1, 1, 1, 1, freshCallTs); + + db.prepare( + "INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ).run( + "old-call", + oldCallTs, + "POST", + "/v1/chat/completions", + 200, + "old", + "openai", + "-", + 1, + 1, + 1, + 0 + ); + db.prepare( + "INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ).run( + "fresh-call", + freshCallTs, + "POST", + "/v1/chat/completions", + 200, + "fresh", + "openai", + "-", + 1, + 1, + 1, + 0 + ); + + db.prepare( + "INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)" + ).run("old-proxy", oldCallTs, "success", "direct", 1); + db.prepare( + "INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)" + ).run("fresh-proxy", freshCallTs, "success", "direct", 1); + + db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run( + "old-detail", + oldCallTs, + 1 + ); + db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run( + "fresh-detail", + freshCallTs, + 1 + ); + + db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run( + oldAppTs, + "old-audit", + "system" + ); + db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run( + freshAppTs, + "fresh-audit", + "system" + ); + + db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run( + "old-tool", + 1, + oldAppTs + ); + db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run( + "fresh-tool", + 1, + freshAppTs + ); + + const result = compliance.cleanupExpiredLogs(); + + assert.equal(result.deletedUsage, 1); + assert.equal(result.deletedCallLogs, 1); + assert.equal(result.deletedProxyLogs, 1); + assert.equal(result.deletedRequestDetailLogs, 1); + assert.equal(result.deletedAuditLogs, 1); + assert.equal(result.deletedMcpAuditLogs, 1); + assert.deepEqual(compliance.getRetentionDays(), { app: 2, call: 1 }); + + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM usage_history").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM call_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM proxy_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM request_detail_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM audit_log").get().cnt, 2); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM mcp_tool_audit").get().cnt, 1); +}); diff --git a/tests/unit/request-log-migration.test.mjs b/tests/unit/request-log-migration.test.mjs new file mode 100644 index 0000000000..95a4c4bce6 --- /dev/null +++ b/tests/unit/request-log-migration.test.mjs @@ -0,0 +1,72 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-migration-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const migrations = await import("../../src/lib/usage/migrations.ts"); + +const LEGACY_LOGS_DIR = path.join(TEST_DATA_DIR, "logs"); +const LEGACY_CALL_LOGS_DIR = path.join(TEST_DATA_DIR, "call_logs"); +const LEGACY_SUMMARY_FILE = path.join(TEST_DATA_DIR, "log.txt"); +const MARKER_PATH = path.join(migrations.LOG_ARCHIVES_DIR, "legacy-request-logs.json"); + +function resetDataDir() { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +function seedLegacyLayout() { + fs.mkdirSync(path.join(LEGACY_LOGS_DIR, "session-a"), { recursive: true }); + fs.writeFileSync( + path.join(LEGACY_LOGS_DIR, "session-a", "1_req_client.json"), + JSON.stringify({ ok: true }, null, 2) + ); + + fs.mkdirSync(path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30"), { recursive: true }); + fs.writeFileSync( + path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30", "123000_model_200.json"), + JSON.stringify({ ok: true }, null, 2) + ); + + fs.writeFileSync(LEGACY_SUMMARY_FILE, "legacy summary\n"); +} + +test.beforeEach(() => { + resetDataDir(); +}); + +test.after(() => { + resetDataDir(); +}); + +test("archives legacy request log layout into a zip and removes old files", async () => { + seedLegacyLayout(); + + const archiveFilename = await migrations.archiveLegacyRequestLogs(); + + assert.match(archiveFilename || "", /_legacy-request-logs\.zip$/); + assert.equal(fs.existsSync(LEGACY_LOGS_DIR), false); + assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), false); + assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), false); + assert.equal(fs.existsSync(MARKER_PATH), true); + + const archivePath = path.join(migrations.LOG_ARCHIVES_DIR, archiveFilename); + assert.equal(fs.existsSync(archivePath), true); + assert.ok(fs.statSync(archivePath).size > 0); +}); + +test("keeps legacy files in place when zip creation fails", async () => { + seedLegacyLayout(); + fs.writeFileSync(migrations.LOG_ARCHIVES_DIR, "not-a-directory"); + + await assert.rejects(() => migrations.archiveLegacyRequestLogs()); + + assert.equal(fs.existsSync(LEGACY_LOGS_DIR), true); + assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), true); + assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), true); + assert.equal(fs.existsSync(MARKER_PATH), false); +}); diff --git a/tests/unit/usage-analytics.test.mjs b/tests/unit/usage-analytics.test.mjs index 69f06bdc23..ed148ac656 100644 --- a/tests/unit/usage-analytics.test.mjs +++ b/tests/unit/usage-analytics.test.mjs @@ -12,8 +12,8 @@ const localDb = await import("../../src/lib/localDb.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const usageHistory = await import("../../src/lib/usage/usageHistory.ts"); const usageStats = await import("../../src/lib/usage/usageStats.ts"); +const callLogs = await import("../../src/lib/usage/callLogs.ts"); const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); -const { LOG_FILE } = await import("../../src/lib/usage/migrations.ts"); function clearPendingRequests() { const pending = usageHistory.getPendingRequests(); @@ -263,7 +263,7 @@ test("getUsageStats aggregates totals, buckets, pending requests, and cost break assert.equal(recentBucketTotal, 1); }); -test("request log appends readable entries and trims to the most recent 200 lines", async () => { +test("recent request summaries are generated from SQLite call logs", async () => { const connection = await providersDb.createProviderConnection({ provider: "log-provider", authType: "apikey", @@ -272,19 +272,23 @@ test("request log appends readable entries and trims to the most recent 200 line }); for (let i = 0; i < 205; i++) { - await usageHistory.appendRequestLog({ + await callLogs.saveCallLog({ + id: `log-${i}`, + timestamp: new Date(Date.now() + i).toISOString(), + method: "POST", + path: "/v1/chat/completions", + status: 200, model: `model-${i}`, provider: "log-provider", connectionId: connection.id, tokens: { input: i + 1, output: i + 2 }, - status: 200, + requestBody: { index: i }, + responseBody: { ok: true, index: i }, }); } const recent = await usageHistory.getRecentLogs(3); - const lines = fs.readFileSync(LOG_FILE, "utf8").trim().split("\n"); - assert.equal(lines.length, 200); assert.equal(recent.length, 3); assert.match(recent[0], /model-204/); assert.match(recent[0], /LOG-PROVIDER/);