Merge PR #851: feat(logging): unify request log retention and artifacts

This commit is contained in:
diegosouzapw
2026-03-31 09:11:57 -03:00
43 changed files with 1437 additions and 862 deletions

View File

@@ -2,6 +2,25 @@
## [Unreleased]
> [!WARNING]
> **BREAKING CHANGE: request logging, retention, and logging environment variables have been redesigned.**
> On the first startup after upgrading, OmniRoute archives legacy request logs from `DATA_DIR/logs/`, legacy `DATA_DIR/call_logs/`, and `DATA_DIR/log.txt` into `DATA_DIR/log_archives/*.zip`, then removes the deprecated layout and switches to the new unified artifact format under `DATA_DIR/call_logs/`.
### ✨ New Features
- **Unified Request Log Artifacts:** Request logging now stores one SQLite index row plus one JSON artifact per request under `DATA_DIR/call_logs/`, with optional pipeline capture embedded in the same file.
### ⚠️ Breaking Changes
- **Request Log Layout:** Removed the old multi-file `DATA_DIR/logs/` request log sessions and the `DATA_DIR/log.txt` summary file. New requests are written as single JSON artifacts in `DATA_DIR/call_logs/YYYY-MM-DD/`.
- **Logging Environment Variables:** Replaced `LOG_*`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` with the new `APP_LOG_*` and `CALL_LOG_RETENTION_DAYS` configuration model.
- **Pipeline Toggle Setting:** Replaced the legacy `detailed_logs_enabled` setting with `call_log_pipeline_enabled`. New pipeline details are embedded inside the request artifact instead of being stored as separate `request_detail_logs` records.
### 🛠️ Maintenance
- **Legacy Request Log Upgrade Backup:** Upgrades now archive old `data/logs/`, legacy `data/call_logs/`, and `data/log.txt` layouts into `DATA_DIR/log_archives/*.zip` before removing the deprecated structure.
- **Streaming Usage Persistence:** Streaming requests now write a single `usage_history` row on completion instead of emitting a duplicate in-progress usage row with empty status metadata.
---
## [3.3.11] - 2026-03-31

View File

@@ -26,6 +26,30 @@ _Your universal API proxy — one endpoint, 67+ providers, zero downtime. Now wi
---
## Breaking Change: Unified Logging Upgrade
> [!WARNING]
> **This release changes both the on-disk request log layout and the logging environment variables.**
>
> If you are upgrading an existing instance:
>
> - Request logs now live in `DATA_DIR/call_logs/YYYY-MM-DD/` as **one JSON artifact per request**.
> - The old `DATA_DIR/logs/` session folders and `DATA_DIR/log.txt` summary file are removed.
> - On the first startup after upgrading, OmniRoute creates a safety backup at `DATA_DIR/log_archives/*.zip` before removing the deprecated request log layout.
> - Legacy logging env vars such as `LOG_TO_FILE`, `LOG_FILE_PATH`, `LOG_MAX_FILE_SIZE`, `LOG_RETENTION_DAYS`, `LOG_LEVEL`, `LOG_FORMAT`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` are no longer supported.
> - Use the new env model instead:
> - `APP_LOG_TO_FILE`
> - `APP_LOG_FILE_PATH`
> - `APP_LOG_MAX_FILE_SIZE`
> - `APP_LOG_RETENTION_DAYS`
> - `APP_LOG_LEVEL`
> - `APP_LOG_FORMAT`
> - `CALL_LOG_RETENTION_DAYS`
>
> For release details and upgrade notes, see the [CHANGELOG](CHANGELOG.md).
---
## 🆕 What's New in v3.0.0
> **Upgrading from v2.9.5?** — See the [full CHANGELOG](CHANGELOG.md#300--2026-03-22-release-candidate--not-yet-merged-to-main) for all changes.
@@ -1811,7 +1835,9 @@ opencode
**No request logs**
- Set `ENABLE_REQUEST_LOGS=true` in `.env`
- Request artifacts are written to `DATA_DIR/call_logs/` as one JSON file per request
- Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads
- Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log`
**Connection test shows "Invalid" for OpenAI-compatible providers**

View File

@@ -23,7 +23,7 @@ import {
import { HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts";
import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts";
import { updateProviderConnection } from "@/lib/db/providers";
import { isDetailedLoggingEnabled, saveRequestDetailLog } from "@/lib/db/detailedLogs";
import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs";
import { logAuditEvent } from "@/lib/compliance";
import { handleBypassRequest } from "../utils/bypassHandler.ts";
import {
@@ -536,6 +536,24 @@ export async function handleChatCore({
claudeCacheUsageMeta?: Record<string, unknown>;
}) => {
const callLogId = generateRequestId();
const pipelinePayloads = detailedLoggingEnabled ? reqLogger?.getPipelinePayloads?.() : null;
if (pipelinePayloads) {
if (providerResponse !== undefined) {
pipelinePayloads.providerResponse = providerResponse as Record<string, unknown>;
}
if (clientResponse !== undefined) {
pipelinePayloads.clientResponse = clientResponse as Record<string, unknown>;
}
if (error) {
pipelinePayloads.error = {
...(typeof pipelinePayloads.error === "object" && pipelinePayloads.error
? (pipelinePayloads.error as Record<string, unknown>)
: {}),
message: error,
};
}
}
saveCallLog({
id: callLogId,
@@ -568,31 +586,8 @@ export async function handleChatCore({
apiKeyId: apiKeyInfo?.id || null,
apiKeyName: apiKeyInfo?.name || null,
noLog: noLogEnabled,
pipelinePayloads,
}).catch(() => {});
if (!detailedLoggingEnabled) {
return;
}
try {
saveRequestDetailLog({
call_log_id: callLogId,
client_request: clientRawRequest?.body ?? body,
translated_request: providerRequest ?? null,
provider_response: providerResponse ?? null,
client_response: clientResponse ?? null,
provider,
model,
source_format: sourceFormat,
target_format: targetFormat,
duration_ms: Date.now() - startTime,
api_key_id: apiKeyInfo?.id || null,
no_log: noLogEnabled,
});
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err);
log?.debug?.("DETAIL_LOG", `Failed to save detailed log: ${errMessage}`);
}
};
// Primary path: merge client model id + alias target so config on either key applies; resolved

View File

@@ -2,7 +2,7 @@
* Structured console logger utility for omniroute.
*
* Provides consistent, machine-parseable log output across the codebase.
* Supports two output formats controlled by LOG_FORMAT env var:
* Supports two output formats controlled by APP_LOG_FORMAT env var:
* - "text" (default): [LEVEL] [TAG] message {metadata}
* - "json": Single-line JSON objects for log aggregators
*
@@ -18,17 +18,16 @@
* reqLog.info("AUTH", "Token refreshed", { provider: "claude" });
*
* Environment variables:
* LOG_LEVEL — minimum level: debug | info | warn | error (default: info)
* LOG_FORMAT — output format: text | json (default: text)
* APP_LOG_LEVEL — minimum level: debug | info | warn | error (default: info)
* APP_LOG_FORMAT — output format: text | json (default: text)
*/
import { getAppLogFormat, getAppLogLevel } from "../../src/lib/logEnv";
const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 };
const currentLevel =
LEVELS[((typeof process !== "undefined" && process.env?.LOG_LEVEL) || "info").toLowerCase()] ??
LEVELS.info;
const currentLevel = LEVELS[getAppLogLevel("info").toLowerCase()] ?? LEVELS.info;
const jsonFormat = typeof process !== "undefined" && process.env?.LOG_FORMAT === "json";
const jsonFormat = getAppLogFormat("text") === "json";
let requestCounter = 0;

View File

@@ -1,92 +1,117 @@
// Check if running in Node.js environment (has fs module)
const isNode =
typeof process !== "undefined" && process.versions?.node && typeof window === "undefined";
type JsonRecord = Record<string, unknown>;
// Check if logging is enabled via environment variable (default: false)
const LOGGING_ENABLED =
typeof process !== "undefined" && process.env?.ENABLE_REQUEST_LOGS === "true";
type HeaderInput =
| Headers
| Record<string, unknown>
| { entries?: () => IterableIterator<[string, string]> }
| null
| undefined;
let fs = null;
let path = null;
let LOGS_DIR = null;
export type RequestPipelinePayloads = {
clientRawRequest?: JsonRecord;
sourceRequest?: JsonRecord;
openaiRequest?: JsonRecord;
providerRequest?: JsonRecord;
providerResponse?: JsonRecord;
clientResponse?: JsonRecord;
error?: JsonRecord;
streamChunks?: {
provider?: string[];
openai?: string[];
client?: string[];
};
};
// Lazy load Node.js modules (avoid top-level await)
async function ensureNodeModules() {
if (!isNode || !LOGGING_ENABLED || fs) return;
try {
fs = await import("fs");
path = await import("path");
const { resolveDataDir } = await import("../../src/lib/dataPaths");
LOGS_DIR = path.join(resolveDataDir(), "logs");
} catch {
// Running in non-Node environment (Worker, Browser, etc.)
}
}
type RequestLogger = {
sessionPath: null;
logClientRawRequest: (endpoint: unknown, body: unknown, headers?: HeaderInput) => void;
logRawRequest: (body: unknown, headers?: HeaderInput) => void;
logOpenAIRequest: (body: unknown) => void;
logTargetRequest: (url: unknown, headers: HeaderInput, body: unknown) => void;
logProviderResponse: (
status: unknown,
statusText: unknown,
headers: HeaderInput,
body: unknown
) => void;
appendProviderChunk: (chunk: string) => void;
appendOpenAIChunk: (chunk: string) => void;
logConvertedResponse: (body: unknown) => void;
appendConvertedChunk: (chunk: string) => void;
logError: (error: unknown, requestBody?: unknown) => void;
getPipelinePayloads: () => RequestPipelinePayloads | null;
};
// Format timestamp for folder name: 20251228_143045
function formatTimestamp(date = new Date()) {
const pad = (n) => String(n).padStart(2, "0");
const y = date.getFullYear();
const m = pad(date.getMonth() + 1);
const d = pad(date.getDate());
const h = pad(date.getHours());
const min = pad(date.getMinutes());
const s = pad(date.getSeconds());
return `${y}${m}${d}_${h}${min}${s}`;
}
// Create log session folder: {sourceFormat}_{targetFormat}_{model}_{timestamp}
async function createLogSession(sourceFormat, targetFormat, model) {
await ensureNodeModules();
if (!fs || !LOGS_DIR) return null;
try {
await fs.promises.mkdir(LOGS_DIR, { recursive: true });
const timestamp = formatTimestamp();
const safeModel = (model || "unknown").replace(/[/:]/g, "-");
const folderName = `${sourceFormat}_${targetFormat}_${safeModel}_${timestamp}`;
const sessionPath = path.join(LOGS_DIR, folderName);
await fs.promises.mkdir(sessionPath, { recursive: true });
return sessionPath;
} catch (err) {
console.log("[LOG] Failed to create log session:", err.message);
return null;
}
}
// Write JSON file (async, fire-and-forget)
function writeJsonFile(sessionPath, filename, data) {
if (!fs || !sessionPath) return;
const filePath = path.join(sessionPath, filename);
fs.promises
.writeFile(filePath, JSON.stringify(data, null, 2))
.catch((err) => console.log(`[LOG] Failed to write ${filename}:`, err.message));
}
// Mask sensitive data in headers before writing to log files
function maskSensitiveHeaders(headers) {
function maskSensitiveHeaders(headers: HeaderInput): Record<string, unknown> {
if (!headers) return {};
const masked = { ...headers };
const headerEntries =
typeof (headers as Headers).entries === "function"
? Object.fromEntries((headers as Headers).entries())
: { ...(headers as Record<string, unknown>) };
const masked = { ...headerEntries };
const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token"];
for (const key of Object.keys(masked)) {
const lowerKey = key.toLowerCase();
if (sensitiveKeys.some((sk) => lowerKey.includes(sk))) {
const value = masked[key];
if (value && value.length > 20) {
masked[key] = value.slice(0, 10) + "..." + value.slice(-5);
}
if (!sensitiveKeys.some((candidate) => lowerKey.includes(candidate))) {
continue;
}
const value = masked[key];
if (typeof value === "string" && value.length > 20) {
masked[key] = `${value.slice(0, 10)}...${value.slice(-5)}`;
} else if (value) {
masked[key] = "[REDACTED]";
}
}
return masked;
}
// No-op logger when logging is disabled
function createNoOpLogger() {
function createEmptyStreamChunks() {
return {
provider: [] as string[],
openai: [] as string[],
client: [] as string[],
};
}
function hasOwnValues(value: unknown): boolean {
return Boolean(value && typeof value === "object" && Object.keys(value as JsonRecord).length > 0);
}
function compactPipelinePayloads(
payloads: RequestPipelinePayloads
): RequestPipelinePayloads | null {
const result: RequestPipelinePayloads = {};
for (const [key, value] of Object.entries(payloads)) {
if (value === null || value === undefined) {
continue;
}
if (key === "streamChunks" && value && typeof value === "object") {
const chunkRecord = value as Record<string, unknown>;
const compactedChunks = Object.fromEntries(
Object.entries(chunkRecord).filter(
([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0
)
);
if (Object.keys(compactedChunks).length > 0) {
result.streamChunks = compactedChunks as RequestPipelinePayloads["streamChunks"];
}
continue;
}
result[key as keyof RequestPipelinePayloads] = value as never;
}
return hasOwnValues(result) ? result : null;
}
function createNoOpLogger(): RequestLogger {
return {
sessionPath: null,
logClientRawRequest() {},
@@ -99,151 +124,106 @@ function createNoOpLogger() {
logConvertedResponse() {},
appendConvertedChunk() {},
logError() {},
getPipelinePayloads() {
return null;
},
};
}
/**
* Create a new log session and return logger functions
* @param {string} sourceFormat - Source format from client (claude, openai, etc.)
* @param {string} targetFormat - Target format to provider (antigravity, gemini-cli, etc.)
* @param {string} model - Model name
* @returns {Promise<object>} Promise that resolves to logger object with methods to log each stage
*/
export async function createRequestLogger(sourceFormat, targetFormat, model) {
// Return no-op logger if logging is disabled
if (!LOGGING_ENABLED) {
return createNoOpLogger();
}
// Wait for session to be created before returning logger
const sessionPath = await createLogSession(sourceFormat, targetFormat, model);
export async function createRequestLogger(
_sourceFormat?: string,
_targetFormat?: string,
_model?: string
): Promise<RequestLogger> {
const streamChunks = createEmptyStreamChunks();
const payloads: RequestPipelinePayloads = {
streamChunks,
};
return {
get sessionPath() {
return sessionPath;
},
sessionPath: null,
// 1. Log client raw request (before all conversion steps)
logClientRawRequest(endpoint, body, headers = {}) {
writeJsonFile(sessionPath, "1_req_client.json", {
payloads.clientRawRequest = {
timestamp: new Date().toISOString(),
endpoint,
headers: maskSensitiveHeaders(headers),
body,
});
};
},
// 2. Log raw request from client (after initial conversion like responsesApi)
logRawRequest(body, headers = {}) {
writeJsonFile(sessionPath, "2_req_source.json", {
payloads.sourceRequest = {
timestamp: new Date().toISOString(),
headers: maskSensitiveHeaders(headers),
body,
});
};
},
// 3. Log OpenAI intermediate format (source → openai)
logOpenAIRequest(body) {
writeJsonFile(sessionPath, "3_req_openai.json", {
payloads.openaiRequest = {
timestamp: new Date().toISOString(),
body,
});
};
},
// 4. Log target format request (openai → target)
logTargetRequest(url, headers, body) {
writeJsonFile(sessionPath, "4_req_target.json", {
payloads.providerRequest = {
timestamp: new Date().toISOString(),
url,
headers: maskSensitiveHeaders(headers),
body,
});
};
},
// 5. Log provider response (for non-streaming or error)
logProviderResponse(status, statusText, headers, body) {
const filename = "5_res_provider.json";
writeJsonFile(sessionPath, filename, {
payloads.providerResponse = {
timestamp: new Date().toISOString(),
status,
statusText,
headers: headers
? typeof headers.entries === "function"
? Object.fromEntries(headers.entries())
: headers
: {},
headers: maskSensitiveHeaders(headers),
body,
});
};
},
// 5. Append streaming chunk to provider response (async)
appendProviderChunk(chunk) {
if (!fs || !sessionPath) return;
const filePath = path.join(sessionPath, "5_res_provider.txt");
fs.promises.appendFile(filePath, chunk).catch(() => {});
if (typeof chunk === "string" && chunk.length > 0) {
streamChunks.provider.push(chunk);
}
},
// 6. Append OpenAI intermediate chunks (async)
appendOpenAIChunk(chunk) {
if (!fs || !sessionPath) return;
const filePath = path.join(sessionPath, "6_res_openai.txt");
fs.promises.appendFile(filePath, chunk).catch(() => {});
if (typeof chunk === "string" && chunk.length > 0) {
streamChunks.openai.push(chunk);
}
},
// 7. Log converted response to client (for non-streaming)
logConvertedResponse(body) {
writeJsonFile(sessionPath, "7_res_client.json", {
payloads.clientResponse = {
timestamp: new Date().toISOString(),
body,
});
};
},
// 7b. Append streaming chunk to converted response (async)
appendConvertedChunk(chunk) {
if (!fs || !sessionPath) return;
const filePath = path.join(sessionPath, "7_res_client.txt");
fs.promises.appendFile(filePath, chunk).catch(() => {});
if (typeof chunk === "string" && chunk.length > 0) {
streamChunks.client.push(chunk);
}
},
// 8. Log error
logError(error, requestBody = null) {
writeJsonFile(sessionPath, "6_error.json", {
payloads.error = {
timestamp: new Date().toISOString(),
error: error?.message || String(error),
stack: error?.stack,
requestBody,
});
},
};
}
// Legacy logError (kept for backward compatibility, converted to async)
export function logError(provider, { error, url, model, requestBody }) {
if (!fs || !LOGS_DIR) return;
const writeLog = async () => {
try {
await fs.promises.mkdir(LOGS_DIR, { recursive: true });
const date = new Date().toISOString().split("T")[0];
const logPath = path.join(LOGS_DIR, `${provider}-${date}.log`);
const logEntry = {
timestamp: new Date().toISOString(),
type: "error",
provider,
model,
url,
error: error?.message || String(error),
stack: error?.stack,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
requestBody,
};
},
await fs.promises.appendFile(logPath, JSON.stringify(logEntry) + "\n");
} catch (err) {
console.log("[LOG] Failed to write error log:", err.message);
}
getPipelinePayloads() {
return compactPipelinePayloads(payloads);
},
};
writeLog();
}
export function logError(_provider: string, _entry: unknown) {}

View File

@@ -2,7 +2,7 @@
* Token Usage Tracking - Extract, normalize, estimate and log token usage
*/
import { saveRequestUsage, appendRequestLog } from "@/lib/usageDb";
import { appendRequestLog } from "@/lib/usageDb";
import {
getLoggedInputTokens,
getLoggedOutputTokens,
@@ -444,7 +444,8 @@ export function logUsage(provider, usage, model = null, connectionId = null, api
console.log(msg);
// Save to usage DB with cache-read tracked separately from the main input counter.
// Streaming requests persist usage once in chatCore's completion callback.
// Keep this helper side-effect free apart from console visibility.
const tokens = {
input: inTokens,
output: outTokens,
@@ -452,13 +453,5 @@ export function logUsage(provider, usage, model = null, connectionId = null, api
cacheCreation: cacheCreation || 0,
reasoning: reasoning || 0,
};
saveRequestUsage({
model,
provider,
connectionId,
apiKeyId: apiKeyInfo?.id || undefined,
apiKeyName: apiKeyInfo?.name || undefined,
tokens,
}).catch(() => {});
appendRequestLog({ model, provider, connectionId, tokens, status: "200 OK" }).catch(() => {});
}

19
package-lock.json generated
View File

@@ -44,6 +44,7 @@
"undici": "^7.19.2",
"uuid": "^13.0.0",
"wreq-js": "^2.0.1",
"yazl": "^3.3.1",
"zod": "^4.3.6",
"zustand": "^5.0.10"
},
@@ -7654,6 +7655,15 @@
"ieee754": "^1.1.13"
}
},
"node_modules/buffer-crc32": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz",
"integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==",
"license": "MIT",
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/bundle-name": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz",
@@ -20226,6 +20236,15 @@
"node": ">=8"
}
},
"node_modules/yazl": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz",
"integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==",
"license": "MIT",
"dependencies": {
"buffer-crc32": "^1.0.0"
}
},
"node_modules/yocto-queue": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

View File

@@ -115,6 +115,7 @@
"undici": "^7.19.2",
"uuid": "^13.0.0",
"wreq-js": "^2.0.1",
"yazl": "^3.3.1",
"zod": "^4.3.6",
"zustand": "^5.0.10"
},

View File

@@ -18,11 +18,6 @@ export default function SystemStorageTab() {
const [importStatus, setImportStatus] = useState({ type: "", message: "" });
const [confirmImport, setConfirmImport] = useState(false);
const [pendingImportFile, setPendingImportFile] = useState<File | null>(null);
const [maxCallLogs, setMaxCallLogs] = useState(10000);
const [maxCallLogsDraft, setMaxCallLogsDraft] = useState("10000");
const [settingsLoading, setSettingsLoading] = useState(true);
const [maxCallLogsSaving, setMaxCallLogsSaving] = useState(false);
const [maxCallLogsStatus, setMaxCallLogsStatus] = useState({ type: "", message: "" });
const fileInputRef = useRef<HTMLInputElement>(null);
const locale = useLocale();
const t = useTranslations("settings");
@@ -31,7 +26,10 @@ export default function SystemStorageTab() {
driver: "sqlite",
dbPath: "~/.omniroute/storage.sqlite",
sizeBytes: 0,
retentionDays: 90,
retentionDays: {
app: 7,
call: 7,
},
lastBackupAt: null,
});
@@ -59,27 +57,6 @@ export default function SystemStorageTab() {
}
};
const loadSettings = async () => {
setSettingsLoading(true);
try {
const res = await fetch("/api/settings");
if (!res.ok) return;
const data = await res.json();
const value =
typeof data.maxCallLogs === "number" &&
Number.isInteger(data.maxCallLogs) &&
data.maxCallLogs > 0
? data.maxCallLogs
: 10000;
setMaxCallLogs(value);
setMaxCallLogsDraft(String(value));
} catch (err) {
console.error("Failed to fetch settings:", err);
} finally {
setSettingsLoading(false);
}
};
const handleManualBackup = async () => {
setManualBackupLoading(true);
setManualBackupStatus({ type: "", message: "" });
@@ -145,47 +122,8 @@ export default function SystemStorageTab() {
useEffect(() => {
loadStorageHealth();
loadSettings();
}, []);
const handleSaveMaxCallLogs = async () => {
const parsed = Number.parseInt(maxCallLogsDraft, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
setMaxCallLogsStatus({
type: "error",
message: "Enter a positive integer for the call log limit.",
});
return;
}
setMaxCallLogsSaving(true);
setMaxCallLogsStatus({ type: "", message: "" });
try {
const res = await fetch("/api/settings", {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ maxCallLogs: parsed }),
});
const data = await res.json();
if (!res.ok) {
throw new Error(data.error || "Failed to save call log limit");
}
setMaxCallLogs(parsed);
setMaxCallLogsDraft(String(parsed));
setMaxCallLogsStatus({
type: "success",
message: "Call log retention limit saved.",
});
} catch (err) {
setMaxCallLogsStatus({
type: "error",
message: (err as Error).message || "Failed to save call log limit",
});
} finally {
setMaxCallLogsSaving(false);
}
};
const handleExport = async () => {
setExportLoading(true);
try {
@@ -345,53 +283,23 @@ export default function SystemStorageTab() {
</div>
<div className="p-3 rounded-lg bg-bg border border-border mb-4">
<div className="flex items-start justify-between gap-3">
<div className="flex items-start justify-between gap-3 flex-wrap">
<div>
<p className="text-sm font-medium text-text-main">Call log retention limit</p>
<p className="text-sm font-medium text-text-main">Log retention policy</p>
<p className="text-xs text-text-muted">
Keep only the most recent call log entries in SQLite. Older entries are pruned
automatically after each new request log is saved.
Request logs follow <code>CALL_LOG_RETENTION_DAYS</code>. Application and audit logs
follow <code>APP_LOG_RETENTION_DAYS</code>.
</p>
</div>
<Badge variant="default" size="sm">
{maxCallLogs.toLocaleString()}
</Badge>
</div>
<div className="flex flex-wrap items-center gap-2 mt-3">
<input
type="number"
min="1"
step="1"
value={maxCallLogsDraft}
onChange={(e) => setMaxCallLogsDraft(e.target.value)}
disabled={settingsLoading || maxCallLogsSaving}
className="w-40 rounded-lg border border-border bg-bg-secondary px-3 py-2 text-sm text-text-main focus:outline-none focus:ring-1 focus:ring-primary/40"
aria-label="Call log retention limit"
/>
<Button
variant="outline"
size="sm"
onClick={handleSaveMaxCallLogs}
loading={maxCallLogsSaving}
disabled={settingsLoading}
>
Save limit
</Button>
</div>
{maxCallLogsStatus.message && (
<div
className={`mt-3 rounded-lg border px-3 py-2 text-sm ${
maxCallLogsStatus.type === "success"
? "border-green-500/20 bg-green-500/10 text-green-500"
: "border-red-500/20 bg-red-500/10 text-red-500"
}`}
role="alert"
>
{maxCallLogsStatus.message}
<div className="flex items-center gap-2">
<Badge variant="default" size="sm">
Call {storageHealth.retentionDays.call}d
</Badge>
<Badge variant="default" size="sm">
App {storageHealth.retentionDays.app}d
</Badge>
</div>
)}
</div>
</div>
{/* Export / Import */}

View File

@@ -12,7 +12,7 @@
import { NextRequest, NextResponse } from "next/server";
import { readFileSync, existsSync } from "fs";
import { join } from "path";
import { getAppLogFilePath } from "@/lib/logEnv";
const LEVEL_ORDER: Record<string, number> = {
trace: 5,
@@ -34,7 +34,7 @@ const NUMERIC_LEVEL_MAP: Record<number, string> = {
};
function getLogFilePath(): string {
return process.env.LOG_FILE_PATH || join(process.cwd(), "logs", "application", "app.log");
return getAppLogFilePath();
}
function parseLevel(raw: string | number): string {

View File

@@ -1,6 +1,6 @@
/**
* GET /api/logs/detail — List detailed request logs + current enabled flag
* POST /api/logs/detail — Enable/disable detailed logging
* GET /api/logs/detail — List legacy detailed request logs + current enabled flag
* POST /api/logs/detail — Enable/disable pipeline capture for unified call log artifacts
*/
import { NextRequest, NextResponse } from "next/server";
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
@@ -35,13 +35,13 @@ export async function POST(req: NextRequest) {
const body = await req.json();
const enabled = body.enabled === true || body.enabled === "1";
await updateSettings({ detailed_logs_enabled: enabled });
await updateSettings({ call_log_pipeline_enabled: enabled });
return NextResponse.json({
success: true,
enabled,
message: enabled
? "Detailed logging enabled. Pipeline bodies will be captured for new requests."
: "Detailed logging disabled.",
? "Pipeline capture enabled. New request artifacts will include per-stage payloads."
: "Pipeline capture disabled.",
});
}

View File

@@ -17,18 +17,12 @@ export async function GET(request: Request) {
let rows: unknown[] = [];
let tableName = "";
if (logType === "call-logs") {
if (logType === "call-logs" || logType === "request-logs") {
tableName = "call_logs";
const stmt = db.prepare(
"SELECT * FROM call_logs WHERE timestamp >= @since ORDER BY timestamp DESC"
);
rows = stmt.all({ since });
} else if (logType === "request-logs") {
tableName = "request_logs";
const stmt = db.prepare(
"SELECT * FROM request_logs WHERE timestamp >= @since ORDER BY timestamp DESC"
);
rows = stmt.all({ since });
} else if (logType === "proxy-logs") {
tableName = "proxy_logs";
const stmt = db.prepare(

View File

@@ -19,14 +19,12 @@ export async function GET() {
setCliCompatProviders(settings.cliCompatProviders as string[]);
}
const enableRequestLogs = process.env.ENABLE_REQUEST_LOGS === "true";
const runtimePorts = getRuntimePorts();
const cloudUrl = process.env.CLOUD_URL || process.env.NEXT_PUBLIC_CLOUD_URL || null;
const machineId = await getConsistentMachineId();
return NextResponse.json({
...safeSettings,
enableRequestLogs,
hasPassword: !!password || !!process.env.INITIAL_PASSWORD,
runtimePorts,
apiPort: runtimePorts.apiPort,
@@ -114,11 +112,6 @@ export async function PATCH(request) {
setCliCompatProviders(body.cliCompatProviders || []);
}
if ("maxCallLogs" in body) {
const { invalidateCallLogsMaxCache } = await import("@/lib/usage/callLogs");
invalidateCallLogsMaxCache();
}
// Sync cache control settings to runtime cache
if ("alwaysPreserveClientCache" in body) {
const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings");

View File

@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
import path from "path";
import fs from "fs";
import { resolveDataDir } from "@/lib/dataPaths";
import { getAppLogRetentionDays, getCallLogRetentionDays } from "@/lib/logEnv";
/**
* GET /api/storage/health — Return database storage information.
@@ -56,7 +57,10 @@ export async function GET() {
sizeBytes,
lastBackupAt,
backupCount,
retentionDays: 90,
retentionDays: {
app: getAppLogRetentionDays(),
call: getCallLogRetentionDays(),
},
dataDir: dataDir.startsWith(homeDir) ? "~" + dataDir.slice(homeDir.length) : dataDir,
});
} catch (error) {

View File

@@ -96,11 +96,10 @@
* @property {boolean} hasPassword - Whether a password has been set
* @property {string} [theme] - UI theme
* @property {string} [language] - UI language
* @property {boolean} [enableRequestLogs] - Whether request logging is enabled
* @property {boolean} [enableSocks5Proxy] - Whether SOCKS5 proxy is allowed
* @property {string} [instanceName] - Instance display name
* @property {string} [corsOrigins] - Allowed CORS origins
* @property {number} [logRetentionDays] - Log retention in days
* @property {boolean} [call_log_pipeline_enabled] - Whether per-request pipeline capture is enabled
* @property {string[]} [hiddenSidebarItems] - Sidebar entry ids hidden for visual decluttering
*/

View File

@@ -65,6 +65,9 @@ async function ensureSecrets(): Promise<void> {
export async function registerNodejs(): Promise<void> {
await ensureSecrets();
// Trigger request-log layout migration during startup, before any request hits usageDb.
await import("@/lib/usage/migrations");
const { initConsoleInterceptor } = await import("@/lib/consoleInterceptor");
initConsoleInterceptor();
@@ -127,7 +130,14 @@ export async function registerNodejs(): Promise<void> {
console.log("[COMPLIANCE] Audit log table initialized");
const cleanup = cleanupExpiredLogs();
if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) {
if (
cleanup.deletedUsage ||
cleanup.deletedCallLogs ||
cleanup.deletedProxyLogs ||
cleanup.deletedRequestDetailLogs ||
cleanup.deletedAuditLogs ||
cleanup.deletedMcpAuditLogs
) {
console.log("[COMPLIANCE] Expired log cleanup:", cleanup);
}
} catch (err: unknown) {

View File

@@ -2,7 +2,7 @@
* Compliance Controls — T-43
*
* Implements compliance features:
* - LOG_RETENTION_DAYS: automatic log cleanup
* - APP_LOG_RETENTION_DAYS / CALL_LOG_RETENTION_DAYS: automatic log cleanup
* - noLog opt-out per API key
* - audit_log table for administrative actions
*
@@ -10,6 +10,7 @@
*/
import { getDbInstance } from "../db/core";
import { getAppLogRetentionDays, getCallLogRetentionDays } from "../logEnv";
/** @returns {import("better-sqlite3").Database | null} */
function getDb() {
@@ -20,8 +21,6 @@ function getDb() {
}
}
const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "90", 10);
/**
* Initialize the audit_log table.
*/
@@ -212,60 +211,122 @@ export function isNoLog(apiKeyId: string) {
// ─── Log Retention / Cleanup ────────────────
/**
* Get the configured retention period.
* @returns {number} Days
* Get the configured retention periods.
*/
export function getRetentionDays() {
return LOG_RETENTION_DAYS;
return {
app: getAppLogRetentionDays(),
call: getCallLogRetentionDays(),
};
}
/**
* Clean up logs older than LOG_RETENTION_DAYS.
* Clean up logs using split APP/CALL retention windows.
* Should be called periodically (e.g. daily cron or on startup).
*
* @returns {{ deletedUsage: number, deletedCallLogs: number, deletedAuditLogs: number }}
* @returns {{
* deletedUsage: number,
* deletedCallLogs: number,
* deletedProxyLogs: number,
* deletedRequestDetailLogs: number,
* deletedAuditLogs: number,
* deletedMcpAuditLogs: number,
* appRetentionDays: number,
* callRetentionDays: number
* }}
*/
export function cleanupExpiredLogs() {
const db = getDb();
if (!db) return { deletedUsage: 0, deletedCallLogs: 0, deletedAuditLogs: 0 };
const appRetentionDays = getAppLogRetentionDays();
const callRetentionDays = getCallLogRetentionDays();
const cutoff = new Date(Date.now() - LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000).toISOString();
if (!db) {
return {
deletedUsage: 0,
deletedCallLogs: 0,
deletedProxyLogs: 0,
deletedRequestDetailLogs: 0,
deletedAuditLogs: 0,
deletedMcpAuditLogs: 0,
appRetentionDays,
callRetentionDays,
};
}
const callCutoff = new Date(Date.now() - callRetentionDays * 24 * 60 * 60 * 1000).toISOString();
const appCutoff = new Date(Date.now() - appRetentionDays * 24 * 60 * 60 * 1000).toISOString();
let deletedUsage = 0;
let deletedCallLogs = 0;
let deletedProxyLogs = 0;
let deletedRequestDetailLogs = 0;
let deletedAuditLogs = 0;
let deletedMcpAuditLogs = 0;
try {
// Clean usage_history
const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(cutoff);
const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(callCutoff);
deletedUsage = r1.changes;
} catch {
/* table may not exist */
}
try {
// Clean call_logs
const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(cutoff);
const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(callCutoff);
deletedCallLogs = r2.changes;
} catch {
/* table may not exist */
}
try {
// Clean audit_log (keep longer, 2x retention)
const auditCutoff = new Date(
Date.now() - LOG_RETENTION_DAYS * 2 * 24 * 60 * 60 * 1000
).toISOString();
const r3 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(auditCutoff);
deletedAuditLogs = r3.changes;
const r3 = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?").run(callCutoff);
deletedProxyLogs = r3.changes;
} catch {
/* table may not exist */
}
try {
const r4 = db.prepare("DELETE FROM request_detail_logs WHERE timestamp < ?").run(callCutoff);
deletedRequestDetailLogs = r4.changes;
} catch {
/* legacy table may not exist */
}
try {
const r5 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(appCutoff);
deletedAuditLogs = r5.changes;
} catch {
/* table may not exist */
}
try {
const r6 = db.prepare("DELETE FROM mcp_tool_audit WHERE created_at < ?").run(appCutoff);
deletedMcpAuditLogs = r6.changes;
} catch {
/* table may not exist */
}
logAuditEvent({
action: "compliance.cleanup",
details: { deletedUsage, deletedCallLogs, deletedAuditLogs, retentionDays: LOG_RETENTION_DAYS },
details: {
deletedUsage,
deletedCallLogs,
deletedProxyLogs,
deletedRequestDetailLogs,
deletedAuditLogs,
deletedMcpAuditLogs,
appRetentionDays,
callRetentionDays,
},
});
return { deletedUsage, deletedCallLogs, deletedAuditLogs };
return {
deletedUsage,
deletedCallLogs,
deletedProxyLogs,
deletedRequestDetailLogs,
deletedAuditLogs,
deletedMcpAuditLogs,
appRetentionDays,
callRetentionDays,
};
}

View File

@@ -12,9 +12,10 @@
import { appendFileSync, existsSync, mkdirSync } from "fs";
import { dirname, resolve } from "path";
import { getAppLogFilePath, getAppLogToFile } from "./logEnv";
const logToFile = process.env.LOG_TO_FILE !== "false";
const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log");
const logToFile = getAppLogToFile();
const logFilePath = resolve(getAppLogFilePath());
declare global {
var __omnirouteConsoleInterceptorInit: boolean | undefined;

View File

@@ -173,7 +173,9 @@ const SCHEMA_SQL = `
combo_name TEXT,
request_body TEXT,
response_body TEXT,
error TEXT
error TEXT,
artifact_relpath TEXT,
has_pipeline_details INTEGER DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp);
CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status);
@@ -376,6 +378,27 @@ function ensureUsageHistoryColumns(db: SqliteDatabase) {
}
}
function ensureCallLogsColumns(db: SqliteDatabase) {
try {
const columns = db.prepare("PRAGMA table_info(call_logs)").all() as Array<{
name?: string;
}>;
const columnNames = new Set(columns.map((column) => String(column.name ?? "")));
if (!columnNames.has("artifact_relpath")) {
db.exec("ALTER TABLE call_logs ADD COLUMN artifact_relpath TEXT");
console.log("[DB] Added call_logs.artifact_relpath column");
}
if (!columnNames.has("has_pipeline_details")) {
db.exec("ALTER TABLE call_logs ADD COLUMN has_pipeline_details INTEGER DEFAULT 0");
console.log("[DB] Added call_logs.has_pipeline_details column");
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : String(error);
console.warn("[DB] Failed to verify call_logs schema:", message);
}
}
export function getDbInstance(): SqliteDatabase {
const existing = getDb();
if (existing) return existing;
@@ -468,6 +491,7 @@ export function getDbInstance(): SqliteDatabase {
db.exec(SCHEMA_SQL);
ensureProviderConnectionsColumns(db);
ensureUsageHistoryColumns(db);
ensureCallLogsColumns(db);
// ── Versioned Migrations ──
// Auto-seed 001 as applied (the inline SCHEMA_SQL already created these tables)

View File

@@ -1,9 +1,9 @@
/**
* Detailed Request Logs DB Layer (#378)
*
* Saves full request/response bodies at each pipeline stage.
* Ring-buffer of 500 entries enforced by SQL trigger in migration 006.
* Only active when settings.detailed_logs_enabled = "1".
* Legacy compatibility layer for detailed request logs.
* New requests now store pipeline details inside unified call log artifacts.
* This module remains available for reading historical request_detail_logs rows.
*/
import { v4 as uuidv4 } from "uuid";
import { getDbInstance } from "./core";
@@ -37,7 +37,7 @@ export interface RequestDetailLog {
export async function isDetailedLoggingEnabled(): Promise<boolean> {
try {
const settings = await getSettings();
const val = settings.detailed_logs_enabled;
const val = settings.call_log_pipeline_enabled;
return val === true || val === "1" || val === "true";
} catch {
return false;

View File

@@ -109,8 +109,8 @@ CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider);
CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model);
CREATE TABLE IF NOT EXISTS call_logs (
id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
method TEXT,
path TEXT,
status INTEGER,
@@ -124,11 +124,13 @@ CREATE TABLE IF NOT EXISTS call_logs (
source_format TEXT,
target_format TEXT,
api_key_id TEXT,
api_key_name TEXT,
combo_name TEXT,
request_body TEXT,
response_body TEXT,
error TEXT
api_key_name TEXT,
combo_name TEXT,
request_body TEXT,
response_body TEXT,
error TEXT,
artifact_relpath TEXT,
has_pipeline_details INTEGER DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp);
CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status);

View File

@@ -0,0 +1,15 @@
-- 013_unified_log_artifacts.sql
-- Switch request logging to unified single-file artifacts and prefixed settings.
INSERT OR REPLACE INTO key_value (namespace, key, value)
VALUES (
'settings',
'call_log_pipeline_enabled',
COALESCE(
(SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'detailed_logs_enabled'),
'false'
)
);
DELETE FROM key_value
WHERE namespace = 'settings' AND key IN ('detailed_logs_enabled', 'maxCallLogs', 'MAX_CALL_LOGS');

61
src/lib/logEnv.ts Normal file
View File

@@ -0,0 +1,61 @@
import path from "path";
const DEFAULT_APP_LOG_RETENTION_DAYS = 7;
const DEFAULT_CALL_LOG_RETENTION_DAYS = 7;
const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024;
const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log");
function parsePositiveInt(value: string | undefined, fallback: number): number {
if (!value) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}
export function parseFileSize(raw: string | undefined): number {
if (!raw) return DEFAULT_APP_LOG_MAX_SIZE;
const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i);
if (!match) return DEFAULT_APP_LOG_MAX_SIZE;
const num = parseInt(match[1], 10);
const unit = (match[2] || "").toLowerCase();
switch (unit) {
case "k":
case "kb":
return num * 1024;
case "m":
case "mb":
return num * 1024 * 1024;
case "g":
case "gb":
return num * 1024 * 1024 * 1024;
default:
return num;
}
}
export function getAppLogToFile(): boolean {
return process.env.APP_LOG_TO_FILE !== "false";
}
export function getAppLogFilePath(): string {
return process.env.APP_LOG_FILE_PATH || DEFAULT_APP_LOG_PATH;
}
export function getAppLogMaxFileSize(): number {
return parseFileSize(process.env.APP_LOG_MAX_FILE_SIZE);
}
export function getAppLogRetentionDays(): number {
return parsePositiveInt(process.env.APP_LOG_RETENTION_DAYS, DEFAULT_APP_LOG_RETENTION_DAYS);
}
export function getCallLogRetentionDays(): number {
return parsePositiveInt(process.env.CALL_LOG_RETENTION_DAYS, DEFAULT_CALL_LOG_RETENTION_DAYS);
}
export function getAppLogLevel(defaultLevel: string): string {
return process.env.APP_LOG_LEVEL || defaultLevel;
}
export function getAppLogFormat(defaultFormat: string): string {
return process.env.APP_LOG_FORMAT || defaultFormat;
}

View File

@@ -7,48 +7,26 @@
* - Creating the log directory on startup
*
* Configuration via env vars:
* - LOG_TO_FILE: enable file logging (default: true)
* - LOG_FILE_PATH: path to log file (default: logs/application/app.log)
* - LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB)
* - LOG_RETENTION_DAYS: days to keep old logs (default: 7)
* - APP_LOG_TO_FILE: enable file logging (default: true)
* - APP_LOG_FILE_PATH: path to log file (default: logs/application/app.log)
* - APP_LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB)
* - APP_LOG_RETENTION_DAYS: days to keep old logs (default: 7)
*/
import { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync } from "fs";
import { dirname, join, basename, extname } from "path";
const DEFAULT_LOG_PATH = "logs/application/app.log";
const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50MB
const DEFAULT_RETENTION_DAYS = 7;
function parseFileSize(raw: string | undefined): number {
if (!raw) return DEFAULT_MAX_SIZE;
const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i);
if (!match) return DEFAULT_MAX_SIZE;
const num = parseInt(match[1], 10);
const unit = (match[2] || "").toLowerCase();
switch (unit) {
case "k":
case "kb":
return num * 1024;
case "m":
case "mb":
return num * 1024 * 1024;
case "g":
case "gb":
return num * 1024 * 1024 * 1024;
default:
return num;
}
}
import {
getAppLogFilePath,
getAppLogMaxFileSize,
getAppLogRetentionDays,
getAppLogToFile,
} from "./logEnv";
export function getLogConfig() {
const logToFile = process.env.LOG_TO_FILE !== "false";
const logFilePath = process.env.LOG_FILE_PATH || join(process.cwd(), DEFAULT_LOG_PATH);
const maxFileSize = parseFileSize(process.env.LOG_MAX_FILE_SIZE);
const retentionDays = parseInt(
process.env.LOG_RETENTION_DAYS || String(DEFAULT_RETENTION_DAYS),
10
);
const logToFile = getAppLogToFile();
const logFilePath = getAppLogFilePath() || join(process.cwd(), "logs/application/app.log");
const maxFileSize = getAppLogMaxFileSize();
const retentionDays = getAppLogRetentionDays();
return { logToFile, logFilePath, maxFileSize, retentionDays };
}

View File

@@ -11,7 +11,7 @@ import { getDbInstance, isCloud, isBuildPhase } from "./db/core";
const shouldPersistToDisk = !isCloud && !isBuildPhase;
const MAX_ENTRIES = parseInt(process.env.PROXY_LOG_MAX_ENTRIES || "200", 10);
const MAX_IN_MEMORY_ENTRIES = 200;
interface ProxyInfo {
type: string;
@@ -56,7 +56,7 @@ function loadFromDb() {
const db = getDbInstance();
const rows = db
.prepare("SELECT * FROM proxy_logs ORDER BY timestamp DESC LIMIT ?")
.all(MAX_ENTRIES) as any[];
.all(MAX_IN_MEMORY_ENTRIES) as any[];
for (const row of rows) {
proxyLogs.push({
@@ -113,8 +113,8 @@ export function logProxyEvent(entry: Partial<ProxyLogEntry>) {
// 1. In-memory ring buffer (newest first)
proxyLogs.unshift(log);
if (proxyLogs.length > MAX_ENTRIES) {
proxyLogs.length = MAX_ENTRIES;
if (proxyLogs.length > MAX_IN_MEMORY_ENTRIES) {
proxyLogs.length = MAX_IN_MEMORY_ENTRIES;
}
// 2. Persist to SQLite
@@ -147,16 +147,6 @@ export function logProxyEvent(entry: Partial<ProxyLogEntry>) {
account: log.account,
tlsFingerprint: log.tlsFingerprint ? 1 : 0,
});
// Trim old entries
const count = (db.prepare("SELECT COUNT(*) as cnt FROM proxy_logs").get() as any)?.cnt || 0;
if (count > MAX_ENTRIES) {
db.prepare(
`DELETE FROM proxy_logs WHERE id IN (
SELECT id FROM proxy_logs ORDER BY timestamp ASC LIMIT ?
)`
).run(count - MAX_ENTRIES);
}
} catch (err: any) {
console.warn("[proxyLogger] Failed to persist:", err.message);
}

View File

@@ -2,24 +2,57 @@
* Call Logs — extracted from usageDb.js (T-15)
*
* Structured call log management: save, query, rotate, and
* full-payload disk storage for the Logger UI.
* unified single-artifact disk storage for the Logger UI.
*
* @module lib/usage/callLogs
*/
import path from "path";
import fs from "fs";
import path from "path";
import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts";
import { getDbInstance } from "../db/core";
import { getSettings } from "../db/settings";
import { getRequestDetailLogByCallLogId } from "../db/detailedLogs";
import { shouldPersistToDisk, CALL_LOGS_DIR } from "./migrations";
import { getLoggedInputTokens, getLoggedOutputTokens } from "./tokenAccounting";
import { isNoLog } from "../compliance";
import { sanitizePII } from "../piiSanitizer";
import { protectPayloadForLog, parseStoredPayload } from "../logPayloads";
import {
protectPayloadForLog,
parseStoredPayload,
serializePayloadForStorage,
} from "../logPayloads";
import { getCallLogRetentionDays } from "../logEnv";
type JsonRecord = Record<string, unknown>;
type CallLogArtifact = {
schemaVersion: 2;
summary: {
id: string;
timestamp: string;
method: string;
path: string;
status: number;
model: string;
requestedModel: string | null;
provider: string;
account: string;
connectionId: string | null;
duration: number;
tokens: { in: number; out: number };
requestType: string | null;
sourceFormat: string | null;
targetFormat: string | null;
apiKeyId: string | null;
apiKeyName: string | null;
comboName: string | null;
};
requestBody: unknown;
responseBody: unknown;
error: unknown;
pipeline?: RequestPipelinePayloads;
};
function asRecord(value: unknown): JsonRecord {
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
}
@@ -34,7 +67,7 @@ function toNumber(value: unknown): number {
}
function toStringOrNull(value: unknown): string | null {
return typeof value === "string" ? value : null;
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
function hasTruncatedFlag(value: unknown): boolean {
@@ -42,74 +75,224 @@ function hasTruncatedFlag(value: unknown): boolean {
return (value as Record<string, unknown>)._truncated === true;
}
const DEFAULT_MAX_CALL_LOGS = 10000;
const CALL_LOGS_MAX_CACHE_TTL_MS = 30_000;
const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "7", 10);
const CALL_LOG_PAYLOAD_MODE = (() => {
const value = (process.env.CALL_LOG_PAYLOAD_MODE || "full").toLowerCase();
return value === "full" || value === "metadata" || value === "none" ? value : "full";
})();
const shouldLogPayloadInDb = CALL_LOG_PAYLOAD_MODE !== "none";
const shouldLogPayloadOnDisk = CALL_LOG_PAYLOAD_MODE === "full";
let callLogsMaxCache = {
value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS,
expiresAt: 0,
};
function resolveCallLogsMaxValue(value: unknown): number | null {
if (typeof value === "number" && Number.isInteger(value) && value > 0) return value;
if (typeof value === "string" && value.trim().length > 0) {
const parsed = Number.parseInt(value, 10);
return Number.isInteger(parsed) && parsed > 0 ? parsed : null;
function sanitizeErrorForLog(error: unknown): unknown {
if (error === null || error === undefined) return null;
if (typeof error === "string") return sanitizePII(error).text;
if (error instanceof Error) {
return {
message: sanitizePII(error.message).text,
stack: sanitizePII(error.stack || "").text || undefined,
name: error.name,
};
}
return null;
return protectPayloadForLog(error);
}
async function getMaxCallLogs(): Promise<number> {
const now = Date.now();
if (callLogsMaxCache.expiresAt > now) {
return callLogsMaxCache.value;
}
let value = resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS;
function toStoredErrorString(error: unknown): string | null {
const sanitized = sanitizeErrorForLog(error);
if (sanitized === null || sanitized === undefined) return null;
if (typeof sanitized === "string") return sanitized;
try {
const { getSettings } = await import("@/lib/localDb");
const settings = await getSettings();
const configured =
resolveCallLogsMaxValue(settings.maxCallLogs) ??
resolveCallLogsMaxValue(settings.MAX_CALL_LOGS);
if (configured !== null) {
value = configured;
}
return JSON.stringify(sanitized);
} catch {
// Fall back to env/default cap when settings are unavailable.
return String(sanitized);
}
}
function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | null {
if (!payloads || typeof payloads !== "object") return null;
const protectedPayloads: RequestPipelinePayloads = {};
for (const [key, value] of Object.entries(payloads as JsonRecord)) {
if (value === null || value === undefined) {
continue;
}
if (key === "streamChunks" && value && typeof value === "object") {
const chunks = value as Record<string, unknown>;
const compacted = Object.fromEntries(
Object.entries(chunks).filter(
([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0
)
);
if (Object.keys(compacted).length > 0) {
protectedPayloads.streamChunks = protectPayloadForLog(
compacted
) as RequestPipelinePayloads["streamChunks"];
}
continue;
}
protectedPayloads[key as keyof RequestPipelinePayloads] = protectPayloadForLog(value) as never;
}
callLogsMaxCache = {
value,
expiresAt: now + CALL_LOGS_MAX_CACHE_TTL_MS,
};
return value;
return Object.keys(protectedPayloads).length > 0 ? protectedPayloads : null;
}
export function invalidateCallLogsMaxCache(): void {
callLogsMaxCache = {
value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS,
expiresAt: 0,
};
}
let logIdCounter = 0;
function generateLogId() {
logIdCounter++;
return `${Date.now()}-${logIdCounter}`;
}
/**
* Save a structured call log entry.
*/
async function resolveAccountName(connectionId: string | null | undefined) {
let account = connectionId ? connectionId.slice(0, 8) : "-";
if (!connectionId) {
return account;
}
try {
const { getProviderConnections } = await import("@/lib/localDb");
const connections = await getProviderConnections();
const conn = connections.find((item) => item.id === connectionId);
if (conn) {
account = conn.name || conn.email || account;
}
} catch {
// Best-effort lookup only.
}
return account;
}
function buildArtifactRelativePath(timestamp: string, id: string) {
const parsed = new Date(timestamp);
const safeTimestamp = (
Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString()
).replace(/[:]/g, "-");
const dateFolder = safeTimestamp.slice(0, 10);
return path.posix.join(dateFolder, `${safeTimestamp}_${id}.json`);
}
function buildArtifact(
logEntry: {
id: string;
timestamp: string;
method: string;
path: string;
status: number;
model: string;
requestedModel: string | null;
provider: string;
account: string;
connectionId: string | null;
duration: number;
tokensIn: number;
tokensOut: number;
requestType: string | null;
sourceFormat: string | null;
targetFormat: string | null;
apiKeyId: string | null;
apiKeyName: string | null;
comboName: string | null;
},
requestBody: unknown,
responseBody: unknown,
error: unknown,
pipelinePayloads: RequestPipelinePayloads | null
): CallLogArtifact {
return {
schemaVersion: 2,
summary: {
id: logEntry.id,
timestamp: logEntry.timestamp,
method: logEntry.method,
path: logEntry.path,
status: logEntry.status,
model: logEntry.model,
requestedModel: logEntry.requestedModel,
provider: logEntry.provider,
account: logEntry.account,
connectionId: logEntry.connectionId,
duration: logEntry.duration,
tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut },
requestType: logEntry.requestType,
sourceFormat: logEntry.sourceFormat,
targetFormat: logEntry.targetFormat,
apiKeyId: logEntry.apiKeyId,
apiKeyName: logEntry.apiKeyName,
comboName: logEntry.comboName,
},
requestBody: requestBody ?? null,
responseBody: responseBody ?? null,
error: error ?? null,
...(pipelinePayloads ? { pipeline: pipelinePayloads } : {}),
};
}
function writeCallArtifact(artifact: CallLogArtifact): string | null {
if (!CALL_LOGS_DIR) return null;
const relPath = buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id);
const absPath = path.join(CALL_LOGS_DIR, relPath);
try {
fs.mkdirSync(path.dirname(absPath), { recursive: true });
fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2));
return relPath;
} catch (error) {
console.error("[callLogs] Failed to write request artifact:", (error as Error).message);
return null;
}
}
function readArtifactFromDisk(relativePath: string | null) {
if (!CALL_LOGS_DIR || !relativePath) return null;
try {
const absPath = path.join(CALL_LOGS_DIR, relativePath);
if (!fs.existsSync(absPath)) return null;
return JSON.parse(fs.readFileSync(absPath, "utf8")) as CallLogArtifact;
} catch (error) {
console.error("[callLogs] Failed to read request artifact:", (error as Error).message);
return null;
}
}
function readLegacyLogFromDisk(entry: {
timestamp: string | null;
model: string | null;
status: number;
}) {
if (!CALL_LOGS_DIR || !entry.timestamp) return null;
try {
const date = new Date(entry.timestamp);
if (Number.isNaN(date.getTime())) return null;
const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(
2,
"0"
)}-${String(date.getDate()).padStart(2, "0")}`;
const dir = path.join(CALL_LOGS_DIR, dateFolder);
if (!fs.existsSync(dir)) return null;
const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart(
2,
"0"
)}${String(date.getSeconds()).padStart(2, "0")}`;
const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-");
const expectedName = `${time}_${safeModel}_${entry.status}.json`;
const exactPath = path.join(dir, expectedName);
if (fs.existsSync(exactPath)) {
return JSON.parse(fs.readFileSync(exactPath, "utf8"));
}
const files = fs
.readdirSync(dir)
.filter((file) => file.startsWith(time) && file.endsWith(`_${entry.status}.json`));
if (files.length > 0) {
return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8"));
}
} catch (error) {
console.error("[callLogs] Failed to read legacy disk log:", (error as Error).message);
}
return null;
}
export async function saveCallLog(entry: any) {
if (!shouldPersistToDisk) return;
@@ -117,44 +300,23 @@ export async function saveCallLog(entry: any) {
const apiKeyId = entry.apiKeyId || null;
const noLogEnabled = Boolean(entry.noLog) || (apiKeyId ? isNoLog(apiKeyId) : false);
const protectedRequestBody =
noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.requestBody);
const protectedResponseBody =
noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.responseBody);
const protectedRequestBody = noLogEnabled ? null : protectPayloadForLog(entry.requestBody);
const protectedResponseBody = noLogEnabled ? null : protectPayloadForLog(entry.responseBody);
const protectedPipelinePayloads = noLogEnabled
? null
: protectPipelinePayloads(entry.pipelinePayloads ?? entry.pipeline ?? null);
const protectedError = sanitizeErrorForLog(entry.error);
// Resolve account name
let account = entry.connectionId ? entry.connectionId.slice(0, 8) : "-";
try {
const { getProviderConnections } = await import("@/lib/localDb");
const connections = await getProviderConnections();
const conn = connections.find((c) => c.id === entry.connectionId);
if (conn) account = conn.name || conn.email || account;
} catch {}
// Truncate large payloads for DB storage (keep under 8KB each)
const truncatePayload = (obj: any) => {
if (!obj) return null;
const str = JSON.stringify(obj);
if (str.length <= 8192) return str;
try {
return JSON.stringify({
_truncated: true,
_originalSize: str.length,
_preview: str.slice(0, 8192) + "...",
});
} catch {
return JSON.stringify({ _truncated: true });
}
};
const account = await resolveAccountName(entry.connectionId || null);
const logEntry = {
id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(),
timestamp: new Date().toISOString(),
timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(),
method: entry.method || "POST",
path: entry.path || "/v1/chat/completions",
status: entry.status || 0,
model: entry.model || "-",
requestedModel: entry.requestedModel || null, // T01: model the client asked for
requestedModel: entry.requestedModel || null,
provider: entry.provider || "-",
account,
connectionId: entry.connectionId || null,
@@ -167,119 +329,82 @@ export async function saveCallLog(entry: any) {
apiKeyId,
apiKeyName: entry.apiKeyName || null,
comboName: entry.comboName || null,
requestBody: truncatePayload(protectedRequestBody),
responseBody: truncatePayload(protectedResponseBody),
error: typeof entry.error === "string" ? sanitizePII(entry.error).text : entry.error || null,
requestBody: serializePayloadForStorage(protectedRequestBody, 8192),
responseBody: serializePayloadForStorage(protectedResponseBody, 8192),
error: toStoredErrorString(protectedError),
};
// 1. Insert into SQLite
const db = getDbInstance();
db.prepare(
`
INSERT INTO call_logs (id, timestamp, method, path, status, model, requested_model, provider,
account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, target_format,
api_key_id, api_key_name, combo_name, request_body, response_body, error)
VALUES (@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider,
@account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, @targetFormat,
@apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error)
INSERT INTO call_logs (
id, timestamp, method, path, status, model, requested_model, provider,
account, connection_id, duration, tokens_in, tokens_out, request_type, source_format,
target_format, api_key_id, api_key_name, combo_name, request_body, response_body, error,
artifact_relpath, has_pipeline_details
)
VALUES (
@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider,
@account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat,
@targetFormat, @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error,
NULL, 0
)
`
).run(logEntry);
// 2. Trim old entries beyond max
const maxLogs = await getMaxCallLogs();
const countRow = asRecord(db.prepare("SELECT COUNT(*) as cnt FROM call_logs").get());
const count = toNumber(countRow.cnt);
if (count > maxLogs) {
db.prepare(
`
DELETE FROM call_logs WHERE id IN (
SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ?
)
`
).run(count - maxLogs);
}
// 3. Write full payload to disk file (untruncated)
// Disabled when no-log is active or payload mode is metadata/none.
if (
shouldLogPayloadOnDisk &&
!noLogEnabled &&
(protectedRequestBody !== null || protectedResponseBody !== null)
) {
writeCallLogToDisk(
{ ...logEntry, tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut } },
if (!noLogEnabled) {
const artifact = buildArtifact(
logEntry,
protectedRequestBody,
protectedResponseBody
protectedResponseBody,
protectedError,
protectedPipelinePayloads
);
const artifactRelPath = writeCallArtifact(artifact);
if (artifactRelPath) {
db.prepare(
`
UPDATE call_logs
SET artifact_relpath = ?, has_pipeline_details = ?
WHERE id = ?
`
).run(artifactRelPath, protectedPipelinePayloads ? 1 : 0, logEntry.id);
}
}
} catch (error: any) {
console.error("[callLogs] Failed to save call log:", error.message);
} catch (error) {
console.error("[callLogs] Failed to save call log:", (error as Error).message);
}
}
/**
* Write call log as JSON file to disk (full payloads, not truncated).
*/
function writeCallLogToDisk(logEntry: any, requestBody: any, responseBody: any) {
if (!CALL_LOGS_DIR) return;
try {
const now = new Date();
const dateFolder = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}`;
const dir = path.join(CALL_LOGS_DIR, dateFolder);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
const safeModel = (logEntry.model || "unknown").replace(/[/:]/g, "-");
const time = `${String(now.getHours()).padStart(2, "0")}${String(now.getMinutes()).padStart(2, "0")}${String(now.getSeconds()).padStart(2, "0")}`;
const filename = `${time}_${safeModel}_${logEntry.status}.json`;
const fullEntry = {
...logEntry,
requestBody: requestBody || null,
responseBody: responseBody || null,
};
fs.writeFileSync(path.join(dir, filename), JSON.stringify(fullEntry, null, 2));
} catch (err: any) {
console.error("[callLogs] Failed to write disk log:", err.message);
}
}
/**
* Rotate old call log directories (keep last 7 days).
*/
export function rotateCallLogs() {
if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return;
try {
const entries = fs.readdirSync(CALL_LOGS_DIR);
const now = Date.now();
const retentionMs = LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000;
const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000;
for (const entry of entries) {
const entryPath = path.join(CALL_LOGS_DIR, entry);
const stat = fs.statSync(entryPath);
if (stat.isDirectory() && now - stat.mtimeMs > retentionMs) {
fs.rmSync(entryPath, { recursive: true, force: true });
console.log(`[callLogs] Rotated old logs: ${entry}`);
}
}
} catch (err: any) {
console.error("[callLogs] Failed to rotate logs:", err.message);
} catch (error) {
console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
}
}
// Run rotation on startup
if (shouldPersistToDisk) {
try {
rotateCallLogs();
} catch {}
} catch {
// Best-effort startup cleanup.
}
}
/**
* Get call logs with optional filtering.
*/
export async function getCallLogs(filter: any = {}) {
const db = getDbInstance();
let sql = "SELECT * FROM call_logs";
@@ -292,8 +417,8 @@ export async function getCallLogs(filter: any = {}) {
} else if (filter.status === "ok") {
conditions.push("status >= 200 AND status < 300");
} else {
const statusCode = parseInt(filter.status);
if (!isNaN(statusCode)) {
const statusCode = parseInt(filter.status, 10);
if (!Number.isNaN(statusCode)) {
conditions.push("status = @statusCode");
params.statusCode = statusCode;
}
@@ -347,7 +472,7 @@ export async function getCallLogs(filter: any = {}) {
path: toStringOrNull(l.path),
status: toNumber(l.status),
model: toStringOrNull(l.model),
requestedModel: toStringOrNull(l.requested_model), // T01: original model from client
requestedModel: toStringOrNull(l.requested_model),
provider: toStringOrNull(l.provider),
account: toStringOrNull(l.account),
duration: toNumber(l.duration),
@@ -360,19 +485,30 @@ export async function getCallLogs(filter: any = {}) {
apiKeyName: toStringOrNull(l.api_key_name),
hasRequestBody: typeof l.request_body === "string" && l.request_body.length > 0,
hasResponseBody: typeof l.response_body === "string" && l.response_body.length > 0,
hasPipelineDetails: toNumber(l.has_pipeline_details) === 1,
};
});
}
/**
* Get a single call log by ID (with full payloads from disk when available).
*/
function buildLegacyPipelinePayloads(id: string) {
const detailed = getRequestDetailLogByCallLogId(id);
if (!detailed) return null;
return {
clientRequest: detailed.client_request ?? null,
providerRequest: detailed.translated_request ?? null,
providerResponse: detailed.provider_response ?? null,
clientResponse: detailed.client_response ?? null,
};
}
export async function getCallLogById(id: string) {
const db = getDbInstance();
const row = db.prepare("SELECT * FROM call_logs WHERE id = ?").get(id);
if (!row) return null;
const entryRow = asRecord(row);
const entryRow = asRecord(row);
const artifactRelPath = toStringOrNull(entryRow.artifact_relpath);
const entry = {
id: toStringOrNull(entryRow.id),
timestamp: toStringOrNull(entryRow.timestamp),
@@ -394,72 +530,47 @@ export async function getCallLogById(id: string) {
requestBody: parseStoredPayload(entryRow.request_body),
responseBody: parseStoredPayload(entryRow.response_body),
error: toStringOrNull(entryRow.error),
artifactRelPath,
hasPipelineDetails: toNumber(entryRow.has_pipeline_details) === 1,
};
// If payloads were truncated, try to read full version from disk
const needsDisk = hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody);
if (needsDisk && CALL_LOGS_DIR) {
try {
const diskEntry = readFullLogFromDisk(entry);
if (diskEntry) {
return {
...entry,
requestBody: diskEntry.requestBody ?? entry.requestBody,
responseBody: diskEntry.responseBody ?? entry.responseBody,
};
}
} catch (err: any) {
console.error("[callLogs] Failed to read full log from disk:", err.message);
const artifact = readArtifactFromDisk(artifactRelPath);
if (artifact) {
return {
...entry,
requestBody: artifact.requestBody ?? entry.requestBody,
responseBody: artifact.responseBody ?? entry.responseBody,
error: artifact.error ?? entry.error,
pipelinePayloads: artifact.pipeline ?? null,
hasPipelineDetails: Boolean(artifact.pipeline) || entry.hasPipelineDetails,
};
}
const needsLegacyDisk =
hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody) || !artifactRelPath;
if (needsLegacyDisk) {
const legacyEntry = readLegacyLogFromDisk(entry);
if (legacyEntry) {
const legacyPipeline = buildLegacyPipelinePayloads(id);
return {
...entry,
requestBody: legacyEntry.requestBody ?? entry.requestBody,
responseBody: legacyEntry.responseBody ?? entry.responseBody,
error: legacyEntry.error ?? entry.error,
pipelinePayloads: legacyPipeline,
hasPipelineDetails: Boolean(legacyPipeline),
};
}
}
const detailed = getRequestDetailLogByCallLogId(id);
if (!detailed) {
return entry;
const legacyPipeline = buildLegacyPipelinePayloads(id);
if (legacyPipeline) {
return {
...entry,
pipelinePayloads: legacyPipeline,
hasPipelineDetails: true,
};
}
return {
...entry,
pipelinePayloads: {
clientRequest: detailed.client_request ?? null,
providerRequest: detailed.translated_request ?? null,
providerResponse: detailed.provider_response ?? null,
clientResponse: detailed.client_response ?? null,
},
};
}
/**
* Read the full (untruncated) log entry from disk.
*/
function readFullLogFromDisk(entry: any) {
if (!CALL_LOGS_DIR || !entry.timestamp) return null;
try {
const date = new Date(entry.timestamp);
const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`;
const dir = path.join(CALL_LOGS_DIR, dateFolder);
if (!fs.existsSync(dir)) return null;
const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart(2, "0")}${String(date.getSeconds()).padStart(2, "0")}`;
const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-");
const expectedName = `${time}_${safeModel}_${entry.status}.json`;
const exactPath = path.join(dir, expectedName);
if (fs.existsSync(exactPath)) {
return JSON.parse(fs.readFileSync(exactPath, "utf8"));
}
const files = fs
.readdirSync(dir)
.filter((f) => f.startsWith(time) && f.endsWith(`_${entry.status}.json`));
if (files.length > 0) {
return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8"));
}
} catch (err: any) {
console.error("[callLogs] Disk log read error:", err.message);
}
return null;
return entry;
}

View File

@@ -1,42 +1,47 @@
/**
* Usage Migrations — extracted from usageDb.js (T-15)
*
* Handles legacy file migration (.data → data/) and JSON → SQLite migration.
* Runs automatically on module load when shouldPersistToDisk is true.
* Handles legacy file migration (.data → data/), JSON → SQLite migration,
* and one-time archival of legacy request log layouts into a zip backup.
*
* @module lib/usage/migrations
*/
import path from "path";
import fs from "fs";
import path from "path";
import { ZipFile } from "yazl";
import { getDbInstance, isCloud, isBuildPhase, DATA_DIR } from "../db/core";
import { getLegacyDotDataDir, isSamePath } from "../dataPaths";
export const shouldPersistToDisk = !isCloud && !isBuildPhase;
// ──────────────── File Paths ────────────────
const LEGACY_DATA_DIR = isCloud ? null : getLegacyDotDataDir();
export const LOG_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt");
export const CALL_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "call_logs");
export const LOG_ARCHIVES_DIR = isCloud ? null : path.join(DATA_DIR, "log_archives");
const LEGACY_LAYOUT_MARKER =
isCloud || !LOG_ARCHIVES_DIR ? null : path.join(LOG_ARCHIVES_DIR, "legacy-request-logs.json");
const CURRENT_REQUEST_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "logs");
const CURRENT_REQUEST_SUMMARY_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt");
// Legacy paths
const LEGACY_DB_FILE =
isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "usage.json");
const LEGACY_LOG_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "log.txt");
const LEGACY_CALL_LOGS_DB_FILE =
isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs.json");
const LEGACY_CALL_LOGS_DIR =
isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs");
// Current-location JSON files (for migration into SQLite)
const USAGE_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "usage.json");
const CALL_LOGS_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "call_logs.json");
// ──────────────── Legacy File Migration ────────────────
type ArchiveTarget = {
sourcePath: string;
archiveRoot: string;
deleteAfterArchive: boolean;
};
function copyIfMissing(fromPath, toPath, label) {
function copyIfMissing(fromPath: string | null, toPath: string | null, label: string) {
if (!fromPath || !toPath) return;
if (!fs.existsSync(fromPath) || fs.existsSync(toPath)) return;
@@ -48,27 +53,185 @@ function copyIfMissing(fromPath, toPath, label) {
console.log(`[usageDb] Migrated ${label}: ${fromPath} -> ${toPath}`);
}
function containsLegacyCallLogLayout(dirPath: string | null): boolean {
if (!dirPath || !fs.existsSync(dirPath)) return false;
try {
const topLevelEntries = fs.readdirSync(dirPath);
for (const topLevelEntry of topLevelEntries) {
const topLevelPath = path.join(dirPath, topLevelEntry);
const stat = fs.statSync(topLevelPath);
if (stat.isFile() && /^\d{6}_.+_\d{3}\.json$/i.test(topLevelEntry)) {
return true;
}
if (!stat.isDirectory()) {
continue;
}
const nestedEntries = fs.readdirSync(topLevelPath);
for (const nestedEntry of nestedEntries) {
if (/^\d{6}_.+_\d{3}\.json$/i.test(nestedEntry)) {
return true;
}
}
}
} catch {
return false;
}
return false;
}
function ensureArchiveDir() {
if (!LOG_ARCHIVES_DIR) return;
fs.mkdirSync(LOG_ARCHIVES_DIR, { recursive: true });
}
function listArchiveTargets(): ArchiveTarget[] {
const targets: ArchiveTarget[] = [];
if (CURRENT_REQUEST_LOGS_DIR && fs.existsSync(CURRENT_REQUEST_LOGS_DIR)) {
targets.push({
sourcePath: CURRENT_REQUEST_LOGS_DIR,
archiveRoot: "data/logs",
deleteAfterArchive: true,
});
}
if (CURRENT_REQUEST_SUMMARY_FILE && fs.existsSync(CURRENT_REQUEST_SUMMARY_FILE)) {
targets.push({
sourcePath: CURRENT_REQUEST_SUMMARY_FILE,
archiveRoot: "data/log.txt",
deleteAfterArchive: true,
});
}
if (CALL_LOGS_DIR && containsLegacyCallLogLayout(CALL_LOGS_DIR)) {
targets.push({
sourcePath: CALL_LOGS_DIR,
archiveRoot: "data/call_logs",
deleteAfterArchive: true,
});
}
return targets;
}
function addPathToZip(zipFile: ZipFile, sourcePath: string, archivePath: string) {
const stat = fs.statSync(sourcePath);
if (stat.isDirectory()) {
const entries = fs.readdirSync(sourcePath);
if (entries.length === 0) {
zipFile.addEmptyDirectory(archivePath);
return;
}
for (const entry of entries) {
addPathToZip(zipFile, path.join(sourcePath, entry), path.posix.join(archivePath, entry));
}
return;
}
zipFile.addFile(sourcePath, archivePath);
}
function createLegacyArchive(targets: ArchiveTarget[]): Promise<string> {
return new Promise((resolve, reject) => {
if (!LOG_ARCHIVES_DIR) {
reject(new Error("LOG_ARCHIVES_DIR is not configured"));
return;
}
ensureArchiveDir();
const timestamp = new Date().toISOString().replace(/[:]/g, "-");
const archiveFilename = `${timestamp}_legacy-request-logs.zip`;
const archivePath = path.join(LOG_ARCHIVES_DIR, archiveFilename);
const zipFile = new ZipFile();
const output = fs.createWriteStream(archivePath);
output.on("close", () => resolve(archiveFilename));
output.on("error", (error) => {
fs.rmSync(archivePath, { force: true });
reject(error);
});
zipFile.outputStream.pipe(output);
try {
for (const target of targets) {
addPathToZip(zipFile, target.sourcePath, target.archiveRoot);
}
zipFile.end();
} catch (error) {
fs.rmSync(archivePath, { force: true });
zipFile.end();
reject(error);
}
});
}
function writeLegacyLayoutMarker(archiveFilename: string) {
if (!LEGACY_LAYOUT_MARKER) return;
ensureArchiveDir();
fs.writeFileSync(
LEGACY_LAYOUT_MARKER,
JSON.stringify(
{
migratedAt: new Date().toISOString(),
archiveFilename,
},
null,
2
)
);
}
function deleteArchivedTargets(targets: ArchiveTarget[]) {
for (const target of targets) {
if (!target.deleteAfterArchive || !fs.existsSync(target.sourcePath)) {
continue;
}
const stat = fs.statSync(target.sourcePath);
if (stat.isDirectory()) {
fs.rmSync(target.sourcePath, { recursive: true, force: true });
} else {
fs.rmSync(target.sourcePath, { force: true });
}
}
}
export function migrateLegacyUsageFiles() {
if (!shouldPersistToDisk || !LEGACY_DATA_DIR) return;
if (isSamePath(DATA_DIR, LEGACY_DATA_DIR)) return;
try {
copyIfMissing(LEGACY_DB_FILE, USAGE_JSON_FILE, "usage history");
copyIfMissing(LEGACY_LOG_FILE, LOG_FILE, "request log");
copyIfMissing(LEGACY_CALL_LOGS_DB_FILE, CALL_LOGS_JSON_FILE, "call log index");
copyIfMissing(LEGACY_CALL_LOGS_DIR, CALL_LOGS_DIR, "call log files");
} catch (error) {
console.error("[usageDb] Legacy migration failed:", error.message);
console.error("[usageDb] Legacy migration failed:", (error as Error).message);
}
}
// ──────────────── JSON → SQLite Migration ────────────────
export async function archiveLegacyRequestLogs() {
if (!shouldPersistToDisk) return null;
if (LEGACY_LAYOUT_MARKER && fs.existsSync(LEGACY_LAYOUT_MARKER)) return null;
const targets = listArchiveTargets();
if (targets.length === 0) return null;
const archiveFilename = await createLegacyArchive(targets);
deleteArchivedTargets(targets);
writeLegacyLayoutMarker(archiveFilename);
console.log(`[usageDb] Archived legacy request logs to ${archiveFilename}`);
return archiveFilename;
}
export function migrateUsageJsonToSqlite() {
if (!shouldPersistToDisk) return;
const db = getDbInstance();
// 1. Migrate usage.json
if (USAGE_JSON_FILE && fs.existsSync(USAGE_JSON_FILE)) {
try {
const raw = fs.readFileSync(USAGE_JSON_FILE, "utf-8");
@@ -118,13 +281,12 @@ export function migrateUsageJsonToSqlite() {
console.log(`[usageDb] ✓ Migrated ${history.length} usage entries`);
}
fs.renameSync(USAGE_JSON_FILE, USAGE_JSON_FILE + ".migrated");
} catch (err) {
console.error("[usageDb] Failed to migrate usage.json:", err.message);
fs.renameSync(USAGE_JSON_FILE, `${USAGE_JSON_FILE}.migrated`);
} catch (error) {
console.error("[usageDb] Failed to migrate usage.json:", (error as Error).message);
}
}
// 2. Migrate call_logs.json
if (CALL_LOGS_JSON_FILE && fs.existsSync(CALL_LOGS_JSON_FILE)) {
try {
const raw = fs.readFileSync(CALL_LOGS_JSON_FILE, "utf-8");
@@ -137,10 +299,12 @@ export function migrateUsageJsonToSqlite() {
const insert = db.prepare(`
INSERT OR IGNORE INTO call_logs (id, timestamp, method, path, status, model, provider,
account, connection_id, duration, tokens_in, tokens_out, source_format, target_format,
api_key_id, api_key_name, combo_name, request_body, response_body, error)
api_key_id, api_key_name, combo_name, request_body, response_body, error,
artifact_relpath, has_pipeline_details)
VALUES (@id, @timestamp, @method, @path, @status, @model, @provider,
@account, @connectionId, @duration, @tokensIn, @tokensOut, @sourceFormat, @targetFormat,
@apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error)
@apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error,
NULL, 0)
`);
const tx = db.transaction(() => {
@@ -173,21 +337,25 @@ export function migrateUsageJsonToSqlite() {
console.log(`[usageDb] ✓ Migrated ${logs.length} call log entries`);
}
fs.renameSync(CALL_LOGS_JSON_FILE, CALL_LOGS_JSON_FILE + ".migrated");
} catch (err) {
console.error("[usageDb] Failed to migrate call_logs.json:", err.message);
fs.renameSync(CALL_LOGS_JSON_FILE, `${CALL_LOGS_JSON_FILE}.migrated`);
} catch (error) {
console.error("[usageDb] Failed to migrate call_logs.json:", (error as Error).message);
}
}
}
// ──────────────── Run on load ────────────────
migrateLegacyUsageFiles();
if (shouldPersistToDisk) {
try {
await archiveLegacyRequestLogs();
} catch (error) {
console.error("[usageDb] Failed to archive legacy request logs:", (error as Error).message);
}
try {
migrateUsageJsonToSqlite();
} catch {
/* ok */
// Best-effort startup migration.
}
}

View File

@@ -378,31 +378,18 @@ export async function getModelLatencyStats(
return stats;
}
// ──────────────── Request Log (log.txt) ────────────────
import fs from "fs";
import { LOG_FILE } from "./migrations";
function formatLogDate(date = new Date()) {
const pad = (n: number) => String(n).padStart(2, "0");
const d = pad(date.getDate());
const m = pad(date.getMonth() + 1);
const y = date.getFullYear();
const h = pad(date.getHours());
const min = pad(date.getMinutes());
const s = pad(date.getSeconds());
return `${d}-${m}-${y} ${h}:${min}:${s}`;
}
// ──────────────── Request Log Compatibility Shim ────────────────
/**
* Append to log.txt.
* Legacy compatibility shim.
* Request summary lines are no longer written to data/log.txt.
*/
export async function appendRequestLog({
model,
provider,
connectionId,
tokens,
status,
model: _model,
provider: _provider,
connectionId: _connectionId,
tokens: _tokens,
status: _status,
}: {
model?: string;
provider?: string;
@@ -410,55 +397,39 @@ export async function appendRequestLog({
tokens?: any;
status?: string | number;
}) {
if (!shouldPersistToDisk) return;
try {
const timestamp = formatLogDate();
const p = provider?.toUpperCase() || "-";
const m = model || "-";
let account = connectionId ? connectionId.slice(0, 8) : "-";
try {
const { getProviderConnections } = await import("@/lib/localDb");
const connections = await getProviderConnections();
const connRaw = connections.find((c) => asRecord(c).id === connectionId);
if (connRaw) {
const conn = asRecord(connRaw);
account = toStringOrNull(conn.name) || toStringOrNull(conn.email) || account;
}
} catch {}
const sent = tokens ? getLoggedInputTokens(tokens) : "-";
const received = tokens ? getLoggedOutputTokens(tokens) : "-";
const line = `${timestamp} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${status}\n`;
fs.appendFileSync(LOG_FILE, line);
const content = fs.readFileSync(LOG_FILE, "utf-8");
const lines = content.trim().split("\n");
if (lines.length > 200) {
fs.writeFileSync(LOG_FILE, lines.slice(-200).join("\n") + "\n");
}
} catch (error: any) {
console.error("Failed to append to log.txt:", error.message);
}
// Deprecated: request summaries now come from SQLite call_logs.
}
/**
* Get last N lines of log.txt.
* Return recent request summaries generated from SQLite call_logs rows.
*/
export async function getRecentLogs(limit = 200) {
if (!shouldPersistToDisk) return [];
if (!fs || typeof fs.existsSync !== "function") return [];
if (!LOG_FILE) return [];
if (!fs.existsSync(LOG_FILE)) return [];
try {
const content = fs.readFileSync(LOG_FILE, "utf-8");
const lines = content.trim().split("\n");
return lines.slice(-limit).reverse();
const db = getDbInstance();
const rows = db
.prepare(
`
SELECT timestamp, model, provider, account, tokens_in, tokens_out, status
FROM call_logs
ORDER BY timestamp DESC
LIMIT ?
`
)
.all(limit) as Array<Record<string, unknown>>;
return rows.map((row) => {
const timestamp =
typeof row.timestamp === "string" ? row.timestamp : new Date().toISOString();
const provider = typeof row.provider === "string" ? row.provider.toUpperCase() : "-";
const model = typeof row.model === "string" ? row.model : "-";
const account = typeof row.account === "string" ? row.account : "-";
const tokensIn = toNumber(row.tokens_in);
const tokensOut = toNumber(row.tokens_out);
const status = typeof row.status === "number" ? row.status : String(row.status || "-");
return `${timestamp} | ${model} | ${provider} | ${account} | ${tokensIn} | ${tokensOut} | ${status}`;
});
} catch (error: any) {
console.error("[usageDb] Failed to read log.txt:", error.message);
console.error("[usageDb] Failed to read recent call logs:", error.message);
return [];
}
}

View File

@@ -5,6 +5,9 @@ import { initAuditLog, cleanupExpiredLogs, logAuditEvent } from "./lib/complianc
import { initConsoleInterceptor } from "./lib/consoleInterceptor";
async function startServer() {
// Trigger request-log layout migration during startup, before serving requests.
await import("./lib/usage/migrations");
// Console interceptor: capture all console output to log file (must be first)
initConsoleInterceptor();
@@ -22,7 +25,14 @@ async function startServer() {
// Compliance: One-time cleanup of expired logs
try {
const cleanup = cleanupExpiredLogs();
if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) {
if (
cleanup.deletedUsage ||
cleanup.deletedCallLogs ||
cleanup.deletedProxyLogs ||
cleanup.deletedRequestDetailLogs ||
cleanup.deletedAuditLogs ||
cleanup.deletedMcpAuditLogs
) {
console.log("[COMPLIANCE] Expired log cleanup:", cleanup);
}
} catch (err) {

View File

@@ -206,7 +206,7 @@ export default function ConsoleLogViewer() {
<span className="material-symbols-outlined text-[16px] align-middle mr-2">error</span>
{error}
<span className="text-xs ml-2 opacity-70">
Make sure the application is writing logs to file (LOG_TO_FILE=true)
Make sure the application is writing logs to file (APP_LOG_TO_FILE=true)
</span>
</div>
)}
@@ -237,7 +237,7 @@ export default function ConsoleLogViewer() {
</span>
<p>{t("noLogEntries")}</p>
<p className="text-[10px] mt-1 opacity-60">
Ensure LOG_TO_FILE=true is set in your .env file
Ensure APP_LOG_TO_FILE=true is set in your .env file
</p>
</div>
) : (

View File

@@ -92,27 +92,21 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC
const pipelinePayloads = detail?.pipelinePayloads || null;
const payloadSections = pipelinePayloads
? [
{
key: "client-request",
title: "Client Request",
json: toPrettyJson(pipelinePayloads.clientRequest),
},
{
key: "provider-request",
title: "Provider Request",
json: toPrettyJson(pipelinePayloads.providerRequest),
},
{
key: "provider-response",
title: "Provider Response",
json: toPrettyJson(pipelinePayloads.providerResponse),
},
{
key: "client-response",
title: "Client Response",
json: toPrettyJson(pipelinePayloads.clientResponse),
},
].filter((section) => section.json)
["clientRawRequest", "Client Raw Request"],
["clientRequest", "Client Request"],
["sourceRequest", "Source Request"],
["openaiRequest", "OpenAI Request"],
["providerRequest", "Provider Request"],
["providerResponse", "Provider Response"],
["clientResponse", "Client Response"],
["error", "Pipeline Error"],
]
.map(([key, title]) => ({
key,
title,
json: toPrettyJson(pipelinePayloads[key]),
}))
.filter((section) => section.json)
: [];
const requestJson = detail?.requestBody ? toPrettyJson(detail.requestBody) : null;
const responseJson = detail?.responseBody ? toPrettyJson(detail.responseBody) : null;

View File

@@ -315,7 +315,7 @@ export default function RequestLoggerV2() {
? "bg-amber-500/10 border-amber-500/30 text-amber-700 dark:text-amber-300"
: "bg-bg-subtle border-border text-text-muted"
}`}
title="Capture four-stage pipeline payloads for new requests"
title="Capture per-request pipeline payloads inside the unified call log artifact"
>
<span
className={`w-2 h-2 rounded-full ${detailLoggingEnabled ? "bg-amber-500" : "bg-text-muted"}`}
@@ -769,8 +769,8 @@ export default function RequestLoggerV2() {
</Card>
<div className="text-[10px] text-text-muted italic">
Call logs are also saved as JSON files to <code>{`{DATA_DIR}/call_logs/`}</code> with 7-day
rotation.
Each request is also saved as a single JSON artifact in{" "}
<code>{`{DATA_DIR}/call_logs/`}</code>.
</div>
{/* Detail Modal */}

View File

@@ -60,8 +60,6 @@ export const settingsSchema = z
requireLogin: z.boolean().optional(),
password: z.string().min(6, "Password must be at least 6 characters").optional(),
defaultModel: z.string().optional(),
enableRequestLogs: z.boolean().optional(),
maxLogRetention: z.number().int().min(1).max(365).optional(),
rateLimitEnabled: z.boolean().optional(),
rateLimitPerMinute: z.number().int().min(0).optional(),
})

View File

@@ -10,17 +10,18 @@
* In development, output is pretty-printed via pino-pretty.
* In production, output is structured JSON for log aggregation.
*
* When LOG_TO_FILE is enabled (default: true), logs are also written
* as JSON lines to the file specified by LOG_FILE_PATH.
* When APP_LOG_TO_FILE is enabled (default: true), logs are also written
* as JSON lines to the file specified by APP_LOG_FILE_PATH.
*/
import pino from "pino";
import { resolve } from "path";
import { getLogConfig, initLogRotation } from "@/lib/logRotation";
import { getAppLogLevel } from "@/lib/logEnv";
const isDev = process.env.NODE_ENV !== "production";
const baseConfig: pino.LoggerOptions = {
level: process.env.LOG_LEVEL || (isDev ? "debug" : "info"),
level: getAppLogLevel(isDev ? "debug" : "info"),
base: { service: "omniroute" },
timestamp: pino.stdTimeFunctions.isoTime,
formatters: {

View File

@@ -5,7 +5,7 @@
* and human-readable output for development. Replaces scattered console.log
* calls with consistent, parseable log entries.
*
* When LOG_TO_FILE is enabled, log entries are also appended as JSON lines
* When APP_LOG_TO_FILE is enabled, log entries are also appended as JSON lines
* to the application log file for the Console Log Viewer.
*
* @module shared/utils/structuredLogger
@@ -14,6 +14,7 @@
import { getCorrelationId } from "../middleware/correlationId";
import { appendFileSync, existsSync, mkdirSync } from "fs";
import { dirname, resolve } from "path";
import { getAppLogFilePath, getAppLogLevel, getAppLogToFile } from "@/lib/logEnv";
const LOG_LEVELS: Record<string, number> = {
debug: 10,
@@ -23,12 +24,12 @@ const LOG_LEVELS: Record<string, number> = {
fatal: 50,
};
const currentLevel = LOG_LEVELS[process.env.LOG_LEVEL?.toLowerCase() || ""] || LOG_LEVELS.info;
const currentLevel = LOG_LEVELS[getAppLogLevel("info").toLowerCase() || ""] || LOG_LEVELS.info;
const isProduction = process.env.NODE_ENV === "production";
// File logging configuration
const logToFile = process.env.LOG_TO_FILE !== "false";
const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log");
const logToFile = getAppLogToFile();
const logFilePath = resolve(getAppLogFilePath());
// Ensure log directory exists once at module load
if (logToFile) {

View File

@@ -148,12 +148,9 @@ export const updateSettingsSchema = z.object({
theme: z.string().max(50).optional(),
language: z.string().max(10).optional(),
requireLogin: z.boolean().optional(),
enableRequestLogs: z.boolean().optional(),
enableSocks5Proxy: z.boolean().optional(),
instanceName: z.string().max(100).optional(),
corsOrigins: z.string().max(500).optional(),
logRetentionDays: z.number().int().min(1).max(365).optional(),
maxCallLogs: z.number().int().min(1).max(1_000_000).optional(),
cloudUrl: z.string().max(500).optional(),
baseUrl: z.string().max(500).optional(),
setupComplete: z.boolean().optional(),

View File

@@ -14,12 +14,9 @@ export const updateSettingsSchema = z.object({
theme: z.string().max(50).optional(),
language: z.string().max(10).optional(),
requireLogin: z.boolean().optional(),
enableRequestLogs: z.boolean().optional(),
enableSocks5Proxy: z.boolean().optional(),
instanceName: z.string().max(100).optional(),
corsOrigins: z.string().max(500).optional(),
logRetentionDays: z.number().int().min(1).max(365).optional(),
maxCallLogs: z.number().int().min(1).max(1_000_000).optional(),
cloudUrl: z.string().max(500).optional(),
baseUrl: z.string().max(500).optional(),
setupComplete: z.boolean().optional(),

View File

@@ -40,7 +40,13 @@ describe("evalRunner", () => {
it("should evaluate exact match", () => {
const result = evaluateCase(
{ id: "t1", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } },
{
id: "t1",
name: "test",
model: "test",
input: {},
expected: { strategy: "exact", value: "hello" },
},
"hello"
);
assert.equal(result.passed, true);
@@ -48,7 +54,13 @@ describe("evalRunner", () => {
it("should fail exact match on mismatch", () => {
const result = evaluateCase(
{ id: "t2", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } },
{
id: "t2",
name: "test",
model: "test",
input: {},
expected: { strategy: "exact", value: "hello" },
},
"world"
);
assert.equal(result.passed, false);
@@ -56,7 +68,13 @@ describe("evalRunner", () => {
it("should evaluate contains (case-insensitive)", () => {
const result = evaluateCase(
{ id: "t3", name: "test", model: "test", input: {}, expected: { strategy: "contains", value: "paris" } },
{
id: "t3",
name: "test",
model: "test",
input: {},
expected: { strategy: "contains", value: "paris" },
},
"The capital is Paris."
);
assert.equal(result.passed, true);
@@ -64,7 +82,13 @@ describe("evalRunner", () => {
it("should evaluate regex", () => {
const result = evaluateCase(
{ id: "t4", name: "test", model: "test", input: {}, expected: { strategy: "regex", value: "\\d+" } },
{
id: "t4",
name: "test",
model: "test",
input: {},
expected: { strategy: "regex", value: "\\d+" },
},
"The answer is 42."
);
assert.equal(result.passed, true);
@@ -73,7 +97,10 @@ describe("evalRunner", () => {
it("should evaluate custom function", () => {
const result = evaluateCase(
{
id: "t5", name: "test", model: "test", input: {},
id: "t5",
name: "test",
model: "test",
input: {},
expected: { strategy: "custom", fn: (output) => output.length > 5 },
},
"this is long enough"
@@ -95,8 +122,20 @@ describe("evalRunner", () => {
id: "test-suite",
name: "Test Suite",
cases: [
{ id: "c1", name: "pass", model: "m", input: {}, expected: { strategy: "contains", value: "yes" } },
{ id: "c2", name: "fail", model: "m", input: {}, expected: { strategy: "contains", value: "no" } },
{
id: "c1",
name: "pass",
model: "m",
input: {},
expected: { strategy: "contains", value: "yes" },
},
{
id: "c2",
name: "fail",
model: "m",
input: {},
expected: { strategy: "contains", value: "no" },
},
],
});
@@ -225,7 +264,7 @@ describe("compliance", () => {
assert.equal(isNoLog("key-1"), false);
});
it("should have default retention of 90 days", () => {
assert.equal(getRetentionDays(), 90);
it("should expose split default retention windows", () => {
assert.deepEqual(getRetentionDays(), { app: 7, call: 7 });
});
});

View File

@@ -4,19 +4,17 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-cap-"));
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-artifacts-"));
process.env.DATA_DIR = TEST_DATA_DIR;
const ORIGINAL_CALL_LOGS_MAX = process.env.CALL_LOGS_MAX;
process.env.CALL_LOG_RETENTION_DAYS = "1";
const core = await import("../../src/lib/db/core.ts");
const localDb = await import("../../src/lib/localDb.ts");
const callLogs = await import("../../src/lib/usage/callLogs.ts");
async function resetStorage() {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
callLogs.invalidateCallLogsMaxCache();
}
test.beforeEach(async () => {
@@ -26,61 +24,69 @@ test.beforeEach(async () => {
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_CALL_LOGS_MAX === undefined) {
delete process.env.CALL_LOGS_MAX;
} else {
process.env.CALL_LOGS_MAX = ORIGINAL_CALL_LOGS_MAX;
}
});
test("call logs respect the configurable maxCallLogs setting", async () => {
await localDb.updateSettings({ maxCallLogs: 3 });
callLogs.invalidateCallLogsMaxCache();
test("call logs store a single per-request artifact with pipeline details", async () => {
const timestamp = "2026-03-30T12:34:56.789Z";
const logId = "req_artifact_1";
for (let i = 1; i <= 5; i++) {
await callLogs.saveCallLog({
method: "POST",
path: "/v1/chat/completions",
status: 200,
model: `model-${i}`,
provider: "openai",
duration: i,
requestBody: { index: i },
responseBody: { ok: true, index: i },
});
}
await callLogs.saveCallLog({
id: logId,
timestamp,
method: "POST",
path: "/v1/chat/completions",
status: 200,
model: "openai/gpt-4.1",
requestedModel: "openai/gpt-5",
provider: "openai",
duration: 42,
requestBody: { messages: [{ role: "user", content: "hello" }] },
responseBody: { id: "resp_1", choices: [{ message: { content: "world" } }] },
pipelinePayloads: {
clientRawRequest: { body: { raw: true } },
providerRequest: { body: { translated: true } },
providerResponse: { body: { upstream: true } },
clientResponse: { body: { final: true } },
},
});
const logs = await callLogs.getCallLogs({ limit: 10 });
const logs = await callLogs.getCallLogs({ limit: 5 });
assert.equal(logs.length, 1);
assert.equal(logs[0].hasPipelineDetails, true);
assert.equal(logs.length, 3);
assert.deepEqual(
logs.map((entry) => entry.model),
["model-5", "model-4", "model-3"]
const detail = await callLogs.getCallLogById(logId);
assert.equal(detail?.requestedModel, "openai/gpt-5");
assert.equal(detail?.pipelinePayloads?.clientRawRequest?.body?.raw, true);
assert.equal(detail?.pipelinePayloads?.providerRequest?.body?.translated, true);
assert.equal(detail?.pipelinePayloads?.providerResponse?.body?.upstream, true);
assert.equal(detail?.pipelinePayloads?.clientResponse?.body?.final, true);
assert.match(
detail?.artifactRelPath || "",
/^2026-03-30\/2026-03-30T12-34-56\.789Z_req_artifact_1\.json$/
);
const artifactPath = path.join(TEST_DATA_DIR, "call_logs", detail.artifactRelPath);
const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8"));
assert.equal(artifact.summary.id, logId);
assert.equal(artifact.summary.requestedModel, "openai/gpt-5");
assert.equal(artifact.pipeline.clientRawRequest.body.raw, true);
});
test("call logs keep honoring CALL_LOGS_MAX when maxCallLogs was never saved", async () => {
process.env.CALL_LOGS_MAX = "2";
callLogs.invalidateCallLogsMaxCache();
test("call log artifact rotation removes directories older than CALL_LOG_RETENTION_DAYS", async () => {
const oldDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-10");
const freshDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-30");
fs.mkdirSync(oldDir, { recursive: true });
fs.mkdirSync(freshDir, { recursive: true });
fs.writeFileSync(path.join(oldDir, "old.json"), "{}");
fs.writeFileSync(path.join(freshDir, "fresh.json"), "{}");
for (let i = 1; i <= 4; i++) {
await callLogs.saveCallLog({
method: "POST",
path: "/v1/chat/completions",
status: 200,
model: `env-model-${i}`,
provider: "openai",
duration: i,
requestBody: { index: i },
responseBody: { ok: true, index: i },
});
}
const oldTime = new Date("2026-03-10T00:00:00.000Z");
const freshTime = new Date();
fs.utimesSync(oldDir, oldTime, oldTime);
fs.utimesSync(freshDir, freshTime, freshTime);
const logs = await callLogs.getCallLogs({ limit: 10 });
callLogs.rotateCallLogs();
assert.equal(logs.length, 2);
assert.deepEqual(
logs.map((entry) => entry.model),
["env-model-4", "env-model-3"]
);
assert.equal(fs.existsSync(oldDir), false);
assert.equal(fs.existsSync(freshDir), true);
});

View File

@@ -7,16 +7,16 @@ import path from "node:path";
const TEST_LOG_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-console-log-levels-"));
const TEST_LOG_PATH = path.join(TEST_LOG_DIR, "app.log");
const originalLogFilePath = process.env.LOG_FILE_PATH;
process.env.LOG_FILE_PATH = TEST_LOG_PATH;
const originalLogFilePath = process.env.APP_LOG_FILE_PATH;
process.env.APP_LOG_FILE_PATH = TEST_LOG_PATH;
const route = await import("../../src/app/api/logs/console/route.ts");
test.after(() => {
if (originalLogFilePath === undefined) {
delete process.env.LOG_FILE_PATH;
delete process.env.APP_LOG_FILE_PATH;
} else {
process.env.LOG_FILE_PATH = originalLogFilePath;
process.env.APP_LOG_FILE_PATH = originalLogFilePath;
}
fs.rmSync(TEST_LOG_DIR, { recursive: true, force: true });
});

View File

@@ -0,0 +1,134 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-retention-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.APP_LOG_RETENTION_DAYS = "2";
process.env.CALL_LOG_RETENTION_DAYS = "1";
const core = await import("../../src/lib/db/core.ts");
const compliance = await import("../../src/lib/compliance/index.ts");
function resetStorage() {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
test.beforeEach(() => {
resetStorage();
});
test.after(() => {
resetStorage();
});
test("cleanupExpiredLogs uses separate APP and CALL retention windows", () => {
compliance.initAuditLog();
const db = core.getDbInstance();
const oldCallTs = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString();
const freshCallTs = new Date().toISOString();
const oldAppTs = new Date(Date.now() - 5 * 24 * 60 * 60 * 1000).toISOString();
const freshAppTs = new Date().toISOString();
db.prepare(
"INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
).run("openai", "old-usage", 1, 1, 1, 1, 1, oldCallTs);
db.prepare(
"INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
).run("openai", "fresh-usage", 1, 1, 1, 1, 1, freshCallTs);
db.prepare(
"INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
).run(
"old-call",
oldCallTs,
"POST",
"/v1/chat/completions",
200,
"old",
"openai",
"-",
1,
1,
1,
0
);
db.prepare(
"INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
).run(
"fresh-call",
freshCallTs,
"POST",
"/v1/chat/completions",
200,
"fresh",
"openai",
"-",
1,
1,
1,
0
);
db.prepare(
"INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)"
).run("old-proxy", oldCallTs, "success", "direct", 1);
db.prepare(
"INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)"
).run("fresh-proxy", freshCallTs, "success", "direct", 1);
db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run(
"old-detail",
oldCallTs,
1
);
db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run(
"fresh-detail",
freshCallTs,
1
);
db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run(
oldAppTs,
"old-audit",
"system"
);
db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run(
freshAppTs,
"fresh-audit",
"system"
);
db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run(
"old-tool",
1,
oldAppTs
);
db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run(
"fresh-tool",
1,
freshAppTs
);
const result = compliance.cleanupExpiredLogs();
assert.equal(result.deletedUsage, 1);
assert.equal(result.deletedCallLogs, 1);
assert.equal(result.deletedProxyLogs, 1);
assert.equal(result.deletedRequestDetailLogs, 1);
assert.equal(result.deletedAuditLogs, 1);
assert.equal(result.deletedMcpAuditLogs, 1);
assert.deepEqual(compliance.getRetentionDays(), { app: 2, call: 1 });
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM usage_history").get().cnt, 1);
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM call_logs").get().cnt, 1);
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM proxy_logs").get().cnt, 1);
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM request_detail_logs").get().cnt, 1);
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM audit_log").get().cnt, 2);
assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM mcp_tool_audit").get().cnt, 1);
});

View File

@@ -0,0 +1,72 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-migration-"));
process.env.DATA_DIR = TEST_DATA_DIR;
const migrations = await import("../../src/lib/usage/migrations.ts");
const LEGACY_LOGS_DIR = path.join(TEST_DATA_DIR, "logs");
const LEGACY_CALL_LOGS_DIR = path.join(TEST_DATA_DIR, "call_logs");
const LEGACY_SUMMARY_FILE = path.join(TEST_DATA_DIR, "log.txt");
const MARKER_PATH = path.join(migrations.LOG_ARCHIVES_DIR, "legacy-request-logs.json");
function resetDataDir() {
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
function seedLegacyLayout() {
fs.mkdirSync(path.join(LEGACY_LOGS_DIR, "session-a"), { recursive: true });
fs.writeFileSync(
path.join(LEGACY_LOGS_DIR, "session-a", "1_req_client.json"),
JSON.stringify({ ok: true }, null, 2)
);
fs.mkdirSync(path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30"), { recursive: true });
fs.writeFileSync(
path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30", "123000_model_200.json"),
JSON.stringify({ ok: true }, null, 2)
);
fs.writeFileSync(LEGACY_SUMMARY_FILE, "legacy summary\n");
}
test.beforeEach(() => {
resetDataDir();
});
test.after(() => {
resetDataDir();
});
test("archives legacy request log layout into a zip and removes old files", async () => {
seedLegacyLayout();
const archiveFilename = await migrations.archiveLegacyRequestLogs();
assert.match(archiveFilename || "", /_legacy-request-logs\.zip$/);
assert.equal(fs.existsSync(LEGACY_LOGS_DIR), false);
assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), false);
assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), false);
assert.equal(fs.existsSync(MARKER_PATH), true);
const archivePath = path.join(migrations.LOG_ARCHIVES_DIR, archiveFilename);
assert.equal(fs.existsSync(archivePath), true);
assert.ok(fs.statSync(archivePath).size > 0);
});
test("keeps legacy files in place when zip creation fails", async () => {
seedLegacyLayout();
fs.writeFileSync(migrations.LOG_ARCHIVES_DIR, "not-a-directory");
await assert.rejects(() => migrations.archiveLegacyRequestLogs());
assert.equal(fs.existsSync(LEGACY_LOGS_DIR), true);
assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), true);
assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), true);
assert.equal(fs.existsSync(MARKER_PATH), false);
});

View File

@@ -12,8 +12,8 @@ const localDb = await import("../../src/lib/localDb.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const usageHistory = await import("../../src/lib/usage/usageHistory.ts");
const usageStats = await import("../../src/lib/usage/usageStats.ts");
const callLogs = await import("../../src/lib/usage/callLogs.ts");
const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts");
const { LOG_FILE } = await import("../../src/lib/usage/migrations.ts");
function clearPendingRequests() {
const pending = usageHistory.getPendingRequests();
@@ -263,7 +263,7 @@ test("getUsageStats aggregates totals, buckets, pending requests, and cost break
assert.equal(recentBucketTotal, 1);
});
test("request log appends readable entries and trims to the most recent 200 lines", async () => {
test("recent request summaries are generated from SQLite call logs", async () => {
const connection = await providersDb.createProviderConnection({
provider: "log-provider",
authType: "apikey",
@@ -272,19 +272,23 @@ test("request log appends readable entries and trims to the most recent 200 line
});
for (let i = 0; i < 205; i++) {
await usageHistory.appendRequestLog({
await callLogs.saveCallLog({
id: `log-${i}`,
timestamp: new Date(Date.now() + i).toISOString(),
method: "POST",
path: "/v1/chat/completions",
status: 200,
model: `model-${i}`,
provider: "log-provider",
connectionId: connection.id,
tokens: { input: i + 1, output: i + 2 },
status: 200,
requestBody: { index: i },
responseBody: { ok: true, index: i },
});
}
const recent = await usageHistory.getRecentLogs(3);
const lines = fs.readFileSync(LOG_FILE, "utf8").trim().split("\n");
assert.equal(lines.length, 200);
assert.equal(recent.length, 3);
assert.match(recent[0], /model-204/);
assert.match(recent[0], /LOG-PROVIDER/);