Refine pipeline logging and add retention caps

src/lib/logEnv.ts

@@ -3,6 +3,8 @@ import path from "path";
 const DEFAULT_APP_LOG_RETENTION_DAYS = 7;
 const DEFAULT_CALL_LOG_RETENTION_DAYS = 7;
 const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024;
+const DEFAULT_APP_LOG_MAX_FILES = 20;
+const DEFAULT_CALL_LOG_MAX_ENTRIES = 10000;
 const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log");
 
 function parsePositiveInt(value: string | undefined, fallback: number): number {
@@ -52,6 +54,14 @@ export function getCallLogRetentionDays(): number {
   return parsePositiveInt(process.env.CALL_LOG_RETENTION_DAYS, DEFAULT_CALL_LOG_RETENTION_DAYS);
 }
 
+export function getAppLogMaxFiles(): number {
+  return parsePositiveInt(process.env.APP_LOG_MAX_FILES, DEFAULT_APP_LOG_MAX_FILES);
+}
+
+export function getCallLogMaxEntries(): number {
+  return parsePositiveInt(process.env.CALL_LOG_MAX_ENTRIES, DEFAULT_CALL_LOG_MAX_ENTRIES);
+}
+
 export function getAppLogLevel(defaultLevel: string): string {
   return process.env.APP_LOG_LEVEL || defaultLevel;
 }

src/lib/logRotation.ts

@@ -4,6 +4,7 @@
  * Handles:
  *   - Rotating log files when they exceed max size
  *   - Cleaning up old log files past retention period
+ *   - Capping the number of rotated log files kept on disk
  *   - Creating the log directory on startup
  *
  * Configuration via env vars:
@@ -11,12 +12,14 @@
  *   - APP_LOG_FILE_PATH: path to log file (default: logs/application/app.log)
  *   - APP_LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB)
  *   - APP_LOG_RETENTION_DAYS: days to keep old logs (default: 7)
+ *   - APP_LOG_MAX_FILES: max number of rotated log files to keep (default: 20)
  */
 
 import { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync } from "fs";
 import { dirname, join, basename, extname } from "path";
 import {
   getAppLogFilePath,
+  getAppLogMaxFiles,
   getAppLogMaxFileSize,
   getAppLogRetentionDays,
   getAppLogToFile,
@@ -27,8 +30,9 @@ export function getLogConfig() {
   const logFilePath = getAppLogFilePath() || join(process.cwd(), "logs/application/app.log");
   const maxFileSize = getAppLogMaxFileSize();
   const retentionDays = getAppLogRetentionDays();
+  const maxFiles = getAppLogMaxFiles();
 
-  return { logToFile, logFilePath, maxFileSize, retentionDays };
+  return { logToFile, logFilePath, maxFileSize, retentionDays, maxFiles };
 }
 
 /**
@@ -100,6 +104,44 @@ export function cleanupOldLogs(logFilePath: string, retentionDays: number): void
   }
 }
 
+/**
+ * Keep only the newest rotated files up to the configured count limit.
+ */
+export function cleanupOverflowLogs(logFilePath: string, maxFiles: number): void {
+  try {
+    const dir = dirname(logFilePath);
+    if (!existsSync(dir) || maxFiles < 1) return;
+
+    const ext = extname(logFilePath);
+    const base = basename(logFilePath, ext);
+    const rotatedFiles = readdirSync(dir)
+      .filter(
+        (file) =>
+          file !== basename(logFilePath) && file.startsWith(base + ".") && file.endsWith(ext)
+      )
+      .map((file) => {
+        const filePath = join(dir, file);
+        try {
+          return { filePath, mtimeMs: statSync(filePath).mtimeMs };
+        } catch {
+          return null;
+        }
+      })
+      .filter((entry): entry is { filePath: string; mtimeMs: number } => !!entry)
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+    for (const entry of rotatedFiles.slice(maxFiles)) {
+      try {
+        unlinkSync(entry.filePath);
+      } catch {
+        // Best effort only.
+      }
+    }
+  } catch {
+    // Cleanup is best-effort
+  }
+}
+
 /**
  * Initialize log rotation — call once at application startup.
  * Creates directories, rotates if needed, and cleans up old files.
@@ -111,4 +153,5 @@ export function initLogRotation(): void {
   ensureLogDir(config.logFilePath);
   rotateIfNeeded(config.logFilePath, config.maxFileSize);
   cleanupOldLogs(config.logFilePath, config.retentionDays);
+  cleanupOverflowLogs(config.logFilePath, config.maxFiles);
 }

src/lib/usage/callLogs.ts

@@ -21,7 +21,7 @@ import {
   parseStoredPayload,
   serializePayloadForStorage,
 } from "../logPayloads";
-import { getCallLogRetentionDays } from "../logEnv";
+import { getCallLogMaxEntries, getCallLogRetentionDays } from "../logEnv";
 
 type JsonRecord = Record<string, unknown>;
 
@@ -230,6 +230,7 @@ function writeCallArtifact(artifact: CallLogArtifact): string | null {
   try {
     fs.mkdirSync(path.dirname(absPath), { recursive: true });
     fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2));
+    rotateCallLogs();
     return relPath;
   } catch (error) {
     console.error("[callLogs] Failed to write request artifact:", (error as Error).message);
@@ -293,6 +294,69 @@ function readLegacyLogFromDisk(entry: {
   return null;
 }
 
+function cleanupEmptyCallLogDirs() {
+  if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return;
+
+  try {
+    for (const entry of fs.readdirSync(CALL_LOGS_DIR)) {
+      const entryPath = path.join(CALL_LOGS_DIR, entry);
+      const stat = fs.statSync(entryPath);
+      if (!stat.isDirectory()) continue;
+      if (fs.readdirSync(entryPath).length === 0) {
+        fs.rmSync(entryPath, { recursive: true, force: true });
+      }
+    }
+  } catch {
+    // Best effort only.
+  }
+}
+
+export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?: number) {
+  if (!baseDir || !fs.existsSync(baseDir)) return;
+
+  const limit = maxEntries ?? getCallLogMaxEntries();
+  if (!Number.isInteger(limit) || limit < 1) return;
+
+  try {
+    const files = fs
+      .readdirSync(baseDir)
+      .flatMap((entry) => {
+        const entryPath = path.join(baseDir, entry);
+        try {
+          const stat = fs.statSync(entryPath);
+          if (!stat.isDirectory()) return [];
+
+          return fs
+            .readdirSync(entryPath)
+            .filter((file) => file.endsWith(".json"))
+            .map((file) => {
+              const filePath = path.join(entryPath, file);
+              const fileStat = fs.statSync(filePath);
+              return { filePath, mtimeMs: fileStat.mtimeMs };
+            });
+        } catch {
+          return [];
+        }
+      })
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+    for (const file of files.slice(limit)) {
+      try {
+        fs.rmSync(file.filePath, { force: true });
+      } catch {
+        // Best effort only.
+      }
+    }
+
+    cleanupEmptyCallLogDirs();
+  } catch (error) {
+    console.error(
+      "[callLogs] Failed to prune overflow request artifacts:",
+      (error as Error).message
+    );
+  }
+}
+
 export async function saveCallLog(entry: any) {
   if (!shouldPersistToDisk) return;
 
@@ -392,6 +456,7 @@ export function rotateCallLogs() {
         fs.rmSync(entryPath, { recursive: true, force: true });
       }
     }
+    cleanupOverflowCallLogFiles(CALL_LOGS_DIR, getCallLogMaxEntries());
   } catch (error) {
     console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
   }