Refine pipeline logging and add retention caps

src/lib/usage/callLogs.ts

@@ -21,7 +21,7 @@ import {
   parseStoredPayload,
   serializePayloadForStorage,
 } from "../logPayloads";
-import { getCallLogRetentionDays } from "../logEnv";
+import { getCallLogMaxEntries, getCallLogRetentionDays } from "../logEnv";
 
 type JsonRecord = Record<string, unknown>;
 
@@ -230,6 +230,7 @@ function writeCallArtifact(artifact: CallLogArtifact): string | null {
   try {
     fs.mkdirSync(path.dirname(absPath), { recursive: true });
     fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2));
+    rotateCallLogs();
     return relPath;
   } catch (error) {
     console.error("[callLogs] Failed to write request artifact:", (error as Error).message);
@@ -293,6 +294,69 @@ function readLegacyLogFromDisk(entry: {
   return null;
 }
 
+function cleanupEmptyCallLogDirs() {
+  if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return;
+
+  try {
+    for (const entry of fs.readdirSync(CALL_LOGS_DIR)) {
+      const entryPath = path.join(CALL_LOGS_DIR, entry);
+      const stat = fs.statSync(entryPath);
+      if (!stat.isDirectory()) continue;
+      if (fs.readdirSync(entryPath).length === 0) {
+        fs.rmSync(entryPath, { recursive: true, force: true });
+      }
+    }
+  } catch {
+    // Best effort only.
+  }
+}
+
+export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?: number) {
+  if (!baseDir || !fs.existsSync(baseDir)) return;
+
+  const limit = maxEntries ?? getCallLogMaxEntries();
+  if (!Number.isInteger(limit) || limit < 1) return;
+
+  try {
+    const files = fs
+      .readdirSync(baseDir)
+      .flatMap((entry) => {
+        const entryPath = path.join(baseDir, entry);
+        try {
+          const stat = fs.statSync(entryPath);
+          if (!stat.isDirectory()) return [];
+
+          return fs
+            .readdirSync(entryPath)
+            .filter((file) => file.endsWith(".json"))
+            .map((file) => {
+              const filePath = path.join(entryPath, file);
+              const fileStat = fs.statSync(filePath);
+              return { filePath, mtimeMs: fileStat.mtimeMs };
+            });
+        } catch {
+          return [];
+        }
+      })
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+    for (const file of files.slice(limit)) {
+      try {
+        fs.rmSync(file.filePath, { force: true });
+      } catch {
+        // Best effort only.
+      }
+    }
+
+    cleanupEmptyCallLogDirs();
+  } catch (error) {
+    console.error(
+      "[callLogs] Failed to prune overflow request artifacts:",
+      (error as Error).message
+    );
+  }
+}
+
 export async function saveCallLog(entry: any) {
   if (!shouldPersistToDisk) return;
 
@@ -392,6 +456,7 @@ export function rotateCallLogs() {
         fs.rmSync(entryPath, { recursive: true, force: true });
       }
     }
+    cleanupOverflowCallLogFiles(CALL_LOGS_DIR, getCallLogMaxEntries());
   } catch (error) {
     console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
   }