Refine pipeline logging and add retention caps

src/app/api/translator/load/route.ts

@@ -17,7 +17,6 @@ export async function GET(request) {
     // Security: only allow specific filenames
     const allowedFiles = [
       "1_req_client.json",
-      "2_req_source.json",
       "3_req_openai.json",
       "4_req_target.json",
       "5_res_provider.txt",

src/app/api/translator/save/route.ts

@@ -31,7 +31,6 @@ export async function POST(request) {
     // Security: only allow specific filenames
     const allowedFiles = [
       "1_req_client.json",
-      "2_req_source.json",
       "3_req_openai.json",
       "4_req_target.json",
       "5_res_provider.txt",

src/lib/logEnv.ts

@@ -3,6 +3,8 @@ import path from "path";
 const DEFAULT_APP_LOG_RETENTION_DAYS = 7;
 const DEFAULT_CALL_LOG_RETENTION_DAYS = 7;
 const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024;
+const DEFAULT_APP_LOG_MAX_FILES = 20;
+const DEFAULT_CALL_LOG_MAX_ENTRIES = 10000;
 const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log");
 
 function parsePositiveInt(value: string | undefined, fallback: number): number {
@@ -52,6 +54,14 @@ export function getCallLogRetentionDays(): number {
   return parsePositiveInt(process.env.CALL_LOG_RETENTION_DAYS, DEFAULT_CALL_LOG_RETENTION_DAYS);
 }
 
+export function getAppLogMaxFiles(): number {
+  return parsePositiveInt(process.env.APP_LOG_MAX_FILES, DEFAULT_APP_LOG_MAX_FILES);
+}
+
+export function getCallLogMaxEntries(): number {
+  return parsePositiveInt(process.env.CALL_LOG_MAX_ENTRIES, DEFAULT_CALL_LOG_MAX_ENTRIES);
+}
+
 export function getAppLogLevel(defaultLevel: string): string {
   return process.env.APP_LOG_LEVEL || defaultLevel;
 }

src/lib/logRotation.ts

@@ -4,6 +4,7 @@
  * Handles:
  *   - Rotating log files when they exceed max size
  *   - Cleaning up old log files past retention period
+ *   - Capping the number of rotated log files kept on disk
  *   - Creating the log directory on startup
  *
  * Configuration via env vars:
@@ -11,12 +12,14 @@
  *   - APP_LOG_FILE_PATH: path to log file (default: logs/application/app.log)
  *   - APP_LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB)
  *   - APP_LOG_RETENTION_DAYS: days to keep old logs (default: 7)
+ *   - APP_LOG_MAX_FILES: max number of rotated log files to keep (default: 20)
  */
 
 import { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync } from "fs";
 import { dirname, join, basename, extname } from "path";
 import {
   getAppLogFilePath,
+  getAppLogMaxFiles,
   getAppLogMaxFileSize,
   getAppLogRetentionDays,
   getAppLogToFile,
@@ -27,8 +30,9 @@ export function getLogConfig() {
   const logFilePath = getAppLogFilePath() || join(process.cwd(), "logs/application/app.log");
   const maxFileSize = getAppLogMaxFileSize();
   const retentionDays = getAppLogRetentionDays();
+  const maxFiles = getAppLogMaxFiles();
 
-  return { logToFile, logFilePath, maxFileSize, retentionDays };
+  return { logToFile, logFilePath, maxFileSize, retentionDays, maxFiles };
 }
 
 /**
@@ -100,6 +104,44 @@ export function cleanupOldLogs(logFilePath: string, retentionDays: number): void
   }
 }
 
+/**
+ * Keep only the newest rotated files up to the configured count limit.
+ */
+export function cleanupOverflowLogs(logFilePath: string, maxFiles: number): void {
+  try {
+    const dir = dirname(logFilePath);
+    if (!existsSync(dir) || maxFiles < 1) return;
+
+    const ext = extname(logFilePath);
+    const base = basename(logFilePath, ext);
+    const rotatedFiles = readdirSync(dir)
+      .filter(
+        (file) =>
+          file !== basename(logFilePath) && file.startsWith(base + ".") && file.endsWith(ext)
+      )
+      .map((file) => {
+        const filePath = join(dir, file);
+        try {
+          return { filePath, mtimeMs: statSync(filePath).mtimeMs };
+        } catch {
+          return null;
+        }
+      })
+      .filter((entry): entry is { filePath: string; mtimeMs: number } => !!entry)
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+    for (const entry of rotatedFiles.slice(maxFiles)) {
+      try {
+        unlinkSync(entry.filePath);
+      } catch {
+        // Best effort only.
+      }
+    }
+  } catch {
+    // Cleanup is best-effort
+  }
+}
+
 /**
  * Initialize log rotation — call once at application startup.
  * Creates directories, rotates if needed, and cleans up old files.
@@ -111,4 +153,5 @@ export function initLogRotation(): void {
   ensureLogDir(config.logFilePath);
   rotateIfNeeded(config.logFilePath, config.maxFileSize);
   cleanupOldLogs(config.logFilePath, config.retentionDays);
+  cleanupOverflowLogs(config.logFilePath, config.maxFiles);
 }

src/lib/usage/callLogs.ts

@@ -21,7 +21,7 @@ import {
   parseStoredPayload,
   serializePayloadForStorage,
 } from "../logPayloads";
-import { getCallLogRetentionDays } from "../logEnv";
+import { getCallLogMaxEntries, getCallLogRetentionDays } from "../logEnv";
 
 type JsonRecord = Record<string, unknown>;
 
@@ -230,6 +230,7 @@ function writeCallArtifact(artifact: CallLogArtifact): string | null {
   try {
     fs.mkdirSync(path.dirname(absPath), { recursive: true });
     fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2));
+    rotateCallLogs();
     return relPath;
   } catch (error) {
     console.error("[callLogs] Failed to write request artifact:", (error as Error).message);
@@ -293,6 +294,69 @@ function readLegacyLogFromDisk(entry: {
   return null;
 }
 
+function cleanupEmptyCallLogDirs() {
+  if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return;
+
+  try {
+    for (const entry of fs.readdirSync(CALL_LOGS_DIR)) {
+      const entryPath = path.join(CALL_LOGS_DIR, entry);
+      const stat = fs.statSync(entryPath);
+      if (!stat.isDirectory()) continue;
+      if (fs.readdirSync(entryPath).length === 0) {
+        fs.rmSync(entryPath, { recursive: true, force: true });
+      }
+    }
+  } catch {
+    // Best effort only.
+  }
+}
+
+export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?: number) {
+  if (!baseDir || !fs.existsSync(baseDir)) return;
+
+  const limit = maxEntries ?? getCallLogMaxEntries();
+  if (!Number.isInteger(limit) || limit < 1) return;
+
+  try {
+    const files = fs
+      .readdirSync(baseDir)
+      .flatMap((entry) => {
+        const entryPath = path.join(baseDir, entry);
+        try {
+          const stat = fs.statSync(entryPath);
+          if (!stat.isDirectory()) return [];
+
+          return fs
+            .readdirSync(entryPath)
+            .filter((file) => file.endsWith(".json"))
+            .map((file) => {
+              const filePath = path.join(entryPath, file);
+              const fileStat = fs.statSync(filePath);
+              return { filePath, mtimeMs: fileStat.mtimeMs };
+            });
+        } catch {
+          return [];
+        }
+      })
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+    for (const file of files.slice(limit)) {
+      try {
+        fs.rmSync(file.filePath, { force: true });
+      } catch {
+        // Best effort only.
+      }
+    }
+
+    cleanupEmptyCallLogDirs();
+  } catch (error) {
+    console.error(
+      "[callLogs] Failed to prune overflow request artifacts:",
+      (error as Error).message
+    );
+  }
+}
+
 export async function saveCallLog(entry: any) {
   if (!shouldPersistToDisk) return;
 
@@ -392,6 +456,7 @@ export function rotateCallLogs() {
         fs.rmSync(entryPath, { recursive: true, force: true });
       }
     }
+    cleanupOverflowCallLogFiles(CALL_LOGS_DIR, getCallLogMaxEntries());
   } catch (error) {
     console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
   }

src/shared/components/RequestLoggerDetail.tsx

@@ -94,7 +94,6 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC
     ? [
         ["clientRawRequest", "Client Raw Request"],
         ["clientRequest", "Client Request"],
-        ["sourceRequest", "Source Request"],
         ["openaiRequest", "OpenAI Request"],
         ["providerRequest", "Provider Request"],
         ["providerResponse", "Provider Response"],

src/shared/components/RequestLoggerV2.tsx

@@ -95,7 +95,6 @@ export default function RequestLoggerV2() {
   const [detailData, setDetailData] = useState(null);
   const [detailLoggingEnabled, setDetailLoggingEnabled] = useState(false);
   const [detailLoggingLoading, setDetailLoggingLoading] = useState(false);
-  const [detailLoggingReady, setDetailLoggingReady] = useState(false);
   const intervalRef = useRef(null);
   const hasLoadedRef = useRef(false);
   const [providerNodes, setProviderNodes] = useState([]);
@@ -173,7 +172,6 @@ export default function RequestLoggerV2() {
       .then((data) => {
         if (!data) return;
         setDetailLoggingEnabled(data.enabled === true);
-        setDetailLoggingReady(true);
       })
       .catch(() => {});
   }, []);
@@ -258,11 +256,10 @@ export default function RequestLoggerV2() {
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ enabled: nextEnabled }),
       });
-      if (!res.ok) throw new Error("Failed to update detailed logging");
+      if (!res.ok) throw new Error("Failed to update pipeline logging");
       setDetailLoggingEnabled(nextEnabled);
-      setDetailLoggingReady(true);
     } catch (error) {
-      console.error("Failed to toggle detailed logging:", error);
+      console.error("Failed to toggle pipeline logging:", error);
     } finally {
       setDetailLoggingLoading(false);
     }
@@ -315,25 +312,18 @@ export default function RequestLoggerV2() {
               ? "bg-amber-500/10 border-amber-500/30 text-amber-700 dark:text-amber-300"
               : "bg-bg-subtle border-border text-text-muted"
           }`}
-          title="Capture per-request pipeline payloads inside the unified call log artifact"
+          title="Capture pipeline payloads for new requests"
         >
           <span
             className={`w-2 h-2 rounded-full ${detailLoggingEnabled ? "bg-amber-500" : "bg-text-muted"}`}
           />
           {detailLoggingLoading
-            ? "Updating detailed logs..."
+            ? "Updating pipeline logs..."
             : detailLoggingEnabled
-              ? "Detailed Logs On"
-              : "Detailed Logs Off"}
+              ? "Pipeline Logs On"
+              : "Pipeline Logs Off"}
         </button>
 
-        {detailLoggingReady && (
-          <span className="text-[11px] text-text-muted">
-            New requests will {detailLoggingEnabled ? "" : "not "}capture client/provider pipeline
-            payloads.
-          </span>
-        )}
-
         {/* Search */}
         <div className="flex-1 min-w-[200px] relative">
           <span className="material-symbols-outlined absolute left-3 top-1/2 -translate-y-1/2 text-text-muted text-[18px]">
@@ -769,8 +759,8 @@ export default function RequestLoggerV2() {
       </Card>
 
       <div className="text-[10px] text-text-muted italic">
-        Each request is also saved as a single JSON artifact in{" "}
-        <code>{`{DATA_DIR}/call_logs/`}</code>.
+        Call logs are also saved as JSON files to <code>{`{DATA_DIR}/call_logs/`}</code> and rotated
+        by <code>CALL_LOG_RETENTION_DAYS</code> and <code>CALL_LOG_MAX_ENTRIES</code>.
       </div>
 
       {/* Detail Modal */}

src/shared/validation/schemas.ts

@@ -767,7 +767,6 @@ const nonEmptyJsonRecordSchema = jsonRecordSchema.refine(
 
 const translatorLogFileSchema = z.enum([
   "1_req_client.json",
-  "2_req_source.json",
   "3_req_openai.json",
   "4_req_target.json",
   "5_res_provider.txt",