From 845e2b3d01170b727a66f0d9cfde30579e9253ca Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Mon, 27 Apr 2026 06:12:34 -0400 Subject: [PATCH] feat: configure call log pipeline artifacts (#1650) Integrated into release/v3.7.2 --- .env.example | 10 ++ README.md | 1 + docs/ENVIRONMENT.md | 28 ++-- docs/TROUBLESHOOTING.md | 2 + open-sse/handlers/chatCore.ts | 5 +- src/lib/logEnv.ts | 22 +++ src/lib/usage/callLogArtifacts.ts | 80 +++++++---- tests/unit/call-log-cap.test.ts | 128 ++++++++++++++++++ tests/unit/chatcore-translation-paths.test.ts | 36 +++++ tests/unit/log-retention.test.ts | 27 ++++ 10 files changed, 297 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index be615356cb..904a56db11 100644 --- a/.env.example +++ b/.env.example @@ -586,6 +586,16 @@ APP_LOG_TO_FILE=true # Default: 100000 # CALL_LOGS_TABLE_MAX_ROWS=100000 +# Whether call log pipeline capture stores stream chunks when enabled in settings. +# Only applies when call_log_pipeline_enabled=true. +# Default: true +# CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS=true + +# Maximum call log artifact size for pipeline captures, in KB. +# Only applies when call_log_pipeline_enabled=true. +# Default: 512 +# CALL_LOG_PIPELINE_MAX_SIZE_KB=512 + # Maximum rows in the proxy_logs SQLite table. # Default: 100000 # PROXY_LOGS_TABLE_MAX_ROWS=100000 diff --git a/README.md b/README.md index 1b7e440391..f619d64816 100644 --- a/README.md +++ b/README.md @@ -2050,6 +2050,7 @@ opencode - `call_logs` in SQLite stores summary metadata for the Request Logs table and analytics views - Detailed request/response payloads are written to `DATA_DIR/call_logs/` as one JSON artifact per request - Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads +- When pipeline capture is enabled, `CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS=false` skips stream chunks and `CALL_LOG_PIPELINE_MAX_SIZE_KB` controls the artifact cap in KB - `Export Logs` reads the artifact files on demand, while `Export All` includes the `call_logs/` directory alongside `storage.sqlite` - Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log` - Adjust `APP_LOG_MAX_FILE_SIZE`, `APP_LOG_RETENTION_DAYS`, `APP_LOG_MAX_FILES`, and `CALL_LOG_MAX_ENTRIES` as needed diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index c6e20dfd73..8d1c5e1175 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -464,19 +464,21 @@ REQUEST_TIMEOUT_MS (global override) The logging system writes to both stdout and rotated log files. All configuration is read by `src/lib/logEnv.ts`. -| Variable | Default | Description | -| --------------------------- | -------------------------- | ---------------------------------------------------------------------------- | -| `APP_LOG_LEVEL` | `info` | Minimum log level: `debug`, `info`, `warn`, `error`. | -| `APP_LOG_FORMAT` | `text` | Output format: `text` (human-readable) or `json` (structured). | -| `APP_LOG_TO_FILE` | `true` | Write logs to file alongside stdout. | -| `APP_LOG_FILE_PATH` | `logs/application/app.log` | Log file path (relative to project root or `DATA_DIR`). | -| `APP_LOG_MAX_FILE_SIZE` | `50M` | Max file size before rotation. Accepts: `50M`, `1G`, `512K`, or plain bytes. | -| `APP_LOG_RETENTION_DAYS` | `7` | Days to keep rotated application log files. | -| `APP_LOG_MAX_FILES` | `20` | Maximum rotated log file backups. | -| `CALL_LOG_RETENTION_DAYS` | `7` | Days to keep request/call log entries in the database. | -| `CALL_LOG_MAX_ENTRIES` | `10000` | Max call log entries in the in-memory buffer. | -| `CALL_LOGS_TABLE_MAX_ROWS` | `100000` | Max rows in the `call_logs` SQLite table before pruning. | -| `PROXY_LOGS_TABLE_MAX_ROWS` | `100000` | Max rows in the `proxy_logs` SQLite table before pruning. | +| Variable | Default | Description | +| ----------------------------------------- | -------------------------- | -------------------------------------------------------------------------------- | +| `APP_LOG_LEVEL` | `info` | Minimum log level: `debug`, `info`, `warn`, `error`. | +| `APP_LOG_FORMAT` | `text` | Output format: `text` (human-readable) or `json` (structured). | +| `APP_LOG_TO_FILE` | `true` | Write logs to file alongside stdout. | +| `APP_LOG_FILE_PATH` | `logs/application/app.log` | Log file path (relative to project root or `DATA_DIR`). | +| `APP_LOG_MAX_FILE_SIZE` | `50M` | Max file size before rotation. Accepts: `50M`, `1G`, `512K`, or plain bytes. | +| `APP_LOG_RETENTION_DAYS` | `7` | Days to keep rotated application log files. | +| `APP_LOG_MAX_FILES` | `20` | Maximum rotated log file backups. | +| `CALL_LOG_RETENTION_DAYS` | `7` | Days to keep request/call log entries in the database. | +| `CALL_LOG_MAX_ENTRIES` | `10000` | Max call log entries in the in-memory buffer. | +| `CALL_LOGS_TABLE_MAX_ROWS` | `100000` | Max rows in the `call_logs` SQLite table before pruning. | +| `CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS` | `true` | Store stream chunks in pipeline artifacts when `call_log_pipeline_enabled=true`. | +| `CALL_LOG_PIPELINE_MAX_SIZE_KB` | `512` | Max pipeline call log artifact size in KB when `call_log_pipeline_enabled=true`. | +| `PROXY_LOGS_TABLE_MAX_ROWS` | `100000` | Max rows in the `proxy_logs` SQLite table before pruning. | --- diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f9a69755bb..5a46315fba 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -191,6 +191,8 @@ curl -s http://localhost:20128/api/cli-tools/openclaw-settings | jq '{installed, Set `APP_LOG_TO_FILE=true` in your `.env` file. Application logs are written under `logs/`. Request artifacts are stored under `${DATA_DIR}/call_logs/` when the call log pipeline is enabled in settings. +When pipeline capture is enabled, set `CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS=false` to omit +stream chunk payloads, or tune `CALL_LOG_PIPELINE_MAX_SIZE_KB` to change the artifact cap in KB. ### Check Provider Health diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 096dd0b340..39aceee320 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -32,6 +32,7 @@ import { } from "../services/errorClassifier.ts"; import { updateProviderConnection } from "@/lib/db/providers"; import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; +import { getCallLogPipelineCaptureStreamChunks } from "@/lib/logEnv"; import { logAuditEvent } from "@/lib/compliance"; import { extractProviderWarnings } from "@/lib/compliance/providerAudit"; import { handleBypassRequest } from "../utils/bypassHandler.ts"; @@ -1129,6 +1130,8 @@ export async function handleChatCore({ } const noLogEnabled = apiKeyInfo?.noLog === true; const detailedLoggingEnabled = !noLogEnabled && (await isDetailedLoggingEnabled()); + const capturePipelineStreamChunks = + detailedLoggingEnabled && getCallLogPipelineCaptureStreamChunks(); const skillRequestId = generateRequestId(); const pipelineSessionId = (clientRawRequest?.headers && typeof clientRawRequest.headers.get === "function" @@ -1310,7 +1313,7 @@ export async function handleChatCore({ // Create request logger for this session: sourceFormat_targetFormat_model const reqLogger = await createRequestLogger(sourceFormat, targetFormat, model, { enabled: detailedLoggingEnabled, - captureStreamChunks: detailedLoggingEnabled, + captureStreamChunks: capturePipelineStreamChunks, }); // 0. Log client raw request (before format conversion) diff --git a/src/lib/logEnv.ts b/src/lib/logEnv.ts index 897b57bbd4..f10d318034 100644 --- a/src/lib/logEnv.ts +++ b/src/lib/logEnv.ts @@ -6,6 +6,7 @@ const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024; const DEFAULT_APP_LOG_MAX_FILES = 20; const DEFAULT_CALL_LOG_MAX_ENTRIES = 10000; const DEFAULT_CALL_LOGS_TABLE_MAX_ROWS = 100000; +const DEFAULT_CALL_LOG_PIPELINE_MAX_SIZE_KB = 512; const DEFAULT_PROXY_LOGS_TABLE_MAX_ROWS = 100000; const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log"); @@ -15,6 +16,14 @@ function parsePositiveInt(value: string | undefined, fallback: number): number { return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } +function parseBoolean(value: string | undefined, fallback: boolean): boolean { + if (!value) return fallback; + const normalized = value.trim().toLowerCase(); + if (["1", "true", "yes", "on"].includes(normalized)) return true; + if (["0", "false", "no", "off"].includes(normalized)) return false; + return fallback; +} + export function parseFileSize(raw: string | undefined): number { if (!raw) return DEFAULT_APP_LOG_MAX_SIZE; const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i); @@ -68,6 +77,19 @@ export function getCallLogsTableMaxRows(): number { return parsePositiveInt(process.env.CALL_LOGS_TABLE_MAX_ROWS, DEFAULT_CALL_LOGS_TABLE_MAX_ROWS); } +export function getCallLogPipelineCaptureStreamChunks(): boolean { + return parseBoolean(process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS, true); +} + +export function getCallLogPipelineMaxSizeBytes(): number { + return ( + parsePositiveInt( + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB, + DEFAULT_CALL_LOG_PIPELINE_MAX_SIZE_KB + ) * 1024 + ); +} + export function getProxyLogsTableMaxRows(): number { return parsePositiveInt(process.env.PROXY_LOGS_TABLE_MAX_ROWS, DEFAULT_PROXY_LOGS_TABLE_MAX_ROWS); } diff --git a/src/lib/usage/callLogArtifacts.ts b/src/lib/usage/callLogArtifacts.ts index b48df4a20a..6ad1f26dcb 100644 --- a/src/lib/usage/callLogArtifacts.ts +++ b/src/lib/usage/callLogArtifacts.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { resolveDataDir } from "../dataPaths"; +import { getCallLogPipelineMaxSizeBytes } from "../logEnv"; const isCloud = typeof globalThis.caches === "object" && globalThis.caches !== null; const isBuildPhase = process.env.NEXT_PHASE === "phase-production-build"; @@ -10,6 +11,11 @@ const DATA_DIR = resolveDataDir({ isCloud }); export const CALL_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "call_logs"); export const MAX_CALL_LOG_ARTIFACT_BYTES = 512 * 1024; +const SIZE_LIMIT_EXCEEDED_REASON = "call_log_artifact_size_limit_exceeded"; +const OMITTED_FOR_SIZE_LIMIT = "[omitted: call log artifact size limit exceeded]"; +const STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT = + "[stream chunks omitted: call log artifact size limit exceeded]"; + export type CallLogDetailState = "none" | "ready" | "missing" | "corrupt" | "legacy-inline"; export type CallLogArtifact = { @@ -83,13 +89,13 @@ function truncateArtifactForStorage(artifact: CallLogArtifact): CallLogArtifact ...pipeline, streamChunks: { provider: pipeline.streamChunks.provider?.length - ? ["[stream chunks omitted: call log artifact size limit exceeded]"] + ? [STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT] : undefined, openai: pipeline.streamChunks.openai?.length - ? ["[stream chunks omitted: call log artifact size limit exceeded]"] + ? [STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT] : undefined, client: pipeline.streamChunks.client?.length - ? ["[stream chunks omitted: call log artifact size limit exceeded]"] + ? [STREAM_CHUNKS_OMITTED_FOR_SIZE_LIMIT] : undefined, }, }, @@ -104,59 +110,77 @@ function omitOversizedPipeline(artifact: CallLogArtifact): CallLogArtifact { pipeline: { error: { _omniroute_truncated: true, - reason: "call_log_artifact_size_limit_exceeded", + reason: SIZE_LIMIT_EXCEEDED_REASON, }, }, }; } +function getArtifactMaxBytes(artifact: CallLogArtifact): number { + return artifact.pipeline ? getCallLogPipelineMaxSizeBytes() : MAX_CALL_LOG_ARTIFACT_BYTES; +} + +function buildMinimalArtifactForSizeLimit(artifact: CallLogArtifact) { + return { + schemaVersion: artifact.schemaVersion, + summary: artifact.summary, + requestBody: OMITTED_FOR_SIZE_LIMIT, + responseBody: OMITTED_FOR_SIZE_LIMIT, + error: artifact.error ? OMITTED_FOR_SIZE_LIMIT : null, + pipeline: { + error: { + _omniroute_truncated: true, + reason: SIZE_LIMIT_EXCEEDED_REASON, + }, + }, + }; +} + +function serializeFinalSizeLimitFallback(artifact: CallLogArtifact, maxBytes: number): string { + const withSummary = JSON.stringify(buildMinimalArtifactForSizeLimit(artifact)); + if (Buffer.byteLength(withSummary) <= maxBytes) { + return withSummary; + } + + return JSON.stringify({ + schemaVersion: artifact.schemaVersion, + _omniroute_truncated: true, + reason: SIZE_LIMIT_EXCEEDED_REASON, + }); +} + function serializeArtifactForStorage(artifact: CallLogArtifact): string { + const maxBytes = getArtifactMaxBytes(artifact); const serialized = JSON.stringify(artifact, null, 2); - if (Buffer.byteLength(serialized) <= MAX_CALL_LOG_ARTIFACT_BYTES) { + if (Buffer.byteLength(serialized) <= maxBytes) { return serialized; } const truncated = JSON.stringify(truncateArtifactForStorage(artifact), null, 2); - if (Buffer.byteLength(truncated) <= MAX_CALL_LOG_ARTIFACT_BYTES) { + if (Buffer.byteLength(truncated) <= maxBytes) { return truncated; } const withoutPipeline = JSON.stringify(omitOversizedPipeline(artifact), null, 2); - if (Buffer.byteLength(withoutPipeline) <= MAX_CALL_LOG_ARTIFACT_BYTES) { + if (Buffer.byteLength(withoutPipeline) <= maxBytes) { return withoutPipeline; } const minimal = JSON.stringify( { ...omitOversizedPipeline(artifact), - requestBody: "[omitted: call log artifact size limit exceeded]", - responseBody: "[omitted: call log artifact size limit exceeded]", - error: artifact.error ? "[omitted: call log artifact size limit exceeded]" : null, + requestBody: OMITTED_FOR_SIZE_LIMIT, + responseBody: OMITTED_FOR_SIZE_LIMIT, + error: artifact.error ? OMITTED_FOR_SIZE_LIMIT : null, }, null, 2 ); - if (Buffer.byteLength(minimal) <= MAX_CALL_LOG_ARTIFACT_BYTES) { + if (Buffer.byteLength(minimal) <= maxBytes) { return minimal; } - return JSON.stringify( - { - schemaVersion: artifact.schemaVersion, - summary: artifact.summary, - requestBody: "[omitted: call log artifact size limit exceeded]", - responseBody: "[omitted: call log artifact size limit exceeded]", - error: artifact.error ? "[omitted: call log artifact size limit exceeded]" : null, - pipeline: { - error: { - _omniroute_truncated: true, - reason: "call_log_artifact_size_limit_exceeded", - }, - }, - }, - null, - 2 - ); + return serializeFinalSizeLimitFallback(artifact, maxBytes); } export function writeCallArtifact( diff --git a/tests/unit/call-log-cap.test.ts b/tests/unit/call-log-cap.test.ts index 7b3ac1fb0c..ed8d108238 100644 --- a/tests/unit/call-log-cap.test.ts +++ b/tests/unit/call-log-cap.test.ts @@ -9,6 +9,8 @@ process.env.DATA_DIR = TEST_DATA_DIR; process.env.CALL_LOG_RETENTION_DAYS = "3650"; process.env.CALL_LOG_MAX_ENTRIES = "100"; +const ORIGINAL_CALL_LOG_PIPELINE_MAX_SIZE_KB = process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB; + const core = await import("../../src/lib/db/core.ts"); const callLogs = await import("../../src/lib/usage/callLogs.ts"); const detailedLogs = await import("../../src/lib/db/detailedLogs.ts"); @@ -19,6 +21,14 @@ async function resetStorage() { fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); } +function restorePipelineEnv() { + if (ORIGINAL_CALL_LOG_PIPELINE_MAX_SIZE_KB === undefined) { + delete process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB; + } else { + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = ORIGINAL_CALL_LOG_PIPELINE_MAX_SIZE_KB; + } +} + function insertCallLog(row) { const db = core.getDbInstance(); db.prepare( @@ -66,11 +76,13 @@ function insertCallLog(row) { } test.beforeEach(async () => { + restorePipelineEnv(); process.env.CALL_LOG_RETENTION_DAYS = "3650"; await resetStorage(); }); test.after(() => { + restorePipelineEnv(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); @@ -541,6 +553,122 @@ test("saveCallLog omits oversized non-stream pipeline payloads to enforce artifa }); }); +test("saveCallLog honors CALL_LOG_PIPELINE_MAX_SIZE_KB for pipeline artifacts", async () => { + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = "8"; + const hugePayload = "x".repeat(32 * 1024); + + await callLogs.saveCallLog({ + id: "configured-pipeline-artifact-cap", + timestamp: "2026-03-31T10:07:00.000Z", + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + requestBody: { payload: "request" }, + responseBody: { output: "response" }, + pipelinePayloads: { + providerRequest: { body: hugePayload }, + providerResponse: { body: hugePayload }, + }, + }); + + const db = core.getDbInstance(); + const row = db + .prepare( + ` + SELECT artifact_relpath, artifact_size_bytes, detail_state + FROM call_logs WHERE id = ? + ` + ) + .get("configured-pipeline-artifact-cap"); + assert.equal((row as any).detail_state, "ready"); + assert.ok((row as any).artifact_size_bytes <= 8 * 1024); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", (row as any).artifact_relpath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.deepEqual(artifact.pipeline, { + error: { + _omniroute_truncated: true, + reason: "call_log_artifact_size_limit_exceeded", + }, + }); +}); + +test("saveCallLog falls back to a compact sentinel when the configured cap is very small", async () => { + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = "1"; + const hugePayload = "x".repeat(32 * 1024); + + await callLogs.saveCallLog({ + id: "tiny-pipeline-artifact-cap", + timestamp: "2026-03-31T10:07:30.000Z", + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: `openai/${"gpt".repeat(512)}`, + provider: "openai", + requestBody: { payload: "request" }, + responseBody: { output: "response" }, + pipelinePayloads: { + providerRequest: { body: hugePayload }, + providerResponse: { body: hugePayload }, + }, + }); + + const db = core.getDbInstance(); + const row = db + .prepare( + ` + SELECT artifact_relpath, artifact_size_bytes, detail_state + FROM call_logs WHERE id = ? + ` + ) + .get("tiny-pipeline-artifact-cap"); + assert.equal((row as any).detail_state, "ready"); + assert.ok((row as any).artifact_size_bytes <= 1024); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", (row as any).artifact_relpath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.deepEqual(artifact, { + schemaVersion: 4, + _omniroute_truncated: true, + reason: "call_log_artifact_size_limit_exceeded", + }); +}); + +test("CALL_LOG_PIPELINE_MAX_SIZE_KB does not cap artifacts without pipeline details", async () => { + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = "8"; + const requestBody = { payload: "x".repeat(16 * 1024) }; + + await callLogs.saveCallLog({ + id: "non-pipeline-artifact-ignores-pipeline-cap", + timestamp: "2026-03-31T10:08:00.000Z", + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + provider: "openai", + requestBody, + responseBody: { output: "response" }, + }); + + const db = core.getDbInstance(); + const row = db + .prepare( + ` + SELECT artifact_relpath, artifact_size_bytes, detail_state + FROM call_logs WHERE id = ? + ` + ) + .get("non-pipeline-artifact-ignores-pipeline-cap"); + assert.equal((row as any).detail_state, "ready"); + assert.ok((row as any).artifact_size_bytes > 8 * 1024); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", (row as any).artifact_relpath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.equal(artifact.requestBody.payload.length, requestBody.payload.length); +}); + test("saveCallLog logs and returns when sqlite persistence throws unexpectedly", async () => { const db = core.getDbInstance(); const originalPrepare = db.prepare; diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index 326de77cd0..1f60a0291a 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -48,6 +48,8 @@ const originalFetch = globalThis.fetch; const originalResponsesToOpenAI = getRequestTranslator(FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI); const originalSetTimeout = globalThis.setTimeout; const originalBackgroundConfig = getBackgroundDegradationConfig(); +const originalCallLogPipelineCaptureStreamChunks = + process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS; function noopLog() { return { @@ -58,6 +60,15 @@ function noopLog() { }; } +function restorePipelineCaptureEnv() { + if (originalCallLogPipelineCaptureStreamChunks === undefined) { + delete process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS; + } else { + process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS = + originalCallLogPipelineCaptureStreamChunks; + } +} + function toPlainHeaders(headers) { if (!headers) return {}; if (headers instanceof Headers) return Object.fromEntries(headers.entries()); @@ -353,6 +364,7 @@ async function invokeChatCore({ test.afterEach(async () => { globalThis.fetch = originalFetch; + restorePipelineCaptureEnv(); resetAccountSemaphores(); await waitForAsyncSideEffects(); await resetStorage(); @@ -360,12 +372,36 @@ test.afterEach(async () => { test.after(async () => { globalThis.fetch = originalFetch; + restorePipelineCaptureEnv(); resetAccountSemaphores(); await waitForAsyncSideEffects(); await resetStorage(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); +test("chatCore can disable pipeline stream chunk capture through environment", async () => { + process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS = "false"; + await settingsDb.updateSettings({ call_log_pipeline_enabled: true }); + + const { result } = await invokeChatCore({ + accept: "text/event-stream", + body: { + model: "gpt-4o-mini", + stream: true, + messages: [{ role: "user", content: "stream without chunk logging" }], + }, + }); + + assert.equal(result.success, true); + await result.response.text(); + await waitForAsyncSideEffects(); + + const detail = await waitFor(getLatestCallLog); + assert.ok(detail, "expected call log detail to be persisted"); + assert.ok(detail.pipelinePayloads, "expected pipeline payloads when capture is enabled"); + assert.equal((detail.pipelinePayloads as any).streamChunks, undefined); +}); + test("chatCore keeps Responses-native Codex payloads in native passthrough mode", async () => { const { call, result } = await invokeChatCore({ provider: "codex", diff --git a/tests/unit/log-retention.test.ts b/tests/unit/log-retention.test.ts index 049a80abfe..c8594c8f4e 100644 --- a/tests/unit/log-retention.test.ts +++ b/tests/unit/log-retention.test.ts @@ -187,3 +187,30 @@ test("getCallLogsTableMaxRows returns configured value", async () => { assert.equal(getCallLogsTableMaxRows(), 5); assert.equal(getProxyLogsTableMaxRows(), 5); }); + +test("call log pipeline env helpers parse stream chunk flag and size cap", async () => { + const originalCapture = process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS; + const originalMaxSize = process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB; + const { getCallLogPipelineCaptureStreamChunks, getCallLogPipelineMaxSizeBytes } = + await import("../../src/lib/logEnv.ts"); + + try { + process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS = "false"; + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = "256"; + + assert.equal(getCallLogPipelineCaptureStreamChunks(), false); + assert.equal(getCallLogPipelineMaxSizeBytes(), 256 * 1024); + } finally { + if (originalCapture === undefined) { + delete process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS; + } else { + process.env.CALL_LOG_PIPELINE_CAPTURE_STREAM_CHUNKS = originalCapture; + } + + if (originalMaxSize === undefined) { + delete process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB; + } else { + process.env.CALL_LOG_PIPELINE_MAX_SIZE_KB = originalMaxSize; + } + } +});