From 70a4d38d040f8e49407ea12a88a481cb1550f5d7 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 31 Mar 2026 10:22:52 -0300 Subject: [PATCH] Release v3.4.0 (Integration) (#861) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(settings): add unit tests for debugMode and hiddenSidebarItems Tests cover: - PATCH debugMode=true/false - PATCH hiddenSidebarItems with array values - Combined updates with both fields * test(e2e): add Playwright tests for settings toggles Tests cover: - Debug mode toggle on/off - Sidebar visibility toggle - Settings persistence after page reload * fix(tests): address code review issues - Unit tests: fix async/await for getSettings, use direct db functions - E2E tests: remove conditional logic, use Playwright auto-waiting assertions * feat(logging): unify request log retention and artifacts * docs: add dashboard settings toggles to CONTRIBUTING Add section documenting: - Debug Mode toggle (Settings → Advanced) - Sidebar Visibility toggle (Settings → General) * fix(cache): only inject prompt_cache_key for supported providers Only inject prompt_cache_key for providers that support prompt caching (Claude, Anthropic, ZAI, Qwen, DeepSeek). This fixes issue #848 where NVIDIA API rejected the parameter. * fix(model-sync): log only channel-level model changes * feat(providers): add 4 free models to opencode-zen * feat(providers): add explicit contextLength for opencode-zen free models * feat(providers): add contextLength for all opencode-zen models * feat: Improve the Chinese translation * fix: preserve client cache_control for all Claude-protocol providers Previously, the cache control preservation logic only recognized a hardcoded list of providers (claude, anthropic, zai, qwen, deepseek). This caused OmniRoute to inject its own cache_control markers for Claude-protocol providers not in that list (bailian-coding-plan, glm, minimax, minimax-cn, etc.), overwriting the client's cache markers. The fix checks both: 1. Known caching providers list (existing behavior) 2. Whether targetFormat === 'claude' (all Claude-protocol providers) This ensures all Claude-compatible providers properly preserve client cache_control headers when appropriate (Claude Code client, deterministic routing, etc.). Also removes unused CacheStatsCard from settings/components (duplicate of the one in cache/ page). Fixes cache token calculation for GLM, Minimax, and other Claude-compatible providers. Co-Authored-By: Claude Sonnet 4.6 * fix: pure passthrough for Claude→Claude when cache_control preserved The Claude passthrough path round-trips through OpenAI format (claude→openai→claude) for structural normalization. This strips cache_control markers from every content block since OpenAI format has no equivalent, causing ~42k cache creation tokens per request with zero cache reads. When preserveCacheControl is true (Claude Code client, "always" setting, or deterministic combo), skip the round-trip entirely and forward the body as-is. Claude Code sends well-formed Messages API payloads — the normalization was only needed for non-Code clients. Co-Authored-By: Claude Sonnet 4.6 * fix: restore CacheStatsCard — was not a duplicate The first commit incorrectly deleted CacheStatsCard from settings/components/ as a "duplicate". It's the only copy — both settings/page.tsx and cache/page.tsx import from this location. Restored the i18n-ized version from main. Co-Authored-By: Claude Sonnet 4.6 * fix(429): parse long quota reset times from error body - Parse XhYmZs format from antigravity error messages (e.g., 27h41m36s) - Dynamic retry-after threshold (60s default) instead of hardcoded 10s - Add parseRetryFromErrorText() in accountFallback.ts for body parsing - Fix 403 'verify your account' to trigger permanent deactivation - Add keyword matching for 'quota will reset', 'exhausted capacity' - Add unit tests for retry parsing and keyword matching Fixes #858 (Antigravity 429 handling) Fixes #832 (Qwen quota 429 - same underlying bug) * chore: bump version to v3.4.0-dev * fix(migrations): rename 013 to 014 to avoid collision with v3.3.11 * chore(docs): update CHANGELOG for v3.4.0 integrations * fix: Claude token refresh, Antigravity quota, and 429 rate-limit handling - Fix Claude OAuth token refresh to use form-urlencoded format (standard OAuth2) - Add anthropic-beta header required by Claude OAuth API - Switch Antigravity quota to use retrieveUserQuota API (same as Gemini CLI) - Parse quota reset time for all providers (not just Antigravity) - Add quota reset keywords to error classifier - Cap maximum retry time at 24 hours to prevent infinite wait Closes #836, #857, #858, #832 * fix(dashboard): resolve /dashboard/limits hanging UI with 70+ accounts via chunk parallelization (#784) --------- Co-authored-by: oyi77 Co-authored-by: R.D. Co-authored-by: kang-heewon Co-authored-by: gmw Co-authored-by: tombii Co-authored-by: Claude Sonnet 4.6 Co-authored-by: diegosouzapw --- .gitignore | 3 + CHANGELOG.md | 31 + CONTRIBUTING.md | 11 + README.md | 28 +- open-sse/config/providerRegistry.ts | 10 +- open-sse/executors/antigravity.ts | 42 +- open-sse/handlers/chatCore.ts | 88 ++- open-sse/services/accountFallback.ts | 47 ++ open-sse/services/errorClassifier.ts | 3 + open-sse/services/tokenRefresh.ts | 16 +- open-sse/services/usage.ts | 78 +-- open-sse/utils/cacheControlPolicy.ts | 17 +- open-sse/utils/error.ts | 11 + open-sse/utils/logger.ts | 13 +- open-sse/utils/requestLogger.ts | 308 +++++----- open-sse/utils/usageTracking.ts | 13 +- package-lock.json | 23 +- package.json | 3 +- .../settings/components/SystemStorageTab.tsx | 124 +--- .../usage/components/ProviderLimits/index.tsx | 11 +- src/app/api/logs/console/route.ts | 4 +- src/app/api/logs/detail/route.ts | 10 +- src/app/api/logs/export/route.ts | 8 +- .../api/providers/[id]/sync-models/route.ts | 145 ++++- src/app/api/settings/route.ts | 7 - src/app/api/storage/health/route.ts | 6 +- src/domain/types.ts | 3 +- src/i18n/messages/zh-CN.json | 265 ++++---- src/instrumentation-node.ts | 12 +- src/lib/compliance/index.ts | 105 +++- src/lib/consoleInterceptor.ts | 5 +- src/lib/db/core.ts | 26 +- src/lib/db/detailedLogs.ts | 8 +- src/lib/db/migrations/001_initial_schema.sql | 16 +- .../migrations/014_unified_log_artifacts.sql | 15 + src/lib/logEnv.ts | 61 ++ src/lib/logRotation.ts | 50 +- src/lib/proxyLogger.ts | 18 +- src/lib/usage/callLogs.ts | 577 +++++++++++------- src/lib/usage/migrations.ts | 226 ++++++- src/lib/usage/usageHistory.ts | 97 ++- src/server-init.ts | 12 +- src/shared/components/ConsoleLogViewer.tsx | 4 +- src/shared/components/RequestLoggerDetail.tsx | 36 +- src/shared/components/RequestLoggerV2.tsx | 6 +- src/shared/schemas/validation.ts | 2 - src/shared/utils/logger.ts | 7 +- src/shared/utils/structuredLogger.ts | 9 +- src/shared/validation/schemas.ts | 3 - src/shared/validation/settingsSchemas.ts | 3 - tests/e2e/settings-toggles.spec.ts | 51 ++ tests/unit/batch-b-final.test.mjs | 57 +- .../cache-control-claude-providers.test.mjs | 141 +++++ tests/unit/call-log-cap.test.mjs | 106 ++-- tests/unit/console-log-levels.test.mjs | 8 +- tests/unit/error-classification.test.mjs | 63 +- tests/unit/log-retention.test.mjs | 134 ++++ tests/unit/model-sync-route.test.mjs | 116 ++++ tests/unit/request-log-migration.test.mjs | 72 +++ tests/unit/settings-api.test.mjs | 67 ++ tests/unit/usage-analytics.test.mjs | 16 +- 61 files changed, 2376 insertions(+), 1081 deletions(-) create mode 100644 src/lib/db/migrations/014_unified_log_artifacts.sql create mode 100644 src/lib/logEnv.ts create mode 100644 tests/e2e/settings-toggles.spec.ts create mode 100644 tests/unit/cache-control-claude-providers.test.mjs create mode 100644 tests/unit/log-retention.test.mjs create mode 100644 tests/unit/model-sync-route.test.mjs create mode 100644 tests/unit/request-log-migration.test.mjs create mode 100644 tests/unit/settings-api.test.mjs diff --git a/.gitignore b/.gitignore index 0e170d179f..df550b50c9 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ vscode-extension/ # Compiled npm-package build artifact (not source, should not be in git) /app + +# IDEA +.idea/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 05f0b51a11..242f19b837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ ## [Unreleased] +> [!WARNING] +> **BREAKING CHANGE: request logging, retention, and logging environment variables have been redesigned.** +> On the first startup after upgrading, OmniRoute archives legacy request logs from `DATA_DIR/logs/`, legacy `DATA_DIR/call_logs/`, and `DATA_DIR/log.txt` into `DATA_DIR/log_archives/*.zip`, then removes the deprecated layout and switches to the new unified artifact format under `DATA_DIR/call_logs/`. + +### ✨ New Features + +- **Unified Request Log Artifacts:** Request logging now stores one SQLite index row plus one JSON artifact per request under `DATA_DIR/call_logs/`, with optional pipeline capture embedded in the same file. +- **Language:** Improved the Chinese translation (#855) +- **Opencode-Zen Models:** Added 4 free models to opencode-zen registry (#854) +- **Tests:** Added unit and E2E tests for settings toggles and bug fixes (#850) + +### 🐛 Bug Fixes + +- **429 Quota Parsing:** Parsed long quota reset times from error bodies to honor correct backoffs and prevent rate-limited account bans (#859) +- **Prompt Caching:** Preserved client `cache_control` headers for all Claude-protocol providers (like Minimax, GLM, and Bailian), correctly recognizing caching support (#856) +- **Model Sync Logs:** Reduced log spam by recording `sync-models` only when the channel actually modifies the list (#853) +- **Provider Quota & Token Parsing:** Switched Antigravity limits to use `retrieveUserQuota` natively and correctly mapped Claude token refresh payloads to URL-encoded forms (#862) +- **Rate-Limiting Stability:** Universalized the 429 Retry-After parsing architecture to cap provider-induced cooldowns at 24 hours max (#862) +- **Dashboard Limit Rendering:** Re-architected `/dashboard/limits` quota mapping to render immediately inside chunks, fixing a major UI freezing delay on accounts exceeding 70 active connections (#784) + +### ⚠️ Breaking Changes + +- **Request Log Layout:** Removed the old multi-file `DATA_DIR/logs/` request log sessions and the `DATA_DIR/log.txt` summary file. New requests are written as single JSON artifacts in `DATA_DIR/call_logs/YYYY-MM-DD/`. +- **Logging Environment Variables:** Replaced `LOG_*`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` with the new `APP_LOG_*` and `CALL_LOG_RETENTION_DAYS` configuration model. +- **Pipeline Toggle Setting:** Replaced the legacy `detailed_logs_enabled` setting with `call_log_pipeline_enabled`. New pipeline details are embedded inside the request artifact instead of being stored as separate `request_detail_logs` records. + +### 🛠️ Maintenance + +- **Legacy Request Log Upgrade Backup:** Upgrades now archive old `data/logs/`, legacy `data/call_logs/`, and `data/log.txt` layouts into `DATA_DIR/log_archives/*.zip` before removing the deprecated structure. +- **Streaming Usage Persistence:** Streaming requests now write a single `usage_history` row on completion instead of emitting a duplicate in-progress usage row with empty status metadata. + --- ## [3.3.11] - 2026-03-31 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f227558a07..c306f5894f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,17 @@ Key variables for development: | `INITIAL_PASSWORD` | `123456` | First login password | | `ENABLE_REQUEST_LOGS` | `false` | Enable debug request logs | +### Dashboard Settings + +The dashboard provides UI toggles for features that can also be configured via environment variables: + +| Setting Location | Toggle | Description | +| ------------------- | ------------------ | ------------------------------ | +| Settings → Advanced | Debug Mode | Enable debug request logs (UI) | +| Settings → General | Sidebar Visibility | Show/hide sidebar sections | + +These settings are stored in the database and persist across restarts, overriding env var defaults when set. + ### Running Locally ```bash diff --git a/README.md b/README.md index 1a5d4b79de..f3ae4c9bae 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,30 @@ _Your universal API proxy — one endpoint, 67+ providers, zero downtime. Now wi --- +## Breaking Change: Unified Logging Upgrade + +> [!WARNING] +> **This release changes both the on-disk request log layout and the logging environment variables.** +> +> If you are upgrading an existing instance: +> +> - Request logs now live in `DATA_DIR/call_logs/YYYY-MM-DD/` as **one JSON artifact per request**. +> - The old `DATA_DIR/logs/` session folders and `DATA_DIR/log.txt` summary file are removed. +> - On the first startup after upgrading, OmniRoute creates a safety backup at `DATA_DIR/log_archives/*.zip` before removing the deprecated request log layout. +> - Legacy logging env vars such as `LOG_TO_FILE`, `LOG_FILE_PATH`, `LOG_MAX_FILE_SIZE`, `LOG_RETENTION_DAYS`, `LOG_LEVEL`, `LOG_FORMAT`, `ENABLE_REQUEST_LOGS`, `CALL_LOGS_MAX`, `CALL_LOG_PAYLOAD_MODE`, and `PROXY_LOG_MAX_ENTRIES` are no longer supported. +> - Use the new env model instead: +> - `APP_LOG_TO_FILE` +> - `APP_LOG_FILE_PATH` +> - `APP_LOG_MAX_FILE_SIZE` +> - `APP_LOG_RETENTION_DAYS` +> - `APP_LOG_LEVEL` +> - `APP_LOG_FORMAT` +> - `CALL_LOG_RETENTION_DAYS` +> +> For release details and upgrade notes, see the [CHANGELOG](CHANGELOG.md). + +--- + ## 🆕 What's New in v3.0.0 > **Upgrading from v2.9.5?** — See the [full CHANGELOG](CHANGELOG.md#300--2026-03-22-release-candidate--not-yet-merged-to-main) for all changes. @@ -1811,7 +1835,9 @@ opencode **No request logs** -- Set `ENABLE_REQUEST_LOGS=true` in `.env` +- Request artifacts are written to `DATA_DIR/call_logs/` as one JSON file per request +- Enable pipeline capture from Dashboard → Logs → Request Logs if you need detailed per-stage payloads +- Set `APP_LOG_TO_FILE=true` if you also want application console logs in `logs/application/app.log` **Connection test shows "Invalid" for OpenAI-compatible providers** diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index 7b38617d74..9afef5ca2a 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -591,9 +591,13 @@ export const REGISTRY: Record = { authPrefix: "Bearer", defaultContextLength: 200000, models: [ - { id: "minimax-m2.5-free", name: "MiniMax M2.5 Free" }, - { id: "big-pickle", name: "Big Pickle" }, - { id: "gpt-5-nano", name: "GPT 5 Nano" }, + { id: "minimax-m2.5-free", name: "MiniMax M2.5 Free", contextLength: 204800 }, + { id: "big-pickle", name: "Big Pickle", contextLength: 200000 }, + { id: "gpt-5-nano", name: "GPT 5 Nano", contextLength: 400000 }, + { id: "mimo-v2-omni-free", name: "MiMo V2 Omni Free", contextLength: 262144 }, + { id: "mimo-v2-pro-free", name: "MiMo V2 Pro Free", contextLength: 1048576 }, + { id: "nemotron-3-super-free", name: "Nemotron 3 Super Free", contextLength: 1000000 }, + { id: "qwen3.6-plus-free", name: "Qwen 3.6 Plus Free", contextLength: 1048576 }, ], }, diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index 80f1e6105e..a0e402892c 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -2,7 +2,8 @@ import crypto from "crypto"; import { BaseExecutor, mergeUpstreamExtraHeaders } from "./base.ts"; import { PROVIDERS, OAUTH_ENDPOINTS, HTTP_STATUS } from "../config/constants.ts"; -const MAX_RETRY_AFTER_MS = 10000; +const MAX_RETRY_AFTER_MS = 60_000; +const LONG_RETRY_THRESHOLD_MS = 60_000; /** * Strip provider prefixes (e.g. "antigravity/model" → "model"). @@ -224,12 +225,15 @@ export class AntigravityExecutor extends BaseExecutor { signal, }); + // Parse retry time for 429/503 responses + let retryMs = null; + if ( response.status === HTTP_STATUS.RATE_LIMITED || response.status === HTTP_STATUS.SERVICE_UNAVAILABLE ) { // Try to get retry time from headers first - let retryMs = this.parseRetryHeaders(response.headers); + retryMs = this.parseRetryHeaders(response.headers); // If no retry time in headers, try to parse from error message body if (!retryMs) { @@ -243,12 +247,13 @@ export class AntigravityExecutor extends BaseExecutor { } } - if (retryMs && retryMs <= MAX_RETRY_AFTER_MS) { + if (retryMs && retryMs <= LONG_RETRY_THRESHOLD_MS) { + const effectiveRetryMs = Math.min(retryMs, MAX_RETRY_AFTER_MS); log?.debug?.( "RETRY", - `${response.status} with Retry-After: ${Math.ceil(retryMs / 1000)}s, waiting...` + `${response.status} with Retry-After: ${Math.ceil(effectiveRetryMs / 1000)}s, waiting...` ); - await new Promise((resolve) => setTimeout(resolve, retryMs)); + await new Promise((resolve) => setTimeout(resolve, effectiveRetryMs)); urlIndex--; continue; } @@ -291,6 +296,33 @@ export class AntigravityExecutor extends BaseExecutor { continue; } + // If we have a 429 with long retry time, embed it in response body + if ( + response.status === HTTP_STATUS.RATE_LIMITED && + retryMs && + retryMs > LONG_RETRY_THRESHOLD_MS + ) { + try { + const respBody = await response.clone().text(); + let obj; + try { + obj = JSON.parse(respBody); + } catch { + obj = {}; + } + obj.retryAfterMs = retryMs; + const modifiedBody = JSON.stringify(obj); + const modifiedResponse = new Response(modifiedBody, { + status: response.status, + headers: response.headers, + }); + return { response: modifiedResponse, url, headers, transformedBody }; + } catch (err) { + log?.warn?.("RETRY", `Failed to embed retryAfterMs: ${err}`); + // Fall back to original response + } + } + return { response, url, headers, transformedBody }; } catch (error) { lastError = error; diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 29fa3d2fcc..d8a46133d3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -23,7 +23,7 @@ import { import { HTTP_STATUS, PROVIDER_MAX_TOKENS } from "../config/constants.ts"; import { classifyProviderError, PROVIDER_ERROR_TYPES } from "../services/errorClassifier.ts"; import { updateProviderConnection } from "@/lib/db/providers"; -import { isDetailedLoggingEnabled, saveRequestDetailLog } from "@/lib/db/detailedLogs"; +import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; import { logAuditEvent } from "@/lib/compliance"; import { handleBypassRequest } from "../utils/bypassHandler.ts"; import { @@ -47,7 +47,10 @@ import { } from "@/lib/localDb"; import { getExecutor } from "../executors/index.ts"; import { getCacheControlSettings } from "@/lib/cacheControlSettings"; -import { shouldPreserveCacheControl } from "../utils/cacheControlPolicy.ts"; +import { + shouldPreserveCacheControl, + providerSupportsCaching, +} from "../utils/cacheControlPolicy.ts"; import { getCacheMetrics } from "@/lib/db/settings.ts"; import { @@ -533,6 +536,24 @@ export async function handleChatCore({ claudeCacheUsageMeta?: Record; }) => { const callLogId = generateRequestId(); + const pipelinePayloads = detailedLoggingEnabled ? reqLogger?.getPipelinePayloads?.() : null; + + if (pipelinePayloads) { + if (providerResponse !== undefined) { + pipelinePayloads.providerResponse = providerResponse as Record; + } + if (clientResponse !== undefined) { + pipelinePayloads.clientResponse = clientResponse as Record; + } + if (error) { + pipelinePayloads.error = { + ...(typeof pipelinePayloads.error === "object" && pipelinePayloads.error + ? (pipelinePayloads.error as Record) + : {}), + message: error, + }; + } + } saveCallLog({ id: callLogId, @@ -565,31 +586,8 @@ export async function handleChatCore({ apiKeyId: apiKeyInfo?.id || null, apiKeyName: apiKeyInfo?.name || null, noLog: noLogEnabled, + pipelinePayloads, }).catch(() => {}); - - if (!detailedLoggingEnabled) { - return; - } - - try { - saveRequestDetailLog({ - call_log_id: callLogId, - client_request: clientRawRequest?.body ?? body, - translated_request: providerRequest ?? null, - provider_response: providerResponse ?? null, - client_response: clientResponse ?? null, - provider, - model, - source_format: sourceFormat, - target_format: targetFormat, - duration_ms: Date.now() - startTime, - api_key_id: apiKeyInfo?.id || null, - no_log: noLogEnabled, - }); - } catch (err) { - const errMessage = err instanceof Error ? err.message : String(err); - log?.debug?.("DETAIL_LOG", `Failed to save detailed log: ${errMessage}`); - } }; // Primary path: merge client model id + alias target so config on either key applies; resolved @@ -697,6 +695,7 @@ export async function handleChatCore({ isCombo, comboStrategy, targetProvider: provider, + targetFormat, settings: { alwaysPreserveClientCache: cacheControlMode }, }); @@ -711,6 +710,15 @@ export async function handleChatCore({ if (nativeCodexPassthrough) { translatedBody = { ...body, _nativeCodexPassthrough: true }; log?.debug?.("FORMAT", "native codex passthrough enabled"); + } else if (isClaudePassthrough && preserveCacheControl) { + // Pure passthrough: when preserveCacheControl is true, forward the body + // as-is without any normalization. The OpenAI round-trip would strip + // cache_control markers; even prepareClaudeRequest can alter structure. + // Claude Code sends well-formed Messages API payloads — trust it. + translatedBody = { ...body }; + translatedBody._disableToolPrefix = true; + + log?.debug?.("FORMAT", "claude passthrough with cache_control preservation"); } else if (isClaudePassthrough) { // Claude OAuth expects the same Claude Code prompt + structural normalization // as the OpenAI-compatible chat path. Round-trip through OpenAI to reuse the @@ -955,9 +963,10 @@ export async function handleChatCore({ ? translatedBody : { ...translatedBody, model: modelToCall }; - // Inject prompt_cache_key for OpenAI providers if not already set + // Inject prompt_cache_key only for providers that support it if ( targetFormat === FORMATS.OPENAI && + providerSupportsCaching(provider) && !bodyToSend.prompt_cache_key && Array.isArray(bodyToSend.messages) && !["nvidia", "codex", "xai"].includes(provider) @@ -1190,6 +1199,31 @@ export async function handleChatCore({ console.warn( `[provider] Node ${connectionId} banned (${statusCode}) — disabling permanently` ); + } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { + await updateProviderConnection(connectionId, { + isActive: false, + testStatus: "deactivated", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` + ); + } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) { + const rateLimitedUntil = new Date(Date.now() + retryAfterMs).toISOString(); + await updateProviderConnection(connectionId, { + rateLimitedUntil: rateLimitedUntil, + testStatus: "credits_exhausted", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + healthCheckInterval: null, + lastHealthCheckAt: null, + }); + console.warn( + `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}` + ); } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { await updateProviderConnection(connectionId, { testStatus: "credits_exhausted", diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index be2be3e8c1..d485184976 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -229,6 +229,39 @@ function parseDelayString(value) { return isNaN(num) ? null : num * 1000; } +/** + * T07: Parse retry time from error text body with combined "XhYmZs" format. + * Examples: "Your quota will reset after 2h30m14s", "reset after 45m", "reset after 30s" + * Returns milliseconds or null if not parseable. + * + * @param {string} errorText - Error message text from response body + * @returns {number|null} Retry duration in milliseconds + */ +export function parseRetryFromErrorText(errorText) { + if (!errorText || typeof errorText !== "string") return null; + + const match = errorText.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i); + if (!match) { + // Also try the variant without "reset after": "will reset after XhYmZs" + const altMatch = errorText.match(/will reset after (\d+h)?(\d+m)?(\d+s)?/i); + if (!altMatch) return null; + return computeDurationMs(altMatch); + } + + return computeDurationMs(match); +} + +/** + * Compute total milliseconds from regex match groups (Xh)(Ym)(Zs) + */ +function computeDurationMs(match) { + let totalMs = 0; + if (match[1]) totalMs += parseInt(match[1], 10) * 3600 * 1000; // hours + if (match[2]) totalMs += parseInt(match[2], 10) * 60 * 1000; // minutes + if (match[3]) totalMs += parseInt(match[3], 10) * 1000; // seconds + return totalMs > 0 ? totalMs : null; +} + // ─── Error Classification ─────────────────────────────────────────────────── /** @@ -241,6 +274,8 @@ export function classifyErrorText(errorText) { if ( lower.includes("quota exceeded") || lower.includes("quota depleted") || + lower.includes("quota will reset") || + lower.includes("your quota will reset") || lower.includes("billing") ) { return RateLimitReason.QUOTA_EXHAUSTED; @@ -428,6 +463,9 @@ export function checkFallbackError( lowerError.includes("rate limit") || lowerError.includes("too many requests") || lowerError.includes("quota exceeded") || + lowerError.includes("quota will reset") || + lowerError.includes("exhausted your capacity") || + lowerError.includes("quota exhausted") || lowerError.includes("capacity") || lowerError.includes("overloaded") ) { @@ -443,6 +481,15 @@ export function checkFallbackError( }; } } + const retryFromBody = parseRetryFromErrorText(errorStr); + if (retryFromBody && retryFromBody > 60_000) { + return { + shouldFallback: true, + cooldownMs: retryFromBody, + newBackoffLevel: 0, + reason: RateLimitReason.RATE_LIMIT_EXCEEDED, + }; + } const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel); const reason = classifyErrorText(errorStr); return { diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index dc61b11390..00e9f368ac 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -46,6 +46,9 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) } if (statusCode === 402) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + if (statusCode === 403 && accountDeactivated) { + return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED; + } if (statusCode === 403) return PROVIDER_ERROR_TYPES.FORBIDDEN; if (statusCode >= 500) return PROVIDER_ERROR_TYPES.SERVER_ERROR; diff --git a/open-sse/services/tokenRefresh.ts b/open-sse/services/tokenRefresh.ts index 26c3e6d270..cb2f020e14 100644 --- a/open-sse/services/tokenRefresh.ts +++ b/open-sse/services/tokenRefresh.ts @@ -207,17 +207,21 @@ export async function refreshKimiCodingToken(refreshToken, log) { */ export async function refreshClaudeOAuthToken(refreshToken, log) { try { + // Standard OAuth2 token refresh uses form-urlencoded (not JSON) + const params = new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + client_id: PROVIDERS.claude.clientId, + }); + const response = await fetch(OAUTH_ENDPOINTS.anthropic.token, { method: "POST", headers: { - "Content-Type": "application/json", + "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json", + "anthropic-beta": "oauth-2025-04-20", }, - body: JSON.stringify({ - grant_type: "refresh_token", - refresh_token: refreshToken, - client_id: PROVIDERS.claude.clientId, - }), + body: params.toString(), }); if (!response.ok) { diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index 49b6ffd255..b4fe38f854 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -658,23 +658,33 @@ function getAntigravityPlanLabel(subscriptionInfo) { /** * Antigravity Usage - Fetch quota from Google Cloud Code API * Now calls loadCodeAssist ONCE (cached) and reuses for projectId + plan. + * Uses retrieveUserQuota API (same as Gemini CLI) for accurate quota data across all tiers. */ async function getAntigravityUsage(accessToken, providerSpecificData) { try { - // Single cached call for subscription info (provides both projectId and plan) const subscriptionInfo = await getAntigravitySubscriptionInfoCached(accessToken); const projectId = subscriptionInfo?.cloudaicompanionProject || null; - // Fetch quota data - const response = await fetch(ANTIGRAVITY_CONFIG.quotaApiUrl, { - method: "POST", - headers: { - Authorization: `Bearer ${accessToken}`, - "User-Agent": ANTIGRAVITY_CONFIG.userAgent, - "Content-Type": "application/json", - }, - body: JSON.stringify(projectId ? { project: projectId } : {}), - }); + if (!projectId) { + return { + plan: getAntigravityPlanLabel(subscriptionInfo), + message: "Antigravity project ID not available.", + }; + } + + // Use retrieveUserQuota API (same as Gemini CLI) - works correctly for both Free and Pro tiers + const response = await fetch( + "https://cloudcode-pa.googleapis.com/v1internal:retrieveUserQuota", + { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ project: projectId }), + signal: AbortSignal.timeout(10000), + } + ); if (response.status === 403) { return { message: "Antigravity access forbidden. Check subscription." }; @@ -685,54 +695,26 @@ async function getAntigravityUsage(accessToken, providerSpecificData) { } const data = await response.json(); - const dataObj = toRecord(data); - const modelEntries = toRecord(dataObj.models); const quotas: Record = {}; - // Parse model quotas (inspired by vscode-antigravity-cockpit) - if (Object.keys(modelEntries).length > 0) { - // Filter only recommended/important models (must match PROVIDER_MODELS ag ids) - const importantModels = [ - "claude-opus-4-6-thinking", - "claude-sonnet-4-6", - "gemini-3.1-pro-high", - "gemini-3.1-pro-low", - "gemini-3-flash", - "gpt-oss-120b-medium", - ]; + // Parse buckets from retrieveUserQuota response (same format as Gemini CLI) + if (Array.isArray(data.buckets)) { + for (const bucket of data.buckets) { + if (!bucket.modelId || bucket.remainingFraction == null) continue; - for (const [modelKey, infoValue] of Object.entries(modelEntries)) { - const info = toRecord(infoValue); - const quotaInfo = toRecord(info.quotaInfo); - // Skip models without quota info - if (Object.keys(quotaInfo).length === 0) { - continue; - } - - // Skip internal models and non-important models - if (info.isInternal === true || !importantModels.includes(modelKey)) { - continue; - } - - const remainingFraction = toNumber(quotaInfo.remainingFraction, 0); + const remainingFraction = toNumber(bucket.remainingFraction, 0); const remainingPercentage = remainingFraction * 100; - - // Convert percentage to used/total for UI compatibility - // QUOTA_NORMALIZED_BASE is an arbitrary base for converting fractions - // to integer used/total pairs that the dashboard UI can display as bars. const QUOTA_NORMALIZED_BASE = 1000; const total = QUOTA_NORMALIZED_BASE; const remaining = Math.round(total * remainingFraction); - const used = total - remaining; + const used = Math.max(0, total - remaining); - // Use modelKey as key (matches PROVIDER_MODELS id) - quotas[modelKey] = { + quotas[bucket.modelId] = { used, total, - resetAt: parseResetTime(quotaInfo.resetTime), + resetAt: parseResetTime(bucket.resetTime), remainingPercentage, unlimited: false, - displayName: typeof info.displayName === "string" ? info.displayName : modelKey, }; } } @@ -743,7 +725,7 @@ async function getAntigravityUsage(accessToken, providerSpecificData) { subscriptionInfo, }; } catch (error) { - return { message: `Antigravity error: ${error.message}` }; + return { message: `Antigravity error: ${(error as Error).message}` }; } } diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index 3f18e2cf6d..0810674bb1 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -90,10 +90,19 @@ export function isClaudeCodeClient(userAgent: string | null | undefined): boolea /** * Check if a provider supports prompt caching + * Supports caching if: + * 1. Provider is in the known caching providers list, OR + * 2. Provider uses Claude protocol (detected via targetFormat) */ -export function providerSupportsCaching(provider: string | null | undefined): boolean { +export function providerSupportsCaching( + provider: string | null | undefined, + targetFormat?: string | null +): boolean { if (!provider) return false; - return CACHING_PROVIDERS.has(provider.toLowerCase()); + if (CACHING_PROVIDERS.has(provider.toLowerCase())) return true; + // All Claude-protocol providers support prompt caching + if (targetFormat === "claude") return true; + return false; } /** @@ -121,12 +130,14 @@ export function shouldPreserveCacheControl({ isCombo, comboStrategy, targetProvider, + targetFormat, settings, }: { userAgent: string | null | undefined; isCombo: boolean; comboStrategy?: RoutingStrategyValue | null; targetProvider: string | null | undefined; + targetFormat?: string | null; settings?: CacheControlSettings; }): boolean { // User override takes precedence @@ -144,7 +155,7 @@ export function shouldPreserveCacheControl({ } // Target provider must support caching - if (!providerSupportsCaching(targetProvider)) { + if (!providerSupportsCaching(targetProvider, targetFormat)) { return false; } diff --git a/open-sse/utils/error.ts b/open-sse/utils/error.ts index 367d5b2730..d39b3c184b 100644 --- a/open-sse/utils/error.ts +++ b/open-sse/utils/error.ts @@ -122,6 +122,17 @@ export async function parseUpstreamError(response, provider = null) { retryAfterMs = parseAntigravityRetryTime(messageStr); } + // Also parse retry time for other providers (Qwen, etc.) with "quota will reset after XhYmZs" format + if (response.status === 429 && !retryAfterMs) { + retryAfterMs = parseAntigravityRetryTime(messageStr); + } + + // Cap maximum retry time at 24 hours to prevent infinite wait + const MAX_RETRY_MS = 24 * 60 * 60 * 1000; + if (retryAfterMs && retryAfterMs > MAX_RETRY_MS) { + retryAfterMs = MAX_RETRY_MS; + } + return { statusCode: response.status, message: messageStr, diff --git a/open-sse/utils/logger.ts b/open-sse/utils/logger.ts index 0620c752e8..b52f0b4128 100644 --- a/open-sse/utils/logger.ts +++ b/open-sse/utils/logger.ts @@ -2,7 +2,7 @@ * Structured console logger utility for omniroute. * * Provides consistent, machine-parseable log output across the codebase. - * Supports two output formats controlled by LOG_FORMAT env var: + * Supports two output formats controlled by APP_LOG_FORMAT env var: * - "text" (default): [LEVEL] [TAG] message {metadata} * - "json": Single-line JSON objects for log aggregators * @@ -18,17 +18,16 @@ * reqLog.info("AUTH", "Token refreshed", { provider: "claude" }); * * Environment variables: - * LOG_LEVEL — minimum level: debug | info | warn | error (default: info) - * LOG_FORMAT — output format: text | json (default: text) + * APP_LOG_LEVEL — minimum level: debug | info | warn | error (default: info) + * APP_LOG_FORMAT — output format: text | json (default: text) */ +import { getAppLogFormat, getAppLogLevel } from "../../src/lib/logEnv"; const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 }; -const currentLevel = - LEVELS[((typeof process !== "undefined" && process.env?.LOG_LEVEL) || "info").toLowerCase()] ?? - LEVELS.info; +const currentLevel = LEVELS[getAppLogLevel("info").toLowerCase()] ?? LEVELS.info; -const jsonFormat = typeof process !== "undefined" && process.env?.LOG_FORMAT === "json"; +const jsonFormat = getAppLogFormat("text") === "json"; let requestCounter = 0; diff --git a/open-sse/utils/requestLogger.ts b/open-sse/utils/requestLogger.ts index be4f7e91ce..c99549f5b0 100644 --- a/open-sse/utils/requestLogger.ts +++ b/open-sse/utils/requestLogger.ts @@ -1,92 +1,117 @@ -// Check if running in Node.js environment (has fs module) -const isNode = - typeof process !== "undefined" && process.versions?.node && typeof window === "undefined"; +type JsonRecord = Record; -// Check if logging is enabled via environment variable (default: false) -const LOGGING_ENABLED = - typeof process !== "undefined" && process.env?.ENABLE_REQUEST_LOGS === "true"; +type HeaderInput = + | Headers + | Record + | { entries?: () => IterableIterator<[string, string]> } + | null + | undefined; -let fs = null; -let path = null; -let LOGS_DIR = null; +export type RequestPipelinePayloads = { + clientRawRequest?: JsonRecord; + sourceRequest?: JsonRecord; + openaiRequest?: JsonRecord; + providerRequest?: JsonRecord; + providerResponse?: JsonRecord; + clientResponse?: JsonRecord; + error?: JsonRecord; + streamChunks?: { + provider?: string[]; + openai?: string[]; + client?: string[]; + }; +}; -// Lazy load Node.js modules (avoid top-level await) -async function ensureNodeModules() { - if (!isNode || !LOGGING_ENABLED || fs) return; - try { - fs = await import("fs"); - path = await import("path"); - const { resolveDataDir } = await import("../../src/lib/dataPaths"); - LOGS_DIR = path.join(resolveDataDir(), "logs"); - } catch { - // Running in non-Node environment (Worker, Browser, etc.) - } -} +type RequestLogger = { + sessionPath: null; + logClientRawRequest: (endpoint: unknown, body: unknown, headers?: HeaderInput) => void; + logRawRequest: (body: unknown, headers?: HeaderInput) => void; + logOpenAIRequest: (body: unknown) => void; + logTargetRequest: (url: unknown, headers: HeaderInput, body: unknown) => void; + logProviderResponse: ( + status: unknown, + statusText: unknown, + headers: HeaderInput, + body: unknown + ) => void; + appendProviderChunk: (chunk: string) => void; + appendOpenAIChunk: (chunk: string) => void; + logConvertedResponse: (body: unknown) => void; + appendConvertedChunk: (chunk: string) => void; + logError: (error: unknown, requestBody?: unknown) => void; + getPipelinePayloads: () => RequestPipelinePayloads | null; +}; -// Format timestamp for folder name: 20251228_143045 -function formatTimestamp(date = new Date()) { - const pad = (n) => String(n).padStart(2, "0"); - const y = date.getFullYear(); - const m = pad(date.getMonth() + 1); - const d = pad(date.getDate()); - const h = pad(date.getHours()); - const min = pad(date.getMinutes()); - const s = pad(date.getSeconds()); - return `${y}${m}${d}_${h}${min}${s}`; -} - -// Create log session folder: {sourceFormat}_{targetFormat}_{model}_{timestamp} -async function createLogSession(sourceFormat, targetFormat, model) { - await ensureNodeModules(); - if (!fs || !LOGS_DIR) return null; - - try { - await fs.promises.mkdir(LOGS_DIR, { recursive: true }); - - const timestamp = formatTimestamp(); - const safeModel = (model || "unknown").replace(/[/:]/g, "-"); - const folderName = `${sourceFormat}_${targetFormat}_${safeModel}_${timestamp}`; - const sessionPath = path.join(LOGS_DIR, folderName); - - await fs.promises.mkdir(sessionPath, { recursive: true }); - - return sessionPath; - } catch (err) { - console.log("[LOG] Failed to create log session:", err.message); - return null; - } -} - -// Write JSON file (async, fire-and-forget) -function writeJsonFile(sessionPath, filename, data) { - if (!fs || !sessionPath) return; - - const filePath = path.join(sessionPath, filename); - fs.promises - .writeFile(filePath, JSON.stringify(data, null, 2)) - .catch((err) => console.log(`[LOG] Failed to write ${filename}:`, err.message)); -} - -// Mask sensitive data in headers before writing to log files -function maskSensitiveHeaders(headers) { +function maskSensitiveHeaders(headers: HeaderInput): Record { if (!headers) return {}; - const masked = { ...headers }; + + const headerEntries = + typeof (headers as Headers).entries === "function" + ? Object.fromEntries((headers as Headers).entries()) + : { ...(headers as Record) }; + + const masked = { ...headerEntries }; const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token"]; for (const key of Object.keys(masked)) { const lowerKey = key.toLowerCase(); - if (sensitiveKeys.some((sk) => lowerKey.includes(sk))) { - const value = masked[key]; - if (value && value.length > 20) { - masked[key] = value.slice(0, 10) + "..." + value.slice(-5); - } + if (!sensitiveKeys.some((candidate) => lowerKey.includes(candidate))) { + continue; + } + + const value = masked[key]; + if (typeof value === "string" && value.length > 20) { + masked[key] = `${value.slice(0, 10)}...${value.slice(-5)}`; + } else if (value) { + masked[key] = "[REDACTED]"; } } + return masked; } -// No-op logger when logging is disabled -function createNoOpLogger() { +function createEmptyStreamChunks() { + return { + provider: [] as string[], + openai: [] as string[], + client: [] as string[], + }; +} + +function hasOwnValues(value: unknown): boolean { + return Boolean(value && typeof value === "object" && Object.keys(value as JsonRecord).length > 0); +} + +function compactPipelinePayloads( + payloads: RequestPipelinePayloads +): RequestPipelinePayloads | null { + const result: RequestPipelinePayloads = {}; + + for (const [key, value] of Object.entries(payloads)) { + if (value === null || value === undefined) { + continue; + } + + if (key === "streamChunks" && value && typeof value === "object") { + const chunkRecord = value as Record; + const compactedChunks = Object.fromEntries( + Object.entries(chunkRecord).filter( + ([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0 + ) + ); + if (Object.keys(compactedChunks).length > 0) { + result.streamChunks = compactedChunks as RequestPipelinePayloads["streamChunks"]; + } + continue; + } + + result[key as keyof RequestPipelinePayloads] = value as never; + } + + return hasOwnValues(result) ? result : null; +} + +function createNoOpLogger(): RequestLogger { return { sessionPath: null, logClientRawRequest() {}, @@ -99,151 +124,106 @@ function createNoOpLogger() { logConvertedResponse() {}, appendConvertedChunk() {}, logError() {}, + getPipelinePayloads() { + return null; + }, }; } -/** - * Create a new log session and return logger functions - * @param {string} sourceFormat - Source format from client (claude, openai, etc.) - * @param {string} targetFormat - Target format to provider (antigravity, gemini-cli, etc.) - * @param {string} model - Model name - * @returns {Promise} Promise that resolves to logger object with methods to log each stage - */ -export async function createRequestLogger(sourceFormat, targetFormat, model) { - // Return no-op logger if logging is disabled - if (!LOGGING_ENABLED) { - return createNoOpLogger(); - } - - // Wait for session to be created before returning logger - const sessionPath = await createLogSession(sourceFormat, targetFormat, model); +export async function createRequestLogger( + _sourceFormat?: string, + _targetFormat?: string, + _model?: string +): Promise { + const streamChunks = createEmptyStreamChunks(); + const payloads: RequestPipelinePayloads = { + streamChunks, + }; return { - get sessionPath() { - return sessionPath; - }, + sessionPath: null, - // 1. Log client raw request (before all conversion steps) logClientRawRequest(endpoint, body, headers = {}) { - writeJsonFile(sessionPath, "1_req_client.json", { + payloads.clientRawRequest = { timestamp: new Date().toISOString(), endpoint, headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 2. Log raw request from client (after initial conversion like responsesApi) logRawRequest(body, headers = {}) { - writeJsonFile(sessionPath, "2_req_source.json", { + payloads.sourceRequest = { timestamp: new Date().toISOString(), headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 3. Log OpenAI intermediate format (source → openai) logOpenAIRequest(body) { - writeJsonFile(sessionPath, "3_req_openai.json", { + payloads.openaiRequest = { timestamp: new Date().toISOString(), body, - }); + }; }, - // 4. Log target format request (openai → target) logTargetRequest(url, headers, body) { - writeJsonFile(sessionPath, "4_req_target.json", { + payloads.providerRequest = { timestamp: new Date().toISOString(), url, headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 5. Log provider response (for non-streaming or error) logProviderResponse(status, statusText, headers, body) { - const filename = "5_res_provider.json"; - writeJsonFile(sessionPath, filename, { + payloads.providerResponse = { timestamp: new Date().toISOString(), status, statusText, - headers: headers - ? typeof headers.entries === "function" - ? Object.fromEntries(headers.entries()) - : headers - : {}, + headers: maskSensitiveHeaders(headers), body, - }); + }; }, - // 5. Append streaming chunk to provider response (async) appendProviderChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "5_res_provider.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.provider.push(chunk); + } }, - // 6. Append OpenAI intermediate chunks (async) appendOpenAIChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "6_res_openai.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.openai.push(chunk); + } }, - // 7. Log converted response to client (for non-streaming) logConvertedResponse(body) { - writeJsonFile(sessionPath, "7_res_client.json", { + payloads.clientResponse = { timestamp: new Date().toISOString(), body, - }); + }; }, - // 7b. Append streaming chunk to converted response (async) appendConvertedChunk(chunk) { - if (!fs || !sessionPath) return; - const filePath = path.join(sessionPath, "7_res_client.txt"); - fs.promises.appendFile(filePath, chunk).catch(() => {}); + if (typeof chunk === "string" && chunk.length > 0) { + streamChunks.client.push(chunk); + } }, - // 8. Log error logError(error, requestBody = null) { - writeJsonFile(sessionPath, "6_error.json", { + payloads.error = { timestamp: new Date().toISOString(), - error: error?.message || String(error), - stack: error?.stack, - requestBody, - }); - }, - }; -} - -// Legacy logError (kept for backward compatibility, converted to async) -export function logError(provider, { error, url, model, requestBody }) { - if (!fs || !LOGS_DIR) return; - - const writeLog = async () => { - try { - await fs.promises.mkdir(LOGS_DIR, { recursive: true }); - - const date = new Date().toISOString().split("T")[0]; - const logPath = path.join(LOGS_DIR, `${provider}-${date}.log`); - - const logEntry = { - timestamp: new Date().toISOString(), - type: "error", - provider, - model, - url, - error: error?.message || String(error), - stack: error?.stack, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, requestBody, }; + }, - await fs.promises.appendFile(logPath, JSON.stringify(logEntry) + "\n"); - } catch (err) { - console.log("[LOG] Failed to write error log:", err.message); - } + getPipelinePayloads() { + return compactPipelinePayloads(payloads); + }, }; - - writeLog(); } + +export function logError(_provider: string, _entry: unknown) {} diff --git a/open-sse/utils/usageTracking.ts b/open-sse/utils/usageTracking.ts index 8c42a62026..ccf08709ce 100644 --- a/open-sse/utils/usageTracking.ts +++ b/open-sse/utils/usageTracking.ts @@ -2,7 +2,7 @@ * Token Usage Tracking - Extract, normalize, estimate and log token usage */ -import { saveRequestUsage, appendRequestLog } from "@/lib/usageDb"; +import { appendRequestLog } from "@/lib/usageDb"; import { getLoggedInputTokens, getLoggedOutputTokens, @@ -444,7 +444,8 @@ export function logUsage(provider, usage, model = null, connectionId = null, api console.log(msg); - // Save to usage DB with cache-read tracked separately from the main input counter. + // Streaming requests persist usage once in chatCore's completion callback. + // Keep this helper side-effect free apart from console visibility. const tokens = { input: inTokens, output: outTokens, @@ -452,13 +453,5 @@ export function logUsage(provider, usage, model = null, connectionId = null, api cacheCreation: cacheCreation || 0, reasoning: reasoning || 0, }; - saveRequestUsage({ - model, - provider, - connectionId, - apiKeyId: apiKeyInfo?.id || undefined, - apiKeyName: apiKeyInfo?.name || undefined, - tokens, - }).catch(() => {}); appendRequestLog({ model, provider, connectionId, tokens, status: "200 OK" }).catch(() => {}); } diff --git a/package-lock.json b/package-lock.json index 6d309a4591..c884d26fcb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "3.3.11", + "version": "3.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "3.3.11", + "version": "3.4.0", "hasInstallScript": true, "license": "MIT", "workspaces": [ @@ -44,6 +44,7 @@ "undici": "^7.19.2", "uuid": "^13.0.0", "wreq-js": "^2.0.1", + "yazl": "^3.3.1", "zod": "^4.3.6", "zustand": "^5.0.10" }, @@ -7654,6 +7655,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -20226,6 +20236,15 @@ "node": ">=8" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 208d434ef3..a67370006b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "3.3.11", + "version": "3.4.0", "description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": { @@ -115,6 +115,7 @@ "undici": "^7.19.2", "uuid": "^13.0.0", "wreq-js": "^2.0.1", + "yazl": "^3.3.1", "zod": "^4.3.6", "zustand": "^5.0.10" }, diff --git a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx index 342cb619e1..96c821dbcc 100644 --- a/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/SystemStorageTab.tsx @@ -18,11 +18,6 @@ export default function SystemStorageTab() { const [importStatus, setImportStatus] = useState({ type: "", message: "" }); const [confirmImport, setConfirmImport] = useState(false); const [pendingImportFile, setPendingImportFile] = useState(null); - const [maxCallLogs, setMaxCallLogs] = useState(10000); - const [maxCallLogsDraft, setMaxCallLogsDraft] = useState("10000"); - const [settingsLoading, setSettingsLoading] = useState(true); - const [maxCallLogsSaving, setMaxCallLogsSaving] = useState(false); - const [maxCallLogsStatus, setMaxCallLogsStatus] = useState({ type: "", message: "" }); const fileInputRef = useRef(null); const locale = useLocale(); const t = useTranslations("settings"); @@ -31,7 +26,10 @@ export default function SystemStorageTab() { driver: "sqlite", dbPath: "~/.omniroute/storage.sqlite", sizeBytes: 0, - retentionDays: 90, + retentionDays: { + app: 7, + call: 7, + }, lastBackupAt: null, }); @@ -59,27 +57,6 @@ export default function SystemStorageTab() { } }; - const loadSettings = async () => { - setSettingsLoading(true); - try { - const res = await fetch("/api/settings"); - if (!res.ok) return; - const data = await res.json(); - const value = - typeof data.maxCallLogs === "number" && - Number.isInteger(data.maxCallLogs) && - data.maxCallLogs > 0 - ? data.maxCallLogs - : 10000; - setMaxCallLogs(value); - setMaxCallLogsDraft(String(value)); - } catch (err) { - console.error("Failed to fetch settings:", err); - } finally { - setSettingsLoading(false); - } - }; - const handleManualBackup = async () => { setManualBackupLoading(true); setManualBackupStatus({ type: "", message: "" }); @@ -145,47 +122,8 @@ export default function SystemStorageTab() { useEffect(() => { loadStorageHealth(); - loadSettings(); }, []); - const handleSaveMaxCallLogs = async () => { - const parsed = Number.parseInt(maxCallLogsDraft, 10); - if (!Number.isInteger(parsed) || parsed <= 0) { - setMaxCallLogsStatus({ - type: "error", - message: "Enter a positive integer for the call log limit.", - }); - return; - } - - setMaxCallLogsSaving(true); - setMaxCallLogsStatus({ type: "", message: "" }); - try { - const res = await fetch("/api/settings", { - method: "PATCH", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ maxCallLogs: parsed }), - }); - const data = await res.json(); - if (!res.ok) { - throw new Error(data.error || "Failed to save call log limit"); - } - setMaxCallLogs(parsed); - setMaxCallLogsDraft(String(parsed)); - setMaxCallLogsStatus({ - type: "success", - message: "Call log retention limit saved.", - }); - } catch (err) { - setMaxCallLogsStatus({ - type: "error", - message: (err as Error).message || "Failed to save call log limit", - }); - } finally { - setMaxCallLogsSaving(false); - } - }; - const handleExport = async () => { setExportLoading(true); try { @@ -345,53 +283,23 @@ export default function SystemStorageTab() {
-
+
-

Call log retention limit

+

Log retention policy

- Keep only the most recent call log entries in SQLite. Older entries are pruned - automatically after each new request log is saved. + Request logs follow CALL_LOG_RETENTION_DAYS. Application and audit logs + follow APP_LOG_RETENTION_DAYS.

- - {maxCallLogs.toLocaleString()} - -
- -
- setMaxCallLogsDraft(e.target.value)} - disabled={settingsLoading || maxCallLogsSaving} - className="w-40 rounded-lg border border-border bg-bg-secondary px-3 py-2 text-sm text-text-main focus:outline-none focus:ring-1 focus:ring-primary/40" - aria-label="Call log retention limit" - /> - -
- - {maxCallLogsStatus.message && ( -
- {maxCallLogsStatus.message} +
+ + Call {storageHealth.retentionDays.call}d + + + App {storageHealth.retentionDays.app}d +
- )} +
{/* Export / Import */} diff --git a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.tsx b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.tsx index 9560014370..ef927c4333 100644 --- a/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.tsx +++ b/src/app/(dashboard)/dashboard/usage/components/ProviderLimits/index.tsx @@ -210,12 +210,16 @@ export default function ProviderLimits() { setCountdown(120); try { const conns = await fetchConnections(); + + // Show table layout immediately once connections are loaded (Issue #784) + setInitialLoading(false); + const usageConnections = conns.filter( (conn) => USAGE_SUPPORTED_PROVIDERS.includes(conn.provider) && (conn.authType === "oauth" || conn.authType === "apikey") ); - // Fix Issue #784: Fetch quotas in chunks of 5 to avoid spamming the backend/provider APIs and hanging the UI. + // Fix: Fetch quotas in chunks of 5 to avoid spamming the backend/provider APIs and hanging the UI. const chunkSize = 5; for (let i = 0; i < usageConnections.length; i += chunkSize) { const chunk = usageConnections.slice(i, i + chunkSize); @@ -225,14 +229,15 @@ export default function ProviderLimits() { console.error("Error refreshing all:", error); } finally { setRefreshingAll(false); + setInitialLoading(false); // Fallback to ensure skeleton is cleared } }, [refreshingAll, fetchConnections, fetchQuota]); useEffect(() => { const init = async () => { setInitialLoading(true); - await refreshAll(); - setInitialLoading(false); + // No longer await refreshAll here so we don't block the UI + refreshAll(); }; init(); }, []); // eslint-disable-line react-hooks/exhaustive-deps diff --git a/src/app/api/logs/console/route.ts b/src/app/api/logs/console/route.ts index 3c7f5a4dbd..96fb706825 100644 --- a/src/app/api/logs/console/route.ts +++ b/src/app/api/logs/console/route.ts @@ -12,7 +12,7 @@ import { NextRequest, NextResponse } from "next/server"; import { readFileSync, existsSync } from "fs"; -import { join } from "path"; +import { getAppLogFilePath } from "@/lib/logEnv"; const LEVEL_ORDER: Record = { trace: 5, @@ -34,7 +34,7 @@ const NUMERIC_LEVEL_MAP: Record = { }; function getLogFilePath(): string { - return process.env.LOG_FILE_PATH || join(process.cwd(), "logs", "application", "app.log"); + return getAppLogFilePath(); } function parseLevel(raw: string | number): string { diff --git a/src/app/api/logs/detail/route.ts b/src/app/api/logs/detail/route.ts index e2734b5cd3..4a85321e42 100644 --- a/src/app/api/logs/detail/route.ts +++ b/src/app/api/logs/detail/route.ts @@ -1,6 +1,6 @@ /** - * GET /api/logs/detail — List detailed request logs + current enabled flag - * POST /api/logs/detail — Enable/disable detailed logging + * GET /api/logs/detail — List legacy detailed request logs + current enabled flag + * POST /api/logs/detail — Enable/disable pipeline capture for unified call log artifacts */ import { NextRequest, NextResponse } from "next/server"; import { requireManagementAuth } from "@/lib/api/requireManagementAuth"; @@ -35,13 +35,13 @@ export async function POST(req: NextRequest) { const body = await req.json(); const enabled = body.enabled === true || body.enabled === "1"; - await updateSettings({ detailed_logs_enabled: enabled }); + await updateSettings({ call_log_pipeline_enabled: enabled }); return NextResponse.json({ success: true, enabled, message: enabled - ? "Detailed logging enabled. Pipeline bodies will be captured for new requests." - : "Detailed logging disabled.", + ? "Pipeline capture enabled. New request artifacts will include per-stage payloads." + : "Pipeline capture disabled.", }); } diff --git a/src/app/api/logs/export/route.ts b/src/app/api/logs/export/route.ts index afb0a24c3c..3ee0390ff9 100644 --- a/src/app/api/logs/export/route.ts +++ b/src/app/api/logs/export/route.ts @@ -17,18 +17,12 @@ export async function GET(request: Request) { let rows: unknown[] = []; let tableName = ""; - if (logType === "call-logs") { + if (logType === "call-logs" || logType === "request-logs") { tableName = "call_logs"; const stmt = db.prepare( "SELECT * FROM call_logs WHERE timestamp >= @since ORDER BY timestamp DESC" ); rows = stmt.all({ since }); - } else if (logType === "request-logs") { - tableName = "request_logs"; - const stmt = db.prepare( - "SELECT * FROM request_logs WHERE timestamp >= @since ORDER BY timestamp DESC" - ); - rows = stmt.all({ since }); } else if (logType === "proxy-logs") { tableName = "proxy_logs"; const stmt = db.prepare( diff --git a/src/app/api/providers/[id]/sync-models/route.ts b/src/app/api/providers/[id]/sync-models/route.ts index 7b6d8b858e..581cc6a73e 100644 --- a/src/app/api/providers/[id]/sync-models/route.ts +++ b/src/app/api/providers/[id]/sync-models/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; import { getProviderConnectionById } from "@/models"; -import { replaceCustomModels } from "@/lib/db/models"; +import { getCustomModels, replaceCustomModels } from "@/lib/db/models"; import { syncManagedAvailableModelAliases, usesManagedAvailableModels, @@ -13,11 +13,108 @@ import { } from "@/shared/services/modelSyncScheduler"; import { getModelsByProviderId } from "@/shared/constants/models"; +type JsonRecord = Record; + +function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +function toNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function normalizeModelForComparison(model: unknown) { + const record = asRecord(model); + const id = toNonEmptyString(record.id) || ""; + const name = toNonEmptyString(record.name) || id; + const source = toNonEmptyString(record.source) || "auto-sync"; + const apiFormat = toNonEmptyString(record.apiFormat) || "chat-completions"; + const supportedEndpoints = Array.isArray(record.supportedEndpoints) + ? Array.from( + new Set( + record.supportedEndpoints + .map((endpoint) => toNonEmptyString(endpoint)) + .filter((endpoint): endpoint is string => Boolean(endpoint)) + ) + ).sort() + : ["chat"]; + + return { + id, + name, + source, + apiFormat, + supportedEndpoints, + }; +} + +function summarizeModelChanges(previousModels: unknown, nextModels: unknown) { + const previousList = Array.isArray(previousModels) ? previousModels : []; + const nextList = Array.isArray(nextModels) ? nextModels : []; + + const previousMap = new Map( + previousList + .map((model) => normalizeModelForComparison(model)) + .filter((model) => model.id) + .map((model) => [model.id, JSON.stringify(model)]) + ); + const nextMap = new Map( + nextList + .map((model) => normalizeModelForComparison(model)) + .filter((model) => model.id) + .map((model) => [model.id, JSON.stringify(model)]) + ); + + let added = 0; + let removed = 0; + let updated = 0; + + for (const [id, nextValue] of nextMap.entries()) { + const previousValue = previousMap.get(id); + if (!previousValue) { + added += 1; + continue; + } + if (previousValue !== nextValue) { + updated += 1; + } + } + + for (const id of previousMap.keys()) { + if (!nextMap.has(id)) { + removed += 1; + } + } + + return { + added, + removed, + updated, + total: added + removed + updated, + }; +} + +function getModelSyncChannelLabel(connection: unknown) { + const record = asRecord(connection); + const providerSpecificData = asRecord(record.providerSpecificData); + + return ( + toNonEmptyString(record.displayName) || + toNonEmptyString(record.email) || + toNonEmptyString(providerSpecificData.tag) || + toNonEmptyString(record.name) || + toNonEmptyString(record.provider) || + (toNonEmptyString(record.id) ? `connection:${String(record.id).slice(0, 8)}` : null) || + "unknown" + ); +} + /** * POST /api/providers/[id]/sync-models * * Fetches the model list from a provider's /models endpoint and replaces the - * full custom models list for that provider. Logs the operation to call_logs. + * full custom models list for that provider. Successful syncs only write a + * call log when the fetched channel actually changes the stored model list. * * Used by: * - modelSyncScheduler (auto-sync on interval) @@ -40,8 +137,7 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: return NextResponse.json({ error: "Connection not found" }, { status: 404 }); } - // Use a human-readable provider name for logs - const providerLabel = connection.name || connection.provider || "unknown"; + const providerLabel = getModelSyncChannelLabel(connection); // Fetch models from the existing /api/providers/[id]/models endpoint const origin = new URL(request.url).origin; @@ -92,7 +188,9 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: })) .filter((m: any) => m.id && !registryIds.has(m.id)); + const previousModels = await getCustomModels(connection.provider); const replaced = await replaceCustomModels(connection.provider, models); + const modelChanges = summarizeModelChanges(previousModels, replaced); let syncedAliases = 0; if (usesManagedAvailableModels(connection.provider)) { @@ -103,29 +201,34 @@ export async function POST(request: Request, { params }: { params: Promise<{ id: syncedAliases = aliasSync.assignedAliases.length; } - // Log the successful sync - await saveCallLog({ - method: "GET", - path: `/api/providers/${id}/models`, - status: 200, - model: "model-sync", - provider: providerLabel, - sourceFormat: "-", - connectionId: id, - duration: Date.now() - start, - requestType: "model-sync", - responseBody: { - syncedModels: models.length, - syncedAliases, - provider: connection.provider, - }, - }); + if (modelChanges.total > 0) { + await saveCallLog({ + method: "GET", + path: `/api/providers/${id}/models`, + status: 200, + model: "model-sync", + provider: providerLabel, + sourceFormat: "-", + connectionId: id, + duration: Date.now() - start, + requestType: "model-sync", + responseBody: { + syncedModels: models.length, + syncedAliases, + provider: connection.provider, + channel: providerLabel, + modelChanges, + }, + }); + } return NextResponse.json({ ok: true, provider: connection.provider, syncedModels: replaced.length, syncedAliases, + modelChanges, + logged: modelChanges.total > 0, models: replaced, }); } catch (error: any) { diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 7bf98a878e..575090df5c 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -19,14 +19,12 @@ export async function GET() { setCliCompatProviders(settings.cliCompatProviders as string[]); } - const enableRequestLogs = process.env.ENABLE_REQUEST_LOGS === "true"; const runtimePorts = getRuntimePorts(); const cloudUrl = process.env.CLOUD_URL || process.env.NEXT_PUBLIC_CLOUD_URL || null; const machineId = await getConsistentMachineId(); return NextResponse.json({ ...safeSettings, - enableRequestLogs, hasPassword: !!password || !!process.env.INITIAL_PASSWORD, runtimePorts, apiPort: runtimePorts.apiPort, @@ -114,11 +112,6 @@ export async function PATCH(request) { setCliCompatProviders(body.cliCompatProviders || []); } - if ("maxCallLogs" in body) { - const { invalidateCallLogsMaxCache } = await import("@/lib/usage/callLogs"); - invalidateCallLogsMaxCache(); - } - // Sync cache control settings to runtime cache if ("alwaysPreserveClientCache" in body) { const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings"); diff --git a/src/app/api/storage/health/route.ts b/src/app/api/storage/health/route.ts index ad8aaa56fd..42a2c5f3e5 100644 --- a/src/app/api/storage/health/route.ts +++ b/src/app/api/storage/health/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server"; import path from "path"; import fs from "fs"; import { resolveDataDir } from "@/lib/dataPaths"; +import { getAppLogRetentionDays, getCallLogRetentionDays } from "@/lib/logEnv"; /** * GET /api/storage/health — Return database storage information. @@ -56,7 +57,10 @@ export async function GET() { sizeBytes, lastBackupAt, backupCount, - retentionDays: 90, + retentionDays: { + app: getAppLogRetentionDays(), + call: getCallLogRetentionDays(), + }, dataDir: dataDir.startsWith(homeDir) ? "~" + dataDir.slice(homeDir.length) : dataDir, }); } catch (error) { diff --git a/src/domain/types.ts b/src/domain/types.ts index ef516fa406..a2dc1b3ade 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -96,11 +96,10 @@ * @property {boolean} hasPassword - Whether a password has been set * @property {string} [theme] - UI theme * @property {string} [language] - UI language - * @property {boolean} [enableRequestLogs] - Whether request logging is enabled * @property {boolean} [enableSocks5Proxy] - Whether SOCKS5 proxy is allowed * @property {string} [instanceName] - Instance display name * @property {string} [corsOrigins] - Allowed CORS origins - * @property {number} [logRetentionDays] - Log retention in days + * @property {boolean} [call_log_pipeline_enabled] - Whether per-request pipeline capture is enabled * @property {string[]} [hiddenSidebarItems] - Sidebar entry ids hidden for visual decluttering */ diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json index b1a0ea54ff..3bce9576a0 100644 --- a/src/i18n/messages/zh-CN.json +++ b/src/i18n/messages/zh-CN.json @@ -36,7 +36,7 @@ "time": "时间", "details": "详情", "created": "已创建", - "lastUsed": "Last Refreshed", + "lastUsed": "最近刷新", "loadMore": "加载更多", "noResults": "没有找到结果", "reloadPage": "重新加载页面", @@ -60,17 +60,17 @@ "maintenanceServerIssues": "服务器当前存在异常，部分功能可能暂时不可用。", "maintenanceServerUnreachable": "服务器暂时无法连接，正在重新连接...", "accept": "接受", - "accountId": "Account ID", + "accountId": "账户 ID", "alias": "别名", - "apiKeyId": "API Key ID", + "apiKeyId": "API 密钥 ID", "apiKeyName": "API Key 名称", "apiKeySecret": "API Key 密钥", - "authorization": "Authorization", - "content-type": "Content Type", - "content-length": "Content Length", + "authorization": "授权", + "content-type": "内容类型", + "content-length": "内容长度", "cookie": "Cookie", "file": "文件", - "host": "Host", + "host": "主机", "id": "ID", "import": "导入", "limit": "限制", @@ -100,25 +100,25 @@ "hex": "Hex", "range": "范围", "component": "组件", - "redirect_uri": "Redirect URI", - "idempotency-key": "Idempotency Key", - "error_description": "Error Description", + "redirect_uri": "重定向 URI", + "idempotency-key": "幂等键", + "error_description": "错误描述", "code": "代码", "compatible": "兼容", - "chat-completions": "Chat Completions", + "chat-completions": "对话补全", "oauth": "OAuth", - "auth_token": "Auth Token", + "auth_token": "认证令牌", "crypto": "加密", "hours": "小时", "selfsigned": "自签名", - "proxy_id": "Proxy ID", - "proxyId": "Proxy ID", - "connectionId": "Connection ID", + "proxy_id": "代理 ID", + "proxyId": "代理 ID", + "connectionId": "连接 ID", "resolveConnectionId": "解析 Connection ID", "resolve_connection_id": "解析 Connection ID", - "scope_id": "Scope ID", - "scopeId": "Scope ID", - "jwtSecret": "JWT Secret", + "scope_id": "范围 ID", + "scopeId": "范围 ID", + "jwtSecret": "JWT 密钥", "keytar": "keytar", "better-sqlite3": "better-sqlite3", "undici": "undici", @@ -135,7 +135,7 @@ "TOOL_DENYLIST": "工具拒绝列表", "Failed to save pricing": "保存定价失败", "Failed to reset pricing": "重置定价失败", - "apikey": "API Key", + "apikey": "API 密钥", "http": "HTTP" }, "sidebar": { @@ -191,8 +191,8 @@ "themeOrange": "橙色", "themeCyan": "青色", "cliToolsShort": "工具", - "cache": "Cache", - "cacheShort": "Cache" + "cache": "缓存", + "cacheShort": "缓存" }, "themesPage": { "title": "主题", @@ -486,7 +486,7 @@ "configureEndpoint": "配置端点", "instructions": "使用说明", "modelMapping": "模型映射", - "baseUrl": "Base URL", + "baseUrl": "基础 URL", "apiKey": "API密钥", "configured": "已配置", "notConfigured": "未配置", @@ -968,11 +968,11 @@ "available": "可用端点", "cloudProxy": "云代理", "disableConfirm": "确定要禁用云代理吗？", - "baseUrl": "Base URL", + "baseUrl": "基础 URL", "apiKeyLabel": "API 密钥", "registeredKeys": "已注册密钥", - "chatCompletions": "Chat Completions", - "responses": "Responses", + "chatCompletions": "对话补全", + "responses": "响应", "listModels": "列出模型", "usingCloudProxy": "当前使用云代理", "usingLocalServer": "当前使用本地服务器", @@ -981,7 +981,7 @@ "enableCloud": "启用云端", "modelsAcrossEndpoints": "{endpoints} 个端点共提供 {models} 个模型", "chatDesc": "支持所有提供商的流式与非流式聊天", - "embeddings": "Embeddings", + "embeddings": "嵌入向量", "embeddingsDesc": "用于搜索和 RAG 流程的文本向量", "imageGeneration": "图像生成", "imageDesc": "根据文本提示生成图像", @@ -1037,7 +1037,7 @@ "cloudWorkerUnreachable": "无法连接到云 Worker。请确认云服务已运行（在 `/cloud` 中执行 `npm run dev`）。", "connectionFailed": "连接失败", "syncFailed": "同步云端数据失败", - "cloudflaredTitle": "Cloudflare Quick Tunnel", + "cloudflaredTitle": "Cloudflare 快速隧道", "cloudflaredDescription": "为当前端点创建 Cloudflare Quick Tunnel。", "cloudflaredUrlNotice": "创建一个临时的 Cloudflare Quick Tunnel。每次重启后 URL 都会变化。", "cloudflaredEnable": "启用 Tunnel", @@ -1357,7 +1357,7 @@ "passwordsMismatch": "密码不匹配", "setupComplete": "设置完成！", "goToDashboard": "转到仪表板→", - "welcomeDesc": "OmniRoute 是您的本地 AI API 代理。它通过负载平衡、故障转移和使用情况跟踪将请求路由到多个 AI 提供商。", + "welcomeDesc": "OmniRoute 是您的本地 AI API 代理。它通过负载均衡、故障转移和使用情况跟踪将请求路由到多个 AI 提供商。", "multiProvider": "多提供商", "usageTracking": "使用情况追踪", "apiKeyMgmt": "API密钥管理", @@ -1473,7 +1473,7 @@ "testSummary": "{passed}/{total} 通过，{failed} 失败", "nameLabel": "名称", "prefixLabel": "前缀", - "baseUrlLabel": "Base URL", + "baseUrlLabel": "基础 URL", "apiTypeLabel": "API类型", "prefixHint": "必填。模型名称使用的唯一前缀。", "nameHint": "必填。该节点的友好标签。", @@ -1677,12 +1677,12 @@ "allModelsAlreadyImported": "所有模型已导入", "noNewModelsToImport": "没有新模型可导入 — 所有模型已在注册表或自定义模型列表中", "skippingExistingModels": "跳过 {count} 个已有模型", - "applyCodexAuthLocal": "Apply auth", - "exportCodexAuthFile": "Export auth", - "codexAuthAppliedLocal": "Codex auth.json applied locally", - "codexAuthApplyFailed": "Failed to apply Codex auth.json locally", - "codexAuthExported": "Codex auth.json exported", - "codexAuthExportFailed": "Failed to export Codex auth.json" + "applyCodexAuthLocal": "应用认证", + "exportCodexAuthFile": "导出认证", + "codexAuthAppliedLocal": "Codex auth.json 已在本地应用", + "codexAuthApplyFailed": "在本地应用 Codex auth.json 失败", + "codexAuthExported": "Codex auth.json 已导出", + "codexAuthExportFailed": "导出 Codex auth.json 失败" }, "settings": { "title": "设置", @@ -1697,7 +1697,7 @@ "proxy": "代理", "pricing": "定价", "storage": "存储", - "policies": "政策", + "policies": "策略", "ipFilter": "IP过滤器", "comboDefaults": "组合默认值", "fallbackChains": "后备链", @@ -1715,7 +1715,7 @@ "hitRate": "命中率", "cacheEntries": "缓存条目", "circuitBreaker": "断路器", - "retryPolicy": "重试政策", + "retryPolicy": "重试策略", "maxRetries": "最大重试次数", "retryDelay": "重试延迟", "timeoutMs": "超时（毫秒）", @@ -1800,11 +1800,11 @@ "customDesc": "为所有请求设置固定的令牌预算", "adaptive": "自适应", "adaptiveDesc": "根据请求复杂性调整预算", - "effortNone": "无（0 代币）", - "effortLow": "低（1K 代币）", - "effortMedium": "中型（10K 代币）", - "effortHigh": "高（128K 代币）", - "tokenBudget": "代币预算", + "effortNone": "无（0 Tokens）", + "effortLow": "低（1K Tokens）", + "effortMedium": "中（10K Tokens）", + "effortHigh": "高（128K Tokens）", + "tokenBudget": "Token 预算", "tokens": "Tokens", "baseEffortLevel": "基本努力水平", "adaptiveHint": "自适应模式根据消息计数、工具使用情况和提示长度从此基本级别进行扩展。", @@ -1948,10 +1948,10 @@ "providerMaxRetriesAria": "{provider} 最大重试次数", "providerTimeoutAria": "{provider} 超时毫秒", "removeProviderOverrideAria": "删除 {provider} 覆盖", - "newProviderNamePlaceholder": "例如谷歌、开放式...", + "newProviderNamePlaceholder": "例如 google、openai...", "newProviderNameAria": "新的提供商名称", "retries": "重试", - "ms": "女士", + "ms": "毫秒", "saveComboDefaults": "保存组合默认值", "maxNestingDepth": "最大嵌套深度", "concurrencyPerModel": "并发/模型", @@ -1987,7 +1987,7 @@ "failures": "{count} 失败", "policiesLocked": "策略和锁定标识符", "allOperational": "所有系统均可运行——无停工或断路器跳闸", - "loadingPolicies": "正在加载政策...", + "loadingPolicies": "正在加载策略...", "lockedIdentifiers": "锁定标识符", "unlockedIdentifier": "解锁：{identifier}", "sinceDate": "自 {date} 起", @@ -2045,7 +2045,7 @@ "errorDuringRestore": "恢复期间发生错误", "errorDuringImport": "导入时发生错误", "modelPricing": "模型定价", - "modelPricingDesc": "配置每个模型的成本费率 • 所有费率均以美元/100 万代币为单位", + "modelPricingDesc": "配置每个模型的成本费率 • 所有费率均以美元/100 万 Tokens 为单位", "providers": "提供商", "registry": "登记处", "priced": "定价", @@ -2062,24 +2062,24 @@ "models": "模型", "moreProviders": "{count} 更多提供商", "withPricing": "配置定价", - "policiesCircuitBreakers": "政策与断路器", + "policiesCircuitBreakers": "策略与断路器", "activeIssuesDetected": "检测到活跃问题", "off": "关闭", "resetPricingConfirm": "将 {provider} 的所有定价重置为默认值？", "pricingDescInput": "输入：发送到模型的令牌", - "pricingDescOutput": "输出：生成的代币", + "pricingDescOutput": "输出：生成的 Tokens", "pricingDescCached": "缓存：重用输入（约输入率的 50%）", "pricingDescReasoning": "推理：思考标记（回到输出）", "pricingDescCacheWrite": "缓存写入：创建缓存条目（回退到输入）", - "pricingDescFormula": "成本 = (输入 × 输入率) + (输出 × 输出率) + (缓存 × 缓存率) 每百万代币。", + "pricingDescFormula": "成本 = (输入 × 输入率) + (输出 × 输出率) + (缓存 × 缓存率) 每百万 Tokens。", "pricingSettingsTitle": "定价设置", "totalModels": "模型总数", "active": "活跃", "costCalculation": "成本计算", "costCalculationDesc": "成本是根据为每个模型配置的令牌使用情况和定价费率计算的。", "pricingFormat": "定价格式", - "pricingFormatDesc": "所有费率均以美元/100 万代币为单位（每百万代币美元）。", - "tokenTypes": "代币类型", + "pricingFormatDesc": "所有费率均以美元/100 万 Tokens 为单位（每百万 Tokens 美元）。", + "tokenTypes": "Token 类型", "inputTokenDesc": "标准提示标记", "outputTokenDesc": "完成/响应标记", "cachedTokenDesc": "缓存输入令牌（通常为输入速率的 50%）", @@ -2095,17 +2095,21 @@ "comboDefaultsGuideTitle": "如何调整组合默认值", "comboDefaultsGuideHint1": "在低延迟流中保持较低的重试次数；仅增加长生成任务的超时。", "comboDefaultsGuideHint2": "当一个提供程序需要与全局默认值不同的超时/重试行为时，请使用提供程序覆盖。", - "debugToggle": "Enable Debug Mode", - "sidebarVisibilityToggle": "Show Sidebar Items", - "cacheSettings": "Cache Settings", - "semanticCache": "Semantic Cache", - "maxEntries": "Max Entries", - "ttlMinutes": "TTL (minutes)", - "strategy": "Strategy", - "preserveClientCache": "Preserve Client Cache", - "enabled": "Enabled", - "loading": "Loading...", - "save": "Save" + "debugToggle": "启用调试模式", + "sidebarVisibilityToggle": "显示侧边栏项目", + "cacheSettings": "缓存设置", + "semanticCache": "语义缓存", + "maxEntries": "最大条目数", + "ttlMinutes": "TTL（分钟）", + "strategy": "策略", + "preserveClientCache": "保留客户端缓存", + "enabled": "已启用", + "loading": "加载中...", + "save": "保存", + "autoDisableBannedAccounts": "自动禁用被封禁账户", + "autoDisableDescription": "若提供商连接返回特定的永久封禁信号（如 HTTP 403\"请验证您的账户\"），则将其永久标记为停用。这会将其从组合轮换中移除。", + "autoDisableThreshold": "封禁阈值", + "autoDisableThresholdDesc": "触发永久停用所需的连续封禁信号次数。" }, "translator": { "title": "翻译者", @@ -2380,7 +2384,7 @@ "loadingQuotas": "正在加载...", "account": "账户", "modelQuotas": "模型配额", - "lastUsed": "Last Refreshed", + "lastUsed": "最近刷新", "actions": "操作", "refreshQuota": "刷新配额", "today": "今天", @@ -2416,7 +2420,7 @@ "chooseAuthMethod": "选择您的身份验证方法：", "awsBuilderId": "AWS 构建器 ID", "awsIamIdentity": "AWS IAM 身份中心", - "googleAccount": "谷歌帐户", + "googleAccount": "Google 帐户", "githubAccount": "GitHub 帐户", "importToken": "导入令牌", "pasteToken": "从 Kiro IDE 粘贴刷新令牌。", @@ -2454,7 +2458,7 @@ }, "stats": { "usageOverview": "使用概述", - "outputTokens": "输出代币", + "outputTokens": "输出 Tokens", "totalCost": "总成本", "usageByModel": "按模型使用", "usageByAccount": "按帐户使用情况", @@ -2479,7 +2483,7 @@ "password": "密码", "unifiedProxy": "统一 AI API 代理", "unifiedAiApiProxy": "统一 AI API 代理", - "unifiedAiApiProxyDesc": "通过单个端点将请求路由到多个 AI 提供商。内置负载平衡、故障转移和使用情况跟踪。", + "unifiedAiApiProxyDesc": "通过单个端点将请求路由到多个 AI 提供商。内置负载均衡、故障转移和使用情况跟踪。", "passwordNotEnabled": "未启用密码保护", "loading": "正在加载...", "invalidPassword": "密码无效", @@ -2501,7 +2505,7 @@ "featureLoadBalancingTitle": "负载均衡", "featureLoadBalancingDesc": "智能分配请求", "featureUsageTrackingTitle": "使用情况追踪", - "featureUsageTrackingDesc": "监控成本和代币", + "featureUsageTrackingDesc": "监控成本和 Tokens", "resetPassword": "重置密码", "resetDescription": "选择一种方法来恢复对仪表板的访问权限", "stopServer": "停止 OmniRoute 服务器", @@ -2666,7 +2670,7 @@ "featureHealthText": "实时健康检查、提供商状态、断路器状态以及具有指数退避功能的自动速率限制检测。", "featureCliTitle": "CLI工具", "featureCliText": "可在仪表板中管理 IDE 配置、导出/导入备份、发现 Codex 配置文件并修改设置。", - "featureSecurityTitle": "安全和政策", + "featureSecurityTitle": "安全与策略", "featureSecurityText": "API 密钥身份验证、IP 过滤、提示注入防护、域策略、会话管理和审核日志记录。", "featureCloudSyncTitle": "云同步", "featureCloudSyncText": "将配置同步到 Cloudflare Workers，以便通过加密凭据和自动故障转移实现远程访问。", @@ -2683,7 +2687,7 @@ "useCaseUsageVisibilityTitle": "使用情况、成本和调试可见性", "useCaseUsageVisibilityText": "在“使用情况”和“分析”选项卡中按提供商、帐户和 API 密钥跟踪令牌和成本。", "clientCherryStudioTitle": "樱桃工作室", - "baseUrlLabel": "Base URL", + "baseUrlLabel": "基础 URL", "chatEndpointLabel": "聊天端点", "modelRecommendationLabel": "模型建议：显式前缀", "clientCodexTitle": "Codex / GitHub Copilot 模型", @@ -2777,7 +2781,7 @@ "privacySection4Text": "当你通过 OmniRoute 发起 API 调用时，请求会被转发到你配置的 AI 提供商（例如：OpenAI、Anthropic、Google）。这些提供商有各自的隐私政策，请查阅：", "privacyOpenAiPolicy": "OpenAI 隐私政策", "privacyAnthropicPolicy": "Anthropic 隐私政策", - "privacyGooglePolicy": "谷歌隐私政策", + "privacyGooglePolicy": "Google 隐私政策", "privacySection5Title": "5. 云同步（可选）", "privacySection5Text": "如果您启用可选的云同步功能，提供商配置和 API 密钥可能会传输到配置的云端点。此功能默认处于禁用状态，需要明确选择加入。", "privacySection6Title": "6. 日志记录", @@ -2900,59 +2904,72 @@ } }, "cache": { - "title": "Cache Management", - "description": "Monitor and manage semantic response cache, hit rates, and token savings.", - "refresh": "Refresh", - "clearAll": "Clear All", - "memoryEntries": "Memory Entries", - "dbEntries": "DB Entries", - "cacheHits": "Cache Hits", - "tokensSaved": "Tokens Saved", - "hitRate": "Hit Rate", - "performance": "Cache Performance", - "behavior": "Cache Behavior", - "idempotency": "Idempotency Layer", - "clearSuccess": "Cache cleared. {count} expired entries removed.", - "clearError": "Failed to clear cache.", - "unavailable": "Cache unavailable", - "unavailableDesc": "Could not fetch cache statistics. Make sure the server is running.", - "memoryEntriesSub": "In-memory LRU", - "dbEntriesSub": "Persisted (SQLite)", - "cacheHitsSub": "of {total} total", - "tokensSavedSub": "Estimated from hits", - "autoRefresh": "Auto-refreshes every {seconds}s", - "hits": "Hits", - "misses": "Misses", - "total": "Total", - "behaviorDeterministic": "Only non-streaming requests with temperature=0 are cached.", - "behaviorBypass": "Bypass with header {header}.", - "behaviorTwoTier": "Two-tier storage: in-memory LRU (fast) + SQLite (persistent across restarts).", - "behaviorTtl": "Default TTL: 30 minutes. Configure via {envVar}.", - "activeDedupKeys": "Active Dedup Keys", - "dedupWindow": "Dedup Window", - "promptCache": "Prompt Cache (Provider-Side)", - "cachedRequests": "Cached Requests", - "cacheHitRate": "Cache Hit Rate", - "cachedTokens": "Cached Tokens", - "cacheCreationTokens": "Cache Creation Tokens", - "byProvider": "Breakdown by Provider", - "provider": "Provider", - "requests": "Requests", - "inputTokens": "Input Tokens", - "cachedTokensCol": "Cached", - "cacheCreation": "Creation", - "trend24h": "Cache Trend (24h)", - "cached": "Cached", - "overview": "Overview", - "entries": "Entries", - "searchEntries": "Search entries...", - "search": "Search", - "loading": "Loading...", - "noEntries": "No cache entries found", - "signature": "Signature", - "model": "Model", - "created": "Created", - "expires": "Expires", - "actions": "Actions" + "title": "缓存管理", + "description": "监控和管理语义响应缓存、命中率及 Tokens 节省情况。", + "refresh": "刷新", + "clearAll": "清空全部", + "memoryEntries": "内存条目", + "dbEntries": "数据库条目", + "cacheHits": "缓存命中数", + "tokensSaved": "节省的 Tokens", + "hitRate": "命中率", + "performance": "缓存性能", + "behavior": "缓存行为", + "idempotency": "幂等层", + "clearSuccess": "缓存已清除。已删除 {count} 条过期条目。", + "clearError": "清除缓存失败。", + "unavailable": "缓存不可用", + "unavailableDesc": "无法获取缓存统计信息。请确保服务器正在运行。", + "memoryEntriesSub": "内存 LRU", + "dbEntriesSub": "已持久化（SQLite）", + "cacheHitsSub": "共 {total} 次", + "tokensSavedSub": "根据命中次数估算", + "autoRefresh": "每 {seconds} 秒自动刷新", + "hits": "命中次数", + "misses": "未命中次数", + "total": "总计", + "behaviorDeterministic": "仅缓存 temperature=0 的非流式请求。", + "behaviorBypass": "通过请求头 {header} 绕过缓存。", + "behaviorTwoTier": "双层存储：内存 LRU（快速）+ SQLite（重启后持久化）。", + "behaviorTtl": "默认 TTL：30 分钟。可通过 {envVar} 配置。", + "activeDedupKeys": "活跃去重键", + "dedupWindow": "去重窗口", + "promptCache": "Prompt 缓存（提供商侧）", + "cachedRequests": "缓存请求数", + "cacheHitRate": "缓存命中率", + "cachedTokens": "缓存 Tokens", + "cacheCreationTokens": "缓存创建 Tokens", + "byProvider": "按提供商分类", + "provider": "提供商", + "requests": "请求数", + "inputTokens": "输入 Tokens", + "cachedTokensCol": "已缓存", + "cacheCreation": "创建", + "trend24h": "缓存趋势（24 小时）", + "cached": "已缓存", + "overview": "概览", + "entries": "条目", + "searchEntries": "搜索条目...", + "search": "搜索", + "loading": "加载中...", + "noEntries": "未找到缓存条目", + "signature": "签名", + "model": "模型", + "created": "创建时间", + "expires": "到期时间", + "actions": "操作", + "cacheCreationWrite": "缓存创建（写入）", + "cacheMetrics": "Prompt 缓存指标", + "cacheReuseRatio": "缓存复用率", + "cacheReuseRatioDesc": "缓存 Tokens / 输入 Tokens 总量", + "cachedShort": "已缓存", + "cachedTokensRead": "缓存 Tokens（读取）", + "estCostSaved": "预估节省费用", + "inputShort": "输入", + "requestsShort": "请求", + "resetMetrics": "重置指标", + "resetting": "正在重置...", + "withCacheControl": "含缓存控制", + "writeShort": "写入" } } diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 529b3aa451..8ecf14c4a3 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -65,6 +65,9 @@ async function ensureSecrets(): Promise { export async function registerNodejs(): Promise { await ensureSecrets(); + // Trigger request-log layout migration during startup, before any request hits usageDb. + await import("@/lib/usage/migrations"); + const { initConsoleInterceptor } = await import("@/lib/consoleInterceptor"); initConsoleInterceptor(); @@ -127,7 +130,14 @@ export async function registerNodejs(): Promise { console.log("[COMPLIANCE] Audit log table initialized"); const cleanup = cleanupExpiredLogs(); - if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) { + if ( + cleanup.deletedUsage || + cleanup.deletedCallLogs || + cleanup.deletedProxyLogs || + cleanup.deletedRequestDetailLogs || + cleanup.deletedAuditLogs || + cleanup.deletedMcpAuditLogs + ) { console.log("[COMPLIANCE] Expired log cleanup:", cleanup); } } catch (err: unknown) { diff --git a/src/lib/compliance/index.ts b/src/lib/compliance/index.ts index f5577497a2..df7ae54407 100644 --- a/src/lib/compliance/index.ts +++ b/src/lib/compliance/index.ts @@ -2,7 +2,7 @@ * Compliance Controls — T-43 * * Implements compliance features: - * - LOG_RETENTION_DAYS: automatic log cleanup + * - APP_LOG_RETENTION_DAYS / CALL_LOG_RETENTION_DAYS: automatic log cleanup * - noLog opt-out per API key * - audit_log table for administrative actions * @@ -10,6 +10,7 @@ */ import { getDbInstance } from "../db/core"; +import { getAppLogRetentionDays, getCallLogRetentionDays } from "../logEnv"; /** @returns {import("better-sqlite3").Database | null} */ function getDb() { @@ -20,8 +21,6 @@ function getDb() { } } -const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "90", 10); - /** * Initialize the audit_log table. */ @@ -212,60 +211,122 @@ export function isNoLog(apiKeyId: string) { // ─── Log Retention / Cleanup ──────────────── /** - * Get the configured retention period. - * @returns {number} Days + * Get the configured retention periods. */ export function getRetentionDays() { - return LOG_RETENTION_DAYS; + return { + app: getAppLogRetentionDays(), + call: getCallLogRetentionDays(), + }; } /** - * Clean up logs older than LOG_RETENTION_DAYS. + * Clean up logs using split APP/CALL retention windows. * Should be called periodically (e.g. daily cron or on startup). * - * @returns {{ deletedUsage: number, deletedCallLogs: number, deletedAuditLogs: number }} + * @returns {{ + * deletedUsage: number, + * deletedCallLogs: number, + * deletedProxyLogs: number, + * deletedRequestDetailLogs: number, + * deletedAuditLogs: number, + * deletedMcpAuditLogs: number, + * appRetentionDays: number, + * callRetentionDays: number + * }} */ export function cleanupExpiredLogs() { const db = getDb(); - if (!db) return { deletedUsage: 0, deletedCallLogs: 0, deletedAuditLogs: 0 }; + const appRetentionDays = getAppLogRetentionDays(); + const callRetentionDays = getCallLogRetentionDays(); - const cutoff = new Date(Date.now() - LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000).toISOString(); + if (!db) { + return { + deletedUsage: 0, + deletedCallLogs: 0, + deletedProxyLogs: 0, + deletedRequestDetailLogs: 0, + deletedAuditLogs: 0, + deletedMcpAuditLogs: 0, + appRetentionDays, + callRetentionDays, + }; + } + + const callCutoff = new Date(Date.now() - callRetentionDays * 24 * 60 * 60 * 1000).toISOString(); + const appCutoff = new Date(Date.now() - appRetentionDays * 24 * 60 * 60 * 1000).toISOString(); let deletedUsage = 0; let deletedCallLogs = 0; + let deletedProxyLogs = 0; + let deletedRequestDetailLogs = 0; let deletedAuditLogs = 0; + let deletedMcpAuditLogs = 0; try { - // Clean usage_history - const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(cutoff); + const r1 = db.prepare("DELETE FROM usage_history WHERE timestamp < ?").run(callCutoff); deletedUsage = r1.changes; } catch { /* table may not exist */ } try { - // Clean call_logs - const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(cutoff); + const r2 = db.prepare("DELETE FROM call_logs WHERE timestamp < ?").run(callCutoff); deletedCallLogs = r2.changes; } catch { /* table may not exist */ } try { - // Clean audit_log (keep longer, 2x retention) - const auditCutoff = new Date( - Date.now() - LOG_RETENTION_DAYS * 2 * 24 * 60 * 60 * 1000 - ).toISOString(); - const r3 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(auditCutoff); - deletedAuditLogs = r3.changes; + const r3 = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?").run(callCutoff); + deletedProxyLogs = r3.changes; + } catch { + /* table may not exist */ + } + + try { + const r4 = db.prepare("DELETE FROM request_detail_logs WHERE timestamp < ?").run(callCutoff); + deletedRequestDetailLogs = r4.changes; + } catch { + /* legacy table may not exist */ + } + + try { + const r5 = db.prepare("DELETE FROM audit_log WHERE timestamp < ?").run(appCutoff); + deletedAuditLogs = r5.changes; + } catch { + /* table may not exist */ + } + + try { + const r6 = db.prepare("DELETE FROM mcp_tool_audit WHERE created_at < ?").run(appCutoff); + deletedMcpAuditLogs = r6.changes; } catch { /* table may not exist */ } logAuditEvent({ action: "compliance.cleanup", - details: { deletedUsage, deletedCallLogs, deletedAuditLogs, retentionDays: LOG_RETENTION_DAYS }, + details: { + deletedUsage, + deletedCallLogs, + deletedProxyLogs, + deletedRequestDetailLogs, + deletedAuditLogs, + deletedMcpAuditLogs, + appRetentionDays, + callRetentionDays, + }, }); - return { deletedUsage, deletedCallLogs, deletedAuditLogs }; + return { + deletedUsage, + deletedCallLogs, + deletedProxyLogs, + deletedRequestDetailLogs, + deletedAuditLogs, + deletedMcpAuditLogs, + appRetentionDays, + callRetentionDays, + }; } diff --git a/src/lib/consoleInterceptor.ts b/src/lib/consoleInterceptor.ts index 21d90a2101..5e8f146ffc 100644 --- a/src/lib/consoleInterceptor.ts +++ b/src/lib/consoleInterceptor.ts @@ -12,9 +12,10 @@ import { appendFileSync, existsSync, mkdirSync } from "fs"; import { dirname, resolve } from "path"; +import { getAppLogFilePath, getAppLogToFile } from "./logEnv"; -const logToFile = process.env.LOG_TO_FILE !== "false"; -const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log"); +const logToFile = getAppLogToFile(); +const logFilePath = resolve(getAppLogFilePath()); declare global { var __omnirouteConsoleInterceptorInit: boolean | undefined; diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 399cf9c09c..14d9ee8448 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -173,7 +173,9 @@ const SCHEMA_SQL = ` combo_name TEXT, request_body TEXT, response_body TEXT, - error TEXT + error TEXT, + artifact_relpath TEXT, + has_pipeline_details INTEGER DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); @@ -376,6 +378,27 @@ function ensureUsageHistoryColumns(db: SqliteDatabase) { } } +function ensureCallLogsColumns(db: SqliteDatabase) { + try { + const columns = db.prepare("PRAGMA table_info(call_logs)").all() as Array<{ + name?: string; + }>; + const columnNames = new Set(columns.map((column) => String(column.name ?? ""))); + + if (!columnNames.has("artifact_relpath")) { + db.exec("ALTER TABLE call_logs ADD COLUMN artifact_relpath TEXT"); + console.log("[DB] Added call_logs.artifact_relpath column"); + } + if (!columnNames.has("has_pipeline_details")) { + db.exec("ALTER TABLE call_logs ADD COLUMN has_pipeline_details INTEGER DEFAULT 0"); + console.log("[DB] Added call_logs.has_pipeline_details column"); + } + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.warn("[DB] Failed to verify call_logs schema:", message); + } +} + export function getDbInstance(): SqliteDatabase { const existing = getDb(); if (existing) return existing; @@ -468,6 +491,7 @@ export function getDbInstance(): SqliteDatabase { db.exec(SCHEMA_SQL); ensureProviderConnectionsColumns(db); ensureUsageHistoryColumns(db); + ensureCallLogsColumns(db); // ── Versioned Migrations ── // Auto-seed 001 as applied (the inline SCHEMA_SQL already created these tables) diff --git a/src/lib/db/detailedLogs.ts b/src/lib/db/detailedLogs.ts index 9de0131e16..d879cf96b1 100644 --- a/src/lib/db/detailedLogs.ts +++ b/src/lib/db/detailedLogs.ts @@ -1,9 +1,9 @@ /** * Detailed Request Logs DB Layer (#378) * - * Saves full request/response bodies at each pipeline stage. - * Ring-buffer of 500 entries enforced by SQL trigger in migration 006. - * Only active when settings.detailed_logs_enabled = "1". + * Legacy compatibility layer for detailed request logs. + * New requests now store pipeline details inside unified call log artifacts. + * This module remains available for reading historical request_detail_logs rows. */ import { v4 as uuidv4 } from "uuid"; import { getDbInstance } from "./core"; @@ -37,7 +37,7 @@ export interface RequestDetailLog { export async function isDetailedLoggingEnabled(): Promise { try { const settings = await getSettings(); - const val = settings.detailed_logs_enabled; + const val = settings.call_log_pipeline_enabled; return val === true || val === "1" || val === "true"; } catch { return false; diff --git a/src/lib/db/migrations/001_initial_schema.sql b/src/lib/db/migrations/001_initial_schema.sql index 4f128b8ab7..49c33a09c3 100644 --- a/src/lib/db/migrations/001_initial_schema.sql +++ b/src/lib/db/migrations/001_initial_schema.sql @@ -109,8 +109,8 @@ CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); CREATE TABLE IF NOT EXISTS call_logs ( - id TEXT PRIMARY KEY, - timestamp TEXT NOT NULL, + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, method TEXT, path TEXT, status INTEGER, @@ -124,11 +124,13 @@ CREATE TABLE IF NOT EXISTS call_logs ( source_format TEXT, target_format TEXT, api_key_id TEXT, - api_key_name TEXT, - combo_name TEXT, - request_body TEXT, - response_body TEXT, - error TEXT + api_key_name TEXT, + combo_name TEXT, + request_body TEXT, + response_body TEXT, + error TEXT, + artifact_relpath TEXT, + has_pipeline_details INTEGER DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_cl_timestamp ON call_logs(timestamp); CREATE INDEX IF NOT EXISTS idx_cl_status ON call_logs(status); diff --git a/src/lib/db/migrations/014_unified_log_artifacts.sql b/src/lib/db/migrations/014_unified_log_artifacts.sql new file mode 100644 index 0000000000..9f98ba7220 --- /dev/null +++ b/src/lib/db/migrations/014_unified_log_artifacts.sql @@ -0,0 +1,15 @@ +-- 013_unified_log_artifacts.sql +-- Switch request logging to unified single-file artifacts and prefixed settings. + +INSERT OR REPLACE INTO key_value (namespace, key, value) +VALUES ( + 'settings', + 'call_log_pipeline_enabled', + COALESCE( + (SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'detailed_logs_enabled'), + 'false' + ) +); + +DELETE FROM key_value +WHERE namespace = 'settings' AND key IN ('detailed_logs_enabled', 'maxCallLogs', 'MAX_CALL_LOGS'); diff --git a/src/lib/logEnv.ts b/src/lib/logEnv.ts new file mode 100644 index 0000000000..381a329408 --- /dev/null +++ b/src/lib/logEnv.ts @@ -0,0 +1,61 @@ +import path from "path"; + +const DEFAULT_APP_LOG_RETENTION_DAYS = 7; +const DEFAULT_CALL_LOG_RETENTION_DAYS = 7; +const DEFAULT_APP_LOG_MAX_SIZE = 50 * 1024 * 1024; +const DEFAULT_APP_LOG_PATH = path.join(process.cwd(), "logs", "application", "app.log"); + +function parsePositiveInt(value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +export function parseFileSize(raw: string | undefined): number { + if (!raw) return DEFAULT_APP_LOG_MAX_SIZE; + const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i); + if (!match) return DEFAULT_APP_LOG_MAX_SIZE; + const num = parseInt(match[1], 10); + const unit = (match[2] || "").toLowerCase(); + switch (unit) { + case "k": + case "kb": + return num * 1024; + case "m": + case "mb": + return num * 1024 * 1024; + case "g": + case "gb": + return num * 1024 * 1024 * 1024; + default: + return num; + } +} + +export function getAppLogToFile(): boolean { + return process.env.APP_LOG_TO_FILE !== "false"; +} + +export function getAppLogFilePath(): string { + return process.env.APP_LOG_FILE_PATH || DEFAULT_APP_LOG_PATH; +} + +export function getAppLogMaxFileSize(): number { + return parseFileSize(process.env.APP_LOG_MAX_FILE_SIZE); +} + +export function getAppLogRetentionDays(): number { + return parsePositiveInt(process.env.APP_LOG_RETENTION_DAYS, DEFAULT_APP_LOG_RETENTION_DAYS); +} + +export function getCallLogRetentionDays(): number { + return parsePositiveInt(process.env.CALL_LOG_RETENTION_DAYS, DEFAULT_CALL_LOG_RETENTION_DAYS); +} + +export function getAppLogLevel(defaultLevel: string): string { + return process.env.APP_LOG_LEVEL || defaultLevel; +} + +export function getAppLogFormat(defaultFormat: string): string { + return process.env.APP_LOG_FORMAT || defaultFormat; +} diff --git a/src/lib/logRotation.ts b/src/lib/logRotation.ts index 80035199de..c4644135ef 100644 --- a/src/lib/logRotation.ts +++ b/src/lib/logRotation.ts @@ -7,48 +7,26 @@ * - Creating the log directory on startup * * Configuration via env vars: - * - LOG_TO_FILE: enable file logging (default: true) - * - LOG_FILE_PATH: path to log file (default: logs/application/app.log) - * - LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB) - * - LOG_RETENTION_DAYS: days to keep old logs (default: 7) + * - APP_LOG_TO_FILE: enable file logging (default: true) + * - APP_LOG_FILE_PATH: path to log file (default: logs/application/app.log) + * - APP_LOG_MAX_FILE_SIZE: max file size before rotation (default: 50MB) + * - APP_LOG_RETENTION_DAYS: days to keep old logs (default: 7) */ import { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync } from "fs"; import { dirname, join, basename, extname } from "path"; - -const DEFAULT_LOG_PATH = "logs/application/app.log"; -const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50MB -const DEFAULT_RETENTION_DAYS = 7; - -function parseFileSize(raw: string | undefined): number { - if (!raw) return DEFAULT_MAX_SIZE; - const match = raw.match(/^(\d+)\s*(k|m|g|kb|mb|gb)?$/i); - if (!match) return DEFAULT_MAX_SIZE; - const num = parseInt(match[1], 10); - const unit = (match[2] || "").toLowerCase(); - switch (unit) { - case "k": - case "kb": - return num * 1024; - case "m": - case "mb": - return num * 1024 * 1024; - case "g": - case "gb": - return num * 1024 * 1024 * 1024; - default: - return num; - } -} +import { + getAppLogFilePath, + getAppLogMaxFileSize, + getAppLogRetentionDays, + getAppLogToFile, +} from "./logEnv"; export function getLogConfig() { - const logToFile = process.env.LOG_TO_FILE !== "false"; - const logFilePath = process.env.LOG_FILE_PATH || join(process.cwd(), DEFAULT_LOG_PATH); - const maxFileSize = parseFileSize(process.env.LOG_MAX_FILE_SIZE); - const retentionDays = parseInt( - process.env.LOG_RETENTION_DAYS || String(DEFAULT_RETENTION_DAYS), - 10 - ); + const logToFile = getAppLogToFile(); + const logFilePath = getAppLogFilePath() || join(process.cwd(), "logs/application/app.log"); + const maxFileSize = getAppLogMaxFileSize(); + const retentionDays = getAppLogRetentionDays(); return { logToFile, logFilePath, maxFileSize, retentionDays }; } diff --git a/src/lib/proxyLogger.ts b/src/lib/proxyLogger.ts index ff8f8caf3b..d3fb2140d4 100644 --- a/src/lib/proxyLogger.ts +++ b/src/lib/proxyLogger.ts @@ -11,7 +11,7 @@ import { getDbInstance, isCloud, isBuildPhase } from "./db/core"; const shouldPersistToDisk = !isCloud && !isBuildPhase; -const MAX_ENTRIES = parseInt(process.env.PROXY_LOG_MAX_ENTRIES || "200", 10); +const MAX_IN_MEMORY_ENTRIES = 200; interface ProxyInfo { type: string; @@ -56,7 +56,7 @@ function loadFromDb() { const db = getDbInstance(); const rows = db .prepare("SELECT * FROM proxy_logs ORDER BY timestamp DESC LIMIT ?") - .all(MAX_ENTRIES) as any[]; + .all(MAX_IN_MEMORY_ENTRIES) as any[]; for (const row of rows) { proxyLogs.push({ @@ -113,8 +113,8 @@ export function logProxyEvent(entry: Partial) { // 1. In-memory ring buffer (newest first) proxyLogs.unshift(log); - if (proxyLogs.length > MAX_ENTRIES) { - proxyLogs.length = MAX_ENTRIES; + if (proxyLogs.length > MAX_IN_MEMORY_ENTRIES) { + proxyLogs.length = MAX_IN_MEMORY_ENTRIES; } // 2. Persist to SQLite @@ -147,16 +147,6 @@ export function logProxyEvent(entry: Partial) { account: log.account, tlsFingerprint: log.tlsFingerprint ? 1 : 0, }); - - // Trim old entries - const count = (db.prepare("SELECT COUNT(*) as cnt FROM proxy_logs").get() as any)?.cnt || 0; - if (count > MAX_ENTRIES) { - db.prepare( - `DELETE FROM proxy_logs WHERE id IN ( - SELECT id FROM proxy_logs ORDER BY timestamp ASC LIMIT ? - )` - ).run(count - MAX_ENTRIES); - } } catch (err: any) { console.warn("[proxyLogger] Failed to persist:", err.message); } diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index 7b1d6aa870..cca04ee932 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -2,24 +2,57 @@ * Call Logs — extracted from usageDb.js (T-15) * * Structured call log management: save, query, rotate, and - * full-payload disk storage for the Logger UI. + * unified single-artifact disk storage for the Logger UI. * * @module lib/usage/callLogs */ -import path from "path"; import fs from "fs"; +import path from "path"; +import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { getDbInstance } from "../db/core"; -import { getSettings } from "../db/settings"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; import { shouldPersistToDisk, CALL_LOGS_DIR } from "./migrations"; import { getLoggedInputTokens, getLoggedOutputTokens } from "./tokenAccounting"; import { isNoLog } from "../compliance"; import { sanitizePII } from "../piiSanitizer"; -import { protectPayloadForLog, parseStoredPayload } from "../logPayloads"; +import { + protectPayloadForLog, + parseStoredPayload, + serializePayloadForStorage, +} from "../logPayloads"; +import { getCallLogRetentionDays } from "../logEnv"; type JsonRecord = Record; +type CallLogArtifact = { + schemaVersion: 2; + summary: { + id: string; + timestamp: string; + method: string; + path: string; + status: number; + model: string; + requestedModel: string | null; + provider: string; + account: string; + connectionId: string | null; + duration: number; + tokens: { in: number; out: number }; + requestType: string | null; + sourceFormat: string | null; + targetFormat: string | null; + apiKeyId: string | null; + apiKeyName: string | null; + comboName: string | null; + }; + requestBody: unknown; + responseBody: unknown; + error: unknown; + pipeline?: RequestPipelinePayloads; +}; + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } @@ -34,7 +67,7 @@ function toNumber(value: unknown): number { } function toStringOrNull(value: unknown): string | null { - return typeof value === "string" ? value : null; + return typeof value === "string" && value.trim().length > 0 ? value : null; } function hasTruncatedFlag(value: unknown): boolean { @@ -42,74 +75,224 @@ function hasTruncatedFlag(value: unknown): boolean { return (value as Record)._truncated === true; } -const DEFAULT_MAX_CALL_LOGS = 10000; -const CALL_LOGS_MAX_CACHE_TTL_MS = 30_000; - -const LOG_RETENTION_DAYS = parseInt(process.env.LOG_RETENTION_DAYS || "7", 10); -const CALL_LOG_PAYLOAD_MODE = (() => { - const value = (process.env.CALL_LOG_PAYLOAD_MODE || "full").toLowerCase(); - return value === "full" || value === "metadata" || value === "none" ? value : "full"; -})(); -const shouldLogPayloadInDb = CALL_LOG_PAYLOAD_MODE !== "none"; -const shouldLogPayloadOnDisk = CALL_LOG_PAYLOAD_MODE === "full"; - -let callLogsMaxCache = { - value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS, - expiresAt: 0, -}; - -function resolveCallLogsMaxValue(value: unknown): number | null { - if (typeof value === "number" && Number.isInteger(value) && value > 0) return value; - if (typeof value === "string" && value.trim().length > 0) { - const parsed = Number.parseInt(value, 10); - return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +function sanitizeErrorForLog(error: unknown): unknown { + if (error === null || error === undefined) return null; + if (typeof error === "string") return sanitizePII(error).text; + if (error instanceof Error) { + return { + message: sanitizePII(error.message).text, + stack: sanitizePII(error.stack || "").text || undefined, + name: error.name, + }; } - return null; + return protectPayloadForLog(error); } -async function getMaxCallLogs(): Promise { - const now = Date.now(); - if (callLogsMaxCache.expiresAt > now) { - return callLogsMaxCache.value; - } - - let value = resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS; - +function toStoredErrorString(error: unknown): string | null { + const sanitized = sanitizeErrorForLog(error); + if (sanitized === null || sanitized === undefined) return null; + if (typeof sanitized === "string") return sanitized; try { - const { getSettings } = await import("@/lib/localDb"); - const settings = await getSettings(); - const configured = - resolveCallLogsMaxValue(settings.maxCallLogs) ?? - resolveCallLogsMaxValue(settings.MAX_CALL_LOGS); - if (configured !== null) { - value = configured; - } + return JSON.stringify(sanitized); } catch { - // Fall back to env/default cap when settings are unavailable. + return String(sanitized); + } +} + +function protectPipelinePayloads(payloads: unknown): RequestPipelinePayloads | null { + if (!payloads || typeof payloads !== "object") return null; + + const protectedPayloads: RequestPipelinePayloads = {}; + for (const [key, value] of Object.entries(payloads as JsonRecord)) { + if (value === null || value === undefined) { + continue; + } + + if (key === "streamChunks" && value && typeof value === "object") { + const chunks = value as Record; + const compacted = Object.fromEntries( + Object.entries(chunks).filter( + ([, chunkValue]) => Array.isArray(chunkValue) && chunkValue.length > 0 + ) + ); + if (Object.keys(compacted).length > 0) { + protectedPayloads.streamChunks = protectPayloadForLog( + compacted + ) as RequestPipelinePayloads["streamChunks"]; + } + continue; + } + + protectedPayloads[key as keyof RequestPipelinePayloads] = protectPayloadForLog(value) as never; } - callLogsMaxCache = { - value, - expiresAt: now + CALL_LOGS_MAX_CACHE_TTL_MS, - }; - return value; + return Object.keys(protectedPayloads).length > 0 ? protectedPayloads : null; } -export function invalidateCallLogsMaxCache(): void { - callLogsMaxCache = { - value: resolveCallLogsMaxValue(process.env.CALL_LOGS_MAX) ?? DEFAULT_MAX_CALL_LOGS, - expiresAt: 0, - }; -} let logIdCounter = 0; function generateLogId() { logIdCounter++; return `${Date.now()}-${logIdCounter}`; } -/** - * Save a structured call log entry. - */ +async function resolveAccountName(connectionId: string | null | undefined) { + let account = connectionId ? connectionId.slice(0, 8) : "-"; + + if (!connectionId) { + return account; + } + + try { + const { getProviderConnections } = await import("@/lib/localDb"); + const connections = await getProviderConnections(); + const conn = connections.find((item) => item.id === connectionId); + if (conn) { + account = conn.name || conn.email || account; + } + } catch { + // Best-effort lookup only. + } + + return account; +} + +function buildArtifactRelativePath(timestamp: string, id: string) { + const parsed = new Date(timestamp); + const safeTimestamp = ( + Number.isNaN(parsed.getTime()) ? new Date().toISOString() : parsed.toISOString() + ).replace(/[:]/g, "-"); + const dateFolder = safeTimestamp.slice(0, 10); + return path.posix.join(dateFolder, `${safeTimestamp}_${id}.json`); +} + +function buildArtifact( + logEntry: { + id: string; + timestamp: string; + method: string; + path: string; + status: number; + model: string; + requestedModel: string | null; + provider: string; + account: string; + connectionId: string | null; + duration: number; + tokensIn: number; + tokensOut: number; + requestType: string | null; + sourceFormat: string | null; + targetFormat: string | null; + apiKeyId: string | null; + apiKeyName: string | null; + comboName: string | null; + }, + requestBody: unknown, + responseBody: unknown, + error: unknown, + pipelinePayloads: RequestPipelinePayloads | null +): CallLogArtifact { + return { + schemaVersion: 2, + summary: { + id: logEntry.id, + timestamp: logEntry.timestamp, + method: logEntry.method, + path: logEntry.path, + status: logEntry.status, + model: logEntry.model, + requestedModel: logEntry.requestedModel, + provider: logEntry.provider, + account: logEntry.account, + connectionId: logEntry.connectionId, + duration: logEntry.duration, + tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut }, + requestType: logEntry.requestType, + sourceFormat: logEntry.sourceFormat, + targetFormat: logEntry.targetFormat, + apiKeyId: logEntry.apiKeyId, + apiKeyName: logEntry.apiKeyName, + comboName: logEntry.comboName, + }, + requestBody: requestBody ?? null, + responseBody: responseBody ?? null, + error: error ?? null, + ...(pipelinePayloads ? { pipeline: pipelinePayloads } : {}), + }; +} + +function writeCallArtifact(artifact: CallLogArtifact): string | null { + if (!CALL_LOGS_DIR) return null; + + const relPath = buildArtifactRelativePath(artifact.summary.timestamp, artifact.summary.id); + const absPath = path.join(CALL_LOGS_DIR, relPath); + + try { + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, JSON.stringify(artifact, null, 2)); + return relPath; + } catch (error) { + console.error("[callLogs] Failed to write request artifact:", (error as Error).message); + return null; + } +} + +function readArtifactFromDisk(relativePath: string | null) { + if (!CALL_LOGS_DIR || !relativePath) return null; + + try { + const absPath = path.join(CALL_LOGS_DIR, relativePath); + if (!fs.existsSync(absPath)) return null; + return JSON.parse(fs.readFileSync(absPath, "utf8")) as CallLogArtifact; + } catch (error) { + console.error("[callLogs] Failed to read request artifact:", (error as Error).message); + return null; + } +} + +function readLegacyLogFromDisk(entry: { + timestamp: string | null; + model: string | null; + status: number; +}) { + if (!CALL_LOGS_DIR || !entry.timestamp) return null; + + try { + const date = new Date(entry.timestamp); + if (Number.isNaN(date.getTime())) return null; + + const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart( + 2, + "0" + )}-${String(date.getDate()).padStart(2, "0")}`; + const dir = path.join(CALL_LOGS_DIR, dateFolder); + if (!fs.existsSync(dir)) return null; + + const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart( + 2, + "0" + )}${String(date.getSeconds()).padStart(2, "0")}`; + const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-"); + const expectedName = `${time}_${safeModel}_${entry.status}.json`; + + const exactPath = path.join(dir, expectedName); + if (fs.existsSync(exactPath)) { + return JSON.parse(fs.readFileSync(exactPath, "utf8")); + } + + const files = fs + .readdirSync(dir) + .filter((file) => file.startsWith(time) && file.endsWith(`_${entry.status}.json`)); + if (files.length > 0) { + return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8")); + } + } catch (error) { + console.error("[callLogs] Failed to read legacy disk log:", (error as Error).message); + } + + return null; +} + export async function saveCallLog(entry: any) { if (!shouldPersistToDisk) return; @@ -117,44 +300,23 @@ export async function saveCallLog(entry: any) { const apiKeyId = entry.apiKeyId || null; const noLogEnabled = Boolean(entry.noLog) || (apiKeyId ? isNoLog(apiKeyId) : false); - const protectedRequestBody = - noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.requestBody); - const protectedResponseBody = - noLogEnabled || !shouldLogPayloadInDb ? null : protectPayloadForLog(entry.responseBody); + const protectedRequestBody = noLogEnabled ? null : protectPayloadForLog(entry.requestBody); + const protectedResponseBody = noLogEnabled ? null : protectPayloadForLog(entry.responseBody); + const protectedPipelinePayloads = noLogEnabled + ? null + : protectPipelinePayloads(entry.pipelinePayloads ?? entry.pipeline ?? null); + const protectedError = sanitizeErrorForLog(entry.error); - // Resolve account name - let account = entry.connectionId ? entry.connectionId.slice(0, 8) : "-"; - try { - const { getProviderConnections } = await import("@/lib/localDb"); - const connections = await getProviderConnections(); - const conn = connections.find((c) => c.id === entry.connectionId); - if (conn) account = conn.name || conn.email || account; - } catch {} - - // Truncate large payloads for DB storage (keep under 8KB each) - const truncatePayload = (obj: any) => { - if (!obj) return null; - const str = JSON.stringify(obj); - if (str.length <= 8192) return str; - try { - return JSON.stringify({ - _truncated: true, - _originalSize: str.length, - _preview: str.slice(0, 8192) + "...", - }); - } catch { - return JSON.stringify({ _truncated: true }); - } - }; + const account = await resolveAccountName(entry.connectionId || null); const logEntry = { id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(), - timestamp: new Date().toISOString(), + timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(), method: entry.method || "POST", path: entry.path || "/v1/chat/completions", status: entry.status || 0, model: entry.model || "-", - requestedModel: entry.requestedModel || null, // T01: model the client asked for + requestedModel: entry.requestedModel || null, provider: entry.provider || "-", account, connectionId: entry.connectionId || null, @@ -167,119 +329,82 @@ export async function saveCallLog(entry: any) { apiKeyId, apiKeyName: entry.apiKeyName || null, comboName: entry.comboName || null, - requestBody: truncatePayload(protectedRequestBody), - responseBody: truncatePayload(protectedResponseBody), - error: typeof entry.error === "string" ? sanitizePII(entry.error).text : entry.error || null, + requestBody: serializePayloadForStorage(protectedRequestBody, 8192), + responseBody: serializePayloadForStorage(protectedResponseBody, 8192), + error: toStoredErrorString(protectedError), }; - // 1. Insert into SQLite const db = getDbInstance(); db.prepare( ` - INSERT INTO call_logs (id, timestamp, method, path, status, model, requested_model, provider, - account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error) - VALUES (@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, - @account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, @targetFormat, - @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error) + INSERT INTO call_logs ( + id, timestamp, method, path, status, model, requested_model, provider, + account, connection_id, duration, tokens_in, tokens_out, request_type, source_format, + target_format, api_key_id, api_key_name, combo_name, request_body, response_body, error, + artifact_relpath, has_pipeline_details + ) + VALUES ( + @id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, + @account, @connectionId, @duration, @tokensIn, @tokensOut, @requestType, @sourceFormat, + @targetFormat, @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, + NULL, 0 + ) ` ).run(logEntry); - // 2. Trim old entries beyond max - const maxLogs = await getMaxCallLogs(); - const countRow = asRecord(db.prepare("SELECT COUNT(*) as cnt FROM call_logs").get()); - const count = toNumber(countRow.cnt); - if (count > maxLogs) { - db.prepare( - ` - DELETE FROM call_logs WHERE id IN ( - SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ? - ) - ` - ).run(count - maxLogs); - } - - // 3. Write full payload to disk file (untruncated) - // Disabled when no-log is active or payload mode is metadata/none. - if ( - shouldLogPayloadOnDisk && - !noLogEnabled && - (protectedRequestBody !== null || protectedResponseBody !== null) - ) { - writeCallLogToDisk( - { ...logEntry, tokens: { in: logEntry.tokensIn, out: logEntry.tokensOut } }, + if (!noLogEnabled) { + const artifact = buildArtifact( + logEntry, protectedRequestBody, - protectedResponseBody + protectedResponseBody, + protectedError, + protectedPipelinePayloads ); + const artifactRelPath = writeCallArtifact(artifact); + + if (artifactRelPath) { + db.prepare( + ` + UPDATE call_logs + SET artifact_relpath = ?, has_pipeline_details = ? + WHERE id = ? + ` + ).run(artifactRelPath, protectedPipelinePayloads ? 1 : 0, logEntry.id); + } } - } catch (error: any) { - console.error("[callLogs] Failed to save call log:", error.message); + } catch (error) { + console.error("[callLogs] Failed to save call log:", (error as Error).message); } } -/** - * Write call log as JSON file to disk (full payloads, not truncated). - */ -function writeCallLogToDisk(logEntry: any, requestBody: any, responseBody: any) { - if (!CALL_LOGS_DIR) return; - - try { - const now = new Date(); - const dateFolder = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}`; - const dir = path.join(CALL_LOGS_DIR, dateFolder); - - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); - - const safeModel = (logEntry.model || "unknown").replace(/[/:]/g, "-"); - const time = `${String(now.getHours()).padStart(2, "0")}${String(now.getMinutes()).padStart(2, "0")}${String(now.getSeconds()).padStart(2, "0")}`; - const filename = `${time}_${safeModel}_${logEntry.status}.json`; - - const fullEntry = { - ...logEntry, - requestBody: requestBody || null, - responseBody: responseBody || null, - }; - - fs.writeFileSync(path.join(dir, filename), JSON.stringify(fullEntry, null, 2)); - } catch (err: any) { - console.error("[callLogs] Failed to write disk log:", err.message); - } -} - -/** - * Rotate old call log directories (keep last 7 days). - */ export function rotateCallLogs() { if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return; try { const entries = fs.readdirSync(CALL_LOGS_DIR); const now = Date.now(); - const retentionMs = LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000; + const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000; for (const entry of entries) { const entryPath = path.join(CALL_LOGS_DIR, entry); const stat = fs.statSync(entryPath); if (stat.isDirectory() && now - stat.mtimeMs > retentionMs) { fs.rmSync(entryPath, { recursive: true, force: true }); - console.log(`[callLogs] Rotated old logs: ${entry}`); } } - } catch (err: any) { - console.error("[callLogs] Failed to rotate logs:", err.message); + } catch (error) { + console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message); } } -// Run rotation on startup if (shouldPersistToDisk) { try { rotateCallLogs(); - } catch {} + } catch { + // Best-effort startup cleanup. + } } -/** - * Get call logs with optional filtering. - */ export async function getCallLogs(filter: any = {}) { const db = getDbInstance(); let sql = "SELECT * FROM call_logs"; @@ -292,8 +417,8 @@ export async function getCallLogs(filter: any = {}) { } else if (filter.status === "ok") { conditions.push("status >= 200 AND status < 300"); } else { - const statusCode = parseInt(filter.status); - if (!isNaN(statusCode)) { + const statusCode = parseInt(filter.status, 10); + if (!Number.isNaN(statusCode)) { conditions.push("status = @statusCode"); params.statusCode = statusCode; } @@ -347,7 +472,7 @@ export async function getCallLogs(filter: any = {}) { path: toStringOrNull(l.path), status: toNumber(l.status), model: toStringOrNull(l.model), - requestedModel: toStringOrNull(l.requested_model), // T01: original model from client + requestedModel: toStringOrNull(l.requested_model), provider: toStringOrNull(l.provider), account: toStringOrNull(l.account), duration: toNumber(l.duration), @@ -360,19 +485,30 @@ export async function getCallLogs(filter: any = {}) { apiKeyName: toStringOrNull(l.api_key_name), hasRequestBody: typeof l.request_body === "string" && l.request_body.length > 0, hasResponseBody: typeof l.response_body === "string" && l.response_body.length > 0, + hasPipelineDetails: toNumber(l.has_pipeline_details) === 1, }; }); } -/** - * Get a single call log by ID (with full payloads from disk when available). - */ +function buildLegacyPipelinePayloads(id: string) { + const detailed = getRequestDetailLogByCallLogId(id); + if (!detailed) return null; + + return { + clientRequest: detailed.client_request ?? null, + providerRequest: detailed.translated_request ?? null, + providerResponse: detailed.provider_response ?? null, + clientResponse: detailed.client_response ?? null, + }; +} + export async function getCallLogById(id: string) { const db = getDbInstance(); const row = db.prepare("SELECT * FROM call_logs WHERE id = ?").get(id); if (!row) return null; - const entryRow = asRecord(row); + const entryRow = asRecord(row); + const artifactRelPath = toStringOrNull(entryRow.artifact_relpath); const entry = { id: toStringOrNull(entryRow.id), timestamp: toStringOrNull(entryRow.timestamp), @@ -394,72 +530,47 @@ export async function getCallLogById(id: string) { requestBody: parseStoredPayload(entryRow.request_body), responseBody: parseStoredPayload(entryRow.response_body), error: toStringOrNull(entryRow.error), + artifactRelPath, + hasPipelineDetails: toNumber(entryRow.has_pipeline_details) === 1, }; - // If payloads were truncated, try to read full version from disk - const needsDisk = hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody); - if (needsDisk && CALL_LOGS_DIR) { - try { - const diskEntry = readFullLogFromDisk(entry); - if (diskEntry) { - return { - ...entry, - requestBody: diskEntry.requestBody ?? entry.requestBody, - responseBody: diskEntry.responseBody ?? entry.responseBody, - }; - } - } catch (err: any) { - console.error("[callLogs] Failed to read full log from disk:", err.message); + const artifact = readArtifactFromDisk(artifactRelPath); + if (artifact) { + return { + ...entry, + requestBody: artifact.requestBody ?? entry.requestBody, + responseBody: artifact.responseBody ?? entry.responseBody, + error: artifact.error ?? entry.error, + pipelinePayloads: artifact.pipeline ?? null, + hasPipelineDetails: Boolean(artifact.pipeline) || entry.hasPipelineDetails, + }; + } + + const needsLegacyDisk = + hasTruncatedFlag(entry.requestBody) || hasTruncatedFlag(entry.responseBody) || !artifactRelPath; + if (needsLegacyDisk) { + const legacyEntry = readLegacyLogFromDisk(entry); + if (legacyEntry) { + const legacyPipeline = buildLegacyPipelinePayloads(id); + return { + ...entry, + requestBody: legacyEntry.requestBody ?? entry.requestBody, + responseBody: legacyEntry.responseBody ?? entry.responseBody, + error: legacyEntry.error ?? entry.error, + pipelinePayloads: legacyPipeline, + hasPipelineDetails: Boolean(legacyPipeline), + }; } } - const detailed = getRequestDetailLogByCallLogId(id); - if (!detailed) { - return entry; + const legacyPipeline = buildLegacyPipelinePayloads(id); + if (legacyPipeline) { + return { + ...entry, + pipelinePayloads: legacyPipeline, + hasPipelineDetails: true, + }; } - return { - ...entry, - pipelinePayloads: { - clientRequest: detailed.client_request ?? null, - providerRequest: detailed.translated_request ?? null, - providerResponse: detailed.provider_response ?? null, - clientResponse: detailed.client_response ?? null, - }, - }; -} - -/** - * Read the full (untruncated) log entry from disk. - */ -function readFullLogFromDisk(entry: any) { - if (!CALL_LOGS_DIR || !entry.timestamp) return null; - - try { - const date = new Date(entry.timestamp); - const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; - const dir = path.join(CALL_LOGS_DIR, dateFolder); - - if (!fs.existsSync(dir)) return null; - - const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart(2, "0")}${String(date.getSeconds()).padStart(2, "0")}`; - const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-"); - const expectedName = `${time}_${safeModel}_${entry.status}.json`; - - const exactPath = path.join(dir, expectedName); - if (fs.existsSync(exactPath)) { - return JSON.parse(fs.readFileSync(exactPath, "utf8")); - } - - const files = fs - .readdirSync(dir) - .filter((f) => f.startsWith(time) && f.endsWith(`_${entry.status}.json`)); - if (files.length > 0) { - return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8")); - } - } catch (err: any) { - console.error("[callLogs] Disk log read error:", err.message); - } - - return null; + return entry; } diff --git a/src/lib/usage/migrations.ts b/src/lib/usage/migrations.ts index 83e5fc9d62..2cd0275aef 100644 --- a/src/lib/usage/migrations.ts +++ b/src/lib/usage/migrations.ts @@ -1,42 +1,47 @@ /** * Usage Migrations — extracted from usageDb.js (T-15) * - * Handles legacy file migration (.data → data/) and JSON → SQLite migration. - * Runs automatically on module load when shouldPersistToDisk is true. + * Handles legacy file migration (.data → data/), JSON → SQLite migration, + * and one-time archival of legacy request log layouts into a zip backup. * * @module lib/usage/migrations */ -import path from "path"; import fs from "fs"; +import path from "path"; +import { ZipFile } from "yazl"; import { getDbInstance, isCloud, isBuildPhase, DATA_DIR } from "../db/core"; import { getLegacyDotDataDir, isSamePath } from "../dataPaths"; export const shouldPersistToDisk = !isCloud && !isBuildPhase; -// ──────────────── File Paths ──────────────── - const LEGACY_DATA_DIR = isCloud ? null : getLegacyDotDataDir(); -export const LOG_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt"); export const CALL_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "call_logs"); +export const LOG_ARCHIVES_DIR = isCloud ? null : path.join(DATA_DIR, "log_archives"); + +const LEGACY_LAYOUT_MARKER = + isCloud || !LOG_ARCHIVES_DIR ? null : path.join(LOG_ARCHIVES_DIR, "legacy-request-logs.json"); + +const CURRENT_REQUEST_LOGS_DIR = isCloud ? null : path.join(DATA_DIR, "logs"); +const CURRENT_REQUEST_SUMMARY_FILE = isCloud ? null : path.join(DATA_DIR, "log.txt"); // Legacy paths const LEGACY_DB_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "usage.json"); -const LEGACY_LOG_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "log.txt"); const LEGACY_CALL_LOGS_DB_FILE = isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs.json"); -const LEGACY_CALL_LOGS_DIR = - isCloud || !LEGACY_DATA_DIR ? null : path.join(LEGACY_DATA_DIR, "call_logs"); - // Current-location JSON files (for migration into SQLite) const USAGE_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "usage.json"); const CALL_LOGS_JSON_FILE = isCloud ? null : path.join(DATA_DIR, "call_logs.json"); -// ──────────────── Legacy File Migration ──────────────── +type ArchiveTarget = { + sourcePath: string; + archiveRoot: string; + deleteAfterArchive: boolean; +}; -function copyIfMissing(fromPath, toPath, label) { +function copyIfMissing(fromPath: string | null, toPath: string | null, label: string) { if (!fromPath || !toPath) return; if (!fs.existsSync(fromPath) || fs.existsSync(toPath)) return; @@ -48,27 +53,185 @@ function copyIfMissing(fromPath, toPath, label) { console.log(`[usageDb] Migrated ${label}: ${fromPath} -> ${toPath}`); } +function containsLegacyCallLogLayout(dirPath: string | null): boolean { + if (!dirPath || !fs.existsSync(dirPath)) return false; + + try { + const topLevelEntries = fs.readdirSync(dirPath); + for (const topLevelEntry of topLevelEntries) { + const topLevelPath = path.join(dirPath, topLevelEntry); + const stat = fs.statSync(topLevelPath); + if (stat.isFile() && /^\d{6}_.+_\d{3}\.json$/i.test(topLevelEntry)) { + return true; + } + if (!stat.isDirectory()) { + continue; + } + + const nestedEntries = fs.readdirSync(topLevelPath); + for (const nestedEntry of nestedEntries) { + if (/^\d{6}_.+_\d{3}\.json$/i.test(nestedEntry)) { + return true; + } + } + } + } catch { + return false; + } + + return false; +} + +function ensureArchiveDir() { + if (!LOG_ARCHIVES_DIR) return; + fs.mkdirSync(LOG_ARCHIVES_DIR, { recursive: true }); +} + +function listArchiveTargets(): ArchiveTarget[] { + const targets: ArchiveTarget[] = []; + + if (CURRENT_REQUEST_LOGS_DIR && fs.existsSync(CURRENT_REQUEST_LOGS_DIR)) { + targets.push({ + sourcePath: CURRENT_REQUEST_LOGS_DIR, + archiveRoot: "data/logs", + deleteAfterArchive: true, + }); + } + + if (CURRENT_REQUEST_SUMMARY_FILE && fs.existsSync(CURRENT_REQUEST_SUMMARY_FILE)) { + targets.push({ + sourcePath: CURRENT_REQUEST_SUMMARY_FILE, + archiveRoot: "data/log.txt", + deleteAfterArchive: true, + }); + } + + if (CALL_LOGS_DIR && containsLegacyCallLogLayout(CALL_LOGS_DIR)) { + targets.push({ + sourcePath: CALL_LOGS_DIR, + archiveRoot: "data/call_logs", + deleteAfterArchive: true, + }); + } + + return targets; +} + +function addPathToZip(zipFile: ZipFile, sourcePath: string, archivePath: string) { + const stat = fs.statSync(sourcePath); + if (stat.isDirectory()) { + const entries = fs.readdirSync(sourcePath); + if (entries.length === 0) { + zipFile.addEmptyDirectory(archivePath); + return; + } + + for (const entry of entries) { + addPathToZip(zipFile, path.join(sourcePath, entry), path.posix.join(archivePath, entry)); + } + return; + } + + zipFile.addFile(sourcePath, archivePath); +} + +function createLegacyArchive(targets: ArchiveTarget[]): Promise { + return new Promise((resolve, reject) => { + if (!LOG_ARCHIVES_DIR) { + reject(new Error("LOG_ARCHIVES_DIR is not configured")); + return; + } + + ensureArchiveDir(); + + const timestamp = new Date().toISOString().replace(/[:]/g, "-"); + const archiveFilename = `${timestamp}_legacy-request-logs.zip`; + const archivePath = path.join(LOG_ARCHIVES_DIR, archiveFilename); + const zipFile = new ZipFile(); + const output = fs.createWriteStream(archivePath); + + output.on("close", () => resolve(archiveFilename)); + output.on("error", (error) => { + fs.rmSync(archivePath, { force: true }); + reject(error); + }); + zipFile.outputStream.pipe(output); + + try { + for (const target of targets) { + addPathToZip(zipFile, target.sourcePath, target.archiveRoot); + } + zipFile.end(); + } catch (error) { + fs.rmSync(archivePath, { force: true }); + zipFile.end(); + reject(error); + } + }); +} + +function writeLegacyLayoutMarker(archiveFilename: string) { + if (!LEGACY_LAYOUT_MARKER) return; + ensureArchiveDir(); + fs.writeFileSync( + LEGACY_LAYOUT_MARKER, + JSON.stringify( + { + migratedAt: new Date().toISOString(), + archiveFilename, + }, + null, + 2 + ) + ); +} + +function deleteArchivedTargets(targets: ArchiveTarget[]) { + for (const target of targets) { + if (!target.deleteAfterArchive || !fs.existsSync(target.sourcePath)) { + continue; + } + + const stat = fs.statSync(target.sourcePath); + if (stat.isDirectory()) { + fs.rmSync(target.sourcePath, { recursive: true, force: true }); + } else { + fs.rmSync(target.sourcePath, { force: true }); + } + } +} + export function migrateLegacyUsageFiles() { if (!shouldPersistToDisk || !LEGACY_DATA_DIR) return; if (isSamePath(DATA_DIR, LEGACY_DATA_DIR)) return; try { copyIfMissing(LEGACY_DB_FILE, USAGE_JSON_FILE, "usage history"); - copyIfMissing(LEGACY_LOG_FILE, LOG_FILE, "request log"); copyIfMissing(LEGACY_CALL_LOGS_DB_FILE, CALL_LOGS_JSON_FILE, "call log index"); - copyIfMissing(LEGACY_CALL_LOGS_DIR, CALL_LOGS_DIR, "call log files"); } catch (error) { - console.error("[usageDb] Legacy migration failed:", error.message); + console.error("[usageDb] Legacy migration failed:", (error as Error).message); } } -// ──────────────── JSON → SQLite Migration ──────────────── +export async function archiveLegacyRequestLogs() { + if (!shouldPersistToDisk) return null; + if (LEGACY_LAYOUT_MARKER && fs.existsSync(LEGACY_LAYOUT_MARKER)) return null; + + const targets = listArchiveTargets(); + if (targets.length === 0) return null; + + const archiveFilename = await createLegacyArchive(targets); + deleteArchivedTargets(targets); + writeLegacyLayoutMarker(archiveFilename); + + console.log(`[usageDb] Archived legacy request logs to ${archiveFilename}`); + return archiveFilename; +} export function migrateUsageJsonToSqlite() { if (!shouldPersistToDisk) return; const db = getDbInstance(); - // 1. Migrate usage.json if (USAGE_JSON_FILE && fs.existsSync(USAGE_JSON_FILE)) { try { const raw = fs.readFileSync(USAGE_JSON_FILE, "utf-8"); @@ -118,13 +281,12 @@ export function migrateUsageJsonToSqlite() { console.log(`[usageDb] ✓ Migrated ${history.length} usage entries`); } - fs.renameSync(USAGE_JSON_FILE, USAGE_JSON_FILE + ".migrated"); - } catch (err) { - console.error("[usageDb] Failed to migrate usage.json:", err.message); + fs.renameSync(USAGE_JSON_FILE, `${USAGE_JSON_FILE}.migrated`); + } catch (error) { + console.error("[usageDb] Failed to migrate usage.json:", (error as Error).message); } } - // 2. Migrate call_logs.json if (CALL_LOGS_JSON_FILE && fs.existsSync(CALL_LOGS_JSON_FILE)) { try { const raw = fs.readFileSync(CALL_LOGS_JSON_FILE, "utf-8"); @@ -137,10 +299,12 @@ export function migrateUsageJsonToSqlite() { const insert = db.prepare(` INSERT OR IGNORE INTO call_logs (id, timestamp, method, path, status, model, provider, account, connection_id, duration, tokens_in, tokens_out, source_format, target_format, - api_key_id, api_key_name, combo_name, request_body, response_body, error) + api_key_id, api_key_name, combo_name, request_body, response_body, error, + artifact_relpath, has_pipeline_details) VALUES (@id, @timestamp, @method, @path, @status, @model, @provider, @account, @connectionId, @duration, @tokensIn, @tokensOut, @sourceFormat, @targetFormat, - @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error) + @apiKeyId, @apiKeyName, @comboName, @requestBody, @responseBody, @error, + NULL, 0) `); const tx = db.transaction(() => { @@ -173,21 +337,25 @@ export function migrateUsageJsonToSqlite() { console.log(`[usageDb] ✓ Migrated ${logs.length} call log entries`); } - fs.renameSync(CALL_LOGS_JSON_FILE, CALL_LOGS_JSON_FILE + ".migrated"); - } catch (err) { - console.error("[usageDb] Failed to migrate call_logs.json:", err.message); + fs.renameSync(CALL_LOGS_JSON_FILE, `${CALL_LOGS_JSON_FILE}.migrated`); + } catch (error) { + console.error("[usageDb] Failed to migrate call_logs.json:", (error as Error).message); } } } -// ──────────────── Run on load ──────────────── - migrateLegacyUsageFiles(); if (shouldPersistToDisk) { + try { + await archiveLegacyRequestLogs(); + } catch (error) { + console.error("[usageDb] Failed to archive legacy request logs:", (error as Error).message); + } + try { migrateUsageJsonToSqlite(); } catch { - /* ok */ + // Best-effort startup migration. } } diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index d8e1630862..8a8ee52f3f 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -378,31 +378,18 @@ export async function getModelLatencyStats( return stats; } -// ──────────────── Request Log (log.txt) ──────────────── - -import fs from "fs"; -import { LOG_FILE } from "./migrations"; - -function formatLogDate(date = new Date()) { - const pad = (n: number) => String(n).padStart(2, "0"); - const d = pad(date.getDate()); - const m = pad(date.getMonth() + 1); - const y = date.getFullYear(); - const h = pad(date.getHours()); - const min = pad(date.getMinutes()); - const s = pad(date.getSeconds()); - return `${d}-${m}-${y} ${h}:${min}:${s}`; -} +// ──────────────── Request Log Compatibility Shim ──────────────── /** - * Append to log.txt. + * Legacy compatibility shim. + * Request summary lines are no longer written to data/log.txt. */ export async function appendRequestLog({ - model, - provider, - connectionId, - tokens, - status, + model: _model, + provider: _provider, + connectionId: _connectionId, + tokens: _tokens, + status: _status, }: { model?: string; provider?: string; @@ -410,55 +397,39 @@ export async function appendRequestLog({ tokens?: any; status?: string | number; }) { - if (!shouldPersistToDisk) return; - - try { - const timestamp = formatLogDate(); - const p = provider?.toUpperCase() || "-"; - const m = model || "-"; - - let account = connectionId ? connectionId.slice(0, 8) : "-"; - try { - const { getProviderConnections } = await import("@/lib/localDb"); - const connections = await getProviderConnections(); - const connRaw = connections.find((c) => asRecord(c).id === connectionId); - if (connRaw) { - const conn = asRecord(connRaw); - account = toStringOrNull(conn.name) || toStringOrNull(conn.email) || account; - } - } catch {} - - const sent = tokens ? getLoggedInputTokens(tokens) : "-"; - const received = tokens ? getLoggedOutputTokens(tokens) : "-"; - - const line = `${timestamp} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${status}\n`; - fs.appendFileSync(LOG_FILE, line); - - const content = fs.readFileSync(LOG_FILE, "utf-8"); - const lines = content.trim().split("\n"); - if (lines.length > 200) { - fs.writeFileSync(LOG_FILE, lines.slice(-200).join("\n") + "\n"); - } - } catch (error: any) { - console.error("Failed to append to log.txt:", error.message); - } + // Deprecated: request summaries now come from SQLite call_logs. } /** - * Get last N lines of log.txt. + * Return recent request summaries generated from SQLite call_logs rows. */ export async function getRecentLogs(limit = 200) { - if (!shouldPersistToDisk) return []; - if (!fs || typeof fs.existsSync !== "function") return []; - if (!LOG_FILE) return []; - if (!fs.existsSync(LOG_FILE)) return []; - try { - const content = fs.readFileSync(LOG_FILE, "utf-8"); - const lines = content.trim().split("\n"); - return lines.slice(-limit).reverse(); + const db = getDbInstance(); + const rows = db + .prepare( + ` + SELECT timestamp, model, provider, account, tokens_in, tokens_out, status + FROM call_logs + ORDER BY timestamp DESC + LIMIT ? + ` + ) + .all(limit) as Array>; + + return rows.map((row) => { + const timestamp = + typeof row.timestamp === "string" ? row.timestamp : new Date().toISOString(); + const provider = typeof row.provider === "string" ? row.provider.toUpperCase() : "-"; + const model = typeof row.model === "string" ? row.model : "-"; + const account = typeof row.account === "string" ? row.account : "-"; + const tokensIn = toNumber(row.tokens_in); + const tokensOut = toNumber(row.tokens_out); + const status = typeof row.status === "number" ? row.status : String(row.status || "-"); + return `${timestamp} | ${model} | ${provider} | ${account} | ${tokensIn} | ${tokensOut} | ${status}`; + }); } catch (error: any) { - console.error("[usageDb] Failed to read log.txt:", error.message); + console.error("[usageDb] Failed to read recent call logs:", error.message); return []; } } diff --git a/src/server-init.ts b/src/server-init.ts index f254547c98..bf83efd0eb 100644 --- a/src/server-init.ts +++ b/src/server-init.ts @@ -5,6 +5,9 @@ import { initAuditLog, cleanupExpiredLogs, logAuditEvent } from "./lib/complianc import { initConsoleInterceptor } from "./lib/consoleInterceptor"; async function startServer() { + // Trigger request-log layout migration during startup, before serving requests. + await import("./lib/usage/migrations"); + // Console interceptor: capture all console output to log file (must be first) initConsoleInterceptor(); @@ -22,7 +25,14 @@ async function startServer() { // Compliance: One-time cleanup of expired logs try { const cleanup = cleanupExpiredLogs(); - if (cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedAuditLogs) { + if ( + cleanup.deletedUsage || + cleanup.deletedCallLogs || + cleanup.deletedProxyLogs || + cleanup.deletedRequestDetailLogs || + cleanup.deletedAuditLogs || + cleanup.deletedMcpAuditLogs + ) { console.log("[COMPLIANCE] Expired log cleanup:", cleanup); } } catch (err) { diff --git a/src/shared/components/ConsoleLogViewer.tsx b/src/shared/components/ConsoleLogViewer.tsx index eb53c00300..97680ffc0f 100644 --- a/src/shared/components/ConsoleLogViewer.tsx +++ b/src/shared/components/ConsoleLogViewer.tsx @@ -206,7 +206,7 @@ export default function ConsoleLogViewer() { error {error} - — Make sure the application is writing logs to file (LOG_TO_FILE=true) + — Make sure the application is writing logs to file (APP_LOG_TO_FILE=true)
)} @@ -237,7 +237,7 @@ export default function ConsoleLogViewer() {

{t("noLogEntries")}

- Ensure LOG_TO_FILE=true is set in your .env file + Ensure APP_LOG_TO_FILE=true is set in your .env file

) : ( diff --git a/src/shared/components/RequestLoggerDetail.tsx b/src/shared/components/RequestLoggerDetail.tsx index e4cf1fbfa0..19a45caad3 100644 --- a/src/shared/components/RequestLoggerDetail.tsx +++ b/src/shared/components/RequestLoggerDetail.tsx @@ -92,27 +92,21 @@ export default function RequestLoggerDetail({ log, detail, loading, onClose, onC const pipelinePayloads = detail?.pipelinePayloads || null; const payloadSections = pipelinePayloads ? [ - { - key: "client-request", - title: "Client Request", - json: toPrettyJson(pipelinePayloads.clientRequest), - }, - { - key: "provider-request", - title: "Provider Request", - json: toPrettyJson(pipelinePayloads.providerRequest), - }, - { - key: "provider-response", - title: "Provider Response", - json: toPrettyJson(pipelinePayloads.providerResponse), - }, - { - key: "client-response", - title: "Client Response", - json: toPrettyJson(pipelinePayloads.clientResponse), - }, - ].filter((section) => section.json) + ["clientRawRequest", "Client Raw Request"], + ["clientRequest", "Client Request"], + ["sourceRequest", "Source Request"], + ["openaiRequest", "OpenAI Request"], + ["providerRequest", "Provider Request"], + ["providerResponse", "Provider Response"], + ["clientResponse", "Client Response"], + ["error", "Pipeline Error"], + ] + .map(([key, title]) => ({ + key, + title, + json: toPrettyJson(pipelinePayloads[key]), + })) + .filter((section) => section.json) : []; const requestJson = detail?.requestBody ? toPrettyJson(detail.requestBody) : null; const responseJson = detail?.responseBody ? toPrettyJson(detail.responseBody) : null; diff --git a/src/shared/components/RequestLoggerV2.tsx b/src/shared/components/RequestLoggerV2.tsx index e8a1ca366d..5b5fba9fc1 100644 --- a/src/shared/components/RequestLoggerV2.tsx +++ b/src/shared/components/RequestLoggerV2.tsx @@ -315,7 +315,7 @@ export default function RequestLoggerV2() { ? "bg-amber-500/10 border-amber-500/30 text-amber-700 dark:text-amber-300" : "bg-bg-subtle border-border text-text-muted" }`} - title="Capture four-stage pipeline payloads for new requests" + title="Capture per-request pipeline payloads inside the unified call log artifact" >
- Call logs are also saved as JSON files to {`{DATA_DIR}/call_logs/`} with 7-day - rotation. + Each request is also saved as a single JSON artifact in{" "} + {`{DATA_DIR}/call_logs/`}.
{/* Detail Modal */} diff --git a/src/shared/schemas/validation.ts b/src/shared/schemas/validation.ts index bcf0bda487..e73a657a38 100644 --- a/src/shared/schemas/validation.ts +++ b/src/shared/schemas/validation.ts @@ -60,8 +60,6 @@ export const settingsSchema = z requireLogin: z.boolean().optional(), password: z.string().min(6, "Password must be at least 6 characters").optional(), defaultModel: z.string().optional(), - enableRequestLogs: z.boolean().optional(), - maxLogRetention: z.number().int().min(1).max(365).optional(), rateLimitEnabled: z.boolean().optional(), rateLimitPerMinute: z.number().int().min(0).optional(), }) diff --git a/src/shared/utils/logger.ts b/src/shared/utils/logger.ts index a5f8f414d9..edb2874200 100644 --- a/src/shared/utils/logger.ts +++ b/src/shared/utils/logger.ts @@ -10,17 +10,18 @@ * In development, output is pretty-printed via pino-pretty. * In production, output is structured JSON for log aggregation. * - * When LOG_TO_FILE is enabled (default: true), logs are also written - * as JSON lines to the file specified by LOG_FILE_PATH. + * When APP_LOG_TO_FILE is enabled (default: true), logs are also written + * as JSON lines to the file specified by APP_LOG_FILE_PATH. */ import pino from "pino"; import { resolve } from "path"; import { getLogConfig, initLogRotation } from "@/lib/logRotation"; +import { getAppLogLevel } from "@/lib/logEnv"; const isDev = process.env.NODE_ENV !== "production"; const baseConfig: pino.LoggerOptions = { - level: process.env.LOG_LEVEL || (isDev ? "debug" : "info"), + level: getAppLogLevel(isDev ? "debug" : "info"), base: { service: "omniroute" }, timestamp: pino.stdTimeFunctions.isoTime, formatters: { diff --git a/src/shared/utils/structuredLogger.ts b/src/shared/utils/structuredLogger.ts index 03b4e98e54..c6d2bd03ba 100644 --- a/src/shared/utils/structuredLogger.ts +++ b/src/shared/utils/structuredLogger.ts @@ -5,7 +5,7 @@ * and human-readable output for development. Replaces scattered console.log * calls with consistent, parseable log entries. * - * When LOG_TO_FILE is enabled, log entries are also appended as JSON lines + * When APP_LOG_TO_FILE is enabled, log entries are also appended as JSON lines * to the application log file for the Console Log Viewer. * * @module shared/utils/structuredLogger @@ -14,6 +14,7 @@ import { getCorrelationId } from "../middleware/correlationId"; import { appendFileSync, existsSync, mkdirSync } from "fs"; import { dirname, resolve } from "path"; +import { getAppLogFilePath, getAppLogLevel, getAppLogToFile } from "@/lib/logEnv"; const LOG_LEVELS: Record = { debug: 10, @@ -23,12 +24,12 @@ const LOG_LEVELS: Record = { fatal: 50, }; -const currentLevel = LOG_LEVELS[process.env.LOG_LEVEL?.toLowerCase() || ""] || LOG_LEVELS.info; +const currentLevel = LOG_LEVELS[getAppLogLevel("info").toLowerCase() || ""] || LOG_LEVELS.info; const isProduction = process.env.NODE_ENV === "production"; // File logging configuration -const logToFile = process.env.LOG_TO_FILE !== "false"; -const logFilePath = resolve(process.env.LOG_FILE_PATH || "logs/application/app.log"); +const logToFile = getAppLogToFile(); +const logFilePath = resolve(getAppLogFilePath()); // Ensure log directory exists once at module load if (logToFile) { diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index d4e12d4181..fbab5451aa 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -148,12 +148,9 @@ export const updateSettingsSchema = z.object({ theme: z.string().max(50).optional(), language: z.string().max(10).optional(), requireLogin: z.boolean().optional(), - enableRequestLogs: z.boolean().optional(), enableSocks5Proxy: z.boolean().optional(), instanceName: z.string().max(100).optional(), corsOrigins: z.string().max(500).optional(), - logRetentionDays: z.number().int().min(1).max(365).optional(), - maxCallLogs: z.number().int().min(1).max(1_000_000).optional(), cloudUrl: z.string().max(500).optional(), baseUrl: z.string().max(500).optional(), setupComplete: z.boolean().optional(), diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index 9d56f2b394..ac174837a6 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -14,12 +14,9 @@ export const updateSettingsSchema = z.object({ theme: z.string().max(50).optional(), language: z.string().max(10).optional(), requireLogin: z.boolean().optional(), - enableRequestLogs: z.boolean().optional(), enableSocks5Proxy: z.boolean().optional(), instanceName: z.string().max(100).optional(), corsOrigins: z.string().max(500).optional(), - logRetentionDays: z.number().int().min(1).max(365).optional(), - maxCallLogs: z.number().int().min(1).max(1_000_000).optional(), cloudUrl: z.string().max(500).optional(), baseUrl: z.string().max(500).optional(), setupComplete: z.boolean().optional(), diff --git a/tests/e2e/settings-toggles.spec.ts b/tests/e2e/settings-toggles.spec.ts new file mode 100644 index 0000000000..fbd4167488 --- /dev/null +++ b/tests/e2e/settings-toggles.spec.ts @@ -0,0 +1,51 @@ +import { test, expect } from "@playwright/test"; + +test.describe("Settings Toggles", () => { + test("Debug mode toggle should work", async ({ page }) => { + await page.goto("/dashboard/settings"); + await page.waitForLoadState("networkidle"); + await page.click("text=Advanced"); + + const debugToggle = page.locator('[aria-label*="debug" i], [data-testid*="debug" i]').first(); + + await expect(debugToggle).toBeVisible({ timeout: 5000 }); + + const initialState = await debugToggle.isChecked(); + await debugToggle.click(); + await expect(debugToggle).not.toBeChecked({ timeout: 5000 }); + }); + + test("Sidebar visibility toggle should work", async ({ page }) => { + await page.goto("/dashboard/settings"); + await page.waitForLoadState("networkidle"); + await page.click("text=General"); + + const sidebarToggle = page + .locator('[aria-label*="sidebar" i], [data-testid*="sidebar" i]') + .first(); + + await expect(sidebarToggle).toBeVisible({ timeout: 5000 }); + + const initialState = await sidebarToggle.isChecked(); + await sidebarToggle.click(); + await expect(sidebarToggle).not.toBeChecked({ timeout: 5000 }); + }); + + test("Debug mode should persist after page reload", async ({ page }) => { + await page.goto("/dashboard/settings"); + await page.waitForLoadState("networkidle"); + await page.click("text=Advanced"); + + const debugToggle = page.locator('[aria-label*="debug" i], [data-testid*="debug" i]').first(); + + await expect(debugToggle).toBeVisible({ timeout: 5000 }); + + const wasChecked = await debugToggle.isChecked(); + await debugToggle.click(); + await expect(debugToggle).not.toBeChecked({ timeout: 5000 }); + await page.reload(); + await page.waitForLoadState("networkidle"); + await page.click("text=Advanced"); + await expect(debugToggle).not.toBeChecked({ timeout: 5000 }); + }); +}); diff --git a/tests/unit/batch-b-final.test.mjs b/tests/unit/batch-b-final.test.mjs index e11b2e87e1..d1843605ca 100644 --- a/tests/unit/batch-b-final.test.mjs +++ b/tests/unit/batch-b-final.test.mjs @@ -40,7 +40,13 @@ describe("evalRunner", () => { it("should evaluate exact match", () => { const result = evaluateCase( - { id: "t1", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } }, + { + id: "t1", + name: "test", + model: "test", + input: {}, + expected: { strategy: "exact", value: "hello" }, + }, "hello" ); assert.equal(result.passed, true); @@ -48,7 +54,13 @@ describe("evalRunner", () => { it("should fail exact match on mismatch", () => { const result = evaluateCase( - { id: "t2", name: "test", model: "test", input: {}, expected: { strategy: "exact", value: "hello" } }, + { + id: "t2", + name: "test", + model: "test", + input: {}, + expected: { strategy: "exact", value: "hello" }, + }, "world" ); assert.equal(result.passed, false); @@ -56,7 +68,13 @@ describe("evalRunner", () => { it("should evaluate contains (case-insensitive)", () => { const result = evaluateCase( - { id: "t3", name: "test", model: "test", input: {}, expected: { strategy: "contains", value: "paris" } }, + { + id: "t3", + name: "test", + model: "test", + input: {}, + expected: { strategy: "contains", value: "paris" }, + }, "The capital is Paris." ); assert.equal(result.passed, true); @@ -64,7 +82,13 @@ describe("evalRunner", () => { it("should evaluate regex", () => { const result = evaluateCase( - { id: "t4", name: "test", model: "test", input: {}, expected: { strategy: "regex", value: "\\d+" } }, + { + id: "t4", + name: "test", + model: "test", + input: {}, + expected: { strategy: "regex", value: "\\d+" }, + }, "The answer is 42." ); assert.equal(result.passed, true); @@ -73,7 +97,10 @@ describe("evalRunner", () => { it("should evaluate custom function", () => { const result = evaluateCase( { - id: "t5", name: "test", model: "test", input: {}, + id: "t5", + name: "test", + model: "test", + input: {}, expected: { strategy: "custom", fn: (output) => output.length > 5 }, }, "this is long enough" @@ -95,8 +122,20 @@ describe("evalRunner", () => { id: "test-suite", name: "Test Suite", cases: [ - { id: "c1", name: "pass", model: "m", input: {}, expected: { strategy: "contains", value: "yes" } }, - { id: "c2", name: "fail", model: "m", input: {}, expected: { strategy: "contains", value: "no" } }, + { + id: "c1", + name: "pass", + model: "m", + input: {}, + expected: { strategy: "contains", value: "yes" }, + }, + { + id: "c2", + name: "fail", + model: "m", + input: {}, + expected: { strategy: "contains", value: "no" }, + }, ], }); @@ -225,7 +264,7 @@ describe("compliance", () => { assert.equal(isNoLog("key-1"), false); }); - it("should have default retention of 90 days", () => { - assert.equal(getRetentionDays(), 90); + it("should expose split default retention windows", () => { + assert.deepEqual(getRetentionDays(), { app: 7, call: 7 }); }); }); diff --git a/tests/unit/cache-control-claude-providers.test.mjs b/tests/unit/cache-control-claude-providers.test.mjs new file mode 100644 index 0000000000..ae437cdf89 --- /dev/null +++ b/tests/unit/cache-control-claude-providers.test.mjs @@ -0,0 +1,141 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { + providerSupportsCaching, + shouldPreserveCacheControl, +} from "../../open-sse/utils/cacheControlPolicy.ts"; + +describe("Cache Control Policy - Claude Protocol Providers", () => { + test("providerSupportsCaching returns true for Claude-format providers", () => { + // Known caching providers + assert.equal(providerSupportsCaching("claude", "claude"), true); + assert.equal(providerSupportsCaching("anthropic", "claude"), true); + assert.equal(providerSupportsCaching("zai", "claude"), true); + assert.equal(providerSupportsCaching("qwen", "openai"), true); + assert.equal(providerSupportsCaching("deepseek", "openai"), true); + + // Claude-protocol providers NOT in CACHING_PROVIDERS set + // These should be detected via targetFormat + assert.equal(providerSupportsCaching("bailian-coding-plan", "claude"), true); + assert.equal(providerSupportsCaching("glm", "claude"), true); + assert.equal(providerSupportsCaching("minimax", "claude"), true); + assert.equal(providerSupportsCaching("minimax-cn", "claude"), true); + assert.equal(providerSupportsCaching("kimi-coding", "claude"), true); + assert.equal(providerSupportsCaching("alicode", "claude"), true); + + // Non-Claude providers without caching support + assert.equal(providerSupportsCaching("openai", "openai"), false); + assert.equal(providerSupportsCaching("gemini", "gemini"), false); + }); + + test("shouldPreserveCacheControl preserves for Claude-format providers with Claude Code client", () => { + const claudeCodeUA = "Claude-Code/1.0.0"; + + // Claude-protocol providers should preserve cache_control + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "bailian-coding-plan", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + true + ); + + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "glm", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + true + ); + + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "zai", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + true + ); + + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "minimax", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + true + ); + }); + + test("shouldPreserveCacheControl respects user override 'always'", () => { + const regularUA = "Mozilla/5.0"; + + // Even with non-Claude Code client, 'always' should preserve + assert.equal( + shouldPreserveCacheControl({ + userAgent: regularUA, + isCombo: false, + targetProvider: "bailian-coding-plan", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "always" }, + }), + true + ); + }); + + test("shouldPreserveCacheControl respects user override 'never'", () => { + const claudeCodeUA = "Claude-Code/1.0.0"; + + // Even with Claude Code client, 'never' should not preserve + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "bailian-coding-plan", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "never" }, + }), + false + ); + }); + + test("shouldPreserveCacheControl does not preserve for non-Claude Code clients in auto mode", () => { + const regularUA = "Mozilla/5.0"; + + assert.equal( + shouldPreserveCacheControl({ + userAgent: regularUA, + isCombo: false, + targetProvider: "bailian-coding-plan", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + false + ); + }); + + test("shouldPreserveCacheControl does not preserve for non-Claude format providers", () => { + const claudeCodeUA = "Claude-Code/1.0.0"; + + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "openai", + targetFormat: "openai", + settings: { alwaysPreserveClientCache: "auto" }, + }), + false + ); + }); +}); diff --git a/tests/unit/call-log-cap.test.mjs b/tests/unit/call-log-cap.test.mjs index 314fe13bdd..a2fe775d49 100644 --- a/tests/unit/call-log-cap.test.mjs +++ b/tests/unit/call-log-cap.test.mjs @@ -4,19 +4,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-cap-")); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-artifacts-")); process.env.DATA_DIR = TEST_DATA_DIR; -const ORIGINAL_CALL_LOGS_MAX = process.env.CALL_LOGS_MAX; +process.env.CALL_LOG_RETENTION_DAYS = "1"; const core = await import("../../src/lib/db/core.ts"); -const localDb = await import("../../src/lib/localDb.ts"); const callLogs = await import("../../src/lib/usage/callLogs.ts"); async function resetStorage() { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); - callLogs.invalidateCallLogsMaxCache(); } test.beforeEach(async () => { @@ -26,61 +24,69 @@ test.beforeEach(async () => { test.after(() => { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); - if (ORIGINAL_CALL_LOGS_MAX === undefined) { - delete process.env.CALL_LOGS_MAX; - } else { - process.env.CALL_LOGS_MAX = ORIGINAL_CALL_LOGS_MAX; - } }); -test("call logs respect the configurable maxCallLogs setting", async () => { - await localDb.updateSettings({ maxCallLogs: 3 }); - callLogs.invalidateCallLogsMaxCache(); +test("call logs store a single per-request artifact with pipeline details", async () => { + const timestamp = "2026-03-30T12:34:56.789Z"; + const logId = "req_artifact_1"; - for (let i = 1; i <= 5; i++) { - await callLogs.saveCallLog({ - method: "POST", - path: "/v1/chat/completions", - status: 200, - model: `model-${i}`, - provider: "openai", - duration: i, - requestBody: { index: i }, - responseBody: { ok: true, index: i }, - }); - } + await callLogs.saveCallLog({ + id: logId, + timestamp, + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + requestedModel: "openai/gpt-5", + provider: "openai", + duration: 42, + requestBody: { messages: [{ role: "user", content: "hello" }] }, + responseBody: { id: "resp_1", choices: [{ message: { content: "world" } }] }, + pipelinePayloads: { + clientRawRequest: { body: { raw: true } }, + providerRequest: { body: { translated: true } }, + providerResponse: { body: { upstream: true } }, + clientResponse: { body: { final: true } }, + }, + }); - const logs = await callLogs.getCallLogs({ limit: 10 }); + const logs = await callLogs.getCallLogs({ limit: 5 }); + assert.equal(logs.length, 1); + assert.equal(logs[0].hasPipelineDetails, true); - assert.equal(logs.length, 3); - assert.deepEqual( - logs.map((entry) => entry.model), - ["model-5", "model-4", "model-3"] + const detail = await callLogs.getCallLogById(logId); + assert.equal(detail?.requestedModel, "openai/gpt-5"); + assert.equal(detail?.pipelinePayloads?.clientRawRequest?.body?.raw, true); + assert.equal(detail?.pipelinePayloads?.providerRequest?.body?.translated, true); + assert.equal(detail?.pipelinePayloads?.providerResponse?.body?.upstream, true); + assert.equal(detail?.pipelinePayloads?.clientResponse?.body?.final, true); + assert.match( + detail?.artifactRelPath || "", + /^2026-03-30\/2026-03-30T12-34-56\.789Z_req_artifact_1\.json$/ ); + + const artifactPath = path.join(TEST_DATA_DIR, "call_logs", detail.artifactRelPath); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")); + assert.equal(artifact.summary.id, logId); + assert.equal(artifact.summary.requestedModel, "openai/gpt-5"); + assert.equal(artifact.pipeline.clientRawRequest.body.raw, true); }); -test("call logs keep honoring CALL_LOGS_MAX when maxCallLogs was never saved", async () => { - process.env.CALL_LOGS_MAX = "2"; - callLogs.invalidateCallLogsMaxCache(); +test("call log artifact rotation removes directories older than CALL_LOG_RETENTION_DAYS", async () => { + const oldDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-10"); + const freshDir = path.join(TEST_DATA_DIR, "call_logs", "2026-03-30"); + fs.mkdirSync(oldDir, { recursive: true }); + fs.mkdirSync(freshDir, { recursive: true }); + fs.writeFileSync(path.join(oldDir, "old.json"), "{}"); + fs.writeFileSync(path.join(freshDir, "fresh.json"), "{}"); - for (let i = 1; i <= 4; i++) { - await callLogs.saveCallLog({ - method: "POST", - path: "/v1/chat/completions", - status: 200, - model: `env-model-${i}`, - provider: "openai", - duration: i, - requestBody: { index: i }, - responseBody: { ok: true, index: i }, - }); - } + const oldTime = new Date("2026-03-10T00:00:00.000Z"); + const freshTime = new Date(); + fs.utimesSync(oldDir, oldTime, oldTime); + fs.utimesSync(freshDir, freshTime, freshTime); - const logs = await callLogs.getCallLogs({ limit: 10 }); + callLogs.rotateCallLogs(); - assert.equal(logs.length, 2); - assert.deepEqual( - logs.map((entry) => entry.model), - ["env-model-4", "env-model-3"] - ); + assert.equal(fs.existsSync(oldDir), false); + assert.equal(fs.existsSync(freshDir), true); }); diff --git a/tests/unit/console-log-levels.test.mjs b/tests/unit/console-log-levels.test.mjs index d268f7aba0..ab90ab1278 100644 --- a/tests/unit/console-log-levels.test.mjs +++ b/tests/unit/console-log-levels.test.mjs @@ -7,16 +7,16 @@ import path from "node:path"; const TEST_LOG_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-console-log-levels-")); const TEST_LOG_PATH = path.join(TEST_LOG_DIR, "app.log"); -const originalLogFilePath = process.env.LOG_FILE_PATH; -process.env.LOG_FILE_PATH = TEST_LOG_PATH; +const originalLogFilePath = process.env.APP_LOG_FILE_PATH; +process.env.APP_LOG_FILE_PATH = TEST_LOG_PATH; const route = await import("../../src/app/api/logs/console/route.ts"); test.after(() => { if (originalLogFilePath === undefined) { - delete process.env.LOG_FILE_PATH; + delete process.env.APP_LOG_FILE_PATH; } else { - process.env.LOG_FILE_PATH = originalLogFilePath; + process.env.APP_LOG_FILE_PATH = originalLogFilePath; } fs.rmSync(TEST_LOG_DIR, { recursive: true, force: true }); }); diff --git a/tests/unit/error-classification.test.mjs b/tests/unit/error-classification.test.mjs index 78136f07e6..9fb24d7181 100644 --- a/tests/unit/error-classification.test.mjs +++ b/tests/unit/error-classification.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { checkFallbackError, getProviderProfile } = +const { checkFallbackError, getProviderProfile, parseRetryFromErrorText } = await import("../../open-sse/services/accountFallback.ts"); const { getProviderCategory } = await import("../../open-sse/config/providerRegistry.ts"); @@ -127,3 +127,64 @@ test("400 bad request: still returns shouldFallback false", () => { const result = checkFallbackError(400, "", 0, null, "groq"); assert.equal(result.shouldFallback, false); }); + +// ─── T07: Retry Time Parsing from Error Text ───────────────────────────────── + +test("parseRetryFromErrorText: parses 27h41m36s format", () => { + const result = parseRetryFromErrorText("Your quota will reset after 27h41m36s"); + assert.equal(result, 27 * 3600 * 1000 + 41 * 60 * 1000 + 36 * 1000); +}); + +test("parseRetryFromErrorText: parses 2h30m format", () => { + const result = parseRetryFromErrorText("quota will reset after 2h30m"); + assert.equal(result, 2 * 3600 * 1000 + 30 * 60 * 1000); +}); + +test("parseRetryFromErrorText: parses 45m format", () => { + const result = parseRetryFromErrorText("reset after 45m"); + assert.equal(result, 45 * 60 * 1000); +}); + +test("parseRetryFromErrorText: parses 30s format", () => { + const result = parseRetryFromErrorText("reset after 30s"); + assert.equal(result, 30 * 1000); +}); + +test("parseRetryFromErrorText: returns null for invalid format", () => { + const result = parseRetryFromErrorText("invalid error message"); + assert.equal(result, null); +}); + +test("parseRetryFromErrorText: parses will reset after variant", () => { + const result = parseRetryFromErrorText("quota will reset after 5h"); + assert.equal(result, 5 * 3600 * 1000); +}); + +// ─── T06: Keyword Matching for Long Cooldowns ──────────────────────────────── + +test("quota will reset keyword triggers long cooldown from body", () => { + const result = checkFallbackError( + 429, + "Your quota will reset after 27h41m36s", + 0, + null, + "antigravity", + null + ); + assert.equal(result.shouldFallback, true); + assert.ok(result.cooldownMs > 60_000, "cooldownMs should be > 60s"); + assert.equal(result.newBackoffLevel, 0, "backoffLevel should reset to 0"); +}); + +test("exhausted your capacity keyword triggers long cooldown", () => { + const result = checkFallbackError( + 429, + "You have exhausted your capacity. Your quota will reset after 2h", + 0, + null, + "antigravity", + null + ); + assert.equal(result.shouldFallback, true); + assert.ok(result.cooldownMs > 60_000); +}); diff --git a/tests/unit/log-retention.test.mjs b/tests/unit/log-retention.test.mjs new file mode 100644 index 0000000000..7f4bc2b931 --- /dev/null +++ b/tests/unit/log-retention.test.mjs @@ -0,0 +1,134 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-retention-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.APP_LOG_RETENTION_DAYS = "2"; +process.env.CALL_LOG_RETENTION_DAYS = "1"; + +const core = await import("../../src/lib/db/core.ts"); +const compliance = await import("../../src/lib/compliance/index.ts"); + +function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => { + resetStorage(); +}); + +test.after(() => { + resetStorage(); +}); + +test("cleanupExpiredLogs uses separate APP and CALL retention windows", () => { + compliance.initAuditLog(); + const db = core.getDbInstance(); + + const oldCallTs = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString(); + const freshCallTs = new Date().toISOString(); + const oldAppTs = new Date(Date.now() - 5 * 24 * 60 * 60 * 1000).toISOString(); + const freshAppTs = new Date().toISOString(); + + db.prepare( + "INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ).run("openai", "old-usage", 1, 1, 1, 1, 1, oldCallTs); + db.prepare( + "INSERT INTO usage_history (provider, model, tokens_input, tokens_output, success, latency_ms, ttft_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ).run("openai", "fresh-usage", 1, 1, 1, 1, 1, freshCallTs); + + db.prepare( + "INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ).run( + "old-call", + oldCallTs, + "POST", + "/v1/chat/completions", + 200, + "old", + "openai", + "-", + 1, + 1, + 1, + 0 + ); + db.prepare( + "INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, account, duration, tokens_in, tokens_out, has_pipeline_details) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ).run( + "fresh-call", + freshCallTs, + "POST", + "/v1/chat/completions", + 200, + "fresh", + "openai", + "-", + 1, + 1, + 1, + 0 + ); + + db.prepare( + "INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)" + ).run("old-proxy", oldCallTs, "success", "direct", 1); + db.prepare( + "INSERT INTO proxy_logs (id, timestamp, status, level, latency_ms) VALUES (?, ?, ?, ?, ?)" + ).run("fresh-proxy", freshCallTs, "success", "direct", 1); + + db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run( + "old-detail", + oldCallTs, + 1 + ); + db.prepare("INSERT INTO request_detail_logs (id, timestamp, duration_ms) VALUES (?, ?, ?)").run( + "fresh-detail", + freshCallTs, + 1 + ); + + db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run( + oldAppTs, + "old-audit", + "system" + ); + db.prepare("INSERT INTO audit_log (timestamp, action, actor) VALUES (?, ?, ?)").run( + freshAppTs, + "fresh-audit", + "system" + ); + + db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run( + "old-tool", + 1, + oldAppTs + ); + db.prepare("INSERT INTO mcp_tool_audit (tool_name, success, created_at) VALUES (?, ?, ?)").run( + "fresh-tool", + 1, + freshAppTs + ); + + const result = compliance.cleanupExpiredLogs(); + + assert.equal(result.deletedUsage, 1); + assert.equal(result.deletedCallLogs, 1); + assert.equal(result.deletedProxyLogs, 1); + assert.equal(result.deletedRequestDetailLogs, 1); + assert.equal(result.deletedAuditLogs, 1); + assert.equal(result.deletedMcpAuditLogs, 1); + assert.deepEqual(compliance.getRetentionDays(), { app: 2, call: 1 }); + + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM usage_history").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM call_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM proxy_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM request_detail_logs").get().cnt, 1); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM audit_log").get().cnt, 2); + assert.equal(db.prepare("SELECT COUNT(*) AS cnt FROM mcp_tool_audit").get().cnt, 1); +}); diff --git a/tests/unit/model-sync-route.test.mjs b/tests/unit/model-sync-route.test.mjs new file mode 100644 index 0000000000..5d1d9e7e18 --- /dev/null +++ b/tests/unit/model-sync-route.test.mjs @@ -0,0 +1,116 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-sync-route-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const callLogs = await import("../../src/lib/usage/callLogs.ts"); +const modelSyncRoute = await import("../../src/app/api/providers/[id]/sync-models/route.ts"); +const scheduler = await import("../../src/shared/services/modelSyncScheduler.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("model sync route skips success log when fetched models do not change stored models", async () => { + await resetStorage(); + + const connection = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "MAIN", + displayName: "channel-east", + apiKey: "test-key", + }); + + await modelsDb.replaceCustomModels("openai", [ + { + id: "custom-model-1", + name: "Custom Model 1", + source: "auto-sync", + }, + ]); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + assert.equal(String(url), `http://localhost/api/providers/${connection.id}/models`); + return Response.json({ + models: [{ id: "custom-model-1", name: "Custom Model 1" }], + }); + }; + + try { + const response = await modelSyncRoute.POST( + new Request(`http://localhost/api/providers/${connection.id}/sync-models`, { + method: "POST", + headers: scheduler.buildModelSyncInternalHeaders(), + }), + { params: { id: connection.id } } + ); + + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.logged, false); + assert.deepEqual(body.modelChanges, { added: 0, removed: 0, updated: 0, total: 0 }); + + const logs = await callLogs.getCallLogs({ model: "model-sync", limit: 10 }); + assert.equal(logs.length, 0); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("model sync route logs changed model syncs against the current channel label", async () => { + await resetStorage(); + + const connection = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "MAIN", + displayName: "channel-west", + apiKey: "test-key", + }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + assert.equal(String(url), `http://localhost/api/providers/${connection.id}/models`); + return Response.json({ + models: [{ id: "custom-model-2", name: "Custom Model 2" }], + }); + }; + + try { + const response = await modelSyncRoute.POST( + new Request(`http://localhost/api/providers/${connection.id}/sync-models`, { + method: "POST", + headers: scheduler.buildModelSyncInternalHeaders(), + }), + { params: { id: connection.id } } + ); + + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.logged, true); + assert.deepEqual(body.modelChanges, { added: 1, removed: 0, updated: 0, total: 1 }); + + const logs = await callLogs.getCallLogs({ model: "model-sync", limit: 10 }); + assert.equal(logs.length, 1); + assert.equal(logs[0].provider, "channel-west"); + assert.equal(logs[0].model, "model-sync"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/request-log-migration.test.mjs b/tests/unit/request-log-migration.test.mjs new file mode 100644 index 0000000000..95a4c4bce6 --- /dev/null +++ b/tests/unit/request-log-migration.test.mjs @@ -0,0 +1,72 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-log-migration-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const migrations = await import("../../src/lib/usage/migrations.ts"); + +const LEGACY_LOGS_DIR = path.join(TEST_DATA_DIR, "logs"); +const LEGACY_CALL_LOGS_DIR = path.join(TEST_DATA_DIR, "call_logs"); +const LEGACY_SUMMARY_FILE = path.join(TEST_DATA_DIR, "log.txt"); +const MARKER_PATH = path.join(migrations.LOG_ARCHIVES_DIR, "legacy-request-logs.json"); + +function resetDataDir() { + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +function seedLegacyLayout() { + fs.mkdirSync(path.join(LEGACY_LOGS_DIR, "session-a"), { recursive: true }); + fs.writeFileSync( + path.join(LEGACY_LOGS_DIR, "session-a", "1_req_client.json"), + JSON.stringify({ ok: true }, null, 2) + ); + + fs.mkdirSync(path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30"), { recursive: true }); + fs.writeFileSync( + path.join(LEGACY_CALL_LOGS_DIR, "2026-03-30", "123000_model_200.json"), + JSON.stringify({ ok: true }, null, 2) + ); + + fs.writeFileSync(LEGACY_SUMMARY_FILE, "legacy summary\n"); +} + +test.beforeEach(() => { + resetDataDir(); +}); + +test.after(() => { + resetDataDir(); +}); + +test("archives legacy request log layout into a zip and removes old files", async () => { + seedLegacyLayout(); + + const archiveFilename = await migrations.archiveLegacyRequestLogs(); + + assert.match(archiveFilename || "", /_legacy-request-logs\.zip$/); + assert.equal(fs.existsSync(LEGACY_LOGS_DIR), false); + assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), false); + assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), false); + assert.equal(fs.existsSync(MARKER_PATH), true); + + const archivePath = path.join(migrations.LOG_ARCHIVES_DIR, archiveFilename); + assert.equal(fs.existsSync(archivePath), true); + assert.ok(fs.statSync(archivePath).size > 0); +}); + +test("keeps legacy files in place when zip creation fails", async () => { + seedLegacyLayout(); + fs.writeFileSync(migrations.LOG_ARCHIVES_DIR, "not-a-directory"); + + await assert.rejects(() => migrations.archiveLegacyRequestLogs()); + + assert.equal(fs.existsSync(LEGACY_LOGS_DIR), true); + assert.equal(fs.existsSync(LEGACY_CALL_LOGS_DIR), true); + assert.equal(fs.existsSync(LEGACY_SUMMARY_FILE), true); + assert.equal(fs.existsSync(MARKER_PATH), false); +}); diff --git a/tests/unit/settings-api.test.mjs b/tests/unit/settings-api.test.mjs new file mode 100644 index 0000000000..e3192e2c76 --- /dev/null +++ b/tests/unit/settings-api.test.mjs @@ -0,0 +1,67 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { getSettings, updateSettings } from "../../src/lib/db/settings.ts"; + +describe("Settings API - debugMode and hiddenSidebarItems", () => { + describe("debugMode", () => { + test("updateSettings with debugMode=true succeeds", async () => { + const result = await updateSettings({ debugMode: true }); + assert.ok(result, "updateSettings should return truthy result"); + + const settings = await getSettings(); + assert.strictEqual(settings.debugMode, true, "debugMode should be true"); + }); + + test("updateSettings with debugMode=false succeeds", async () => { + const result = await updateSettings({ debugMode: false }); + assert.ok(result, "updateSettings should return truthy result"); + + const settings = await getSettings(); + assert.strictEqual(settings.debugMode, false, "debugMode should be false"); + }); + }); + + describe("hiddenSidebarItems", () => { + test("updateSettings with hiddenSidebarItems=['translator'] succeeds", async () => { + const result = await updateSettings({ hiddenSidebarItems: ["translator"] }); + assert.ok(result, "updateSettings should return truthy result"); + + const settings = await getSettings(); + assert.deepStrictEqual( + settings.hiddenSidebarItems, + ["translator"], + "hiddenSidebarItems should contain translator" + ); + }); + + test("updateSettings with empty hiddenSidebarItems succeeds", async () => { + const result = await updateSettings({ hiddenSidebarItems: [] }); + assert.ok(result, "updateSettings should return truthy result"); + + const settings = await getSettings(); + assert.deepStrictEqual( + settings.hiddenSidebarItems, + [], + "hiddenSidebarItems should be empty array" + ); + }); + }); + + describe("combined updates", () => { + test("updateSettings with both debugMode and hiddenSidebarItems succeeds", async () => { + const result = await updateSettings({ + debugMode: true, + hiddenSidebarItems: ["translator"], + }); + assert.ok(result, "updateSettings should return truthy result"); + + const settings = await getSettings(); + assert.strictEqual(settings.debugMode, true, "debugMode should be true"); + assert.deepStrictEqual( + settings.hiddenSidebarItems, + ["translator"], + "hiddenSidebarItems should be updated" + ); + }); + }); +}); diff --git a/tests/unit/usage-analytics.test.mjs b/tests/unit/usage-analytics.test.mjs index 69f06bdc23..ed148ac656 100644 --- a/tests/unit/usage-analytics.test.mjs +++ b/tests/unit/usage-analytics.test.mjs @@ -12,8 +12,8 @@ const localDb = await import("../../src/lib/localDb.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const usageHistory = await import("../../src/lib/usage/usageHistory.ts"); const usageStats = await import("../../src/lib/usage/usageStats.ts"); +const callLogs = await import("../../src/lib/usage/callLogs.ts"); const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); -const { LOG_FILE } = await import("../../src/lib/usage/migrations.ts"); function clearPendingRequests() { const pending = usageHistory.getPendingRequests(); @@ -263,7 +263,7 @@ test("getUsageStats aggregates totals, buckets, pending requests, and cost break assert.equal(recentBucketTotal, 1); }); -test("request log appends readable entries and trims to the most recent 200 lines", async () => { +test("recent request summaries are generated from SQLite call logs", async () => { const connection = await providersDb.createProviderConnection({ provider: "log-provider", authType: "apikey", @@ -272,19 +272,23 @@ test("request log appends readable entries and trims to the most recent 200 line }); for (let i = 0; i < 205; i++) { - await usageHistory.appendRequestLog({ + await callLogs.saveCallLog({ + id: `log-${i}`, + timestamp: new Date(Date.now() + i).toISOString(), + method: "POST", + path: "/v1/chat/completions", + status: 200, model: `model-${i}`, provider: "log-provider", connectionId: connection.id, tokens: { input: i + 1, output: i + 2 }, - status: 200, + requestBody: { index: i }, + responseBody: { ok: true, index: i }, }); } const recent = await usageHistory.getRecentLogs(3); - const lines = fs.readFileSync(LOG_FILE, "utf8").trim().split("\n"); - assert.equal(lines.length, 200); assert.equal(recent.length, 3); assert.match(recent[0], /model-204/); assert.match(recent[0], /LOG-PROVIDER/);