mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
967 lines
32 KiB
TypeScript
967 lines
32 KiB
TypeScript
/**
|
|
* Structured call log management.
|
|
*
|
|
* SQLite stores only summary metadata. Detailed request/response payloads live in
|
|
* filesystem artifacts and are loaded only for explicit detail/export flows.
|
|
*/
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts";
|
|
import { getDbInstance } from "../db/core";
|
|
import { collectReferencedArtifacts, selectCallLogIdsBefore } from "./callLogsBoundedQueries";
|
|
import { getRequestDetailLogByCallLogId } from "../db/detailedLogs";
|
|
import { shouldPersistToDisk } from "./migrations";
|
|
import {
|
|
getLoggedInputTokens,
|
|
getLoggedOutputTokens,
|
|
getPromptCacheReadTokensOrNull,
|
|
getPromptCacheCreationTokensOrNull,
|
|
getReasoningTokensOrNull,
|
|
getObservedReasoning,
|
|
} from "./tokenAccounting";
|
|
import { isNoLog } from "../compliance/noLog";
|
|
import { protectPayloadForLog, parseStoredPayload } from "../logPayloads";
|
|
import { getCallLogMaxEntries, getCallLogRetentionDays, getCallLogsTableMaxRows } from "../logEnv";
|
|
import { pickDisplayValue } from "@/shared/utils/maskEmail";
|
|
import {
|
|
CALL_LOGS_DIR,
|
|
cleanupEmptyCallLogDirs,
|
|
deleteCallArtifact,
|
|
listCallLogArtifactFiles,
|
|
readCallArtifact,
|
|
writeCallArtifact,
|
|
type CallLogArtifact,
|
|
type CallLogDetailState,
|
|
} from "./callLogArtifacts";
|
|
import {
|
|
toNumber,
|
|
toStringOrNull,
|
|
parseInlineError,
|
|
normalizeDetailState,
|
|
sanitizeErrorForLog,
|
|
toStoredErrorSummary,
|
|
protectPipelinePayloads,
|
|
buildRequestSummary,
|
|
} from "./callLogs/format";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
const CALL_LOG_ROTATE_THROTTLE_MS = 60_000;
|
|
let lastCallLogRotationScheduledAt = 0;
|
|
let callLogRotateInFlight = false;
|
|
let callLogRotateScheduled = false;
|
|
|
|
type CallLogSummaryRow = {
|
|
id: string;
|
|
timestamp: string | null;
|
|
method: string | null;
|
|
path: string | null;
|
|
status: number | null;
|
|
model: string | null;
|
|
requested_model: string | null;
|
|
provider: string | null;
|
|
account: string | null;
|
|
connection_id: string | null;
|
|
duration: number | null;
|
|
tokens_in: number | null;
|
|
tokens_out: number | null;
|
|
tokens_cache_read: number | null;
|
|
tokens_cache_creation: number | null;
|
|
tokens_reasoning: number | null;
|
|
tokens_compressed: number | null;
|
|
cache_source: string | null;
|
|
request_type: string | null;
|
|
source_format: string | null;
|
|
target_format: string | null;
|
|
api_key_id: string | null;
|
|
api_key_name: string | null;
|
|
combo_name: string | null;
|
|
combo_step_id: string | null;
|
|
combo_execution_key: string | null;
|
|
error_summary: string | null;
|
|
detail_state: string | null;
|
|
artifact_relpath: string | null;
|
|
artifact_size_bytes: number | null;
|
|
artifact_sha256: string | null;
|
|
has_request_body: number | null;
|
|
has_response_body: number | null;
|
|
has_pipeline_details: number | null;
|
|
request_summary: string | null;
|
|
provider_node_name?: string | null;
|
|
provider_node_prefix?: string | null;
|
|
resolved_account?: string | null;
|
|
correlation_id?: string | null;
|
|
model_pinned?: number | null;
|
|
session_tag?: string | null;
|
|
};
|
|
|
|
const RESOLVED_ACCOUNT_SQL = "COALESCE(NULLIF(pc.name, ''), NULLIF(pc.email, ''), cl.account)";
|
|
|
|
type LegacyInlineRow = {
|
|
request_body: string | null;
|
|
response_body: string | null;
|
|
error: string | null;
|
|
};
|
|
|
|
type DeleteResult = {
|
|
deletedRows: number;
|
|
deletedArtifacts: number;
|
|
};
|
|
|
|
let logIdCounter = 0;
|
|
|
|
function generateLogId() {
|
|
logIdCounter++;
|
|
return `${Date.now()}-${logIdCounter}`;
|
|
}
|
|
|
|
async function resolveAccountName(connectionId: string | null | undefined) {
|
|
let account = connectionId ? connectionId.slice(0, 8) : "-";
|
|
|
|
if (!connectionId) {
|
|
return account;
|
|
}
|
|
|
|
try {
|
|
const { getProviderConnections } = await import("@/lib/localDb");
|
|
const connections = await getProviderConnections();
|
|
const conn = connections.find((item) => item.id === connectionId);
|
|
if (conn) {
|
|
account = pickDisplayValue(
|
|
[toStringOrNull(conn.name), toStringOrNull(conn.email)],
|
|
true,
|
|
account
|
|
);
|
|
}
|
|
} catch {
|
|
// Best-effort lookup only.
|
|
}
|
|
|
|
return account;
|
|
}
|
|
|
|
async function resolveProviderPrefix(providerId: string): Promise<string | null> {
|
|
if (!providerId) return null;
|
|
try {
|
|
const { getProviderNodeById } = await import("@/lib/localDb");
|
|
const node = await getProviderNodeById(providerId);
|
|
if (node && typeof node.prefix === "string" && node.prefix.trim().length > 0) {
|
|
return node.prefix.trim();
|
|
}
|
|
} catch {
|
|
// Best-effort lookup only.
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isCompatibleProviderId(providerId: string | null): boolean {
|
|
if (!providerId) return false;
|
|
return (
|
|
providerId.startsWith("openai-compatible-") || providerId.startsWith("anthropic-compatible-")
|
|
);
|
|
}
|
|
|
|
function applyNodePrefix(
|
|
requestedModel: string | null,
|
|
provider: string | null,
|
|
nodePrefix: string | null
|
|
): string | null {
|
|
if (!requestedModel || !provider || !nodePrefix) return requestedModel;
|
|
if (requestedModel.startsWith(provider + "/")) {
|
|
return nodePrefix + "/" + requestedModel.slice(provider.length + 1);
|
|
}
|
|
return requestedModel;
|
|
}
|
|
function buildArtifact(
|
|
logEntry: {
|
|
id: string;
|
|
timestamp: string;
|
|
method: string;
|
|
path: string;
|
|
status: number;
|
|
model: string;
|
|
requestedModel: string | null;
|
|
provider: string;
|
|
account: string;
|
|
connectionId: string | null;
|
|
duration: number;
|
|
tokensIn: number;
|
|
tokensOut: number;
|
|
tokensCacheRead: number | null;
|
|
tokensCacheCreation: number | null;
|
|
tokensReasoning: number | null;
|
|
tokensCompressed: number | null;
|
|
requestType: string | null;
|
|
sourceFormat: string | null;
|
|
targetFormat: string | null;
|
|
apiKeyId: string | null;
|
|
apiKeyName: string | null;
|
|
comboName: string | null;
|
|
comboStepId: string | null;
|
|
comboExecutionKey: string | null;
|
|
},
|
|
requestBody: unknown,
|
|
responseBody: unknown,
|
|
error: unknown,
|
|
pipelinePayloads: RequestPipelinePayloads | null
|
|
): CallLogArtifact {
|
|
return {
|
|
schemaVersion: 5,
|
|
summary: {
|
|
id: logEntry.id,
|
|
timestamp: logEntry.timestamp,
|
|
method: logEntry.method,
|
|
path: logEntry.path,
|
|
status: logEntry.status,
|
|
model: logEntry.model,
|
|
requestedModel: logEntry.requestedModel,
|
|
provider: logEntry.provider,
|
|
account: logEntry.account,
|
|
connectionId: logEntry.connectionId,
|
|
duration: logEntry.duration,
|
|
tokens: {
|
|
in: logEntry.tokensIn,
|
|
out: logEntry.tokensOut,
|
|
cacheRead: logEntry.tokensCacheRead,
|
|
cacheWrite: logEntry.tokensCacheCreation,
|
|
reasoning: logEntry.tokensReasoning,
|
|
compressed: logEntry.tokensCompressed,
|
|
},
|
|
requestType: logEntry.requestType,
|
|
sourceFormat: logEntry.sourceFormat,
|
|
targetFormat: logEntry.targetFormat,
|
|
apiKeyId: logEntry.apiKeyId,
|
|
apiKeyName: logEntry.apiKeyName,
|
|
comboName: logEntry.comboName,
|
|
comboStepId: logEntry.comboStepId,
|
|
comboExecutionKey: logEntry.comboExecutionKey,
|
|
},
|
|
requestBody: requestBody ?? null,
|
|
responseBody: responseBody ?? null,
|
|
error: error ?? null,
|
|
...(pipelinePayloads ? { pipeline: pipelinePayloads } : {}),
|
|
};
|
|
}
|
|
|
|
// #6187: extract the assistant message from a chat-completion-shaped response
|
|
// body so we can inspect its reasoning_content / <think> content.
|
|
function extractAssistantMessage(responseBody: unknown): unknown {
|
|
if (!responseBody || typeof responseBody !== "object") return responseBody;
|
|
const choices = (responseBody as JsonRecord).choices;
|
|
if (Array.isArray(choices) && choices.length > 0) {
|
|
const first = choices[0] as JsonRecord;
|
|
return first?.message ?? first?.delta ?? first;
|
|
}
|
|
return responseBody;
|
|
}
|
|
|
|
// #6187: decide the reasoning SOURCE and (char-only) count recorded alongside
|
|
// the usage-derived tokens_reasoning. Usage is authoritative when it reports
|
|
// non-zero reasoning tokens; otherwise we fall back to observed reasoning
|
|
// content so "reasoned but metered 0" stays distinguishable. reasoning_chars is
|
|
// a CHARACTER count, never a token count — it must not touch cost math.
|
|
function resolveReasoningObservation(
|
|
usageReasoning: number | null,
|
|
responseBody: unknown
|
|
): { source: string | null; chars: number | null } {
|
|
if (usageReasoning != null && usageReasoning > 0) {
|
|
return { source: "usage", chars: null };
|
|
}
|
|
const observed = getObservedReasoning(extractAssistantMessage(responseBody));
|
|
if (observed.chars > 0) {
|
|
return { source: observed.source, chars: observed.chars };
|
|
}
|
|
return { source: null, chars: null };
|
|
}
|
|
|
|
function hasTable(tableName: string): boolean {
|
|
const db = getDbInstance();
|
|
return Boolean(
|
|
db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName)
|
|
);
|
|
}
|
|
|
|
function readLegacyLogFromDisk(entry: {
|
|
timestamp: string | null;
|
|
model: string | null;
|
|
status: number;
|
|
}) {
|
|
if (!CALL_LOGS_DIR || !entry.timestamp) return null;
|
|
|
|
try {
|
|
const date = new Date(entry.timestamp);
|
|
if (Number.isNaN(date.getTime())) return null;
|
|
|
|
const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(
|
|
2,
|
|
"0"
|
|
)}-${String(date.getDate()).padStart(2, "0")}`;
|
|
const dir = path.join(CALL_LOGS_DIR, dateFolder);
|
|
if (!fs.existsSync(dir)) return null;
|
|
|
|
const time = `${String(date.getHours()).padStart(2, "0")}${String(date.getMinutes()).padStart(
|
|
2,
|
|
"0"
|
|
)}${String(date.getSeconds()).padStart(2, "0")}`;
|
|
const safeModel = (entry.model || "unknown").replace(/[/:]/g, "-");
|
|
const expectedName = `${time}_${safeModel}_${entry.status}.json`;
|
|
|
|
const exactPath = path.join(dir, expectedName);
|
|
if (fs.existsSync(exactPath)) {
|
|
return JSON.parse(fs.readFileSync(exactPath, "utf8"));
|
|
}
|
|
|
|
const files = fs
|
|
.readdirSync(dir)
|
|
.filter((file) => file.startsWith(time) && file.endsWith(`_${entry.status}.json`));
|
|
if (files.length > 0) {
|
|
return JSON.parse(fs.readFileSync(path.join(dir, files[0]), "utf8"));
|
|
}
|
|
} catch (error) {
|
|
console.error("[callLogs] Failed to read legacy disk log:", (error as Error).message);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function clearArtifactReference(relativePath: string, nextState: CallLogDetailState) {
|
|
const db = getDbInstance();
|
|
db.prepare(
|
|
`
|
|
UPDATE call_logs
|
|
SET detail_state = ?,
|
|
artifact_relpath = NULL,
|
|
artifact_size_bytes = NULL,
|
|
artifact_sha256 = NULL
|
|
WHERE artifact_relpath = ?
|
|
`
|
|
).run(nextState, relativePath);
|
|
}
|
|
|
|
function listReferencedArtifacts() {
|
|
// #5618: paged to avoid an unbounded `.all()` OOM on large call_logs tables.
|
|
return collectReferencedArtifacts();
|
|
}
|
|
|
|
// #5217: SQLite caps a statement at SQLITE_MAX_VARIABLE_NUMBER bound params
|
|
// (~999 on many builds). Callers like trimCallLogsToMaxRows() passed up to 5000
|
|
// ids in one `IN (...)` → "too many SQL variables" aborted trimming. Chunk well
|
|
// under the limit so each DELETE/SELECT stays valid.
|
|
const DELETE_ID_CHUNK_SIZE = 500;
|
|
|
|
function deleteCallLogRowsByIds(ids: string[]): DeleteResult {
|
|
if (ids.length === 0) {
|
|
return { deletedRows: 0, deletedArtifacts: 0 };
|
|
}
|
|
|
|
const db = getDbInstance();
|
|
let deletedRows = 0;
|
|
let deletedArtifacts = 0;
|
|
|
|
for (let i = 0; i < ids.length; i += DELETE_ID_CHUNK_SIZE) {
|
|
const chunk = ids.slice(i, i + DELETE_ID_CHUNK_SIZE);
|
|
const placeholders = chunk.map(() => "?").join(", ");
|
|
const rows = db
|
|
.prepare(`SELECT artifact_relpath FROM call_logs WHERE id IN (${placeholders})`)
|
|
.all(...chunk) as Array<{ artifact_relpath: string | null }>;
|
|
|
|
const result = db.prepare(`DELETE FROM call_logs WHERE id IN (${placeholders})`).run(...chunk);
|
|
deletedRows += result.changes;
|
|
for (const row of rows) {
|
|
if (deleteCallArtifact(row.artifact_relpath)) {
|
|
deletedArtifacts++;
|
|
}
|
|
}
|
|
}
|
|
cleanupEmptyCallLogDirs();
|
|
|
|
return {
|
|
deletedRows,
|
|
deletedArtifacts,
|
|
};
|
|
}
|
|
|
|
export function cleanupOrphanCallLogFiles(baseDir = CALL_LOGS_DIR) {
|
|
if (!baseDir || !fs.existsSync(baseDir)) return 0;
|
|
|
|
try {
|
|
const referenced = listReferencedArtifacts();
|
|
let deleted = 0;
|
|
for (const file of listCallLogArtifactFiles(baseDir)) {
|
|
if (referenced.has(file.relativePath)) continue;
|
|
if (deleteCallArtifact(file.relativePath)) {
|
|
deleted++;
|
|
}
|
|
}
|
|
cleanupEmptyCallLogDirs(baseDir);
|
|
return deleted;
|
|
} catch (error) {
|
|
console.error("[callLogs] Failed to prune orphan request artifacts:", (error as Error).message);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?: number) {
|
|
if (!baseDir || !fs.existsSync(baseDir)) return 0;
|
|
|
|
const limit = maxEntries ?? getCallLogMaxEntries();
|
|
if (!Number.isInteger(limit) || limit < 1) return 0;
|
|
|
|
try {
|
|
let deleted = 0;
|
|
const files = listCallLogArtifactFiles(baseDir);
|
|
for (const file of files.slice(limit)) {
|
|
if (deleteCallArtifact(file.relativePath)) {
|
|
clearArtifactReference(file.relativePath, "missing");
|
|
deleted++;
|
|
}
|
|
}
|
|
cleanupEmptyCallLogDirs(baseDir);
|
|
return deleted;
|
|
} catch (error) {
|
|
console.error(
|
|
"[callLogs] Failed to prune overflow request artifacts:",
|
|
(error as Error).message
|
|
);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
export function deleteCallLogsBefore(cutoff: string): DeleteResult {
|
|
// #5618: page the id selection so a large backlog never loads in one `.all()`.
|
|
let deletedRows = 0;
|
|
let deletedArtifacts = 0;
|
|
for (;;) {
|
|
const ids = selectCallLogIdsBefore(cutoff);
|
|
if (ids.length === 0) break;
|
|
const result = deleteCallLogRowsByIds(ids);
|
|
deletedRows += result.deletedRows;
|
|
deletedArtifacts += result.deletedArtifacts;
|
|
if (result.deletedRows === 0) break;
|
|
}
|
|
return { deletedRows, deletedArtifacts };
|
|
}
|
|
|
|
export function trimCallLogsToMaxRows(maxRows = getCallLogsTableMaxRows()) {
|
|
if (!Number.isInteger(maxRows) || maxRows < 1) {
|
|
return { deletedRows: 0, deletedArtifacts: 0 };
|
|
}
|
|
|
|
const db = getDbInstance();
|
|
let deletedRows = 0;
|
|
let deletedArtifacts = 0;
|
|
const batchSize = 5000;
|
|
|
|
while (true) {
|
|
const currentCount = db.prepare("SELECT COUNT(*) AS cnt FROM call_logs").get() as {
|
|
cnt: number;
|
|
};
|
|
if (currentCount.cnt <= maxRows) break;
|
|
|
|
const toDelete = Math.min(currentCount.cnt - maxRows, batchSize);
|
|
const ids = db
|
|
.prepare("SELECT id FROM call_logs ORDER BY timestamp ASC LIMIT ?")
|
|
.all(toDelete)
|
|
.map((row) => String((row as { id: string }).id));
|
|
const result = deleteCallLogRowsByIds(ids);
|
|
deletedRows += result.deletedRows;
|
|
deletedArtifacts += result.deletedArtifacts;
|
|
if (result.deletedRows === 0) break;
|
|
}
|
|
|
|
return { deletedRows, deletedArtifacts };
|
|
}
|
|
|
|
function resolveProviderDisplay(
|
|
provider: string | null,
|
|
nodeName: string | null,
|
|
nodePrefix: string | null
|
|
): string | null {
|
|
const rawProvider = toStringOrNull(provider);
|
|
if (!rawProvider) return null;
|
|
|
|
const name = toStringOrNull(nodeName)?.trim();
|
|
if (name) return name;
|
|
|
|
const prefix = toStringOrNull(nodePrefix)?.trim();
|
|
if (prefix) return prefix;
|
|
|
|
return null;
|
|
}
|
|
|
|
function mapSummaryRow(row: CallLogSummaryRow) {
|
|
const detailState = normalizeDetailState(row.detail_state);
|
|
const provider = row.provider;
|
|
const nodeName = row.provider_node_name ?? null;
|
|
const nodePrefix = row.provider_node_prefix ?? null;
|
|
return {
|
|
id: row.id,
|
|
timestamp: row.timestamp,
|
|
method: row.method,
|
|
path: row.path,
|
|
status: toNumber(row.status),
|
|
model: row.model,
|
|
requestedModel: applyNodePrefix(row.requested_model, provider, nodePrefix),
|
|
provider,
|
|
providerDisplay: resolveProviderDisplay(provider, nodeName, nodePrefix),
|
|
account: row.resolved_account || row.account,
|
|
connectionId: row.connection_id,
|
|
duration: toNumber(row.duration),
|
|
tokens: {
|
|
in: toNumber(row.tokens_in),
|
|
out: toNumber(row.tokens_out),
|
|
cacheRead: row.tokens_cache_read != null ? toNumber(row.tokens_cache_read) : null,
|
|
cacheWrite: row.tokens_cache_creation != null ? toNumber(row.tokens_cache_creation) : null,
|
|
reasoning: row.tokens_reasoning != null ? toNumber(row.tokens_reasoning) : null,
|
|
compressed: row.tokens_compressed != null ? toNumber(row.tokens_compressed) : null,
|
|
},
|
|
cacheSource: row.cache_source || "upstream",
|
|
requestType: row.request_type,
|
|
sourceFormat: row.source_format,
|
|
targetFormat: row.target_format,
|
|
apiKeyId: row.api_key_id,
|
|
apiKeyName: row.api_key_name,
|
|
comboName: row.combo_name,
|
|
comboStepId: row.combo_step_id,
|
|
comboExecutionKey: row.combo_execution_key,
|
|
error: row.error_summary,
|
|
detailState,
|
|
artifactRelPath: row.artifact_relpath,
|
|
artifactSizeBytes: row.artifact_size_bytes,
|
|
artifactSha256: row.artifact_sha256,
|
|
requestSummary: row.request_summary ? parseStoredPayload(row.request_summary) : null,
|
|
hasRequestBody: toNumber(row.has_request_body) === 1,
|
|
hasResponseBody: toNumber(row.has_response_body) === 1,
|
|
hasPipelineDetails: toNumber(row.has_pipeline_details) === 1,
|
|
correlationId: row.correlation_id || null,
|
|
modelPinned: toNumber(row.model_pinned) === 1,
|
|
sessionTag: row.session_tag || null,
|
|
};
|
|
}
|
|
|
|
function buildLegacyPipelinePayloads(id: string) {
|
|
const detailed = getRequestDetailLogByCallLogId(id);
|
|
if (!detailed) return null;
|
|
|
|
return {
|
|
clientRequest: detailed.client_request ?? null,
|
|
providerRequest: detailed.translated_request ?? null,
|
|
providerResponse: detailed.provider_response ?? null,
|
|
clientResponse: detailed.client_response ?? null,
|
|
};
|
|
}
|
|
|
|
function getLegacyInlineDetail(id: string) {
|
|
if (!hasTable("call_logs_v1_legacy")) return null;
|
|
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare("SELECT request_body, response_body, error FROM call_logs_v1_legacy WHERE id = ?")
|
|
.get(id) as LegacyInlineRow | undefined;
|
|
if (!row) return null;
|
|
|
|
return {
|
|
requestBody: parseStoredPayload(row.request_body),
|
|
responseBody: parseStoredPayload(row.response_body),
|
|
error: parseInlineError(row.error),
|
|
};
|
|
}
|
|
|
|
export async function saveCallLog(entry: any) {
|
|
if (!shouldPersistToDisk) return;
|
|
|
|
try {
|
|
const apiKeyId = entry.apiKeyId || null;
|
|
const noLogEnabled = Boolean(entry.noLog) || (apiKeyId ? isNoLog(apiKeyId) : false);
|
|
|
|
const protectedRequestBody = noLogEnabled ? null : protectPayloadForLog(entry.requestBody);
|
|
const protectedResponseBody = noLogEnabled ? null : protectPayloadForLog(entry.responseBody);
|
|
const protectedPipelinePayloads = noLogEnabled
|
|
? null
|
|
: protectPipelinePayloads(entry.pipelinePayloads ?? entry.pipeline ?? null);
|
|
const protectedError = sanitizeErrorForLog(entry.error);
|
|
|
|
const account = await resolveAccountName(entry.connectionId || null);
|
|
const rawProvider: string = entry.provider || "-";
|
|
const rawRequestedModel: string | null = entry.requestedModel || null;
|
|
let resolvedRequestedModel = rawRequestedModel;
|
|
if (rawRequestedModel && isCompatibleProviderId(rawProvider)) {
|
|
const nodePrefix = await resolveProviderPrefix(rawProvider);
|
|
resolvedRequestedModel = applyNodePrefix(rawRequestedModel, rawProvider, nodePrefix);
|
|
}
|
|
// #6187: usage-derived reasoning tokens stay UNCHANGED (cost math reads this),
|
|
// while reasoning source/char-count are recorded separately for observability.
|
|
const tokensReasoning = getReasoningTokensOrNull(entry.tokens);
|
|
const reasoningObservation = resolveReasoningObservation(tokensReasoning, entry.responseBody);
|
|
const logEntry = {
|
|
id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(),
|
|
timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(),
|
|
method: entry.method || "POST",
|
|
path: entry.path || "/v1/chat/completions",
|
|
status: entry.status || 0,
|
|
model: entry.model || "-",
|
|
requestedModel: resolvedRequestedModel,
|
|
provider: rawProvider,
|
|
account,
|
|
connectionId: entry.connectionId || null,
|
|
duration: entry.duration || 0,
|
|
tokensIn: toNumber(getLoggedInputTokens(entry.tokens)),
|
|
tokensOut: toNumber(getLoggedOutputTokens(entry.tokens)),
|
|
tokensCacheRead: getPromptCacheReadTokensOrNull(entry.tokens),
|
|
tokensCacheCreation: getPromptCacheCreationTokensOrNull(entry.tokens),
|
|
tokensReasoning,
|
|
reasoningSource: reasoningObservation.source,
|
|
reasoningChars: reasoningObservation.chars,
|
|
tokensCompressed: entry.tokensCompressed != null ? toNumber(entry.tokensCompressed) : null,
|
|
cacheSource: entry.cacheSource === "semantic" ? "semantic" : "upstream",
|
|
requestType: entry.requestType || null,
|
|
sourceFormat: entry.sourceFormat || null,
|
|
targetFormat: entry.targetFormat || null,
|
|
apiKeyId,
|
|
apiKeyName: entry.apiKeyName || null,
|
|
comboName: entry.comboName || null,
|
|
comboStepId: toStringOrNull(entry.comboStepId),
|
|
comboExecutionKey:
|
|
toStringOrNull(entry.comboExecutionKey) || toStringOrNull(entry.comboStepId),
|
|
correlationId: entry.correlationId || null,
|
|
modelPinned: entry.modelPinned ? 1 : 0,
|
|
sessionTag: entry.sessionTag || null,
|
|
};
|
|
|
|
const requestSummary = noLogEnabled
|
|
? null
|
|
: buildRequestSummary(logEntry.requestType, protectedRequestBody);
|
|
const detailExpected =
|
|
!noLogEnabled &&
|
|
(protectedRequestBody !== null ||
|
|
protectedResponseBody !== null ||
|
|
protectedError !== null ||
|
|
protectedPipelinePayloads !== null);
|
|
|
|
let detailState: CallLogDetailState = "none";
|
|
let artifactRelPath: string | null = null;
|
|
let artifactSizeBytes: number | null = null;
|
|
let artifactSha256: string | null = null;
|
|
|
|
if (detailExpected) {
|
|
const artifact = buildArtifact(
|
|
logEntry,
|
|
protectedRequestBody,
|
|
protectedResponseBody,
|
|
protectedError,
|
|
protectedPipelinePayloads
|
|
);
|
|
const artifactResult = writeCallArtifact(artifact);
|
|
if (artifactResult) {
|
|
detailState = "ready";
|
|
artifactRelPath = artifactResult.relPath;
|
|
artifactSizeBytes = artifactResult.sizeBytes;
|
|
artifactSha256 = artifactResult.sha256;
|
|
} else {
|
|
detailState = "missing";
|
|
}
|
|
}
|
|
|
|
const db = getDbInstance();
|
|
db.prepare(
|
|
`
|
|
INSERT INTO call_logs (
|
|
id, timestamp, method, path, status, model, requested_model, provider,
|
|
account, connection_id, duration, tokens_in, tokens_out,
|
|
tokens_cache_read, tokens_cache_creation, tokens_reasoning, tokens_compressed,
|
|
reasoning_source, reasoning_chars,
|
|
cache_source, request_type, source_format, target_format, api_key_id, api_key_name,
|
|
combo_name, combo_step_id, combo_execution_key, error_summary, detail_state,
|
|
artifact_relpath, artifact_size_bytes, artifact_sha256,
|
|
has_request_body, has_response_body, has_pipeline_details, request_summary,
|
|
correlation_id, model_pinned, session_tag
|
|
)
|
|
VALUES (
|
|
@id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider,
|
|
@account, @connectionId, @duration, @tokensIn, @tokensOut,
|
|
@tokensCacheRead, @tokensCacheCreation, @tokensReasoning, @tokensCompressed,
|
|
@reasoningSource, @reasoningChars,
|
|
@cacheSource, @requestType, @sourceFormat, @targetFormat, @apiKeyId, @apiKeyName,
|
|
@comboName, @comboStepId, @comboExecutionKey, @errorSummary, @detailState,
|
|
@artifactRelPath, @artifactSizeBytes, @artifactSha256,
|
|
@hasRequestBody, @hasResponseBody, @hasPipelineDetails, @requestSummary,
|
|
@correlationId, @modelPinned, @sessionTag
|
|
)
|
|
`
|
|
).run({
|
|
...logEntry,
|
|
errorSummary: toStoredErrorSummary(protectedError),
|
|
detailState,
|
|
artifactRelPath,
|
|
artifactSizeBytes,
|
|
artifactSha256,
|
|
hasRequestBody: protectedRequestBody !== null ? 1 : 0,
|
|
hasResponseBody: protectedResponseBody !== null ? 1 : 0,
|
|
hasPipelineDetails: protectedPipelinePayloads ? 1 : 0,
|
|
requestSummary,
|
|
});
|
|
|
|
scheduleCallLogRotation();
|
|
} catch (error) {
|
|
console.error("[callLogs] Failed to save call log:", (error as Error).message);
|
|
}
|
|
}
|
|
|
|
export function rotateCallLogs() {
|
|
try {
|
|
if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return;
|
|
|
|
const retentionMs = getCallLogRetentionDays() * 24 * 60 * 60 * 1000;
|
|
const cutoff = new Date(Date.now() - retentionMs).toISOString();
|
|
|
|
deleteCallLogsBefore(cutoff);
|
|
trimCallLogsToMaxRows(getCallLogsTableMaxRows());
|
|
cleanupOverflowCallLogFiles(CALL_LOGS_DIR, getCallLogMaxEntries());
|
|
cleanupOrphanCallLogFiles(CALL_LOGS_DIR);
|
|
} catch (error) {
|
|
console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
|
|
}
|
|
}
|
|
|
|
function runScheduledCallLogRotation() {
|
|
if (callLogRotateInFlight) return;
|
|
callLogRotateInFlight = true;
|
|
setImmediate(() => {
|
|
try {
|
|
rotateCallLogs();
|
|
} catch (error) {
|
|
console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message);
|
|
} finally {
|
|
callLogRotateInFlight = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
export function scheduleCallLogRotation() {
|
|
if (!CALL_LOGS_DIR) return;
|
|
const elapsed = Date.now() - lastCallLogRotationScheduledAt;
|
|
if (elapsed >= CALL_LOG_ROTATE_THROTTLE_MS) {
|
|
lastCallLogRotationScheduledAt = Date.now();
|
|
runScheduledCallLogRotation();
|
|
return;
|
|
}
|
|
if (callLogRotateScheduled) return;
|
|
callLogRotateScheduled = true;
|
|
lastCallLogRotationScheduledAt = Date.now();
|
|
const timer = setTimeout(() => {
|
|
callLogRotateScheduled = false;
|
|
runScheduledCallLogRotation();
|
|
}, CALL_LOG_ROTATE_THROTTLE_MS - elapsed);
|
|
timer.unref?.();
|
|
}
|
|
|
|
if (shouldPersistToDisk && process.env.NODE_ENV !== "test") {
|
|
scheduleCallLogRotation();
|
|
}
|
|
|
|
/**
|
|
* Pushes a `column LIKE %value%` condition (mirrors the correlationId/sessionTag substring-match
|
|
* precedent). Extracted so getCallLogs stays under the max-lines-per-function ratchet — #8249.
|
|
*/
|
|
function pushLikeFilter(
|
|
conditions: string[],
|
|
params: Record<string, unknown>,
|
|
column: string,
|
|
paramKey: string,
|
|
value: unknown
|
|
) {
|
|
if (!value) return;
|
|
conditions.push(`cl.${column} LIKE @${paramKey}`);
|
|
params[paramKey] = `%${value}%`;
|
|
}
|
|
|
|
export async function getCallLogs(filter: any = {}) {
|
|
const db = getDbInstance();
|
|
let sql = `
|
|
SELECT cl.*,
|
|
pn.name AS provider_node_name, pn.prefix AS provider_node_prefix,
|
|
${RESOLVED_ACCOUNT_SQL} AS resolved_account
|
|
FROM call_logs cl
|
|
LEFT JOIN provider_nodes pn ON pn.id = cl.provider
|
|
LEFT JOIN provider_connections pc ON pc.id = cl.connection_id
|
|
`;
|
|
const conditions: string[] = [];
|
|
const params: Record<string, unknown> = {};
|
|
|
|
if (filter.status) {
|
|
if (filter.status === "error") {
|
|
conditions.push("(cl.status >= 400 OR cl.error_summary IS NOT NULL)");
|
|
} else if (filter.status === "ok") {
|
|
conditions.push("cl.status >= 200 AND cl.status < 300");
|
|
} else {
|
|
const statusCode = Number.parseInt(filter.status, 10);
|
|
if (!Number.isNaN(statusCode)) {
|
|
conditions.push("cl.status = @statusCode");
|
|
params.statusCode = statusCode;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (filter.model) {
|
|
conditions.push("(cl.model LIKE @modelQ OR cl.requested_model LIKE @modelQ)");
|
|
params.modelQ = `%${filter.model}%`;
|
|
}
|
|
if (filter.provider) {
|
|
conditions.push("cl.provider LIKE @providerQ");
|
|
params.providerQ = `%${filter.provider}%`;
|
|
}
|
|
if (filter.account) {
|
|
conditions.push(`(cl.account LIKE @accountQ OR ${RESOLVED_ACCOUNT_SQL} LIKE @accountQ)`);
|
|
params.accountQ = `%${filter.account}%`;
|
|
}
|
|
if (filter.apiKey) {
|
|
conditions.push("(cl.api_key_name LIKE @apiKeyQ OR cl.api_key_id LIKE @apiKeyQ)");
|
|
params.apiKeyQ = `%${filter.apiKey}%`;
|
|
}
|
|
pushLikeFilter(conditions, params, "correlation_id", "correlationId", filter.correlationId);
|
|
pushLikeFilter(conditions, params, "session_tag", "sessionTag", filter.sessionTag);
|
|
if (filter.combo) {
|
|
conditions.push("cl.combo_name IS NOT NULL");
|
|
}
|
|
if (filter.since) {
|
|
conditions.push("cl.timestamp >= @since");
|
|
params.since = filter.since instanceof Date ? filter.since.toISOString() : String(filter.since);
|
|
}
|
|
if (filter.until) {
|
|
conditions.push("cl.timestamp <= @until");
|
|
params.until = filter.until instanceof Date ? filter.until.toISOString() : String(filter.until);
|
|
}
|
|
if (filter.search) {
|
|
conditions.push(`(
|
|
cl.model LIKE @searchQ OR cl.path LIKE @searchQ OR cl.account LIKE @searchQ OR
|
|
${RESOLVED_ACCOUNT_SQL} LIKE @searchQ OR
|
|
cl.requested_model LIKE @searchQ OR cl.provider LIKE @searchQ OR
|
|
cl.api_key_name LIKE @searchQ OR cl.api_key_id LIKE @searchQ OR
|
|
cl.combo_name LIKE @searchQ OR CAST(cl.status AS TEXT) LIKE @searchQ
|
|
OR cl.combo_step_id LIKE @searchQ OR cl.combo_execution_key LIKE @searchQ
|
|
OR cl.error_summary LIKE @searchQ
|
|
OR cl.correlation_id LIKE @searchQ
|
|
)`);
|
|
params.searchQ = `%${filter.search}%`;
|
|
}
|
|
|
|
if (conditions.length > 0) {
|
|
sql += " WHERE " + conditions.join(" AND ");
|
|
}
|
|
|
|
const limit = Number.isInteger(filter.limit) && filter.limit > 0 ? filter.limit : 200;
|
|
const offset = Number.isInteger(filter.offset) && filter.offset > 0 ? filter.offset : 0;
|
|
sql += ` ORDER BY cl.timestamp DESC LIMIT @__limit OFFSET @__offset`;
|
|
params.__limit = limit;
|
|
params.__offset = offset;
|
|
|
|
const rows = db.prepare(sql).all(params) as CallLogSummaryRow[];
|
|
return rows.map(mapSummaryRow);
|
|
}
|
|
|
|
export async function getCallLogById(id: string) {
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare(
|
|
`SELECT cl.*,
|
|
pn.name AS provider_node_name,
|
|
pn.prefix AS provider_node_prefix,
|
|
${RESOLVED_ACCOUNT_SQL} AS resolved_account
|
|
FROM call_logs cl
|
|
LEFT JOIN provider_nodes pn ON pn.id = cl.provider
|
|
LEFT JOIN provider_connections pc ON pc.id = cl.connection_id
|
|
WHERE cl.id = ?`
|
|
)
|
|
.get(id) as CallLogSummaryRow | undefined;
|
|
if (!row) return null;
|
|
|
|
const entry = mapSummaryRow(row);
|
|
let detailState = entry.detailState;
|
|
let artifactRelPath = entry.artifactRelPath;
|
|
|
|
if (artifactRelPath) {
|
|
const artifactResult = readCallArtifact(artifactRelPath);
|
|
if (artifactResult.state === "ready" && artifactResult.artifact) {
|
|
return {
|
|
...entry,
|
|
detailState: "ready" as const,
|
|
requestBody: artifactResult.artifact.requestBody ?? null,
|
|
responseBody: artifactResult.artifact.responseBody ?? null,
|
|
error: artifactResult.artifact.error ?? entry.error,
|
|
pipelinePayloads: artifactResult.artifact.pipeline ?? buildLegacyPipelinePayloads(id),
|
|
hasPipelineDetails: Boolean(artifactResult.artifact.pipeline) || entry.hasPipelineDetails,
|
|
active: false,
|
|
};
|
|
}
|
|
|
|
detailState = artifactResult.state;
|
|
if (artifactResult.state === "missing") {
|
|
clearArtifactReference(artifactRelPath, "missing");
|
|
artifactRelPath = null;
|
|
} else {
|
|
db.prepare("UPDATE call_logs SET detail_state = ? WHERE id = ?").run("corrupt", id);
|
|
}
|
|
}
|
|
|
|
if (detailState === "legacy-inline") {
|
|
const legacyInline = getLegacyInlineDetail(id);
|
|
if (legacyInline) {
|
|
const legacyPipeline = buildLegacyPipelinePayloads(id);
|
|
return {
|
|
...entry,
|
|
detailState,
|
|
artifactRelPath,
|
|
...legacyInline,
|
|
pipelinePayloads: legacyPipeline,
|
|
hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails,
|
|
active: false,
|
|
};
|
|
}
|
|
}
|
|
|
|
const legacyDisk = readLegacyLogFromDisk(entry);
|
|
if (legacyDisk) {
|
|
const legacyPipeline = buildLegacyPipelinePayloads(id);
|
|
return {
|
|
...entry,
|
|
detailState,
|
|
artifactRelPath,
|
|
requestBody: legacyDisk.requestBody ?? null,
|
|
responseBody: legacyDisk.responseBody ?? null,
|
|
error: legacyDisk.error ?? entry.error,
|
|
pipelinePayloads: legacyPipeline,
|
|
hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails,
|
|
active: false,
|
|
};
|
|
}
|
|
|
|
const legacyPipeline = buildLegacyPipelinePayloads(id);
|
|
return {
|
|
...entry,
|
|
detailState,
|
|
artifactRelPath,
|
|
requestBody: null,
|
|
responseBody: null,
|
|
error: entry.error,
|
|
pipelinePayloads: legacyPipeline,
|
|
hasPipelineDetails: Boolean(legacyPipeline) || entry.hasPipelineDetails,
|
|
active: false,
|
|
};
|
|
}
|
|
|
|
export async function exportCallLogsSince(since: string) {
|
|
const db = getDbInstance();
|
|
const ids = db
|
|
.prepare("SELECT id FROM call_logs WHERE timestamp >= ? ORDER BY timestamp DESC")
|
|
.all(since)
|
|
.map((row) => String((row as { id: string }).id));
|
|
|
|
const logs: unknown[] = [];
|
|
for (const id of ids) {
|
|
const log = await getCallLogById(id);
|
|
if (log) logs.push(log);
|
|
}
|
|
return logs;
|
|
}
|