diff --git a/changelog.d/features/10670-call-logs-error-type.md b/changelog.d/features/10670-call-logs-error-type.md new file mode 100644 index 0000000000..1ffd94bd22 --- /dev/null +++ b/changelog.d/features/10670-call-logs-error-type.md @@ -0,0 +1 @@ +- **feat(call_logs):** persist the per-call error family in `call_logs.error_type` and expose a failure breakdown (`errorBreakdown`) in the usage analytics endpoint, reusing the existing production classifier ([#10670](https://github.com/diegosouzapw/OmniRoute/issues/10670)) diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts index d90480964e..306292865a 100644 --- a/src/app/api/usage/analytics/route.ts +++ b/src/app/api/usage/analytics/route.ts @@ -21,7 +21,7 @@ import { getWeeklyPatternRows, getPresetCostModelRows, } from "@/lib/db/usageAnalytics"; -import { getFallbackStats } from "@/lib/db/callLogStats"; +import { getFallbackStats, getErrorTypeBreakdown } from "@/lib/db/callLogStats"; import { buildByProviderRows } from "@/lib/usage/providerDisplayNames"; import { toNumber } from "@/shared/utils/numeric"; @@ -481,6 +481,7 @@ export async function GET(request: Request) { const weeklyRows = getWeeklyPatternRows(unifiedSource, unifiedParams) as UsageRows; const fallbackRow = getFallbackStats(whereClause, params) as Record; + const errorBreakdown = getErrorTypeBreakdown(whereClause, params); const summary = { totalRequests: Number(summaryRow?.totalRequests || 0), @@ -869,6 +870,7 @@ export async function GET(request: Request) { weeklyCounts, dailyByModel, modelNames, + errorBreakdown, range, } as any; diff --git a/src/lib/db/callLogStats.ts b/src/lib/db/callLogStats.ts index e238215e66..7df9908bd1 100644 --- a/src/lib/db/callLogStats.ts +++ b/src/lib/db/callLogStats.ts @@ -239,3 +239,34 @@ export function getFallbackStats( .get(params) as FallbackStatsRow | undefined; return row ?? { total: 0, with_requested: 0, fallback_eligible: 0, fallbacks: 0 }; } + +/** + * Failure-family breakdown over `call_logs` for the usage analytics endpoint. + * Failures are rows with status >= 400 or a non-empty error summary; successes + * are excluded in SQL. Pre-migration rows and failures the classifier does not + * recognize (null family) land in the explicit `unclassified` bucket. + * + * @param whereClause - SQL WHERE clause (may be empty string) using the same + * named params as the usage_history queries. + * @param params - Named params object (string values). + */ +export function getErrorTypeBreakdown( + whereClause: string, + params: Record +): Array<{ errorType: string; count: number }> { + const db = getDbInstance(); + const rows = db + .prepare( + ` + SELECT + COALESCE(error_type, 'unclassified') AS errorType, + COUNT(*) AS count + FROM call_logs + ${whereClause} ${whereClause ? "AND" : "WHERE"} (status >= 400 OR error_summary IS NOT NULL) + GROUP BY 1 + ORDER BY count DESC, errorType ASC + ` + ) + .all(params) as Array<{ errorType: string; count: number }>; + return rows.map((row) => ({ errorType: String(row.errorType), count: Number(row.count) })); +} diff --git a/src/lib/db/migrations/158_call_logs_error_type.sql b/src/lib/db/migrations/158_call_logs_error_type.sql new file mode 100644 index 0000000000..6aa0bc9956 --- /dev/null +++ b/src/lib/db/migrations/158_call_logs_error_type.sql @@ -0,0 +1,4 @@ +-- #10670: per-call error family, set at the single write point in +-- src/lib/usage/callLogs.ts from classifyProviderError. NULL for successes +-- (analytics filters by failure in SQL, so no need for a sentinel value). +ALTER TABLE call_logs ADD COLUMN error_type TEXT DEFAULT NULL; diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index 8caaedc31d..cea52ba196 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -39,6 +39,7 @@ import { toStoredErrorSummary, protectPipelinePayloads, buildRequestSummary, + classifyCallLogError, } from "./callLogs/format"; import { clearArtifactReference, @@ -464,12 +465,14 @@ async function saveCallLogOperation(entry: any): Promise { // while reasoning source/char-count are recorded separately for observability. const tokensReasoning = getReasoningTokensOrNull(entry.tokens); const reasoningObservation = resolveReasoningObservation(tokensReasoning, entry.responseBody); + const errorType = classifyCallLogError(entry.status, entry.error, entry.provider); const logEntry = { id: typeof entry.id === "string" && entry.id.length > 0 ? entry.id : generateLogId(), timestamp: typeof entry.timestamp === "string" ? entry.timestamp : new Date().toISOString(), method: entry.method || "POST", path: entry.path || "/v1/chat/completions", status: entry.status || 0, + errorType, model: entry.model || "-", requestedModel: resolvedRequestedModel, provider: rawProvider, @@ -550,7 +553,7 @@ async function saveCallLogOperation(entry: any): Promise { combo_name, combo_step_id, combo_execution_key, error_summary, detail_state, artifact_relpath, artifact_size_bytes, artifact_sha256, has_request_body, has_response_body, has_pipeline_details, request_summary, - correlation_id, model_pinned, session_tag, response_id + correlation_id, model_pinned, session_tag, response_id, error_type ) VALUES ( @id, @timestamp, @method, @path, @status, @model, @requestedModel, @provider, @@ -561,7 +564,7 @@ async function saveCallLogOperation(entry: any): Promise { @comboName, @comboStepId, @comboExecutionKey, @errorSummary, @detailState, @artifactRelPath, @artifactSizeBytes, @artifactSha256, @hasRequestBody, @hasResponseBody, @hasPipelineDetails, @requestSummary, - @correlationId, @modelPinned, @sessionTag, @responseId + @correlationId, @modelPinned, @sessionTag, @responseId, @errorType ) ` ).run({ diff --git a/src/lib/usage/callLogs/format.ts b/src/lib/usage/callLogs/format.ts index b4aa72de64..40054c3a37 100644 --- a/src/lib/usage/callLogs/format.ts +++ b/src/lib/usage/callLogs/format.ts @@ -1,4 +1,5 @@ import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; +import { classifyProviderError } from "@omniroute/open-sse/services/errorClassifier.ts"; import { sanitizePII } from "../../piiSanitizer"; import { omitEncryptedReasoningFromLogChunks, protectPayloadForLog } from "../../logPayloads"; import type { CallLogDetailState } from "../callLogArtifacts"; @@ -124,3 +125,22 @@ export function buildRequestSummary( if (Object.keys(summary).length === 0) return null; return JSON.stringify(summary); } + +// #10670: per-call error family at the single write point. Reuses the +// production classifier (chatCore.ts:3974, auth.ts:2598) so the persisted +// vocabulary is exactly PROVIDER_ERROR_TYPES. Successes (status < 400 with no +// error text) short-circuit to null โ€” the classifier never returns a family +// for them anyway, this only skips the call. +// Normalization: strings pass through, Error objects yield .message, any other +// object yields "" (no caller passes plain objects โ€” verified: 35 callers use +// strings and Error only). Deliberate deviation from design ยง4 ("objet โ†’ +// JSON.stringify"): a stringified object carries no classifier signal. +export function classifyCallLogError( + status: number, + error: unknown, + provider?: string | null +): string | null { + const errorText = typeof error === "string" ? error : error instanceof Error ? error.message : ""; + if (status < 400 && errorText.length === 0) return null; + return classifyProviderError(status, errorText, provider); +} diff --git a/tests/unit/call-log-error-type.test.ts b/tests/unit/call-log-error-type.test.ts new file mode 100644 index 0000000000..848e51952c --- /dev/null +++ b/tests/unit/call-log-error-type.test.ts @@ -0,0 +1,192 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { getDbInstance } from "../../src/lib/db/core.ts"; +import { classifyCallLogError } from "../../src/lib/usage/callLogs/format.ts"; +import { saveCallLog } from "../../src/lib/usage/callLogs.ts"; +import { getErrorTypeBreakdown } from "../../src/lib/db/callLogStats.ts"; + +test("call_logs table has error_type column", () => { + const db = getDbInstance(); + const columns = db.prepare("PRAGMA table_info(call_logs)").all() as { name: string }[]; + const colNames = columns.map((c) => c.name); + assert.ok(colNames.includes("error_type"), "call_logs should have error_type column"); +}); + +test("classifyCallLogError maps status+body to the provider error family", () => { + assert.equal(classifyCallLogError(402, "whatever body", "openai"), "quota_exhausted"); + assert.equal(classifyCallLogError(500, "Internal Server Error", "openai"), "server_error"); + assert.equal(classifyCallLogError(429, "rate limit", "openai"), "rate_limited"); + assert.equal(classifyCallLogError(404, "model not found", "openai"), "model_not_found"); + assert.equal(classifyCallLogError(401, "bad key", "openai"), "unauthorized"); +}); + +test("classifyCallLogError classifies only failures", () => { + assert.equal(classifyCallLogError(200, "", "openai"), null); + assert.equal(classifyCallLogError(200, "some body", "openai"), null); + assert.equal(classifyCallLogError(0, "boom", "test-provider"), null); +}); + +test("classifyCallLogError extracts message from Error object", () => { + assert.equal( + classifyCallLogError(403, new Error("browser_signature_banned"), "openai"), + "fingerprint_rejection" + ); +}); + +test("classifyCallLogError returns null for unclassifiable provider-403 (api key)", () => { + assert.equal(classifyCallLogError(403, "some other 403 body", "openai"), null); +}); + +test("saveCallLog persists error_type from failure", async () => { + const db = getDbInstance(); + const testId = `test-errtype-${Date.now()}`; + + await saveCallLog({ + id: testId, + method: "POST", + path: "/v1/chat/completions", + status: 402, + error: "exceeded your current quota", + model: "test-model", + provider: "test-provider", + duration: 100, + tokens: { in: 10, out: 5 }, + }); + + const row = db.prepare("SELECT error_type FROM call_logs WHERE id = ?").get(testId) as { + error_type: string | null; + }; + assert.equal(row.error_type, "quota_exhausted"); + + db.prepare("DELETE FROM call_logs WHERE id = ?").run(testId); +}); + +test("saveCallLog persists null error_type for success", async () => { + const db = getDbInstance(); + const testId = `test-errtype-ok-${Date.now()}`; + + await saveCallLog({ + id: testId, + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "test-model", + provider: "test-provider", + duration: 100, + tokens: { in: 10, out: 5 }, + }); + + const row = db.prepare("SELECT error_type FROM call_logs WHERE id = ?").get(testId) as { + error_type: string | null; + }; + assert.equal(row.error_type, null); + + db.prepare("DELETE FROM call_logs WHERE id = ?").run(testId); +}); + +test("saveCallLog normalizes Error object before classifying", async () => { + const db = getDbInstance(); + const testId = `test-errtype-err-${Date.now()}`; + + await saveCallLog({ + id: testId, + method: "POST", + path: "/v1/chat/completions", + status: 403, + error: new Error("browser_signature_banned"), + model: "test-model", + provider: "test-provider", + duration: 100, + tokens: { in: 10, out: 5 }, + }); + + const row = db.prepare("SELECT error_type FROM call_logs WHERE id = ?").get(testId) as { + error_type: string | null; + }; + assert.equal(row.error_type, "fingerprint_rejection"); + + db.prepare("DELETE FROM call_logs WHERE id = ?").run(testId); +}); + +test("getErrorTypeBreakdown groups failures by family, excludes successes", async () => { + const db = getDbInstance(); + const ids = [ + `test-errbd-q1-${Date.now()}`, + `test-errbd-q2-${Date.now()}`, + `test-errbd-s5-${Date.now()}`, + `test-errbd-403-${Date.now()}`, + `test-errbd-ok-${Date.now()}`, + ]; + + await saveCallLog({ + id: ids[0], + method: "POST", + path: "/v1/chat/completions", + status: 402, + error: "exceeded your current quota", + model: "m", + provider: "test-provider", + duration: 100, + tokens: { in: 1, out: 1 }, + }); + await saveCallLog({ + id: ids[1], + method: "POST", + path: "/v1/chat/completions", + status: 402, + error: "insufficient balance", + model: "m", + provider: "test-provider", + duration: 100, + tokens: { in: 1, out: 1 }, + }); + await saveCallLog({ + id: ids[2], + method: "POST", + path: "/v1/chat/completions", + status: 500, + error: "Internal Server Error", + model: "m", + provider: "test-provider", + duration: 100, + tokens: { in: 1, out: 1 }, + }); + await saveCallLog({ + id: ids[3], + method: "POST", + path: "/v1/chat/completions", + status: 403, + error: "some other 403 body", + model: "m", + provider: "test-provider", + duration: 100, + tokens: { in: 1, out: 1 }, + }); + await saveCallLog({ + id: ids[4], + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "m", + provider: "test-provider", + duration: 100, + tokens: { in: 1, out: 1 }, + }); + + const whereClause = `WHERE id IN (${ids.map((_, i) => `@id${i}`).join(", ")})`; + const params = Object.fromEntries(ids.map((id, i) => [`id${i}`, id])); + const breakdown = getErrorTypeBreakdown(whereClause, params); + + assert.deepEqual(breakdown, [ + { errorType: "quota_exhausted", count: 2 }, + { errorType: "server_error", count: 1 }, + { errorType: "unclassified", count: 1 }, + ]); + + ids.forEach((id) => db.prepare("DELETE FROM call_logs WHERE id = ?").run(id)); +}); + +test("getErrorTypeBreakdown with empty whereClause does not crash", () => { + const breakdown = getErrorTypeBreakdown("", {}); + assert.ok(Array.isArray(breakdown)); +});