import { getDbInstance } from "./core"; import { ERROR_TYPE_CONTRACT } from "@omniroute/open-sse/services/errorClassifier.ts"; import { SEARCH_CREDENTIAL_FALLBACKS, SEARCH_PROVIDERS, } from "@omniroute/open-sse/config/searchRegistry.ts"; import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags"; /** * Aggregation queries over `call_logs` extracted from route handlers. * * Hard Rule #5: routes must not embed raw SQL — these queries live here so the * /api/provider-metrics, /api/search/stats, and /api/v1/search/analytics routes * can delegate. Read-only aggregation; no writes. * * Sliced out of #3500 (call_logs cluster). */ // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface ProviderMetricRow { provider: string; totalRequests: number; totalSuccesses: number; avgLatencyMs: number | null; lastRequestAt: string | null; lastErrorAt: string | null; lastStatus: number | null; lastErrorStatus: number | null; } /** One provider's traffic over a bounded window. See `getProviderUsageSince`. */ export interface ProviderUsageRow { provider: string; requests: number; successes: number; avgLatencyMs: number | null; lastRequestAt: string | null; } export interface SearchProviderStatRow { provider: string; requests: number; avg_latency_ms: number | null; } export interface SearchRecentRow { request_summary: string | null; provider: string; timestamp: string; } export interface SearchAggregateStats { total: number; today: number; errors: number; avg_duration: number | null; cached: number; } export interface SearchProviderCountRow { provider: string; cnt: number; } // --------------------------------------------------------------------------- // /api/provider-metrics — aggregate per-provider stats // --------------------------------------------------------------------------- /** * Returns one row per provider with call-level aggregates plus last-status * subselects. Excludes rows where provider is NULL or '-', and excludes * providers with no live row in `provider_connections` — a deleted provider * connection must not keep surfacing as a ghost topology node forever from * its retained historical call_logs rows. See #10714. */ export function getProviderMetrics(): ProviderMetricRow[] { const db = getDbInstance(); return db .prepare( `SELECT c.provider, COUNT(*) as totalRequests, SUM(CASE WHEN status >= 200 AND status < 400 THEN 1 ELSE 0 END) as totalSuccesses, ROUND(AVG(duration)) as avgLatencyMs, MAX(timestamp) as lastRequestAt, MAX( CASE WHEN (status IS NOT NULL AND (status < 200 OR status >= 400)) OR error_summary IS NOT NULL THEN timestamp ELSE NULL END ) as lastErrorAt, ( SELECT c2.status FROM call_logs c2 WHERE c2.provider = c.provider ORDER BY c2.timestamp DESC, c2.id DESC LIMIT 1 ) as lastStatus, ( SELECT c3.status FROM call_logs c3 WHERE c3.provider = c.provider AND ( (c3.status IS NOT NULL AND (c3.status < 200 OR c3.status >= 400)) OR c3.error_summary IS NOT NULL ) ORDER BY c3.timestamp DESC, c3.id DESC LIMIT 1 ) as lastErrorStatus FROM call_logs c WHERE c.provider IS NOT NULL AND c.provider != '-' AND EXISTS ( SELECT 1 FROM provider_connections pc WHERE pc.provider = c.provider ) GROUP BY c.provider` ) .all() as ProviderMetricRow[]; } // --------------------------------------------------------------------------- // /api/free-provider-rankings — windowed usage aggregate // --------------------------------------------------------------------------- /** * Per-provider usage over a time window: how much traffic a provider actually * served, and how much of it succeeded. * * Deliberately NOT `getProviderMetrics()` with a `since` parameter: that query * carries two correlated subqueries (`lastStatus`, `lastErrorStatus`) which a * ranking never displays, and they dominate its cost. Here a single bounded * `GROUP BY` leans on `idx_cl_timestamp` plus `idx_cl_provider_timestamp` / * `idx_cl_request_provider` (migration 174) and stops there. The rules are shared with its neighbour, not the query: same success * definition, same `#10714` guard against providers whose connections are gone. */ export function getProviderUsageSince(since: string): ProviderUsageRow[] { const db = getDbInstance(); return db .prepare( `SELECT c.provider, COUNT(*) as requests, SUM(CASE WHEN c.status >= 200 AND c.status < 400 THEN 1 ELSE 0 END) as successes, ROUND(AVG(c.duration)) as avgLatencyMs, MAX(c.timestamp) as lastRequestAt FROM call_logs c WHERE c.provider IS NOT NULL AND c.provider != '-' AND c.timestamp >= @since AND EXISTS ( SELECT 1 FROM provider_connections pc WHERE pc.provider = c.provider ) GROUP BY c.provider` ) .all({ since }) as ProviderUsageRow[]; } // --------------------------------------------------------------------------- // /api/search/stats — search provider aggregates + recent entries // --------------------------------------------------------------------------- function sqlStringLiteral(value: string): string { return `'${value.replace(/'/g, "''")}'`; } let searchLiveProviderGuardSql: string | null = null; /** Always applied: never surface a NULL provider or the '-' sentinel. */ const SEARCH_PROVIDER_PRESENT_SQL = "c.provider IS NOT NULL AND c.provider != '-'"; /** * WHERE fragment shared by every search query below (alias `c` = call_logs). * A search row is surfaced only when its provider is still servable: * - never a NULL provider or the '-' sentinel; * - with SEARCH_STATS_HIDE_DELETED_CONNECTIONS on, a keyed provider also needs a * provider_connections row, for itself or for one of its credential fallbacks * (perplexity-search reuses a `perplexity` key), so a deleted connection stops * resurfacing from its retained call_logs rows; * - keyless providers (`authType: "none"` in the search registry, e.g. * duckduckgo-free, searxng-search, anonymous context7) are always live — * they are served without any provider_connections row. * The flag defaults to off, which keeps the historical stats: every retained row * with a real provider id counts. Built from registry constants on first use. */ function getSearchLiveProviderGuardSql(): string { if (!isSearchStatsHideDeletedConnectionsEnabled()) return SEARCH_PROVIDER_PRESENT_SQL; if (searchLiveProviderGuardSql !== null) return searchLiveProviderGuardSql; const keyless = Object.values(SEARCH_PROVIDERS) .filter((provider) => provider.authType === "none") .map((provider) => sqlStringLiteral(provider.id)); const fallbackPairs = Object.entries(SEARCH_CREDENTIAL_FALLBACKS).flatMap( ([searchId, fallback]) => (Array.isArray(fallback) ? fallback : [fallback]).map( (fallbackId) => `(${sqlStringLiteral(searchId)}, ${sqlStringLiteral(fallbackId)})` ) ); const keylessClause = keyless.length > 0 ? `c.provider IN (${keyless.join(", ")}) OR ` : ""; const fallbackClause = fallbackPairs.length > 0 ? ` OR EXISTS ( SELECT 1 FROM (VALUES ${fallbackPairs.join(", ")}) fb JOIN provider_connections pcf ON pcf.provider = fb.column2 WHERE fb.column1 = c.provider )` : ""; searchLiveProviderGuardSql = `${SEARCH_PROVIDER_PRESENT_SQL} AND ( ${keylessClause}EXISTS ( SELECT 1 FROM provider_connections pc WHERE pc.provider = c.provider )${fallbackClause} )`; return searchLiveProviderGuardSql; } /** Fail closed to the historical behavior when the flag cannot be resolved. */ function isSearchStatsHideDeletedConnectionsEnabled(): boolean { try { return isFeatureFlagEnabled("SEARCH_STATS_HIDE_DELETED_CONNECTIONS"); } catch { return false; } } /** * Per-provider request count and average latency for search requests. * Rows pass the search live-provider guard (see getSearchLiveProviderGuardSql). */ export function getSearchProviderStats(): SearchProviderStatRow[] { const db = getDbInstance(); return db .prepare( ` SELECT c.provider, COUNT(*) as requests, CAST(AVG(c.duration) AS INTEGER) as avg_latency_ms FROM call_logs c WHERE c.request_type = 'search' AND ${getSearchLiveProviderGuardSql()} GROUP BY c.provider ` ) .all() as SearchProviderStatRow[]; } /** * Most recent 10 search entries (request_summary + provider + timestamp). * Only rows from providers with a live connection are surfaced. */ export function getRecentSearchLogs(): SearchRecentRow[] { const db = getDbInstance(); return db .prepare( ` SELECT c.request_summary, c.provider, c.timestamp FROM call_logs c WHERE c.request_type = 'search' AND ${getSearchLiveProviderGuardSql()} ORDER BY c.timestamp DESC LIMIT 10 ` ) .all() as SearchRecentRow[]; } // --------------------------------------------------------------------------- // /api/v1/search/analytics — aggregated search analytics // --------------------------------------------------------------------------- /** * Single-pass scalar aggregations for all search entries since `todayIso`. * `todayIso` is the ISO-8601 UTC start-of-day string used for the "today" count. * Uses the same live-provider guard as the per-provider breakdown, so `total` * always equals the sum of `getSearchProviderCounts()`. */ export function getSearchAggregateStats(todayIso: string): SearchAggregateStats { const db = getDbInstance(); const row = db .prepare( `SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN c.timestamp >= ? THEN 1 ELSE 0 END), 0) as today, COALESCE(SUM(CASE WHEN c.status >= 400 OR c.error_summary IS NOT NULL THEN 1 ELSE 0 END), 0) as errors, AVG(CASE WHEN c.duration > 0 THEN c.duration END) as avg_duration, COALESCE(SUM(CASE WHEN c.duration > 0 AND c.duration < 5 THEN 1 ELSE 0 END), 0) as cached FROM call_logs c WHERE c.request_type = 'search' AND ${getSearchLiveProviderGuardSql()}` ) .get(todayIso) as SearchAggregateStats | undefined; return row ?? { total: 0, today: 0, errors: 0, avg_duration: null, cached: 0 }; } /** * Per-provider request count for search entries, ordered by count descending. * Rows pass the search live-provider guard (see getSearchLiveProviderGuardSql). */ export function getSearchProviderCounts(): SearchProviderCountRow[] { const db = getDbInstance(); return db .prepare( `SELECT c.provider, COUNT(*) as cnt FROM call_logs c WHERE c.request_type = 'search' AND ${getSearchLiveProviderGuardSql()} GROUP BY c.provider ORDER BY cnt DESC` ) .all() as SearchProviderCountRow[]; } // --------------------------------------------------------------------------- // /api/usage/analytics — fallback-rate aggregates over call_logs // --------------------------------------------------------------------------- export interface FallbackStatsRow { total: number; with_requested: number; fallback_eligible: number; fallbacks: number; } /** * Scalar fallback-rate stats over `call_logs` for the usage analytics endpoint. * * @param whereClause - SQL WHERE clause (may be empty string) using the same * named params as the usage_history queries. * @param params - Named params object (string values). */ export function getFallbackStats( whereClause: string, params: Record ): FallbackStatsRow { const db = getDbInstance(); const row = db .prepare( ` SELECT COALESCE(SUM(CASE WHEN (combo_name IS NULL OR combo_name = '') THEN 1 ELSE 0 END), 0) as total, COALESCE(SUM(CASE WHEN requested_model IS NOT NULL AND requested_model != '' AND (combo_name IS NULL OR combo_name = '') THEN 1 ELSE 0 END), 0) as with_requested, COALESCE(SUM(CASE WHEN (combo_name IS NULL OR combo_name = '') AND requested_model IS NOT NULL AND requested_model != '' AND model IS NOT NULL AND model != '' THEN 1 ELSE 0 END ), 0) as fallback_eligible, COALESCE(SUM(CASE WHEN (combo_name IS NULL OR combo_name = '') AND requested_model IS NOT NULL AND requested_model != '' AND model IS NOT NULL AND model != '' AND LOWER(CASE WHEN instr(requested_model, '/') > 0 THEN substr(requested_model, instr(requested_model, '/') + 1) ELSE requested_model END) != LOWER(model) THEN 1 ELSE 0 END ), 0) as fallbacks FROM call_logs ${whereClause} ` ) .get(params) as FallbackStatsRow | undefined; return row ?? { total: 0, with_requested: 0, fallback_eligible: 0, fallbacks: 0 }; } // ERROR_TYPE_CUTOVER_ISO — single source of truth for the cutover: date of // migration 158 (commit 4c15c05f9) that added `call_logs.error_type`. export const ERROR_TYPE_CUTOVER_ISO = "2026-08-20"; // SQL `IN (...)` list of the persisted vocabulary. Built lazily (not at module // evaluation) so an import cycle through the classifier can never observe the // contract before it is initialised. Values are fixed identifiers; quotes are // still escaped defensively. let errorTypeVocabSql: string | null = null; function getErrorTypeVocabSql(): string { if (errorTypeVocabSql === null) { errorTypeVocabSql = ERROR_TYPE_CONTRACT.map((v) => `'${v.replace(/'/g, "''")}'`).join(", "); } return errorTypeVocabSql; } /** * Failure-family breakdown over `call_logs` for the usage analytics endpoint. * Failures are rows with status >= 400 or a non-empty error summary; successes * are excluded in SQL. Rows predating migration 158 (`error_type` NULL, * `timestamp` before ERROR_TYPE_CUTOVER_ISO) land in `pre_migration`; other * NULL families land in `unclassified`, and so does any stored value outside * ERROR_TYPE_CONTRACT (free text written out of band). Every failure row lands * in exactly one bucket, so the counts always sum to the failure total. * * @param whereClause - SQL WHERE clause (may be empty string) using the same * named params as the usage_history queries. * @param params - Named params object (string values). */ export function getErrorTypeBreakdown( whereClause: string, params: Record ): Array<{ errorType: string; count: number }> { const db = getDbInstance(); const rows = db .prepare( ` SELECT -- ERROR_TYPE_CUTOVER_ISO (migration 158). Lower bound, not exact: late -- upgraders have post-cutoff rows with NULL values. CASE WHEN error_type IS NULL AND timestamp < '${ERROR_TYPE_CUTOVER_ISO}' THEN 'pre_migration' WHEN error_type IS NULL THEN 'unclassified' WHEN error_type NOT IN (${getErrorTypeVocabSql()}) THEN 'unclassified' ELSE error_type END AS errorType, COUNT(*) AS count FROM call_logs ${whereClause} ${whereClause ? "AND" : "WHERE"} (status >= 400 OR error_summary IS NOT NULL) GROUP BY 1 ORDER BY count DESC, errorType ASC ` ) .all(params) as Array<{ errorType: string; count: number }>; return rows.map((row) => ({ errorType: String(row.errorType), count: Number(row.count) })); }