+ -

Page not found

+

+ Page not found +

The page you're looking for doesn't exist or has been moved.

Go to Dashboard diff --git a/src/lib/container.ts b/src/lib/container.ts new file mode 100644 index 0000000000..86f722d09b --- /dev/null +++ b/src/lib/container.ts @@ -0,0 +1,117 @@ +/** + * Simple DI Container — Factory-pattern service locator + * + * Provides a lightweight dependency injection container using factory + * functions (no heavy frameworks). Services are lazily instantiated + * and cached as singletons. + * + * Usage: + * import { container } from '@/lib/container'; + * const settings = container.resolve('settings'); + * + * Registration: + * container.register('myService', () => new MyService()); + * + * @module lib/container + */ + +type Factory = () => T; + +class Container { + private _factories = new Map(); + private _instances = new Map(); + + /** + * Register a factory for a service. Does NOT instantiate until resolve(). + */ + register(name: string, factory: Factory): void { + this._factories.set(name, factory); + // Clear cached instance if re-registering (useful for testing) + this._instances.delete(name); + } + + /** + * Resolve a service by name. Lazy-creates via factory on first call, + * then returns the cached singleton. + */ + resolve(name: string): T { + if (this._instances.has(name)) { + return this._instances.get(name) as T; + } + + const factory = this._factories.get(name); + if (!factory) { + throw new Error(`[Container] No factory registered for "${name}"`); + } + + const instance = factory(); + this._instances.set(name, instance); + return instance as T; + } + + /** + * Check if a service is registered (factory exists). + */ + has(name: string): boolean { + return this._factories.has(name); + } + + /** + * List all registered service names. + */ + list(): string[] { + return Array.from(this._factories.keys()); + } + + /** + * Reset all factories and instances (for testing). + */ + reset(): void { + this._factories.clear(); + this._instances.clear(); + } +} + +// ── Singleton container instance ── +export const container = new Container(); + +// ── Default registrations ── +// These lazy-load the actual modules so the container can be imported +// without triggering all side effects at module load time. + +container.register("settings", () => { + const { getSettings } = require("@/lib/localDb"); + return { get: getSettings }; +}); + +container.register("db", () => { + const { getDbInstance } = require("@/lib/db/core"); + return getDbInstance(); +}); + +container.register("encryption", () => { + const enc = require("@/lib/db/encryption"); + return { + encrypt: enc.encrypt, + decrypt: enc.decrypt, + encryptConnectionFields: enc.encryptConnectionFields, + decryptConnectionFields: enc.decryptConnectionFields, + }; +}); + +container.register("policyEngine", () => { + const { evaluateRequest, evaluateFirstAllowed, PolicyEngine } = require("@/domain/policyEngine"); + return { evaluateRequest, evaluateFirstAllowed, PolicyEngine }; +}); + +container.register("circuitBreaker", () => { + const { getCircuitBreaker } = require("@/shared/utils/circuitBreaker"); + return { get: getCircuitBreaker }; +}); + +container.register("telemetry", () => { + const { RequestTelemetry, recordTelemetry } = require("@/shared/utils/requestTelemetry"); + return { RequestTelemetry, recordTelemetry }; +}); + +export default container; diff --git a/src/lib/db/prompts.ts b/src/lib/db/prompts.ts new file mode 100644 index 0000000000..b2a27f22ce --- /dev/null +++ b/src/lib/db/prompts.ts @@ -0,0 +1,241 @@ +/** + * Prompt Template Versioning — L-6 + * + * SQLite-backed prompt template storage with version tracking. + * Each prompt has a unique `slug`, and every save creates a new version + * (content-addressed via SHA-256 hash). Previous versions are retained + * for rollback and audit. + * + * @module lib/db/prompts + */ + +import crypto from "node:crypto"; +import { getDbInstance } from "./core"; + +// ── Schema (auto-created on first access) ── + +const PROMPT_SCHEMA = ` + CREATE TABLE IF NOT EXISTS prompt_templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + slug TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + variables TEXT, + description TEXT, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(slug, version) + ); + CREATE INDEX IF NOT EXISTS idx_pt_slug ON prompt_templates(slug); + CREATE INDEX IF NOT EXISTS idx_pt_active ON prompt_templates(slug, is_active); + CREATE INDEX IF NOT EXISTS idx_pt_hash ON prompt_templates(content_hash); +`; + +let _initialized = false; + +function ensureSchema(): void { + if (_initialized) return; + try { + const db = getDbInstance(); + db.exec(PROMPT_SCHEMA); + _initialized = true; + } catch { + // Schema creation is best-effort during build phase + } +} + +function hashContent(content: string): string { + return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16); +} + +// ── Public API ── + +export interface PromptTemplate { + id: number; + slug: string; + version: number; + content: string; + contentHash: string; + variables: string[] | null; + description: string | null; + isActive: boolean; + createdAt: string; +} + +/** + * Save a prompt template. If the slug already exists and the content + * has changed, a new version is created. If content is identical, + * returns the existing version without duplicating. + */ +export function savePrompt( + slug: string, + content: string, + options: { variables?: string[]; description?: string } = {} +): PromptTemplate { + ensureSchema(); + const db = getDbInstance(); + const hash = hashContent(content); + + // Check if identical content already exists for this slug + const existing = db + .prepare("SELECT * FROM prompt_templates WHERE slug = ? AND content_hash = ?") + .get(slug, hash) as any; + + if (existing) { + return rowToPrompt(existing); + } + + // Deactivate previous active version + db.prepare("UPDATE prompt_templates SET is_active = 0 WHERE slug = ? AND is_active = 1").run( + slug + ); + + // Get next version number + const maxVersion = db + .prepare("SELECT MAX(version) as max_v FROM prompt_templates WHERE slug = ?") + .get(slug) as any; + const nextVersion = (maxVersion?.max_v || 0) + 1; + + // Insert new version + const result = db + .prepare( + `INSERT INTO prompt_templates (slug, version, content, content_hash, variables, description, is_active) + VALUES (?, ?, ?, ?, ?, ?, 1)` + ) + .run( + slug, + nextVersion, + content, + hash, + options.variables ? JSON.stringify(options.variables) : null, + options.description || null + ); + + return { + id: Number(result.lastInsertRowid), + slug, + version: nextVersion, + content, + contentHash: hash, + variables: options.variables || null, + description: options.description || null, + isActive: true, + createdAt: new Date().toISOString(), + }; +} + +/** + * Get the active (latest) version of a prompt by slug. + */ +export function getActivePrompt(slug: string): PromptTemplate | null { + ensureSchema(); + const db = getDbInstance(); + const row = db + .prepare("SELECT * FROM prompt_templates WHERE slug = ? AND is_active = 1") + .get(slug) as any; + return row ? rowToPrompt(row) : null; +} + +/** + * Get a specific version of a prompt. + */ +export function getPromptVersion(slug: string, version: number): PromptTemplate | null { + ensureSchema(); + const db = getDbInstance(); + const row = db + .prepare("SELECT * FROM prompt_templates WHERE slug = ? AND version = ?") + .get(slug, version) as any; + return row ? rowToPrompt(row) : null; +} + +/** + * List all versions of a prompt (newest first). + */ +export function listPromptVersions(slug: string): PromptTemplate[] { + ensureSchema(); + const db = getDbInstance(); + const rows = db + .prepare("SELECT * FROM prompt_templates WHERE slug = ? ORDER BY version DESC") + .all(slug) as any[]; + return rows.map(rowToPrompt); +} + +/** + * List all prompt slugs with their active version info. + */ +export function listPrompts(): Array<{ slug: string; activeVersion: number; totalVersions: number }> { + ensureSchema(); + const db = getDbInstance(); + const rows = db + .prepare( + `SELECT slug, + MAX(CASE WHEN is_active = 1 THEN version ELSE 0 END) as active_version, + COUNT(*) as total_versions + FROM prompt_templates + GROUP BY slug + ORDER BY slug` + ) + .all() as any[]; + + return rows.map((r) => ({ + slug: r.slug, + activeVersion: r.active_version, + totalVersions: r.total_versions, + })); +} + +/** + * Rollback to a previous version (makes it the active one). + */ +export function rollbackPrompt(slug: string, version: number): PromptTemplate | null { + ensureSchema(); + const db = getDbInstance(); + + const target = db + .prepare("SELECT * FROM prompt_templates WHERE slug = ? AND version = ?") + .get(slug, version) as any; + + if (!target) return null; + + const rollback = db.transaction(() => { + db.prepare("UPDATE prompt_templates SET is_active = 0 WHERE slug = ?").run(slug); + db.prepare("UPDATE prompt_templates SET is_active = 1 WHERE slug = ? AND version = ?").run( + slug, + version + ); + }); + rollback(); + + return rowToPrompt({ ...target, is_active: 1 }); +} + +/** + * Render a prompt template by substituting variables. + */ +export function renderPrompt(slug: string, vars: Record = {}): string | null { + const prompt = getActivePrompt(slug); + if (!prompt) return null; + + let content = prompt.content; + for (const [key, value] of Object.entries(vars)) { + content = content.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), value); + } + return content; +} + +// ── Internal ── + +function rowToPrompt(row: any): PromptTemplate { + return { + id: row.id, + slug: row.slug, + version: row.version, + content: row.content, + contentHash: row.content_hash, + variables: row.variables ? JSON.parse(row.variables) : null, + description: row.description, + isActive: row.is_active === 1, + createdAt: row.created_at, + }; +} diff --git a/src/lib/evals/scheduler.ts b/src/lib/evals/scheduler.ts new file mode 100644 index 0000000000..6b4886698a --- /dev/null +++ b/src/lib/evals/scheduler.ts @@ -0,0 +1,276 @@ +/** + * Eval Scheduler — L-7 + * + * Cron-based scheduling for golden set evaluation runs. + * Uses a simple interval timer (no external cron dependency). + * Results are persisted to SQLite for trend tracking. + * + * @module lib/evals/scheduler + */ + +import { runSuite, listSuites, createScorecard } from "./evalRunner"; + +// ── Types ── + +export interface ScheduledEval { + suiteId: string; + intervalMs: number; + lastRunAt: number | null; + nextRunAt: number; + enabled: boolean; +} + +export interface EvalRunResult { + suiteId: string; + suiteName: string; + timestamp: number; + passRate: number; + total: number; + passed: number; + failed: number; + results: any[]; +} + +// ── State ── + +const _schedules = new Map(); +const _timers = new Map(); +const _history: EvalRunResult[] = []; +let _outputProvider: ((suiteId: string, caseId: string) => Promise) | null = null; + +// ── Configuration ── + +/** + * Set the output provider function — called to get actual LLM output + * for each eval case. This decouples the scheduler from the chat pipeline. + * + * @param fn - Async function(suiteId, caseId) → actual output string + */ +export function setOutputProvider( + fn: (suiteId: string, caseId: string) => Promise +): void { + _outputProvider = fn; +} + +// ── Scheduling ── + +/** + * Schedule a suite to run at a fixed interval. + * + * @param suiteId - ID of a registered eval suite + * @param intervalMs - Interval between runs in milliseconds (min 60000 = 1 min) + */ +export function schedule(suiteId: string, intervalMs: number): ScheduledEval { + const safeInterval = Math.max(intervalMs, 60_000); // Min 1 minute + const now = Date.now(); + + // Clear existing timer if re-scheduling + if (_timers.has(suiteId)) { + clearInterval(_timers.get(suiteId) as any); + } + + const entry: ScheduledEval = { + suiteId, + intervalMs: safeInterval, + lastRunAt: null, + nextRunAt: now + safeInterval, + enabled: true, + }; + + _schedules.set(suiteId, entry); + + const timer = setInterval(() => { + executeScheduledRun(suiteId).catch((err) => { + console.error(`[EvalScheduler] Failed to run suite ${suiteId}:`, err.message); + }); + }, safeInterval); + + _timers.set(suiteId, timer); + + console.log( + `[EvalScheduler] Scheduled "${suiteId}" every ${Math.round(safeInterval / 1000)}s` + ); + + return entry; +} + +/** + * Unschedule a suite. + */ +export function unschedule(suiteId: string): boolean { + const timer = _timers.get(suiteId); + if (timer) { + clearInterval(timer as any); + _timers.delete(suiteId); + } + return _schedules.delete(suiteId); +} + +/** + * Pause a scheduled suite without removing it. + */ +export function pause(suiteId: string): boolean { + const entry = _schedules.get(suiteId); + if (!entry) return false; + entry.enabled = false; + const timer = _timers.get(suiteId); + if (timer) { + clearInterval(timer as any); + _timers.delete(suiteId); + } + return true; +} + +/** + * Resume a paused scheduled suite. + */ +export function resume(suiteId: string): boolean { + const entry = _schedules.get(suiteId); + if (!entry) return false; + entry.enabled = true; + return !!schedule(suiteId, entry.intervalMs); +} + +// ── Execution ── + +/** + * Execute a scheduled run for a suite. + */ +async function executeScheduledRun(suiteId: string): Promise { + const entry = _schedules.get(suiteId); + if (!entry?.enabled) return null; + + if (!_outputProvider) { + console.warn(`[EvalScheduler] No output provider set — skipping ${suiteId}`); + return null; + } + + console.log(`[EvalScheduler] Running suite: ${suiteId}`); + + try { + // Collect outputs for all cases in the suite + const suites = listSuites(); + const suiteInfo = suites.find((s) => s.id === suiteId); + if (!suiteInfo) { + console.warn(`[EvalScheduler] Suite not found: ${suiteId}`); + return null; + } + + // Get outputs from provider + const outputs: Record = {}; + // We use the suite's cases to get the case IDs + const { getSuite } = require("./evalRunner"); + const suite = getSuite(suiteId); + if (!suite?.cases) return null; + + for (const evalCase of suite.cases) { + try { + outputs[evalCase.id] = await _outputProvider(suiteId, evalCase.id); + } catch (err: any) { + console.warn(`[EvalScheduler] Failed to get output for ${evalCase.id}: ${err.message}`); + outputs[evalCase.id] = `[ERROR] ${err.message}`; + } + } + + // Run evaluation + const result = runSuite(suiteId, outputs); + const now = Date.now(); + + const runResult: EvalRunResult = { + suiteId: result.suiteId, + suiteName: result.suiteName, + timestamp: now, + passRate: result.summary.passRate, + total: result.summary.total, + passed: result.summary.passed, + failed: result.summary.failed, + results: result.results, + }; + + // Update schedule state + entry.lastRunAt = now; + entry.nextRunAt = now + entry.intervalMs; + + // Store in history + _history.push(runResult); + // Keep last 100 runs + if (_history.length > 100) _history.shift(); + + console.log( + `[EvalScheduler] ${suiteId}: ${result.summary.passed}/${result.summary.total} passed (${(result.summary.passRate * 100).toFixed(1)}%)` + ); + + return runResult; + } catch (err: any) { + console.error(`[EvalScheduler] Error running ${suiteId}:`, err.message); + return null; + } +} + +/** + * Run a suite immediately (outside of schedule). + */ +export async function runNow(suiteId: string): Promise { + const entry = _schedules.get(suiteId) || { + suiteId, + intervalMs: 0, + lastRunAt: null, + nextRunAt: 0, + enabled: true, + }; + _schedules.set(suiteId, entry); + return executeScheduledRun(suiteId); +} + +// ── Query ── + +/** + * Get all scheduled suites and their status. + */ +export function getSchedules(): ScheduledEval[] { + return Array.from(_schedules.values()); +} + +/** + * Get run history for a suite (newest first). + */ +export function getHistory(suiteId?: string): EvalRunResult[] { + const filtered = suiteId ? _history.filter((r) => r.suiteId === suiteId) : _history; + return [...filtered].reverse(); +} + +/** + * Get a scorecard across all recent runs. + */ +export function getScorecard(): ReturnType | null { + if (_history.length === 0) return null; + + // Get latest run per suite + const latestBySuite = new Map(); + for (const run of _history) { + latestBySuite.set(run.suiteId, run); + } + + // Build scorecard from latest runs + const runs = Array.from(latestBySuite.values()).map((r) => ({ + suiteId: r.suiteId, + suiteName: r.suiteName, + results: r.results, + summary: { total: r.total, passed: r.passed, failed: r.failed, passRate: r.passRate }, + })); + + return createScorecard(runs); +} + +/** + * Stop all scheduled evaluations and clear state. + */ +export function stopAll(): void { + for (const timer of _timers.values()) { + clearInterval(timer as any); + } + _timers.clear(); + _schedules.clear(); + _history.length = 0; + _outputProvider = null; +} diff --git a/src/lib/piiSanitizer.ts b/src/lib/piiSanitizer.ts new file mode 100644 index 0000000000..dd439ace18 --- /dev/null +++ b/src/lib/piiSanitizer.ts @@ -0,0 +1,179 @@ +/** + * Output PII Sanitization — L-3 + * + * Scans LLM response text for PII patterns and optionally redacts them. + * This is the OUTPUT-side counterpart to the input sanitizer. + * Configurable via environment variables: + * + * PII_RESPONSE_SANITIZATION=true|false (default: false) + * PII_RESPONSE_SANITIZATION_MODE=redact|warn|block (default: redact) + * + * @module lib/piiSanitizer + */ + +// ── Configuration ── + +const isEnabled = () => process.env.PII_RESPONSE_SANITIZATION === "true"; +const getMode = (): "redact" | "warn" | "block" => + (process.env.PII_RESPONSE_SANITIZATION_MODE as "redact" | "warn" | "block") || "redact"; + +// ── PII Patterns ── + +interface PIIPattern { + name: string; + regex: RegExp; + replacement: string; + severity: "high" | "medium" | "low"; +} + +const PII_PATTERNS: PIIPattern[] = [ + { + name: "email", + regex: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, + replacement: "[EMAIL_REDACTED]", + severity: "medium", + }, + { + name: "ssn", + regex: /\b\d{3}-\d{2}-\d{4}\b/g, + replacement: "[SSN_REDACTED]", + severity: "high", + }, + { + name: "credit_card", + regex: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g, + replacement: "[CC_REDACTED]", + severity: "high", + }, + { + name: "phone_us", + regex: /\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g, + replacement: "[PHONE_REDACTED]", + severity: "medium", + }, + { + name: "phone_br", + regex: /\b(?:\+?55[-.\s]?)?\(?\d{2}\)?[-.\s]?\d{4,5}[-.\s]?\d{4}\b/g, + replacement: "[PHONE_REDACTED]", + severity: "medium", + }, + { + name: "cpf", + regex: /\b\d{3}\.\d{3}\.\d{3}-\d{2}\b/g, + replacement: "[CPF_REDACTED]", + severity: "high", + }, + { + name: "cnpj", + regex: /\b\d{2}\.\d{3}\.\d{3}\/\d{4}-\d{2}\b/g, + replacement: "[CNPJ_REDACTED]", + severity: "high", + }, + { + name: "ip_address", + regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, + replacement: "[IP_REDACTED]", + severity: "low", + }, + { + name: "aws_key", + regex: /\bAKIA[0-9A-Z]{16}\b/g, + replacement: "[AWS_KEY_REDACTED]", + severity: "high", + }, + { + name: "api_key_generic", + regex: /\b(?:sk|pk|api|key|token)[_-][a-zA-Z0-9]{20,}\b/gi, + replacement: "[API_KEY_REDACTED]", + severity: "high", + }, +]; + +// ── Public API ── + +export interface SanitizeResult { + text: string; + detections: Array<{ + pattern: string; + count: number; + severity: string; + }>; + redacted: boolean; +} + +/** + * Scan and optionally redact PII from LLM response text. + */ +export function sanitizePII(text: string): SanitizeResult { + if (!isEnabled() || !text || typeof text !== "string") { + return { text, detections: [], redacted: false }; + } + + const mode = getMode(); + const detections: SanitizeResult["detections"] = []; + let sanitized = text; + + for (const pattern of PII_PATTERNS) { + // Reset lastIndex for global regexes + pattern.regex.lastIndex = 0; + const matches = text.match(pattern.regex); + if (matches && matches.length > 0) { + detections.push({ + pattern: pattern.name, + count: matches.length, + severity: pattern.severity, + }); + + if (mode === "redact") { + pattern.regex.lastIndex = 0; + sanitized = sanitized.replace(pattern.regex, pattern.replacement); + } + } + } + + if (detections.length > 0 && mode === "warn") { + console.warn( + `[PII] Detected PII in response: ${detections.map((d) => `${d.pattern}(${d.count})`).join(", ")}` + ); + } + + return { + text: mode === "redact" ? sanitized : text, + detections, + redacted: mode === "redact" && detections.length > 0, + }; +} + +/** + * Sanitize a streaming chunk (text content only). + */ +export function sanitizePIIChunk(chunk: string): string { + if (!isEnabled()) return chunk; + const { text } = sanitizePII(chunk); + return text; +} + +/** + * Sanitize PII in a full response object (OpenAI-compatible format). + */ +export function sanitizePIIResponse(response: any): any { + if (!isEnabled() || !response) return response; + + try { + const choices = response.choices || []; + for (const choice of choices) { + if (choice.message?.content) { + const result = sanitizePII(choice.message.content); + choice.message.content = result.text; + } + if (choice.delta?.content) { + const result = sanitizePII(choice.delta.content); + choice.delta.content = result.text; + } + } + } catch { + // Fail open — don't break the response + } + + return response; +} diff --git a/src/lib/plugins/index.ts b/src/lib/plugins/index.ts new file mode 100644 index 0000000000..492719bbf8 --- /dev/null +++ b/src/lib/plugins/index.ts @@ -0,0 +1,214 @@ +/** + * Plugin/Middleware Architecture — L-8 + * + * Pre/post hooks on the request pipeline. Plugins are registered + * with a priority (lower = runs first) and can intercept requests + * before they reach the chat handler or modify responses after. + * + * Lifecycle: + * onRequest → runs BEFORE chat handler (can block/modify request) + * onResponse → runs AFTER chat handler (can modify/log response) + * onError → runs on handler errors (can recover or re-throw) + * + * @module lib/plugins + */ + +// ── Types ── + +export interface PluginContext { + /** Unique request ID */ + requestId: string; + /** Request body (parsed JSON) */ + body: any; + /** Model string */ + model: string; + /** Provider (if resolved) */ + provider?: string; + /** API key info */ + apiKeyInfo?: any; + /** Arbitrary metadata plugins can share */ + metadata: Record; +} + +export interface PluginResult { + /** If true, stop processing further plugins and return immediately */ + blocked?: boolean; + /** Optional response to return if blocked */ + response?: any; + /** Modified body (if any) */ + body?: any; + /** Modified metadata */ + metadata?: Record; +} + +export interface Plugin { + /** Unique plugin name */ + name: string; + /** Priority (lower = runs first, default 100) */ + priority?: number; + /** Whether the plugin is enabled */ + enabled?: boolean; + /** Called before the chat handler */ + onRequest?: (ctx: PluginContext) => Promise | PluginResult | void; + /** Called after the chat handler */ + onResponse?: ( + ctx: PluginContext, + response: any + ) => Promise | any | void; + /** Called on handler error */ + onError?: ( + ctx: PluginContext, + error: Error + ) => Promise | any | void; +} + +// ── Registry ── + +const _plugins: Plugin[] = []; + +/** + * Register a plugin. Plugins are sorted by priority on each registration. + */ +export function registerPlugin(plugin: Plugin): void { + // Set defaults + plugin.priority = plugin.priority ?? 100; + plugin.enabled = plugin.enabled ?? true; + + // Remove existing plugin with same name (re-registration) + const idx = _plugins.findIndex((p) => p.name === plugin.name); + if (idx !== -1) _plugins.splice(idx, 1); + + _plugins.push(plugin); + _plugins.sort((a, b) => (a.priority || 100) - (b.priority || 100)); + + console.log( + `[Plugins] Registered "${plugin.name}" (priority: ${plugin.priority}, enabled: ${plugin.enabled})` + ); +} + +/** + * Unregister a plugin by name. + */ +export function unregisterPlugin(name: string): boolean { + const idx = _plugins.findIndex((p) => p.name === name); + if (idx === -1) return false; + _plugins.splice(idx, 1); + return true; +} + +/** + * Enable/disable a plugin at runtime. + */ +export function setPluginEnabled(name: string, enabled: boolean): boolean { + const plugin = _plugins.find((p) => p.name === name); + if (!plugin) return false; + plugin.enabled = enabled; + return true; +} + +/** + * List all registered plugins. + */ +export function listPlugins(): Array<{ + name: string; + priority: number; + enabled: boolean; + hooks: string[]; +}> { + return _plugins.map((p) => ({ + name: p.name, + priority: p.priority || 100, + enabled: p.enabled !== false, + hooks: [ + p.onRequest ? "onRequest" : "", + p.onResponse ? "onResponse" : "", + p.onError ? "onError" : "", + ].filter(Boolean), + })); +} + +// ── Execution ── + +/** + * Run all onRequest hooks. Returns the (possibly modified) context, + * or a blocked response if any plugin blocked the request. + */ +export async function runOnRequest( + ctx: PluginContext +): Promise<{ blocked: boolean; response?: any; ctx: PluginContext }> { + let currentCtx = { ...ctx }; + + for (const plugin of _plugins) { + if (!plugin.enabled || !plugin.onRequest) continue; + + try { + const result = await plugin.onRequest(currentCtx); + if (result) { + if (result.blocked) { + console.log(`[Plugins] Request blocked by "${plugin.name}"`); + return { blocked: true, response: result.response, ctx: currentCtx }; + } + if (result.body) currentCtx.body = result.body; + if (result.metadata) { + currentCtx.metadata = { ...currentCtx.metadata, ...result.metadata }; + } + } + } catch (err: any) { + console.error(`[Plugins] onRequest error in "${plugin.name}": ${err.message}`); + // Plugin errors don't block the pipeline by default + } + } + + return { blocked: false, ctx: currentCtx }; +} + +/** + * Run all onResponse hooks. Returns the (possibly modified) response. + */ +export async function runOnResponse(ctx: PluginContext, response: any): Promise { + let currentResponse = response; + + for (const plugin of _plugins) { + if (!plugin.enabled || !plugin.onResponse) continue; + + try { + const modified = await plugin.onResponse(ctx, currentResponse); + if (modified !== undefined && modified !== null) { + currentResponse = modified; + } + } catch (err: any) { + console.error(`[Plugins] onResponse error in "${plugin.name}": ${err.message}`); + } + } + + return currentResponse; +} + +/** + * Run all onError hooks. Returns a recovery response if any plugin handles it, + * or null to let the error propagate. + */ +export async function runOnError(ctx: PluginContext, error: Error): Promise { + for (const plugin of _plugins) { + if (!plugin.enabled || !plugin.onError) continue; + + try { + const recovery = await plugin.onError(ctx, error); + if (recovery !== undefined && recovery !== null) { + console.log(`[Plugins] Error recovered by "${plugin.name}"`); + return recovery; + } + } catch (err: any) { + console.error(`[Plugins] onError error in "${plugin.name}": ${err.message}`); + } + } + + return null; // No recovery — let error propagate +} + +/** + * Reset all plugins (for testing). + */ +export function resetPlugins(): void { + _plugins.length = 0; +} diff --git a/src/lib/semanticCache.ts b/src/lib/semanticCache.ts index 2ad92d86e2..d6ab80e3a3 100644 --- a/src/lib/semanticCache.ts +++ b/src/lib/semanticCache.ts @@ -160,6 +160,90 @@ export function cleanExpiredEntries() { } } +/** + * Invalidate cache entries by model name. + * Useful when a model is updated/changed and cached responses are stale. + * @param {string} model - Model name to invalidate (exact match) + * @returns {number} Number of entries removed + */ +export function invalidateByModel(model: string): number { + getMemoryCache().clear(); // Memory cache doesn't track model; full clear + try { + const db = getDbInstance(); + const result = db + .prepare("DELETE FROM semantic_cache WHERE model = ?") + .run(model); + return result.changes || 0; + } catch { + return 0; + } +} + +/** + * Invalidate a single cache entry by its signature. + * @param {string} signature - Cache signature to invalidate + * @returns {boolean} Whether the entry was found and removed + */ +export function invalidateBySignature(signature: string): boolean { + getMemoryCache().delete(signature); + try { + const db = getDbInstance(); + const result = db + .prepare("DELETE FROM semantic_cache WHERE signature = ?") + .run(signature); + return (result.changes || 0) > 0; + } catch { + return false; + } +} + +/** + * Invalidate entries older than a given age. + * @param {number} maxAgeMs - Maximum age in milliseconds + * @returns {number} Number of entries removed + */ +export function invalidateStale(maxAgeMs: number): number { + getMemoryCache().clear(); + try { + const db = getDbInstance(); + const cutoff = new Date(Date.now() - maxAgeMs).toISOString(); + const result = db + .prepare("DELETE FROM semantic_cache WHERE created_at < ?") + .run(cutoff); + return result.changes || 0; + } catch { + return 0; + } +} + +// ── Auto-cleanup timer ── + +let _cleanupTimer: ReturnType | null = null; + +/** + * Start periodic auto-cleanup of expired entries. + * @param {number} intervalMs - Cleanup interval (default: 5 minutes) + */ +export function startAutoCleanup(intervalMs = 300_000): void { + stopAutoCleanup(); + _cleanupTimer = setInterval(() => { + const removed = cleanExpiredEntries(); + if (removed > 0) { + console.log(`[SemanticCache] Auto-cleaned ${removed} expired entries`); + } + }, intervalMs); +} + +/** + * Stop periodic auto-cleanup. + */ +export function stopAutoCleanup(): void { + if (_cleanupTimer) { + clearInterval(_cleanupTimer); + _cleanupTimer = null; + } +} + /** * Clear all cache entries. */ diff --git a/src/lib/toolPolicy.ts b/src/lib/toolPolicy.ts new file mode 100644 index 0000000000..1197f764cb --- /dev/null +++ b/src/lib/toolPolicy.ts @@ -0,0 +1,150 @@ +/** + * Tool-Calling Policy — L-4 + * + * Allowlist/denylist for tool (function) calling in LLM requests. + * Controls which tool names can be invoked, preventing dangerous + * tool use via prompt injection or misconfiguration. + * + * Configuration via environment variables: + * TOOL_POLICY_MODE=allowlist|denylist|disabled (default: disabled) + * TOOL_ALLOWLIST=tool1,tool2,tool3 + * TOOL_DENYLIST=dangerous_tool,exec_command + * + * @module lib/toolPolicy + */ + +// ── Types ── + +export interface ToolPolicyResult { + allowed: boolean; + denied: string[]; + reason?: string; +} + +type PolicyMode = "allowlist" | "denylist" | "disabled"; + +// ── Configuration ── + +function getMode(): PolicyMode { + return (process.env.TOOL_POLICY_MODE as PolicyMode) || "disabled"; +} + +function parseList(envKey: string): Set { + const raw = process.env[envKey]; + if (!raw) return new Set(); + return new Set( + raw + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean) + ); +} + +// ── Runtime overrides (for dashboard/API configuration) ── + +let _runtimeAllowlist: Set | null = null; +let _runtimeDenylist: Set | null = null; +let _runtimeMode: PolicyMode | null = null; + +/** + * Override the policy at runtime (e.g., from dashboard settings). + */ +export function setRuntimePolicy(config: { + mode?: PolicyMode; + allowlist?: string[]; + denylist?: string[]; +}): void { + if (config.mode) _runtimeMode = config.mode; + if (config.allowlist) _runtimeAllowlist = new Set(config.allowlist.map((s) => s.toLowerCase())); + if (config.denylist) _runtimeDenylist = new Set(config.denylist.map((s) => s.toLowerCase())); +} + +/** + * Reset runtime overrides. + */ +export function resetRuntimePolicy(): void { + _runtimeMode = null; + _runtimeAllowlist = null; + _runtimeDenylist = null; +} + +// ── Core Logic ── + +/** + * Evaluate a list of tool names against the policy. + */ +export function evaluateToolPolicy(toolNames: string[]): ToolPolicyResult { + const mode = _runtimeMode || getMode(); + + if (mode === "disabled" || !toolNames || toolNames.length === 0) { + return { allowed: true, denied: [] }; + } + + const normalizedNames = toolNames.map((n) => n.toLowerCase()); + + if (mode === "allowlist") { + const allowlist = _runtimeAllowlist || parseList("TOOL_ALLOWLIST"); + if (allowlist.size === 0) { + return { allowed: true, denied: [], reason: "Allowlist is empty — all tools permitted" }; + } + + const denied = normalizedNames.filter((name) => !allowlist.has(name)); + return { + allowed: denied.length === 0, + denied, + reason: denied.length > 0 ? `Tools not in allowlist: ${denied.join(", ")}` : undefined, + }; + } + + if (mode === "denylist") { + const denylist = _runtimeDenylist || parseList("TOOL_DENYLIST"); + const denied = normalizedNames.filter((name) => denylist.has(name)); + return { + allowed: denied.length === 0, + denied, + reason: denied.length > 0 ? `Tools in denylist: ${denied.join(", ")}` : undefined, + }; + } + + return { allowed: true, denied: [] }; +} + +/** + * Extract tool names from an OpenAI-compatible request body. + */ +export function extractToolNames(body: any): string[] { + const tools: string[] = []; + + // tools array (new format) + if (Array.isArray(body?.tools)) { + for (const tool of body.tools) { + if (tool?.function?.name) { + tools.push(tool.function.name); + } + } + } + + // functions array (legacy format) + if (Array.isArray(body?.functions)) { + for (const fn of body.functions) { + if (fn?.name) { + tools.push(fn.name); + } + } + } + + // tool_choice (if specific tool is forced) + if (body?.tool_choice?.function?.name) { + tools.push(body.tool_choice.function.name); + } + + return tools; +} + +/** + * Convenience: validate an entire request body against the tool policy. + */ +export function validateToolsInRequest(body: any): ToolPolicyResult { + const toolNames = extractToolNames(body); + return evaluateToolPolicy(toolNames); +} diff --git a/tests/integration/proxy-pipeline.test.mjs b/tests/integration/proxy-pipeline.test.mjs new file mode 100644 index 0000000000..ac50a66d04 --- /dev/null +++ b/tests/integration/proxy-pipeline.test.mjs @@ -0,0 +1,399 @@ +/** + * Proxy Pipeline Integration Tests — T-3 + * + * Tests the proxy pipeline wiring: format detection, credential retry loop, + * circuit breaker integration, and the new Phase 2 modules (DI container, + * prompt versioning, plugin architecture, eval scheduler). + * + * @module tests/integration/proxy-pipeline.test.mjs + */ + +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, existsSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, "..", ".."); + +function readSrc(relPath) { + const full = join(ROOT, "src", relPath); + if (!existsSync(full)) return null; + return readFileSync(full, "utf8"); +} + +// ═══════════════════════════════════════════════════ +// 1. Chat Handler Pipeline Wiring +// ═══════════════════════════════════════════════════ + +describe("Chat Pipeline — handleSingleModelChat decomposition", () => { + const src = readSrc("sse/handlers/chat.ts"); + + it("should define resolveModelOrError helper", () => { + assert.ok(src, "chat.ts should exist"); + assert.match(src, /function\s+resolveModelOrError/); + }); + + it("should define checkPipelineGates helper", () => { + assert.match(src, /function\s+checkPipelineGates/); + }); + + it("should define executeChatWithBreaker helper", () => { + assert.match(src, /function\s+executeChatWithBreaker/); + }); + + it("should define recordCostIfNeeded helper", () => { + assert.match(src, /function\s+recordCostIfNeeded/); + }); + + it("handleSingleModelChat should use resolveModelOrError", () => { + // Extract handleSingleModelChat body + assert.match(src, /resolveModelOrError\(modelStr/); + }); + + it("handleSingleModelChat should use checkPipelineGates", () => { + assert.match(src, /checkPipelineGates\(provider/); + }); + + it("handleSingleModelChat should use executeChatWithBreaker", () => { + assert.match(src, /executeChatWithBreaker\(/); + }); + + it("handleSingleModelChat should use recordCostIfNeeded", () => { + assert.match(src, /recordCostIfNeeded\(/); + }); +}); + +describe("Chat Pipeline — combo fallback support", () => { + const src = readSrc("sse/handlers/chat.ts"); + + it("should import handleComboChat", () => { + assert.ok(src, "chat.ts should exist"); + assert.match(src, /handleComboChat/); + }); + + it("should delegate to handleSingleModelChat for each combo model", () => { + assert.match(src, /handleSingleModel.*handleSingleModelChat/s); + }); + + it("should check model availability before attempting combo models", () => { + assert.match(src, /isModelAvailable/); + }); +}); + +describe("Chat Pipeline — circuit breaker integration", () => { + const src = readSrc("sse/handlers/chat.ts"); + + it("should import CircuitBreakerOpenError", () => { + assert.ok(src, "chat.ts should exist"); + assert.match(src, /CircuitBreakerOpenError/); + }); + + it("should handle CircuitBreakerOpenError with retry-after", () => { + assert.match(src, /retryAfterMs/); + }); + + it("should reject requests when circuit is open", () => { + assert.match(src, /circuit breaker is open/i); + }); +}); + +// ═══════════════════════════════════════════════════ +// 2. DI Container (A-5) +// ═══════════════════════════════════════════════════ + +describe("DI Container — container.ts", () => { + let container; + + beforeEach(async () => { + const mod = await import("../../src/lib/container.ts"); + container = mod.container; + }); + + afterEach(() => { + // Don't reset — keep default registrations + }); + + it("should export a container singleton", () => { + assert.ok(container); + assert.equal(typeof container.register, "function"); + assert.equal(typeof container.resolve, "function"); + assert.equal(typeof container.has, "function"); + }); + + it("should register and resolve a custom service", () => { + container.register("testService", () => ({ greeting: "hello" })); + const svc = container.resolve("testService"); + assert.deepEqual(svc, { greeting: "hello" }); + }); + + it("should return cached singleton on repeated resolve", () => { + let count = 0; + container.register("counterService", () => ({ value: ++count })); + const a = container.resolve("counterService"); + const b = container.resolve("counterService"); + assert.strictEqual(a, b); + assert.equal(a.value, 1); + }); + + it("should throw on resolving unregistered service", () => { + assert.throws(() => container.resolve("nonExistent"), /No factory registered/); + }); + + it("should have default registrations", () => { + const names = container.list(); + assert.ok(names.includes("settings"), "should have settings"); + assert.ok(names.includes("db"), "should have db"); + assert.ok(names.includes("encryption"), "should have encryption"); + assert.ok(names.includes("policyEngine"), "should have policyEngine"); + assert.ok(names.includes("circuitBreaker"), "should have circuitBreaker"); + assert.ok(names.includes("telemetry"), "should have telemetry"); + }); + + it("should support re-registration (overwrite)", () => { + container.register("testOverwrite", () => "v1"); + assert.equal(container.resolve("testOverwrite"), "v1"); + container.register("testOverwrite", () => "v2"); + assert.equal(container.resolve("testOverwrite"), "v2"); + }); +}); + +// ═══════════════════════════════════════════════════ +// 3. Plugin Architecture (L-8) +// ═══════════════════════════════════════════════════ + +describe("Plugin Architecture — plugins/index.ts", () => { + let plugins; + + beforeEach(async () => { + plugins = await import("../../src/lib/plugins/index.ts"); + plugins.resetPlugins(); + }); + + afterEach(() => { + plugins.resetPlugins(); + }); + + it("should register and list plugins", () => { + plugins.registerPlugin({ + name: "test-logger", + priority: 10, + onRequest: () => {}, + }); + + const list = plugins.listPlugins(); + assert.equal(list.length, 1); + assert.equal(list[0].name, "test-logger"); + assert.equal(list[0].priority, 10); + assert.deepEqual(list[0].hooks, ["onRequest"]); + }); + + it("should sort plugins by priority", () => { + plugins.registerPlugin({ name: "low", priority: 200 }); + plugins.registerPlugin({ name: "high", priority: 1 }); + plugins.registerPlugin({ name: "mid", priority: 50 }); + + const list = plugins.listPlugins(); + assert.deepEqual( + list.map((p) => p.name), + ["high", "mid", "low"] + ); + }); + + it("should run onRequest hooks in order", async () => { + const order = []; + plugins.registerPlugin({ + name: "first", + priority: 1, + onRequest: () => { + order.push("first"); + }, + }); + plugins.registerPlugin({ + name: "second", + priority: 2, + onRequest: () => { + order.push("second"); + }, + }); + + const ctx = { requestId: "r1", body: {}, model: "test", metadata: {} }; + await plugins.runOnRequest(ctx); + assert.deepEqual(order, ["first", "second"]); + }); + + it("should support request blocking", async () => { + plugins.registerPlugin({ + name: "blocker", + priority: 1, + onRequest: () => ({ blocked: true, response: { error: "denied" } }), + }); + plugins.registerPlugin({ + name: "never-runs", + priority: 2, + onRequest: () => { + throw new Error("should not run"); + }, + }); + + const ctx = { requestId: "r2", body: {}, model: "test", metadata: {} }; + const result = await plugins.runOnRequest(ctx); + assert.equal(result.blocked, true); + assert.deepEqual(result.response, { error: "denied" }); + }); + + it("should enable/disable plugins at runtime", () => { + plugins.registerPlugin({ + name: "toggle-me", + onRequest: () => {}, + }); + + assert.ok(plugins.setPluginEnabled("toggle-me", false)); + const list = plugins.listPlugins(); + assert.equal(list[0].enabled, false); + }); + + it("should unregister plugins", () => { + plugins.registerPlugin({ name: "removable" }); + assert.equal(plugins.listPlugins().length, 1); + assert.ok(plugins.unregisterPlugin("removable")); + assert.equal(plugins.listPlugins().length, 0); + }); + + it("should run onResponse hooks", async () => { + plugins.registerPlugin({ + name: "response-modifier", + onResponse: (_ctx, response) => ({ ...response, modified: true }), + }); + + const ctx = { requestId: "r3", body: {}, model: "test", metadata: {} }; + const result = await plugins.runOnResponse(ctx, { data: "original" }); + assert.equal(result.modified, true); + assert.equal(result.data, "original"); + }); + + it("should run onError hooks and allow recovery", async () => { + plugins.registerPlugin({ + name: "error-handler", + onError: (_ctx, _error) => ({ recovered: true }), + }); + + const ctx = { requestId: "r4", body: {}, model: "test", metadata: {} }; + const result = await plugins.runOnError(ctx, new Error("test error")); + assert.deepEqual(result, { recovered: true }); + }); + + it("should return null from onError if no recovery", async () => { + const ctx = { requestId: "r5", body: {}, model: "test", metadata: {} }; + const result = await plugins.runOnError(ctx, new Error("unhandled")); + assert.equal(result, null); + }); +}); + +// ═══════════════════════════════════════════════════ +// 4. Prompt Template Versioning (L-6) +// ═══════════════════════════════════════════════════ + +describe("Prompt Template Versioning — prompts.ts module existence", () => { + it("prompts.ts should exist", () => { + const full = join(ROOT, "src", "lib", "db", "prompts.ts"); + assert.ok(existsSync(full), "prompts.ts should exist"); + }); + + it("should export CRUD functions", () => { + const src = readFileSync(join(ROOT, "src", "lib", "db", "prompts.ts"), "utf8"); + assert.match(src, /export function savePrompt/); + assert.match(src, /export function getActivePrompt/); + assert.match(src, /export function getPromptVersion/); + assert.match(src, /export function listPromptVersions/); + assert.match(src, /export function listPrompts/); + assert.match(src, /export function rollbackPrompt/); + assert.match(src, /export function renderPrompt/); + }); + + it("should define PromptTemplate interface", () => { + const src = readFileSync(join(ROOT, "src", "lib", "db", "prompts.ts"), "utf8"); + assert.match(src, /export interface PromptTemplate/); + }); + + it("should use content hashing for deduplication", () => { + const src = readFileSync(join(ROOT, "src", "lib", "db", "prompts.ts"), "utf8"); + assert.match(src, /content_hash/); + assert.match(src, /sha256/); + }); +}); + +// ═══════════════════════════════════════════════════ +// 5. Eval Scheduler (L-7) +// ═══════════════════════════════════════════════════ + +describe("Eval Scheduler — scheduler.ts module existence", () => { + it("scheduler.ts should exist", () => { + const full = join(ROOT, "src", "lib", "evals", "scheduler.ts"); + assert.ok(existsSync(full), "scheduler.ts should exist"); + }); + + it("should export scheduling functions", () => { + const src = readFileSync(join(ROOT, "src", "lib", "evals", "scheduler.ts"), "utf8"); + assert.match(src, /export function schedule/); + assert.match(src, /export function unschedule/); + assert.match(src, /export function pause/); + assert.match(src, /export function resume/); + assert.match(src, /export\s+(async\s+)?function\s+runNow/); + assert.match(src, /export function getSchedules/); + assert.match(src, /export function getHistory/); + assert.match(src, /export function stopAll/); + }); + + it("should define ScheduledEval and EvalRunResult types", () => { + const src = readFileSync(join(ROOT, "src", "lib", "evals", "scheduler.ts"), "utf8"); + assert.match(src, /export interface ScheduledEval/); + assert.match(src, /export interface EvalRunResult/); + }); + + it("should have pluggable output provider", () => { + const src = readFileSync(join(ROOT, "src", "lib", "evals", "scheduler.ts"), "utf8"); + assert.match(src, /export function setOutputProvider/); + }); +}); + +// ═══════════════════════════════════════════════════ +// 6. Migration Runner (E-5) +// ═══════════════════════════════════════════════════ + +describe("Migration System — files exist", () => { + it("migrationRunner.ts should exist", () => { + const full = join(ROOT, "src", "lib", "db", "migrationRunner.ts"); + assert.ok(existsSync(full), "migrationRunner.ts should exist"); + }); + + it("001_initial_schema.sql should exist", () => { + const full = join(ROOT, "src", "lib", "db", "migrations", "001_initial_schema.sql"); + assert.ok(existsSync(full), "001_initial_schema.sql should exist"); + }); + + it("core.ts should reference migration runner", () => { + const src = readSrc("lib/db/core.ts"); + assert.ok(src); + assert.match(src, /runMigrations/); + assert.match(src, /_omniroute_migrations/); + }); +}); + +// ═══════════════════════════════════════════════════ +// 7. CORS Configuration (L-5) +// ═══════════════════════════════════════════════════ + +describe("CORS — centralized configuration", () => { + it("shared/utils/cors.ts should exist", () => { + const full = join(ROOT, "src", "shared", "utils", "cors.ts"); + assert.ok(existsSync(full), "shared/utils/cors.ts should exist"); + }); + + it("should export CORS_HEADERS and CORS_ORIGIN", () => { + const src = readSrc("shared/utils/cors.ts"); + assert.match(src, /CORS_HEADERS/); + assert.match(src, /CORS_ORIGIN/); + }); +});