diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index 34f1e744cc..0b6982d8c3 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -230,6 +230,11 @@ function attachToolNameMap(payload: T, toolNameMap: Map | nul return copy; } +function getRequestTargetModel(body: Record): string { + const target = body.model; + return typeof target === "string" && target.length > 0 ? target : "unknown"; +} + export class AntigravityExecutor extends BaseExecutor { constructor() { super("antigravity", PROVIDERS.antigravity); @@ -635,7 +640,7 @@ export class AntigravityExecutor extends BaseExecutor { log?.debug?.( "TELEMETRY", - `[Antigravity] Execute - URL: ${url}, Model: ${model}, Target: ${(transformedBody as any)?.model || "unknown"}, RetryAttempt: ${retryAttemptsByUrl[urlIndex]}` + `[Antigravity] Execute - URL: ${url}, Model: ${model}, Target: ${getRequestTargetModel(transformedBody)}, RetryAttempt: ${retryAttemptsByUrl[urlIndex]}` ); const response = await fetch(url, { diff --git a/open-sse/services/compression/caveman.ts b/open-sse/services/compression/caveman.ts index b6050edf26..6a0b092bef 100644 --- a/open-sse/services/compression/caveman.ts +++ b/open-sse/services/compression/caveman.ts @@ -204,17 +204,188 @@ export function applyRulesToText( function cleanupArtifacts(text: string): string { let result = text; - if (result.includes(" ")) result = result.replace(/[ \t]{2,}/g, " "); - if (/[\t ]+[,.;:!?]/.test(result)) result = result.replace(/[ \t]([,.;:!?])/g, "$1"); - if (/[.!?]{2,}/.test(result)) result = result.replace(/([.!?]){2,}/g, "$1"); - if (/[ \t]\n/.test(result)) result = result.replace(/[ \t]+$/gm, ""); + if (hasRepeatedHorizontalWhitespace(result)) { + result = collapseHorizontalWhitespaceRuns(result); + } + result = removeHorizontalWhitespaceBeforePunctuation(result); + result = collapseRepeatedSentencePunctuation(result); + if (result.includes(" \n") || result.includes("\t\n")) { + result = stripLineTrailingHorizontalWhitespace(result); + } if (result.endsWith(" ") || result.endsWith("\t")) result = result.trimEnd(); - if (result.includes("\n\n\n")) result = result.replace(/\n{3,}/g, "\n\n"); - if (result.startsWith("\n")) result = result.replace(/^\n+/, ""); - if (result.endsWith("\n")) result = result.replace(/\n+$/, ""); + if (result.includes("\n\n\n")) result = collapseExcessNewlines(result); + if (result.startsWith("\n")) result = trimLeadingNewlines(result); + if (result.endsWith("\n")) result = trimTrailingNewlines(result); return result; } +function isHorizontalWhitespace(char: string): boolean { + return char === " " || char === "\t"; +} + +function isSentencePunctuation(char: string): boolean { + return char === "." || char === "!" || char === "?"; +} + +function isCleanupPunctuation(char: string): boolean { + return ( + char === "," || char === "." || char === ";" || char === ":" || char === "!" || char === "?" + ); +} + +function hasRepeatedHorizontalWhitespace(text: string): boolean { + let previousWasWhitespace = false; + for (const char of text) { + const currentIsWhitespace = isHorizontalWhitespace(char); + if (currentIsWhitespace && previousWasWhitespace) return true; + previousWasWhitespace = currentIsWhitespace; + } + return false; +} + +function collapseHorizontalWhitespaceRuns(text: string): string { + let output = ""; + let changed = false; + + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (!isHorizontalWhitespace(char)) { + output += char; + continue; + } + + const start = index; + while (index + 1 < text.length && isHorizontalWhitespace(text[index + 1])) { + index++; + } + + if (index > start) { + output += " "; + changed = true; + } else { + output += char; + } + } + + return changed ? output : text; +} + +function removeHorizontalWhitespaceBeforePunctuation(text: string): string { + let output = ""; + let changed = false; + + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (!isHorizontalWhitespace(char)) { + output += char; + continue; + } + + const start = index; + while (index + 1 < text.length && isHorizontalWhitespace(text[index + 1])) { + index++; + } + + const nextChar = text[index + 1]; + if (nextChar && isCleanupPunctuation(nextChar)) { + changed = true; + continue; + } + + output += text.slice(start, index + 1); + } + + return changed ? output : text; +} + +function collapseRepeatedSentencePunctuation(text: string): string { + let output = ""; + let changed = false; + + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (!isSentencePunctuation(char)) { + output += char; + continue; + } + + let lastPunctuation = char; + const start = index; + while (index + 1 < text.length && isSentencePunctuation(text[index + 1])) { + index++; + lastPunctuation = text[index]; + } + + if (index > start) changed = true; + output += lastPunctuation; + } + + return changed ? output : text; +} + +function trimEndHorizontalWhitespace(text: string): string { + let end = text.length; + while (end > 0 && isHorizontalWhitespace(text[end - 1])) { + end--; + } + return end === text.length ? text : text.slice(0, end); +} + +function stripLineTrailingHorizontalWhitespace(text: string): string { + const lines = text.split("\n"); + let changed = false; + const cleanedLines = lines.map((line) => { + const cleaned = trimEndHorizontalWhitespace(line); + if (cleaned !== line) changed = true; + return cleaned; + }); + return changed ? cleanedLines.join("\n") : text; +} + +function collapseExcessNewlines(text: string): string { + let output = ""; + let changed = false; + + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (char !== "\n") { + output += char; + continue; + } + + const start = index; + while (index + 1 < text.length && text[index + 1] === "\n") { + index++; + } + + const newlineCount = index - start + 1; + if (newlineCount > 2) { + output += "\n\n"; + changed = true; + } else { + output += text.slice(start, index + 1); + } + } + + return changed ? output : text; +} + +function trimLeadingNewlines(text: string): string { + let start = 0; + while (start < text.length && text[start] === "\n") { + start++; + } + return start === 0 ? text : text.slice(start); +} + +function trimTrailingNewlines(text: string): string { + let end = text.length; + while (end > 0 && text[end - 1] === "\n") { + end--; + } + return end === text.length ? text : text.slice(0, end); +} + function recapitalizeSentences(text: string): string { return text.replace(/(^|[.!?][ \t]|\n[ \t]*)([a-z])/g, (_match, prefix: string, char: string) => { return `${prefix}${char.toUpperCase()}`; diff --git a/open-sse/services/compression/validation.ts b/open-sse/services/compression/validation.ts index 152b8a2221..b359ff2f7c 100644 --- a/open-sse/services/compression/validation.ts +++ b/open-sse/services/compression/validation.ts @@ -26,12 +26,47 @@ function requireExactPresence( ) { for (const item of originalItems) { if (!compressed.includes(item)) { - const preview = item.replace(/\s+/g, " ").slice(0, 80); + const preview = collapseWhitespaceForPreview(item).slice(0, 80); errors.push(`${label} changed or missing: ${preview}`); } } } +function isPreviewWhitespace(char: string): boolean { + return ( + char === " " || + char === "\t" || + char === "\n" || + char === "\r" || + char === "\f" || + char === "\v" + ); +} + +function collapseWhitespaceForPreview(text: string): string { + let output = ""; + let previousWasWhitespace = false; + let changed = false; + + for (const char of text) { + if (isPreviewWhitespace(char)) { + if (!previousWasWhitespace) { + output += " "; + } else { + changed = true; + } + if (char !== " ") changed = true; + previousWasWhitespace = true; + continue; + } + + output += char; + previousWasWhitespace = false; + } + + return changed ? output : text; +} + export function validateCompression(original: string, compressed: string): ValidationResult { const errors: string[] = []; const warnings: string[] = []; diff --git a/open-sse/translator/request/openai-to-gemini.ts b/open-sse/translator/request/openai-to-gemini.ts index 4af955986d..018c6fdb5a 100644 --- a/open-sse/translator/request/openai-to-gemini.ts +++ b/open-sse/translator/request/openai-to-gemini.ts @@ -58,7 +58,7 @@ type GeminiFunctionDeclaration = { type GeminiRequest = { model: string; contents?: GeminiContent[]; - [key: string]: any; + [key: string]: unknown; generationConfig: GeminiGenerationConfig; safetySettings: unknown; systemInstruction?: GeminiContent; @@ -81,7 +81,7 @@ type CloudCodeEnvelope = { session_id?: string; sessionId?: string; contents?: GeminiContent[]; - [key: string]: any; + [key: string]: unknown; systemInstruction?: GeminiContent; generationConfig: GeminiGenerationConfig; tools?: Array<{ @@ -101,6 +101,15 @@ type GeminiToolNameOptions = { functionResponseShape?: "result" | "output"; }; +type OpenAIToolCallLike = { + thoughtSignature?: unknown; + thought_signature?: unknown; + function?: { + thoughtSignature?: unknown; + thought_signature?: unknown; + }; +}; + function buildChangedToolNameMap(toolNameMap: Map): Map | null { const changedEntries = [...toolNameMap.entries()].filter( ([sanitizedName, originalName]) => sanitizedName !== originalName @@ -108,16 +117,17 @@ function buildChangedToolNameMap(toolNameMap: Map): Map 0 ? new Map(changedEntries) : null; } -function extractClientThoughtSignature(toolCall) { +function extractClientThoughtSignature(toolCall: unknown): string | null { if (!toolCall || typeof toolCall !== "object") return null; + const candidate = toolCall as OpenAIToolCallLike; - return ( - toolCall.thoughtSignature || - toolCall.thought_signature || - toolCall.function?.thoughtSignature || - toolCall.function?.thought_signature || - null - ); + const signature = + candidate.thoughtSignature || + candidate.thought_signature || + candidate.function?.thoughtSignature || + candidate.function?.thought_signature || + null; + return typeof signature === "string" && signature.length > 0 ? signature : null; } // Core: Convert OpenAI request to Gemini format (base for all variants) diff --git a/package-lock.json b/package-lock.json index 9583daaead..2a8b6b45c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9720,9 +9720,9 @@ "license": "MIT" }, "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.1.tgz", + "integrity": "sha512-1FMu8/N15Ck1BL551Jf42NYIoin2unWjLQ2Fze/DXryJRl5twqtwNHlO39qERGbIOcKYWHdgRryhOC+NG4eaLw==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/package.json b/package.json index 6f55bf3045..3eabb2539e 100644 --- a/package.json +++ b/package.json @@ -216,6 +216,7 @@ "dompurify": "^3.4.2", "path-to-regexp": "^8.4.0", "postcss": "^8.5.10", + "ip-address": "10.1.1", "hono": "^4.12.14", "@hono/node-server": "^1.19.13", "react": "$react", diff --git a/src/app/api/assess/route.ts b/src/app/api/assess/route.ts index dba89d00a8..993d21a77c 100644 --- a/src/app/api/assess/route.ts +++ b/src/app/api/assess/route.ts @@ -1,8 +1,14 @@ import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; import { Assessor } from "@/domain/assessment/assessor"; import { Categorizer } from "@/domain/assessment/categorizer"; import { SelfHealer } from "@/domain/assessment/selfHealer"; -import { type AssessmentScope, type AssessmentTrigger } from "@/domain/assessment/types"; +import { + type AssessmentScope, + type AssessmentTrigger, + type ModelCategory, +} from "@/domain/assessment/types"; +import { validateBody } from "@/shared/validation/helpers"; const assessor = new Assessor( process.env.OMNIROUTe_API_KEY ?? process.env.API_KEY ?? "", @@ -12,11 +18,56 @@ const assessor = new Assessor( const categorizer = new Categorizer(); const healer = new SelfHealer(); +const modelCategories = new Set([ + "coding", + "reasoning", + "reasoning_deep", + "chat", + "fast", + "vision", + "tool_call", + "structured_output", +]); + +const assessmentScopeSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("all") }), + z.object({ type: z.literal("provider"), providerId: z.string().min(1) }), + z.object({ type: z.literal("model"), modelId: z.string().min(1) }), +]); + +const assessmentPostSchema = z.object({ + scope: assessmentScopeSchema.optional().default({ type: "all" }), + trigger: z + .enum(["scheduled", "on_demand", "on_provider_change", "on_error", "startup"]) + .optional() + .default("on_demand"), +}); + +type ModelListItem = { id: string }; + +function isModelCategory(value: string): value is ModelCategory { + return modelCategories.has(value as ModelCategory); +} + +function isModelListItem(value: unknown): value is ModelListItem { + return ( + !!value && + typeof value === "object" && + "id" in value && + typeof (value as { id?: unknown }).id === "string" + ); +} + export async function POST(request: NextRequest) { try { - const body = await request.json(); - const scope: AssessmentScope = body.scope ?? { type: "all" }; - const trigger: AssessmentTrigger = body.trigger ?? "on_demand"; + const rawBody = await request.json(); + const validation = validateBody(assessmentPostSchema, rawBody); + if (!validation.success) { + return NextResponse.json({ error: validation.error }, { status: 400 }); + } + + const scope: AssessmentScope = validation.data.scope; + const trigger: AssessmentTrigger = validation.data.trigger; let models: Array<{ providerId: string; modelId: string }>; @@ -63,7 +114,9 @@ export async function GET(request: NextRequest) { let results = assessor.getAllAssessments(); if (status) results = results.filter((a) => a.status === status); if (provider) results = results.filter((a) => a.providerId === provider); - if (category) results = results.filter((a) => a.categories.includes(category as any)); + if (category && isModelCategory(category)) { + results = results.filter((a) => a.categories.includes(category)); + } return NextResponse.json({ models: results }); } @@ -93,10 +146,12 @@ async function getAllModels(): Promise m.id?.startsWith("auto/")) - .map((m: any) => ({ providerId: "auto", modelId: m.id.replace("auto/", "") })); + const data = (await resp.json()) as { data?: unknown }; + const models = Array.isArray(data.data) ? data.data : []; + return models + .filter(isModelListItem) + .filter((model) => model.id.startsWith("auto/")) + .map((model) => ({ providerId: "auto", modelId: model.id.replace("auto/", "") })); } catch { return []; } diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index dad1bc3110..62d8a67d70 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -16,7 +16,8 @@ import { writeCallArtifact, type CallLogArtifact, } from "../usage/callLogArtifacts"; -import { autoMigrateLegacyEncryptedConnections } from "./providers"; +import { migrateLegacyEncryptedString } from "./encryption"; +import { invalidateDbCache } from "./readCache"; type SqliteDatabase = import("better-sqlite3").Database; type JsonRecord = Record; @@ -94,10 +95,16 @@ function isNativeSqliteLoadError(error: unknown): boolean { message.includes("Module did not self-register") || message.includes("NODE_MODULE_VERSION") || message.includes("ERR_DLOPEN_FAILED") || - (error as any)?.code === "ERR_DLOPEN_FAILED" + getErrorCode(error) === "ERR_DLOPEN_FAILED" ); } +function getErrorCode(error: unknown): string | undefined { + if (!error || typeof error !== "object" || !("code" in error)) return undefined; + const code = (error as { code?: unknown }).code; + return typeof code === "string" ? code : undefined; +} + function createNativeSqliteLoadError(error: unknown): Error { const message = error instanceof Error ? error.message : String(error); const detail = @@ -105,10 +112,10 @@ function createNativeSqliteLoadError(error: unknown): Error { "This usually happens after switching Node.js versions without rebuilding native modules. " + "Run `npm rebuild better-sqlite3` in the OmniRoute project and start again. " + `Original error: ${message}`; - const wrapped = new Error(detail); + const wrapped = new Error(detail) as Error & { cause?: unknown; code?: string }; wrapped.name = "NativeSqliteLoadError"; - (wrapped as any).cause = error; - (wrapped as any).code = (error as any)?.code || "ERR_DLOPEN_FAILED"; + wrapped.cause = error; + wrapped.code = getErrorCode(error) || "ERR_DLOPEN_FAILED"; return wrapped; } @@ -967,7 +974,7 @@ function shouldRunStartupDbHealthCheck(): boolean { return !isAutomatedTestProcess(); } -function createHealthCheckBackup(db: SqliteDatabase): boolean { +function createManagedDbBackup(db: SqliteDatabase, reason: string): boolean { const isTest = isAutomatedTestProcess(); if (isTest) return false; @@ -978,19 +985,72 @@ function createHealthCheckBackup(db: SqliteDatabase): boolean { } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const backupPath = path.join(backupDir, `db_${timestamp}_health-check-repair.sqlite`); + const backupPath = path.join(backupDir, `db_${timestamp}_${reason}.sqlite`); const escapedBackupPath = backupPath.replace(/'/g, "''"); db.exec(`VACUUM INTO '${escapedBackupPath}'`); - console.log(`[DB] Health-check backup created: ${backupPath}`); + console.log(`[DB] Backup created (${reason}): ${backupPath}`); return true; } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); - console.warn("[DB] Failed to create health-check backup:", message); + console.warn(`[DB] Failed to create ${reason} backup:`, message); return false; } } +function createHealthCheckBackup(db: SqliteDatabase): boolean { + return createManagedDbBackup(db, "health-check-repair"); +} + +function autoMigrateLegacyEncryptedConnections(db: SqliteDatabase): number { + const rows = db.prepare("SELECT * FROM provider_connections").all() as JsonRecord[]; + const updateStmt = db.prepare( + "UPDATE provider_connections SET api_key = @apiKey, id_token = @idToken, access_token = @accessToken, refresh_token = @refreshToken, updated_at = @updatedAt WHERE id = @id" + ); + const encryptedFields = ["apiKey", "idToken", "accessToken", "refreshToken"] as const; + let migratedCount = 0; + let backupCreated = false; + + for (const row of rows) { + const camelRow = rowToCamel(row); + if (!camelRow) continue; + + let updatedRow = false; + for (const field of encryptedFields) { + if (typeof camelRow[field] !== "string") continue; + + const { updated, value } = migrateLegacyEncryptedString(camelRow[field]); + if (updated) { + camelRow[field] = value; + updatedRow = true; + } + } + + if (!updatedRow) continue; + if (!backupCreated) { + createManagedDbBackup(db, "legacy-encryption-migration"); + backupCreated = true; + } + + updateStmt.run({ + id: camelRow.id, + apiKey: camelRow.apiKey ?? null, + idToken: camelRow.idToken ?? null, + accessToken: camelRow.accessToken ?? null, + refreshToken: camelRow.refreshToken ?? null, + updatedAt: new Date().toISOString(), + }); + migratedCount++; + } + + if (migratedCount > 0) { + invalidateDbCache("connections"); + console.log(`[DB] Auto-migrated ${migratedCount} connection(s) to new static-salt encryption.`); + } + + return migratedCount; +} + let dbHealthCheckTimer: NodeJS.Timeout | null = null; function getDbHealthCheckIntervalMs(): number { @@ -1285,7 +1345,7 @@ export function getDbInstance(): SqliteDatabase { // Re-encrypt any tokens using the legacy dynamic salt to canonical static salt try { - autoMigrateLegacyEncryptedConnections(); + autoMigrateLegacyEncryptedConnections(db); } catch (err: unknown) { const message = err instanceof Error ? err.message : String(err); console.error(`[DB] Legacy encryption migration failed: ${message}`); @@ -1577,10 +1637,11 @@ export function runManualVacuum(): { success: boolean; duration: number; error?: const duration = Date.now() - startTime; console.log(`[DB] Manual VACUUM completed in ${duration}ms`); return { success: true, duration }; - } catch (err: any) { + } catch (err: unknown) { const duration = Date.now() - startTime; console.error("[DB] Manual VACUUM failed:", err); - return { success: false, duration, error: err.message }; + const message = err instanceof Error ? err.message : String(err); + return { success: false, duration, error: message }; } } diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index c254477fd2..f9fa21b109 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -126,7 +126,7 @@ function mergeDatabaseSettingsNamespace( for (const section of DATABASE_SETTINGS_SECTIONS) { const defaultSection = DEFAULT_DATABASE_SETTINGS[section] as Record; const sectionTarget = target[section] as Record; - const flatAliases = LEGACY_FLAT_KEYS[section]; + const flatAliases = LEGACY_FLAT_KEYS[section] as Partial>; for (const key of Object.keys(defaultSection)) { for (const alias of flatAliases[key] ?? []) { diff --git a/tests/unit/compression/caveman-engine.test.ts b/tests/unit/compression/caveman-engine.test.ts index 94f2fe9eab..f97457eecc 100644 --- a/tests/unit/compression/caveman-engine.test.ts +++ b/tests/unit/compression/caveman-engine.test.ts @@ -229,4 +229,29 @@ describe("caveman engine", () => { }); assert.ok(result.stats.durationMs < 25, `Expected <25ms, got ${result.stats.durationMs}ms`); }); + + it("cleans whitespace and punctuation artifacts without regex backtracking", () => { + const body = { + messages: [ + { + role: "user", + content: "\n\nPlease\t make sure to keep this stable !!! \n\n\n\nThank you.", + }, + ], + }; + const result = cavemanCompress(body, { + enabled: true, + compressRoles: ["user"], + skipRules: [], + minMessageLength: 0, + preservePatterns: [], + }); + const text = result.body.messages[0].content as string; + + assert.doesNotMatch(text, /^\n/); + assert.doesNotMatch(text, /\n$/); + assert.doesNotMatch(text, /\n\n\n/); + assert.doesNotMatch(text, /[ \t]+[,.!?;:]/); + assert.doesNotMatch(text, /[ \t]{2,}/); + }); }); diff --git a/tests/unit/compression/validation.test.ts b/tests/unit/compression/validation.test.ts index f20d8862e2..423aad4d28 100644 --- a/tests/unit/compression/validation.test.ts +++ b/tests/unit/compression/validation.test.ts @@ -117,6 +117,13 @@ describe("compression preservation and validation", () => { assert.ok(validation.errors.length >= 2); }); + it("formats validation previews with linear whitespace collapsing", () => { + const original = `Use \`${"\t".repeat(5000)}exact_token\` in the request.`; + const validation = validateCompression(original, "Use token."); + assert.equal(validation.valid, false); + assert.match(validation.errors.join("\n"), /inline code changed or missing: ` exact_token`/); + }); + it("falls back to original when validation detects structural loss", () => { const original = "Please make sure to use `the exact token` in the request."; const result = cavemanCompress(