fix(core): harden input handling and compression cleanup

Replace regex-based compression artifact cleanup with linear helpers to
avoid pathological backtracking and normalize whitespace safely.
Tighten request and response parsing in assess, Gemini translation, and
executor telemetry to avoid unsafe property access and invalid category
filtering.

Also add managed database backup support for legacy encrypted
connection migration, improve SQLite load error handling, and cover the
regressions with unit tests.
This commit is contained in:
diegosouzapw
2026-05-06 02:28:51 -03:00
parent 28b7172c0a
commit 171081dbbb
11 changed files with 414 additions and 44 deletions

View File

@@ -230,6 +230,11 @@ function attachToolNameMap<T>(payload: T, toolNameMap: Map<string, string> | nul
return copy;
}
function getRequestTargetModel(body: Record<string, unknown>): string {
const target = body.model;
return typeof target === "string" && target.length > 0 ? target : "unknown";
}
export class AntigravityExecutor extends BaseExecutor {
constructor() {
super("antigravity", PROVIDERS.antigravity);
@@ -635,7 +640,7 @@ export class AntigravityExecutor extends BaseExecutor {
log?.debug?.(
"TELEMETRY",
`[Antigravity] Execute - URL: ${url}, Model: ${model}, Target: ${(transformedBody as any)?.model || "unknown"}, RetryAttempt: ${retryAttemptsByUrl[urlIndex]}`
`[Antigravity] Execute - URL: ${url}, Model: ${model}, Target: ${getRequestTargetModel(transformedBody)}, RetryAttempt: ${retryAttemptsByUrl[urlIndex]}`
);
const response = await fetch(url, {

View File

@@ -204,17 +204,188 @@ export function applyRulesToText(
function cleanupArtifacts(text: string): string {
let result = text;
if (result.includes(" ")) result = result.replace(/[ \t]{2,}/g, " ");
if (/[\t ]+[,.;:!?]/.test(result)) result = result.replace(/[ \t]([,.;:!?])/g, "$1");
if (/[.!?]{2,}/.test(result)) result = result.replace(/([.!?]){2,}/g, "$1");
if (/[ \t]\n/.test(result)) result = result.replace(/[ \t]+$/gm, "");
if (hasRepeatedHorizontalWhitespace(result)) {
result = collapseHorizontalWhitespaceRuns(result);
}
result = removeHorizontalWhitespaceBeforePunctuation(result);
result = collapseRepeatedSentencePunctuation(result);
if (result.includes(" \n") || result.includes("\t\n")) {
result = stripLineTrailingHorizontalWhitespace(result);
}
if (result.endsWith(" ") || result.endsWith("\t")) result = result.trimEnd();
if (result.includes("\n\n\n")) result = result.replace(/\n{3,}/g, "\n\n");
if (result.startsWith("\n")) result = result.replace(/^\n+/, "");
if (result.endsWith("\n")) result = result.replace(/\n+$/, "");
if (result.includes("\n\n\n")) result = collapseExcessNewlines(result);
if (result.startsWith("\n")) result = trimLeadingNewlines(result);
if (result.endsWith("\n")) result = trimTrailingNewlines(result);
return result;
}
function isHorizontalWhitespace(char: string): boolean {
return char === " " || char === "\t";
}
function isSentencePunctuation(char: string): boolean {
return char === "." || char === "!" || char === "?";
}
function isCleanupPunctuation(char: string): boolean {
return (
char === "," || char === "." || char === ";" || char === ":" || char === "!" || char === "?"
);
}
function hasRepeatedHorizontalWhitespace(text: string): boolean {
let previousWasWhitespace = false;
for (const char of text) {
const currentIsWhitespace = isHorizontalWhitespace(char);
if (currentIsWhitespace && previousWasWhitespace) return true;
previousWasWhitespace = currentIsWhitespace;
}
return false;
}
function collapseHorizontalWhitespaceRuns(text: string): string {
let output = "";
let changed = false;
for (let index = 0; index < text.length; index++) {
const char = text[index];
if (!isHorizontalWhitespace(char)) {
output += char;
continue;
}
const start = index;
while (index + 1 < text.length && isHorizontalWhitespace(text[index + 1])) {
index++;
}
if (index > start) {
output += " ";
changed = true;
} else {
output += char;
}
}
return changed ? output : text;
}
function removeHorizontalWhitespaceBeforePunctuation(text: string): string {
let output = "";
let changed = false;
for (let index = 0; index < text.length; index++) {
const char = text[index];
if (!isHorizontalWhitespace(char)) {
output += char;
continue;
}
const start = index;
while (index + 1 < text.length && isHorizontalWhitespace(text[index + 1])) {
index++;
}
const nextChar = text[index + 1];
if (nextChar && isCleanupPunctuation(nextChar)) {
changed = true;
continue;
}
output += text.slice(start, index + 1);
}
return changed ? output : text;
}
function collapseRepeatedSentencePunctuation(text: string): string {
let output = "";
let changed = false;
for (let index = 0; index < text.length; index++) {
const char = text[index];
if (!isSentencePunctuation(char)) {
output += char;
continue;
}
let lastPunctuation = char;
const start = index;
while (index + 1 < text.length && isSentencePunctuation(text[index + 1])) {
index++;
lastPunctuation = text[index];
}
if (index > start) changed = true;
output += lastPunctuation;
}
return changed ? output : text;
}
function trimEndHorizontalWhitespace(text: string): string {
let end = text.length;
while (end > 0 && isHorizontalWhitespace(text[end - 1])) {
end--;
}
return end === text.length ? text : text.slice(0, end);
}
function stripLineTrailingHorizontalWhitespace(text: string): string {
const lines = text.split("\n");
let changed = false;
const cleanedLines = lines.map((line) => {
const cleaned = trimEndHorizontalWhitespace(line);
if (cleaned !== line) changed = true;
return cleaned;
});
return changed ? cleanedLines.join("\n") : text;
}
function collapseExcessNewlines(text: string): string {
let output = "";
let changed = false;
for (let index = 0; index < text.length; index++) {
const char = text[index];
if (char !== "\n") {
output += char;
continue;
}
const start = index;
while (index + 1 < text.length && text[index + 1] === "\n") {
index++;
}
const newlineCount = index - start + 1;
if (newlineCount > 2) {
output += "\n\n";
changed = true;
} else {
output += text.slice(start, index + 1);
}
}
return changed ? output : text;
}
function trimLeadingNewlines(text: string): string {
let start = 0;
while (start < text.length && text[start] === "\n") {
start++;
}
return start === 0 ? text : text.slice(start);
}
function trimTrailingNewlines(text: string): string {
let end = text.length;
while (end > 0 && text[end - 1] === "\n") {
end--;
}
return end === text.length ? text : text.slice(0, end);
}
function recapitalizeSentences(text: string): string {
return text.replace(/(^|[.!?][ \t]|\n[ \t]*)([a-z])/g, (_match, prefix: string, char: string) => {
return `${prefix}${char.toUpperCase()}`;

View File

@@ -26,12 +26,47 @@ function requireExactPresence(
) {
for (const item of originalItems) {
if (!compressed.includes(item)) {
const preview = item.replace(/\s+/g, " ").slice(0, 80);
const preview = collapseWhitespaceForPreview(item).slice(0, 80);
errors.push(`${label} changed or missing: ${preview}`);
}
}
}
function isPreviewWhitespace(char: string): boolean {
return (
char === " " ||
char === "\t" ||
char === "\n" ||
char === "\r" ||
char === "\f" ||
char === "\v"
);
}
function collapseWhitespaceForPreview(text: string): string {
let output = "";
let previousWasWhitespace = false;
let changed = false;
for (const char of text) {
if (isPreviewWhitespace(char)) {
if (!previousWasWhitespace) {
output += " ";
} else {
changed = true;
}
if (char !== " ") changed = true;
previousWasWhitespace = true;
continue;
}
output += char;
previousWasWhitespace = false;
}
return changed ? output : text;
}
export function validateCompression(original: string, compressed: string): ValidationResult {
const errors: string[] = [];
const warnings: string[] = [];

View File

@@ -58,7 +58,7 @@ type GeminiFunctionDeclaration = {
type GeminiRequest = {
model: string;
contents?: GeminiContent[];
[key: string]: any;
[key: string]: unknown;
generationConfig: GeminiGenerationConfig;
safetySettings: unknown;
systemInstruction?: GeminiContent;
@@ -81,7 +81,7 @@ type CloudCodeEnvelope = {
session_id?: string;
sessionId?: string;
contents?: GeminiContent[];
[key: string]: any;
[key: string]: unknown;
systemInstruction?: GeminiContent;
generationConfig: GeminiGenerationConfig;
tools?: Array<{
@@ -101,6 +101,15 @@ type GeminiToolNameOptions = {
functionResponseShape?: "result" | "output";
};
type OpenAIToolCallLike = {
thoughtSignature?: unknown;
thought_signature?: unknown;
function?: {
thoughtSignature?: unknown;
thought_signature?: unknown;
};
};
function buildChangedToolNameMap(toolNameMap: Map<string, string>): Map<string, string> | null {
const changedEntries = [...toolNameMap.entries()].filter(
([sanitizedName, originalName]) => sanitizedName !== originalName
@@ -108,16 +117,17 @@ function buildChangedToolNameMap(toolNameMap: Map<string, string>): Map<string,
return changedEntries.length > 0 ? new Map(changedEntries) : null;
}
function extractClientThoughtSignature(toolCall) {
function extractClientThoughtSignature(toolCall: unknown): string | null {
if (!toolCall || typeof toolCall !== "object") return null;
const candidate = toolCall as OpenAIToolCallLike;
return (
toolCall.thoughtSignature ||
toolCall.thought_signature ||
toolCall.function?.thoughtSignature ||
toolCall.function?.thought_signature ||
null
);
const signature =
candidate.thoughtSignature ||
candidate.thought_signature ||
candidate.function?.thoughtSignature ||
candidate.function?.thought_signature ||
null;
return typeof signature === "string" && signature.length > 0 ? signature : null;
}
// Core: Convert OpenAI request to Gemini format (base for all variants)

6
package-lock.json generated
View File

@@ -9720,9 +9720,9 @@
"license": "MIT"
},
"node_modules/ip-address": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz",
"integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==",
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.1.tgz",
"integrity": "sha512-1FMu8/N15Ck1BL551Jf42NYIoin2unWjLQ2Fze/DXryJRl5twqtwNHlO39qERGbIOcKYWHdgRryhOC+NG4eaLw==",
"license": "MIT",
"engines": {
"node": ">= 12"

View File

@@ -216,6 +216,7 @@
"dompurify": "^3.4.2",
"path-to-regexp": "^8.4.0",
"postcss": "^8.5.10",
"ip-address": "10.1.1",
"hono": "^4.12.14",
"@hono/node-server": "^1.19.13",
"react": "$react",

View File

@@ -1,8 +1,14 @@
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { Assessor } from "@/domain/assessment/assessor";
import { Categorizer } from "@/domain/assessment/categorizer";
import { SelfHealer } from "@/domain/assessment/selfHealer";
import { type AssessmentScope, type AssessmentTrigger } from "@/domain/assessment/types";
import {
type AssessmentScope,
type AssessmentTrigger,
type ModelCategory,
} from "@/domain/assessment/types";
import { validateBody } from "@/shared/validation/helpers";
const assessor = new Assessor(
process.env.OMNIROUTe_API_KEY ?? process.env.API_KEY ?? "",
@@ -12,11 +18,56 @@ const assessor = new Assessor(
const categorizer = new Categorizer();
const healer = new SelfHealer();
const modelCategories = new Set<ModelCategory>([
"coding",
"reasoning",
"reasoning_deep",
"chat",
"fast",
"vision",
"tool_call",
"structured_output",
]);
const assessmentScopeSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal("all") }),
z.object({ type: z.literal("provider"), providerId: z.string().min(1) }),
z.object({ type: z.literal("model"), modelId: z.string().min(1) }),
]);
const assessmentPostSchema = z.object({
scope: assessmentScopeSchema.optional().default({ type: "all" }),
trigger: z
.enum(["scheduled", "on_demand", "on_provider_change", "on_error", "startup"])
.optional()
.default("on_demand"),
});
type ModelListItem = { id: string };
function isModelCategory(value: string): value is ModelCategory {
return modelCategories.has(value as ModelCategory);
}
function isModelListItem(value: unknown): value is ModelListItem {
return (
!!value &&
typeof value === "object" &&
"id" in value &&
typeof (value as { id?: unknown }).id === "string"
);
}
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const scope: AssessmentScope = body.scope ?? { type: "all" };
const trigger: AssessmentTrigger = body.trigger ?? "on_demand";
const rawBody = await request.json();
const validation = validateBody(assessmentPostSchema, rawBody);
if (!validation.success) {
return NextResponse.json({ error: validation.error }, { status: 400 });
}
const scope: AssessmentScope = validation.data.scope;
const trigger: AssessmentTrigger = validation.data.trigger;
let models: Array<{ providerId: string; modelId: string }>;
@@ -63,7 +114,9 @@ export async function GET(request: NextRequest) {
let results = assessor.getAllAssessments();
if (status) results = results.filter((a) => a.status === status);
if (provider) results = results.filter((a) => a.providerId === provider);
if (category) results = results.filter((a) => a.categories.includes(category as any));
if (category && isModelCategory(category)) {
results = results.filter((a) => a.categories.includes(category));
}
return NextResponse.json({ models: results });
}
@@ -93,10 +146,12 @@ async function getAllModels(): Promise<Array<{ providerId: string; modelId: stri
Authorization: `Bearer ${process.env.OMNIROUTe_API_KEY ?? process.env.API_KEY ?? ""}`,
},
});
const data = await resp.json();
return (data.data ?? [])
.filter((m: any) => m.id?.startsWith("auto/"))
.map((m: any) => ({ providerId: "auto", modelId: m.id.replace("auto/", "") }));
const data = (await resp.json()) as { data?: unknown };
const models = Array.isArray(data.data) ? data.data : [];
return models
.filter(isModelListItem)
.filter((model) => model.id.startsWith("auto/"))
.map((model) => ({ providerId: "auto", modelId: model.id.replace("auto/", "") }));
} catch {
return [];
}

View File

@@ -16,7 +16,8 @@ import {
writeCallArtifact,
type CallLogArtifact,
} from "../usage/callLogArtifacts";
import { autoMigrateLegacyEncryptedConnections } from "./providers";
import { migrateLegacyEncryptedString } from "./encryption";
import { invalidateDbCache } from "./readCache";
type SqliteDatabase = import("better-sqlite3").Database;
type JsonRecord = Record<string, unknown>;
@@ -94,10 +95,16 @@ function isNativeSqliteLoadError(error: unknown): boolean {
message.includes("Module did not self-register") ||
message.includes("NODE_MODULE_VERSION") ||
message.includes("ERR_DLOPEN_FAILED") ||
(error as any)?.code === "ERR_DLOPEN_FAILED"
getErrorCode(error) === "ERR_DLOPEN_FAILED"
);
}
function getErrorCode(error: unknown): string | undefined {
if (!error || typeof error !== "object" || !("code" in error)) return undefined;
const code = (error as { code?: unknown }).code;
return typeof code === "string" ? code : undefined;
}
function createNativeSqliteLoadError(error: unknown): Error {
const message = error instanceof Error ? error.message : String(error);
const detail =
@@ -105,10 +112,10 @@ function createNativeSqliteLoadError(error: unknown): Error {
"This usually happens after switching Node.js versions without rebuilding native modules. " +
"Run `npm rebuild better-sqlite3` in the OmniRoute project and start again. " +
`Original error: ${message}`;
const wrapped = new Error(detail);
const wrapped = new Error(detail) as Error & { cause?: unknown; code?: string };
wrapped.name = "NativeSqliteLoadError";
(wrapped as any).cause = error;
(wrapped as any).code = (error as any)?.code || "ERR_DLOPEN_FAILED";
wrapped.cause = error;
wrapped.code = getErrorCode(error) || "ERR_DLOPEN_FAILED";
return wrapped;
}
@@ -967,7 +974,7 @@ function shouldRunStartupDbHealthCheck(): boolean {
return !isAutomatedTestProcess();
}
function createHealthCheckBackup(db: SqliteDatabase): boolean {
function createManagedDbBackup(db: SqliteDatabase, reason: string): boolean {
const isTest = isAutomatedTestProcess();
if (isTest) return false;
@@ -978,19 +985,72 @@ function createHealthCheckBackup(db: SqliteDatabase): boolean {
}
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
const backupPath = path.join(backupDir, `db_${timestamp}_health-check-repair.sqlite`);
const backupPath = path.join(backupDir, `db_${timestamp}_${reason}.sqlite`);
const escapedBackupPath = backupPath.replace(/'/g, "''");
db.exec(`VACUUM INTO '${escapedBackupPath}'`);
console.log(`[DB] Health-check backup created: ${backupPath}`);
console.log(`[DB] Backup created (${reason}): ${backupPath}`);
return true;
} catch (error: unknown) {
const message = error instanceof Error ? error.message : String(error);
console.warn("[DB] Failed to create health-check backup:", message);
console.warn(`[DB] Failed to create ${reason} backup:`, message);
return false;
}
}
function createHealthCheckBackup(db: SqliteDatabase): boolean {
return createManagedDbBackup(db, "health-check-repair");
}
function autoMigrateLegacyEncryptedConnections(db: SqliteDatabase): number {
const rows = db.prepare("SELECT * FROM provider_connections").all() as JsonRecord[];
const updateStmt = db.prepare(
"UPDATE provider_connections SET api_key = @apiKey, id_token = @idToken, access_token = @accessToken, refresh_token = @refreshToken, updated_at = @updatedAt WHERE id = @id"
);
const encryptedFields = ["apiKey", "idToken", "accessToken", "refreshToken"] as const;
let migratedCount = 0;
let backupCreated = false;
for (const row of rows) {
const camelRow = rowToCamel(row);
if (!camelRow) continue;
let updatedRow = false;
for (const field of encryptedFields) {
if (typeof camelRow[field] !== "string") continue;
const { updated, value } = migrateLegacyEncryptedString(camelRow[field]);
if (updated) {
camelRow[field] = value;
updatedRow = true;
}
}
if (!updatedRow) continue;
if (!backupCreated) {
createManagedDbBackup(db, "legacy-encryption-migration");
backupCreated = true;
}
updateStmt.run({
id: camelRow.id,
apiKey: camelRow.apiKey ?? null,
idToken: camelRow.idToken ?? null,
accessToken: camelRow.accessToken ?? null,
refreshToken: camelRow.refreshToken ?? null,
updatedAt: new Date().toISOString(),
});
migratedCount++;
}
if (migratedCount > 0) {
invalidateDbCache("connections");
console.log(`[DB] Auto-migrated ${migratedCount} connection(s) to new static-salt encryption.`);
}
return migratedCount;
}
let dbHealthCheckTimer: NodeJS.Timeout | null = null;
function getDbHealthCheckIntervalMs(): number {
@@ -1285,7 +1345,7 @@ export function getDbInstance(): SqliteDatabase {
// Re-encrypt any tokens using the legacy dynamic salt to canonical static salt
try {
autoMigrateLegacyEncryptedConnections();
autoMigrateLegacyEncryptedConnections(db);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
console.error(`[DB] Legacy encryption migration failed: ${message}`);
@@ -1577,10 +1637,11 @@ export function runManualVacuum(): { success: boolean; duration: number; error?:
const duration = Date.now() - startTime;
console.log(`[DB] Manual VACUUM completed in ${duration}ms`);
return { success: true, duration };
} catch (err: any) {
} catch (err: unknown) {
const duration = Date.now() - startTime;
console.error("[DB] Manual VACUUM failed:", err);
return { success: false, duration, error: err.message };
const message = err instanceof Error ? err.message : String(err);
return { success: false, duration, error: message };
}
}

View File

@@ -126,7 +126,7 @@ function mergeDatabaseSettingsNamespace(
for (const section of DATABASE_SETTINGS_SECTIONS) {
const defaultSection = DEFAULT_DATABASE_SETTINGS[section] as Record<string, unknown>;
const sectionTarget = target[section] as Record<string, unknown>;
const flatAliases = LEGACY_FLAT_KEYS[section];
const flatAliases = LEGACY_FLAT_KEYS[section] as Partial<Record<string, string[]>>;
for (const key of Object.keys(defaultSection)) {
for (const alias of flatAliases[key] ?? []) {

View File

@@ -229,4 +229,29 @@ describe("caveman engine", () => {
});
assert.ok(result.stats.durationMs < 25, `Expected <25ms, got ${result.stats.durationMs}ms`);
});
it("cleans whitespace and punctuation artifacts without regex backtracking", () => {
const body = {
messages: [
{
role: "user",
content: "\n\nPlease\t make sure to keep this stable !!! \n\n\n\nThank you.",
},
],
};
const result = cavemanCompress(body, {
enabled: true,
compressRoles: ["user"],
skipRules: [],
minMessageLength: 0,
preservePatterns: [],
});
const text = result.body.messages[0].content as string;
assert.doesNotMatch(text, /^\n/);
assert.doesNotMatch(text, /\n$/);
assert.doesNotMatch(text, /\n\n\n/);
assert.doesNotMatch(text, /[ \t]+[,.!?;:]/);
assert.doesNotMatch(text, /[ \t]{2,}/);
});
});

View File

@@ -117,6 +117,13 @@ describe("compression preservation and validation", () => {
assert.ok(validation.errors.length >= 2);
});
it("formats validation previews with linear whitespace collapsing", () => {
const original = `Use \`${"\t".repeat(5000)}exact_token\` in the request.`;
const validation = validateCompression(original, "Use token.");
assert.equal(validation.valid, false);
assert.match(validation.errors.join("\n"), /inline code changed or missing: ` exact_token`/);
});
it("falls back to original when validation detects structural loss", () => {
const original = "Please make sure to use `the exact token` in the request.";
const result = cavemanCompress(