mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-13 18:52:18 +03:00
fix(memory): sanitize FTS5 MATCH in vector hybrid search too
searchHybrid in vectorStore.ts runs its own raw FTS5 MATCH — the earlier retrieval.ts fix missed it, so hybrid queries with punctuation still threw 'fts5: syntax error'. Extracted the sanitizer to src/lib/memory/ftsQuery.ts (no import cycle) and applied it at all three MATCH sites; punctuation-only queries now yield a non-matching '""' phrase instead of an error.
This commit is contained in:
14
src/lib/memory/ftsQuery.ts
Normal file
14
src/lib/memory/ftsQuery.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Sanitize a free-text query into an FTS5 MATCH expression.
|
||||
* Strips FTS5 syntax characters (?, !, ", *, :, parentheses, brackets, etc.)
|
||||
* and quotes each whitespace-separated term so natural-language queries with
|
||||
* punctuation don't raise "fts5: syntax error" from SQLite.
|
||||
*/
|
||||
export function toFts5MatchQuery(query: string): string {
|
||||
const terms = query
|
||||
.split(/\s+/)
|
||||
.map((term) => term.replace(/["*():\[\]{}!?\^~+.-]/g, ""))
|
||||
.filter((term) => term.length > 0);
|
||||
if (terms.length === 0) return '""';
|
||||
return terms.map((term) => `"${term}"`).join(" AND ");
|
||||
}
|
||||
@@ -84,13 +84,7 @@ function fetchMemoriesByIds(ids: string[]): Memory[] {
|
||||
* each whitespace-separated term so natural-language queries with punctuation
|
||||
* don't raise "fts5: syntax error".
|
||||
*/
|
||||
export function toFts5MatchQuery(query: string): string {
|
||||
const terms = query
|
||||
.split(/\s+/)
|
||||
.map((term) => term.replace(/["*():\[\]{}!?\^~+-]/g, ""))
|
||||
.filter((term) => term.length > 0);
|
||||
return terms.map((term) => `"${term}"`).join(" AND ");
|
||||
}
|
||||
export { toFts5MatchQuery } from "./ftsQuery";
|
||||
|
||||
interface FtsColConfig { apiKeyCol: string;
|
||||
expiresCol: string;
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
countMemoryReindexPending,
|
||||
} from "../localDb";
|
||||
import { getDbInstance } from "../db/core";
|
||||
import { toFts5MatchQuery } from "./ftsQuery";
|
||||
import { logger } from "../../../open-sse/utils/logger.ts";
|
||||
import { sanitizeErrorMessage } from "../../../open-sse/utils/error.ts";
|
||||
|
||||
@@ -344,7 +345,7 @@ class VectorStoreImpl implements VectorStore {
|
||||
encodeVector(vector),
|
||||
{ apiKeyId: apiKeyId ?? null },
|
||||
k,
|
||||
queryText,
|
||||
toFts5MatchQuery(queryText),
|
||||
k,
|
||||
k,
|
||||
) as Array<{
|
||||
|
||||
@@ -21,11 +21,11 @@ describe("memory retrieval toFts5MatchQuery", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps punctuation-only input as a single quoted term", () => {
|
||||
assert.strictEqual(toFts5MatchQuery("???! ..."), '"..."');
|
||||
it("keeps punctuation-only input as a non-matching quoted phrase", () => {
|
||||
assert.strictEqual(toFts5MatchQuery("???! ..."), '""');
|
||||
});
|
||||
|
||||
it("returns empty string for empty input", () => {
|
||||
assert.strictEqual(toFts5MatchQuery(""), "");
|
||||
it("returns non-matching quoted phrase for empty input", () => {
|
||||
assert.strictEqual(toFts5MatchQuery(""), '""');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user