diff --git a/src/lib/memory/ftsQuery.ts b/src/lib/memory/ftsQuery.ts new file mode 100644 index 0000000000..f99af4f703 --- /dev/null +++ b/src/lib/memory/ftsQuery.ts @@ -0,0 +1,14 @@ +/** + * Sanitize a free-text query into an FTS5 MATCH expression. + * Strips FTS5 syntax characters (?, !, ", *, :, parentheses, brackets, etc.) + * and quotes each whitespace-separated term so natural-language queries with + * punctuation don't raise "fts5: syntax error" from SQLite. + */ +export function toFts5MatchQuery(query: string): string { + const terms = query + .split(/\s+/) + .map((term) => term.replace(/["*():\[\]{}!?\^~+.-]/g, "")) + .filter((term) => term.length > 0); + if (terms.length === 0) return '""'; + return terms.map((term) => `"${term}"`).join(" AND "); +} diff --git a/src/lib/memory/retrieval.ts b/src/lib/memory/retrieval.ts index bbb5e7c3da..eec63ee6aa 100644 --- a/src/lib/memory/retrieval.ts +++ b/src/lib/memory/retrieval.ts @@ -84,13 +84,7 @@ function fetchMemoriesByIds(ids: string[]): Memory[] { * each whitespace-separated term so natural-language queries with punctuation * don't raise "fts5: syntax error". */ -export function toFts5MatchQuery(query: string): string { - const terms = query - .split(/\s+/) - .map((term) => term.replace(/["*():\[\]{}!?\^~+-]/g, "")) - .filter((term) => term.length > 0); - return terms.map((term) => `"${term}"`).join(" AND "); -} +export { toFts5MatchQuery } from "./ftsQuery"; interface FtsColConfig { apiKeyCol: string; expiresCol: string; diff --git a/src/lib/memory/vectorStore.ts b/src/lib/memory/vectorStore.ts index 92f706a3d4..87f3ce187c 100644 --- a/src/lib/memory/vectorStore.ts +++ b/src/lib/memory/vectorStore.ts @@ -16,6 +16,7 @@ import { countMemoryReindexPending, } from "../localDb"; import { getDbInstance } from "../db/core"; +import { toFts5MatchQuery } from "./ftsQuery"; import { logger } from "../../../open-sse/utils/logger.ts"; import { sanitizeErrorMessage } from "../../../open-sse/utils/error.ts"; @@ -344,7 +345,7 @@ class VectorStoreImpl implements VectorStore { encodeVector(vector), { apiKeyId: apiKeyId ?? null }, k, - queryText, + toFts5MatchQuery(queryText), k, k, ) as Array<{ diff --git a/tests/unit/memory/fts5-query-sanitize.test.ts b/tests/unit/memory/fts5-query-sanitize.test.ts index ca8ee46a46..51a0b96404 100644 --- a/tests/unit/memory/fts5-query-sanitize.test.ts +++ b/tests/unit/memory/fts5-query-sanitize.test.ts @@ -21,11 +21,11 @@ describe("memory retrieval toFts5MatchQuery", () => { ); }); - it("keeps punctuation-only input as a single quoted term", () => { - assert.strictEqual(toFts5MatchQuery("???! ..."), '"..."'); + it("keeps punctuation-only input as a non-matching quoted phrase", () => { + assert.strictEqual(toFts5MatchQuery("???! ..."), '""'); }); - it("returns empty string for empty input", () => { - assert.strictEqual(toFts5MatchQuery(""), ""); + it("returns non-matching quoted phrase for empty input", () => { + assert.strictEqual(toFts5MatchQuery(""), '""'); }); });