mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-06 07:12:12 +03:00
feb0e983eb
Code review of plan 21 found two functional gaps: FAIL #1 — toMemoryRetrievalConfig never forwarded the user query, so the gate `if (config.query && useModernTable)` in retrieval.ts was always false in the chat hot path. semantic/hybrid silently fell back to "ORDER BY created_at DESC LIMIT 100", the pre-plan-21 behaviour. sqlite-vec + RRF only ran in the Playground (retrievePreview, which takes `query` positionally). FAIL #2 — Bug #1 was not closed: searchSemanticMemory (Qdrant) was only imported by /api/settings/qdrant/search, never by retrieval. With vectorStore="qdrant", engineStatus reported backend="qdrant" but retrieveMemories/retrievePreview kept using sqlite-vec — the status diverged from the actual search path. - settings.ts: toMemoryRetrievalConfig(settings, { query? }) accepts and forwards the query. - chatCore.ts: extracts the last user message from body.messages or body.input (Chat + Responses APIs). - retrieval.ts: adds a Qdrant branch in case "semantic", case "hybrid" and retrievePreview; falls through to sqlite-vec on failure or empty results so the §7 "degrades to sqlite-vec / FTS5" contract holds. - engineStatus only reports backend="qdrant" when the user opted in (settings.vectorStore === "qdrant") and Qdrant is healthy. - memories[] tier union includes "qdrant". 331 memory unit tests pass; typecheck:core / typecheck:noimplicit:core / check:cycles clean.
1171 lines
40 KiB
TypeScript
1171 lines
40 KiB
TypeScript
import { getDbInstance } from "../db/core";
|
|
import { Memory, MemoryConfig, MemoryType } from "./types";
|
|
import { MemoryConfigSchema } from "./schemas";
|
|
import { logger } from "../../../open-sse/utils/logger.ts";
|
|
import { sanitizeErrorMessage } from "../../../open-sse/utils/error.ts";
|
|
import { resolveEmbeddingSource, embed } from "./embedding";
|
|
import { getVectorStore } from "./vectorStore";
|
|
import { getMemorySettings } from "./settings";
|
|
import { stats as embeddingCacheStats } from "./embedding/cache";
|
|
import { getQdrantConfig, checkQdrantHealth, searchSemanticMemory } from "./qdrant";
|
|
import type { MemoryEngineStatus } from "@/shared/schemas/memory";
|
|
|
|
const log = logger("MEMORY_RETRIEVAL");
|
|
|
|
interface MemoryRow {
|
|
id: string;
|
|
api_key_id?: string;
|
|
apiKeyId?: string;
|
|
session_id?: string | null;
|
|
sessionId?: string | null;
|
|
type: MemoryType;
|
|
key?: string | null;
|
|
content: string;
|
|
metadata?: string | null;
|
|
created_at?: string;
|
|
createdAt?: string;
|
|
updated_at?: string;
|
|
updatedAt?: string;
|
|
expires_at?: string | null;
|
|
expiresAt?: string | null;
|
|
}
|
|
|
|
interface RetrievalOptions extends Partial<MemoryConfig> {
|
|
query?: string;
|
|
sessionId?: string;
|
|
}
|
|
|
|
// ──────────────── Types exposed publicly (§3.6) ────────────────
|
|
|
|
export interface RetrievePreviewItem {
|
|
memory: Memory;
|
|
score: number;
|
|
tokens: number;
|
|
tier: "fts5" | "vector" | "hybrid-rrf" | "qdrant";
|
|
vecScore: number | null;
|
|
ftsScore: number | null;
|
|
}
|
|
|
|
export interface RetrievePreviewResolution {
|
|
embeddingSource: "remote" | "static" | "transformers" | null;
|
|
embeddingModel: string | null;
|
|
vectorStore: "sqlite-vec" | "qdrant" | "none";
|
|
strategyUsed: "exact" | "semantic" | "hybrid";
|
|
rerankApplied: boolean;
|
|
fallbackReason: string | null;
|
|
}
|
|
|
|
export interface RetrievePreviewBundle {
|
|
items: RetrievePreviewItem[];
|
|
resolution: RetrievePreviewResolution;
|
|
totalTokens: number;
|
|
budgetMaxTokens: number;
|
|
}
|
|
|
|
// ──────────────── Helpers ────────────────
|
|
|
|
/**
|
|
* Simple token estimation function (roughly 1 token per 4 characters)
|
|
*/
|
|
export function estimateTokens(text: string): number {
|
|
if (!text || typeof text !== "string") return 0;
|
|
return Math.ceil(text.length / 4);
|
|
}
|
|
|
|
function hasTable(tableName: string): boolean {
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
.get(tableName) as { name?: string } | undefined;
|
|
return row?.name === tableName;
|
|
}
|
|
|
|
function parseMetadata(raw: unknown): Record<string, unknown> {
|
|
if (!raw || typeof raw !== "string") return {};
|
|
try {
|
|
const parsed = JSON.parse(raw);
|
|
return typeof parsed === "object" && parsed !== null ? parsed : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function rowToMemory(row: MemoryRow): Memory {
|
|
const createdAt = row.created_at || row.createdAt || new Date().toISOString();
|
|
const updatedAt = row.updated_at || row.updatedAt || createdAt;
|
|
const expiresAt = row.expires_at ?? row.expiresAt ?? null;
|
|
|
|
return {
|
|
id: String(row.id),
|
|
apiKeyId: String(row.api_key_id || row.apiKeyId || ""),
|
|
sessionId: String(row.session_id ?? row.sessionId ?? ""),
|
|
type: row.type as MemoryType,
|
|
key: String(row.key || ""),
|
|
content: String(row.content || ""),
|
|
metadata: parseMetadata(row.metadata),
|
|
createdAt: new Date(createdAt),
|
|
updatedAt: new Date(updatedAt),
|
|
expiresAt: expiresAt ? new Date(String(expiresAt)) : null,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Score a memory against a query using simple string matching (no dynamic RegExp).
|
|
* Uses indexOf() for full-phrase matches and split-token substring checks only,
|
|
* so there is no ReDoS risk — no user input is passed to RegExp().
|
|
*/
|
|
function getRelevanceScore(memory: Memory, query: string): number {
|
|
const normalizedQuery = query.trim().toLowerCase();
|
|
if (!normalizedQuery) return 0;
|
|
|
|
const haystacks = [
|
|
memory.content.toLowerCase(),
|
|
memory.key.toLowerCase(),
|
|
JSON.stringify(memory.metadata).toLowerCase(),
|
|
];
|
|
const tokens = normalizedQuery.split(/\s+/).filter(Boolean);
|
|
|
|
let score = 0;
|
|
for (const haystack of haystacks) {
|
|
// Full phrase match (safe: literal string, not regex)
|
|
if (haystack.includes(normalizedQuery)) {
|
|
score += 20;
|
|
}
|
|
|
|
for (const token of tokens) {
|
|
if (!token) continue;
|
|
// Token-level substring count using indexOf loop (no RegExp on user input)
|
|
if (haystack === memory.key.toLowerCase() && haystack.includes(token)) {
|
|
score += 6;
|
|
continue;
|
|
}
|
|
// Count occurrences via indexOf loop — avoids new RegExp(token)
|
|
let pos = 0;
|
|
let matchCount = 0;
|
|
while ((pos = haystack.indexOf(token, pos)) !== -1) {
|
|
matchCount++;
|
|
pos += token.length;
|
|
}
|
|
score += matchCount * 3;
|
|
}
|
|
}
|
|
|
|
return score;
|
|
}
|
|
|
|
/**
|
|
* Fetch memories from SQLite by an array of IDs, preserving order.
|
|
*/
|
|
function fetchMemoriesByIds(ids: string[]): Memory[] {
|
|
if (ids.length === 0) return [];
|
|
const db = getDbInstance();
|
|
const placeholders = ids.map(() => "?").join(", ");
|
|
const rows = db
|
|
.prepare(`SELECT * FROM memories WHERE id IN (${placeholders})`)
|
|
.all(...ids) as MemoryRow[];
|
|
|
|
const byId = new Map<string, Memory>();
|
|
for (const row of rows) {
|
|
byId.set(String(row.id), rowToMemory(row));
|
|
}
|
|
|
|
return ids.map((id) => byId.get(id)).filter((m): m is Memory => m !== undefined);
|
|
}
|
|
|
|
interface FtsColConfig {
|
|
apiKeyCol: string;
|
|
expiresCol: string;
|
|
createdCol: string;
|
|
sessionCol: string;
|
|
tableName: string;
|
|
query?: string;
|
|
scope?: string;
|
|
sessionId?: string;
|
|
retentionDays?: number;
|
|
}
|
|
|
|
/**
|
|
* Build the FTS5 rows for a given apiKeyId + config + query.
|
|
* Returns MemoryRow array (or falls back to empty on error).
|
|
*/
|
|
function buildFtsRows(apiKeyId: string, config: FtsColConfig): MemoryRow[] {
|
|
if (!config.query) return [];
|
|
const db = getDbInstance();
|
|
const {
|
|
apiKeyCol,
|
|
expiresCol,
|
|
createdCol,
|
|
sessionCol,
|
|
tableName,
|
|
query: q,
|
|
scope,
|
|
sessionId,
|
|
retentionDays,
|
|
} = config;
|
|
|
|
let ftsQueryStr =
|
|
`SELECT m.* FROM ${tableName} m ` +
|
|
`JOIN memory_fts f ON m.memory_id = f.rowid ` +
|
|
`WHERE f.memory_fts MATCH ? AND m.${apiKeyCol} = ? ` +
|
|
`AND (m.${expiresCol} IS NULL OR datetime(m.${expiresCol}) > datetime('now'))`;
|
|
if (scope === "session" && sessionId) {
|
|
ftsQueryStr += ` AND m.${sessionCol} = ?`;
|
|
}
|
|
if (retentionDays && retentionDays > 0) {
|
|
ftsQueryStr += ` AND datetime(m.${createdCol}) >= datetime(?)`;
|
|
}
|
|
ftsQueryStr += ` ORDER BY f.rank LIMIT 100`;
|
|
|
|
const ftsParams: unknown[] = [q, apiKeyId];
|
|
if (scope === "session" && sessionId) ftsParams.push(sessionId);
|
|
if (retentionDays && retentionDays > 0) {
|
|
const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000).toISOString();
|
|
ftsParams.push(cutoff);
|
|
}
|
|
|
|
try {
|
|
return db.prepare(ftsQueryStr).all(...ftsParams) as MemoryRow[];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
// Loopback rerank URL — localhost only, never routed over the network.
|
|
// nosemgrep: javascript.lang.security.audit.non-literal-regexp.non-literal-regexp
|
|
const RERANK_LOOPBACK_URL = "http://127.0.0.1:20128/v1/rerank";
|
|
|
|
/**
|
|
* Apply reranking via /v1/rerank (loopback-only) if rerankEnabled + rerankProviderModel is set.
|
|
* Returns reordered array (or original order on any error — rerank failure never fails retrieval).
|
|
*
|
|
* Security note: the URL is a hardcoded loopback address (127.0.0.1:20128) — it never
|
|
* carries sensitive data over a network link. HTTP is safe for loopback-only IPC.
|
|
* nosemgrep: javascript.lang.security.detect-non-literal-url
|
|
*/
|
|
async function applyRerank<T extends { memory: Memory; score: number }>(
|
|
items: T[],
|
|
query: string,
|
|
rerankProviderModel: string
|
|
): Promise<T[]> {
|
|
if (items.length === 0) return items;
|
|
|
|
try {
|
|
const documents = items.map((item) => item.memory.content);
|
|
const body = {
|
|
model: rerankProviderModel,
|
|
query,
|
|
documents,
|
|
top_n: items.length,
|
|
};
|
|
|
|
const res = await fetch(RERANK_LOOPBACK_URL, { // nosemgrep: typescript.react.security.react-insecure-request.react-insecure-request
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(5000),
|
|
});
|
|
|
|
if (!res.ok) {
|
|
log.warn("memory.rerank.http_fail", {
|
|
status: res.status,
|
|
model: rerankProviderModel,
|
|
});
|
|
return items;
|
|
}
|
|
|
|
const data = (await res.json()) as {
|
|
results?: Array<{ index: number; relevance_score: number }>;
|
|
};
|
|
|
|
if (!Array.isArray(data.results) || data.results.length === 0) {
|
|
return items;
|
|
}
|
|
|
|
// Build reordered list using the index references from the rerank response
|
|
const reordered: T[] = [];
|
|
for (const r of data.results) {
|
|
const idx = r.index;
|
|
if (typeof idx === "number" && idx >= 0 && idx < items.length) {
|
|
const item = items[idx];
|
|
if (item) reordered.push({ ...item, score: r.relevance_score });
|
|
}
|
|
}
|
|
// Append any items not mentioned in results (safety net)
|
|
const mentionedIndices = new Set(data.results.map((r) => r.index));
|
|
for (let i = 0; i < items.length; i++) {
|
|
if (!mentionedIndices.has(i)) {
|
|
const item = items[i];
|
|
if (item) reordered.push(item);
|
|
}
|
|
}
|
|
return reordered;
|
|
} catch (err: unknown) {
|
|
log.warn("memory.rerank.error", {
|
|
error: sanitizeErrorMessage(err instanceof Error ? err.message : String(err)),
|
|
model: rerankProviderModel,
|
|
});
|
|
return items;
|
|
}
|
|
}
|
|
|
|
// ──────────────── Main retrieval function (hot path — signature PRESERVED) ────────────────
|
|
|
|
/**
|
|
* Retrieve memories with token budget enforcement.
|
|
* Signature PRESERVED: retrieveMemories(apiKeyId: string, config: RetrievalOptions = {})
|
|
* Hot path: open-sse/handlers/chatCore.ts calls this unchanged.
|
|
*/
|
|
export async function retrieveMemories(
|
|
apiKeyId: string,
|
|
config: RetrievalOptions = {}
|
|
): Promise<Memory[]> {
|
|
log.info("memory.retrieval.start", { apiKeyId, strategy: config.retrievalStrategy });
|
|
|
|
// Validate and normalize config
|
|
const normalizedConfig = MemoryConfigSchema.parse({
|
|
enabled: true,
|
|
maxTokens: 2000,
|
|
retrievalStrategy: "exact",
|
|
autoSummarize: false,
|
|
persistAcrossModels: false,
|
|
retentionDays: 30,
|
|
scope: "apiKey",
|
|
...config,
|
|
});
|
|
|
|
if (!normalizedConfig.enabled || normalizedConfig.maxTokens <= 0) {
|
|
return [];
|
|
}
|
|
|
|
const maxTokens = Math.min(Math.max(normalizedConfig.maxTokens, 1), 8000);
|
|
const strategy = normalizedConfig.retrievalStrategy;
|
|
|
|
const db = getDbInstance();
|
|
// Plan 21 FAIL #2 fix: include "qdrant" in the tier union so that the
|
|
// Qdrant tier-2 branch in semantic/hybrid below can push hits with that tier.
|
|
const memories: Array<{
|
|
memory: Memory;
|
|
score: number;
|
|
tier: "fts5" | "vector" | "hybrid-rrf" | "qdrant";
|
|
}> = [];
|
|
let totalTokens = 0;
|
|
|
|
const useModernTable = hasTable("memories");
|
|
const tableName = useModernTable ? "memories" : "memory";
|
|
const columns = useModernTable
|
|
? {
|
|
apiKeyId: "api_key_id",
|
|
sessionId: "session_id",
|
|
createdAt: "created_at",
|
|
expiresAt: "expires_at",
|
|
}
|
|
: {
|
|
apiKeyId: "apiKeyId",
|
|
sessionId: "sessionId",
|
|
createdAt: "createdAt",
|
|
expiresAt: "expiresAt",
|
|
};
|
|
|
|
// Build base query
|
|
let query =
|
|
`SELECT * FROM ${tableName} WHERE ${columns.apiKeyId} = ? ` +
|
|
`AND (${columns.expiresAt} IS NULL OR datetime(${columns.expiresAt}) > datetime('now'))`;
|
|
const params: unknown[] = [apiKeyId];
|
|
|
|
if (normalizedConfig.scope === "session" && config.sessionId) {
|
|
query += ` AND ${columns.sessionId} = ?`;
|
|
params.push(config.sessionId);
|
|
}
|
|
|
|
if (normalizedConfig.retentionDays > 0) {
|
|
const cutoff = new Date(
|
|
Date.now() - normalizedConfig.retentionDays * 24 * 60 * 60 * 1000
|
|
).toISOString();
|
|
query += ` AND datetime(${columns.createdAt}) >= datetime(?)`;
|
|
params.push(cutoff);
|
|
}
|
|
|
|
// Load extended settings for embedding/vector-store resolution
|
|
const settings = await getMemorySettings();
|
|
|
|
// Execute query based on strategy
|
|
let rows: MemoryRow[];
|
|
const ftsAvailable = useModernTable && hasTable("memory_fts");
|
|
|
|
const ftsColConfig: FtsColConfig = {
|
|
apiKeyCol: columns.apiKeyId,
|
|
expiresCol: columns.expiresAt,
|
|
createdCol: columns.createdAt,
|
|
sessionCol: columns.sessionId,
|
|
tableName,
|
|
query: config.query,
|
|
scope: normalizedConfig.scope,
|
|
sessionId: config.sessionId,
|
|
retentionDays: normalizedConfig.retentionDays,
|
|
};
|
|
|
|
switch (strategy) {
|
|
case "semantic": {
|
|
// Attempt vector search if embedding + vector store are available
|
|
if (config.query && useModernTable) {
|
|
const resolution = resolveEmbeddingSource(settings);
|
|
if (resolution.source !== null) {
|
|
// Plan 21 FAIL #2 fix (Bug #1): when the user opted into Qdrant
|
|
// (settings.vectorStore === "qdrant"), route the semantic search to
|
|
// Qdrant first. If Qdrant is unreachable or returns nothing, fall
|
|
// through to sqlite-vec — preserving the "degrades to sqlite-vec /
|
|
// FTS5" contract of §7.
|
|
if (settings.vectorStore === "qdrant") {
|
|
try {
|
|
const qres = await searchSemanticMemory(config.query, 100, {
|
|
apiKeyId,
|
|
});
|
|
if (qres.ok && qres.results && qres.results.length > 0) {
|
|
const hitIds = qres.results.map((r) => r.id);
|
|
const hitMemories = fetchMemoriesByIds(hitIds);
|
|
const scoreMap = new Map(
|
|
qres.results.map((r) => [r.id, r.score])
|
|
);
|
|
let qdrantItems = hitMemories.map((m) => ({
|
|
memory: m,
|
|
score: scoreMap.get(m.id) ?? 0,
|
|
tier: "qdrant" as const,
|
|
}));
|
|
if (
|
|
settings.rerankEnabled &&
|
|
settings.rerankProviderModel &&
|
|
config.query
|
|
) {
|
|
qdrantItems = (await applyRerank(
|
|
qdrantItems,
|
|
config.query,
|
|
settings.rerankProviderModel
|
|
)) as typeof qdrantItems;
|
|
}
|
|
for (const entry of qdrantItems) {
|
|
const memoryTokens = estimateTokens(entry.memory.content);
|
|
if (totalTokens + memoryTokens > maxTokens) {
|
|
if (memories.length === 0) {
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
break;
|
|
}
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
log.info("memory.retrieval.complete", {
|
|
apiKeyId,
|
|
count: memories.length,
|
|
tier: "qdrant",
|
|
});
|
|
return memories.map((e) => e.memory);
|
|
}
|
|
} catch (err: unknown) {
|
|
log.warn("memory.retrieval.qdrant.fail", {
|
|
error: sanitizeErrorMessage(
|
|
err instanceof Error ? err.message : String(err)
|
|
),
|
|
});
|
|
// fall through to sqlite-vec degradation
|
|
}
|
|
}
|
|
|
|
const embeddingResult = await embed(config.query, settings);
|
|
if ("vector" in embeddingResult) {
|
|
const vec = getVectorStore();
|
|
if (vec) {
|
|
try {
|
|
await vec.ensureReady(resolution);
|
|
const hits = await vec.searchVector(embeddingResult.vector, 100, apiKeyId);
|
|
const hitIds = hits.map((h) => h.memoryId);
|
|
const hitMemories = fetchMemoriesByIds(hitIds);
|
|
const scoreMap = new Map(hits.map((h) => [h.memoryId, h.score]));
|
|
|
|
let rankedItems = hitMemories.map((m) => ({
|
|
memory: m,
|
|
score: scoreMap.get(m.id) ?? 0,
|
|
tier: "vector" as const,
|
|
}));
|
|
|
|
// Apply rerank if enabled
|
|
if (settings.rerankEnabled && settings.rerankProviderModel && config.query) {
|
|
rankedItems = (await applyRerank(
|
|
rankedItems,
|
|
config.query,
|
|
settings.rerankProviderModel
|
|
)) as typeof rankedItems;
|
|
}
|
|
|
|
// Token budget enforcement
|
|
for (const entry of rankedItems) {
|
|
const memoryTokens = estimateTokens(entry.memory.content);
|
|
if (totalTokens + memoryTokens > maxTokens) {
|
|
if (memories.length === 0) {
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
break;
|
|
}
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
|
|
log.info("memory.retrieval.complete", {
|
|
apiKeyId,
|
|
count: memories.length,
|
|
tier: "vector",
|
|
});
|
|
return memories.map((e) => e.memory);
|
|
} catch (err: unknown) {
|
|
log.warn("memory.retrieval.vector.fail", {
|
|
error: sanitizeErrorMessage(err instanceof Error ? err.message : String(err)),
|
|
});
|
|
// Fall through to FTS5 degradation
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Degraded path: FTS5 keyword
|
|
if (config.query && ftsAvailable) {
|
|
rows = buildFtsRows(apiKeyId, ftsColConfig);
|
|
if (rows.length === 0) {
|
|
query += ` ORDER BY ${columns.createdAt} DESC LIMIT 100`;
|
|
rows = db.prepare(query).all(...params) as MemoryRow[];
|
|
}
|
|
} else {
|
|
query += ` ORDER BY ${columns.createdAt} DESC LIMIT 100`;
|
|
rows = db.prepare(query).all(...params) as MemoryRow[];
|
|
}
|
|
break;
|
|
}
|
|
|
|
case "hybrid": {
|
|
// Attempt hybrid vector+FTS5 search if embedding + vector store are available
|
|
if (config.query && useModernTable) {
|
|
const resolution = resolveEmbeddingSource(settings);
|
|
if (resolution.source !== null) {
|
|
// Plan 21 FAIL #2 fix: tier-2 Qdrant route also covers hybrid strategy.
|
|
// Qdrant is vector-only (no FTS5 fusion), so this acts as the vector
|
|
// half of the hybrid contract. If Qdrant is down or empty, fall through
|
|
// to sqlite-vec's hybrid RRF (FTS5 + vector).
|
|
if (settings.vectorStore === "qdrant") {
|
|
try {
|
|
const qres = await searchSemanticMemory(config.query, 100, {
|
|
apiKeyId,
|
|
});
|
|
if (qres.ok && qres.results && qres.results.length > 0) {
|
|
const hitIds = qres.results.map((r) => r.id);
|
|
const hitMemories = fetchMemoriesByIds(hitIds);
|
|
const scoreMap = new Map(
|
|
qres.results.map((r) => [r.id, r.score])
|
|
);
|
|
let qdrantItems = hitMemories.map((m) => ({
|
|
memory: m,
|
|
score: scoreMap.get(m.id) ?? 0,
|
|
tier: "qdrant" as const,
|
|
}));
|
|
if (
|
|
settings.rerankEnabled &&
|
|
settings.rerankProviderModel &&
|
|
config.query
|
|
) {
|
|
qdrantItems = (await applyRerank(
|
|
qdrantItems,
|
|
config.query,
|
|
settings.rerankProviderModel
|
|
)) as typeof qdrantItems;
|
|
}
|
|
for (const entry of qdrantItems) {
|
|
const memoryTokens = estimateTokens(entry.memory.content);
|
|
if (totalTokens + memoryTokens > maxTokens) {
|
|
if (memories.length === 0) {
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
break;
|
|
}
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
log.info("memory.retrieval.complete", {
|
|
apiKeyId,
|
|
count: memories.length,
|
|
tier: "qdrant",
|
|
});
|
|
return memories.map((e) => e.memory);
|
|
}
|
|
} catch (err: unknown) {
|
|
log.warn("memory.retrieval.qdrant.fail", {
|
|
error: sanitizeErrorMessage(
|
|
err instanceof Error ? err.message : String(err)
|
|
),
|
|
});
|
|
// fall through to sqlite-vec hybrid RRF
|
|
}
|
|
}
|
|
|
|
const embeddingResult = await embed(config.query, settings);
|
|
if ("vector" in embeddingResult) {
|
|
const vec = getVectorStore();
|
|
if (vec) {
|
|
try {
|
|
await vec.ensureReady(resolution);
|
|
const hybridHits = await vec.searchHybrid(
|
|
embeddingResult.vector,
|
|
config.query,
|
|
100,
|
|
apiKeyId
|
|
);
|
|
const hitIds = hybridHits.map((h) => h.memoryId);
|
|
const hitMemories = fetchMemoriesByIds(hitIds);
|
|
const scoreMap = new Map(
|
|
hybridHits.map((h) => [
|
|
h.memoryId,
|
|
{ rrfScore: h.rrfScore, vecDistance: h.vecDistance, ftsScore: h.ftsScore },
|
|
])
|
|
);
|
|
|
|
let rankedHybridItems = hitMemories.map((m) => {
|
|
const sc = scoreMap.get(m.id);
|
|
return {
|
|
memory: m,
|
|
score: sc?.rrfScore ?? 0,
|
|
tier: "hybrid-rrf" as const,
|
|
};
|
|
});
|
|
|
|
// Apply rerank if enabled
|
|
if (settings.rerankEnabled && settings.rerankProviderModel && config.query) {
|
|
rankedHybridItems = (await applyRerank(
|
|
rankedHybridItems,
|
|
config.query,
|
|
settings.rerankProviderModel
|
|
)) as typeof rankedHybridItems;
|
|
}
|
|
|
|
// Token budget enforcement
|
|
for (const entry of rankedHybridItems) {
|
|
const memoryTokens = estimateTokens(entry.memory.content);
|
|
if (totalTokens + memoryTokens > maxTokens) {
|
|
if (memories.length === 0) {
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
break;
|
|
}
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
|
|
log.info("memory.retrieval.complete", {
|
|
apiKeyId,
|
|
count: memories.length,
|
|
tier: "hybrid-rrf",
|
|
});
|
|
return memories.map((e) => e.memory);
|
|
} catch (err: unknown) {
|
|
log.warn("memory.retrieval.hybrid.fail", {
|
|
error: sanitizeErrorMessage(err instanceof Error ? err.message : String(err)),
|
|
});
|
|
// Fall through to FTS5 degradation
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Degraded path: FTS5 + keyword union
|
|
let ftsRows: MemoryRow[] = [];
|
|
if (config.query && ftsAvailable) {
|
|
ftsRows = buildFtsRows(apiKeyId, ftsColConfig);
|
|
}
|
|
// Get chronological results for keyword scoring
|
|
query += ` ORDER BY ${columns.createdAt} DESC LIMIT 100`;
|
|
const keywordRows = db.prepare(query).all(...params) as MemoryRow[];
|
|
|
|
// Union: FTS5 results first (higher relevance), then keyword results, dedup by id
|
|
const seen = new Set<string>();
|
|
rows = [];
|
|
for (const row of [...ftsRows, ...keywordRows]) {
|
|
const rowId = String(row.id);
|
|
if (!seen.has(rowId)) {
|
|
seen.add(rowId);
|
|
rows.push(row);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
case "exact":
|
|
default: {
|
|
query += ` ORDER BY ${columns.createdAt} DESC LIMIT 100`;
|
|
rows = db.prepare(query).all(...params) as MemoryRow[];
|
|
}
|
|
}
|
|
|
|
const rankedRows = rows
|
|
.map((row) => {
|
|
const memory = rowToMemory(row);
|
|
const score = config.query ? getRelevanceScore(memory, config.query) : 0;
|
|
return { memory, score, tier: "fts5" as const };
|
|
})
|
|
.filter((entry) => !config.query || entry.score > 0)
|
|
.sort((a, b) => {
|
|
if (b.score !== a.score) return b.score - a.score;
|
|
return b.memory.createdAt.getTime() - a.memory.createdAt.getTime();
|
|
});
|
|
|
|
// Process memories until budget exceeded
|
|
for (const entry of rankedRows) {
|
|
const memory = entry.memory;
|
|
const memoryTokens = estimateTokens(memory.content);
|
|
|
|
if (totalTokens + memoryTokens > maxTokens) {
|
|
if (memories.length === 0) {
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
break;
|
|
}
|
|
|
|
memories.push(entry);
|
|
totalTokens += memoryTokens;
|
|
}
|
|
|
|
const result = memories.map((entry) => entry.memory);
|
|
log.info("memory.retrieval.complete", { apiKeyId, count: result.length });
|
|
log.debug("memory.retrieval.selected", { ids: result.map((m) => m.id) });
|
|
return result;
|
|
}
|
|
|
|
// ──────────────── retrievePreview (§3.6 — dry-run for Playground) ────────────────
|
|
|
|
/**
|
|
* Dry-run of retrieveMemories.
|
|
* Returns the full bundle (items + resolution metadata) WITHOUT injecting into chat.
|
|
* If apiKeyId is null, tests against all memories (global scope).
|
|
*/
|
|
export async function retrievePreview(
|
|
apiKeyId: string | null,
|
|
query: string,
|
|
options: { strategy: "exact" | "semantic" | "hybrid"; maxTokens: number; limit: number }
|
|
): Promise<RetrievePreviewBundle> {
|
|
const { strategy, maxTokens, limit } = options;
|
|
|
|
const settings = await getMemorySettings();
|
|
const resolution = resolveEmbeddingSource(settings);
|
|
|
|
let fallbackReason: string | null = null;
|
|
let rerankApplied = false;
|
|
|
|
const result: RetrievePreviewItem[] = [];
|
|
let totalTokens = 0;
|
|
|
|
const useModernTable = hasTable("memories");
|
|
const ftsAvailable = useModernTable && hasTable("memory_fts");
|
|
const db = getDbInstance();
|
|
|
|
const tableName = useModernTable ? "memories" : "memory";
|
|
const apiKeyCol = useModernTable ? "api_key_id" : "apiKeyId";
|
|
const expiresCol = useModernTable ? "expires_at" : "expiresAt";
|
|
const createdCol = useModernTable ? "created_at" : "createdAt";
|
|
|
|
// Determine vector store backend
|
|
let vectorStoreBackend: "sqlite-vec" | "qdrant" | "none" = "none";
|
|
const vec = getVectorStore();
|
|
if (vec) vectorStoreBackend = "sqlite-vec";
|
|
|
|
if (strategy === "semantic" || strategy === "hybrid") {
|
|
if (resolution.source !== null && query) {
|
|
// Plan 21 FAIL #2 fix: when settings.vectorStore === "qdrant",
|
|
// the Playground must show what production retrieval would actually
|
|
// see — Qdrant tier-2. Falls through to sqlite-vec on failure.
|
|
if (settings.vectorStore === "qdrant") {
|
|
try {
|
|
const qres = await searchSemanticMemory(
|
|
query,
|
|
limit,
|
|
apiKeyId ? { apiKeyId } : undefined
|
|
);
|
|
if (qres.ok && qres.results && qres.results.length > 0) {
|
|
const hitIds = qres.results.map((r) => r.id);
|
|
const hitMemories = fetchMemoriesByIds(hitIds).slice(0, limit);
|
|
const scoreMap = new Map(
|
|
qres.results.map((r) => [r.id, r.score])
|
|
);
|
|
let items: Array<{
|
|
memory: Memory;
|
|
score: number;
|
|
tier: "qdrant";
|
|
vecScore: number | null;
|
|
ftsScore: null;
|
|
}> = hitMemories.map((m) => ({
|
|
memory: m,
|
|
score: scoreMap.get(m.id) ?? 0,
|
|
tier: "qdrant" as const,
|
|
vecScore: scoreMap.get(m.id) ?? null,
|
|
ftsScore: null,
|
|
}));
|
|
|
|
if (settings.rerankEnabled && settings.rerankProviderModel) {
|
|
items = (await applyRerank(
|
|
items,
|
|
query,
|
|
settings.rerankProviderModel
|
|
)) as typeof items;
|
|
rerankApplied = true;
|
|
}
|
|
|
|
for (const item of items) {
|
|
if (result.length >= limit) break;
|
|
const tokens = estimateTokens(item.memory.content);
|
|
if (totalTokens + tokens > maxTokens && result.length > 0) break;
|
|
result.push({ ...item, tokens });
|
|
totalTokens += tokens;
|
|
}
|
|
|
|
return {
|
|
items: result,
|
|
resolution: {
|
|
embeddingSource: resolution.source,
|
|
embeddingModel: resolution.model,
|
|
vectorStore: "qdrant",
|
|
strategyUsed: strategy,
|
|
rerankApplied,
|
|
fallbackReason: null,
|
|
},
|
|
totalTokens,
|
|
budgetMaxTokens: maxTokens,
|
|
};
|
|
}
|
|
// Qdrant returned nothing — fall through to sqlite-vec for parity
|
|
// with production (so the Playground reflects the same fallback).
|
|
fallbackReason = "Qdrant retornou 0 resultados — fallback p/ sqlite-vec";
|
|
} catch (err: unknown) {
|
|
fallbackReason = sanitizeErrorMessage(
|
|
err instanceof Error ? err.message : String(err)
|
|
);
|
|
log.warn("memory.preview.qdrant.fail", { error: fallbackReason });
|
|
}
|
|
}
|
|
|
|
const embeddingResult = await embed(query, settings);
|
|
|
|
if ("vector" in embeddingResult) {
|
|
if (vec) {
|
|
try {
|
|
await vec.ensureReady(resolution);
|
|
|
|
if (strategy === "semantic") {
|
|
const hits = await vec.searchVector(
|
|
embeddingResult.vector,
|
|
limit,
|
|
apiKeyId ?? undefined
|
|
);
|
|
const hitIds = hits.map((h) => h.memoryId);
|
|
const hitMemories = fetchMemoriesByIds(hitIds).slice(0, limit);
|
|
const scoreMap = new Map(hits.map((h) => [h.memoryId, h.score]));
|
|
|
|
let items: Array<{
|
|
memory: Memory;
|
|
score: number;
|
|
tier: "vector";
|
|
vecScore: number | null;
|
|
ftsScore: null;
|
|
}> = hitMemories.map((m) => ({
|
|
memory: m,
|
|
score: scoreMap.get(m.id) ?? 0,
|
|
tier: "vector" as const,
|
|
vecScore: scoreMap.get(m.id) ?? null,
|
|
ftsScore: null,
|
|
}));
|
|
|
|
if (settings.rerankEnabled && settings.rerankProviderModel) {
|
|
items = (await applyRerank(
|
|
items,
|
|
query,
|
|
settings.rerankProviderModel
|
|
)) as typeof items;
|
|
rerankApplied = true;
|
|
}
|
|
|
|
for (const item of items) {
|
|
if (result.length >= limit) break;
|
|
const tokens = estimateTokens(item.memory.content);
|
|
if (totalTokens + tokens > maxTokens && result.length > 0) break;
|
|
result.push({ ...item, tokens });
|
|
totalTokens += tokens;
|
|
}
|
|
} else {
|
|
// hybrid
|
|
const hybridHits = await vec.searchHybrid(
|
|
embeddingResult.vector,
|
|
query,
|
|
limit,
|
|
apiKeyId ?? undefined
|
|
);
|
|
const hitIds = hybridHits.map((h) => h.memoryId);
|
|
const hitMemories = fetchMemoriesByIds(hitIds);
|
|
const scoreMap = new Map(
|
|
hybridHits.map((h) => [
|
|
h.memoryId,
|
|
{
|
|
rrfScore: h.rrfScore,
|
|
vecDistance: h.vecDistance,
|
|
ftsScore: h.ftsScore,
|
|
},
|
|
])
|
|
);
|
|
|
|
let items = hitMemories.slice(0, limit).map((m) => {
|
|
const sc = scoreMap.get(m.id);
|
|
return {
|
|
memory: m,
|
|
score: sc?.rrfScore ?? 0,
|
|
tier: "hybrid-rrf" as const,
|
|
vecScore: sc?.vecDistance != null ? 1 / (1 + sc.vecDistance) : null,
|
|
ftsScore: sc?.ftsScore ?? null,
|
|
};
|
|
});
|
|
|
|
if (settings.rerankEnabled && settings.rerankProviderModel) {
|
|
items = (await applyRerank(
|
|
items,
|
|
query,
|
|
settings.rerankProviderModel
|
|
)) as typeof items;
|
|
rerankApplied = true;
|
|
}
|
|
|
|
for (const item of items) {
|
|
if (result.length >= limit) break;
|
|
const tokens = estimateTokens(item.memory.content);
|
|
if (totalTokens + tokens > maxTokens && result.length > 0) break;
|
|
result.push({ ...item, tokens });
|
|
totalTokens += tokens;
|
|
}
|
|
}
|
|
|
|
return {
|
|
items: result,
|
|
resolution: {
|
|
embeddingSource: resolution.source,
|
|
embeddingModel: resolution.model,
|
|
vectorStore: vectorStoreBackend,
|
|
strategyUsed: strategy,
|
|
rerankApplied,
|
|
fallbackReason: null,
|
|
},
|
|
totalTokens,
|
|
budgetMaxTokens: maxTokens,
|
|
};
|
|
} catch (err: unknown) {
|
|
fallbackReason = sanitizeErrorMessage(
|
|
err instanceof Error ? err.message : String(err)
|
|
);
|
|
log.warn("memory.preview.vector.fail", { error: fallbackReason });
|
|
}
|
|
} else {
|
|
fallbackReason = "sqlite-vec não disponível (degradado para FTS5)";
|
|
}
|
|
} else {
|
|
// EmbeddingError
|
|
fallbackReason =
|
|
"message" in embeddingResult ? (embeddingResult.message as string) : "embedding falhou";
|
|
}
|
|
} else if (!query) {
|
|
fallbackReason = "query vazia — usando FTS5";
|
|
} else {
|
|
fallbackReason = resolution.reason;
|
|
}
|
|
}
|
|
|
|
// FTS5 fallback path (or strategy=exact)
|
|
let baseQuery = `SELECT * FROM ${tableName}`;
|
|
const baseParams: unknown[] = [];
|
|
|
|
if (apiKeyId) {
|
|
baseQuery += ` WHERE ${apiKeyCol} = ?`;
|
|
baseParams.push(apiKeyId);
|
|
baseQuery += ` AND (${expiresCol} IS NULL OR datetime(${expiresCol}) > datetime('now'))`;
|
|
} else {
|
|
baseQuery += ` WHERE (${expiresCol} IS NULL OR datetime(${expiresCol}) > datetime('now'))`;
|
|
}
|
|
|
|
if (strategy === "exact") {
|
|
baseQuery += ` ORDER BY ${createdCol} DESC LIMIT ?`;
|
|
baseParams.push(limit);
|
|
|
|
const rows = db.prepare(baseQuery).all(...baseParams) as MemoryRow[];
|
|
for (const row of rows) {
|
|
if (result.length >= limit) break;
|
|
const memory = rowToMemory(row);
|
|
const score = query ? getRelevanceScore(memory, query) : 0;
|
|
const tokens = estimateTokens(memory.content);
|
|
if (totalTokens + tokens > maxTokens && result.length > 0) break;
|
|
result.push({ memory, score, tokens, tier: "fts5", vecScore: null, ftsScore: null });
|
|
totalTokens += tokens;
|
|
}
|
|
} else {
|
|
// Semantic/hybrid degraded to FTS5
|
|
let ftsRows: MemoryRow[] = [];
|
|
if (query && ftsAvailable) {
|
|
const ftsQueryStr = apiKeyId
|
|
? `SELECT m.* FROM ${tableName} m JOIN memory_fts f ON m.memory_id = f.rowid WHERE f.memory_fts MATCH ? AND m.${apiKeyCol} = ? ORDER BY f.rank LIMIT ?`
|
|
: `SELECT m.* FROM ${tableName} m JOIN memory_fts f ON m.memory_id = f.rowid WHERE f.memory_fts MATCH ? ORDER BY f.rank LIMIT ?`;
|
|
const ftsP: unknown[] = apiKeyId ? [query, apiKeyId, limit] : [query, limit];
|
|
try {
|
|
ftsRows = db.prepare(ftsQueryStr).all(...ftsP) as MemoryRow[];
|
|
} catch {
|
|
ftsRows = [];
|
|
}
|
|
}
|
|
|
|
if (ftsRows.length === 0) {
|
|
baseQuery += ` ORDER BY ${createdCol} DESC LIMIT ?`;
|
|
baseParams.push(limit);
|
|
ftsRows = db.prepare(baseQuery).all(...baseParams) as MemoryRow[];
|
|
}
|
|
|
|
for (const row of ftsRows) {
|
|
if (result.length >= limit) break;
|
|
const memory = rowToMemory(row);
|
|
const score = query ? getRelevanceScore(memory, query) : 0;
|
|
const tokens = estimateTokens(memory.content);
|
|
if (totalTokens + tokens > maxTokens && result.length > 0) break;
|
|
result.push({ memory, score, tokens, tier: "fts5", vecScore: null, ftsScore: null });
|
|
totalTokens += tokens;
|
|
}
|
|
}
|
|
|
|
return {
|
|
items: result,
|
|
resolution: {
|
|
embeddingSource: resolution.source,
|
|
embeddingModel: resolution.model,
|
|
vectorStore: vectorStoreBackend,
|
|
strategyUsed: strategy,
|
|
rerankApplied,
|
|
fallbackReason,
|
|
},
|
|
totalTokens,
|
|
budgetMaxTokens: maxTokens,
|
|
};
|
|
}
|
|
|
|
// ──────────────── engineStatus (§3.2) ────────────────
|
|
|
|
/**
|
|
* Returns the current status of the memory engine (for the Engine tab in the UI).
|
|
* Matches MemoryEngineStatusSchema from @/shared/schemas/memory.
|
|
*/
|
|
export async function engineStatus(): Promise<MemoryEngineStatus> {
|
|
const settings = await getMemorySettings();
|
|
const resolution = resolveEmbeddingSource(settings);
|
|
const cacheStats = embeddingCacheStats();
|
|
|
|
// Vector store
|
|
const vec = getVectorStore();
|
|
let vecBackend: "sqlite-vec" | "qdrant" | "none" = "none";
|
|
let vecAvailable = false;
|
|
let vecRowCount = 0;
|
|
let vecNeedsReindex = 0;
|
|
let vecReason = "sqlite-vec não disponível";
|
|
|
|
if (vec) {
|
|
vecBackend = "sqlite-vec";
|
|
vecAvailable = true;
|
|
try {
|
|
const s = await vec.stats();
|
|
vecRowCount = s.rowCount;
|
|
vecNeedsReindex = s.needsReindex;
|
|
vecReason = `sqlite-vec ativo, dim=${s.activeDim ?? "null"}`;
|
|
} catch {
|
|
vecReason = "sqlite-vec ativo mas stats falharam";
|
|
}
|
|
} else {
|
|
vecReason = "sqlite-vec não disponível — usando apenas FTS5";
|
|
}
|
|
|
|
// Qdrant
|
|
let qdrantEnabled = false;
|
|
let qdrantHealthy: boolean | null = null;
|
|
let qdrantLatencyMs: number | null = null;
|
|
let qdrantError: string | null = null;
|
|
|
|
try {
|
|
const qdrantCfg = await getQdrantConfig();
|
|
qdrantEnabled = qdrantCfg.enabled;
|
|
if (qdrantEnabled) {
|
|
const health = await checkQdrantHealth();
|
|
qdrantHealthy = health.ok;
|
|
qdrantLatencyMs = health.latencyMs;
|
|
qdrantError = health.error ? sanitizeErrorMessage(health.error) : null;
|
|
|
|
// Plan 21 FAIL #2 fix: only claim vectorStore=qdrant when the user
|
|
// explicitly opted into it (settings.vectorStore === "qdrant"). Before,
|
|
// the status reported "qdrant" whenever the cluster was healthy even
|
|
// though the retrieval path always used sqlite-vec — engineStatus lied.
|
|
if (qdrantHealthy && settings.vectorStore === "qdrant") {
|
|
vecBackend = "qdrant";
|
|
vecAvailable = true;
|
|
vecReason = `Qdrant configurado em ${qdrantCfg.host}:${qdrantCfg.port}`;
|
|
}
|
|
}
|
|
} catch (err: unknown) {
|
|
qdrantError = sanitizeErrorMessage(err instanceof Error ? err.message : String(err));
|
|
}
|
|
|
|
// Rerank
|
|
let rerankAvailable = false;
|
|
let rerankReason = "rerank desabilitado";
|
|
if (settings.rerankEnabled && settings.rerankProviderModel) {
|
|
rerankAvailable = true;
|
|
rerankReason = `rerank ativo: ${settings.rerankProviderModel}`;
|
|
} else if (settings.rerankEnabled && !settings.rerankProviderModel) {
|
|
rerankReason = "rerank habilitado mas provider não configurado";
|
|
}
|
|
|
|
const rerankParts = settings.rerankProviderModel?.split("/") ?? [];
|
|
const rerankProvider = rerankParts.length >= 2 ? (rerankParts[0] ?? null) : null;
|
|
const rerankModel =
|
|
rerankParts.length >= 2
|
|
? rerankParts.slice(1).join("/")
|
|
: (settings.rerankProviderModel ?? null);
|
|
|
|
return {
|
|
keyword: { available: true, backend: "FTS5" },
|
|
embedding: {
|
|
source: resolution.source,
|
|
model: resolution.model,
|
|
dimensions: resolution.dimensions,
|
|
available: resolution.source !== null,
|
|
reason: resolution.reason,
|
|
cacheStats,
|
|
},
|
|
vectorStore: {
|
|
backend: vecBackend,
|
|
available: vecAvailable,
|
|
rowCount: vecRowCount,
|
|
needsReindex: vecNeedsReindex,
|
|
reason: vecReason,
|
|
},
|
|
qdrant: {
|
|
enabled: qdrantEnabled,
|
|
healthy: qdrantHealthy,
|
|
latencyMs: qdrantLatencyMs,
|
|
error: qdrantError,
|
|
},
|
|
rerank: {
|
|
enabled: settings.rerankEnabled,
|
|
provider: rerankProvider,
|
|
model: rerankModel,
|
|
available: rerankAvailable,
|
|
reason: rerankReason,
|
|
},
|
|
};
|
|
}
|