mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-21 14:22:14 +03:00
ad633c8440
* fix(memory): word/sentence-boundary aware truncation in extraction sanitizeMatch() and capExtractionText() previously did raw character-offset slices (slice(0, MAX_FACT_LENGTH) / slice(-MAX_EXTRACTION_TEXT_LENGTH)) with no boundary awareness, producing garbled mid-word/mid-clause fragments that get injected into LLM context as memory facts. - sanitizeMatch() now backs the cut off to the nearest sentence-ending punctuation (. ! ?) within a lookback window, falling back to a plain whitespace boundary, falling back to the original hard cut only when no boundary exists nearby. - capExtractionText() applies the equivalent boundary-aware trim on the front edge of the kept tail. Mirrors the boundary-aware truncation pattern already used by open-sse/services/compression/lite.ts (#8169) for tool-result truncation. Adds tests/unit/memory-extraction-boundary-truncation.test.ts covering word-boundary cuts, sentence-boundary preference, short-string passthrough, the no-boundary-available fallback, and capExtractionText's tail behavior. * docs(changelog): add fragment for word/sentence-boundary fact truncation Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
72 lines
3.3 KiB
TypeScript
72 lines
3.3 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
const { extractFactsFromText } = await import("../../src/lib/memory/extraction.ts");
|
|
|
|
// ─── sanitizeMatch (via extractFactsFromText): word/sentence boundary cuts ──
|
|
|
|
test("sanitizeMatch: long match is cut at a word boundary, not mid-word", () => {
|
|
const words = "lorem ".repeat(120); // well over 500 chars, plenty of spaces
|
|
const facts = extractFactsFromText(`I prefer ${words}.`);
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref, "Should extract a preference fact");
|
|
assert.ok(pref.content.length <= 500, "Content should be capped at 500 chars");
|
|
// Source is "lorem " repeated, so a clean word-boundary cut must always end
|
|
// exactly on a full "lorem" token, never a truncated fragment like "lore".
|
|
assert.ok(
|
|
pref.content.endsWith("lorem"),
|
|
`Cut should land on a word boundary, got: "${pref.content.slice(-10)}"`
|
|
);
|
|
});
|
|
|
|
test("sanitizeMatch: prefers cutting at sentence-ending punctuation near the limit", () => {
|
|
// The capture group patterns exclude "." and "," (they stop the match), so
|
|
// the only sentence-ending punctuation that can appear mid-match is "!"/"?".
|
|
// Place one just before the 500-char cap, with more content past the cap.
|
|
const before = "a".repeat(490);
|
|
const after = "b".repeat(50);
|
|
const raw = `${before}! ${after}`;
|
|
const facts = extractFactsFromText(`I prefer ${raw}`);
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref);
|
|
assert.ok(
|
|
pref.content.endsWith("!"),
|
|
`Expected a sentence-boundary cut, got: "${pref.content.slice(-30)}"`
|
|
);
|
|
});
|
|
|
|
test("sanitizeMatch: short content is left untouched", () => {
|
|
const facts = extractFactsFromText("I prefer dark mode in my editor.");
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref);
|
|
assert.equal(pref.content, "dark mode in my editor");
|
|
});
|
|
|
|
test("sanitizeMatch: falls back to a hard cut when no boundary exists in the lookback window", () => {
|
|
const noBoundary = "a".repeat(600); // no whitespace/punctuation anywhere
|
|
const facts = extractFactsFromText(`I prefer ${noBoundary}.`);
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref);
|
|
assert.ok(pref.content.length <= 500);
|
|
assert.ok(/^a+$/.test(pref.content), "Should still cap even with no boundary available");
|
|
});
|
|
|
|
// ─── capExtractionText (via extractFactsFromText tail-scan behavior) ───────
|
|
|
|
test("extractFactsFromText: capExtractionText does not truncate the kept tail mid-word", () => {
|
|
// Push the "I prefer" match itself to straddle the 64KB tail-cut boundary.
|
|
const padding = "x ".repeat(40000); // > 64KB of padding before the real fact
|
|
const text = `${padding}I prefer boundary-safe-editor for daily work.`;
|
|
const facts = extractFactsFromText(text);
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref, "Fact near the tail boundary should still be extracted intact");
|
|
assert.ok(pref.content.includes("boundary-safe-editor"));
|
|
});
|
|
|
|
test("extractFactsFromText: capExtractionText leaves text under the limit untouched", () => {
|
|
const facts = extractFactsFromText("I prefer short text under the cap.");
|
|
const pref = facts.find((f) => f.category === "preference");
|
|
assert.ok(pref);
|
|
assert.ok(pref.content.includes("short text under the cap"));
|
|
});
|