diff --git a/README.md b/README.md index 0d5ea9f25f..390acbe472 100644 --- a/README.md +++ b/README.md @@ -1497,7 +1497,7 @@ OmniRoute stands on the shoulders of giants. It started as a fork of **[9router] - + diff --git a/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts b/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts index 9c9a7774d9..f9597e7b2e 100644 --- a/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts +++ b/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts @@ -1,7 +1,8 @@ /** * GCF generic-profile decoder (decodeGeneric). * Vendored from gcf-typescript — generic profile only. Current with GCF spec v3.2 - * (nested object flattening) and the [N]: inline-array quoting fix. + * (nested object flattening), the [N]: inline-array quoting fix, the int64/2^53 numeric- + * domain rendering (SPEC 2.3.1), and the root-array surplus count check (SPEC 13). * https://github.com/blackwell-systems/gcf-typescript * * SPDX-License-Identifier: MIT @@ -78,7 +79,14 @@ export function decodeGeneric(input: string): any { // Root array. if (first.startsWith("## [")) { - const [arr] = parseArrayFromHeader(contentLines, 0, 0, first.slice(3)); + const [arr, consumed] = parseArrayFromHeader(contentLines, 0, 0, first.slice(3)); + // A root array spans the whole document, so any structural line past the consumed + // rows is a surplus item, not sibling content. The row loop stops at the declared + // count, so the count assert only catches the deficit; surplus is caught here (SPEC + // Section 13: a mismatch, fewer OR more items than declared, is an error). + if (consumed < contentLines.length) { + throw new Error("count_mismatch: declared count is fewer than the rows present"); + } return arr; } diff --git a/open-sse/services/compression/engines/headroom/gcf/index.ts b/open-sse/services/compression/engines/headroom/gcf/index.ts index 5671ced952..6be512f61e 100644 --- a/open-sse/services/compression/engines/headroom/gcf/index.ts +++ b/open-sse/services/compression/engines/headroom/gcf/index.ts @@ -1,7 +1,8 @@ /** * GCF (Graph Compact Format) — generic profile encoder/decoder. * Vendored from gcf-typescript for zero-dependency integration. Current with - * GCF spec v3.2 (nested object flattening) + [N]: inline-array quoting fix. + * GCF spec v3.2 (nested object flattening) + [N]: inline-array quoting fix + int64/2^53 + * numeric-domain rendering (SPEC 2.3.1) + root-array surplus count check (SPEC 13). * https://github.com/blackwell-systems/gcf-typescript * * SPDX-License-Identifier: MIT diff --git a/open-sse/services/compression/engines/headroom/gcf/scalar.ts b/open-sse/services/compression/engines/headroom/gcf/scalar.ts index f7e82d4419..f3b5082362 100644 --- a/open-sse/services/compression/engines/headroom/gcf/scalar.ts +++ b/open-sse/services/compression/engines/headroom/gcf/scalar.ts @@ -1,7 +1,8 @@ /** * Common scalar grammar for GCF (Graph Compact Format). * Vendored from gcf-typescript — generic profile only. Current with GCF spec v3.2 - * (nested object flattening) and the [N]: inline-array quoting fix. + * (nested object flattening), the [N]: inline-array quoting fix, the int64/2^53 numeric- + * domain rendering (SPEC 2.3.1), and the root-array surplus count check (SPEC 13). * https://github.com/blackwell-systems/gcf-typescript * * SPDX-License-Identifier: MIT @@ -107,7 +108,12 @@ export function formatNumber(f: number): string { if (Object.is(f, -0)) return "-0"; if (f === 0) return "0"; const abs = Math.abs(f); - if (abs >= 1e-6 && abs < 1e21) { + // Plain decimal only below 2^53. Every double at or above 2^53 is integer-valued, so a + // plain rendering emits a bare-integer token: indistinguishable from an int64 on the wire + // and beyond a JavaScript decoder's safe-integer range (2^53-1), so it is rejected/misread + // on decode. Exponent shape keeps bare tokens int64 and decimal/exponent tokens doubles + // (SPEC 2.3.1). 2^53 = 9007199254740992. + if (abs >= 1e-6 && abs < 9007199254740992) { return toPreciseDecimal(f); } // Exponent notation. diff --git a/tests/unit/compression/gcf-count-mismatch.test.ts b/tests/unit/compression/gcf-count-mismatch.test.ts new file mode 100644 index 0000000000..737e559de0 --- /dev/null +++ b/tests/unit/compression/gcf-count-mismatch.test.ts @@ -0,0 +1,41 @@ +/** + * Regression guard for GCF root-array surplus (SPEC Section 13). + * + * The root-array decode discarded parseArrayFromHeader's `consumed` count, so a wire whose + * declared count was fewer than the rows actually present silently dropped the surplus rows + * (the row loop breaks at the declared count and the extra lines were never read). SPEC 13 + * makes a count mismatch — fewer OR more items than declared — an error. The fix checks that + * the consumed lines cover the whole document and throws otherwise. + * + * Reachable in prod: headroomEngine round-trips the encoded blob; a truncated decode is a + * silent losslessness violation. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { encodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/generic.ts"; +import { decodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts"; + +test("surplus rows beyond the declared root-array count throw count_mismatch (SPEC 13)", () => { + const rows = [ + { a: 1, b: "x" }, + { a: 2, b: "y" }, + { a: 3, b: "z" }, + ]; + const wire = encodeGeneric(rows); // header declares [3] + const understated = wire.replace("[3]", "[2]"); // declare fewer than present + assert.throws( + () => decodeGeneric(understated), + /count_mismatch/, + "a root array carrying more rows than declared must error, not silently truncate" + ); +}); + +test("a root array with the exact declared count decodes normally", () => { + const rows = [ + { a: 1, b: "x" }, + { a: 2, b: "y" }, + { a: 3, b: "z" }, + ]; + const wire = encodeGeneric(rows); + assert.deepEqual(decodeGeneric(wire), rows); +}); diff --git a/tests/unit/compression/gcf-numeric-domain.test.ts b/tests/unit/compression/gcf-numeric-domain.test.ts new file mode 100644 index 0000000000..82929bb1d3 --- /dev/null +++ b/tests/unit/compression/gcf-numeric-domain.test.ts @@ -0,0 +1,40 @@ +/** + * Regression guard for the GCF numeric domain (SPEC 2.3.1 / 2.3.2). + * + * formatNumber() gated plain-decimal rendering at `abs < 1e21`, so an integer-valued double + * in [2^53, 1e21) was emitted as a bare-integer token (e.g. `1e18` -> `1000000000000000000`). + * That token is indistinguishable from an int64 on the wire and beyond a JavaScript decoder's + * safe-integer range (2^53-1), so a spec-compliant decoder rejects it (unsafe_integer) or a + * cross-language decoder reads it as an exact int64 it never was. The fix gates at 2^53, so + * such values render in exponent form and stay typed as doubles. + * + * Reachable in prod: headroomEngine.apply() ships the encoded blob. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { encodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/generic.ts"; +import { decodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts"; + +test("a double >= 2^53 renders in exponent form, not a bare integer (SPEC 2.3.1)", () => { + const wire = encodeGeneric([{ v: 1e18 }]); + assert.match(wire, /1e\+18/, `expected exponent notation, got:\n${wire}`); + assert.doesNotMatch( + wire, + /1000000000000000000/, + `a bare-integer token for a double is ambiguous with int64:\n${wire}` + ); + assert.deepEqual(decodeGeneric(wire), [{ v: 1e18 }]); +}); + +test("2^53 itself renders as exponent and round-trips", () => { + const wire = encodeGeneric([{ v: 9007199254740992 }]); // 2^53 + assert.doesNotMatch(wire, /9007199254740992/, `2^53 must not emit as a bare integer:\n${wire}`); + assert.deepEqual(decodeGeneric(wire), [{ v: 9007199254740992 }]); +}); + +test("integers below 2^53 still render as plain decimal (int64 domain) and round-trip", () => { + const rows = [{ v: 42 }, { v: 1234567 }, { v: 9007199254740991 }]; // 2^53-1 + const wire = encodeGeneric(rows); + assert.match(wire, /9007199254740991/, `2^53-1 is an exact int64 and must stay plain:\n${wire}`); + assert.deepEqual(decodeGeneric(wire), rows); +});
Project⭐How it inspired OmniRoute
TOON24.9kToken-Oriented Object Notation — its columnar, header-plus-rows model shaped our tabular compaction stage.
GCF – Graph Compact Format22First inspired our tabular compaction stage; now its zero-dependency, lossless generic-profile encoder is vendored directly as the Headroom codec (MIT, SPDX-marked), current with GCF spec v3.2.
GCF – Graph Compact Format22First inspired our tabular compaction stage; now its zero-dependency, lossless generic-profile encoder is vendored directly as the Headroom codec (MIT, SPDX-marked), with later numeric-domain and count-mismatch correctness fixes.
token-optimizer-mcp444Brotli/SQLite cache + per-session context-delta — inspired our session-dedup engine.
token-savior1.1kBash-output compaction + MCP profiles — inspired our compression bail-out discipline and MCP tool-manifest reduction.
token-saver117Content-aware, per-file-type output compression with failure-aware bail-out — validated our per-type dispatch and minimum-gain skip.