From 931afe3482772eb26258081d173b34fbb4e593d3 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:02:51 -0300 Subject: [PATCH] =?UTF-8?q?Fase=208=20=C2=B7=20Bloco=20B=20=E2=80=94=20su?= =?UTF-8?q?=C3=ADte=20de=20corre=C3=A7=C3=A3o=20(property=20+=20golden=20+?= =?UTF-8?q?=20SSE-correctness)=20(#3808)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.25 — Fase 8 Bloco B (property + golden + SSE-correctness). --- .github/workflows/ci.yml | 8 +- .github/workflows/nightly-property.yml | 33 +++++ dependency-allowlist.json | 1 + package-lock.json | 41 ++++++ package.json | 14 +- scripts/check/check-test-discovery.mjs | 7 +- scripts/quality/property-seed.mjs | 3 + .../translation/claude-to-openai.json | 23 ++++ .../translation/gemini-to-openai.json | 24 ++++ .../translation/openai-to-claude.json | 37 ++++++ .../translation/openai-to-gemini.json | 24 ++++ .../translation/sse-chunk-sequences.json | 26 ++++ tests/helpers/fakeUpstreamStream.ts | 28 ++++ tests/helpers/goldenSnapshot.ts | 56 ++++++++ tests/helpers/propertyConfig.ts | 13 ++ tests/helpers/translationFixtures.ts | 31 +++++ tests/integration/sse-correctness.test.ts | 120 ++++++++++++++++++ .../claude-to-openai/basic-message.json | 11 ++ .../claude-to-openai/with-system.json | 15 +++ .../gemini-to-openai/basic-chat.json | 10 ++ .../gemini-to-openai/multi-turn.json | 18 +++ .../openai-to-claude/basic-chat.json | 16 +++ .../openai-to-claude/multi-turn.json | 34 +++++ .../openai-to-claude/with-system.json | 22 ++++ .../openai-to-gemini/basic-chat.json | 38 ++++++ .../openai-to-gemini/with-system.json | 47 +++++++ tests/unit/correctness/combo.property.test.ts | 66 ++++++++++ .../correctness/fakeUpstreamStream.test.ts | 39 ++++++ tests/unit/correctness/fixtures.test.ts | 18 +++ tests/unit/correctness/goldenSnapshot.test.ts | 41 ++++++ .../correctness/sanitizers.property.test.ts | 42 ++++++ .../correctness/sse-parser.property.test.ts | 63 +++++++++ .../correctness/translation.golden.test.ts | 19 +++ .../correctness/translators.property.test.ts | 78 ++++++++++++ 34 files changed, 1054 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/nightly-property.yml create mode 100644 scripts/quality/property-seed.mjs create mode 100644 tests/fixtures/translation/claude-to-openai.json create mode 100644 tests/fixtures/translation/gemini-to-openai.json create mode 100644 tests/fixtures/translation/openai-to-claude.json create mode 100644 tests/fixtures/translation/openai-to-gemini.json create mode 100644 tests/fixtures/translation/sse-chunk-sequences.json create mode 100644 tests/helpers/fakeUpstreamStream.ts create mode 100644 tests/helpers/goldenSnapshot.ts create mode 100644 tests/helpers/propertyConfig.ts create mode 100644 tests/helpers/translationFixtures.ts create mode 100644 tests/integration/sse-correctness.test.ts create mode 100644 tests/snapshots/translation/claude-to-openai/basic-message.json create mode 100644 tests/snapshots/translation/claude-to-openai/with-system.json create mode 100644 tests/snapshots/translation/gemini-to-openai/basic-chat.json create mode 100644 tests/snapshots/translation/gemini-to-openai/multi-turn.json create mode 100644 tests/snapshots/translation/openai-to-claude/basic-chat.json create mode 100644 tests/snapshots/translation/openai-to-claude/multi-turn.json create mode 100644 tests/snapshots/translation/openai-to-claude/with-system.json create mode 100644 tests/snapshots/translation/openai-to-gemini/basic-chat.json create mode 100644 tests/snapshots/translation/openai-to-gemini/with-system.json create mode 100644 tests/unit/correctness/combo.property.test.ts create mode 100644 tests/unit/correctness/fakeUpstreamStream.test.ts create mode 100644 tests/unit/correctness/fixtures.test.ts create mode 100644 tests/unit/correctness/goldenSnapshot.test.ts create mode 100644 tests/unit/correctness/sanitizers.property.test.ts create mode 100644 tests/unit/correctness/sse-parser.property.test.ts create mode 100644 tests/unit/correctness/translation.golden.test.ts create mode 100644 tests/unit/correctness/translators.property.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c2fdb4847..8ff70a2d93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -348,7 +348,7 @@ jobs: cache: npm - run: npm ci - run: npm run check:node-runtime - - run: node --max-old-space-size=4096 --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --max-old-space-size=4096 --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" test-vitest: name: Vitest (MCP / autoCombo / UI components) @@ -397,7 +397,7 @@ jobs: - run: npm ci - run: npm run check:node-runtime - run: npm run build - - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" node-26-compat: name: Node 26 Compatibility (${{ matrix.shard }}/2) @@ -421,7 +421,7 @@ jobs: - run: npm ci - run: npm run check:node-runtime - run: npm run build - - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" test-coverage-shard: name: Coverage Shard (${{ matrix.shard }}/8) @@ -460,7 +460,7 @@ jobs: --exclude=tests/** \ --exclude=**/*.test.* \ node --max-old-space-size=4096 --import tsx --test --test-force-exit --test-concurrency=4 \ - --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" - name: Upload raw shard coverage if: always() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/nightly-property.yml b/.github/workflows/nightly-property.yml new file mode 100644 index 0000000000..0e1f28b3d5 --- /dev/null +++ b/.github/workflows/nightly-property.yml @@ -0,0 +1,33 @@ +name: Nightly Property Discovery +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: +permissions: + contents: read + issues: write +jobs: + property-random-seed: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - name: fast-check random seed (high runs) + id: prop + run: FC_SEED=random FC_NUM_RUNS=2000 npm run test:property + - name: Open issue on failure + if: failure() + uses: actions/github-script@v7 + with: + script: | + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Nightly property-test failure (Fase 8 B)", + body: "fast-check found a counterexample with a random seed. Check the run logs for the reproducible seed + minimal case, then add it as a fixture.\n\nRun: " + context.serverUrl + "/" + context.repo.owner + "/" + context.repo.repo + "/actions/runs/" + context.runId, + labels: ["quality-gate-finding"], + }); diff --git a/dependency-allowlist.json b/dependency-allowlist.json index 79abd59235..e43b815de5 100644 --- a/dependency-allowlist.json +++ b/dependency-allowlist.json @@ -45,6 +45,7 @@ "eslint-config-next", "eslint-plugin-sonarjs", "express", + "fast-check", "fetch-socks", "fflate", "fumadocs-core", diff --git a/package-lock.json b/package-lock.json index 0e35ed59be..e053269520 100644 --- a/package-lock.json +++ b/package-lock.json @@ -109,6 +109,7 @@ "eslint": "^9.39.4", "eslint-config-next": "16.2.7", "eslint-plugin-sonarjs": "^4.0.3", + "fast-check": "^4.8.0", "glob": "^13.0.6", "husky": "^9.1.7", "jscpd": "^4.2.5", @@ -13489,6 +13490,29 @@ "node": ">=0.10.0" } }, + "node_modules/fast-check": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.8.0.tgz", + "integrity": "sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-copy": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.2.tgz", @@ -21758,6 +21782,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pure-rand": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.0.tgz", + "integrity": "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/pvtsutils": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", diff --git a/package.json b/package.json index a6703ccb97..617e5f72ca 100644 --- a/package.json +++ b/package.json @@ -84,15 +84,16 @@ "electron:build:mac": "npm run build && cd electron && npm run build:mac", "electron:build:linux": "npm run build && cd electron && npm run build:linux", "electron:smoke:packaged": "node scripts/dev/smoke-electron-packaged.mjs", - "test": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit:fast": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-isolation=none tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:fast": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-isolation=none tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", "test:unit:shard": "concurrently --kill-others-on-fail -n s1,s2 \"npm:test:unit:shard:1\" \"npm:test:unit:shard:2\"", - "test:unit:shard:1": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit:shard:2": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:shard:1": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:shard:2": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", "test:plan3": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/plan3-p0.test.ts", "test:fixes": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/fixes-p1.test.ts", "test:security": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/security-fase01.test.ts", + "test:property": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --test --test-force-exit tests/unit/correctness/*.property.test.ts", "check:cycles": "node scripts/check/check-cycles.mjs", "check:route-validation:t06": "node scripts/check/check-route-validation.mjs", "check:any-budget:t11": "node scripts/check/check-t11-any-budget.mjs", @@ -164,7 +165,7 @@ "test:vitest:ui": "vitest run --config vitest.config.ts tests/unit/ui", "test:ecosystem": "node scripts/dev/run-ecosystem-tests.mjs", "test:system": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/e2e/system-failover.test.ts", - "test:coverage": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true c8 --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:coverage": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true c8 --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", "test:coverage:legacy": "c8 --output-dir coverage --exclude=open-sse --check-coverage --lines 50 --functions 50 --branches 50 node --import tsx --test tests/unit/*.test.ts", "coverage:report": "c8 report --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov", "coverage:summary": "node scripts/check/test-report-summary.mjs --input coverage/coverage-summary.json --output coverage/coverage-report.md", @@ -279,6 +280,7 @@ "eslint": "^9.39.4", "eslint-config-next": "16.2.7", "eslint-plugin-sonarjs": "^4.0.3", + "fast-check": "^4.8.0", "glob": "^13.0.6", "husky": "^9.1.7", "jscpd": "^4.2.5", diff --git a/scripts/check/check-test-discovery.mjs b/scripts/check/check-test-discovery.mjs index bc8a539165..99948e6a2d 100644 --- a/scripts/check/check-test-discovery.mjs +++ b/scripts/check/check-test-discovery.mjs @@ -53,7 +53,7 @@ export const COLLECTORS = [ // "vitest" e explodem no node runner). Subdir novo: adicione aqui E nos scripts // (o drift-check + o gate de órfãos forçam a manutenção em sincronia). { - glob: "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts", + glob: "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts", sources: ["package.json", ".github/workflows/ci.yml"], }, // Node native runner — test:integration (top-level only; tests/integration/services/ NÃO roda) @@ -79,7 +79,10 @@ export const COLLECTORS = [ { glob: "tests/e2e/*.spec.ts", sources: ["package.json"] }, // Runners custom — test:ecosystem / test:protocols:e2e (spawnam vitest com o arquivo) { glob: "tests/e2e/ecosystem.test.ts", sources: ["scripts/dev/run-ecosystem-tests.mjs"] }, - { glob: "tests/e2e/protocol-clients.test.ts", sources: ["scripts/dev/run-protocol-clients-tests.mjs"] }, + { + glob: "tests/e2e/protocol-clients.test.ts", + sources: ["scripts/dev/run-protocol-clients-tests.mjs"], + }, ]; const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); diff --git a/scripts/quality/property-seed.mjs b/scripts/quality/property-seed.mjs new file mode 100644 index 0000000000..f94de30711 --- /dev/null +++ b/scripts/quality/property-seed.mjs @@ -0,0 +1,3 @@ +// Fixed seed for PR runs (deterministic). Nightly overrides via FC_SEED=random. +export const PROPERTY_SEED = 42424242; +export const PROPERTY_NUM_RUNS = 200; diff --git a/tests/fixtures/translation/claude-to-openai.json b/tests/fixtures/translation/claude-to-openai.json new file mode 100644 index 0000000000..08707b76d7 --- /dev/null +++ b/tests/fixtures/translation/claude-to-openai.json @@ -0,0 +1,23 @@ +[ + { + "name": "claude-to-openai/basic-message", + "sourceFormat": "claude", + "targetFormat": "openai", + "input": { + "model": "gpt-4o", + "messages": [{ "role": "user", "content": [{ "type": "text", "text": "Hello" }] }], + "max_tokens": 512 + } + }, + { + "name": "claude-to-openai/with-system", + "sourceFormat": "claude", + "targetFormat": "openai", + "input": { + "model": "gpt-4o-mini", + "system": [{ "type": "text", "text": "Be helpful." }], + "messages": [{ "role": "user", "content": [{ "type": "text", "text": "What is 1+1?" }] }], + "max_tokens": 256 + } + } +] diff --git a/tests/fixtures/translation/gemini-to-openai.json b/tests/fixtures/translation/gemini-to-openai.json new file mode 100644 index 0000000000..7a7ff9c44d --- /dev/null +++ b/tests/fixtures/translation/gemini-to-openai.json @@ -0,0 +1,24 @@ +[ + { + "name": "gemini-to-openai/basic-chat", + "sourceFormat": "gemini", + "targetFormat": "openai", + "input": { + "model": "gpt-4o", + "contents": [{ "role": "user", "parts": [{ "text": "Hello from Gemini format!" }] }] + } + }, + { + "name": "gemini-to-openai/multi-turn", + "sourceFormat": "gemini", + "targetFormat": "openai", + "input": { + "model": "gpt-4o-mini", + "contents": [ + { "role": "user", "parts": [{ "text": "What is 2+2?" }] }, + { "role": "model", "parts": [{ "text": "4" }] }, + { "role": "user", "parts": [{ "text": "And 3+3?" }] } + ] + } + } +] diff --git a/tests/fixtures/translation/openai-to-claude.json b/tests/fixtures/translation/openai-to-claude.json new file mode 100644 index 0000000000..52a02d71c3 --- /dev/null +++ b/tests/fixtures/translation/openai-to-claude.json @@ -0,0 +1,37 @@ +[ + { + "name": "openai-to-claude/basic-chat", + "sourceFormat": "openai", + "targetFormat": "claude", + "input": { + "model": "claude-opus-4-5", + "messages": [{ "role": "user", "content": "hi" }] + } + }, + { + "name": "openai-to-claude/with-system", + "sourceFormat": "openai", + "targetFormat": "claude", + "input": { + "model": "claude-sonnet-4-6", + "messages": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "Hello!" } + ], + "max_tokens": 1024 + } + }, + { + "name": "openai-to-claude/multi-turn", + "sourceFormat": "openai", + "targetFormat": "claude", + "input": { + "model": "claude-haiku-4-5", + "messages": [ + { "role": "user", "content": "What is 2+2?" }, + { "role": "assistant", "content": "4" }, + { "role": "user", "content": "And 3+3?" } + ] + } + } +] diff --git a/tests/fixtures/translation/openai-to-gemini.json b/tests/fixtures/translation/openai-to-gemini.json new file mode 100644 index 0000000000..3c91b8cd4f --- /dev/null +++ b/tests/fixtures/translation/openai-to-gemini.json @@ -0,0 +1,24 @@ +[ + { + "name": "openai-to-gemini/basic-chat", + "sourceFormat": "openai", + "targetFormat": "gemini", + "input": { + "model": "gemini-2.5-flash", + "messages": [{ "role": "user", "content": "Hello Gemini!" }] + } + }, + { + "name": "openai-to-gemini/with-system", + "sourceFormat": "openai", + "targetFormat": "gemini", + "input": { + "model": "gemini-2.5-pro", + "messages": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "Tell me a joke." } + ], + "temperature": 0.7 + } + } +] diff --git a/tests/fixtures/translation/sse-chunk-sequences.json b/tests/fixtures/translation/sse-chunk-sequences.json new file mode 100644 index 0000000000..86de18afac --- /dev/null +++ b/tests/fixtures/translation/sse-chunk-sequences.json @@ -0,0 +1,26 @@ +[ + { + "name": "openai-stream/two-deltas-then-done", + "chunks": [ + "data: {\"choices\":[{\"delta\":{\"content\":\"He\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"llo\"}}]}\n\n", + "data: [DONE]\n\n" + ], + "expectedText": "Hello" + }, + { + "name": "openai-stream/single-delta-then-done", + "chunks": ["data: {\"choices\":[{\"delta\":{\"content\":\"World\"}}]}\n\n", "data: [DONE]\n\n"], + "expectedText": "World" + }, + { + "name": "openai-stream/multiple-deltas", + "chunks": [ + "data: {\"choices\":[{\"delta\":{\"content\":\"foo\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\" \"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"bar\"}}]}\n\n", + "data: [DONE]\n\n" + ], + "expectedText": "foo bar" + } +] diff --git a/tests/helpers/fakeUpstreamStream.ts b/tests/helpers/fakeUpstreamStream.ts new file mode 100644 index 0000000000..04bf130f39 --- /dev/null +++ b/tests/helpers/fakeUpstreamStream.ts @@ -0,0 +1,28 @@ +/** + * Controllable fake upstream ReadableStream for integration tests. + * Allows pushing chunks, closing, erroring, and observing cancel. + */ +export function fakeUpstreamStream() { + let controllerRef: ReadableStreamDefaultController | null = null; + let cancelCb: ((reason?: unknown) => void) | null = null; + const enc = new TextEncoder(); + + const stream = new ReadableStream({ + start(controller) { + controllerRef = controller; + }, + cancel(reason) { + cancelCb?.(reason); + }, + }); + + return { + stream, + push: (s: string) => controllerRef?.enqueue(enc.encode(s)), + close: () => controllerRef?.close(), + error: (e: unknown) => controllerRef?.error(e), + onCancel: (cb: (reason?: unknown) => void) => { + cancelCb = cb; + }, + }; +} diff --git a/tests/helpers/goldenSnapshot.ts b/tests/helpers/goldenSnapshot.ts new file mode 100644 index 0000000000..7170cd17c7 --- /dev/null +++ b/tests/helpers/goldenSnapshot.ts @@ -0,0 +1,56 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../snapshots"); + +/** Stable JSON serialization: object keys sorted recursively. */ +function stable(v: unknown): string { + return JSON.stringify( + v, + (_k, val) => + val && typeof val === "object" && !Array.isArray(val) + ? Object.fromEntries( + Object.keys(val as Record) + .sort() + .map((k) => [k, (val as Record)[k]]) + ) + : val, + 2 + ); +} + +class GoldenMismatchError extends Error { + constructor(name: string, expected: string, actual: string) { + super( + `golden mismatch for "${name}". Run with UPDATE_GOLDEN=1 to regenerate.\n--- expected\n${expected}\n--- actual\n${actual}` + ); + this.name = "GoldenMismatchError"; + } +} + +/** + * Compare `value` with a stable JSON snapshot at `tests/snapshots/.json`. + * - First run (file missing): writes the file and passes. + * - UPDATE_GOLDEN=1: rewrites the file and passes. + * - Mismatch: throws GoldenMismatchError with a diff-friendly message. + * + * @param name - Slash-separated path under tests/snapshots/ (e.g. "translation/openai-to-claude/basic-chat") + * @param value - The value to serialize and compare. + * @param dir - Optional override for snapshot root directory (used in tests to isolate to tmpdir). + */ +export function goldenSnapshot(name: string, value: unknown, dir = DEFAULT_DIR): void { + const file = path.join(dir, `${name}.json`); + const serialized = stable(value); + + if (process.env.UPDATE_GOLDEN === "1" || !fs.existsSync(file)) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, serialized + "\n"); + return; + } + + const expected = fs.readFileSync(file, "utf8").trimEnd(); + if (serialized !== expected) { + throw new GoldenMismatchError(name, expected, serialized); + } +} diff --git a/tests/helpers/propertyConfig.ts b/tests/helpers/propertyConfig.ts new file mode 100644 index 0000000000..e43e88a6b5 --- /dev/null +++ b/tests/helpers/propertyConfig.ts @@ -0,0 +1,13 @@ +import fc from "fast-check"; +import { PROPERTY_SEED, PROPERTY_NUM_RUNS } from "../../scripts/quality/property-seed.mjs"; + +const envSeed = process.env.FC_SEED; +export const seed = envSeed === "random" ? undefined : envSeed ? Number(envSeed) : PROPERTY_SEED; +export const numRuns = process.env.FC_NUM_RUNS + ? Number(process.env.FC_NUM_RUNS) + : PROPERTY_NUM_RUNS; + +/** Apply the shared deterministic config; call once at top of each property test file. */ +export function configureProperties(): void { + fc.configureGlobal({ seed, numRuns }); +} diff --git a/tests/helpers/translationFixtures.ts b/tests/helpers/translationFixtures.ts new file mode 100644 index 0000000000..c60e4844ea --- /dev/null +++ b/tests/helpers/translationFixtures.ts @@ -0,0 +1,31 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../fixtures/translation"); + +export type TranslationCase = { + name: string; + sourceFormat: string; + targetFormat: string; + input: Record; + expected?: unknown; +}; + +export type SseSequence = { + name: string; + chunks: string[]; + expectedText: string; +}; + +export function loadTranslationFixtures(): TranslationCase[] { + return ["openai-to-claude", "claude-to-openai", "openai-to-gemini", "gemini-to-openai"].flatMap( + (f) => JSON.parse(fs.readFileSync(path.join(DIR, `${f}.json`), "utf8")) as TranslationCase[] + ); +} + +export function loadSseSequences(): SseSequence[] { + return JSON.parse( + fs.readFileSync(path.join(DIR, "sse-chunk-sequences.json"), "utf8") + ) as SseSequence[]; +} diff --git a/tests/integration/sse-correctness.test.ts b/tests/integration/sse-correctness.test.ts new file mode 100644 index 0000000000..d5874861ca --- /dev/null +++ b/tests/integration/sse-correctness.test.ts @@ -0,0 +1,120 @@ +/** + * SSE-correctness integration tests (Task 11, Fase 8 B). + * + * Drives the real createSSEStream pipeline through a controllable fake upstream. + * Run with: node --import tsx/esm --test --test-concurrency=1 tests/integration/sse-correctness.test.ts + * + * Notes on observed createSSEStream behavior (calibrated invariants): + * - The TransformStream processes SSE events and emits translated chunks to the client. + * - [DONE] is consumed by the pipeline: it closes the upstream readable and the + * TransformStream flushes + terminates, but does NOT re-emit "data: [DONE]" to the output. + * The output stream closes naturally (reader.read() returns {done:true}). + * - Upstream errors propagate as TransformStream errors (reader.read() rejects). + * - Cancel propagates via ReadableStream cancel callback (pipeThrough wires it). + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import { fakeUpstreamStream } from "../helpers/fakeUpstreamStream.ts"; +import { createSSEStream } from "../../open-sse/utils/stream.ts"; +import { FORMATS } from "../../open-sse/translator/formats.ts"; + +/** Drain a ReadableStream to a string, with optional timeout guard. */ +async function drain(out: ReadableStream, timeoutMs = 5000): Promise { + const r = out.getReader(); + const dec = new TextDecoder(); + let s = ""; + const timeout = new Promise((_, reject) => + setTimeout(() => reject(new Error(`drain timeout after ${timeoutMs}ms`)), timeoutMs) + ); + const read = async () => { + for (;;) { + const { done, value } = await r.read(); + if (done) break; + s += dec.decode(value); + } + return s; + }; + return Promise.race([read(), timeout]); +} + +function makeStream(extraOpts: Record = {}) { + const up = fakeUpstreamStream(); + const transform = createSSEStream({ + sourceFormat: FORMATS.OPENAI, + targetFormat: FORMATS.OPENAI, + model: "m", + ...extraOpts, + }); + const out = up.stream.pipeThrough(transform as TransformStream); + return { up, out }; +} + +test("1. stream closes after [DONE] (no hang)", async () => { + const { up, out } = makeStream(); + up.push('data: {"choices":[{"delta":{"content":"hi"}}]}\n\n'); + up.push("data: [DONE]\n\n"); + up.close(); + // drain() must return within the timeout — proves the stream closed + const text = await drain(out); + assert.ok(text.includes("hi"), `expected 'hi' in output: ${JSON.stringify(text)}`); +}); + +test("2. client cancel propagates to upstream (abort propagation)", async () => { + const { up, out } = makeStream(); + let cancelled = false; + up.onCancel(() => { + cancelled = true; + }); + const r = out.getReader(); + await r.cancel("client-abort"); + // Allow microtask queue to flush + await new Promise((res) => setTimeout(res, 50)); + assert.equal(cancelled, true, "upstream cancel callback must have been called"); +}); + +test("3. no leaked idle timers across N sequential streams", async () => { + // createSSEStream installs a setInterval idle watchdog per stream. + // If cleanup (clearInterval) does not run on stream close, timers accumulate. + // This test creates 10 streams and drains them; it acts as a smoke test that + // the process does not hang (a leaked setInterval that fires 10s later would + // prevent the test process from exiting cleanly in --test-force-exit mode). + for (let i = 0; i < 10; i++) { + const { up, out } = makeStream(); + up.push("data: [DONE]\n\n"); + up.close(); + await drain(out); + } + // If we reach here without a timeout, no blocking resources were leaked. + assert.ok(true, "all 10 streams completed without hanging"); +}); + +test("4. final snapshot does not duplicate tail text", async () => { + // Regression guard for the SSE snapshot bug (CLAUDE.md §2, Fase 8 B spec §4.2): + // the text 'Hello' should appear EXACTLY ONCE in the output, not duplicated. + const { up, out } = makeStream(); + up.push('data: {"choices":[{"delta":{"content":"Hello"}}]}\n\n'); + up.push("data: [DONE]\n\n"); + up.close(); + const text = await drain(out); + const occurrences = (text.match(/Hello/g) ?? []).length; + assert.equal( + occurrences, + 1, + `'Hello' appeared ${occurrences} times; expected 1. Output: ${JSON.stringify(text)}` + ); +}); + +test("5. upstream error propagates and closes stream (no hang, Hard Rule #6)", async () => { + // If upstream errors, the TransformStream must propagate the error so the + // consumer sees a rejection — never silently swallow and never hang. + const { up, out } = makeStream(); + up.error(new Error("upstream boom")); + await assert.rejects( + async () => { + await drain(out, 2000); + }, + undefined, // any error is acceptable — just must not hang + "upstream error must propagate as a rejection to the consumer" + ); +}); diff --git a/tests/snapshots/translation/claude-to-openai/basic-message.json b/tests/snapshots/translation/claude-to-openai/basic-message.json new file mode 100644 index 0000000000..606e12e686 --- /dev/null +++ b/tests/snapshots/translation/claude-to-openai/basic-message.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 512, + "messages": [ + { + "content": "Hello", + "role": "user" + } + ], + "model": "gpt-4o", + "stream": true +} diff --git a/tests/snapshots/translation/claude-to-openai/with-system.json b/tests/snapshots/translation/claude-to-openai/with-system.json new file mode 100644 index 0000000000..4b9be272f5 --- /dev/null +++ b/tests/snapshots/translation/claude-to-openai/with-system.json @@ -0,0 +1,15 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": "Be helpful.", + "role": "system" + }, + { + "content": "What is 1+1?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stream": true +} diff --git a/tests/snapshots/translation/gemini-to-openai/basic-chat.json b/tests/snapshots/translation/gemini-to-openai/basic-chat.json new file mode 100644 index 0000000000..43297d969e --- /dev/null +++ b/tests/snapshots/translation/gemini-to-openai/basic-chat.json @@ -0,0 +1,10 @@ +{ + "messages": [ + { + "content": "Hello from Gemini format!", + "role": "user" + } + ], + "model": "gpt-4o", + "stream": true +} diff --git a/tests/snapshots/translation/gemini-to-openai/multi-turn.json b/tests/snapshots/translation/gemini-to-openai/multi-turn.json new file mode 100644 index 0000000000..487346b806 --- /dev/null +++ b/tests/snapshots/translation/gemini-to-openai/multi-turn.json @@ -0,0 +1,18 @@ +{ + "messages": [ + { + "content": "What is 2+2?", + "role": "user" + }, + { + "content": "4", + "role": "assistant" + }, + { + "content": "And 3+3?", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stream": true +} diff --git a/tests/snapshots/translation/openai-to-claude/basic-chat.json b/tests/snapshots/translation/openai-to-claude/basic-chat.json new file mode 100644 index 0000000000..9a28f5ca3f --- /dev/null +++ b/tests/snapshots/translation/openai-to-claude/basic-chat.json @@ -0,0 +1,16 @@ +{ + "max_tokens": 32768, + "messages": [ + { + "content": [ + { + "text": "hi", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-opus-4-5", + "stream": true +} diff --git a/tests/snapshots/translation/openai-to-claude/multi-turn.json b/tests/snapshots/translation/openai-to-claude/multi-turn.json new file mode 100644 index 0000000000..c565c4c70d --- /dev/null +++ b/tests/snapshots/translation/openai-to-claude/multi-turn.json @@ -0,0 +1,34 @@ +{ + "max_tokens": 8192, + "messages": [ + { + "content": [ + { + "text": "What is 2+2?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "4", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And 3+3?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-haiku-4-5", + "stream": true +} diff --git a/tests/snapshots/translation/openai-to-claude/with-system.json b/tests/snapshots/translation/openai-to-claude/with-system.json new file mode 100644 index 0000000000..7abdc103a6 --- /dev/null +++ b/tests/snapshots/translation/openai-to-claude/with-system.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 1024, + "messages": [ + { + "content": [ + { + "text": "Hello!", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-6", + "stream": true, + "system": [ + { + "text": "You are a helpful assistant.", + "type": "text" + } + ] +} diff --git a/tests/snapshots/translation/openai-to-gemini/basic-chat.json b/tests/snapshots/translation/openai-to-gemini/basic-chat.json new file mode 100644 index 0000000000..7c03dfa908 --- /dev/null +++ b/tests/snapshots/translation/openai-to-gemini/basic-chat.json @@ -0,0 +1,38 @@ +{ + "contents": [ + { + "parts": [ + { + "text": "Hello Gemini!" + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 65536 + }, + "model": "gemini-2.5-flash", + "safetySettings": [ + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_CIVIC_INTEGRITY", + "threshold": "OFF" + } + ] +} diff --git a/tests/snapshots/translation/openai-to-gemini/with-system.json b/tests/snapshots/translation/openai-to-gemini/with-system.json new file mode 100644 index 0000000000..4306e1f464 --- /dev/null +++ b/tests/snapshots/translation/openai-to-gemini/with-system.json @@ -0,0 +1,47 @@ +{ + "contents": [ + { + "parts": [ + { + "text": "Tell me a joke." + } + ], + "role": "user" + } + ], + "generationConfig": { + "maxOutputTokens": 8192, + "temperature": 0.7 + }, + "model": "gemini-2.5-pro", + "safetySettings": [ + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "threshold": "OFF" + }, + { + "category": "HARM_CATEGORY_CIVIC_INTEGRITY", + "threshold": "OFF" + } + ], + "systemInstruction": { + "parts": [ + { + "text": "You are a helpful assistant." + } + ], + "role": "system" + } +} diff --git a/tests/unit/correctness/combo.property.test.ts b/tests/unit/correctness/combo.property.test.ts new file mode 100644 index 0000000000..97438ba55f --- /dev/null +++ b/tests/unit/correctness/combo.property.test.ts @@ -0,0 +1,66 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { configureProperties } from "../../helpers/propertyConfig.ts"; +import { validateComboDAG, resolveNestedComboModels } from "../../../open-sse/services/combo.ts"; + +configureProperties(); + +// ComboLike shape: { name: string, models: unknown[] } +// normalizeModelEntry extracts .model (string) from each entry — plain strings work directly. +// "combo:X" notation is NOT used; the resolver checks if models[i] matches combo.name in allCombos. + +test("validateComboDAG throws on self-cycle", () => { + // c0 references itself directly — must throw + const combos = [{ name: "c0", models: ["c0"] }]; + assert.throws(() => validateComboDAG("c0", combos)); +}); + +test("validateComboDAG throws on indirect cycle (a->b->a)", () => { + const combos = [ + { name: "a", models: ["b"] }, + { name: "b", models: ["a"] }, + ]; + assert.throws(() => validateComboDAG("a", combos)); +}); + +test("validateComboDAG does not throw on acyclic graph", () => { + const combos = [ + { name: "a", models: ["b", "openai/gpt-4o"] }, + { name: "b", models: ["anthropic/claude-3-5-sonnet"] }, + ]; + assert.doesNotThrow(() => validateComboDAG("a", combos)); +}); + +test("resolveNestedComboModels never throws and is cycle-safe", () => { + // Arbitrary combo graph: names c0..cN where each combo's models reference + // other combo names by their exact name string (no "combo:" prefix needed). + const graphArb = fc.integer({ min: 1, max: 5 }).chain((n) => { + const names = Array.from({ length: n }, (_, i) => `c${i}`); + const comboArr = names.map((name, i) => { + // Each combo references up to 2 other combos by name plus one real model + const refs = names.filter((_, j) => j !== i).slice(0, 2); + return { name, models: [...refs, "openai/gpt-4o"] }; + }); + return fc.constant(comboArr); + }); + + fc.assert( + fc.property(graphArb, (combos) => { + // resolveNestedComboModels must always return an array (never throw, never loop) + const out = resolveNestedComboModels(combos[0], combos); + assert.ok(Array.isArray(out), "must return array"); + }) + ); +}); + +test("resolveNestedComboModels returns [] on direct cycle (cycle-safety)", () => { + // Cycle: c0->c1->c0 + const combos = [ + { name: "c0", models: ["c1"] }, + { name: "c1", models: ["c0"] }, + ]; + const out = resolveNestedComboModels(combos[0], combos); + // When a cycle is detected, the visited guard returns [] — no infinite loop + assert.ok(Array.isArray(out)); +}); diff --git a/tests/unit/correctness/fakeUpstreamStream.test.ts b/tests/unit/correctness/fakeUpstreamStream.test.ts new file mode 100644 index 0000000000..3a85c1692d --- /dev/null +++ b/tests/unit/correctness/fakeUpstreamStream.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { fakeUpstreamStream } from "../../helpers/fakeUpstreamStream.ts"; + +test("fakeUpstreamStream emits pushed chunks then closes", async () => { + const { stream, push, close } = fakeUpstreamStream(); + push("a"); + push("b"); + close(); + const reader = stream.getReader(); + const dec = new TextDecoder(); + let out = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + out += dec.decode(value); + } + assert.equal(out, "ab"); +}); + +test("fakeUpstreamStream signals cancel when consumer aborts", async () => { + const { stream, onCancel } = fakeUpstreamStream(); + let cancelled = false; + onCancel(() => { + cancelled = true; + }); + const reader = stream.getReader(); + await reader.cancel("client-abort"); + assert.equal(cancelled, true); +}); + +test("fakeUpstreamStream propagates error to reader", async () => { + const { stream, error } = fakeUpstreamStream(); + error(new Error("upstream boom")); + const reader = stream.getReader(); + await assert.rejects(async () => { + await reader.read(); + }, /upstream boom/); +}); diff --git a/tests/unit/correctness/fixtures.test.ts b/tests/unit/correctness/fixtures.test.ts new file mode 100644 index 0000000000..5f5bfa4d06 --- /dev/null +++ b/tests/unit/correctness/fixtures.test.ts @@ -0,0 +1,18 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { loadTranslationFixtures, loadSseSequences } from "../../helpers/translationFixtures.ts"; + +test("fixtures: each pair file has at least one well-formed case", () => { + const cases = loadTranslationFixtures(); + assert.ok(cases.length >= 4); + for (const c of cases) { + assert.ok(c.name && c.sourceFormat && c.targetFormat && c.input); + } +}); +test("fixtures: sse sequences have chunks and expectedText", () => { + const seqs = loadSseSequences(); + assert.ok(seqs.length >= 1); + for (const s of seqs) { + assert.ok(Array.isArray(s.chunks) && typeof s.expectedText === "string"); + } +}); diff --git a/tests/unit/correctness/goldenSnapshot.test.ts b/tests/unit/correctness/goldenSnapshot.test.ts new file mode 100644 index 0000000000..d7f804cea5 --- /dev/null +++ b/tests/unit/correctness/goldenSnapshot.test.ts @@ -0,0 +1,41 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { goldenSnapshot } from "../../helpers/goldenSnapshot.ts"; + +// Use an isolated tmpdir so the selftest does not pollute tests/snapshots/ +let tmpDir: string; + +test("goldenSnapshot writes on first run then matches", (t) => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "golden-selftest-")); + + // First run: UPDATE_GOLDEN=1 → writes + process.env.UPDATE_GOLDEN = "1"; + goldenSnapshot("selftest/sample", { b: 2, a: 1 }, tmpDir); + delete process.env.UPDATE_GOLDEN; + + // Re-run without flag: same value (keys re-ordered) → should pass + assert.doesNotThrow(() => goldenSnapshot("selftest/sample", { a: 1, b: 2 }, tmpDir)); + + // Mismatch: different value → should throw + assert.throws(() => goldenSnapshot("selftest/sample", { a: 1, b: 3 }, tmpDir)); + + // Cleanup + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("goldenSnapshot first-run (no UPDATE_GOLDEN) writes and passes", () => { + const td = fs.mkdtempSync(path.join(os.tmpdir(), "golden-first-run-")); + try { + // File does not exist yet: should write and not throw + assert.doesNotThrow(() => goldenSnapshot("test/value", { x: 42 }, td)); + // File now exists: same value should pass + assert.doesNotThrow(() => goldenSnapshot("test/value", { x: 42 }, td)); + // File exists: different value should throw + assert.throws(() => goldenSnapshot("test/value", { x: 99 }, td)); + } finally { + fs.rmSync(td, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/correctness/sanitizers.property.test.ts b/tests/unit/correctness/sanitizers.property.test.ts new file mode 100644 index 0000000000..e8a8aad04e --- /dev/null +++ b/tests/unit/correctness/sanitizers.property.test.ts @@ -0,0 +1,42 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { configureProperties } from "../../helpers/propertyConfig.ts"; +import { sanitizeErrorMessage } from "../../../open-sse/utils/error.ts"; + +configureProperties(); + +test("sanitizeErrorMessage never leaks a file path / stack frame", () => { + // sanitizeErrorMessage takes only the FIRST LINE of input (observed real behavior: + // it splits on \n and processes only the part before the first newline). + // Stack frames are on subsequent lines and thus already stripped. + // The invariant we test: single-line content containing "at /path/file.ts" has the + // absolute path replaced with "", so the output never contains "at /". + const firstLineWithPath = fc + .string() + .map((s) => s.replace(/\n/g, " ")) // ensure single line + .chain((prefix) => + fc + .string() + .map((s) => s.replace(/\n/g, " ")) + .map((suffix) => `${prefix} at /home/app/open-sse/foo.ts:42:10 ${suffix}`) + ); + + fc.assert( + fc.property(firstLineWithPath, (input) => { + const out = sanitizeErrorMessage(input); + assert.ok(!out.includes("at /"), `leaked path in: ${JSON.stringify(out)}`); + }) + ); +}); + +test("sanitizeErrorMessage terminates on long adversarial input (ReDoS guard)", () => { + fc.assert( + fc.property(fc.integer({ min: 1000, max: 20000 }), (len) => { + const start = process.hrtime.bigint(); + sanitizeErrorMessage("a".repeat(len) + "@" + "b".repeat(len) + ".com " + "1".repeat(len)); + const ms = Number(process.hrtime.bigint() - start) / 1e6; + assert.ok(ms < 250, `too slow: ${ms}ms for len=${len}`); + }) + ); +}); diff --git a/tests/unit/correctness/sse-parser.property.test.ts b/tests/unit/correctness/sse-parser.property.test.ts new file mode 100644 index 0000000000..02fb4bf2c1 --- /dev/null +++ b/tests/unit/correctness/sse-parser.property.test.ts @@ -0,0 +1,63 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { configureProperties } from "../../helpers/propertyConfig.ts"; +import { parseSSELine } from "../../../open-sse/utils/streamHelpers.ts"; +import { loadSseSequences } from "../../helpers/translationFixtures.ts"; + +configureProperties(); + +// parseSSELine is line-oriented: it processes one line at a time. +// The split-mid-line scenario (splitting in the middle of "data: {...}") belongs to +// the stream-layer harness (Task 11). Here we test line-level reconstruction invariance: +// given a complete SSE event line, parseSSELine extracts the correct content regardless +// of what other lines surround it. + +function extractTextFromLines(rawContent: string): string { + let text = ""; + for (const line of rawContent.split("\n")) { + const parsed = parseSSELine(line); + if (parsed && !(parsed as { done?: boolean }).done) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const delta = (parsed as any)?.choices?.[0]?.delta?.content; + if (typeof delta === "string") text += delta; + } + } + return text; +} + +test("SSE text reconstruction is invariant to chunk boundaries (line-level)", () => { + const seqs = loadSseSequences(); + // Property: joining all chunks and re-splitting by line yields the same text + // as processing the chunks individually. This tests that parseSSELine is stateless + // and position-invariant (no dependency on which "chunk" a line comes from). + fc.assert( + fc.property(fc.integer({ min: 0, max: seqs.length - 1 }), (idx) => { + const seq = seqs[idx]; + const full = seq.chunks.join(""); + const reconstructed = extractTextFromLines(full); + assert.equal(reconstructed, seq.expectedText); + }) + ); +}); + +test("parseSSELine returns null for non-data lines", () => { + // Non-data lines (comment, event:, id:, retry:, empty) must return null + const nonDataLines = ["", ": comment", "event: update", "id: 42", "retry: 5000"]; + for (const line of nonDataLines) { + assert.equal(parseSSELine(line), null, `expected null for line: ${JSON.stringify(line)}`); + } +}); + +test("parseSSELine returns {done:true} for [DONE] sentinel", () => { + const parsed = parseSSELine("data: [DONE]"); + assert.deepEqual(parsed, { done: true }); +}); + +test("parseSSELine correctly parses valid JSON data line", () => { + const line = 'data: {"choices":[{"delta":{"content":"hello"}}]}'; + const parsed = parseSSELine(line) as Record; + assert.ok(parsed && typeof parsed === "object"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert.equal((parsed as any)?.choices?.[0]?.delta?.content, "hello"); +}); diff --git a/tests/unit/correctness/translation.golden.test.ts b/tests/unit/correctness/translation.golden.test.ts new file mode 100644 index 0000000000..6a7dd8fe80 --- /dev/null +++ b/tests/unit/correctness/translation.golden.test.ts @@ -0,0 +1,19 @@ +import test from "node:test"; +import { goldenSnapshot } from "../../helpers/goldenSnapshot.ts"; +import { loadTranslationFixtures } from "../../helpers/translationFixtures.ts"; +import { translateRequest } from "../../../open-sse/translator/index.ts"; + +// Golden-file tests: freeze translateRequest output per fixture. +// Regenerate with: UPDATE_GOLDEN=1 node --import tsx/esm --test tests/unit/correctness/translation.golden.test.ts +for (const c of loadTranslationFixtures()) { + test(`golden: ${c.name}`, () => { + const out = translateRequest( + c.sourceFormat, + c.targetFormat, + (c.input as { model?: string }).model ?? "m", + c.input, + true + ); + goldenSnapshot(`translation/${c.name}`, out); + }); +} diff --git a/tests/unit/correctness/translators.property.test.ts b/tests/unit/correctness/translators.property.test.ts new file mode 100644 index 0000000000..178865fe35 --- /dev/null +++ b/tests/unit/correctness/translators.property.test.ts @@ -0,0 +1,78 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { configureProperties } from "../../helpers/propertyConfig.ts"; +import { translateRequest } from "../../../open-sse/translator/index.ts"; +import { FORMATS } from "../../../open-sse/translator/formats.ts"; + +configureProperties(); + +// Non-empty, non-whitespace content required: translator drops messages with empty/whitespace +// content (observed real behavior — empty-content messages produce invalid Claude/Gemini requests). +const messageArb = fc.record({ + role: fc.constantFrom("user", "assistant", "system"), + content: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0), +}); +const bodyArb = fc.record({ + model: fc.string({ minLength: 1 }), + messages: fc.array(messageArb, { minLength: 1, maxLength: 6 }), +}); + +test("translateRequest openai->claude never throws and keeps messages", () => { + fc.assert( + fc.property(bodyArb, (body) => { + const out = translateRequest( + FORMATS.OPENAI, + FORMATS.CLAUDE, + body.model, + body, + true + ) as Record; + assert.ok(out && typeof out === "object"); + // Either messages array is non-empty, or content went to system field (system-only body) + const hasMessages = Array.isArray(out.messages) && (out.messages as unknown[]).length > 0; + const hasSystem = Array.isArray(out.system) && (out.system as unknown[]).length > 0; + assert.ok(hasMessages || hasSystem, "output must have messages or non-empty system"); + }) + ); +}); + +test("translateRequest round-trip openai->claude->openai preserves message count for well-formed input", () => { + // Generate bodies with properly alternating user/assistant roles (no consecutive same role), + // optionally preceded by a system message. This is the well-formed case where Claude's + // merge-consecutive-same-role normalization does NOT collapse messages. + // Invariant-calibration note: arbitrary bodies with consecutive same-role messages get merged + // by the OpenAI→Claude translator (correct behavior — Claude doesn't allow consecutive same-role). + // We test only the well-formed alternating case for an exact-count invariant. + const altBodyArb = fc.record({ + model: fc.string({ minLength: 1 }), + messages: fc.integer({ min: 1, max: 5 }).chain((n) => { + // Build alternating user/assistant sequence, starting with user + const msgs = Array.from({ length: n }, (_, i) => ({ + role: (i % 2 === 0 ? "user" : "assistant") as "user" | "assistant", + content: "msg" + i, + })); + return fc.constant(msgs); + }), + }); + fc.assert( + fc.property(altBodyArb, (body) => { + const claude = translateRequest( + FORMATS.OPENAI, + FORMATS.CLAUDE, + body.model, + body, + true + ) as Record; + const back = translateRequest( + FORMATS.CLAUDE, + FORMATS.OPENAI, + body.model, + claude, + true + ) as Record; + assert.ok(Array.isArray(back.messages)); + assert.equal((back.messages as unknown[]).length, body.messages.length); + }) + ); +});