From 69b3e234000214b610541d44ecc0cd15d618b1e7 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 28 Mar 2026 12:58:31 -0300 Subject: [PATCH] test(tests): introduce feature-tests suite and update coverage tooling - add unit tests for API auth, display/error utilities, login bootstrap, model combo mappings, provider validation branches, and usage analytics - add COVERAGE_PLAN.md and extend CONTRIBUTING.md with coverage notes and workflow guidance - update package.json to adjust test:coverage thresholds and add coverage:report; include c8 as a devDependency - introduce test scaffolding and ensure compatibility with existing test runners - align tests with open-sse changes and improve overall test coverage planning --- CONTRIBUTING.md | 9 +- COVERAGE_PLAN.md | 166 +++++++++ package-lock.json | 317 ++++++++++++++++++ package.json | 6 +- tests/unit/api-auth.test.mjs | 162 +++++++++ tests/unit/display-and-error-utils.test.mjs | 180 ++++++++++ tests/unit/login-bootstrap-route.test.mjs | 92 +++++ tests/unit/model-combo-mappings-db.test.mjs | 186 ++++++++++ .../provider-validation-branches.test.mjs | 202 +++++++++++ tests/unit/usage-analytics.test.mjs | 293 ++++++++++++++++ 10 files changed, 1611 insertions(+), 2 deletions(-) create mode 100644 COVERAGE_PLAN.md create mode 100644 tests/unit/api-auth.test.mjs create mode 100644 tests/unit/display-and-error-utils.test.mjs create mode 100644 tests/unit/model-combo-mappings-db.test.mjs create mode 100644 tests/unit/provider-validation-branches.test.mjs create mode 100644 tests/unit/usage-analytics.test.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b20d06fe2..f227558a07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,6 +114,7 @@ npm run test:fixes # Fix verification tests # With coverage npm run test:coverage +npm run coverage:report # E2E tests (requires Playwright) npm run test:e2e @@ -123,7 +124,13 @@ npm run lint npm run check ``` -Current test status: **368+ unit tests** covering: +Coverage notes: + +- `npm run test:coverage` measures source coverage for the main unit test suite, excludes `tests/**`, and includes `open-sse/**` +- `npm run coverage:report` prints the detailed file-by-file report from the latest coverage run +- `npm run test:coverage:legacy` preserves the older metric for historical comparison + +Current test status: **968+ unit tests** covering: - Provider translators and format conversion - Rate limiting, circuit breaker, and resilience diff --git a/COVERAGE_PLAN.md b/COVERAGE_PLAN.md new file mode 100644 index 0000000000..2c639045fe --- /dev/null +++ b/COVERAGE_PLAN.md @@ -0,0 +1,166 @@ +# Test Coverage Plan + +Last updated: 2026-03-28 + +## Baseline + +There are multiple coverage numbers depending on how the report is computed. For planning, only one of them is useful. + +| Metric | Scope | Statements / Lines | Branches | Functions | Notes | +| -------------------- | ----------------------------------------------------- | -----------------: | -------: | --------: | --------------------------------------------------- | +| Legacy | Old `npm run test:coverage` | 79.42% | 75.15% | 67.94% | Inflated: counts test files and excludes `open-sse` | +| Diagnostic | Source-only, excluding tests and excluding `open-sse` | 68.16% | 63.55% | 64.06% | Useful only to isolate `src/**` | +| Recommended baseline | Source-only, excluding tests and including `open-sse` | 56.95% | 66.05% | 57.80% | This is the project-wide baseline to improve | + +The recommended baseline is the number to optimize against. + +## Rules + +- Coverage targets apply to source files, not to `tests/**`. +- `open-sse/**` is part of the product and must remain in scope. +- New code should not reduce coverage in touched areas. +- Prefer testing behavior and branch outcomes over implementation details. +- Prefer temp SQLite databases and small fixtures over broad mocks for `src/lib/db/**`. + +## Current command set + +- `npm run test:coverage` + - Main source coverage gate for the unit test suite + - Generates `text-summary`, `html`, `json-summary`, and `lcov` +- `npm run coverage:report` + - Detailed file-by-file report from the latest run +- `npm run test:coverage:legacy` + - Historical comparison only + +## Milestones + +| Phase | Target | Focus | +| ------- | ---------------------: | ------------------------------------------------- | +| Phase 1 | 60% statements / lines | Quick wins and low-risk utility coverage | +| Phase 2 | 65% statements / lines | DB and route foundations | +| Phase 3 | 70% statements / lines | Provider validation and usage analytics | +| Phase 4 | 75% statements / lines | `open-sse` translators and helpers | +| Phase 5 | 80% statements / lines | `open-sse` handlers and executor branches | +| Phase 6 | 85% statements / lines | Harder edge cases, branch debt, regression suites | +| Phase 7 | 90% statements / lines | Final sweep, gap closure, strict ratchet | + +Branches and functions should ratchet upward with each phase, but the primary hard target is statements / lines. + +## Priority hotspots + +These files or areas offer the best return for the next phases: + +1. `open-sse/handlers` + - `chatCore.ts` at 7.57% + - Overall directory at 29.07% +2. `open-sse/translator/request` + - Overall directory at 36.39% + - Many translators are still near single-digit coverage +3. `open-sse/translator/response` + - Overall directory at 8.07% +4. `open-sse/executors` + - Overall directory at 36.62% +5. `src/lib/db` + - `models.ts` at 20.66% + - `registeredKeys.ts` at 34.46% + - `modelComboMappings.ts` at 36.25% + - `settings.ts` at 46.40% + - `webhooks.ts` at 33.33% +6. `src/lib/usage` + - `usageHistory.ts` at 21.12% + - `usageStats.ts` at 9.56% + - `costCalculator.ts` at 30.00% +7. `src/lib/providers` + - `validation.ts` at 41.16% +8. Low-risk utility and API files for early gains + - `src/shared/utils/upstreamError.ts` + - `src/shared/utils/apiAuth.ts` + - `src/lib/api/errorResponse.ts` + - `src/app/api/settings/require-login/route.ts` + - `src/app/api/providers/[id]/models/route.ts` + +## Execution checklist + +### Phase 1: 56.95% -> 60% + +- [x] Fix coverage metric so it reflects source code instead of test files +- [x] Keep a legacy coverage script for comparison +- [x] Record the baseline and hotspots in-repo +- [ ] Add focused tests for low-risk utilities: + - `src/shared/utils/upstreamError.ts` + - `src/shared/utils/fetchTimeout.ts` + - `src/lib/api/errorResponse.ts` + - `src/shared/utils/apiAuth.ts` + - `src/lib/display/names.ts` +- [ ] Add route tests for: + - `src/app/api/settings/require-login/route.ts` + - `src/app/api/providers/[id]/models/route.ts` + +### Phase 2: 60% -> 65% + +- [ ] Add DB-backed tests for: + - `src/lib/db/modelComboMappings.ts` + - `src/lib/db/settings.ts` + - `src/lib/db/registeredKeys.ts` +- [ ] Cover branch behavior in: + - `src/lib/providers/validation.ts` + - `src/app/api/v1/embeddings/route.ts` + - `src/app/api/v1/moderations/route.ts` + +### Phase 3: 65% -> 70% + +- [ ] Add usage analytics tests for: + - `src/lib/usage/usageHistory.ts` + - `src/lib/usage/usageStats.ts` + - `src/lib/usage/costCalculator.ts` +- [ ] Expand route coverage for proxy management and settings branches + +### Phase 4: 70% -> 75% + +- [ ] Cover translator helpers and central translation paths: + - `open-sse/translator/index.ts` + - `open-sse/translator/helpers/*` + - `open-sse/translator/request/*` + - `open-sse/translator/response/*` + +### Phase 5: 75% -> 80% + +- [ ] Add handler-level tests for: + - `open-sse/handlers/chatCore.ts` + - `open-sse/handlers/responsesHandler.js` + - `open-sse/handlers/imageGeneration.js` + - `open-sse/handlers/embeddings.js` +- [ ] Add executor branch coverage for provider-specific auth, retries, and endpoint overrides + +### Phase 6: 80% -> 85% + +- [ ] Merge more edge-case suites into the main coverage path +- [ ] Increase function coverage for DB modules with weak constructor/helper coverage +- [ ] Close branch gaps in `settings.ts`, `registeredKeys.ts`, `validation.ts`, and translator helpers + +### Phase 7: 85% -> 90% + +- [ ] Treat the remaining low-coverage files as blockers +- [ ] Add regression tests for every uncovered production bug fixed during the push to 90% +- [ ] Raise the coverage gate in CI only after the local baseline is stable for at least two consecutive runs + +## Ratchet policy + +Update `npm run test:coverage` thresholds only after the project actually exceeds the next milestone with a comfortable buffer. + +Recommended ratchet sequence: + +1. 55/60/55 +2. 60/62/58 +3. 65/64/62 +4. 70/66/66 +5. 75/70/72 +6. 80/75/78 +7. 85/80/84 +8. 90/85/88 + +Order is `statements-lines / branches / functions`. + +## Known gap + +The current coverage command measures the main Node unit suite and includes source reached from it, including `open-sse`. It does not yet merge Vitest coverage into a single unified report. That merge is worth doing later, but it is not a blocker for starting the 60% -> 80% climb. diff --git a/package-lock.json b/package-lock.json index 50ad87b98e..52f1ea8e43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,6 +60,7 @@ "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "c8": "^11.0.0", "concurrently": "^9.2.1", "cross-env": "^10.1.0", "eslint": "^9.39.2", @@ -511,6 +512,16 @@ } } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@braintree/sanitize-url": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", @@ -2160,6 +2171,16 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -6007,6 +6028,13 @@ "@types/node": "*" } }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/js-cookie": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.6.tgz", @@ -7659,6 +7687,40 @@ "node": ">=6.0.0" } }, + "node_modules/c8": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/c8/-/c8-11.0.0.tgz", + "integrity": "sha512-e/uRViGHSVIJv7zsaDKM7VRn2390TgHXqUSvYwPHBQaU6L7E9L0n9JbdkwdYPvshDT0KymBmmlwSpms3yBaMNg==", + "dev": true, + "license": "ISC", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.1", + "@istanbuljs/schema": "^0.1.3", + "find-up": "^5.0.0", + "foreground-child": "^3.1.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.6", + "test-exclude": "^8.0.0", + "v8-to-istanbul": "^9.0.0", + "yargs": "^17.7.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "c8": "bin/c8.js" + }, + "engines": { + "node": "20 || >=22" + }, + "peerDependencies": { + "monocart-coverage-reports": "^2" + }, + "peerDependenciesMeta": { + "monocart-coverage-reports": { + "optional": true + } + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -10522,6 +10584,23 @@ "node": ">=0.10.0" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", @@ -10803,6 +10882,24 @@ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", "license": "MIT" }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -10816,6 +10913,45 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -11282,6 +11418,13 @@ "node": ">=16.9.0" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -12210,6 +12353,45 @@ "node": ">=0.10.0" } }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/iterator.prototype": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz", @@ -13059,6 +13241,35 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/markdown-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", @@ -14461,6 +14672,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mixin-deep": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", @@ -15278,6 +15499,33 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "license": "MIT" }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.2.7", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", + "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/path-to-regexp": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", @@ -18144,6 +18392,60 @@ "node": ">=6" } }, + "node_modules/test-exclude": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-8.0.0.tgz", + "integrity": "sha512-ZOffsNrXYggvU1mDGHk54I96r26P8SyMjO5slMKSc7+IWmtB/MQKnEC2fP51imB3/pT6YK5cT5E8f+Dd9KdyOQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^13.0.6", + "minimatch": "^10.2.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/test-exclude/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/thread-stream": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", @@ -18902,6 +19204,21 @@ "uuid": "dist-node/bin/uuid" } }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/v8n": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/v8n/-/v8n-1.5.1.tgz", diff --git a/package.json b/package.json index b38794b590..ef7f8ff1c4 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,10 @@ "test:protocols:e2e": "node scripts/run-protocol-clients-tests.mjs", "test:vitest": "vitest run open-sse/mcp-server/__tests__/*.test.ts open-sse/services/autoCombo/__tests__/*.test.ts", "test:ecosystem": "node scripts/run-ecosystem-tests.mjs", - "test:coverage": "npx c8 --exclude=open-sse --check-coverage --lines 50 --functions 50 --branches 50 node --import tsx/esm --test tests/unit/*.test.mjs", + "test:coverage": "c8 --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 55 --lines 55 --functions 55 --branches 60 node --import tsx/esm --test tests/unit/*.test.mjs", + "test:coverage:legacy": "c8 --exclude=open-sse --check-coverage --lines 50 --functions 50 --branches 50 node --import tsx/esm --test tests/unit/*.test.mjs", + "coverage:report": "c8 report --exclude=tests/** --exclude=**/*.test.* --reporter=text --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov", + "coverage:report:legacy": "c8 report --exclude=open-sse --reporter=text --reporter=text-summary", "test:all": "npm run test:unit && npm run test:vitest && npm run test:ecosystem && npm run test:e2e", "check": "npm run lint && npm run test", "prepublishOnly": "npm run build:cli", @@ -124,6 +127,7 @@ "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "c8": "^11.0.0", "concurrently": "^9.2.1", "cross-env": "^10.1.0", "eslint": "^9.39.2", diff --git a/tests/unit/api-auth.test.mjs b/tests/unit/api-auth.test.mjs new file mode 100644 index 0000000000..11e069d34d --- /dev/null +++ b/tests/unit/api-auth.test.mjs @@ -0,0 +1,162 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { SignJWT } from "jose"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-api-auth-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = "test-api-key-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const localDb = await import("../../src/lib/localDb.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const apiAuth = await import("../../src/shared/utils/apiAuth.ts"); + +const ORIGINAL_JWT_SECRET = process.env.JWT_SECRET; +const ORIGINAL_INITIAL_PASSWORD = process.env.INITIAL_PASSWORD; + +async function resetStorage() { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + delete process.env.JWT_SECRET; + delete process.env.INITIAL_PASSWORD; +} + +function makeCookieRequest(token) { + return { + cookies: { + get(name) { + return name === "auth_token" && token ? { value: token } : undefined; + }, + }, + headers: new Headers(), + }; +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + + if (ORIGINAL_JWT_SECRET === undefined) { + delete process.env.JWT_SECRET; + } else { + process.env.JWT_SECRET = ORIGINAL_JWT_SECRET; + } + + if (ORIGINAL_INITIAL_PASSWORD === undefined) { + delete process.env.INITIAL_PASSWORD; + } else { + process.env.INITIAL_PASSWORD = ORIGINAL_INITIAL_PASSWORD; + } +}); + +test("isPublicRoute recognizes allowed API prefixes", () => { + assert.equal(apiAuth.isPublicRoute("/api/auth/login"), true); + assert.equal(apiAuth.isPublicRoute("/api/v1/chat/completions"), true); + assert.equal(apiAuth.isPublicRoute("/api/settings"), false); +}); + +test("verifyAuth accepts a valid JWT session cookie", async () => { + process.env.JWT_SECRET = "jwt-secret-for-tests"; + const secret = new TextEncoder().encode(process.env.JWT_SECRET); + const token = await new SignJWT({ authenticated: true }) + .setProtectedHeader({ alg: "HS256" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(secret); + + const result = await apiAuth.verifyAuth(makeCookieRequest(token)); + + assert.equal(result, null); +}); + +test("verifyAuth falls back to bearer API key validation after a bad JWT", async () => { + process.env.JWT_SECRET = "jwt-secret-for-tests"; + const key = await apiKeysDb.createApiKey("integration", "machine1234567890"); + const request = { + cookies: { + get() { + return { value: "definitely-not-a-valid-jwt" }; + }, + }, + headers: new Headers({ authorization: `Bearer ${key.key}` }), + }; + + const result = await apiAuth.verifyAuth(request); + + assert.equal(result, null); +}); + +test("verifyAuth rejects requests without valid credentials", async () => { + const result = await apiAuth.verifyAuth({ + cookies: { + get() { + return undefined; + }, + }, + headers: new Headers({ authorization: "Bearer sk-invalid" }), + }); + + assert.equal(result, "Authentication required"); +}); + +test("isAuthenticated accepts bearer API keys", async () => { + const key = await apiKeysDb.createApiKey("integration", "machine1234567890"); + const request = new Request("https://example.com/api/providers", { + headers: { authorization: `Bearer ${key.key}` }, + }); + + const result = await apiAuth.isAuthenticated(request); + + assert.equal(result, true); +}); + +test("isAuthenticated returns false without valid credentials", async () => { + const request = new Request("https://example.com/api/providers"); + + const result = await apiAuth.isAuthenticated(request); + + assert.equal(result, false); +}); + +test("isAuthRequired is disabled when requireLogin is false", async () => { + await localDb.updateSettings({ requireLogin: false }); + + const result = await apiAuth.isAuthRequired(); + + assert.equal(result, false); +}); + +test("isAuthRequired is disabled while no password exists", async () => { + await localDb.updateSettings({ requireLogin: true, password: "" }); + + const result = await apiAuth.isAuthRequired(); + + assert.equal(result, false); +}); + +test("isAuthRequired stays enabled when a password exists", async () => { + await localDb.updateSettings({ requireLogin: true, password: "hashed-password" }); + + const result = await apiAuth.isAuthRequired(); + + assert.equal(result, true); +}); + +test("isAuthRequired stays enabled when INITIAL_PASSWORD is present", async () => { + process.env.INITIAL_PASSWORD = "bootstrap-password"; + await localDb.updateSettings({ requireLogin: true, password: "" }); + + const result = await apiAuth.isAuthRequired(); + + assert.equal(result, true); +}); diff --git a/tests/unit/display-and-error-utils.test.mjs b/tests/unit/display-and-error-utils.test.mjs new file mode 100644 index 0000000000..5725ab6a6c --- /dev/null +++ b/tests/unit/display-and-error-utils.test.mjs @@ -0,0 +1,180 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { toJsonErrorPayload } = await import("../../src/shared/utils/upstreamError.ts"); +const { createErrorResponse, createErrorResponseFromUnknown } = + await import("../../src/lib/api/errorResponse.ts"); +const { getAccountDisplayName, getProviderDisplayName } = + await import("../../src/lib/display/names.ts"); + +test("toJsonErrorPayload: preserves upstream error objects that already have error payloads", () => { + const payload = { + error: { + message: "provider exploded", + code: "quota_exceeded", + }, + }; + + assert.deepEqual(toJsonErrorPayload(payload), payload); +}); + +test("toJsonErrorPayload: normalizes object payloads with string error", () => { + assert.deepEqual(toJsonErrorPayload({ error: "plain provider error" }), { + error: { + message: "plain provider error", + type: "upstream_error", + code: "upstream_error", + }, + }); +}); + +test("toJsonErrorPayload: wraps plain objects under error key", () => { + assert.deepEqual(toJsonErrorPayload({ status: 503, message: "backend down" }), { + error: { + status: 503, + message: "backend down", + }, + }); +}); + +test("toJsonErrorPayload: parses JSON strings recursively", () => { + const raw = JSON.stringify({ error: { message: "nested json", code: "bad_request" } }); + assert.deepEqual(toJsonErrorPayload(raw), { + error: { + message: "nested json", + code: "bad_request", + }, + }); +}); + +test("toJsonErrorPayload: falls back for blank strings and unsupported values", () => { + const fallback = { + error: { + message: "custom fallback", + type: "upstream_error", + code: "upstream_error", + }, + }; + + assert.deepEqual(toJsonErrorPayload(" ", "custom fallback"), fallback); + assert.deepEqual(toJsonErrorPayload(null, "custom fallback"), fallback); +}); + +test("toJsonErrorPayload: converts non-JSON strings into normalized error payloads", () => { + assert.deepEqual(toJsonErrorPayload("gateway timeout"), { + error: { + message: "gateway timeout", + type: "upstream_error", + code: "upstream_error", + }, + }); +}); + +test("createErrorResponse: infers error types from status and preserves details", async () => { + const response = createErrorResponse({ + status: 409, + message: "Conflict detected", + details: { field: "name" }, + }); + const body = await response.json(); + + assert.equal(response.status, 409); + assert.equal(body.error.message, "Conflict detected"); + assert.equal(body.error.type, "conflict"); + assert.deepEqual(body.error.details, { field: "name" }); + assert.match(body.requestId, /^[0-9a-f-]{36}$/i); +}); + +test("createErrorResponse: uses explicit type when provided", async () => { + const response = createErrorResponse({ + status: 418, + message: "teapot", + type: "not_found", + }); + const body = await response.json(); + + assert.equal(body.error.type, "not_found"); +}); + +test("createErrorResponseFromUnknown: normalizes typed errors", async () => { + const response = createErrorResponseFromUnknown({ + message: "db exploded", + status: 503, + type: "server_error", + details: { retryable: true }, + }); + const body = await response.json(); + + assert.equal(response.status, 503); + assert.equal(body.error.message, "db exploded"); + assert.equal(body.error.type, "server_error"); + assert.deepEqual(body.error.details, { retryable: true }); +}); + +test("createErrorResponseFromUnknown: falls back for non-object errors", async () => { + const response = createErrorResponseFromUnknown("boom", "fallback message"); + const body = await response.json(); + + assert.equal(response.status, 500); + assert.equal(body.error.message, "fallback message"); + assert.equal(body.error.type, "server_error"); +}); + +test("getAccountDisplayName: respects priority order and fallback", () => { + assert.equal( + getAccountDisplayName({ + id: "abcdef123456", + name: "Primary Name", + displayName: "Display Name", + email: "account@example.com", + }), + "Primary Name" + ); + assert.equal( + getAccountDisplayName({ + id: "abcdef123456", + name: " ", + displayName: "Display Name", + email: "account@example.com", + }), + "Display Name" + ); + assert.equal( + getAccountDisplayName({ + id: "abcdef123456", + name: null, + displayName: " ", + email: "account@example.com", + }), + "account@example.com" + ); + assert.equal(getAccountDisplayName({ id: "abcdef123456" }), "Account #abcdef"); + assert.equal(getAccountDisplayName(null), "Unknown Account"); +}); + +test("getProviderDisplayName: prefers node metadata and simplifies compatible IDs", () => { + assert.equal( + getProviderDisplayName("openai-compatible-chat-02669115-2545-4896-b003-cb4dac09d441", { + name: "Friendly Node", + prefix: "ignored-prefix", + }), + "Friendly Node" + ); + assert.equal( + getProviderDisplayName("anthropic-compatible-responses-02669115-2545-4896-b003-cb4dac09d441", { + name: " ", + prefix: "Anthropic Prefix", + }), + "Anthropic Prefix" + ); + assert.equal( + getProviderDisplayName("openai-compatible-chat-02669115-2545-4896-b003-cb4dac09d441"), + "Compatible (openai)" + ); + assert.equal( + getProviderDisplayName("anthropic-compatible-responses-02669115-2545-4896-b003-cb4dac09d441"), + "Compatible (anthropic)" + ); + assert.equal(getProviderDisplayName(undefined), "Unknown Provider"); + assert.equal(getProviderDisplayName("plain-provider-id"), "plain-provider-id"); +}); diff --git a/tests/unit/login-bootstrap-route.test.mjs b/tests/unit/login-bootstrap-route.test.mjs index fc56cbd2ab..05c74fb30b 100644 --- a/tests/unit/login-bootstrap-route.test.mjs +++ b/tests/unit/login-bootstrap-route.test.mjs @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import bcrypt from "bcryptjs"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-login-bootstrap-")); process.env.DATA_DIR = TEST_DATA_DIR; @@ -22,12 +23,18 @@ test.beforeEach(async () => { await resetStorage(); }); +test.afterEach(() => { + bcrypt.hash = originalHash; +}); + test.after(() => { delete process.env.INITIAL_PASSWORD; core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); +const originalHash = bcrypt.hash; + test("public login bootstrap route exposes the metadata the login page consumes", async () => { await settingsDb.updateSettings({ requireLogin: true, @@ -81,3 +88,88 @@ test("public login bootstrap route reports stored password metadata and disabled setupComplete: true, }); }); + +test("public login bootstrap route POST rejects invalid JSON bodies", async () => { + const request = new Request("http://localhost/api/settings/require-login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ invalid json", + }); + + const response = await route.POST(request); + const body = await response.json(); + + assert.equal(response.status, 400); + assert.equal(body.error.message, "Invalid request"); + assert.deepEqual(body.error.details, [{ field: "body", message: "Invalid JSON body" }]); +}); + +test("public login bootstrap route POST rejects empty updates", async () => { + const request = new Request("http://localhost/api/settings/require-login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }); + + const response = await route.POST(request); + const body = await response.json(); + + assert.equal(response.status, 400); + assert.equal(body.error.message, "Invalid request"); + assert.match(body.error.details[0].message, /No valid fields to update/); +}); + +test("public login bootstrap route POST updates requireLogin without forcing password", async () => { + const request = new Request("http://localhost/api/settings/require-login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ requireLogin: false }), + }); + + const response = await route.POST(request); + const body = await response.json(); + const settings = await settingsDb.getSettings(); + + assert.equal(response.status, 200); + assert.deepEqual(body, { success: true }); + assert.equal(settings.requireLogin, false); + assert.equal(settings.password, undefined); +}); + +test("public login bootstrap route POST hashes and stores passwords", async () => { + const password = "super-secret"; + const request = new Request("http://localhost/api/settings/require-login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ requireLogin: true, password }), + }); + + const response = await route.POST(request); + const body = await response.json(); + const settings = await settingsDb.getSettings(); + + assert.equal(response.status, 200); + assert.deepEqual(body, { success: true }); + assert.equal(settings.requireLogin, true); + assert.ok(settings.password); + assert.notEqual(settings.password, password); + assert.equal(await bcrypt.compare(password, settings.password), true); +}); + +test("public login bootstrap route POST returns 500 when hashing fails", async () => { + bcrypt.hash = async () => { + throw new Error("hash failed"); + }; + + const request = new Request("http://localhost/api/settings/require-login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ password: "super-secret" }), + }); + + const response = await route.POST(request); + const body = await response.json(); + + assert.equal(response.status, 500); + assert.deepEqual(body, { error: "hash failed" }); +}); diff --git a/tests/unit/model-combo-mappings-db.test.mjs b/tests/unit/model-combo-mappings-db.test.mjs new file mode 100644 index 0000000000..d95beba93a --- /dev/null +++ b/tests/unit/model-combo-mappings-db.test.mjs @@ -0,0 +1,186 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-combo-db-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const combosDb = await import("../../src/lib/db/combos.ts"); +const mappingsDb = await import("../../src/lib/db/modelComboMappings.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +async function createCombo(name, model) { + return combosDb.createCombo({ + name, + models: [{ provider: "openai", model }], + strategy: "priority", + config: { temperature: 0 }, + }); +} + +test("model combo mappings CRUD joins combo names and preserves ordering", async () => { + const comboA = await createCombo("alpha", "gpt-4o"); + const comboB = await createCombo("beta", "claude-sonnet-4"); + + const first = await mappingsDb.createModelComboMapping({ + pattern: "gpt-*", + comboId: comboA.id, + priority: 20, + description: "primary", + }); + const second = await mappingsDb.createModelComboMapping({ + pattern: "claude-*", + comboId: comboB.id, + priority: 20, + enabled: false, + }); + + const all = await mappingsDb.getModelComboMappings(); + + assert.equal(all.length, 2); + assert.equal(all[0].id, first.id); + assert.equal(all[0].comboName, "alpha"); + assert.equal(all[0].enabled, true); + assert.equal(all[0].description, "primary"); + assert.equal(all[1].id, second.id); + assert.equal(all[1].comboName, "beta"); + assert.equal(all[1].enabled, false); + + const fetched = await mappingsDb.getModelComboMappingById(first.id); + assert.equal(fetched?.pattern, "gpt-*"); + assert.equal(fetched?.comboName, "alpha"); + assert.equal(fetched?.priority, 20); +}); + +test("updateModelComboMapping merges fields and returns the refreshed mapping", async () => { + const comboA = await createCombo("alpha", "gpt-4o"); + const comboB = await createCombo("beta", "claude-sonnet-4"); + + const created = await mappingsDb.createModelComboMapping({ + pattern: "gpt-*", + comboId: comboA.id, + priority: 1, + }); + + const updated = await mappingsDb.updateModelComboMapping(created.id, { + pattern: "claude-*", + comboId: comboB.id, + priority: 99, + enabled: false, + description: "rerouted", + }); + + assert.ok(updated); + assert.equal(updated?.id, created.id); + assert.equal(updated?.pattern, "claude-*"); + assert.equal(updated?.comboId, comboB.id); + assert.equal(updated?.comboName, "beta"); + assert.equal(updated?.priority, 99); + assert.equal(updated?.enabled, false); + assert.equal(updated?.description, "rerouted"); + assert.notEqual(updated?.updatedAt, created.updatedAt); +}); + +test("updateModelComboMapping returns null for unknown ids", async () => { + const updated = await mappingsDb.updateModelComboMapping("missing-id", { + pattern: "gpt-*", + }); + + assert.equal(updated, null); +}); + +test("deleteModelComboMapping reports whether a row existed", async () => { + const combo = await createCombo("alpha", "gpt-4o"); + const created = await mappingsDb.createModelComboMapping({ + pattern: "gpt-*", + comboId: combo.id, + }); + + assert.equal(await mappingsDb.deleteModelComboMapping(created.id), true); + assert.equal(await mappingsDb.deleteModelComboMapping(created.id), false); + assert.equal(await mappingsDb.getModelComboMappingById(created.id), null); +}); + +test("resolveComboForModel returns the highest-priority enabled combo", async () => { + const fallbackCombo = await createCombo("fallback", "gpt-4o-mini"); + const priorityCombo = await createCombo("priority", "gpt-4o"); + const disabledCombo = await createCombo("disabled", "gpt-4.1"); + + await mappingsDb.createModelComboMapping({ + pattern: "*", + comboId: fallbackCombo.id, + priority: 1, + }); + await mappingsDb.createModelComboMapping({ + pattern: "gpt-4*", + comboId: priorityCombo.id, + priority: 10, + }); + await mappingsDb.createModelComboMapping({ + pattern: "gpt-*", + comboId: disabledCombo.id, + priority: 100, + enabled: false, + }); + + const resolved = await mappingsDb.resolveComboForModel("gpt-4o"); + + assert.ok(resolved); + assert.equal(resolved.name, "priority"); + assert.deepEqual(resolved.models, [{ provider: "openai", model: "gpt-4o" }]); +}); + +test("resolveComboForModel skips corrupted combo payloads and keeps scanning", async () => { + const brokenCombo = await createCombo("broken", "gpt-4o"); + const fallbackCombo = await createCombo("fallback", "gpt-4o-mini"); + + await mappingsDb.createModelComboMapping({ + pattern: "gpt-4*", + comboId: brokenCombo.id, + priority: 10, + }); + await mappingsDb.createModelComboMapping({ + pattern: "gpt-*", + comboId: fallbackCombo.id, + priority: 1, + }); + + const db = core.getDbInstance(); + db.prepare("UPDATE combos SET data = ? WHERE id = ?").run("{not-json", brokenCombo.id); + + const resolved = await mappingsDb.resolveComboForModel("gpt-4o"); + + assert.ok(resolved); + assert.equal(resolved.name, "fallback"); +}); + +test("resolveComboForModel returns null when nothing matches", async () => { + const combo = await createCombo("alpha", "gpt-4o"); + + await mappingsDb.createModelComboMapping({ + pattern: "claude-*", + comboId: combo.id, + enabled: false, + }); + + const resolved = await mappingsDb.resolveComboForModel("gpt-4o"); + + assert.equal(resolved, null); +}); diff --git a/tests/unit/provider-validation-branches.test.mjs b/tests/unit/provider-validation-branches.test.mjs new file mode 100644 index 0000000000..ad45301e91 --- /dev/null +++ b/tests/unit/provider-validation-branches.test.mjs @@ -0,0 +1,202 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { validateProviderApiKey } = await import("../../src/lib/providers/validation.ts"); + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +test("validateProviderApiKey rejects missing provider or API key", async () => { + const result = await validateProviderApiKey({ provider: "", apiKey: "" }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Provider and API key required"); + assert.equal(result.unsupported, false); +}); + +test("validateProviderApiKey returns unsupported for unknown providers", async () => { + const result = await validateProviderApiKey({ + provider: "definitely-unknown-provider", + apiKey: "sk-test", + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Provider validation not supported"); + assert.equal(result.unsupported, true); +}); + +test("openai-compatible validation reports missing base URL", async () => { + const result = await validateProviderApiKey({ + provider: "openai-compatible-missing-base", + apiKey: "sk-test", + providerSpecificData: {}, + }); + + assert.equal(result.valid, false); + assert.match(result.error, /No base URL configured/i); +}); + +test("openai-compatible validation accepts rate-limited /models responses", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + return new Response(JSON.stringify({ error: "rate limited" }), { status: 429 }); + }; + + const result = await validateProviderApiKey({ + provider: "openai-compatible-rate-limit", + apiKey: "sk-test", + providerSpecificData: { baseUrl: "https://api.example.com/v1" }, + }); + + assert.equal(result.valid, true); + assert.equal(result.method, "models_endpoint"); + assert.match(result.warning, /Rate limited/i); + assert.deepEqual(calls, ["https://api.example.com/v1/models"]); +}); + +test("openai-compatible validation treats chat 400 as authenticated fallback", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + if (String(url).endsWith("/models")) { + return new Response(JSON.stringify({ error: "server error" }), { status: 500 }); + } + + return new Response(JSON.stringify({ error: "bad model" }), { status: 400 }); + }; + + const result = await validateProviderApiKey({ + provider: "openai-compatible-fallback-chat", + apiKey: "sk-test", + providerSpecificData: { + baseUrl: "https://api.example.com/v1", + validationModelId: "custom-model", + }, + }); + + assert.equal(result.valid, true); + assert.equal(result.method, "inference_available"); + assert.match(result.warning, /Model ID may be invalid/i); + assert.deepEqual(calls, [ + "https://api.example.com/v1/models", + "https://api.example.com/v1/chat/completions", + ]); +}); + +test("openai-compatible validation returns actionable connection failure when probes fail", async () => { + globalThis.fetch = async () => { + throw new Error("socket hang up"); + }; + + const result = await validateProviderApiKey({ + provider: "openai-compatible-network-error", + apiKey: "sk-test", + providerSpecificData: { + baseUrl: "https://api.example.com/v1", + validationModelId: "custom-model", + }, + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Connection failed while testing /chat/completions"); +}); + +test("anthropic-compatible validation requires a base URL", async () => { + const result = await validateProviderApiKey({ + provider: "anthropic-compatible-no-base", + apiKey: "sk-test", + providerSpecificData: {}, + }); + + assert.equal(result.valid, false); + assert.match(result.error, /No base URL configured/i); +}); + +test("anthropic-compatible validation rejects invalid keys from /models", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + return new Response(JSON.stringify({ error: "forbidden" }), { status: 403 }); + }; + + const result = await validateProviderApiKey({ + provider: "anthropic-compatible-bad-key", + apiKey: "sk-test", + providerSpecificData: { baseUrl: "https://api.example.com/v1/messages" }, + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Invalid API key"); + assert.deepEqual(calls, ["https://api.example.com/v1/models"]); +}); + +test("anthropic-compatible validation falls back to /messages and treats 400 as auth success", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + if (String(url).endsWith("/models")) { + throw new Error("models endpoint unavailable"); + } + + return new Response(JSON.stringify({ error: "bad request" }), { status: 400 }); + }; + + const result = await validateProviderApiKey({ + provider: "anthropic-compatible-fallback", + apiKey: "sk-test", + providerSpecificData: { + baseUrl: "https://api.example.com/v1/messages", + validationModelId: "claude-custom", + }, + }); + + assert.equal(result.valid, true); + assert.equal(result.error, null); + assert.deepEqual(calls, [ + "https://api.example.com/v1/models", + "https://api.example.com/v1/messages", + ]); +}); + +test("registry openai-like providers report unsupported validation endpoints on 404 chat probes", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + return new Response(JSON.stringify({ error: "not found" }), { status: 404 }); + }; + + const result = await validateProviderApiKey({ + provider: "openai", + apiKey: "sk-test", + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Provider validation endpoint not supported"); + assert.deepEqual(calls, [ + "https://api.openai.com/v1/models", + "https://api.openai.com/v1/chat/completions", + ]); +}); + +test("gemini validation rejects invalid API keys", async () => { + const calls = []; + globalThis.fetch = async (url) => { + calls.push(String(url)); + return new Response(JSON.stringify({ error: "unauthorized" }), { status: 401 }); + }; + + const result = await validateProviderApiKey({ + provider: "gemini", + apiKey: "bad-key", + }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Invalid API key"); + assert.equal(calls.length, 1); + assert.match(calls[0], /generativelanguage\.googleapis\.com/); + assert.match(calls[0], /key=bad-key/); +}); diff --git a/tests/unit/usage-analytics.test.mjs b/tests/unit/usage-analytics.test.mjs new file mode 100644 index 0000000000..69f06bdc23 --- /dev/null +++ b/tests/unit/usage-analytics.test.mjs @@ -0,0 +1,293 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-usage-analytics-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const localDb = await import("../../src/lib/localDb.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const usageHistory = await import("../../src/lib/usage/usageHistory.ts"); +const usageStats = await import("../../src/lib/usage/usageStats.ts"); +const { calculateCost } = await import("../../src/lib/usage/costCalculator.ts"); +const { LOG_FILE } = await import("../../src/lib/usage/migrations.ts"); + +function clearPendingRequests() { + const pending = usageHistory.getPendingRequests(); + for (const key of Object.keys(pending.byModel)) delete pending.byModel[key]; + for (const key of Object.keys(pending.byAccount)) delete pending.byAccount[key]; +} + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + clearPendingRequests(); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("usage history persists entries and supports filtering and usageDb compatibility", async () => { + const recentTimestamp = new Date().toISOString(); + const olderTimestamp = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(); + + await usageHistory.saveRequestUsage({ + provider: "provider-a", + model: "model-a", + connectionId: "conn-a", + apiKeyId: "key-a", + apiKeyName: "Key A", + tokens: { + input: 10, + output: 5, + cacheRead: 2, + cacheCreation: 1, + reasoning: 3, + }, + status: "success", + success: true, + latencyMs: 120, + timeToFirstTokenMs: 30, + timestamp: recentTimestamp, + }); + + await usageHistory.saveRequestUsage({ + provider: "provider-b", + model: "model-b", + connectionId: "conn-b", + tokens: { + prompt_tokens: 20, + completion_tokens: 7, + cached_tokens: 4, + cache_creation_input_tokens: 2, + reasoning_tokens: 1, + }, + status: "error", + success: false, + latencyMs: 400, + errorCode: "rate_limited", + timestamp: olderTimestamp, + }); + + const filtered = await usageHistory.getUsageHistory({ + provider: "provider-a", + startDate: new Date(Date.now() - 5 * 60 * 1000).toISOString(), + }); + const all = await usageHistory.getUsageDb(); + + assert.equal(filtered.length, 1); + assert.equal(filtered[0].provider, "provider-a"); + assert.equal(filtered[0].tokens.input, 10); + assert.equal(filtered[0].tokens.output, 5); + assert.equal(filtered[0].tokens.cacheRead, 2); + assert.equal(filtered[0].tokens.cacheCreation, 1); + assert.equal(filtered[0].tokens.reasoning, 3); + assert.equal(filtered[0].timeToFirstTokenMs, 30); + + assert.equal(all.data.history.length, 2); + assert.equal(all.data.history[0].provider, "provider-b"); + assert.equal(all.data.history[1].provider, "provider-a"); + assert.equal(all.data.history[0].success, false); + assert.equal(all.data.history[1].success, true); +}); + +test("getModelLatencyStats aggregates success rate and latency percentiles", async () => { + const now = Date.now(); + const entries = [ + { latencyMs: 100, success: true }, + { latencyMs: 200, success: true }, + { latencyMs: 400, success: true }, + { latencyMs: 900, success: false }, + ]; + + for (const [index, entry] of entries.entries()) { + await usageHistory.saveRequestUsage({ + provider: "latency-provider", + model: "latency-model", + success: entry.success, + latencyMs: entry.latencyMs, + timestamp: new Date(now - index * 60 * 1000).toISOString(), + }); + } + + const stats = await usageHistory.getModelLatencyStats({ + windowHours: 1, + minSamples: 2, + maxRows: 50, + }); + + const entry = stats["latency-provider/latency-model"]; + assert.ok(entry); + assert.equal(entry.totalRequests, 4); + assert.equal(entry.successfulRequests, 3); + assert.equal(entry.successRate, 0.75); + assert.equal(entry.avgLatencyMs, 233); + assert.equal(entry.p50LatencyMs, 200); + assert.equal(entry.p95LatencyMs, 400); + assert.equal(entry.p99LatencyMs, 400); + assert.ok(entry.latencyStdDev > 0); +}); + +test("getModelLatencyStats falls back to all latencies when successful sample count is too small", async () => { + await usageHistory.saveRequestUsage({ + provider: "fallback-provider", + model: "fallback-model", + success: true, + latencyMs: 100, + timestamp: new Date().toISOString(), + }); + await usageHistory.saveRequestUsage({ + provider: "fallback-provider", + model: "fallback-model", + success: false, + latencyMs: 500, + timestamp: new Date().toISOString(), + }); + + const stats = await usageHistory.getModelLatencyStats({ + windowHours: 1, + minSamples: 2, + }); + + const entry = stats["fallback-provider/fallback-model"]; + assert.ok(entry); + assert.equal(entry.successRate, 0.5); + assert.equal(entry.avgLatencyMs, 300); + assert.equal(entry.p50LatencyMs, 500); +}); + +test("getUsageStats aggregates totals, buckets, pending requests, and cost breakdowns", async () => { + await localDb.updatePricing({ + "pricing-provider": { + "pricing-model": { + input: 1000, + cached: 100, + output: 2000, + reasoning: 3000, + cache_creation: 1500, + }, + }, + }); + + const connection = await providersDb.createProviderConnection({ + provider: "pricing-provider", + authType: "apikey", + name: "Primary Account", + apiKey: "sk-test", + }); + + const recentTokens = { + input: 100, + output: 50, + cacheRead: 20, + cacheCreation: 10, + reasoning: 5, + }; + const oldTokens = { + input: 40, + output: 10, + cacheRead: 0, + cacheCreation: 0, + reasoning: 0, + }; + + await usageHistory.saveRequestUsage({ + provider: "pricing-provider", + model: "pricing-model", + connectionId: connection.id, + apiKeyId: "api-key-1", + apiKeyName: "Service Key", + tokens: recentTokens, + success: true, + latencyMs: 150, + timestamp: new Date().toISOString(), + }); + await usageHistory.saveRequestUsage({ + provider: "pricing-provider", + model: "pricing-model", + connectionId: connection.id, + apiKeyId: "api-key-1", + apiKeyName: "Service Key", + tokens: oldTokens, + success: true, + latencyMs: 80, + timestamp: new Date(Date.now() - 20 * 60 * 1000).toISOString(), + }); + + usageHistory.trackPendingRequest("pricing-model", "pricing-provider", connection.id, true); + usageHistory.trackPendingRequest("pricing-model", "pricing-provider", connection.id, true); + usageHistory.trackPendingRequest("pricing-model", "pricing-provider", connection.id, false); + + const stats = await usageStats.getUsageStats(); + const expectedCost = + (await calculateCost("pricing-provider", "pricing-model", recentTokens)) + + (await calculateCost("pricing-provider", "pricing-model", oldTokens)); + + assert.equal(stats.totalRequests, 2); + assert.equal(stats.totalPromptTokens, 140); + assert.equal(stats.totalCompletionTokens, 60); + assert.ok(Math.abs(stats.totalCost - expectedCost) < 1e-9); + + assert.equal(stats.byProvider["pricing-provider"].requests, 2); + assert.equal(stats.byProvider["pricing-provider"].promptTokens, 140); + assert.equal(stats.byModel["pricing-model (pricing-provider)"].requests, 2); + + const accountKey = "pricing-model (pricing-provider - Primary Account)"; + assert.equal(stats.byAccount[accountKey].requests, 2); + assert.equal(stats.byAccount[accountKey].accountName, "Primary Account"); + + assert.equal(stats.byApiKey["Service Key (api-key-1)"].requests, 2); + assert.equal(stats.pending.byModel["pricing-model (pricing-provider)"], 1); + assert.equal(stats.pending.byAccount[connection.id]["pricing-model (pricing-provider)"], 1); + assert.deepEqual(stats.activeRequests, [ + { + model: "pricing-model", + provider: "pricing-provider", + account: "Primary Account", + count: 1, + }, + ]); + + assert.equal(stats.last10Minutes.length, 10); + const recentBucketTotal = stats.last10Minutes.reduce((sum, bucket) => sum + bucket.requests, 0); + assert.equal(recentBucketTotal, 1); +}); + +test("request log appends readable entries and trims to the most recent 200 lines", async () => { + const connection = await providersDb.createProviderConnection({ + provider: "log-provider", + authType: "apikey", + name: "Named Account", + apiKey: "sk-test", + }); + + for (let i = 0; i < 205; i++) { + await usageHistory.appendRequestLog({ + model: `model-${i}`, + provider: "log-provider", + connectionId: connection.id, + tokens: { input: i + 1, output: i + 2 }, + status: 200, + }); + } + + const recent = await usageHistory.getRecentLogs(3); + const lines = fs.readFileSync(LOG_FILE, "utf8").trim().split("\n"); + + assert.equal(lines.length, 200); + assert.equal(recent.length, 3); + assert.match(recent[0], /model-204/); + assert.match(recent[0], /LOG-PROVIDER/); + assert.match(recent[0], /Named Account/); + assert.match(recent[0], /205 \| 206 \| 200$/); +});