diff --git a/AGENTS.md b/AGENTS.md index e26971c64e..5707caf104 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,10 +3,44 @@ ## Project Unified AI proxy/router — route any LLM through one endpoint. Multi-provider support -with **212 providers** (OpenAI, Anthropic, Gemini, DeepSeek, Groq, xAI, Mistral, Fireworks, +with **229 provider entries** (OpenAI, Anthropic, Gemini, DeepSeek, Groq, xAI, Mistral, Fireworks, Cohere, NVIDIA, Cerebras, Pollinations, Puter, Cloudflare AI, HuggingFace, DeepInfra, SambaNova, Meta Llama API, Moonshot AI, AI21 Labs, Databricks, Snowflake, and many more) -with **MCP Server** (37 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. +with **MCP Server** (69 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. + +> **Live counts (v3.8.16)**: providers 229 · MCP tools 69 · MCP scopes 13 · A2A skills 6 · +> open-sse services 111 · routing strategies 15 · auto-combo scoring factors 12 · +> DB modules 76 · DB migrations 94 · base tables 17 · search providers 12 · +> i18n locales 42. **Refresh with `npm run check:docs-all`.** + +## Doc Accuracy Discipline (read before writing any doc) + +> **If `grep -rn "name" src/ open-sse/ bin/` returns nothing, the name does not exist. Do not document it.** + +The recurring failure mode in AI-generated docs is _plausible-but-unverified specifics_. +Every claim in a `.md` file under `docs/` should be verifiable against the source. + +**Rules (enforced by `npm run check:fabricated-docs`):** + +1. **Never state an API name, endpoint, path, CLI command, or env var without grepping for it first.** + ```bash + grep -rn "theName" src/ open-sse/ bin/ + # 0 hits → do not document + ``` +2. **Never write a line count, file size, migration count, provider count, or strategy count from memory.** + ```bash + wc -l # exact line count + ls /*.ts | wc -l # file count + ``` +3. **Every code example should be copy-pasted from real usage or actually run** — not synthesized. + Link to a real call site (`path:line`) instead of inventing a signature. +4. **Prefer citing real source (`file.ts:line`) over paraphrasing behavior** — verifiable and self-correcting. +5. **A shorter doc that is 100% accurate beats a comprehensive one with fabrications.** + Wrong docs cost more than missing docs, because people trust and act on them. + +The script `scripts/check/check-fabricated-docs.mjs` extracts every route path, env var, hook +name, function name, and file reference from `docs/**/*.md` and verifies each one against the +codebase. Run it locally before pushing docs; it runs in CI via `npm run check:docs-all`. ## Stack @@ -15,7 +49,7 @@ with **MCP Server** (37 tools), **A2A v0.3 Protocol**, and **Electron desktop ap - **Database**: better-sqlite3 (SQLite) — `DATA_DIR` configurable, default `~/.omniroute/` - **Streaming**: SSE via `open-sse` internal workspace package - **Styling**: Tailwind CSS v4 -- **i18n**: next-intl with 40+ languages +- **i18n**: next-intl with 42 locales (`src/i18n/messages/`) — refresh with `ls src/i18n/messages/*.json | wc -l` - **Desktop**: Electron (cross-platform: Windows, macOS, Linux) - **Schemas**: Zod v4 for all API / MCP input validation @@ -23,28 +57,28 @@ with **MCP Server** (37 tools), **A2A v0.3 Protocol**, and **Electron desktop ap ## Build, Lint, and Test Commands -| Command | Description | -| ----------------------------------- | ---------------------------------------------------------------- | -| `npm run dev` | Start Next.js dev server | +| Command | Description | +| ----------------------------------- | ------------------------------------------------------------------ | +| `npm run dev` | Start Next.js dev server | | `npm run build` | Production build: `next build` → `.build/next/` + assemble `dist/` | -| `npm run build:release` | Clean rebuild + HEAD sentinel (`dist/BUILD_SHA`) — use for deploy | -| `npm run start` | Run production build | -| `npm run build:cli` | Build CLI package | -| `npm run lint` | ESLint on all source files | -| `npm run typecheck:core` | TypeScript core type checking | -| `npm run typecheck:noimplicit:core` | Strict checking (no implicit any) | -| `npm run check` | Run lint + test | -| `npm run check:cycles` | Check for circular dependencies | -| `npm run electron:dev` | Run Electron app in dev mode | -| `npm run electron:build` | Build Electron app for current OS | +| `npm run build:release` | Clean rebuild + HEAD sentinel (`dist/BUILD_SHA`) — use for deploy | +| `npm run start` | Run production build | +| `npm run build:cli` | Build CLI package | +| `npm run lint` | ESLint on all source files | +| `npm run typecheck:core` | TypeScript core type checking | +| `npm run typecheck:noimplicit:core` | Strict checking (no implicit any) | +| `npm run check` | Run lint + test | +| `npm run check:cycles` | Check for circular dependencies | +| `npm run electron:dev` | Run Electron app in dev mode | +| `npm run electron:build` | Build Electron app for current OS | **Build output layout:** -| Directory | Purpose | Gitignored | -| --------- | --------------------------------------------------- | ---------- | -| `src/` | Application source (TypeScript / TSX) | No | -| `.build/` | Build intermediates (`distDir = .build/next`) | Yes | -| `dist/` | Shippable bundle assembled by `assembleStandalone` | Yes | +| Directory | Purpose | Gitignored | +| --------- | -------------------------------------------------- | ---------- | +| `src/` | Application source (TypeScript / TSX) | No | +| `.build/` | Build intermediates (`distDir = .build/next`) | Yes | +| `dist/` | Shippable bundle assembled by `assembleStandalone` | Yes | The pipeline is a single `next build` pass — intermediates land in `.build/next/`, the assembled bundle in `dist/`. VPS deploys rsync `dist/` into the remote @@ -144,7 +178,7 @@ Always run `prettier --write` on changed files. ### Data Layer (`src/lib/db/`) -All persistence uses SQLite through **45+ domain-specific modules** in `src/lib/db/`. Top modules: +All persistence uses SQLite through **76 domain-specific modules** in `src/lib/db/`. Top modules: - Core: `core.ts`, `migrationRunner.ts`, `encryption.ts`, `stateReset.ts` - Providers / catalog: `providers.ts`, `models.ts`, `providerLimits.ts`, `compressionAnalytics.ts` @@ -154,17 +188,17 @@ All persistence uses SQLite through **45+ domain-specific modules** in `src/lib/ - Storage: `backup.ts`, `cleanup.ts`, `jsonMigration.ts`, `healthCheck.ts`, `databaseSettings.ts` - Extension modules: `evals.ts`, `webhooks.ts`, `reasoningCache.ts`, `readCache.ts`, `tierConfig.ts`, `compressionCombos.ts`, `compressionScheduler.ts`, `batches.ts`, `files.ts`, `syncTokens.ts`, `proxies.ts`, `oneproxy.ts`, `upstreamProxy.ts`, `versionManager.ts`, `cliToolState.ts`, `prompts.ts`, `detailedLogs.ts`, `contextHandoffs.ts`, `compression.ts`, `stats.ts` -Live count: `ls src/lib/db/*.ts | wc -l` (currently 45). -Schema migrations live in `db/migrations/` (55 files) and run via `migrationRunner.ts`. +Live count: `ls src/lib/db/*.ts | wc -l` (currently 76). Drift detection: `npm run check:docs-counts`. +Schema migrations live in `db/migrations/` (**94 files** as of v3.8.16) and run via `migrationRunner.ts`. `src/lib/localDb.ts` is a **re-export layer only** — never add logic there. #### DB Internals - **`core.ts`**: `getDbInstance()` returns a singleton `better-sqlite3` instance with WAL - journaling. `SCHEMA_SQL` defines 15 base tables. Helpers: `rowToCamel`, `encryptConnectionFields`. + journaling. `SCHEMA_SQL` defines **17 base tables** (verify with `grep -c "CREATE TABLE" src/lib/db/core.ts` minus 1 for the bookkeeping `_omniroute_migrations` table). Helpers: `rowToCamel`, `encryptConnectionFields`. - **`migrationRunner.ts`**: Applies versioned SQL files from `db/migrations/` inside transactions. Tracks applied migrations in `_omniroute_migrations` table. -- **Migrations**: 55 files (`001_initial_schema.sql` → `055_command_code_auth_sessions.sql`). +- **Migrations**: 94 files (`001_initial_schema.sql` → `094_*.sql`). Each migration is idempotent and runs in a transaction. Live count: `ls src/lib/db/migrations/*.sql | wc -l`. - **Domain modules** import `getDbInstance()` from `core.ts` for all CRUD operations. Each module owns a specific table/set of tables (e.g., `providers.ts` → `provider_connections`, @@ -180,19 +214,19 @@ Route → CORS preflight → Body validation (Zod) → Optional auth (extractApi → API key policy enforcement (enforceApiKeyPolicy) → Handler delegation (open-sse) ``` -| Route | Handler | Notes | -| ------------------------------- | ------------------------- | ----------------------------------------- | -| `chat/completions/route.ts` | `handleChat()` | + prompt injection guard (clones request) | -| `responses/route.ts` | `handleChat()` (unified) | Responses API format | -| `embeddings/route.ts` | `handleEmbedding()` | Model listing + creation | -| `images/generations/route.ts` | `handleImageGeneration()` | Model listing + creation | -| `audio/transcriptions/route.ts` | audio handler | Multipart form data | -| `audio/speech/route.ts` | TTS handler | Binary audio response | -| `videos/generations/route.ts` | video handler | ComfyUI/SD WebUI | -| `music/generations/route.ts` | music handler | ComfyUI workflows | -| `moderations/route.ts` | moderation handler | Content safety | -| `rerank/route.ts` | rerank handler | Document relevance | -| `search/route.ts` | search handler | Web search (5 providers) | +| Route | Handler | Notes | +| ------------------------------- | ------------------------- | ------------------------------------------------------------- | +| `chat/completions/route.ts` | `handleChat()` | + prompt injection guard (clones request) | +| `responses/route.ts` | `handleChat()` (unified) | Responses API format | +| `embeddings/route.ts` | `handleEmbedding()` | Model listing + creation | +| `images/generations/route.ts` | `handleImageGeneration()` | Model listing + creation | +| `audio/transcriptions/route.ts` | audio handler | Multipart form data | +| `audio/speech/route.ts` | TTS handler | Binary audio response | +| `videos/generations/route.ts` | video handler | ComfyUI/SD WebUI | +| `music/generations/route.ts` | music handler | ComfyUI workflows | +| `moderations/route.ts` | moderation handler | Content safety | +| `rerank/route.ts` | rerank handler | Document relevance | +| `search/route.ts` | search handler | Web search (12 providers per `open-sse/handlers/search.ts:6`) | **No global Next.js middleware file** — interception is route-specific. Auth is optional (controlled by `REQUIRE_API_KEY` env). Prompt injection guard is unique to chat completions. @@ -302,7 +336,8 @@ Includes request/response translators with helpers for image handling. ### Services (`open-sse/services/`) -36+ service modules including: `combo.ts` (routing engine), `usage.ts`, `tokenRefresh.ts`, +111 service modules in `open-sse/services/` (top-level only; 171 including sub-dirs like `autoCombo/` and `compression/`). Refresh: `ls open-sse/services/*.ts | wc -l`. Key modules: +`combo.ts` (routing engine), `usage.ts`, `tokenRefresh.ts`, `rateLimitManager.ts`, `accountFallback.ts`, `sessionManager.ts`, `wildcardRouter.ts`, `autoCombo/`, `intentClassifier.ts`, `taskAwareRouter.ts`, `thinkingBudget.ts`, `contextManager.ts`, `modelDeprecation.ts`, `modelFamilyFallback.ts`, @@ -343,8 +378,8 @@ Modular prompt compression that runs proactively before the existing reactive co and iterates through targets in order until one succeeds or all fail. - **`resolveComboTargets()`**: Expands a combo configuration into an ordered array of `ResolvedComboTarget[]`, each specifying provider + model + account + credentials. -- **Strategies** (14): priority, weighted, fill-first, round-robin, P2C, random, least-used, reset-aware (v3.8), - cost-optimized, strict-random, auto, lkgp, context-optimized, context-relay. +- **Strategies** (15): priority, weighted, fill-first, round-robin, P2C, random, least-used, reset-aware (v3.8), + reset-window, cost-optimized, strict-random, auto, lkgp, context-optimized, context-relay. Source: `ROUTING_STRATEGY_VALUES` in `src/shared/constants/routingStrategies.ts`. - Each target calls **`handleSingleModel()`** which wraps `handleChatCore()` with per-target error handling and circuit breaker checks. @@ -356,7 +391,7 @@ Policy engine modules: `policyEngine.ts`, `comboResolver.ts`, `costRules.ts`, ### MCP Server (`open-sse/mcp-server/`) -37 tools (30 base + 3 memory + 4 skills), 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (~13 scopes), Zod schemas. See [`docs/frameworks/MCP-SERVER.md`](docs/frameworks/MCP-SERVER.md). +69 tools across 10 tool files (advancedTools: 5, agentSkillTools: 3, compressionTools: 5, gamificationTools: 8, memoryTools: 3, notionTools: 6, obsidianTools: 22, pluginTools: 8, poolTools: 5, skillTools: 4), 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (13 scopes — see `OMNIROUTE_MCP_SCOPES` in `open-sse/mcp-server/README.md:51`), Zod schemas. See [`docs/frameworks/MCP-SERVER.md`](docs/frameworks/MCP-SERVER.md). **Core tools** (20): get_health, list_combos, get_combo_metrics, switch_combo, check_quota, route_request, cost_report, list_models_catalog, web_search, simulate_route, set_budget_guard, @@ -382,7 +417,7 @@ handler: async (args) => {...} }`. Zod validates inputs before the handler fires `createMcpServer()` wires all tool sets; `startMcpStdio()` launches the stdio transport. - **Transports**: stdio (CLI `omniroute --mcp`), SSE (`/api/mcp/sse`), Streamable HTTP (`/api/mcp/stream`). All share the same tool/scope engine. -- **Scopes** (10): Control which tool categories an API key can access. Enforcement happens +- **Scopes** (13): Control which tool categories an API key can access. Enforcement happens before handler dispatch. - **Audit**: Every tool invocation is logged to SQLite (`mcp_audit` table) with tool name, args, success/failure, API key attribution, and timestamp. @@ -391,7 +426,7 @@ handler: async (args) => {...} }`. Zod validates inputs before the handler fires JSON-RPC 2.0, SSE streaming, Task Manager with TTL cleanup. Agent Card at `/.well-known/agent.json`. -Skills (5): `smartRouting.ts`, `quotaManagement.ts`, `providerDiscovery.ts`, `costAnalysis.ts`, `healthReport.ts`. +Skills (6): `smartRouting.ts`, `quotaManagement.ts`, `providerDiscovery.ts`, `costAnalysis.ts`, `healthReport.ts`, `listCapabilities.ts`. #### A2A Internals @@ -492,31 +527,31 @@ Cloudflare Quick/Named, ngrok, Tailscale Funnel. See [`docs/ops/TUNNELS_GUIDE.md For any non-trivial change, read the matching deep-dive first: -| Area | Doc | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | -| Repo navigation | [`docs/architecture/REPOSITORY_MAP.md`](docs/architecture/REPOSITORY_MAP.md) | -| Architecture | [`docs/architecture/ARCHITECTURE.md`](docs/architecture/ARCHITECTURE.md) | -| Engineering reference | [`docs/architecture/CODEBASE_DOCUMENTATION.md`](docs/architecture/CODEBASE_DOCUMENTATION.md) | -| Auto-Combo (9-factor, 14 strategies) | [`docs/routing/AUTO-COMBO.md`](docs/routing/AUTO-COMBO.md) | -| Resilience (3 layers) | [`docs/architecture/RESILIENCE_GUIDE.md`](docs/architecture/RESILIENCE_GUIDE.md) | -| Skills | [`docs/frameworks/SKILLS.md`](docs/frameworks/SKILLS.md) | -| Memory | [`docs/frameworks/MEMORY.md`](docs/frameworks/MEMORY.md) | -| Cloud agents | [`docs/frameworks/CLOUD_AGENT.md`](docs/frameworks/CLOUD_AGENT.md) | -| Guardrails | [`docs/security/GUARDRAILS.md`](docs/security/GUARDRAILS.md) | -| Evals | [`docs/frameworks/EVALS.md`](docs/frameworks/EVALS.md) | -| Compliance | [`docs/security/COMPLIANCE.md`](docs/security/COMPLIANCE.md) | -| Webhooks | [`docs/frameworks/WEBHOOKS.md`](docs/frameworks/WEBHOOKS.md) | -| Authz | [`docs/architecture/AUTHZ_GUIDE.md`](docs/architecture/AUTHZ_GUIDE.md) | -| Stealth | [`docs/security/STEALTH_GUIDE.md`](docs/security/STEALTH_GUIDE.md) | -| Reasoning replay | [`docs/routing/REASONING_REPLAY.md`](docs/routing/REASONING_REPLAY.md) | -| Agent protocols (A2A / ACP / Cloud) | [`docs/frameworks/AGENT_PROTOCOLS_GUIDE.md`](docs/frameworks/AGENT_PROTOCOLS_GUIDE.md) | -| MCP server | [`docs/frameworks/MCP-SERVER.md`](docs/frameworks/MCP-SERVER.md) | -| A2A server | [`docs/frameworks/A2A-SERVER.md`](docs/frameworks/A2A-SERVER.md) | -| API reference | [`docs/reference/API_REFERENCE.md`](docs/reference/API_REFERENCE.md) + [`docs/reference/openapi.yaml`](docs/reference/openapi.yaml) | -| Provider catalog (auto-generated) | [`docs/reference/PROVIDER_REFERENCE.md`](docs/reference/PROVIDER_REFERENCE.md) | -| Tunnels | [`docs/ops/TUNNELS_GUIDE.md`](docs/ops/TUNNELS_GUIDE.md) | -| Electron desktop | [`docs/guides/ELECTRON_GUIDE.md`](docs/guides/ELECTRON_GUIDE.md) | -| Release flow | [`docs/ops/RELEASE_CHECKLIST.md`](docs/ops/RELEASE_CHECKLIST.md) | +| Area | Doc | +| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| Repo navigation | [`docs/architecture/REPOSITORY_MAP.md`](docs/architecture/REPOSITORY_MAP.md) | +| Architecture | [`docs/architecture/ARCHITECTURE.md`](docs/architecture/ARCHITECTURE.md) | +| Engineering reference | [`docs/architecture/CODEBASE_DOCUMENTATION.md`](docs/architecture/CODEBASE_DOCUMENTATION.md) | +| Auto-Combo (12-factor, 15 strategies) | [`docs/routing/AUTO-COMBO.md`](docs/routing/AUTO-COMBO.md) | +| Resilience (3 layers) | [`docs/architecture/RESILIENCE_GUIDE.md`](docs/architecture/RESILIENCE_GUIDE.md) | +| Skills | [`docs/frameworks/SKILLS.md`](docs/frameworks/SKILLS.md) | +| Memory | [`docs/frameworks/MEMORY.md`](docs/frameworks/MEMORY.md) | +| Cloud agents | [`docs/frameworks/CLOUD_AGENT.md`](docs/frameworks/CLOUD_AGENT.md) | +| Guardrails | [`docs/security/GUARDRAILS.md`](docs/security/GUARDRAILS.md) | +| Evals | [`docs/frameworks/EVALS.md`](docs/frameworks/EVALS.md) | +| Compliance | [`docs/security/COMPLIANCE.md`](docs/security/COMPLIANCE.md) | +| Webhooks | [`docs/frameworks/WEBHOOKS.md`](docs/frameworks/WEBHOOKS.md) | +| Authz | [`docs/architecture/AUTHZ_GUIDE.md`](docs/architecture/AUTHZ_GUIDE.md) | +| Stealth | [`docs/security/STEALTH_GUIDE.md`](docs/security/STEALTH_GUIDE.md) | +| Reasoning replay | [`docs/routing/REASONING_REPLAY.md`](docs/routing/REASONING_REPLAY.md) | +| Agent protocols (A2A / ACP / Cloud) | [`docs/frameworks/AGENT_PROTOCOLS_GUIDE.md`](docs/frameworks/AGENT_PROTOCOLS_GUIDE.md) | +| MCP server | [`docs/frameworks/MCP-SERVER.md`](docs/frameworks/MCP-SERVER.md) | +| A2A server | [`docs/frameworks/A2A-SERVER.md`](docs/frameworks/A2A-SERVER.md) | +| API reference | [`docs/reference/API_REFERENCE.md`](docs/reference/API_REFERENCE.md) + [`docs/reference/openapi.yaml`](docs/reference/openapi.yaml) | +| Provider catalog (auto-generated) | [`docs/reference/PROVIDER_REFERENCE.md`](docs/reference/PROVIDER_REFERENCE.md) | +| Tunnels | [`docs/ops/TUNNELS_GUIDE.md`](docs/ops/TUNNELS_GUIDE.md) | +| Electron desktop | [`docs/guides/ELECTRON_GUIDE.md`](docs/guides/ELECTRON_GUIDE.md) | +| Release flow | [`docs/ops/RELEASE_CHECKLIST.md`](docs/ops/RELEASE_CHECKLIST.md) | --- diff --git a/package.json b/package.json index 1dc3c8ec0d..6ff9d2cb28 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,8 @@ "check:docs-counts": "node scripts/check/check-docs-counts-sync.mjs", "check:deprecated-versions": "node scripts/check/check-deprecated-versions.mjs", "check:doc-links": "node scripts/check/check-doc-links.mjs", - "check:docs-all": "npm run check:docs-sync && npm run check:docs-counts && npm run check:env-doc-sync && npm run check:deprecated-versions && npm run check:doc-links", + "check:fabricated-docs": "node scripts/check/check-fabricated-docs.mjs", + "check:docs-all": "npm run check:docs-sync && npm run check:docs-counts && npm run check:env-doc-sync && npm run check:deprecated-versions && npm run check:doc-links && npm run check:fabricated-docs", "docs:render-diagrams": "node scripts/docs/render-diagrams.mjs", "i18n:run": "node scripts/i18n/run-translation.mjs", "i18n:run:dry": "node scripts/i18n/run-translation.mjs --dry-run", diff --git a/scripts/check/check-fabricated-docs.mjs b/scripts/check/check-fabricated-docs.mjs new file mode 100644 index 0000000000..36f1a9d64b --- /dev/null +++ b/scripts/check/check-fabricated-docs.mjs @@ -0,0 +1,701 @@ +#!/usr/bin/env node +// Doc accuracy gate — catches fabricated API/endpoint/function/env-var claims in docs. +// +// Scans every `docs/{*,*/*}.md` and `AGENTS.md` for concrete code references and +// verifies each one against the source. Reports drift as warnings (soft-fail +// by default) and exits 1 with `--strict` so CI can block fabricated claims. +// +// What it checks: +// 1. /api/... endpoint paths → must match a route.ts file under src/app/api/ +// 2. UPPER_SNAKE env var names → must have a process.env.X or env.X read +// 3. CLI commands `omniroute ...` → must exist in bin/cli/commands/ or bin/ +// 4. BUILTIN_EVENTS hook names → must be exported from hooks.ts +// 5. `src/.../foo.ts` file refs → must exist (relative to repo root) +// 6. `open-sse/.../bar.ts` file refs → must exist +// 7. `bin/...` file refs → must exist +// +// Out of scope (covered by other scripts): +// - File-size / line-count claims → scripts/check/check-docs-counts-sync.mjs +// - Env var → doc table sync → scripts/check/check-env-doc-sync.mjs +// - Cross-doc link integrity → scripts/check/check-doc-links.mjs +// - openapi.yaml ↔ routes sync → scripts/check/check-openapi-coverage.mjs +// +// Exit codes: +// 0 no drift (or soft warnings only) +// 1 strict mode and any drift was found +// +// Usage: +// node scripts/check/check-fabricated-docs.mjs # soft report +// node scripts/check/check-fabricated-docs.mjs --strict # fail on any drift +// node scripts/check/check-fabricated-docs.mjs --json # machine-readable output +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, "..", ".."); + +const ARGS = new Set(process.argv.slice(2)); +const STRICT = ARGS.has("--strict"); +const JSON_OUT = ARGS.has("--json"); + +// ── Config ───────────────────────────────────────────────────────────────── + +/** Paths to scan recursively. AGENTS.md is checked too. */ +const SCAN_PATHS = ["docs", "AGENTS.md", "open-sse/AGENTS.md", "src/lib/db/AGENTS.md"]; + +/** Built-in event names that AGENTS.md / docs are allowed to mention. */ +const KNOWN_HOOKS = new Set([ + "onRequest", + "onResponse", + "onError", + "onModelSelect", + "onComboResolve", + "onRateLimit", + "onQuotaExhaust", + "onProviderError", + "onStreamStart", + "onStreamEnd", + "onInstall", + "onActivate", + "onDeactivate", + "onUninstall", +]); + +// Common false-positives the heuristic would otherwise flag. Add to this +// list as the script matures — keep it small and well-justified. +const ENV_VAR_ALLOWLIST = new Set([ + "PATH", + "HOME", + "USER", + "SHELL", + "PWD", + "LANG", + "NODE_ENV", + "NODE_PATH", + "NODE_OPTIONS", + "DEBUG", + "VERBOSE", + "LOG_LEVEL", + "PORT", // generic, not OmniRoute-specific + "DATA_DIR", + "REQUIRE_API_KEY", + "OMNIROUTE_BUILD_PROFILE", // build-time only + "OMNIROUTE_BUILD_SHA", + "OMNIROUTE_URL", // used by ad-hoc tooling, validated elsewhere + "OMNIROUTE_KEY", // ditto + "OPENCODE_API_KEY", // ditto +]); + +// Common pluralized / column-header all-caps that aren't env vars +const ENV_VAR_DENYLIST = new Set([ + "API_DOCS", + "API_REFERENCE", + "API_GUIDE", + "PROVIDERS", + "FREE_TIERS", + "CHANGELOG", + "CONTRIBUTING", + "ARCHITECTURE", + "CODEBASE_DOCUMENTATION", + "REPOSITORY_MAP", + "AUTHZ_GUIDE", + "RESILIENCE_GUIDE", + "MCP_SERVER", + "MCP_AUDIT", + "MCP_TOOLS", + "MCP_SCOPES", + "BUILTIN_EVENTS", + "LIFECYCLE_HOOKS", + "OBSERVABILITY", + "TELEMETRY", + "TRACING", + "METRICS", + "WEB_COOKIE_PROVIDERS", + "WEB_SEARCH", + "WEB_FETCH", + "WEB_SOCKET", + "WEBSOCKET", + "WEBHOOKS", + "WEBHOOK_EVENTS", + "GUARDRAILS", + "PROVIDER_NODES", + "PROVIDER_NODES_VALIDATE", + "PROVIDER_HEALTH_AUTOPILOT", + "PROVIDER_HEALTH_MATRIX", + "PROVIDER_HEALTH_PROBE", + "PROVIDER_HEALTH_HISTORY", + "PROVIDER_QUOTA_WINDOWS", + "PROVIDER_STATS", + "PROVIDER_MODELS", + "PROVIDER_TYPE", + "PROVIDER_CREDENTIALS", + "PROVIDER_BULK", + "PROVIDER_VALIDATE", + "PROVIDER_TEST_BATCH", + "PROVIDER_TEST_ALL", + "PROVIDER_BULK_WEB_SESSION", + "PROVIDER_EXPIRATION", + "FREE_PROVIDERS", + "FREE_PROXIES", + "PROXY_POOLS", + "ONE_PROXY", + "ONE_PROXY_FETCH", + "ONE_PROXY_STATS", + "ONE_PROXY_ROTATE", + "PROXY_FALLBACK", + "PROXY_HEALTH", + "PROXY_STATS", + "PROXY_MARKETPLACE", + "PROVIDER_REGISTRY", + "PROVIDER_CATALOG", + "PROVIDER_COST", + "PROVIDER_LIMITS", + "PROVIDER_CONFIG", + "PROVIDER_CONNECTION", + "PROVIDER_CONNECTIONS", + "PROVIDER_REFRESH", + "PROVIDER_SYNC_MODELS", + "PROVIDER_TEST", + "PROVIDER_TESTS", + "PROVIDER_FETCH", + "PROVIDER_IMPORT", + "PROVIDER_EXPORT", + "PROVIDER_LIST", + "PROVIDER_ADD", + "PROVIDER_REMOVE", + "PROVIDER_CREATE", + "PROVIDER_UPDATE", + "PROVIDER_DELETE", + "PROVIDER_DISABLE", + "PROVIDER_ENABLE", + "PROVIDER_RESET", + "PROVIDER_RUN", + "PROVIDER_GET", + "PROVIDER_SET", + "PROVIDER_REVOKE", + "MODEL_REGISTRY", + "MODEL_COMBO_MAPPINGS", + "MODEL_ALIASES", + "COMBO_TARGETS", + "COMBO_HEALTH", + "COMBO_DEFAULTS", + "COMBO_FORECAST", + "COMBO_SCORING", + "COMBO_INSPECTOR", + "RATE_LIMITS", + "RATE_LIMIT_CONFIG", + "TASK_FACTORY", + "TASK_MANAGER", + "AGENT_BASE", + "AGENT_BUILDER", + "AGENT_SKILL", + "AGENT_SKILLS", + "AGENT_BRIDGE", + "MENU_ITEM", + "MENU_ITEMS", + "MENU_ICON", + "MENU_ICONS", + "FAVICON", + "FEATURE_FLAG", + "FEATURE_FLAGS", + "VERSION_MANAGER", + "VM_DEPLOY", + "VPS_DEPLOY", + "I18N_CONFIG", + "I18N_LOCALES", + "PROXY_GUIDE", + "OPENAPI_SPEC", + "OPENAPI_GUIDE", + "WAF_RULES", + "WAF_BYPASS", + "WAF_PROTECTION", + "SOCIAL_OAUTH", + "OAUTH_FLOWS", + "OAUTH_TOKENS", + "STORAGE_BACKEND", + "STORAGE_HEALTH", + "DATABASE_SETTINGS", + "TUNNELS", + "TUNNEL_CLOUDFLARED", + "TUNNEL_NGROK", + "TUNNEL_TAILSCALE", + "PRICING_CATALOG", + "PRICING_SYNC", + "PRICING_DEFAULTS", + "USAGE_ANALYTICS", + "USAGE_QUOTA", + "USAGE_BUDGET", + "QUOTA_SNAPSHOT", + "QUOTA_SNAPSHOTS", + "QUOTA_POOL", + "QUOTA_POOLS", + "QUOTA_PLAN", + "QUOTA_PLANS", + "QUOTA_MONITOR", + "QUOTA_MONITORS", + "DOMAIN_BUDGET", + "DOMAIN_BUDGETS", + "DOMAIN_COST", + "DOMAIN_COSTS", + "DOMAIN_FALLBACK", + "DOMAIN_FALLBACKS", + "DOMAIN_LOCKOUT", + "DOMAIN_LOCKOUTS", + "DOMAIN_CIRCUIT", + "DOMAIN_CIRCUITS", + "DOMAIN_RESET", + "DOMAIN_RESETS", + "PROVIDER_HEALTH_AUTOPILOT_ACTIONS", + "PROVIDER_HEALTH_AUTOPILOT_HISTORY", + "PROVIDER_HEALTH_AUTOPILOT_STATS", + "PROVIDER_HEALTH_AUTOPILOT_CONFIG", + "PROVIDER_HEALTH_AUTOPILOT_INTERVAL", + "PROVIDER_HEALTH_AUTOPILOT_TIMEOUT", + "PROVIDER_HEALTH_AUTOPILOT_THRESHOLD", + "PROVIDER_HEALTH_AUTOPILOT_COOLDOWN", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_TIMEOUT", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_THRESHOLD", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_COOLDOWN", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_MAX", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_MIN", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_BASE", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_FACTOR", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_JITTER", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_THRESHOLD", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_LIMIT", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_FLOOR", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_CEILING", + "PROVIDER_HEALTH_AUTOPILOT_RECOVERY_RETRY_BACKOFF_CAP", +]); + +/** Endpoints that don't follow the standard route.ts pattern. */ +const ENDPOINT_ALLOWLIST = new Set([ + "/api/v1/models", + "/api/v1/chat/completions", + "/api/v1/embeddings", + "/api/v1/responses", + "/api/v1/images/generations", + "/api/v1/audio/transcriptions", + "/api/v1/audio/speech", + "/api/v1/videos/generations", + "/api/v1/music/generations", + "/api/v1/moderations", + "/api/v1/rerank", + "/api/v1/search", + "/api/v1/messages", + "/api/v1/agents/tasks", + "/api/v1/agents/tasks/{id}", + "/api/v1/agents/credentials", + "/api/v1/agents/health", + "/.well-known/agent.json", + "/v1/models", + "/v1/chat/completions", + "/v1/embeddings", + "/v1/responses", + "/v1/ws", // WebSocket bridge, not standard route.ts + "/a2a", // JSON-RPC 2.0 entry + "/api/mcp/stream", // Streamable HTTP MCP transport + "/api/mcp/sse", // SSE MCP transport + "/api/health", +]); + +/** Doc files to skip (auto-generated, vendored, or third-party). */ +const SKIP_DOC_FILES = new Set([ + "docs/reference/PROVIDER_REFERENCE.md", // auto-generated from providers.ts + "docs/reference/openapi.yaml", + "docs/i18n", // translations — separate workflow +]); + +// ── File discovery ───────────────────────────────────────────────────────── + +function walkMarkdown(dir, out = []) { + const abs = path.join(ROOT, dir); + if (!fs.existsSync(abs)) return out; + const stat = fs.statSync(abs); + if (stat.isFile()) { + if (abs.endsWith(".md") || abs.endsWith(".mdx")) out.push(abs); + return out; + } + for (const name of fs.readdirSync(abs)) { + if (name === "node_modules" || name.startsWith(".")) continue; + const childAbs = path.join(abs, name); + const s = fs.statSync(childAbs); + if (s.isDirectory()) walkMarkdown(path.relative(ROOT, childAbs), out); + else if (childAbs.endsWith(".md") || childAbs.endsWith(".mdx")) out.push(childAbs); + } + return out; +} + +function allScanFiles() { + const files = []; + for (const p of SCAN_PATHS) walkMarkdown(p, files); + return files.filter((f) => { + const rel = path.relative(ROOT, f); + for (const skip of SKIP_DOC_FILES) { + if (rel === skip || rel.startsWith(skip + path.sep)) return false; + } + return true; + }); +} + +// ── Codebase index ───────────────────────────────────────────────────────── + +function buildCodebaseIndex() { + // Set of /api/... paths that have a route.ts handler. + const apiRoutes = new Set(); + // Map of /api/... → methods implemented in route.ts + const apiMethods = new Map(); + + function walkApiRoutes(dir) { + const abs = path.join(ROOT, dir); + if (!fs.existsSync(abs)) return; + for (const name of fs.readdirSync(abs)) { + const child = path.join(abs, name); + const s = fs.statSync(child); + if (s.isDirectory()) walkApiRoutes(path.relative(ROOT, child)); + else if (name === "route.ts" || name === "route.mjs") { + // Build the route path from the directory hierarchy + const rel = path.relative(ROOT, child).replace(/\\/g, "/"); + const parts = rel.split("/"); + // drop "src/app/api" and "route.ts" + parts.shift(); // src + parts.shift(); // app + parts.shift(); // api + parts.pop(); // route.ts + const routePath = "/api/" + parts.join("/"); + apiRoutes.add(routePath); + apiRoutes.add(routePath + "/"); // trailing slash variant + + // Read the file to find exported HTTP methods + try { + const content = fs.readFileSync(child, "utf8"); + const methods = new Set(); + for (const m of ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"]) { + const re = new RegExp(`export\\s+(?:async\\s+)?function\\s+${m}\\b`); + if (re.test(content)) methods.add(m); + const re2 = new RegExp(`export\\s+const\\s+${m}\\b`); + if (re2.test(content)) methods.add(m); + } + if (methods.size > 0) apiMethods.set(routePath, methods); + } catch { + /* ignore read errors */ + } + } + } + } + walkApiRoutes("src/app/api"); + + // Set of env var names that are actually read in code. + const envVars = new Set(); + function walkForEnv(dir) { + const abs = path.join(ROOT, dir); + if (!fs.existsSync(abs)) return; + const skipDirs = new Set(["node_modules", ".next", "dist", ".build", "coverage"]); + for (const name of fs.readdirSync(abs)) { + if (skipDirs.has(name)) continue; + const child = path.join(abs, name); + const s = fs.statSync(child); + if (s.isDirectory()) walkForEnv(path.relative(ROOT, child)); + else if (/\.(ts|tsx|js|mjs|cjs)$/.test(name)) { + try { + const content = fs.readFileSync(child, "utf8"); + // process.env.X + const m1 = content.matchAll(/process\.env\.([A-Z][A-Z0-9_]+)/g); + for (const m of m1) envVars.add(m[1]); + // env.X (destructured in some handlers) + const m2 = content.matchAll(/\benv\.([A-Z][A-Z0-9_]+)\b/g); + for (const m of m2) envVars.add(m[1]); + // import.meta.env.X (Vite-style, unlikely here but cheap) + const m3 = content.matchAll(/import\.meta\.env\.([A-Z][A-Z0-9_]+)/g); + for (const m of m3) envVars.add(m[1]); + } catch { + /* ignore */ + } + } + } + } + walkForEnv("src"); + walkForEnv("open-sse"); + walkForEnv("bin"); + walkForEnv("scripts"); + + // Set of `omniroute ` strings that exist in bin/ + const cliCommands = new Set(); + function walkCli(dir) { + const abs = path.join(ROOT, dir); + if (!fs.existsSync(abs)) return; + for (const name of fs.readdirSync(abs)) { + const child = path.join(abs, name); + const s = fs.statSync(child); + if (s.isDirectory()) walkCli(path.relative(ROOT, child)); + else if (/\.(mjs|js|ts)$/.test(name)) { + try { + const content = fs.readFileSync(child, "utf8"); + // Programmatic API: `command('foo', ...)`, `.command('bar')` + const m1 = content.matchAll(/\.command\(\s*['"`]([a-z][a-z0-9-]+)['"`]/g); + for (const m of m1) cliCommands.add(m[1]); + // Subcommand names: `${name}Cmd`, `name = "foo"`, etc. + const m2 = content.matchAll(/name:\s*['"`]([a-z][a-z0-9-]+)['"`]/g); + for (const m of m2) cliCommands.add(m[1]); + // `.name('foo')` (commander pattern) + const m3 = content.matchAll(/\.name\(\s*['"`]([a-z][a-z0-9-]+)['"`]\s*\)/g); + for (const m of m3) cliCommands.add(m[1]); + } catch { + /* ignore */ + } + } + } + } + walkCli("bin"); + + return { apiRoutes, apiMethods, envVars, cliCommands }; +} + +// ── Doc scanning ─────────────────────────────────────────────────────────── + +const COARSE_PATTERNS = { + apiPath: /(?= 3 + envVar: /\b([A-Z][A-Z0-9_]{2,})\b/g, + // omniroute ... — only on the same line, captures first 2 tokens + cliCmd: /\bomniroute\s+([a-z][a-z0-9-]+)(?:\s+([a-z][a-z0-9-]+))?/g, + // Built-in event names like onRequest, onFoo + hookName: /\b(on[A-Z][a-zA-Z]+)\b/g, + // File references like src/lib/foo.ts, open-sse/handlers/bar.ts, bin/cli/baz.mjs + fileRef: + /\b((?:src|open-sse|bin|scripts|tests|electron)\/[A-Za-z0-9_\-\/\.]+\.(?:ts|tsx|mjs|js|cjs|sh|sql))\b/g, +}; + +function stripCodeBlocksAndFences(text) { + // Remove fenced code blocks (``` ... ```) but KEEP inline backticks so + // we can still detect `BACKTICKED_LIKE_THIS` env-var/hook/CLI claims. + return text.replace(/```[\s\S]*?```/g, ""); +} + +function lineOf(text, idx) { + let line = 1; + for (let i = 0; i < idx && i < text.length; i++) if (text[i] === "\n") line++; + return line; +} + +function scanDocFile(absPath, index) { + const rel = path.relative(ROOT, absPath); + const text = fs.readFileSync(absPath, "utf8"); + const textNoCode = stripCodeBlocksAndFences(text); + const findings = []; + + // 1) API endpoints + for (const m of textNoCode.matchAll(COARSE_PATTERNS.apiPath)) { + const p = m[0].replace(/[\[\]\{\}]/g, ""); // strip wildcards for lookup + const candidate = p.replace(/\/$/, ""); + if (ENDPOINT_ALLOWLIST.has(candidate) || ENDPOINT_ALLOWLIST.has(candidate + "/")) continue; + if (index.apiRoutes.has(candidate) || index.apiRoutes.has(candidate + "/")) continue; + // Allow docs that describe intended-but-not-yet-shipped routes by skipping lines that say "planned" / "TBD" / "future" + const ln = lineOf(text, m.index); + const lineText = text.split("\n")[ln - 1] || ""; + if (/\b(planned|tbd|future|coming|proposed|not yet|will be)\b/i.test(lineText)) continue; + findings.push({ + kind: "api-path", + value: m[0], + line: ln, + msg: `endpoint ${m[0]} not found in src/app/api/`, + }); + } + + // 2) Env vars — only flag names wrapped in backticks AND containing an + // underscore. The maintainer's actual fabricated env vars (PR #3456) + // were always in `BACKTICKS` inside tables; bare all-caps tokens + // inside markdown link display text are doc references, not env vars. + // Example of TRUE positive: | `ACP_MAX_CONCURRENT_SESSIONS` | 5 | ... | + // Example of false positive: | [STEALTH_GUIDE](security/...) | + for (const m of textNoCode.matchAll(/`([A-Z][A-Z0-9_]{4,})`/g)) { + const name = m[1]; + if (ENV_VAR_ALLOWLIST.has(name)) continue; + if (!/_/.test(name)) continue; // real env vars have an underscore + if (index.envVars.has(name)) continue; + if (/^X-[A-Z]/.test(name)) continue; + if (ENV_VAR_DENYLIST.has(name)) continue; + const ln = lineOf(text, m.index); + const lineText = text.split("\n")[ln - 1] || ""; + if (/example|placeholder|todo|tbd|\.\.\./i.test(lineText)) continue; + findings.push({ + kind: "env-var", + value: name, + line: ln, + msg: `env var \`${name}\` is never read via process.env / env / import.meta.env`, + }); + } + + // 3) CLI commands: `omniroute foo bar` — only flag when the line is in + // a code-like context (inside backticks or a shell block). Bare prose + // like "we use omniroute and..." is not a command claim. + for (const m of textNoCode.matchAll(COARSE_PATTERNS.cliCmd)) { + const sub = m[1]; + if (index.cliCommands.has(sub)) continue; + if (["help", "--help", "-h", "version", "--version", "doctor", "setup", "chat"].includes(sub)) + continue; + const ln = lineOf(text, m.index); + const lineText = text.split("\n")[ln - 1] || ""; + // Only flag when on a line that looks like a shell command (starts with $, or + // inside a shell block, or wrapped in `code`) + const isShellLike = /^[ \t]*\$\s|^```sh|^```bash|^```shell|`omniroute/.test(lineText); + if (!isShellLike) continue; + if (/example|placeholder|tbd/i.test(lineText)) continue; + findings.push({ + kind: "cli-cmd", + value: `omniroute ${sub}`, + line: ln, + msg: `omniroute subcommand '${sub}' not registered in bin/`, + }); + } + + // 4) Hook names — only flag when wrapped in backticks/code, since bare + // "onFoo" prose is common English. + for (const m of textNoCode.matchAll(/`?(on[A-Z][a-zA-Z]+)`?/g)) { + const name = m[1]; + if (KNOWN_HOOKS.has(name)) continue; + // Require backticks to reduce noise (text mentions are usually casual) + if (!m[0].startsWith("`")) continue; + const ln = lineOf(text, m.index); + const lineText = text.split("\n")[ln - 1] || ""; + if (/example|placeholder|tbd/i.test(lineText)) continue; + findings.push({ + kind: "hook", + value: name, + line: ln, + msg: `hook ${name} not in BUILTIN_EVENTS (hooks.ts) — is this a real hook?`, + }); + } + + // 5) File references + for (const m of textNoCode.matchAll(COARSE_PATTERNS.fileRef)) { + const ref = m[1].replace(/\\/g, "/"); + const abs = path.join(ROOT, ref); + if (fs.existsSync(abs)) continue; + // Allow README/AGENTS to mention example files explicitly in a non-verified way + if (/\{\{|\.\.\./.test(ref)) continue; // templated / placeholder + const ln = lineOf(text, m.index); + findings.push({ + kind: "file-ref", + value: ref, + line: ln, + msg: `file ${ref} does not exist`, + }); + } + + return { rel, findings }; +} + +// ── Main ─────────────────────────────────────────────────────────────────── + +export function runFabricatedDocsCheck(opts = {}) { + const index = buildCodebaseIndex(); + const files = allScanFiles(); + + const allFindings = []; + for (const f of files) { + const result = scanDocFile(f, index); + if (result.findings.length > 0) { + allFindings.push(result); + } + } + + const totalFindings = allFindings.reduce((acc, r) => acc + r.findings.length, 0); + return { totalFindings, files: allFindings, fileCount: files.length, index }; +} + +export function formatHumanReport(result) { + const { totalFindings, files, fileCount, index } = result; + const lines = []; + lines.push("Doc accuracy gate — fabricated-claim detection"); + lines.push("================================================"); + lines.push(`Scanned ${fileCount} markdown file(s)`); + lines.push( + `Codebase: ${index.apiRoutes.size} api routes · ${index.envVars.size} env vars · ${index.cliCommands.size} cli commands` + ); + lines.push(""); + + if (totalFindings === 0) { + lines.push("✓ No fabricated API/env/CLI/hook/file references found."); + return lines.join("\n"); + } + + // Dedupe identical findings across files (report once, with file list) + const deduped = new Map(); + for (const r of files) { + for (const f of r.findings) { + const key = `${f.kind}::${f.value}::${f.msg}`; + if (!deduped.has(key)) deduped.set(key, { ...f, files: new Set() }); + deduped.get(key).files.add(r.rel); + } + } + + const groups = { "api-path": [], "env-var": [], "cli-cmd": [], hook: [], "file-ref": [] }; + for (const f of deduped.values()) groups[f.kind].push(f); + + const KIND_LABELS = { + "api-path": "API endpoint paths not in src/app/api/", + "env-var": "Env vars never read in code", + "cli-cmd": "omniroute subcommands not registered", + hook: "Hook names not in BUILTIN_EVENTS", + "file-ref": "File references that don't exist", + }; + + for (const [kind, items] of Object.entries(groups)) { + if (items.length === 0) continue; + lines.push(`\n## ${KIND_LABELS[kind]} (${items.length})`); + for (const f of items.slice(0, 20)) { + const fileList = [...f.files] + .slice(0, 3) + .map((r) => `${r}:${f.line}`) + .join(", "); + const more = f.files.size > 3 ? ` (+${f.files.size - 3} more)` : ""; + lines.push(` • ${f.value.padEnd(40)} ${f.msg}`); + lines.push(` ${fileList}${more}`); + } + if (items.length > 20) lines.push(` ... and ${items.length - 20} more`); + } + + return lines.join("\n"); +} + +// CLI entry — only run when invoked directly (not when imported for tests). +const isMain = import.meta.url === `file://${process.argv[1]}`; +if (isMain) { + main(); +} + +function main() { + const result = runFabricatedDocsCheck(); + const { totalFindings, files } = result; + + if (JSON_OUT) { + console.log( + JSON.stringify( + { + totalFindings, + files: files.length, + results: files, + }, + null, + 2 + ) + ); + if (STRICT && totalFindings > 0) process.exit(1); + process.exit(0); + } + + console.log(formatHumanReport(result)); + console.log(); + if (STRICT) { + console.error(`✗ ${totalFindings} claim(s) drift from source. Failing (--strict).`); + process.exit(1); + } else { + console.warn(`⚠ ${totalFindings} claim(s) drift from source. Re-run with --strict to fail.`); + process.exit(0); + } +} diff --git a/tests/unit/check-fabricated-docs.test.ts b/tests/unit/check-fabricated-docs.test.ts new file mode 100644 index 0000000000..c421851d73 --- /dev/null +++ b/tests/unit/check-fabricated-docs.test.ts @@ -0,0 +1,75 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; + +import { + runFabricatedDocsCheck, + formatHumanReport, +} from "../../scripts/check/check-fabricated-docs.mjs"; + +// We can't easily mock the buildCodebaseIndex() which walks src/app/api etc. +// Instead, we test the report-formatting logic and the no-findings path on +// the real repo, which acts as a smoke test that the script runs end-to-end. + +test("runFabricatedDocsCheck: runs without throwing on the real repo", () => { + const result = runFabricatedDocsCheck(); + assert.ok(result); + assert.ok(typeof result.totalFindings === "number"); + assert.ok(result.fileCount > 0, "should scan at least AGENTS.md"); + assert.ok(result.index); + assert.ok(result.index.apiRoutes instanceof Set); + assert.ok(result.index.envVars instanceof Set); + assert.ok(result.index.cliCommands instanceof Set); +}); + +test("runFabricatedDocsCheck: index contains real OmniRoute routes", () => { + const result = runFabricatedDocsCheck(); + // The real repo has /api/v1/chat/completions — a known truth + assert.ok(result.index.apiRoutes.has("/api/v1/chat/completions")); + // The real repo has /api/monitoring/health + assert.ok(result.index.apiRoutes.has("/api/monitoring/health")); + // The real repo reads PORT via process.env + assert.ok(result.index.envVars.has("PORT")); +}); + +test("formatHumanReport: no-drift case produces a checkmark", () => { + const result = { + totalFindings: 0, + files: [], + fileCount: 10, + index: { + apiRoutes: new Set(), + envVars: new Set(), + cliCommands: new Set(), + }, + }; + const out = formatHumanReport(result); + assert.match(out, /No fabricated API\/env\/CLI\/hook\/file references found/); +}); + +test("formatHumanReport: groups findings by kind", () => { + const result = { + totalFindings: 3, + files: [ + { + rel: "docs/test.md", + findings: [ + { kind: "api-path", value: "/api/fake/1", line: 1, msg: "fake" }, + { kind: "env-var", value: "FAKE_VAR_X", line: 2, msg: "fake" }, + { kind: "hook", value: "onFake", line: 3, msg: "fake" }, + ], + }, + ], + fileCount: 1, + index: { apiRoutes: new Set(), envVars: new Set(), cliCommands: new Set() }, + }; + const out = formatHumanReport(result); + assert.match(out, /API endpoint paths/); + assert.match(out, /Env vars never read/); + assert.match(out, /Hook names/); + assert.match(out, /\/api\/fake\/1/); + assert.match(out, /FAKE_VAR_X/); + assert.match(out, /onFake/); +});