diff --git a/docs/architecture/CODEBASE_DOCUMENTATION.md b/docs/architecture/CODEBASE_DOCUMENTATION.md index 091da95dd8..d222b94b90 100644 --- a/docs/architecture/CODEBASE_DOCUMENTATION.md +++ b/docs/architecture/CODEBASE_DOCUMENTATION.md @@ -274,7 +274,7 @@ table groups the actual directories and notable top-level files. | Module | Purpose | | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `a2a/` | A2A protocol server: `taskManager.ts`, `streaming.ts`, `taskExecution.ts`, `routingLogger.ts`, `skills/` (5 skills: cost analysis, health report, provider discovery, quota management, smart routing) | +| `a2a/` | A2A protocol server: `taskManager.ts`, `streaming.ts`, `taskExecution.ts`, `routingLogger.ts`, `skills/` (6 skills: cost analysis, health report, provider discovery, quota management, smart routing, list-capabilities) | | `acp/` | Agent-Control-Protocol: `index.ts`, `manager.ts`, `registry.ts` | | `api/` | Internal API helpers: `requireManagementAuth.ts`, `requireCliToolsAuth.ts`, `errorResponse.ts` | | `auth/` | `managementPassword.ts` (password reset / hashing) | @@ -302,6 +302,7 @@ table groups the actual directories and notable top-level files. | `runtime/` | Runtime feature detection | | `search/` | `executeWebSearch.ts` | | `services/` | Embedded services framework: `ServiceSupervisor.ts` (generic child-process supervisor with operation lock, ring buffer, health checker), `bootstrap.ts` (process-level registration and auto-start), `registry.ts` (tool → supervisor map), `apiKey.ts` (AES-256-GCM key store), `modelSync.ts` (periodic model sync), `ringBuffer.ts` (5 MB circular log buffer), `healthCheck.ts` (HTTP health probe), `types.ts`, `embedWsProxy.ts` (WebSocket proxy), `installers/{ninerouter,cliproxy}.ts`. See `docs/frameworks/EMBEDDED-SERVICES.md` | +| `agentSkills/` | Agent Skills catalog + generator: `catalog.ts` (getCatalog/getSkillById/filterCatalog/computeCoverage), `generator.ts` (generateAgentSkills → writes `skills/{id}/SKILL.md`), `openapiParser.ts` (extracts REST endpoints from OpenAPI spec), `cliRegistryParser.ts` (extracts CLI subcommands from bin/cli-registry), `schemas.ts` (Zod: AgentSkillSchema, SkillCoverageSchema, ListQuerySchema, GenerateBodySchema), `types.ts` (AgentSkill, SkillCoverage, SkillMarkdown, GeneratorReport). Consumed by REST routes (`/api/agent-skills/*`), MCP tools (`omniroute_agent_skills_*`), and A2A skill `list-capabilities`. See [AGENT-SKILLS.md](../frameworks/AGENT-SKILLS.md). | | `skills/` | Skill framework: `registry.ts`, `executor.ts`, `interception.ts`, `injection.ts`, `sandbox.ts`, `custom.ts`, `hybrid.ts`, `builtins.ts`, `a2a.ts`, `providerSettings.ts`, `schemas.ts`, `skillssh.ts`, `types.ts`, plus `builtin/browser.ts` | | `spend/` | `batchWriter.ts` (write-behind buffer) | | `sync/` | `bundle.ts`, `tokens.ts` (Cloud Sync) | diff --git a/docs/frameworks/A2A-SERVER.md b/docs/frameworks/A2A-SERVER.md index e8cfbc0a1f..296bce224f 100644 --- a/docs/frameworks/A2A-SERVER.md +++ b/docs/frameworks/A2A-SERVER.md @@ -139,18 +139,32 @@ curl -X POST http://localhost:20128/a2a \ ## Available Skills -OmniRoute exposes 5 A2A skills wired in `src/lib/a2a/taskExecution.ts::A2A_SKILL_HANDLERS`. Each skill module lives in `src/lib/a2a/skills/`. +OmniRoute exposes 6 A2A skills wired in `src/lib/a2a/taskExecution.ts::A2A_SKILL_HANDLERS`. Each skill module lives in `src/lib/a2a/skills/`. -| Skill | ID | Description | -| :----------------- | :------------------- | :------------------------------------------------------------------------------------------ | -| Smart Routing | `smart-routing` | Routes a prompt through the optimal provider/combo using OmniRoute's combo engine + scoring | -| Quota Management | `quota-management` | Reports per-provider quota state, helps callers decide when to throttle/switch | -| Provider Discovery | `provider-discovery` | Lists installed providers with capabilities, free-tier flags, OAuth status | -| Cost Analysis | `cost-analysis` | Estimates cost of a request/conversation given the catalog + recent usage | -| Health Report | `health-report` | Aggregates circuit breaker, cooldown, lockout state per provider | +| Skill | ID | Description | Tags | Examples | +| :------------------ | :-------------------- | :------------------------------------------------------------------------------------------------------------ | :---------------------- | :-------------------------------------- | +| Smart Routing | `smart-routing` | Routes a prompt through the optimal provider/combo using OmniRoute's combo engine + scoring | routing, providers | "Route this prompt via the best model" | +| Quota Management | `quota-management` | Reports per-provider quota state, helps callers decide when to throttle/switch | quota, providers | "Check quota for anthropic" | +| Provider Discovery | `provider-discovery` | Lists installed providers with capabilities, free-tier flags, OAuth status | providers, discovery | "What providers are available?" | +| Cost Analysis | `cost-analysis` | Estimates cost of a request/conversation given the catalog + recent usage | cost, usage | "Estimate cost for this conversation" | +| Health Report | `health-report` | Aggregates circuit breaker, cooldown, lockout state per provider | health, resilience | "Show health status of all providers" | +| List Capabilities | `list-capabilities` | Returns the full 42-entry Agent Skills catalog as a markdown table with raw SKILL.md URLs for context injection | catalog, discovery, skills | "List all OmniRoute capabilities" | > Note: the Agent Card description currently advertises "36+ providers" (`src/app/.well-known/agent.json/route.ts:26` and `:55`). The actual catalog has grown to 180+ providers — the string should be updated in a follow-up change (tracked as a separate doc/code TODO; not modified here). +### `list-capabilities` Skill Detail + +The `list-capabilities` skill is particularly useful for external agents that need to discover what OmniRoute exposes before sending API calls. It returns a structured markdown table artifact: + +``` +| ID | Name | Category | Area | Endpoints/Commands | Raw URL | +| --- | --- | --- | --- | --- | --- | +| omni-auth | Auth & Sessions | api | auth | POST /api/auth/login, ... | https://raw.githubusercontent.com/... | +... +``` + +Each row includes the `rawUrl` column so agents can immediately fetch the full SKILL.md. The `metadata.totalSkills` field is always `42`. Implementation: `src/lib/a2a/skills/listCapabilities.ts`. See also [AGENT-SKILLS.md](./AGENT-SKILLS.md). + --- ## REST API (auxiliary) diff --git a/docs/frameworks/AGENT-SKILLS.md b/docs/frameworks/AGENT-SKILLS.md new file mode 100644 index 0000000000..79d8258eef --- /dev/null +++ b/docs/frameworks/AGENT-SKILLS.md @@ -0,0 +1,301 @@ +--- +title: "OmniRoute Agent Skills Catalog" +version: 3.8.6 +lastUpdated: 2026-05-28 +--- + +# OmniRoute Agent Skills Catalog + +> **Source of truth:** `src/lib/agentSkills/` (catalog, generator, parsers) + `skills/` directory (SKILL.md files) +> **Last updated:** 2026-05-28 — v3.8.6 + +Agent Skills are structured SKILL.md files that teach external agents, MCP clients, and A2A orchestrators how to use OmniRoute's REST API and CLI. Unlike [Omni Skills](./SKILLS.md) (which are LLM tool definitions executed inside OmniRoute), Agent Skills are a *documentation catalog* — static markdown that can be fed directly into agent context. + +--- + +## Overview + +The catalog contains **42 canonical Agent Skills** (22 REST API + 20 CLI). Each skill has: + +- A **canonical ID** (`omni-auth`, `cli-serve`, etc.) +- A **SKILL.md** file in `skills/{id}/SKILL.md` with YAML frontmatter (`name`, `description`) + rich markdown body +- **REST endpoints** (API skills) or **CLI subcommands** (CLI skills) derived from the OpenAPI spec and CLI registry +- A **GitHub raw URL** for live fetch: `https://raw.githubusercontent.com/diegosouzapw/OmniRoute/refs/heads/main/skills/{id}/SKILL.md` + +--- + +## Architecture + +``` +src/shared/constants/agentSkills.ts — 42-entry curated list (name/desc/category/area/icon) +src/lib/agentSkills/ + catalog.ts — getCatalog(), getSkillById(), filterCatalog(), computeCoverage() + generator.ts — generateAgentSkills() writes SKILL.md to skills/{id}/ + openapiParser.ts — extracts REST endpoints from docs/reference/openapi.yaml + cliRegistryParser.ts — extracts CLI subcommands from bin/cli-registry.ts + schemas.ts — Zod schemas: AgentSkillSchema, SkillCoverageSchema, etc. + types.ts — TypeScript interfaces: AgentSkill, SkillCoverage, etc. + +skills/{id}/SKILL.md — Generated + curated markdown files (42 total) + +src/app/api/agent-skills/ + route.ts — GET /api/agent-skills + [id]/route.ts — GET /api/agent-skills/{id} + [id]/raw/route.ts — GET /api/agent-skills/{id}/raw (text/markdown) + coverage/route.ts — GET /api/agent-skills/coverage + generate/route.ts — POST /api/agent-skills/generate (auth required) + +open-sse/mcp-server/tools/agentSkillTools.ts — 3 MCP tools (list, get, coverage) +src/lib/a2a/skills/listCapabilities.ts — A2A skill: list-capabilities +``` + +--- + +## SKILL.md Format + +```markdown +--- +name: omni-providers +description: "Manage provider connections: add, test, rotate, and remove credentials." +--- + + +## Overview +... + +## Authentication +... + +## Endpoints +... + + +## Custom Section (preserved across regeneration) +... + +``` + +The generator preserves content between `` and `` on regeneration. Ten skills have curated custom blocks: + +`omni-mcp`, `omni-compression`, `cli-providers`, `cli-eval`, `omni-agents-a2a`, `omni-combos-routing`, `omni-auth`, `omni-resilience`, `omni-inference`, `cli-serve`. + +--- + +## REST API Discovery + +| Endpoint | Method | Description | Auth | +| :--- | :--- | :--- | :--- | +| `/api/agent-skills` | GET | List catalog (optional `?category=api\|cli&area=`) | none | +| `/api/agent-skills/{id}` | GET | Get single skill metadata | none | +| `/api/agent-skills/{id}/raw` | GET | Fetch SKILL.md as `text/markdown` | none | +| `/api/agent-skills/coverage` | GET | Coverage stats (how many SKILL.md files exist) | none | +| `/api/agent-skills/generate` | POST | Trigger generator (dryRun/prune/onlyIds) | management | + +Example — list all API skills: + +```bash +curl "http://localhost:20128/api/agent-skills?category=api" +``` + +Example — fetch a single SKILL.md: + +```bash +curl -H "Accept: text/markdown" "http://localhost:20128/api/agent-skills/omni-providers/raw" +``` + +--- + +## MCP Discovery + +Three MCP tools are registered under scope `read:catalog`: + +| Tool | Description | +| :--- | :--- | +| `omniroute_agent_skills_list` | List skills (optional `category` / `area` filters) | +| `omniroute_agent_skills_get` | Get metadata + SKILL.md for one skill by `id` | +| `omniroute_agent_skills_coverage` | Coverage stats (API/CLI have/total) | + +See [MCP-SERVER.md](./MCP-SERVER.md) for scope wiring and authentication. + +--- + +## A2A Discovery + +The A2A skill `list-capabilities` returns the full 42-skill catalog as a markdown table artifact. External orchestrators can invoke it via: + +```json +{ + "jsonrpc": "2.0", "id": "1", + "method": "message/send", + "params": { + "skill": "list-capabilities", + "messages": [{"role": "user", "content": "List all capabilities"}] + } +} +``` + +See [A2A-SERVER.md](./A2A-SERVER.md) for protocol details. + +--- + +## Catalog — 42 Skill IDs + +### API Skills (22) + +| ID | Area | Entry Point | +| :--- | :--- | :--- | +| `omni-auth` | auth | Auth + session management | +| `omni-providers` | providers | Provider connection management | +| `omni-models` | models | Model catalog and capabilities | +| `omni-combos-routing` | combos-routing | Combo routing strategies | +| `omni-api-keys` | api-keys | API key management | +| `omni-usage-logs` | usage-logs | Usage and cost logs | +| `omni-budget` | budget | Budget guards | +| `omni-settings` | settings | Global settings | +| `omni-proxies` | proxies | Proxy pool management | +| `omni-cache` | cache | Semantic + prompt cache | +| `omni-compression` | compression | Context compression engines | +| `omni-context-rtk` | context-rtk | RTK compression | +| `omni-resilience` | resilience | Circuit breakers + cooldowns | +| `omni-cli-tools` | cli-tools | CLI tools REST proxy | +| `omni-tunnels` | tunnels | Tunnel management | +| `omni-sync-cloud` | sync-cloud | Cloud sync | +| `omni-db-backups` | db-backups | Database backups | +| `omni-webhooks` | webhooks | Webhook event dispatcher | +| `omni-mcp` | mcp | MCP server (37 tools, 3 transports) | +| `omni-agents-a2a` | agents-a2a | A2A agent protocol | +| `omni-version-manager` | version-manager | Version and update management | +| `omni-inference` | inference | Direct inference / completions | + +### CLI Skills (20) + +| ID | Area | CLI Command Root | +| :--- | :--- | :--- | +| `cli-serve` | cli-serve | `omniroute serve` | +| `cli-health` | cli-health | `omniroute health` | +| `cli-providers` | cli-providers | `omniroute providers` | +| `cli-keys` | cli-keys | `omniroute keys` | +| `cli-models` | cli-models | `omniroute models` | +| `cli-chat` | cli-chat | `omniroute chat` | +| `cli-routing` | cli-routing | `omniroute routing` | +| `cli-resilience` | cli-resilience | `omniroute resilience` | +| `cli-compression` | cli-compression | `omniroute compression` | +| `cli-contexts` | cli-contexts | `omniroute contexts` | +| `cli-cost-usage` | cli-cost-usage | `omniroute cost` | +| `cli-mcp` | cli-mcp | `omniroute mcp` | +| `cli-a2a` | cli-a2a | `omniroute a2a` | +| `cli-tunnel` | cli-tunnel | `omniroute tunnel` | +| `cli-backup-sync` | cli-backup-sync | `omniroute backup` | +| `cli-policy-audit` | cli-policy-audit | `omniroute policy` | +| `cli-batches` | cli-batches | `omniroute batch` | +| `cli-eval` | cli-eval | `omniroute eval` | +| `cli-plugins-skills` | cli-plugins-skills | `omniroute plugins` | +| `cli-setup` | cli-setup | `omniroute setup` | + +--- + +## How External Agents Consume Skills + +### 1. Discovery via REST + +```bash +# Get the full catalog +curl "http://your-omniroute/api/agent-skills" | jq '.skills[] | {id, name, category}' + +# Get SKILL.md for context injection +curl "http://your-omniroute/api/agent-skills/omni-providers/raw" > omni-providers.md +``` + +### 2. Discovery via MCP + +```typescript +// In a Claude Desktop / Cursor MCP client: +const result = await client.callTool("omniroute_agent_skills_list", { category: "api" }); +// result.skills → array of AgentSkill with rawUrl for each +``` + +### 3. Discovery via A2A + +```python +import requests + +resp = requests.post("http://your-omniroute/a2a", json={ + "jsonrpc": "2.0", "id": "1", + "method": "message/send", + "params": {"skill": "list-capabilities", "messages": [{"role": "user", "content": "list"}]} +}) +table = resp.json()["result"]["artifacts"][0]["content"] +# table is a markdown table with all 42 skill IDs + rawUrl columns +``` + +### 4. Direct GitHub raw fetch (no server required) + +```bash +BASE="https://raw.githubusercontent.com/diegosouzapw/OmniRoute/refs/heads/main/skills" +curl "${BASE}/omni-providers/SKILL.md" +``` + +--- + +## Generator + +The generator reads the curated catalog + OpenAPI spec + CLI registry and writes `skills/{id}/SKILL.md` for each entry: + +```bash +# Preview (dry run, no writes) +curl -X POST http://localhost:20128/api/agent-skills/generate \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"dryRun":true}' + +# Full regeneration +curl -X POST http://localhost:20128/api/agent-skills/generate \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"dryRun":false,"prune":false}' + +# Regenerate specific IDs +curl -X POST http://localhost:20128/api/agent-skills/generate \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"dryRun":false,"onlyIds":["omni-providers","cli-serve"]}' +``` + +The generator response is a `GeneratorReport`: + +```json +{ + "generated": ["omni-providers", "cli-serve"], + "unchanged": [], + "pruned": [], + "orphansDetected": [], + "errors": [] +} +``` + +--- + +## Coverage API + +```bash +curl "http://localhost:20128/api/agent-skills/coverage" +``` + +```json +{ + "api": {"have": 22, "total": 22}, + "cli": {"have": 20, "total": 20}, + "totalSkills": 42, + "generatedAt": "2026-05-28T00:00:00.000Z" +} +``` + +--- + +## Related + +- [SKILLS.md](./SKILLS.md) — Omni Skills framework (LLM tool injection + marketplace) +- [MCP-SERVER.md](./MCP-SERVER.md) — MCP tool catalog (`omniroute_agent_skills_*` tools) +- [A2A-SERVER.md](./A2A-SERVER.md) — A2A protocol (`list-capabilities` skill) +- `src/lib/agentSkills/` — catalog, generator, parsers +- `skills/` — generated SKILL.md files (42 entries) diff --git a/docs/frameworks/MCP-SERVER.md b/docs/frameworks/MCP-SERVER.md index 1dae671b76..ecc47fee8b 100644 --- a/docs/frameworks/MCP-SERVER.md +++ b/docs/frameworks/MCP-SERVER.md @@ -175,6 +175,18 @@ Defined in `open-sse/mcp-server/tools/skillTools.ts`. Backed by `src/lib/skills/ | `omniroute_skills_execute` | Execute a skill with provided input and return the execution record | | `omniroute_skills_executions` | List recent skill execution history | +## Agent Skill Catalog Tools (3) + +Defined in `open-sse/mcp-server/tools/agentSkillTools.ts`. Backed by `src/lib/agentSkills/catalog`. These tools expose the 42-entry Agent Skills documentation catalog to MCP clients and external agents. Scope: `read:catalog`. + +| Tool | Scopes | Description | +| :--------------------------------- | :------------- | :--------------------------------------------------------------------------------------------------------------- | +| `omniroute_agent_skills_list` | `read:catalog` | List all 42 agent skills with optional `category` (api\|cli) and `area` filters; returns metadata + coverage | +| `omniroute_agent_skills_get` | `read:catalog` | Get full metadata + SKILL.md content for a single skill by canonical `id` | +| `omniroute_agent_skills_coverage` | `read:catalog` | Coverage stats: how many of the 22 API and 20 CLI skills have SKILL.md files on the filesystem vs catalog totals | + +See [AGENT-SKILLS.md](./AGENT-SKILLS.md) for the full catalog and how external agents consume it. + ## Related Frameworks (v3.8.0) The MCP tool inventory above (37 tools = 30 base + 3 memory + 4 skills) is intentionally @@ -248,9 +260,11 @@ MCP tools are authenticated through API key scopes. Scope enforcement is central | `write:compression` | `compression_configure`, `set_compression_engine` | | `read:proxies` | `oneproxy_fetch`, `oneproxy_rotate`, `oneproxy_stats` | +| `read:catalog` | `agent_skills_list`, `agent_skills_get`, `agent_skills_coverage` | + Wildcard scopes are supported: `read:*` grants all read-scopes, `*` grants full access. -Memory and Skill tools currently do not declare static scope requirements in their definitions; access is gated by the caller's API key and audited through the standard MCP audit pipeline. +Memory and Skill tools currently do not declare static scope requirements in their definitions; access is gated by the caller's API key and audited through the standard MCP audit pipeline. Agent Skill Catalog tools require `read:catalog`. --- diff --git a/docs/frameworks/SKILLS.md b/docs/frameworks/SKILLS.md index 313d8d82c1..748a842ca1 100644 --- a/docs/frameworks/SKILLS.md +++ b/docs/frameworks/SKILLS.md @@ -1,13 +1,13 @@ --- title: "Skills Framework" -version: 3.8.2 -lastUpdated: 2026-05-13 +version: 3.8.6 +lastUpdated: 2026-05-28 --- # Skills Framework > **Source of truth:** `src/lib/skills/` and `src/app/api/skills/` -> **Last updated:** 2026-05-13 — v3.8.0 +> **Last updated:** 2026-05-28 — v3.8.6 OmniRoute exposes an extensible Skills framework that lets language models (and operators) compose reusable capabilities — from filesystem reads and HTTP requests to sandboxed code execution and curated marketplace skills. @@ -15,6 +15,28 @@ A skill is a versioned, schema-defined unit of work. OmniRoute can inject skills --- +## Agent Skills vs Omni Skills + +OmniRoute has two distinct but complementary skill systems: + +| Dimension | **Omni Skills** (this doc) | **Agent Skills** | +| :--- | :--- | :--- | +| Purpose | LLM tool injection + sandboxed execution | SKILL.md catalog for external agents to discover and consume | +| Source of truth | `src/lib/skills/` + marketplace | `src/lib/agentSkills/` + `skills/` directory | +| Runtime mode | Injected into outbound requests, executed on tool-call events | Static markdown catalog + REST/MCP/A2A discovery endpoints | +| Who uses it | OmniRoute itself (combo routing, inbound LLM calls) | External agents, MCP clients, A2A orchestrators | +| Count | Variable (marketplace-driven) | 42 canonical entries (22 API + 20 CLI) | +| Format | `SkillDefinition` with tool schema + handler | `SKILL.md` frontmatter + markdown body | +| Discovery | `/api/skills/*` REST + `omniroute_skills_*` MCP tools | `/api/agent-skills/*` REST + `omniroute_agent_skills_*` MCP tools + A2A `list-capabilities` | + +**Omni Skills** are the execution engine — they define what OmniRoute *can do* when an LLM invokes a tool. + +**Agent Skills** are the documentation catalog — they explain to external agents *how to use* OmniRoute's REST API and CLI, with structured SKILL.md files that can be fed directly into agent prompts. + +For the Agent Skills catalog, generator, MCP tools, and A2A skill, see [docs/frameworks/AGENT-SKILLS.md](./AGENT-SKILLS.md). + +--- + ## Concepts ### Skill Sources diff --git a/docs/reference/openapi.yaml b/docs/reference/openapi.yaml index faa3b50d07..f8a8f77733 100644 --- a/docs/reference/openapi.yaml +++ b/docs/reference/openapi.yaml @@ -75,6 +75,10 @@ tags: description: Fallback chain management - name: Telemetry description: Telemetry and token health monitoring + - name: Agent Skills + description: >- + Agent Skills catalog — 42 SKILL.md files (22 REST API + 20 CLI) for external agents, + MCP clients, and A2A orchestrators to discover OmniRoute capabilities. paths: # ─── Proxy Endpoints ────────────────────────────────────────── @@ -2730,6 +2734,232 @@ paths: "500": description: Failed to parse OpenAPI spec + # ─── Agent Skills Catalog ──────────────────────────────────────────────────── + + /api/agent-skills: + get: + tags: [Agent Skills] + summary: List agent skills catalog + description: | + Returns the full 42-entry Agent Skills catalog with optional filtering. + Skills describe how to use OmniRoute's REST API and CLI — they are structured + SKILL.md documentation files discoverable by external agents, MCP clients, and + A2A orchestrators. No authentication required. + parameters: + - name: category + in: query + required: false + schema: + type: string + enum: [api, cli] + description: Filter by category (api = REST API skills, cli = CLI skills) + - name: area + in: query + required: false + schema: + type: string + description: Filter by area slug (e.g. "providers", "models", "cli-serve") + responses: + "200": + description: Catalog list + content: + application/json: + schema: + type: object + required: [skills, count, coverage] + properties: + skills: + type: array + items: + $ref: "#/components/schemas/AgentSkill" + count: + type: integer + coverage: + $ref: "#/components/schemas/SkillCoverage" + "400": + $ref: "#/components/responses/BadRequest" + "500": + $ref: "#/components/responses/InternalError" + + /api/agent-skills/{id}: + get: + tags: [Agent Skills] + summary: Get a single agent skill + description: | + Returns metadata for a single agent skill by its canonical ID + (e.g. `omni-providers`, `cli-serve`). No authentication required. + parameters: + - name: id + in: path + required: true + schema: + type: string + pattern: "^[a-z][a-z0-9-]*$" + description: Canonical skill ID + example: omni-providers + responses: + "200": + description: Agent skill metadata + content: + application/json: + schema: + $ref: "#/components/schemas/AgentSkill" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalError" + + /api/agent-skills/{id}/raw: + get: + tags: [Agent Skills] + summary: Get raw SKILL.md content + description: | + Returns the SKILL.md content for a skill as `text/markdown`. + Resolution order: local filesystem `skills/{id}/SKILL.md` → GitHub raw URL (1-hour cache). + No authentication required. + parameters: + - name: id + in: path + required: true + schema: + type: string + pattern: "^[a-z][a-z0-9-]*$" + description: Canonical skill ID + example: omni-providers + responses: + "200": + description: SKILL.md content as Markdown + headers: + X-Skill-Source: + schema: + type: string + enum: [filesystem, github, generated] + description: Where the content was loaded from + X-Skill-Fetched-At: + schema: + type: string + format: date-time + description: ISO timestamp of when the content was fetched + Cache-Control: + schema: + type: string + description: "public, max-age=3600" + content: + text/markdown: + schema: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "502": + description: Upstream GitHub fetch failed + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "500": + $ref: "#/components/responses/InternalError" + + /api/agent-skills/coverage: + get: + tags: [Agent Skills] + summary: Get SKILL.md coverage stats + description: | + Returns how many of the 22 API skills and 20 CLI skills have SKILL.md + files on the local filesystem vs the catalog totals. No authentication required. + responses: + "200": + description: Coverage stats + content: + application/json: + schema: + $ref: "#/components/schemas/SkillCoverage" + "500": + $ref: "#/components/responses/InternalError" + + /api/agent-skills/generate: + post: + tags: [Agent Skills] + summary: Trigger SKILL.md generator + description: | + Runs the Agent Skills generator which writes `skills/{id}/SKILL.md` for + all 42 catalog entries (or a subset via `onlyIds`). Preserves + ` ... ` blocks. + **Requires management authentication.** + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + dryRun: + type: boolean + default: true + description: "If true, reports what would be generated without writing files" + prune: + type: boolean + default: false + description: "If true, deletes skill directories not in the catalog" + onlyIds: + type: array + items: + type: string + description: "If provided, only regenerate these skill IDs" + responses: + "200": + description: Generator report + content: + application/json: + schema: + type: object + required: [generated, unchanged, pruned, orphansDetected, errors] + properties: + generated: + type: array + items: + type: string + description: IDs that got new/updated SKILL.md + unchanged: + type: array + items: + type: string + description: IDs whose content was already up to date + pruned: + type: array + items: + type: string + description: IDs whose directories were deleted (prune mode) + orphansDetected: + type: array + items: + type: string + description: Directories found in skills/ not in the catalog + errors: + type: array + items: + type: object + required: [id, error] + properties: + id: + type: string + error: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "500": + $ref: "#/components/responses/InternalError" + "503": + description: Generator module not available + components: securitySchemes: BearerAuth: @@ -2791,6 +3021,131 @@ components: $ref: "#/components/schemas/ValidationErrorResponse" schemas: + AgentSkill: + type: object + description: >- + Single entry in the Agent Skills catalog. Describes one OmniRoute REST API surface + (category: api) or CLI subcommand group (category: cli) with a canonical ID and a + link to its SKILL.md documentation file. + required: [id, name, description, category, area, rawUrl, githubUrl] + properties: + id: + type: string + pattern: "^[a-z][a-z0-9-]*$" + description: Canonical skill ID (e.g. "omni-providers", "cli-serve") + example: omni-providers + name: + type: string + minLength: 1 + maxLength: 100 + description: Human-readable skill name + example: Provider Management + description: + type: string + minLength: 1 + maxLength: 2000 + description: One-paragraph description of what the skill covers + category: + type: string + enum: [api, cli] + description: "api = REST API skill; cli = CLI subcommand skill" + area: + type: string + minLength: 1 + maxLength: 50 + description: Functional area slug (e.g. "providers", "combos-routing", "cli-serve") + example: providers + endpoints: + type: array + items: + type: string + description: REST API endpoints (present for api-category skills only) + example: ["POST /api/providers", "GET /api/providers/:id"] + cliCommands: + type: array + items: + type: string + description: CLI subcommand names (present for cli-category skills only) + example: ["providers list", "providers test", "providers rotate"] + icon: + type: string + description: Material symbol icon name for dashboard display + isEntry: + type: boolean + description: Whether this is a recommended starting point + isNew: + type: boolean + description: Whether this skill was added in a recent release + rawUrl: + type: string + format: uri + description: GitHub raw URL of the SKILL.md file + example: "https://raw.githubusercontent.com/diegosouzapw/OmniRoute/refs/heads/main/skills/omni-providers/SKILL.md" + githubUrl: + type: string + format: uri + description: GitHub blob URL for viewing the SKILL.md in the browser + example: "https://github.com/diegosouzapw/OmniRoute/blob/main/skills/omni-providers/SKILL.md" + + SkillCoverage: + type: object + description: >- + Coverage statistics for the Agent Skills catalog: how many of the 22 REST API + skills and 20 CLI skills have generated SKILL.md files on the local filesystem. + required: [api, cli, totalSkills, generatedAt] + properties: + api: + type: object + required: [have, total] + properties: + have: + type: integer + minimum: 0 + maximum: 22 + description: Number of API skills with SKILL.md on disk + total: + type: integer + enum: [22] + description: Canonical API skill count (always 22) + cli: + type: object + required: [have, total] + properties: + have: + type: integer + minimum: 0 + maximum: 20 + description: Number of CLI skills with SKILL.md on disk + total: + type: integer + enum: [20] + description: Canonical CLI skill count (always 20) + totalSkills: + type: integer + minimum: 0 + maximum: 42 + description: Sum of api.have + cli.have + generatedAt: + type: string + format: date-time + description: ISO datetime when coverage was last computed + + ErrorResponse: + type: object + description: Standard error response body + required: [error] + properties: + error: + type: object + required: [message] + properties: + message: + type: string + description: Human-readable error message (never includes stack traces) + code: + type: string + description: Machine-readable error code + ServiceStatus: type: object description: Live supervisor state for an embedded service diff --git a/tests/integration/agent-skills-content.test.ts b/tests/integration/agent-skills-content.test.ts new file mode 100644 index 0000000000..9b6631017c --- /dev/null +++ b/tests/integration/agent-skills-content.test.ts @@ -0,0 +1,147 @@ +/** + * Integration tests for Agent Skills content integrity. + * + * Verifies: + * 1. All 42 skill IDs from catalog have skills/{id}/ folder with SKILL.md. + * 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed). + * 3. 10 specific IDs have ... blocks: + * omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a, + * omni-combos-routing, omni-auth, omni-resilience, omni-inference, cli-serve. + * + * Does NOT spin up a server. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts"); + +const SKILLS_DIR = path.resolve(process.cwd(), "skills"); +const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[]; + +// IDs that must have a custom block +const CUSTOM_BLOCK_IDS = [ + "omni-mcp", + "omni-compression", + "cli-providers", + "cli-eval", + "omni-agents-a2a", + "omni-combos-routing", + "omni-auth", + "omni-resilience", + "omni-inference", + "cli-serve", +] as const; + +// ── §1: All 42 catalog IDs have skills/{id}/SKILL.md ───────────────────────── + +test("all 42 catalog IDs have a skills/{id}/ directory", () => { + const missing: string[] = []; + for (const id of ALL_IDS) { + const dirPath = path.join(SKILLS_DIR, id); + if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`); +}); + +test("all 42 catalog IDs have a skills/{id}/SKILL.md file", () => { + const missing: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing SKILL.md files: ${missing.join(", ")}`); +}); + +// ── §2: No omniroute-* directories remain ──────────────────────────────────── + +test("skills/ has zero omniroute-* directories (all pruned)", () => { + if (!fs.existsSync(SKILLS_DIR)) { + // If skills dir doesn't exist at all, nothing to prune + return; + } + const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true }); + const omniRouteDirs = entries + .filter((e) => e.isDirectory() && e.name.startsWith("omniroute-")) + .map((e) => e.name); + assert.deepEqual( + omniRouteDirs, + [], + `Found omniroute-* directories that should have been pruned: ${omniRouteDirs.join(", ")}`, + ); +}); + +test("skills/ directory only contains expected catalog IDs plus README", () => { + if (!fs.existsSync(SKILLS_DIR)) return; + const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true }); + const dirs = entries.filter((e) => e.isDirectory()).map((e) => e.name); + const expectedSet = new Set(ALL_IDS); + const unexpected = dirs.filter((d) => !expectedSet.has(d)); + assert.deepEqual( + unexpected, + [], + `Unexpected directories in skills/: ${unexpected.join(", ")}`, + ); +}); + +// ── §3: 10 specific IDs have custom blocks ─────────────────────────────────── + +for (const id of CUSTOM_BLOCK_IDS) { + test(`skills/${id}/SKILL.md has block`, () => { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + assert.ok( + fs.existsSync(skillPath), + `skills/${id}/SKILL.md does not exist`, + ); + const content = fs.readFileSync(skillPath, "utf-8"); + assert.ok( + content.includes(""), + `skills/${id}/SKILL.md missing block`, + ); + assert.ok( + content.includes(""), + `skills/${id}/SKILL.md missing block`, + ); + }); +} + +// ── Additional integrity checks ─────────────────────────────────────────────── + +test("exactly 10 skills have custom blocks", () => { + const withCustomBlocks: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + if (content.includes("")) { + withCustomBlocks.push(id); + } + } + // Verify exactly the expected 10 IDs have custom blocks + const expectedIds = [...CUSTOM_BLOCK_IDS].sort(); + assert.deepEqual( + withCustomBlocks.sort(), + expectedIds, + `Expected exactly these 10 custom-block IDs: ${expectedIds.join(", ")}\nActual: ${withCustomBlocks.join(", ")}`, + ); +}); + +test("no skill has custom-start without custom-end (unclosed blocks)", () => { + const unclosed: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const hasStart = content.includes(""); + const hasEnd = content.includes(""); + if (hasStart !== hasEnd) { + unclosed.push(id); + } + } + assert.deepEqual(unclosed, [], `Skills with unclosed custom blocks: ${unclosed.join(", ")}`); +}); diff --git a/tests/integration/agent-skills-discovery.test.ts b/tests/integration/agent-skills-discovery.test.ts new file mode 100644 index 0000000000..ec29864a82 --- /dev/null +++ b/tests/integration/agent-skills-discovery.test.ts @@ -0,0 +1,174 @@ +/** + * Integration tests for Agent Skills discovery. + * + * Verifies: + * 1. Every ID in API_SKILL_IDS + CLI_SKILL_IDS has a skills//SKILL.md on disk. + * 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars. + * 3. MCP tool omniroute_agent_skills_list handler returns 42 entries. + * 4. A2A skill list-capabilities returns 1 artifact with 42 lines. + * + * Does NOT spin up a server — tests handlers directly via imports. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +// Dynamic imports for ESM + tsx compatibility +const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts"); +const { agentSkillTools } = await import("../../open-sse/mcp-server/tools/agentSkillTools.ts"); +const { executeListCapabilities } = await import("../../src/lib/a2a/skills/listCapabilities.ts"); +import type { A2ATask } from "../../src/lib/a2a/taskManager.ts"; + +const SKILLS_DIR = path.resolve(process.cwd(), "skills"); + +// ── Frontmatter parser (mirrors catalog.ts parseMarkdownFrontmatter) ───────── + +function parseSkillMarkdown(content: string): { name: string; description: string; body: string } { + const FM_REGEX = /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/; + const match = FM_REGEX.exec(content); + if (!match) return { name: "", description: "", body: content }; + const yamlBlock = match[1]; + const body = match[2] ?? ""; + const nameMatch = /^name:\s*(.+)$/m.exec(yamlBlock); + const descMatch = /^description:\s*(.+)$/m.exec(yamlBlock); + return { + name: nameMatch ? nameMatch[1].trim().replace(/^["']|["']$/g, "") : "", + description: descMatch ? descMatch[1].trim().replace(/^["']|["']$/g, "") : "", + body, + }; +} + +// ── §1: Filesystem — every skill ID has a SKILL.md ─────────────────────────── + +const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[]; + +test("skills/ directory exists and is readable", () => { + assert.ok(fs.existsSync(SKILLS_DIR), `skills/ directory not found at ${SKILLS_DIR}`); +}); + +test("every API skill ID has skills//SKILL.md on disk", () => { + const missing: string[] = []; + for (const id of API_SKILL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing API SKILL.md files: ${missing.join(", ")}`); +}); + +test("every CLI skill ID has skills//SKILL.md on disk", () => { + const missing: string[] = []; + for (const id of CLI_SKILL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing CLI SKILL.md files: ${missing.join(", ")}`); +}); + +test("total skill count is exactly 42 (22 API + 20 CLI)", () => { + assert.equal(API_SKILL_IDS.length + CLI_SKILL_IDS.length, 42); +}); + +// ── §2: Frontmatter validation ──────────────────────────────────────────────── + +test("each SKILL.md has non-empty name in frontmatter", () => { + const failures: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; // covered by disk test above + const content = fs.readFileSync(skillPath, "utf-8"); + const { name } = parseSkillMarkdown(content); + if (!name || name.length === 0) { + failures.push(id); + } + } + assert.deepEqual(failures, [], `SKILL.md files with empty name: ${failures.join(", ")}`); +}); + +test("each SKILL.md has non-empty description in frontmatter", () => { + const failures: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const { description } = parseSkillMarkdown(content); + if (!description || description.length === 0) { + failures.push(id); + } + } + assert.deepEqual(failures, [], `SKILL.md files with empty description: ${failures.join(", ")}`); +}); + +test("each SKILL.md body is at least 100 chars", () => { + const failures: Array<{ id: string; len: number }> = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const { body } = parseSkillMarkdown(content); + if (body.length < 100) { + failures.push({ id, len: body.length }); + } + } + const msg = failures.map((f) => `${f.id}(${f.len})`).join(", "); + assert.deepEqual(failures, [], `SKILL.md files with body < 100 chars: ${msg}`); +}); + +// ── §3: MCP tool omniroute_agent_skills_list ───────────────────────────────── + +test("MCP omniroute_agent_skills_list handler returns count 42", async () => { + const result = await agentSkillTools.omniroute_agent_skills_list.handler({}); + assert.equal(result.count, 42, `Expected 42 but got ${result.count}`); + assert.ok(Array.isArray(result.skills)); + assert.equal(result.skills.length, 42); +}); + +test("MCP omniroute_agent_skills_list result has all 42 IDs", async () => { + const result = await agentSkillTools.omniroute_agent_skills_list.handler({}); + const returnedIds = new Set(result.skills.map((s: { id: string }) => s.id)); + for (const id of ALL_IDS) { + assert.ok(returnedIds.has(id), `MCP result missing skill ID: ${id}`); + } +}); + +// ── §4: A2A list-capabilities ──────────────────────────────────────────────── + +const stubTask = {} as A2ATask; + +test("A2A list-capabilities returns exactly 1 artifact", async () => { + const result = await executeListCapabilities(stubTask); + assert.equal(result.artifacts.length, 1, "Expected exactly 1 artifact"); + assert.equal(result.artifacts[0].type, "text", "Artifact type should be 'text'"); +}); + +test("A2A list-capabilities artifact content contains 42 skill IDs as table rows", async () => { + const result = await executeListCapabilities(stubTask); + const content = result.artifacts[0].content; + const rows = content.split("\n").filter((line) => line.startsWith("| ") && !line.startsWith("| ID") && !line.startsWith("| ---")); + // Each skill row starts with "| |" + assert.ok( + rows.length >= 42, + `Expected at least 42 data rows but got ${rows.length}`, + ); +}); + +test("A2A list-capabilities metadata.totalSkills === 42", async () => { + const result = await executeListCapabilities(stubTask); + assert.equal(result.metadata.totalSkills, 42); +}); + +test("A2A list-capabilities artifact contains all 42 skill IDs", async () => { + const result = await executeListCapabilities(stubTask); + const content = result.artifacts[0].content; + const missing: string[] = []; + for (const id of ALL_IDS) { + if (!content.includes(id)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `A2A artifact missing skill IDs: ${missing.join(", ")}`); +});