diff --git a/tests/integration/agent-skills-content.test.ts b/tests/integration/agent-skills-content.test.ts new file mode 100644 index 0000000000..9b6631017c --- /dev/null +++ b/tests/integration/agent-skills-content.test.ts @@ -0,0 +1,147 @@ +/** + * Integration tests for Agent Skills content integrity. + * + * Verifies: + * 1. All 42 skill IDs from catalog have skills/{id}/ folder with SKILL.md. + * 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed). + * 3. 10 specific IDs have ... blocks: + * omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a, + * omni-combos-routing, omni-auth, omni-resilience, omni-inference, cli-serve. + * + * Does NOT spin up a server. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts"); + +const SKILLS_DIR = path.resolve(process.cwd(), "skills"); +const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[]; + +// IDs that must have a custom block +const CUSTOM_BLOCK_IDS = [ + "omni-mcp", + "omni-compression", + "cli-providers", + "cli-eval", + "omni-agents-a2a", + "omni-combos-routing", + "omni-auth", + "omni-resilience", + "omni-inference", + "cli-serve", +] as const; + +// ── §1: All 42 catalog IDs have skills/{id}/SKILL.md ───────────────────────── + +test("all 42 catalog IDs have a skills/{id}/ directory", () => { + const missing: string[] = []; + for (const id of ALL_IDS) { + const dirPath = path.join(SKILLS_DIR, id); + if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`); +}); + +test("all 42 catalog IDs have a skills/{id}/SKILL.md file", () => { + const missing: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing SKILL.md files: ${missing.join(", ")}`); +}); + +// ── §2: No omniroute-* directories remain ──────────────────────────────────── + +test("skills/ has zero omniroute-* directories (all pruned)", () => { + if (!fs.existsSync(SKILLS_DIR)) { + // If skills dir doesn't exist at all, nothing to prune + return; + } + const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true }); + const omniRouteDirs = entries + .filter((e) => e.isDirectory() && e.name.startsWith("omniroute-")) + .map((e) => e.name); + assert.deepEqual( + omniRouteDirs, + [], + `Found omniroute-* directories that should have been pruned: ${omniRouteDirs.join(", ")}`, + ); +}); + +test("skills/ directory only contains expected catalog IDs plus README", () => { + if (!fs.existsSync(SKILLS_DIR)) return; + const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true }); + const dirs = entries.filter((e) => e.isDirectory()).map((e) => e.name); + const expectedSet = new Set(ALL_IDS); + const unexpected = dirs.filter((d) => !expectedSet.has(d)); + assert.deepEqual( + unexpected, + [], + `Unexpected directories in skills/: ${unexpected.join(", ")}`, + ); +}); + +// ── §3: 10 specific IDs have custom blocks ─────────────────────────────────── + +for (const id of CUSTOM_BLOCK_IDS) { + test(`skills/${id}/SKILL.md has block`, () => { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + assert.ok( + fs.existsSync(skillPath), + `skills/${id}/SKILL.md does not exist`, + ); + const content = fs.readFileSync(skillPath, "utf-8"); + assert.ok( + content.includes(""), + `skills/${id}/SKILL.md missing block`, + ); + assert.ok( + content.includes(""), + `skills/${id}/SKILL.md missing block`, + ); + }); +} + +// ── Additional integrity checks ─────────────────────────────────────────────── + +test("exactly 10 skills have custom blocks", () => { + const withCustomBlocks: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + if (content.includes("")) { + withCustomBlocks.push(id); + } + } + // Verify exactly the expected 10 IDs have custom blocks + const expectedIds = [...CUSTOM_BLOCK_IDS].sort(); + assert.deepEqual( + withCustomBlocks.sort(), + expectedIds, + `Expected exactly these 10 custom-block IDs: ${expectedIds.join(", ")}\nActual: ${withCustomBlocks.join(", ")}`, + ); +}); + +test("no skill has custom-start without custom-end (unclosed blocks)", () => { + const unclosed: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const hasStart = content.includes(""); + const hasEnd = content.includes(""); + if (hasStart !== hasEnd) { + unclosed.push(id); + } + } + assert.deepEqual(unclosed, [], `Skills with unclosed custom blocks: ${unclosed.join(", ")}`); +}); diff --git a/tests/integration/agent-skills-discovery.test.ts b/tests/integration/agent-skills-discovery.test.ts new file mode 100644 index 0000000000..ec29864a82 --- /dev/null +++ b/tests/integration/agent-skills-discovery.test.ts @@ -0,0 +1,174 @@ +/** + * Integration tests for Agent Skills discovery. + * + * Verifies: + * 1. Every ID in API_SKILL_IDS + CLI_SKILL_IDS has a skills//SKILL.md on disk. + * 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars. + * 3. MCP tool omniroute_agent_skills_list handler returns 42 entries. + * 4. A2A skill list-capabilities returns 1 artifact with 42 lines. + * + * Does NOT spin up a server — tests handlers directly via imports. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +// Dynamic imports for ESM + tsx compatibility +const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts"); +const { agentSkillTools } = await import("../../open-sse/mcp-server/tools/agentSkillTools.ts"); +const { executeListCapabilities } = await import("../../src/lib/a2a/skills/listCapabilities.ts"); +import type { A2ATask } from "../../src/lib/a2a/taskManager.ts"; + +const SKILLS_DIR = path.resolve(process.cwd(), "skills"); + +// ── Frontmatter parser (mirrors catalog.ts parseMarkdownFrontmatter) ───────── + +function parseSkillMarkdown(content: string): { name: string; description: string; body: string } { + const FM_REGEX = /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/; + const match = FM_REGEX.exec(content); + if (!match) return { name: "", description: "", body: content }; + const yamlBlock = match[1]; + const body = match[2] ?? ""; + const nameMatch = /^name:\s*(.+)$/m.exec(yamlBlock); + const descMatch = /^description:\s*(.+)$/m.exec(yamlBlock); + return { + name: nameMatch ? nameMatch[1].trim().replace(/^["']|["']$/g, "") : "", + description: descMatch ? descMatch[1].trim().replace(/^["']|["']$/g, "") : "", + body, + }; +} + +// ── §1: Filesystem — every skill ID has a SKILL.md ─────────────────────────── + +const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[]; + +test("skills/ directory exists and is readable", () => { + assert.ok(fs.existsSync(SKILLS_DIR), `skills/ directory not found at ${SKILLS_DIR}`); +}); + +test("every API skill ID has skills//SKILL.md on disk", () => { + const missing: string[] = []; + for (const id of API_SKILL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing API SKILL.md files: ${missing.join(", ")}`); +}); + +test("every CLI skill ID has skills//SKILL.md on disk", () => { + const missing: string[] = []; + for (const id of CLI_SKILL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `Missing CLI SKILL.md files: ${missing.join(", ")}`); +}); + +test("total skill count is exactly 42 (22 API + 20 CLI)", () => { + assert.equal(API_SKILL_IDS.length + CLI_SKILL_IDS.length, 42); +}); + +// ── §2: Frontmatter validation ──────────────────────────────────────────────── + +test("each SKILL.md has non-empty name in frontmatter", () => { + const failures: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; // covered by disk test above + const content = fs.readFileSync(skillPath, "utf-8"); + const { name } = parseSkillMarkdown(content); + if (!name || name.length === 0) { + failures.push(id); + } + } + assert.deepEqual(failures, [], `SKILL.md files with empty name: ${failures.join(", ")}`); +}); + +test("each SKILL.md has non-empty description in frontmatter", () => { + const failures: string[] = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const { description } = parseSkillMarkdown(content); + if (!description || description.length === 0) { + failures.push(id); + } + } + assert.deepEqual(failures, [], `SKILL.md files with empty description: ${failures.join(", ")}`); +}); + +test("each SKILL.md body is at least 100 chars", () => { + const failures: Array<{ id: string; len: number }> = []; + for (const id of ALL_IDS) { + const skillPath = path.join(SKILLS_DIR, id, "SKILL.md"); + if (!fs.existsSync(skillPath)) continue; + const content = fs.readFileSync(skillPath, "utf-8"); + const { body } = parseSkillMarkdown(content); + if (body.length < 100) { + failures.push({ id, len: body.length }); + } + } + const msg = failures.map((f) => `${f.id}(${f.len})`).join(", "); + assert.deepEqual(failures, [], `SKILL.md files with body < 100 chars: ${msg}`); +}); + +// ── §3: MCP tool omniroute_agent_skills_list ───────────────────────────────── + +test("MCP omniroute_agent_skills_list handler returns count 42", async () => { + const result = await agentSkillTools.omniroute_agent_skills_list.handler({}); + assert.equal(result.count, 42, `Expected 42 but got ${result.count}`); + assert.ok(Array.isArray(result.skills)); + assert.equal(result.skills.length, 42); +}); + +test("MCP omniroute_agent_skills_list result has all 42 IDs", async () => { + const result = await agentSkillTools.omniroute_agent_skills_list.handler({}); + const returnedIds = new Set(result.skills.map((s: { id: string }) => s.id)); + for (const id of ALL_IDS) { + assert.ok(returnedIds.has(id), `MCP result missing skill ID: ${id}`); + } +}); + +// ── §4: A2A list-capabilities ──────────────────────────────────────────────── + +const stubTask = {} as A2ATask; + +test("A2A list-capabilities returns exactly 1 artifact", async () => { + const result = await executeListCapabilities(stubTask); + assert.equal(result.artifacts.length, 1, "Expected exactly 1 artifact"); + assert.equal(result.artifacts[0].type, "text", "Artifact type should be 'text'"); +}); + +test("A2A list-capabilities artifact content contains 42 skill IDs as table rows", async () => { + const result = await executeListCapabilities(stubTask); + const content = result.artifacts[0].content; + const rows = content.split("\n").filter((line) => line.startsWith("| ") && !line.startsWith("| ID") && !line.startsWith("| ---")); + // Each skill row starts with "| |" + assert.ok( + rows.length >= 42, + `Expected at least 42 data rows but got ${rows.length}`, + ); +}); + +test("A2A list-capabilities metadata.totalSkills === 42", async () => { + const result = await executeListCapabilities(stubTask); + assert.equal(result.metadata.totalSkills, 42); +}); + +test("A2A list-capabilities artifact contains all 42 skill IDs", async () => { + const result = await executeListCapabilities(stubTask); + const content = result.artifacts[0].content; + const missing: string[] = []; + for (const id of ALL_IDS) { + if (!content.includes(id)) { + missing.push(id); + } + } + assert.deepEqual(missing, [], `A2A artifact missing skill IDs: ${missing.join(", ")}`); +});