mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
test(agent-skills): integration tests for discovery + content + MCP + A2A
Add tests/integration/agent-skills-discovery.test.ts (13 tests):
- Verifies every API_SKILL_IDS + CLI_SKILL_IDS has skills/<id>/SKILL.md on disk
- Validates frontmatter name + description present in each SKILL.md
- Validates body >= 100 chars per SKILL.md
- Verifies MCP omniroute_agent_skills_list handler returns 42 entries
- Verifies A2A list-capabilities returns 1 artifact containing all 42 IDs
Add tests/integration/agent-skills-content.test.ts (16 tests):
- Confirms all 42 catalog IDs have skills/{id}/ directory + SKILL.md
- Confirms 0 omniroute-* directories remain (post-prune)
- Confirms exactly 10 IDs have skill:custom-start/end blocks
(omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a,
omni-combos-routing, omni-auth, omni-resilience, omni-inference, cli-serve)
- Confirms no unclosed custom blocks
This commit is contained in:
diegosouzapw
147
tests/integration/agent-skills-content.test.ts
Normal file
147
tests/integration/agent-skills-content.test.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Integration tests for Agent Skills content integrity.
|
||||
*
|
||||
* Verifies:
|
||||
* 1. All 42 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
|
||||
* 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed).
|
||||
* 3. 10 specific IDs have <!-- skill:custom-start --> ... <!-- skill:custom-end --> blocks:
|
||||
* omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a,
|
||||
* omni-combos-routing, omni-auth, omni-resilience, omni-inference, cli-serve.
|
||||
*
|
||||
* Does NOT spin up a server.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
|
||||
|
||||
const SKILLS_DIR = path.resolve(process.cwd(), "skills");
|
||||
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
|
||||
|
||||
// IDs that must have a custom block
|
||||
const CUSTOM_BLOCK_IDS = [
|
||||
"omni-mcp",
|
||||
"omni-compression",
|
||||
"cli-providers",
|
||||
"cli-eval",
|
||||
"omni-agents-a2a",
|
||||
"omni-combos-routing",
|
||||
"omni-auth",
|
||||
"omni-resilience",
|
||||
"omni-inference",
|
||||
"cli-serve",
|
||||
] as const;
|
||||
|
||||
// ── §1: All 42 catalog IDs have skills/{id}/SKILL.md ─────────────────────────
|
||||
|
||||
test("all 42 catalog IDs have a skills/{id}/ directory", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const dirPath = path.join(SKILLS_DIR, id);
|
||||
if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) {
|
||||
missing.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`);
|
||||
});
|
||||
|
||||
test("all 42 catalog IDs have a skills/{id}/SKILL.md file", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) {
|
||||
missing.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(missing, [], `Missing SKILL.md files: ${missing.join(", ")}`);
|
||||
});
|
||||
|
||||
// ── §2: No omniroute-* directories remain ────────────────────────────────────
|
||||
|
||||
test("skills/ has zero omniroute-* directories (all pruned)", () => {
|
||||
if (!fs.existsSync(SKILLS_DIR)) {
|
||||
// If skills dir doesn't exist at all, nothing to prune
|
||||
return;
|
||||
}
|
||||
const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true });
|
||||
const omniRouteDirs = entries
|
||||
.filter((e) => e.isDirectory() && e.name.startsWith("omniroute-"))
|
||||
.map((e) => e.name);
|
||||
assert.deepEqual(
|
||||
omniRouteDirs,
|
||||
[],
|
||||
`Found omniroute-* directories that should have been pruned: ${omniRouteDirs.join(", ")}`,
|
||||
);
|
||||
});
|
||||
|
||||
test("skills/ directory only contains expected catalog IDs plus README", () => {
|
||||
if (!fs.existsSync(SKILLS_DIR)) return;
|
||||
const entries = fs.readdirSync(SKILLS_DIR, { withFileTypes: true });
|
||||
const dirs = entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
||||
const expectedSet = new Set(ALL_IDS);
|
||||
const unexpected = dirs.filter((d) => !expectedSet.has(d));
|
||||
assert.deepEqual(
|
||||
unexpected,
|
||||
[],
|
||||
`Unexpected directories in skills/: ${unexpected.join(", ")}`,
|
||||
);
|
||||
});
|
||||
|
||||
// ── §3: 10 specific IDs have custom blocks ───────────────────────────────────
|
||||
|
||||
for (const id of CUSTOM_BLOCK_IDS) {
|
||||
test(`skills/${id}/SKILL.md has <!-- skill:custom-start --> block`, () => {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
assert.ok(
|
||||
fs.existsSync(skillPath),
|
||||
`skills/${id}/SKILL.md does not exist`,
|
||||
);
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
assert.ok(
|
||||
content.includes("<!-- skill:custom-start -->"),
|
||||
`skills/${id}/SKILL.md missing <!-- skill:custom-start --> block`,
|
||||
);
|
||||
assert.ok(
|
||||
content.includes("<!-- skill:custom-end -->"),
|
||||
`skills/${id}/SKILL.md missing <!-- skill:custom-end --> block`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Additional integrity checks ───────────────────────────────────────────────
|
||||
|
||||
test("exactly 10 skills have custom blocks", () => {
|
||||
const withCustomBlocks: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) continue;
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
if (content.includes("<!-- skill:custom-start -->")) {
|
||||
withCustomBlocks.push(id);
|
||||
}
|
||||
}
|
||||
// Verify exactly the expected 10 IDs have custom blocks
|
||||
const expectedIds = [...CUSTOM_BLOCK_IDS].sort();
|
||||
assert.deepEqual(
|
||||
withCustomBlocks.sort(),
|
||||
expectedIds,
|
||||
`Expected exactly these 10 custom-block IDs: ${expectedIds.join(", ")}\nActual: ${withCustomBlocks.join(", ")}`,
|
||||
);
|
||||
});
|
||||
|
||||
test("no skill has custom-start without custom-end (unclosed blocks)", () => {
|
||||
const unclosed: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) continue;
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
const hasStart = content.includes("<!-- skill:custom-start -->");
|
||||
const hasEnd = content.includes("<!-- skill:custom-end -->");
|
||||
if (hasStart !== hasEnd) {
|
||||
unclosed.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(unclosed, [], `Skills with unclosed custom blocks: ${unclosed.join(", ")}`);
|
||||
});
|
||||
174
tests/integration/agent-skills-discovery.test.ts
Normal file
174
tests/integration/agent-skills-discovery.test.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Integration tests for Agent Skills discovery.
|
||||
*
|
||||
* Verifies:
|
||||
* 1. Every ID in API_SKILL_IDS + CLI_SKILL_IDS has a skills/<id>/SKILL.md on disk.
|
||||
* 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars.
|
||||
* 3. MCP tool omniroute_agent_skills_list handler returns 42 entries.
|
||||
* 4. A2A skill list-capabilities returns 1 artifact with 42 lines.
|
||||
*
|
||||
* Does NOT spin up a server — tests handlers directly via imports.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
// Dynamic imports for ESM + tsx compatibility
|
||||
const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
|
||||
const { agentSkillTools } = await import("../../open-sse/mcp-server/tools/agentSkillTools.ts");
|
||||
const { executeListCapabilities } = await import("../../src/lib/a2a/skills/listCapabilities.ts");
|
||||
import type { A2ATask } from "../../src/lib/a2a/taskManager.ts";
|
||||
|
||||
const SKILLS_DIR = path.resolve(process.cwd(), "skills");
|
||||
|
||||
// ── Frontmatter parser (mirrors catalog.ts parseMarkdownFrontmatter) ─────────
|
||||
|
||||
function parseSkillMarkdown(content: string): { name: string; description: string; body: string } {
|
||||
const FM_REGEX = /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/;
|
||||
const match = FM_REGEX.exec(content);
|
||||
if (!match) return { name: "", description: "", body: content };
|
||||
const yamlBlock = match[1];
|
||||
const body = match[2] ?? "";
|
||||
const nameMatch = /^name:\s*(.+)$/m.exec(yamlBlock);
|
||||
const descMatch = /^description:\s*(.+)$/m.exec(yamlBlock);
|
||||
return {
|
||||
name: nameMatch ? nameMatch[1].trim().replace(/^["']|["']$/g, "") : "",
|
||||
description: descMatch ? descMatch[1].trim().replace(/^["']|["']$/g, "") : "",
|
||||
body,
|
||||
};
|
||||
}
|
||||
|
||||
// ── §1: Filesystem — every skill ID has a SKILL.md ───────────────────────────
|
||||
|
||||
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
|
||||
|
||||
test("skills/ directory exists and is readable", () => {
|
||||
assert.ok(fs.existsSync(SKILLS_DIR), `skills/ directory not found at ${SKILLS_DIR}`);
|
||||
});
|
||||
|
||||
test("every API skill ID has skills/<id>/SKILL.md on disk", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of API_SKILL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) {
|
||||
missing.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(missing, [], `Missing API SKILL.md files: ${missing.join(", ")}`);
|
||||
});
|
||||
|
||||
test("every CLI skill ID has skills/<id>/SKILL.md on disk", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of CLI_SKILL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) {
|
||||
missing.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(missing, [], `Missing CLI SKILL.md files: ${missing.join(", ")}`);
|
||||
});
|
||||
|
||||
test("total skill count is exactly 42 (22 API + 20 CLI)", () => {
|
||||
assert.equal(API_SKILL_IDS.length + CLI_SKILL_IDS.length, 42);
|
||||
});
|
||||
|
||||
// ── §2: Frontmatter validation ────────────────────────────────────────────────
|
||||
|
||||
test("each SKILL.md has non-empty name in frontmatter", () => {
|
||||
const failures: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) continue; // covered by disk test above
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
const { name } = parseSkillMarkdown(content);
|
||||
if (!name || name.length === 0) {
|
||||
failures.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(failures, [], `SKILL.md files with empty name: ${failures.join(", ")}`);
|
||||
});
|
||||
|
||||
test("each SKILL.md has non-empty description in frontmatter", () => {
|
||||
const failures: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) continue;
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
const { description } = parseSkillMarkdown(content);
|
||||
if (!description || description.length === 0) {
|
||||
failures.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(failures, [], `SKILL.md files with empty description: ${failures.join(", ")}`);
|
||||
});
|
||||
|
||||
test("each SKILL.md body is at least 100 chars", () => {
|
||||
const failures: Array<{ id: string; len: number }> = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
if (!fs.existsSync(skillPath)) continue;
|
||||
const content = fs.readFileSync(skillPath, "utf-8");
|
||||
const { body } = parseSkillMarkdown(content);
|
||||
if (body.length < 100) {
|
||||
failures.push({ id, len: body.length });
|
||||
}
|
||||
}
|
||||
const msg = failures.map((f) => `${f.id}(${f.len})`).join(", ");
|
||||
assert.deepEqual(failures, [], `SKILL.md files with body < 100 chars: ${msg}`);
|
||||
});
|
||||
|
||||
// ── §3: MCP tool omniroute_agent_skills_list ─────────────────────────────────
|
||||
|
||||
test("MCP omniroute_agent_skills_list handler returns count 42", async () => {
|
||||
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
|
||||
assert.equal(result.count, 42, `Expected 42 but got ${result.count}`);
|
||||
assert.ok(Array.isArray(result.skills));
|
||||
assert.equal(result.skills.length, 42);
|
||||
});
|
||||
|
||||
test("MCP omniroute_agent_skills_list result has all 42 IDs", async () => {
|
||||
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
|
||||
const returnedIds = new Set(result.skills.map((s: { id: string }) => s.id));
|
||||
for (const id of ALL_IDS) {
|
||||
assert.ok(returnedIds.has(id), `MCP result missing skill ID: ${id}`);
|
||||
}
|
||||
});
|
||||
|
||||
// ── §4: A2A list-capabilities ────────────────────────────────────────────────
|
||||
|
||||
const stubTask = {} as A2ATask;
|
||||
|
||||
test("A2A list-capabilities returns exactly 1 artifact", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
assert.equal(result.artifacts.length, 1, "Expected exactly 1 artifact");
|
||||
assert.equal(result.artifacts[0].type, "text", "Artifact type should be 'text'");
|
||||
});
|
||||
|
||||
test("A2A list-capabilities artifact content contains 42 skill IDs as table rows", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
const content = result.artifacts[0].content;
|
||||
const rows = content.split("\n").filter((line) => line.startsWith("| ") && !line.startsWith("| ID") && !line.startsWith("| ---"));
|
||||
// Each skill row starts with "| <id> |"
|
||||
assert.ok(
|
||||
rows.length >= 42,
|
||||
`Expected at least 42 data rows but got ${rows.length}`,
|
||||
);
|
||||
});
|
||||
|
||||
test("A2A list-capabilities metadata.totalSkills === 42", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
assert.equal(result.metadata.totalSkills, 42);
|
||||
});
|
||||
|
||||
test("A2A list-capabilities artifact contains all 42 skill IDs", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
const content = result.artifacts[0].content;
|
||||
const missing: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
if (!content.includes(id)) {
|
||||
missing.push(id);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(missing, [], `A2A artifact missing skill IDs: ${missing.join(", ")}`);
|
||||
});
|
||||
Reference in New Issue
Block a user