fix(release): align agent skills catalog tests

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
adevwithpurpose
2026-08-17 05:18:57 -03:00
parent 810c6b9843
commit be364d2c70
5 changed files with 33 additions and 30 deletions

View File

@@ -94,7 +94,7 @@ function deriveCatalog(): AgentSkill[] {
// ── Public API ───────────────────────────────────────────────────────────────
/**
* Returns the full catalog (45 entries). Cached in module scope after first call.
* Returns the full catalog (46 entries). Cached in module scope after first call.
* Safe to call multiple times — re-derives only after `refreshCatalog()`.
*/
export function getCatalog(): AgentSkill[] {

View File

@@ -69,7 +69,7 @@ export interface AgentSkill {
}
export interface SkillCoverage {
// Totals are derived from the catalog id lists (literal types went stale the
// Totals are derived from the canonical catalog id lists (literal types went stale the
// first time the catalog grew — cli-skill-collector, 2026-07-15).
api: { have: number; total: number };
cli: { have: number; total: number };

View File

@@ -35,9 +35,9 @@ export interface CuratedSkillEntry {
isNew?: boolean;
}
// ── Canonical 45-entry curated list (D28) ────────────────────────────────────
// ── Canonical 46-entry curated list (D28) ────────────────────────────────────
/** Curated metadata for all 45 agent skills. Source-of-truth for the catalog. */
/** Curated metadata for all 46 agent skills. Source-of-truth for the catalog. */
export const CURATED_SKILLS: CuratedSkillEntry[] = [
// ── API Skills (23) ─────────────────────────────────────────────────────────

View File

@@ -2,9 +2,9 @@
* Integration tests for Agent Skills content integrity.
*
* Verifies:
* 1. All 45 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
* 1. All 46 skill IDs from the catalog have a skills/{id}/ folder with SKILL.md.
* 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed).
* 3. 12 specific IDs have <!-- skill:custom-start --> ... <!-- skill:custom-end --> blocks:
* 3. 14 specific IDs have <!-- skill:custom-start --> ... <!-- skill:custom-end --> blocks:
* omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a,
* omni-combos-routing, omni-auth, omni-resilience, omni-inference, cli-serve.
*
@@ -15,11 +15,11 @@ import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS } =
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS, getCatalog } =
await import("../../src/lib/agentSkills/catalog.ts");
const SKILLS_DIR = path.resolve(process.cwd(), "skills");
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS, ...CONFIG_SKILL_IDS] as string[];
const ALL_IDS = getCatalog().map((skill) => skill.id);
// IDs that must have a custom block
const CUSTOM_BLOCK_IDS = [
@@ -35,12 +35,14 @@ const CUSTOM_BLOCK_IDS = [
"omni-inference",
"cli-serve",
"omni-providers",
"omni-settings",
"config-codex-cli",
"ponytail",
] as const;
// ── §1: All 45 catalog IDs have skills/{id}/SKILL.md ─────────────────────────
// ── §1: All 46 catalog IDs have skills/{id}/SKILL.md ─────────────────────────
test("all 45 catalog IDs have a skills/{id}/ directory", () => {
test("all 46 catalog IDs have a skills/{id}/ directory", () => {
const missing: string[] = [];
for (const id of ALL_IDS) {
const dirPath = path.join(SKILLS_DIR, id);
@@ -51,7 +53,7 @@ test("all 45 catalog IDs have a skills/{id}/ directory", () => {
assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`);
});
test("all 45 catalog IDs have a skills/{id}/SKILL.md file", () => {
test("all 46 catalog IDs have a skills/{id}/SKILL.md file", () => {
const missing: string[] = [];
for (const id of ALL_IDS) {
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
@@ -89,7 +91,7 @@ test("skills/ directory only contains expected catalog IDs plus README", () => {
assert.deepEqual(unexpected, [], `Unexpected directories in skills/: ${unexpected.join(", ")}`);
});
// ── §3: 10 specific IDs have custom blocks ───────────────────────────────────
// ── §3: 14 specific IDs have custom blocks ───────────────────────────────────
for (const id of CUSTOM_BLOCK_IDS) {
test(`skills/${id}/SKILL.md has <!-- skill:custom-start --> block`, () => {
@@ -109,7 +111,7 @@ for (const id of CUSTOM_BLOCK_IDS) {
// ── Additional integrity checks ───────────────────────────────────────────────
test("exactly 13 skills have custom blocks", () => {
test("exactly 14 skills have custom blocks", () => {
const withCustomBlocks: string[] = [];
for (const id of ALL_IDS) {
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
@@ -119,12 +121,12 @@ test("exactly 13 skills have custom blocks", () => {
withCustomBlocks.push(id);
}
}
// Verify exactly the expected 13 IDs have custom blocks
// Verify exactly the expected 14 IDs have custom blocks
const expectedIds = [...CUSTOM_BLOCK_IDS].sort();
assert.deepEqual(
withCustomBlocks.sort(),
expectedIds,
`Expected exactly these 13 custom-block IDs: ${expectedIds.join(", ")}\nActual: ${withCustomBlocks.join(", ")}`
`Expected exactly these 14 custom-block IDs: ${expectedIds.join(", ")}\nActual: ${withCustomBlocks.join(", ")}`
);
});

View File

@@ -4,8 +4,8 @@
* Verifies:
* 1. Every catalog ID has a skills/<id>/SKILL.md on disk.
* 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars.
* 3. MCP tool omniroute_agent_skills_list handler returns 45 entries.
* 4. A2A skill list-capabilities returns one artifact containing all 45 entries.
* 3. MCP tool omniroute_agent_skills_list handler returns 46 entries.
* 4. A2A skill list-capabilities returns one artifact containing all 46 entries.
*
* Does NOT spin up a server — tests handlers directly via imports.
*/
@@ -15,7 +15,7 @@ import fs from "node:fs";
import path from "node:path";
// Dynamic imports for ESM + tsx compatibility
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS } =
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS, getCatalog } =
await import("../../src/lib/agentSkills/catalog.ts");
const { agentSkillTools } = await import("../../open-sse/mcp-server/tools/agentSkillTools.ts");
const { executeListCapabilities } = await import("../../src/lib/a2a/skills/listCapabilities.ts");
@@ -42,7 +42,8 @@ function parseSkillMarkdown(content: string): { name: string; description: strin
// ── §1: Filesystem — every skill ID has a SKILL.md ───────────────────────────
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS, ...CONFIG_SKILL_IDS] as string[];
const CANONICAL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS, ...CONFIG_SKILL_IDS] as string[];
const ALL_IDS = getCatalog().map((skill) => skill.id);
test("skills/ directory exists and is readable", () => {
assert.ok(fs.existsSync(SKILLS_DIR), `skills/ directory not found at ${SKILLS_DIR}`);
@@ -81,8 +82,8 @@ test("every config skill ID has skills/<id>/SKILL.md on disk", () => {
assert.deepEqual(missing, [], `Missing config SKILL.md files: ${missing.join(", ")}`);
});
test("total skill count is exactly 45 (23 API + 21 CLI + 1 config)", () => {
assert.equal(ALL_IDS.length, 45);
test("canonical skill count is exactly 45 (23 API + 21 CLI + 1 config)", () => {
assert.equal(CANONICAL_IDS.length, 45);
});
// ── §2: Frontmatter validation ────────────────────────────────────────────────
@@ -132,14 +133,14 @@ test("each SKILL.md body is at least 100 chars", () => {
// ── §3: MCP tool omniroute_agent_skills_list ─────────────────────────────────
test("MCP omniroute_agent_skills_list handler returns count 45", async () => {
test("MCP omniroute_agent_skills_list handler returns count 46", async () => {
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
assert.equal(result.count, 45, `Expected 45 but got ${result.count}`);
assert.equal(result.count, 46, `Expected 46 but got ${result.count}`);
assert.ok(Array.isArray(result.skills));
assert.equal(result.skills.length, 45);
assert.equal(result.skills.length, 46);
});
test("MCP omniroute_agent_skills_list result has all 45 IDs", async () => {
test("MCP omniroute_agent_skills_list result has all 46 IDs", async () => {
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
const returnedIds = new Set(result.skills.map((s: { id: string }) => s.id));
for (const id of ALL_IDS) {
@@ -157,7 +158,7 @@ test("A2A list-capabilities returns exactly 1 artifact", async () => {
assert.equal(result.artifacts[0].type, "text", "Artifact type should be 'text'");
});
test("A2A list-capabilities artifact content contains 45 skill IDs as table rows", async () => {
test("A2A list-capabilities artifact content contains 46 skill IDs as table rows", async () => {
const result = await executeListCapabilities(stubTask);
const content = result.artifacts[0].content;
const rows = content
@@ -166,15 +167,15 @@ test("A2A list-capabilities artifact content contains 45 skill IDs as table rows
(line) => line.startsWith("| ") && !line.startsWith("| ID") && !line.startsWith("| ---")
);
// Each skill row starts with "| <id> |"
assert.equal(rows.length, 45, `Expected 45 data rows but got ${rows.length}`);
assert.equal(rows.length, 46, `Expected 46 data rows but got ${rows.length}`);
});
test("A2A list-capabilities metadata.totalSkills === 45", async () => {
test("A2A list-capabilities metadata.totalSkills === 46", async () => {
const result = await executeListCapabilities(stubTask);
assert.equal(result.metadata.totalSkills, 45);
assert.equal(result.metadata.totalSkills, 46);
});
test("A2A list-capabilities artifact contains all 45 skill IDs", async () => {
test("A2A list-capabilities artifact contains all 46 skill IDs", async () => {
const result = await executeListCapabilities(stubTask);
const content = result.artifacts[0].content;
const missing: string[] = [];