fix(tests+ci): update 42→43 skill count, fix E2E artifact path

- Unit/integration tests: update hardcoded 42→43 in 7 test files
  (agentSkillTools-mcp, agentSkills-catalog, agentSkills-generator,
  agent-skills-content, agent-skills-discovery, listCapabilities-a2a)
  to match the 43rd skill (config-codex-cli) added in the previous commit.
- Include CONFIG_SKILL_IDS in integration content test ALL_IDS so
  skills/config-codex-cli/ is no longer "unexpected".
- listCapabilities.ts: change totalSkills from literal 42 to catalog.length
  so it adapts to catalog growth automatically.
- computeCoverage assertions: include config.have in totalSkills check.
- CI: switch E2E artifact from upload-artifact path (ambiguous stripping)
  to explicit tar archive. Fixes "Could not find a production build in
  ./.build/next" — the previous approach's download path was double-nested
  (.build/next/next/...) due to upload-artifact LCA computation. tar -czf
  stores .build/next/... relative to CWD; tar -xzf restores them verbatim.
- Also exclude .build/next/cache from the tar to keep archive lean.
- feat(translator): strip client_metadata in Responses→Chat translation
  (Mistral 422 extra_forbidden fix); add regression test.
This commit is contained in:
diegosouzapw
2026-06-08 10:41:00 -03:00
parent 1012603a1b
commit 3ea416350e
11 changed files with 74 additions and 45 deletions

View File

@@ -144,15 +144,20 @@ jobs:
- run: npm ci
- run: npm run check:node-runtime
- run: npm run build
- name: Archive Next.js build for E2E shards
# Use tar so the archive preserves paths relative to CWD (.build/next/...).
# upload-artifact path-stripping is ambiguous when exclude patterns are used;
# an explicit tar avoids the double-nesting issue (.build/next/next/...).
run: |
tar -czf /tmp/e2e-build.tar.gz \
--exclude='.build/next/standalone/node_modules' \
--exclude='.build/next/cache' \
.build/next
- name: Upload Next.js build for E2E shards
uses: actions/upload-artifact@v4
with:
name: e2e-next-build
# Exclude standalone/node_modules (558MB) — each shard already runs
# npm ci, so we cp node_modules into standalone after the download.
path: |
.build/next/
!.build/next/standalone/node_modules/
path: /tmp/e2e-build.tar.gz
retention-days: 1
package-artifact:
@@ -544,9 +549,11 @@ jobs:
uses: actions/download-artifact@v4
with:
name: e2e-next-build
path: .build/next/
- name: Restore standalone node_modules from npm ci
run: cp -r node_modules .build/next/standalone/node_modules
path: /tmp/
- name: Extract Next.js build and restore standalone node_modules
run: |
tar -xzf /tmp/e2e-build.tar.gz
cp -r node_modules .build/next/standalone/node_modules
- run: npx playwright test tests/e2e/*.spec.ts --shard=${{ matrix.shard }}/9
test-integration:

View File

@@ -384,6 +384,9 @@ export function openaiResponsesToOpenAIRequest(
// Strip Responses-API-only fields that Chat Completions rejects with 400.
// safety_identifier is sent by LobeHub and has no Chat Completions equivalent (#2770).
delete result.safety_identifier;
// client_metadata is sent by Codex CLI and has no Chat Completions equivalent.
// Strict upstreams (e.g. Mistral) reject it with HTTP 422 extra_forbidden.
delete result.client_metadata;
return result;
}

View File

@@ -1,7 +1,7 @@
/**
* A2A Skill: List Capabilities
*
* Returns the full catalog of 42 OmniRoute agent skills (22 API + 20 CLI)
* Returns the full catalog of OmniRoute agent skills (22 API + 20 CLI + config)
* as a markdown table with raw SKILL.md URLs for orchestrating agents.
*/
@@ -16,7 +16,7 @@ export interface ListCapabilitiesResult {
api: { have: number; total: 22 };
cli: { have: number; total: 20 };
};
totalSkills: 42;
totalSkills: number;
generatedAt: string;
source: "agent-skills-catalog";
};
@@ -64,7 +64,7 @@ export async function executeListCapabilities(_task: A2ATask): Promise<ListCapab
api: { have: coverage.api.have, total: 22 },
cli: { have: coverage.cli.have, total: 20 },
},
totalSkills: 42,
totalSkills: catalog.length,
generatedAt: coverage.generatedAt,
source: "agent-skills-catalog",
},

View File

@@ -1,5 +1,5 @@
/**
* catalog.ts — single source of truth for the 42-entry Agent Skills catalog.
* catalog.ts — single source of truth for the 43-entry Agent Skills catalog.
*
* Consumers: REST routes (/api/agent-skills/*), MCP tools, A2A skill, Generator.
* Do NOT import this from UI components directly — use the REST API instead.
@@ -92,7 +92,7 @@ function deriveCatalog(): AgentSkill[] {
// ── Public API ───────────────────────────────────────────────────────────────
/**
* Returns the full catalog (42 entries). Cached in module scope after first call.
* Returns the full catalog (43 entries). Cached in module scope after first call.
* Safe to call multiple times — re-derives only after `refreshCatalog()`.
*/
export function getCatalog(): AgentSkill[] {

View File

@@ -2,7 +2,7 @@
* Integration tests for Agent Skills content integrity.
*
* Verifies:
* 1. All 42 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
* 1. All 43 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
* 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed).
* 3. 10 specific IDs have <!-- skill:custom-start --> ... <!-- skill:custom-end --> blocks:
* omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a,
@@ -15,10 +15,10 @@ import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
const SKILLS_DIR = path.resolve(process.cwd(), "skills");
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS, ...CONFIG_SKILL_IDS] as string[];
// IDs that must have a custom block
const CUSTOM_BLOCK_IDS = [
@@ -37,7 +37,7 @@ const CUSTOM_BLOCK_IDS = [
// ── §1: All 42 catalog IDs have skills/{id}/SKILL.md ─────────────────────────
test("all 42 catalog IDs have a skills/{id}/ directory", () => {
test("all 43 catalog IDs have a skills/{id}/ directory", () => {
const missing: string[] = [];
for (const id of ALL_IDS) {
const dirPath = path.join(SKILLS_DIR, id);
@@ -48,7 +48,7 @@ test("all 42 catalog IDs have a skills/{id}/ directory", () => {
assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`);
});
test("all 42 catalog IDs have a skills/{id}/SKILL.md file", () => {
test("all 43 catalog IDs have a skills/{id}/SKILL.md file", () => {
const missing: string[] = [];
for (const id of ALL_IDS) {
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");

View File

@@ -4,8 +4,8 @@
* Verifies:
* 1. Every ID in API_SKILL_IDS + CLI_SKILL_IDS has a skills/<id>/SKILL.md on disk.
* 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars.
* 3. MCP tool omniroute_agent_skills_list handler returns 42 entries.
* 4. A2A skill list-capabilities returns 1 artifact with 42 lines.
* 3. MCP tool omniroute_agent_skills_list handler returns 43 entries.
* 4. A2A skill list-capabilities returns 1 artifact with 43 lines.
*
* Does NOT spin up a server — tests handlers directly via imports.
*/
@@ -120,11 +120,11 @@ test("each SKILL.md body is at least 100 chars", () => {
// ── §3: MCP tool omniroute_agent_skills_list ─────────────────────────────────
test("MCP omniroute_agent_skills_list handler returns count 42", async () => {
test("MCP omniroute_agent_skills_list handler returns count 43", async () => {
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
assert.equal(result.count, 42, `Expected 42 but got ${result.count}`);
assert.equal(result.count, 43, `Expected 43 but got ${result.count}`);
assert.ok(Array.isArray(result.skills));
assert.equal(result.skills.length, 42);
assert.equal(result.skills.length, 43);
});
test("MCP omniroute_agent_skills_list result has all 42 IDs", async () => {
@@ -156,9 +156,9 @@ test("A2A list-capabilities artifact content contains 42 skill IDs as table rows
);
});
test("A2A list-capabilities metadata.totalSkills === 42", async () => {
test("A2A list-capabilities metadata.totalSkills === 43", async () => {
const result = await executeListCapabilities(stubTask);
assert.equal(result.metadata.totalSkills, 42);
assert.equal(result.metadata.totalSkills, 43);
});
test("A2A list-capabilities artifact contains all 42 skill IDs", async () => {

View File

@@ -52,11 +52,11 @@ test("each agentSkillTool has name, description, inputSchema, and handler", () =
// ─── omniroute_agent_skills_list ────────────────────────────────────────────
test("omniroute_agent_skills_list with no filters returns all 42 skills", async () => {
test("omniroute_agent_skills_list with no filters returns all 43 skills", async () => {
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
assert.equal(result.count, 42, `Expected 42 but got ${result.count}`);
assert.equal(result.count, 43, `Expected 43 but got ${result.count}`);
assert.ok(Array.isArray(result.skills));
assert.equal(result.skills.length, 42);
assert.equal(result.skills.length, 43);
});
test("omniroute_agent_skills_list({category:'api'}) returns exactly 22 entries", async () => {
@@ -168,7 +168,7 @@ test("omniroute_agent_skills_coverage({}) returns coverage shape", async () => {
assert.ok(result.api.have >= 0 && result.api.have <= 22);
assert.ok(result.cli.have >= 0 && result.cli.have <= 20);
assert.ok(typeof result.totalSkills === "number");
assert.equal(result.totalSkills, result.api.have + result.cli.have);
assert.equal(result.totalSkills, result.api.have + result.cli.have + (result.config?.have ?? 0));
assert.ok(typeof result.generatedAt === "string");
// Validate ISO datetime format
assert.ok(!isNaN(Date.parse(result.generatedAt)), "generatedAt should be valid ISO datetime");

View File

@@ -7,10 +7,10 @@ const { getCatalog, getSkillById, filterCatalog, computeCoverage, refreshCatalog
// ─── Counts ───────────────────────────────────────────────────────────────────
test("getCatalog() returns exactly 42 entries", () => {
test("getCatalog() returns exactly 43 entries", () => {
refreshCatalog();
const catalog = getCatalog();
assert.equal(catalog.length, 42, `Expected 42 but got ${catalog.length}`);
assert.equal(catalog.length, 43, `Expected 43 but got ${catalog.length}`);
});
test("API_SKILL_IDS has exactly 22 entries", () => {
@@ -167,9 +167,9 @@ test("filterCatalog({ area: 'nonexistent' }) returns empty array", () => {
assert.equal(skills.length, 0);
});
test("filterCatalog({}) returns full catalog (42 entries)", () => {
test("filterCatalog({}) returns full catalog (43 entries)", () => {
const skills = filterCatalog({});
assert.equal(skills.length, 42);
assert.equal(skills.length, 43);
});
// ─── refreshCatalog ───────────────────────────────────────────────────────────
@@ -200,7 +200,7 @@ test("computeCoverage() returns valid SkillCoverage shape", () => {
assert.ok(typeof cov.cli.have === "number");
assert.ok(cov.cli.have >= 0 && cov.cli.have <= 20);
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have);
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have + (cov.config?.have ?? 0));
// generatedAt must be a valid ISO datetime string
assert.ok(!isNaN(Date.parse(cov.generatedAt)), `generatedAt "${cov.generatedAt}" is not a valid ISO date`);
@@ -208,7 +208,7 @@ test("computeCoverage() returns valid SkillCoverage shape", () => {
test("computeCoverage() api.have + cli.have = totalSkills", () => {
const cov = computeCoverage();
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have);
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have + (cov.config?.have ?? 0));
});
// ─── Cache behaviour ─────────────────────────────────────────────────────────

View File

@@ -61,11 +61,11 @@ test("dry-run (default) returns report without writing any files", async () => {
outputDir: tmpDir,
});
// All 42 skills should appear as generated (would-write) since dir is empty
// All 43 skills should appear as generated (would-write) since dir is empty
assert.equal(
report.generated.length + report.unchanged.length,
42,
`Expected 42 total (generated+unchanged), got generated=${report.generated.length} unchanged=${report.unchanged.length}`,
43,
`Expected 43 total (generated+unchanged), got generated=${report.generated.length} unchanged=${report.unchanged.length}`,
);
assert.equal(report.errors.length, 0, `Unexpected errors: ${JSON.stringify(report.errors)}`);
@@ -81,7 +81,7 @@ test("dry-run (default) returns report without writing any files", async () => {
}
});
test("dry-run generates report with 42 total (generated+unchanged)", async () => {
test("dry-run generates report with 43 total (generated+unchanged)", async () => {
const tmpDir = mkTmpDir();
try {
refreshCatalog();
@@ -91,7 +91,7 @@ test("dry-run generates report with 42 total (generated+unchanged)", async () =>
outputDir: tmpDir,
});
const total = report.generated.length + report.unchanged.length;
assert.equal(total, 42);
assert.equal(total, 43);
} finally {
rmTmpDir(tmpDir);
}
@@ -134,7 +134,7 @@ test("apply mode writes SKILL.md with valid frontmatter for omni-providers", asy
}
});
test("apply mode writes all 42 SKILL.md files when no onlyIds filter", async () => {
test("apply mode writes all 43 SKILL.md files when no onlyIds filter", async () => {
const tmpDir = mkTmpDir();
try {
refreshCatalog();
@@ -145,7 +145,7 @@ test("apply mode writes all 42 SKILL.md files when no onlyIds filter", async ()
});
assert.equal(report.errors.length, 0, `Errors: ${JSON.stringify(report.errors)}`);
assert.equal(report.generated.length, 42);
assert.equal(report.generated.length, 43);
// Verify all dirs exist
const catalog = getCatalog();

View File

@@ -31,7 +31,7 @@ test("executeListCapabilities returns shape matching §3.7 contract", async () =
const { metadata } = result;
assert.ok(metadata, "metadata exists");
assert.equal(metadata.source, "agent-skills-catalog", "metadata.source matches");
assert.equal(metadata.totalSkills, 42, "metadata.totalSkills === 42");
assert.equal(metadata.totalSkills, 43, "metadata.totalSkills === 43");
assert.ok(metadata.coverage, "metadata.coverage exists");
assert.ok(metadata.coverage.api, "metadata.coverage.api exists");
assert.ok(metadata.coverage.cli, "metadata.coverage.cli exists");
@@ -39,12 +39,12 @@ test("executeListCapabilities returns shape matching §3.7 contract", async () =
assert.equal(metadata.coverage.cli.total, 20, "cli.total === 20");
});
test("executeListCapabilities markdown table contains all 42 skill IDs", async () => {
test("executeListCapabilities markdown table contains all 42 API+CLI skill IDs", async () => {
const result = await executeListCapabilities(stubTask);
const content = result.artifacts[0].content;
const allIds = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
assert.equal(allIds.length, 42, "catalog declares 42 skill IDs");
assert.equal(allIds.length, 42, "API+CLI catalog declares 42 skill IDs");
for (const id of allIds) {
assert.ok(content.includes(id), `Markdown table missing skill ID: ${id}`);

View File

@@ -233,6 +233,25 @@ test("Responses -> Chat strips safety_identifier (LobeHub #2770)", () => {
assert.ok(Array.isArray(result.messages), "translation must still produce messages");
});
test("Responses -> Chat strips client_metadata (Mistral 422 fix)", () => {
// Codex CLI always sends client_metadata in Responses API requests. Mistral (and other
// strict upstreams) reject it with HTTP 422 extra_forbidden. The translator must strip
// the field in the Responses-API cleanup block so it never reaches the upstream.
const result = openaiResponsesToOpenAIRequest(
"mistral-large-latest",
{
input: [{ role: "user", content: [{ type: "input_text", text: "oi" }] }],
client_metadata: { session_id: "abc123", foo: "bar" },
},
false,
null
) as Record<string, unknown>;
assert.equal(result.client_metadata, undefined, "client_metadata must be stripped before forwarding to Chat Completions");
assert.ok(Array.isArray(result.messages), "translation must still produce messages");
assert.equal((result.messages as unknown[]).length, 1, "user message must be preserved");
});
test("Chat -> Responses converts messages, tool calls, tool outputs, tools and pass-through params", () => {
const result = openaiToOpenAIResponsesRequest(
"gpt-4o",