mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(tests+ci): update 42→43 skill count, fix E2E artifact path
- Unit/integration tests: update hardcoded 42→43 in 7 test files (agentSkillTools-mcp, agentSkills-catalog, agentSkills-generator, agent-skills-content, agent-skills-discovery, listCapabilities-a2a) to match the 43rd skill (config-codex-cli) added in the previous commit. - Include CONFIG_SKILL_IDS in integration content test ALL_IDS so skills/config-codex-cli/ is no longer "unexpected". - listCapabilities.ts: change totalSkills from literal 42 to catalog.length so it adapts to catalog growth automatically. - computeCoverage assertions: include config.have in totalSkills check. - CI: switch E2E artifact from upload-artifact path (ambiguous stripping) to explicit tar archive. Fixes "Could not find a production build in ./.build/next" — the previous approach's download path was double-nested (.build/next/next/...) due to upload-artifact LCA computation. tar -czf stores .build/next/... relative to CWD; tar -xzf restores them verbatim. - Also exclude .build/next/cache from the tar to keep archive lean. - feat(translator): strip client_metadata in Responses→Chat translation (Mistral 422 extra_forbidden fix); add regression test.
This commit is contained in:
23
.github/workflows/ci.yml
vendored
23
.github/workflows/ci.yml
vendored
@@ -144,15 +144,20 @@ jobs:
|
||||
- run: npm ci
|
||||
- run: npm run check:node-runtime
|
||||
- run: npm run build
|
||||
- name: Archive Next.js build for E2E shards
|
||||
# Use tar so the archive preserves paths relative to CWD (.build/next/...).
|
||||
# upload-artifact path-stripping is ambiguous when exclude patterns are used;
|
||||
# an explicit tar avoids the double-nesting issue (.build/next/next/...).
|
||||
run: |
|
||||
tar -czf /tmp/e2e-build.tar.gz \
|
||||
--exclude='.build/next/standalone/node_modules' \
|
||||
--exclude='.build/next/cache' \
|
||||
.build/next
|
||||
- name: Upload Next.js build for E2E shards
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-next-build
|
||||
# Exclude standalone/node_modules (558MB) — each shard already runs
|
||||
# npm ci, so we cp node_modules into standalone after the download.
|
||||
path: |
|
||||
.build/next/
|
||||
!.build/next/standalone/node_modules/
|
||||
path: /tmp/e2e-build.tar.gz
|
||||
retention-days: 1
|
||||
|
||||
package-artifact:
|
||||
@@ -544,9 +549,11 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: e2e-next-build
|
||||
path: .build/next/
|
||||
- name: Restore standalone node_modules from npm ci
|
||||
run: cp -r node_modules .build/next/standalone/node_modules
|
||||
path: /tmp/
|
||||
- name: Extract Next.js build and restore standalone node_modules
|
||||
run: |
|
||||
tar -xzf /tmp/e2e-build.tar.gz
|
||||
cp -r node_modules .build/next/standalone/node_modules
|
||||
- run: npx playwright test tests/e2e/*.spec.ts --shard=${{ matrix.shard }}/9
|
||||
|
||||
test-integration:
|
||||
|
||||
@@ -384,6 +384,9 @@ export function openaiResponsesToOpenAIRequest(
|
||||
// Strip Responses-API-only fields that Chat Completions rejects with 400.
|
||||
// safety_identifier is sent by LobeHub and has no Chat Completions equivalent (#2770).
|
||||
delete result.safety_identifier;
|
||||
// client_metadata is sent by Codex CLI and has no Chat Completions equivalent.
|
||||
// Strict upstreams (e.g. Mistral) reject it with HTTP 422 extra_forbidden.
|
||||
delete result.client_metadata;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* A2A Skill: List Capabilities
|
||||
*
|
||||
* Returns the full catalog of 42 OmniRoute agent skills (22 API + 20 CLI)
|
||||
* Returns the full catalog of OmniRoute agent skills (22 API + 20 CLI + config)
|
||||
* as a markdown table with raw SKILL.md URLs for orchestrating agents.
|
||||
*/
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface ListCapabilitiesResult {
|
||||
api: { have: number; total: 22 };
|
||||
cli: { have: number; total: 20 };
|
||||
};
|
||||
totalSkills: 42;
|
||||
totalSkills: number;
|
||||
generatedAt: string;
|
||||
source: "agent-skills-catalog";
|
||||
};
|
||||
@@ -64,7 +64,7 @@ export async function executeListCapabilities(_task: A2ATask): Promise<ListCapab
|
||||
api: { have: coverage.api.have, total: 22 },
|
||||
cli: { have: coverage.cli.have, total: 20 },
|
||||
},
|
||||
totalSkills: 42,
|
||||
totalSkills: catalog.length,
|
||||
generatedAt: coverage.generatedAt,
|
||||
source: "agent-skills-catalog",
|
||||
},
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* catalog.ts — single source of truth for the 42-entry Agent Skills catalog.
|
||||
* catalog.ts — single source of truth for the 43-entry Agent Skills catalog.
|
||||
*
|
||||
* Consumers: REST routes (/api/agent-skills/*), MCP tools, A2A skill, Generator.
|
||||
* Do NOT import this from UI components directly — use the REST API instead.
|
||||
@@ -92,7 +92,7 @@ function deriveCatalog(): AgentSkill[] {
|
||||
// ── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Returns the full catalog (42 entries). Cached in module scope after first call.
|
||||
* Returns the full catalog (43 entries). Cached in module scope after first call.
|
||||
* Safe to call multiple times — re-derives only after `refreshCatalog()`.
|
||||
*/
|
||||
export function getCatalog(): AgentSkill[] {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* Integration tests for Agent Skills content integrity.
|
||||
*
|
||||
* Verifies:
|
||||
* 1. All 42 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
|
||||
* 1. All 43 skill IDs from catalog have skills/{id}/ folder with SKILL.md.
|
||||
* 2. Zero omniroute-* folders remain (post-prune: old omniroute-* skill dirs were removed).
|
||||
* 3. 10 specific IDs have <!-- skill:custom-start --> ... <!-- skill:custom-end --> blocks:
|
||||
* omni-mcp, omni-compression, cli-providers, cli-eval, omni-agents-a2a,
|
||||
@@ -15,10 +15,10 @@ import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const { API_SKILL_IDS, CLI_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
|
||||
const { API_SKILL_IDS, CLI_SKILL_IDS, CONFIG_SKILL_IDS } = await import("../../src/lib/agentSkills/catalog.ts");
|
||||
|
||||
const SKILLS_DIR = path.resolve(process.cwd(), "skills");
|
||||
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
|
||||
const ALL_IDS = [...API_SKILL_IDS, ...CLI_SKILL_IDS, ...CONFIG_SKILL_IDS] as string[];
|
||||
|
||||
// IDs that must have a custom block
|
||||
const CUSTOM_BLOCK_IDS = [
|
||||
@@ -37,7 +37,7 @@ const CUSTOM_BLOCK_IDS = [
|
||||
|
||||
// ── §1: All 42 catalog IDs have skills/{id}/SKILL.md ─────────────────────────
|
||||
|
||||
test("all 42 catalog IDs have a skills/{id}/ directory", () => {
|
||||
test("all 43 catalog IDs have a skills/{id}/ directory", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const dirPath = path.join(SKILLS_DIR, id);
|
||||
@@ -48,7 +48,7 @@ test("all 42 catalog IDs have a skills/{id}/ directory", () => {
|
||||
assert.deepEqual(missing, [], `Missing skill directories: ${missing.join(", ")}`);
|
||||
});
|
||||
|
||||
test("all 42 catalog IDs have a skills/{id}/SKILL.md file", () => {
|
||||
test("all 43 catalog IDs have a skills/{id}/SKILL.md file", () => {
|
||||
const missing: string[] = [];
|
||||
for (const id of ALL_IDS) {
|
||||
const skillPath = path.join(SKILLS_DIR, id, "SKILL.md");
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
* Verifies:
|
||||
* 1. Every ID in API_SKILL_IDS + CLI_SKILL_IDS has a skills/<id>/SKILL.md on disk.
|
||||
* 2. Each SKILL.md has valid frontmatter (name + description) and body ≥ 100 chars.
|
||||
* 3. MCP tool omniroute_agent_skills_list handler returns 42 entries.
|
||||
* 4. A2A skill list-capabilities returns 1 artifact with 42 lines.
|
||||
* 3. MCP tool omniroute_agent_skills_list handler returns 43 entries.
|
||||
* 4. A2A skill list-capabilities returns 1 artifact with 43 lines.
|
||||
*
|
||||
* Does NOT spin up a server — tests handlers directly via imports.
|
||||
*/
|
||||
@@ -120,11 +120,11 @@ test("each SKILL.md body is at least 100 chars", () => {
|
||||
|
||||
// ── §3: MCP tool omniroute_agent_skills_list ─────────────────────────────────
|
||||
|
||||
test("MCP omniroute_agent_skills_list handler returns count 42", async () => {
|
||||
test("MCP omniroute_agent_skills_list handler returns count 43", async () => {
|
||||
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
|
||||
assert.equal(result.count, 42, `Expected 42 but got ${result.count}`);
|
||||
assert.equal(result.count, 43, `Expected 43 but got ${result.count}`);
|
||||
assert.ok(Array.isArray(result.skills));
|
||||
assert.equal(result.skills.length, 42);
|
||||
assert.equal(result.skills.length, 43);
|
||||
});
|
||||
|
||||
test("MCP omniroute_agent_skills_list result has all 42 IDs", async () => {
|
||||
@@ -156,9 +156,9 @@ test("A2A list-capabilities artifact content contains 42 skill IDs as table rows
|
||||
);
|
||||
});
|
||||
|
||||
test("A2A list-capabilities metadata.totalSkills === 42", async () => {
|
||||
test("A2A list-capabilities metadata.totalSkills === 43", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
assert.equal(result.metadata.totalSkills, 42);
|
||||
assert.equal(result.metadata.totalSkills, 43);
|
||||
});
|
||||
|
||||
test("A2A list-capabilities artifact contains all 42 skill IDs", async () => {
|
||||
|
||||
@@ -52,11 +52,11 @@ test("each agentSkillTool has name, description, inputSchema, and handler", () =
|
||||
|
||||
// ─── omniroute_agent_skills_list ────────────────────────────────────────────
|
||||
|
||||
test("omniroute_agent_skills_list with no filters returns all 42 skills", async () => {
|
||||
test("omniroute_agent_skills_list with no filters returns all 43 skills", async () => {
|
||||
const result = await agentSkillTools.omniroute_agent_skills_list.handler({});
|
||||
assert.equal(result.count, 42, `Expected 42 but got ${result.count}`);
|
||||
assert.equal(result.count, 43, `Expected 43 but got ${result.count}`);
|
||||
assert.ok(Array.isArray(result.skills));
|
||||
assert.equal(result.skills.length, 42);
|
||||
assert.equal(result.skills.length, 43);
|
||||
});
|
||||
|
||||
test("omniroute_agent_skills_list({category:'api'}) returns exactly 22 entries", async () => {
|
||||
@@ -168,7 +168,7 @@ test("omniroute_agent_skills_coverage({}) returns coverage shape", async () => {
|
||||
assert.ok(result.api.have >= 0 && result.api.have <= 22);
|
||||
assert.ok(result.cli.have >= 0 && result.cli.have <= 20);
|
||||
assert.ok(typeof result.totalSkills === "number");
|
||||
assert.equal(result.totalSkills, result.api.have + result.cli.have);
|
||||
assert.equal(result.totalSkills, result.api.have + result.cli.have + (result.config?.have ?? 0));
|
||||
assert.ok(typeof result.generatedAt === "string");
|
||||
// Validate ISO datetime format
|
||||
assert.ok(!isNaN(Date.parse(result.generatedAt)), "generatedAt should be valid ISO datetime");
|
||||
|
||||
@@ -7,10 +7,10 @@ const { getCatalog, getSkillById, filterCatalog, computeCoverage, refreshCatalog
|
||||
|
||||
// ─── Counts ───────────────────────────────────────────────────────────────────
|
||||
|
||||
test("getCatalog() returns exactly 42 entries", () => {
|
||||
test("getCatalog() returns exactly 43 entries", () => {
|
||||
refreshCatalog();
|
||||
const catalog = getCatalog();
|
||||
assert.equal(catalog.length, 42, `Expected 42 but got ${catalog.length}`);
|
||||
assert.equal(catalog.length, 43, `Expected 43 but got ${catalog.length}`);
|
||||
});
|
||||
|
||||
test("API_SKILL_IDS has exactly 22 entries", () => {
|
||||
@@ -167,9 +167,9 @@ test("filterCatalog({ area: 'nonexistent' }) returns empty array", () => {
|
||||
assert.equal(skills.length, 0);
|
||||
});
|
||||
|
||||
test("filterCatalog({}) returns full catalog (42 entries)", () => {
|
||||
test("filterCatalog({}) returns full catalog (43 entries)", () => {
|
||||
const skills = filterCatalog({});
|
||||
assert.equal(skills.length, 42);
|
||||
assert.equal(skills.length, 43);
|
||||
});
|
||||
|
||||
// ─── refreshCatalog ───────────────────────────────────────────────────────────
|
||||
@@ -200,7 +200,7 @@ test("computeCoverage() returns valid SkillCoverage shape", () => {
|
||||
assert.ok(typeof cov.cli.have === "number");
|
||||
assert.ok(cov.cli.have >= 0 && cov.cli.have <= 20);
|
||||
|
||||
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have);
|
||||
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have + (cov.config?.have ?? 0));
|
||||
|
||||
// generatedAt must be a valid ISO datetime string
|
||||
assert.ok(!isNaN(Date.parse(cov.generatedAt)), `generatedAt "${cov.generatedAt}" is not a valid ISO date`);
|
||||
@@ -208,7 +208,7 @@ test("computeCoverage() returns valid SkillCoverage shape", () => {
|
||||
|
||||
test("computeCoverage() api.have + cli.have = totalSkills", () => {
|
||||
const cov = computeCoverage();
|
||||
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have);
|
||||
assert.equal(cov.totalSkills, cov.api.have + cov.cli.have + (cov.config?.have ?? 0));
|
||||
});
|
||||
|
||||
// ─── Cache behaviour ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -61,11 +61,11 @@ test("dry-run (default) returns report without writing any files", async () => {
|
||||
outputDir: tmpDir,
|
||||
});
|
||||
|
||||
// All 42 skills should appear as generated (would-write) since dir is empty
|
||||
// All 43 skills should appear as generated (would-write) since dir is empty
|
||||
assert.equal(
|
||||
report.generated.length + report.unchanged.length,
|
||||
42,
|
||||
`Expected 42 total (generated+unchanged), got generated=${report.generated.length} unchanged=${report.unchanged.length}`,
|
||||
43,
|
||||
`Expected 43 total (generated+unchanged), got generated=${report.generated.length} unchanged=${report.unchanged.length}`,
|
||||
);
|
||||
assert.equal(report.errors.length, 0, `Unexpected errors: ${JSON.stringify(report.errors)}`);
|
||||
|
||||
@@ -81,7 +81,7 @@ test("dry-run (default) returns report without writing any files", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("dry-run generates report with 42 total (generated+unchanged)", async () => {
|
||||
test("dry-run generates report with 43 total (generated+unchanged)", async () => {
|
||||
const tmpDir = mkTmpDir();
|
||||
try {
|
||||
refreshCatalog();
|
||||
@@ -91,7 +91,7 @@ test("dry-run generates report with 42 total (generated+unchanged)", async () =>
|
||||
outputDir: tmpDir,
|
||||
});
|
||||
const total = report.generated.length + report.unchanged.length;
|
||||
assert.equal(total, 42);
|
||||
assert.equal(total, 43);
|
||||
} finally {
|
||||
rmTmpDir(tmpDir);
|
||||
}
|
||||
@@ -134,7 +134,7 @@ test("apply mode writes SKILL.md with valid frontmatter for omni-providers", asy
|
||||
}
|
||||
});
|
||||
|
||||
test("apply mode writes all 42 SKILL.md files when no onlyIds filter", async () => {
|
||||
test("apply mode writes all 43 SKILL.md files when no onlyIds filter", async () => {
|
||||
const tmpDir = mkTmpDir();
|
||||
try {
|
||||
refreshCatalog();
|
||||
@@ -145,7 +145,7 @@ test("apply mode writes all 42 SKILL.md files when no onlyIds filter", async ()
|
||||
});
|
||||
|
||||
assert.equal(report.errors.length, 0, `Errors: ${JSON.stringify(report.errors)}`);
|
||||
assert.equal(report.generated.length, 42);
|
||||
assert.equal(report.generated.length, 43);
|
||||
|
||||
// Verify all dirs exist
|
||||
const catalog = getCatalog();
|
||||
|
||||
@@ -31,7 +31,7 @@ test("executeListCapabilities returns shape matching §3.7 contract", async () =
|
||||
const { metadata } = result;
|
||||
assert.ok(metadata, "metadata exists");
|
||||
assert.equal(metadata.source, "agent-skills-catalog", "metadata.source matches");
|
||||
assert.equal(metadata.totalSkills, 42, "metadata.totalSkills === 42");
|
||||
assert.equal(metadata.totalSkills, 43, "metadata.totalSkills === 43");
|
||||
assert.ok(metadata.coverage, "metadata.coverage exists");
|
||||
assert.ok(metadata.coverage.api, "metadata.coverage.api exists");
|
||||
assert.ok(metadata.coverage.cli, "metadata.coverage.cli exists");
|
||||
@@ -39,12 +39,12 @@ test("executeListCapabilities returns shape matching §3.7 contract", async () =
|
||||
assert.equal(metadata.coverage.cli.total, 20, "cli.total === 20");
|
||||
});
|
||||
|
||||
test("executeListCapabilities markdown table contains all 42 skill IDs", async () => {
|
||||
test("executeListCapabilities markdown table contains all 42 API+CLI skill IDs", async () => {
|
||||
const result = await executeListCapabilities(stubTask);
|
||||
const content = result.artifacts[0].content;
|
||||
|
||||
const allIds = [...API_SKILL_IDS, ...CLI_SKILL_IDS] as string[];
|
||||
assert.equal(allIds.length, 42, "catalog declares 42 skill IDs");
|
||||
assert.equal(allIds.length, 42, "API+CLI catalog declares 42 skill IDs");
|
||||
|
||||
for (const id of allIds) {
|
||||
assert.ok(content.includes(id), `Markdown table missing skill ID: ${id}`);
|
||||
|
||||
@@ -233,6 +233,25 @@ test("Responses -> Chat strips safety_identifier (LobeHub #2770)", () => {
|
||||
assert.ok(Array.isArray(result.messages), "translation must still produce messages");
|
||||
});
|
||||
|
||||
test("Responses -> Chat strips client_metadata (Mistral 422 fix)", () => {
|
||||
// Codex CLI always sends client_metadata in Responses API requests. Mistral (and other
|
||||
// strict upstreams) reject it with HTTP 422 extra_forbidden. The translator must strip
|
||||
// the field in the Responses-API cleanup block so it never reaches the upstream.
|
||||
const result = openaiResponsesToOpenAIRequest(
|
||||
"mistral-large-latest",
|
||||
{
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: "oi" }] }],
|
||||
client_metadata: { session_id: "abc123", foo: "bar" },
|
||||
},
|
||||
false,
|
||||
null
|
||||
) as Record<string, unknown>;
|
||||
|
||||
assert.equal(result.client_metadata, undefined, "client_metadata must be stripped before forwarding to Chat Completions");
|
||||
assert.ok(Array.isArray(result.messages), "translation must still produce messages");
|
||||
assert.equal((result.messages as unknown[]).length, 1, "user message must be preserved");
|
||||
});
|
||||
|
||||
test("Chat -> Responses converts messages, tool calls, tool outputs, tools and pass-through params", () => {
|
||||
const result = openaiToOpenAIResponsesRequest(
|
||||
"gpt-4o",
|
||||
|
||||
Reference in New Issue
Block a user