diff --git a/bin/cli/provider-catalog.mjs b/bin/cli/provider-catalog.mjs index 873f933d18..27cfab1a9c 100644 --- a/bin/cli/provider-catalog.mjs +++ b/bin/cli/provider-catalog.mjs @@ -1,11 +1,9 @@ -import { existsSync, readFileSync } from "node:fs"; -import { createRequire } from "node:module"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const CLI_DIR = dirname(fileURLToPath(import.meta.url)); const DEFAULT_ROOT_DIR = join(CLI_DIR, "..", ".."); -const require = createRequire(import.meta.url); export const COMMON_PROVIDERS = [ { id: "openai", name: "OpenAI" }, @@ -17,94 +15,201 @@ export const COMMON_PROVIDERS = [ ]; function normalizeCatalogCategory(exportName) { - const raw = exportName - .replace(/_PROVIDERS$/, "") - .toLowerCase() - .replaceAll("_", "-"); + const raw = exportName.split("_PROVIDERS")[0].toLowerCase().replaceAll("_", "-"); if (raw === "apikey") return "api-key"; return raw; } -function loadTypeScript() { - try { - return require("typescript"); - } catch { - return null; - } -} +/** + * Advance past a string literal, template literal, or comment starting at `i`. + * Returns the index just after it, or -1 when `i` does not start one. Keeping + * the scanner string/comment aware is what lets it walk braces safely — provider + * notes routinely contain `{`, `}` and apostrophes. + */ +function skipNonCode(source, i) { + const c = source[i]; -function getPropertyName(ts, name) { - if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) { - return name.text; - } - return null; -} - -function getObjectProperty(ts, objectLiteral, propertyName) { - return objectLiteral.properties.find( - (property) => - ts.isPropertyAssignment(property) && getPropertyName(ts, property.name) === propertyName - ); -} - -function getStringProperty(ts, objectLiteral, propertyName) { - const property = getObjectProperty(ts, objectLiteral, propertyName); - const initializer = property?.initializer; - if (!initializer) return null; - if (ts.isStringLiteral(initializer) || ts.isNoSubstitutionTemplateLiteral(initializer)) { - return initializer.text; - } - return null; -} - -function getBooleanProperty(ts, objectLiteral, propertyName) { - const property = getObjectProperty(ts, objectLiteral, propertyName); - const initializer = property?.initializer; - return initializer?.kind === ts.SyntaxKind.TrueKeyword; -} - -function extractProviderBlocks(source, filePath) { - const ts = loadTypeScript(); - if (!ts) return []; - - const providers = []; - const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true); - - sourceFile.forEachChild((node) => { - if (!ts.isVariableStatement(node)) return; - - for (const declaration of node.declarationList.declarations) { - if (!ts.isIdentifier(declaration.name)) continue; - const exportName = declaration.name.text; - if (!exportName.endsWith("_PROVIDERS")) continue; - if (!declaration.initializer || !ts.isObjectLiteralExpression(declaration.initializer)) { + if (c === '"' || c === "'" || c === "`") { + for (let j = i + 1; j < source.length; j++) { + if (source[j] === "\\") { + j++; continue; } - - const category = normalizeCatalogCategory(exportName); - for (const property of declaration.initializer.properties) { - if (!ts.isPropertyAssignment(property)) continue; - if (!ts.isObjectLiteralExpression(property.initializer)) continue; - - const key = getPropertyName(ts, property.name); - if (!key) continue; - - const id = getStringProperty(ts, property.initializer, "id") || key; - const name = getStringProperty(ts, property.initializer, "name") || id; - - providers.push({ - id, - name, - category, - alias: getStringProperty(ts, property.initializer, "alias"), - website: getStringProperty(ts, property.initializer, "website"), - deprecated: getBooleanProperty(ts, property.initializer, "deprecated"), - hasFree: getBooleanProperty(ts, property.initializer, "hasFree"), - passthroughModels: getBooleanProperty(ts, property.initializer, "passthroughModels"), - }); - } + if (source[j] === c) return j + 1; } - }); + return source.length; + } + + if (c === "/" && source[i + 1] === "/") { + const nl = source.indexOf("\n", i); + return nl === -1 ? source.length : nl; + } + + if (c === "/" && source[i + 1] === "*") { + const close = source.indexOf("*/", i + 2); + return close === -1 ? source.length : close + 2; + } + + return -1; +} + +/** Index of the `}` matching the `{` at `openIdx`, or -1. */ +function findMatchingBrace(source, openIdx) { + let depth = 0; + for (let i = openIdx; i < source.length; i++) { + const skipped = skipNonCode(source, i); + if (skipped !== -1) { + i = skipped - 1; + continue; + } + if (source[i] === "{") depth++; + else if (source[i] === "}") { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + +const MEMBER_KEY = /(?:([A-Za-z_$][\w$]*)|"([^"]*)"|'([^']*)')\s*:/y; + +/** + * Parse the direct members of the object literal whose `{` is at `openIdx`. + * Returns `[{ key, value }]` with `value` as the raw source slice. + */ +function parseObjectMembers(source, openIdx) { + // An unbalanced literal (a missing `},` in a large data file — see #10093) + // should not blank the whole catalog: scan to end-of-source so the entries + // before the damage are still recovered. + const matching = findMatchingBrace(source, openIdx); + const close = matching === -1 ? source.length : matching; + + const members = []; + let i = openIdx + 1; + + while (i < close) { + if (/[\s,;]/.test(source[i])) { + i++; + continue; + } + + // The key match MUST be attempted before skipNonCode: quoted keys such as + // `"duckduckgo-web":` start with a quote, and skipping them as string + // literals both loses the entry and desynchronizes the walk, which then + // reports nested keys (`notice`, …) as top-level providers. + MEMBER_KEY.lastIndex = i; + const match = MEMBER_KEY.exec(source); + if (!match) { + const skipped = skipNonCode(source, i); + i = skipped !== -1 ? skipped : i + 1; + continue; + } + + const key = match[1] ?? match[2] ?? match[3]; + let valueStart = MEMBER_KEY.lastIndex; + while (valueStart < close && /\s/.test(source[valueStart])) valueStart++; + + let valueEnd; + if (source[valueStart] === "{" || source[valueStart] === "[") { + const openChar = source[valueStart]; + const closeChar = openChar === "{" ? "}" : "]"; + let depth = 0; + let j = valueStart; + for (; j < close; j++) { + const s2 = skipNonCode(source, j); + if (s2 !== -1) { + j = s2 - 1; + continue; + } + if (source[j] === openChar) depth++; + else if (source[j] === closeChar) { + depth--; + if (depth === 0) break; + } + } + valueEnd = j + 1; + } else { + let j = valueStart; + for (; j < close; j++) { + const s2 = skipNonCode(source, j); + if (s2 !== -1) { + j = s2 - 1; + continue; + } + if (source[j] === ",") break; + } + valueEnd = j; + } + + members.push({ key, value: source.slice(valueStart, valueEnd), valueStart }); + // Guarantee forward progress even on malformed input. + i = valueEnd > i ? valueEnd : i + 1; + } + + return members; +} + +/** First string literal in a raw value (handles `"a" + "b"` continuations). */ +function readString(raw) { + if (raw == null) return null; + const match = raw.match(/"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'/); + if (!match) return null; + return (match[1] ?? match[2]).replace(/\\(.)/g, "$1"); +} + +function readBoolean(raw) { + return String(raw).trim() === "true"; +} + +const PROVIDER_EXPORT = + /(?:export\s+)?const\s+([A-Z0-9_]*_PROVIDERS[A-Z0-9_]*)\s*(?::[^=]+)?=\s*\{/g; + +/** + * Extract provider entries from a catalog source file. + * + * Deliberately dependency-free: `typescript` is a devDependency, so requiring it + * at runtime made this silently return [] on every published install (#10080). + * These files are pure data literals, so a string/comment-aware brace walk is + * both sufficient and stable. + */ +export function extractProviderBlocks(source) { + const providers = []; + PROVIDER_EXPORT.lastIndex = 0; + + let exportMatch; + while ((exportMatch = PROVIDER_EXPORT.exec(source)) !== null) { + const exportName = exportMatch[1]; + const openIdx = source.indexOf("{", exportMatch.index + exportMatch[0].length - 1); + if (openIdx === -1) continue; + + const category = normalizeCatalogCategory(exportName); + + for (const entry of parseObjectMembers(source, openIdx)) { + if (!entry.value.startsWith("{")) continue; // spread / non-object member + const fields = new Map( + parseObjectMembers(source, entry.valueStart).map((f) => [f.key, f.value]) + ); + + const id = readString(fields.get("id")) || entry.key; + providers.push({ + id, + name: readString(fields.get("name")) || id, + category, + alias: readString(fields.get("alias")), + website: readString(fields.get("website")), + deprecated: readBoolean(fields.get("deprecated")), + hasFree: readBoolean(fields.get("hasFree")), + passthroughModels: readBoolean(fields.get("passthroughModels")), + }); + } + + // An unbalanced literal (see #10093) yields -1 here. Resetting lastIndex to + // 0 would restart the scan from the top forever, so stop instead — the + // entries recovered above are still returned. + const closeIdx = findMatchingBrace(source, openIdx); + if (closeIdx === -1) break; + PROVIDER_EXPORT.lastIndex = closeIdx + 1; + } return providers; } @@ -126,9 +231,31 @@ function resolveProviderCatalogPath(rootDir, options = {}) { if (configuredPath) { return isAbsolute(configuredPath) ? configuredPath : resolve(rootDir, configuredPath); } + + // The catalog used to be one god-file at constants/providers.ts. It was + // decomposed into constants/providers/**, leaving the barrel with nothing but + // re-exports and an empty `FREE_PROVIDERS = {}` — so parsing it alone yielded + // zero providers and the CLI silently fell back to COMMON_PROVIDERS (#10080). + // Prefer the directory; keep the legacy file for older trees. + const catalogDir = join(rootDir, "src", "shared", "constants", "providers"); + if (existsSync(catalogDir)) return catalogDir; return join(rootDir, "src", "shared", "constants", "providers.ts"); } +/** Every .ts catalog file under `dir`, one level of subdirectories deep. */ +function collectCatalogFiles(dir) { + const files = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...collectCatalogFiles(full)); + } else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".d.ts")) { + files.push(full); + } + } + return files.sort(); +} + export function loadAvailableProviders(options = {}) { const rootDir = typeof options === "string" ? options : options.rootDir || DEFAULT_ROOT_DIR; const providersPath = resolveProviderCatalogPath(rootDir, options); @@ -138,8 +265,10 @@ export function loadAvailableProviders(options = {}) { } try { - const source = readFileSync(providersPath, "utf-8"); - const providers = extractProviderBlocks(source, providersPath); + const sources = statSync(providersPath).isDirectory() + ? collectCatalogFiles(providersPath) + : [providersPath]; + const providers = sources.flatMap((file) => extractProviderBlocks(readFileSync(file, "utf-8"))); if (providers.length === 0) return fallbackAvailableProviders(); const seen = new Set(); diff --git a/tests/unit/cli-provider-catalog-full-10080.test.ts b/tests/unit/cli-provider-catalog-full-10080.test.ts new file mode 100644 index 0000000000..f7a907c690 --- /dev/null +++ b/tests/unit/cli-provider-catalog-full-10080.test.ts @@ -0,0 +1,152 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// #10080 — `loadAvailableProviders()` always returned the 6-entry +// COMMON_PROVIDERS fallback, so `omniroute keys add ` rejected ~290 of +// the ~296 catalog providers with "Unknown provider". Two independent causes: +// 1. the parser required `typescript`, a devDependency absent from published +// installs, and swallowed the failure; +// 2. it read constants/providers.ts, which after the god-file decomposition +// holds only re-exports plus an empty `FREE_PROVIDERS = {}`. + +const ROOT_DIR = path.resolve("."); +const { extractProviderBlocks, loadAvailableProviders, COMMON_PROVIDERS } = + await import("../../bin/cli/provider-catalog.mjs"); + +test("extractProviderBlocks handles the shapes the real catalog files use", () => { + const source = ` + // A comment with a stray { brace } that must not confuse the walk. + export const NOAUTH_PROVIDERS = { + opencode: { + id: "opencode", + alias: "oc", + name: "OpenCode Free", + noAuth: true, + hasFree: true, + /* block comment with } */ + notice: { + text: "Nested object — its keys must not surface as providers.", + }, + }, + "duckduckgo-web": { + id: "duckduckgo-web", + name: "DuckDuckGo AI Chat", + hasFree: true, + }, + legacy: { + id: "legacy", + name: "Legacy" + " Provider", + website: "https://example.com/a{b}c", + deprecated: true, + passthroughModels: true, + }, + }; + `; + + const providers = extractProviderBlocks(source); + + assert.deepEqual( + providers.map((p) => p.id), + ["opencode", "duckduckgo-web", "legacy"], + "quoted keys must parse, nested object keys must not leak" + ); + assert.equal(providers[0].alias, "oc"); + assert.equal(providers[0].hasFree, true); + assert.equal(providers[0].category, "noauth"); + // A quoted key with no explicit alias/website yields nulls, not undefined. + assert.equal(providers[1].alias, null); + assert.equal(providers[1].website, null); + // Concatenated strings keep the first literal; braces inside strings survive. + assert.equal(providers[2].name, "Legacy"); + assert.equal(providers[2].website, "https://example.com/a{b}c"); + assert.equal(providers[2].deprecated, true); + assert.equal(providers[2].passthroughModels, true); +}); + +test("category comes from the prefix before _PROVIDERS, including suffixed exports", () => { + const mk = (name: string) => `export const ${name} = { a: { id: "a", name: "A" } };`; + + assert.equal(extractProviderBlocks(mk("NOAUTH_PROVIDERS"))[0].category, "noauth"); + assert.equal(extractProviderBlocks(mk("WEB_COOKIE_PROVIDERS"))[0].category, "web-cookie"); + assert.equal(extractProviderBlocks(mk("APIKEY_PROVIDERS"))[0].category, "api-key"); + // The decomposition introduced suffixed family exports; these are still api-key. + assert.equal(extractProviderBlocks(mk("APIKEY_PROVIDERS_GATEWAYS"))[0].category, "api-key"); + assert.equal(extractProviderBlocks(mk("APIKEY_PROVIDERS_REGIONAL"))[0].category, "api-key"); +}); + +test("an unbalanced literal terminates and keeps what was parsed", () => { + // Regression for an infinite loop: a missing `},` (see #10093) left the outer + // scan resetting its regex lastIndex to 0 forever. + const broken = ` + export const APIKEY_PROVIDERS_GATEWAYS = { + good: { id: "good", name: "Good" }, + broken: { + id: "broken", + name: "Broken", + other: { + id: "other", + name: "Other", + }, + `; + + const providers = extractProviderBlocks(broken); + assert.ok(providers.length >= 1, "entries before the damage are still recovered"); + assert.equal(providers[0].id, "good"); +}); + +test("loadAvailableProviders reads the decomposed catalog, not the 6-entry fallback", () => { + const providers = loadAvailableProviders({ rootDir: ROOT_DIR }); + + assert.ok( + providers.length > COMMON_PROVIDERS.length, + `expected the real catalog, got the ${providers.length}-entry fallback` + ); + // Lower bound rather than an exact count so adding providers does not break this. + assert.ok(providers.length > 200, `expected 200+ providers, got ${providers.length}`); + + const byId = new Map(providers.map((p) => [p.id, p])); + for (const id of ["cerebras", "groq", "gemini", "siliconflow", "opencode", "kiro"]) { + assert.ok(byId.has(id), `${id} missing from the catalog`); + } + assert.equal(byId.get("opencode")?.category, "noauth"); + assert.equal(byId.get("kiro")?.category, "oauth"); + assert.equal(byId.get("cerebras")?.category, "api-key"); + assert.ok(providers.filter((p) => p.hasFree).length > 50, "free-tier flags should survive"); + + // ids are unique — the apikey barrel spreads family files that this walk also + // visits directly, so duplicates must be collapsed. + assert.equal(new Set(providers.map((p) => p.id)).size, providers.length); +}); + +test("falls back to COMMON_PROVIDERS when no catalog is present", () => { + const emptyRoot = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-empty-root-")); + try { + const providers = loadAvailableProviders({ rootDir: emptyRoot }); + assert.equal(providers.length, COMMON_PROVIDERS.length); + assert.equal(providers[0].id, "openai"); + } finally { + fs.rmSync(emptyRoot, { recursive: true, force: true }); + } +}); + +test("an explicit catalogPath still overrides the directory walk", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-catalog-")); + try { + const file = path.join(dir, "custom.ts"); + fs.writeFileSync( + file, + `export const APIKEY_PROVIDERS = { only: { id: "only", name: "Only" } };` + ); + + const providers = loadAvailableProviders({ rootDir: ROOT_DIR, catalogPath: file }); + assert.deepEqual( + providers.map((p) => p.id), + ["only"] + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +});