From 25ef8d05c04dcd2b48199da60c86bc5bee923d67 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sun, 19 Jul 2026 16:10:12 -0300 Subject: [PATCH] test(combo): classify dispatches by host and close the auto test pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit providerFromUrl() classified the upstream by URL path shape. With hundreds of providers in the catalog that is wrong in both directions: '/chat/completions' matched every OpenAI-compatible upstream (opencode.ai/zen was being reported as 'openai'), while OpenAI's own GPT-5.6 dispatches go to '/v1/responses' and came back 'unknown'. Classify by hostname instead, and return 'host:' for unrecognised hosts so an unexpected dispatch names itself in the failure message. combo-matrix/auto.test.ts additionally assumed the auto pool contained only the connections it seeded, but no-auth providers (#6557/#7622) legitimately join the pool without a seeded connection. Block those in beforeEach so the pool is closed to the seeded connections — preserving what the assertions are actually about (LKGP pinning, variant pool resolution) rather than relaxing them to accept whatever an open pool picks. Verified: auto.test.ts 2/2, and all 9 combo-matrix files 27/27 with no regressions (no test was passing on the old helper's false 'openai' label). --- .../combo-harness-host-classification.md | 1 + tests/integration/_comboRoutingHarness.ts | 35 +++++++++++++++---- tests/integration/combo-matrix/auto.test.ts | 24 +++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 changelog.d/maintenance/combo-harness-host-classification.md diff --git a/changelog.d/maintenance/combo-harness-host-classification.md b/changelog.d/maintenance/combo-harness-host-classification.md new file mode 100644 index 0000000000..6a7243c456 --- /dev/null +++ b/changelog.d/maintenance/combo-harness-host-classification.md @@ -0,0 +1 @@ +- Classify upstream dispatches by host instead of URL path shape in the combo-routing integration harness, and close the `auto` test pool to its seeded connections. The path heuristic mislabelled every OpenAI-compatible upstream as `openai` and stopped recognising OpenAI's own `/v1/responses` calls, which made `combo-matrix/auto.test.ts` fail on a release tip whose routing was in fact correct. diff --git a/tests/integration/_comboRoutingHarness.ts b/tests/integration/_comboRoutingHarness.ts index cd05dc7c6f..971106a71b 100644 --- a/tests/integration/_comboRoutingHarness.ts +++ b/tests/integration/_comboRoutingHarness.ts @@ -5,13 +5,36 @@ import { createChatPipelineHarness } from "./_chatPipelineHarness.ts"; -// Map an upstream request URL to the provider id it targets. Mirrors the URL -// shapes asserted in combo-routing-e2e.test.ts. Extend as providers are added. +// Map an upstream request URL to the provider id it targets. +// +// Classification is by HOST, not by path shape. Path shapes are ambiguous in +// both directions now that the catalog has hundreds of providers: +// - `/chat/completions` is served by dozens of OpenAI-compatible upstreams +// (e.g. https://opencode.ai/zen/v1/chat/completions), so matching on it +// mislabelled other providers as "openai"; +// - OpenAI itself dispatches the GPT-5.6 family to `/v1/responses`, so its +// own calls stopped matching and came back "unknown". +// The host is what actually identifies the upstream, so we key on that. +// +// Unrecognised hosts return `host:` rather than a bare "unknown", so +// an unexpected dispatch names itself in the assertion message instead of +// forcing a debugging round-trip. +const PROVIDER_BY_HOST: Record = { + "api.openai.com": "openai", + "api.anthropic.com": "claude", + "generativelanguage.googleapis.com": "gemini", + "chatgpt.com": "codex", + "auth.openai.com": "codex", +}; + export function providerFromUrl(url: string): string { - if (url.includes("?beta=true")) return "claude"; - if (url.endsWith("generateContent") || url.includes(":generateContent")) return "gemini"; - if (url.includes("/chat/completions")) return "openai"; - return "unknown"; + let host: string; + try { + host = new URL(url).hostname; + } catch { + return "unknown"; + } + return PROVIDER_BY_HOST[host] ?? `host:${host}`; } export type DispatchCall = { diff --git a/tests/integration/combo-matrix/auto.test.ts b/tests/integration/combo-matrix/auto.test.ts index 59a91cf51b..dd19a76599 100644 --- a/tests/integration/combo-matrix/auto.test.ts +++ b/tests/integration/combo-matrix/auto.test.ts @@ -32,9 +32,33 @@ function body(model: string) { return { model, stream: false, messages: [{ role: "user", content: "hello" }] }; } +// No-auth providers (#6557 / #7622) join the auto-combo pool without any seeded +// connection, because their credential is synthetic — see NOAUTH_PROVIDERS in +// src/shared/constants/providers.ts and the candidate build in +// open-sse/services/autoCombo/virtualFactory.ts. That is intended production +// behavior, but it makes these tests non-deterministic: the pool would contain +// providers this file never seeded, and the dispatch could legitimately land on +// one of them instead of the connection under test. +// +// Blocking them via settings.blockedProviders closes the pool to exactly the +// connections each test seeds, which is what these assertions are actually +// about (LKGP pinning and variant pool resolution) — rather than weakening the +// assertions to accept whatever the open pool happens to pick. +const NO_AUTH_PROVIDER_IDS = [ + "opencode", + "duckduckgo-web", + "felo-web", + "theoldllm", + "chipotle", + "veoaifree-web", + "mimocode", + "auggie", +]; + test.beforeEach(async () => { BaseExecutor.RETRY_CONFIG.delayMs = 0; await resetStorage(); + await settingsDb.updateSettings({ blockedProviders: NO_AUTH_PROVIDER_IDS }); }); test.afterEach(async () => { BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;