diff --git a/.env.example b/.env.example index 4086714a6f..be34e27634 100644 --- a/.env.example +++ b/.env.example @@ -1683,6 +1683,15 @@ APP_LOG_TO_FILE=true # FREE_PROXY_IPLOCATE_ENABLED=false # FREE_PROXY_IPLOCATE_BASE_URL=https://raw.githubusercontent.com/iplocate/free-proxy-list/main/protocols +# ── Free Proxy Pool (Webshare source) ── +# Used by: src/lib/freeProxyProviders/webshare.ts +# Paid, per-account proxy list — requires FREE_PROXY_WEBSHARE_API_KEY to activate, +# regardless of FREE_PROXY_WEBSHARE_ENABLED. +# FREE_PROXY_WEBSHARE_ENABLED=true +# FREE_PROXY_WEBSHARE_API_KEY= +# FREE_PROXY_WEBSHARE_API_URL=https://proxy.webshare.io/api/v2/proxy/list/ +# FREE_PROXY_WEBSHARE_MAX=500 + # ── Vercel Relay ── # Used by: src/app/api/settings/proxy/vercel-deploy/route.ts # Hides the "Deploy Relay" button when set to false. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6be28923..88bcac19ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - **feat(api):** add `/v1/ocr` endpoint (Mistral OCR), an OCR provider category, and Mistral moderation support. (thanks @waguriagentic) - **Discovery tool (Phase 2):** add the `discoveryResults` DB module (CRUD over the `discovery_results` table, migration 074) and wire the opt-in provider-discovery service to persist and read findings through it (`persistDiscoveryResult`, `getDiscoveryResults`, `getDiscoveryResultById`, `markVerified`, `deleteDiscoveryResult`) with `(provider, method, endpoint)` upsert de-duplication. Adds the `/api/discovery/*` HTTP surface — `GET /results`, `GET|DELETE /results/:id`, `POST /scan`, `POST /verify/:id` — under **strict loopback-only** authorization (`/api/discovery/` is in `LOCAL_ONLY_API_PREFIXES` and is NOT manage-scope-bypassable, so the `scan` route's outbound probes can never be reached from a tunnel/remote origin). Adds a **dashboard UI tab** (Tools → Discovery, `/dashboard/discovery`) to run scans and review, verify, or delete findings. The service stays **opt-in / default-off**. +- **feat(proxy):** add Webshare proxy pool import and sync — a `WebshareProvider` (`FreeProxyProvider`) that paginates `proxy.webshare.io/api/v2/proxy/list/` gated on `FREE_PROXY_WEBSHARE_API_KEY`, SSRF-guards imported hosts, and tombstones retired proxy IDs via `pruneStaleFreeProxies()`. (thanks @ricatix) ### 🔧 Bug Fixes diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index ff634a7f6e..e0006c6180 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -1008,6 +1008,10 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `FREE_PROXY_PROXIFLY_ANONYMITY` | `elite` | `src/lib/freeProxyProviders/proxifly.ts` | Anonymity level filter for Proxifly (`elite`, `anonymous`, `transparent`). | | `FREE_PROXY_IPLOCATE_ENABLED` | `false` | `src/lib/freeProxyProviders/iplocate.ts` | Enable the IPLocate free proxy source. Opt-in only. | | `FREE_PROXY_IPLOCATE_BASE_URL` | `https://raw.githubusercontent.com/iplocate/free-proxy-list/main/protocols` | `src/lib/freeProxyProviders/iplocate.ts` | IPLocate proxy list base URL override. | +| `FREE_PROXY_WEBSHARE_ENABLED` | `true` | `src/lib/freeProxyProviders/webshare.ts` | Enable the Webshare proxy pool source. Set to `false` to disable; also requires `FREE_PROXY_WEBSHARE_API_KEY` to be set. | +| `FREE_PROXY_WEBSHARE_API_KEY` | _(none)_ | `src/lib/freeProxyProviders/webshare.ts` | Webshare account API token (`Authorization: Token `). Required — the provider stays disabled without it. | +| `FREE_PROXY_WEBSHARE_API_URL` | `https://proxy.webshare.io/api/v2/proxy/list/` | `src/lib/freeProxyProviders/webshare.ts` | Webshare proxy list API URL override. | +| `FREE_PROXY_WEBSHARE_MAX` | `500` | `src/lib/freeProxyProviders/webshare.ts` | Maximum proxies imported per Webshare sync. | | `NEXT_PUBLIC_VERCEL_RELAY_ENABLED` | `true` | `src/app/(dashboard)/…/ProxyPoolTab.tsx` | Show/hide the Deploy Vercel Relay button in the Proxy Pool tab. | | `VERCEL_API_BASE` | `https://api.vercel.com` | `src/app/api/settings/proxy/vercel-deploy/route.ts` | Vercel API base URL override (for testing). | | `NEXT_PUBLIC_VERCEL_RELAY_DEFAULT_PROJECT` | `omniroute-relay` | `src/app/(dashboard)/…/VercelRelayModal.tsx` | Default project name pre-filled in the Vercel Relay deploy modal. | diff --git a/src/app/(dashboard)/dashboard/settings/components/proxy/SourceToggleBar.tsx b/src/app/(dashboard)/dashboard/settings/components/proxy/SourceToggleBar.tsx index 74a6a4d976..78152b18bf 100644 --- a/src/app/(dashboard)/dashboard/settings/components/proxy/SourceToggleBar.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/proxy/SourceToggleBar.tsx @@ -1,8 +1,8 @@ "use client"; -export type SourceId = "1proxy" | "proxifly" | "iplocate"; +export type SourceId = "1proxy" | "proxifly" | "iplocate" | "webshare"; -export const ALL_SOURCE_IDS: SourceId[] = ["1proxy", "proxifly", "iplocate"]; +export const ALL_SOURCE_IDS: SourceId[] = ["1proxy", "proxifly", "iplocate", "webshare"]; export const FREE_POOL_DISABLED_SOURCES_KEY = "freePool.disabledSources"; @@ -32,6 +32,7 @@ const SOURCES: Array<{ id: SourceId; label: string }> = [ { id: "1proxy", label: "1proxy" }, { id: "proxifly", label: "Proxifly" }, { id: "iplocate", label: "IPLocate" }, + { id: "webshare", label: "Webshare" }, ]; export default function SourceToggleBar({ disabledSources, onToggle }: SourceToggleBarProps) { diff --git a/src/lib/db/freeProxies.ts b/src/lib/db/freeProxies.ts index 57593c98ef..c75642d491 100644 --- a/src/lib/db/freeProxies.ts +++ b/src/lib/db/freeProxies.ts @@ -270,6 +270,34 @@ export async function clearFreeProxiesBySource(source: FreeProxySourceId): Promi return result.changes; } +/** + * Tombstone rows for `source` whose `host:port` is no longer present in the + * provider's latest list — e.g. Webshare recycles/retires proxy IDs between + * syncs. Rows already promoted to the pool (`in_pool = 1`) are left alone so + * runtime resolution of an in-use proxy is never disturbed; only stale, + * not-yet-pooled candidates are pruned. Returns the number of rows removed. + */ +export async function pruneStaleFreeProxies( + source: FreeProxySourceId, + activeKeys: ReadonlySet +): Promise { + const db = getDbInstance(); + const rows = db + .prepare("SELECT id, host, port FROM free_proxies WHERE source = ? AND in_pool = 0") + .all(source) as Array<{ id: string; host: string; port: number }>; + + const staleIds = rows.filter((r) => !activeKeys.has(`${r.host}:${r.port}`)).map((r) => r.id); + + if (staleIds.length === 0) return 0; + + const placeholders = staleIds.map(() => "?").join(","); + const result = db + .prepare(`DELETE FROM free_proxies WHERE id IN (${placeholders})`) + .run(...staleIds); + backupDbFile("pre-write"); + return result.changes; +} + // #4878: the displayed "last sync" used to be derived from MAX(last_validated), // which only advances when a provider returns at least one new/updated proxy. A // sync that returns zero rows (or whose providers all fail) left the timestamp diff --git a/src/lib/freeProxyProviders/index.ts b/src/lib/freeProxyProviders/index.ts index 8a29d71eb0..3f7115069d 100644 --- a/src/lib/freeProxyProviders/index.ts +++ b/src/lib/freeProxyProviders/index.ts @@ -2,11 +2,13 @@ import type { FreeProxyProvider, FreeProxySourceId } from "./types"; import { OneproxyProvider } from "./oneproxy"; import { ProxiflyProvider } from "./proxifly"; import { IplocateProvider } from "./iplocate"; +import { WebshareProvider } from "./webshare"; const ALL_PROVIDERS: FreeProxyProvider[] = [ new OneproxyProvider(), new ProxiflyProvider(), new IplocateProvider(), + new WebshareProvider(), ]; export function getProvider(id: FreeProxySourceId): FreeProxyProvider | undefined { diff --git a/src/lib/freeProxyProviders/types.ts b/src/lib/freeProxyProviders/types.ts index 81bcf86bac..e050a36880 100644 --- a/src/lib/freeProxyProviders/types.ts +++ b/src/lib/freeProxyProviders/types.ts @@ -1,4 +1,4 @@ -export type FreeProxySourceId = "1proxy" | "proxifly" | "iplocate"; +export type FreeProxySourceId = "1proxy" | "proxifly" | "iplocate" | "webshare"; export interface FreeProxyItem { source: FreeProxySourceId; diff --git a/src/lib/freeProxyProviders/webshare.ts b/src/lib/freeProxyProviders/webshare.ts new file mode 100644 index 0000000000..0ca393a680 --- /dev/null +++ b/src/lib/freeProxyProviders/webshare.ts @@ -0,0 +1,148 @@ +import type { FreeProxyItem, FreeProxySyncResult, FreeProxyProvider } from "./types"; +import { isPrivateHost } from "@/shared/network/outboundUrlGuard"; + +const DEFAULT_API_URL = "https://proxy.webshare.io/api/v2/proxy/list/"; +const DEFAULT_MAX = 500; +const DEFAULT_PAGE_SIZE = 100; +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_PAGES = 50; // hard stop so a misbehaving `next` cursor can't loop forever + +type WebshareApiProxy = { + proxy_address?: string; + port?: number; + valid?: boolean; + country_code?: string | null; + last_verification?: string | null; +}; + +type WebshareApiResponse = { + count?: number; + next?: string | null; + previous?: string | null; + results?: WebshareApiProxy[]; +}; + +/** + * Webshare (https://proxy.webshare.io) proxy pool — imports the operator's + * purchased/rotating proxy list via the account API. Unlike the other + * free-proxy sources this is a paid, per-account list, so it is gated on an + * API key rather than a plain on/off flag: `isEnabled()` returns false + * whenever no key is configured, regardless of `FREE_PROXY_WEBSHARE_ENABLED`. + */ +export class WebshareProvider implements FreeProxyProvider { + readonly id = "webshare" as const; + readonly name = "Webshare"; + + isEnabled(): boolean { + if (process.env.FREE_PROXY_WEBSHARE_ENABLED === "false") return false; + return Boolean(process.env.FREE_PROXY_WEBSHARE_API_KEY); + } + + private getConfig() { + return { + apiUrl: process.env.FREE_PROXY_WEBSHARE_API_URL || DEFAULT_API_URL, + apiKey: process.env.FREE_PROXY_WEBSHARE_API_KEY || "", + maxProxies: parseInt(process.env.FREE_PROXY_WEBSHARE_MAX || "", 10) || DEFAULT_MAX, + }; + } + + async sync(): Promise { + if (!this.isEnabled()) { + return { + fetched: 0, + added: 0, + updated: 0, + errors: ["Webshare provider disabled (set FREE_PROXY_WEBSHARE_API_KEY to enable)"], + }; + } + + const { upsertFreeProxy } = await import("../db/freeProxies"); + const { pruneStaleFreeProxies } = await import("../db/freeProxies"); + const { apiUrl, apiKey, maxProxies } = this.getConfig(); + + const errors: string[] = []; + const activeKeys = new Set(); + let added = 0; + let updated = 0; + let fetched = 0; + let page = 1; + + try { + while (fetched < maxProxies && page <= MAX_PAGES) { + const url = new URL(apiUrl); + url.searchParams.set("mode", "direct"); + url.searchParams.set("page", String(page)); + url.searchParams.set("page_size", String(Math.min(DEFAULT_PAGE_SIZE, maxProxies - fetched))); + + const res = await fetch(url, { + signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), + headers: { Authorization: `Token ${apiKey}` }, + }); + + if (!res.ok) { + // Response bodies never contain the request's own Authorization + // header, so this is safe to surface verbatim (truncated) without + // leaking the API key. + const text = await res.text().catch(() => ""); + errors.push(`HTTP ${res.status}: ${text.slice(0, 100)}`); + break; + } + + const json = (await res.json()) as WebshareApiResponse; + const results = Array.isArray(json.results) ? json.results : []; + if (results.length === 0) break; + + for (const p of results) { + if (!p.proxy_address || !p.port) continue; + if (isPrivateHost(p.proxy_address)) { + errors.push(`Webshare: skipped private/loopback host ${p.proxy_address}`); + continue; + } + if (p.valid === false) continue; + + activeKeys.add(`${p.proxy_address}:${p.port}`); + + const item: FreeProxyItem = { + source: "webshare", + host: p.proxy_address, + port: p.port, + type: "http", + countryCode: p.country_code || null, + qualityScore: null, + latencyMs: null, + anonymity: null, + lastValidated: p.last_verification || new Date().toISOString(), + }; + const result = await upsertFreeProxy(item); + if (result.action === "created") added++; + else updated++; + fetched++; + } + + if (!json.next) break; + page++; + } + + // Tombstone rows this account no longer lists (recycled/retired IDs). + // Only run this when the sync completed without a hard error, so a + // failed fetch never wipes out a previously-good candidate list. + if (errors.length === 0 && fetched > 0) { + await pruneStaleFreeProxies("webshare", activeKeys); + } + } catch (err) { + errors.push(err instanceof Error ? err.message : String(err)); + } + + return { fetched, added, updated, errors }; + } + + async list(filters: { + protocol?: string; + country?: string; + minQuality?: number; + limit?: number; + }): Promise { + const { listFreeProxiesBySource } = await import("../db/freeProxies"); + return listFreeProxiesBySource("webshare", filters); + } +} diff --git a/src/lib/localDb.ts b/src/lib/localDb.ts index e2478a946b..b22f879d0f 100755 --- a/src/lib/localDb.ts +++ b/src/lib/localDb.ts @@ -557,6 +557,7 @@ export { promoteFreeProxyToPool, deleteFreeProxy, clearFreeProxiesBySource, + pruneStaleFreeProxies, getFreeProxyStats, recordFreeProxySync, } from "./db/freeProxies"; diff --git a/src/shared/validation/freeProxySchemas.ts b/src/shared/validation/freeProxySchemas.ts index ab91b72810..445e9c90c0 100644 --- a/src/shared/validation/freeProxySchemas.ts +++ b/src/shared/validation/freeProxySchemas.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -export const freeProxySourceSchema = z.enum(["1proxy", "proxifly", "iplocate"]); +export const freeProxySourceSchema = z.enum(["1proxy", "proxifly", "iplocate", "webshare"]); export const freeProxyListSchema = z.object({ sources: z diff --git a/tests/unit/free-proxy-providers.test.ts b/tests/unit/free-proxy-providers.test.ts index a63d53087c..1faa53b9cf 100644 --- a/tests/unit/free-proxy-providers.test.ts +++ b/tests/unit/free-proxy-providers.test.ts @@ -29,13 +29,14 @@ test.after(() => { // ── Registry ───────────────────────────────────────────────────────────────── -test("getAllProviders returns exactly 3 providers", () => { +test("getAllProviders returns exactly 4 providers", () => { const providers = getAllProviders(); - assert.equal(providers.length, 3); + assert.equal(providers.length, 4); const ids = providers.map((p) => p.id); assert.ok(ids.includes("1proxy")); assert.ok(ids.includes("proxifly")); assert.ok(ids.includes("iplocate")); + assert.ok(ids.includes("webshare")); }); test("getProvider returns the correct provider by id", () => { diff --git a/tests/unit/webshare-sync.test.ts b/tests/unit/webshare-sync.test.ts new file mode 100644 index 0000000000..184641bf08 --- /dev/null +++ b/tests/unit/webshare-sync.test.ts @@ -0,0 +1,367 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-webshare-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +// Providers read process.env at call-time, so we can set flags before import. +// Keep the other free-proxy sources disabled so this file only exercises Webshare. +process.env.FREE_PROXY_1PROXY_ENABLED = "false"; +process.env.FREE_PROXY_PROXIFLY_ENABLED = "false"; +process.env.FREE_PROXY_IPLOCATE_ENABLED = "false"; + +const FAKE_API_KEY = "wsk_test_super_secret_token_1234567890"; + +const core = await import("../../src/lib/db/core.ts"); +const { getProvider } = await import("../../src/lib/freeProxyProviders/index.ts"); +const freeProxiesDb = await import("../../src/lib/db/freeProxies.ts"); + +async function reset() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function webshareResponse(results: unknown[], next: string | null = null) { + return new Response(JSON.stringify({ count: results.length, next, previous: null, results }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +// ── isEnabled ──────────────────────────────────────────────────────────────── + +test("WebshareProvider.isEnabled is false without an API key", () => { + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + + const p = getProvider("webshare")!; + assert.equal(p.isEnabled(), false); + + if (originalKey !== undefined) process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey; +}); + +test("WebshareProvider.isEnabled is true once an API key is configured", () => { + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + const p = getProvider("webshare")!; + assert.equal(p.isEnabled(), true); + + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; +}); + +test("WebshareProvider.isEnabled is false when explicitly disabled, even with a key", () => { + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalEnabled = process.env.FREE_PROXY_WEBSHARE_ENABLED; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + process.env.FREE_PROXY_WEBSHARE_ENABLED = "false"; + + const p = getProvider("webshare")!; + assert.equal(p.isEnabled(), false); + + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + process.env.FREE_PROXY_WEBSHARE_ENABLED = originalEnabled ?? ""; + if (originalEnabled === undefined) delete process.env.FREE_PROXY_WEBSHARE_ENABLED; +}); + +// ── sync — disabled path ──────────────────────────────────────────────────── + +test("WebshareProvider.sync returns a disabled error (no key leak) when no API key is set", async () => { + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + await reset(); + + const p = getProvider("webshare")!; + const result = await p.sync(); + + assert.equal(result.fetched, 0); + assert.ok(result.errors.length > 0); + assert.ok(result.errors[0].includes("disabled")); + + if (originalKey !== undefined) process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey; +}); + +// ── sync — pagination + upsert ────────────────────────────────────────────── + +test("WebshareProvider.sync paginates via `next` and upserts proxies", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + const seenAuthHeaders: string[] = []; + const seenPages: string[] = []; + + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)); + seenPages.push(url.searchParams.get("page") || ""); + const headers = new Headers(init?.headers); + seenAuthHeaders.push(headers.get("authorization") || ""); + + const page = url.searchParams.get("page"); + if (page === "1") { + return webshareResponse( + [ + { + proxy_address: "45.10.20.1", + port: 6001, + valid: true, + country_code: "US", + last_verification: "2026-01-01T00:00:00.000Z", + }, + { + proxy_address: "45.10.20.2", + port: 6002, + valid: true, + country_code: "DE", + last_verification: "2026-01-01T00:00:00.000Z", + }, + ], + "https://proxy.webshare.io/api/v2/proxy/list/?page=2" + ); + } + return webshareResponse( + [ + { + proxy_address: "45.10.20.3", + port: 6003, + valid: true, + country_code: "FR", + last_verification: "2026-01-01T00:00:00.000Z", + }, + ], + null + ); + }) as typeof fetch; + + try { + const p = getProvider("webshare")!; + const result = await p.sync(); + + assert.deepEqual(seenPages, ["1", "2"]); + assert.ok( + seenAuthHeaders.every((h) => h === `Token ${FAKE_API_KEY}`), + "every request must carry the Webshare Authorization token" + ); + assert.equal(result.fetched, 3); + assert.equal(result.added, 3); + assert.equal(result.updated, 0); + assert.deepEqual(result.errors, []); + + const items = await p.list({ limit: 10 }); + assert.equal(items.length, 3); + assert.ok(items.every((item) => item.source === "webshare")); + assert.ok(items.some((item) => item.host === "45.10.20.1" && item.port === 6001)); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +test("WebshareProvider.sync updates existing rows instead of duplicating them", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + let call = 0; + globalThis.fetch = (async () => { + call++; + return webshareResponse([ + { + proxy_address: "45.10.20.1", + port: 6001, + valid: true, + country_code: "US", + last_verification: new Date().toISOString(), + }, + ]); + }) as typeof fetch; + + try { + const p = getProvider("webshare")!; + const first = await p.sync(); + const second = await p.sync(); + + assert.equal(first.added, 1); + assert.equal(second.added, 0); + assert.equal(second.updated, 1); + assert.equal(call, 2); + + const items = await p.list({ limit: 10 }); + assert.equal(items.length, 1); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +// ── sync — tombstone ───────────────────────────────────────────────────────── + +test("WebshareProvider.sync tombstones proxies no longer returned by the account list", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + let responses: unknown[][] = [ + [ + { proxy_address: "45.10.20.1", port: 6001, valid: true, country_code: "US" }, + { proxy_address: "45.10.20.2", port: 6002, valid: true, country_code: "DE" }, + ], + ]; + let callIndex = 0; + globalThis.fetch = (async () => { + const batch = responses[callIndex] ?? []; + callIndex++; + return webshareResponse(batch); + }) as typeof fetch; + + try { + const p = getProvider("webshare")!; + await p.sync(); + let items = await p.list({ limit: 10 }); + assert.equal(items.length, 2); + + // Second sync: the account list now only has 45.10.20.1 — .2 was retired. + responses = [[{ proxy_address: "45.10.20.1", port: 6001, valid: true, country_code: "US" }]]; + callIndex = 0; + await p.sync(); + + items = await p.list({ limit: 10 }); + assert.equal(items.length, 1); + assert.equal(items[0].host, "45.10.20.1"); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +test("WebshareProvider.sync never tombstones proxies already promoted to the pool", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + globalThis.fetch = (async () => + webshareResponse([ + { proxy_address: "45.10.20.1", port: 6001, valid: true, country_code: "US" }, + ])) as typeof fetch; + + try { + const p = getProvider("webshare")!; + await p.sync(); + const [item] = await freeProxiesDb.listFreeProxies({ sources: ["webshare"] }); + assert.ok(item); + await freeProxiesDb.markFreeProxyInPool(item.id, "some-registry-id"); + + // Next sync returns an empty list — if pruning ignored `in_pool`, this row + // would be deleted even though it's actively in use by the proxy pool. + globalThis.fetch = (async () => webshareResponse([])) as typeof fetch; + await p.sync(); + + const stillThere = await freeProxiesDb.getFreeProxyById(item.id); + assert.ok(stillThere, "in-pool proxy must survive a sync where it is no longer listed"); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +// ── sync — error sanitization / key masking ───────────────────────────────── + +test("WebshareProvider.sync never leaks the API key in error messages on an HTTP failure", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + globalThis.fetch = (async () => + new Response("Unauthorized", { status: 401 })) as typeof fetch; + + try { + const p = getProvider("webshare")!; + const result = await p.sync(); + + assert.equal(result.fetched, 0); + assert.ok(result.errors.length > 0); + for (const err of result.errors) { + assert.ok(!err.includes(FAKE_API_KEY), `error must not leak the API key: ${err}`); + assert.ok(!err.toLowerCase().includes("authorization"), `error must not leak the auth header: ${err}`); + } + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +test("WebshareProvider.sync never leaks the API key in error messages on a network exception", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + globalThis.fetch = (async () => { + throw new Error(`connect ECONNREFUSED (key was ${FAKE_API_KEY})`); + }) as typeof fetch; + + try { + const p = getProvider("webshare")!; + const result = await p.sync(); + + // The provider itself must not additionally embed the key; it only + // forwards err.message. This guards the call site never re-injects the + // key into a wrapper string. + assert.equal(result.fetched, 0); + assert.ok(result.errors.length > 0); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +}); + +test("WebshareProvider.sync skips private/loopback hosts", async () => { + await reset(); + const originalKey = process.env.FREE_PROXY_WEBSHARE_API_KEY; + const originalFetch = globalThis.fetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = FAKE_API_KEY; + + globalThis.fetch = (async () => + webshareResponse([ + { proxy_address: "127.0.0.1", port: 6001, valid: true, country_code: "US" }, + { proxy_address: "45.10.20.9", port: 6009, valid: true, country_code: "US" }, + ])) as typeof fetch; + + try { + const p = getProvider("webshare")!; + const result = await p.sync(); + + assert.equal(result.fetched, 1); + assert.ok(result.errors.some((e) => e.includes("private/loopback"))); + + const items = await p.list({ limit: 10 }); + assert.equal(items.length, 1); + assert.equal(items[0].host, "45.10.20.9"); + } finally { + globalThis.fetch = originalFetch; + process.env.FREE_PROXY_WEBSHARE_API_KEY = originalKey ?? ""; + if (originalKey === undefined) delete process.env.FREE_PROXY_WEBSHARE_API_KEY; + } +});