mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-18 05:02:15 +03:00
feat(api): add opt-in auto-sync scheduler for free-proxy sources (#7079)
This commit is contained in:
1
changelog.d/features/7079-free-proxy-autosync.md
Normal file
1
changelog.d/features/7079-free-proxy-autosync.md
Normal file
@@ -0,0 +1 @@
|
||||
- **feat(api):** opt-in scheduled auto-sync for free-proxy sources (`src/lib/freeProxyProviders/scheduler.ts`) — periodically re-runs the free-proxy provider `sync()` calls (iplocate, proxifly, oneproxy, webshare) that previously only ran via a manual `POST /api/settings/free-proxies/sync`, so a seeded pool no longer goes stale as free-proxy lists rotate; gated by `FREE_PROXY_AUTO_SYNC_ENABLED` (default `false`) and `FREE_PROXY_AUTO_SYNC_INTERVAL_MS` (default 30min, floor-clamped to 5min), reuses the existing `isBuildProcess()`/`OMNIROUTE_DISABLE_BACKGROUND_SERVICES` guards, and shares one `runFreeProxySyncCycle()` code path with the manual route; `GET /api/settings/free-proxies/stats` now also reports `autoSync: { enabled, intervalMs }` (#7079 — thanks @chirag127).
|
||||
@@ -2,6 +2,10 @@ import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { createErrorResponseFromUnknown } from "@/lib/api/errorResponse";
|
||||
import { getFreeProxyStats } from "@/lib/localDb";
|
||||
import { getAllProviders } from "@/lib/freeProxyProviders";
|
||||
import {
|
||||
isFreeProxyAutoSyncEnabled,
|
||||
getFreeProxyAutoSyncIntervalMs,
|
||||
} from "@/lib/freeProxyProviders/scheduler";
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const authError = await requireManagementAuth(request);
|
||||
@@ -14,7 +18,11 @@ export async function GET(request: Request) {
|
||||
name: p.name,
|
||||
enabled: p.isEnabled(),
|
||||
}));
|
||||
return Response.json({ stats, providers });
|
||||
const autoSync = {
|
||||
enabled: isFreeProxyAutoSyncEnabled(),
|
||||
intervalMs: getFreeProxyAutoSyncIntervalMs(),
|
||||
};
|
||||
return Response.json({ stats, providers, autoSync });
|
||||
} catch (error) {
|
||||
return createErrorResponseFromUnknown(error, "Failed to get free proxy stats");
|
||||
}
|
||||
|
||||
@@ -2,12 +2,8 @@ import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import { createErrorResponse, createErrorResponseFromUnknown } from "@/lib/api/errorResponse";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { freeProxySyncSchema } from "@/shared/validation/freeProxySchemas";
|
||||
import { getEnabledProviders, getProvider } from "@/lib/freeProxyProviders";
|
||||
import {
|
||||
recordFreeProxySync,
|
||||
clearFreeProxySyncErrors,
|
||||
recordFreeProxySyncErrors,
|
||||
} from "@/lib/localDb";
|
||||
import { getProvider } from "@/lib/freeProxyProviders";
|
||||
import { runFreeProxySyncCycle } from "@/lib/freeProxyProviders/syncCycle";
|
||||
import type { FreeProxyProvider, FreeProxySourceId } from "@/lib/freeProxyProviders/types";
|
||||
|
||||
let _providersOverrideForTests: FreeProxyProvider[] | null = null;
|
||||
@@ -43,38 +39,15 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
try {
|
||||
const providers =
|
||||
const providers: FreeProxyProvider[] | undefined =
|
||||
_providersOverrideForTests ??
|
||||
(validation.data.sources && validation.data.sources.length > 0
|
||||
? validation.data.sources
|
||||
.map((id) => getProvider(id as FreeProxySourceId))
|
||||
.filter((p): p is NonNullable<typeof p> => p != null)
|
||||
: getEnabledProviders());
|
||||
: undefined);
|
||||
|
||||
const results: Record<string, unknown> = {};
|
||||
for (const provider of providers) {
|
||||
try {
|
||||
results[provider.id] = await provider.sync();
|
||||
await clearFreeProxySyncErrors(provider.id);
|
||||
} catch (error) {
|
||||
// #5595: isolate per-source failures so one provider throwing doesn't
|
||||
// abort the whole sync — the other sources still populate the pool and
|
||||
// the failure is surfaced in `results` instead of a blanket 500.
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
results[provider.id] = {
|
||||
fetched: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
errors: [errorMessage],
|
||||
};
|
||||
await recordFreeProxySyncErrors(provider.id, [errorMessage]);
|
||||
}
|
||||
}
|
||||
|
||||
// #4878: persist the sync timestamp so the UI's "last sync" advances even
|
||||
// when a sync returns zero new/updated proxies (otherwise it stayed frozen
|
||||
// at MAX(last_validated)).
|
||||
const lastSyncAt = await recordFreeProxySync();
|
||||
const { results, lastSyncAt } = await runFreeProxySyncCycle(providers);
|
||||
|
||||
return Response.json({ success: true, results, lastSyncAt });
|
||||
} catch (error) {
|
||||
|
||||
@@ -281,6 +281,9 @@ export async function registerNodejs(): Promise<void> {
|
||||
// Proxy health scheduler (auto-removes dead proxies on interval)
|
||||
await import("@/lib/proxyHealth/scheduler");
|
||||
|
||||
// Free-proxy auto-sync scheduler (re-fetches free-proxy sources on interval, #7079)
|
||||
await import("@/lib/freeProxyProviders/scheduler");
|
||||
|
||||
initGracefulShutdown();
|
||||
initApiBridgeServer();
|
||||
startSpendBatchWriter();
|
||||
|
||||
150
src/lib/freeProxyProviders/scheduler.ts
Normal file
150
src/lib/freeProxyProviders/scheduler.ts
Normal file
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* Free-Proxy Auto-Sync Scheduler (#7079)
|
||||
*
|
||||
* Periodically re-runs the free-proxy provider `sync()` calls (iplocate,
|
||||
* proxifly, oneproxy, webshare) that otherwise only run via a manual
|
||||
* `POST /api/settings/free-proxies/sync`. Free-proxy lists rotate hourly, so a
|
||||
* manually-seeded pool goes stale fast without this.
|
||||
*
|
||||
* Combines two scheduler idioms already used elsewhere in this codebase:
|
||||
* - `proxyHealth/scheduler.ts`: `globalThis`-guarded interval,
|
||||
* `isBuildProcess()` / `isBackgroundServicesDisabled()` guards.
|
||||
* - `providerLimitsSyncScheduler.ts`: `isRunning` reentrancy guard,
|
||||
* elapsed-since-last-run initial delay, `.unref()`'d timers.
|
||||
*
|
||||
* Config via environment (opt-in, off by default — parallels Hard Rule #20's
|
||||
* default-off posture for another data-mutating background feature):
|
||||
* FREE_PROXY_AUTO_SYNC_ENABLED — set "true" to enable (default: off)
|
||||
* FREE_PROXY_AUTO_SYNC_INTERVAL_MS — sync interval in ms (default: 1_800_000
|
||||
* = 30min; floor-clamped to 300_000 =
|
||||
* 5min — outbound courtesy to free-proxy
|
||||
* sources without their own TTL guard)
|
||||
*/
|
||||
|
||||
import { getEnabledProviders } from "@/lib/freeProxyProviders";
|
||||
import { getFreeProxyStats } from "@/lib/localDb";
|
||||
import { runFreeProxySyncCycle, type FreeProxySyncCycleResult } from "./syncCycle";
|
||||
|
||||
const STARTUP_DELAY_MS = 5_000;
|
||||
const DEFAULT_INTERVAL_MS = 1_800_000;
|
||||
const MIN_INTERVAL_MS = 300_000;
|
||||
const LOG_PREFIX = "[FreeProxyAutoSync]";
|
||||
|
||||
declare global {
|
||||
var __freeProxyAutoSyncInterval: ReturnType<typeof setInterval> | undefined;
|
||||
var __freeProxyAutoSyncStartupTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
}
|
||||
|
||||
let isRunning = false;
|
||||
|
||||
type SyncCycleRunner = () => Promise<FreeProxySyncCycleResult>;
|
||||
let _syncCycleRunner: SyncCycleRunner = () => runFreeProxySyncCycle();
|
||||
|
||||
/** Test-only seam: override the cycle body so tests never hit real providers. */
|
||||
export function _setSyncCycleRunnerForTests(runner: SyncCycleRunner | null): void {
|
||||
_syncCycleRunner = runner ?? (() => runFreeProxySyncCycle());
|
||||
}
|
||||
|
||||
export function isFreeProxyAutoSyncEnabled(): boolean {
|
||||
return process.env.FREE_PROXY_AUTO_SYNC_ENABLED === "true";
|
||||
}
|
||||
|
||||
export function getFreeProxyAutoSyncIntervalMs(): number {
|
||||
const raw = parseInt(process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS ?? "", 10);
|
||||
const candidate = Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_INTERVAL_MS;
|
||||
return Math.max(candidate, MIN_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function isBuildProcess(): boolean {
|
||||
return typeof process !== "undefined" && process.env.NEXT_PHASE === "phase-production-build";
|
||||
}
|
||||
|
||||
function isBackgroundServicesDisabled(): boolean {
|
||||
const raw = process.env.OMNIROUTE_DISABLE_BACKGROUND_SERVICES;
|
||||
if (!raw) return false;
|
||||
return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase());
|
||||
}
|
||||
|
||||
async function runCycle(): Promise<void> {
|
||||
if (isRunning) {
|
||||
console.log(`${LOG_PREFIX} Skipping cycle — previous run still in progress`);
|
||||
return;
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
const start = Date.now();
|
||||
try {
|
||||
const { results, lastSyncAt } = await _syncCycleRunner();
|
||||
console.log(
|
||||
`${LOG_PREFIX} Cycle complete in ${Date.now() - start}ms ` +
|
||||
`(lastSyncAt=${lastSyncAt}, sources=${Object.keys(results).length})`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`${LOG_PREFIX} Cycle error:`, error);
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleInterval(intervalMs: number): void {
|
||||
globalThis.__freeProxyAutoSyncInterval = setInterval(() => {
|
||||
void runCycle();
|
||||
}, intervalMs);
|
||||
globalThis.__freeProxyAutoSyncInterval.unref?.();
|
||||
}
|
||||
|
||||
async function computeInitialDelayMs(intervalMs: number): Promise<number> {
|
||||
const { lastSyncAt } = await getFreeProxyStats();
|
||||
if (!lastSyncAt) return STARTUP_DELAY_MS;
|
||||
|
||||
const lastRunMs = Date.parse(lastSyncAt);
|
||||
if (!Number.isFinite(lastRunMs)) return STARTUP_DELAY_MS;
|
||||
|
||||
const elapsedMs = Date.now() - lastRunMs;
|
||||
if (elapsedMs >= intervalMs) return STARTUP_DELAY_MS;
|
||||
return Math.max(intervalMs - elapsedMs, STARTUP_DELAY_MS);
|
||||
}
|
||||
|
||||
/** Guarded entrypoint — auto-called at module bottom, matching `proxyHealth/scheduler.ts`. */
|
||||
export function initFreeProxyAutoSync(): void {
|
||||
if (!isFreeProxyAutoSyncEnabled() || isBuildProcess() || isBackgroundServicesDisabled()) return;
|
||||
if (globalThis.__freeProxyAutoSyncInterval || globalThis.__freeProxyAutoSyncStartupTimer) return;
|
||||
|
||||
if (getEnabledProviders().length === 0) {
|
||||
console.log(`${LOG_PREFIX} No enabled providers — skipping scheduling`);
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalMs = getFreeProxyAutoSyncIntervalMs();
|
||||
console.log(`${LOG_PREFIX} Starting scheduler (interval: ${intervalMs}ms)`);
|
||||
|
||||
void (async () => {
|
||||
const initialDelayMs = await computeInitialDelayMs(intervalMs);
|
||||
globalThis.__freeProxyAutoSyncStartupTimer = setTimeout(() => {
|
||||
globalThis.__freeProxyAutoSyncStartupTimer = undefined;
|
||||
void runCycle();
|
||||
scheduleInterval(intervalMs);
|
||||
}, initialDelayMs);
|
||||
globalThis.__freeProxyAutoSyncStartupTimer.unref?.();
|
||||
})();
|
||||
}
|
||||
|
||||
/** Test/shutdown seam — clears both the startup timer and the recurring interval. */
|
||||
export function stopFreeProxyAutoSync(): void {
|
||||
if (globalThis.__freeProxyAutoSyncInterval) {
|
||||
clearInterval(globalThis.__freeProxyAutoSyncInterval);
|
||||
globalThis.__freeProxyAutoSyncInterval = undefined;
|
||||
}
|
||||
if (globalThis.__freeProxyAutoSyncStartupTimer) {
|
||||
clearTimeout(globalThis.__freeProxyAutoSyncStartupTimer);
|
||||
globalThis.__freeProxyAutoSyncStartupTimer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Test seam — runs one cycle immediately, still honoring the reentrancy guard. */
|
||||
export async function forceFreeProxySyncCycle(): Promise<void> {
|
||||
await runCycle();
|
||||
}
|
||||
|
||||
// Auto-initialize on first import
|
||||
initFreeProxyAutoSync();
|
||||
56
src/lib/freeProxyProviders/syncCycle.ts
Normal file
56
src/lib/freeProxyProviders/syncCycle.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { getEnabledProviders } from "@/lib/freeProxyProviders";
|
||||
import {
|
||||
recordFreeProxySync,
|
||||
clearFreeProxySyncErrors,
|
||||
recordFreeProxySyncErrors,
|
||||
} from "@/lib/localDb";
|
||||
import type { FreeProxyProvider } from "@/lib/freeProxyProviders/types";
|
||||
|
||||
export interface FreeProxySyncCycleResult {
|
||||
results: Record<string, unknown>;
|
||||
lastSyncAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one free-proxy sync cycle: call `sync()` on each provider, isolate
|
||||
* per-provider failures, and persist the cycle timestamp.
|
||||
*
|
||||
* Shared by both trigger paths — the manual `POST /api/settings/free-proxies/sync`
|
||||
* route and the automatic scheduler (`freeProxyProviders/scheduler.ts`, #7079) —
|
||||
* so both go through the exact same code path.
|
||||
*
|
||||
* `providers` defaults to `getEnabledProviders()` when omitted, which is what the
|
||||
* scheduler always uses (it has no notion of a per-request `sources` filter).
|
||||
*/
|
||||
export async function runFreeProxySyncCycle(
|
||||
providers?: FreeProxyProvider[]
|
||||
): Promise<FreeProxySyncCycleResult> {
|
||||
const resolvedProviders = providers ?? getEnabledProviders();
|
||||
const results: Record<string, unknown> = {};
|
||||
|
||||
for (const provider of resolvedProviders) {
|
||||
try {
|
||||
results[provider.id] = await provider.sync();
|
||||
await clearFreeProxySyncErrors(provider.id);
|
||||
} catch (error) {
|
||||
// #5595: isolate per-source failures so one provider throwing doesn't
|
||||
// abort the whole sync — the other sources still populate the pool and
|
||||
// the failure is surfaced in `results` instead of a blanket 500.
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
results[provider.id] = {
|
||||
fetched: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
errors: [errorMessage],
|
||||
};
|
||||
await recordFreeProxySyncErrors(provider.id, [errorMessage]);
|
||||
}
|
||||
}
|
||||
|
||||
// #4878: persist the sync timestamp so the UI's "last sync" advances even
|
||||
// when a sync returns zero new/updated proxies (otherwise it stayed frozen
|
||||
// at MAX(last_validated)).
|
||||
const lastSyncAt = await recordFreeProxySync();
|
||||
|
||||
return { results, lastSyncAt };
|
||||
}
|
||||
213
tests/unit/free-proxy-auto-sync-scheduler.test.ts
Normal file
213
tests/unit/free-proxy-auto-sync-scheduler.test.ts
Normal file
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* #7079 — free-proxy auto-sync scheduler. Network-free: the sync-cycle body
|
||||
* is swapped out via `_setSyncCycleRunnerForTests()` so no test hits a real
|
||||
* provider or the network. Timers use `node:test`'s `mock.timers`.
|
||||
*/
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-free-proxy-autosync-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
|
||||
// The module auto-initializes on import (matches `proxyHealth/scheduler.ts`);
|
||||
// keep it disabled at import time so the very first import doesn't schedule
|
||||
// anything before a test can control the env.
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "false";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const freeProxiesDb = await import("../../src/lib/db/freeProxies.ts");
|
||||
const scheduler = await import("../../src/lib/freeProxyProviders/scheduler.ts");
|
||||
|
||||
const ENV_KEYS = [
|
||||
"FREE_PROXY_AUTO_SYNC_ENABLED",
|
||||
"FREE_PROXY_AUTO_SYNC_INTERVAL_MS",
|
||||
"NEXT_PHASE",
|
||||
"OMNIROUTE_DISABLE_BACKGROUND_SERVICES",
|
||||
"FREE_PROXY_1PROXY_ENABLED",
|
||||
"FREE_PROXY_PROXIFLY_ENABLED",
|
||||
] as const;
|
||||
const savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
|
||||
function restoreEnv() {
|
||||
for (const key of ENV_KEYS) {
|
||||
const val = savedEnv[key];
|
||||
if (val === undefined) delete process.env[key];
|
||||
else process.env[key] = val;
|
||||
}
|
||||
}
|
||||
|
||||
function reset() {
|
||||
scheduler.stopFreeProxyAutoSync();
|
||||
scheduler._setSyncCycleRunnerForTests(null);
|
||||
restoreEnv();
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "false";
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
reset();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
scheduler.stopFreeProxyAutoSync();
|
||||
scheduler._setSyncCycleRunnerForTests(null);
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
restoreEnv();
|
||||
});
|
||||
|
||||
test("disabled by default (unset) — initFreeProxyAutoSync schedules nothing", () => {
|
||||
delete process.env.FREE_PROXY_AUTO_SYNC_ENABLED;
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
assert.equal(globalThis.__freeProxyAutoSyncInterval, undefined);
|
||||
assert.equal(globalThis.__freeProxyAutoSyncStartupTimer, undefined);
|
||||
});
|
||||
|
||||
test('FREE_PROXY_AUTO_SYNC_ENABLED="false" schedules nothing, sync never called', async () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "false";
|
||||
let calls = 0;
|
||||
scheduler._setSyncCycleRunnerForTests(async () => {
|
||||
calls++;
|
||||
return { results: {}, lastSyncAt: new Date().toISOString() };
|
||||
});
|
||||
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
assert.equal(globalThis.__freeProxyAutoSyncStartupTimer, undefined);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test("enabled but no providers are enabled — scheduling is skipped", () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "true";
|
||||
process.env.FREE_PROXY_1PROXY_ENABLED = "false";
|
||||
process.env.FREE_PROXY_PROXIFLY_ENABLED = "false";
|
||||
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
|
||||
assert.equal(globalThis.__freeProxyAutoSyncStartupTimer, undefined);
|
||||
assert.equal(globalThis.__freeProxyAutoSyncInterval, undefined);
|
||||
});
|
||||
|
||||
test("enabled with an enabled provider — startup timer is scheduled", async () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "true";
|
||||
// Leave oneproxy/proxifly at their default-enabled state.
|
||||
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
// The startup timer is armed after an async initial-delay DB read resolves.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
assert.ok(globalThis.__freeProxyAutoSyncStartupTimer, "expected a startup timer to be armed");
|
||||
});
|
||||
|
||||
test("an interval below the 5-minute floor is raised to the floor", () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS = "1000";
|
||||
assert.equal(scheduler.getFreeProxyAutoSyncIntervalMs(), 300_000);
|
||||
});
|
||||
|
||||
test("a valid interval above the floor is respected as-is", () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS = "900000";
|
||||
assert.equal(scheduler.getFreeProxyAutoSyncIntervalMs(), 900_000);
|
||||
});
|
||||
|
||||
test("an unset/invalid interval falls back to the 30-minute default", () => {
|
||||
delete process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS;
|
||||
assert.equal(scheduler.getFreeProxyAutoSyncIntervalMs(), 1_800_000);
|
||||
|
||||
process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS = "not-a-number";
|
||||
assert.equal(scheduler.getFreeProxyAutoSyncIntervalMs(), 1_800_000);
|
||||
});
|
||||
|
||||
test("isBuildProcess() (NEXT_PHASE=phase-production-build) suppresses scheduling", () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "true";
|
||||
process.env.NEXT_PHASE = "phase-production-build";
|
||||
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
|
||||
assert.equal(globalThis.__freeProxyAutoSyncStartupTimer, undefined);
|
||||
});
|
||||
|
||||
test("OMNIROUTE_DISABLE_BACKGROUND_SERVICES=true suppresses scheduling", () => {
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "true";
|
||||
process.env.OMNIROUTE_DISABLE_BACKGROUND_SERVICES = "true";
|
||||
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
|
||||
assert.equal(globalThis.__freeProxyAutoSyncStartupTimer, undefined);
|
||||
});
|
||||
|
||||
test("reentrancy guard: a second forced cycle is skipped while the first is still running", async () => {
|
||||
let concurrentCalls = 0;
|
||||
let maxConcurrent = 0;
|
||||
let releaseFirst: (() => void) | undefined;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
|
||||
scheduler._setSyncCycleRunnerForTests(async () => {
|
||||
concurrentCalls++;
|
||||
maxConcurrent = Math.max(maxConcurrent, concurrentCalls);
|
||||
await gate;
|
||||
concurrentCalls--;
|
||||
return { results: {}, lastSyncAt: new Date().toISOString() };
|
||||
});
|
||||
|
||||
const firstCycle = scheduler.forceFreeProxySyncCycle();
|
||||
// Fire the second cycle while the first is still awaiting the gate. The
|
||||
// reentrancy guard must skip it entirely — the runner must not be invoked
|
||||
// a second time while isRunning is true.
|
||||
const secondCycle = scheduler.forceFreeProxySyncCycle();
|
||||
|
||||
releaseFirst?.();
|
||||
await Promise.all([firstCycle, secondCycle]);
|
||||
|
||||
assert.equal(maxConcurrent, 1, "the cycle runner must never overlap itself");
|
||||
});
|
||||
|
||||
test("cycle delegates to the shared sync-cycle runner (same path as the manual route)", async () => {
|
||||
let called = false;
|
||||
scheduler._setSyncCycleRunnerForTests(async () => {
|
||||
called = true;
|
||||
return { results: { "1proxy": { fetched: 1, added: 1, updated: 0, errors: [] } }, lastSyncAt: "x" };
|
||||
});
|
||||
|
||||
await scheduler.forceFreeProxySyncCycle();
|
||||
|
||||
assert.equal(called, true);
|
||||
});
|
||||
|
||||
test("initial delay: a recent lastSyncAt shortens the first tick below a full interval", async () => {
|
||||
const intervalMs = 300_000; // floor
|
||||
process.env.FREE_PROXY_AUTO_SYNC_INTERVAL_MS = String(intervalMs);
|
||||
process.env.FREE_PROXY_AUTO_SYNC_ENABLED = "true";
|
||||
|
||||
// Record a sync that "happened" 100s ago — well inside the 5-minute interval.
|
||||
const elapsedMs = 100_000;
|
||||
await freeProxiesDb.recordFreeProxySync(new Date(Date.now() - elapsedMs).toISOString());
|
||||
|
||||
let cycleRuns = 0;
|
||||
scheduler._setSyncCycleRunnerForTests(async () => {
|
||||
cycleRuns++;
|
||||
return { results: {}, lastSyncAt: new Date().toISOString() };
|
||||
});
|
||||
|
||||
mock.timers.enable({ apis: ["setTimeout", "setInterval"] });
|
||||
try {
|
||||
scheduler.initFreeProxyAutoSync();
|
||||
// Let the async initial-delay computation (a DB read) resolve and arm the timer.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
// Advancing by less than (intervalMs - elapsedMs) must not fire yet.
|
||||
mock.timers.tick(intervalMs - elapsedMs - 5_000);
|
||||
assert.equal(cycleRuns, 0, "must not fire before the shortened initial delay elapses");
|
||||
|
||||
// Advancing past the remaining delay fires the first cycle.
|
||||
mock.timers.tick(10_000);
|
||||
assert.equal(cycleRuns, 1, "expected exactly one cycle once the initial delay elapses");
|
||||
} finally {
|
||||
mock.timers.reset();
|
||||
}
|
||||
});
|
||||
130
tests/unit/free-proxy-sync-cycle.test.ts
Normal file
130
tests/unit/free-proxy-sync-cycle.test.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* #7079 — `runFreeProxySyncCycle()` is the shared helper extracted out of the
|
||||
* manual `POST /api/settings/free-proxies/sync` route so both the manual route
|
||||
* and the new auto-sync scheduler go through the exact same code path.
|
||||
*
|
||||
* This file proves the helper's own boundary preserves the two invariants the
|
||||
* extraction must not regress:
|
||||
* - #5595: one provider throwing does not abort the others.
|
||||
* - #4878: the sync timestamp advances even when every provider fails/no-ops.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { FreeProxyProvider } from "../../src/lib/freeProxyProviders/types.ts";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-free-proxy-cycle-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const freeProxiesDb = await import("../../src/lib/db/freeProxies.ts");
|
||||
const { runFreeProxySyncCycle } = await import("../../src/lib/freeProxyProviders/syncCycle.ts");
|
||||
|
||||
function reset() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
reset();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function makeProvider(
|
||||
id: FreeProxyProvider["id"],
|
||||
sync: FreeProxyProvider["sync"]
|
||||
): FreeProxyProvider {
|
||||
return { id, name: id, isEnabled: () => true, sync, list: async () => [] };
|
||||
}
|
||||
|
||||
test("#5595 one provider rejecting does not prevent the others from syncing", async () => {
|
||||
const good = makeProvider("1proxy", async () => ({
|
||||
fetched: 2,
|
||||
added: 2,
|
||||
updated: 0,
|
||||
errors: [],
|
||||
}));
|
||||
const bad = makeProvider("proxifly", async () => {
|
||||
throw new Error("upstream unreachable");
|
||||
});
|
||||
|
||||
const { results } = await runFreeProxySyncCycle([bad, good]);
|
||||
|
||||
assert.deepEqual(results["1proxy"], { fetched: 2, added: 2, updated: 0, errors: [] });
|
||||
assert.ok(
|
||||
(results["proxifly"] as { errors: string[] }).errors.some((e) =>
|
||||
e.includes("upstream unreachable")
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test("#5595 a throwing provider records its error via recordFreeProxySyncErrors", async () => {
|
||||
const bad = makeProvider("proxifly", async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
|
||||
await runFreeProxySyncCycle([bad]);
|
||||
|
||||
const errors = await freeProxiesDb.getFreeProxySyncErrors();
|
||||
assert.ok(errors["proxifly"]?.some((e) => e.includes("boom")));
|
||||
});
|
||||
|
||||
test("#5595 a subsequent successful sync clears a source's stored error", async () => {
|
||||
const bad = makeProvider("proxifly", async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
await runFreeProxySyncCycle([bad]);
|
||||
assert.ok((await freeProxiesDb.getFreeProxySyncErrors())["proxifly"]?.length);
|
||||
|
||||
const nowGood = makeProvider("proxifly", async () => ({
|
||||
fetched: 1,
|
||||
added: 1,
|
||||
updated: 0,
|
||||
errors: [],
|
||||
}));
|
||||
await runFreeProxySyncCycle([nowGood]);
|
||||
|
||||
assert.equal((await freeProxiesDb.getFreeProxySyncErrors())["proxifly"], undefined);
|
||||
});
|
||||
|
||||
test("#4878 the sync timestamp advances even when every provider fails", async () => {
|
||||
const before = await freeProxiesDb.getFreeProxyStats();
|
||||
assert.equal(before.lastSyncAt, null);
|
||||
|
||||
const bad = makeProvider("proxifly", async () => {
|
||||
throw new Error("dead upstream");
|
||||
});
|
||||
const { lastSyncAt } = await runFreeProxySyncCycle([bad]);
|
||||
|
||||
assert.ok(typeof lastSyncAt === "string" && lastSyncAt.length > 0);
|
||||
const after = await freeProxiesDb.getFreeProxyStats();
|
||||
assert.equal(after.lastSyncAt, lastSyncAt);
|
||||
});
|
||||
|
||||
test("omitting `providers` still records a sync timestamp with zero providers enabled", async () => {
|
||||
// Disable every default-enabled provider so this stays network-free while
|
||||
// still exercising the `providers === undefined` → `getEnabledProviders()`
|
||||
// default path the scheduler relies on.
|
||||
const prevOneproxy = process.env.FREE_PROXY_1PROXY_ENABLED;
|
||||
const prevProxifly = process.env.FREE_PROXY_PROXIFLY_ENABLED;
|
||||
process.env.FREE_PROXY_1PROXY_ENABLED = "false";
|
||||
process.env.FREE_PROXY_PROXIFLY_ENABLED = "false";
|
||||
try {
|
||||
const { lastSyncAt, results } = await runFreeProxySyncCycle();
|
||||
assert.ok(typeof lastSyncAt === "string" && lastSyncAt.length > 0);
|
||||
assert.deepEqual(results, {});
|
||||
} finally {
|
||||
if (prevOneproxy === undefined) delete process.env.FREE_PROXY_1PROXY_ENABLED;
|
||||
else process.env.FREE_PROXY_1PROXY_ENABLED = prevOneproxy;
|
||||
if (prevProxifly === undefined) delete process.env.FREE_PROXY_PROXIFLY_ENABLED;
|
||||
else process.env.FREE_PROXY_PROXIFLY_ENABLED = prevProxifly;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user