mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-15 19:52:50 +03:00
* feat(proxy-subscriptions): src/lib/proxySubscription/parse.ts * feat(proxy-subscriptions): src/lib/proxySubscription/subscriptionService.ts * feat(proxy-subscriptions): src/lib/proxySubscription/index.ts * feat(proxy-subscriptions): src/lib/db/migrations/123_proxy_subscriptions.sql * feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/route.ts * feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/route.ts * feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/refresh/route.ts * feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/nodes/route.ts * feat(proxy-subscriptions): src/app/(dashboard)/dashboard/settings/components/proxy/SubscriptionTab.tsx * feat(proxy-subscriptions): tests/unit/proxySubscription.parse.test.ts * feat(proxy-subscriptions): tests/unit/proxySubscription.service.test.ts * feat(proxy-subscriptions): docs/proxy-subscriptions.md * feat(proxy-subscriptions): src/lib/db/proxies/types.ts * feat(proxy-subscriptions): src/lib/db/proxies/mappers.ts * feat(proxy-subscriptions): src/lib/db/proxies.ts * feat(proxy-subscriptions): src/app/(dashboard)/dashboard/settings/components/ProxyTab.tsx * i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab * i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab * i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab * i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab * test(proxy-subscriptions): extract isSubscriptionDue + unit tests * test(proxy-subscriptions): extract isSubscriptionDue + unit tests * test(proxy-subscriptions): extract isSubscriptionDue + unit tests * test(proxy-subscriptions): add global->rule switch re-bind integration test * feat(proxy-subscriptions): inline needs-local-core guidance in SubscriptionTab * i18n: add proxySubscriptionsTab to en (rebased on current main) * i18n: add proxySubscriptionsTab to zh-CN (rebased on current main) * i18n: add proxySubscriptionsTab to pt-BR (rebased on current main) * test(proxy-subscriptions): extract needsCore detection into pure module + unit tests * test(proxy-subscriptions): extract needsCore detection into pure module + unit tests * test(proxy-subscriptions): extract needsCore detection into pure module + unit tests * refactor(proxy-subscriptions): extract scopes.ts into pure module + unit tests (#65) * refactor(proxy-subscriptions): extract coreEndpoint.ts into pure module + unit tests (#65) * refactor(proxy-subscriptions): extract subscriptionService.ts into pure module + unit tests (#65) * refactor(proxy-subscriptions): extract proxySubscription.scopes.test.ts into pure module + unit tests (#65) * refactor(proxy-subscriptions): extract proxySubscription.coreEndpoint.test.ts into pure module + unit tests (#65) * security(proxy-subscriptions): fetchGuard.ts — SSRF guard + core scheme (#P0) * security(proxy-subscriptions): coreEndpoint.ts — SSRF guard + core scheme (#P0) * security(proxy-subscriptions): subscriptionService.ts — SSRF guard + core scheme (#P0) * security(proxy-subscriptions): proxySubscription.fetchGuard.test.ts — SSRF guard + core scheme (#P0) * security(proxy-subscriptions): proxySubscription.coreEndpoint.test.ts — SSRF guard + core scheme (#P0) * refactor(proxy-subscriptions): subscriptionService.ts — concurrency lock / resilience / url redaction (#P1) * refactor(proxy-subscriptions): url.ts — concurrency lock / resilience / url redaction (#P1) * refactor(proxy-subscriptions): index.ts — concurrency lock / resilience / url redaction (#P1) * refactor(proxy-subscriptions): route.ts — concurrency lock / resilience / url redaction (#P1) * refactor(proxy-subscriptions): route.ts — concurrency lock / resilience / url redaction (#P1) * refactor(proxy-subscriptions): proxySubscription.url.test.ts — concurrency lock / resilience / url redaction (#P1) * i18n(proxy-subscriptions): subscriptionService.ts — stable error codes + locale keys (#P2-6) * i18n(proxy-subscriptions): SubscriptionTab.tsx — stable error codes + locale keys (#P2-6) * i18n(proxy-subscriptions): en.json — stable error codes + locale keys (#P2-6) * i18n(proxy-subscriptions): zh-CN.json — stable error codes + locale keys (#P2-6) * i18n(proxy-subscriptions): pt-BR.json — stable error codes + locale keys (#P2-6) * i18n(proxy-subscriptions): en.json — normalize to LF line endings (#P2-6) * i18n(proxy-subscriptions): zh-CN.json — normalize to LF line endings (#P2-6) * i18n(proxy-subscriptions): pt-BR.json — normalize to LF line endings (#P2-6) * enhance(proxy-subscriptions): reject subscription fetch if ANY resolved DNS address is blocked (P3-1) * enhance(proxy-subscriptions): add withRetry() exponential-backoff helper (P3-2) * enhance(proxy-subscriptions): DNS multi-record guard + retry/backoff fetch + batch scope writes + observability (P3-1..P3-4) * enhance(proxy-subscriptions): batch addProxiesToScopePool() to drop N+1 writes (P3-3) * enhance(proxy-subscriptions): add last_error_at + consecutive_failures observability columns (P3-4) * enhance(proxy-subscriptions): surface consecutive failures + last error time in subscription cards (P3-4) * test(proxy-subscriptions): cover multi-record DNS SSRF (block if ANY address internal) (P3-1) * test(proxy-subscriptions): cover withRetry() first-success / retries / backoff / non-retryable stop (P3-2) * fix(proxy-subscriptions): resolve js-yaml import + missing backup/generation-bump imports Two bugs made the feature non-functional and its own test suite false: 1. parse.ts used `import yaml from "js-yaml"` (default import), but js-yaml@^5 is ESM-only with no default export — this threw a SyntaxError at module load, crashing every caller (index.ts re-exports parse.ts, so every API route hit this too). Switch to `import * as yaml`, matching how the rest of the codebase already imports js-yaml (hermes-agent.ts, openapiParser.ts, openapi/spec route.ts, guide-settings route.ts). 2. subscriptionService.ts's unapplySubscription() called backupDbFile() and bumpProxyRegistryGeneration() without importing either — backupDbFile exists but wasn't imported; bumpProxyRegistryGeneration was a private, non-exported function in db/proxies.ts. This threw a ReferenceError whenever a subscription with bound proxies was disabled/deleted (the normal path). Import backupDbFile from ../db/backup and export+import bumpProxyRegistryGeneration from ../db/proxies, matching the identical backup+bump pattern already used by deleteProxyById for the same proxy_assignments/proxy_registry mutation shape. Fixing both unmasked a third, previously-unreachable bug (both crashes happened before any test assertion could run): recomputeProxyEnabled() checked `proxy_subscriptions.enabled = 1` alone, which stays true across an unapply/disable cycle since unapplySubscription() never touches that column — the proxyEnabled flag would get stuck on `true` even after the subscription's proxies were fully detached. Changed the check to require an actually-bound proxy_assignments row for an enabled subscription, matching hasNonSubscriptionGlobalProxy()'s existing bound-check pattern. Traced all 3 production call sites (mode/rule switch, disable, delete) to confirm this doesn't change their outcome — only the previously-wrong "unapply in isolation" case. Also fixed the test file's own pre-existing bug: its provider_connections inserts omitted created_at/updated_at (NOT NULL, no default in the schema since 001_initial_schema.sql), which 0 assertions had ever reached before because the SyntaxError always crashed the file first. All 9 proxySubscription test files now run clean: 49/49 pass (previously 2 files crashed outright at import time, 0 assertions ever ran). Separately confirmed via testing against the pristine PR head: this PR has 3 more pre-existing gate failures unrelated to the above (file-size on db/proxies.ts, cognitive-complexity, complexity, changelog-integrity vs the current release tip) plus 3 pre-existing TS2345 errors in parse.ts (lines 228/233/263, unrelated to the yaml import). All are the PR's own scope/base-drift, out of scope for this fix — the PR has never had a real CI run (base=main), so no gate has ever surfaced them; they need the base retarget + a full CI pass called out in the plan file's own remaining mandatory items. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> * fix(db): renumber proxy-subscriptions migrations to avoid version collision release/v3.8.49 already ships 123_quota_auto_ping.sql and 124_generic_session_affinity_ttl.sql; this PR's 123/124 files collided, tripping migrationRunner's version-collision guard and failing all proxySubscription.service tests. Renumber to 127/128 (next free slots after 126_reasoning_routing_rules.sql). Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com> * refactor(db): extract proxySubscriptions + registryGeneration to keep proxies.ts under file-size cap Moves addProxiesToScopePool to ./proxySubscriptions.ts and the registry-generation helpers to ./proxies/registryGeneration.ts (re-exported from proxies.ts), keeping the module under its frozen 1177-line cap after the operator-proxy-subscriptions feature. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> * fix(db): renumber proxy-subscriptions migrations past release collision Rebasing onto release/v3.8.49 surfaced a migration version collision: this branch's 127_proxy_subscriptions.sql and 128_proxy_subscriptions_meta.sql now collide with 127_usage_history_account_identity.sql and 128_auto_candidate_overrides.sql that landed on release since this branch last synced. Renumbered to 131/132 (next free prefixes after the current 130_remove_unregistered_qwen_data.sql) and updated the in-file header comments to match. No schema/behavior change. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouza.pw@gmail.com> Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com> Co-authored-by: xier2012 <xier2012@users.noreply.github.com>
701 lines
27 KiB
TypeScript
701 lines
27 KiB
TypeScript
/**
|
|
* Proxy subscription service (Karing-style, operator-supplied).
|
|
*
|
|
* A subscription is a URL the operator pastes. We fetch + parse it into a pool
|
|
* of proxy nodes, sync those nodes into `proxy_registry` (source='subscription',
|
|
* subscription_id set), and bind the pool through the EXISTING scope resolution
|
|
* (account/provider/global) — so subscriptions inherit rotation, health checks,
|
|
* and the fail-closed guard for free.
|
|
*
|
|
* Modes:
|
|
* - 'global': pool bound to the global scope (all provider traffic proxied).
|
|
* - 'rule': pool bound only to the selected provider scopes (others direct).
|
|
*
|
|
* Protocol support:
|
|
* - http/https/socks5 nodes are used directly.
|
|
* - ss/vmess/vless/trojan/tuic/hysteria/wireguard nodes need a local proxy
|
|
* core (sing-box/clash) exposing a SOCKS5/HTTP endpoint; supply it via
|
|
* `localCoreEndpoint` and we bind that single endpoint (the core does the
|
|
* protocol translation + node selection). Without it, those nodes are
|
|
* reported but not routed.
|
|
*/
|
|
import { randomUUID } from "crypto";
|
|
import { getDbInstance } from "../db/core";
|
|
import { backupDbFile } from "../db/backup";
|
|
import {
|
|
addProxiesToScopePool,
|
|
bumpProxyRegistryGeneration,
|
|
deleteProxyById,
|
|
upsertProxy,
|
|
} from "../db/proxies";
|
|
import { bumpProxyConfigGeneration } from "../db/settings";
|
|
import { isSubscriptionDue } from "./due";
|
|
import { isLocalCoreEndpointAllowed } from "./coreEndpoint";
|
|
import { resolveTargetScopes } from "./scopes";
|
|
import {
|
|
isSubscriptionFetchUrlAllowed,
|
|
isIpLiteral,
|
|
isAnyResolvedAddressBlocked,
|
|
} from "./fetchGuard";
|
|
import { withRetry } from "./fetchRetry";
|
|
import { parseSubscription, redactedNodeSummary, type ParsedSubscription } from "./parse";
|
|
|
|
export type ProxySubscriptionMode = "global" | "rule";
|
|
export type ProxySubscriptionStatus = "ok" | "error" | "empty";
|
|
|
|
/** Stable, language-neutral error codes stored in the subscription `error`
|
|
* column (as JSON) so the dashboard can localize them via i18n instead of
|
|
* showing server-side strings. */
|
|
export type ProxySubscriptionErrorCode =
|
|
| "LOCAL_CORE_ENDPOINT_INVALID"
|
|
| "NEEDS_CORE_NOT_CONFIGURED"
|
|
| "NO_USABLE_NODES";
|
|
|
|
/** Encode a user-facing error as `{ code, detail? }` for i18n on the client. */
|
|
export function subscriptionErrorCode(code: ProxySubscriptionErrorCode, detail?: string): string {
|
|
return JSON.stringify(detail ? { code, detail } : { code });
|
|
}
|
|
|
|
export interface ProxySubscriptionRecord {
|
|
id: string;
|
|
name: string;
|
|
url: string;
|
|
enabled: boolean;
|
|
mode: ProxySubscriptionMode;
|
|
ruleProviders: string[] | null;
|
|
localCoreEndpoint: string | null;
|
|
updateIntervalMinutes: number;
|
|
lastFetchedAt: string | null;
|
|
status: ProxySubscriptionStatus;
|
|
error: string | null;
|
|
lastNodes: unknown[] | null;
|
|
lastErrorAt: string | null;
|
|
consecutiveFailures: number;
|
|
createdAt: string;
|
|
updatedAt: string;
|
|
}
|
|
|
|
export interface ProxySubscriptionPayload {
|
|
name: string;
|
|
url: string;
|
|
enabled?: boolean;
|
|
mode?: ProxySubscriptionMode;
|
|
ruleProviders?: string[] | null;
|
|
localCoreEndpoint?: string | null;
|
|
updateIntervalMinutes?: number;
|
|
}
|
|
|
|
export interface SyncResult {
|
|
subscriptionId: string;
|
|
nodes: number;
|
|
needsCore: number;
|
|
boundProxies: number;
|
|
status: ProxySubscriptionStatus;
|
|
error: string | null;
|
|
applied: boolean;
|
|
}
|
|
|
|
const SUBSCRIPTION_FETCH_TIMEOUT_MS = 15_000;
|
|
|
|
// ───────────────────────────── Row mapping ─────────────────────────────
|
|
|
|
function mapSubscriptionRow(row: unknown): ProxySubscriptionRecord {
|
|
const r = (row && typeof row === "object" ? row : {}) as Record<string, unknown>;
|
|
const parseList = (v: unknown): string[] | null => {
|
|
if (typeof v !== "string" || !v.trim()) return null;
|
|
try {
|
|
const arr = JSON.parse(v);
|
|
return Array.isArray(arr) ? arr.filter((x) => typeof x === "string") : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
const parseNodes = (v: unknown): unknown[] | null => {
|
|
if (typeof v !== "string" || !v.trim()) return null;
|
|
try {
|
|
const arr = JSON.parse(v);
|
|
return Array.isArray(arr) ? arr : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
return {
|
|
id: typeof r.id === "string" ? r.id : "",
|
|
name: typeof r.name === "string" ? r.name : "",
|
|
url: typeof r.url === "string" ? r.url : "",
|
|
enabled: Number(r.enabled) !== 0,
|
|
mode: r.mode === "rule" ? "rule" : "global",
|
|
ruleProviders: parseList(r.rule_providers),
|
|
localCoreEndpoint: typeof r.local_core_endpoint === "string" ? r.local_core_endpoint : null,
|
|
updateIntervalMinutes: Number(r.update_interval_minutes) || 60,
|
|
lastFetchedAt: typeof r.last_fetched_at === "string" ? r.last_fetched_at : null,
|
|
status: (r.status as ProxySubscriptionStatus) || "empty",
|
|
error: typeof r.error === "string" ? r.error : null,
|
|
lastNodes: parseNodes(r.last_nodes),
|
|
lastErrorAt: typeof r.last_error_at === "string" ? r.last_error_at : null,
|
|
consecutiveFailures: Number(r.consecutive_failures) || 0,
|
|
createdAt: typeof r.created_at === "string" ? r.created_at : "",
|
|
updatedAt: typeof r.updated_at === "string" ? r.updated_at : "",
|
|
};
|
|
}
|
|
|
|
// ───────────────────────────── CRUD ─────────────────────────────
|
|
|
|
export async function listSubscriptions(): Promise<ProxySubscriptionRecord[]> {
|
|
const db = getDbInstance();
|
|
const rows = db
|
|
.prepare(
|
|
"SELECT id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, last_fetched_at, status, error, last_nodes, last_error_at, consecutive_failures, created_at, updated_at FROM proxy_subscriptions ORDER BY datetime(updated_at) DESC, name ASC"
|
|
)
|
|
.all();
|
|
return rows.map(mapSubscriptionRow);
|
|
}
|
|
|
|
export async function getSubscriptionById(id: string): Promise<ProxySubscriptionRecord | null> {
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare(
|
|
"SELECT id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, last_fetched_at, status, error, last_nodes, last_error_at, consecutive_failures, created_at, updated_at FROM proxy_subscriptions WHERE id = ?"
|
|
)
|
|
.get(id);
|
|
return row ? mapSubscriptionRow(row) : null;
|
|
}
|
|
|
|
export async function createSubscription(
|
|
payload: ProxySubscriptionPayload
|
|
): Promise<ProxySubscriptionRecord> {
|
|
const id = randomUUID();
|
|
const now = new Date().toISOString();
|
|
const enabled = payload.enabled === true ? 1 : 0;
|
|
const db = getDbInstance();
|
|
db.prepare(
|
|
`INSERT INTO proxy_subscriptions
|
|
(id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, status, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'empty', ?, ?)`
|
|
).run(
|
|
id,
|
|
payload.name,
|
|
payload.url,
|
|
enabled,
|
|
payload.mode || "global",
|
|
payload.ruleProviders ? JSON.stringify(payload.ruleProviders) : null,
|
|
payload.localCoreEndpoint || null,
|
|
payload.updateIntervalMinutes || 60,
|
|
now,
|
|
now
|
|
);
|
|
const created = (await getSubscriptionById(id))!;
|
|
if (created.enabled) {
|
|
await syncSubscription(id);
|
|
}
|
|
// Re-read so the returned record reflects the post-sync status/error/lastNodes.
|
|
return (await getSubscriptionById(id))!;
|
|
}
|
|
|
|
export async function updateSubscription(
|
|
id: string,
|
|
payload: Partial<ProxySubscriptionPayload>
|
|
): Promise<ProxySubscriptionRecord | null> {
|
|
const existing = await getSubscriptionById(id);
|
|
if (!existing) return null;
|
|
const db = getDbInstance();
|
|
const name = payload.name ?? existing.name;
|
|
const url = payload.url ?? existing.url;
|
|
const mode = payload.mode ?? existing.mode;
|
|
const ruleProviders = payload.ruleProviders !== undefined ? payload.ruleProviders : existing.ruleProviders;
|
|
const localCoreEndpoint =
|
|
payload.localCoreEndpoint !== undefined ? payload.localCoreEndpoint : existing.localCoreEndpoint;
|
|
const updateIntervalMinutes = payload.updateIntervalMinutes ?? existing.updateIntervalMinutes;
|
|
const now = new Date().toISOString();
|
|
|
|
const enabledChanged = payload.enabled !== undefined && payload.enabled !== existing.enabled;
|
|
const enabled = payload.enabled !== undefined ? (payload.enabled ? 1 : 0) : existing.enabled ? 1 : 0;
|
|
|
|
db.prepare(
|
|
`UPDATE proxy_subscriptions
|
|
SET name = ?, url = ?, enabled = ?, mode = ?, rule_providers = ?, local_core_endpoint = ?, update_interval_minutes = ?, updated_at = ?
|
|
WHERE id = ?`
|
|
).run(
|
|
name,
|
|
url,
|
|
enabled,
|
|
mode,
|
|
ruleProviders ? JSON.stringify(ruleProviders) : null,
|
|
localCoreEndpoint || null,
|
|
updateIntervalMinutes,
|
|
now,
|
|
id
|
|
);
|
|
|
|
const updated = (await getSubscriptionById(id))!;
|
|
|
|
// Re-evaluate binding.
|
|
if (updated.enabled) {
|
|
if (payload.mode !== undefined || payload.ruleProviders !== undefined) {
|
|
// Routing targets changed: detach from the old scopes first, then
|
|
// re-fetch + bind to the new targets. applySubscription is idempotent per
|
|
// scope, but a global→rule switch must drop the previous global binding.
|
|
await unapplySubscription(id);
|
|
await syncSubscription(id);
|
|
} else if (
|
|
enabledChanged ||
|
|
payload.url !== undefined ||
|
|
payload.localCoreEndpoint !== undefined
|
|
) {
|
|
// Same scopes: just refresh nodes (re-applies idempotently to same scopes).
|
|
await syncSubscription(id);
|
|
}
|
|
} else {
|
|
// Disabled: detach everything.
|
|
await unapplySubscription(id);
|
|
}
|
|
return getSubscriptionById(id);
|
|
}
|
|
|
|
export async function setSubscriptionEnabled(id: string, enabled: boolean): Promise<ProxySubscriptionRecord | null> {
|
|
return updateSubscription(id, { enabled });
|
|
}
|
|
|
|
export async function deleteSubscription(id: string): Promise<boolean> {
|
|
await unapplySubscription(id);
|
|
// Remove subscription-sourced proxy rows (force-clears their assignments).
|
|
const db = getDbInstance();
|
|
const rows = db
|
|
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
|
|
.all(id) as Array<{ id: string }>;
|
|
for (const r of rows) {
|
|
try {
|
|
await deleteProxyById(r.id, { force: true });
|
|
} catch {
|
|
// ignore individual failures
|
|
}
|
|
}
|
|
const res = db.prepare("DELETE FROM proxy_subscriptions WHERE id = ?").run(id);
|
|
await recomputeProxyEnabled();
|
|
return res.changes > 0;
|
|
}
|
|
|
|
// ───────────────────────────── Scope resolution ─────────────────────────────
|
|
|
|
// ───────────────────────────── Sync + apply ─────────────────────────────
|
|
|
|
/**
|
|
* Refuse to fetch a subscription URL unless it is http/https to a non-internal
|
|
* host. IP literals are checked structurally; hostnames are resolved and the
|
|
* resolved addresses are re-checked (fail closed on resolution errors). This
|
|
* blocks SSRF to internal services / cloud metadata (169.254.169.254).
|
|
*/
|
|
async function assertSafeFetchTarget(url: string): Promise<void> {
|
|
if (!isSubscriptionFetchUrlAllowed(url)) {
|
|
throw new Error("Subscription URL is not allowed (scheme or host blocked)");
|
|
}
|
|
const host = new URL(url).hostname.toLowerCase();
|
|
const bare = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
|
if (!isIpLiteral(bare)) {
|
|
// Hostname: resolve ALL records and refuse if ANY address is internal
|
|
// (fail closed). A hostname can resolve to multiple records; checking only
|
|
// the first would let an internal IP slip through if a public record also
|
|
// exists. `lookup(..., { all: true })` returns every A/AAAA record.
|
|
try {
|
|
const dns = await import("node:dns");
|
|
const addrs = await dns.promises.lookup(bare, { all: true });
|
|
if (isAnyResolvedAddressBlocked(addrs)) {
|
|
throw new Error("Subscription host resolves to a blocked (internal) address");
|
|
}
|
|
} catch (e) {
|
|
if (e instanceof Error && e.message.includes("blocked")) throw e;
|
|
throw new Error(`Subscription host resolution failed: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Single fetch attempt: SSRF-validate the target, fetch with manual redirect
|
|
* handling, and re-validate any redirect target. Throws on non-2xx or a
|
|
* blocked target. Each attempt owns its own timeout so a retry after a fast
|
|
* failure isn't killed by the previous attempt's timer.
|
|
*/
|
|
async function doSafeFetch(url: string, headers: Record<string, string>): Promise<string> {
|
|
await assertSafeFetchTarget(url);
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), SUBSCRIPTION_FETCH_TIMEOUT_MS);
|
|
try {
|
|
const res = await fetch(url, { redirect: "manual", signal: controller.signal, headers });
|
|
if (res.status >= 300 && res.status < 400) {
|
|
const loc = res.headers.get("location");
|
|
if (!loc) throw new Error("Subscription fetch redirected without Location");
|
|
// Resolve relative redirects and re-validate the target (SSRF guard).
|
|
const next = new URL(loc, url).toString();
|
|
await assertSafeFetchTarget(next);
|
|
const res2 = await fetch(next, { redirect: "manual", signal: controller.signal, headers });
|
|
if (res2.status >= 300 && res2.status < 400) {
|
|
throw new Error("Subscription fetch: too many redirects");
|
|
}
|
|
if (!res2.ok) {
|
|
throw new Error(`Subscription fetch failed: HTTP ${res2.status}`);
|
|
}
|
|
return await res2.text();
|
|
}
|
|
if (!res.ok) {
|
|
throw new Error(`Subscription fetch failed: HTTP ${res.status}`);
|
|
}
|
|
return await res.text();
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Classify a fetch error as retryable. Transient (retry): network/timeout/DNS
|
|
* failures, HTTP 5xx, and HTTP 429. Permanent (no retry): 4xx client errors
|
|
* (except 429) and any SSRF-guard block — those will never succeed on retry.
|
|
*/
|
|
function isSubscriptionFetchRetryable(e: unknown): boolean {
|
|
const msg = e instanceof Error ? e.message : String(e);
|
|
if (msg.includes("not allowed") || msg.includes("blocked (internal)")) return false;
|
|
const m = msg.match(/HTTP (\d{3})/);
|
|
if (m) {
|
|
const code = Number(m[1]);
|
|
if (code === 429) return true;
|
|
if (code >= 500 && code < 600) return true;
|
|
return false; // 4xx (except 429) — permanent client error
|
|
}
|
|
return true; // network error / timeout / DNS failure — transient
|
|
}
|
|
|
|
async function fetchSubscriptionContent(url: string): Promise<string> {
|
|
const headers = { "User-Agent": "OmniRoute-ProxySubscription" };
|
|
// Retry transient failures (timeouts, 5xx, 429) with bounded exponential
|
|
// backoff; give up fast on permanent errors (4xx, SSRF block).
|
|
return withRetry(() => doSafeFetch(url, headers), {
|
|
maxAttempts: 3,
|
|
baseDelayMs: 500,
|
|
maxDelayMs: 5000,
|
|
isRetryable: isSubscriptionFetchRetryable,
|
|
});
|
|
}
|
|
|
|
/** Fetch + parse + sync nodes into proxy_registry, then (if enabled) (re)bind. */
|
|
async function syncSubscriptionUnsafe(id: string): Promise<SyncResult> {
|
|
const sub = await getSubscriptionById(id);
|
|
if (!sub) {
|
|
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: "not found", applied: false };
|
|
}
|
|
|
|
let body: string;
|
|
try {
|
|
body = await fetchSubscriptionContent(sub.url);
|
|
} catch (e) {
|
|
const msg = e instanceof Error ? e.message : String(e);
|
|
const fetchConsec = (sub.consecutiveFailures || 0) + 1;
|
|
await updateSubscriptionStatus(id, "error", `Fetch failed: ${msg}`, null, new Date().toISOString(), fetchConsec);
|
|
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: msg, applied: false };
|
|
}
|
|
|
|
const parsed: ParsedSubscription = parseSubscription(body);
|
|
const db = getDbInstance();
|
|
|
|
const keptIds: string[] = [];
|
|
let warning: string | null = null;
|
|
|
|
// The upsert loop + stale-removal are the multi-write section. upsertProxy /
|
|
// deleteProxyById each run their own internal db.transaction, so each write
|
|
// is atomic, and a re-sync is idempotent (self-heals partial state). A single
|
|
// outer db.transaction around `await` calls would NOT be atomic under
|
|
// better-sqlite3, so instead we guard against an unexpected DB error so a
|
|
// half-completed sync can never be left flagged "ok".
|
|
try {
|
|
// Directly-usable nodes → upsert into the registry as a pool.
|
|
for (const node of parsed.nodes) {
|
|
const upserted = await upsertProxy({
|
|
name: node.name || `${sub.name} (${node.host}:${node.port})`,
|
|
type: node.type,
|
|
host: node.host,
|
|
port: node.port,
|
|
username: node.username,
|
|
password: node.password,
|
|
source: "subscription",
|
|
subscriptionId: id,
|
|
status: "active",
|
|
});
|
|
if (upserted.proxy?.id) keptIds.push(upserted.proxy.id);
|
|
}
|
|
|
|
// needsCore nodes → bind the operator-supplied local core endpoint (single).
|
|
if (parsed.needsCore.length > 0) {
|
|
if (sub.localCoreEndpoint && isLocalCoreEndpointAllowed(sub.localCoreEndpoint)) {
|
|
try {
|
|
const coreUrl = new URL(sub.localCoreEndpoint);
|
|
const coreType = coreUrl.protocol === "https:" ? "https" : coreUrl.protocol === "socks5:" ? "socks5" : "http";
|
|
const upserted = await upsertProxy({
|
|
name: `${sub.name} (local core)`,
|
|
type: coreType,
|
|
host: coreUrl.hostname,
|
|
port: Number(coreUrl.port) || (coreType === "https" ? 443 : 8080),
|
|
username: coreUrl.username ? decodeURIComponent(coreUrl.username) : undefined,
|
|
password: coreUrl.password ? decodeURIComponent(coreUrl.password) : undefined,
|
|
source: "subscription",
|
|
subscriptionId: id,
|
|
status: "active",
|
|
});
|
|
if (upserted.proxy?.id) keptIds.push(upserted.proxy.id);
|
|
} catch {
|
|
warning = subscriptionErrorCode("LOCAL_CORE_ENDPOINT_INVALID");
|
|
}
|
|
} else {
|
|
const nodes = parsed.needsCore
|
|
.map((n) => `${n.rawProtocol}://${n.host ?? ""}${n.port ? ":" + n.port : ""}`)
|
|
.join(", ");
|
|
warning = subscriptionErrorCode("NEEDS_CORE_NOT_CONFIGURED", nodes);
|
|
}
|
|
}
|
|
|
|
// Remove stale subscription nodes no longer present in the fetched set.
|
|
if (keptIds.length > 0) {
|
|
const placeholders = keptIds.map(() => "?").join(",");
|
|
const stale = db
|
|
.prepare(`SELECT id FROM proxy_registry WHERE subscription_id = ? AND id NOT IN (${placeholders})`)
|
|
.all(id, ...keptIds) as Array<{ id: string }>;
|
|
for (const r of stale) {
|
|
try {
|
|
await deleteProxyById(r.id, { force: true });
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
} else {
|
|
const stale = db
|
|
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
|
|
.all(id) as Array<{ id: string }>;
|
|
for (const r of stale) {
|
|
try {
|
|
await deleteProxyById(r.id, { force: true });
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
} catch (e) {
|
|
const msg = e instanceof Error ? e.message : String(e);
|
|
const writeConsec = (sub.consecutiveFailures || 0) + 1;
|
|
await updateSubscriptionStatus(id, "error", `Sync write failed: ${msg}`, null, new Date().toISOString(), writeConsec);
|
|
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: msg, applied: false };
|
|
}
|
|
|
|
const lastNodes = redactedNodeSummary(parsed);
|
|
|
|
// Determine status.
|
|
let status: ProxySubscriptionStatus;
|
|
let error: string | null = warning;
|
|
if (keptIds.length === 0) {
|
|
status = "error";
|
|
error = warning || subscriptionErrorCode("NO_USABLE_NODES");
|
|
} else if (warning) {
|
|
status = "ok";
|
|
} else {
|
|
status = parsed.nodes.length === 0 && parsed.needsCore.length === 0 ? "empty" : "ok";
|
|
}
|
|
|
|
// Reset the consecutive-failure counter on a successful (ok/empty) sync; bump
|
|
// it on error. Record the error timestamp only when there is an error.
|
|
const isErr = status === "error";
|
|
const newConsec = isErr ? (sub.consecutiveFailures || 0) + 1 : 0;
|
|
const errAt = isErr ? new Date().toISOString() : null;
|
|
await updateSubscriptionStatus(id, status, error, lastNodes, errAt, newConsec);
|
|
|
|
// Bind if enabled.
|
|
let applied = false;
|
|
let boundProxies = 0;
|
|
if (sub.enabled && keptIds.length > 0) {
|
|
await applySubscription(id);
|
|
applied = true;
|
|
boundProxies = keptIds.length;
|
|
}
|
|
|
|
// Ensure the background auto-refresh ticker is running once any subscription
|
|
// has actually synced. startSubscriptionScheduler is idempotent and a no-op
|
|
// in the browser and in NODE_ENV=test.
|
|
startSubscriptionScheduler();
|
|
|
|
return {
|
|
subscriptionId: id,
|
|
nodes: parsed.nodes.length,
|
|
needsCore: parsed.needsCore.length,
|
|
boundProxies,
|
|
status,
|
|
error,
|
|
applied,
|
|
};
|
|
}
|
|
|
|
// Deduplicate concurrent syncs for the same subscription. A manual refresh and
|
|
// the scheduled ticker can otherwise fire `syncSubscription` for the same id at
|
|
// the same time and race on the upsert / stale-removal writes.
|
|
const syncInFlight = new Map<string, Promise<SyncResult>>();
|
|
|
|
/** Public entry point: de-dupes concurrent syncs, then runs the unsafe body. */
|
|
export async function syncSubscription(id: string): Promise<SyncResult> {
|
|
const existing = syncInFlight.get(id);
|
|
if (existing) return existing;
|
|
const run = syncSubscriptionUnsafe(id).finally(() => {
|
|
syncInFlight.delete(id);
|
|
});
|
|
syncInFlight.set(id, run);
|
|
return run;
|
|
}
|
|
|
|
async function updateSubscriptionStatus(
|
|
id: string,
|
|
status: ProxySubscriptionStatus,
|
|
error: string | null,
|
|
lastNodes: unknown[] | null,
|
|
lastErrorAt: string | null,
|
|
consecutiveFailures: number
|
|
): Promise<void> {
|
|
const db = getDbInstance();
|
|
const now = new Date().toISOString();
|
|
db.prepare(
|
|
`UPDATE proxy_subscriptions SET status = ?, error = ?, last_nodes = ?, last_fetched_at = ?, last_error_at = ?, consecutive_failures = ?, updated_at = ? WHERE id = ?`
|
|
).run(
|
|
status,
|
|
error,
|
|
lastNodes ? JSON.stringify(lastNodes) : null,
|
|
now,
|
|
lastErrorAt,
|
|
consecutiveFailures,
|
|
now,
|
|
id
|
|
);
|
|
}
|
|
|
|
/** Bind the subscription's synced proxy pool into the target scope(s). */
|
|
export async function applySubscription(id: string): Promise<void> {
|
|
const sub = await getSubscriptionById(id);
|
|
if (!sub || !sub.enabled) return;
|
|
|
|
const db = getDbInstance();
|
|
const rows = db
|
|
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ? AND status != 'error'")
|
|
.all(id) as Array<{ id: string }>;
|
|
const ids = rows.map((r) => r.id);
|
|
if (ids.length === 0) return;
|
|
|
|
const targets = resolveTargetScopes(sub);
|
|
for (const t of targets) {
|
|
// Add the whole subscription pool to this scope in one batched, idempotent
|
|
// write (preserves any manual proxies already in the pool).
|
|
await addProxiesToScopePool(t.scope, t.scopeId, ids);
|
|
}
|
|
|
|
await setProxyEnabledFlag(true);
|
|
}
|
|
|
|
/** Remove the subscription's proxies from their bound scope(s). */
|
|
export async function unapplySubscription(id: string): Promise<void> {
|
|
const db = getDbInstance();
|
|
const rows = db
|
|
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
|
|
.all(id) as Array<{ id: string }>;
|
|
|
|
// Detach every subscription proxy from ALL scopes in one batched delete.
|
|
// Replaces the previous per-proxy getProxyWhereUsed + removeProxyFromScopePool
|
|
// loop (N+1 queries) — the subscription's proxies must leave every pool they
|
|
// were added to, regardless of which scope resolved them.
|
|
const ids = rows.map((r) => r.id);
|
|
if (ids.length > 0) {
|
|
const placeholders = ids.map(() => "?").join(",");
|
|
const res = db
|
|
.prepare(`DELETE FROM proxy_assignments WHERE proxy_id IN (${placeholders})`)
|
|
.run(...ids);
|
|
if (res.changes > 0) {
|
|
backupDbFile("pre-write");
|
|
bumpProxyRegistryGeneration();
|
|
}
|
|
}
|
|
|
|
await recomputeProxyEnabled();
|
|
}
|
|
|
|
// ───────────────────────────── proxyEnabled flag ─────────────────────────────
|
|
|
|
async function hasNonSubscriptionGlobalProxy(): Promise<boolean> {
|
|
const db = getDbInstance();
|
|
const row = db
|
|
.prepare(
|
|
"SELECT 1 FROM proxy_assignments a JOIN proxy_registry p ON p.id = a.proxy_id WHERE a.scope = 'global' AND (p.subscription_id IS NULL OR p.subscription_id = '') LIMIT 1"
|
|
)
|
|
.get();
|
|
return !!row;
|
|
}
|
|
|
|
async function recomputeProxyEnabled(): Promise<void> {
|
|
const db = getDbInstance();
|
|
// Must check for an ACTUALLY BOUND subscription proxy, not just an enabled
|
|
// subscription row: unapplySubscription() detaches proxy_assignments without
|
|
// touching `enabled`, so a bare `enabled = 1` check would leave the flag
|
|
// permanently stuck on `true` after an unapply/disable cycle even though
|
|
// nothing is bound anymore (resolveProxyForConnectionFromRegistry correctly
|
|
// returns null, but the operator-facing proxyEnabled flag would lie).
|
|
const enabledSubWithBoundProxy = db
|
|
.prepare(
|
|
`SELECT 1 FROM proxy_subscriptions s
|
|
JOIN proxy_registry p ON p.subscription_id = s.id
|
|
JOIN proxy_assignments a ON a.proxy_id = p.id
|
|
WHERE s.enabled = 1
|
|
LIMIT 1`
|
|
)
|
|
.get();
|
|
const shouldEnable = !!enabledSubWithBoundProxy || (await hasNonSubscriptionGlobalProxy());
|
|
await setProxyEnabledFlag(shouldEnable);
|
|
}
|
|
|
|
async function setProxyEnabledFlag(value: boolean): Promise<void> {
|
|
const db = getDbInstance();
|
|
db.prepare(
|
|
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('settings', 'proxyEnabled', ?)"
|
|
).run(JSON.stringify(value));
|
|
bumpProxyConfigGeneration();
|
|
}
|
|
|
|
// ───────────────────────────── Auto-refresh scheduler ─────────────────────────────
|
|
|
|
let schedulerStarted = false;
|
|
let schedulerTimer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
/** Start a background ticker that refreshes enabled subscriptions on their interval. */
|
|
export function startSubscriptionScheduler(): void {
|
|
if (schedulerStarted) return;
|
|
if (typeof window !== "undefined") return; // never in the browser
|
|
if (process.env.NODE_ENV === "test") return; // no timers during tests
|
|
schedulerStarted = true;
|
|
|
|
const tick = async () => {
|
|
try {
|
|
const subs = await listSubscriptions();
|
|
const now = Date.now();
|
|
for (const s of subs) {
|
|
if (!isSubscriptionDue(s, now)) continue;
|
|
try {
|
|
await syncSubscription(s.id);
|
|
} catch (e) {
|
|
console.warn(`[ProxySubscription] refresh failed for ${s.id}: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.warn(`[ProxySubscription] scheduler tick error: ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
};
|
|
|
|
// Check every minute; each subscription self-throttles by its interval.
|
|
schedulerTimer = setInterval(tick, 60_000);
|
|
if (typeof schedulerTimer.unref === "function") schedulerTimer.unref();
|
|
}
|
|
|
|
export function stopSubscriptionScheduler(): void {
|
|
if (schedulerTimer) {
|
|
clearInterval(schedulerTimer);
|
|
schedulerTimer = null;
|
|
}
|
|
schedulerStarted = false;
|
|
}
|