Files
OmniRoute/src/lib/proxySubscription/subscriptionService.ts
Adam f31f3c081e feat(proxy): operator-level proxy subscriptions (Karing-style) — hardened, ready for review (#7299)
* feat(proxy-subscriptions): src/lib/proxySubscription/parse.ts

* feat(proxy-subscriptions): src/lib/proxySubscription/subscriptionService.ts

* feat(proxy-subscriptions): src/lib/proxySubscription/index.ts

* feat(proxy-subscriptions): src/lib/db/migrations/123_proxy_subscriptions.sql

* feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/route.ts

* feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/route.ts

* feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/refresh/route.ts

* feat(proxy-subscriptions): src/app/api/v1/management/proxy-subscriptions/[id]/nodes/route.ts

* feat(proxy-subscriptions): src/app/(dashboard)/dashboard/settings/components/proxy/SubscriptionTab.tsx

* feat(proxy-subscriptions): tests/unit/proxySubscription.parse.test.ts

* feat(proxy-subscriptions): tests/unit/proxySubscription.service.test.ts

* feat(proxy-subscriptions): docs/proxy-subscriptions.md

* feat(proxy-subscriptions): src/lib/db/proxies/types.ts

* feat(proxy-subscriptions): src/lib/db/proxies/mappers.ts

* feat(proxy-subscriptions): src/lib/db/proxies.ts

* feat(proxy-subscriptions): src/app/(dashboard)/dashboard/settings/components/ProxyTab.tsx

* i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab

* i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab

* i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab

* i18n(proxy-subscriptions): add proxySubscriptionsTab key + use in ProxyTab

* test(proxy-subscriptions): extract isSubscriptionDue + unit tests

* test(proxy-subscriptions): extract isSubscriptionDue + unit tests

* test(proxy-subscriptions): extract isSubscriptionDue + unit tests

* test(proxy-subscriptions): add global->rule switch re-bind integration test

* feat(proxy-subscriptions): inline needs-local-core guidance in SubscriptionTab

* i18n: add proxySubscriptionsTab to en (rebased on current main)

* i18n: add proxySubscriptionsTab to zh-CN (rebased on current main)

* i18n: add proxySubscriptionsTab to pt-BR (rebased on current main)

* test(proxy-subscriptions): extract needsCore detection into pure module + unit tests

* test(proxy-subscriptions): extract needsCore detection into pure module + unit tests

* test(proxy-subscriptions): extract needsCore detection into pure module + unit tests

* refactor(proxy-subscriptions): extract scopes.ts into pure module + unit tests (#65)

* refactor(proxy-subscriptions): extract coreEndpoint.ts into pure module + unit tests (#65)

* refactor(proxy-subscriptions): extract subscriptionService.ts into pure module + unit tests (#65)

* refactor(proxy-subscriptions): extract proxySubscription.scopes.test.ts into pure module + unit tests (#65)

* refactor(proxy-subscriptions): extract proxySubscription.coreEndpoint.test.ts into pure module + unit tests (#65)

* security(proxy-subscriptions): fetchGuard.ts — SSRF guard + core scheme (#P0)

* security(proxy-subscriptions): coreEndpoint.ts — SSRF guard + core scheme (#P0)

* security(proxy-subscriptions): subscriptionService.ts — SSRF guard + core scheme (#P0)

* security(proxy-subscriptions): proxySubscription.fetchGuard.test.ts — SSRF guard + core scheme (#P0)

* security(proxy-subscriptions): proxySubscription.coreEndpoint.test.ts — SSRF guard + core scheme (#P0)

* refactor(proxy-subscriptions): subscriptionService.ts — concurrency lock / resilience / url redaction (#P1)

* refactor(proxy-subscriptions): url.ts — concurrency lock / resilience / url redaction (#P1)

* refactor(proxy-subscriptions): index.ts — concurrency lock / resilience / url redaction (#P1)

* refactor(proxy-subscriptions): route.ts — concurrency lock / resilience / url redaction (#P1)

* refactor(proxy-subscriptions): route.ts — concurrency lock / resilience / url redaction (#P1)

* refactor(proxy-subscriptions): proxySubscription.url.test.ts — concurrency lock / resilience / url redaction (#P1)

* i18n(proxy-subscriptions): subscriptionService.ts — stable error codes + locale keys (#P2-6)

* i18n(proxy-subscriptions): SubscriptionTab.tsx — stable error codes + locale keys (#P2-6)

* i18n(proxy-subscriptions): en.json — stable error codes + locale keys (#P2-6)

* i18n(proxy-subscriptions): zh-CN.json — stable error codes + locale keys (#P2-6)

* i18n(proxy-subscriptions): pt-BR.json — stable error codes + locale keys (#P2-6)

* i18n(proxy-subscriptions): en.json — normalize to LF line endings (#P2-6)

* i18n(proxy-subscriptions): zh-CN.json — normalize to LF line endings (#P2-6)

* i18n(proxy-subscriptions): pt-BR.json — normalize to LF line endings (#P2-6)

* enhance(proxy-subscriptions): reject subscription fetch if ANY resolved DNS address is blocked (P3-1)

* enhance(proxy-subscriptions): add withRetry() exponential-backoff helper (P3-2)

* enhance(proxy-subscriptions): DNS multi-record guard + retry/backoff fetch + batch scope writes + observability (P3-1..P3-4)

* enhance(proxy-subscriptions): batch addProxiesToScopePool() to drop N+1 writes (P3-3)

* enhance(proxy-subscriptions): add last_error_at + consecutive_failures observability columns (P3-4)

* enhance(proxy-subscriptions): surface consecutive failures + last error time in subscription cards (P3-4)

* test(proxy-subscriptions): cover multi-record DNS SSRF (block if ANY address internal) (P3-1)

* test(proxy-subscriptions): cover withRetry() first-success / retries / backoff / non-retryable stop (P3-2)

* fix(proxy-subscriptions): resolve js-yaml import + missing backup/generation-bump imports

Two bugs made the feature non-functional and its own test suite false:

1. parse.ts used `import yaml from "js-yaml"` (default import), but
   js-yaml@^5 is ESM-only with no default export — this threw a
   SyntaxError at module load, crashing every caller (index.ts
   re-exports parse.ts, so every API route hit this too). Switch to
   `import * as yaml`, matching how the rest of the codebase already
   imports js-yaml (hermes-agent.ts, openapiParser.ts, openapi/spec
   route.ts, guide-settings route.ts).

2. subscriptionService.ts's unapplySubscription() called backupDbFile()
   and bumpProxyRegistryGeneration() without importing either —
   backupDbFile exists but wasn't imported; bumpProxyRegistryGeneration
   was a private, non-exported function in db/proxies.ts. This threw a
   ReferenceError whenever a subscription with bound proxies was
   disabled/deleted (the normal path). Import backupDbFile from
   ../db/backup and export+import bumpProxyRegistryGeneration from
   ../db/proxies, matching the identical backup+bump pattern already
   used by deleteProxyById for the same proxy_assignments/proxy_registry
   mutation shape.

Fixing both unmasked a third, previously-unreachable bug (both crashes
happened before any test assertion could run): recomputeProxyEnabled()
checked `proxy_subscriptions.enabled = 1` alone, which stays true across
an unapply/disable cycle since unapplySubscription() never touches that
column — the proxyEnabled flag would get stuck on `true` even after the
subscription's proxies were fully detached. Changed the check to require
an actually-bound proxy_assignments row for an enabled subscription,
matching hasNonSubscriptionGlobalProxy()'s existing bound-check pattern.
Traced all 3 production call sites (mode/rule switch, disable, delete)
to confirm this doesn't change their outcome — only the previously-wrong
"unapply in isolation" case.

Also fixed the test file's own pre-existing bug: its provider_connections
inserts omitted created_at/updated_at (NOT NULL, no default in the
schema since 001_initial_schema.sql), which 0 assertions had ever
reached before because the SyntaxError always crashed the file first.

All 9 proxySubscription test files now run clean: 49/49 pass (previously
2 files crashed outright at import time, 0 assertions ever ran).

Separately confirmed via testing against the pristine PR head: this PR
has 3 more pre-existing gate failures unrelated to the above (file-size
on db/proxies.ts, cognitive-complexity, complexity, changelog-integrity
vs the current release tip) plus 3 pre-existing TS2345 errors in
parse.ts (lines 228/233/263, unrelated to the yaml import). All are the
PR's own scope/base-drift, out of scope for this fix — the PR has never
had a real CI run (base=main), so no gate has ever surfaced them; they
need the base retarget + a full CI pass called out in the plan file's
own remaining mandatory items.

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

* fix(db): renumber proxy-subscriptions migrations to avoid version collision

release/v3.8.49 already ships 123_quota_auto_ping.sql and
124_generic_session_affinity_ttl.sql; this PR's 123/124 files collided,
tripping migrationRunner's version-collision guard and failing all
proxySubscription.service tests. Renumber to 127/128 (next free slots
after 126_reasoning_routing_rules.sql).

Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com>

* refactor(db): extract proxySubscriptions + registryGeneration to keep proxies.ts under file-size cap

Moves addProxiesToScopePool to ./proxySubscriptions.ts and the registry-generation
helpers to ./proxies/registryGeneration.ts (re-exported from proxies.ts), keeping the
module under its frozen 1177-line cap after the operator-proxy-subscriptions feature.

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

* fix(db): renumber proxy-subscriptions migrations past release collision

Rebasing onto release/v3.8.49 surfaced a migration version collision: this
branch's 127_proxy_subscriptions.sql and 128_proxy_subscriptions_meta.sql
now collide with 127_usage_history_account_identity.sql and
128_auto_candidate_overrides.sql that landed on release since this branch
last synced. Renumbered to 131/132 (next free prefixes after the current
130_remove_unregistered_qwen_data.sql) and updated the in-file header
comments to match. No schema/behavior change.

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

---------

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouza.pw@gmail.com>
Co-authored-by: Diego Rodrigues de Sa e Souza <diegosouzapw@users.noreply.github.com>
Co-authored-by: xier2012 <xier2012@users.noreply.github.com>
2026-07-21 13:16:41 -03:00

701 lines
27 KiB
TypeScript

/**
* Proxy subscription service (Karing-style, operator-supplied).
*
* A subscription is a URL the operator pastes. We fetch + parse it into a pool
* of proxy nodes, sync those nodes into `proxy_registry` (source='subscription',
* subscription_id set), and bind the pool through the EXISTING scope resolution
* (account/provider/global) — so subscriptions inherit rotation, health checks,
* and the fail-closed guard for free.
*
* Modes:
* - 'global': pool bound to the global scope (all provider traffic proxied).
* - 'rule': pool bound only to the selected provider scopes (others direct).
*
* Protocol support:
* - http/https/socks5 nodes are used directly.
* - ss/vmess/vless/trojan/tuic/hysteria/wireguard nodes need a local proxy
* core (sing-box/clash) exposing a SOCKS5/HTTP endpoint; supply it via
* `localCoreEndpoint` and we bind that single endpoint (the core does the
* protocol translation + node selection). Without it, those nodes are
* reported but not routed.
*/
import { randomUUID } from "crypto";
import { getDbInstance } from "../db/core";
import { backupDbFile } from "../db/backup";
import {
addProxiesToScopePool,
bumpProxyRegistryGeneration,
deleteProxyById,
upsertProxy,
} from "../db/proxies";
import { bumpProxyConfigGeneration } from "../db/settings";
import { isSubscriptionDue } from "./due";
import { isLocalCoreEndpointAllowed } from "./coreEndpoint";
import { resolveTargetScopes } from "./scopes";
import {
isSubscriptionFetchUrlAllowed,
isIpLiteral,
isAnyResolvedAddressBlocked,
} from "./fetchGuard";
import { withRetry } from "./fetchRetry";
import { parseSubscription, redactedNodeSummary, type ParsedSubscription } from "./parse";
export type ProxySubscriptionMode = "global" | "rule";
export type ProxySubscriptionStatus = "ok" | "error" | "empty";
/** Stable, language-neutral error codes stored in the subscription `error`
* column (as JSON) so the dashboard can localize them via i18n instead of
* showing server-side strings. */
export type ProxySubscriptionErrorCode =
| "LOCAL_CORE_ENDPOINT_INVALID"
| "NEEDS_CORE_NOT_CONFIGURED"
| "NO_USABLE_NODES";
/** Encode a user-facing error as `{ code, detail? }` for i18n on the client. */
export function subscriptionErrorCode(code: ProxySubscriptionErrorCode, detail?: string): string {
return JSON.stringify(detail ? { code, detail } : { code });
}
export interface ProxySubscriptionRecord {
id: string;
name: string;
url: string;
enabled: boolean;
mode: ProxySubscriptionMode;
ruleProviders: string[] | null;
localCoreEndpoint: string | null;
updateIntervalMinutes: number;
lastFetchedAt: string | null;
status: ProxySubscriptionStatus;
error: string | null;
lastNodes: unknown[] | null;
lastErrorAt: string | null;
consecutiveFailures: number;
createdAt: string;
updatedAt: string;
}
export interface ProxySubscriptionPayload {
name: string;
url: string;
enabled?: boolean;
mode?: ProxySubscriptionMode;
ruleProviders?: string[] | null;
localCoreEndpoint?: string | null;
updateIntervalMinutes?: number;
}
export interface SyncResult {
subscriptionId: string;
nodes: number;
needsCore: number;
boundProxies: number;
status: ProxySubscriptionStatus;
error: string | null;
applied: boolean;
}
const SUBSCRIPTION_FETCH_TIMEOUT_MS = 15_000;
// ───────────────────────────── Row mapping ─────────────────────────────
function mapSubscriptionRow(row: unknown): ProxySubscriptionRecord {
const r = (row && typeof row === "object" ? row : {}) as Record<string, unknown>;
const parseList = (v: unknown): string[] | null => {
if (typeof v !== "string" || !v.trim()) return null;
try {
const arr = JSON.parse(v);
return Array.isArray(arr) ? arr.filter((x) => typeof x === "string") : null;
} catch {
return null;
}
};
const parseNodes = (v: unknown): unknown[] | null => {
if (typeof v !== "string" || !v.trim()) return null;
try {
const arr = JSON.parse(v);
return Array.isArray(arr) ? arr : null;
} catch {
return null;
}
};
return {
id: typeof r.id === "string" ? r.id : "",
name: typeof r.name === "string" ? r.name : "",
url: typeof r.url === "string" ? r.url : "",
enabled: Number(r.enabled) !== 0,
mode: r.mode === "rule" ? "rule" : "global",
ruleProviders: parseList(r.rule_providers),
localCoreEndpoint: typeof r.local_core_endpoint === "string" ? r.local_core_endpoint : null,
updateIntervalMinutes: Number(r.update_interval_minutes) || 60,
lastFetchedAt: typeof r.last_fetched_at === "string" ? r.last_fetched_at : null,
status: (r.status as ProxySubscriptionStatus) || "empty",
error: typeof r.error === "string" ? r.error : null,
lastNodes: parseNodes(r.last_nodes),
lastErrorAt: typeof r.last_error_at === "string" ? r.last_error_at : null,
consecutiveFailures: Number(r.consecutive_failures) || 0,
createdAt: typeof r.created_at === "string" ? r.created_at : "",
updatedAt: typeof r.updated_at === "string" ? r.updated_at : "",
};
}
// ───────────────────────────── CRUD ─────────────────────────────
export async function listSubscriptions(): Promise<ProxySubscriptionRecord[]> {
const db = getDbInstance();
const rows = db
.prepare(
"SELECT id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, last_fetched_at, status, error, last_nodes, last_error_at, consecutive_failures, created_at, updated_at FROM proxy_subscriptions ORDER BY datetime(updated_at) DESC, name ASC"
)
.all();
return rows.map(mapSubscriptionRow);
}
export async function getSubscriptionById(id: string): Promise<ProxySubscriptionRecord | null> {
const db = getDbInstance();
const row = db
.prepare(
"SELECT id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, last_fetched_at, status, error, last_nodes, last_error_at, consecutive_failures, created_at, updated_at FROM proxy_subscriptions WHERE id = ?"
)
.get(id);
return row ? mapSubscriptionRow(row) : null;
}
export async function createSubscription(
payload: ProxySubscriptionPayload
): Promise<ProxySubscriptionRecord> {
const id = randomUUID();
const now = new Date().toISOString();
const enabled = payload.enabled === true ? 1 : 0;
const db = getDbInstance();
db.prepare(
`INSERT INTO proxy_subscriptions
(id, name, url, enabled, mode, rule_providers, local_core_endpoint, update_interval_minutes, status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'empty', ?, ?)`
).run(
id,
payload.name,
payload.url,
enabled,
payload.mode || "global",
payload.ruleProviders ? JSON.stringify(payload.ruleProviders) : null,
payload.localCoreEndpoint || null,
payload.updateIntervalMinutes || 60,
now,
now
);
const created = (await getSubscriptionById(id))!;
if (created.enabled) {
await syncSubscription(id);
}
// Re-read so the returned record reflects the post-sync status/error/lastNodes.
return (await getSubscriptionById(id))!;
}
export async function updateSubscription(
id: string,
payload: Partial<ProxySubscriptionPayload>
): Promise<ProxySubscriptionRecord | null> {
const existing = await getSubscriptionById(id);
if (!existing) return null;
const db = getDbInstance();
const name = payload.name ?? existing.name;
const url = payload.url ?? existing.url;
const mode = payload.mode ?? existing.mode;
const ruleProviders = payload.ruleProviders !== undefined ? payload.ruleProviders : existing.ruleProviders;
const localCoreEndpoint =
payload.localCoreEndpoint !== undefined ? payload.localCoreEndpoint : existing.localCoreEndpoint;
const updateIntervalMinutes = payload.updateIntervalMinutes ?? existing.updateIntervalMinutes;
const now = new Date().toISOString();
const enabledChanged = payload.enabled !== undefined && payload.enabled !== existing.enabled;
const enabled = payload.enabled !== undefined ? (payload.enabled ? 1 : 0) : existing.enabled ? 1 : 0;
db.prepare(
`UPDATE proxy_subscriptions
SET name = ?, url = ?, enabled = ?, mode = ?, rule_providers = ?, local_core_endpoint = ?, update_interval_minutes = ?, updated_at = ?
WHERE id = ?`
).run(
name,
url,
enabled,
mode,
ruleProviders ? JSON.stringify(ruleProviders) : null,
localCoreEndpoint || null,
updateIntervalMinutes,
now,
id
);
const updated = (await getSubscriptionById(id))!;
// Re-evaluate binding.
if (updated.enabled) {
if (payload.mode !== undefined || payload.ruleProviders !== undefined) {
// Routing targets changed: detach from the old scopes first, then
// re-fetch + bind to the new targets. applySubscription is idempotent per
// scope, but a global→rule switch must drop the previous global binding.
await unapplySubscription(id);
await syncSubscription(id);
} else if (
enabledChanged ||
payload.url !== undefined ||
payload.localCoreEndpoint !== undefined
) {
// Same scopes: just refresh nodes (re-applies idempotently to same scopes).
await syncSubscription(id);
}
} else {
// Disabled: detach everything.
await unapplySubscription(id);
}
return getSubscriptionById(id);
}
export async function setSubscriptionEnabled(id: string, enabled: boolean): Promise<ProxySubscriptionRecord | null> {
return updateSubscription(id, { enabled });
}
export async function deleteSubscription(id: string): Promise<boolean> {
await unapplySubscription(id);
// Remove subscription-sourced proxy rows (force-clears their assignments).
const db = getDbInstance();
const rows = db
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
.all(id) as Array<{ id: string }>;
for (const r of rows) {
try {
await deleteProxyById(r.id, { force: true });
} catch {
// ignore individual failures
}
}
const res = db.prepare("DELETE FROM proxy_subscriptions WHERE id = ?").run(id);
await recomputeProxyEnabled();
return res.changes > 0;
}
// ───────────────────────────── Scope resolution ─────────────────────────────
// ───────────────────────────── Sync + apply ─────────────────────────────
/**
* Refuse to fetch a subscription URL unless it is http/https to a non-internal
* host. IP literals are checked structurally; hostnames are resolved and the
* resolved addresses are re-checked (fail closed on resolution errors). This
* blocks SSRF to internal services / cloud metadata (169.254.169.254).
*/
async function assertSafeFetchTarget(url: string): Promise<void> {
if (!isSubscriptionFetchUrlAllowed(url)) {
throw new Error("Subscription URL is not allowed (scheme or host blocked)");
}
const host = new URL(url).hostname.toLowerCase();
const bare = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
if (!isIpLiteral(bare)) {
// Hostname: resolve ALL records and refuse if ANY address is internal
// (fail closed). A hostname can resolve to multiple records; checking only
// the first would let an internal IP slip through if a public record also
// exists. `lookup(..., { all: true })` returns every A/AAAA record.
try {
const dns = await import("node:dns");
const addrs = await dns.promises.lookup(bare, { all: true });
if (isAnyResolvedAddressBlocked(addrs)) {
throw new Error("Subscription host resolves to a blocked (internal) address");
}
} catch (e) {
if (e instanceof Error && e.message.includes("blocked")) throw e;
throw new Error(`Subscription host resolution failed: ${e instanceof Error ? e.message : e}`);
}
}
}
/**
* Single fetch attempt: SSRF-validate the target, fetch with manual redirect
* handling, and re-validate any redirect target. Throws on non-2xx or a
* blocked target. Each attempt owns its own timeout so a retry after a fast
* failure isn't killed by the previous attempt's timer.
*/
async function doSafeFetch(url: string, headers: Record<string, string>): Promise<string> {
await assertSafeFetchTarget(url);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), SUBSCRIPTION_FETCH_TIMEOUT_MS);
try {
const res = await fetch(url, { redirect: "manual", signal: controller.signal, headers });
if (res.status >= 300 && res.status < 400) {
const loc = res.headers.get("location");
if (!loc) throw new Error("Subscription fetch redirected without Location");
// Resolve relative redirects and re-validate the target (SSRF guard).
const next = new URL(loc, url).toString();
await assertSafeFetchTarget(next);
const res2 = await fetch(next, { redirect: "manual", signal: controller.signal, headers });
if (res2.status >= 300 && res2.status < 400) {
throw new Error("Subscription fetch: too many redirects");
}
if (!res2.ok) {
throw new Error(`Subscription fetch failed: HTTP ${res2.status}`);
}
return await res2.text();
}
if (!res.ok) {
throw new Error(`Subscription fetch failed: HTTP ${res.status}`);
}
return await res.text();
} finally {
clearTimeout(timeout);
}
}
/**
* Classify a fetch error as retryable. Transient (retry): network/timeout/DNS
* failures, HTTP 5xx, and HTTP 429. Permanent (no retry): 4xx client errors
* (except 429) and any SSRF-guard block — those will never succeed on retry.
*/
function isSubscriptionFetchRetryable(e: unknown): boolean {
const msg = e instanceof Error ? e.message : String(e);
if (msg.includes("not allowed") || msg.includes("blocked (internal)")) return false;
const m = msg.match(/HTTP (\d{3})/);
if (m) {
const code = Number(m[1]);
if (code === 429) return true;
if (code >= 500 && code < 600) return true;
return false; // 4xx (except 429) — permanent client error
}
return true; // network error / timeout / DNS failure — transient
}
async function fetchSubscriptionContent(url: string): Promise<string> {
const headers = { "User-Agent": "OmniRoute-ProxySubscription" };
// Retry transient failures (timeouts, 5xx, 429) with bounded exponential
// backoff; give up fast on permanent errors (4xx, SSRF block).
return withRetry(() => doSafeFetch(url, headers), {
maxAttempts: 3,
baseDelayMs: 500,
maxDelayMs: 5000,
isRetryable: isSubscriptionFetchRetryable,
});
}
/** Fetch + parse + sync nodes into proxy_registry, then (if enabled) (re)bind. */
async function syncSubscriptionUnsafe(id: string): Promise<SyncResult> {
const sub = await getSubscriptionById(id);
if (!sub) {
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: "not found", applied: false };
}
let body: string;
try {
body = await fetchSubscriptionContent(sub.url);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
const fetchConsec = (sub.consecutiveFailures || 0) + 1;
await updateSubscriptionStatus(id, "error", `Fetch failed: ${msg}`, null, new Date().toISOString(), fetchConsec);
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: msg, applied: false };
}
const parsed: ParsedSubscription = parseSubscription(body);
const db = getDbInstance();
const keptIds: string[] = [];
let warning: string | null = null;
// The upsert loop + stale-removal are the multi-write section. upsertProxy /
// deleteProxyById each run their own internal db.transaction, so each write
// is atomic, and a re-sync is idempotent (self-heals partial state). A single
// outer db.transaction around `await` calls would NOT be atomic under
// better-sqlite3, so instead we guard against an unexpected DB error so a
// half-completed sync can never be left flagged "ok".
try {
// Directly-usable nodes → upsert into the registry as a pool.
for (const node of parsed.nodes) {
const upserted = await upsertProxy({
name: node.name || `${sub.name} (${node.host}:${node.port})`,
type: node.type,
host: node.host,
port: node.port,
username: node.username,
password: node.password,
source: "subscription",
subscriptionId: id,
status: "active",
});
if (upserted.proxy?.id) keptIds.push(upserted.proxy.id);
}
// needsCore nodes → bind the operator-supplied local core endpoint (single).
if (parsed.needsCore.length > 0) {
if (sub.localCoreEndpoint && isLocalCoreEndpointAllowed(sub.localCoreEndpoint)) {
try {
const coreUrl = new URL(sub.localCoreEndpoint);
const coreType = coreUrl.protocol === "https:" ? "https" : coreUrl.protocol === "socks5:" ? "socks5" : "http";
const upserted = await upsertProxy({
name: `${sub.name} (local core)`,
type: coreType,
host: coreUrl.hostname,
port: Number(coreUrl.port) || (coreType === "https" ? 443 : 8080),
username: coreUrl.username ? decodeURIComponent(coreUrl.username) : undefined,
password: coreUrl.password ? decodeURIComponent(coreUrl.password) : undefined,
source: "subscription",
subscriptionId: id,
status: "active",
});
if (upserted.proxy?.id) keptIds.push(upserted.proxy.id);
} catch {
warning = subscriptionErrorCode("LOCAL_CORE_ENDPOINT_INVALID");
}
} else {
const nodes = parsed.needsCore
.map((n) => `${n.rawProtocol}://${n.host ?? ""}${n.port ? ":" + n.port : ""}`)
.join(", ");
warning = subscriptionErrorCode("NEEDS_CORE_NOT_CONFIGURED", nodes);
}
}
// Remove stale subscription nodes no longer present in the fetched set.
if (keptIds.length > 0) {
const placeholders = keptIds.map(() => "?").join(",");
const stale = db
.prepare(`SELECT id FROM proxy_registry WHERE subscription_id = ? AND id NOT IN (${placeholders})`)
.all(id, ...keptIds) as Array<{ id: string }>;
for (const r of stale) {
try {
await deleteProxyById(r.id, { force: true });
} catch {
// ignore
}
}
} else {
const stale = db
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
.all(id) as Array<{ id: string }>;
for (const r of stale) {
try {
await deleteProxyById(r.id, { force: true });
} catch {
// ignore
}
}
}
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
const writeConsec = (sub.consecutiveFailures || 0) + 1;
await updateSubscriptionStatus(id, "error", `Sync write failed: ${msg}`, null, new Date().toISOString(), writeConsec);
return { subscriptionId: id, nodes: 0, needsCore: 0, boundProxies: 0, status: "error", error: msg, applied: false };
}
const lastNodes = redactedNodeSummary(parsed);
// Determine status.
let status: ProxySubscriptionStatus;
let error: string | null = warning;
if (keptIds.length === 0) {
status = "error";
error = warning || subscriptionErrorCode("NO_USABLE_NODES");
} else if (warning) {
status = "ok";
} else {
status = parsed.nodes.length === 0 && parsed.needsCore.length === 0 ? "empty" : "ok";
}
// Reset the consecutive-failure counter on a successful (ok/empty) sync; bump
// it on error. Record the error timestamp only when there is an error.
const isErr = status === "error";
const newConsec = isErr ? (sub.consecutiveFailures || 0) + 1 : 0;
const errAt = isErr ? new Date().toISOString() : null;
await updateSubscriptionStatus(id, status, error, lastNodes, errAt, newConsec);
// Bind if enabled.
let applied = false;
let boundProxies = 0;
if (sub.enabled && keptIds.length > 0) {
await applySubscription(id);
applied = true;
boundProxies = keptIds.length;
}
// Ensure the background auto-refresh ticker is running once any subscription
// has actually synced. startSubscriptionScheduler is idempotent and a no-op
// in the browser and in NODE_ENV=test.
startSubscriptionScheduler();
return {
subscriptionId: id,
nodes: parsed.nodes.length,
needsCore: parsed.needsCore.length,
boundProxies,
status,
error,
applied,
};
}
// Deduplicate concurrent syncs for the same subscription. A manual refresh and
// the scheduled ticker can otherwise fire `syncSubscription` for the same id at
// the same time and race on the upsert / stale-removal writes.
const syncInFlight = new Map<string, Promise<SyncResult>>();
/** Public entry point: de-dupes concurrent syncs, then runs the unsafe body. */
export async function syncSubscription(id: string): Promise<SyncResult> {
const existing = syncInFlight.get(id);
if (existing) return existing;
const run = syncSubscriptionUnsafe(id).finally(() => {
syncInFlight.delete(id);
});
syncInFlight.set(id, run);
return run;
}
async function updateSubscriptionStatus(
id: string,
status: ProxySubscriptionStatus,
error: string | null,
lastNodes: unknown[] | null,
lastErrorAt: string | null,
consecutiveFailures: number
): Promise<void> {
const db = getDbInstance();
const now = new Date().toISOString();
db.prepare(
`UPDATE proxy_subscriptions SET status = ?, error = ?, last_nodes = ?, last_fetched_at = ?, last_error_at = ?, consecutive_failures = ?, updated_at = ? WHERE id = ?`
).run(
status,
error,
lastNodes ? JSON.stringify(lastNodes) : null,
now,
lastErrorAt,
consecutiveFailures,
now,
id
);
}
/** Bind the subscription's synced proxy pool into the target scope(s). */
export async function applySubscription(id: string): Promise<void> {
const sub = await getSubscriptionById(id);
if (!sub || !sub.enabled) return;
const db = getDbInstance();
const rows = db
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ? AND status != 'error'")
.all(id) as Array<{ id: string }>;
const ids = rows.map((r) => r.id);
if (ids.length === 0) return;
const targets = resolveTargetScopes(sub);
for (const t of targets) {
// Add the whole subscription pool to this scope in one batched, idempotent
// write (preserves any manual proxies already in the pool).
await addProxiesToScopePool(t.scope, t.scopeId, ids);
}
await setProxyEnabledFlag(true);
}
/** Remove the subscription's proxies from their bound scope(s). */
export async function unapplySubscription(id: string): Promise<void> {
const db = getDbInstance();
const rows = db
.prepare("SELECT id FROM proxy_registry WHERE subscription_id = ?")
.all(id) as Array<{ id: string }>;
// Detach every subscription proxy from ALL scopes in one batched delete.
// Replaces the previous per-proxy getProxyWhereUsed + removeProxyFromScopePool
// loop (N+1 queries) — the subscription's proxies must leave every pool they
// were added to, regardless of which scope resolved them.
const ids = rows.map((r) => r.id);
if (ids.length > 0) {
const placeholders = ids.map(() => "?").join(",");
const res = db
.prepare(`DELETE FROM proxy_assignments WHERE proxy_id IN (${placeholders})`)
.run(...ids);
if (res.changes > 0) {
backupDbFile("pre-write");
bumpProxyRegistryGeneration();
}
}
await recomputeProxyEnabled();
}
// ───────────────────────────── proxyEnabled flag ─────────────────────────────
async function hasNonSubscriptionGlobalProxy(): Promise<boolean> {
const db = getDbInstance();
const row = db
.prepare(
"SELECT 1 FROM proxy_assignments a JOIN proxy_registry p ON p.id = a.proxy_id WHERE a.scope = 'global' AND (p.subscription_id IS NULL OR p.subscription_id = '') LIMIT 1"
)
.get();
return !!row;
}
async function recomputeProxyEnabled(): Promise<void> {
const db = getDbInstance();
// Must check for an ACTUALLY BOUND subscription proxy, not just an enabled
// subscription row: unapplySubscription() detaches proxy_assignments without
// touching `enabled`, so a bare `enabled = 1` check would leave the flag
// permanently stuck on `true` after an unapply/disable cycle even though
// nothing is bound anymore (resolveProxyForConnectionFromRegistry correctly
// returns null, but the operator-facing proxyEnabled flag would lie).
const enabledSubWithBoundProxy = db
.prepare(
`SELECT 1 FROM proxy_subscriptions s
JOIN proxy_registry p ON p.subscription_id = s.id
JOIN proxy_assignments a ON a.proxy_id = p.id
WHERE s.enabled = 1
LIMIT 1`
)
.get();
const shouldEnable = !!enabledSubWithBoundProxy || (await hasNonSubscriptionGlobalProxy());
await setProxyEnabledFlag(shouldEnable);
}
async function setProxyEnabledFlag(value: boolean): Promise<void> {
const db = getDbInstance();
db.prepare(
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('settings', 'proxyEnabled', ?)"
).run(JSON.stringify(value));
bumpProxyConfigGeneration();
}
// ───────────────────────────── Auto-refresh scheduler ─────────────────────────────
let schedulerStarted = false;
let schedulerTimer: ReturnType<typeof setInterval> | null = null;
/** Start a background ticker that refreshes enabled subscriptions on their interval. */
export function startSubscriptionScheduler(): void {
if (schedulerStarted) return;
if (typeof window !== "undefined") return; // never in the browser
if (process.env.NODE_ENV === "test") return; // no timers during tests
schedulerStarted = true;
const tick = async () => {
try {
const subs = await listSubscriptions();
const now = Date.now();
for (const s of subs) {
if (!isSubscriptionDue(s, now)) continue;
try {
await syncSubscription(s.id);
} catch (e) {
console.warn(`[ProxySubscription] refresh failed for ${s.id}: ${e instanceof Error ? e.message : e}`);
}
}
} catch (e) {
console.warn(`[ProxySubscription] scheduler tick error: ${e instanceof Error ? e.message : e}`);
}
};
// Check every minute; each subscription self-throttles by its interval.
schedulerTimer = setInterval(tick, 60_000);
if (typeof schedulerTimer.unref === "function") schedulerTimer.unref();
}
export function stopSubscriptionScheduler(): void {
if (schedulerTimer) {
clearInterval(schedulerTimer);
schedulerTimer = null;
}
schedulerStarted = false;
}