mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-16 03:42:21 +03:00
892 lines
30 KiB
TypeScript
892 lines
30 KiB
TypeScript
/**
|
|
* Shared browser-impersonating HTTP transport for the six web-cookie providers.
|
|
*
|
|
* Provider wrappers keep their existing `tlsFetch*` APIs while this module owns
|
|
* wreq-js loading, transport pooling, proxy selection, deadlines, byte responses,
|
|
* SSE/NDJSON validation, EOF handling, and cancellation.
|
|
*
|
|
* Every wreq request uses an ephemeral cookie scope. Transports are reused only
|
|
* within one provider module and are keyed by browser profile, emulated OS, and
|
|
* resolved proxy; no cookie jar or session identifier is shared between calls.
|
|
*/
|
|
|
|
import { createRequire } from "node:module";
|
|
|
|
import { resolveProxyForRequest } from "../utils/proxyFetch.ts";
|
|
import { resolveTlsClientProxyUrl } from "./tlsClientProxy.ts";
|
|
|
|
const runtimeRequire = createRequire(import.meta.url);
|
|
const transportClosers = new Set<() => void>();
|
|
let exitHookInstalled = false;
|
|
|
|
type EmulationOs = "windows" | "macos" | "linux" | "android" | "ios";
|
|
|
|
export type IterableHeaders = Iterable<[string, string]> & {
|
|
getSetCookie?: () => string[];
|
|
};
|
|
|
|
export interface ReadableBodyLike {
|
|
getReader: () => ReadableStreamDefaultReader<Uint8Array>;
|
|
cancel?: (reason?: unknown) => Promise<void>;
|
|
}
|
|
|
|
export interface TlsResponseLike {
|
|
status: number;
|
|
headers: Record<string, string[]> | IterableHeaders;
|
|
body: string | ReadableBodyLike | null;
|
|
text?: () => Promise<string>;
|
|
bytes?: () => Promise<Uint8Array>;
|
|
}
|
|
|
|
interface WreqTransportLike {
|
|
close: () => Promise<void> | void;
|
|
}
|
|
|
|
export interface WreqRuntimeLike {
|
|
createTransport: (options: Record<string, unknown>) => Promise<WreqTransportLike>;
|
|
fetch: (url: string, options: Record<string, unknown>) => Promise<TlsResponseLike>;
|
|
}
|
|
|
|
export type WreqRuntimeLoader = () => Promise<WreqRuntimeLike>;
|
|
|
|
export interface TlsFetchResult {
|
|
status: number;
|
|
headers: Headers;
|
|
text: string | null;
|
|
body: ReadableStream<Uint8Array> | null;
|
|
}
|
|
|
|
export interface TlsFetchOptions {
|
|
method?: string;
|
|
headers?: Record<string, string>;
|
|
body?: string;
|
|
signal?: AbortSignal;
|
|
timeoutMs?: number;
|
|
stream?: boolean;
|
|
streamEofSymbol?: string;
|
|
byteResponse?: boolean;
|
|
proxyUrl?: string;
|
|
}
|
|
|
|
export interface TlsClientConfig {
|
|
/** Human-readable provider name for logs and error messages. */
|
|
providerName: string;
|
|
/** Browser profile identifier, for example `chrome_146` or `firefox_148`. */
|
|
tlsProfile: string;
|
|
/** Operating system paired with the browser profile. */
|
|
emulationOs?: EmulationOs;
|
|
/** Default upstream domain used by proxy resolution. */
|
|
domain: string;
|
|
/** @deprecated wreq-js streams directly and ignores this compatibility field. */
|
|
tempDirPrefix?: string;
|
|
/** Default EOF marker. An empty string disables marker filtering. */
|
|
streamEofSymbol?: string;
|
|
/** Native request timeout in milliseconds. */
|
|
defaultTimeoutMs?: number;
|
|
/** Additional JavaScript-side hard-timeout grace period. */
|
|
hardTimeoutGraceMs?: number;
|
|
/** Delay after which a late first byte is returned as a buffered response. */
|
|
firstByteTimeoutMs?: number;
|
|
/** How a detected EOF marker is exposed; `none` disables marker filtering. */
|
|
streamEofPolicy?: "include" | "exclude" | "none";
|
|
/** @deprecated Compatibility alias: `A` includes EOF; `B1`/`B2` exclude it. */
|
|
tailFileVariant?: "A" | "B1" | "B2";
|
|
/** `sse` validates SSE prefixes; `cf` rejects Cloudflare/HTML responses. */
|
|
responseValidation: "sse" | "cf";
|
|
/** Optional proxy-resolution domain override (LMArena uses arena.ai). */
|
|
proxyDomainOverride?: string;
|
|
/** Whether the provider module exposes the Cloudflare detection helper. */
|
|
exportCloudflareCheck: boolean;
|
|
/** Whether to expose the direct-stream dependency-injection seam. */
|
|
exposeStreamingForTesting?: boolean;
|
|
/** External-runtime seam used by focused tests; production loads wreq-js lazily. */
|
|
wreqRuntimeLoader?: WreqRuntimeLoader;
|
|
}
|
|
|
|
export class TlsClientUnavailableError extends Error {
|
|
override name = "TlsClientUnavailableError";
|
|
}
|
|
|
|
export class TlsClientHangError extends Error {
|
|
override name = "TlsClientHangError";
|
|
}
|
|
|
|
export function makeAbortError(signal: AbortSignal): Error {
|
|
const reason = signal.reason;
|
|
if (reason instanceof Error) return reason;
|
|
const error = new Error(typeof reason === "string" ? reason : "The operation was aborted");
|
|
error.name = "AbortError";
|
|
return error;
|
|
}
|
|
|
|
export function toHeaders(
|
|
raw: Record<string, string[]> | IterableHeaders | null | undefined
|
|
): Headers {
|
|
const headers = new Headers();
|
|
if (!raw) return headers;
|
|
|
|
const iterator = (raw as Partial<IterableHeaders>)[Symbol.iterator];
|
|
if (typeof iterator === "function") {
|
|
const iterable = raw as IterableHeaders;
|
|
const setCookies = typeof iterable.getSetCookie === "function" ? iterable.getSetCookie() : [];
|
|
for (const [name, value] of iterable) {
|
|
if (name.toLowerCase() !== "set-cookie" || setCookies.length === 0) {
|
|
headers.append(name, value);
|
|
}
|
|
}
|
|
for (const value of setCookies) headers.append("set-cookie", value);
|
|
return headers;
|
|
}
|
|
|
|
for (const [name, values] of Object.entries(raw)) {
|
|
for (const value of values) headers.append(name, value);
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
function isReadableBody(body: TlsResponseLike["body"]): body is ReadableBodyLike {
|
|
return body !== null && typeof body !== "string" && typeof body.getReader === "function";
|
|
}
|
|
|
|
function concatChunks(chunks: Uint8Array[]): Uint8Array {
|
|
const length = chunks.reduce((total, chunk) => total + chunk.byteLength, 0);
|
|
const combined = new Uint8Array(length);
|
|
let offset = 0;
|
|
for (const chunk of chunks) {
|
|
combined.set(chunk, offset);
|
|
offset += chunk.byteLength;
|
|
}
|
|
return combined;
|
|
}
|
|
|
|
async function readAllChunks(
|
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
initialChunks: Uint8Array[] = [],
|
|
readNext: () => Promise<ReadableStreamReadResult<Uint8Array>> = () => reader.read()
|
|
): Promise<Uint8Array> {
|
|
const chunks = [...initialChunks];
|
|
while (true) {
|
|
const next = await readNext();
|
|
if (next.done) return concatChunks(chunks);
|
|
chunks.push(next.value);
|
|
}
|
|
}
|
|
|
|
async function readTlsResponseText(
|
|
response: TlsResponseLike,
|
|
onReader?: (reader: ReadableStreamDefaultReader<Uint8Array>) => void
|
|
): Promise<string> {
|
|
if (typeof response.body === "string") return response.body;
|
|
if (isReadableBody(response.body)) {
|
|
const reader = response.body.getReader();
|
|
onReader?.(reader);
|
|
return new TextDecoder().decode(await readAllChunks(reader));
|
|
}
|
|
if (typeof response.text === "function") return response.text();
|
|
return "";
|
|
}
|
|
|
|
async function readTlsResponseBytes(
|
|
response: TlsResponseLike,
|
|
onReader?: (reader: ReadableStreamDefaultReader<Uint8Array>) => void
|
|
): Promise<Uint8Array> {
|
|
if (isReadableBody(response.body)) {
|
|
const reader = response.body.getReader();
|
|
onReader?.(reader);
|
|
return readAllChunks(reader);
|
|
}
|
|
if (typeof response.bytes === "function") return response.bytes();
|
|
if (typeof response.body === "string") return Buffer.from(response.body, "binary");
|
|
return new Uint8Array();
|
|
}
|
|
|
|
export async function raceWithTimeout<T>(
|
|
promise: Promise<T>,
|
|
timeoutMs: number,
|
|
signal: AbortSignal | null | undefined
|
|
): Promise<T> {
|
|
return await new Promise<T>((resolve, reject) => {
|
|
let settled = false;
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
|
|
const onAbort = (): void => {
|
|
settle(() => reject(makeAbortError(signal!)));
|
|
};
|
|
const cleanup = (): void => {
|
|
if (timer) clearTimeout(timer);
|
|
signal?.removeEventListener("abort", onAbort);
|
|
};
|
|
const settle = (complete: () => void): void => {
|
|
if (settled) return;
|
|
settled = true;
|
|
cleanup();
|
|
complete();
|
|
};
|
|
|
|
timer = setTimeout(
|
|
() => settle(() => reject(new TlsClientHangError())),
|
|
Math.max(0, timeoutMs)
|
|
);
|
|
if (signal?.aborted) {
|
|
onAbort();
|
|
} else {
|
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
}
|
|
|
|
promise.then(
|
|
(value) => settle(() => resolve(value)),
|
|
(error) => settle(() => reject(error))
|
|
);
|
|
});
|
|
}
|
|
|
|
/** Return true when a prefix begins with an SSE field or comment marker. */
|
|
export function looksLikeSse(text: string): boolean {
|
|
const trimmed = text.replace(/^[\s\r\n]+/, "");
|
|
if (!trimmed) return false;
|
|
if (trimmed.startsWith(":")) return true;
|
|
return /^(data|event|id|retry):/i.test(trimmed);
|
|
}
|
|
|
|
/** Return true when a response prefix is a Cloudflare challenge/interstitial. */
|
|
export function isCloudflareChallenge(text: string | null | undefined): boolean {
|
|
if (!text) return false;
|
|
return /just a moment|window\._cf_chl_opt|challenges\.cloudflare\.com|attention required|cf-chl/i.test(
|
|
text
|
|
);
|
|
}
|
|
|
|
function couldBecomeSsePrefix(text: string): boolean {
|
|
const trimmed = text.replace(/^[\s\r\n]+/, "").toLowerCase();
|
|
return ["data:", "event:", "id:", "retry:", ":"].some((marker) => marker.startsWith(trimmed));
|
|
}
|
|
|
|
function couldBecomeCloudflareChallenge(text: string): boolean {
|
|
const trimmed = text.trimStart().toLowerCase();
|
|
return [
|
|
"just a moment",
|
|
"window._cf_chl_opt",
|
|
"challenges.cloudflare.com",
|
|
"attention required",
|
|
"cf-chl",
|
|
].some((marker) => marker.startsWith(trimmed));
|
|
}
|
|
|
|
type EofControlCandidate = "possible" | "matched" | "not-control";
|
|
|
|
function classifyEofControlCandidate(bytes: number[], eofSymbol: string): EofControlCandidate {
|
|
const decoded = new TextDecoder().decode(Uint8Array.from(bytes), { stream: true });
|
|
const candidate = decoded.replace(/^[\t\r ]+/, "");
|
|
if (candidate.startsWith(eofSymbol)) return "matched";
|
|
if (eofSymbol.startsWith(candidate)) return "possible";
|
|
|
|
const dataPrefix = "data:";
|
|
const lowerCandidate = candidate.toLowerCase();
|
|
if (dataPrefix.startsWith(lowerCandidate)) return "possible";
|
|
if (!lowerCandidate.startsWith(dataPrefix)) return "not-control";
|
|
|
|
const dataValue = candidate.slice(dataPrefix.length).replace(/^[\t ]+/, "");
|
|
if (dataValue.startsWith(eofSymbol)) return "matched";
|
|
return eofSymbol.startsWith(dataValue) ? "possible" : "not-control";
|
|
}
|
|
|
|
function createEofFilteredStream(
|
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
initialChunks: Uint8Array[],
|
|
eofSymbol: string,
|
|
includeEof: boolean,
|
|
readNext: () => Promise<ReadableStreamReadResult<Uint8Array>>,
|
|
onReadError: (error: unknown) => void,
|
|
signal: AbortSignal | null,
|
|
hardDeadlineAt: number
|
|
): ReadableStream<Uint8Array> {
|
|
const queued = [...initialChunks];
|
|
const eofBytes = new TextEncoder().encode(eofSymbol);
|
|
let controlCandidate: number[] = [];
|
|
let atLineStart = true;
|
|
let closed = false;
|
|
let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
|
|
let removeAbortListener = (): void => {};
|
|
|
|
const cleanupLifecycle = (): void => {
|
|
if (deadlineTimer) clearTimeout(deadlineTimer);
|
|
deadlineTimer = undefined;
|
|
removeAbortListener();
|
|
removeAbortListener = (): void => {};
|
|
};
|
|
|
|
const errorStream = (
|
|
controller: ReadableStreamDefaultController<Uint8Array>,
|
|
error: Error,
|
|
notifyReadError: boolean
|
|
): void => {
|
|
if (closed) return;
|
|
closed = true;
|
|
controlCandidate = [];
|
|
cleanupLifecycle();
|
|
if (notifyReadError) onReadError(error);
|
|
void reader.cancel(error).catch(() => {});
|
|
try {
|
|
controller.error(error);
|
|
} catch {
|
|
// The consumer may have closed the stream concurrently.
|
|
}
|
|
};
|
|
|
|
return new ReadableStream<Uint8Array>({
|
|
start(controller) {
|
|
if (signal) {
|
|
const onAbort = (): void => errorStream(controller, makeAbortError(signal), false);
|
|
if (signal.aborted) {
|
|
onAbort();
|
|
return;
|
|
}
|
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
removeAbortListener = (): void => signal.removeEventListener("abort", onAbort);
|
|
}
|
|
|
|
if (Number.isFinite(hardDeadlineAt)) {
|
|
deadlineTimer = setTimeout(
|
|
() => {
|
|
errorStream(controller, new TlsClientHangError(), true);
|
|
},
|
|
Math.max(0, hardDeadlineAt - Date.now())
|
|
);
|
|
deadlineTimer.unref?.();
|
|
}
|
|
},
|
|
async pull(controller) {
|
|
while (!closed) {
|
|
let chunk = queued.shift();
|
|
if (!chunk) {
|
|
let next: ReadableStreamReadResult<Uint8Array>;
|
|
try {
|
|
next = await readNext();
|
|
} catch (error) {
|
|
if (closed) return;
|
|
closed = true;
|
|
cleanupLifecycle();
|
|
onReadError(error);
|
|
await reader.cancel(error).catch(() => {});
|
|
controller.error(error);
|
|
return;
|
|
}
|
|
if (closed) return;
|
|
if (next.done) {
|
|
if (controlCandidate.length > 0) {
|
|
controller.enqueue(Uint8Array.from(controlCandidate));
|
|
}
|
|
controlCandidate = [];
|
|
closed = true;
|
|
cleanupLifecycle();
|
|
controller.close();
|
|
return;
|
|
}
|
|
chunk = next.value;
|
|
}
|
|
|
|
if (eofBytes.byteLength === 0) {
|
|
controller.enqueue(chunk);
|
|
return;
|
|
}
|
|
|
|
const output: number[] = [];
|
|
let eofReached = false;
|
|
for (const byte of chunk) {
|
|
if (!atLineStart) {
|
|
output.push(byte);
|
|
if (byte === 0x0a || byte === 0x0d) atLineStart = true;
|
|
continue;
|
|
}
|
|
|
|
if (byte === 0x0a || byte === 0x0d) {
|
|
for (const candidateByte of controlCandidate) output.push(candidateByte);
|
|
controlCandidate = [];
|
|
output.push(byte);
|
|
continue;
|
|
}
|
|
|
|
controlCandidate.push(byte);
|
|
const classification = classifyEofControlCandidate(controlCandidate, eofSymbol);
|
|
if (classification === "matched") {
|
|
if (includeEof) {
|
|
for (const candidateByte of controlCandidate) output.push(candidateByte);
|
|
}
|
|
controlCandidate = [];
|
|
eofReached = true;
|
|
break;
|
|
}
|
|
if (classification === "not-control") {
|
|
for (const candidateByte of controlCandidate) output.push(candidateByte);
|
|
controlCandidate = [];
|
|
atLineStart = false;
|
|
}
|
|
}
|
|
|
|
if (eofReached) {
|
|
if (output.length > 0) controller.enqueue(Uint8Array.from(output));
|
|
closed = true;
|
|
cleanupLifecycle();
|
|
await reader.cancel("TLS stream EOF reached").catch(() => {});
|
|
controller.close();
|
|
return;
|
|
}
|
|
|
|
if (output.length > 0) {
|
|
controller.enqueue(Uint8Array.from(output));
|
|
return;
|
|
}
|
|
}
|
|
},
|
|
async cancel(reason) {
|
|
closed = true;
|
|
controlCandidate = [];
|
|
cleanupLifecycle();
|
|
await reader.cancel(reason).catch(() => {});
|
|
},
|
|
});
|
|
}
|
|
|
|
async function loadWreqRuntime(): Promise<WreqRuntimeLike> {
|
|
const moduleName = "wreq-js";
|
|
const loaded = Reflect.apply(runtimeRequire, undefined, [moduleName]) as Partial<WreqRuntimeLike>;
|
|
if (typeof loaded.createTransport !== "function" || typeof loaded.fetch !== "function") {
|
|
throw new Error("wreq-js runtime is missing createTransport/fetch");
|
|
}
|
|
return loaded as WreqRuntimeLike;
|
|
}
|
|
|
|
export interface TlsRequestClient {
|
|
request: (url: string, options: Record<string, unknown>) => Promise<TlsResponseLike>;
|
|
invalidateTransport?: (options: Record<string, unknown>) => void;
|
|
}
|
|
|
|
function transportKey(browser: string, os: EmulationOs, options: Record<string, unknown>): string {
|
|
const proxy = typeof options.proxyUrl === "string" ? options.proxyUrl : "";
|
|
return `${browser}\0${os}\0${proxy}`;
|
|
}
|
|
|
|
/** Create a lazy wreq-js client with transport-only pooling and no shared cookie jar. */
|
|
export function createGetClient(config: {
|
|
providerName: string;
|
|
tlsProfile?: string;
|
|
emulationOs?: EmulationOs;
|
|
wreqRuntimeLoader?: WreqRuntimeLoader;
|
|
}): () => Promise<TlsRequestClient> {
|
|
const browser = config.tlsProfile ?? "chrome_146";
|
|
const os = config.emulationOs ?? "macos";
|
|
const transports = new Map<string, Promise<WreqTransportLike>>();
|
|
let runtimePromise: Promise<WreqRuntimeLike> | null = null;
|
|
|
|
const getRuntime = (): Promise<WreqRuntimeLike> => {
|
|
if (!runtimePromise) {
|
|
const pending = (config.wreqRuntimeLoader ?? loadWreqRuntime)().catch(() => {
|
|
if (runtimePromise === pending) runtimePromise = null;
|
|
throw new TlsClientUnavailableError(
|
|
`wreq-js 3.0.0 is not installed or unsupported on this platform — ` +
|
|
`cannot start browser transport for ${config.providerName}`
|
|
);
|
|
});
|
|
runtimePromise = pending;
|
|
}
|
|
return runtimePromise;
|
|
};
|
|
|
|
const closeTransport = (key: string): void => {
|
|
const pending = transports.get(key);
|
|
if (!pending) return;
|
|
transports.delete(key);
|
|
void pending.then((transport) => transport.close()).catch(() => {});
|
|
};
|
|
|
|
const closeAllTransports = (): void => {
|
|
for (const key of [...transports.keys()]) closeTransport(key);
|
|
};
|
|
transportClosers.add(closeAllTransports);
|
|
if (!exitHookInstalled) {
|
|
exitHookInstalled = true;
|
|
process.once("exit", () => {
|
|
for (const close of transportClosers) close();
|
|
transportClosers.clear();
|
|
});
|
|
}
|
|
|
|
const client: TlsRequestClient = {
|
|
async request(url, options) {
|
|
const runtime = await getRuntime();
|
|
const key = transportKey(browser, os, options);
|
|
let pending = transports.get(key);
|
|
if (!pending) {
|
|
const proxy = typeof options.proxyUrl === "string" ? options.proxyUrl : undefined;
|
|
const transportOptions: Record<string, unknown> = { browser, os };
|
|
if (proxy) transportOptions.proxy = proxy;
|
|
pending = runtime.createTransport(transportOptions);
|
|
transports.set(key, pending);
|
|
void pending.catch(() => {
|
|
if (transports.get(key) === pending) transports.delete(key);
|
|
});
|
|
}
|
|
const transport = await pending;
|
|
return runtime.fetch(url, {
|
|
method: options.method,
|
|
headers: options.headers,
|
|
body: options.body,
|
|
redirect: "follow",
|
|
timeout: options.timeoutMilliseconds,
|
|
signal: options.signal,
|
|
transport,
|
|
cookieMode: "ephemeral",
|
|
});
|
|
},
|
|
invalidateTransport(options) {
|
|
closeTransport(transportKey(browser, os, options));
|
|
},
|
|
};
|
|
|
|
return async () => client;
|
|
}
|
|
|
|
/** Resolve a per-call/provider/dashboard proxy for a browser-transport request. */
|
|
export function resolveProxyUrl(domain: string, perCall: string | undefined): string | undefined {
|
|
return resolveTlsClientProxyUrl(domain, perCall, resolveProxyForRequest);
|
|
}
|
|
|
|
export interface TlsClientModule {
|
|
tlsFetch: (url: string, options?: TlsFetchOptions) => Promise<TlsFetchResult>;
|
|
__setTlsFetchOverrideForTesting: (
|
|
fn: ((url: string, options: TlsFetchOptions) => Promise<TlsFetchResult>) | null
|
|
) => void;
|
|
isCloudflareChallenge?: (text: string | null | undefined) => boolean;
|
|
__tlsFetchStreamingForTesting?: (
|
|
client: TlsRequestClient,
|
|
url: string,
|
|
requestOptions: Record<string, unknown>,
|
|
eofSymbol?: string,
|
|
signal?: AbortSignal | null,
|
|
hardTimeoutMs?: number,
|
|
firstByteTimeoutMs?: number
|
|
) => Promise<TlsFetchResult>;
|
|
}
|
|
|
|
/** Build one provider-specific facade over the shared wreq-js transport. */
|
|
export function createTlsClientModule(config: TlsClientConfig): TlsClientModule {
|
|
const {
|
|
providerName,
|
|
tlsProfile,
|
|
emulationOs = "macos",
|
|
domain,
|
|
streamEofSymbol = "[DONE]",
|
|
defaultTimeoutMs = 60_000,
|
|
hardTimeoutGraceMs = 10_000,
|
|
firstByteTimeoutMs = 5_000,
|
|
responseValidation,
|
|
proxyDomainOverride,
|
|
exportCloudflareCheck,
|
|
wreqRuntimeLoader,
|
|
} = config;
|
|
const streamEofPolicy =
|
|
config.streamEofPolicy ?? (config.tailFileVariant === "A" ? "include" : "exclude");
|
|
|
|
const getClient = createGetClient({
|
|
providerName,
|
|
tlsProfile,
|
|
emulationOs,
|
|
wreqRuntimeLoader,
|
|
});
|
|
let testOverride: ((url: string, options: TlsFetchOptions) => Promise<TlsFetchResult>) | null =
|
|
null;
|
|
|
|
const invalidateOnHang = (
|
|
client: TlsRequestClient,
|
|
requestOptions: Record<string, unknown>,
|
|
error: unknown
|
|
): void => {
|
|
if (error instanceof TlsClientHangError) client.invalidateTransport?.(requestOptions);
|
|
};
|
|
|
|
async function tlsFetchStreaming(
|
|
client: TlsRequestClient,
|
|
url: string,
|
|
requestOptions: Record<string, unknown>,
|
|
eofSymbol: string,
|
|
signal: AbortSignal | null,
|
|
hardTimeoutMs: number,
|
|
firstByteMs: number = firstByteTimeoutMs
|
|
): Promise<TlsFetchResult> {
|
|
const startedAt = Date.now();
|
|
const hardDeadlineAt = startedAt + hardTimeoutMs;
|
|
const firstByteDeadlineAt = Number.isFinite(firstByteMs)
|
|
? startedAt + Math.max(0, firstByteMs)
|
|
: Number.POSITIVE_INFINITY;
|
|
const remainingHardTimeoutMs = (): number => Math.max(0, hardDeadlineAt - Date.now());
|
|
let reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
|
|
|
|
try {
|
|
const response = await raceWithTimeout(
|
|
client.request(url, requestOptions),
|
|
remainingHardTimeoutMs(),
|
|
signal
|
|
);
|
|
if (!isReadableBody(response.body)) {
|
|
const text = await raceWithTimeout(
|
|
readTlsResponseText(response),
|
|
remainingHardTimeoutMs(),
|
|
signal
|
|
);
|
|
return { status: response.status, headers: toHeaders(response.headers), text, body: null };
|
|
}
|
|
|
|
reader = response.body.getReader();
|
|
const activeReader = reader;
|
|
const readBeforeHardDeadline = (): Promise<ReadableStreamReadResult<Uint8Array>> =>
|
|
raceWithTimeout(activeReader.read(), remainingHardTimeoutMs(), signal);
|
|
const initialChunks: Uint8Array[] = [];
|
|
const readFirstNonEmptyChunk = async (): Promise<ReadableStreamReadResult<Uint8Array>> => {
|
|
while (true) {
|
|
const result = await readBeforeHardDeadline();
|
|
if (result.done || result.value.byteLength > 0) return result;
|
|
}
|
|
};
|
|
const firstRead = readFirstNonEmptyChunk();
|
|
let firstResult: ReadableStreamReadResult<Uint8Array>;
|
|
let firstByteTimedOut = Date.now() >= firstByteDeadlineAt;
|
|
|
|
if (Number.isFinite(firstByteMs) && !firstByteTimedOut) {
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
const timed = await Promise.race([
|
|
firstRead.then((result) => ({ kind: "read" as const, result })),
|
|
new Promise<{ kind: "timeout" }>((resolve) => {
|
|
timer = setTimeout(
|
|
() => resolve({ kind: "timeout" }),
|
|
Math.max(0, firstByteDeadlineAt - Date.now())
|
|
);
|
|
}),
|
|
]);
|
|
if (timed.kind === "timeout") {
|
|
firstByteTimedOut = true;
|
|
firstResult = await firstRead;
|
|
} else {
|
|
firstResult = timed.result;
|
|
}
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
} else {
|
|
firstResult = await firstRead;
|
|
}
|
|
|
|
if (!firstResult.done) initialChunks.push(firstResult.value);
|
|
if (firstByteTimedOut) {
|
|
const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline);
|
|
return {
|
|
status: response.status,
|
|
headers: toHeaders(response.headers),
|
|
text: new TextDecoder().decode(bytes),
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
let previewBytes = concatChunks(initialChunks).subarray(0, 256);
|
|
let preview = new TextDecoder().decode(previewBytes, { stream: true });
|
|
let previewReachedEof = firstResult.done;
|
|
while (
|
|
!previewReachedEof &&
|
|
previewBytes.byteLength < 256 &&
|
|
((responseValidation === "sse" &&
|
|
!looksLikeSse(preview) &&
|
|
couldBecomeSsePrefix(preview)) ||
|
|
(responseValidation === "cf" &&
|
|
!isCloudflareChallenge(preview) &&
|
|
(preview.trimStart().startsWith("<") || couldBecomeCloudflareChallenge(preview))))
|
|
) {
|
|
const next = await readBeforeHardDeadline();
|
|
if (next.done) {
|
|
previewReachedEof = true;
|
|
break;
|
|
}
|
|
initialChunks.push(next.value);
|
|
previewBytes = concatChunks(initialChunks).subarray(0, 256);
|
|
preview = new TextDecoder().decode(previewBytes, { stream: true });
|
|
}
|
|
|
|
if (previewReachedEof && previewBytes.byteLength === 0) {
|
|
return {
|
|
status: response.status,
|
|
headers: toHeaders(response.headers),
|
|
text: "",
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
if (responseValidation === "cf" && isCloudflareChallenge(preview)) {
|
|
await activeReader.cancel("Cloudflare challenge").catch(() => {});
|
|
return {
|
|
status: 403,
|
|
headers: new Headers({ "Content-Type": "text/html" }),
|
|
text: preview,
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
if (responseValidation === "cf" && preview.trimStart().startsWith("<")) {
|
|
await activeReader.cancel("HTML response").catch(() => {});
|
|
return {
|
|
status: 502,
|
|
headers: new Headers({ "Content-Type": "text/html" }),
|
|
text: preview,
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
if (response.status < 200 || response.status >= 300) {
|
|
const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline);
|
|
return {
|
|
status: response.status,
|
|
headers: toHeaders(response.headers),
|
|
text: new TextDecoder().decode(bytes),
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
if (responseValidation === "sse" && !looksLikeSse(preview)) {
|
|
const bytes = await readAllChunks(activeReader, initialChunks, readBeforeHardDeadline);
|
|
return {
|
|
status: response.status,
|
|
headers: toHeaders(response.headers),
|
|
text: new TextDecoder().decode(bytes),
|
|
body: null,
|
|
};
|
|
}
|
|
|
|
const headers = toHeaders(response.headers);
|
|
headers.set(
|
|
"Content-Type",
|
|
responseValidation === "cf" ? "application/x-ndjson" : "text/event-stream"
|
|
);
|
|
headers.set("Cache-Control", "no-cache");
|
|
const stream = createEofFilteredStream(
|
|
activeReader,
|
|
initialChunks,
|
|
streamEofPolicy === "none" ? "" : eofSymbol,
|
|
streamEofPolicy === "include",
|
|
readBeforeHardDeadline,
|
|
(error) => invalidateOnHang(client, requestOptions, error),
|
|
signal,
|
|
hardDeadlineAt
|
|
);
|
|
reader = null;
|
|
return { status: 200, headers, text: null, body: stream };
|
|
} catch (error) {
|
|
invalidateOnHang(client, requestOptions, error);
|
|
if (reader) await reader.cancel(error).catch(() => {});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function tlsFetch(url: string, options: TlsFetchOptions = {}): Promise<TlsFetchResult> {
|
|
const resolvedProxyUrl = resolveProxyUrl(proxyDomainOverride ?? domain, options.proxyUrl);
|
|
if (testOverride) return testOverride(url, { ...options, proxyUrl: resolvedProxyUrl });
|
|
if (options.signal?.aborted) throw makeAbortError(options.signal);
|
|
|
|
const client = await getClient();
|
|
if (options.signal?.aborted) throw makeAbortError(options.signal);
|
|
|
|
const requestOptions: Record<string, unknown> = {
|
|
method: options.method || "GET",
|
|
headers: options.headers || {},
|
|
body: options.body,
|
|
timeoutMilliseconds: options.timeoutMs ?? defaultTimeoutMs,
|
|
proxyUrl: resolvedProxyUrl,
|
|
signal: options.signal,
|
|
};
|
|
const hardTimeoutMs = (options.timeoutMs ?? defaultTimeoutMs) + hardTimeoutGraceMs;
|
|
|
|
if (options.stream) {
|
|
return tlsFetchStreaming(
|
|
client,
|
|
url,
|
|
requestOptions,
|
|
options.streamEofSymbol ?? streamEofSymbol,
|
|
options.signal ?? null,
|
|
hardTimeoutMs,
|
|
firstByteTimeoutMs
|
|
);
|
|
}
|
|
|
|
const hardDeadlineAt = Date.now() + hardTimeoutMs;
|
|
const remainingHardTimeoutMs = (): number => Math.max(0, hardDeadlineAt - Date.now());
|
|
let response: TlsResponseLike | null = null;
|
|
let bodyReader: ReadableStreamDefaultReader<Uint8Array> | null = null;
|
|
try {
|
|
response = await raceWithTimeout(
|
|
client.request(url, requestOptions),
|
|
remainingHardTimeoutMs(),
|
|
options.signal ?? null
|
|
);
|
|
if (options.signal?.aborted) throw makeAbortError(options.signal);
|
|
const headers = toHeaders(response.headers);
|
|
if (options.byteResponse) {
|
|
const bytes = await raceWithTimeout(
|
|
readTlsResponseBytes(response, (reader) => {
|
|
bodyReader = reader;
|
|
}),
|
|
remainingHardTimeoutMs(),
|
|
options.signal ?? null
|
|
);
|
|
bodyReader = null;
|
|
const mime =
|
|
headers.get("content-type")?.split(";", 1)[0]?.trim() || "application/octet-stream";
|
|
return {
|
|
status: response.status,
|
|
headers,
|
|
text: `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`,
|
|
body: null,
|
|
};
|
|
}
|
|
const text = await raceWithTimeout(
|
|
readTlsResponseText(response, (reader) => {
|
|
bodyReader = reader;
|
|
}),
|
|
remainingHardTimeoutMs(),
|
|
options.signal ?? null
|
|
);
|
|
bodyReader = null;
|
|
return { status: response.status, headers, text, body: null };
|
|
} catch (error) {
|
|
invalidateOnHang(client, requestOptions, error);
|
|
if (bodyReader) {
|
|
await bodyReader.cancel(error).catch(() => {});
|
|
} else if (
|
|
response &&
|
|
isReadableBody(response.body) &&
|
|
typeof response.body.cancel === "function"
|
|
) {
|
|
await response.body.cancel(error).catch(() => {});
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
const module: TlsClientModule = {
|
|
tlsFetch,
|
|
__setTlsFetchOverrideForTesting(fn) {
|
|
testOverride = fn;
|
|
},
|
|
};
|
|
if (exportCloudflareCheck) module.isCloudflareChallenge = isCloudflareChallenge;
|
|
if (config.exposeStreamingForTesting) {
|
|
module.__tlsFetchStreamingForTesting = (
|
|
client,
|
|
url,
|
|
requestOptions,
|
|
eofSymbol = "[DONE]",
|
|
signal = null,
|
|
hardTimeoutMs = defaultTimeoutMs + hardTimeoutGraceMs,
|
|
firstByteMs = firstByteTimeoutMs
|
|
) =>
|
|
tlsFetchStreaming(client, url, requestOptions, eofSymbol, signal, hardTimeoutMs, firstByteMs);
|
|
}
|
|
return module;
|
|
}
|