mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
feat(network): add guarded remote image fetch utility
Centralize remote image downloads behind a shared helper that validates outbound URLs, enforces redirect and size limits, and applies request timeouts before bytes are read. Wire the helper into image generation and vision bridge flows so remote image inputs and result URLs follow the same fetch policy and block redirects to private hosts. Update key management routes to use structured logging and document the WebSocket bridge secret in the example environment file.
This commit is contained in:
@@ -110,6 +110,13 @@ REQUIRE_API_KEY=false
|
||||
# Default: false | Security risk if enabled on shared instances.
|
||||
ALLOW_API_KEY_REVEAL=false
|
||||
|
||||
# Shared secret for the internal Codex Responses WebSocket bridge.
|
||||
# Used by: src/app/api/internal/codex-responses-ws/route.ts — authenticates
|
||||
# bridge requests between the Electron/browser WS relay and OmniRoute.
|
||||
# ⚠️ REQUIRED for production — if unset, all WS bridge requests are rejected.
|
||||
# Generate: openssl rand -base64 32
|
||||
# OMNIROUTE_WS_BRIDGE_SECRET=
|
||||
|
||||
# Comma-separated API key IDs that skip request logging (GDPR/compliance).
|
||||
# Used by: src/lib/compliance/index.ts — suppresses logs for specific keys.
|
||||
# NO_LOG_API_KEY_IDS=key_abc123,key_def456
|
||||
|
||||
@@ -20,10 +20,7 @@ import { getImageProvider, parseImageModel } from "../config/imageRegistry.ts";
|
||||
import { mapImageSize } from "../translator/image/sizeMapper.ts";
|
||||
import { getCodexClientVersion, getCodexUserAgent } from "../config/codexClient.ts";
|
||||
import { ChatGptWebExecutor } from "../executors/chatgpt-web.ts";
|
||||
import {
|
||||
getChatGptImage,
|
||||
findChatGptImageBySha256,
|
||||
} from "../services/chatgptImageCache.ts";
|
||||
import { getChatGptImage, findChatGptImageBySha256 } from "../services/chatgptImageCache.ts";
|
||||
import { createHash } from "node:crypto";
|
||||
import { saveCallLog } from "@/lib/usageDb";
|
||||
import {
|
||||
@@ -32,6 +29,7 @@ import {
|
||||
fetchComfyOutput,
|
||||
extractComfyOutputFiles,
|
||||
} from "../utils/comfyuiClient.ts";
|
||||
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
|
||||
|
||||
const OPENAI_IMAGE_TO_IMAGE_MODELS = new Set([
|
||||
"black-forest-labs/FLUX.1-redux",
|
||||
@@ -625,8 +623,7 @@ async function handleChatGptWebImageGeneration({
|
||||
// own limit for image-1 / dall-e-3) so a stray n=1000 doesn't pin the
|
||||
// executor for hours before the upstream HTTP timeout fires.
|
||||
const CHATGPT_WEB_IMAGE_N_MAX = 4;
|
||||
const rawCount =
|
||||
Number.isInteger(body.n) && (body.n as number) > 0 ? (body.n as number) : 1;
|
||||
const rawCount = Number.isInteger(body.n) && (body.n as number) > 0 ? (body.n as number) : 1;
|
||||
if (rawCount > CHATGPT_WEB_IMAGE_N_MAX) {
|
||||
return saveImageErrorResult({
|
||||
provider,
|
||||
@@ -1547,15 +1544,11 @@ async function resolveImageSource(source) {
|
||||
}
|
||||
|
||||
if (isHttpUrl(trimmed)) {
|
||||
const response = await fetch(trimmed);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch image source (${response.status})`);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const remoteImage = await fetchRemoteImage(trimmed);
|
||||
return {
|
||||
buffer,
|
||||
base64: buffer.toString("base64"),
|
||||
contentType: response.headers.get("content-type") || "application/octet-stream",
|
||||
buffer: remoteImage.buffer,
|
||||
base64: remoteImage.buffer.toString("base64"),
|
||||
contentType: remoteImage.contentType,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2437,12 +2430,8 @@ async function normalizeNanoBananaTaskResult(taskData, body, log) {
|
||||
|
||||
if (urlCandidates.length > 0) {
|
||||
const firstUrl = urlCandidates[0];
|
||||
const resp = await fetch(firstUrl);
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Failed to fetch NanoBanana result image URL (${resp.status})`);
|
||||
}
|
||||
const arrayBuffer = await resp.arrayBuffer();
|
||||
const base64 = Buffer.from(arrayBuffer).toString("base64");
|
||||
const remoteImage = await fetchRemoteImage(firstUrl);
|
||||
const base64 = remoteImage.buffer.toString("base64");
|
||||
return [{ b64_json: base64, revised_prompt: body.prompt }];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
|
||||
import { getApiKeyById } from "@/lib/localDb";
|
||||
import { isApiKeyRevealEnabled } from "@/lib/apiKeyExposure";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import * as log from "@/sse/utils/logger";
|
||||
|
||||
// GET /api/keys/[id]/reveal - Reveal full API key for explicit copy actions
|
||||
export async function GET(request, { params }) {
|
||||
@@ -22,7 +23,7 @@ export async function GET(request, { params }) {
|
||||
|
||||
return NextResponse.json({ key: key.key });
|
||||
} catch (error) {
|
||||
console.log("Error revealing key:", error);
|
||||
log.error("keys", "Error revealing key", error);
|
||||
return NextResponse.json({ error: "Failed to reveal key" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { syncToCloud } from "@/lib/cloudSync";
|
||||
import { updateKeyPermissionsSchema } from "@/shared/validation/schemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import * as log from "@/sse/utils/logger";
|
||||
|
||||
// GET /api/keys/[id] - Get single API key
|
||||
export async function GET(request, { params }) {
|
||||
@@ -31,7 +32,7 @@ export async function GET(request, { params }) {
|
||||
key: keyValue ? keyValue.slice(0, 8) + "****" + keyValue.slice(-4) : null,
|
||||
});
|
||||
} catch (error) {
|
||||
console.log("Error fetching key:", error);
|
||||
log.error("keys", "Error fetching key", error);
|
||||
return NextResponse.json({ error: "Failed to fetch key" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -103,7 +104,7 @@ export async function PATCH(request, { params }) {
|
||||
...(accessSchedule !== undefined && { accessSchedule }),
|
||||
});
|
||||
} catch (error) {
|
||||
console.log("Error updating key permissions:", error);
|
||||
log.error("keys", "Error updating key permissions", error);
|
||||
return NextResponse.json({ error: "Failed to update permissions" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -126,7 +127,7 @@ export async function DELETE(request, { params }) {
|
||||
|
||||
return NextResponse.json({ message: "Key deleted successfully" });
|
||||
} catch (error) {
|
||||
console.log("Error deleting key:", error);
|
||||
log.error("keys", "Error deleting key", error);
|
||||
return NextResponse.json({ error: "Failed to delete key" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -142,6 +143,6 @@ async function syncKeysToCloudIfEnabled() {
|
||||
const machineId = await getConsistentMachineId();
|
||||
await syncToCloud(machineId);
|
||||
} catch (error) {
|
||||
console.log("Error syncing keys to cloud:", error);
|
||||
log.error("keys", "Error syncing keys to cloud", error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { createKeySchema } from "@/shared/validation/schemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
import { isApiKeyRevealEnabled, maskStoredApiKey } from "@/lib/apiKeyExposure";
|
||||
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
|
||||
import * as log from "@/sse/utils/logger";
|
||||
|
||||
function parsePagination(request: Request) {
|
||||
const url = new URL(request.url);
|
||||
@@ -43,7 +44,7 @@ export async function GET(request: Request) {
|
||||
allowKeyReveal: isApiKeyRevealEnabled(),
|
||||
});
|
||||
} catch (error) {
|
||||
console.log("Error fetching keys:", error);
|
||||
log.error("keys", "Error fetching keys", error);
|
||||
return NextResponse.json({ error: "Failed to fetch keys" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -84,7 +85,7 @@ export async function POST(request) {
|
||||
{ status: 201 }
|
||||
);
|
||||
} catch (error) {
|
||||
console.log("Error creating key:", error);
|
||||
log.error("keys", "Error creating key", error);
|
||||
return NextResponse.json({ error: "Failed to create key" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -100,6 +101,6 @@ async function syncKeysToCloudIfEnabled() {
|
||||
const machineId = await getConsistentMachineId();
|
||||
await syncToCloud(machineId);
|
||||
} catch (error) {
|
||||
console.log("Error syncing keys to cloud:", error);
|
||||
log.error("keys", "Error syncing keys to cloud", error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* Vision Bridge helper functions for image processing.
|
||||
*/
|
||||
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
|
||||
|
||||
/**
|
||||
* Provider to environment variable mapping for API key resolution.
|
||||
@@ -111,14 +112,9 @@ export function resolveImageAsDataUri(imageUrl: string): string {
|
||||
}
|
||||
|
||||
async function fetchRemoteImageAsDataUri(imageUrl: string, signal: AbortSignal): Promise<string> {
|
||||
const response = await fetch(imageUrl, { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Vision image fetch error ${response.status}`);
|
||||
}
|
||||
|
||||
const mediaType = response.headers.get("content-type")?.split(";")[0]?.trim() || "image/png";
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
return `data:${mediaType};base64,${bytes.toString("base64")}`;
|
||||
const remoteImage = await fetchRemoteImage(imageUrl, { signal });
|
||||
const mediaType = remoteImage.contentType.split(";")[0]?.trim() || "image/png";
|
||||
return `data:${mediaType};base64,${remoteImage.buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
async function normalizeVisionImageInput(
|
||||
|
||||
118
src/shared/network/remoteImageFetch.ts
Normal file
118
src/shared/network/remoteImageFetch.ts
Normal file
@@ -0,0 +1,118 @@
|
||||
import {
|
||||
type OutboundUrlGuardMode,
|
||||
getProviderOutboundGuard,
|
||||
parseAndValidatePublicUrl,
|
||||
parseOutboundUrl,
|
||||
} from "@/shared/network/outboundUrlGuard";
|
||||
|
||||
const DEFAULT_MAX_REMOTE_IMAGE_BYTES = 20 * 1024 * 1024;
|
||||
const DEFAULT_MAX_REDIRECTS = 3;
|
||||
const DEFAULT_TIMEOUT_MS = 15000;
|
||||
|
||||
export interface RemoteImageFetchOptions {
|
||||
fetchImpl?: typeof fetch;
|
||||
guard?: OutboundUrlGuardMode;
|
||||
maxBytes?: number;
|
||||
maxRedirects?: number;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface RemoteImageFetchResult {
|
||||
buffer: Buffer;
|
||||
contentType: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function validateRemoteImageUrl(input: string | URL, guard: OutboundUrlGuardMode) {
|
||||
return guard === "public-only" ? parseAndValidatePublicUrl(input) : parseOutboundUrl(input);
|
||||
}
|
||||
|
||||
function combineSignals(signal: AbortSignal | undefined, timeoutMs: number) {
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
if (!signal) return timeoutSignal;
|
||||
return AbortSignal.any([signal, timeoutSignal]);
|
||||
}
|
||||
|
||||
async function readResponseBuffer(response: Response, maxBytes: number) {
|
||||
const contentLengthHeader = response.headers.get("content-length");
|
||||
const contentLength = contentLengthHeader ? Number.parseInt(contentLengthHeader, 10) : null;
|
||||
if (contentLength !== null && Number.isFinite(contentLength) && contentLength > maxBytes) {
|
||||
throw new Error(`Remote image exceeds ${maxBytes} byte limit`);
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (buffer.byteLength > maxBytes) {
|
||||
throw new Error(`Remote image exceeds ${maxBytes} byte limit`);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Buffer[] = [];
|
||||
let totalBytes = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = Buffer.from(value);
|
||||
totalBytes += chunk.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
await reader.cancel();
|
||||
throw new Error(`Remote image exceeds ${maxBytes} byte limit`);
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
return Buffer.concat(chunks, totalBytes);
|
||||
}
|
||||
|
||||
export async function fetchRemoteImage(
|
||||
input: string | URL,
|
||||
options: RemoteImageFetchOptions = {}
|
||||
): Promise<RemoteImageFetchResult> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
const guard = options.guard ?? getProviderOutboundGuard();
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_REMOTE_IMAGE_BYTES;
|
||||
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
||||
const signal = combineSignals(options.signal, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
||||
|
||||
let currentUrl = validateRemoteImageUrl(input, guard);
|
||||
for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount++) {
|
||||
const response = await fetchImpl(currentUrl.toString(), {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal,
|
||||
});
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const location = response.headers.get("location");
|
||||
if (!location) {
|
||||
throw new Error(`Remote image redirect missing Location header (${response.status})`);
|
||||
}
|
||||
if (redirectCount >= maxRedirects) {
|
||||
throw new Error(`Remote image exceeded ${maxRedirects} redirect limit`);
|
||||
}
|
||||
currentUrl = validateRemoteImageUrl(new URL(location, currentUrl), guard);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Remote image fetch error ${response.status}`);
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: await readResponseBuffer(response, maxBytes),
|
||||
contentType: response.headers.get("content-type") || "application/octet-stream",
|
||||
url: currentUrl.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Remote image exceeded ${maxRedirects} redirect limit`);
|
||||
}
|
||||
51
tests/unit/remote-image-fetch.test.ts
Normal file
51
tests/unit/remote-image-fetch.test.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { fetchRemoteImage } from "@/shared/network/remoteImageFetch";
|
||||
|
||||
test("fetchRemoteImage reads public image bytes", async () => {
|
||||
const result = await fetchRemoteImage("https://cdn.example.com/image.png", {
|
||||
fetchImpl: async () =>
|
||||
new Response(new Uint8Array([1, 2, 3]), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
guard: "public-only",
|
||||
});
|
||||
|
||||
assert.equal(result.buffer.toString("base64"), "AQID");
|
||||
assert.equal(result.contentType, "image/png");
|
||||
});
|
||||
|
||||
test("fetchRemoteImage blocks private image hosts before fetch", async () => {
|
||||
let called = false;
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
fetchRemoteImage("http://127.0.0.1:20128/private.png", {
|
||||
fetchImpl: async () => {
|
||||
called = true;
|
||||
return new Response("unexpected");
|
||||
},
|
||||
guard: "public-only",
|
||||
}),
|
||||
/Blocked private or local provider URL/
|
||||
);
|
||||
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test("fetchRemoteImage blocks redirects to private image hosts", async () => {
|
||||
await assert.rejects(
|
||||
() =>
|
||||
fetchRemoteImage("https://cdn.example.com/redirect.png", {
|
||||
fetchImpl: async () =>
|
||||
new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "http://169.254.169.254/latest/meta-data" },
|
||||
}),
|
||||
guard: "public-only",
|
||||
}),
|
||||
/Blocked private or local provider URL/
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user