diff --git a/.env.example b/.env.example index 40ce624b7c..0ffc425c1b 100644 --- a/.env.example +++ b/.env.example @@ -1559,6 +1559,13 @@ APP_LOG_TO_FILE=true # DESIGNER_WEB_POLL_TIMEOUT_MS=60000 # Max wait for job completion (default: 60s) # DESIGNER_WEB_POLL_INTERVAL_MS=2000 # Poll frequency (default: 2s) +# ── Adobe Firefly (Image Upscale) ── +# Base delay (ms) for the submit-retry exponential backoff when Adobe Firefly's +# upscale job submission is rate-limited. Used by: +# open-sse/services/adobeFireflyUpscale.ts::submitRetryDelayMs. +# Default: 8000 (20 under NODE_ENV=test/VITEST/NODE_TEST_CONTEXT). +# ADOBE_FIREFLY_SUBMIT_BASE_DELAY_MS=8000 + # ── AWS Bedrock (Kiro / Audio) ── # Region used to construct AWS Bedrock endpoints. Used by: # src/lib/providers/validation.ts and open-sse/handlers/audioSpeech.ts. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 52b379ec42..96288962b9 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -850,6 +850,7 @@ Reverse-engineered session bridge for hyperagent.com (`src/shared/constants/prov | `NANOBANANA_POLL_INTERVAL_MS` | `2500` | `open-sse/handlers/imageGeneration.ts` | NanoBanana job polling frequency. | | `DESIGNER_WEB_POLL_TIMEOUT_MS` | `60000` | `open-sse/handlers/imageGeneration/providers/designerWeb.ts` | Max wait for microsoft-designer-web image generation jobs. | | `DESIGNER_WEB_POLL_INTERVAL_MS` | `2000` | `open-sse/handlers/imageGeneration/providers/designerWeb.ts` | microsoft-designer-web job polling frequency. | +| `ADOBE_FIREFLY_SUBMIT_BASE_DELAY_MS` | `8000` | `open-sse/services/adobeFireflyUpscale.ts` | Base delay for the Adobe Firefly upscale submit-retry exponential backoff. | | `AWS_REGION` | _(unset)_ | `src/lib/providers/validation.ts`, `open-sse/handlers/audioSpeech.ts` | Region used to construct AWS Bedrock endpoints (Kiro, audio). | | `AWS_DEFAULT_REGION` | _(unset)_ | `src/lib/providers/validation.ts`, `open-sse/handlers/audioSpeech.ts` | Fallback when `AWS_REGION` is not set. | | `CLOUDFLARE_ACCOUNT_ID` | _(unset)_ | `open-sse/executors/cloudflare-ai.ts` | Account ID for Cloudflare Workers AI. | diff --git a/open-sse/config/imageRegistry.ts b/open-sse/config/imageRegistry.ts index becb18e008..386dbf1485 100644 --- a/open-sse/config/imageRegistry.ts +++ b/open-sse/config/imageRegistry.ts @@ -711,6 +711,20 @@ export const IMAGE_PROVIDERS: Record = { name: "Firefly Runway Gen-4 Image", inputModalities: ["text", "image"], }, + // Topaz Labs upscalers (inputMediaUseCase: ["upscaling"]). + // Served by firefly-3p /v2/3p-images/upsample — see config/upscaleRegistry.ts. + { + id: "topaz-standard", + name: "Firefly Topaz Upscale (Standard)", + inputModalities: ["image"], + imageRequired: true, + }, + { + id: "topaz-bloom", + name: "Firefly Topaz Bloom (Creative Upscale)", + inputModalities: ["image"], + imageRequired: true, + }, ], supportedSizes: ["1:1", "16:9", "9:16", "4:3", "3:4", "1024x1024", "1792x1024", "1024x1792"], }, @@ -943,7 +957,6 @@ export function getImageModelAliases() { export function isRegisteredImageModel(providerId, modelId) { return Boolean(findImageModelConfig(providerId, modelId)); } - export function getImageModelEntry(modelStr) { if (!modelStr) return null; diff --git a/open-sse/config/upscaleRegistry.ts b/open-sse/config/upscaleRegistry.ts new file mode 100644 index 0000000000..fd383e5252 --- /dev/null +++ b/open-sse/config/upscaleRegistry.ts @@ -0,0 +1,228 @@ +/** + * Image Upscale Provider Registry + * + * Providers that serve `POST /v1/images/upscale` — image→image super-resolution. + * Upscaling is a distinct capability from generation: there is no text-to-image + * path, an input image is always mandatory, and the meaningful controls are the + * scale factor and (for generative upscalers) a creativity level. + * + * Only providers whose upscale API is already implemented here are listed: + * - adobe-firefly → Topaz models on firefly-3p `/v2/3p-images/upsample` + * - stability-ai → `/v2beta/stable-image/upscale/{fast,conservative,creative}` + * - topaz → Topaz Labs `/image/v1/enhance` (native API key) + * + * Credentials/proxy resolution reuses each provider's existing connection, so a + * configured Adobe Firefly / Stability AI / Topaz Labs account works with no + * extra setup. + */ + +import { parseModelFromRegistry, getAllModelsFromRegistry } from "./registryUtils.ts"; + +/** Scale factors offered by default when a model does not restrict them. */ +export const DEFAULT_UPSCALE_FACTORS: readonly number[] = Object.freeze([2, 4]); + +export interface UpscaleModelEntry { + id: string; + name: string; + /** Discrete scale factors the upstream accepts (in x). */ + factors: number[]; + /** Model exposes a creativity / re-imagine control (0-100 % on the wire-agnostic API). */ + supportsCreativity?: boolean; + /** Model accepts an optional guidance prompt. */ + supportsPrompt?: boolean; + /** Upstream rejects the request without a prompt. */ + promptRequired?: boolean; + description?: string; +} + +export interface UpscaleProviderConfig { + id: string; + alias?: string; + baseUrl: string; + authType: "apikey" | "none"; + authHeader: string; + format: "adobe-firefly-upscale" | "stability-upscale" | "topaz-upscale"; + models: UpscaleModelEntry[]; +} + +export const UPSCALE_PROVIDERS: Record = { + // Adobe Firefly (unofficial) — Topaz Labs models exposed through the Firefly 3P + // async upsample job API. Live capture: web_providers/upsample.txt. + // Discovery (web_providers/upscale.txt) lists modelId "topaz" with the image + // modelVersions default/standard/reimagine carrying inputMediaUseCase ["upscaling"]; + // starlight-*/astra-2 are video upscalers and intentionally excluded here. + "adobe-firefly": { + id: "adobe-firefly", + alias: "firefly", + baseUrl: "https://firefly-3p.ff.adobe.io/v2/3p-images/upsample", + authType: "apikey", + authHeader: "bearer", + format: "adobe-firefly-upscale", + models: [ + { + id: "topaz", + name: "Firefly Topaz Upscale", + factors: [2, 4], + description: "Topaz Labs detail-preserving upscale (standard).", + }, + { + id: "topaz-standard", + name: "Firefly Topaz Upscale (Standard)", + factors: [2, 4], + description: "Topaz Labs detail-preserving upscale — no invented detail.", + }, + { + id: "topaz-bloom", + name: "Firefly Topaz Bloom (Creative)", + factors: [2, 4], + supportsCreativity: true, + description: "Topaz Bloom generative upscale — creativity adds synthesized detail.", + }, + ], + }, + + // Stability AI stable-image upscale family. `fast` is a 4x deterministic pass; + // `conservative` and `creative` are prompt-guided (creative is an async job). + "stability-ai": { + id: "stability-ai", + baseUrl: "https://api.stability.ai", + authType: "apikey", + authHeader: "bearer", + format: "stability-upscale", + models: [ + { + id: "fast", + name: "Stability Fast Upscale (4x)", + factors: [4], + description: "Lightweight 4x upscale, no prompt.", + }, + { + id: "conservative", + name: "Stability Conservative Upscale", + factors: [4], + supportsPrompt: true, + promptRequired: true, + description: "Up to ~4 MP while preserving every detail. Prompt required upstream.", + }, + { + id: "creative", + name: "Stability Creative Upscale", + factors: [4], + supportsCreativity: true, + supportsPrompt: true, + promptRequired: true, + description: "Heavily reimagines low-quality inputs (async job). Prompt required upstream.", + }, + ], + }, + + // Topaz Labs native Image API (own api key, synchronous). + topaz: { + id: "topaz", + baseUrl: "https://api.topazlabs.com", + authType: "apikey", + authHeader: "x-api-key", + format: "topaz-upscale", + models: [ + { + id: "topaz-enhance", + name: "Topaz Labs Enhance", + factors: [2, 4], + description: "Topaz Labs Image Enhance (auto model selection).", + }, + ], + }, +}; + +export function getUpscaleProvider(providerId: string | null | undefined): UpscaleProviderConfig | null { + if (!providerId) return null; + return UPSCALE_PROVIDERS[providerId] || null; +} + +/** Parse `provider/model` (or a bare, unambiguous model id) against the upscale registry. */ +export function parseUpscaleModel(modelStr: string | null) { + return parseModelFromRegistry(modelStr, UPSCALE_PROVIDERS); +} + +/** Flat catalog for `GET /v1/images/upscale`. */ +export function getAllUpscaleModels() { + return getAllModelsFromRegistry(UPSCALE_PROVIDERS, (_providerId, config) => ({ + format: config.format, + })); +} + +/** Registry row for a `provider/model` string, or null when unknown. */ +export function getUpscaleModelEntry( + modelStr: string | null +): { provider: string; providerConfig: UpscaleProviderConfig; entry: UpscaleModelEntry } | null { + const { provider, model } = parseUpscaleModel(modelStr); + if (!provider || !model) return null; + const providerConfig = UPSCALE_PROVIDERS[provider]; + if (!providerConfig) return null; + const entry = providerConfig.models.find((m) => m.id === model); + if (!entry) return null; + return { provider, providerConfig, entry }; +} + +/** True when `provider/model` (or bare id) names a registered upscale model. */ +export function isRegisteredUpscaleModel(modelStr: string | null): boolean { + return getUpscaleModelEntry(modelStr) !== null; +} + +/** + * Normalize a requested scale factor to one the model actually supports. + * + * Accepts numbers and the loose strings clients send (`"2"`, `"2x"`, `"x4"`, `"4X"`). + * Unparseable/out-of-range values snap to the nearest allowed factor rather than + * failing the request — a 3x ask on a {2,4} model is better served at 4x than 400ed. + */ +export function normalizeUpscaleFactor( + value: unknown, + allowed: readonly number[] = DEFAULT_UPSCALE_FACTORS +): number { + const factors = allowed.length > 0 ? [...allowed] : [...DEFAULT_UPSCALE_FACTORS]; + const fallback = factors.includes(2) ? 2 : factors[0]!; + + let n: number = NaN; + if (typeof value === "number") { + n = value; + } else if (typeof value === "string") { + const match = /(\d+(?:\.\d+)?)/.exec(value.trim()); + if (match) n = Number(match[1]); + } + if (!Number.isFinite(n) || n <= 0) return fallback; + + let best = factors[0]!; + let bestDelta = Math.abs(factors[0]! - n); + for (const f of factors) { + const delta = Math.abs(f - n); + if (delta < bestDelta) { + best = f; + bestDelta = delta; + } + } + return best; +} + +/** + * Normalize a creativity input to a 0-100 percentage. + * + * The public API is percentage-based so every provider gets the same control + * regardless of its native scale (Firefly uses an integer level, Stability a + * 0.1-0.5 float). A fractional value strictly between 0 and 1 is read as a + * fraction (0.35 → 35 %); everything else is read as a percentage, so an + * integer `1` stays 1 % instead of silently becoming 100 %. + */ +export function normalizeCreativityPercent(value: unknown, fallback = 0): number { + let n: number = NaN; + if (typeof value === "number") n = value; + else if (typeof value === "string" && value.trim()) n = Number(value.trim().replace("%", "")); + if (!Number.isFinite(n)) return clampPercent(fallback); + if (n > 0 && n < 1) return clampPercent(n * 100); + return clampPercent(n); +} + +function clampPercent(n: number): number { + if (!Number.isFinite(n)) return 0; + return Math.max(0, Math.min(100, Math.round(n))); +} diff --git a/open-sse/handlers/imageGeneration/providers/adobeFirefly.ts b/open-sse/handlers/imageGeneration/providers/adobeFirefly.ts index 33f6990e51..24f2ba361d 100644 --- a/open-sse/handlers/imageGeneration/providers/adobeFirefly.ts +++ b/open-sse/handlers/imageGeneration/providers/adobeFirefly.ts @@ -21,6 +21,8 @@ import { resolveAdobeSourceImageIds, resolveAdobeImageModel, } from "../../../services/adobeFireflyClient.ts"; +import { isAdobeFireflyUpscaleModel } from "../../../services/adobeFireflyUpscale.ts"; +import { handleAdobeFireflyImageUpscale } from "../../imageUpscale/adobeFirefly.ts"; export async function handleAdobeFireflyImageGeneration({ model, @@ -54,6 +56,19 @@ export async function handleAdobeFireflyImageGeneration({ }) { const startTime = Date.now(); const prompt = typeof body.prompt === "string" ? body.prompt.trim() : ""; + + // Topaz upscalers share adobe-firefly but use /v2/3p-images/upsample (no prompt). + if (isAdobeFireflyUpscaleModel(model)) { + return handleAdobeFireflyImageUpscale({ + model, + provider, + body: body as Record, + credentials, + log, + fetchImpl, + }); + } + if (!prompt) { return saveImageErrorResult({ provider, diff --git a/open-sse/handlers/imageUpscale.ts b/open-sse/handlers/imageUpscale.ts new file mode 100644 index 0000000000..0c8956a18d --- /dev/null +++ b/open-sse/handlers/imageUpscale.ts @@ -0,0 +1,110 @@ +/** + * Image Upscale Handler + * + * Handles `POST /v1/images/upscale` — image→image super-resolution. + * + * Request (OpenAI-adjacent, deliberately minimal): + * { + * "model": "adobe-firefly/topaz-bloom", + * "image": "data:image/png;base64,...", // or image_url / http(s) URL + * "factor": 2, // 2 | 4 (snapped to what the model supports) + * "creativity": 40, // 0-100 % (generative upscalers only) + * "prompt": "…", // required by Stability conservative/creative + * "response_format": "url" | "b64_json" + * } + * + * Response is shaped like `/v1/images/generations` (`{ created, data: [{ url | b64_json }] }`) + * plus an `upscale` metadata block, so existing image clients need no changes. + */ + +import { getUpscaleProvider, parseUpscaleModel } from "../config/upscaleRegistry.ts"; +import { handleAdobeFireflyImageUpscale } from "./imageUpscale/adobeFirefly.ts"; +import { handleStabilityImageUpscale } from "./imageUpscale/stability.ts"; +import { handleTopazImageUpscale } from "./imageUpscale/topaz.ts"; +import type { + UpscaleCredentials, + UpscaleHandlerResult, + UpscaleLogger, +} from "./imageUpscale/shared.ts"; + +export type { UpscaleHandlerResult } from "./imageUpscale/shared.ts"; + +export async function handleImageUpscale({ + body, + credentials, + log, + fetchImpl, +}: { + body: Record; + credentials: UpscaleCredentials | null; + log?: UpscaleLogger; + fetchImpl?: typeof fetch; +}): Promise { + const requestedModel = typeof body.model === "string" ? body.model : ""; + const { provider, model } = parseUpscaleModel(requestedModel); + + if (!provider || !model) { + return { + success: false, + status: 400, + error: + `Invalid upscale model: ${requestedModel || "(missing)"}. ` + + `Use format: provider/model (e.g. adobe-firefly/topaz-bloom).`, + }; + } + + const providerConfig = getUpscaleProvider(provider); + if (!providerConfig) { + return { success: false, status: 400, error: `Unknown upscale provider: ${provider}` }; + } + + if (!providerConfig.models.some((entry) => entry.id === model)) { + return { + success: false, + status: 400, + error: + `Unsupported upscale model for ${provider}: ${model}. ` + + `Available: ${providerConfig.models.map((entry) => entry.id).join(", ")}.`, + }; + } + + const resolvedCredentials = credentials ?? {}; + + switch (providerConfig.format) { + case "adobe-firefly-upscale": + return handleAdobeFireflyImageUpscale({ + model, + provider, + body, + credentials: resolvedCredentials, + log, + ...(fetchImpl ? { fetchImpl } : {}), + }); + case "stability-upscale": + return handleStabilityImageUpscale({ + model, + provider, + providerConfig, + body, + credentials: resolvedCredentials, + log, + ...(fetchImpl ? { fetchImpl } : {}), + }); + case "topaz-upscale": + return handleTopazImageUpscale({ + model, + provider, + providerConfig, + body, + credentials: resolvedCredentials, + log, + ...(fetchImpl ? { fetchImpl } : {}), + }); + default: + return { + success: false, + status: 400, + error: `Upscale is not implemented for provider format: ${providerConfig.format}`, + }; + } +} diff --git a/open-sse/handlers/imageUpscale/adobeFirefly.ts b/open-sse/handlers/imageUpscale/adobeFirefly.ts new file mode 100644 index 0000000000..63b8b7c277 --- /dev/null +++ b/open-sse/handlers/imageUpscale/adobeFirefly.ts @@ -0,0 +1,177 @@ +/** + * Adobe Firefly upscale handler — Topaz Labs models on firefly-3p `/v2/3p-images/upsample`. + * + * Flow (mirrors the SPA and the Firefly generate path): + * 1. Resolve the durable session (JWT + Cookie → ARP rebuild, sticky ARP, submit gate). + * 2. Upload the source image to `/v2/storage/image` → blob id, reusing that ARP. + * 3. POST the upsample job, poll the BKS result link, return the presigned URL. + */ + +import { + AdobeFireflyError, + resolveAdobeAccessToken, + resolveAdobeSourceImageIds, +} from "../../services/adobeFireflyClient.ts"; +import { + adobeFireflyUpscaleImage, + resolveAdobeUpscaleModel, +} from "../../services/adobeFireflyUpscale.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; +import { + extractUpscaleSourceImage, + saveUpscaleErrorResult, + saveUpscaleSuccessResult, + type UpscaleCredentials, + type UpscaleHandlerResult, + type UpscaleLogger, +} from "./shared.ts"; + +export async function handleAdobeFireflyImageUpscale({ + model, + provider, + body, + credentials, + log, + fetchImpl = fetch, +}: { + model: string; + provider: string; + body: Record; + credentials: UpscaleCredentials; + log?: UpscaleLogger; + fetchImpl?: typeof fetch; +}): Promise { + const startTime = Date.now(); + + const resolved = resolveAdobeUpscaleModel(model); + if (!resolved) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: `Unsupported Adobe Firefly upscale model: ${model}. Use topaz-standard or topaz-bloom.`, + }); + } + + if (!extractUpscaleSourceImage(body)) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: "Adobe Firefly upscale requires a source image", + }); + } + + try { + const accessToken = await resolveAdobeAccessToken(credentials, fetchImpl); + // Keep the raw credential blob for Cookie + sherlockToken (x-arp-session-id). + const psd = (credentials as { providerSpecificData?: { cookie?: string } })?.providerSpecificData; + const sessionCookie = + (typeof psd?.cookie === "string" && psd.cookie.trim()) || + (typeof credentials?.apiKey === "string" && credentials.apiKey.trim()) || + (typeof credentials?.accessToken === "string" && credentials.accessToken.includes(";") + ? credentials.accessToken + : undefined); + + // Upscale consumes exactly one source; upload it under the same ARP as submit. + const blobIds = await resolveAdobeSourceImageIds({ + accessToken, + body, + max: 1, + sessionCookie, + prompt: "upsample", + fetchImpl, + log, + }); + + if (blobIds.length === 0) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: "Adobe Firefly upscale could not resolve the source image", + }); + } + + const timeoutMs = normalizePositiveNumber(body.timeout_ms, 0); + const result = await adobeFireflyUpscaleImage({ + accessToken, + model, + blobId: blobIds[0]!, + upsamplerFactor: readFactor(body), + creativityPercent: readCreativityPercent(body), + creativityLevel: body.creativity_level ?? body.creativityLevel, + sessionCookie, + ...(timeoutMs > 0 ? { timeoutMs } : {}), + fetchImpl, + log, + }); + + log?.info?.( + "IMAGE", + `${provider}/${model} (adobe-firefly upsample) | ${result.factor}x` + + (resolved.spec.supportsCreativity ? ` | creativityLevel=${result.creativityLevel}` : "") + ); + + return saveUpscaleSuccessResult({ + provider, + model, + startTime, + images: [{ url: result.url }], + meta: { + provider, + model, + factor: result.factor, + ...(resolved.spec.supportsCreativity ? { creativity_level: result.creativityLevel } : {}), + }, + }); + } catch (err) { + if (err instanceof AdobeFireflyError) { + log?.error?.("IMAGE", `${provider} adobe-firefly upscale error ${err.status}: ${err.message}`); + return saveUpscaleErrorResult({ + provider, + model, + status: err.status, + startTime, + error: err.message, + }); + } + const errorText = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + log?.error?.("IMAGE", `${provider} adobe-firefly upscale exception: ${errorText}`); + return saveUpscaleErrorResult({ + provider, + model, + status: 500, + startTime, + error: errorText, + }); + } +} + +function readFactor(body: Record): unknown { + return ( + body.factor ?? + body.scale ?? + body.upscale_factor ?? + body.upscaleFactor ?? + body.upsampler_factor ?? + body.upsamplerFactor + ); +} + +function readCreativityPercent(body: Record): number | undefined { + const raw = body.creativity ?? body.creativity_percent ?? body.creativityPercent; + if (raw === undefined || raw === null) return undefined; + const n = typeof raw === "number" ? raw : Number(String(raw).replace("%", "").trim()); + if (!Number.isFinite(n)) return undefined; + if (n > 0 && n < 1) return Math.max(0, Math.min(100, n * 100)); + return Math.max(0, Math.min(100, n)); +} + +function normalizePositiveNumber(value: unknown, fallback: number): number { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} diff --git a/open-sse/handlers/imageUpscale/shared.ts b/open-sse/handlers/imageUpscale/shared.ts new file mode 100644 index 0000000000..cf37e99910 --- /dev/null +++ b/open-sse/handlers/imageUpscale/shared.ts @@ -0,0 +1,391 @@ +/** + * Shared plumbing for the `/v1/images/upscale` provider handlers. + * + * Kept separate from `handlers/imageGeneration.ts` on purpose: upscaling needs raw + * source bytes + pixel dimensions (to turn a 2x/4x factor into an output size for + * providers that only accept absolute targets), neither of which the generation + * handler exposes. + */ + +import { saveCallLog } from "@/lib/usageDb"; +import { fetchRemoteImage } from "@/shared/network/remoteImageFetch"; + +export const UPSCALE_CALL_LOG_PATH = "/v1/images/upscale"; + +/** Hard cap on a decoded source image (matches the Firefly storage upload limit). */ +export const MAX_UPSCALE_SOURCE_BYTES = 20 * 1024 * 1024; + +export interface UpscaleImageSource { + buffer: Buffer; + base64: string; + contentType: string; +} + +export interface UpscaleHandlerResult { + success: boolean; + status?: number; + error?: unknown; + data?: unknown; +} + +export interface UpscaleLogger { + info?: (scope: string, message: string) => void; + error?: (scope: string, message: string) => void; +} + +/** + * Credential shape the upscale handlers need. Mirrors what + * `getProviderCredentialsWithQuotaPreflight` yields for these providers: an API key or + * access token, plus (for Adobe Firefly) the connection's `providerSpecificData`, which + * is where a pasted firefly.adobe.com Cookie lives. + */ +export interface UpscaleCredentials { + apiKey?: string; + accessToken?: string; + providerSpecificData?: { + cookie?: unknown; + access_token?: unknown; + accessToken?: unknown; + } | null; +} + +/** + * `Buffer` is typed as `Buffer`, which TypeScript will not accept as a + * `BlobPart` (a Blob part must be backed by a plain `ArrayBuffer`). Copy the bytes into a + * fresh `ArrayBuffer` so multipart bodies typecheck without an unsafe cast. + */ +export function toBlobBytes(buffer: Buffer): ArrayBuffer { + const out = new ArrayBuffer(buffer.byteLength); + new Uint8Array(out).set(buffer); + return out; +} + +/** + * Collect the source image from an OpenAI-ish / Media-page body. + * + * Only ONE image is meaningful for an upscale, so the first resolvable candidate + * wins. Field order mirrors `extractAdobeSourceImageSources` so a body built for + * generation keeps working here. + */ +export function extractUpscaleSourceImage(body: unknown): string | null { + if (!body || typeof body !== "object") return null; + const b = body as Record; + const providerOptions = + b.provider_options && typeof b.provider_options === "object" && !Array.isArray(b.provider_options) + ? (b.provider_options as Record) + : {}; + + const keys = [ + "image_url", + "imageUrl", + "input_image", + "source_image", + "promptImage", + "prompt_image", + "image", + "images", + "image_urls", + "imageUrls", + "input_images", + "reference_images", + "referenceImages", + "reference_image", + ]; + + for (const key of keys) { + const found = firstImageCandidate(b[key]) || firstImageCandidate(providerOptions[key]); + if (found) return found; + } + + if (Array.isArray(b.messages)) { + for (const msg of b.messages) { + if (!msg || typeof msg !== "object") continue; + const content = (msg as Record).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (!part || typeof part !== "object") continue; + const p = part as Record; + if (p.type === "image_url" || p.type === "image") { + const found = firstImageCandidate(p.image_url ?? p.image ?? p.url); + if (found) return found; + } + } + } + } + + return null; +} + +function firstImageCandidate(value: unknown): string | null { + if (typeof value === "string") { + const trimmed = value.trim(); + if (!trimmed || trimmed === "null" || trimmed === "undefined") return null; + return trimmed; + } + if (Array.isArray(value)) { + for (const item of value) { + const found = firstImageCandidate(item); + if (found) return found; + } + return null; + } + if (value && typeof value === "object") { + const o = value as Record; + if (typeof o.url === "string") return firstImageCandidate(o.url); + if (typeof o.image_url === "string") return firstImageCandidate(o.image_url); + if (o.image_url && typeof o.image_url === "object") { + return firstImageCandidate((o.image_url as Record).url); + } + if (typeof o.b64_json === "string") return `data:image/png;base64,${o.b64_json}`; + if (typeof o.base64 === "string") return `data:image/png;base64,${o.base64}`; + } + return null; +} + +/** Decode a data URL / http(s) URL / bare base64 string into bytes. */ +export async function resolveUpscaleImageSource(source: string): Promise { + const trimmed = String(source || "").trim(); + if (!trimmed) throw new Error("Invalid image source"); + + const dataUri = /^data:([^;,]+)?(?:;charset=[^;,]+)?;base64,([\s\S]+)$/i.exec(trimmed); + if (dataUri) { + const contentType = (dataUri[1] || "image/png").trim().toLowerCase(); + const base64 = (dataUri[2] || "").replace(/\s/g, ""); + const buffer = Buffer.from(base64, "base64"); + assertSourceBytes(buffer); + return { + buffer, + base64, + contentType: contentType.startsWith("image/") ? contentType : "image/png", + }; + } + + if (/^https?:\/\//i.test(trimmed)) { + const remote = await fetchRemoteImage(trimmed); + assertSourceBytes(remote.buffer); + // fetchRemoteImage falls back to application/octet-stream; sniff whenever the + // server did not send a usable image/* type so multipart uploads stay correct. + const declared = (remote.contentType || "").split(";")[0]!.trim().toLowerCase(); + return { + buffer: remote.buffer, + base64: remote.buffer.toString("base64"), + contentType: declared.startsWith("image/") ? declared : sniffImageMime(remote.buffer), + }; + } + + const buffer = Buffer.from(trimmed.replace(/\s/g, ""), "base64"); + assertSourceBytes(buffer); + return { buffer, base64: buffer.toString("base64"), contentType: sniffImageMime(buffer) }; +} + +function assertSourceBytes(buffer: Buffer): void { + if (!buffer.length) throw new Error("Source image decoded to empty bytes"); + if (buffer.length > MAX_UPSCALE_SOURCE_BYTES) { + throw new Error( + `Source image too large (${buffer.length} bytes; max ${MAX_UPSCALE_SOURCE_BYTES})` + ); + } +} + +/** Best-effort MIME sniff from the magic bytes (falls back to PNG). */ +export function sniffImageMime(buffer: Buffer): string { + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return "image/jpeg"; + } + if (buffer.length >= 8 && buffer[0] === 0x89 && buffer.toString("ascii", 1, 4) === "PNG") { + return "image/png"; + } + if (buffer.length >= 6 && buffer.toString("ascii", 0, 3) === "GIF") return "image/gif"; + if ( + buffer.length >= 12 && + buffer.toString("ascii", 0, 4) === "RIFF" && + buffer.toString("ascii", 8, 12) === "WEBP" + ) { + return "image/webp"; + } + if (buffer.length >= 2 && buffer.toString("ascii", 0, 2) === "BM") return "image/bmp"; + return "image/png"; +} + +/** + * Read pixel dimensions straight from the container header — no image library needed. + * Supports PNG, JPEG (SOFn scan), GIF, WebP (VP8 / VP8L / VP8X) and BMP. + * Returns null when the format is unknown or the header is truncated. + */ +export function readImageDimensions(buffer: Buffer): { width: number; height: number } | null { + try { + if ( + buffer.length >= 24 && + buffer[0] === 0x89 && + buffer.toString("ascii", 1, 4) === "PNG" + ) { + // IHDR is always the first chunk: 8-byte signature + 4 length + 4 "IHDR". + return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) }; + } + + if (buffer.length >= 6 && buffer.toString("ascii", 0, 3) === "GIF") { + return { width: buffer.readUInt16LE(6), height: buffer.readUInt16LE(8) }; + } + + if (buffer.length >= 26 && buffer.toString("ascii", 0, 2) === "BM") { + return { width: buffer.readInt32LE(18), height: Math.abs(buffer.readInt32LE(22)) }; + } + + if ( + buffer.length >= 30 && + buffer.toString("ascii", 0, 4) === "RIFF" && + buffer.toString("ascii", 8, 12) === "WEBP" + ) { + return readWebpDimensions(buffer); + } + + if (buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8) { + return readJpegDimensions(buffer); + } + } catch { + return null; + } + return null; +} + +function readWebpDimensions(buffer: Buffer): { width: number; height: number } | null { + const chunk = buffer.toString("ascii", 12, 16); + if (chunk === "VP8 " && buffer.length >= 30) { + // Lossy: 3-byte frame tag + 3-byte sync code, then 14-bit width/height. + return { + width: buffer.readUInt16LE(26) & 0x3fff, + height: buffer.readUInt16LE(28) & 0x3fff, + }; + } + if (chunk === "VP8L" && buffer.length >= 25) { + const bits = buffer.readUInt32LE(21); + return { width: (bits & 0x3fff) + 1, height: ((bits >> 14) & 0x3fff) + 1 }; + } + if (chunk === "VP8X" && buffer.length >= 30) { + const width = 1 + (buffer[24]! | (buffer[25]! << 8) | (buffer[26]! << 16)); + const height = 1 + (buffer[27]! | (buffer[28]! << 8) | (buffer[29]! << 16)); + return { width, height }; + } + return null; +} + +function readJpegDimensions(buffer: Buffer): { width: number; height: number } | null { + let offset = 2; + while (offset + 9 < buffer.length) { + if (buffer[offset] !== 0xff) { + offset += 1; + continue; + } + const marker = buffer[offset + 1]!; + // Standalone markers (no length payload). + if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { + offset += 2; + continue; + } + const length = buffer.readUInt16BE(offset + 2); + // SOF0..SOF15 except DHT(c4)/JPGA(c8)/DAC(cc) carry the frame dimensions. + const isSof = + marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; + if (isSof) { + return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) }; + } + if (length <= 0) return null; + offset += 2 + length; + } + return null; +} + +/** + * Absolute output size for a scale factor, clamped to `maxEdge` so a 4x pass on an + * already-large source cannot ask for an impossible canvas. Returns null when the + * source dimensions could not be read. + */ +export function scaleDimensions( + buffer: Buffer, + factor: number, + maxEdge = 32000 +): { width: number; height: number } | null { + const source = readImageDimensions(buffer); + if (!source || source.width <= 0 || source.height <= 0) return null; + const safeFactor = Number.isFinite(factor) && factor > 0 ? factor : 2; + const scale = Math.min( + safeFactor, + maxEdge / Math.max(source.width, source.height) + ); + return { + width: Math.max(1, Math.round(source.width * Math.max(1, scale))), + height: Math.max(1, Math.round(source.height * Math.max(1, scale))), + }; +} + +/** OpenAI-images-shaped success envelope + call log. */ +export function saveUpscaleSuccessResult(opts: { + provider: string; + model: string; + startTime: number; + images: Array>; + requestBody?: unknown; + responseBody?: unknown; + meta?: Record; +}): UpscaleHandlerResult { + saveCallLog({ + method: "POST", + path: UPSCALE_CALL_LOG_PATH, + status: 200, + model: `${opts.provider}/${opts.model}`, + provider: opts.provider, + duration: Date.now() - opts.startTime, + requestBody: opts.requestBody ?? null, + responseBody: opts.responseBody ?? { images_count: opts.images.length }, + }).catch(() => {}); + + return { + success: true, + data: { + created: Math.floor(Date.now() / 1000), + data: opts.images, + ...(opts.meta ? { upscale: opts.meta } : {}), + }, + }; +} + +export function saveUpscaleErrorResult(opts: { + provider: string; + model: string; + status: number; + startTime: number; + error: unknown; + requestBody?: unknown; +}): UpscaleHandlerResult { + saveCallLog({ + method: "POST", + path: UPSCALE_CALL_LOG_PATH, + status: opts.status, + model: `${opts.provider}/${opts.model}`, + provider: opts.provider, + duration: Date.now() - opts.startTime, + error: + typeof opts.error === "string" + ? opts.error.slice(0, 500) + : String(opts.error).slice(0, 500), + requestBody: opts.requestBody ?? null, + }).catch(() => {}); + + return { success: false, status: opts.status, error: opts.error }; +} + +/** `{ url }` or `{ b64_json }` depending on the requested response_format. */ +export function buildUpscaleImageEntry(opts: { + buffer?: Buffer | null; + contentType?: string; + url?: string | null; + responseFormat?: unknown; +}): Record { + const wantsBase64 = String(opts.responseFormat ?? "").toLowerCase() === "b64_json"; + if (opts.buffer && opts.buffer.length > 0) { + const base64 = opts.buffer.toString("base64"); + const mime = opts.contentType || sniffImageMime(opts.buffer); + return wantsBase64 ? { b64_json: base64 } : { url: `data:${mime};base64,${base64}` }; + } + return { url: String(opts.url || "") }; +} diff --git a/open-sse/handlers/imageUpscale/stability.ts b/open-sse/handlers/imageUpscale/stability.ts new file mode 100644 index 0000000000..4b323ea222 --- /dev/null +++ b/open-sse/handlers/imageUpscale/stability.ts @@ -0,0 +1,335 @@ +/** + * Stability AI upscale handler — `/v2beta/stable-image/upscale/{fast,conservative,creative}`. + * + * Wire contract (platform.stability.ai): + * - all three take multipart/form-data with an `image` part + * - `Accept: application/json` → `{ image: , finish_reason, seed }` + * - `fast` : no prompt, fixed 4x + * - `conservative` : prompt REQUIRED, `creativity` 0.2-0.5 (default 0.35), synchronous + * - `creative` : prompt REQUIRED, `creativity` 0-0.35 (default 0.3), **async** — + * responds `{ id }`, then `GET /v2beta/results/{id}` returns 202 while + * running and 200 with the base64 image when finished. + * + * The generation handler's stability path does not poll, so the async `creative` + * variant is implemented here rather than delegated. + */ + +import { + buildUpscaleImageEntry, + extractUpscaleSourceImage, + resolveUpscaleImageSource, + saveUpscaleErrorResult, + saveUpscaleSuccessResult, + toBlobBytes, + type UpscaleCredentials, + type UpscaleHandlerResult, + type UpscaleLogger, +} from "./shared.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; + +const UPSCALE_ENDPOINTS: Record = { + fast: "/v2beta/stable-image/upscale/fast", + conservative: "/v2beta/stable-image/upscale/conservative", + creative: "/v2beta/stable-image/upscale/creative", +}; + +/** Documented `creativity` range per model — a 0-100 % request is mapped into it. */ +const CREATIVITY_RANGES: Record = { + conservative: { min: 0.2, max: 0.5, fallback: 0.35 }, + creative: { min: 0, max: 0.35, fallback: 0.3 }, +}; + +/** Models whose upstream rejects a request without a prompt. */ +const PROMPT_REQUIRED = new Set(["conservative", "creative"]); + +/** `creative` is an async job. */ +const ASYNC_MODELS = new Set(["creative"]); + +const RESULT_POLL_INTERVAL_MS = 3000; +const DEFAULT_RESULT_TIMEOUT_MS = 300_000; +const ALLOWED_OUTPUT_FORMATS = ["png", "jpeg", "webp"]; + +export async function handleStabilityImageUpscale({ + model, + provider, + providerConfig, + body, + credentials, + log, + fetchImpl = fetch, +}: { + model: string; + provider: string; + providerConfig: { baseUrl: string }; + body: Record; + credentials: UpscaleCredentials; + log?: UpscaleLogger; + fetchImpl?: typeof fetch; +}): Promise { + const startTime = Date.now(); + const endpoint = UPSCALE_ENDPOINTS[model]; + if (!endpoint) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: `Unsupported Stability AI upscale model: ${model}. Use fast, conservative or creative.`, + }); + } + + const token = credentials.apiKey || credentials.accessToken; + if (!token) { + return saveUpscaleErrorResult({ + provider, + model, + status: 401, + startTime, + error: "Missing Stability AI API key", + }); + } + + const source = extractUpscaleSourceImage(body); + if (!source) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: `Stability AI upscale model ${model} requires a source image`, + }); + } + + const prompt = typeof body.prompt === "string" ? body.prompt.trim() : ""; + if (PROMPT_REQUIRED.has(model) && !prompt) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: + `Stability AI "${model}" upscale requires a prompt describing the image. ` + + `Use the "fast" model for a prompt-free 4x upscale.`, + }); + } + + const outputFormat = normalizeOutputFormat(body.output_format ?? body.format); + const creativity = CREATIVITY_RANGES[model] + ? mapCreativity(body, CREATIVITY_RANGES[model]!) + : null; + + const requestSummary: Record = { model, output_format: outputFormat }; + if (prompt) requestSummary.prompt = prompt; + if (creativity !== null) requestSummary.creativity = creativity; + + try { + const imageSource = await resolveUpscaleImageSource(source); + + const formData = new FormData(); + formData.append( + "image", + new Blob([toBlobBytes(imageSource.buffer)], { type: imageSource.contentType || "image/png" }), + "image" + ); + formData.append("output_format", outputFormat); + if (prompt) formData.append("prompt", prompt); + if (typeof body.negative_prompt === "string" && body.negative_prompt.trim()) { + formData.append("negative_prompt", body.negative_prompt.trim()); + } + if (creativity !== null) formData.append("creativity", String(creativity)); + if (body.seed !== undefined && body.seed !== null && String(body.seed).trim()) { + formData.append("seed", String(body.seed)); + } + if (typeof body.style_preset === "string" && body.style_preset.trim()) { + formData.append("style_preset", body.style_preset.trim()); + } + + log?.info?.( + "IMAGE", + `${provider}/${model} (stability upscale)` + + (creativity !== null ? ` | creativity=${creativity}` : "") + + ` | output=${outputFormat}` + ); + + const baseUrl = providerConfig.baseUrl.replace(/\/$/, ""); + const response = await fetchImpl(`${baseUrl}${endpoint}`, { + method: "POST", + headers: { Accept: "application/json", Authorization: `Bearer ${token}` }, + body: formData, + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => ""); + log?.error?.( + "IMAGE", + `${provider} stability upscale error ${response.status}: ${errorText.slice(0, 200)}` + ); + return saveUpscaleErrorResult({ + provider, + model, + status: response.status, + startTime, + error: errorText || `HTTP ${response.status}`, + requestBody: requestSummary, + }); + } + + const payload = (await response.json().catch(() => ({}))) as Record; + + let finalPayload = payload; + if (ASYNC_MODELS.has(model) && typeof payload.id === "string" && payload.id) { + finalPayload = await pollStabilityResult({ + baseUrl, + token, + id: payload.id, + timeoutMs: normalizePositiveNumber(body.timeout_ms, DEFAULT_RESULT_TIMEOUT_MS), + fetchImpl, + log, + }); + } + + const finishReason = String(finalPayload.finish_reason ?? "").toUpperCase(); + if (finishReason === "CONTENT_FILTERED") { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: "Stability AI filtered the upscale result (CONTENT_FILTERED)", + requestBody: requestSummary, + }); + } + + const base64 = typeof finalPayload.image === "string" ? finalPayload.image : ""; + if (!base64) { + return saveUpscaleErrorResult({ + provider, + model, + status: 502, + startTime, + error: "Stability AI upscale returned no image", + requestBody: requestSummary, + }); + } + + const buffer = Buffer.from(base64, "base64"); + return saveUpscaleSuccessResult({ + provider, + model, + startTime, + requestBody: requestSummary, + images: [ + buildUpscaleImageEntry({ + buffer, + contentType: `image/${outputFormat === "jpeg" ? "jpeg" : outputFormat}`, + responseFormat: body.response_format, + }), + ], + meta: { + provider, + model, + factor: 4, + ...(creativity !== null ? { creativity } : {}), + ...(finalPayload.seed !== undefined ? { seed: finalPayload.seed } : {}), + }, + }); + } catch (err) { + const errorText = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + log?.error?.("IMAGE", `${provider} stability upscale exception: ${errorText}`); + return saveUpscaleErrorResult({ + provider, + model, + status: 502, + startTime, + error: `Image upscale provider error: ${errorText}`, + requestBody: requestSummary, + }); + } +} + +/** Poll `GET /v2beta/results/{id}` until the async creative upscale finishes. */ +async function pollStabilityResult(opts: { + baseUrl: string; + token: string; + id: string; + timeoutMs: number; + fetchImpl: typeof fetch; + log?: UpscaleLogger; +}): Promise> { + const deadline = Date.now() + opts.timeoutMs; + let attempt = 0; + + while (Date.now() < deadline) { + attempt += 1; + const response = await opts.fetchImpl( + `${opts.baseUrl}/v2beta/results/${encodeURIComponent(opts.id)}`, + { + method: "GET", + headers: { Accept: "application/json", Authorization: `Bearer ${opts.token}` }, + } + ); + + if (response.status === 202) { + opts.log?.info?.("IMAGE", `stability creative upscale pending #${attempt}`); + await sleep(RESULT_POLL_INTERVAL_MS); + continue; + } + + if (!response.ok) { + const text = await response.text().catch(() => ""); + if (response.status === 429 || response.status >= 500) { + await sleep(RESULT_POLL_INTERVAL_MS); + continue; + } + throw new Error( + `Stability AI upscale result failed (${response.status}): ${text.slice(0, 300)}` + ); + } + + return (await response.json().catch(() => ({}))) as Record; + } + + throw new Error("Stability AI creative upscale timed out"); +} + +function normalizeOutputFormat(value: unknown): string { + const raw = String(value ?? "").trim().toLowerCase(); + if (raw === "jpg") return "jpeg"; + return ALLOWED_OUTPUT_FORMATS.includes(raw) ? raw : "png"; +} + +/** + * Map the API's 0-100 % creativity onto the model's documented float range. + * An explicit in-range float (`creativity: 0.4`) is passed through untouched so + * power users keep exact control. + */ +function mapCreativity( + body: Record, + range: { min: number; max: number; fallback: number } +): number { + const raw = body.creativity ?? body.creativity_percent ?? body.creativityPercent; + if (raw === undefined || raw === null || String(raw).trim() === "") return range.fallback; + + const n = typeof raw === "number" ? raw : Number(String(raw).replace("%", "").trim()); + if (!Number.isFinite(n)) return range.fallback; + + // Values that already look like a native Stability creativity float (< 1 and not a + // whole percent) are honored as-is, clamped to the documented range. + if (n > 0 && n < 1) return round2(Math.max(range.min, Math.min(range.max, n))); + + const percent = Math.max(0, Math.min(100, n)); + return round2(range.min + ((range.max - range.min) * percent) / 100); +} + +function round2(n: number): number { + return Math.round(n * 100) / 100; +} + +function normalizePositiveNumber(value: unknown, fallback: number): number { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +async function sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/open-sse/handlers/imageUpscale/topaz.ts b/open-sse/handlers/imageUpscale/topaz.ts new file mode 100644 index 0000000000..100102a37b --- /dev/null +++ b/open-sse/handlers/imageUpscale/topaz.ts @@ -0,0 +1,271 @@ +/** + * Topaz Labs upscale handler — native Image API `POST /image/v1/enhance`. + * + * Wire contract (docs.topazlabs.com Image API v1): + * headers: X-API-Key: , accept: image/ + * multipart/form-data: + * image (required) source bytes + * model (optional) e.g. "Standard V2" / "High Fidelity V2" / "Low Resolution V2" + * output_width (optional) absolute target width + * output_height (optional) absolute target height + * output_format (optional) jpeg | png | webp + * sharpen / denoise / fix_compression (optional) 0-1 strengths + * face_enhancement (optional) boolean + * → raw image bytes of the enhanced result. + * + * The endpoint only accepts an ABSOLUTE target size, so a 2x/4x factor is turned into + * `output_width`/`output_height` by reading the source dimensions out of the container + * header (`scaleDimensions`). When the dimensions cannot be read the factor is dropped + * and Topaz's own default upscale applies, rather than failing the request. + */ + +import { + buildUpscaleImageEntry, + extractUpscaleSourceImage, + resolveUpscaleImageSource, + saveUpscaleErrorResult, + saveUpscaleSuccessResult, + scaleDimensions, + sniffImageMime, + toBlobBytes, + type UpscaleCredentials, + type UpscaleHandlerResult, + type UpscaleLogger, +} from "./shared.ts"; +import { sanitizeErrorMessage } from "../../utils/error.ts"; + +/** Topaz caps a single output edge well below this; keeps a 4x pass on a huge source sane. */ +const MAX_OUTPUT_EDGE = 16000; +const ALLOWED_OUTPUT_FORMATS = ["png", "jpeg", "webp"]; + +export async function handleTopazImageUpscale({ + model, + provider, + providerConfig, + body, + credentials, + log, + fetchImpl = fetch, +}: { + model: string; + provider: string; + providerConfig: { baseUrl: string }; + body: Record; + credentials: UpscaleCredentials; + log?: UpscaleLogger; + fetchImpl?: typeof fetch; +}): Promise { + const startTime = Date.now(); + const token = credentials.apiKey || credentials.accessToken; + if (!token) { + return saveUpscaleErrorResult({ + provider, + model, + status: 401, + startTime, + error: "Missing Topaz Labs API key", + }); + } + + const source = extractUpscaleSourceImage(body); + if (!source) { + return saveUpscaleErrorResult({ + provider, + model, + status: 400, + startTime, + error: `Topaz Labs upscale model ${model} requires a source image`, + }); + } + + const factor = normalizeFactor(body); + const outputFormat = normalizeOutputFormat(body.output_format ?? body.format); + const requestSummary: Record = { model, factor, output_format: outputFormat }; + + try { + const imageSource = await resolveUpscaleImageSource(source); + + const formData = new FormData(); + formData.append( + "image", + new Blob([toBlobBytes(imageSource.buffer)], { type: imageSource.contentType || "image/png" }), + "image" + ); + formData.append("output_format", outputFormat); + + const explicitSize = parseExplicitSize(body.size ?? body.output_size); + const target = explicitSize ?? scaleDimensions(imageSource.buffer, factor, MAX_OUTPUT_EDGE); + if (target) { + formData.append("output_width", String(target.width)); + formData.append("output_height", String(target.height)); + requestSummary.output_width = target.width; + requestSummary.output_height = target.height; + } else { + log?.info?.( + "IMAGE", + `${provider}/${model} (topaz upscale) | source dimensions unknown — using Topaz default scale` + ); + } + + const topazModel = typeof body.topaz_model === "string" ? body.topaz_model.trim() : ""; + if (topazModel) { + formData.append("model", topazModel); + requestSummary.topaz_model = topazModel; + } + + appendUnitFloat(formData, "sharpen", body.sharpen, requestSummary); + appendUnitFloat(formData, "denoise", body.denoise, requestSummary); + appendUnitFloat(formData, "fix_compression", body.fix_compression, requestSummary); + + if (body.face_enhancement !== undefined && body.face_enhancement !== null) { + const enabled = toBoolean(body.face_enhancement); + formData.append("face_enhancement", enabled ? "true" : "false"); + requestSummary.face_enhancement = enabled; + // Topaz exposes creativity/strength only when face enhancement is on. + if (enabled) { + appendUnitFloat( + formData, + "face_enhancement_creativity", + body.creativity ?? body.face_enhancement_creativity, + requestSummary, + /* percentAware */ true + ); + appendUnitFloat( + formData, + "face_enhancement_strength", + body.face_enhancement_strength, + requestSummary + ); + } + } + + log?.info?.( + "IMAGE", + `${provider}/${model} (topaz upscale) | ${factor}x` + + (target ? ` → ${target.width}x${target.height}` : "") + + ` | output=${outputFormat}` + ); + + const baseUrl = providerConfig.baseUrl.replace(/\/$/, ""); + const response = await fetchImpl(`${baseUrl}/image/v1/enhance`, { + method: "POST", + headers: { + Accept: `image/${outputFormat}`, + "X-API-Key": token, + }, + body: formData, + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => ""); + log?.error?.( + "IMAGE", + `${provider} topaz upscale error ${response.status}: ${errorText.slice(0, 200)}` + ); + return saveUpscaleErrorResult({ + provider, + model, + status: response.status, + startTime, + error: errorText || `HTTP ${response.status}`, + requestBody: requestSummary, + }); + } + + const buffer = Buffer.from(await response.arrayBuffer()); + if (!buffer.length) { + return saveUpscaleErrorResult({ + provider, + model, + status: 502, + startTime, + error: "Topaz Labs upscale returned an empty body", + requestBody: requestSummary, + }); + } + + const declared = (response.headers.get("content-type") || "").split(";")[0]!.trim().toLowerCase(); + const contentType = declared.startsWith("image/") ? declared : sniffImageMime(buffer); + + return saveUpscaleSuccessResult({ + provider, + model, + startTime, + requestBody: requestSummary, + images: [ + buildUpscaleImageEntry({ buffer, contentType, responseFormat: body.response_format }), + ], + meta: { provider, model, factor, ...(target ? { width: target.width, height: target.height } : {}) }, + }); + } catch (err) { + const errorText = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + log?.error?.("IMAGE", `${provider} topaz upscale exception: ${errorText}`); + return saveUpscaleErrorResult({ + provider, + model, + status: 502, + startTime, + error: `Image upscale provider error: ${errorText}`, + requestBody: requestSummary, + }); + } +} + +function normalizeFactor(body: Record): number { + const raw = + body.factor ?? + body.scale ?? + body.upscale_factor ?? + body.upscaleFactor ?? + body.upsampler_factor ?? + body.upsamplerFactor; + let n = typeof raw === "number" ? raw : Number(String(raw ?? "").replace(/[^\d.]/g, "")); + if (!Number.isFinite(n) || n <= 0) return 2; + return Math.abs(n - 4) < Math.abs(n - 2) ? 4 : 2; +} + +function normalizeOutputFormat(value: unknown): string { + const raw = String(value ?? "").trim().toLowerCase(); + if (raw === "jpg") return "jpeg"; + return ALLOWED_OUTPUT_FORMATS.includes(raw) ? raw : "png"; +} + +function parseExplicitSize(value: unknown): { width: number; height: number } | null { + if (typeof value !== "string") return null; + const match = /^(\d+)\s*[x×]\s*(\d+)$/i.exec(value.trim()); + if (!match) return null; + const width = Number(match[1]); + const height = Number(match[2]); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return null; + return { + width: Math.min(width, MAX_OUTPUT_EDGE), + height: Math.min(height, MAX_OUTPUT_EDGE), + }; +} + +/** + * Append a 0-1 strength. Percent-aware fields also accept 0-100 (the shared UI + * creativity slider), which is divided down; anything non-numeric is skipped. + */ +function appendUnitFloat( + formData: FormData, + key: string, + value: unknown, + summary: Record, + percentAware = false +): void { + if (value === undefined || value === null || String(value).trim() === "") return; + let n = typeof value === "number" ? value : Number(String(value).replace("%", "").trim()); + if (!Number.isFinite(n)) return; + if (percentAware && n > 1) n = n / 100; + n = Math.max(0, Math.min(1, n)); + const rounded = Math.round(n * 100) / 100; + formData.append(key, String(rounded)); + summary[key] = rounded; +} + +function toBoolean(value: unknown): boolean { + if (typeof value === "boolean") return value; + const raw = String(value ?? "").trim().toLowerCase(); + return raw === "true" || raw === "1" || raw === "yes" || raw === "on"; +} diff --git a/open-sse/services/adobeFireflyClient.ts b/open-sse/services/adobeFireflyClient.ts index b3b31cc46a..16d491637a 100644 --- a/open-sse/services/adobeFireflyClient.ts +++ b/open-sse/services/adobeFireflyClient.ts @@ -2026,7 +2026,7 @@ async function sleep(ms: number): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } -async function pollAdobeJob(opts: { +export async function pollAdobeJob(opts: { pollUrl: string; accessToken: string; kind: "image" | "video"; diff --git a/open-sse/services/adobeFireflyUpscale.ts b/open-sse/services/adobeFireflyUpscale.ts new file mode 100644 index 0000000000..92edc6b340 --- /dev/null +++ b/open-sse/services/adobeFireflyUpscale.ts @@ -0,0 +1,437 @@ +/** + * Adobe Firefly (unofficial) image **upsample** client — Topaz Labs models. + * + * Wire contract from a live firefly.adobe.com capture (web_providers/upsample.txt): + * + * POST https://firefly-3p.ff.adobe.io/v2/3p-images/upsample + * headers: Authorization: Bearer + * x-api-key: clio-playground-web + * x-arp-session-id: (NO x-nonce on this endpoint) + * content-type: application/json + * body: { + * "modelId": "topaz", + * "modelVersion": "reimagine", + * "generationMetadata": { "module": "image-editing", "submodule": "ff-image-editor", ... }, + * "referenceBlobs": [{ "id": "", "usage": "general" }], + * "upsamplerFactor": 2, + * "creativityLevel": 0 + * } + * → 200 { "links": { "cancel": {...}, "result": { "href": ".../jobs/result/" } } } + * + * The job link is polled with the same BKS rewrite + status semantics as + * generate-async, so `pollAdobeJob` from `adobeFireflyClient.ts` is reused verbatim. + * + * Model discovery (web_providers/upscale.txt) lists modelId `topaz` with image + * modelVersions `default` / `standard` / `reimagine`, each carrying + * `inputMediaUseCase: ["upscaling"]`. `starlight-*` and `astra-2` are the VIDEO + * upscalers of the same family (`acModelFamilyId: topaz-video`) and are not served + * by this image endpoint, so they are deliberately absent. + */ + +import { + AdobeFireflyError, + buildAdobeArpSessionId, + buildAdobeSubmitHeaders, + extractAdobeArpSessionId, + extractAdobeCookieHeader, + extractAdobeResultLink, + formatAdobeSystemUnderLoadError, + isAdobeTransientSubmitError, + normalizeAdobePollUrl, + pollAdobeJob, +} from "./adobeFireflyClient.ts"; +import { sanitizeErrorMessage } from "../utils/error.ts"; + +export const ADOBE_FIREFLY_IMAGE_UPSAMPLE_URL = + "https://firefly-3p.ff.adobe.io/v2/3p-images/upsample"; + +/** Firefly image upscale timeout — Topaz jobs are slower than a 1K generate. */ +export const ADOBE_FIREFLY_UPSCALE_TIMEOUT_MS = 300_000; + +/** Same submit-retry budget as generate-async (colligo 408 recovery). */ +const SUBMIT_MAX_ATTEMPTS = 5; + +/** + * Firefly Topaz upsample wire range for `creativityLevel`. + * + * Live colligo on `/v2/3p-images/upsample` rejects values > 1 + * (`less_than_equal`, `le: 1.0`). The browser capture sends `0` (off). + * Discovery docs mention a 1–5 integer scale for *other* Topaz endpoints — + * that scale is NOT accepted by upsample, so we stay on 0–1. + */ +export const ADOBE_FIREFLY_MAX_CREATIVITY_LEVEL = 1; + +export type AdobeFireflyUpscaleModelId = "topaz" | "topaz-standard" | "topaz-bloom"; + +export interface AdobeFireflyUpscaleModelSpec { + upstreamModelId: string; + upstreamModelVersion: string; + /** Scale factors accepted for this version. */ + factors: number[]; + /** `creativityLevel` is only meaningful on the generative (reimagine) version. */ + supportsCreativity: boolean; +} + +export const ADOBE_FIREFLY_UPSCALE_MODELS: Record< + AdobeFireflyUpscaleModelId, + AdobeFireflyUpscaleModelSpec +> = { + // Bare `topaz` maps to the standard version rather than the discovery-listed + // "default" alias: both resolve to bksGenerationModel firefly_3p:external:topaz_standard, + // and pinning the explicit version avoids depending on an alias we have not captured. + topaz: { + upstreamModelId: "topaz", + upstreamModelVersion: "standard", + factors: [2, 4], + supportsCreativity: false, + }, + "topaz-standard": { + upstreamModelId: "topaz", + upstreamModelVersion: "standard", + factors: [2, 4], + supportsCreativity: false, + }, + "topaz-bloom": { + upstreamModelId: "topaz", + upstreamModelVersion: "reimagine", + factors: [2, 4], + supportsCreativity: true, + }, +}; + +/** + * Resolve a catalog id (with or without an `adobe-firefly/` prefix) to its upstream + * modelId/modelVersion pair. Returns null for anything that is not a Firefly image + * upscaler, so callers can fall through instead of silently upscaling with a default. + */ +export function resolveAdobeUpscaleModel(model: string): { + id: AdobeFireflyUpscaleModelId; + spec: AdobeFireflyUpscaleModelSpec; +} | null { + const raw = String(model || "") + .trim() + .toLowerCase() + .replace(/^adobe-firefly\//, "") + .replace(/^firefly\//, ""); + + if (!raw) return null; + if (raw in ADOBE_FIREFLY_UPSCALE_MODELS) { + const id = raw as AdobeFireflyUpscaleModelId; + return { id, spec: ADOBE_FIREFLY_UPSCALE_MODELS[id] }; + } + + // Accept the upstream version names and common spellings. + if (raw.includes("bloom") || raw.includes("reimagine")) { + return { id: "topaz-bloom", spec: ADOBE_FIREFLY_UPSCALE_MODELS["topaz-bloom"] }; + } + if (raw.includes("topaz")) { + return { id: "topaz-standard", spec: ADOBE_FIREFLY_UPSCALE_MODELS["topaz-standard"] }; + } + return null; +} + +/** True when the model id names a Firefly image upscaler (used to split the generate path). */ +export function isAdobeFireflyUpscaleModel(model: string): boolean { + return resolveAdobeUpscaleModel(model) !== null; +} + +/** + * Map a 0-100 creativity percentage onto Firefly upsample's `creativityLevel` (0–1 float). + * + * Precedence: + * 1. explicit `creativityLevel` — if in (1, 5] treat as legacy 1–5 integer scale + * and map onto 0–1 (`level / 5`); otherwise clamp to 0–1 + * 2. `creativityPercent` 0–100 → 0–1 + * 3. default 0 (browser default / off) + */ +export function resolveAdobeCreativityLevel(opts: { + creativityPercent?: number | null; + creativityLevel?: unknown; +}): number { + const explicit = opts.creativityLevel; + if (typeof explicit === "number" && Number.isFinite(explicit)) { + return clampLevel(normalizeExplicitCreativity(explicit)); + } + if (typeof explicit === "string" && explicit.trim() && Number.isFinite(Number(explicit))) { + return clampLevel(normalizeExplicitCreativity(Number(explicit))); + } + + const percent = typeof opts.creativityPercent === "number" && Number.isFinite(opts.creativityPercent) + ? Math.max(0, Math.min(100, opts.creativityPercent)) + : 0; + return clampLevel(percent / 100); +} + +/** Legacy 1–5 integer scale (discovery docs) → 0–1 wire float. Values already in 0–1 pass through. */ +function normalizeExplicitCreativity(value: number): number { + if (value > ADOBE_FIREFLY_MAX_CREATIVITY_LEVEL && value <= 5) { + return value / 5; + } + return value; +} + +/** Clamp to the upsample wire range [0, 1], two decimal places. */ +function clampLevel(value: number): number { + if (!Number.isFinite(value)) return 0; + const clamped = Math.max(0, Math.min(ADOBE_FIREFLY_MAX_CREATIVITY_LEVEL, value)); + return Math.round(clamped * 100) / 100; +} + +/** + * Headers for the upsample submit. + * + * Identical to generate-async EXCEPT `x-nonce`, which the live upsample request does + * not send (there is no prompt to derive a deterministic nonce from). We mirror the + * capture exactly rather than adding a header colligo never sees from the SPA. + */ +export function buildAdobeUpsampleHeaders( + accessToken: string, + extras?: { arpSessionId?: string; cookie?: string } +): Record { + const headers = buildAdobeSubmitHeaders(accessToken, { + arpSessionId: extras?.arpSessionId, + cookie: extras?.cookie, + prompt: "upsample", + }); + delete headers["x-nonce"]; + return headers; +} + +export function buildAdobeUpsamplePayload(opts: { + modelSpec: AdobeFireflyUpscaleModelSpec; + blobId: string; + upsamplerFactor: number; + creativityLevel?: number; +}): Record { + const payload: Record = { + modelId: opts.modelSpec.upstreamModelId, + modelVersion: opts.modelSpec.upstreamModelVersion, + generationMetadata: { + module: "image-editing", + submodule: "ff-image-editor", + sourceDocumentId: null, + originalPrompt: null, + filterString: null, + subPrompts: null, + canvasImageReference: null, + }, + referenceBlobs: [{ id: String(opts.blobId), usage: "general" }], + upsamplerFactor: opts.upsamplerFactor, + }; + + // creativityLevel is optional/nullable upstream — only the generative version + // consumes it, so the standard pass omits it entirely. + if (opts.modelSpec.supportsCreativity) { + payload.creativityLevel = Number.isFinite(opts.creativityLevel as number) + ? (opts.creativityLevel as number) + : 0; + } + + return payload; +} + +/** + * Submit + poll a Firefly Topaz upscale job. + * + * `blobId` must already be a Firefly storage id — callers upload the source image with + * `resolveAdobeSourceImageIds`/`uploadAdobeFireflyImage` first, reusing the same ARP so + * colligo sees one coherent risk session for upload + submit. + */ +export async function adobeFireflyUpscaleImage(opts: { + accessToken: string; + model: string; + blobId: string; + upsamplerFactor?: unknown; + creativityPercent?: number; + creativityLevel?: unknown; + sessionCookie?: string; + arpSessionId?: string; + sessionFingerprint?: string; + timeoutMs?: number; + fetchImpl?: typeof fetch; + log?: { info?: (...args: unknown[]) => void; error?: (...args: unknown[]) => void }; +}): Promise<{ url: string; latest: unknown; factor: number; creativityLevel: number }> { + const fetchImpl = opts.fetchImpl || fetch; + const resolved = resolveAdobeUpscaleModel(opts.model); + if (!resolved) { + throw new AdobeFireflyError( + `Unsupported Adobe Firefly upscale model: ${opts.model}. ` + + `Use topaz-standard or topaz-bloom.`, + 400, + "bad_model" + ); + } + const { spec } = resolved; + + const blobId = String(opts.blobId || "").trim(); + if (!blobId) { + throw new AdobeFireflyError( + "Adobe Firefly upscale requires a source image", + 400, + "bad_image" + ); + } + + const factor = normalizeFactor(opts.upsamplerFactor, spec.factors); + const creativityLevel = spec.supportsCreativity + ? resolveAdobeCreativityLevel({ + creativityPercent: opts.creativityPercent ?? null, + creativityLevel: opts.creativityLevel, + }) + : 0; + + const payload = buildAdobeUpsamplePayload({ + modelSpec: spec, + blobId, + upsamplerFactor: factor, + creativityLevel, + }); + + const sessionCookie = String(opts.sessionCookie || "").trim(); + const cookieHeader = extractAdobeCookieHeader(sessionCookie); + const browserArp = extractAdobeArpSessionId(cookieHeader || sessionCookie); + const hadBrowserArp = Boolean(browserArp); + let arpSessionId = + (opts.arpSessionId && String(opts.arpSessionId).trim()) || + browserArp || + buildAdobeArpSessionId(); + const accessToken = opts.accessToken; + let submitData: unknown = {}; + let submitHeaders: Headers | Record = new Headers(); + let lastSubmitError = ""; + let sawSystemUnderLoad = false; + let submitted = false; + + for (let attempt = 1; attempt <= SUBMIT_MAX_ATTEMPTS; attempt++) { + const submitResp = await fetchImpl(ADOBE_FIREFLY_IMAGE_UPSAMPLE_URL, { + method: "POST", + headers: buildAdobeUpsampleHeaders(accessToken, { + arpSessionId, + cookie: cookieHeader || undefined, + }), + body: JSON.stringify(payload), + }); + + if (submitResp.status === 401 || submitResp.status === 403) { + if ((submitResp.headers.get("x-access-error") || "") === "taste_exhausted") { + throw new AdobeFireflyError( + "Adobe Firefly quota exhausted for this account", + 429, + "quota_exhausted" + ); + } + throw new AdobeFireflyError( + "Adobe Firefly token invalid or expired. Paste a fresh IMS JWT (Authorization: Bearer on " + + "firefly-3p) plus the firefly.adobe.com Cookie once.", + 401, + "auth" + ); + } + + if (!submitResp.ok) { + const text = await submitResp.text().catch(() => ""); + if (isAdobeTransientSubmitError(submitResp.status, text)) sawSystemUnderLoad = true; + lastSubmitError = + `Adobe Firefly image upscale submit failed (${submitResp.status}): ` + + sanitizeErrorMessage(text.slice(0, 300)); + + if (isAdobeTransientSubmitError(submitResp.status, text) && attempt < SUBMIT_MAX_ATTEMPTS) { + // Rotate synthetic ARP on transient 408; real browser ARP is reused as-is. + if (!hadBrowserArp) { + arpSessionId = buildAdobeArpSessionId(); + } + const delay = submitRetryDelayMs(attempt); + opts.log?.info?.( + "ADOBE-FIREFLY", + `upscale submit transient ${submitResp.status}, retry ${attempt}/${SUBMIT_MAX_ATTEMPTS} in ${delay}ms` + ); + await sleep(delay); + continue; + } + + if (sawSystemUnderLoad && isAdobeTransientSubmitError(submitResp.status, text)) { + throw new AdobeFireflyError( + formatAdobeSystemUnderLoadError("image", attempt), + 408, + "system_under_load" + ); + } + throw new AdobeFireflyError( + lastSubmitError, + submitResp.status >= 400 && submitResp.status < 500 ? submitResp.status : 502 + ); + } + + submitData = await submitResp.json().catch(() => ({})); + submitHeaders = submitResp.headers; + submitted = true; + break; + } + + if (!submitted) { + throw new AdobeFireflyError( + lastSubmitError || "Adobe Firefly upscale submit failed after retries", + 502 + ); + } + + let pollUrl = extractAdobeResultLink(submitHeaders, submitData); + if (!pollUrl) { + if (sawSystemUnderLoad) { + throw new AdobeFireflyError( + formatAdobeSystemUnderLoadError("image", SUBMIT_MAX_ATTEMPTS), + 408, + "system_under_load" + ); + } + throw new AdobeFireflyError( + lastSubmitError || "Adobe Firefly upscale submit succeeded but no poll URL was returned", + 502 + ); + } + pollUrl = normalizeAdobePollUrl(pollUrl); + + const { mediaUrl, latest } = await pollAdobeJob({ + pollUrl, + accessToken, + kind: "image", + timeoutMs: + opts.timeoutMs && opts.timeoutMs > 0 ? opts.timeoutMs : ADOBE_FIREFLY_UPSCALE_TIMEOUT_MS, + fetchImpl, + log: opts.log, + }); + + return { url: mediaUrl, latest, factor, creativityLevel }; +} + +function normalizeFactor(value: unknown, allowed: readonly number[]): number { + const factors = allowed.length > 0 ? [...allowed] : [2, 4]; + let n = typeof value === "number" ? value : Number(String(value ?? "").replace(/[^\d.]/g, "")); + if (!Number.isFinite(n) || n <= 0) n = 2; + let best = factors[0]!; + let bestDelta = Math.abs(best - n); + for (const f of factors) { + const delta = Math.abs(f - n); + if (delta < bestDelta) { + best = f; + bestDelta = delta; + } + } + return best; +} + +function submitRetryDelayMs(attempt: number): number { + const raw = process.env.ADOBE_FIREFLY_SUBMIT_BASE_DELAY_MS; + const base = + raw != null && raw !== "" + ? Math.max(0, Number(raw) || 0) + : process.env.NODE_ENV === "test" || process.env.VITEST || process.env.NODE_TEST_CONTEXT + ? 20 + : 8000; + if (base <= 50) return base; + return Math.min(90_000, base * Math.pow(2, attempt - 1)) + Math.floor(Math.random() * 1500); +} + +async function sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/app/api/v1/images/upscale/route.ts b/src/app/api/v1/images/upscale/route.ts new file mode 100644 index 0000000000..e3ea105b93 --- /dev/null +++ b/src/app/api/v1/images/upscale/route.ts @@ -0,0 +1,274 @@ +import { handleImageUpscale } from "@omniroute/open-sse/handlers/imageUpscale.ts"; +import { + getUpscaleProvider, + getAllUpscaleModels, + parseUpscaleModel, +} from "@omniroute/open-sse/config/upscaleRegistry.ts"; +import { extractUpscaleSourceImage } from "@omniroute/open-sse/handlers/imageUpscale/shared.ts"; +import { withInjectionGuard } from "@/middleware/promptInjectionGuard"; +import { + getProviderCredentialsWithQuotaPreflight, + clearRecoveredProviderState, +} from "@/sse/services/auth"; +import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts"; +import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; +import * as log from "@/sse/utils/logger"; +import { toJsonErrorPayload } from "@/shared/utils/upstreamError"; +import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; +import { v1ImageUpscaleSchema } from "@/shared/validation/schemas"; +import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { resolveProxyForConnection } from "@/lib/db/settings"; +import { runWithProxyContext } from "@omniroute/open-sse/utils/proxyFetch.ts"; +import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta"; +import { calculateModalCost } from "@/lib/usage/costCalculator"; +import { generateRequestId } from "@/shared/utils/requestId"; + +export const dynamic = "force-dynamic"; + +/** + * `/v1/images/upscale` — image→image super-resolution. + * + * A dedicated endpoint rather than a `/v1/images/generations` model: upscaling has no + * text-to-image path, always needs a source image, and its meaningful controls (scale + * factor, creativity level) do not exist on the generation contract. + * + * Providers are declared in `open-sse/config/upscaleRegistry.ts`: + * - `adobe-firefly/topaz-standard` · `adobe-firefly/topaz-bloom` (Topaz via Firefly 3P) + * - `stability-ai/fast` · `stability-ai/conservative` · `stability-ai/creative` + * - `topaz/topaz-enhance` (Topaz Labs native API) + * + * Accepts JSON (data-URL / http(s) image) or multipart/form-data (`image` file part), + * since OpenAI-style clients send the latter for image inputs. + */ + +export async function OPTIONS() { + return new Response(null, { + headers: { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); +} + +/** GET /v1/images/upscale — list the upscale models this instance can serve. */ +export async function GET() { + const data = getAllUpscaleModels().map((model) => { + const providerConfig = getUpscaleProvider(model.provider); + const entry = providerConfig?.models.find((candidate) => model.id.endsWith(`/${candidate.id}`)); + return { + id: model.id, + object: "model", + owned_by: model.provider, + name: model.name, + type: "image", + subtype: "upscale", + input_modalities: ["image"], + output_modalities: ["image"], + factors: entry?.factors ?? [], + supports_creativity: Boolean(entry?.supportsCreativity), + supports_prompt: Boolean(entry?.supportsPrompt), + prompt_required: Boolean(entry?.promptRequired), + ...(entry?.description ? { description: entry.description } : {}), + }; + }); + + return new Response(JSON.stringify({ object: "list", data }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +/** + * Read the request body as a plain object from either JSON or multipart/form-data. + * Multipart file parts become data URLs so every downstream handler sees one shape. + */ +async function readUpscaleBody(request: Request): Promise | null> { + const contentType = request.headers.get("content-type") || ""; + + if (contentType.includes("multipart/form-data")) { + try { + const formData = await request.formData(); + const body: Record = {}; + for (const [key, value] of formData.entries()) { + if (typeof value === "string") { + body[key] = value; + continue; + } + const file = value as File; + const bytes = Buffer.from(await file.arrayBuffer()); + if (!bytes.length) continue; + const mime = file.type && file.type.startsWith("image/") ? file.type : "image/png"; + body[key] = `data:${mime};base64,${bytes.toString("base64")}`; + } + return body; + } catch (err) { + log.warn("IMAGE", `Invalid multipart upscale body: ${err instanceof Error ? err.message : err}`); + return null; + } + } + + try { + const parsed = await request.json(); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } +} + +async function postHandler(request: Request) { + const rawBody = await readUpscaleBody(request); + if (!rawBody) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + "Invalid request body. Send JSON or multipart/form-data with an image." + ); + } + + const validation = validateBody(v1ImageUpscaleSchema, rawBody); + if (isValidationFailure(validation)) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message); + } + const body = validation.data as Record; + const startTime = Date.now(); + + const policy = await enforceApiKeyPolicy(request, String(body.model ?? "")); + if (policy.rejection) return policy.rejection; + + const allowedConnections = + policy.apiKeyInfo?.allowedConnections && policy.apiKeyInfo.allowedConnections.length > 0 + ? policy.apiKeyInfo.allowedConnections + : null; + + const { provider, model } = parseUpscaleModel(String(body.model ?? "")); + if (!provider || !model) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `Invalid upscale model: ${body.model}. Use format: provider/model ` + + `(e.g. adobe-firefly/topaz-bloom).` + ); + } + + const providerConfig = getUpscaleProvider(provider); + if (!providerConfig) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `Unknown upscale provider: ${provider}`); + } + + const entry = providerConfig.models.find((candidate) => candidate.id === model); + if (!entry) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `Unsupported upscale model for ${provider}: ${model}. ` + + `Available: ${providerConfig.models.map((m) => m.id).join(", ")}.` + ); + } + + if (!extractUpscaleSourceImage(body)) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `A source image is required for upscaling. Send "image" or "image_url" ` + + `(data URL, http(s) URL, or a multipart file part).` + ); + } + + if (entry.promptRequired && !(typeof body.prompt === "string" && body.prompt.trim())) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `Upscale model ${provider}/${model} requires a prompt describing the image.` + ); + } + + const credentialsResult = await getProviderCredentialsWithQuotaPreflight( + provider, + null, + allowedConnections, + `${provider}/${model}` + ); + if (!credentialsResult) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `No credentials for upscale provider: ${provider}` + ); + } + + // getProviderCredentialsWithQuotaPreflight returns either a credential record or an + // all-rate-limited marker; read both through one loose view (the union has no common + // discriminant) and narrow explicitly afterwards. + const creds = credentialsResult as { + allRateLimited?: boolean; + retryAfter?: string; + retryAfterHuman?: string; + apiKey?: string | null; + accessToken?: string | null; + connectionId?: string | null; + providerSpecificData?: Record | null; + }; + + if (creds.allRateLimited) { + return unavailableResponse( + HTTP_STATUS.RATE_LIMITED, + `[${provider}] All accounts rate limited`, + creds.retryAfter, + creds.retryAfterHuman + ); + } + + const upscaleCredentials = { + ...(typeof creds.apiKey === "string" && creds.apiKey ? { apiKey: creds.apiKey } : {}), + ...(typeof creds.accessToken === "string" && creds.accessToken + ? { accessToken: creds.accessToken } + : {}), + // Adobe Firefly keeps a pasted firefly.adobe.com Cookie here. + ...(creds.providerSpecificData ? { providerSpecificData: creds.providerSpecificData } : {}), + }; + + let proxyInfo: { proxy?: unknown } | null = null; + if (creds.connectionId) { + try { + proxyInfo = (await resolveProxyForConnection(creds.connectionId)) as { proxy?: unknown } | null; + } catch { + log.debug("PROXY", `Failed to resolve proxy for upscale provider: ${provider}`); + } + } + + const runUpscale = () => handleImageUpscale({ body, credentials: upscaleCredentials, log }); + + const result = await (creds.connectionId + ? runWithProxyContext((proxyInfo?.proxy as never) || null, runUpscale).catch( + (err: { statusCode?: number; message?: string }) => ({ + success: false, + status: err.statusCode || 500, + error: err.message, + }) + ) + : runUpscale()); + + if (result.success) { + await clearRecoveredProviderState(credentialsResult); + const costUsd = await calculateModalCost("image", provider, `${provider}/${model}`, { n: 1 }); + const headers = new Headers({ "Content-Type": "application/json" }); + attachOmniRouteMetaHeaders(headers, { + provider, + model: `${provider}/${model}`, + costUsd, + latencyMs: Date.now() - startTime, + requestId: generateRequestId(), + }); + return new Response(JSON.stringify((result as { data: unknown }).data), { + status: 200, + headers, + }); + } + + const errorPayload = toJsonErrorPayload( + (result as { error?: unknown }).error, + "Image upscale provider error" + ); + return new Response(JSON.stringify(errorPayload), { + status: (result as { status?: number }).status ?? HTTP_STATUS.BAD_GATEWAY, + headers: { "Content-Type": "application/json" }, + }); +} + +export const POST = withInjectionGuard(postHandler); diff --git a/src/shared/constants/endpointCategories.ts b/src/shared/constants/endpointCategories.ts index fd761f9987..abaaeb7a2f 100644 --- a/src/shared/constants/endpointCategories.ts +++ b/src/shared/constants/endpointCategories.ts @@ -39,7 +39,7 @@ export const ENDPOINT_CATEGORIES: readonly EndpointCategory[] = [ { id: "images", label: "Images", - description: "Image generation and editing", + description: "Image generation, editing and upscaling", prefixes: ["/v1/images"], }, { diff --git a/src/shared/validation/schemas/apiV1.ts b/src/shared/validation/schemas/apiV1.ts index f470d9d7b9..4c740cb42d 100644 --- a/src/shared/validation/schemas/apiV1.ts +++ b/src/shared/validation/schemas/apiV1.ts @@ -178,6 +178,20 @@ export const v1ImageGenerationSchema = z }) .catchall(z.unknown()); +// POST /v1/images/upscale — image→image super-resolution. `prompt` is optional here +// (only Stability conservative/creative need one, enforced by the route/handler), but a +// resolvable source image is mandatory and validated by the route after extraction. +export const v1ImageUpscaleSchema = z + .object({ + model: modelIdSchema, + prompt: nonEmptyStringSchema.optional(), + factor: z.union([z.number(), z.string()]).optional(), + creativity: z.union([z.number(), z.string()]).optional(), + response_format: z.enum(["url", "b64_json"]).optional(), + }) + .catchall(z.unknown()); + + export const v1AudioSpeechSchema = z .object({ model: modelIdSchema, diff --git a/tests/unit/image-upscale.test.ts b/tests/unit/image-upscale.test.ts new file mode 100644 index 0000000000..b6eda4c365 --- /dev/null +++ b/tests/unit/image-upscale.test.ts @@ -0,0 +1,635 @@ +import { test } from "node:test"; +import assert from "node:assert"; +import { + DEFAULT_UPSCALE_FACTORS, + UPSCALE_PROVIDERS, + getAllUpscaleModels, + getUpscaleModelEntry, + getUpscaleProvider, + isRegisteredUpscaleModel, + normalizeCreativityPercent, + normalizeUpscaleFactor, + parseUpscaleModel, +} from "../../open-sse/config/upscaleRegistry.ts"; +import { + ADOBE_FIREFLY_IMAGE_UPSAMPLE_URL, + ADOBE_FIREFLY_MAX_CREATIVITY_LEVEL, + ADOBE_FIREFLY_UPSCALE_MODELS, + adobeFireflyUpscaleImage, + buildAdobeUpsampleHeaders, + buildAdobeUpsamplePayload, + isAdobeFireflyUpscaleModel, + resolveAdobeCreativityLevel, + resolveAdobeUpscaleModel, +} from "../../open-sse/services/adobeFireflyUpscale.ts"; +import { + extractUpscaleSourceImage, + readImageDimensions, + scaleDimensions, + sniffImageMime, +} from "../../open-sse/handlers/imageUpscale/shared.ts"; +import { handleImageUpscale } from "../../open-sse/handlers/imageUpscale.ts"; +import { handleStabilityImageUpscale } from "../../open-sse/handlers/imageUpscale/stability.ts"; +import { handleTopazImageUpscale } from "../../open-sse/handlers/imageUpscale/topaz.ts"; +import { IMAGE_PROVIDERS } from "../../open-sse/config/imageRegistry.ts"; + +// ── Fixtures ─────────────────────────────────────────────────────────────── + +/** Minimal but real 1x1 PNG (valid IHDR so dimension reads work). */ +const PNG_1X1 = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg==", + "base64" +); +const PNG_1X1_DATA_URL = `data:image/png;base64,${PNG_1X1.toString("base64")}`; + +/** 640x480 PNG header only — enough for readImageDimensions. */ +function pngHeader(width: number, height: number): Buffer { + const buf = Buffer.alloc(24); + buf[0] = 0x89; + buf.write("PNG", 1, "ascii"); + buf.writeUInt32BE(width, 16); + buf.writeUInt32BE(height, 20); + return buf; +} + +/** JPEG with a single SOF0 marker declaring width/height. */ +function jpegHeader(width: number, height: number): Buffer { + const sof = Buffer.alloc(11); + sof[0] = 0xff; + sof[1] = 0xc0; + sof.writeUInt16BE(8, 2); // segment length + sof[4] = 8; // precision + sof.writeUInt16BE(height, 5); + sof.writeUInt16BE(width, 7); + return Buffer.concat([Buffer.from([0xff, 0xd8]), sof, Buffer.alloc(4)]); +} + +const FAKE_JWT = (() => { + const header = Buffer.from(JSON.stringify({ alg: "RS256" })).toString("base64url"); + const payload = Buffer.from( + JSON.stringify({ user_id: "TESTUSER@AdobeID", type: "access_token", created_at: "1", expires_in: "86400000" }) + ).toString("base64url"); + return `${header}.${payload}.sig`; +})(); + +/** `new Response(buffer)` does not typecheck (Buffer); copy to an ArrayBuffer. */ +function bytes(buffer: Buffer): ArrayBuffer { + const out = new ArrayBuffer(buffer.byteLength); + new Uint8Array(out).set(buffer); + return out; +} + +function jsonResponse(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json", ...headers }, + }); +} + +// ── Registry ─────────────────────────────────────────────────────────────── + +test("upscale registry exposes adobe-firefly, stability-ai and topaz", () => { + assert.deepEqual(Object.keys(UPSCALE_PROVIDERS).sort(), [ + "adobe-firefly", + "stability-ai", + "topaz", + ]); + assert.equal(getUpscaleProvider("adobe-firefly")?.format, "adobe-firefly-upscale"); + assert.equal(getUpscaleProvider("stability-ai")?.format, "stability-upscale"); + assert.equal(getUpscaleProvider("topaz")?.format, "topaz-upscale"); + assert.equal(getUpscaleProvider("nope"), null); +}); + +test("adobe-firefly upscale models are Topaz only (video starlight/astra excluded)", () => { + const ids = UPSCALE_PROVIDERS["adobe-firefly"]!.models.map((m) => m.id); + assert.deepEqual(ids, ["topaz", "topaz-standard", "topaz-bloom"]); + for (const id of ids) assert.ok(id.startsWith("topaz"), `${id} must be a Topaz model`); + for (const forbidden of ["starlight-quality", "starlight-creative", "starlight-fast", "astra-2"]) { + assert.ok(!ids.includes(forbidden), `${forbidden} is a video upscaler and must not be listed`); + } +}); + +test("only topaz-bloom advertises creativity; stability creative/conservative take prompts", () => { + const firefly = UPSCALE_PROVIDERS["adobe-firefly"]!.models; + assert.equal(firefly.find((m) => m.id === "topaz-bloom")?.supportsCreativity, true); + assert.notEqual(firefly.find((m) => m.id === "topaz-standard")?.supportsCreativity, true); + + const stability = UPSCALE_PROVIDERS["stability-ai"]!.models; + assert.equal(stability.find((m) => m.id === "creative")?.promptRequired, true); + assert.equal(stability.find((m) => m.id === "conservative")?.promptRequired, true); + assert.notEqual(stability.find((m) => m.id === "fast")?.promptRequired, true); +}); + +test("parseUpscaleModel accepts provider prefix, alias and bare model ids", () => { + assert.deepEqual(parseUpscaleModel("adobe-firefly/topaz-bloom"), { + provider: "adobe-firefly", + model: "topaz-bloom", + }); + assert.deepEqual(parseUpscaleModel("firefly/topaz-standard"), { + provider: "adobe-firefly", + model: "topaz-standard", + }); + assert.deepEqual(parseUpscaleModel("stability-ai/creative"), { + provider: "stability-ai", + model: "creative", + }); + assert.deepEqual(parseUpscaleModel("topaz-enhance"), { provider: "topaz", model: "topaz-enhance" }); + assert.equal(parseUpscaleModel("openai/gpt-image-2").provider, null); + assert.deepEqual(parseUpscaleModel(null), { provider: null, model: null }); +}); + +test("getUpscaleModelEntry / isRegisteredUpscaleModel resolve registry rows", () => { + const hit = getUpscaleModelEntry("adobe-firefly/topaz-bloom"); + assert.ok(hit); + assert.equal(hit.provider, "adobe-firefly"); + assert.equal(hit.entry.supportsCreativity, true); + assert.equal(getUpscaleModelEntry("adobe-firefly/nope"), null); + assert.equal(isRegisteredUpscaleModel("stability-ai/fast"), true); + assert.equal(isRegisteredUpscaleModel("stability-ai/ultra"), false); +}); + +test("getAllUpscaleModels lists prefixed ids for every provider and alias", () => { + const ids = getAllUpscaleModels().map((m) => m.id); + assert.ok(ids.includes("adobe-firefly/topaz-bloom")); + assert.ok(ids.includes("firefly/topaz-bloom"), "alias-prefixed id must be listed too"); + assert.ok(ids.includes("stability-ai/fast")); + assert.ok(ids.includes("topaz/topaz-enhance")); +}); + +test("adobe-firefly image registry now carries the Topaz upscale models as image-only", () => { + const models = IMAGE_PROVIDERS["adobe-firefly"]!.models as unknown as Array< + Record + >; + const bloom = models.find((m) => m.id === "topaz-bloom"); + assert.ok(bloom, "topaz-bloom must be registered on the adobe-firefly image provider"); + assert.deepEqual(bloom.inputModalities, ["image"]); + assert.equal(bloom.imageRequired, true); + const standard = models.find((m) => m.id === "topaz-standard"); + assert.ok(standard); + assert.deepEqual(standard.inputModalities, ["image"]); +}); + +// ── Factor / creativity normalization ────────────────────────────────────── + +test("normalizeUpscaleFactor snaps loose input onto supported factors", () => { + assert.deepEqual([...DEFAULT_UPSCALE_FACTORS], [2, 4]); + assert.equal(normalizeUpscaleFactor(2), 2); + assert.equal(normalizeUpscaleFactor(4), 4); + assert.equal(normalizeUpscaleFactor("4x"), 4); + assert.equal(normalizeUpscaleFactor("x2"), 2); + assert.equal(normalizeUpscaleFactor("4X"), 4); + // 3 is equidistant; the first-listed (2) wins because ties keep the earlier entry. + assert.equal(normalizeUpscaleFactor(3), 2); + assert.equal(normalizeUpscaleFactor(3.6), 4); + assert.equal(normalizeUpscaleFactor(99), 4); + assert.equal(normalizeUpscaleFactor("nonsense"), 2); + assert.equal(normalizeUpscaleFactor(undefined), 2); + assert.equal(normalizeUpscaleFactor(0), 2); + assert.equal(normalizeUpscaleFactor(-4), 2); + // Single-factor models always report that factor. + assert.equal(normalizeUpscaleFactor(2, [4]), 4); +}); + +test("normalizeCreativityPercent clamps and distinguishes fractions from percents", () => { + assert.equal(normalizeCreativityPercent(0), 0); + assert.equal(normalizeCreativityPercent(40), 40); + assert.equal(normalizeCreativityPercent("60%"), 60); + assert.equal(normalizeCreativityPercent(0.35), 35); + assert.equal(normalizeCreativityPercent(1), 1, "integer 1 stays 1 %, not 100 %"); + assert.equal(normalizeCreativityPercent(140), 100); + assert.equal(normalizeCreativityPercent(-5), 0); + assert.equal(normalizeCreativityPercent("abc", 25), 25); +}); + +// ── Adobe Firefly upsample wire contract ─────────────────────────────────── + +test("resolveAdobeUpscaleModel maps ids to upstream topaz versions and rejects others", () => { + assert.equal(resolveAdobeUpscaleModel("topaz-bloom")?.spec.upstreamModelVersion, "reimagine"); + assert.equal(resolveAdobeUpscaleModel("topaz-standard")?.spec.upstreamModelVersion, "standard"); + assert.equal(resolveAdobeUpscaleModel("topaz")?.spec.upstreamModelVersion, "standard"); + assert.equal( + resolveAdobeUpscaleModel("adobe-firefly/topaz-bloom")?.spec.upstreamModelId, + "topaz" + ); + assert.equal(resolveAdobeUpscaleModel("firefly/reimagine")?.spec.upstreamModelVersion, "reimagine"); + assert.equal(resolveAdobeUpscaleModel("nano-banana-pro"), null); + assert.equal(resolveAdobeUpscaleModel(""), null); + assert.equal(isAdobeFireflyUpscaleModel("topaz-bloom"), true); + assert.equal(isAdobeFireflyUpscaleModel("gpt-image-2"), false); + // Every registered spec targets the image family (never topaz-video). + for (const spec of Object.values(ADOBE_FIREFLY_UPSCALE_MODELS)) { + assert.equal(spec.upstreamModelId, "topaz"); + assert.deepEqual(spec.factors, [2, 4]); + } +}); + +test("resolveAdobeCreativityLevel maps 0-100 % onto the 0-1 upsample wire float", () => { + // Live colligo on /v2/3p-images/upsample rejects creativityLevel > 1. + assert.equal(ADOBE_FIREFLY_MAX_CREATIVITY_LEVEL, 1); + assert.equal(resolveAdobeCreativityLevel({ creativityPercent: 0 }), 0); + assert.equal(resolveAdobeCreativityLevel({ creativityPercent: 100 }), 1); + assert.equal(resolveAdobeCreativityLevel({ creativityPercent: 50 }), 0.5); + assert.equal(resolveAdobeCreativityLevel({ creativityPercent: 40 }), 0.4); + assert.equal(resolveAdobeCreativityLevel({}), 0); + // Explicit 0-1 wins over percent. + assert.equal(resolveAdobeCreativityLevel({ creativityPercent: 100, creativityLevel: 0.25 }), 0.25); + // Legacy 1-5 integer scale (discovery docs) is mapped onto 0-1. + assert.equal(resolveAdobeCreativityLevel({ creativityLevel: "4" }), 0.8); + assert.equal(resolveAdobeCreativityLevel({ creativityLevel: 5 }), 1); + assert.equal(resolveAdobeCreativityLevel({ creativityLevel: 99 }), 1); + assert.equal(resolveAdobeCreativityLevel({ creativityLevel: -3 }), 0); +}); + +test("buildAdobeUpsamplePayload matches the live upsample capture", () => { + const payload = buildAdobeUpsamplePayload({ + modelSpec: ADOBE_FIREFLY_UPSCALE_MODELS["topaz-bloom"], + blobId: "a99ffe89-ba67-478e-bd22-bb686506006e", + upsamplerFactor: 2, + creativityLevel: 0, + }); + + assert.equal(payload.modelId, "topaz"); + assert.equal(payload.modelVersion, "reimagine"); + assert.equal(payload.upsamplerFactor, 2); + assert.equal(payload.creativityLevel, 0); + assert.deepEqual(payload.referenceBlobs, [ + { id: "a99ffe89-ba67-478e-bd22-bb686506006e", usage: "general" }, + ]); + assert.deepEqual(payload.generationMetadata, { + module: "image-editing", + submodule: "ff-image-editor", + sourceDocumentId: null, + originalPrompt: null, + filterString: null, + subPrompts: null, + canvasImageReference: null, + }); + // No prompt / size / n keys — the upsample contract has none. + assert.ok(!("prompt" in payload)); + assert.ok(!("n" in payload)); +}); + +test("buildAdobeUpsamplePayload omits creativityLevel for the non-generative version", () => { + const payload = buildAdobeUpsamplePayload({ + modelSpec: ADOBE_FIREFLY_UPSCALE_MODELS["topaz-standard"], + blobId: "blob-1", + upsamplerFactor: 4, + creativityLevel: 3, + }); + assert.equal(payload.upsamplerFactor, 4); + assert.ok(!("creativityLevel" in payload), "standard upscale must not send creativityLevel"); +}); + +test("buildAdobeUpsampleHeaders mirrors the capture (ARP present, x-nonce absent)", () => { + const headers = buildAdobeUpsampleHeaders(FAKE_JWT, { arpSessionId: "arp-test-1" }); + assert.equal(headers.Authorization, `Bearer ${FAKE_JWT}`); + assert.equal(headers["x-arp-session-id"], "arp-test-1"); + assert.equal(headers["content-type"], "application/json"); + assert.ok(headers["x-api-key"], "x-api-key must be sent"); + assert.equal(headers["x-nonce"], undefined, "upsample capture sends no x-nonce"); + assert.equal(headers.Cookie, undefined, "page cookies never go to firefly-3p"); +}); + +test("adobeFireflyUpscaleImage submits to /v2/3p-images/upsample and polls the result link", async () => { + const calls: Array<{ url: string; init?: RequestInit }> = []; + const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => { + const href = String(url); + calls.push({ url: href, init }); + if (href === ADOBE_FIREFLY_IMAGE_UPSAMPLE_URL) { + return jsonResponse({ + links: { + result: { href: "https://firefly-epo855232.adobe.io/jobs/result/job-42" }, + }, + }); + } + return jsonResponse({ + status: "COMPLETED", + outputs: [{ image: { presignedUrl: "https://s3.example/upscaled.png?X-Amz=1" } }], + }); + }) as unknown as typeof fetch; + + const result = await adobeFireflyUpscaleImage({ + accessToken: FAKE_JWT, + model: "adobe-firefly/topaz-bloom", + blobId: "blob-9", + upsamplerFactor: 4, + creativityPercent: 100, + fetchImpl, + }); + + assert.equal(result.url, "https://s3.example/upscaled.png?X-Amz=1"); + assert.equal(result.factor, 4); + assert.equal(result.creativityLevel, 1); + + assert.equal(calls[0]!.url, ADOBE_FIREFLY_IMAGE_UPSAMPLE_URL); + const submitted = JSON.parse(String(calls[0]!.init?.body)); + assert.equal(submitted.modelVersion, "reimagine"); + assert.equal(submitted.upsamplerFactor, 4); + assert.equal(submitted.creativityLevel, 1); + assert.deepEqual(submitted.referenceBlobs, [{ id: "blob-9", usage: "general" }]); + + // Poll URL is rewritten to the BKS host, exactly like generate-async. + assert.equal( + calls[1]!.url, + "https://bks-epo8552.adobe.io/v2/jobs/result/job-42?host=firefly-epo855232.adobe.io" + ); +}); + +test("adobeFireflyUpscaleImage rejects a non-upscale model and a missing blob", async () => { + await assert.rejects( + () => + adobeFireflyUpscaleImage({ + accessToken: FAKE_JWT, + model: "nano-banana-pro", + blobId: "blob-1", + }), + /Unsupported Adobe Firefly upscale model/ + ); + await assert.rejects( + () => + adobeFireflyUpscaleImage({ + accessToken: FAKE_JWT, + model: "topaz-bloom", + blobId: " ", + }), + /requires a source image/ + ); +}); + +// ── Shared helpers ───────────────────────────────────────────────────────── + +test("extractUpscaleSourceImage finds the first image across every alias", () => { + assert.equal(extractUpscaleSourceImage({ image: "data:image/png;base64,AAA" }), "data:image/png;base64,AAA"); + assert.equal(extractUpscaleSourceImage({ image_url: "https://x/y.png" }), "https://x/y.png"); + assert.equal(extractUpscaleSourceImage({ images: ["https://a/1.png", "https://a/2.png"] }), "https://a/1.png"); + assert.equal( + extractUpscaleSourceImage({ image_url: { url: "https://obj/u.png" } }), + "https://obj/u.png" + ); + assert.equal( + extractUpscaleSourceImage({ provider_options: { image_urls: ["https://po/1.png"] } }), + "https://po/1.png" + ); + assert.equal( + extractUpscaleSourceImage({ + messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: "https://m/1.png" } }] }], + }), + "https://m/1.png" + ); + assert.equal(extractUpscaleSourceImage({ image: " " }), null); + assert.equal(extractUpscaleSourceImage({ image: "null" }), null); + assert.equal(extractUpscaleSourceImage(null), null); + assert.equal(extractUpscaleSourceImage({ prompt: "hi" }), null); +}); + +test("readImageDimensions parses PNG and JPEG headers", () => { + assert.deepEqual(readImageDimensions(pngHeader(640, 480)), { width: 640, height: 480 }); + assert.deepEqual(readImageDimensions(PNG_1X1), { width: 1, height: 1 }); + assert.deepEqual(readImageDimensions(jpegHeader(1920, 1080)), { width: 1920, height: 1080 }); + assert.equal(readImageDimensions(Buffer.from("not an image")), null); + assert.equal(readImageDimensions(Buffer.alloc(0)), null); +}); + +test("sniffImageMime recognizes PNG and JPEG magic bytes", () => { + assert.equal(sniffImageMime(PNG_1X1), "image/png"); + assert.equal(sniffImageMime(jpegHeader(2, 2)), "image/jpeg"); + assert.equal(sniffImageMime(Buffer.from("zzzz")), "image/png"); +}); + +test("scaleDimensions multiplies the source size and clamps the long edge", () => { + assert.deepEqual(scaleDimensions(pngHeader(640, 480), 2), { width: 1280, height: 960 }); + assert.deepEqual(scaleDimensions(pngHeader(640, 480), 4), { width: 2560, height: 1920 }); + // Clamp: a 4x pass on a 5000px edge with maxEdge 8000 scales by 1.6, not 4. + assert.deepEqual(scaleDimensions(pngHeader(5000, 2500), 4, 8000), { width: 8000, height: 4000 }); + // Never downscale, even when the source already exceeds maxEdge. + assert.deepEqual(scaleDimensions(pngHeader(9000, 9000), 4, 8000), { width: 9000, height: 9000 }); + assert.equal(scaleDimensions(Buffer.from("nope"), 2), null); +}); + +// ── Dispatcher ───────────────────────────────────────────────────────────── + +test("handleImageUpscale rejects unknown / mismatched models before any network call", async () => { + const badModel = await handleImageUpscale({ body: { model: "openai/gpt-image-2" }, credentials: {} }); + assert.equal(badModel.success, false); + assert.equal(badModel.status, 400); + assert.match(String(badModel.error), /Invalid upscale model/); + + const badPair = await handleImageUpscale({ + body: { model: "stability-ai/topaz-bloom" }, + credentials: {}, + }); + assert.equal(badPair.success, false); + assert.equal(badPair.status, 400); + assert.match(String(badPair.error), /Unsupported upscale model for stability-ai/); + + const missing = await handleImageUpscale({ body: {}, credentials: {} }); + assert.equal(missing.success, false); + assert.equal(missing.status, 400); +}); + +test("handleImageUpscale requires a source image for every provider", async () => { + for (const model of ["adobe-firefly/topaz-standard", "stability-ai/fast", "topaz/topaz-enhance"]) { + const result = await handleImageUpscale({ + body: { model }, + credentials: { apiKey: "k" }, + }); + assert.equal(result.success, false, `${model} must fail without an image`); + assert.equal(result.status, 400); + assert.match(String(result.error), /source image/i); + } +}); + +// ── Stability AI ─────────────────────────────────────────────────────────── + +test("stability fast upscale posts multipart and returns the base64 image", async () => { + let captured: { url: string; form?: FormData } | null = null; + const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => { + captured = { url: String(url), form: init?.body as FormData }; + return jsonResponse({ image: PNG_1X1.toString("base64"), finish_reason: "SUCCESS", seed: 7 }); + }) as unknown as typeof fetch; + + const result = await handleStabilityImageUpscale({ + model: "fast", + provider: "stability-ai", + providerConfig: { baseUrl: "https://api.stability.ai" }, + body: { image: PNG_1X1_DATA_URL, response_format: "b64_json" }, + credentials: { apiKey: "sk-test" }, + fetchImpl, + }); + + assert.equal(result.success, true); + assert.equal(captured!.url, "https://api.stability.ai/v2beta/stable-image/upscale/fast"); + assert.ok(captured!.form instanceof FormData); + assert.ok(captured!.form!.get("image"), "image part must be present"); + assert.equal(captured!.form!.get("output_format"), "png"); + assert.equal(captured!.form!.get("creativity"), null, "fast takes no creativity"); + const data = (result.data as { data: Array<{ b64_json?: string }> }).data; + assert.equal(data[0]!.b64_json, PNG_1X1.toString("base64")); +}); + +test("stability conservative/creative demand a prompt and map creativity into range", async () => { + const noPrompt = await handleStabilityImageUpscale({ + model: "conservative", + provider: "stability-ai", + providerConfig: { baseUrl: "https://api.stability.ai" }, + body: { image: PNG_1X1_DATA_URL }, + credentials: { apiKey: "sk-test" }, + fetchImpl: (async () => jsonResponse({})) as unknown as typeof fetch, + }); + assert.equal(noPrompt.success, false); + assert.equal(noPrompt.status, 400); + assert.match(String(noPrompt.error), /requires a prompt/); + + let form: FormData | null = null; + const ok = await handleStabilityImageUpscale({ + model: "conservative", + provider: "stability-ai", + providerConfig: { baseUrl: "https://api.stability.ai" }, + body: { image: PNG_1X1_DATA_URL, prompt: "a cat", creativity: 100 }, + credentials: { apiKey: "sk-test" }, + fetchImpl: (async (_url: unknown, init?: RequestInit) => { + form = init?.body as FormData; + return jsonResponse({ image: PNG_1X1.toString("base64") }); + }) as unknown as typeof fetch, + }); + assert.equal(ok.success, true); + // conservative range is 0.2-0.5 → 100 % maps to the max. + assert.equal(form!.get("creativity"), "0.5"); + assert.equal(form!.get("prompt"), "a cat"); +}); + +test("stability creative polls /v2beta/results until the job completes", async () => { + const urls: string[] = []; + let pollCount = 0; + const fetchImpl = (async (url: string | URL | Request) => { + const href = String(url); + urls.push(href); + if (href.includes("/upscale/creative")) return jsonResponse({ id: "job-77" }); + pollCount += 1; + if (pollCount === 1) return new Response(null, { status: 202 }); + return jsonResponse({ image: PNG_1X1.toString("base64"), finish_reason: "SUCCESS" }); + }) as unknown as typeof fetch; + + const result = await handleStabilityImageUpscale({ + model: "creative", + provider: "stability-ai", + providerConfig: { baseUrl: "https://api.stability.ai" }, + body: { image: PNG_1X1_DATA_URL, prompt: "a cat", creativity: 0 }, + credentials: { apiKey: "sk-test" }, + fetchImpl, + }); + + assert.equal(result.success, true); + assert.equal(urls[1], "https://api.stability.ai/v2beta/results/job-77"); + assert.equal(urls[2], "https://api.stability.ai/v2beta/results/job-77"); + const entry = (result.data as { data: Array<{ url?: string }> }).data[0]!; + assert.match(String(entry.url), /^data:image\/png;base64,/); +}); + +test("stability surfaces CONTENT_FILTERED as a 400 instead of an empty image", async () => { + const result = await handleStabilityImageUpscale({ + model: "fast", + provider: "stability-ai", + providerConfig: { baseUrl: "https://api.stability.ai" }, + body: { image: PNG_1X1_DATA_URL }, + credentials: { apiKey: "sk-test" }, + fetchImpl: (async () => + jsonResponse({ finish_reason: "CONTENT_FILTERED" })) as unknown as typeof fetch, + }); + assert.equal(result.success, false); + assert.equal(result.status, 400); + assert.match(String(result.error), /CONTENT_FILTERED/); +}); + +// ── Topaz Labs ───────────────────────────────────────────────────────────── + +test("topaz enhance converts the factor into an absolute output size", async () => { + let form: FormData | null = null; + let headers: Record | null = null; + const source = Buffer.concat([pngHeader(800, 600), Buffer.alloc(8)]); + + const result = await handleTopazImageUpscale({ + model: "topaz-enhance", + provider: "topaz", + providerConfig: { baseUrl: "https://api.topazlabs.com" }, + body: { + image: `data:image/png;base64,${source.toString("base64")}`, + factor: 4, + output_format: "jpeg", + }, + credentials: { apiKey: "topaz-key" }, + fetchImpl: (async (_url: unknown, init?: RequestInit) => { + form = init?.body as FormData; + headers = init?.headers as Record; + return new Response(bytes(jpegHeader(3200, 2400)), { + status: 200, + headers: { "content-type": "image/jpeg" }, + }); + }) as unknown as typeof fetch, + }); + + assert.equal(result.success, true); + assert.equal(form!.get("output_width"), "3200"); + assert.equal(form!.get("output_height"), "2400"); + assert.equal(form!.get("output_format"), "jpeg"); + assert.equal(headers!["X-API-Key"], "topaz-key"); + assert.equal(headers!.Accept, "image/jpeg"); + const entry = (result.data as { data: Array<{ url?: string }> }).data[0]!; + assert.match(String(entry.url), /^data:image\/jpeg;base64,/); + assert.equal((result.data as { upscale: { factor: number } }).upscale.factor, 4); +}); + +test("topaz falls back to its own scale when the source dimensions are unreadable", async () => { + let form: FormData | null = null; + const result = await handleTopazImageUpscale({ + model: "topaz-enhance", + provider: "topaz", + providerConfig: { baseUrl: "https://api.topazlabs.com" }, + // A valid base64 payload whose bytes are not a recognizable image container. + body: { image: Buffer.from("x".repeat(200)).toString("base64"), factor: 2 }, + credentials: { apiKey: "topaz-key" }, + fetchImpl: (async (_url: unknown, init?: RequestInit) => { + form = init?.body as FormData; + return new Response(bytes(PNG_1X1), { status: 200, headers: { "content-type": "image/png" } }); + }) as unknown as typeof fetch, + }); + + assert.equal(result.success, true); + assert.equal(form!.get("output_width"), null); + assert.equal(form!.get("output_height"), null); +}); + +test("topaz honors an explicit WxH size over the factor and propagates upstream errors", async () => { + let form: FormData | null = null; + const source = Buffer.concat([pngHeader(100, 100), Buffer.alloc(8)]); + await handleTopazImageUpscale({ + model: "topaz-enhance", + provider: "topaz", + providerConfig: { baseUrl: "https://api.topazlabs.com" }, + body: { + image: `data:image/png;base64,${source.toString("base64")}`, + factor: 4, + size: "1500x1200", + }, + credentials: { apiKey: "topaz-key" }, + fetchImpl: (async (_url: unknown, init?: RequestInit) => { + form = init?.body as FormData; + return new Response(bytes(PNG_1X1), { status: 200, headers: { "content-type": "image/png" } }); + }) as unknown as typeof fetch, + }); + assert.equal(form!.get("output_width"), "1500"); + assert.equal(form!.get("output_height"), "1200"); + + const failed = await handleTopazImageUpscale({ + model: "topaz-enhance", + provider: "topaz", + providerConfig: { baseUrl: "https://api.topazlabs.com" }, + body: { image: PNG_1X1_DATA_URL }, + credentials: { apiKey: "topaz-key" }, + fetchImpl: (async () => + new Response("quota exceeded", { status: 402 })) as unknown as typeof fetch, + }); + assert.equal(failed.success, false); + assert.equal(failed.status, 402); + assert.match(String(failed.error), /quota exceeded/); +});