mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-26 09:02:11 +03:00
fix(ollama): route models by advertised capability (#11088)
Landed with the design call resolved per the owner's pick — **option 1**: the synced store is now endpoint-agnostic (persistDiscoveredModels and managedModelImport no longer drop non-chat models at write time), and chat selectability moved to read time (auto-pool expansion in autoStrategy applies filterChatSelectableModels; the models-route projection already had its chatOnly filter). Your discovery test now passes end-to-end (3/3): /api/show capabilities persist per connection and image/embedding requests route through the advertising host. Reconciliation notes: conflicted areas merged onto the current tip (adobe discovery import, requestedModel preflight signature, resolvedProvider fast-path coexists with the synced-route override — explicit resolution wins); carried base-red drains (#10055 memoization, #11071 test variants) dropped as already-landed; the managed-model-import exclusion test was propagated to the new contract (image/video models persist; the read filter still hides them from chat pickers — pinned by a new assertion). Full battery: 205/206 focused (the one red is a confirmed periodic-timer timing flake on the loaded devbox — 20/20 isolated), autoCombo vitest 30/30, combo suites 46/46, gates + typecheck clean. Thank you @yourspraveen — the capability probe + routing design was right; it just needed the store contract opened up. Fixes #11087.
This commit is contained in:
committed by
GitHub
parent
c68cda7dfb
commit
65e81158ab
94
scripts/dev/codex-ws.sh
Executable file
94
scripts/dev/codex-ws.sh
Executable file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# codex-ws — roda a OpenAI Codex CLI contra um OmniRoute LOCAL usando o
|
||||
# transporte Responses-over-WebSocket (em vez do HTTP da Cloud).
|
||||
#
|
||||
# POR QUE ESTE WRAPPER EXISTE
|
||||
# ---------------------------
|
||||
# O OmniRoute expõe um proxy WebSocket para a API de Responses do Codex em
|
||||
# ws(s)://<host>/v1/responses
|
||||
# A Codex CLI sabe falar esse transporte quando o provider tem
|
||||
# `supports_websockets = true` + `wire_api = "responses"`. Mas há DOIS detalhes
|
||||
# que quebram o uso ingênuo:
|
||||
#
|
||||
# 1) A Codex CLI valida o NOME do modelo no cliente. Ids com prefixo de provider
|
||||
# (ex.: "codex/gpt-5.5") são REJEITADOS ("model is not supported ... ChatGPT
|
||||
# account"). É preciso mandar o id "puro" -> "gpt-5.5". (O OmniRoute, no
|
||||
# bridge WS, re-resolve "gpt-5.5" -> provider codex internamente.)
|
||||
#
|
||||
# 2) A Codex CLI v0.136 carrega TAMBÉM "$CWD/.codex/config.toml" como config
|
||||
# "project-local". Se você rodar de um diretório que tenha um .codex (ex.:
|
||||
# /root, onde mora a config da Cloud), o `model` daquele arquivo SOBRESCREVE
|
||||
# o `model` do seu CODEX_HOME -> você acaba mandando o modelo errado.
|
||||
# Por isso forçamos model + model_provider via `-c` (precedência máxima),
|
||||
# que vence qualquer config de arquivo (user-level OU project-local).
|
||||
#
|
||||
# Além disso, no modo `exec` (headless) a CLI exige um diretório "confiável" ou
|
||||
# a flag --skip-git-repo-check; o wrapper adiciona a flag automaticamente.
|
||||
#
|
||||
# USO
|
||||
# ---
|
||||
# codex-ws "sua pergunta" # abre a TUI interativa (precisa de terminal)
|
||||
# codex-ws exec "Responda: PONG" # one-shot headless (CI/validação)
|
||||
# codex-ws --help # repassa flags pra Codex CLI
|
||||
#
|
||||
# Variáveis de ambiente (todas com default; sobrescreva exportando antes):
|
||||
# OMNIROUTE_WS_BASE base URL do OmniRoute local (default abaixo)
|
||||
# OMNIROUTE_WS_MODEL modelo codex (id puro) (default "gpt-5.5")
|
||||
# OMNIROUTE_LOCAL_KEY API key (qualquer valor se REQUIRE_API_KEY=false)
|
||||
# CODEX_WS_HOME dir de config isolado da Codex (default ~/.codex-ws)
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configuração (com defaults seguros) ------------------------------------
|
||||
OMNIROUTE_WS_BASE="${OMNIROUTE_WS_BASE:-http://127.0.0.1:20128/v1}" # base do OmniRoute local
|
||||
OMNIROUTE_WS_MODEL="${OMNIROUTE_WS_MODEL:-gpt-5.5}" # id PURO (sem "codex/")
|
||||
CODEX_WS_HOME="${CODEX_WS_HOME:-$HOME/.codex-ws}" # CODEX_HOME isolado da Cloud
|
||||
|
||||
# A Codex CLI lê a key Bearer da env var nomeada em `env_key`. Mantemos um nome
|
||||
# próprio para não colidir com OPENAI_API_KEY da config da Cloud.
|
||||
export OMNIROUTE_LOCAL_KEY="${OMNIROUTE_LOCAL_KEY:-local}"
|
||||
|
||||
# CODEX_HOME isolado: a Codex CLI usa ESTE diretório como config "user-level",
|
||||
# deixando a sua ~/.codex (Cloud) totalmente intacta.
|
||||
export CODEX_HOME="$CODEX_WS_HOME"
|
||||
|
||||
# ---- Garante que a config do CODEX_HOME exista (auto-bootstrap) --------------
|
||||
# Só o bloco [model_providers.*] precisa estar aqui; model/model_provider são
|
||||
# forçados via -c logo abaixo (por causa do detalhe #2 do cabeçalho).
|
||||
if [ ! -f "$CODEX_HOME/config.toml" ]; then
|
||||
mkdir -p "$CODEX_HOME"
|
||||
cat > "$CODEX_HOME/config.toml" <<EOF
|
||||
# Gerado por codex-ws.sh — config isolada para o WS local do OmniRoute.
|
||||
model = "$OMNIROUTE_WS_MODEL"
|
||||
model_provider = "omniroute-local"
|
||||
|
||||
[model_providers.omniroute-local]
|
||||
name = "OmniRoute Local (WS)"
|
||||
base_url = "$OMNIROUTE_WS_BASE" # a URL WebSocket é derivada desta base pela CLI
|
||||
wire_api = "responses" # único valor suportado desde fev/2026
|
||||
supports_websockets = true # <- liga o transporte Responses-over-WebSocket
|
||||
env_key = "OMNIROUTE_LOCAL_KEY" # a CLI lê a key Bearer desta env var
|
||||
|
||||
# Marca o HOME como diretório confiável para o modo exec.
|
||||
[projects."$HOME"]
|
||||
trust_level = "trusted"
|
||||
EOF
|
||||
fi
|
||||
|
||||
# ---- Overrides de precedência máxima ----------------------------------------
|
||||
# Vencem qualquer config de arquivo (inclusive a project-local da Cloud em
|
||||
# $CWD/.codex/config.toml). É o que garante o modelo certo no transporte certo.
|
||||
overrides=(-c model="$OMNIROUTE_WS_MODEL" -c model_provider="omniroute-local")
|
||||
|
||||
# ---- Dispatch ---------------------------------------------------------------
|
||||
# No modo headless (`exec`) injeta --skip-git-repo-check (senão a CLI recusa
|
||||
# rodar fora de um repo git "confiável"). O `shift` remove o "exec" duplicado.
|
||||
if [ "${1:-}" = "exec" ]; then
|
||||
shift
|
||||
exec codex exec --skip-git-repo-check "${overrides[@]}" "$@"
|
||||
fi
|
||||
|
||||
# Modo interativo (TUI) ou qualquer outro subcomando/flag: repassa direto.
|
||||
exec codex "${overrides[@]}" "$@"
|
||||
207
scripts/dev/generate-adobe-firefly-snapshot.mjs
Normal file
207
scripts/dev/generate-adobe-firefly-snapshot.mjs
Normal file
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
function usage() {
|
||||
console.error(
|
||||
"Usage: node scripts/dev/generate-adobe-firefly-snapshot.mjs <discovery.json> <output.ts>"
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const [, , inputArg, outputArg] = process.argv;
|
||||
if (!inputArg || !outputArg) usage();
|
||||
|
||||
const inputPath = path.resolve(inputArg);
|
||||
const outputPath = path.resolve(outputArg);
|
||||
const inputBytes = fs.readFileSync(inputPath);
|
||||
const sourceHash = createHash("sha256").update(inputBytes).digest("hex");
|
||||
const root = JSON.parse(inputBytes.toString("utf8"));
|
||||
|
||||
function mergeObjectSchema(schema) {
|
||||
const merged = { properties: {}, required: [] };
|
||||
const visit = (node) => {
|
||||
if (!node || typeof node !== "object") return;
|
||||
if (node.properties && typeof node.properties === "object") {
|
||||
Object.assign(merged.properties, node.properties);
|
||||
}
|
||||
if (Array.isArray(node.required)) merged.required.push(...node.required);
|
||||
if (Array.isArray(node.allOf)) node.allOf.forEach(visit);
|
||||
};
|
||||
visit(schema);
|
||||
merged.required = [...new Set(merged.required)];
|
||||
return merged;
|
||||
}
|
||||
|
||||
function branches(schema) {
|
||||
if (!schema || typeof schema !== "object") return [];
|
||||
return [schema, ...(schema.anyOf || []), ...(schema.oneOf || [])];
|
||||
}
|
||||
|
||||
function stringEnums(schema) {
|
||||
return [
|
||||
...new Set(
|
||||
branches(schema)
|
||||
.flatMap((branch) => (Array.isArray(branch.enum) ? branch.enum : []))
|
||||
.filter((value) => typeof value === "string")
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function integerSchema(schema) {
|
||||
return branches(schema).find((branch) => branch.type === "integer") || {};
|
||||
}
|
||||
|
||||
function publicModelId(modelId, modelVersion) {
|
||||
const slug = (value, allowDot = false) =>
|
||||
String(value || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(allowDot ? /[^a-z0-9.]+/g : /[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
const family = slug(modelId);
|
||||
const publicVersion =
|
||||
family === "kling" ? String(modelVersion).replace(/^kling_v3_omni/i, "kling_o3") : modelVersion;
|
||||
const version = slug(publicVersion, true);
|
||||
if (!version || version === "default" || version === family) return family || "model";
|
||||
return `${family}-${version}`;
|
||||
}
|
||||
|
||||
function normalizeModel(family, modelVersion, version) {
|
||||
const schema = mergeObjectSchema(version.requestSchema);
|
||||
const properties = schema.properties;
|
||||
const referenceSchema = properties.referenceBlobs || {};
|
||||
const referenceInputs = [];
|
||||
for (const media of referenceSchema["x-capabilities"] || []) {
|
||||
for (const usage of media.usageConstraints || []) {
|
||||
if (usage.deprecated === true) continue;
|
||||
referenceInputs.push({
|
||||
mediaType: String(media.mediaType || ""),
|
||||
usageType: String(usage.usageType || ""),
|
||||
minItems: Number.isInteger(usage.minItems) ? usage.minItems : 0,
|
||||
maxItems: Number.isInteger(usage.maxItems) ? usage.maxItems : null,
|
||||
maxFileSizeBytes: Number.isInteger(media.maxFileSizeBytes) ? media.maxFileSizeBytes : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const supportedSizes = [
|
||||
...new Set(
|
||||
branches(properties.size)
|
||||
.flatMap((branch) => (Array.isArray(branch.enum) ? branch.enum : []))
|
||||
.filter(
|
||||
(size) =>
|
||||
size &&
|
||||
Number.isInteger(size.width) &&
|
||||
size.width > 0 &&
|
||||
Number.isInteger(size.height) &&
|
||||
size.height > 0
|
||||
)
|
||||
.map((size) => `${size.width}x${size.height}`)
|
||||
),
|
||||
];
|
||||
const supportedAspectRatios = [
|
||||
...new Set(
|
||||
branches(properties.generationSettings).flatMap((branch) =>
|
||||
stringEnums(branch?.properties?.aspectRatio)
|
||||
)
|
||||
),
|
||||
];
|
||||
const duration = integerSchema(properties.duration);
|
||||
const supportedDurations = [
|
||||
...new Set(
|
||||
branches(properties.duration)
|
||||
.flatMap((branch) => (Array.isArray(branch.enum) ? branch.enum : []))
|
||||
.filter(Number.isInteger)
|
||||
),
|
||||
];
|
||||
const prompt = branches(properties.prompt).find((branch) => branch.type === "string") || {};
|
||||
const outputCount = integerSchema(properties.n);
|
||||
|
||||
return {
|
||||
id: publicModelId(family.modelId, modelVersion),
|
||||
name: String(version.modelDisplayName || version.modelCaiDisplayName || modelVersion),
|
||||
modality: version.outputModality[0],
|
||||
upstreamModelId: family.modelId,
|
||||
upstreamModelVersion: modelVersion,
|
||||
providerName: String(family.acModelFamilyProviderDisplayName || ""),
|
||||
releaseReadiness: String(version.releaseReadiness || ""),
|
||||
healthStatus: String(version.healthStatus || ""),
|
||||
inputMediaUseCases: (version.inputMediaUseCase || []).map(String),
|
||||
schemaProperties: Object.keys(properties),
|
||||
requiredProperties: schema.required,
|
||||
referenceInputs,
|
||||
maxReferenceItems: Number.isInteger(referenceSchema.maxItems) ? referenceSchema.maxItems : null,
|
||||
supportedSizes,
|
||||
supportedAspectRatios,
|
||||
supportedResolutions: stringEnums(properties.resolution),
|
||||
supportedDurations,
|
||||
durationMin: Number.isInteger(duration.minimum) ? duration.minimum : null,
|
||||
durationMax: Number.isInteger(duration.maximum) ? duration.maximum : null,
|
||||
durationDefault: Number.isInteger(duration.default) ? duration.default : null,
|
||||
outputCountMin: Number.isInteger(outputCount.minimum) ? outputCount.minimum : null,
|
||||
outputCountMax: Number.isInteger(outputCount.maximum) ? outputCount.maximum : null,
|
||||
promptMaxLength: Number.isInteger(prompt.maxLength) ? prompt.maxLength : null,
|
||||
backingModel: String(version.bksGenerationModel || ""),
|
||||
};
|
||||
}
|
||||
|
||||
const rawModels = [];
|
||||
for (const family of Array.isArray(root.models) ? root.models : []) {
|
||||
for (const [modelVersion, version] of Object.entries(family.modelVersions || {})) {
|
||||
if (!version || version.enabled === false) continue;
|
||||
const modality = Array.isArray(version.outputModality)
|
||||
? version.outputModality.map((value) => String(value).toLowerCase())[0]
|
||||
: "";
|
||||
if (modality !== "image" && modality !== "video") continue;
|
||||
|
||||
const schema = mergeObjectSchema(version.requestSchema);
|
||||
if (!schema.properties.prompt) continue;
|
||||
const useCases = (version.inputMediaUseCase || []).map((value) => String(value).toLowerCase());
|
||||
if (useCases.some((value) => ["upscaling", "sharpening", "denoising"].includes(value))) {
|
||||
continue;
|
||||
}
|
||||
rawModels.push(normalizeModel(family, modelVersion, version));
|
||||
}
|
||||
}
|
||||
|
||||
// Discovery currently repeats a few exact aliases (for example flux/fluxPro and
|
||||
// fluxPro/1.1). Keep the first canonical wire pair and suppress duplicate cards.
|
||||
const seen = new Set();
|
||||
const models = [];
|
||||
for (const model of rawModels) {
|
||||
const semanticKey = JSON.stringify({
|
||||
backingModel: model.backingModel,
|
||||
name: model.name,
|
||||
modality: model.modality,
|
||||
schemaProperties: model.schemaProperties,
|
||||
requiredProperties: model.requiredProperties,
|
||||
referenceInputs: model.referenceInputs,
|
||||
maxReferenceItems: model.maxReferenceItems,
|
||||
supportedSizes: model.supportedSizes,
|
||||
supportedAspectRatios: model.supportedAspectRatios,
|
||||
supportedResolutions: model.supportedResolutions,
|
||||
supportedDurations: model.supportedDurations,
|
||||
durationMin: model.durationMin,
|
||||
durationMax: model.durationMax,
|
||||
});
|
||||
if (seen.has(semanticKey)) continue;
|
||||
seen.add(semanticKey);
|
||||
models.push(model);
|
||||
}
|
||||
|
||||
const source = `/**
|
||||
* Generated from Adobe Firefly POST /v2/models/discovery with resolveSchema=true.
|
||||
* Source SHA-256: ${sourceHash}
|
||||
* Regenerate with scripts/dev/generate-adobe-firefly-snapshot.mjs; do not edit by hand.
|
||||
* The generated literal stays compact to satisfy the repository's line-count gate.
|
||||
*/
|
||||
// prettier-ignore
|
||||
export const ADOBE_FIREFLY_DISCOVERY_SNAPSHOT = ${JSON.stringify(models)} as const;
|
||||
`;
|
||||
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, source, "utf8");
|
||||
console.log(`Wrote ${models.length} models to ${outputPath}`);
|
||||
@@ -2,9 +2,21 @@
|
||||
|
||||
/**
|
||||
* Docker healthcheck script for OmniRoute.
|
||||
* Probes the /api/monitoring/health endpoint on the dashboard port.
|
||||
* Probes the lightweight /healthz endpoint on the dashboard port.
|
||||
* /api/monitoring/health is the deep human/dashboard check (SQLite ping);
|
||||
* using it as Docker HEALTHCHECK marks the container Unhealthy whenever the
|
||||
* event loop is busy (#10052) and can restart the only replica mid-session.
|
||||
* Used by Dockerfile and docker-compose files.
|
||||
*
|
||||
* #10311 — the container HEALTHCHECK previously probed the heavy
|
||||
* /api/monitoring/health path (synchronous SQLite reads + deep monitoring
|
||||
* aggregation) on the same single-process event loop as catalog rebuild /
|
||||
* long-context compression. Under load that probe could stall past the 5s
|
||||
* timeout and flip the container `unhealthy`, restarting it mid-session and
|
||||
* killing active SSE streams. /healthz is a pure in-memory lifecycle check
|
||||
* with no DB access. Operators who want the deep monitoring probe can opt
|
||||
* back in with OMNIROUTE_HEALTHCHECK_PATH.
|
||||
*
|
||||
* #3151 — in some Docker network setups the server binds to a container IP and
|
||||
* a probe against `127.0.0.1` is not reachable, while `localhost`/`::1` (or vice
|
||||
* versa) is. The previous version probed ONLY `127.0.0.1` and swallowed every
|
||||
@@ -21,7 +33,7 @@ import { networkInterfaces } from "node:os";
|
||||
|
||||
const DEFAULT_HOSTS = ["127.0.0.1", "localhost", "::1"];
|
||||
const DEFAULT_TIMEOUT_MS = 4000;
|
||||
const DEFAULT_HEALTH_PATH = "/api/monitoring/health";
|
||||
const DEFAULT_HEALTH_PATH = "/healthz";
|
||||
|
||||
function normalizeBasePath(value) {
|
||||
const trimmed = typeof value === "string" ? value.trim() : "";
|
||||
@@ -32,10 +44,34 @@ function normalizeBasePath(value) {
|
||||
return `/${segments.join("/")}`;
|
||||
}
|
||||
|
||||
/** Prefixes the health route with the configured Next.js basePath. */
|
||||
export function resolveHealthPath(basePathValue) {
|
||||
/**
|
||||
* Normalize an explicit health-check path override (OMNIROUTE_HEALTHCHECK_PATH).
|
||||
* Returns "" when absent/invalid so callers fall back to DEFAULT_HEALTH_PATH.
|
||||
* Mirrors normalizeBasePath's safety rules (no query/hash/backslash, no "." /
|
||||
* ".." segments, must start with "/").
|
||||
*/
|
||||
function normalizeHealthPath(value) {
|
||||
const trimmed = typeof value === "string" ? value.trim() : "";
|
||||
if (!trimmed) return "";
|
||||
if (!trimmed.startsWith("/") || /[?#\\]/.test(trimmed)) return "";
|
||||
const segments = trimmed.split("/").filter(Boolean);
|
||||
if (segments.some((segment) => segment === "." || segment === "..")) return "";
|
||||
return `/${segments.join("/")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the health route to probe. By default the lightweight /healthz
|
||||
* lifecycle endpoint (pure in-memory, no DB reads). An explicit
|
||||
* OMNIROUTE_HEALTHCHECK_PATH override opts back into the deep monitoring
|
||||
* probe. The configured Next.js basePath is always prefixed.
|
||||
*
|
||||
* @param {string} [basePathValue] value of OMNIROUTE_BASE_PATH
|
||||
* @param {string} [healthPathValue] value of OMNIROUTE_HEALTHCHECK_PATH
|
||||
*/
|
||||
export function resolveHealthPath(basePathValue, healthPathValue) {
|
||||
const basePath = normalizeBasePath(basePathValue);
|
||||
return basePath ? `${basePath}${DEFAULT_HEALTH_PATH}` : DEFAULT_HEALTH_PATH;
|
||||
const healthPath = normalizeHealthPath(healthPathValue) || DEFAULT_HEALTH_PATH;
|
||||
return basePath ? `${basePath}${healthPath}` : healthPath;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -115,7 +151,10 @@ async function main() {
|
||||
}
|
||||
|
||||
try {
|
||||
const healthPath = resolveHealthPath(process.env.OMNIROUTE_BASE_PATH);
|
||||
const healthPath = resolveHealthPath(
|
||||
process.env.OMNIROUTE_BASE_PATH,
|
||||
process.env.OMNIROUTE_HEALTHCHECK_PATH
|
||||
);
|
||||
await probeHealth({ port, hosts, healthPath });
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
|
||||
@@ -317,6 +317,17 @@ function getAuthHeaders(requestUrl, requestHeaders) {
|
||||
if (isText(requestHeaders["x-forwarded-for"])) {
|
||||
headers["x-forwarded-for"] = requestHeaders["x-forwarded-for"];
|
||||
}
|
||||
for (const key of [
|
||||
"session-id",
|
||||
"session_id",
|
||||
"x-codex-installation-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-turn-metadata",
|
||||
"originator",
|
||||
"user-agent",
|
||||
]) {
|
||||
if (isText(requestHeaders[key])) headers[key] = requestHeaders[key];
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
@@ -585,12 +596,18 @@ class ResponsesWsSession {
|
||||
// preparedContext, but never touches this.upstream/this.upstreamReady; the caller decides
|
||||
// whether a new upstream socket is needed.
|
||||
async runPrepare(message, responseBody) {
|
||||
const prepared = await callInternal(this.fetchImpl, this.baseUrl, this.bridgeSecret, "prepare", {
|
||||
requestUrl: this.requestUrl,
|
||||
headers: getAuthHeaders(this.requestUrl, this.requestHeaders),
|
||||
message,
|
||||
response: responseBody,
|
||||
});
|
||||
const prepared = await callInternal(
|
||||
this.fetchImpl,
|
||||
this.baseUrl,
|
||||
this.bridgeSecret,
|
||||
"prepare",
|
||||
{
|
||||
requestUrl: this.requestUrl,
|
||||
headers: getAuthHeaders(this.requestUrl, this.requestHeaders),
|
||||
message,
|
||||
response: responseBody,
|
||||
}
|
||||
);
|
||||
|
||||
if (!prepared.ok) {
|
||||
const message2 =
|
||||
@@ -602,6 +619,7 @@ class ResponsesWsSession {
|
||||
const error = new Error(message2);
|
||||
error.code = code;
|
||||
error.status = prepared.status;
|
||||
if (code === "responses_websocket_http_fallback") error.httpFallback = true;
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -716,11 +734,28 @@ class ResponsesWsSession {
|
||||
// otherwise every turn after the first bypasses the whole pipeline. This reuses
|
||||
// the already-established upstream transport; it must NOT recreate the socket.
|
||||
const prepared = await this.runPrepare(message, nextTurnBody);
|
||||
this.upstream.send(jsonStringifySafe(withPreparedResponseCreate(message, prepared.json.response)));
|
||||
this.upstream.send(
|
||||
jsonStringifySafe(withPreparedResponseCreate(message, prepared.json.response))
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.upstream.send(jsonStringifySafe(message));
|
||||
} catch (error) {
|
||||
if (error?.httpFallback) {
|
||||
const failurePayload = this.sendFailure(
|
||||
"responses_websocket_http_fallback",
|
||||
"Retry this request over HTTP/SSE Responses"
|
||||
);
|
||||
void this.persistHistory({
|
||||
status: 426,
|
||||
success: false,
|
||||
errorCode: "responses_websocket_http_fallback",
|
||||
errorMessage: "HTTP/SSE Responses transport required",
|
||||
terminalMessage: failurePayload,
|
||||
});
|
||||
this.close(1013, "http_fallback_required");
|
||||
return;
|
||||
}
|
||||
const code = error?.code || "upstream_websocket_connect_failed";
|
||||
const messageText = error instanceof Error ? error.message : String(error);
|
||||
const failurePayload = this.sendFailure(code, messageText);
|
||||
|
||||
@@ -74,7 +74,15 @@ async function main() {
|
||||
|
||||
const vitestProcess = spawn(
|
||||
process.execPath,
|
||||
["./node_modules/vitest/vitest.mjs", "run", "tests/e2e/ecosystem.test.ts"],
|
||||
[
|
||||
"./node_modules/vitest/vitest.mjs",
|
||||
"run",
|
||||
// Without --config, Vitest loads vitest.config.ts, whose exclude list drops
|
||||
// this file — the run then dies with "No test files found".
|
||||
"--config",
|
||||
"vitest.e2e-live.config.ts",
|
||||
"tests/e2e/ecosystem.test.ts",
|
||||
],
|
||||
{
|
||||
stdio: "inherit",
|
||||
env: testEnv,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { ensureNativeSqlite } from "./ensure-native-sqlite.mjs";
|
||||
import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopackCacheHeal.mjs";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
|
||||
const { maybeHandleDisallowedMethod } = methodGuard;
|
||||
const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard;
|
||||
@@ -60,6 +61,13 @@ for (const [key, value] of Object.entries(mergedEnv)) {
|
||||
}
|
||||
}
|
||||
|
||||
// systemd sd_notify (Type=notify / WatchdogSec=): this process owns the
|
||||
// watchdog pings — if its event loop blocks (freeze), the pings stop and
|
||||
// systemd kills the service. No-op outside systemd (no NOTIFY_SOCKET).
|
||||
// Created AFTER .env is merged so the OMNIROUTE_DISABLE_SD_NOTIFY opt-out
|
||||
// documented in .env is honored on this path too.
|
||||
const systemdNotifier = createSystemdNotifier();
|
||||
|
||||
// The mergedEnv copy above pulls NODE_ENV straight from `.env` — and the shipped
|
||||
// `.env.example` default is `NODE_ENV=production`. Next's programmatic `next()`
|
||||
// entry (unlike the `next` CLI) trusts that value verbatim, so `npm run dev`
|
||||
@@ -75,8 +83,10 @@ const { dashboardPort } = runtimePorts;
|
||||
const hostname = process.env.HOST || "0.0.0.0";
|
||||
// Turbopack by default in dev (matches the Next 16 CLI default and the production
|
||||
// build default in build-next-isolated.mjs); OMNIROUTE_USE_TURBOPACK=0 is the
|
||||
// webpack escape hatch.
|
||||
const useTurbopack = dev && mergedEnv.OMNIROUTE_USE_TURBOPACK !== "0";
|
||||
// webpack escape hatch. Under Bun, Turbopack native V8 bindings are unavailable,
|
||||
// so Bun automatically disables Turbopack and uses Webpack.
|
||||
const isBun = Boolean(process.versions.bun);
|
||||
const useTurbopack = dev && mergedEnv.OMNIROUTE_USE_TURBOPACK !== "0" && !isBun;
|
||||
process.env.OMNIROUTE_WS_BRIDGE_SECRET ||= randomUUID();
|
||||
// Per-process secret used to prove the trusted peer-IP stamp came from this
|
||||
// server (read by the authz middleware in the same process). See peer-stamp.mjs.
|
||||
@@ -184,6 +194,7 @@ async function start() {
|
||||
});
|
||||
|
||||
const shutdown = async (signal) => {
|
||||
systemdNotifier.stopping();
|
||||
try {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
await nextApp.close();
|
||||
@@ -202,6 +213,8 @@ async function start() {
|
||||
console.log(
|
||||
`[Next] ${mode} server listening on http://${hostname}:${dashboardPort} (${bundler})`
|
||||
);
|
||||
systemdNotifier.ready();
|
||||
systemdNotifier.startWatchdog();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -73,8 +73,11 @@ async function main() {
|
||||
[
|
||||
"./node_modules/vitest/vitest.mjs",
|
||||
"run",
|
||||
"--environment",
|
||||
"node",
|
||||
// Without --config, Vitest loads vitest.config.ts, whose exclude list drops
|
||||
// this file — the run then dies with "No test files found". The config also
|
||||
// sets environment: node, so the flag is no longer needed here.
|
||||
"--config",
|
||||
"vitest.e2e-live.config.ts",
|
||||
"tests/e2e/protocol-clients.test.ts",
|
||||
],
|
||||
{
|
||||
|
||||
@@ -5,6 +5,8 @@ import {
|
||||
resolveRuntimePorts,
|
||||
withRuntimePortEnv,
|
||||
resolveMaxOldSpaceMb,
|
||||
warnConflictingHeapLimits,
|
||||
buildStandaloneNodeOptions,
|
||||
spawnWithForwardedSignals,
|
||||
} from "../build/runtime-env.mjs";
|
||||
import { bootstrapEnv } from "../build/bootstrap-env.mjs";
|
||||
@@ -13,13 +15,13 @@ const env = bootstrapEnv();
|
||||
const runtimePorts = resolveRuntimePorts(env);
|
||||
const childEnv = withRuntimePortEnv(env, runtimePorts);
|
||||
|
||||
// #2939: honor OMNIROUTE_MEMORY_MB (default 512), the same knob
|
||||
// `omniroute serve` uses, so Docker users can control the server heap under
|
||||
// load / large SQLite DBs. A trailing --max-old-space-size wins, so this
|
||||
// overrides the image fallback without clobbering any other NODE_OPTIONS flags.
|
||||
// #2939 / #10353: OMNIROUTE_MEMORY_MB is the Docker/standalone heap knob.
|
||||
// When it is set, we append --max-old-space-size last (V8 last-flag wins).
|
||||
// When it is unset and NODE_OPTIONS already pins the heap, keep NODE_OPTIONS
|
||||
// (#5238). Warn when both are set and the numbers disagree.
|
||||
const maxOldSpaceMb = resolveMaxOldSpaceMb(childEnv.OMNIROUTE_MEMORY_MB);
|
||||
childEnv.NODE_OPTIONS =
|
||||
`${childEnv.NODE_OPTIONS || ""} --max-old-space-size=${maxOldSpaceMb}`.trim();
|
||||
warnConflictingHeapLimits(childEnv, maxOldSpaceMb);
|
||||
childEnv.NODE_OPTIONS = buildStandaloneNodeOptions(childEnv, maxOldSpaceMb);
|
||||
|
||||
// Prefer the WS-aware wrapper (server-ws.mjs) over the bare Next standalone
|
||||
// server.js: it installs the trusted peer-IP stamp (scripts/dev/peer-stamp.mjs)
|
||||
|
||||
@@ -255,20 +255,33 @@ async function signalProcessTree(child, signal) {
|
||||
}
|
||||
}
|
||||
|
||||
async function stopApp(child) {
|
||||
export async function stopApp(
|
||||
child,
|
||||
{
|
||||
currentPlatform = platform(),
|
||||
signalProcessTreeFn = signalProcessTree,
|
||||
waitForProcessTreeExitFn = waitForProcessTreeExit,
|
||||
} = {}
|
||||
) {
|
||||
if (!child.pid) return;
|
||||
|
||||
await signalProcessTree(child, "SIGTERM");
|
||||
await waitForProcessTreeExit(child, 5_000);
|
||||
// On Windows, terminating only the direct Electron process can orphan the
|
||||
// packaged server when the parent exits before the follow-up liveness check.
|
||||
// Kill the process tree in one operation while the root PID is still valid.
|
||||
if (currentPlatform === "win32") {
|
||||
await signalProcessTreeFn(child, "SIGKILL");
|
||||
await waitForProcessTreeExitFn(child, 2_000);
|
||||
return;
|
||||
}
|
||||
|
||||
const isStillRunning =
|
||||
platform() === "win32"
|
||||
? child.exitCode === null && child.signalCode === null
|
||||
: isProcessGroupAlive(child.pid);
|
||||
await signalProcessTreeFn(child, "SIGTERM");
|
||||
await waitForProcessTreeExitFn(child, 5_000);
|
||||
|
||||
const isStillRunning = isProcessGroupAlive(child.pid);
|
||||
|
||||
if (isStillRunning) {
|
||||
await signalProcessTree(child, "SIGKILL");
|
||||
await waitForProcessTreeExit(child, 2_000);
|
||||
await signalProcessTreeFn(child, "SIGKILL");
|
||||
await waitForProcessTreeExitFn(child, 2_000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -396,45 +409,115 @@ async function settleAfterReady({ getExitState, logs, settleMs }) {
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const appExecutable = discoverPackagedExecutable();
|
||||
if (!existsSync(appExecutable)) {
|
||||
function assertExecutableExists(appExecutable) {
|
||||
if (existsSync(appExecutable)) return;
|
||||
|
||||
throw new Error(
|
||||
`Packaged OmniRoute executable not found at ${appExecutable}. Build it first with \`npm run build:<target> --prefix electron\` or set ELECTRON_SMOKE_APP_EXECUTABLE.`
|
||||
);
|
||||
}
|
||||
|
||||
// ── CI sandbox workaround ──────────────────────────────────
|
||||
// GitHub Actions runners cannot set SUID on chrome-sandbox (Linux)
|
||||
// and Windows runners may fail silently without --no-sandbox.
|
||||
function buildCiSpawnArgs(currentPlatform = platform()) {
|
||||
if (!process.env.CI) return [];
|
||||
|
||||
const spawnArgs = ["--no-sandbox", "--disable-gpu"];
|
||||
if (currentPlatform === "linux") {
|
||||
spawnArgs.push("--disable-dev-shm-usage");
|
||||
}
|
||||
return spawnArgs;
|
||||
}
|
||||
|
||||
const NATIVE_DRIVER_LOG_PATTERN = /\[DB\] Driver: (bun:sqlite|better-sqlite3|node:sqlite) \|/;
|
||||
const SQLJS_DRIVER_LOG_PATTERN = /\[DB\] Driver: sql\.js \|/;
|
||||
|
||||
/**
|
||||
* Regression guard for #7592: on a packaged app's SECOND launch against an
|
||||
* already-persisted DATA_DIR, a stale-ABI better-sqlite3 binary (resolved via
|
||||
* a Turbopack-hashed import) used to fail to load and silently fall through
|
||||
* to the sql.js (WASM) driver — which then OOMs/retry-loops on real-sized
|
||||
* databases. Asserts the startup log shows a native driver was selected.
|
||||
*/
|
||||
export function assertNativeDriverSelected(logs) {
|
||||
if (NATIVE_DRIVER_LOG_PATTERN.test(logs)) return;
|
||||
|
||||
if (SQLJS_DRIVER_LOG_PATTERN.test(logs)) {
|
||||
throw new Error(
|
||||
`Packaged OmniRoute executable not found at ${appExecutable}. Build it first with \`npm run build:<target> --prefix electron\` or set ELECTRON_SMOKE_APP_EXECUTABLE.`
|
||||
"Packaged Electron app fell back to the sql.js (WASM) driver instead of a native SQLite " +
|
||||
"driver — this is the regression #7592 guards against (stale-ABI better-sqlite3 binary)."
|
||||
);
|
||||
}
|
||||
|
||||
const smokeUrl = process.env.ELECTRON_SMOKE_URL || DEFAULT_URL;
|
||||
const timeoutMs = parsePositiveInteger(process.env.ELECTRON_SMOKE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS);
|
||||
const settleMs = parsePositiveInteger(process.env.ELECTRON_SMOKE_SETTLE_MS, DEFAULT_SETTLE_MS);
|
||||
const dataDir =
|
||||
process.env.ELECTRON_SMOKE_DATA_DIR ||
|
||||
(await mkdtemp(join(tmpdir(), "omniroute-electron-smoke-")));
|
||||
const removeDataDir =
|
||||
!process.env.ELECTRON_SMOKE_DATA_DIR && process.env.ELECTRON_SMOKE_KEEP_DATA !== "1";
|
||||
const smokeEnv = buildSmokeEnv({ dataDir });
|
||||
throw new Error(
|
||||
"Packaged Electron app logs contain no '[DB] Driver: ...' line — cannot confirm which SQLite " +
|
||||
"driver loaded."
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForReady({ logs, smokeUrl, timeoutMs, settleMs, exitState }) {
|
||||
const startedAt = Date.now();
|
||||
let lastError = null;
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
assertNoFatalLogs(logs.value);
|
||||
|
||||
if (exitState.spawnError !== null) {
|
||||
throw new Error(`Packaged Electron app failed to launch: ${exitState.spawnError.message}`);
|
||||
}
|
||||
if (exitState.exitCode !== null || exitState.signalCode !== null) {
|
||||
throw new Error(
|
||||
`Packaged Electron app exited before readiness: code=${exitState.exitCode} signal=${exitState.signalCode}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithTimeout(smokeUrl, 1_000);
|
||||
if (response.status === 200) {
|
||||
assertNoFatalLogs(logs.value);
|
||||
console.log(`[electron-smoke] ready: ${smokeUrl} returned HTTP 200`);
|
||||
await settleAfterReady({
|
||||
getExitState: () => ({ exitCode: exitState.exitCode, signalCode: exitState.signalCode }),
|
||||
logs,
|
||||
settleMs,
|
||||
});
|
||||
console.log(`[electron-smoke] stable for ${settleMs}ms after readiness`);
|
||||
return;
|
||||
}
|
||||
lastError = new Error(`HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Packaged Electron app did not serve ${smokeUrl} within ${timeoutMs}ms. Last error: ${
|
||||
lastError instanceof Error ? lastError.message : String(lastError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Launches the packaged app once against `dataDir`, waits for readiness +
|
||||
* settle, tears it down, and returns the captured stdout/stderr text. Shared
|
||||
* by the single-launch path and the cold-restart (two-launch) path so both
|
||||
* exercise identical spawn/readiness/shutdown behavior.
|
||||
*/
|
||||
async function launchAndCollectLogs({ appExecutable, smokeUrl, dataDir, timeoutMs, settleMs, streamLogs }) {
|
||||
const smokeEnv = buildSmokeEnv({ dataDir });
|
||||
await assertPortIsFree(smokeUrl);
|
||||
await ensureSmokeEnvDirs(smokeEnv, dataDir);
|
||||
|
||||
// ── CI sandbox workaround ──────────────────────────────────
|
||||
// GitHub Actions runners cannot set SUID on chrome-sandbox (Linux)
|
||||
// and Windows runners may fail silently without --no-sandbox.
|
||||
const spawnArgs = [];
|
||||
if (process.env.CI) {
|
||||
spawnArgs.push("--no-sandbox", "--disable-gpu");
|
||||
if (platform() === "linux") {
|
||||
spawnArgs.push("--disable-dev-shm-usage");
|
||||
}
|
||||
}
|
||||
|
||||
const spawnArgs = buildCiSpawnArgs();
|
||||
console.log(`[electron-smoke] launching ${appExecutable}`);
|
||||
if (spawnArgs.length) console.log(`[electron-smoke] CI args: ${spawnArgs.join(" ")}`);
|
||||
console.log(`[electron-smoke] DATA_DIR=${dataDir}`);
|
||||
console.log(`[electron-smoke] waiting for ${smokeUrl}`);
|
||||
|
||||
const logs = { value: "" };
|
||||
const streamLogs = process.env.ELECTRON_SMOKE_STREAM_LOGS === "1";
|
||||
const child = spawn(appExecutable, spawnArgs, {
|
||||
detached: platform() !== "win32",
|
||||
env: smokeEnv,
|
||||
@@ -444,60 +527,18 @@ async function main() {
|
||||
child.stdout?.on("data", (chunk) => appendLog(logs, chunk, "[electron] ", streamLogs));
|
||||
child.stderr?.on("data", (chunk) => appendLog(logs, chunk, "[electron:err] ", streamLogs));
|
||||
|
||||
let exitCode = null;
|
||||
let signalCode = null;
|
||||
let spawnError = null;
|
||||
const exitState = { exitCode: null, signalCode: null, spawnError: null };
|
||||
child.once("exit", (code, signal) => {
|
||||
exitCode = code;
|
||||
signalCode = signal;
|
||||
exitState.exitCode = code;
|
||||
exitState.signalCode = signal;
|
||||
});
|
||||
child.once("error", (error) => {
|
||||
spawnError = error;
|
||||
exitState.spawnError = error;
|
||||
});
|
||||
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
let lastError = null;
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
assertNoFatalLogs(logs.value);
|
||||
|
||||
if (spawnError !== null) {
|
||||
throw new Error(`Packaged Electron app failed to launch: ${spawnError.message}`);
|
||||
}
|
||||
|
||||
if (exitCode !== null || signalCode !== null) {
|
||||
throw new Error(
|
||||
`Packaged Electron app exited before readiness: code=${exitCode} signal=${signalCode}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithTimeout(smokeUrl, 1_000);
|
||||
if (response.status === 200) {
|
||||
assertNoFatalLogs(logs.value);
|
||||
console.log(`[electron-smoke] ready: ${smokeUrl} returned HTTP 200`);
|
||||
await settleAfterReady({
|
||||
getExitState: () => ({ exitCode, signalCode }),
|
||||
logs,
|
||||
settleMs,
|
||||
});
|
||||
console.log(`[electron-smoke] stable for ${settleMs}ms after readiness`);
|
||||
return;
|
||||
}
|
||||
lastError = new Error(`HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Packaged Electron app did not serve ${smokeUrl} within ${timeoutMs}ms. Last error: ${
|
||||
lastError instanceof Error ? lastError.message : String(lastError)
|
||||
}`
|
||||
);
|
||||
await waitForReady({ logs, smokeUrl, timeoutMs, settleMs, exitState });
|
||||
return logs.value;
|
||||
} catch (error) {
|
||||
if (!streamLogs) {
|
||||
printLogTail(logs.value);
|
||||
@@ -506,6 +547,43 @@ async function main() {
|
||||
} finally {
|
||||
await stopApp(child);
|
||||
await waitForPortClosed(smokeUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const appExecutable = discoverPackagedExecutable();
|
||||
assertExecutableExists(appExecutable);
|
||||
|
||||
const smokeUrl = process.env.ELECTRON_SMOKE_URL || DEFAULT_URL;
|
||||
const timeoutMs = parsePositiveInteger(process.env.ELECTRON_SMOKE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS);
|
||||
const settleMs = parsePositiveInteger(process.env.ELECTRON_SMOKE_SETTLE_MS, DEFAULT_SETTLE_MS);
|
||||
const streamLogs = process.env.ELECTRON_SMOKE_STREAM_LOGS === "1";
|
||||
// #7592: rerun against the SAME (persisted) DATA_DIR and assert the second
|
||||
// launch selected a native SQLite driver, not the sql.js WASM fallback.
|
||||
const coldRestart = process.env.ELECTRON_SMOKE_COLD_RESTART === "1";
|
||||
const dataDir =
|
||||
process.env.ELECTRON_SMOKE_DATA_DIR ||
|
||||
(await mkdtemp(join(tmpdir(), "omniroute-electron-smoke-")));
|
||||
const removeDataDir =
|
||||
!process.env.ELECTRON_SMOKE_DATA_DIR && process.env.ELECTRON_SMOKE_KEEP_DATA !== "1";
|
||||
|
||||
try {
|
||||
await launchAndCollectLogs({ appExecutable, smokeUrl, dataDir, timeoutMs, settleMs, streamLogs });
|
||||
|
||||
if (!coldRestart) return;
|
||||
|
||||
console.log("[electron-smoke] cold-restart: relaunching against the same DATA_DIR");
|
||||
const secondLaunchLogs = await launchAndCollectLogs({
|
||||
appExecutable,
|
||||
smokeUrl,
|
||||
dataDir,
|
||||
timeoutMs,
|
||||
settleMs,
|
||||
streamLogs,
|
||||
});
|
||||
assertNativeDriverSelected(secondLaunchLogs);
|
||||
console.log("[electron-smoke] cold-restart: native SQLite driver confirmed on second launch");
|
||||
} finally {
|
||||
if (removeDataDir) {
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -3,11 +3,25 @@ import net from "node:net";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createResponsesWsProxy } from "./responses-ws-proxy.mjs";
|
||||
import { ensurePeerStampToken, wrapRequestListenerWithPeerStamp } from "./peer-stamp.mjs";
|
||||
import { maybeHandleWebdav } from "./webdav-handler.mjs";
|
||||
import { maybeHandleWebdav, WEBDAV_PREFIX } from "./webdav-handler.mjs";
|
||||
import methodGuard from "./http-method-guard.cjs";
|
||||
import headResponseGuard from "./head-response-guard.cjs";
|
||||
import { resolveTlsOptions, createServerListener } from "./tls-options.mjs";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
|
||||
// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one
|
||||
// whose event loop can freeze (cold /v1/models rebuild), so it must own the
|
||||
// watchdog pings — a blocked loop stops the pings and systemd kills the
|
||||
// service. No-op outside systemd (no NOTIFY_SOCKET).
|
||||
const systemdNotifier = createSystemdNotifier();
|
||||
let systemdReadySent = false;
|
||||
// NOTE: if an operator sets NEXT_MANUAL_SIG_HANDLE=1, Next never registers its
|
||||
// own signal cleanup and these once() handlers would suppress Node's default
|
||||
// signal exit (process lingers until systemd's stop-timeout SIGKILL). Nothing
|
||||
// in this repo sets that var; acceptable, documented behavior.
|
||||
process.once("SIGINT", () => systemdNotifier.stopping());
|
||||
process.once("SIGTERM", () => systemdNotifier.stopping());
|
||||
|
||||
const originalCreateServer = http.createServer.bind(http);
|
||||
const proxiesByPort = new Map();
|
||||
@@ -122,14 +136,20 @@ function wrapUpgradeListener(server, listener) {
|
||||
* Returns true if the request was handled; the wrapped listener is never called.
|
||||
*/
|
||||
function wrapRequestListenerWithWebdav(listener) {
|
||||
return async function webdavAwareRequestHandler(req, res) {
|
||||
try {
|
||||
const handled = await maybeHandleWebdav(req, res);
|
||||
if (handled) return;
|
||||
} catch {
|
||||
// Never block a request on WebDAV errors — fall through to Next
|
||||
return function webdavAwareRequestHandler(req, res) {
|
||||
if (!(req.url || "").startsWith(WEBDAV_PREFIX)) {
|
||||
return listener.call(this, req, res);
|
||||
}
|
||||
return listener.call(this, req, res);
|
||||
const self = this;
|
||||
(async () => {
|
||||
try {
|
||||
const handled = await maybeHandleWebdav(req, res);
|
||||
if (handled) return;
|
||||
} catch {
|
||||
// Never block a request on WebDAV errors — fall through to Next
|
||||
}
|
||||
return listener.call(self, req, res);
|
||||
})();
|
||||
};
|
||||
}
|
||||
|
||||
@@ -203,6 +223,15 @@ http.createServer = function createServerWithResponsesWs(...args) {
|
||||
return originalAddListener(eventName, listener);
|
||||
};
|
||||
|
||||
// sd_notify READY once the main listener is actually accepting, then arm
|
||||
// the watchdog keep-alive interval (unref'd — never keeps the process up).
|
||||
server.once("listening", () => {
|
||||
if (systemdReadySent) return;
|
||||
systemdReadySent = true;
|
||||
systemdNotifier.ready();
|
||||
systemdNotifier.startWatchdog();
|
||||
});
|
||||
|
||||
return server;
|
||||
};
|
||||
|
||||
|
||||
98
scripts/dev/systemd-notify.mjs
Normal file
98
scripts/dev/systemd-notify.mjs
Normal file
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Minimal systemd sd_notify integration (sd_notify(3) protocol).
|
||||
*
|
||||
* Node's stable API has no AF_UNIX datagram socket support (node:dgram is
|
||||
* udp4/udp6 only), so notifications are sent by spawning the `systemd-notify`
|
||||
* binary — present on every systemd host, no extra dependency.
|
||||
*
|
||||
* Everything is guarded: without a NOTIFY_SOCKET (plain terminal, Docker,
|
||||
* Electron, Windows) the notifier is a no-op and costs nothing. Set
|
||||
* OMNIROUTE_DISABLE_SD_NOTIFY=1 to force-disable even under systemd.
|
||||
*
|
||||
* A watchdog keep-alive interval lives in the main event loop of the process
|
||||
* that runs it: if that loop is ever blocked (frozen server, cf. the cold
|
||||
* /v1/models rebuild freeze), the pings stop and systemd kills the service
|
||||
* after WatchdogSec=.
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
export const SD_NOTIFY_BINARY = "systemd-notify";
|
||||
export const SD_NOTIFY_SOCKET_ENV = "NOTIFY_SOCKET";
|
||||
export const SD_NOTIFY_DISABLE_ENV = "OMNIROUTE_DISABLE_SD_NOTIFY";
|
||||
// Ping every 60s — satisfies any systemd WatchdogSec= >= 120s (systemd
|
||||
// requires keep-alive pings at most every WatchdogSec/2).
|
||||
export const SD_NOTIFY_WATCHDOG_INTERVAL_MS = 60_000;
|
||||
|
||||
export function isSystemdNotifyEnabled(env = process.env) {
|
||||
return Boolean(env[SD_NOTIFY_SOCKET_ENV]) && env[SD_NOTIFY_DISABLE_ENV] !== "1";
|
||||
}
|
||||
|
||||
export function buildNotifyMessage(kind) {
|
||||
switch (kind) {
|
||||
case "ready":
|
||||
return "READY=1";
|
||||
case "watchdog":
|
||||
return "WATCHDOG=1";
|
||||
case "stopping":
|
||||
return "STOPPING=1";
|
||||
default:
|
||||
throw new Error(`[omniroute][sd_notify] unknown message kind: ${kind}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function createSystemdNotifier({
|
||||
env = process.env,
|
||||
binary = SD_NOTIFY_BINARY,
|
||||
watchdogIntervalMs = SD_NOTIFY_WATCHDOG_INTERVAL_MS,
|
||||
spawnFn = spawn,
|
||||
onWarn = (message) => console.warn(message),
|
||||
} = {}) {
|
||||
const enabled = isSystemdNotifyEnabled(env);
|
||||
let disabled = false;
|
||||
let watchdogTimer = null;
|
||||
|
||||
const send = (kind) => {
|
||||
if (!enabled || disabled) return;
|
||||
const child = spawnFn(binary, [buildNotifyMessage(kind)], { env, stdio: "ignore" });
|
||||
// Never let a hung systemd-notify keep the process alive.
|
||||
child.unref?.();
|
||||
child.on("error", (err) => {
|
||||
// A failed send means systemd never sees the keep-alive: the service
|
||||
// would be killed as unhealthy anyway, so disabling loudly (one
|
||||
// warning) is safer than spamming errors forever.
|
||||
disabled = true;
|
||||
if (watchdogTimer) {
|
||||
clearInterval(watchdogTimer);
|
||||
watchdogTimer = null;
|
||||
}
|
||||
onWarn(
|
||||
`[omniroute][sd_notify] failed to send '${kind}' (${err?.code ?? err?.message ?? err}); sd_notify disabled for this process`
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
enabled,
|
||||
ready() {
|
||||
send("ready");
|
||||
},
|
||||
watchdog() {
|
||||
send("watchdog");
|
||||
},
|
||||
stopping() {
|
||||
send("stopping");
|
||||
},
|
||||
startWatchdog() {
|
||||
if (!enabled || disabled || watchdogTimer) return;
|
||||
watchdogTimer = setInterval(() => send("watchdog"), watchdogIntervalMs);
|
||||
watchdogTimer.unref?.();
|
||||
},
|
||||
dispose() {
|
||||
if (watchdogTimer) {
|
||||
clearInterval(watchdogTimer);
|
||||
watchdogTimer = null;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -185,6 +185,18 @@ function getForwardHeaders(requestUrl, requestHeaders) {
|
||||
headers.origin = origin;
|
||||
}
|
||||
|
||||
for (const key of [
|
||||
"session-id",
|
||||
"session_id",
|
||||
"x-codex-installation-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-turn-metadata",
|
||||
"originator",
|
||||
"user-agent",
|
||||
]) {
|
||||
if (isText(requestHeaders[key])) headers[key] = requestHeaders[key];
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user