mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-13 10:43:43 +03:00
fix: harden Devin bridge runtime boundaries
This commit is contained in:
@@ -3,17 +3,34 @@ set -euo pipefail
|
||||
BRIDGE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
BRIDGE_COMPOSE="$BRIDGE_ROOT/docker/devin-bridge/compose.yml"
|
||||
BRIDGE_SANDBOX="$BRIDGE_ROOT/.sandbox"
|
||||
BRIDGE_GUARD_AUDIT_ROOT="$BRIDGE_SANDBOX/guard-audit"
|
||||
BRIDGE_CLAUDE_AUDIT="$BRIDGE_GUARD_AUDIT_ROOT/claude/egress.jsonl"
|
||||
BRIDGE_DEVIN_AUDIT="$BRIDGE_GUARD_AUDIT_ROOT/devin/egress.jsonl"
|
||||
BRIDGE_RUNTIME_POLICY="$BRIDGE_ROOT/scripts/devin-bridge/runtime-policy.mjs"
|
||||
bridge_prepare_sandbox() {
|
||||
mkdir -p "$BRIDGE_SANDBOX/home" "$BRIDGE_SANDBOX/test-data" \
|
||||
"$BRIDGE_SANDBOX/e2e-workspace" "$BRIDGE_SANDBOX/live-workspace" \
|
||||
"$BRIDGE_SANDBOX/evidence"
|
||||
"$BRIDGE_SANDBOX/evidence" "$BRIDGE_GUARD_AUDIT_ROOT/claude" \
|
||||
"$BRIDGE_GUARD_AUDIT_ROOT/devin"
|
||||
chmod 0777 "$BRIDGE_SANDBOX/e2e-workspace" "$BRIDGE_SANDBOX/live-workspace" \
|
||||
"$BRIDGE_SANDBOX/evidence"
|
||||
chmod 01777 "$BRIDGE_GUARD_AUDIT_ROOT/claude" "$BRIDGE_GUARD_AUDIT_ROOT/devin"
|
||||
}
|
||||
bridge_reset_guard_audit() {
|
||||
local audit_path="$1"
|
||||
local audit_dir
|
||||
local temp_path
|
||||
bridge_prepare_sandbox
|
||||
audit_dir="$(dirname "$audit_path")"
|
||||
temp_path="$(mktemp "$audit_dir/.egress.jsonl.XXXXXX")"
|
||||
chmod 0666 "$temp_path"
|
||||
mv -f "$temp_path" "$audit_path"
|
||||
}
|
||||
bridge_reset_claude_egress_audit() {
|
||||
bridge_prepare_sandbox
|
||||
: >"$BRIDGE_SANDBOX/evidence/claude-egress.jsonl"
|
||||
chmod 0666 "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl"
|
||||
bridge_reset_guard_audit "$BRIDGE_CLAUDE_AUDIT"
|
||||
}
|
||||
bridge_reset_devin_egress_audit() {
|
||||
bridge_reset_guard_audit "$BRIDGE_DEVIN_AUDIT"
|
||||
}
|
||||
bridge_reset_e2e_fixture() {
|
||||
bridge_prepare_sandbox
|
||||
@@ -35,6 +52,7 @@ bridge_reset_live_fixture() {
|
||||
"$BRIDGE_SANDBOX/evidence/live-models.json" \
|
||||
"$BRIDGE_SANDBOX/evidence/egress.jsonl"
|
||||
bridge_reset_claude_egress_audit
|
||||
bridge_reset_devin_egress_audit
|
||||
}
|
||||
bridge_test_env() {
|
||||
bridge_prepare_sandbox
|
||||
@@ -55,18 +73,13 @@ bridge_run_devin() {
|
||||
bridge_assert_devin_auth_status() {
|
||||
local exit_status="$1"
|
||||
local output="$2"
|
||||
[[ "$exit_status" == 0 ]] || {
|
||||
printf 'FAIL: Devin auth status command failed\n' >&2
|
||||
return 1
|
||||
}
|
||||
[[ "$output" == *"Logged in (via Devin)"* ]] || {
|
||||
printf 'FAIL: Devin auth status did not confirm login\n' >&2
|
||||
return 1
|
||||
}
|
||||
if grep -Fqi 'Failed to fetch from server' <<<"$output"; then
|
||||
printf 'FAIL: Devin auth status could not confirm server access\n' >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s' "$output" | node --input-type=module -e '
|
||||
import { pathToFileURL } from "node:url";
|
||||
import fs from "node:fs";
|
||||
const policy = await import(pathToFileURL(process.argv[1]));
|
||||
const result = policy.validateDevinAuthStatus(process.argv[2], fs.readFileSync(0, "utf8"));
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
' "$BRIDGE_RUNTIME_POLICY" "$exit_status"
|
||||
}
|
||||
|
||||
bridge_check_devin_auth() {
|
||||
@@ -82,34 +95,37 @@ bridge_check_devin_auth() {
|
||||
|
||||
bridge_assert_zero_claude_egress() {
|
||||
local audit_path="$1"
|
||||
[[ -f "$audit_path" ]] || {
|
||||
printf 'FAIL: Claude egress audit file is missing\n' >&2
|
||||
return 1
|
||||
}
|
||||
[[ ! -s "$audit_path" ]] || {
|
||||
printf 'FAIL: Claude attempted external egress during the real run\n' >&2
|
||||
return 1
|
||||
}
|
||||
bridge_validate_guard_audit claude-zero "$audit_path"
|
||||
}
|
||||
|
||||
bridge_assert_claude_guard_denials() {
|
||||
local audit_path="$1"
|
||||
[[ -s "$audit_path" ]] || {
|
||||
printf 'FAIL: Claude egress denial audit is missing or empty\n' >&2
|
||||
return 1
|
||||
}
|
||||
node -e '
|
||||
const fs = require("node:fs");
|
||||
const entries = fs.readFileSync(process.argv[1], "utf8")
|
||||
.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
||||
if (!entries.length) throw new Error("Claude egress audit has no records");
|
||||
if (entries.some((entry) => entry.decision !== "deny")) {
|
||||
throw new Error("Claude egress audit contains a non-deny decision");
|
||||
}
|
||||
for (const hostname of ["api.anthropic.com", "claude.ai"]) {
|
||||
if (!entries.some((entry) => entry.hostname === hostname && entry.decision === "deny")) {
|
||||
throw new Error(`Claude egress audit is missing deny for ${hostname}`);
|
||||
}
|
||||
}
|
||||
' "$audit_path"
|
||||
bridge_validate_guard_audit claude-denials "$audit_path"
|
||||
}
|
||||
|
||||
bridge_assert_devin_guard_audit() {
|
||||
local audit_path="$1"
|
||||
bridge_validate_guard_audit devin-allowed "$audit_path"
|
||||
}
|
||||
|
||||
bridge_validate_guard_audit() {
|
||||
local kind="$1"
|
||||
local audit_path="$2"
|
||||
node --input-type=module -e '
|
||||
import { pathToFileURL } from "node:url";
|
||||
const policy = await import(pathToFileURL(process.argv[1]));
|
||||
policy.validateAuditFile(process.argv[2], process.argv[3], process.argv[4]);
|
||||
' "$BRIDGE_RUNTIME_POLICY" "$kind" "$audit_path" "$(id -u)"
|
||||
}
|
||||
|
||||
bridge_export_guard_audit() {
|
||||
local audit_path="$1"
|
||||
local evidence_name="$2"
|
||||
cp "$audit_path" "$BRIDGE_SANDBOX/evidence/$evidence_name"
|
||||
chmod 0644 "$BRIDGE_SANDBOX/evidence/$evidence_name"
|
||||
}
|
||||
|
||||
bridge_cleanup_compose() {
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline --profile live-devin \
|
||||
down --remove-orphans >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
source "$(dirname "$0")/common"
|
||||
trap bridge_cleanup_compose EXIT
|
||||
bridge_cleanup_compose
|
||||
bridge_prepare_sandbox
|
||||
"$(dirname "$0")/verify-anthropic-isolation"
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline --profile live-devin down --remove-orphans
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d network-guard claude-egress-guard
|
||||
bridge_reset_claude_egress_audit
|
||||
bridge_reset_devin_egress_audit
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d --wait network-guard claude-egress-guard
|
||||
bridge_check_devin_auth
|
||||
bridge_run_devin models list --format json >"$BRIDGE_SANDBOX/evidence/live-models.json"
|
||||
devin_model="$(node --import tsx/esm "$BRIDGE_ROOT/scripts/devin-bridge/select-live-model.mjs" \
|
||||
@@ -15,5 +18,11 @@ export DEVIN_BRIDGE_OPUS_MODEL="${DEVIN_BRIDGE_OPUS_MODEL:-$DEVIN_BRIDGE_MODEL}"
|
||||
export DEVIN_BRIDGE_HAIKU_MODEL="${DEVIN_BRIDGE_HAIKU_MODEL:-$DEVIN_BRIDGE_MODEL}"
|
||||
export DEVIN_BRIDGE_SUBAGENT_MODEL="${DEVIN_BRIDGE_SUBAGENT_MODEL:-$DEVIN_BRIDGE_MODEL}"
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d --wait omniroute-live
|
||||
exec docker compose -f "$BRIDGE_COMPOSE" --profile live-devin run --rm --no-deps \
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin run --rm --no-deps \
|
||||
claude-live claude
|
||||
bridge_cleanup_compose
|
||||
bridge_assert_devin_guard_audit "$BRIDGE_DEVIN_AUDIT"
|
||||
bridge_assert_zero_claude_egress "$BRIDGE_CLAUDE_AUDIT"
|
||||
bridge_export_guard_audit "$BRIDGE_DEVIN_AUDIT" egress.jsonl
|
||||
bridge_export_guard_audit "$BRIDGE_CLAUDE_AUDIT" claude-egress.jsonl
|
||||
trap - EXIT
|
||||
|
||||
@@ -2,8 +2,12 @@
|
||||
set -euo pipefail
|
||||
source "$(dirname "$0")/common"
|
||||
[[ "${ENABLE_LIVE_DEVIN_TESTS:-}" == 1 ]] || { echo 'Set ENABLE_LIVE_DEVIN_TESTS=1' >&2; exit 1; }
|
||||
trap bridge_cleanup_compose EXIT
|
||||
bridge_cleanup_compose
|
||||
bridge_prepare_sandbox
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline --profile live-devin down --remove-orphans
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d network-guard
|
||||
bridge_reset_devin_egress_audit
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d --wait network-guard
|
||||
bridge_run_devin auth login --force-manual-token-flow
|
||||
bridge_cleanup_compose
|
||||
trap - EXIT
|
||||
exec env ENABLE_LIVE_DEVIN_TESTS=1 "$(dirname "$0")/test-live-devin"
|
||||
|
||||
101
scripts/devin-bridge/runtime-policy.mjs
Normal file
101
scripts/devin-bridge/runtime-policy.mjs
Normal file
@@ -0,0 +1,101 @@
|
||||
import fs from "node:fs";
|
||||
|
||||
const ALLOWED_DEVIN_SUFFIXES = [".devin.ai", ".cognition.ai"];
|
||||
const ALLOWED_DEVIN_EXACT = ["server.codeium.com", "unleash.codeium.com"];
|
||||
|
||||
function normalizedHostname(value) {
|
||||
return String(value || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/\.$/, "");
|
||||
}
|
||||
|
||||
export function isAllowedDevinAuditHostname(hostname) {
|
||||
const value = normalizedHostname(hostname);
|
||||
return (
|
||||
ALLOWED_DEVIN_EXACT.includes(value) ||
|
||||
ALLOWED_DEVIN_SUFFIXES.some((suffix) => value === suffix.slice(1) || value.endsWith(suffix))
|
||||
);
|
||||
}
|
||||
|
||||
export function validateDevinAuthStatus(exitStatus, output) {
|
||||
if (Number(exitStatus) !== 0) return { ok: false, error: "auth status command failed" };
|
||||
const lines = String(output)
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim());
|
||||
if (!lines.includes("Logged in (via Devin)")) {
|
||||
return { ok: false, error: "auth status did not confirm login" };
|
||||
}
|
||||
if (lines.some((line) => /failed to fetch from server/i.test(line))) {
|
||||
return { ok: false, error: "auth status could not confirm server access" };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function parseAuditEntries(text) {
|
||||
const lines = String(text)
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.trim().length > 0);
|
||||
return lines.map((line) => JSON.parse(line));
|
||||
}
|
||||
|
||||
export function validateZeroClaudeEgress(text) {
|
||||
if (String(text).length !== 0) {
|
||||
return { ok: false, error: "Claude attempted external egress during the real run" };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function validateClaudeGuardDenials(text) {
|
||||
const entries = parseAuditEntries(text);
|
||||
if (!entries.length) return { ok: false, error: "Claude egress audit has no records" };
|
||||
if (entries.some((entry) => entry.decision !== "deny")) {
|
||||
return { ok: false, error: "Claude egress audit contains a non-deny decision" };
|
||||
}
|
||||
for (const hostname of ["api.anthropic.com", "claude.ai"]) {
|
||||
if (!entries.some((entry) => entry.hostname === hostname && entry.decision === "deny")) {
|
||||
return { ok: false, error: `Claude egress audit is missing deny for ${hostname}` };
|
||||
}
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function validateDevinGuardAudit(text) {
|
||||
const entries = parseAuditEntries(text);
|
||||
if (!entries.length) return { ok: false, error: "Devin egress audit has no records" };
|
||||
for (const entry of entries) {
|
||||
if (entry.decision !== "allow" || !isAllowedDevinAuditHostname(entry.hostname)) {
|
||||
return { ok: false, error: `unexpected Devin egress record: ${String(entry.hostname)}` };
|
||||
}
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export function validateAuditFileStat(stat, expectedUid) {
|
||||
if (!stat || !stat.isFile() || stat.isSymbolicLink()) return "audit path is not a regular file";
|
||||
if (stat.nlink !== 1) return "audit file link count is not one";
|
||||
if (stat.uid !== Number(expectedUid)) return "audit file owner mismatch";
|
||||
if ((stat.mode & 0o777) !== 0o666) return "audit file mode mismatch";
|
||||
return null;
|
||||
}
|
||||
|
||||
export function readValidatedAuditFile(path, expectedUid) {
|
||||
const stat = fs.lstatSync(path);
|
||||
const statError = validateAuditFileStat(stat, expectedUid);
|
||||
if (statError) throw new Error(statError);
|
||||
return fs.readFileSync(path, "utf8");
|
||||
}
|
||||
|
||||
export function validateAuditFile(kind, path, expectedUid) {
|
||||
const text = readValidatedAuditFile(path, expectedUid);
|
||||
const result =
|
||||
kind === "claude-zero"
|
||||
? validateZeroClaudeEgress(text)
|
||||
: kind === "claude-denials"
|
||||
? validateClaudeGuardDenials(text)
|
||||
: kind === "devin-allowed"
|
||||
? validateDevinGuardAudit(text)
|
||||
: { ok: false, error: `unknown audit validation kind: ${kind}` };
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
return text;
|
||||
}
|
||||
@@ -1,19 +1,15 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { DEVIN_MODEL_CATALOG } from "../../open-sse/config/providers/registry/devin/catalog.ts";
|
||||
|
||||
const document = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
const candidates = [];
|
||||
const candidateFields = new Set([
|
||||
"id",
|
||||
"model",
|
||||
"model_id",
|
||||
"modelid",
|
||||
"modelId",
|
||||
"model_uid",
|
||||
"modeluid",
|
||||
"modelUid",
|
||||
"family_uid",
|
||||
"familyuid",
|
||||
"slug",
|
||||
"familyUid",
|
||||
]);
|
||||
|
||||
function normalizeModelId(value) {
|
||||
@@ -24,56 +20,82 @@ function normalizeModelId(value) {
|
||||
.replace(/^-+|-+$/g, "");
|
||||
}
|
||||
|
||||
function collect(value) {
|
||||
function collect(value, candidates) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(collect);
|
||||
value.forEach((item) => collect(item, candidates));
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== "object") return;
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
if (
|
||||
typeof nested === "string" &&
|
||||
candidateFields.has(key.toLowerCase()) &&
|
||||
candidateFields.has(key) &&
|
||||
/^[a-z0-9][a-z0-9._/-]*$/i.test(nested)
|
||||
) {
|
||||
candidates.push(nested);
|
||||
}
|
||||
collect(nested);
|
||||
collect(nested, candidates);
|
||||
}
|
||||
}
|
||||
|
||||
collect(document);
|
||||
const unique = [...new Set(candidates)];
|
||||
const catalogIds = new Set(DEVIN_MODEL_CATALOG.map((entry) => entry.id));
|
||||
const available = [
|
||||
...new Set(
|
||||
unique
|
||||
.map((candidate) => normalizeModelId(candidate))
|
||||
.filter((candidate) => catalogIds.has(candidate))
|
||||
),
|
||||
];
|
||||
|
||||
for (const [name, configured] of [
|
||||
["DEVIN_BRIDGE_SONNET_MODEL", process.env.DEVIN_BRIDGE_SONNET_MODEL],
|
||||
["DEVIN_BRIDGE_OPUS_MODEL", process.env.DEVIN_BRIDGE_OPUS_MODEL],
|
||||
["DEVIN_BRIDGE_HAIKU_MODEL", process.env.DEVIN_BRIDGE_HAIKU_MODEL],
|
||||
["DEVIN_BRIDGE_SUBAGENT_MODEL", process.env.DEVIN_BRIDGE_SUBAGENT_MODEL],
|
||||
]) {
|
||||
if (!configured) continue;
|
||||
const prefix = "devin-cli-agentic/";
|
||||
const modelId = configured.startsWith(prefix) ? configured.slice(prefix.length) : "";
|
||||
if (!modelId || !available.includes(modelId)) {
|
||||
throw new Error(`${name} is not a model returned by Devin and present in OmniRoute`);
|
||||
export function selectLiveModel(
|
||||
document,
|
||||
environment = process.env,
|
||||
catalog = DEVIN_MODEL_CATALOG
|
||||
) {
|
||||
const candidates = [];
|
||||
collect(document, candidates);
|
||||
const unique = [...new Set(candidates)];
|
||||
const normalizedCatalog = new Map();
|
||||
for (const entry of catalog) {
|
||||
const normalized = normalizeModelId(entry.id);
|
||||
const existing = normalizedCatalog.get(normalized) || [];
|
||||
existing.push(entry.id);
|
||||
normalizedCatalog.set(normalized, existing);
|
||||
}
|
||||
for (const [normalized, ids] of normalizedCatalog) {
|
||||
if (ids.length > 1) {
|
||||
throw new Error(
|
||||
`Ambiguous OmniRoute catalog normalization for ${normalized}: ${ids.join(", ")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
const catalogIds = new Set(catalog.map((entry) => entry.id));
|
||||
const available = [
|
||||
...new Set(
|
||||
unique
|
||||
.map((candidate) => normalizeModelId(candidate))
|
||||
.filter((candidate) => normalizedCatalog.has(candidate))
|
||||
),
|
||||
];
|
||||
|
||||
for (const [name, configured] of [
|
||||
["DEVIN_BRIDGE_SONNET_MODEL", environment.DEVIN_BRIDGE_SONNET_MODEL],
|
||||
["DEVIN_BRIDGE_OPUS_MODEL", environment.DEVIN_BRIDGE_OPUS_MODEL],
|
||||
["DEVIN_BRIDGE_HAIKU_MODEL", environment.DEVIN_BRIDGE_HAIKU_MODEL],
|
||||
["DEVIN_BRIDGE_SUBAGENT_MODEL", environment.DEVIN_BRIDGE_SUBAGENT_MODEL],
|
||||
]) {
|
||||
if (!configured) continue;
|
||||
const prefix = "devin-cli-agentic/";
|
||||
const modelId = configured.startsWith(prefix) ? configured.slice(prefix.length) : "";
|
||||
if (!modelId || !catalogIds.has(modelId) || !available.includes(modelId)) {
|
||||
throw new Error(`${name} is not a model returned by Devin and present in OmniRoute`);
|
||||
}
|
||||
}
|
||||
|
||||
const selected =
|
||||
available.find((candidate) => candidate === "swe-1-7-lightning") ||
|
||||
available.find((candidate) => candidate === "swe-1-7") ||
|
||||
available.find((candidate) => /swe|claude|gpt|gemini/i.test(candidate)) ||
|
||||
available[0];
|
||||
|
||||
if (!selected) {
|
||||
throw new Error("Devin returned no model identifier present in OmniRoute's Devin catalog");
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
const selected =
|
||||
available.find((candidate) => candidate === "swe-1-7-lightning") ||
|
||||
available.find((candidate) => candidate === "swe-1-7") ||
|
||||
available.find((candidate) => /swe|claude|gpt|gemini/i.test(candidate)) ||
|
||||
available[0];
|
||||
|
||||
if (!selected) {
|
||||
throw new Error("Devin returned no model identifier present in OmniRoute's Devin catalog");
|
||||
if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
|
||||
const document = JSON.parse(fs.readFileSync(0, "utf8"));
|
||||
process.stdout.write(selectLiveModel(document));
|
||||
}
|
||||
process.stdout.write(selected);
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
source "$(dirname "$0")/common"
|
||||
trap bridge_cleanup_compose EXIT
|
||||
bridge_cleanup_compose
|
||||
bridge_reset_e2e_fixture
|
||||
"$(dirname "$0")/verify-anthropic-isolation" --static
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline down --remove-orphans
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline up --abort-on-container-exit \
|
||||
--exit-code-from claude claude
|
||||
grep -q '"action":"final"' "$BRIDGE_SANDBOX/evidence/mock-acp.jsonl"
|
||||
grep -q 'BRIDGE_E2E_COMPLETE' "$BRIDGE_SANDBOX/evidence/claude-stream.jsonl"
|
||||
bridge_assert_zero_claude_egress "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl"
|
||||
bridge_cleanup_compose
|
||||
bridge_assert_zero_claude_egress "$BRIDGE_CLAUDE_AUDIT"
|
||||
bridge_export_guard_audit "$BRIDGE_CLAUDE_AUDIT" claude-egress.jsonl
|
||||
trap - EXIT
|
||||
printf 'PASS: real Claude Code completed the offline agentic fixture\n'
|
||||
|
||||
@@ -2,10 +2,11 @@
|
||||
set -euo pipefail
|
||||
source "$(dirname "$0")/common"
|
||||
[[ "${ENABLE_LIVE_DEVIN_TESTS:-}" == 1 ]] || { echo 'Set ENABLE_LIVE_DEVIN_TESTS=1' >&2; exit 1; }
|
||||
trap bridge_cleanup_compose EXIT
|
||||
bridge_cleanup_compose
|
||||
bridge_reset_live_fixture
|
||||
"$(dirname "$0")/verify-anthropic-isolation" --static
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline --profile live-devin down --remove-orphans
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d network-guard
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up -d --wait network-guard
|
||||
bridge_check_devin_auth
|
||||
bridge_run_devin models list --format json >"$BRIDGE_SANDBOX/evidence/live-models.json"
|
||||
devin_model="$(node --import tsx/esm "$BRIDGE_ROOT/scripts/devin-bridge/select-live-model.mjs" \
|
||||
@@ -16,18 +17,10 @@ export DEVIN_BRIDGE_OPUS_MODEL="$DEVIN_BRIDGE_MODEL"
|
||||
export DEVIN_BRIDGE_HAIKU_MODEL="$DEVIN_BRIDGE_MODEL"
|
||||
export DEVIN_BRIDGE_SUBAGENT_MODEL="$DEVIN_BRIDGE_MODEL"
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile live-devin up --abort-on-container-exit --exit-code-from claude-live claude-live
|
||||
node -e '
|
||||
const fs = require("node:fs");
|
||||
const path = process.argv[1];
|
||||
if (!fs.existsSync(path)) process.exit(0);
|
||||
for (const line of fs.readFileSync(path, "utf8").trim().split("\n").filter(Boolean)) {
|
||||
const entry = JSON.parse(line);
|
||||
const suffixAllowed = /(^|\.)(devin\.ai|cognition\.ai)$/.test(entry.hostname);
|
||||
const exactAllowed = ["server.codeium.com", "unleash.codeium.com"].includes(entry.hostname);
|
||||
if (entry.decision === "allow" && !suffixAllowed && !exactAllowed) {
|
||||
throw new Error(`unexpected allowed egress: ${entry.hostname}`);
|
||||
}
|
||||
}
|
||||
' "$BRIDGE_SANDBOX/evidence/egress.jsonl"
|
||||
bridge_assert_zero_claude_egress "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl"
|
||||
bridge_cleanup_compose
|
||||
bridge_assert_devin_guard_audit "$BRIDGE_DEVIN_AUDIT"
|
||||
bridge_assert_zero_claude_egress "$BRIDGE_CLAUDE_AUDIT"
|
||||
bridge_export_guard_audit "$BRIDGE_DEVIN_AUDIT" egress.jsonl
|
||||
bridge_export_guard_audit "$BRIDGE_CLAUDE_AUDIT" claude-egress.jsonl
|
||||
trap - EXIT
|
||||
printf 'PASS: live model %s was discovered and validated by three scenarios\n' "$devin_model"
|
||||
|
||||
@@ -4,4 +4,5 @@ source "$(dirname "$0")/common"
|
||||
cd "$BRIDGE_ROOT"
|
||||
bridge_test_env node --import tsx/esm --test \
|
||||
tests/unit/executor-devin-cli-agentic-core.test.ts \
|
||||
tests/unit/executor-devin-cli-agentic-acp.test.ts
|
||||
tests/unit/executor-devin-cli-agentic-acp.test.ts \
|
||||
tests/unit/devin-bridge-network-guard.test.ts
|
||||
|
||||
@@ -35,10 +35,16 @@ node -e '
|
||||
throw new Error("Claude egress guard is not deny-all");
|
||||
}
|
||||
for (const guardName of ["network-guard", "claude-egress-guard"]) {
|
||||
const guard = config.services[guardName];
|
||||
const env = config.services[guardName].environment;
|
||||
if (env.GUARD_ALLOW_SUFFIXES || env.GUARD_ALLOW_HOSTS) {
|
||||
throw new Error(`${guardName} exposes mutable host allowlists`);
|
||||
}
|
||||
if (!guard.healthcheck?.test) throw new Error(`${guardName} has no healthcheck`);
|
||||
const auditMount = (guard.volumes || []).find((mount) => mount.target === "/guard-audit");
|
||||
if (!auditMount || auditMount.type !== "bind" || !auditMount.source.includes("/.sandbox/guard-audit/")) {
|
||||
throw new Error(`${guardName} does not use its guard-only audit bind`);
|
||||
}
|
||||
}
|
||||
const runtimeNames = ["omniroute", "claude", "contract", "omniroute-live", "claude-live"];
|
||||
for (const serviceName of [...runtimeNames, "network-guard", "claude-egress-guard"]) {
|
||||
@@ -49,6 +55,9 @@ node -e '
|
||||
}
|
||||
for (const serviceName of runtimeNames) {
|
||||
const service = config.services[serviceName];
|
||||
if ((service.volumes || []).some((mount) => mount.target === "/guard-audit")) {
|
||||
throw new Error(`${serviceName} can mutate guard audit evidence`);
|
||||
}
|
||||
const namedVolumes = (service.volumes || []).filter((mount) => mount.type === "volume");
|
||||
const hasClaudeConfig = namedVolumes.some(
|
||||
(mount) => mount.target === "/home/bridge/.claude-devin-isolated",
|
||||
@@ -76,6 +85,14 @@ node -e '
|
||||
}
|
||||
}
|
||||
}
|
||||
if (config.services["omniroute-live"].depends_on["network-guard"].condition !== "service_healthy") {
|
||||
throw new Error("omniroute-live does not wait for a healthy Devin guard");
|
||||
}
|
||||
for (const serviceName of ["claude", "claude-live"]) {
|
||||
if (config.services[serviceName].depends_on["claude-egress-guard"].condition !== "service_healthy") {
|
||||
throw new Error(`${serviceName} does not wait for a healthy Claude guard`);
|
||||
}
|
||||
}
|
||||
const liveEnv = config.services["omniroute-live"].environment;
|
||||
if (liveEnv.DEVIN_BRIDGE_PROXY_URL !== "http://network-guard:8080") {
|
||||
throw new Error("trusted Devin bridge proxy is missing");
|
||||
@@ -167,6 +184,9 @@ fi
|
||||
if bridge_assert_devin_auth_status 0 $'Logged out\n' 2>/dev/null; then
|
||||
fail "logged-out auth fixture was accepted"
|
||||
fi
|
||||
if bridge_assert_devin_auth_status 0 $'Not Logged in (via Devin)\n' 2>/dev/null; then
|
||||
fail "misleading auth fixture was accepted"
|
||||
fi
|
||||
selected_model="$(printf '%s' '{"models":[{"family_uid":"swe-1.7"},{"modelUid":"swe-1.7-lightning"}]}' | \
|
||||
node --import tsx/esm "$BRIDGE_ROOT/scripts/devin-bridge/select-live-model.mjs")"
|
||||
[[ "$selected_model" == swe-1-7-lightning ]] || fail "live model normalization or preference failed"
|
||||
@@ -184,15 +204,14 @@ grep -q 'bridge_check_devin_auth' "$BRIDGE_ROOT/scripts/devin-bridge/test-live-d
|
||||
fail "live test bypasses strict auth status"
|
||||
grep -q 'bridge_check_devin_auth' "$BRIDGE_ROOT/scripts/devin-bridge/launch" || \
|
||||
fail "normal launch bypasses strict auth status"
|
||||
grep -q 'up -d network-guard claude-egress-guard' "$BRIDGE_ROOT/scripts/devin-bridge/launch" || \
|
||||
grep -q 'up -d --wait network-guard claude-egress-guard' "$BRIDGE_ROOT/scripts/devin-bridge/launch" || \
|
||||
fail "normal launch does not start the audited Claude egress guard"
|
||||
grep -qx '\.sandbox' "$BRIDGE_ROOT/.dockerignore" || fail ".sandbox is not excluded from builds"
|
||||
if [[ "${1:-}" == --static ]]; then printf 'PASS: static bridge isolation checks passed\n'; exit 0; fi
|
||||
trap bridge_cleanup_compose EXIT
|
||||
bridge_cleanup_compose
|
||||
bridge_reset_claude_egress_audit
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline up -d claude-egress-guard
|
||||
cleanup_claude_guard() {
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline stop claude-egress-guard >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_claude_guard EXIT
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline up -d --wait claude-egress-guard
|
||||
docker compose -f "$BRIDGE_COMPOSE" --profile offline run --rm --no-deps claude bash -ceu '
|
||||
test "$(id -u)" = 10001
|
||||
test "$HOME" = /home/bridge
|
||||
@@ -231,9 +250,10 @@ docker compose -f "$BRIDGE_COMPOSE" --profile offline run --rm --no-deps claude
|
||||
signal: AbortSignal.timeout(3000),
|
||||
}));
|
||||
'
|
||||
bridge_assert_claude_guard_denials "$BRIDGE_SANDBOX/evidence/claude-egress.jsonl" || \
|
||||
bridge_cleanup_compose
|
||||
bridge_assert_claude_guard_denials "$BRIDGE_CLAUDE_AUDIT" || \
|
||||
fail "Claude proxy denial audit proof failed"
|
||||
cleanup_claude_guard
|
||||
bridge_export_guard_audit "$BRIDGE_CLAUDE_AUDIT" claude-egress-verifier.jsonl
|
||||
trap - EXIT
|
||||
bridge_reset_claude_egress_audit
|
||||
printf 'PASS: runtime bridge isolation checks passed\n'
|
||||
|
||||
Reference in New Issue
Block a user