OmniRoute/.github/workflows/nightly-llm-security.yml

name: Nightly LLM Security
on:
  schedule:
    - cron: "53 5 * * *"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  promptfoo-guard:
    name: promptfoo — injection guard (block mode, no secret)
    # #11965: this job runs a backend-only `next build`; the hosted 7 GB runner cannot build
    # release/v3.8.51 (VM shutdown ~7 min in), so it targets the box's light pool (`omni-light`:
    # two listeners, jobs ≤ ~6 GB). Falls back to hosted when USE_VPS_RUNNER is off.
    runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-light"]')) || 'ubuntu-latest' }}
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
      - uses: actions/setup-node@v7
        with: { node-version: "24", cache: npm }
      - run: npm ci
      - name: Build CLI bundle
        env:
          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
          OMNIROUTE_BUILD_BACKEND_ONLY: "1"
        run: npm run build:cli
      - name: Start OmniRoute (block mode)
        env:
          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
          PORT: "20128"
          INJECTION_GUARD_MODE: block
        run: |
          node dist/server.js > server.log 2>&1 &
          echo $! > server.pid
          for i in $(seq 1 30); do
            if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo up; break; fi
            sleep 2
          done
      - name: promptfoo guard-validation
        run: npx --yes promptfoo@latest eval -c promptfooconfig.yaml --no-cache
        env:
          OMNIROUTE_URL: http://localhost:20128
          OMNIROUTE_API_KEY: not-needed-blocked-before-upstream
      - name: Stop server
        if: always()
        run: kill "$(cat server.pid)" || true

  garak:
    name: garak probes (skip without provider secret)
    # #11965: this job runs a backend-only `next build`; the hosted 7 GB runner cannot build
    # release/v3.8.51 (VM shutdown ~7 min in), so it targets the box's light pool (`omni-light`:
    # two listeners, jobs ≤ ~6 GB). Falls back to hosted when USE_VPS_RUNNER is off.
    runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-light"]')) || 'ubuntu-latest' }}
    # NOTE: the `secrets` context is NOT available in a job-level `if:` — referencing
    # it there makes GitHub reject the file on push (startup_failure on every push).
    # Map the secret into a job-level env and gate each step on a presence check, so
    # the job stays green and simply skips the probes when the secret is absent.
    env:
      PROMPTFOO_PROVIDER_KEY: ${{ secrets.PROMPTFOO_PROVIDER_KEY }}
    steps:
      - name: Gate on provider secret
        id: gate
        run: |
          if [ -n "$PROMPTFOO_PROVIDER_KEY" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "run=false" >> "$GITHUB_OUTPUT"
            echo "::notice::PROMPTFOO_PROVIDER_KEY not set — skipping garak probes (advisory)."
          fi
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
        if: steps.gate.outputs.run == 'true'
      - uses: actions/setup-node@v7
        if: steps.gate.outputs.run == 'true'
        with: { node-version: "24", cache: npm }
      - run: npm ci
        if: steps.gate.outputs.run == 'true'
      - name: Build CLI bundle
        if: steps.gate.outputs.run == 'true'
        env:
          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
          OMNIROUTE_BUILD_BACKEND_ONLY: "1"
        run: npm run build:cli
      - name: Start OmniRoute
        if: steps.gate.outputs.run == 'true'
        env:
          JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
          PORT: "20128"
        run: |
          node dist/server.js > server.log 2>&1 &
          echo $! > server.pid
          for i in $(seq 1 30); do
            if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo up; break; fi
            sleep 2
          done
      - uses: actions/setup-python@v7
        if: steps.gate.outputs.run == 'true'
        with: { python-version: "3.12" }
      - run: pip install garak
        if: steps.gate.outputs.run == 'true'
      - name: garak limited probes
        if: steps.gate.outputs.run == 'true'
        env:
          OPENAI_API_KEY: ${{ secrets.PROMPTFOO_PROVIDER_KEY }}
          OPENAI_BASE_URL: http://localhost:20128/v1
        run: garak --model_type openai --model_name gpt-4o-mini --probes promptinject,dan,leakreplay --report_prefix garak-omniroute || true
      - name: Stop server
        if: always() && steps.gate.outputs.run == 'true'
        run: kill "$(cat server.pid)" || true