Caddy vs Traefik vs NGINX vs NPM: reverse proxy measured

# Backend + one proxy (selected by profile) on one network. The proxy answers to the alias bench.lab.
# PROXY_CONF is the config file for the nginx/caddy/traefik variants (set by bench.sh).
services:
  backend:
    image: nginx:1.31.8-alpine
    volumes:
      - ./backend.conf:/etc/nginx/conf.d/default.conf:ro
      - ./index.html:/usr/share/nginx/html/index.html:ro

  caddy:
    profiles: [caddy]
    image: caddy:2.11.7
    volumes:
      - ./${PROXY_CONF:-x}:/etc/caddy/Caddyfile:ro
      - ./certs:/certs:ro
    networks: { default: { aliases: [bench.lab] } }

  nginx:
    profiles: [nginx]
    image: nginx:1.31.8-alpine
    volumes:
      - ./${PROXY_CONF:-x}:/etc/nginx/conf.d/default.conf:ro
      - ./certs:/certs:ro
    networks: { default: { aliases: [bench.lab] } }

  traefik:
    profiles: [traefik]
    image: traefik:v3.7.17
    command:
      - --providers.file.filename=/dynamic.yml
      - --entrypoints.web.address=:80
      - --entrypoints.websecure.address=:443
    volumes:
      - ./${PROXY_CONF:-x}:/dynamic.yml:ro
      - ./certs:/certs:ro
    networks: { default: { aliases: [bench.lab] } }

  npm:
    profiles: [npm]
    image: jc21/nginx-proxy-manager:2.16.0
    ports: ["127.0.0.1:8081:81"]
    volumes:
      - npm_data:/data
      - npm_certs:/etc/letsencrypt
    networks: { default: { aliases: [bench.lab] } }

volumes:
  npm_data:
  npm_certs:

#!/usr/bin/env bash
# Runs inside the VM, from ~/pb. Writes raw outputs to ~/pb/raw. One proxy at a time, backend always up.
set -u
cd "$HOME/pb"
export COMPOSE_PROJECT_NAME=pb
OHA=ghcr.io/hatoo/oha:1.16.0
CONC=64; DUR=15s; WARM=5s; RUNS=3
NET=pb_default
mkdir -p raw certs
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -keyout certs/leaf.key -out certs/leaf.crt \
  -days 2 -subj /CN=bench.lab -addext subjectAltName=DNS:bench.lab >/dev/null 2>&1
chmod 644 certs/*
{ docker image ls --digests --format '{{.Repository}}:{{.Tag}} {{.Digest}}'; } > raw/images.txt 2>&1

oha() { docker run --rm --network "$NET" "$OHA" "$@"; }

wait_ready() { # wait_ready <url> [oha flags]
  local url="$1"; shift
  for _ in $(seq 1 60); do
    if oha -n 1 --no-tui --output-format json "$@" "$url" 2>/dev/null | jq -e '.statusCodeDistribution["200"] == 1' >/dev/null 2>&1; then return 0; fi
    sleep 2
  done
  echo "NOT READY: $url" >&2; return 1
}

sample() { # sample <container> <outfile> <flagfile>: docker stats until flag disappears
  while [ -e "$3" ]; do
    docker stats --no-stream --format '{{.MemUsage}} {{.CPUPerc}}' "$1" >> "$2" 2>/dev/null
  done
}

measure() { # measure <label> <container> <url> [oha flags]
  local label="$1" cid="$2" url="$3"; shift 3
  echo "== $label $(date -u +%T)"
  sleep 8
  for _ in 1 2 3; do docker stats --no-stream --format '{{.MemUsage}} {{.CPUPerc}}' "$cid"; done > "raw/$label.idle.txt"
  oha -c "$CONC" -z "$WARM" --no-tui --output-format json "$@" "$url" > /dev/null 2>&1
  for i in $(seq 1 $RUNS); do
    local flag="raw/.sampling"; touch "$flag"; : > "raw/$label.run$i.stats.txt"
    sample "$cid" "raw/$label.run$i.stats.txt" "$flag" &
    local sp=$!
    oha -c "$CONC" -z "$DUR" --no-tui --output-format json "$@" "$url" > "raw/$label.run$i.json" 2> "raw/$label.run$i.err"
    rm -f "$flag"; wait "$sp"
    if [ "$i" = 1 ]; then
      { echo "-- proxy /proc/net/sockstat"; docker exec "$cid" cat /proc/net/sockstat 2>&1
        echo "-- VM nf_conntrack_count"; cat /proc/sys/net/netfilter/nf_conntrack_count 2>&1; } > "raw/$label.sockets.txt"
    fi
    jq -c '{rps:.summary.requestsPerSec,p50:.latencyPercentiles.p50,p99:.latencyPercentiles.p99,ok:.statusCodeDistribution}' "raw/$label.run$i.json"
  done
}

cid() { docker compose ps -q "$1"; }

# --- direct baseline (backend only)
docker compose up -d backend >/dev/null 2>&1
wait_ready http://backend/
measure direct.http "$(cid backend)" http://backend/

# --- caddy / nginx / traefik: file-configured, one config per scheme
run_file_proxy() { # run_file_proxy <service> <label> <conf-prefix>
  local svc="$1" label="$2" prefix="$3" v
  # throwaway start: the first start of a binary reads it into the page cache, which docker stats counts as memory
  PROXY_CONF="$( [ "$svc" = caddy ] && echo caddy/Caddyfile.http || { [ "$svc" = traefik ] && echo traefik/dynamic.http.yml || echo nginx/$prefix.http.conf; } )" docker compose up -d "$svc" >/dev/null 2>&1
  sleep 5; docker compose rm -sf "$svc" >/dev/null 2>&1
  for v in http tls; do
    case "$svc" in
      caddy) conf="caddy/Caddyfile.$v" ;;
      traefik) conf="traefik/dynamic.$v.yml" ;;
      nginx) conf="nginx/$prefix.$v.conf" ;;
    esac
    PROXY_CONF="$conf" docker compose up -d --force-recreate "$svc" >/dev/null 2>&1
    if [ "$v" = http ]; then url=http://bench.lab/; fl=(); else url=https://bench.lab/; fl=(--insecure); fi
    if wait_ready "$url" "${fl[@]}"; then measure "$label.$v" "$(cid "$svc")" "$url" "${fl[@]}"; fi
    docker compose logs --no-color --tail=15 "$svc" > "raw/$label.$v.logs-tail.txt" 2>&1
    docker compose rm -sf "$svc" >/dev/null 2>&1
  done
}
run_file_proxy caddy caddy x
run_file_proxy traefik traefik x
run_file_proxy nginx nginx-minimal minimal
run_file_proxy nginx nginx-keepalive keepalive
run_file_proxy nginx nginx-keepalive-nolog keepalive-nolog

# --- Nginx Proxy Manager: configured through its API
docker compose up -d npm >/dev/null 2>&1; sleep 40; docker compose rm -sfv npm >/dev/null 2>&1; docker volume rm -f pb_npm_data pb_npm_certs >/dev/null 2>&1   # throwaway start, same reason as above; the backend stays up
docker compose up -d npm >/dev/null 2>&1
for _ in $(seq 1 90); do curl -s -m3 localhost:8081/api/ | jq -e '.status=="OK"' >/dev/null 2>&1 && break; sleep 2; done
api() { curl -s -m20 "$@"; }
api -XPOST localhost:8081/api/users -H 'content-type: application/json' \
  -d '{"name":"Lab","nickname":"lab","email":"lab@example.com","roles":["admin"],"is_disabled":false,"auth":{"type":"password","secret":"lab-only-not-a-secret"}}' > raw/npm.create-user.json
TOKEN=$(api -XPOST localhost:8081/api/tokens -H 'content-type: application/json' -d '{"identity":"lab@example.com","secret":"lab-only-not-a-secret"}' | jq -r .token)
H="Authorization: Bearer $TOKEN"
HOSTBODY='{"domain_names":["bench.lab"],"forward_scheme":"http","forward_host":"backend","forward_port":80,"enabled":true,"meta":{}}'
echo "$HOSTBODY" > raw/npm.host-body.json
api -XPOST localhost:8081/api/nginx/proxy-hosts -H "$H" -H 'content-type: application/json' -d "$HOSTBODY" > raw/npm.create-host.json
HID=$(jq -r .id raw/npm.create-host.json)
sleep 4
wait_ready http://bench.lab/ && measure npm.http "$(cid npm)" http://bench.lab/
CID=$(api -XPOST localhost:8081/api/nginx/certificates -H "$H" -H 'content-type: application/json' -d '{"provider":"other","nice_name":"bench"}' | jq -r .id)
api -XPOST "localhost:8081/api/nginx/certificates/$CID/upload" -H "$H" -F certificate=@certs/leaf.crt -F certificate_key=@certs/leaf.key > raw/npm.cert-upload.json
api -XPUT "localhost:8081/api/nginx/proxy-hosts/$HID" -H "$H" -H 'content-type: application/json' \
  -d "{\"domain_names\":[\"bench.lab\"],\"forward_scheme\":\"http\",\"forward_host\":\"backend\",\"forward_port\":80,\"certificate_id\":$CID}" > raw/npm.update-host.json
sleep 4
wait_ready https://bench.lab/ --insecure && measure npm.tls "$(cid npm)" https://bench.lab/ --insecure
docker compose exec -T npm sh -c 'cat /data/nginx/proxy_host/*.conf; echo ---- proxy.conf; cat /etc/nginx/conf.d/include/proxy.conf' > raw/npm.generated-nginx-host.conf 2>&1
docker compose logs --no-color --tail=15 npm > raw/npm.logs-tail.txt 2>&1
docker compose down -v >/dev/null 2>&1
echo done > raw/DONE

upstream app {
    server backend:80;
    keepalive 64;
}
server {
    listen 80;
    server_name bench.lab;
    location / {
        proxy_pass http://app;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
    }
}