Uptime Kuma vs Gatus vs Beszel on 10 checks

#!/usr/bin/env bash
# monitors: Uptime Kuma 2.5.9, Gatus 5.38.0 and Beszel 0.21.1 checking the same 10 local nginx targets every 60 s.
# Measures RAM (baseline vs 10 checks), disk growth over 50 min, and how fast each reports a killed target.
set -uo pipefail
source "$(dirname "$0")/../_dlib.sh"
T=monitors; OUT="$ROOT/evidence/tests/$T"; RAW="$OUT/raw"
# DETECT_ONLY=1 re-runs only the detection part in a fresh VM (5 min of checks first) and keeps everything else in evidence/tests/monitors.
if [ -z "${DETECT_ONLY:-}" ]; then rm -rf "$OUT"; fi
mkdir -p "$RAW"
DUR_MIN=${DUR_MIN:-50}
start_vm d-$T
D="$ROOT/lab/tests/$T"
gx 'mkdir -p ~/m'
for f in gen-compose.py kuma-setup.js beszel-setup.py sample.py detect.py; do vm scp-to "$D/$f" "m/$f" >/dev/null; done
dpull louislam/uptime-kuma:2.5.9 twinproduction/gatus:v5.38.0 henrygd/beszel:0.21.1 henrygd/beszel-agent:0.21.1 nginx:1.29-alpine
gx 'cd ~/m; python3 gen-compose.py 0 && docker compose up -d 2>&1 | tail -1'
sleep 20
# Baseline: apps configured but with no checks (Kuma: admin, 0 monitors; Beszel hub: one user, no system; Gatus: one self-check)
gx 'cd ~/m; docker cp kuma-setup.js mon-uptime-kuma-1:/app/kuma-setup.js; docker exec mon-uptime-kuma-1 node /app/kuma-setup.js 0; python3 beszel-setup.py hub' > "$RAW/setup-baseline${DETECT_ONLY:+-detect}.txt" 2>&1
if [ -z "${DETECT_ONLY:-}" ]; then sleep 120
gx 'cd ~/m; python3 sample.py mon-uptime-kuma-1 mon-gatus-1 mon-beszel-1' > "$RAW/ram-baseline.json"; fi
# Phase B: 10 checks at 60 s on every tool
gx 'cd ~/m; python3 gen-compose.py 10 && docker compose up -d --force-recreate gatus 2>&1 | tail -2; docker compose up -d 2>&1 | tail -1; cp compose.yml ~/m/compose.used.yml
 docker exec mon-uptime-kuma-1 node /app/kuma-setup.js 10
 KEY=$(python3 - <<P
import json,urllib.request as u
H="http://127.0.0.1:8083"
r=lambda p,b=None,t=None: json.load(u.urlopen(u.Request(H+p,data=json.dumps(b).encode() if b else None,headers={"content-type":"application/json",**({"Authorization":t} if t else {})})))
t=r("/api/collections/_superusers/auth-with-password",{"identity":"admin@lab.internal","password":"lab-only-not-a-secret"})["token"]
print(r("/api/beszel/getkey",t=t)["key"])
P
)
 docker run -d --name beszel-agent --network mon_default --restart unless-stopped -e KEY="$KEY" -e PORT=45876 -v /var/run/docker.sock:/var/run/docker.sock:ro henrygd/beszel-agent:0.21.1 >/dev/null
 python3 beszel-setup.py full' > "$RAW/setup-checks${DETECT_ONLY:+-detect}.txt" 2>&1
[ -z "${DETECT_ONLY:-}" ] && gx 'cat ~/m/compose.used.yml' > "$OUT/compose.used.yml"
sleep 5; N=$(gx 'curl -s localhost:8082/api/v1/endpoints/statuses | python3 -c "import json,sys;print(len(json.load(sys.stdin)))"'); echo "gatus endpoints loaded: $N" > "$RAW/gatus-endpoints${DETECT_ONLY:+-detect}.txt"; [ "$N" = 10 ] || { echo "gatus did not load 10 endpoints" >&2; exit 1; }
if [ -n "${DETECT_ONLY:-}" ]; then
  sleep 330
  for n in 3 6 9; do
    gx "cd ~/m; python3 detect.py $n" > "$RAW/detect-$n.json" 2> "$RAW/detect-$n.err"
    sleep $(( n * 7 + 23 ))
  done
  mkdir -p "$RAW/detect-env"; collect_env "$RAW/detect-env"
  python3 "$D/summarize.py" "$RAW" "$OUT"; exit 0
fi
T0=$(date +%s); echo "T0=$T0" > "$RAW/t0.txt"
disk() { gx "for v in kuma_data gatus_data beszel_data; do echo \"$(date +%s) \$v \$(sudo du -sb /var/lib/docker/volumes/mon_\$v/_data | cut -f1)\"; done" >> "$RAW/disk.tsv"; }
disk
sleep 180
gx 'cd ~/m; python3 sample.py mon-uptime-kuma-1 mon-gatus-1 mon-beszel-1 beszel-agent' > "$RAW/ram-early.json"
shot_all=0
while [ $(( $(date +%s) - T0 )) -lt $(( DUR_MIN * 60 )) ]; do
  sleep $(( 300 - ( $(date +%s) - T0 ) % 300 )); disk
done
gx 'cd ~/m; python3 sample.py mon-uptime-kuma-1 mon-gatus-1 mon-beszel-1 beszel-agent' > "$RAW/ram-late.json"
gx 'docker ps --format "{{.Names}} {{.Status}}"; docker inspect -f "{{.Name}} restarts={{.RestartCount}}" $(docker ps -aq)' > "$RAW/containers-end.txt"
# Detection: kill target 3, 6, 9 at different phases of the 60 s cycle
for n in 3 6 9; do
  gx "cd ~/m; python3 detect.py $n" > "$RAW/detect-$n.json" 2> "$RAW/detect-$n.err"
  sleep $(( n * 7 + 23 ))
done
# Screenshots
S="$ROOT/public/shots/$T"
shot 8081 / "$S/uptime-kuma.png" '[{"fill":"input[name=username],#floatingInput","value":"admin"},{"fill":"input[type=password]","value":"lab-only-not-a-secret"},{"press":"Enter"},{"wait":6000}]'
shot 8082 / "$S/gatus.png"
shot 8083 / "$S/beszel.png" '[{"fill":"input[placeholder=\"name@example.com\"]","value":"lab@lab.internal"},{"fill":"input[type=password]","value":"lab-only-not-a-secret"},{"press":"Enter"},{"wait":6000}]'
gx 'docker exec mon-gatus-1 /gatus --help 2>&1 | head -2; docker images --format "{{.Repository}}:{{.Tag}}" | grep -E "kuma|gatus|beszel|nginx"' > "$RAW/versions.txt" 2>&1
collect_env "$OUT"
python3 "$D/summarize.py" "$RAW" "$OUT"

// Runs inside the uptime-kuma container (socket.io-client is bundled in the image).
// Creates the admin account and 10 HTTP monitors at a 60 s interval.
const { io } = require("socket.io-client");
const s = io("http://127.0.0.1:3001");
const call = (ev, ...a) => new Promise((res) => s.emit(ev, ...a, res));
s.on("connect", async () => {
  console.log("setup", JSON.stringify(await call("setup", "admin", "lab-only-not-a-secret")));
  const l = await call("login", { username: "admin", password: "lab-only-not-a-secret", token: "" });
  console.log("login", l.ok);
  for (let i = 1; i <= (parseInt(process.argv[2] || "10")); i++) {
    const r = await call("add", {
      type: "http", name: `target-${i}`, url: `http://t${i}/`, method: "GET", interval: 60, retryInterval: 60,
      resendInterval: 0, maxretries: 0, timeout: 48, notificationIDList: {}, ignoreTls: false, upsideDown: false,
      maxredirects: 10, accepted_statuscodes: ["200-299"], dns_resolve_type: "A", dns_resolve_server: "1.1.1.1",
      expiryNotification: false, conditions: [], packetSize: 56, kafkaProducerBrokers: [], kafkaProducerSaslOptions: { mechanism: "None" },
      rabbitmqNodes: [], authMethod: null, httpBodyEncoding: "json", active: true,
    });
    console.log(i, JSON.stringify(r));
  }
  process.exit(0);
});
setTimeout(() => { console.log("timeout"); process.exit(1); }, 60000);

#!/usr/bin/env python3
"""Runs in the VM. Creates a Beszel user, registers the agent as a system and adds 10 HTTP network monitors (60 s)."""
import json, subprocess, time, urllib.parse, urllib.request
H = "http://127.0.0.1:8083"
def api(path, body=None, tok=None, method=None):
    req = urllib.request.Request(H + path, data=json.dumps(body).encode() if body is not None else None, method=method or ("POST" if body is not None else "GET"),
                                 headers={"content-type": "application/json", **({"Authorization": tok} if tok else {})})
    try:
        return json.load(urllib.request.urlopen(req, timeout=20))
    except urllib.error.HTTPError as e:
        return {"error": e.code, "body": e.read().decode()[:400]}
subprocess.run(["docker", "exec", "mon-beszel-1", "/beszel", "superuser", "upsert", "admin@lab.internal", "lab-only-not-a-secret"], check=True, capture_output=True)
tok = api("/api/collections/_superusers/auth-with-password", {"identity": "admin@lab.internal", "password": "lab-only-not-a-secret"})["token"]
u = api("/api/collections/users/records", {"email": "lab@lab.internal", "password": "lab-only-not-a-secret", "passwordConfirm": "lab-only-not-a-secret", "role": "admin", "verified": True}, tok)
if "id" not in u:  # already created in the baseline phase
    u = api("/api/collections/users/records?filter=" + urllib.parse.quote('email="lab@lab.internal"'), tok=tok)["items"][0]
print("user", u.get("id") or u)
import sys
if len(sys.argv) > 1 and sys.argv[1] == "hub":
    sys.exit(0)  # baseline: hub with a user, no system, no monitors
sysr = api("/api/collections/systems/records", {"name": "labbox", "host": "beszel-agent", "port": "45876", "users": [u["id"]]}, tok)
print("system", sysr.get("id") or sysr)
for i in range(1, 11):
    r = api("/api/collections/network_monitors/records", {"system": sysr["id"], "target": f"http://t{i}", "protocol": "http", "port": 80, "interval": 60, "enabled": True}, tok)
    print("monitor", i, r.get("id") or r)
time.sleep(5)
print(json.dumps(api("/api/collections/systems/records/" + sysr["id"], tok=tok))[:300])

#!/usr/bin/env python3
"""Generates compose.yml: 10 nginx targets plus Uptime Kuma, Gatus and the Beszel hub (versions as pinned in lab/stacks).
Usage: gen-compose.py [gatus_endpoints]. 0 (default) = the single self-check of lab/stacks/gatus (baseline); 10 = the ten targets."""
import sys
N = int(sys.argv[1]) if len(sys.argv) > 1 else 0
targets = "\n".join(f"  t{i}:\n    image: nginx:1.29-alpine\n    restart: unless-stopped" for i in range(1, 11))
gat = "        - name: self\n          url: http://127.0.0.1:8080/health\n          interval: 60s\n          conditions:\n            - \"[STATUS] == 200\"" if N == 0 else "\n".join(f"        - name: target-{i}\n          url: http://t{i}/\n          interval: 60s\n          conditions:\n            - \"[STATUS] == 200\"" for i in range(1, 11))
open("compose.yml", "w").write(f"""name: mon
services:
{targets}
  uptime-kuma:
    image: louislam/uptime-kuma:2.5.9
    restart: unless-stopped
    ports: ["127.0.0.1:8081:3001"]
    environment: {{UPTIME_KUMA_DB_TYPE: sqlite}}
    volumes: [kuma_data:/app/data]
  gatus:
    image: twinproduction/gatus:v5.38.0
    restart: unless-stopped
    ports: ["127.0.0.1:8082:8080"]
    volumes: [gatus_data:/data]
    configs:
      - source: gatus
        target: /config/config.yaml
  beszel:
    image: henrygd/beszel:0.21.1
    restart: unless-stopped
    ports: ["127.0.0.1:8083:8090"]
    volumes: [beszel_data:/beszel_data]
volumes:
  kuma_data:
  gatus_data:
  beszel_data:
configs:
  gatus:
    content: |
      storage:
        type: sqlite
        path: /data/gatus.db
      endpoints:
{gat}
""")

#!/usr/bin/env python3
"""Runs in the VM: kills target container t<N>, then polls the three tools once a second until each reports it down (max 200 s).
Usage: detect.py <N>. Prints JSON with seconds from the kill to the first poll that shows the target as failed."""
import base64, json, subprocess, sys, time, urllib.request
n = sys.argv[1]
def get(url, headers=None):
    return urllib.request.urlopen(urllib.request.Request(url, headers=headers or {}), timeout=10).read().decode()
def post(url, body):
    return json.load(urllib.request.urlopen(urllib.request.Request(url, data=json.dumps(body).encode(), headers={"content-type": "application/json"}), timeout=10))
kauth = {"Authorization": "Basic " + base64.b64encode(b"admin:lab-only-not-a-secret").decode()}
tok = post("http://127.0.0.1:8083/api/collections/_superusers/auth-with-password", {"identity": "admin@lab.internal", "password": "lab-only-not-a-secret"})["token"]
def kuma():
    for l in get("http://127.0.0.1:8081/metrics", kauth).splitlines():
        if l.startswith("monitor_status") and f'monitor_name="target-{n}"' in l:
            return l.rsplit(" ", 1)[1]
def gatus():
    for e in json.loads(get("http://127.0.0.1:8082/api/v1/endpoints/statuses")):
        if e["name"] == f"target-{n}":
            r = e["results"][-1]
            return "ok" if r["success"] else "FAIL", r["timestamp"]
def beszel():
    q = urllib.parse.quote(f'target="http://t{n}"')
    it = json.loads(get(f"http://127.0.0.1:8083/api/collections/network_monitors/records?filter={q}", {"Authorization": tok}))["items"][0]
    return {"loss1h": it["loss1h"], "res": it["res"], "updated": it["updated"]}
import urllib.parse
def gatus_dbg():
    g = gatus()
    if g is None:
        print("gatus: target-%s not found in statuses: %s" % (n, get("http://127.0.0.1:8082/api/v1/endpoints/statuses")[:400]), file=sys.stderr); g = ("MISSING", "")
    return g
before = {"kuma": kuma(), "gatus": gatus_dbg(), "beszel": beszel()}
t0 = time.time()
subprocess.run(["docker", "kill", f"mon-t{n}-1"], capture_output=True, check=True)
det = {}; log = []
while time.time() - t0 < 200 and len(det) < 3:
    t = round(time.time() - t0, 1)
    k, g, b = kuma(), gatus_dbg(), beszel()
    log.append({"t": t, "kuma": k, "gatus": g, "beszel": b})
    if "kuma" not in det and k == "0": det["kuma"] = t
    if "gatus" not in det and g[0] == "FAIL": det["gatus"] = t
    if "beszel" not in det and b["loss1h"] > 0: det["beszel"] = t
    time.sleep(1)
subprocess.run(["docker", "start", f"mon-t{n}-1"], capture_output=True)
# keep only the lines around changes to stay small
compact = [x for i, x in enumerate(log) if i == 0 or (x["kuma"], x["gatus"][0], x["beszel"]["loss1h"]) != (log[i-1]["kuma"], log[i-1]["gatus"][0], log[i-1]["beszel"]["loss1h"])]
print(json.dumps({"target": n, "before": before, "detected_seconds_after_kill": det, "changes": compact}, indent=1))