Prometheus + Grafana vs Netdata: RAM, disk, setup

#!/usr/bin/env bash
# metrics: Prometheus 3.15.1 + Grafana 13.3.0 + node_exporter vs Netdata 2.12.2 on the same VM.
# Clean start to first useful dashboard (steps counted), then 50 min of data growth and RAM with both running.
set -uo pipefail
source "$(dirname "$0")/../_dlib.sh"
T=metrics; OUT="$ROOT/evidence/tests/$T"; RAW="$OUT/raw"
rm -rf "$OUT"; mkdir -p "$RAW"
DUR_MIN=${DUR_MIN:-50}
start_vm d-$T
D="$ROOT/lab/tests/$T"
gx 'mkdir -p ~/nd ~/prom'
vm scp-to "$ROOT/lab/stacks/netdata/compose.yml" nd/compose.yml >/dev/null
vm scp-to "$ROOT/lab/stacks/prometheus-grafana/compose.yml" prom/compose.yml >/dev/null
vm scp-to "$D/timeto.py" timeto.py >/dev/null; vm scp-to "$ROOT/lab/tests/_dsample.py" sample.py >/dev/null
dpull netdata/netdata:v2.12.2 prom/prometheus:v3.15.1 prom/node-exporter:v1.10.2 grafana/grafana:13.3.0
gx 'docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}"' > "$RAW/images.txt"
# Clean-start timings, one stack at a time (images already pulled, so this is start-up only)
gx 'python3 ~/timeto.py netdata' > "$RAW/timeto-netdata.json" 2> "$RAW/timeto-netdata.err"
gx 'python3 ~/timeto.py prom' > "$RAW/timeto-prom.json" 2> "$RAW/timeto-prom.err"
T0=$(date +%s); echo "T0=$T0" > "$RAW/t0.txt"
VOLS="prom_prom_data prom_grafana_data nd_netdataconfig nd_netdatalib nd_netdatacache"
disk() { gx "for v in $VOLS; do echo \"$(date +%s) \$v \$(sudo du -sb /var/lib/docker/volumes/\$v/_data | cut -f1)\"; done" >> "$RAW/disk.tsv"; }
disk
sleep 240
gx 'python3 ~/sample.py prom-prometheus-1 prom-grafana-1 prom-node-exporter-1 nd-netdata-1' > "$RAW/ram-early.json"
while [ $(( $(date +%s) - T0 )) -lt $(( DUR_MIN * 60 )) ]; do sleep $(( 300 - ( $(date +%s) - T0 ) % 300 )); disk; done
gx 'python3 ~/sample.py prom-prometheus-1 prom-grafana-1 prom-node-exporter-1 nd-netdata-1' > "$RAW/ram-late.json"
{ echo "query: count all series"; gx 'docker exec prom-prometheus-1 wget -qO- "http://127.0.0.1:9090/api/v1/query?query=count({__name__=~\".%2B\"})"'
  echo; echo "netdata charts/dimensions:"; gx 'curl -s http://127.0.0.1:19999/api/v1/charts | python3 -c "import json,sys;d=json.load(sys.stdin)[\"charts\"];print(len(d),sum(len(c[\"dimensions\"]) for c in d.values()))"'
  gx 'docker ps --format "{{.Names}} {{.Status}}"; docker inspect -f "{{.Name}} restarts={{.RestartCount}}" $(docker ps -aq)'; } > "$RAW/end-state.txt" 2>&1
S="$ROOT/public/shots/$T"
shot 8080 "/d/rYdddlPWk/node-exporter-full?orgId=1&from=now-30m&to=now&kiosk" "$S/grafana-node-exporter.png" '[{"fill":"input[name=user]","value":"admin"},{"fill":"input[name=password]","value":"admin"},{"press":"Enter"},{"wait":4000},{"click":"text=Skip"},{"goto":"http://127.0.0.1:18391/d/rYdddlPWk/node-exporter-full?orgId=1&from=now-30m&to=now&kiosk"},{"wait":8000}]'
shot 19999 / "$S/netdata.png" '[{"wait":6000},{"click":"text=Skip and use the dashboard anonymously"},{"wait":8000}]'
collect_env "$OUT"
python3 "$D/summarize.py" "$RAW" "$OUT"

#!/usr/bin/env python3
"""Runs in the VM. Times a clean start of each stack to its first useful dashboard and counts the steps.
Usage: timeto.py netdata|prom. Prints JSON with elapsed seconds (from the start of the first step) after every step."""
import base64, json, subprocess, sys, time, urllib.request, urllib.parse
which = sys.argv[1]
t0 = time.time(); steps = []
def mark(name, cmd_or_action):
    steps.append({"step": len(steps) + 1, "what": name, "action": cmd_or_action, "elapsed_s": round(time.time() - t0, 1)})
def http(url, data=None, headers=None, method=None):
    req = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
    return urllib.request.urlopen(req, timeout=20).read()
def wait(fn, timeout=600):
    end = time.time() + timeout
    while time.time() < end:
        try:
            r = fn()
            if r: return r
        except Exception:
            pass
        time.sleep(1)
    raise SystemExit(json.dumps({"error": "timeout", "steps": steps}))
if which == "netdata":
    subprocess.run("cd ~/nd && docker compose -p nd up -d", shell=True, check=True, capture_output=True)
    mark("start the stack", "docker compose up -d")
    def cpu():
        d = json.loads(http("http://127.0.0.1:19999/api/v1/data?chart=system.cpu&after=-5&points=1&format=json"))
        return d if d.get("data") and d["data"][0] and any(v for v in d["data"][0][1:]) else None
    wait(cpu)
    mark("dashboard serves live CPU data (system.cpu chart returned values)", "open http://host:19999 (no configuration)")
else:
    subprocess.run("cd ~/prom && docker compose -p prom up -d", shell=True, check=True, capture_output=True)
    mark("start the stack", "docker compose up -d")
    auth = {"Authorization": "Basic " + base64.b64encode(b"admin:admin").decode()}
    wait(lambda: b"ok" in http("http://127.0.0.1:8080/api/health"))
    mark("Grafana answers /api/health", "wait")
    http("http://127.0.0.1:8080/api/org", headers=auth)
    mark("log in with the default admin account", "UI: sign in as admin/admin (the UI then asks for a new password; skipped via API basic auth)")
    ds = json.loads(http("http://127.0.0.1:8080/api/datasources", json.dumps({"name": "Prometheus", "type": "prometheus", "url": "http://prometheus:9090", "access": "proxy", "isDefault": True}).encode(), {**auth, "content-type": "application/json"}))
    mark("add the Prometheus data source", "UI: Connections > Data sources > Add > Prometheus > URL http://prometheus:9090 > Save")
    rev = json.loads(http("https://grafana.com/api/dashboards/1860/revisions"))["items"][0]["revision"] if False else None
    dash = json.loads(http("https://grafana.com/api/dashboards/1860/revisions/latest/download"))
    mark("download dashboard 1860 (Node Exporter Full) from grafana.com", "UI: Dashboards > New > Import > 1860")
    out = json.loads(http("http://127.0.0.1:8080/api/dashboards/import", json.dumps({"dashboard": dash, "overwrite": True, "inputs": [{"name": "DS_PROMETHEUS", "type": "datasource", "pluginId": "prometheus", "value": ds["datasource"]["uid"]}], "folderId": 0}).encode(), {**auth, "content-type": "application/json"}))
    mark("import the dashboard", "UI: select the data source > Import")
    q = urllib.parse.quote('100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)')
    def cpu():
        d = json.loads(http(f"http://127.0.0.1:8080/api/datasources/proxy/uid/{ds['datasource']['uid']}/api/v1/query?query={q}", headers=auth))
        return d if d["data"]["result"] else None
    wait(cpu)
    mark("a dashboard query (1-minute CPU rate) returns data", "open the dashboard")
    steps.append({"dashboard_uid": out.get("uid"), "dashboard_title": dash.get("title"), "dashboard_revision_version": dash.get("version"), "dashboard_sha256": __import__("hashlib").sha256(json.dumps(dash, sort_keys=True).encode()).hexdigest()})
print(json.dumps(steps, indent=1))