Jellyfin CPU transcoding on 4 vCPU: how many streams?

#!/usr/bin/env python3
"""Runs in the VM. bench.py <outdir> [max_streams=14] [reps=3]
For each source (H.264 1080p, HEVC 1080p; 60 s excerpts of Big Buck Bunny) and N = 1, 2, 3 ... simultaneous software transcodes to
720p H.264 + AAC stereo (GET /Videos/{id}/stream.ts, progressive, consumed as fast as the server produces it), 3 reps each.
Real-time factor (RTF) of a stream = media duration / wall time of the request. Stops raising N for a source when the slowest
stream of the median rep drops below 1.0x. Direct play: GET /Videos/{id}/stream?static=true paced at the file's own bitrate."""
import sys, os, json, time, subprocess, uuid, statistics as st
out = sys.argv[1]; MAXN = int(sys.argv[2]) if len(sys.argv) > 2 else 14; REPS = int(sys.argv[3]) if len(sys.argv) > 3 else 3
os.makedirs(out, exist_ok=True); d = json.load(open(os.path.expanduser("~/jf/ids.json"))); TOK = d["token"]; DUR = 60.04
items = {"h264": d["items"]["Big Buck Bunny"], "hevc": d["items"]["Big Buck Bunny HEVC"]}
def url(it, transcode=True):
    base = f"http://127.0.0.1:8080/Videos/{it['id']}/"
    if not transcode: return base + f"stream?static=true&DeviceId=lab-dp&MediaSourceId={it['source']}&api_key={TOK}"
    return (base + f"stream.ts?DeviceId=lab-{uuid.uuid4().hex[:8]}&MediaSourceId={it['source']}&VideoCodec=h264&AudioCodec=aac&MaxWidth=1280&MaxHeight=720"
            f"&VideoBitRate=4000000&AudioBitRate=128000&TranscodingMaxAudioChannels=2&PlaySessionId={uuid.uuid4().hex}&api_key={TOK}")
def sh(c): return subprocess.run(c, shell=True, capture_output=True, text=True).stdout
cname = sh("docker ps --format '{{.Names}}' | grep jellyfin").strip().splitlines()[0]
def stats_loop(path):  # CPU/RAM samples of the Jellyfin container, about 2 s apart (docker stats --no-stream takes about 2 s)
    return subprocess.Popen(f"while true; do echo $(date +%s.%N),$(docker stats --no-stream --format '{{{{.CPUPerc}}}},{{{{.MemUsage}}}}' {cname}); sleep 0.5; done > {path}", shell=True, preexec_fn=os.setsid)
def stop(p):
    import signal; os.killpg(os.getpgid(p.pid), signal.SIGTERM)
res = {"runs": [], "container": cname}
def run_n(codec, n, rep):
    csv = f"{out}/cpu-{codec}-n{n}-r{rep}.csv"; sp = stats_loop(csv); time.sleep(2)
    t0 = time.time(); procs = [subprocess.Popen(["curl", "-s", "-o", "/dev/null", "-w", "%{http_code} %{time_total} %{size_download}", url(items[codec])], stdout=subprocess.PIPE, text=True) for _ in range(n)]
    rows = []
    for p in procs:
        o = p.communicate()[0].split(); rows.append({"http": o[0], "secs": float(o[1]), "bytes": int(o[2]), "rtf": DUR / float(o[1])})
    t1 = time.time(); time.sleep(2); stop(sp)
    cpu = []
    for l in open(csv):
        try: ts, c, m = l.strip().split(",", 2); ts = float(ts)
        except ValueError: continue
        if t0 <= ts <= t1: cpu.append((float(c.strip("%")), m))
    r = {"codec": codec, "n": n, "rep": rep, "t_start": t0, "t_end": t1, "streams": rows, "min_rtf": min(x["rtf"] for x in rows), "mean_rtf": st.mean(x["rtf"] for x in rows),
         "cpu_mean": st.mean(c for c, _ in cpu) if cpu else None, "cpu_peak": max(c for c, _ in cpu) if cpu else None, "cpu_samples": len(cpu)}
    res["runs"].append(r); print(json.dumps({k: v for k, v in r.items() if k != "streams"}), flush=True); return r
for codec in ("h264", "hevc"):
    for n in range(1, MAXN + 1):
        rr = [run_n(codec, n, rep) for rep in range(1, REPS + 1)]
        if st.median(r["min_rtf"] for r in rr) < 1.0: break
    sh("sleep 3")
# ffmpeg command lines and the speed it reported, from Jellyfin's own transcode logs
sh(f"docker exec {cname} sh -c 'cp /config/encoding.xml /tmp/enc.xml 2>/dev/null; true'")
open(f"{out}/encoding.xml", "w").write(sh(f"docker exec {cname} cat /config/config/encoding.xml 2>&1 || docker exec {cname} find /config -name encoding.xml"))
open(f"{out}/ffmpeg-cmdline-sample.txt", "w").write(sh(f"docker exec {cname} sh -c 'f=$(ls -t /config/log/FFmpeg.Transcode-*.log | head -1); head -n 12 $f | cut -c1-900'"))
open(f"{out}/ffmpeg-speed-lines.txt", "w").write(sh(f"docker exec {cname} sh -c 'for f in $(ls -t /config/log/FFmpeg.Transcode-*.log | head -20); do echo $f; tr \"\\r\" \"\\n\" < $f | grep -o \"speed=[0-9.]*x\" | tail -1; done'"))
# direct play at real-time pace: curl limited to the file's average byte rate
dp = []
for codec in ("h264", "hevc"):
    size = os.path.getsize(os.path.expanduser(f"~/jf/media/Big Buck Bunny {codec.upper()} (2008).mp4")); rate = int(size / DUR)
    csv = f"{out}/cpu-directplay-{codec}.csv"; sp = stats_loop(csv); time.sleep(2); t0 = time.time()
    o = subprocess.run(["curl", "-s", "-o", "/dev/null", "--limit-rate", str(rate), "-w", "%{http_code} %{time_total} %{size_download}", url(items[codec], False)], capture_output=True, text=True).stdout.split()
    t1 = time.time(); time.sleep(2); stop(sp)
    cpu = [float(l.split(",")[1].strip("%")) for l in open(csv) if l.count(",") >= 2 and t0 <= float(l.split(",")[0]) <= t1]
    dp.append({"codec": codec, "http": o[0], "secs": float(o[1]), "bytes": int(o[2]), "rate_Bps": rate, "cpu_mean": st.mean(cpu), "cpu_peak": max(cpu), "samples": len(cpu)}); print(json.dumps(dp[-1]), flush=True)
res["direct_play"] = dp
json.dump(res, open(f"{out}/bench.json", "w"), indent=1)