Self-hosted email: RAM of Stalwart and docker-mailserver

# lab/stacks/stalwart/compose.yml with the volumes moved to the paths Stalwart actually uses (/etc/stalwart, /var/lib/stalwart),
# and the IMAPS and implicit-TLS submission (465) ports published (587 is not exposed by the image). First start is bootstrap mode; stalwart-setup.py finishes it through the JMAP API.
services:
  stalwart:
    image: stalwartlabs/stalwart:v1.0.1
    restart: unless-stopped
    environment:
      STALWART_RECOVERY_ADMIN: admin:lab-only-not-a-secret
    ports:
      - "127.0.0.1:8080:8080"
      - "127.0.0.1:2525:25"
      - "127.0.0.1:1465:465"
      - "127.0.0.1:1993:993"
    volumes:
      - stalwart-etc:/etc/stalwart
      - stalwart-data:/var/lib/stalwart
volumes:
  stalwart-etc:
  stalwart-data:

#!/usr/bin/env bash
# mail: docker-mailserver 16.0.1 (4 anti-spam configurations) and Stalwart 1.0.1, inside one VM each.
# Create two mailboxes, send alice -> bob over SMTP (port 25 and authenticated submission), read over IMAP, sample RAM before and after.
# Nothing leaves the VM: no DNS, no outbound mail, no real domain.
set -uo pipefail
source "$(dirname "$0")/../_dlib.sh"
T=mail; OUT="$ROOT/evidence/tests/$T"; RAW="$OUT/raw"; D="$ROOT/lab/tests/$T"
TOOLS=("$@"); [ ${#TOOLS[@]} -eq 0 ] && TOOLS=(dms stalwart)
[ $# -eq 0 ] && rm -rf "$OUT"; mkdir -p "$RAW"
dms() {
  start_vm d-$T-dms
  gx 'mkdir -p ~/d'; vm scp-to "$ROOT/lab/stacks/docker-mailserver/compose.yml" d/compose.yml >/dev/null
  vm scp-to "$D/mailtest.py" mailtest.py >/dev/null; vm scp-to "$D/dms-cfg.py" d/dms-cfg.py >/dev/null; vm scp-to "$ROOT/lab/tests/_dsample.py" sample.py >/dev/null
  cpull '~/d' dms
  # name | extra environment for the mailserver service (the lab stack already sets ENABLE_RSPAMD=1, CLAMAV=0, FAIL2BAN=0)
  local cfgs=("A-rspamd-on-legacy-on|" "B-rspamd-on-legacy-off|ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 ENABLE_POLICYD_SPF=0 ENABLE_AMAVIS=0 ENABLE_SPAMASSASSIN=0" "C-rspamd-off|ENABLE_RSPAMD=0" "D-all-antispam-off|ENABLE_RSPAMD=0 ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 ENABLE_POLICYD_SPF=0 ENABLE_AMAVIS=0 ENABLE_SPAMASSASSIN=0")
  for c in "${cfgs[@]}"; do
    local name="${c%%|*}"; local extra="${c#*|}"; local out="$RAW/dms-$name"; mkdir -p "$out"
    gx "cd ~/d && docker compose -p dms down -v >/dev/null 2>&1; python3 dms-cfg.py $extra && docker compose -p dms -f compose.cfg.yml up -d 2>&1 | tail -1; grep -E 'ENABLE_' compose.cfg.yml" > "$out/env-lines.txt" 2>&1
    local t0; t0=$(date +%s)
    until gx 'docker exec dms-mailserver-1 setup email add alice@lab.internal lab-only-not-a-secret' >> "$out/accounts.txt" 2>&1; do sleep 3; [ $(( $(date +%s) - t0 )) -gt 100 ] && break; done
    gx 'docker exec dms-mailserver-1 setup email add bob@lab.internal lab-only-not-a-secret' >> "$out/accounts.txt" 2>&1
    until gx 'curl -s --max-time 3 telnet://127.0.0.1:1143 </dev/null | head -c 20 | grep -q OK'; do sleep 3; [ $(( $(date +%s) - t0 )) -gt 240 ] && { echo "dovecot not answering in 240 s" > "$out/FAILED.txt"; break; }; done
    echo "ready_s_since_up=$(( $(date +%s) - t0 ))" > "$out/ready.txt"
    sleep 60
    gx 'python3 ~/sample.py dms-mailserver-1' > "$out/ram-before-mail.json"
    gx 'python3 ~/mailtest.py dms' > "$out/mailtest.json" 2> "$out/mailtest.err"
    gx 'python3 ~/sample.py dms-mailserver-1' > "$out/ram-after-mail.json"
    gx 'docker exec dms-mailserver-1 supervisorctl status | grep RUNNING | awk "{print \$1}" | tr "\n" " "; docker exec dms-mailserver-1 ps -eo rss,comm --sort=-rss | head -8; docker inspect -f "restarts={{.RestartCount}}" dms-mailserver-1' > "$out/processes.txt" 2>&1
  done
  mkdir -p "$RAW/dms-env"; collect_env "$RAW/dms-env"
}
stalwart() {
  start_vm d-$T-stalwart
  gx 'mkdir -p ~/s'; vm scp-to "$D/files/stalwart.yml" s/compose.yml >/dev/null
  vm scp-to "$D/mailtest.py" mailtest.py >/dev/null; vm scp-to "$D/stalwart-setup.py" stalwart-setup.py >/dev/null; vm scp-to "$ROOT/lab/tests/_dsample.py" sample.py >/dev/null
  cpull '~/s' stalwart
  local out="$RAW/stalwart"; mkdir -p "$out"
  gx 'cd ~/s && docker compose -p stalwart up -d 2>&1 | tail -1'
  until gx 'curl -s -o /dev/null http://127.0.0.1:8080/jmap/session'; do sleep 2; done; sleep 60
  gx 'python3 ~/sample.py stalwart-stalwart-1' > "$out/ram-bootstrap-mode.json"
  gx 'python3 ~/stalwart-setup.py 1' > "$out/setup1.json" 2> "$out/setup1.err"
  gx 'cd ~/s && sed -i "/environment:/d;/STALWART_RECOVERY_ADMIN/d" compose.yml && docker compose -p stalwart up -d 2>&1 | tail -1; grep -c RECOVERY compose.yml' > "$out/recreate.txt" 2>&1
  gx 'python3 ~/stalwart-setup.py 2' > "$out/setup2.json" 2> "$out/setup2.err"
  sleep 60
  gx 'python3 ~/sample.py stalwart-stalwart-1' > "$out/ram-before-mail.json"
  gx 'python3 ~/mailtest.py stalwart' > "$out/mailtest.json" 2> "$out/mailtest.err"
  gx 'python3 ~/sample.py stalwart-stalwart-1' > "$out/ram-after-mail.json"
  gx 'docker inspect -f "restarts={{.RestartCount}}" stalwart-stalwart-1; docker logs stalwart-stalwart-1 2>&1 | grep -ci "error"' > "$out/state.txt" 2>&1
  mkdir -p "$RAW/stalwart-env"; collect_env "$RAW/stalwart-env"
}
for t in "${TOOLS[@]}"; do ( $t ); done
cp "$D/files/stalwart.yml" "$OUT/"
python3 "$D/summarize.py" "$RAW" "$OUT"

#!/usr/bin/env python3
"""Runs in the VM. Usage: mailtest.py dms|stalwart. Sends 5 messages alice -> bob over SMTP port 25 (no login, as another server would)
and 5 over submission (port 587, with login), then reads them over IMAP. Per message: SMTP time, seconds until it is visible over IMAP, and
the folder it landed in. Prints JSON."""
import email.utils, imaplib, json, smtplib, ssl, sys, time
kind = sys.argv[1]; PW = "lab-only-not-a-secret"; ctx = ssl._create_unverified_context()
def imap():
    if kind == "stalwart": return imaplib.IMAP4_SSL("127.0.0.1", 1993, ssl_context=ctx)
    return imaplib.IMAP4("127.0.0.1", 1143)
def counts(i):
    out = {}
    for line in i.list()[1]:
        name = line.decode().rsplit(' "', 1)[-1].strip('"') if line.decode().endswith('"') else line.decode().split()[-1]
        t, d = i.status(f'"{name}"', "(MESSAGES)")
        if t == "OK": out[name] = int(d[0].decode().split("MESSAGES")[1].strip(" )"))
    return out
def msg(n, path):
    from email.message import EmailMessage
    m = EmailMessage(); m["From"] = "alice@lab.internal"; m["To"] = "bob@lab.internal"; m["Subject"] = f"{path} test {n}"
    m["Date"] = email.utils.formatdate(); m["Message-ID"] = email.utils.make_msgid(domain="lab.internal"); m.set_content("A short generated message body for the lab test.\n" * 5); return m
res = []; excerpt = {}
for path in ("smtp25", "submission"):
    for n in range(1, 6):
        i = imap(); i.login("bob@lab.internal", PW); before = counts(i); attempts = 0; waited = 0.0
        while True:  # a connection that is closed straight away is retried every 5 s (Stalwart did this intermittently; see the guide)
            attempts += 1; t = time.perf_counter()
            try:
                if path == "smtp25":
                    s = smtplib.SMTP("127.0.0.1", 2525, timeout=60); s.ehlo("lab.internal")
                elif kind == "stalwart":  # port 465, TLS from the first byte (the image does not expose 587)
                    s = smtplib.SMTP_SSL("127.0.0.1", 1465, timeout=60, context=ctx); s.ehlo("lab.internal"); s.login("alice@lab.internal", PW)
                else:
                    s = smtplib.SMTP("127.0.0.1", 1587, timeout=60); s.ehlo("lab.internal")
                    if "starttls" in s.esmtp_features: s.starttls(context=ctx); s.ehlo("lab.internal")
                    s.login("alice@lab.internal", PW)
                break
            except smtplib.SMTPServerDisconnected:
                waited += 5; time.sleep(5)
                if attempts >= 30: raise
        s.send_message(msg(n, path)); smtp_s = time.perf_counter() - t; s.quit(); folder = None
        while time.perf_counter() - t < 60:
            now = counts(i); diff = [k for k in now if now[k] > before.get(k, 0)]
            if diff: folder = diff[0]; break
            time.sleep(0.1)
        vis = time.perf_counter() - t
        if folder and n == 1:
            i.select(f'"{folder}"'); typ, d = i.search(None, "ALL"); h = i.fetch(d[0].split()[-1], "(BODY.PEEK[HEADER])")[1][0][1].decode()
            excerpt[path] = "\n".join(l for l in h.splitlines() if l.split(":")[0] in ("X-Spam-Status", "X-Rspamd-Result", "X-Spamd-Result", "X-Rspamd-Server", "Authentication-Results", "Received-SPF", "DKIM-Signature", "X-Spam-Flag", "X-Spam-Result"))[:1200]
        i.logout(); res.append({"path": path, "n": n, "attempts": attempts, "waited_s": waited, "smtp_s": round(smtp_s, 3), "visible_s": round(vis, 3) if folder else None, "folder": folder})
print(json.dumps({"kind": kind, "messages": res, "header_excerpt": excerpt}, indent=1))

#!/usr/bin/env python3
"""Runs in the VM. Usage: stalwart-setup.py 1|2.
1: finishes Stalwart's bootstrap mode through JMAP (x:Bootstrap/set) and saves the generated permanent admin login to /tmp/stalwart-admin.txt.
   The caller must then recreate the container WITHOUT STALWART_RECOVERY_ADMIN (with it set, Stalwart stays in recovery mode and starts no mail listeners).
2: creates alice and bob at lab.internal. Prints a JSON log (the admin secret is not printed)."""
import json, subprocess, sys, time, urllib.request, base64
H = "http://127.0.0.1:8080/jmap/"; log = {}
def jmap(user, pw, calls):
    req = urllib.request.Request(H, data=json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"], "methodCalls": calls}).encode(),
        headers={"content-type": "application/json", "Authorization": "Basic " + base64.b64encode(f"{user}:{pw}".encode()).decode()})
    return json.load(urllib.request.urlopen(req, timeout=30))["methodResponses"]
phase = sys.argv[1]
if phase == "1":
    for _ in range(60):
        try: jmap("admin", "lab-only-not-a-secret", [["x:Bootstrap/get", {"ids": None}, "c"]]); break
        except Exception: time.sleep(2)
    t = time.time()
    r = jmap("admin", "lab-only-not-a-secret", [["x:Bootstrap/set", {"update": {"singleton": {"defaultHostname": "mail.lab.internal", "defaultDomain": "lab.internal", "requestTlsCertificate": False, "generateDkimKeys": True}}}, "c"]])
    upd = r[0][1]["updated"]["singleton"]; open("/tmp/stalwart-admin.txt", "w").write(f"{upd['username']}:{upd['secret']}")
    log["bootstrap_set_s"] = round(time.time() - t, 2); log["admin_user"] = upd["username"]
else:
    admin_user, admin_pw = open("/tmp/stalwart-admin.txt").read().split(":", 1); t = time.time()
    for _ in range(90):
        try: dom = jmap(admin_user, admin_pw, [["x:Domain/get", {"ids": None, "properties": ["name"]}, "c"]])[0][1]["list"]; break
        except Exception: time.sleep(1)
    log["api_after_recreate_s"] = round(time.time() - t, 2); did = dom[0]["id"]; log["domain"] = dom[0]["name"]
    for u in ("alice", "bob"):
        r = jmap(admin_user, admin_pw, [["x:Account/set", {"create": {"a": {"@type": "User", "name": u, "domainId": did, "credentials": {"0": {"@type": "Password", "secret": "lab-only-not-a-secret"}}}}}, "c"]])
        log["create_" + u] = "created" if r[0][1].get("created") else r[0][1]
print(json.dumps(log, indent=1))

#!/usr/bin/env python3
"""Runs in the VM. Usage: dms-cfg.py KEY=VAL ... Writes compose.cfg.yml from compose.yml with those ENABLE_* values set."""
import re, sys
s = open("compose.yml").read()
for kv in sys.argv[1:]:
    k, v = kv.split("=")
    if re.search(rf"^\s+{k}:", s, re.M): s = re.sub(rf'({k}: )"\d"', rf'\1"{v}"', s)
    else: s = s.replace("      ENABLE_CLAMAV:", f'      {k}: "{v}"\n      ENABLE_CLAMAV:')
open("compose.cfg.yml", "w").write(s)