Headscale 0.29.7 with two Tailscale clients, no DERP relay

services:
  headscale:
    image: headscale/headscale:0.29.7
    container_name: headscale
    restart: unless-stopped
    command: serve
    volumes:
      - ./${HS_CONFIG:-config.yaml}:/etc/headscale/config.yaml:ro
      - ./derp.yaml:/etc/headscale/derp.yaml:ro
      - headscale_lib:/var/lib/headscale
volumes:
  headscale_lib:

server_url: http://headscale:8080
listen_addr: 0.0.0.0:8080
metrics_listen_addr: 127.0.0.1:9090
noise:
  private_key_path: /var/lib/headscale/noise_private.key
prefixes:
  v4: 100.64.0.0/10
  v6: fd7a:115c:a1e0::/48
derp:
  server:
    enabled: false
  urls: []
  paths:
    - /etc/headscale/derp.yaml
  auto_update_enabled: false
database:
  type: sqlite
  sqlite:
    path: /var/lib/headscale/db.sqlite
dns:
  magic_dns: false
  override_local_dns: false
unix_socket: /var/run/headscale/headscale.sock

# Placeholder DERP map: one region whose only node (TEST-NET-1 address) is not running. There is no relay in this lab.
regions:
  900:
    regionid: 900
    regioncode: lab
    regionname: Lab placeholder, no relay running
    nodes:
      - name: 900a
        regionid: 900
        hostname: derp.lab.invalid
        ipv4: 192.0.2.1
        stunport: -1
        stunonly: false
        derpport: 443

#!/usr/bin/env bash
# Runs inside the VM from ~/hs. Raw outputs in ~/hs/raw.
set -u
cd "$HOME/hs"; mkdir -p raw
export COMPOSE_PROJECT_NAME=hs
TS=tailscale/tailscale:v1.104.1
NET=hs_default
HSC="docker exec headscale headscale"
now() { date +%s.%N; }
el() { python3 -c "print(round($2 - $1, 2))"; }
docker pull -q curlimages/curl:8.11.1 > /dev/null; docker pull -q "$TS" > /dev/null; docker pull -q headscale/headscale:0.29.7 > /dev/null
docker image inspect "$TS" headscale/headscale:0.29.7 --format '{{index .RepoTags 0}} {{.Id}}' > raw/images.txt

echo "== attempt 1: DERP disabled and no DERP map at all $(date -u +%T)"
HS_CONFIG=config.derp-empty.yaml docker compose up -d > raw/derp-empty.compose-up.txt 2>&1
sleep 20
docker inspect headscale --format 'restart_count={{.RestartCount}} state={{.State.Status}} exit_code={{.State.ExitCode}}' > raw/derp-empty.state.txt 2>&1
docker logs headscale > raw/derp-empty.logs.txt 2>&1
docker compose down -v > /dev/null 2>&1

echo "== start headscale with the placeholder DERP map $(date -u +%T)"
T0=$(now)
docker compose up -d > raw/compose-up.txt 2>&1
for _ in $(seq 1 60); do
  docker run --rm --network "$NET" curlimages/curl:8.11.1 -s -o /dev/null -w '%{http_code}' -m3 http://headscale:8080/health 2>/dev/null | grep -q 200 && break
  sleep 1
done
echo "ready_seconds $(el "$T0" "$(now)")" > raw/headscale-ready.txt
sleep 20
for _ in $(seq 1 12); do docker stats --no-stream --format '{{.MemUsage}} {{.CPUPerc}}' headscale; done > raw/headscale-idle.txt
$HSC version > raw/headscale-version.txt 2>&1
$HSC users create lab > raw/users-create.txt 2>&1
$HSC users list > raw/users-list.txt 2>&1
UID_=$($HSC users list -o json | jq -r '.[0].id')
$HSC preauthkeys create --user "$UID_" --reusable --expiration 1h -o json > raw/preauthkey.json 2>&1
KEY=$(jq -r '.key' raw/preauthkey.json)
[ -n "$KEY" ] && [ "$KEY" != null ] || { echo "no key"; cat raw/preauthkey.json; }

start_node() { # start_node <name>
  docker run -d --name "$1" --hostname "$1" --network "$NET" --cap-add NET_ADMIN --device /dev/net/tun \
    -e TS_USERSPACE=false -e TS_AUTHKEY="$KEY" -e TS_HOSTNAME="$1" -e TS_STATE_DIR=/var/lib/tailscale \
    -e TS_EXTRA_ARGS="--login-server=http://headscale:8080" "$TS" > /dev/null
}
running() { docker exec "$1" tailscale status --json 2>/dev/null | jq -e '.BackendState=="Running"' > /dev/null 2>&1; }

: > raw/cycles.tsv
for c in 1 2 3; do
  echo "== cycle $c $(date -u +%T)"
  A0=$(now)
  start_node node-a & start_node node-b & wait
  timeout 120 bash -c 'until docker exec node-a tailscale status --json 2>/dev/null | jq -e ".BackendState==\"Running\"" >/dev/null 2>&1 && docker exec node-b tailscale status --json 2>/dev/null | jq -e ".BackendState==\"Running\"" >/dev/null 2>&1; do sleep 0.25; done' || echo "cycle $c: nodes not Running in 120 s" >> raw/cycle-errors.txt
  A1=$(now)
  IPB=$(docker exec node-b tailscale ip -4 | head -1)
  timeout 120 bash -c "until docker exec node-a tailscale ping -c 1 --timeout 3s $IPB >/dev/null 2>&1; do sleep 0.25; done" || echo "cycle $c: no ping in 120 s" >> raw/cycle-errors.txt
  A2=$(now)
  # tailscale ping is answered by tailscaled itself; plain ICMP needs the WireGuard data path too, so time that separately
  timeout 120 bash -c "until docker exec node-a ping -c 1 -W 1 $IPB >/dev/null 2>&1; do sleep 0.25; done" || echo "cycle $c: no plain ping in 120 s" >> raw/cycle-errors.txt
  A3=$(now)
  echo -e "$c\t$(el "$A0" "$A1")\t$(el "$A0" "$A2")\t$(el "$A0" "$A3")" >> raw/cycles.tsv
  if [ "$c" -lt 3 ]; then
    docker rm -f node-a node-b > /dev/null
    for id in $($HSC nodes list -o json | jq -r '.[].id'); do $HSC nodes delete --identifier "$id" --force > /dev/null 2>&1; done
  fi
done

echo "== inspect $(date -u +%T)"
$HSC nodes list > raw/nodes-list.txt 2>&1
$HSC nodes list -o json > raw/nodes-list.json 2>&1
for n in node-a node-b; do
  docker exec $n tailscale version > raw/$n.version.txt 2>&1
  docker exec $n tailscale ip -4 > raw/$n.ip4.txt 2>&1
  docker exec $n tailscale status > raw/$n.status.txt 2>&1
  docker exec $n tailscale status --json > raw/$n.status.json 2>&1
done
IPB=$(cat raw/node-b.ip4.txt | head -1); IPA=$(cat raw/node-a.ip4.txt | head -1)
docker exec node-a ping -c 10 -i 0.2 "$IPB" > raw/ping.a-to-b.txt 2>&1
docker exec node-b ping -c 10 -i 0.2 "$IPA" > raw/ping.b-to-a.txt 2>&1
docker exec node-a tailscale ping -c 5 --timeout 5s "$IPB" > raw/tailscale-ping.txt 2>&1
docker exec node-a tailscale netcheck > raw/netcheck.txt 2>&1
docker exec node-a tailscale debug derp-map > raw/derp-map.txt 2>&1
for _ in $(seq 1 12); do docker stats --no-stream --format '{{.MemUsage}} {{.CPUPerc}}' headscale; done > raw/headscale-with-2-nodes.txt
for _ in $(seq 1 6); do docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' node-a node-b; done > raw/clients-stats.txt

echo "== block the direct path $(date -u +%T)"
docker exec node-b iptables -I INPUT -p udp -j DROP > raw/block.txt 2>&1
docker exec node-b iptables -I OUTPUT -p udp -j DROP >> raw/block.txt 2>&1
sleep 3
docker exec node-a tailscale ping -c 5 --timeout 4s "$IPB" > raw/blocked-tailscale-ping.txt 2>&1
docker exec node-a ping -c 5 -W 2 "$IPB" > raw/blocked-ping.txt 2>&1
docker exec node-a tailscale status > raw/blocked-status.txt 2>&1
docker exec node-b iptables -F INPUT; docker exec node-b iptables -F OUTPUT
R0=$(now)
timeout 240 bash -c "until docker exec node-a tailscale ping -c 1 --timeout 2s $IPB >/dev/null 2>&1; do sleep 1; done" && echo "recovered_after_seconds $(el "$R0" "$(now)")" > raw/recovered.txt || echo "not recovered within 240 s" > raw/recovered.txt
docker exec node-a tailscale ping -c 3 --timeout 3s "$IPB" > raw/recovered-tailscale-ping.txt 2>&1
docker exec node-a tailscale status > raw/recovered-status.txt 2>&1
docker exec node-a ping -c 5 -W 2 "$IPB" > raw/recovered-ping.txt 2>&1
docker compose logs --no-color --tail=40 headscale > raw/headscale-logs-tail.txt 2>&1
docker rm -f node-a node-b > /dev/null; docker compose down -v > /dev/null 2>&1
echo done > raw/DONE