WireGuard Networking — Mesh, Peer-to-Peer & Multi-Site

# Usage
chmod +x generate-mesh.sh
./generate-mesh.sh nodes.txt
# Output: mesh-configs/node-1.conf, mesh-configs/node-2.conf, etc.

# Deploy to each node
for node in node-1 node-2 node-3 node-4; do
  scp "mesh-configs/${node}.conf" "root@${node}:/etc/wireguard/wg0.conf"
  ssh "root@${node}" "systemctl enable --now wg-quick@wg0"
done

Multi-site — connecting offices with subnet routing

Network layout

HQ gateway — /etc/wireguard/wg0.conf

[Interface]
Address    = 10.99.0.1/24
ListenPort = 51820
PrivateKey = <hq-private-key>

PostUp   = sysctl -w net.ipv4.ip_forward=1
PostDown = sysctl -w net.ipv4.ip_forward=0

[Peer]
# Branch A — route their entire LAN through this peer
PublicKey  = <branch-a-pubkey>
AllowedIPs = 10.99.0.2/32, 192.168.2.0/24
Endpoint   = 198.51.100.2:51820
PersistentKeepalive = 25

[Peer]
# Branch B
PublicKey  = <branch-b-pubkey>
AllowedIPs = 10.99.0.3/32, 192.168.3.0/24
Endpoint   = 203.0.113.1:51820
PersistentKeepalive = 25

Branch A gateway — /etc/wireguard/wg0.conf

[Interface]
Address    = 10.99.0.2/24
ListenPort = 51820
PrivateKey = <branch-a-private-key>

PostUp   = sysctl -w net.ipv4.ip_forward=1
PostDown = sysctl -w net.ipv4.ip_forward=0

[Peer]
# HQ — route HQ LAN + Branch B LAN through HQ
PublicKey  = <hq-pubkey>
AllowedIPs = 10.99.0.1/32, 192.168.1.0/24, 10.99.0.3/32, 192.168.3.0/24
Endpoint   = 198.51.100.1:51820
PersistentKeepalive = 25

Branch B gateway — /etc/wireguard/wg0.conf

[Interface]
Address    = 10.99.0.3/24
ListenPort = 51820
PrivateKey = <branch-b-private-key>

PostUp   = sysctl -w net.ipv4.ip_forward=1
PostDown = sysctl -w net.ipv4.ip_forward=0

[Peer]
# HQ — route HQ LAN + Branch A LAN through HQ
PublicKey  = <hq-pubkey>
AllowedIPs = 10.99.0.1/32, 192.168.1.0/24, 10.99.0.2/32, 192.168.2.0/24
Endpoint   = 198.51.100.1:51820
PersistentKeepalive = 25

LAN routing — tell other machines about the tunnels

# On each LAN, machines need a route to the remote subnets.
# Option 1: add a static route on each machine
ip route add 192.168.2.0/24 via 192.168.1.1   # on HQ LAN machines
ip route add 192.168.3.0/24 via 192.168.1.1   # on HQ LAN machines

# Option 2: set the WG gateway as the default gateway (simpler)
# Or configure the routes on your DHCP server / router

Split tunneling

# To only route specific traffic through the tunnel (not everything),
# use precise AllowedIPs on the client side.
#
# Example: only route traffic to the remote LAN through the tunnel
AllowedIPs = 192.168.1.0/24, 10.99.0.0/24
# Internet traffic goes direct — not through the tunnel

WireGuard + ZFS replication

Setup

# Assumptions:
# - Primary server: 10.99.0.1 (WireGuard tunnel IP)
# - Backup server:  10.99.0.2 (WireGuard tunnel IP)
# - WireGuard tunnel is already up (see peer-to-peer section above)
# - Both servers run kldloadOS with sanoid/syncoid installed

# On the PRIMARY server — configure syncoid to replicate over the tunnel
# Use the tunnel IP, not the public IP
syncoid --recursive \
  rpool/data \
  root@10.99.0.2:backup/data

# Automate with a systemd timer
cat > /etc/systemd/system/zfs-replicate.service <<'EOF'
[Unit]
Description=ZFS replication to offsite backup
After=wg-quick@wg0.service
Requires=wg-quick@wg0.service

[Service]
Type=oneshot
ExecStart=/usr/sbin/syncoid --recursive --no-privilege-elevation rpool/data root@10.99.0.2:backup/data
EOF

cat > /etc/systemd/system/zfs-replicate.timer <<'EOF'
[Unit]
Description=Run ZFS replication every hour

[Timer]
OnCalendar=hourly
Persistent=true
RandomizedDelaySec=300

[Install]
WantedBy=timers.target
EOF

systemctl daemon-reload
systemctl enable --now zfs-replicate.timer

Verify replication

# On the backup server — check that snapshots are arriving
zfs list -t snapshot -r backup/data | tail -5

# Verify data integrity
zpool scrub backup
zpool status backup

WireGuard + kldload fleet

How auto-mesh works on firstboot

# When a kldloadOS node boots in cluster mode:
# 1. It generates keys for all 4 interfaces
# 2. It connects to the cluster manager over wg0 (enrollment plane)
# 3. The cluster manager distributes peer configs for wg1, wg2, wg3
# 4. The node brings up all 4 interfaces
# 5. Full mesh is established — every node can reach every other node on all 4 planes

# You can inspect the mesh at any time:
for iface in wg0 wg1 wg2 wg3; do
  echo "=== $iface ==="
  wg show "$iface"
  echo
done

Manual 4-plane setup

# If you want the same isolation without cluster mode, create 4 configs:

for i in 0 1 2 3; do
  port=$((51820 + i))
  subnet=$((77 + i))
  wg genkey | tee "/etc/wireguard/wg${i}-private.key" | wg pubkey > "/etc/wireguard/wg${i}-public.key"
  chmod 600 "/etc/wireguard/wg${i}-private.key"

  cat > "/etc/wireguard/wg${i}.conf" <

# /etc/nftables.conf

table inet filter {
  chain input {
    type filter hook input priority 0; policy drop;

    # Allow established connections
    ct state established,related accept

    # Allow loopback
    iifname "lo" accept

    # Allow WireGuard UDP port from anywhere
    udp dport 51820 accept

    # Allow SSH only from the management tunnel (wg1)
    iifname "wg1" tcp dport 22 accept

    # Allow Prometheus scrapes only from the monitoring tunnel (wg2)
    iifname "wg2" tcp dport 9090 accept
    iifname "wg2" tcp dport 9100 accept

    # Allow ZFS replication (SSH) only from the storage tunnel (wg3)
    iifname "wg3" tcp dport 22 accept

    # Allow ICMP (ping) on all WireGuard interfaces
    iifname "wg*" icmp type echo-request accept

    # Drop everything else
    log prefix "nft-drop: " drop
  }

  chain forward {
    type filter hook forward priority 0; policy drop;

    # Allow forwarding between WireGuard interfaces (for site-to-site routing)
    iifname "wg0" oifname "eth0" accept
    iifname "eth0" oifname "wg0" ct state established,related accept

    # Allow inter-site traffic
    iifname "wg0" oifname "wg0" accept
  }
}

table inet nat {
  chain postrouting {
    type nat hook postrouting priority 100;
    # Masquerade WireGuard traffic going out to the LAN
    oifname "eth0" masquerade
  }
}

# Apply the rules
nft -f /etc/nftables.conf

# Make them persistent
systemctl enable nftables

# Verify
nft list ruleset

# 1. Is the interface up?
ip link show wg0
# Should say "UP" — if not:
wg-quick up wg0

# 2. Is there a handshake?
wg show wg0
# Look for "latest handshake: X seconds ago"
# If it says "none" — the tunnel is NOT working. Continue debugging.

# 3. Is the firewall blocking UDP?
# On the RECEIVING end:
nft list ruleset | grep 51820
# Make sure UDP 51820 is allowed inbound
# Quick test — temporarily open everything:
nft flush ruleset
# Then try again. If it works, your firewall is the problem.

# 4. Is the endpoint reachable?
# From the side that CANNOT handshake:
nc -zuv 198.51.100.10 51820
# Should say "succeeded" or "open"

# 5. Are the keys correct?
# The public key on Node A's [Peer] section must match Node B's private key (and vice versa)
# Regenerate if unsure:
wg genkey | tee /etc/wireguard/private.key | wg pubkey > /etc/wireguard/public.key

# 6. Is the kernel module loaded?
lsmod | grep wireguard
# If not:
modprobe wireguard

# 7. Check system logs
journalctl -u wg-quick@wg0 --no-pager -n 50
dmesg | grep wireguard

# A healthy tunnel shows a recent handshake:
wg show wg0
# peer: abc123...
#   endpoint: 198.51.100.10:51820
#   allowed ips: 10.99.0.2/32
#   latest handshake: 12 seconds ago     <-- GOOD
#   transfer: 1.24 MiB received, 3.48 MiB sent

# If "latest handshake" is missing or says "none":
# - The peer has never successfully connected
# - Check firewall, endpoint, and keys

# If "latest handshake" is more than 2-3 minutes old:
# - The peer may be offline
# - PersistentKeepalive may not be set (needed behind NAT)

# WireGuard adds 60 bytes of overhead (IPv4) or 80 bytes (IPv6)
# Default MTU is 1420, which works for most networks
# If you are seeing packet fragmentation or slow transfers:

# Check current MTU
ip link show wg0 | grep mtu

# Lower the MTU if needed (common behind PPPoE or other encapsulation)
# In wg0.conf:
# [Interface]
# MTU = 1380

# Or set it live:
ip link set wg0 mtu 1380

# Test with a specific packet size
ping -c 3 -s 1400 -M do 10.99.0.2
# If this fails but smaller sizes work, you have an MTU problem

# Install iperf3 (already in kldloadOS darksite)
dnf install -y iperf3   # CentOS/RHEL
apt install -y iperf3    # Debian

# === BASELINE — no tunnel ===
# On Node B (server):
iperf3 -s

# On Node A (client):
iperf3 -c 198.51.100.20 -t 30 -P 4
# Note the bandwidth

# === THROUGH THE TUNNEL ===
# On Node B (server):
iperf3 -s -B 10.99.0.2

# On Node A (client):
iperf3 -c 10.99.0.2 -t 30 -P 4
# Compare with baseline — expect less than 5% drop on modern hardware