{ writeShellScriptBin , coreutils , findutils , gnugrep , iproute2 , systemd }: # For user services that only make sense while online (jellyfin-mpv-shim, # nextcloud-client, ...). They keep a persistent socket to their server; when # the machine drops off the network that socket dies, but the process stays # alive and the unit stays "active" — running yet deaf, the "up but not # working, restart it by hand" case. systemd can't see this: nothing crashes # and the programs speak no health protocol, so we probe the real thing — is # the MainPID still holding a live (established, non-loopback) connection? # # wait-online UNIT block until a real address exists; used as ExecStartPre # so UNIT only starts once online. Also stamps the start # time so `check` leaves a freshly (re)started UNIT alone. # check UNIT... one tick, run by a 60s timer: restart every UNIT that is # active but not connected, at most once per 5 minutes. # # Wired up by the services.network-watchdog home-manager module. writeShellScriptBin "network-watchdog" '' set -u stampdir="''${XDG_RUNTIME_DIR:-/run/user/$(${coreutils}/bin/id -u)}/network-watchdog" ${coreutils}/bin/mkdir -p "$stampdir" usage() { echo "usage: network-watchdog wait-online UNIT | check UNIT..." >&2; exit 64; } # A global-scope address on a non-loopback interface — true for a LAN IP too, # so this needs no server URL and works whether the server is local or remote. network_up() { ${iproute2}/bin/ip -o addr show scope global up 2>/dev/null | ${gnugrep}/bin/grep -q .; } # Healthy: MainPID owns an established socket to something other than loopback. connected() { pid="$(${systemd}/bin/systemctl --user show -p MainPID --value "$1" 2>/dev/null)" [ "''${pid:-0}" != 0 ] && ${iproute2}/bin/ss -tnpH state established 2>/dev/null \ | ${gnugrep}/bin/grep "pid=$pid," \ | ${gnugrep}/bin/grep -qv -e '127\.0\.0\.1' -e '\[::1\]' } # Stamped within the last 5 minutes: still connecting, or already restarted. recently_started() { [ -n "$(${findutils}/bin/find "$stampdir/$1" -mmin -5 2>/dev/null)" ]; } [ $# -ge 1 ] || usage cmd="$1"; shift case "$cmd" in wait-online) [ $# -eq 1 ] || usage while ! network_up; do ${coreutils}/bin/sleep 5; done : > "$stampdir/$1" ;; check) # Nothing can be connected while offline, and restarting wouldn't help. network_up || exit 0 for unit; do # Not "active" (inactive, failed, or still in wait-online): systemd's job, not ours. ${systemd}/bin/systemctl --user is-active --quiet "$unit" || continue recently_started "$unit" && continue connected "$unit" && continue echo "network-watchdog: $unit is online but has no live connection -> restarting" ${systemd}/bin/systemctl --user restart "$unit" # its wait-online refreshes the stamp done ;; *) usage ;; esac ''