70 lines
3.0 KiB
Nix
70 lines
3.0 KiB
Nix
{ writeShellScriptBin
|
|
, coreutils
|
|
, findutils
|
|
, gnugrep
|
|
, iproute2
|
|
, systemd
|
|
}:
|
|
|
|
# For user services that only make sense while online (jellyfin-mpv-shim,
|
|
# nextcloud-client, ...). They keep a persistent socket to their server; when
|
|
# the machine drops off the network that socket dies, but the process stays
|
|
# alive and the unit stays "active" — running yet deaf, the "up but not
|
|
# working, restart it by hand" case. systemd can't see this: nothing crashes
|
|
# and the programs speak no health protocol, so we probe the real thing — is
|
|
# the MainPID still holding a live (established, non-loopback) connection?
|
|
#
|
|
# wait-online UNIT block until a default route exists; used as ExecStartPre
|
|
# so UNIT only starts once online. Also stamps the start
|
|
# time so `check` leaves a freshly (re)started UNIT alone.
|
|
# check UNIT... one tick, run by a 60s timer: restart every UNIT that is
|
|
# active but not connected, at most once per 2 minutes.
|
|
#
|
|
# Wired up by the services.network-watchdog home-manager module.
|
|
|
|
writeShellScriptBin "network-watchdog" ''
|
|
set -u
|
|
stampdir="''${XDG_RUNTIME_DIR:-/run/user/$(${coreutils}/bin/id -u)}/network-watchdog"
|
|
${coreutils}/bin/mkdir -p "$stampdir"
|
|
|
|
usage() { echo "usage: network-watchdog wait-online UNIT | check UNIT..." >&2; exit 64; }
|
|
|
|
# Online = a default route. An address alone is not enough: wg0 and virtual
|
|
# bridges carry global-scope addresses whether or not there is a way out.
|
|
network_up() { { ${iproute2}/bin/ip -4 route show default; ${iproute2}/bin/ip -6 route show default; } 2>/dev/null | ${gnugrep}/bin/grep -q .; }
|
|
|
|
# Healthy: MainPID owns an established socket to something other than loopback.
|
|
connected() {
|
|
pid="$(${systemd}/bin/systemctl --user show -p MainPID --value "$1" 2>/dev/null)"
|
|
[ "''${pid:-0}" != 0 ] && ${iproute2}/bin/ss -tnpH state established 2>/dev/null \
|
|
| ${gnugrep}/bin/grep "pid=$pid," \
|
|
| ${gnugrep}/bin/grep -qv -e '127\.0\.0\.1' -e '\[::1\]'
|
|
}
|
|
|
|
# Stamped within the last 2 minutes: still connecting, or already restarted.
|
|
recently_started() { [ -n "$(${findutils}/bin/find "$stampdir/$1" -mmin -2 2>/dev/null)" ]; }
|
|
|
|
[ $# -ge 1 ] || usage
|
|
cmd="$1"; shift
|
|
case "$cmd" in
|
|
wait-online)
|
|
[ $# -eq 1 ] || usage
|
|
while ! network_up; do ${coreutils}/bin/sleep 5; done
|
|
: > "$stampdir/$1"
|
|
;;
|
|
check)
|
|
# Nothing can be connected while offline, and restarting wouldn't help.
|
|
network_up || exit 0
|
|
for unit; do
|
|
# Not "active" (inactive, failed, or still in wait-online): systemd's job, not ours.
|
|
${systemd}/bin/systemctl --user is-active --quiet "$unit" || continue
|
|
recently_started "$unit" && continue
|
|
connected "$unit" && continue
|
|
echo "network-watchdog: $unit is online but has no live connection -> restarting"
|
|
${systemd}/bin/systemctl --user restart "$unit" # its wait-online refreshes the stamp
|
|
done
|
|
;;
|
|
*) usage ;;
|
|
esac
|
|
''
|