more telegraf

modules/nixos/tools/telegraf/default.nix

@@ -28,11 +28,11 @@ let
   '';
   baseConfig = {
     agent = {
-      interval = "10s";
+      interval = cfg.interval;
       round_interval = true;
       metric_batch_size = 1000;
       metric_buffer_limit = 10000;
-      flush_interval = "10s";
+      flush_interval = cfg.interval;
       hostname = config.syscfg.hostname;
       omit_hostname = false;
     };
@@ -258,6 +258,14 @@ let
         }
       ];
     })
+    (lib.mkIf (hasCollector "systemd_units") {
+      inputs.systemd_units = {
+        pattern = "*";
+        unittype = "service";
+        details = true;
+        timeout = "5s";
+      };
+    })
     (lib.mkIf (hasCollector "ping") {
       inputs.ping = {
         urls = [ "1.1.1.1" ];
@@ -278,11 +286,39 @@ let
         memory_saving_mode = true;
       };
     })
-    (lib.mkIf (hasCollector "gpu") {
+    (lib.mkIf (hasCollector "gpu" || hasCollector "nix") {
-      inputs.exec = [{
+      inputs.exec =
+        lib.optionals (hasCollector "gpu") [{
           commands = [ amdgpuMetricsScript ];
           timeout = "5s";
           data_format = "influx";
+        }]
+        ++ lib.optionals (hasCollector "nix") [{
+          commands = [
+            (pkgs.writeShellScript "telegraf-nix-metrics" ''
+              set -euo pipefail
+
+              current="$(${lib.getExe pkgs.nixos-rebuild} list-generations | ${lib.getExe pkgs.gawk} '$NF == "True" {print $1 "|" $2 " " $3; exit}')"
+              [ -n "$current" ]
+
+              generation="''${current%%|*}"
+              build_datetime="''${current#*|}"
+              build_timestamp="$(${lib.getExe' pkgs.coreutils "date"} -d "$build_datetime" +%s)"
+              now="$(${lib.getExe' pkgs.coreutils "date"} +%s)"
+              store_bytes="$(${lib.getExe' pkgs.coreutils "du"} -sb /nix/store | ${lib.getExe pkgs.gawk} '{print $1}')"
+
+              printf 'nix generation=%si,configured_packages=%si,store_bytes=%si,build_timestamp=%si,seconds_since_build=%si,build_datetime="%s"\n' \
+                "$generation" \
+                ${toString (builtins.length config.environment.systemPackages)} \
+                "$store_bytes" \
+                "$build_timestamp" \
+                "$((now - build_timestamp))" \
+                "$build_datetime"
+            '')
+          ];
+          interval = "1h";
+          timeout = "30s";
+          data_format = "influx";
         }];
     })
   ];

modules/shared/syscfg/monitoring.nix

@@ -6,6 +6,11 @@ with lib; {
       default = false;
     };
 
+    interval = mkOption {
+      type = types.str;
+      default = "10s";
+    };
+
     collectors = mkOption {
       type = types.listOf (types.enum [
         "cpu"
@@ -22,6 +27,8 @@ with lib; {
         "mdstat"
         "smart"
         "docker"
+        "systemd_units"
+        "nix"
         "ping"
         "internet_speed"
         "gpu"

systems/sandbox/cfg.nix

@@ -90,6 +90,7 @@
     };
     monitoring.telegraf = {
       enable = true;
+      interval = "15s";
       collectors = [
         "cpu"
         "mem"
@@ -102,6 +103,8 @@
         #"netstat"
         #"processes"
         "docker"
+        "systemd_units"
+        "nix"
         "ping"
         "internet_speed"
       ];