add blackbox exporter

flake.lock

@@ -539,11 +539,11 @@
     },
     "nix-secrets": {
       "locked": {
-        "lastModified": 1737494768,
+        "lastModified": 1737643624,
-        "narHash": "sha256-a1Wy0e7E6xpPgF2q3ysBMKN+0qoPZ0umdaNYXO+MP+4=",
+        "narHash": "sha256-RAnbZSi2yagPCpNcm3U3wA6FAzbhGUi9ifvnu6Du3Rs=",
         "ref": "refs/heads/master",
-        "rev": "512145a45785b730dab4cecc441680c7dd3eca5e",
+        "rev": "5260822187ce58af680e5aceba8fb01f10415def",
-        "revCount": 247,
+        "revCount": 248,
         "type": "git",
         "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
       },

hosts/common/optional/nixos-containers/backup-server.nix

@@ -65,6 +65,16 @@ in {
         pkgs.apacheHttpd
       ];
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/common/optional/nixos-containers/baseddata-worker.nix

@@ -295,6 +295,16 @@ in {
         };
       };
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/common/optional/nixos-containers/metrics-server.nix

@@ -6,8 +6,18 @@
 }: let
   containerName = "metrics-server";
   containerIp = configVars.networking.addresses.metrics-server.ip;
+
   dockerContainerIp = configVars.networking.addresses.docker.ip;
-  semitaIp = configVars.networking.addresses.semita.ip;
+  smWorkerIp = configVars.networking.addresses.sm-worker.ip;
+  merlinIp = configVars.networking.addresses.merlin.ip;
+  bdWorker = configVars.networking.addresses.bd-worker.ip;
+  pihole = configVars.networking.addresses.pihole.ip;
+  bitcoinNode = configVars.networking.addresses.bitcoin-node.ip;
+  postres = configVars.networking.addresses.postgres.ip;
+  backupServer = configVars.networking.addresses.backup-server.ip;
+
+  http_endpoints = configVars.metrics-server.blackbox.http_endpoints;
+
   gatewayIp = configVars.networking.addresses.gateway.ip;
   metricsServerContainerData = configVars.locations.metricsServerContainerData;
   pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
@@ -63,6 +73,7 @@ in {
           allowedTCPPorts = [
             config.services.prometheus.port
             config.services.grafana.port
+            config.services.prometheus.exporters.blackbox.port
           ];
         };
         useHostResolvConf = lib.mkForce false;
@@ -88,22 +99,77 @@ in {
               {
                 targets = [
                   "${dockerContainerIp}:9100"
-                  "${semitaIp}:9100"
+                  "${smWorkerIp}:9100"
+                  "${merlinIp}:9100"
+                  "${bdWorker}:9100"
+                  "${pihole}:9100"
+                  "${bitcoinNode}:9100"
+                  "${postres}:9100"
+                  "${backupServer}:9100"
                 ];
               }
             ];
           }
+
+          {
+            job_name = "blackbox";
+            scrape_interval = "30s";
+            scrape_timeout = "15s";
+            metrics_path = "/probe";
+            params.module = ["http_basic"];
+            relabel_configs = [
+              {
+                source_labels = ["__address__"];
+                target_label = "__param_target";
+              }
+              {
+                source_labels = ["__param_target"];
+                target_label = "instance";
+              }
+              {
+                target_label = "__address__";
+                replacement = "${config.services.prometheus.exporters.blackbox.listenAddress}:${toString config.services.prometheus.exporters.blackbox.port}";
+              }
+            ];
+            static_configs = [
+              {targets = http_endpoints;}
+            ];
+          }
         ];
       };
 
       services.grafana = {
         enable = true;
-        port = 2342;
+        settings.server = {
-        addr = "0.0.0.0";
+          http_port = 2342;
+          http_addr = "0.0.0.0";
+        };
       };
 
       services.prometheus = {
         exporters = {
+          blackbox = {
+            enable = true;
+            configFile = pkgs.writeText "blackbox-conf.yaml" ''
+              modules:
+                http_basic:
+                  prober: http
+                  timeout: 5s
+                  http:
+                    preferred_ip_protocol: ip4
+                    valid_http_versions: ["HTTP/1.1", "HTTP/2"]
+                    method: GET
+                    fail_if_ssl: false
+                    fail_if_not_ssl: true
+                    tls_config:
+                      insecure_skip_verify: true
+                tcp_connect:
+                  prober: tcp
+                  tcp:
+                    preferred_ip_protocol: ip4
+
+            '';
+          };
           node = {
             enable = true;
             enabledCollectors = ["systemd"];

hosts/common/optional/nixos-containers/nix-bitcoin.nix

@@ -186,6 +186,16 @@ in {
         lnd.public = true;
       };
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/common/optional/nixos-containers/pihole.nix

@@ -94,6 +94,16 @@ in {
 
       networking.firewall.interfaces."podman+".allowedUDPPorts = [53];
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/common/optional/nixos-containers/postgres.nix

@@ -123,6 +123,16 @@ in {
       #   EOF
       # '';
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/common/optional/nixos-containers/semitamaps-worker.nix

@@ -137,6 +137,16 @@ in {
         };
       };
 
+      services.prometheus = {
+        exporters = {
+          node = {
+            enable = true;
+            enabledCollectors = ["systemd"];
+            openFirewall = true;
+          };
+        };
+      };
+
       services.openssh = {
         enable = true;
         settings.PasswordAuthentication = false;

hosts/merlin/default.nix

@@ -156,6 +156,16 @@ in {
     user = "admin";
   };
 
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = ["systemd"];
+        openFirewall = true;
+      };
+    };
+  };
+
   boot.supportedFilesystems = ["zfs"];
   boot.zfs.forceImportRoot = false;
   networking.hostId = "18aec5d7";

vars/default.nix

@@ -3,6 +3,7 @@
   inherit (inputs.nix-secrets)
     networking
     email
+    metrics-server
     ;
   locations = {
     mediaDataMountPoint = "/media/media";