diff --git a/flake.lock b/flake.lock index 47d0560..9786635 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737494768, - "narHash": "sha256-a1Wy0e7E6xpPgF2q3ysBMKN+0qoPZ0umdaNYXO+MP+4=", + "lastModified": 1737643624, + "narHash": "sha256-RAnbZSi2yagPCpNcm3U3wA6FAzbhGUi9ifvnu6Du3Rs=", "ref": "refs/heads/master", - "rev": "512145a45785b730dab4cecc441680c7dd3eca5e", - "revCount": 247, + "rev": "5260822187ce58af680e5aceba8fb01f10415def", + "revCount": 248, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/backup-server.nix b/hosts/common/optional/nixos-containers/backup-server.nix index d0cbaee..3a7a45f 100644 --- a/hosts/common/optional/nixos-containers/backup-server.nix +++ b/hosts/common/optional/nixos-containers/backup-server.nix @@ -65,6 +65,16 @@ in { pkgs.apacheHttpd ]; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/common/optional/nixos-containers/baseddata-worker.nix b/hosts/common/optional/nixos-containers/baseddata-worker.nix index f6f5038..2cd11ed 100644 --- a/hosts/common/optional/nixos-containers/baseddata-worker.nix +++ b/hosts/common/optional/nixos-containers/baseddata-worker.nix @@ -295,6 +295,16 @@ in { }; }; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index e777228..4b2ac05 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -6,8 +6,18 @@ }: let containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + dockerContainerIp = configVars.networking.addresses.docker.ip; - semitaIp = configVars.networking.addresses.semita.ip; + smWorkerIp = configVars.networking.addresses.sm-worker.ip; + merlinIp = configVars.networking.addresses.merlin.ip; + bdWorker = configVars.networking.addresses.bd-worker.ip; + pihole = configVars.networking.addresses.pihole.ip; + bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; + postres = configVars.networking.addresses.postgres.ip; + backupServer = configVars.networking.addresses.backup-server.ip; + + http_endpoints = configVars.metrics-server.blackbox.http_endpoints; + gatewayIp = configVars.networking.addresses.gateway.ip; metricsServerContainerData = configVars.locations.metricsServerContainerData; pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; @@ -63,6 +73,7 @@ in { allowedTCPPorts = [ config.services.prometheus.port config.services.grafana.port + config.services.prometheus.exporters.blackbox.port ]; }; useHostResolvConf = lib.mkForce false; @@ -88,22 +99,77 @@ in { { targets = [ "${dockerContainerIp}:9100" - "${semitaIp}:9100" + "${smWorkerIp}:9100" + "${merlinIp}:9100" + "${bdWorker}:9100" + "${pihole}:9100" + "${bitcoinNode}:9100" + "${postres}:9100" + "${backupServer}:9100" ]; } ]; } + + { + job_name = "blackbox"; + scrape_interval = "30s"; + scrape_timeout = "15s"; + metrics_path = "/probe"; + params.module = ["http_basic"]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "${config.services.prometheus.exporters.blackbox.listenAddress}:${toString config.services.prometheus.exporters.blackbox.port}"; + } + ]; + static_configs = [ + {targets = http_endpoints;} + ]; + } ]; }; services.grafana = { enable = true; - port = 2342; - addr = "0.0.0.0"; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; }; services.prometheus = { exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + fail_if_ssl: false + fail_if_not_ssl: true + tls_config: + insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; node = { enable = true; enabledCollectors = ["systemd"]; diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 93f16f1..0bfd532 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -186,6 +186,16 @@ in { lnd.public = true; }; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/common/optional/nixos-containers/pihole.nix b/hosts/common/optional/nixos-containers/pihole.nix index 96c5ed2..1f648fd 100644 --- a/hosts/common/optional/nixos-containers/pihole.nix +++ b/hosts/common/optional/nixos-containers/pihole.nix @@ -94,6 +94,16 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/common/optional/nixos-containers/postgres.nix b/hosts/common/optional/nixos-containers/postgres.nix index 068563c..a823087 100644 --- a/hosts/common/optional/nixos-containers/postgres.nix +++ b/hosts/common/optional/nixos-containers/postgres.nix @@ -123,6 +123,16 @@ in { # EOF # ''; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/common/optional/nixos-containers/semitamaps-worker.nix b/hosts/common/optional/nixos-containers/semitamaps-worker.nix index 23d7ab8..9270136 100644 --- a/hosts/common/optional/nixos-containers/semitamaps-worker.nix +++ b/hosts/common/optional/nixos-containers/semitamaps-worker.nix @@ -137,6 +137,16 @@ in { }; }; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + services.openssh = { enable = true; settings.PasswordAuthentication = false; diff --git a/hosts/merlin/default.nix b/hosts/merlin/default.nix index fa692fa..dae6cca 100644 --- a/hosts/merlin/default.nix +++ b/hosts/merlin/default.nix @@ -156,6 +156,16 @@ in { user = "admin"; }; + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + openFirewall = true; + }; + }; + }; + boot.supportedFilesystems = ["zfs"]; boot.zfs.forceImportRoot = false; networking.hostId = "18aec5d7"; diff --git a/vars/default.nix b/vars/default.nix index 3acbb18..ab16e35 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -3,6 +3,7 @@ inherit (inputs.nix-secrets) networking email + metrics-server ; locations = { mediaDataMountPoint = "/media/media";