setup prometheus alertmanager

flake.lock

@@ -539,11 +539,11 @@
     },
     "nix-secrets": {
       "locked": {
-        "lastModified": 1738358831,
+        "lastModified": 1738685297,
-        "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=",
+        "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=",
         "ref": "refs/heads/master",
-        "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45",
+        "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9",
-        "revCount": 268,
+        "revCount": 269,
         "type": "git",
         "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
       },

hosts/common/optional/nixos-containers/metrics-server.nix

@@ -8,6 +8,9 @@
   containerName = "metrics-server";
   containerIp = configVars.networking.addresses.metrics-server.ip;
 
+  notifybotJid = configVars.xmpp.notifybotJid;
+  receiverJid = configVars.xmpp.personalAccount;
+
   dockerContainerIp = configVars.networking.addresses.docker.ip;
   smWorkerIp = configVars.networking.addresses.sm-worker.ip;
   merlinIp = configVars.networking.addresses.merlin.ip;
@@ -83,6 +86,7 @@ in {
             config.services.prometheus.port
             config.services.grafana.port
             config.services.prometheus.exporters.blackbox.port
+            9199 #xmpp listen port
           ];
         };
         useHostResolvConf = lib.mkForce false;
@@ -98,6 +102,9 @@ in {
         secrets = {
           "software/restic-passphrase" = {};
           "software/restic-exporter-credentials" = {};
+          "comms/xmpp/notifybot/password" = {
+            mode = "0644";
+          };
         };
       };
 
@@ -112,9 +119,48 @@ in {
         pkgs.git
       ];
 
+      services.grafana = {
+        enable = true;
+        settings.server = {
+          http_port = 2342;
+          http_addr = "0.0.0.0";
+        };
+      };
+
+      # main prometheus service
       services.prometheus = {
         enable = true;
         port = 9001;
+        alertmanagers = [
+          {
+            scheme = "http";
+            path_prefix = "/";
+            static_configs = [
+              {
+                targets = [
+                  "0.0.0.0:9093"
+                ];
+              }
+            ];
+          }
+        ];
+        ruleFiles = [
+          "${pkgs.writeText
+            "alert_rule.yml"
+            ''
+              groups:
+              - name: blackbox_alert
+                rules:
+                - alert: EndpointDown
+                  expr: probe_success{job="blackbox"} == 0
+                  for: 1m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Endpoint {{ $labels.instance }} down"
+                    description: "An endpoint has been down for more than 1 minute."
+            ''}"
+        ];
         scrapeConfigs = [
           {
             job_name = "node_exporter";
@@ -171,16 +217,40 @@ in {
         ];
       };
 
-      services.grafana = {
+      # setup alertmanager
+      services.prometheus.xmpp-alerts = {
         enable = true;
-        settings.server = {
+        settings = {
-          http_port = 2342;
+          jid = notifybotJid;
-          http_addr = "0.0.0.0";
+          password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}";
+          to_jid = receiverJid;
+          listen_address = "0.0.0.0";
+          listen_port = 9199;
         };
       };
+      services.prometheus.alertmanager = {
+        webExternalUrl = containerIp;
+        enable = true;
+        openFirewall = true;
+        port = 9093;
+        configText = ''
+          global:
+            resolve_timeout: 1m
+
+          route:
+            group_by: ['...']
+            repeat_interval: 1h
+            receiver: 'xmpp-alerts'
+
+          receivers:
+          - name: 'xmpp-alerts'
+            webhook_configs:
+            - url: 'http://0.0.0.0:9199/alert'
+        '';
+      };
 
-      services.prometheus = {
+      # prometheus exporters
-        exporters = {
+      services.prometheus.exporters = {
         blackbox = {
           enable = true;
           configFile = pkgs.writeText "blackbox-conf.yaml" ''
@@ -217,7 +287,6 @@ in {
           port = 8001;
         };
       };
-      };
 
       services.openssh = {
         enable = true;

vars/default.nix

@@ -4,6 +4,7 @@
     networking
     email
     metrics-server
+    xmpp
     ;
   locations = {
     mediaDataMountPoint = "/media/media";