nixos/hosts/common/optional/nixos-containers/metrics-server.nix

{
  pkgs,
  lib,
  configVars,
  inputs,
  ...
}: let
  containerName = "metrics-server";
  containerIp = configVars.networking.addresses.metrics-server.ip;

  notifybotJid = configVars.xmpp.notifybotJid;
  receiverJid = configVars.xmpp.personalAccount;

  dockerContainerIp = configVars.networking.addresses.docker.ip;
  smWorkerIp = configVars.networking.addresses.sm-worker.ip;
  merlinIp = configVars.networking.addresses.merlin.ip;
  bdWorker = configVars.networking.addresses.bd-worker.ip;
  pihole = configVars.networking.addresses.pihole.ip;
  bitcoinNode = configVars.networking.addresses.bitcoin-node.ip;
  postres = configVars.networking.addresses.postgres.ip;
  backupServer = configVars.networking.addresses.backup-server.ip;
  sops-nix = inputs.sops-nix;

  http_endpoints = configVars.metrics-server.blackbox.http_endpoints;

  gatewayIp = configVars.networking.addresses.gateway.ip;
  metricsServerContainerData = configVars.locations.metricsServerContainerData;
  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
in {
  networking.nat.enable = true;
  networking.nat.internalInterfaces = ["ve-+"];
  networking.nat.externalInterface = "br0";

  services.restic.backups = {
    daily = {
      paths = [
        metricsServerContainerData
      ];
    };
  };

  environment.persistence."/persist" = {
    hideMounts = true;
    directories = [
      "/var/lib/nixos-containers/${containerName}"
    ];
  };

  containers."${containerName}" = {
    enableTun = true;
    autoStart = true;
    privateNetwork = true;
    hostBridge = "br0";
    nixpkgs = pkgs.path;
    bindMounts = {
      "/var/lib/" = {
        hostPath = metricsServerContainerData;
        isReadOnly = false;
      };
      "/etc/ssh/ssh_host_ed25519_key" = {
        hostPath = "/etc/ssh/ssh_host_ed25519_key";
        isReadOnly = true;
      };
    };

    config = {
      pkgs,
      lib,
      config,
      ...
    }: let
      secretsDirectory = builtins.toString inputs.nix-secrets;
      secretsFile = "${secretsDirectory}/secrets.yaml";
    in {
      networking = {
        defaultGateway = "${gatewayIp}";
        interfaces.eth0.ipv4.addresses = [
          {
            "address" = "${containerIp}";
            "prefixLength" = 24;
          }
        ];
        firewall = {
          enable = true;
          allowedTCPPorts = [
            config.services.prometheus.port
            config.services.grafana.port
            config.services.prometheus.exporters.blackbox.port
            9199 #xmpp listen port
          ];
        };
        useHostResolvConf = lib.mkForce false;
      };

      sops = {
        defaultSopsFile = "${secretsFile}";
        validateSopsFiles = false;

        age = {
          sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"];
        };
        secrets = {
          "software/restic-passphrase" = {};
          "software/restic-exporter-credentials" = {};
          "comms/xmpp/notifybot/password" = {
            mode = "0644";
          };
        };
      };

      services.resolved.enable = true;

      imports = [
        sops-nix.nixosModules.sops
      ];

      environment.systemPackages = [
        pkgs.vim
        pkgs.git
      ];

      services.grafana = {
        enable = true;
        settings.server = {
          http_port = 2342;
          http_addr = "0.0.0.0";
        };
      };

      # main prometheus service
      services.prometheus = {
        enable = true;
        webExternalUrl = "http://${containerIp}:9001";
        port = 9001;
        alertmanagers = [
          {
            scheme = "http";
            path_prefix = "/";
            static_configs = [
              {
                targets = [
                  "0.0.0.0:9093"
                ];
              }
            ];
          }
        ];
        ruleFiles = [
          "${pkgs.writeText
            "alert_rule.yml"
            ''
              groups:
              - name: blackbox_alert
                rules:
                - alert: EndpointDown
                  expr: probe_success{job="blackbox"} == 0
                  for: 1m
                  labels:
                    severity: critical
                  annotations:
                    summary: "Endpoint {{ $labels.instance }} down"
                    description: "An endpoint has been down for more than 1 minute."
            ''}"
        ];
        scrapeConfigs = [
          {
            job_name = "node_exporter";
            static_configs = [
              {
                targets = [
                  "${dockerContainerIp}:9100"
                  "${smWorkerIp}:9100"
                  "${merlinIp}:9100"
                  "${bdWorker}:9100"
                  "${pihole}:9100"
                  "${bitcoinNode}:9100"
                  "${postres}:9100"
                  "${backupServer}:9100"
                ];
              }
            ];
          }
          {
            job_name = "restic-exporter";
            static_configs = [
              {
                targets = [
                  "0.0.0.0:8001"
                ];
              }
            ];
          }

          {
            job_name = "blackbox";
            scrape_interval = "30s";
            scrape_timeout = "15s";
            metrics_path = "/probe";
            params.module = ["http_basic"];
            relabel_configs = [
              {
                source_labels = ["__address__"];
                target_label = "__param_target";
              }
              {
                source_labels = ["__param_target"];
                target_label = "instance";
              }
              {
                target_label = "__address__";
                replacement = "${config.services.prometheus.exporters.blackbox.listenAddress}:${toString config.services.prometheus.exporters.blackbox.port}";
              }
            ];
            static_configs = [
              {targets = http_endpoints;}
            ];
          }
        ];
      };

      # setup alertmanager
      services.prometheus.xmpp-alerts = {
        enable = true;
        settings = {
          jid = notifybotJid;
          password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}";
          to_jid = receiverJid;
          listen_address = "0.0.0.0";
          listen_port = 9199;
        };
      };
      services.prometheus.alertmanager = {
        webExternalUrl = "http://${containerIp}:9093";
        enable = true;
        openFirewall = true;
        port = 9093;
        configText = ''
          global:
            resolve_timeout: 1m

          route:
            group_by: ['...']
            repeat_interval: 1h
            receiver: 'xmpp-alerts'

          receivers:
          - name: 'xmpp-alerts'
            webhook_configs:
            - url: 'http://0.0.0.0:9199/alert'
        '';
      };

      # prometheus exporters
      services.prometheus.exporters = {
        blackbox = {
          enable = true;
          configFile = pkgs.writeText "blackbox-conf.yaml" ''
            modules:
              http_basic:
                prober: http
                timeout: 5s
                http:
                  preferred_ip_protocol: ip4
                  valid_http_versions: ["HTTP/1.1", "HTTP/2"]
                  method: GET
                  # fail_if_ssl: false
                  # fail_if_not_ssl: true
                  # tls_config:
                  #   insecure_skip_verify: true
              tcp_connect:
                prober: tcp
                tcp:
                  preferred_ip_protocol: ip4

          '';
        };
        node = {
          enable = true;
          enabledCollectors = ["systemd"];
          port = 9002;
        };
        restic = {
          enable = true;
          repository = "";
          environmentFile = config.sops.secrets."software/restic-exporter-credentials".path;
          passwordFile = config.sops.secrets."software/restic-passphrase".path;
          refreshInterval = 10800; # refresh every 3 hours
          port = 8001;
        };
      };

      services.openssh = {
        enable = true;
        settings.PasswordAuthentication = false;
      };

      users.users = {
        root = {
          openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
        };
      };

      system.stateVersion = "24.05";
    };
  };
}