nixos/hosts/common/optional/nixos-containers/metrics-server.nix

{
  pkgs,
  lib,
  configVars,
  ...
}: let
  containerName = "metrics-server";
  containerIp = configVars.networking.addresses.metrics-server.ip;

  dockerContainerIp = configVars.networking.addresses.docker.ip;
  smWorkerIp = configVars.networking.addresses.sm-worker.ip;
  merlinIp = configVars.networking.addresses.merlin.ip;
  bdWorker = configVars.networking.addresses.bd-worker.ip;
  pihole = configVars.networking.addresses.pihole.ip;
  bitcoinNode = configVars.networking.addresses.bitcoin-node.ip;
  postres = configVars.networking.addresses.postgres.ip;
  backupServer = configVars.networking.addresses.backup-server.ip;

  http_endpoints = configVars.metrics-server.blackbox.http_endpoints;

  gatewayIp = configVars.networking.addresses.gateway.ip;
  metricsServerContainerData = configVars.locations.metricsServerContainerData;
  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
in {
  networking.nat.enable = true;
  networking.nat.internalInterfaces = ["ve-+"];
  networking.nat.externalInterface = "br0";

  services.restic.backups = {
    daily = {
      paths = [
        metricsServerContainerData
      ];
    };
  };

  environment.persistence."/persist" = {
    hideMounts = true;
    directories = [
      "/var/lib/nixos-containers/${containerName}"
    ];
  };

  containers."${containerName}" = {
    enableTun = true;
    autoStart = true;
    privateNetwork = true;
    hostBridge = "br0";
    nixpkgs = pkgs.path;
    bindMounts = {
      "/var/lib/" = {
        hostPath = metricsServerContainerData;
        isReadOnly = false;
      };
    };

    config = {
      pkgs,
      lib,
      config,
      ...
    }: {
      networking = {
        defaultGateway = "${gatewayIp}";
        interfaces.eth0.ipv4.addresses = [
          {
            "address" = "${containerIp}";
            "prefixLength" = 24;
          }
        ];
        firewall = {
          enable = true;
          allowedTCPPorts = [
            config.services.prometheus.port
            config.services.grafana.port
            config.services.prometheus.exporters.blackbox.port
          ];
        };
        useHostResolvConf = lib.mkForce false;
      };

      services.resolved.enable = true;

      imports = [
      ];

      environment.systemPackages = [
        pkgs.vim
        pkgs.git
      ];

      services.prometheus = {
        enable = true;
        port = 9001;
        scrapeConfigs = [
          {
            job_name = "node_exporter";
            static_configs = [
              {
                targets = [
                  "${dockerContainerIp}:9100"
                  "${smWorkerIp}:9100"
                  "${merlinIp}:9100"
                  "${bdWorker}:9100"
                  "${pihole}:9100"
                  "${bitcoinNode}:9100"
                  "${postres}:9100"
                  "${backupServer}:9100"
                ];
              }
            ];
          }

          {
            job_name = "blackbox";
            scrape_interval = "30s";
            scrape_timeout = "15s";
            metrics_path = "/probe";
            params.module = ["http_basic"];
            relabel_configs = [
              {
                source_labels = ["__address__"];
                target_label = "__param_target";
              }
              {
                source_labels = ["__param_target"];
                target_label = "instance";
              }
              {
                target_label = "__address__";
                replacement = "${config.services.prometheus.exporters.blackbox.listenAddress}:${toString config.services.prometheus.exporters.blackbox.port}";
              }
            ];
            static_configs = [
              {targets = http_endpoints;}
            ];
          }
        ];
      };

      services.grafana = {
        enable = true;
        settings.server = {
          http_port = 2342;
          http_addr = "0.0.0.0";
        };
      };

      services.prometheus = {
        exporters = {
          blackbox = {
            enable = true;
            configFile = pkgs.writeText "blackbox-conf.yaml" ''
              modules:
                http_basic:
                  prober: http
                  timeout: 5s
                  http:
                    preferred_ip_protocol: ip4
                    valid_http_versions: ["HTTP/1.1", "HTTP/2"]
                    method: GET
                    fail_if_ssl: false
                    fail_if_not_ssl: true
                    tls_config:
                      insecure_skip_verify: true
                tcp_connect:
                  prober: tcp
                  tcp:
                    preferred_ip_protocol: ip4

            '';
          };
          node = {
            enable = true;
            enabledCollectors = ["systemd"];
            port = 9002;
          };
        };
      };

      services.openssh = {
        enable = true;
        settings.PasswordAuthentication = false;
      };

      users.users = {
        root = {
          openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
        };
      };

      system.stateVersion = "24.05";
    };
  };
}