nixos/hosts/common/optional/nixos-containers/baseddata-worker.nix

{
  lib,
  pkgs,
  inputs,
  configVars,
  ...
}: let
  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
  containerName = "bd-worker";
  sops-nix = inputs.sops-nix;
  baseddataData = configVars.locations.baseddataData;
in {
  environment.persistence."/persist" = {
    hideMounts = true;
    directories = [
      "/var/lib/nixos-containers/${containerName}"
    ];
  };

  networking.nat.enable = true;
  networking.nat.internalInterfaces = ["ve-+"];
  networking.nat.externalInterface = "br0";

  containers.${containerName} = {
    autoStart = true;
    privateNetwork = true;
    hostBridge = "br0";
    nixpkgs = pkgs.path;

    allowedDevices = [
      {
        node = "/dev/net/tun";
        modifier = "rw";
      }
    ];

    bindMounts = {
      "/etc/ssh/ssh_host_ed25519_key" = {
        hostPath = "/etc/ssh/ssh_host_ed25519_key";
        isReadOnly = true;
      };
      "/data/baseddata-data" = {
        hostPath = baseddataData;
        isReadOnly = false;
      };
    };

    config = {
      pkgs,
      lib,
      config,
      ...
    }: let
      configVars = import ../../../../vars {inherit inputs lib;};
      secretsDirectory = builtins.toString inputs.nix-secrets;
      secretsFile = "${secretsDirectory}/secrets.yaml";

      # define ip addresses
      containerIp = configVars.networking.addresses.bd-worker.ip;
      gatewayIp = configVars.networking.addresses.gateway.ip;
      postgresIp = configVars.networking.addresses.postgres.ip;
      postgresRemoteIp = configVars.networking.addresses.postgres-remote.ip;
      postgresPort = toString configVars.networking.addresses.postgres.port;
      bitcoindIp = configVars.networking.addresses.bitcoin-node.ip;
      bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port;

      # define secret paths
      notifybotUsername = config.sops.secrets."comms/xmpp/notifybot/username".path;
      notifybotPassword = config.sops.secrets."comms/xmpp/notifybot/password".path;
      recipientUsername = config.sops.secrets."comms/xmpp/mrsu/username".path;
      postgresUser = config.sops.secrets."software/postgres/baseddata/user_username".path;
      postgresPassword = config.sops.secrets."software/postgres/baseddata/user_password".path;
      bitcoindRPCUsername = config.sops.secrets."software/bitcoind/username".path;
      bitcoindRPCPassword = config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path;
      dbtProfiles = config.sops.templates."profiles.yml".path;
      pgsyncConfig = config.sops.templates."pgsync.yml".path;
      baseddataEnv = "dev";
    in {
      networking = {
        defaultGateway = "${gatewayIp}";
        interfaces.eth0.ipv4.addresses = [
          {
            "address" = "${containerIp}";
            "prefixLength" = 24;
          }
        ];
        firewall = {
          enable = true;
          allowedTCPPorts = [
            4200
          ];
        };
        useHostResolvConf = lib.mkForce false;
      };

      sops = {
        defaultSopsFile = "${secretsFile}";
        validateSopsFiles = false;

        age = {
          sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"];
        };
      };

      sops.secrets = {
        "ssh_keys/baseddata-models-access/id_ed25519" = {
          mode = "0400";
          path = "/root/.ssh/id_ed25519";
        };
        "comms/xmpp/notifybot/username" = {};
        "comms/xmpp/notifybot/password" = {};
        "comms/xmpp/mrsu/username" = {};
        "software/postgres/baseddata/user_password" = {};
        "software/postgres/baseddata/user_username" = {};
        "software/bitcoind/username" = {};
        "software/bitcoind/bitcoin-rpcpassword-public" = {};
      };

      sops.templates."profiles.yml" = {
        mode = "0600";
        path = "/root/.dbt/profiles.yml";
        content = ''
          baseddata:
            target: prod
            outputs:
              prod:
                dbname: baseddata
                host: ${postgresIp}
                pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
                port: 5432
                schema: models
                threads: 6
                type: postgres
                user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}

        '';
      };

      sops.templates."pgsync.yml" = {
        mode = "0600";
        path = "/root/.pgsync.yml";
        content = ''
            from: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresIp}/baseddata
            to: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresRemoteIp}/baseddata
            schemas:
              - models_final
            to_safe: true

        '';
      };

      imports = [
        sops-nix.nixosModules.sops
      ];

      services.resolved.enable = true;
      services.tailscale.enable = true;

      environment.systemPackages = [
        pkgs.vim
        pkgs.git
        pkgs.python311
        pkgs.poetry
        pkgs.aria2
        pkgs.osmctools
        pkgs.osmium-tool
        pkgs.osm2pgsql
        pkgs.pgsync
        pkgs.postgresql_16
        pkgs.htop
      ];

      environment.variables = {
        BASEDDATA_ENVIRONMENT = "dev";
        NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
        NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
        LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
      };

      systemd.services.baseddata-deploy-service = {
        wantedBy = ["multi-user.target"];
        after = ["network.target"];
        description = "Initiates deployment of application and builds python environment using Poetry";
        environment = {
          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
        };
        serviceConfig = {
          ExecStart = pkgs.writeShellScript "baseddata-deploy-service" ''
            GITCMD="${pkgs.openssh}/bin/ssh -i /root/.ssh/id_ed25519"
            if [ ! -d "/srv/baseddata-models" ]; then
              GIT_SSH_COMMAND=$GITCMD  ${pkgs.git}/bin/git clone --branch $BASEDDATA_ENVIRONMENT git@git.bitlab21.com:sam/baseddata-models.git /srv/baseddata-models
            else
              cd /srv/baseddata-models
              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git stash --include-untracked
              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git pull
            fi

            cd /srv/baseddata-models
            mkdir .venv
            ${pkgs.poetry}/bin/poetry lock --no-update
            ${pkgs.poetry}/bin/poetry install
          '';
          Restart = "on-failure";
        };
      };

      systemd.services.baseddata-prefect-server = {
        wantedBy = ["multi-user.target"];
        after = ["baseddata-deploy-service.target"];
        description = "Initates the Prefect server";
        environment = {
          NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
          NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
          LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
          PREFECT_API_URL = "http://${containerIp}:4200/api";
          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
        };
        serviceConfig = {
          WorkingDirectory = "/srv/baseddata-models";
          ExecStart = pkgs.writeShellScript "baseddata-prefect-server" ''

            # run prefect server
            .venv/bin/prefect server start --host ${containerIp}

          '';
          Restart = "on-failure";
        };
      };

      systemd.services.baseddata-serve-flows = {
        wantedBy = ["multi-user.target"];
        after = ["baseddata-prefect-server.target"];
        description = "Serves the Prefect flows";
        path = ["/run/current-system/sw" "/srv/baseddata-models/.venv"];
        environment = {
          PREFECT_API_URL = "http://${containerIp}:4200/api";
          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
          NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
          NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
          LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
        };
        serviceConfig = {
          WorkingDirectory = "/srv/baseddata-models";
          ExecStartPre = "${pkgs.coreutils}/bin/timeout 120 ${pkgs.bash}/bin/bash -c 'until ${pkgs.netcat-openbsd}/bin/nc -z ${containerIp} 4200; do sleep 3; done'";
          ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''

            # set prefect environment variables
            .venv/bin/prefect variable set "xmpp_jid" $(cat ${notifybotUsername}) --overwrite
            .venv/bin/prefect variable set "xmpp_password" $(cat ${notifybotPassword}) --overwrite
            .venv/bin/prefect variable set "xmpp_recipient" $(cat ${recipientUsername}) --overwrite

            .venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite
            .venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite
            .venv/bin/prefect variable set "postgres_user" $(cat ${postgresUser}) --overwrite
            .venv/bin/prefect variable set "postgres_pwd" $(cat ${postgresPassword}) --overwrite
            .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
            .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite

            .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat ${bitcoindRPCPassword}) --overwrite
            .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat ${bitcoindRPCUsername}) --overwrite
            .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite
            .venv/bin/prefect variable set "bitcoind_port" ${bitcoindPort} --overwrite

            .venv/bin/prefect variable set "osm_dir" "/data/baseddata-data/osm" --overwrite
            .venv/bin/prefect variable set "wdpa_dir" "/data/baseddata-data/wdpa" --overwrite

            .venv/bin/prefect variable set "dbt_profiles_dir" $(dirname ${dbtProfiles}) --overwrite
            .venv/bin/prefect variable set "pgsync_config" ${pgsyncConfig} --overwrite


            # serve flows
            .venv/bin/python automation/flows/serve-flows.py
          '';
          Restart = "on-failure";
        };
      };

      programs.nix-ld.enable = true;
      programs.nix-ld.libraries = with pkgs; [
        zlib
        libgcc
      ];

      programs.ssh.knownHosts = {
        "git.bitlab21.com" = {
          publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALNd2BGf64heYjWT9yt0fVmngepiHRIMsL7au/MRteg";
        };
      };

      services.openssh = {
        enable = true;
        settings.PasswordAuthentication = false;
      };

      users.users.root = {
        openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
      };

      system.stateVersion = "24.05";
    };
  };
}