small changes and updates

flake.lock

@@ -427,11 +427,11 @@
     },
     "nix-secrets": {
       "locked": {
-        "lastModified": 1728426148,
-        "narHash": "sha256-NN1eE1bEnUcmd0u7EzmzrbsuyY2Nfwqh+kDfKEPrmRM=",
+        "lastModified": 1728676183,
+        "narHash": "sha256-zs7GE1hB5Jyjf+me5V6g9OkK+cHqRJeDe1BuTYHLVFs=",
         "ref": "refs/heads/master",
-        "rev": "6152199b861b3534ab5075de9f9cdc2590150fe4",
-        "revCount": 179,
+        "rev": "25feeaad694df4e737e365fcee5f0783be94d3e9",
+        "revCount": 185,
         "type": "git",
         "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
       },

flake.nix

@@ -80,7 +80,7 @@
     in
     {
       packages = forAllSystems (system: import ./pkgs nixpkgs.legacyPackages.${system});
-      formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.alejandra);
+#formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.alejandra);
       overlays = import ./overlays { inherit inputs; };
       nixosModules = import ./modules/nixos;
       homeManagerModules = import ./modules/home-manager;

hosts/common/optional/nixos-containers/baseddata-worker.nix

@@ -0,0 +1,251 @@
+{
+  lib,
+  pkgs,
+  configVars,
+  inputs,
+  config,
+  ...
+}: let
+  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
+  containerName = "bd-worker";
+  containerIp = configVars.networking.addresses.bd-worker.ip;
+  mongodbIp = configVars.networking.addresses.mongodb.ip;
+  mongodbPort = toString configVars.networking.addresses.mongodb.port;
+  gatewayIp = configVars.networking.addresses.gateway.ip;
+  postgresIp = configVars.networking.addresses.postgres.ip;
+  postgresPort = toString configVars.networking.addresses.postgres.port;
+
+  #secrets
+  sshKeyFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."ssh_keys/baseddata-models-access/id_ed25519".path;
+  notifybotUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/username".path;
+  notifybotPwd = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/password".path;
+  recipientUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/mrsu/username".path;
+  mongoclientAuth = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/mongodb/baseddata/auth".path;
+  mongoclientUser = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/mongodb/baseddata/username".path;
+  mongoclientPassword = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/mongodb/baseddata/password".path;
+  postgresUser = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_username".path;
+  postgresPassword = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_password".path;
+  baseddataEnv = "dev";
+in {
+  sops.secrets = {
+    "ssh_keys/baseddata-models-access/id_ed25519" = {};
+    "comms/xmpp/notifybot/username" = {};
+    "comms/xmpp/notifybot/password" = {};
+    "comms/xmpp/mrsu/username" = {};
+    "software/mongodb/baseddata/auth" = {};
+    "software/mongodb/baseddata/username" = {};
+    "software/mongodb/baseddata/password" = {};
+    "software/postgres/baseddata/user_password" = {};
+    "software/postgres/baseddata/user_username" = {};
+  };
+
+  environment.persistence."/persist" = {
+    hideMounts = true;
+    directories = [
+      "/var/lib/nixos-containers/${containerName}"
+    ];
+  };
+
+  networking.nat.enable = true;
+  networking.nat.internalInterfaces = ["ve-+"];
+  networking.nat.externalInterface = "br0";
+
+  containers.${containerName} = {
+    autoStart = true;
+    privateNetwork = true;
+    hostBridge = "br0";
+    nixpkgs = pkgs.path;
+    bindMounts = {
+      "/root/.ssh/id_ed25519" = {
+        hostPath = "${sshKeyFile}";
+        isReadOnly = true;
+      };
+      "/run/secrets/notifybotUsername" = {
+        hostPath = "${notifybotUsername}";
+        isReadOnly = true;
+      };
+      "/run/secrets/notifybotPassword" = {
+        hostPath = "${notifybotPwd}";
+        isReadOnly = true;
+      };
+      "/run/secrets/recipientUsername" = {
+        hostPath = "${recipientUsername}";
+        isReadOnly = true;
+      };
+      "/run/secrets/mongoclientAuth" = {
+        hostPath = "${mongoclientAuth}";
+        isReadOnly = true;
+      };
+      "/run/secrets/mongoclientUser" = {
+        hostPath = "${mongoclientUser}";
+        isReadOnly = true;
+      };
+      "/run/secrets/mongoclientPassword" = {
+        hostPath = "${mongoclientPassword}";
+        isReadOnly = true;
+      };
+      "/run/secrets/postgresPassword" = {
+        hostPath = "${postgresPassword}";
+        isReadOnly = true;
+      };
+      "/run/secrets/postgresUser" = {
+        hostPath = "${postgresUser}";
+        isReadOnly = true;
+      };
+      "/media/baseddata-data" = {
+        hostPath = "/media/main-ssd/baseddata-data";
+        isReadOnly = false;
+      };
+    };
+
+    config = {
+      pkgs,
+      lib,
+      ...
+    }: {
+      networking = {
+        defaultGateway = "${gatewayIp}";
+        interfaces.eth0.ipv4.addresses = [
+          {
+            "address" = "${containerIp}";
+            "prefixLength" = 24;
+          }
+        ];
+        firewall = {
+          enable = true;
+          allowedTCPPorts = [
+            4200
+          ];
+        };
+        useHostResolvConf = lib.mkForce false;
+      };
+
+      services.resolved.enable = true;
+
+      environment.systemPackages = [
+        pkgs.vim
+        pkgs.git
+        pkgs.python311
+        pkgs.poetry
+      ];
+
+      environment.variables = {
+        BASEDDATA_ENVIRONMENT = "dev";
+        NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
+        NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
+        LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
+      };
+
+      systemd.services.baseddata-deploy-service = {
+        wantedBy = ["multi-user.target"];
+        after = ["network.target"];
+        description = "Initiates deployment of application and builds python environment using Poetry";
+        environment = {
+          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
+        };
+        serviceConfig = {
+          ExecStart = pkgs.writeShellScript "baseddata-deploy-service" ''
+            GITCMD="${pkgs.openssh}/bin/ssh -i /root/.ssh/id_ed25519"
+            if [ ! -d "/srv/baseddata-models" ]; then
+              GIT_SSH_COMMAND=$GITCMD  ${pkgs.git}/bin/git clone --branch $BASEDDATA_ENVIRONMENT git@git.bitlab21.com:sam/baseddata-models.git /srv/baseddata-models
+            else
+              cd /srv/baseddata-models
+              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git stash --include-untracked
+              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git pull
+            fi
+
+            cd /srv/baseddata-models
+            mkdir .venv
+            ${pkgs.poetry}/bin/poetry install
+          '';
+          Restart = "on-failure";
+        };
+      };
+
+      systemd.services.baseddata-prefect-server = {
+        wantedBy = ["multi-user.target"];
+        after = ["baseddata-deploy-service.target"];
+        description = "Initates the Prefect server";
+        environment = {
+          NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
+          NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
+          LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
+          PREFECT_API_URL = "http://${containerIp}:4200/api";
+          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
+        };
+        serviceConfig = {
+          WorkingDirectory = "/srv/baseddata-models";
+          ExecStart = pkgs.writeShellScript "baseddata-prefect-server" ''
+
+            # run prefect server
+            .venv/bin/prefect server start --host 0.0.0.0
+
+          '';
+          Restart = "on-failure";
+        };
+      };
+
+      systemd.services.baseddata-serve-flows = {
+        wantedBy = ["multi-user.target"];
+        after = ["baseddata-prefect-server.target"];
+        description = "Serves the Prefect flows";
+        environment = {
+          PREFECT_API_URL = "http://${containerIp}:4200/api";
+          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
+        };
+        serviceConfig = {
+          Environment = "PATH=/run/current-system/sw/bin/";
+          WorkingDirectory = "/srv/baseddata-models";
+          ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
+
+            # set prefect environment variables
+            .venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername)
+            .venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword)
+            .venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername)
+            .venv/bin/prefect variable set "mongoclient_auth" $(cat /run/secrets/mongoclientAuth)
+            .venv/bin/prefect variable set "mongoclient_host" "${mongodbIp}:${mongodbPort}"
+            .venv/bin/prefect variable set "mongoclient_user" $(cat /run/secrets/mongoclientUser)
+            .venv/bin/prefect variable set "mongoclient_pwd" $(cat /run/secrets/mongoclientPassword)
+            .venv/bin/prefect variable set "postgres_host" ${postgresIp}
+            .venv/bin/prefect variable set "postgres_port" ${postgresPort}
+            .venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser)
+            .venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword)
+
+            .venv/bin/prefect variable set "osm_history_dir" "/media/baseddata-data/osm-history"
+            .venv/bin/prefect variable set "mongo_db_name" "baseddata"
+            .venv/bin/prefect variable set "postgres_dbname" "dev_baseddata_models"
+            .venv/bin/prefect variable set "postgres_schema" "models_final"
+            .venv/bin/prefect variable set "unique_key" "row_uuid"
+
+            # serve flows
+            .venv/bin/python automation/flows/serve-flows.py
+          '';
+          Restart = "on-failure";
+        };
+      };
+
+      programs.nix-ld.enable = true;
+      programs.nix-ld.libraries = with pkgs; [
+        zlib
+        libgcc
+      ];
+
+      programs.ssh.knownHosts = {
+        "git.bitlab21.com" = {
+          publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALNd2BGf64heYjWT9yt0fVmngepiHRIMsL7au/MRteg";
+        };
+      };
+
+      services.openssh = {
+        enable = true;
+        settings.PasswordAuthentication = false;
+      };
+
+      users.users.root = {
+        openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
+      };
+
+      system.stateVersion = "24.05";
+    };
+  };
+}

hosts/common/optional/nixos-containers/mongodb.nix

@@ -0,0 +1,98 @@
+{
+  inputs,
+  lib,
+  config,
+  configVars,
+  pkgs,
+  ...
+}: let
+  mongodbPasswordPath = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/mongodb/baseddata/password".path;
+  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
+  containerName = "mongodb";
+  containerIp = configVars.networking.addresses.mongodb.ip;
+  gatewayIp = configVars.networking.addresses.gateway.ip;
+in {
+  sops.secrets = {
+    "software/postgres/postgres/password" = {
+    };
+  };
+
+  environment.persistence."/persist" = {
+    hideMounts = true;
+    directories = [
+      "/var/lib/nixos-containers/${containerName}"
+    ];
+  };
+
+  networking.nat.enable = true;
+  networking.nat.internalInterfaces = ["ve-+"];
+  networking.nat.externalInterface = "br0";
+
+  containers.${containerName} = {
+    autoStart = true;
+    privateNetwork = true;
+    hostBridge = "br0";
+    nixpkgs = pkgs.path;
+    bindMounts = {
+      # "/var/db/mongodb" = {
+      #   hostPath = "/media/main-ssd/mongodb";
+      #   isReadOnly = false;
+      # };
+    };
+
+    config = {
+      pkgs,
+      lib,
+      ...
+    }: {
+      networking = {
+        defaultGateway = "${gatewayIp}";
+        interfaces.eth0.ipv4.addresses = [
+          {
+            "address" = "${containerIp}";
+            "prefixLength" = 24;
+          }
+        ];
+        firewall = {
+          enable = true;
+          allowedTCPPorts = [
+            27017
+          ];
+        };
+        useHostResolvConf = lib.mkForce false;
+      };
+
+      services.resolved.enable = true;
+
+      environment.systemPackages = with pkgs; [
+        mongosh
+      ];
+
+      # allow unfree packages
+      nixpkgs.config.allowUnfreePredicate = let
+        whitelist = map lib.getName [
+          pkgs.mongodb
+        ];
+      in
+        pkg: builtins.elem (lib.getName pkg) whitelist;
+
+      services.mongodb = {
+        enable = true;
+        #        enableAuth = true;
+        #        initialRootPassword = mongodbPasswordPath;
+        bind_ip = "0.0.0.0";
+      };
+
+      services.openssh = {
+        enable = true;
+        settings.PasswordAuthentication = false;
+      };
+
+      users.users.root = {
+        openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
+      };
+
+      system.stateVersion = "24.05";
+    };
+  };
+}

hosts/common/optional/nixos-containers/postgres.nix

@@ -10,6 +10,7 @@
   pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
   containerName = "postgres";
   containerIp = configVars.networking.addresses.postgres.ip;
+  subnetIp = configVars.networking.addresses.subnet.ip;
   gatewayIp = configVars.networking.addresses.gateway.ip;
 in {
   sops.secrets = {
@@ -73,6 +74,7 @@ in {
         enableJIT = true;
         package = pkgs.postgresql_16;
         extraPlugins = with pkgs.postgresql_16.pkgs; [ postgis ];
+        enableTCPIP = true;
         settings = {
           max_worker_processes = "12";
           max_parallel_workers = "8";
@@ -84,8 +86,11 @@ in {
           maintenance_work_mem = "64MB";
         };
         authentication = pkgs.lib.mkOverride 10 ''
-          #type database  DBuser  auth-method
-          local all       all     trust
+          #type database DBuser origin-address auth-method
+          local   all             postgres                                peer
+          host    all             all             ${subnetIp}/24         scram-sha-256
+          local   replication     all                                    peer
+          host    replication     all             127.0.0.1/32           scram-sha-256
         '';
       };
 

hosts/common/optional/nixos-containers/worker.nix

@@ -1,163 +0,0 @@
-{
-  lib,
-  pkgs,
-  configVars,
-  inputs,
-  config,
-  ...
-}: let
-  pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
-  containerName = "worker";
-  containerIp = configVars.networking.addresses.worker.ip;
-  gatewayIp = configVars.networking.addresses.gateway.ip;
-  sopsHashedPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."ssh_keys/baseddata-models-access/id_ed25519".path;
-  baseddataEnv = "dev";
-in {
-  sops.secrets = {
-    "ssh_keys/baseddata-models-access/id_ed25519" = {};
-  };
-
-  environment.persistence."/persist" = {
-    hideMounts = true;
-    directories = [
-      "/var/lib/nixos-containers/${containerName}"
-    ];
-  };
-
-  networking.nat.enable = true;
-  networking.nat.internalInterfaces = ["ve-+"];
-  networking.nat.externalInterface = "br0";
-
-  containers.${containerName} = {
-    autoStart = true;
-    privateNetwork = true;
-    hostBridge = "br0";
-    nixpkgs = pkgs.path;
-    bindMounts = {
-      "/root/.ssh/id_ed25519" = {
-        hostPath = "${sopsHashedPasswordFile}";
-        isReadOnly = true;
-      };
-    };
-
-    config = {
-      pkgs,
-      lib,
-      ...
-    }: {
-      networking = {
-        defaultGateway = "${gatewayIp}";
-        interfaces.eth0.ipv4.addresses = [
-          {
-            "address" = "${containerIp}";
-            "prefixLength" = 24;
-          }
-        ];
-        firewall = {
-          enable = true;
-          allowedTCPPorts = [
-            4200
-          ];
-        };
-        useHostResolvConf = lib.mkForce false;
-      };
-
-      services.resolved.enable = true;
-
-      environment.systemPackages = [
-        pkgs.vim
-        pkgs.git
-        pkgs.python311
-        pkgs.poetry
-      ];
-
-      environment.variables = {
-        BASEDDATA_ENVIRONMENT = "dev";
-        NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
-        NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
-        LD_LIBRARY_PATH = "$NIX_LD_LIBRARY_PATH";
-      };
-
-      systemd.services.baseddata-deploy-service = {
-        wantedBy = ["multi-user.target"];
-        after = ["network.target"];
-        description = "Initiates deployment of application and builds python environment using Poetry";
-        environment = {
-          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
-        };
-        serviceConfig = {
-          ExecStart = pkgs.writeShellScript "baseddata-deploy-service" ''
-            GITCMD="${pkgs.openssh}/bin/ssh -i /root/.ssh/id_ed25519"
-            if [ ! -d "/srv/baseddata-models" ]; then
-              GIT_SSH_COMMAND=$GITCMD  ${pkgs.git}/bin/git clone --branch $BASEDDATA_ENVIRONMENT git@git.bitlab21.com:sam/baseddata-models.git /srv/baseddata-models
-            else
-              cd /srv/baseddata-models
-              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git stash --include-untracked
-              GIT_SSH_COMMAND=$GITCMD ${pkgs.git}/bin/git pull
-            fi
-
-            cd /srv/baseddata-models
-            mkdir .venv
-            ${pkgs.poetry}/bin/poetry install
-          '';
-          Restart = "on-failure";
-        };
-      };
-
-      systemd.services.baseddata-prefect-server = {
-        wantedBy = ["multi-user.target"];
-        after = ["baseddata-deploy-service.target"];
-        description = "Initates the Prefect server";
-        environment = {
-          NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
-          NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
-          LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
-          PREFECT_API_URL = "http://${containerIp}:4200/api";
-          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
-        };
-        serviceConfig = {
-          WorkingDirectory = "/srv/baseddata-models";
-          ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
-            .venv/bin/prefect server start --host 0.0.0.0
-          '';
-          Restart = "on-failure";
-        };
-      };
-
-      systemd.services.baseddata-serve-flows = {
-        wantedBy = ["multi-user.target"];
-        after = ["baseddata-prefect-server.target"];
-        description = "Serves the Prefect flows";
-        environment = {
-          PREFECT_API_URL = "http://${containerIp}:4200/api";
-          BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
-        };
-        serviceConfig = {
-          Environment="PATH=/run/current-system/sw/bin/";
-          WorkingDirectory = "/srv/baseddata-models";
-          ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
-            .venv/bin/python automation/flows/serve-flows.py
-          '';
-          Restart = "on-failure";
-        };
-      };
-
-      programs.nix-ld.enable = true;
-      programs.nix-ld.libraries = with pkgs; [
-        zlib
-        libgcc
-      ];
-
-      services.openssh = {
-        enable = true;
-        settings.PasswordAuthentication = false;
-      };
-
-      users.users.root = {
-        openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key);
-      };
-
-      system.stateVersion = "24.05";
-    };
-  };
-}

hosts/semita/default.nix

@@ -55,7 +55,8 @@ in {
     ../common/optional/nixos-containers/nix-bitcoin.nix
     ../common/optional/nixos-containers/postgres.nix
     ../common/optional/nixos-containers/jellyfin.nix
-    ../common/optional/nixos-containers/worker.nix
+    ../common/optional/nixos-containers/baseddata-worker.nix
+    ../common/optional/nixos-containers/mongodb.nix
   ];
 
   fileSystems."/media/main-ssd" = {