major refactor of baseddata container

- included config for dbt and pgsync as sops templates
- import sops-nix into container
- remove unnecessary bind mounts
- update nix-secrets in flake.lock

flake.lock

@@ -538,11 +538,11 @@
     },
     "nix-secrets": {
       "locked": {
-        "lastModified": 1736678262,
+        "lastModified": 1736770787,
-        "narHash": "sha256-LLKE6KboWC8l9llNrrLU/o6kVaHFHDsWEW8fKpX/6NI=",
+        "narHash": "sha256-TEDwOm0ZnRioT01DdfS5XaFPYXA9uHA+P8pTOSFK0bI=",
         "ref": "refs/heads/master",
-        "rev": "7a43e9814b20d7eac2a44d514402c64a54597f35",
+        "rev": "a88c4ab6fb54527aabeeee2a2954338f81016f98",
-        "revCount": 198,
+        "revCount": 201,
         "type": "git",
         "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
       },

hosts/common/optional/dbt_profiles.nix

@@ -1,25 +0,0 @@
-{configVars, config, ...}:
-let
-  baseddataPostgresIp = configVars.networking.addresses.postgres.ip;
-in
-{
-  sops.templates."dbt_profiles.yml" = {
-    mode = "0600";
-    content = ''
-      baseddata:
-        target: prod
-        outputs:
-          prod:
-            dbname: baseddata
-            host: ${baseddataPostgresIp}
-            pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
-            port: 5432
-            schema: models
-            threads: 6
-            type: postgres
-            user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
-
-    '';
-  };
-
-      }

hosts/common/optional/nixos-containers/baseddata-worker.nix

@@ -1,41 +1,13 @@
 {
   lib,
   pkgs,
-  configVars,
   inputs,
-  config,
   ...
 }: let
   pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
   containerName = "bd-worker";
-  containerIp = configVars.networking.addresses.bd-worker.ip;
+  sops-nix = inputs.sops-nix;
-  gatewayIp = configVars.networking.addresses.gateway.ip;
-  postgresIp = configVars.networking.addresses.postgres.ip;
-  postgresPort = toString configVars.networking.addresses.postgres.port;
-  bitcoindIp = configVars.networking.addresses.bitcoin-node.ip;
-  bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port;
-
-  #secrets
-  sshKeyFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."ssh_keys/baseddata-models-access/id_ed25519".path;
-  notifybotUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/username".path;
-  notifybotPwd = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/password".path;
-  recipientUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/mrsu/username".path;
-  postgresUser = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_username".path;
-  postgresPassword = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_password".path;
-  bitcoindRPCUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/username".path;
-  bitcoindRPCPassword= lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path;
-  baseddataEnv = "dev";
 in {
-  sops.secrets = {
-    "ssh_keys/baseddata-models-access/id_ed25519" = {};
-    "comms/xmpp/notifybot/username" = {};
-    "comms/xmpp/notifybot/password" = {};
-    "comms/xmpp/mrsu/username" = {};
-    "software/postgres/baseddata/user_password" = {};
-    "software/postgres/baseddata/user_username" = {};
-    "software/bitcoind/username" = {};
-    "software/bitcoind/bitcoin-rpcpassword-public" = {};
-  };
   environment.persistence."/persist" = {
     hideMounts = true;
     directories = [
@@ -43,10 +15,6 @@ in {
     ];
   };
 
-  imports = [
-    ../dbt_profiles.nix
-  ];
-
   networking.nat.enable = true;
   networking.nat.internalInterfaces = ["ve-+"];
   networking.nat.externalInterface = "br0";
@@ -56,54 +24,56 @@ in {
     privateNetwork = true;
     hostBridge = "br0";
     nixpkgs = pkgs.path;
+
+    allowedDevices = [
+      {
+        node = "/dev/net/tun";
+        modifier = "rw";
+      }
+    ];
+
     bindMounts = {
-      "/root/.ssh/id_ed25519" = {
+      "/etc/ssh/ssh_host_ed25519_key" = {
-        hostPath = "${sshKeyFile}";
+        hostPath = "/etc/ssh/ssh_host_ed25519_key";
-        isReadOnly = true;
-      };
-      "/run/secrets/notifybotUsername" = {
-        hostPath = "${notifybotUsername}";
-        isReadOnly = true;
-      };
-      "/run/secrets/notifybotPassword" = {
-        hostPath = "${notifybotPwd}";
-        isReadOnly = true;
-      };
-      "/run/secrets/recipientUsername" = {
-        hostPath = "${recipientUsername}";
-        isReadOnly = true;
-      };
-      "/run/secrets/postgresPassword" = {
-        hostPath = "${postgresPassword}";
-        isReadOnly = true;
-      };
-      "/run/secrets/postgresUser" = {
-        hostPath = "${postgresUser}";
-        isReadOnly = true;
-      };
-      "/run/secrets/bitcoindRPCPassword" = {
-        hostPath = "${bitcoindRPCPassword}";
-        isReadOnly = true;
-      };
-      "/run/secrets/bitcoindRPCUsername" = {
-        hostPath = "${bitcoindRPCUsername}";
         isReadOnly = true;
       };
       "/media/baseddata-data" = {
         hostPath = "/media/main-ssd/baseddata-data";
         isReadOnly = false;
       };
-      "/root/.dbt/profiles.yml" = {
-        hostPath = "/run/secrets/rendered/dbt_profiles.yml";
-        isReadOnly = true;
-      };
     };
 
     config = {
       pkgs,
       lib,
+      config,
       ...
-    }: {
+    }: let
+      configVars = import ../../../../vars {inherit inputs lib;};
+      secretsDirectory = builtins.toString inputs.nix-secrets;
+      secretsFile = "${secretsDirectory}/secrets.yaml";
+
+      # define ip addresses
+      containerIp = configVars.networking.addresses.bd-worker.ip;
+      gatewayIp = configVars.networking.addresses.gateway.ip;
+      postgresIp = configVars.networking.addresses.postgres.ip;
+      postgresRemoteIp = configVars.networking.addresses.postgres-remote.ip;
+      postgresPort = toString configVars.networking.addresses.postgres.port;
+      bitcoindIp = configVars.networking.addresses.bitcoin-node.ip;
+      bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port;
+
+      # define secret paths
+      notifybotUsername = config.sops.secrets."comms/xmpp/notifybot/username".path;
+      notifybotPassword = config.sops.secrets."comms/xmpp/notifybot/password".path;
+      recipientUsername = config.sops.secrets."comms/xmpp/mrsu/username".path;
+      postgresUser = config.sops.secrets."software/postgres/baseddata/user_username".path;
+      postgresPassword = config.sops.secrets."software/postgres/baseddata/user_password".path;
+      bitcoindRPCUsername = config.sops.secrets."software/bitcoind/username".path;
+      bitcoindRPCPassword = config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path;
+      dbtProfiles = config.sops.templates."profiles.yml".path;
+      pgsyncConfig = config.sops.templates."pgsync.yml".path;
+      baseddataEnv = "dev";
+    in {
       networking = {
         defaultGateway = "${gatewayIp}";
         interfaces.eth0.ipv4.addresses = [
@@ -121,7 +91,68 @@ in {
         useHostResolvConf = lib.mkForce false;
       };
 
+      sops = {
+        defaultSopsFile = "${secretsFile}";
+        validateSopsFiles = false;
+
+        age = {
+          sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"];
+        };
+      };
+
+      sops.secrets = {
+        "ssh_keys/baseddata-models-access/id_ed25519" = {
+          mode = "0400";
+          path = "/root/.ssh/id_ed25519";
+        };
+        "comms/xmpp/notifybot/username" = {};
+        "comms/xmpp/notifybot/password" = {};
+        "comms/xmpp/mrsu/username" = {};
+        "software/postgres/baseddata/user_password" = {};
+        "software/postgres/baseddata/user_username" = {};
+        "software/bitcoind/username" = {};
+        "software/bitcoind/bitcoin-rpcpassword-public" = {};
+      };
+
+      sops.templates."profiles.yml" = {
+        mode = "0600";
+        path = "/root/.dbt/profiles.yml";
+        content = ''
+          baseddata:
+            target: prod
+            outputs:
+              prod:
+                dbname: baseddata
+                host: ${postgresIp}
+                pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
+                port: 5432
+                schema: models
+                threads: 6
+                type: postgres
+                user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
+
+        '';
+      };
+
+      sops.templates."pgsync.yml" = {
+        mode = "0600";
+        path = "/root/.pgsync.yml";
+        content = ''
+            from: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresIp}/baseddata
+            to: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresRemoteIp}/baseddata
+            schemas:
+              - models_final
+            to_safe: true
+
+        '';
+      };
+
+      imports = [
+        sops-nix.nixosModules.sops
+      ];
+
       services.resolved.enable = true;
+      services.tailscale.enable = true;
 
       environment.systemPackages = [
         pkgs.vim
@@ -132,6 +163,9 @@ in {
         pkgs.osmctools
         pkgs.osmium-tool
         pkgs.osm2pgsql
+        pkgs.pgsync
+        pkgs.postgresql_16
+        pkgs.htop
       ];
 
       environment.variables = {
@@ -184,7 +218,7 @@ in {
           ExecStart = pkgs.writeShellScript "baseddata-prefect-server" ''
 
             # run prefect server
-            .venv/bin/prefect server start --host 0.0.0.0
+            .venv/bin/prefect server start --host ${containerIp}
 
           '';
           Restart = "on-failure";
@@ -209,26 +243,27 @@ in {
           ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
 
             # set prefect environment variables
-            .venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite
+            .venv/bin/prefect variable set "xmpp_jid" $(cat ${notifybotUsername}) --overwrite
-            .venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite
+            .venv/bin/prefect variable set "xmpp_password" $(cat ${notifybotPassword}) --overwrite
-            .venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite
+            .venv/bin/prefect variable set "xmpp_recipient" $(cat ${recipientUsername}) --overwrite
 
             .venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite
             .venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite
-            .venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite
+            .venv/bin/prefect variable set "postgres_user" $(cat ${postgresUser}) --overwrite
-            .venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite
+            .venv/bin/prefect variable set "postgres_pwd" $(cat ${postgresPassword}) --overwrite
             .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
             .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
 
-            .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite
+            .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat ${bitcoindRPCPassword}) --overwrite
-            .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite
+            .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat ${bitcoindRPCUsername}) --overwrite
             .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite
             .venv/bin/prefect variable set "bitcoind_port" ${bitcoindPort} --overwrite
 
             .venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite
             .venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite
 
-            .venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite
+            .venv/bin/prefect variable set "dbt_profiles_dir" $(dirname ${dbtProfiles}) --overwrite
+            .venv/bin/prefect variable set "pgsync_config" ${pgsyncConfig} --overwrite
 
 
             # serve flows

hosts/common/optional/nixos-containers/docker.nix

@@ -1,8 +1,8 @@
 {
   pkgs,
-  configVars,
   lib,
   inputs,
+  configVars,
   ...
 }: let
   containerName = "docker";
@@ -66,6 +66,10 @@ in {
         node = "/dev/dri/renderD128";
         modifier = "rwm";
       }
+      {
+        node = "/dev/net/tun";
+        modifier = "rw";
+      }
     ];
     ######
 

hosts/common/optional/nixos-containers/postgres.nix

@@ -77,14 +77,31 @@ in {
         extensions = with pkgs.postgresql_16.pkgs; [postgis];
         enableTCPIP = true;
         settings = {
-          max_worker_processes = "12";
+          # max_worker_processes = "12";
-          max_parallel_workers = "8";
+          # max_parallel_workers = "8";
-          max_parallel_workers_per_gather = "4";
+          # max_parallel_workers_per_gather = "4";
-          max_connections = "100";
+          # max_connections = "100";
-          autovacuum_work_mem = "2GB";
+          # autovacuum_work_mem = "2GB";
-          shared_buffers = "32GB";
+          # shared_buffers = "32GB";
-          work_mem = "0.32GB";
+          # work_mem = "0.32GB";
-          maintenance_work_mem = "64MB";
+          # maintenance_work_mem = "64MB";
+          max_connections = "20";
+          shared_buffers = "2GB";
+          effective_cache_size = "6GB";
+          maintenance_work_mem = "1GB";
+          checkpoint_completion_target = "0.9";
+          wal_buffers = "16MB";
+          default_statistics_target = "500";
+          random_page_cost = "1.1";
+          effective_io_concurrency = "200";
+          work_mem = "17476kB";
+          huge_pages = "off";
+          min_wal_size = "4GB";
+          max_wal_size = "16GB";
+          max_worker_processes = "6";
+          max_parallel_workers_per_gather = "3";
+          max_parallel_workers = "6";
+          max_parallel_maintenance_workers = "3";
         };
         authentication = pkgs.lib.mkOverride 10 ''
           #type database DBuser origin-address auth-method
@@ -95,16 +112,16 @@ in {
         '';
       };
 
-      systemd.services.postgresql.postStart = ''
+      # systemd.services.postgresql.postStart = ''
-        $PSQL -tA <<'EOF'
+      #   $PSQL -tA <<'EOF'
-          DO $$
+      #     DO $$
-          DECLARE password TEXT;
+      #     DECLARE password TEXT;
-          BEGIN
+      #     BEGIN
-            password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', '''));
+      #       password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', '''));
-            EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password);
+      #       EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password);
-          END $$;
+      #     END $$;
-        EOF
+      #   EOF
-      '';
+      # '';
 
       services.openssh = {
         enable = true;

hosts/common/optional/secrets.nix

@@ -0,0 +1,12 @@
+{
+  sops.secrets = {
+    "ssh_keys/baseddata-models-access/id_ed25519" = {};
+    "comms/xmpp/notifybot/username" = {};
+    "comms/xmpp/notifybot/password" = {};
+    "comms/xmpp/mrsu/username" = {};
+    "software/postgres/baseddata/user_password" = {};
+    "software/postgres/baseddata/user_username" = {};
+    "software/bitcoind/username" = {};
+    "software/bitcoind/bitcoin-rpcpassword-public" = {};
+  };
+}