From aad9334639d3d5fcc8c8d95822a52ef633f58086 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 13 Jan 2025 21:15:17 +0000 Subject: [PATCH] major refactor of baseddata container - included config for dbt and pgsync as sops templates - import sops-nix into container - remove unnecessary bind mounts - update nix-secrets in flake.lock --- flake.lock | 8 +- hosts/common/optional/dbt_profiles.nix | 25 --- .../nixos-containers/baseddata-worker.nix | 189 +++++++++++------- .../optional/nixos-containers/docker.nix | 6 +- .../optional/nixos-containers/postgres.nix | 53 +++-- hosts/common/optional/secrets.nix | 12 ++ 6 files changed, 168 insertions(+), 125 deletions(-) delete mode 100644 hosts/common/optional/dbt_profiles.nix create mode 100644 hosts/common/optional/secrets.nix diff --git a/flake.lock b/flake.lock index db91f09..c8a8891 100644 --- a/flake.lock +++ b/flake.lock @@ -538,11 +538,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1736678262, - "narHash": "sha256-LLKE6KboWC8l9llNrrLU/o6kVaHFHDsWEW8fKpX/6NI=", + "lastModified": 1736770787, + "narHash": "sha256-TEDwOm0ZnRioT01DdfS5XaFPYXA9uHA+P8pTOSFK0bI=", "ref": "refs/heads/master", - "rev": "7a43e9814b20d7eac2a44d514402c64a54597f35", - "revCount": 198, + "rev": "a88c4ab6fb54527aabeeee2a2954338f81016f98", + "revCount": 201, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/dbt_profiles.nix b/hosts/common/optional/dbt_profiles.nix deleted file mode 100644 index 9acb62a..0000000 --- a/hosts/common/optional/dbt_profiles.nix +++ /dev/null @@ -1,25 +0,0 @@ -{configVars, config, ...}: -let - baseddataPostgresIp = configVars.networking.addresses.postgres.ip; -in -{ - sops.templates."dbt_profiles.yml" = { - mode = "0600"; - content = '' - baseddata: - target: prod - outputs: - prod: - dbname: baseddata - host: ${baseddataPostgresIp} - pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}' - port: 5432 - schema: models - threads: 6 - type: postgres - user: ${config.sops.placeholder."software/postgres/baseddata/user_username"} - - ''; - }; - - } diff --git a/hosts/common/optional/nixos-containers/baseddata-worker.nix b/hosts/common/optional/nixos-containers/baseddata-worker.nix index c645b59..beb6411 100644 --- a/hosts/common/optional/nixos-containers/baseddata-worker.nix +++ b/hosts/common/optional/nixos-containers/baseddata-worker.nix @@ -1,41 +1,13 @@ { lib, pkgs, - configVars, inputs, - config, ... }: let pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; containerName = "bd-worker"; - containerIp = configVars.networking.addresses.bd-worker.ip; - gatewayIp = configVars.networking.addresses.gateway.ip; - postgresIp = configVars.networking.addresses.postgres.ip; - postgresPort = toString configVars.networking.addresses.postgres.port; - bitcoindIp = configVars.networking.addresses.bitcoin-node.ip; - bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port; - - #secrets - sshKeyFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."ssh_keys/baseddata-models-access/id_ed25519".path; - notifybotUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/username".path; - notifybotPwd = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/password".path; - recipientUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/mrsu/username".path; - postgresUser = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_username".path; - postgresPassword = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_password".path; - bitcoindRPCUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/username".path; - bitcoindRPCPassword= lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path; - baseddataEnv = "dev"; + sops-nix = inputs.sops-nix; in { - sops.secrets = { - "ssh_keys/baseddata-models-access/id_ed25519" = {}; - "comms/xmpp/notifybot/username" = {}; - "comms/xmpp/notifybot/password" = {}; - "comms/xmpp/mrsu/username" = {}; - "software/postgres/baseddata/user_password" = {}; - "software/postgres/baseddata/user_username" = {}; - "software/bitcoind/username" = {}; - "software/bitcoind/bitcoin-rpcpassword-public" = {}; - }; environment.persistence."/persist" = { hideMounts = true; directories = [ @@ -43,10 +15,6 @@ in { ]; }; - imports = [ - ../dbt_profiles.nix - ]; - networking.nat.enable = true; networking.nat.internalInterfaces = ["ve-+"]; networking.nat.externalInterface = "br0"; @@ -56,54 +24,56 @@ in { privateNetwork = true; hostBridge = "br0"; nixpkgs = pkgs.path; + + allowedDevices = [ + { + node = "/dev/net/tun"; + modifier = "rw"; + } + ]; + bindMounts = { - "/root/.ssh/id_ed25519" = { - hostPath = "${sshKeyFile}"; - isReadOnly = true; - }; - "/run/secrets/notifybotUsername" = { - hostPath = "${notifybotUsername}"; - isReadOnly = true; - }; - "/run/secrets/notifybotPassword" = { - hostPath = "${notifybotPwd}"; - isReadOnly = true; - }; - "/run/secrets/recipientUsername" = { - hostPath = "${recipientUsername}"; - isReadOnly = true; - }; - "/run/secrets/postgresPassword" = { - hostPath = "${postgresPassword}"; - isReadOnly = true; - }; - "/run/secrets/postgresUser" = { - hostPath = "${postgresUser}"; - isReadOnly = true; - }; - "/run/secrets/bitcoindRPCPassword" = { - hostPath = "${bitcoindRPCPassword}"; - isReadOnly = true; - }; - "/run/secrets/bitcoindRPCUsername" = { - hostPath = "${bitcoindRPCUsername}"; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; isReadOnly = true; }; "/media/baseddata-data" = { hostPath = "/media/main-ssd/baseddata-data"; isReadOnly = false; }; - "/root/.dbt/profiles.yml" = { - hostPath = "/run/secrets/rendered/dbt_profiles.yml"; - isReadOnly = true; - }; }; config = { pkgs, lib, + config, ... - }: { + }: let + configVars = import ../../../../vars {inherit inputs lib;}; + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + + # define ip addresses + containerIp = configVars.networking.addresses.bd-worker.ip; + gatewayIp = configVars.networking.addresses.gateway.ip; + postgresIp = configVars.networking.addresses.postgres.ip; + postgresRemoteIp = configVars.networking.addresses.postgres-remote.ip; + postgresPort = toString configVars.networking.addresses.postgres.port; + bitcoindIp = configVars.networking.addresses.bitcoin-node.ip; + bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port; + + # define secret paths + notifybotUsername = config.sops.secrets."comms/xmpp/notifybot/username".path; + notifybotPassword = config.sops.secrets."comms/xmpp/notifybot/password".path; + recipientUsername = config.sops.secrets."comms/xmpp/mrsu/username".path; + postgresUser = config.sops.secrets."software/postgres/baseddata/user_username".path; + postgresPassword = config.sops.secrets."software/postgres/baseddata/user_password".path; + bitcoindRPCUsername = config.sops.secrets."software/bitcoind/username".path; + bitcoindRPCPassword = config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path; + dbtProfiles = config.sops.templates."profiles.yml".path; + pgsyncConfig = config.sops.templates."pgsync.yml".path; + baseddataEnv = "dev"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -121,7 +91,68 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + }; + + sops.secrets = { + "ssh_keys/baseddata-models-access/id_ed25519" = { + mode = "0400"; + path = "/root/.ssh/id_ed25519"; + }; + "comms/xmpp/notifybot/username" = {}; + "comms/xmpp/notifybot/password" = {}; + "comms/xmpp/mrsu/username" = {}; + "software/postgres/baseddata/user_password" = {}; + "software/postgres/baseddata/user_username" = {}; + "software/bitcoind/username" = {}; + "software/bitcoind/bitcoin-rpcpassword-public" = {}; + }; + + sops.templates."profiles.yml" = { + mode = "0600"; + path = "/root/.dbt/profiles.yml"; + content = '' + baseddata: + target: prod + outputs: + prod: + dbname: baseddata + host: ${postgresIp} + pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}' + port: 5432 + schema: models + threads: 6 + type: postgres + user: ${config.sops.placeholder."software/postgres/baseddata/user_username"} + + ''; + }; + + sops.templates."pgsync.yml" = { + mode = "0600"; + path = "/root/.pgsync.yml"; + content = '' + from: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresIp}/baseddata + to: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresRemoteIp}/baseddata + schemas: + - models_final + to_safe: true + + ''; + }; + + imports = [ + sops-nix.nixosModules.sops + ]; + services.resolved.enable = true; + services.tailscale.enable = true; environment.systemPackages = [ pkgs.vim @@ -132,6 +163,9 @@ in { pkgs.osmctools pkgs.osmium-tool pkgs.osm2pgsql + pkgs.pgsync + pkgs.postgresql_16 + pkgs.htop ]; environment.variables = { @@ -184,7 +218,7 @@ in { ExecStart = pkgs.writeShellScript "baseddata-prefect-server" '' # run prefect server - .venv/bin/prefect server start --host 0.0.0.0 + .venv/bin/prefect server start --host ${containerIp} ''; Restart = "on-failure"; @@ -209,26 +243,27 @@ in { ExecStart = pkgs.writeShellScript "baseddata-serve-flows" '' # set prefect environment variables - .venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite - .venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite - .venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite + .venv/bin/prefect variable set "xmpp_jid" $(cat ${notifybotUsername}) --overwrite + .venv/bin/prefect variable set "xmpp_password" $(cat ${notifybotPassword}) --overwrite + .venv/bin/prefect variable set "xmpp_recipient" $(cat ${recipientUsername}) --overwrite .venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite .venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite - .venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite - .venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite + .venv/bin/prefect variable set "postgres_user" $(cat ${postgresUser}) --overwrite + .venv/bin/prefect variable set "postgres_pwd" $(cat ${postgresPassword}) --overwrite .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite - .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite - .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite + .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat ${bitcoindRPCPassword}) --overwrite + .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat ${bitcoindRPCUsername}) --overwrite .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite .venv/bin/prefect variable set "bitcoind_port" ${bitcoindPort} --overwrite .venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite .venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite - .venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite + .venv/bin/prefect variable set "dbt_profiles_dir" $(dirname ${dbtProfiles}) --overwrite + .venv/bin/prefect variable set "pgsync_config" ${pgsyncConfig} --overwrite # serve flows diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 64fdf01..5628856 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -1,8 +1,8 @@ { pkgs, - configVars, lib, inputs, + configVars, ... }: let containerName = "docker"; @@ -66,6 +66,10 @@ in { node = "/dev/dri/renderD128"; modifier = "rwm"; } + { + node = "/dev/net/tun"; + modifier = "rw"; + } ]; ###### diff --git a/hosts/common/optional/nixos-containers/postgres.nix b/hosts/common/optional/nixos-containers/postgres.nix index cfbcf9e..0aac4f4 100644 --- a/hosts/common/optional/nixos-containers/postgres.nix +++ b/hosts/common/optional/nixos-containers/postgres.nix @@ -77,14 +77,31 @@ in { extensions = with pkgs.postgresql_16.pkgs; [postgis]; enableTCPIP = true; settings = { - max_worker_processes = "12"; - max_parallel_workers = "8"; - max_parallel_workers_per_gather = "4"; - max_connections = "100"; - autovacuum_work_mem = "2GB"; - shared_buffers = "32GB"; - work_mem = "0.32GB"; - maintenance_work_mem = "64MB"; + # max_worker_processes = "12"; + # max_parallel_workers = "8"; + # max_parallel_workers_per_gather = "4"; + # max_connections = "100"; + # autovacuum_work_mem = "2GB"; + # shared_buffers = "32GB"; + # work_mem = "0.32GB"; + # maintenance_work_mem = "64MB"; + max_connections = "20"; + shared_buffers = "2GB"; + effective_cache_size = "6GB"; + maintenance_work_mem = "1GB"; + checkpoint_completion_target = "0.9"; + wal_buffers = "16MB"; + default_statistics_target = "500"; + random_page_cost = "1.1"; + effective_io_concurrency = "200"; + work_mem = "17476kB"; + huge_pages = "off"; + min_wal_size = "4GB"; + max_wal_size = "16GB"; + max_worker_processes = "6"; + max_parallel_workers_per_gather = "3"; + max_parallel_workers = "6"; + max_parallel_maintenance_workers = "3"; }; authentication = pkgs.lib.mkOverride 10 '' #type database DBuser origin-address auth-method @@ -95,16 +112,16 @@ in { ''; }; - systemd.services.postgresql.postStart = '' - $PSQL -tA <<'EOF' - DO $$ - DECLARE password TEXT; - BEGIN - password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', ''')); - EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password); - END $$; - EOF - ''; + # systemd.services.postgresql.postStart = '' + # $PSQL -tA <<'EOF' + # DO $$ + # DECLARE password TEXT; + # BEGIN + # password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', ''')); + # EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password); + # END $$; + # EOF + # ''; services.openssh = { enable = true; diff --git a/hosts/common/optional/secrets.nix b/hosts/common/optional/secrets.nix new file mode 100644 index 0000000..2dd0744 --- /dev/null +++ b/hosts/common/optional/secrets.nix @@ -0,0 +1,12 @@ +{ + sops.secrets = { + "ssh_keys/baseddata-models-access/id_ed25519" = {}; + "comms/xmpp/notifybot/username" = {}; + "comms/xmpp/notifybot/password" = {}; + "comms/xmpp/mrsu/username" = {}; + "software/postgres/baseddata/user_password" = {}; + "software/postgres/baseddata/user_username" = {}; + "software/bitcoind/username" = {}; + "software/bitcoind/bitcoin-rpcpassword-public" = {}; + }; +}