From eaebfcaeb4e93c4dbc0432a3861c1a4c18ab6342 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 28 Dec 2024 10:07:25 +0000 Subject: [PATCH] Add dbt profiles configuration and update baseddata-worker - Add 'dbt_profiles.nix' file to configure dbt profiles for baseddata - Update 'baseddata-worker.nix' to import dbt profiles and modify environment variables - Remove dbt profiles configuration from user-specific default.nix for sam --- hosts/common/optional/dbt_profiles.nix | 26 ++++++++++++++++ .../nixos-containers/baseddata-worker.nix | 27 +++++++++++++---- hosts/common/users/sam/default.nix | 30 ------------------- 3 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 hosts/common/optional/dbt_profiles.nix diff --git a/hosts/common/optional/dbt_profiles.nix b/hosts/common/optional/dbt_profiles.nix new file mode 100644 index 0000000..49a6e1b --- /dev/null +++ b/hosts/common/optional/dbt_profiles.nix @@ -0,0 +1,26 @@ +{configVars, config, ...}: +let + baseddataPostgresIp = configVars.networking.addresses.postgres.ip; +in +{ + sops.templates."dbt_profiles.yml" = { + path = "/run/secrets/templates/dbt/profiles.yml"; + mode = "0600"; + content = '' + baseddata: + target: prod + outputs: + prod: + dbname: baseddata + host: ${baseddataPostgresIp} + pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}' + port: 5432 + schema: models + threads: 6 + type: postgres + user: ${config.sops.placeholder."software/postgres/baseddata/user_username"} + + ''; + }; + + } diff --git a/hosts/common/optional/nixos-containers/baseddata-worker.nix b/hosts/common/optional/nixos-containers/baseddata-worker.nix index 9480d61..8c4059f 100644 --- a/hosts/common/optional/nixos-containers/baseddata-worker.nix +++ b/hosts/common/optional/nixos-containers/baseddata-worker.nix @@ -36,7 +36,6 @@ in { "software/bitcoind/username" = {}; "software/bitcoind/bitcoin-rpcpassword-public" = {}; }; - environment.persistence."/persist" = { hideMounts = true; directories = [ @@ -44,6 +43,10 @@ in { ]; }; + imports = [ + ../dbt_profiles.nix + ]; + networking.nat.enable = true; networking.nat.internalInterfaces = ["ve-+"]; networking.nat.externalInterface = "br0"; @@ -90,6 +93,10 @@ in { hostPath = "/media/main-ssd/baseddata-data"; isReadOnly = false; }; + "/root/.dbt/profiles.yml" = { + hostPath = "/run/secrets/templates/dbt/profiles.yml"; + isReadOnly = false; + }; }; config = { @@ -124,6 +131,7 @@ in { pkgs.aria2 pkgs.osmctools pkgs.osmium-tool + pkgs.osm2pgsql ]; environment.variables = { @@ -153,6 +161,7 @@ in { cd /srv/baseddata-models mkdir .venv + ${pkgs.poetry}/bin/poetry lock --no-update ${pkgs.poetry}/bin/poetry install ''; Restart = "on-failure"; @@ -186,12 +195,15 @@ in { wantedBy = ["multi-user.target"]; after = ["baseddata-prefect-server.target"]; description = "Serves the Prefect flows"; + path = ["/run/current-system/sw" "/srv/baseddata-models/.venv"]; environment = { PREFECT_API_URL = "http://${containerIp}:4200/api"; BASEDDATA_ENVIRONMENT = "${baseddataEnv}"; + NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib"; + NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so"; + LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib"; }; serviceConfig = { - Environment = "PATH=/run/current-system/sw/bin/"; WorkingDirectory = "/srv/baseddata-models"; ExecStartPre = "${pkgs.coreutils}/bin/timeout 120 ${pkgs.bash}/bin/bash -c 'until ${pkgs.netcat-openbsd}/bin/nc -z ${containerIp} 4200; do sleep 3; done'"; ExecStart = pkgs.writeShellScript "baseddata-serve-flows" '' @@ -200,10 +212,14 @@ in { .venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite .venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite .venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite + .venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite .venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite .venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite .venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite + .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite + .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite + .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite @@ -211,10 +227,9 @@ in { .venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite .venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite - .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite - .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite - .venv/bin/prefect variable set "unique_key" "row_uuid" --overwrite - .venv/bin/prefect variable set "dbt_profiles_dir" "/srv/baseddata-models/data-models" --overwrite + + .venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite + # serve flows .venv/bin/python automation/flows/serve-flows.py diff --git a/hosts/common/users/sam/default.nix b/hosts/common/users/sam/default.nix index acf31a7..e8d669a 100644 --- a/hosts/common/users/sam/default.nix +++ b/hosts/common/users/sam/default.nix @@ -98,36 +98,6 @@ in { ''; }; - sops.templates."dbt_profiles.yml" = { - path = "/home/${username}/.config/dbt/profiles.yml"; - owner = "${username}"; - mode = "0600"; - content = '' - baseddata: - target: dev - outputs: - dev: - dbname: dev_baseddata - host: ${baseddataPostgresIp} - pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}' - port: 5432 - schema: models - threads: 6 - type: postgres - user: ${config.sops.placeholder."software/postgres/baseddata/user_username"} - prod: - dbname: baseddata - host: ${baseddataPostgresIp} - pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}' - port: 5432 - schema: models - threads: 6 - type: postgres - user: ${config.sops.placeholder."software/postgres/baseddata/user_username"} - - ''; - }; - nix = { extraOptions = '' experimental-features = nix-command flakes