Add dbt profiles configuration and update baseddata-worker

- Add 'dbt_profiles.nix' file to configure dbt profiles for baseddata
- Update 'baseddata-worker.nix' to import dbt profiles and modify environment variables
- Remove dbt profiles configuration from user-specific default.nix for sam

hosts/common/optional/dbt_profiles.nix

@@ -0,0 +1,26 @@
+{configVars, config, ...}:
+let
+  baseddataPostgresIp = configVars.networking.addresses.postgres.ip;
+in
+{
+  sops.templates."dbt_profiles.yml" = {
+    path = "/run/secrets/templates/dbt/profiles.yml";
+    mode = "0600";
+    content = ''
+      baseddata:
+        target: prod
+        outputs:
+          prod:
+            dbname: baseddata
+            host: ${baseddataPostgresIp}
+            pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
+            port: 5432
+            schema: models
+            threads: 6
+            type: postgres
+            user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
+
+    '';
+  };
+
+      }

hosts/common/optional/nixos-containers/baseddata-worker.nix

@@ -36,7 +36,6 @@ in {
     "software/bitcoind/username" = {};
     "software/bitcoind/bitcoin-rpcpassword-public" = {};
   };
-
   environment.persistence."/persist" = {
     hideMounts = true;
     directories = [
@@ -44,6 +43,10 @@ in {
     ];
   };
 
+  imports = [
+    ../dbt_profiles.nix
+  ];
+
   networking.nat.enable = true;
   networking.nat.internalInterfaces = ["ve-+"];
   networking.nat.externalInterface = "br0";
@@ -90,6 +93,10 @@ in {
         hostPath = "/media/main-ssd/baseddata-data";
         isReadOnly = false;
       };
+      "/root/.dbt/profiles.yml" = {
+        hostPath = "/run/secrets/templates/dbt/profiles.yml";
+        isReadOnly = false;
+      };
     };
 
     config = {
@@ -124,6 +131,7 @@ in {
         pkgs.aria2
         pkgs.osmctools
         pkgs.osmium-tool
+        pkgs.osm2pgsql
       ];
 
       environment.variables = {
@@ -153,6 +161,7 @@ in {
 
             cd /srv/baseddata-models
             mkdir .venv
+            ${pkgs.poetry}/bin/poetry lock --no-update
             ${pkgs.poetry}/bin/poetry install
           '';
           Restart = "on-failure";
@@ -186,12 +195,15 @@ in {
         wantedBy = ["multi-user.target"];
         after = ["baseddata-prefect-server.target"];
         description = "Serves the Prefect flows";
+        path = ["/run/current-system/sw" "/srv/baseddata-models/.venv"];
         environment = {
           PREFECT_API_URL = "http://${containerIp}:4200/api";
           BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
+          NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
+          NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
+          LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
         };
         serviceConfig = {
-          Environment = "PATH=/run/current-system/sw/bin/";
           WorkingDirectory = "/srv/baseddata-models";
           ExecStartPre = "${pkgs.coreutils}/bin/timeout 120 ${pkgs.bash}/bin/bash -c 'until ${pkgs.netcat-openbsd}/bin/nc -z ${containerIp} 4200; do sleep 3; done'";
           ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
@@ -200,10 +212,14 @@ in {
             .venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite
             .venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite
             .venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite
+
             .venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite
             .venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite
             .venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite
             .venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite
+            .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
+            .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
+
             .venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite
             .venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite
             .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite
@@ -211,10 +227,9 @@ in {
 
             .venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite
             .venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite
-            .venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
+
-            .venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
+            .venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite
-            .venv/bin/prefect variable set "unique_key" "row_uuid" --overwrite
+
-            .venv/bin/prefect variable set "dbt_profiles_dir" "/srv/baseddata-models/data-models" --overwrite
 
             # serve flows
             .venv/bin/python automation/flows/serve-flows.py

hosts/common/users/sam/default.nix

@@ -98,36 +98,6 @@ in {
     '';
   };
 
-  sops.templates."dbt_profiles.yml" = {
-    path = "/home/${username}/.config/dbt/profiles.yml";
-    owner = "${username}";
-    mode = "0600";
-    content = ''
-      baseddata:
-        target: dev
-        outputs:
-          dev:
-            dbname: dev_baseddata
-            host: ${baseddataPostgresIp}
-            pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
-            port: 5432
-            schema: models
-            threads: 6
-            type: postgres
-            user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
-          prod:
-            dbname: baseddata
-            host: ${baseddataPostgresIp}
-            pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
-            port: 5432
-            schema: models
-            threads: 6
-            type: postgres
-            user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
-
-    '';
-  };
-
   nix = {
     extraOptions = ''
       experimental-features = nix-command flakes