sam
/
nixos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add dbt profiles configuration and update baseddata-worker 

- Add 'dbt_profiles.nix' file to configure dbt profiles for baseddata
- Update 'baseddata-worker.nix' to import dbt profiles and modify environment variables
- Remove dbt profiles configuration from user-specific default.nix for sam
This commit is contained in:
Sam 2024-12-28 10:07:25 +00:00
parent 8778496ca4
commit eaebfcaeb4
3 changed files with 47 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hosts/common/optional/dbt_profiles.nix Normal file
View File

@ -0,0 +1,26 @@
{configVars, config, ...}:
let
baseddataPostgresIp = configVars.networking.addresses.postgres.ip;
in
{
sops.templates."dbt_profiles.yml" = {
path = "/run/secrets/templates/dbt/profiles.yml";
mode = "0600";
content = ''
baseddata:
target: prod
outputs:
prod:
dbname: baseddata
host: ${baseddataPostgresIp}
pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
port: 5432
schema: models
threads: 6
type: postgres
user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
'';
};
}

27
hosts/common/optional/nixos-containers/baseddata-worker.nix
View File

@ -36,7 +36,6 @@ in {
"software/bitcoind/username" = {};
"software/bitcoind/bitcoin-rpcpassword-public" = {};
};
environment.persistence."/persist" = {
hideMounts = true;
directories = [
@ -44,6 +43,10 @@ in {
];
};
imports = [
../dbt_profiles.nix
];
networking.nat.enable = true;
networking.nat.internalInterfaces = ["ve-+"];
networking.nat.externalInterface = "br0";
@ -90,6 +93,10 @@ in {
hostPath = "/media/main-ssd/baseddata-data";
isReadOnly = false;
};
"/root/.dbt/profiles.yml" = {
hostPath = "/run/secrets/templates/dbt/profiles.yml";
isReadOnly = false;
};
};
config = {
@ -124,6 +131,7 @@ in {
pkgs.aria2
pkgs.osmctools
pkgs.osmium-tool
pkgs.osm2pgsql
];
environment.variables = {
@ -153,6 +161,7 @@ in {
cd /srv/baseddata-models
mkdir .venv
${pkgs.poetry}/bin/poetry lock --no-update
${pkgs.poetry}/bin/poetry install
'';
Restart = "on-failure";
@ -186,12 +195,15 @@ in {
wantedBy = ["multi-user.target"];
after = ["baseddata-prefect-server.target"];
description = "Serves the Prefect flows";
path = ["/run/current-system/sw" "/srv/baseddata-models/.venv"];
environment = {
PREFECT_API_URL = "http://${containerIp}:4200/api";
BASEDDATA_ENVIRONMENT = "${baseddataEnv}";
NIX_LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
NIX_LD = "/run/current-system/sw/share/nix-ld/lib/ld.so";
LD_LIBRARY_PATH = "/run/current-system/sw/share/nix-ld/lib";
};
serviceConfig = {
Environment = "PATH=/run/current-system/sw/bin/";
WorkingDirectory = "/srv/baseddata-models";
ExecStartPre = "${pkgs.coreutils}/bin/timeout 120 ${pkgs.bash}/bin/bash -c 'until ${pkgs.netcat-openbsd}/bin/nc -z ${containerIp} 4200; do sleep 3; done'";
ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
@ -200,10 +212,14 @@ in {
.venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite
.venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite
.venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite
.venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite
.venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite
.venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite
.venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite
.venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
.venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite
.venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite
@ -211,10 +227,9 @@ in {
.venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite
.venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite
.venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
.venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
.venv/bin/prefect variable set "unique_key" "row_uuid" --overwrite
.venv/bin/prefect variable set "dbt_profiles_dir" "/srv/baseddata-models/data-models" --overwrite
.venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite
# serve flows
.venv/bin/python automation/flows/serve-flows.py

30
hosts/common/users/sam/default.nix
View File

@ -98,36 +98,6 @@ in {
'';
};
sops.templates."dbt_profiles.yml" = {
path = "/home/${username}/.config/dbt/profiles.yml";
owner = "${username}";
mode = "0600";
content = ''
baseddata:
target: dev
outputs:
dev:
dbname: dev_baseddata
host: ${baseddataPostgresIp}
pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
port: 5432
schema: models
threads: 6
type: postgres
user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
prod:
dbname: baseddata
host: ${baseddataPostgresIp}
pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
port: 5432
schema: models
threads: 6
type: postgres
user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
'';
};
nix = {
extraOptions = ''
experimental-features = nix-command flakes