sam
/
nixos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

major refactor of baseddata container 

- included config for dbt and pgsync as sops templates
- import sops-nix into container
- remove unnecessary bind mounts
- update nix-secrets in flake.lock
This commit is contained in:
Sam 2025-01-13 21:15:17 +00:00
parent 6d4b95fd2c
commit aad9334639
6 changed files with 168 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
flake.lock
View File

@ -538,11 +538,11 @@
},
"nix-secrets": {
"locked": {
"lastModified": 1736678262,
"narHash": "sha256-LLKE6KboWC8l9llNrrLU/o6kVaHFHDsWEW8fKpX/6NI=",
"lastModified": 1736770787,
"narHash": "sha256-TEDwOm0ZnRioT01DdfS5XaFPYXA9uHA+P8pTOSFK0bI=",
"ref": "refs/heads/master",
"rev": "7a43e9814b20d7eac2a44d514402c64a54597f35",
"revCount": 198,
"rev": "a88c4ab6fb54527aabeeee2a2954338f81016f98",
"revCount": 201,
"type": "git",
"url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
},

25
hosts/common/optional/dbt_profiles.nix
View File

@ -1,25 +0,0 @@
{configVars, config, ...}:
let
baseddataPostgresIp = configVars.networking.addresses.postgres.ip;
in
{
sops.templates."dbt_profiles.yml" = {
mode = "0600";
content = ''
baseddata:
target: prod
outputs:
prod:
dbname: baseddata
host: ${baseddataPostgresIp}
pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
port: 5432
schema: models
threads: 6
type: postgres
user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
'';
};
}

189
hosts/common/optional/nixos-containers/baseddata-worker.nix
View File

@ -1,41 +1,13 @@
{
lib,
pkgs,
configVars,
inputs,
config,
...
}: let
pubKeys = lib.filesystem.listFilesRecursive ../../users/keys;
containerName = "bd-worker";
containerIp = configVars.networking.addresses.bd-worker.ip;
gatewayIp = configVars.networking.addresses.gateway.ip;
postgresIp = configVars.networking.addresses.postgres.ip;
postgresPort = toString configVars.networking.addresses.postgres.port;
bitcoindIp = configVars.networking.addresses.bitcoin-node.ip;
bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port;
#secrets
sshKeyFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."ssh_keys/baseddata-models-access/id_ed25519".path;
notifybotUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/username".path;
notifybotPwd = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/notifybot/password".path;
recipientUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."comms/xmpp/mrsu/username".path;
postgresUser = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_username".path;
postgresPassword = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/baseddata/user_password".path;
bitcoindRPCUsername = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/username".path;
bitcoindRPCPassword= lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path;
baseddataEnv = "dev";
sops-nix = inputs.sops-nix;
in {
sops.secrets = {
"ssh_keys/baseddata-models-access/id_ed25519" = {};
"comms/xmpp/notifybot/username" = {};
"comms/xmpp/notifybot/password" = {};
"comms/xmpp/mrsu/username" = {};
"software/postgres/baseddata/user_password" = {};
"software/postgres/baseddata/user_username" = {};
"software/bitcoind/username" = {};
"software/bitcoind/bitcoin-rpcpassword-public" = {};
};
environment.persistence."/persist" = {
hideMounts = true;
directories = [
@ -43,10 +15,6 @@ in {
];
};
imports = [
../dbt_profiles.nix
];
networking.nat.enable = true;
networking.nat.internalInterfaces = ["ve-+"];
networking.nat.externalInterface = "br0";
@ -56,54 +24,56 @@ in {
privateNetwork = true;
hostBridge = "br0";
nixpkgs = pkgs.path;
allowedDevices = [
{
node = "/dev/net/tun";
modifier = "rw";
}
];
bindMounts = {
"/root/.ssh/id_ed25519" = {
hostPath = "${sshKeyFile}";
isReadOnly = true;
};
"/run/secrets/notifybotUsername" = {
hostPath = "${notifybotUsername}";
isReadOnly = true;
};
"/run/secrets/notifybotPassword" = {
hostPath = "${notifybotPwd}";
isReadOnly = true;
};
"/run/secrets/recipientUsername" = {
hostPath = "${recipientUsername}";
isReadOnly = true;
};
"/run/secrets/postgresPassword" = {
hostPath = "${postgresPassword}";
isReadOnly = true;
};
"/run/secrets/postgresUser" = {
hostPath = "${postgresUser}";
isReadOnly = true;
};
"/run/secrets/bitcoindRPCPassword" = {
hostPath = "${bitcoindRPCPassword}";
isReadOnly = true;
};
"/run/secrets/bitcoindRPCUsername" = {
hostPath = "${bitcoindRPCUsername}";
"/etc/ssh/ssh_host_ed25519_key" = {
hostPath = "/etc/ssh/ssh_host_ed25519_key";
isReadOnly = true;
};
"/media/baseddata-data" = {
hostPath = "/media/main-ssd/baseddata-data";
isReadOnly = false;
};
"/root/.dbt/profiles.yml" = {
hostPath = "/run/secrets/rendered/dbt_profiles.yml";
isReadOnly = true;
};
};
config = {
pkgs,
lib,
config,
...
}: {
}: let
configVars = import ../../../../vars {inherit inputs lib;};
secretsDirectory = builtins.toString inputs.nix-secrets;
secretsFile = "${secretsDirectory}/secrets.yaml";
# define ip addresses
containerIp = configVars.networking.addresses.bd-worker.ip;
gatewayIp = configVars.networking.addresses.gateway.ip;
postgresIp = configVars.networking.addresses.postgres.ip;
postgresRemoteIp = configVars.networking.addresses.postgres-remote.ip;
postgresPort = toString configVars.networking.addresses.postgres.port;
bitcoindIp = configVars.networking.addresses.bitcoin-node.ip;
bitcoindPort = toString configVars.networking.addresses.bitcoin-node.services.bitcoind.port;
# define secret paths
notifybotUsername = config.sops.secrets."comms/xmpp/notifybot/username".path;
notifybotPassword = config.sops.secrets."comms/xmpp/notifybot/password".path;
recipientUsername = config.sops.secrets."comms/xmpp/mrsu/username".path;
postgresUser = config.sops.secrets."software/postgres/baseddata/user_username".path;
postgresPassword = config.sops.secrets."software/postgres/baseddata/user_password".path;
bitcoindRPCUsername = config.sops.secrets."software/bitcoind/username".path;
bitcoindRPCPassword = config.sops.secrets."software/bitcoind/bitcoin-rpcpassword-public".path;
dbtProfiles = config.sops.templates."profiles.yml".path;
pgsyncConfig = config.sops.templates."pgsync.yml".path;
baseddataEnv = "dev";
in {
networking = {
defaultGateway = "${gatewayIp}";
interfaces.eth0.ipv4.addresses = [
@ -121,7 +91,68 @@ in {
useHostResolvConf = lib.mkForce false;
};
sops = {
defaultSopsFile = "${secretsFile}";
validateSopsFiles = false;
age = {
sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"];
};
};
sops.secrets = {
"ssh_keys/baseddata-models-access/id_ed25519" = {
mode = "0400";
path = "/root/.ssh/id_ed25519";
};
"comms/xmpp/notifybot/username" = {};
"comms/xmpp/notifybot/password" = {};
"comms/xmpp/mrsu/username" = {};
"software/postgres/baseddata/user_password" = {};
"software/postgres/baseddata/user_username" = {};
"software/bitcoind/username" = {};
"software/bitcoind/bitcoin-rpcpassword-public" = {};
};
sops.templates."profiles.yml" = {
mode = "0600";
path = "/root/.dbt/profiles.yml";
content = ''
baseddata:
target: prod
outputs:
prod:
dbname: baseddata
host: ${postgresIp}
pass: '${config.sops.placeholder."software/postgres/baseddata/user_password"}'
port: 5432
schema: models
threads: 6
type: postgres
user: ${config.sops.placeholder."software/postgres/baseddata/user_username"}
'';
};
sops.templates."pgsync.yml" = {
mode = "0600";
path = "/root/.pgsync.yml";
content = ''
from: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresIp}/baseddata
to: postgresql://${config.sops.placeholder."software/postgres/baseddata/user_username"}:${config.sops.placeholder."software/postgres/baseddata/user_password"}@${postgresRemoteIp}/baseddata
schemas:
- models_final
to_safe: true
'';
};
imports = [
sops-nix.nixosModules.sops
];
services.resolved.enable = true;
services.tailscale.enable = true;
environment.systemPackages = [
pkgs.vim
@ -132,6 +163,9 @@ in {
pkgs.osmctools
pkgs.osmium-tool
pkgs.osm2pgsql
pkgs.pgsync
pkgs.postgresql_16
pkgs.htop
];
environment.variables = {
@ -184,7 +218,7 @@ in {
ExecStart = pkgs.writeShellScript "baseddata-prefect-server" ''
# run prefect server
.venv/bin/prefect server start --host 0.0.0.0
.venv/bin/prefect server start --host ${containerIp}
'';
Restart = "on-failure";
@ -209,26 +243,27 @@ in {
ExecStart = pkgs.writeShellScript "baseddata-serve-flows" ''
# set prefect environment variables
.venv/bin/prefect variable set "xmpp_jid" $(cat /run/secrets/notifybotUsername) --overwrite
.venv/bin/prefect variable set "xmpp_password" $(cat /run/secrets/notifybotPassword) --overwrite
.venv/bin/prefect variable set "xmpp_recipient" $(cat /run/secrets/recipientUsername) --overwrite
.venv/bin/prefect variable set "xmpp_jid" $(cat ${notifybotUsername}) --overwrite
.venv/bin/prefect variable set "xmpp_password" $(cat ${notifybotPassword}) --overwrite
.venv/bin/prefect variable set "xmpp_recipient" $(cat ${recipientUsername}) --overwrite
.venv/bin/prefect variable set "postgres_host" ${postgresIp} --overwrite
.venv/bin/prefect variable set "postgres_port" ${postgresPort} --overwrite
.venv/bin/prefect variable set "postgres_user" $(cat /run/secrets/postgresUser) --overwrite
.venv/bin/prefect variable set "postgres_pwd" $(cat /run/secrets/postgresPassword) --overwrite
.venv/bin/prefect variable set "postgres_user" $(cat ${postgresUser}) --overwrite
.venv/bin/prefect variable set "postgres_pwd" $(cat ${postgresPassword}) --overwrite
.venv/bin/prefect variable set "postgres_dbname" "baseddata" --overwrite
.venv/bin/prefect variable set "postgres_schema" "models_final" --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_password" $(cat /run/secrets/bitcoindRPCPassword) --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_username" $(cat /run/secrets/bitcoindRPCUsername) --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_password" $(cat ${bitcoindRPCPassword}) --overwrite
.venv/bin/prefect variable set "bitcoin_rpc_username" $(cat ${bitcoindRPCUsername}) --overwrite
.venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite
.venv/bin/prefect variable set "bitcoind_port" ${bitcoindPort} --overwrite
.venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite
.venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite
.venv/bin/prefect variable set "dbt_profiles_dir" "/root/.dbt" --overwrite
.venv/bin/prefect variable set "dbt_profiles_dir" $(dirname ${dbtProfiles}) --overwrite
.venv/bin/prefect variable set "pgsync_config" ${pgsyncConfig} --overwrite
# serve flows

6
hosts/common/optional/nixos-containers/docker.nix
View File

@ -1,8 +1,8 @@
{
pkgs,
configVars,
lib,
inputs,
configVars,
...
}: let
containerName = "docker";
@ -66,6 +66,10 @@ in {
node = "/dev/dri/renderD128";
modifier = "rwm";
}
{
node = "/dev/net/tun";
modifier = "rw";
}
];
######

53
hosts/common/optional/nixos-containers/postgres.nix
View File

@ -77,14 +77,31 @@ in {
extensions = with pkgs.postgresql_16.pkgs; [postgis];
enableTCPIP = true;
settings = {
max_worker_processes = "12";
max_parallel_workers = "8";
max_parallel_workers_per_gather = "4";
max_connections = "100";
autovacuum_work_mem = "2GB";
shared_buffers = "32GB";
work_mem = "0.32GB";
maintenance_work_mem = "64MB";
# max_worker_processes = "12";
# max_parallel_workers = "8";
# max_parallel_workers_per_gather = "4";
# max_connections = "100";
# autovacuum_work_mem = "2GB";
# shared_buffers = "32GB";
# work_mem = "0.32GB";
# maintenance_work_mem = "64MB";
max_connections = "20";
shared_buffers = "2GB";
effective_cache_size = "6GB";
maintenance_work_mem = "1GB";
checkpoint_completion_target = "0.9";
wal_buffers = "16MB";
default_statistics_target = "500";
random_page_cost = "1.1";
effective_io_concurrency = "200";
work_mem = "17476kB";
huge_pages = "off";
min_wal_size = "4GB";
max_wal_size = "16GB";
max_worker_processes = "6";
max_parallel_workers_per_gather = "3";
max_parallel_workers = "6";
max_parallel_maintenance_workers = "3";
};
authentication = pkgs.lib.mkOverride 10 ''
#type database DBuser origin-address auth-method
@ -95,16 +112,16 @@ in {
'';
};
systemd.services.postgresql.postStart = ''
$PSQL -tA <<'EOF'
DO $$
DECLARE password TEXT;
BEGIN
password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', '''));
EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password);
END $$;
EOF
'';
# systemd.services.postgresql.postStart = ''
# $PSQL -tA <<'EOF'
# DO $$
# DECLARE password TEXT;
# BEGIN
# password := trim(both from replace(pg_read_file('${postgresPasswordPath}'), E'\n', '''));
# EXECUTE format('ALTER ROLE postgres WITH PASSWORD '''%s''';', password);
# END $$;
# EOF
# '';
services.openssh = {
enable = true;

12
hosts/common/optional/secrets.nix Normal file
View File

@ -0,0 +1,12 @@
{
sops.secrets = {
"ssh_keys/baseddata-models-access/id_ed25519" = {};
"comms/xmpp/notifybot/username" = {};
"comms/xmpp/notifybot/password" = {};
"comms/xmpp/mrsu/username" = {};
"software/postgres/baseddata/user_password" = {};
"software/postgres/baseddata/user_username" = {};
"software/bitcoind/username" = {};
"software/bitcoind/bitcoin-rpcpassword-public" = {};
};
}