From bd719c72fa1cc5d63d9c6dada05e28d749ec6fa3 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:56:18 +0100 Subject: [PATCH 01/30] Update flake secrets --- flake.lock | 143 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 5809752..72ca555 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,27 @@ { "nodes": { + "arion": { + "inputs": { + "flake-parts": "flake-parts", + "haskell-flake": "haskell-flake", + "hercules-ci-effects": "hercules-ci-effects", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1720147808, + "narHash": "sha256-hlWEQGUbIwYb+vnd8egzlW/P++yKu3HjV/rOdOPVank=", + "owner": "hercules-ci", + "repo": "arion", + "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "arion", + "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318", + "type": "github" + } + }, "base16-schemes": { "flake": false, "locked": { @@ -90,6 +112,48 @@ } }, "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "arion", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "arion", + "hercules-ci-effects", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-parts_3": { "inputs": { "nixpkgs-lib": [ "nixvim", @@ -177,6 +241,44 @@ "type": "github" } }, + "haskell-flake": { + "locked": { + "lastModified": 1675296942, + "narHash": "sha256-u1X1sblozi5qYEcLp1hxcyo8FfDHnRUVX3dJ/tW19jY=", + "owner": "srid", + "repo": "haskell-flake", + "rev": "c2cafce9d57bfca41794dc3b99c593155006c71e", + "type": "github" + }, + "original": { + "owner": "srid", + "ref": "0.1.0", + "repo": "haskell-flake", + "type": "github" + } + }, + "hercules-ci-effects": { + "inputs": { + "flake-parts": "flake-parts_2", + "nixpkgs": [ + "arion", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719226092, + "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=", + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "type": "github" + } + }, "home-manager": { "inputs": { "nixpkgs": [ @@ -278,11 +380,11 @@ "nix-secrets": { "flake": false, "locked": { - "lastModified": 1719601133, - "narHash": "sha256-2+e92LyX1fFj3mIZft+K8OzR9NT/1xtheO8hO/3DyRc=", + "lastModified": 1719686367, + "narHash": "sha256-zQ/Mgrg3GjE4QkweXPLAtbO8SnfzTXZrqmm8oZwXBV4=", "ref": "refs/heads/master", - "rev": "278ccbbd646e86cab5fd38d43d9134270d8123d0", - "revCount": 141, + "rev": "eb8d568c7e30a8c45148fa5c235ebd49bc8effee", + "revCount": 148, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, @@ -293,16 +395,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1719426051, - "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=", - "owner": "nixos", + "lastModified": 1720031269, + "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd", + "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", "type": "github" }, "original": { - "owner": "nixos", - "ref": "nixos-24.05", + "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } @@ -354,11 +456,27 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1719426051, + "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, "nixvim": { "inputs": { "devshell": "devshell", "flake-compat": "flake-compat", - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_3", "git-hooks": "git-hooks", "home-manager": "home-manager_2", "nix-darwin": "nix-darwin", @@ -399,12 +517,13 @@ }, "root": { "inputs": { + "arion": "arion", "disko": "disko", "home-manager": "home-manager", "impermanence": "impermanence", "nix-colors": "nix-colors", "nix-secrets": "nix-secrets", - "nixpkgs": "nixpkgs", + "nixpkgs": "nixpkgs_2", "nixpkgs-unstable": "nixpkgs-unstable", "nixvim": "nixvim", "nur": "nur", From 33981eea6d1738cf8f6b6913ee5ca64e57a729af Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:56:41 +0100 Subject: [PATCH 02/30] Remove deploy_key from sops --- hosts/common/core/sops.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/hosts/common/core/sops.nix b/hosts/common/core/sops.nix index 6cb172f..e217bcc 100644 --- a/hosts/common/core/sops.nix +++ b/hosts/common/core/sops.nix @@ -21,9 +21,6 @@ in }; secrets = { "passwords/root".neededForUsers = true; - "ssh_keys/deploy_key/id_ed25519" = { - path = "/etc/ssh/deploy_key-ssh-ed25519"; - }; }; }; } From 92d09646fa70df7d72599450bbf592f280023d8b Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:57:17 +0100 Subject: [PATCH 03/30] Add postgres btrfs zvol --- hosts/common/disks/zfs/zspeed.nix | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hosts/common/disks/zfs/zspeed.nix b/hosts/common/disks/zfs/zspeed.nix index ead5709..9133e71 100644 --- a/hosts/common/disks/zfs/zspeed.nix +++ b/hosts/common/disks/zfs/zspeed.nix @@ -53,14 +53,11 @@ postgres = { type = "zfs_volume"; size = "10G -s"; - options = { - "com.sun:auto-snapshot:daily" = "true"; - "volblocksize" = "8k"; + content = { + type = "filesystem"; + format = "btrfs"; + mountpoint = "/postgres"; }; - }; - lxc = { - type = "zfs_volume"; - size = "10G -s"; options = { "com.sun:auto-snapshot:daily" = "true"; "volblocksize" = "8k"; From 9ace130029582859aa256d023b0cdd9af8314191 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:58:03 +0100 Subject: [PATCH 04/30] Docker and postgres config --- hosts/common/optional/docker/default.nix | 20 ++++++++++++++++ hosts/common/optional/docker/postgres.nix | 29 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 hosts/common/optional/docker/default.nix create mode 100644 hosts/common/optional/docker/postgres.nix diff --git a/hosts/common/optional/docker/default.nix b/hosts/common/optional/docker/default.nix new file mode 100644 index 0000000..05426f8 --- /dev/null +++ b/hosts/common/optional/docker/default.nix @@ -0,0 +1,20 @@ +{ inputs, ... }: { + + imports = [ inputs.arion.nixosModules.arion ]; + # Arion works with Docker, but for NixOS-based containers, you need Podman + # since NixOS 21.05. + virtualisation = { + podman = { + enable = true; + defaultNetwork.settings.dns_enabled = true; + }; + docker = { + storageDriver = "btrfs"; + rootless = { + enable = true; + setSocketVariable = true; + }; + }; + }; + +} diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix new file mode 100644 index 0000000..8138b22 --- /dev/null +++ b/hosts/common/optional/docker/postgres.nix @@ -0,0 +1,29 @@ +{ pkgs, ... }: +{ + virtualisation.arion = { + backend = "docker"; + projects = { + "db".settings.services."db".service = { + image = "postgres:16"; + restart = "unless-stopped"; + environment = { + POSTGRES_PASSWORD = "balls1234"; + POSTGRES_USER = "admin"; + POSTGRES_DATABASE = "test_db"; + }; + ports = [ "5432:5432" ]; + volumes = [ + "/mnt/postgres/data:/var/lib/postgres/data" + "${pkgs.writeScript "load_extensions" '' + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" < Date: Fri, 5 Jul 2024 18:58:30 +0100 Subject: [PATCH 05/30] Arion flake input --- flake.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index 2698fb2..a933d0b 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; + # Arion for docker + arion = { + url = "github:hercules-ci/arion/236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318"; + }; + # Nix colors nix-colors.url = "github:misterio77/nix-colors"; From a71ee506d3bbbd4c7bf093ddcbb51c79f7e749c4 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:58:46 +0100 Subject: [PATCH 06/30] Podman to user groups --- hosts/common/users/admin/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/common/users/admin/default.nix b/hosts/common/users/admin/default.nix index 670f5b2..462a97a 100644 --- a/hosts/common/users/admin/default.nix +++ b/hosts/common/users/admin/default.nix @@ -22,6 +22,7 @@ in "docker" "lxc" "git" + "podman" ]; packages = with pkgs; [ From f71ece31f16cccdb3d232f20851a4718837466f6 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:59:10 +0100 Subject: [PATCH 07/30] Minor fixes --- hosts/nebula/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hosts/nebula/default.nix b/hosts/nebula/default.nix index 366c277..9897458 100644 --- a/hosts/nebula/default.nix +++ b/hosts/nebula/default.nix @@ -2,9 +2,9 @@ let # Disko setup fsType = "btrfs"; # one of ext4 or btrfs. Use btrfs if using impermanence - dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; # depends on target hardware + dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; encrypted = false; # currrently only applies to btrfs - impermanence = false; # currrently only applies to btrfs + impermanence = false; user = "admin"; in { @@ -23,7 +23,8 @@ in # Import optional options ../common/optional/openssh.nix - ../common/optional/lxd + ../common/optional/docker + ../common/optional/docker/postgres.nix ]; From d29250a2a69a9b3a031a75b7da134d706d8ffc97 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:26:08 +0100 Subject: [PATCH 08/30] Build postgres using dockerfile - use dockerfile to install postgis during build --- hosts/common/optional/docker/postgres.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 8138b22..2b78fea 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -4,8 +4,16 @@ backend = "docker"; projects = { "db".settings.services."db".service = { - image = "postgres:16"; restart = "unless-stopped"; + build.context = "/nix/store"; + build.dockerfile = builtins.baseNameOf "${pkgs.writeScript "pgDockerfile" '' + FROM postgres:16 + # install packages + RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-16-postgis \ + && rm -rf /var/lib/apt/lists/* + ''}"; environment = { POSTGRES_PASSWORD = "balls1234"; POSTGRES_USER = "admin"; From 1e95ba6c36ec7839ae9f26b78db6657eee7da80f Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:27:15 +0100 Subject: [PATCH 09/30] pgdata dir and admin_db default database --- hosts/common/optional/docker/postgres.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 2b78fea..22e0293 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -17,7 +17,8 @@ environment = { POSTGRES_PASSWORD = "balls1234"; POSTGRES_USER = "admin"; - POSTGRES_DATABASE = "test_db"; + POSTGRES_DB = "admin_db"; + PGDATA = "/var/lib/postgresql/data/pgdata"; }; ports = [ "5432:5432" ]; volumes = [ From 3b7a597d8f139ec67be18d46aefb51f9240207e7 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:28:09 +0100 Subject: [PATCH 10/30] pg init script to configure db on start - create users & dbs - setup db permissions - install extensions --- hosts/common/optional/docker/postgres.nix | 38 ++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 22e0293..7ccc75f 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -22,13 +22,37 @@ }; ports = [ "5432:5432" ]; volumes = [ - "/mnt/postgres/data:/var/lib/postgres/data" - "${pkgs.writeScript "load_extensions" '' - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" < Date: Sat, 6 Jul 2024 16:01:17 +0100 Subject: [PATCH 11/30] Add arion package --- hosts/common/optional/docker/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/default.nix b/hosts/common/optional/docker/default.nix index 05426f8..b6dba80 100644 --- a/hosts/common/optional/docker/default.nix +++ b/hosts/common/optional/docker/default.nix @@ -1,6 +1,11 @@ -{ inputs, ... }: { +{ pkgs, inputs, ... }: +{ imports = [ inputs.arion.nixosModules.arion ]; + environment.systemPackages = [ + pkgs.arion + ]; + # Arion works with Docker, but for NixOS-based containers, you need Podman # since NixOS 21.05. virtualisation = { @@ -9,6 +14,7 @@ defaultNetwork.settings.dns_enabled = true; }; docker = { + enable = true; storageDriver = "btrfs"; rootless = { enable = true; From e419389862c215334076610c590e59069935c351 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 16:01:40 +0100 Subject: [PATCH 12/30] Update nix-secrets --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 72ca555..9f30b93 100644 --- a/flake.lock +++ b/flake.lock @@ -380,11 +380,11 @@ "nix-secrets": { "flake": false, "locked": { - "lastModified": 1719686367, - "narHash": "sha256-zQ/Mgrg3GjE4QkweXPLAtbO8SnfzTXZrqmm8oZwXBV4=", + "lastModified": 1720263046, + "narHash": "sha256-6tJLK4EtB4IXBO4i6P/Ulf03Bd7GaEezT7AebN3VPHA=", "ref": "refs/heads/master", - "rev": "eb8d568c7e30a8c45148fa5c235ebd49bc8effee", - "revCount": 148, + "rev": "33d677fea187322e503f8a56d9c75ff7e7df057c", + "revCount": 151, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, From 2f0ddf8375db2b487d27ecaa474825de69f8198c Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 16:02:10 +0100 Subject: [PATCH 13/30] Modify postgres docker container - add sops-secrets for admin pwd - POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions - initdb docker entrypoint script to create dbs, users and extensions from json --- hosts/common/optional/docker/postgres.nix | 103 ++++++++++++++++------ 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 7ccc75f..c5d7c3e 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -1,5 +1,11 @@ -{ pkgs, ... }: +{ pkgs, lib, inputs, config, ... }: +let + admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path; +in { + sops.secrets = { + "software/postgres/admin_db/password" = { }; + }; virtualisation.arion = { backend = "docker"; projects = { @@ -12,51 +18,92 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ postgresql-16-postgis \ + jq \ && rm -rf /var/lib/apt/lists/* ''}"; environment = { - POSTGRES_PASSWORD = "balls1234"; + POSTGRES_PASSWORD_FILE = admin_dbPasswordFile; POSTGRES_USER = "admin"; POSTGRES_DB = "admin_db"; PGDATA = "/var/lib/postgresql/data/pgdata"; + POSTGRES_MULTIPLE_DATABASES = '' + [ + { + "osm": { + "user": "gis", + "extensions": [ + "hstore", + "postgis" + ] + }, + "bitcoin": { + "user": "satoshi", + "extensions": [] + }, + "btc_models": { + "user": "dbt", + "extensions": [] + }, + "dev_btc_models": { + "user": "dbt", + "extensions": [] + } + "test": { + "user": "test", + "extensions": [hstore] + } + } + ] + ''; }; ports = [ "5432:5432" ]; volumes = [ "/mnt/postgres:/var/lib/postgresql/data" - # PG init script + # Need to mount secret file + "${admin_dbPasswordFile}:${admin_dbPasswordFile}" + + # PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES + # creates databases, users and installs extensions for each database. "${pkgs.writeScript "init.sh" '' #!/bin/bash - - # Create additional databases - psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - create database bitcoin; - create database osm; + function create_user_and_database() { + local database=$1 + local user=$2 + local extensions=$3 + echo "### admin user: $POSTGRES_USER ###" + echo " Creating database '$database'" + echo " Creating user '$user'" + psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL + CREATE USER $user; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $user; EOSQL - - # Create additional users - psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - create user gis; - create user satoshi; - EOSQL - - # Setup bitcoin db - psql -v --username "$POSTGRES_USER" --dbname "bitcoin" <<-EOSQL - grant all privileges on database bitcoin to satoshi; - EOSQL - - # Setup osm db - psql -v --username "$POSTGRES_USER" --dbname "osm" <<-EOSQL - grant all privileges on database osm to gis; - create extension if not exists postgis; - create extension if not exists hstore; - EOSQL - + + # Loop through extensions and create them + for ext in $(echo "$extensions" | tr ',' ' '); do + echo " - Installing extention $ext" + psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" + done + } + + if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + + # Parse the JSON string + database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') + echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" + + # Loop through each database and create it + for db_name in $database_names; do + user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") + extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") + create_user_and_database "$db_name" "$user" "$extensions" + done + fi ''}:/docker-entrypoint-initdb.d/init.sh" ]; }; }; }; - } From 271b5958b8962afa737f8ce9c00362aa602a7840 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 20:53:26 +0100 Subject: [PATCH 14/30] Postgres docker configuration --- hosts/common/optional/docker/postgres.nix | 116 ++++++++++++++-------- 1 file changed, 75 insertions(+), 41 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index c5d7c3e..f698c4d 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -1,6 +1,72 @@ { pkgs, lib, inputs, config, ... }: let admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path; + initScript = pkgs.writeText "init.sh" '' + #!/bin/bash + function create_user_and_database() { + local database=$1 + local user=$2 + local extensions=$3 + echo "### admin user: $POSTGRES_USER ###" + echo " Creating database '$database'" + echo " Creating user '$user'" + psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL + CREATE USER $user; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $user; + EOSQL + + # Loop through extensions and create them + for ext in $(echo "$extensions" | tr ',' ' '); do + echo " - Installing extention $ext" + psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" + done + } + + if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + + # Parse the JSON string + database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') + echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" + + # Loop through each database and create it + for db_name in $database_names; do + user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") + extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") + create_user_and_database "$db_name" "$user" "$extensions" + done + fi + ''; + + pg_hbaConfig = pkgs.writeText "pg_hba.conf" '' + none + ''; + + pgsqlConfig = pkgs.writeText "postgresql.conf" '' + listen_addresses = '*' + port = 5432 + max_connections = 100 + shared_buffers = 24GB + work_mem = 1GB + maintenance_work_mem = 10GB + autovacuum_work_mem = 2GB + dynamic_shared_memory_type = posix + wal_level = minimal + checkpoint_timeout = 60min + checkpoint_completion_target = 0.9 + max_wal_size = 10GB + min_wal_size = 80MB + max_wal_senders = 0 + random_page_cost = 1.0 + effective_cache_size = 25GB + jit = off + log_line_prefix = '%m [%p] %q%u@%d ' + log_timezone = 'Etc/UTC' + cluster_name = 'postgres-docker' + datestyle = 'iso, dmy' + timezone = 'Etc/UTC' + default_text_search_config = 'pg_catalog.english' + ''; in { sops.secrets = { @@ -12,7 +78,7 @@ in "db".settings.services."db".service = { restart = "unless-stopped"; build.context = "/nix/store"; - build.dockerfile = builtins.baseNameOf "${pkgs.writeScript "pgDockerfile" '' + build.dockerfile = builtins.baseNameOf "${pkgs.writeText "pgDockerfile" '' FROM postgres:16 # install packages RUN apt-get update \ @@ -21,6 +87,7 @@ in jq \ && rm -rf /var/lib/apt/lists/* ''}"; + command = [ "postgres" "-c" "config_file=/etc/postgresql/postgresql.conf" ]; environment = { POSTGRES_PASSWORD_FILE = admin_dbPasswordFile; POSTGRES_USER = "admin"; @@ -48,59 +115,26 @@ in "user": "dbt", "extensions": [] } - "test": { - "user": "test", - "extensions": [hstore] - } } ] ''; }; ports = [ "5432:5432" ]; volumes = [ + + # Mount pgdata to external zfs volume "/mnt/postgres:/var/lib/postgresql/data" + # Mount config files + # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" + "${pgsqlConfig}:/etc/postgresql/postgresql.conf" + # Need to mount secret file "${admin_dbPasswordFile}:${admin_dbPasswordFile}" # PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES # creates databases, users and installs extensions for each database. - "${pkgs.writeScript "init.sh" '' - #!/bin/bash - function create_user_and_database() { - local database=$1 - local user=$2 - local extensions=$3 - echo "### admin user: $POSTGRES_USER ###" - echo " Creating database '$database'" - echo " Creating user '$user'" - psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL - CREATE USER $user; - CREATE DATABASE $database; - GRANT ALL PRIVILEGES ON DATABASE $database TO $user; - EOSQL - - # Loop through extensions and create them - for ext in $(echo "$extensions" | tr ',' ' '); do - echo " - Installing extention $ext" - psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" - done - } - - if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then - - # Parse the JSON string - database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') - echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" - - # Loop through each database and create it - for db_name in $database_names; do - user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") - extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") - create_user_and_database "$db_name" "$user" "$extensions" - done - fi - ''}:/docker-entrypoint-initdb.d/init.sh" + "${initScript}:/docker-entrypoint-initdb.d/init.sh" ]; }; }; From b8f85256a7b546d96a176259e0ebbdc418681a52 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 21:17:32 +0100 Subject: [PATCH 15/30] small fix --- hosts/common/optional/docker/postgres.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index f698c4d..75e2ac8 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -126,7 +126,7 @@ in "/mnt/postgres:/var/lib/postgresql/data" # Mount config files - # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" + # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" "${pgsqlConfig}:/etc/postgresql/postgresql.conf" # Need to mount secret file From 67e3d9ddedb2547826a55e15a2869d99aacdba8e Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:56:18 +0100 Subject: [PATCH 16/30] Update flake secrets --- flake.lock | 143 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 5809752..72ca555 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,27 @@ { "nodes": { + "arion": { + "inputs": { + "flake-parts": "flake-parts", + "haskell-flake": "haskell-flake", + "hercules-ci-effects": "hercules-ci-effects", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1720147808, + "narHash": "sha256-hlWEQGUbIwYb+vnd8egzlW/P++yKu3HjV/rOdOPVank=", + "owner": "hercules-ci", + "repo": "arion", + "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "arion", + "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318", + "type": "github" + } + }, "base16-schemes": { "flake": false, "locked": { @@ -90,6 +112,48 @@ } }, "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "arion", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "arion", + "hercules-ci-effects", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-parts_3": { "inputs": { "nixpkgs-lib": [ "nixvim", @@ -177,6 +241,44 @@ "type": "github" } }, + "haskell-flake": { + "locked": { + "lastModified": 1675296942, + "narHash": "sha256-u1X1sblozi5qYEcLp1hxcyo8FfDHnRUVX3dJ/tW19jY=", + "owner": "srid", + "repo": "haskell-flake", + "rev": "c2cafce9d57bfca41794dc3b99c593155006c71e", + "type": "github" + }, + "original": { + "owner": "srid", + "ref": "0.1.0", + "repo": "haskell-flake", + "type": "github" + } + }, + "hercules-ci-effects": { + "inputs": { + "flake-parts": "flake-parts_2", + "nixpkgs": [ + "arion", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719226092, + "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=", + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "hercules-ci-effects", + "type": "github" + } + }, "home-manager": { "inputs": { "nixpkgs": [ @@ -278,11 +380,11 @@ "nix-secrets": { "flake": false, "locked": { - "lastModified": 1719601133, - "narHash": "sha256-2+e92LyX1fFj3mIZft+K8OzR9NT/1xtheO8hO/3DyRc=", + "lastModified": 1719686367, + "narHash": "sha256-zQ/Mgrg3GjE4QkweXPLAtbO8SnfzTXZrqmm8oZwXBV4=", "ref": "refs/heads/master", - "rev": "278ccbbd646e86cab5fd38d43d9134270d8123d0", - "revCount": 141, + "rev": "eb8d568c7e30a8c45148fa5c235ebd49bc8effee", + "revCount": 148, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, @@ -293,16 +395,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1719426051, - "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=", - "owner": "nixos", + "lastModified": 1720031269, + "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd", + "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", "type": "github" }, "original": { - "owner": "nixos", - "ref": "nixos-24.05", + "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } @@ -354,11 +456,27 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1719426051, + "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, "nixvim": { "inputs": { "devshell": "devshell", "flake-compat": "flake-compat", - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_3", "git-hooks": "git-hooks", "home-manager": "home-manager_2", "nix-darwin": "nix-darwin", @@ -399,12 +517,13 @@ }, "root": { "inputs": { + "arion": "arion", "disko": "disko", "home-manager": "home-manager", "impermanence": "impermanence", "nix-colors": "nix-colors", "nix-secrets": "nix-secrets", - "nixpkgs": "nixpkgs", + "nixpkgs": "nixpkgs_2", "nixpkgs-unstable": "nixpkgs-unstable", "nixvim": "nixvim", "nur": "nur", From fc2f6f4ca39f67d46c23a7820dec8f8e2e1e825b Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:56:41 +0100 Subject: [PATCH 17/30] Remove deploy_key from sops --- hosts/common/core/sops.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/hosts/common/core/sops.nix b/hosts/common/core/sops.nix index 6cb172f..e217bcc 100644 --- a/hosts/common/core/sops.nix +++ b/hosts/common/core/sops.nix @@ -21,9 +21,6 @@ in }; secrets = { "passwords/root".neededForUsers = true; - "ssh_keys/deploy_key/id_ed25519" = { - path = "/etc/ssh/deploy_key-ssh-ed25519"; - }; }; }; } From c9ee7c7e80eb392a2ba2c8ef16ad8d4b2098ed78 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:57:17 +0100 Subject: [PATCH 18/30] Add postgres btrfs zvol --- hosts/common/disks/zfs/zspeed.nix | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hosts/common/disks/zfs/zspeed.nix b/hosts/common/disks/zfs/zspeed.nix index ead5709..9133e71 100644 --- a/hosts/common/disks/zfs/zspeed.nix +++ b/hosts/common/disks/zfs/zspeed.nix @@ -53,14 +53,11 @@ postgres = { type = "zfs_volume"; size = "10G -s"; - options = { - "com.sun:auto-snapshot:daily" = "true"; - "volblocksize" = "8k"; + content = { + type = "filesystem"; + format = "btrfs"; + mountpoint = "/postgres"; }; - }; - lxc = { - type = "zfs_volume"; - size = "10G -s"; options = { "com.sun:auto-snapshot:daily" = "true"; "volblocksize" = "8k"; From 52a3b85c8fbee2c03eaa19abd31dbea4e52e2f73 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:58:03 +0100 Subject: [PATCH 19/30] Docker and postgres config --- hosts/common/optional/docker/default.nix | 20 ++++++++++++++++ hosts/common/optional/docker/postgres.nix | 29 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 hosts/common/optional/docker/default.nix create mode 100644 hosts/common/optional/docker/postgres.nix diff --git a/hosts/common/optional/docker/default.nix b/hosts/common/optional/docker/default.nix new file mode 100644 index 0000000..05426f8 --- /dev/null +++ b/hosts/common/optional/docker/default.nix @@ -0,0 +1,20 @@ +{ inputs, ... }: { + + imports = [ inputs.arion.nixosModules.arion ]; + # Arion works with Docker, but for NixOS-based containers, you need Podman + # since NixOS 21.05. + virtualisation = { + podman = { + enable = true; + defaultNetwork.settings.dns_enabled = true; + }; + docker = { + storageDriver = "btrfs"; + rootless = { + enable = true; + setSocketVariable = true; + }; + }; + }; + +} diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix new file mode 100644 index 0000000..8138b22 --- /dev/null +++ b/hosts/common/optional/docker/postgres.nix @@ -0,0 +1,29 @@ +{ pkgs, ... }: +{ + virtualisation.arion = { + backend = "docker"; + projects = { + "db".settings.services."db".service = { + image = "postgres:16"; + restart = "unless-stopped"; + environment = { + POSTGRES_PASSWORD = "balls1234"; + POSTGRES_USER = "admin"; + POSTGRES_DATABASE = "test_db"; + }; + ports = [ "5432:5432" ]; + volumes = [ + "/mnt/postgres/data:/var/lib/postgres/data" + "${pkgs.writeScript "load_extensions" '' + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" < Date: Fri, 5 Jul 2024 18:58:30 +0100 Subject: [PATCH 20/30] Arion flake input --- flake.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index 2698fb2..a933d0b 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,11 @@ inputs.nixpkgs.follows = "nixpkgs"; }; + # Arion for docker + arion = { + url = "github:hercules-ci/arion/236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318"; + }; + # Nix colors nix-colors.url = "github:misterio77/nix-colors"; From 052c941e81fe80092975405d1100737dabdbea9c Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:58:46 +0100 Subject: [PATCH 21/30] Podman to user groups --- hosts/common/users/admin/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/common/users/admin/default.nix b/hosts/common/users/admin/default.nix index 670f5b2..462a97a 100644 --- a/hosts/common/users/admin/default.nix +++ b/hosts/common/users/admin/default.nix @@ -22,6 +22,7 @@ in "docker" "lxc" "git" + "podman" ]; packages = with pkgs; [ From 7df7970414306fede54ec767a14e35d599f81c1e Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jul 2024 18:59:10 +0100 Subject: [PATCH 22/30] Minor fixes --- hosts/nebula/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hosts/nebula/default.nix b/hosts/nebula/default.nix index 366c277..9897458 100644 --- a/hosts/nebula/default.nix +++ b/hosts/nebula/default.nix @@ -2,9 +2,9 @@ let # Disko setup fsType = "btrfs"; # one of ext4 or btrfs. Use btrfs if using impermanence - dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; # depends on target hardware + dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; encrypted = false; # currrently only applies to btrfs - impermanence = false; # currrently only applies to btrfs + impermanence = false; user = "admin"; in { @@ -23,7 +23,8 @@ in # Import optional options ../common/optional/openssh.nix - ../common/optional/lxd + ../common/optional/docker + ../common/optional/docker/postgres.nix ]; From 591a9ce48f066f98d1c8f54708ae74de25783faa Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:26:08 +0100 Subject: [PATCH 23/30] Build postgres using dockerfile - use dockerfile to install postgis during build --- hosts/common/optional/docker/postgres.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 8138b22..2b78fea 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -4,8 +4,16 @@ backend = "docker"; projects = { "db".settings.services."db".service = { - image = "postgres:16"; restart = "unless-stopped"; + build.context = "/nix/store"; + build.dockerfile = builtins.baseNameOf "${pkgs.writeScript "pgDockerfile" '' + FROM postgres:16 + # install packages + RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-16-postgis \ + && rm -rf /var/lib/apt/lists/* + ''}"; environment = { POSTGRES_PASSWORD = "balls1234"; POSTGRES_USER = "admin"; From baaaa3e8d60a44c449154c447a8ec819b4908fc4 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:27:15 +0100 Subject: [PATCH 24/30] pgdata dir and admin_db default database --- hosts/common/optional/docker/postgres.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 2b78fea..22e0293 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -17,7 +17,8 @@ environment = { POSTGRES_PASSWORD = "balls1234"; POSTGRES_USER = "admin"; - POSTGRES_DATABASE = "test_db"; + POSTGRES_DB = "admin_db"; + PGDATA = "/var/lib/postgresql/data/pgdata"; }; ports = [ "5432:5432" ]; volumes = [ From 491350bc58243df52e4c76fbaea653541ef658fe Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 10:28:09 +0100 Subject: [PATCH 25/30] pg init script to configure db on start - create users & dbs - setup db permissions - install extensions --- hosts/common/optional/docker/postgres.nix | 38 ++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 22e0293..7ccc75f 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -22,13 +22,37 @@ }; ports = [ "5432:5432" ]; volumes = [ - "/mnt/postgres/data:/var/lib/postgres/data" - "${pkgs.writeScript "load_extensions" '' - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" < Date: Sat, 6 Jul 2024 16:01:17 +0100 Subject: [PATCH 26/30] Add arion package --- hosts/common/optional/docker/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/default.nix b/hosts/common/optional/docker/default.nix index 05426f8..b6dba80 100644 --- a/hosts/common/optional/docker/default.nix +++ b/hosts/common/optional/docker/default.nix @@ -1,6 +1,11 @@ -{ inputs, ... }: { +{ pkgs, inputs, ... }: +{ imports = [ inputs.arion.nixosModules.arion ]; + environment.systemPackages = [ + pkgs.arion + ]; + # Arion works with Docker, but for NixOS-based containers, you need Podman # since NixOS 21.05. virtualisation = { @@ -9,6 +14,7 @@ defaultNetwork.settings.dns_enabled = true; }; docker = { + enable = true; storageDriver = "btrfs"; rootless = { enable = true; From 89646a5d6a1e999aed321ea84235a1bcd623d665 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 16:01:40 +0100 Subject: [PATCH 27/30] Update nix-secrets --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 72ca555..9f30b93 100644 --- a/flake.lock +++ b/flake.lock @@ -380,11 +380,11 @@ "nix-secrets": { "flake": false, "locked": { - "lastModified": 1719686367, - "narHash": "sha256-zQ/Mgrg3GjE4QkweXPLAtbO8SnfzTXZrqmm8oZwXBV4=", + "lastModified": 1720263046, + "narHash": "sha256-6tJLK4EtB4IXBO4i6P/Ulf03Bd7GaEezT7AebN3VPHA=", "ref": "refs/heads/master", - "rev": "eb8d568c7e30a8c45148fa5c235ebd49bc8effee", - "revCount": 148, + "rev": "33d677fea187322e503f8a56d9c75ff7e7df057c", + "revCount": 151, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, From 529fc394effbeadcbbfa39d2035d8753c763666e Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 16:02:10 +0100 Subject: [PATCH 28/30] Modify postgres docker container - add sops-secrets for admin pwd - POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions - initdb docker entrypoint script to create dbs, users and extensions from json --- hosts/common/optional/docker/postgres.nix | 103 ++++++++++++++++------ 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index 7ccc75f..c5d7c3e 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -1,5 +1,11 @@ -{ pkgs, ... }: +{ pkgs, lib, inputs, config, ... }: +let + admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path; +in { + sops.secrets = { + "software/postgres/admin_db/password" = { }; + }; virtualisation.arion = { backend = "docker"; projects = { @@ -12,51 +18,92 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ postgresql-16-postgis \ + jq \ && rm -rf /var/lib/apt/lists/* ''}"; environment = { - POSTGRES_PASSWORD = "balls1234"; + POSTGRES_PASSWORD_FILE = admin_dbPasswordFile; POSTGRES_USER = "admin"; POSTGRES_DB = "admin_db"; PGDATA = "/var/lib/postgresql/data/pgdata"; + POSTGRES_MULTIPLE_DATABASES = '' + [ + { + "osm": { + "user": "gis", + "extensions": [ + "hstore", + "postgis" + ] + }, + "bitcoin": { + "user": "satoshi", + "extensions": [] + }, + "btc_models": { + "user": "dbt", + "extensions": [] + }, + "dev_btc_models": { + "user": "dbt", + "extensions": [] + } + "test": { + "user": "test", + "extensions": [hstore] + } + } + ] + ''; }; ports = [ "5432:5432" ]; volumes = [ "/mnt/postgres:/var/lib/postgresql/data" - # PG init script + # Need to mount secret file + "${admin_dbPasswordFile}:${admin_dbPasswordFile}" + + # PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES + # creates databases, users and installs extensions for each database. "${pkgs.writeScript "init.sh" '' #!/bin/bash - - # Create additional databases - psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - create database bitcoin; - create database osm; + function create_user_and_database() { + local database=$1 + local user=$2 + local extensions=$3 + echo "### admin user: $POSTGRES_USER ###" + echo " Creating database '$database'" + echo " Creating user '$user'" + psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL + CREATE USER $user; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $user; EOSQL - - # Create additional users - psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - create user gis; - create user satoshi; - EOSQL - - # Setup bitcoin db - psql -v --username "$POSTGRES_USER" --dbname "bitcoin" <<-EOSQL - grant all privileges on database bitcoin to satoshi; - EOSQL - - # Setup osm db - psql -v --username "$POSTGRES_USER" --dbname "osm" <<-EOSQL - grant all privileges on database osm to gis; - create extension if not exists postgis; - create extension if not exists hstore; - EOSQL - + + # Loop through extensions and create them + for ext in $(echo "$extensions" | tr ',' ' '); do + echo " - Installing extention $ext" + psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" + done + } + + if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + + # Parse the JSON string + database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') + echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" + + # Loop through each database and create it + for db_name in $database_names; do + user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") + extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") + create_user_and_database "$db_name" "$user" "$extensions" + done + fi ''}:/docker-entrypoint-initdb.d/init.sh" ]; }; }; }; - } From 947ddaca43ad567cb468892748f3cd6833c36da4 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 20:53:26 +0100 Subject: [PATCH 29/30] Postgres docker configuration --- hosts/common/optional/docker/postgres.nix | 116 ++++++++++++++-------- 1 file changed, 75 insertions(+), 41 deletions(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index c5d7c3e..f698c4d 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -1,6 +1,72 @@ { pkgs, lib, inputs, config, ... }: let admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path; + initScript = pkgs.writeText "init.sh" '' + #!/bin/bash + function create_user_and_database() { + local database=$1 + local user=$2 + local extensions=$3 + echo "### admin user: $POSTGRES_USER ###" + echo " Creating database '$database'" + echo " Creating user '$user'" + psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL + CREATE USER $user; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $user; + EOSQL + + # Loop through extensions and create them + for ext in $(echo "$extensions" | tr ',' ' '); do + echo " - Installing extention $ext" + psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" + done + } + + if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + + # Parse the JSON string + database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') + echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" + + # Loop through each database and create it + for db_name in $database_names; do + user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") + extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") + create_user_and_database "$db_name" "$user" "$extensions" + done + fi + ''; + + pg_hbaConfig = pkgs.writeText "pg_hba.conf" '' + none + ''; + + pgsqlConfig = pkgs.writeText "postgresql.conf" '' + listen_addresses = '*' + port = 5432 + max_connections = 100 + shared_buffers = 24GB + work_mem = 1GB + maintenance_work_mem = 10GB + autovacuum_work_mem = 2GB + dynamic_shared_memory_type = posix + wal_level = minimal + checkpoint_timeout = 60min + checkpoint_completion_target = 0.9 + max_wal_size = 10GB + min_wal_size = 80MB + max_wal_senders = 0 + random_page_cost = 1.0 + effective_cache_size = 25GB + jit = off + log_line_prefix = '%m [%p] %q%u@%d ' + log_timezone = 'Etc/UTC' + cluster_name = 'postgres-docker' + datestyle = 'iso, dmy' + timezone = 'Etc/UTC' + default_text_search_config = 'pg_catalog.english' + ''; in { sops.secrets = { @@ -12,7 +78,7 @@ in "db".settings.services."db".service = { restart = "unless-stopped"; build.context = "/nix/store"; - build.dockerfile = builtins.baseNameOf "${pkgs.writeScript "pgDockerfile" '' + build.dockerfile = builtins.baseNameOf "${pkgs.writeText "pgDockerfile" '' FROM postgres:16 # install packages RUN apt-get update \ @@ -21,6 +87,7 @@ in jq \ && rm -rf /var/lib/apt/lists/* ''}"; + command = [ "postgres" "-c" "config_file=/etc/postgresql/postgresql.conf" ]; environment = { POSTGRES_PASSWORD_FILE = admin_dbPasswordFile; POSTGRES_USER = "admin"; @@ -48,59 +115,26 @@ in "user": "dbt", "extensions": [] } - "test": { - "user": "test", - "extensions": [hstore] - } } ] ''; }; ports = [ "5432:5432" ]; volumes = [ + + # Mount pgdata to external zfs volume "/mnt/postgres:/var/lib/postgresql/data" + # Mount config files + # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" + "${pgsqlConfig}:/etc/postgresql/postgresql.conf" + # Need to mount secret file "${admin_dbPasswordFile}:${admin_dbPasswordFile}" # PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES # creates databases, users and installs extensions for each database. - "${pkgs.writeScript "init.sh" '' - #!/bin/bash - function create_user_and_database() { - local database=$1 - local user=$2 - local extensions=$3 - echo "### admin user: $POSTGRES_USER ###" - echo " Creating database '$database'" - echo " Creating user '$user'" - psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL - CREATE USER $user; - CREATE DATABASE $database; - GRANT ALL PRIVILEGES ON DATABASE $database TO $user; - EOSQL - - # Loop through extensions and create them - for ext in $(echo "$extensions" | tr ',' ' '); do - echo " - Installing extention $ext" - psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;" - done - } - - if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then - - # Parse the JSON string - database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]') - echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")" - - # Loop through each database and create it - for db_name in $database_names; do - user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user") - extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")") - create_user_and_database "$db_name" "$user" "$extensions" - done - fi - ''}:/docker-entrypoint-initdb.d/init.sh" + "${initScript}:/docker-entrypoint-initdb.d/init.sh" ]; }; }; From 2f99d05406e79fc60ebd88639dc8d9de43aa7d4a Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 6 Jul 2024 21:17:32 +0100 Subject: [PATCH 30/30] small fix --- hosts/common/optional/docker/postgres.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/common/optional/docker/postgres.nix b/hosts/common/optional/docker/postgres.nix index f698c4d..75e2ac8 100644 --- a/hosts/common/optional/docker/postgres.nix +++ b/hosts/common/optional/docker/postgres.nix @@ -126,7 +126,7 @@ in "/mnt/postgres:/var/lib/postgresql/data" # Mount config files - # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" + # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf" "${pgsqlConfig}:/etc/postgresql/postgresql.conf" # Need to mount secret file