merge master

- add sops-secrets for admin pwd
- POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions
- initdb docker entrypoint script to create dbs, users and extensions
  from json

flake.lock

@@ -1,5 +1,27 @@
 {
   "nodes": {
+    "arion": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "haskell-flake": "haskell-flake",
+        "hercules-ci-effects": "hercules-ci-effects",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1720147808,
+        "narHash": "sha256-hlWEQGUbIwYb+vnd8egzlW/P++yKu3HjV/rOdOPVank=",
+        "owner": "hercules-ci",
+        "repo": "arion",
+        "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "arion",
+        "rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318",
+        "type": "github"
+      }
+    },
     "base16-schemes": {
       "flake": false,
       "locked": {
@@ -90,6 +112,48 @@
       }
     },
     "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "arion",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719994518,
+        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "arion",
+          "hercules-ci-effects",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
+    "flake-parts_3": {
       "inputs": {
         "nixpkgs-lib": [
           "nixvim",
@@ -177,6 +241,44 @@
         "type": "github"
       }
     },
+    "haskell-flake": {
+      "locked": {
+        "lastModified": 1675296942,
+        "narHash": "sha256-u1X1sblozi5qYEcLp1hxcyo8FfDHnRUVX3dJ/tW19jY=",
+        "owner": "srid",
+        "repo": "haskell-flake",
+        "rev": "c2cafce9d57bfca41794dc3b99c593155006c71e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "srid",
+        "ref": "0.1.0",
+        "repo": "haskell-flake",
+        "type": "github"
+      }
+    },
+    "hercules-ci-effects": {
+      "inputs": {
+        "flake-parts": "flake-parts_2",
+        "nixpkgs": [
+          "arion",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719226092,
+        "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-effects",
+        "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-effects",
+        "type": "github"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -278,11 +380,11 @@
     "nix-secrets": {
       "flake": false,
       "locked": {
-        "lastModified": 1719601133,
+        "lastModified": 1720263046,
-        "narHash": "sha256-2+e92LyX1fFj3mIZft+K8OzR9NT/1xtheO8hO/3DyRc=",
+        "narHash": "sha256-6tJLK4EtB4IXBO4i6P/Ulf03Bd7GaEezT7AebN3VPHA=",
         "ref": "refs/heads/master",
-        "rev": "278ccbbd646e86cab5fd38d43d9134270d8123d0",
+        "rev": "33d677fea187322e503f8a56d9c75ff7e7df057c",
-        "revCount": 141,
+        "revCount": 151,
         "type": "git",
         "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
       },
@@ -293,16 +395,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1719426051,
+        "lastModified": 1720031269,
-        "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=",
+        "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=",
-        "owner": "nixos",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd",
+        "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -354,11 +456,27 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1719426051,
+        "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixvim": {
       "inputs": {
         "devshell": "devshell",
         "flake-compat": "flake-compat",
-        "flake-parts": "flake-parts",
+        "flake-parts": "flake-parts_3",
         "git-hooks": "git-hooks",
         "home-manager": "home-manager_2",
         "nix-darwin": "nix-darwin",
@@ -399,12 +517,13 @@
     },
     "root": {
       "inputs": {
+        "arion": "arion",
         "disko": "disko",
         "home-manager": "home-manager",
         "impermanence": "impermanence",
         "nix-colors": "nix-colors",
         "nix-secrets": "nix-secrets",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-unstable": "nixpkgs-unstable",
         "nixvim": "nixvim",
         "nur": "nur",

flake.nix

@@ -21,6 +21,11 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    # Arion for docker
+    arion = {
+      url = "github:hercules-ci/arion/236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318";
+    };
+
     # Nix colors
     nix-colors.url = "github:misterio77/nix-colors";
 

hosts/common/core/sops.nix

@@ -21,9 +21,6 @@ in
     };
     secrets = {
       "passwords/root".neededForUsers = true;
-      "ssh_keys/deploy_key/id_ed25519" = {
-        path = "/etc/ssh/deploy_key-ssh-ed25519";
-      };
     };
   };
 }

hosts/common/disks/zfs/zspeed.nix

@@ -53,14 +53,11 @@
           postgres = {
             type = "zfs_volume";
             size = "10G -s";
-            options = {
+            content = {
-              "com.sun:auto-snapshot:daily" = "true";
+              type = "filesystem";
-              "volblocksize" = "8k";
+              format = "btrfs";
+              mountpoint = "/postgres";
             };
-          };
-          lxc = {
-            type = "zfs_volume";
-            size = "10G -s";
             options = {
               "com.sun:auto-snapshot:daily" = "true";
               "volblocksize" = "8k";

hosts/common/optional/docker/default.nix

@@ -0,0 +1,26 @@
+{ pkgs, inputs, ... }:
+{
+
+  imports = [ inputs.arion.nixosModules.arion ];
+  environment.systemPackages = [
+    pkgs.arion
+  ];
+
+  # Arion works with Docker, but for NixOS-based containers, you need Podman
+  # since NixOS 21.05.
+  virtualisation = {
+    podman = {
+      enable = true;
+      defaultNetwork.settings.dns_enabled = true;
+    };
+    docker = {
+      enable = true;
+      storageDriver = "btrfs";
+      rootless = {
+        enable = true;
+        setSocketVariable = true;
+      };
+    };
+  };
+
+}

hosts/common/optional/docker/postgres.nix

@@ -0,0 +1,143 @@
+{ pkgs, lib, inputs, config, ... }:
+let
+  admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path;
+  initScript = pkgs.writeText "init.sh" ''
+    #!/bin/bash
+    function create_user_and_database() {
+      local database=$1
+      local user=$2
+      local extensions=$3
+      echo "### admin user: $POSTGRES_USER ###"
+      echo "  Creating database '$database'"
+      echo "  Creating user  '$user'"
+      psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL
+          CREATE USER $user;
+          CREATE DATABASE $database;
+          GRANT ALL PRIVILEGES ON DATABASE $database TO $user;
+    EOSQL
+            
+    # Loop through extensions and create them
+    for ext in $(echo "$extensions" | tr ',' ' '); do
+      echo "     - Installing extention $ext"
+      psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;"
+    done
+    }
+            
+    if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
+            
+      # Parse the JSON string
+      database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]')
+      echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")"
+              
+      # Loop through each database and create it
+      for db_name in $database_names; do
+        user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user")
+        extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")")
+        create_user_and_database "$db_name" "$user" "$extensions"
+      done
+    fi
+  '';
+
+  pg_hbaConfig = pkgs.writeText "pg_hba.conf" ''
+    none
+  '';
+
+  pgsqlConfig = pkgs.writeText "postgresql.conf" ''
+    listen_addresses = '*'
+    port = 5432
+    max_connections = 100
+    shared_buffers = 24GB
+    work_mem = 1GB
+    maintenance_work_mem = 10GB
+    autovacuum_work_mem = 2GB
+    dynamic_shared_memory_type = posix
+    wal_level = minimal
+    checkpoint_timeout = 60min
+    checkpoint_completion_target = 0.9
+    max_wal_size = 10GB
+    min_wal_size = 80MB
+    max_wal_senders = 0
+    random_page_cost = 1.0
+    effective_cache_size = 25GB
+    jit = off
+    log_line_prefix = '%m [%p] %q%u@%d '
+    log_timezone = 'Etc/UTC'
+    cluster_name = 'postgres-docker'
+    datestyle = 'iso, dmy'
+    timezone = 'Etc/UTC'
+    default_text_search_config = 'pg_catalog.english'
+  '';
+in
+{
+  sops.secrets = {
+    "software/postgres/admin_db/password" = { };
+  };
+  virtualisation.arion = {
+    backend = "docker";
+    projects = {
+      "db".settings.services."db".service = {
+        restart = "unless-stopped";
+        build.context = "/nix/store";
+        build.dockerfile = builtins.baseNameOf "${pkgs.writeText "pgDockerfile" ''
+            FROM postgres:16
+            # install packages
+            RUN apt-get update \
+                && apt-get install -y --no-install-recommends \
+                    postgresql-16-postgis \
+                    jq \
+                && rm -rf /var/lib/apt/lists/*
+          ''}";
+        command = [ "postgres" "-c" "config_file=/etc/postgresql/postgresql.conf" ];
+        environment = {
+          POSTGRES_PASSWORD_FILE = admin_dbPasswordFile;
+          POSTGRES_USER = "admin";
+          POSTGRES_DB = "admin_db";
+          PGDATA = "/var/lib/postgresql/data/pgdata";
+          POSTGRES_MULTIPLE_DATABASES = ''
+            [
+              {
+                "osm": {
+                  "user": "gis",
+                  "extensions": [
+                    "hstore",
+                    "postgis"
+                  ]
+                },
+                "bitcoin": {
+                  "user": "satoshi",
+                  "extensions": []
+                },
+                "btc_models": {
+                  "user": "dbt",
+                  "extensions": []
+                },
+                "dev_btc_models": {
+                  "user": "dbt",
+                  "extensions": []
+                }
+              }
+            ]
+          '';
+        };
+        ports = [ "5432:5432" ];
+        volumes = [
+
+          # Mount pgdata to external zfs volume
+          "/mnt/postgres:/var/lib/postgresql/data"
+
+          # Mount config files
+          # "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf"
+          "${pgsqlConfig}:/etc/postgresql/postgresql.conf"
+
+          # Need to mount secret file 
+          "${admin_dbPasswordFile}:${admin_dbPasswordFile}"
+
+          # PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES
+          # creates databases, users and installs extensions for each database.
+          "${initScript}:/docker-entrypoint-initdb.d/init.sh"
+        ];
+      };
+    };
+  };
+}
+

hosts/common/users/admin/default.nix

@@ -22,6 +22,7 @@ in
       "docker"
       "lxc"
       "git"
+      "podman"
     ];
 
     packages = with pkgs; [

hosts/nebula/default.nix

@@ -2,9 +2,9 @@
 let
   # Disko setup
   fsType = "btrfs"; # one of ext4 or btrfs. Use btrfs if using impermanence
-  dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; # depends on target hardware
+  dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005";
   encrypted = false; # currrently only applies to btrfs
-  impermanence = false; # currrently only applies to btrfs
+  impermanence = false;
   user = "admin";
 in
 {
@@ -23,6 +23,8 @@ in
 
       # Import optional options
       ../common/optional/openssh.nix
+      ../common/optional/docker
+      ../common/optional/docker/postgres.nix
 
     ];