Compare commits

...

51 Commits

Author SHA1 Message Date
Sam fb7cf9e280 merge master 2024-07-07 16:48:11 +01:00
sam 539ac37b8a Merge branch 'master' into docker 2024-07-07 16:25:47 +01:00
sam 8e527473ac Merge branch 'docker' of git.bitlab21.com:sam/nixos into docker 2024-07-07 16:24:00 +01:00
Sam db7bce57e7 Add arion package 2024-07-07 16:23:36 +01:00
Sam 614b9765dd Docker and postgres config 2024-07-07 16:23:36 +01:00
Sam 4b85810128 small fix 2024-07-07 16:23:36 +01:00
Sam a7c8b86b1f Postgres docker configuration 2024-07-07 16:23:36 +01:00
Sam 89ab4e8f9d Modify postgres docker container
- add sops-secrets for admin pwd
- POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions
- initdb docker entrypoint script to create dbs, users and extensions
  from json
2024-07-07 16:23:36 +01:00
Sam 01ad0238a7 Update nix-secrets 2024-07-07 16:23:36 +01:00
Sam 688c2c9bcd Add arion package 2024-07-07 16:23:36 +01:00
Sam b8973040d5 pg init script to configure db on start
- create users & dbs
- setup db permissions
- install extensions
2024-07-07 16:23:36 +01:00
Sam ba9f593bcd pgdata dir and admin_db default database 2024-07-07 16:23:36 +01:00
Sam 3dbe85853e Build postgres using dockerfile
- use dockerfile to install postgis during build
2024-07-07 16:23:36 +01:00
Sam ba19ee9125 Minor fixes 2024-07-07 16:23:36 +01:00
Sam 8173a0dc94 Podman to user groups 2024-07-07 16:23:36 +01:00
Sam 600160bd9a Arion flake input 2024-07-07 16:23:36 +01:00
Sam 5205e606c1 Docker and postgres config 2024-07-07 16:23:36 +01:00
Sam 5b8a1430fe Add postgres btrfs zvol 2024-07-07 16:23:36 +01:00
Sam 8f458590e2 Remove deploy_key from sops 2024-07-07 16:23:36 +01:00
Sam bcea6919fb Update flake secrets 2024-07-07 16:23:36 +01:00
sam febc33faee Merge branch 'docker' of git.bitlab21.com:sam/nixos into docker 2024-07-07 15:36:51 +01:00
Sam 2f99d05406 small fix 2024-07-07 15:36:34 +01:00
Sam 947ddaca43 Postgres docker configuration 2024-07-07 15:36:34 +01:00
Sam 529fc394ef Modify postgres docker container
- add sops-secrets for admin pwd
- POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions
- initdb docker entrypoint script to create dbs, users and extensions
  from json
2024-07-07 15:36:34 +01:00
Sam 89646a5d6a Update nix-secrets 2024-07-07 15:36:34 +01:00
Sam 804d6bf4d0 Add arion package 2024-07-07 15:36:34 +01:00
Sam 491350bc58 pg init script to configure db on start
- create users & dbs
- setup db permissions
- install extensions
2024-07-07 15:36:34 +01:00
Sam baaaa3e8d6 pgdata dir and admin_db default database 2024-07-07 15:36:34 +01:00
Sam 591a9ce48f Build postgres using dockerfile
- use dockerfile to install postgis during build
2024-07-07 15:36:34 +01:00
Sam 7df7970414 Minor fixes 2024-07-07 15:36:34 +01:00
Sam 052c941e81 Podman to user groups 2024-07-07 15:36:34 +01:00
Sam f7695f4d15 Arion flake input 2024-07-07 15:36:34 +01:00
Sam 52a3b85c8f Docker and postgres config 2024-07-07 15:36:34 +01:00
Sam c9ee7c7e80 Add postgres btrfs zvol 2024-07-07 15:36:34 +01:00
Sam fc2f6f4ca3 Remove deploy_key from sops 2024-07-07 15:36:34 +01:00
Sam 67e3d9dded Update flake secrets 2024-07-07 15:36:34 +01:00
Sam b8f85256a7 small fix 2024-07-06 21:17:32 +01:00
Sam 271b5958b8 Postgres docker configuration 2024-07-06 20:53:26 +01:00
Sam 2f0ddf8375 Modify postgres docker container
- add sops-secrets for admin pwd
- POSTGRES_MULTIPLE_DATABASES as json to specify users and extensions
- initdb docker entrypoint script to create dbs, users and extensions
  from json
2024-07-06 16:02:10 +01:00
Sam e419389862 Update nix-secrets 2024-07-06 16:01:40 +01:00
Sam fec1dae750 Add arion package 2024-07-06 16:01:17 +01:00
Sam 3b7a597d8f pg init script to configure db on start
- create users & dbs
- setup db permissions
- install extensions
2024-07-06 10:28:09 +01:00
Sam 1e95ba6c36 pgdata dir and admin_db default database 2024-07-06 10:27:15 +01:00
Sam d29250a2a6 Build postgres using dockerfile
- use dockerfile to install postgis during build
2024-07-06 10:26:08 +01:00
Sam f71ece31f1 Minor fixes 2024-07-05 18:59:10 +01:00
Sam a71ee506d3 Podman to user groups 2024-07-05 18:58:46 +01:00
Sam 7f9c3535ef Arion flake input 2024-07-05 18:58:30 +01:00
Sam 9ace130029 Docker and postgres config 2024-07-05 18:58:03 +01:00
Sam 92d09646fa Add postgres btrfs zvol 2024-07-05 18:57:17 +01:00
Sam 33981eea6d Remove deploy_key from sops 2024-07-05 18:56:41 +01:00
Sam bd719c72fa Update flake secrets 2024-07-05 18:56:18 +01:00
8 changed files with 314 additions and 24 deletions

View File

@ -1,5 +1,27 @@
{
"nodes": {
"arion": {
"inputs": {
"flake-parts": "flake-parts",
"haskell-flake": "haskell-flake",
"hercules-ci-effects": "hercules-ci-effects",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1720147808,
"narHash": "sha256-hlWEQGUbIwYb+vnd8egzlW/P++yKu3HjV/rOdOPVank=",
"owner": "hercules-ci",
"repo": "arion",
"rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "arion",
"rev": "236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318",
"type": "github"
}
},
"base16-schemes": {
"flake": false,
"locked": {
@ -90,6 +112,48 @@
}
},
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
"arion",
"nixpkgs"
]
},
"locked": {
"lastModified": 1719994518,
"narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-parts_2": {
"inputs": {
"nixpkgs-lib": [
"arion",
"hercules-ci-effects",
"nixpkgs"
]
},
"locked": {
"lastModified": 1712014858,
"narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
"type": "github"
},
"original": {
"id": "flake-parts",
"type": "indirect"
}
},
"flake-parts_3": {
"inputs": {
"nixpkgs-lib": [
"nixvim",
@ -177,6 +241,44 @@
"type": "github"
}
},
"haskell-flake": {
"locked": {
"lastModified": 1675296942,
"narHash": "sha256-u1X1sblozi5qYEcLp1hxcyo8FfDHnRUVX3dJ/tW19jY=",
"owner": "srid",
"repo": "haskell-flake",
"rev": "c2cafce9d57bfca41794dc3b99c593155006c71e",
"type": "github"
},
"original": {
"owner": "srid",
"ref": "0.1.0",
"repo": "haskell-flake",
"type": "github"
}
},
"hercules-ci-effects": {
"inputs": {
"flake-parts": "flake-parts_2",
"nixpkgs": [
"arion",
"nixpkgs"
]
},
"locked": {
"lastModified": 1719226092,
"narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
"owner": "hercules-ci",
"repo": "hercules-ci-effects",
"rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "hercules-ci-effects",
"type": "github"
}
},
"home-manager": {
"inputs": {
"nixpkgs": [
@ -278,11 +380,11 @@
"nix-secrets": {
"flake": false,
"locked": {
"lastModified": 1719601133,
"narHash": "sha256-2+e92LyX1fFj3mIZft+K8OzR9NT/1xtheO8hO/3DyRc=",
"lastModified": 1720263046,
"narHash": "sha256-6tJLK4EtB4IXBO4i6P/Ulf03Bd7GaEezT7AebN3VPHA=",
"ref": "refs/heads/master",
"rev": "278ccbbd646e86cab5fd38d43d9134270d8123d0",
"revCount": 141,
"rev": "33d677fea187322e503f8a56d9c75ff7e7df057c",
"revCount": 151,
"type": "git",
"url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git"
},
@ -293,16 +395,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1719426051,
"narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=",
"owner": "nixos",
"lastModified": 1720031269,
"narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd",
"rev": "9f4128e00b0ae8ec65918efeba59db998750ead6",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-24.05",
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
@ -354,11 +456,27 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1719426051,
"narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-24.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixvim": {
"inputs": {
"devshell": "devshell",
"flake-compat": "flake-compat",
"flake-parts": "flake-parts",
"flake-parts": "flake-parts_3",
"git-hooks": "git-hooks",
"home-manager": "home-manager_2",
"nix-darwin": "nix-darwin",
@ -399,12 +517,13 @@
},
"root": {
"inputs": {
"arion": "arion",
"disko": "disko",
"home-manager": "home-manager",
"impermanence": "impermanence",
"nix-colors": "nix-colors",
"nix-secrets": "nix-secrets",
"nixpkgs": "nixpkgs",
"nixpkgs": "nixpkgs_2",
"nixpkgs-unstable": "nixpkgs-unstable",
"nixvim": "nixvim",
"nur": "nur",

View File

@ -21,6 +21,11 @@
inputs.nixpkgs.follows = "nixpkgs";
};
# Arion for docker
arion = {
url = "github:hercules-ci/arion/236f9dd82d6ef6a2d9987c7a7df3e75f1bc8b318";
};
# Nix colors
nix-colors.url = "github:misterio77/nix-colors";

View File

@ -21,9 +21,6 @@ in
};
secrets = {
"passwords/root".neededForUsers = true;
"ssh_keys/deploy_key/id_ed25519" = {
path = "/etc/ssh/deploy_key-ssh-ed25519";
};
};
};
}

View File

@ -53,14 +53,11 @@
postgres = {
type = "zfs_volume";
size = "10G -s";
options = {
"com.sun:auto-snapshot:daily" = "true";
"volblocksize" = "8k";
content = {
type = "filesystem";
format = "btrfs";
mountpoint = "/postgres";
};
};
lxc = {
type = "zfs_volume";
size = "10G -s";
options = {
"com.sun:auto-snapshot:daily" = "true";
"volblocksize" = "8k";

View File

@ -0,0 +1,26 @@
{ pkgs, inputs, ... }:
{
imports = [ inputs.arion.nixosModules.arion ];
environment.systemPackages = [
pkgs.arion
];
# Arion works with Docker, but for NixOS-based containers, you need Podman
# since NixOS 21.05.
virtualisation = {
podman = {
enable = true;
defaultNetwork.settings.dns_enabled = true;
};
docker = {
enable = true;
storageDriver = "btrfs";
rootless = {
enable = true;
setSocketVariable = true;
};
};
};
}

View File

@ -0,0 +1,143 @@
{ pkgs, lib, inputs, config, ... }:
let
admin_dbPasswordFile = lib.optionalString (lib.hasAttr "sops-nix" inputs) config.sops.secrets."software/postgres/admin_db/password".path;
initScript = pkgs.writeText "init.sh" ''
#!/bin/bash
function create_user_and_database() {
local database=$1
local user=$2
local extensions=$3
echo "### admin user: $POSTGRES_USER ###"
echo " Creating database '$database'"
echo " Creating user '$user'"
psql -v --username "$POSTGRES_USER" -d "$POSTGRES_DB" <<-EOSQL
CREATE USER $user;
CREATE DATABASE $database;
GRANT ALL PRIVILEGES ON DATABASE $database TO $user;
EOSQL
# Loop through extensions and create them
for ext in $(echo "$extensions" | tr ',' ' '); do
echo " - Installing extention $ext"
psql -v --username "$POSTGRES_USER" -d "$database" -c "CREATE EXTENSION $ext;"
done
}
if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
# Parse the JSON string
database_names=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r '.[0] | keys[]')
echo "Multiple database creation requested: $(echo "$database_names" | tr "\n" " ")"
# Loop through each database and create it
for db_name in $database_names; do
user=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .user")
extensions=$(echo "$POSTGRES_MULTIPLE_DATABASES" | jq -r ".[0] | .''${db_name} | .extensions | join(\",\")")
create_user_and_database "$db_name" "$user" "$extensions"
done
fi
'';
pg_hbaConfig = pkgs.writeText "pg_hba.conf" ''
none
'';
pgsqlConfig = pkgs.writeText "postgresql.conf" ''
listen_addresses = '*'
port = 5432
max_connections = 100
shared_buffers = 24GB
work_mem = 1GB
maintenance_work_mem = 10GB
autovacuum_work_mem = 2GB
dynamic_shared_memory_type = posix
wal_level = minimal
checkpoint_timeout = 60min
checkpoint_completion_target = 0.9
max_wal_size = 10GB
min_wal_size = 80MB
max_wal_senders = 0
random_page_cost = 1.0
effective_cache_size = 25GB
jit = off
log_line_prefix = '%m [%p] %q%u@%d '
log_timezone = 'Etc/UTC'
cluster_name = 'postgres-docker'
datestyle = 'iso, dmy'
timezone = 'Etc/UTC'
default_text_search_config = 'pg_catalog.english'
'';
in
{
sops.secrets = {
"software/postgres/admin_db/password" = { };
};
virtualisation.arion = {
backend = "docker";
projects = {
"db".settings.services."db".service = {
restart = "unless-stopped";
build.context = "/nix/store";
build.dockerfile = builtins.baseNameOf "${pkgs.writeText "pgDockerfile" ''
FROM postgres:16
# install packages
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
postgresql-16-postgis \
jq \
&& rm -rf /var/lib/apt/lists/*
''}";
command = [ "postgres" "-c" "config_file=/etc/postgresql/postgresql.conf" ];
environment = {
POSTGRES_PASSWORD_FILE = admin_dbPasswordFile;
POSTGRES_USER = "admin";
POSTGRES_DB = "admin_db";
PGDATA = "/var/lib/postgresql/data/pgdata";
POSTGRES_MULTIPLE_DATABASES = ''
[
{
"osm": {
"user": "gis",
"extensions": [
"hstore",
"postgis"
]
},
"bitcoin": {
"user": "satoshi",
"extensions": []
},
"btc_models": {
"user": "dbt",
"extensions": []
},
"dev_btc_models": {
"user": "dbt",
"extensions": []
}
}
]
'';
};
ports = [ "5432:5432" ];
volumes = [
# Mount pgdata to external zfs volume
"/mnt/postgres:/var/lib/postgresql/data"
# Mount config files
# "${pg_hbaConfig}:/var/lib/postgres/data/pgdata/pg_hba.conf"
"${pgsqlConfig}:/etc/postgresql/postgresql.conf"
# Need to mount secret file
"${admin_dbPasswordFile}:${admin_dbPasswordFile}"
# PG init script to parse json specified in POSTGRES_MULTIPLE_DATABASES
# creates databases, users and installs extensions for each database.
"${initScript}:/docker-entrypoint-initdb.d/init.sh"
];
};
};
};
}

View File

@ -22,6 +22,7 @@ in
"docker"
"lxc"
"git"
"podman"
];
packages = with pkgs; [

View File

@ -2,9 +2,9 @@
let
# Disko setup
fsType = "btrfs"; # one of ext4 or btrfs. Use btrfs if using impermanence
dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005"; # depends on target hardware
dev = "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00005";
encrypted = false; # currrently only applies to btrfs
impermanence = false; # currrently only applies to btrfs
impermanence = false;
user = "admin";
in
{
@ -23,6 +23,8 @@ in
# Import optional options
../common/optional/openssh.nix
../common/optional/docker
../common/optional/docker/postgres.nix
];