From 9a12850075df0355acbf8733b23d605562ec1df6 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 15 Jan 2025 13:26:25 +0000 Subject: [PATCH] add semita-worker container --- flake.lock | 8 +- .../nixos-containers/baseddata-worker.nix | 10 +- .../optional/nixos-containers/postgres.nix | 2 +- .../nixos-containers/semitamaps-worker.nix | 356 ++++++++++++++++++ hosts/semita/default.nix | 3 +- vars/default.nix | 11 +- 6 files changed, 376 insertions(+), 14 deletions(-) create mode 100644 hosts/common/optional/nixos-containers/semitamaps-worker.nix diff --git a/flake.lock b/flake.lock index c8a8891..1887182 100644 --- a/flake.lock +++ b/flake.lock @@ -538,11 +538,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1736770787, - "narHash": "sha256-TEDwOm0ZnRioT01DdfS5XaFPYXA9uHA+P8pTOSFK0bI=", + "lastModified": 1736868005, + "narHash": "sha256-ijKkZEq9W5L6ymFQd6O21657ws6FRroNrfaIVZnuW+c=", "ref": "refs/heads/master", - "rev": "a88c4ab6fb54527aabeeee2a2954338f81016f98", - "revCount": 201, + "rev": "657823c04eb7b392d3f71cf9047470e93fbed61a", + "revCount": 202, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/baseddata-worker.nix b/hosts/common/optional/nixos-containers/baseddata-worker.nix index beb6411..3e7a98c 100644 --- a/hosts/common/optional/nixos-containers/baseddata-worker.nix +++ b/hosts/common/optional/nixos-containers/baseddata-worker.nix @@ -2,11 +2,13 @@ lib, pkgs, inputs, + configVars, ... }: let pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; containerName = "bd-worker"; sops-nix = inputs.sops-nix; + baseddataData = configVars.locations.baseddataData; in { environment.persistence."/persist" = { hideMounts = true; @@ -37,8 +39,8 @@ in { hostPath = "/etc/ssh/ssh_host_ed25519_key"; isReadOnly = true; }; - "/media/baseddata-data" = { - hostPath = "/media/main-ssd/baseddata-data"; + "/data/baseddata-data" = { + hostPath = baseddataData; isReadOnly = false; }; }; @@ -259,8 +261,8 @@ in { .venv/bin/prefect variable set "bitcoind_ip" ${bitcoindIp} --overwrite .venv/bin/prefect variable set "bitcoind_port" ${bitcoindPort} --overwrite - .venv/bin/prefect variable set "osm_dir" "/media/baseddata-data/osm" --overwrite - .venv/bin/prefect variable set "wdpa_dir" "/media/baseddata-data/wdpa" --overwrite + .venv/bin/prefect variable set "osm_dir" "/data/baseddata-data/osm" --overwrite + .venv/bin/prefect variable set "wdpa_dir" "/data/baseddata-data/wdpa" --overwrite .venv/bin/prefect variable set "dbt_profiles_dir" $(dirname ${dbtProfiles}) --overwrite .venv/bin/prefect variable set "pgsync_config" ${pgsyncConfig} --overwrite diff --git a/hosts/common/optional/nixos-containers/postgres.nix b/hosts/common/optional/nixos-containers/postgres.nix index 0aac4f4..068563c 100644 --- a/hosts/common/optional/nixos-containers/postgres.nix +++ b/hosts/common/optional/nixos-containers/postgres.nix @@ -85,7 +85,7 @@ in { # shared_buffers = "32GB"; # work_mem = "0.32GB"; # maintenance_work_mem = "64MB"; - max_connections = "20"; + max_connections = "100"; shared_buffers = "2GB"; effective_cache_size = "6GB"; maintenance_work_mem = "1GB"; diff --git a/hosts/common/optional/nixos-containers/semitamaps-worker.nix b/hosts/common/optional/nixos-containers/semitamaps-worker.nix new file mode 100644 index 0000000..34d5036 --- /dev/null +++ b/hosts/common/optional/nixos-containers/semitamaps-worker.nix @@ -0,0 +1,356 @@ +# { +# lib, +# pkgs, +# inputs, +# configVars, +# ... +# }: let +# pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; +# containerName = "sm-worker"; +# sops-nix = inputs.sops-nix; +# semitamapsData = configVars.locations.semitamapsData; +# in { +# environment.persistence."/persist" = { +# hideMounts = true; +# directories = [ +# "/var/lib/nixos-containers/${containerName}" +# ]; +# }; +# +# networking.nat.enable = true; +# networking.nat.internalInterfaces = ["ve-+"]; +# networking.nat.externalInterface = "br0"; +# +# containers.${containerName} = { +# enableTun = true; +# +# # configuration to run docker/podman in systemd-nspawn container +# # https://discourse.nixos.org/t/podman-docker-in-nixos-container-ideally-in-unprivileged-one/22909/12 +# additionalCapabilities = [ +# ''all" --system-call-filter="add_key keyctl bpf" --capability="all'' +# ]; +# extraFlags = ["--private-users-ownership=chown"]; +# allowedDevices = [ +# { +# node = "/dev/fuse"; +# modifier = "rwm"; +# } +# { +# node = "/dev/mapper/control"; +# modifier = "rw"; +# } +# { +# node = "/dev/console"; +# modifier = "rwm"; +# } +# { +# node = "/dev/net/tun"; +# modifier = "rw"; +# } +# ]; +# ###### +# +# autoStart = true; +# privateNetwork = true; +# hostBridge = "br0"; +# nixpkgs = pkgs.path; +# bindMounts = { +# "/etc/ssh/ssh_host_ed25519_key" = { +# hostPath = "/etc/ssh/ssh_host_ed25519_key"; +# isReadOnly = true; +# }; +# "/data/semitamaps-data" = { +# hostPath = semitamapsData; +# isReadOnly = false; +# }; +# }; +# +# config = { +# pkgs, +# lib, +# ... +# }: let +# configVars = import ../../../../vars {inherit inputs lib;}; +# secretsDirectory = builtins.toString inputs.nix-secrets; +# secretsFile = "${secretsDirectory}/secrets.yaml"; +# +# # define ip addresses +# containerIp = configVars.networking.addresses.sm-worker.ip; +# gatewayIp = configVars.networking.addresses.gateway.ip; +# in { +# networking = { +# defaultGateway = "${gatewayIp}"; +# interfaces.eth0.ipv4.addresses = [ +# { +# "address" = "${containerIp}"; +# "prefixLength" = 24; +# } +# ]; +# firewall = { +# enable = true; +# allowedTCPPorts = [ +# 2322 +# 8080 +# 8081 +# ]; +# }; +# useHostResolvConf = lib.mkForce false; +# }; +# +# sops = { +# defaultSopsFile = "${secretsFile}"; +# validateSopsFiles = false; +# +# age = { +# sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; +# }; +# }; +# +# sops.secrets = { +# }; +# +# imports = [ +# sops-nix.nixosModules.sops +# ]; +# +# services.resolved.enable = true; +# +# environment.systemPackages = [ +# pkgs.vim +# pkgs.git +# pkgs.python311 +# pkgs.poetry +# pkgs.htop +# pkgs.podman-compose +# pkgs.jdk +# ]; +# +# virtualisation = { +# podman = { +# enable = true; +# dockerSocket.enable = true; +# defaultNetwork.settings.dns_enabled = true; +# dockerCompat = true; +# }; +# }; +# +# systemd.services.photon = { +# wantedBy = ["multi-user.target"]; +# after = ["network.target"]; +# description = "Photon Service"; +# path = ["/run/current-system/sw"]; +# serviceConfig = { +# WorkingDirectory = "/data/semitamaps-data/photon"; +# ExecStart = pkgs.writeShellScript "photon" '' +# java -jar photon-*.jar -cors-any +# ''; +# Restart = "on-failure"; +# }; +# }; +# +# programs.nix-ld.enable = true; +# programs.nix-ld.libraries = with pkgs; [ +# zlib +# libgcc +# ]; +# +# programs.ssh.knownHosts = { +# "git.bitlab21.com" = { +# publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALNd2BGf64heYjWT9yt0fVmngepiHRIMsL7au/MRteg"; +# }; +# }; +# +# services.openssh = { +# enable = true; +# settings.PasswordAuthentication = false; +# }; +# +# users.users.root = { +# openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); +# }; +# +# system.stateVersion = "24.05"; +# }; +# }; +# } +{ + pkgs, + lib, + inputs, + configVars, + ... +}: let + pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + containerName = "sm-worker"; + sops-nix = inputs.sops-nix; + semitamapsData = configVars.locations.semitamapsData; + containerIp = configVars.networking.addresses.sm-worker.ip; + gatewayIp = configVars.networking.addresses.gateway.ip; +in { + networking.nat.enable = true; + networking.nat.internalInterfaces = ["ve-+"]; + networking.nat.externalInterface = "br0"; + + environment.persistence."/persist" = { + hideMounts = true; + directories = [ + "/var/lib/nixos-containers/${containerName}" + ]; + }; + + containers."${containerName}" = { + enableTun = true; + + # configuration to run docker/podman in systemd-nspawn container + # https://discourse.nixos.org/t/podman-docker-in-nixos-container-ideally-in-unprivileged-one/22909/12 + additionalCapabilities = [ + ''all" --system-call-filter="add_key keyctl bpf" --capability="all'' + ]; + extraFlags = ["--private-users-ownership=chown"]; + allowedDevices = [ + { + node = "/dev/fuse"; + modifier = "rwm"; + } + { + node = "/dev/mapper/control"; + modifier = "rw"; + } + { + node = "/dev/console"; + modifier = "rwm"; + } + { + node = "/dev/dri/card1"; + modifier = "rwm"; + } + { + node = "/dev/dri/renderD128"; + modifier = "rwm"; + } + { + node = "/dev/net/tun"; + modifier = "rw"; + } + ]; + ###### + + autoStart = true; + privateNetwork = true; + hostBridge = "br0"; + nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + "/data/semitamaps-data" = { + hostPath = semitamapsData; + isReadOnly = false; + }; + }; + + config = { + pkgs, + lib, + ... + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { + networking = { + defaultGateway = "${gatewayIp}"; + interfaces.eth0.ipv4.addresses = [ + { + "address" = "${containerIp}"; + "prefixLength" = 24; + } + ]; + firewall = { + enable = true; + allowedTCPPorts = [ + 2322 + 8080 + 8081 + ]; + }; + useHostResolvConf = lib.mkForce false; + }; + + services.resolved.enable = true; + + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + }; + + imports = [ + sops-nix.nixosModules.sops + ]; + + environment.systemPackages = [ + pkgs.vim + pkgs.git + pkgs.arion + pkgs.dive + pkgs.podman-tui + pkgs.podman-compose + pkgs.jdk + ]; + + virtualisation = { + podman = { + enable = true; + dockerSocket.enable = true; + defaultNetwork.settings.dns_enabled = true; + dockerCompat = true; + }; + }; + + networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; + + systemd.services.podman-autostart = { + enable = true; + after = ["podman.service"]; + wantedBy = ["multi-user.target"]; + description = "Automatically start containers with --restart=always tag"; + serviceConfig = { + Type = "idle"; + ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; + ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; + }; + }; + + systemd.services.photon = { + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + description = "Photon Service"; + path = ["/run/current-system/sw"]; + serviceConfig = { + WorkingDirectory = "/data/semitamaps-data/photon"; + ExecStart = pkgs.writeShellScript "photon" '' + java -jar photon-*.jar -cors-any + ''; + Restart = "on-failure"; + }; + }; + + services.openssh = { + enable = true; + settings.PasswordAuthentication = false; + }; + + users.users = { + root = { + openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); + }; + }; + + system.stateVersion = "24.05"; + }; + }; +} diff --git a/hosts/semita/default.nix b/hosts/semita/default.nix index e3f428a..63ec365 100644 --- a/hosts/semita/default.nix +++ b/hosts/semita/default.nix @@ -62,6 +62,7 @@ in { ../common/optional/nixos-containers/nix-bitcoin.nix ../common/optional/nixos-containers/postgres.nix ../common/optional/nixos-containers/baseddata-worker.nix + ../common/optional/nixos-containers/semitamaps-worker.nix ../common/optional/nixos-containers/backup-server.nix ../common/optional/nixos-containers/docker.nix @@ -69,7 +70,7 @@ in { # ../common/optional/distributed-builds/local-machine.nix ]; - fileSystems."/media/main-ssd" = { + fileSystems."/mnt/main-ssd" = { device = "/dev/disk/by-uuid/ba884006-e813-4b67-9fe6-62aea08b3b59"; fsType = "ext4"; }; diff --git a/vars/default.nix b/vars/default.nix index 830b61f..8307b2e 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -5,12 +5,15 @@ email ; locations = { - dockerContainerData = "/media/main-ssd/docker"; mediaDataMountPoint = "/media/media"; - bitcoinNodeContainerData = "/media/main-ssd/nix-bitcoin"; - backupContainerData = "/media/main-ssd/backup"; - postgresContainerData = "/media/main-ssd/postgresql"; photosDataMountPoint = "/media/photos"; + + dockerContainerData = "/mnt/main-ssd/docker"; + bitcoinNodeContainerData = "/mnt/main-ssd/nix-bitcoin"; + backupContainerData = "/mnt/main-ssd/backup"; + postgresContainerData = "/mnt/main-ssd/postgresql"; + semitamapsData = "/mnt/main-ssd/semitamaps-data"; + baseddataData = "/mnt/main-ssd/baseddata-data"; }; }