From 3c0baaed18810faddaddc0854759af965f6c2206 Mon Sep 17 00:00:00 2001 From: Sam Date: Sun, 26 Jan 2025 23:59:29 +0000 Subject: [PATCH 01/31] add ssl certs to reverse proxy --- .../nixos-containers/reverse-proxy.nix | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/reverse-proxy.nix b/hosts/common/optional/nixos-containers/reverse-proxy.nix index bbb57d2..ea82408 100644 --- a/hosts/common/optional/nixos-containers/reverse-proxy.nix +++ b/hosts/common/optional/nixos-containers/reverse-proxy.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "reverse-proxy"; @@ -10,6 +11,7 @@ gatewayIp = configVars.networking.addresses.gateway.ip; pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + sops-nix = inputs.sops-nix; dockerContainerIp = configVars.networking.addresses.docker.ip; bdWorker = configVars.networking.addresses.bd-worker.ip; pihole = configVars.networking.addresses.pihole.ip; @@ -33,13 +35,40 @@ in { privateNetwork = true; hostBridge = "br0"; nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + }; config = { pkgs, lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + }; + + sops.secrets = { + "ssl_keys/lan-selfsigned.crt" = { + mode = "0644"; + }; + "ssl_keys/lan-selfsigned.key" = { + mode = "0644"; + }; + }; + networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -52,6 +81,7 @@ in { enable = true; allowedTCPPorts = [ 80 + 443 ]; }; useHostResolvConf = lib.mkForce false; @@ -60,6 +90,7 @@ in { services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -72,27 +103,51 @@ in { enable = true; virtualHosts = { "jellyfin.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${dockerContainerIp}:8096"; }; "mempool.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${bitcoinNode}:4080"; + extraConfig = '' + proxy_set_header Host mempool.lan; + ''; }; "grafana.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${metricsServer}:2342"; extraConfig = '' proxy_set_header Host grafana.lan; ''; }; "metrics.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${metricsServer}:9001"; }; "searx.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${dockerContainerIp}:8855"; }; "dns.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${pihole}:80"; }; "prefect.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${bdWorker}:4200"; }; }; From 05012c2056a9b70faf8e8108031904b999f16e80 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 10:41:43 +0000 Subject: [PATCH 02/31] change jellyfin from arion to oci container --- .../optional/arion-containers/jellyfin.nix | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 65a075c..0792ea3 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -1,31 +1,27 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.jellyfin = { - settings = { - services.jellyfin.service = { - ports = [ - "8096:8096" - ]; - container_name = "jellyfin"; - image = "lscr.io/linuxserver/jellyfin:latest"; - restart = "always"; - volumes = [ - "/srv/docker/media-server/jellyfin/config:/config" - "/media/media/tv:/data/tvshows:ro" - "/media/media/movies:/data/movies:ro" - "/media/media/music/music_data:/data/music:ro" - "/media/media/youtube:/data/youtube:ro" - "/media/media/podcasts:/data/podcasts:ro" - "/srv/docker/media-server/jellyfin/config/custom-cont-init.d:/custom-cont-init.d:ro" - ]; - environment = { - PUID = "1000"; - PGID = "1000"; - DOCKER_MODS="linuxserver/mods:jellyfin-opencl-intel"; - }; - devices = ["/dev/dri:/dev/dri"]; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + jellyfin = { + image = "lscr.io/linuxserver/jellyfin:latest"; + ports = [ + "8096:8096" + ]; + volumes = [ + "/srv/docker/media-server/jellyfin/config:/config" + "/media/media/tv:/data/tvshows:ro" + "/media/media/movies:/data/movies:ro" + "/media/media/music/music_data:/data/music:ro" + "/media/media/youtube:/data/youtube:ro" + "/media/media/podcasts:/data/podcasts:ro" + "/srv/docker/media-server/jellyfin/config/custom-cont-init.d:/custom-cont-init.d:ro" + ]; + environment = { + PUID = "1000"; + PGID = "1000"; + DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; }; + extraOptions = ["--gpus=all"]; }; }; }; From 2ad550029420d0c7dedaa7d279f30a1665489542 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 13:37:52 +0000 Subject: [PATCH 03/31] change jellyfin to oci container --- .../optional/arion-containers/jellyfin.nix | 5 ++++- .../common/optional/nixos-containers/docker.nix | 16 ++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 0792ea3..600eefe 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -20,8 +20,11 @@ PUID = "1000"; PGID = "1000"; DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; + NVIDIA_VISIBLE_DEVICES = "all"; }; - extraOptions = ["--gpus=all"]; + extraOptions = [ + "--device=nvidia.com/gpu=all" + ]; }; }; }; diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index cc74b46..6b1f7b2 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -53,6 +53,10 @@ in { node = "/dev/nvidiactl"; modifier = "rwm"; } + { + node = "/dev/nvidia-uvm"; + modifier = "rwm"; + } { node = "/dev/fuse"; modifier = "rwm"; @@ -204,18 +208,6 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; - systemd.services.podman-autostart = { - enable = true; - after = ["podman.service"]; - wantedBy = ["multi-user.target"]; - description = "Automatically start containers with --restart=always tag"; - serviceConfig = { - Type = "idle"; - ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; - ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; - }; - }; - services.prometheus = { exporters = { node = { From 3f28197cafb19ee3fef4f3f1db354f0a58182a79 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:07:48 +0000 Subject: [PATCH 04/31] rm nvim add ccrypt --- hosts/iso/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/iso/default.nix b/hosts/iso/default.nix index 4e8fd4c..7d9736b 100644 --- a/hosts/iso/default.nix +++ b/hosts/iso/default.nix @@ -3,7 +3,7 @@ in { imports = [(modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix")]; environment.systemPackages = [ pkgs.openssl - pkgs.nvim + pkgs.ccrypt ]; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; } From 31e98b52591e45513f46bd30a3ef3b7cb5ffb55d Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:09:34 +0000 Subject: [PATCH 05/31] add vlc --- home/common/optional/desktop/common/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/home/common/optional/desktop/common/default.nix b/home/common/optional/desktop/common/default.nix index 1c0bbad..79ccb19 100644 --- a/home/common/optional/desktop/common/default.nix +++ b/home/common/optional/desktop/common/default.nix @@ -29,5 +29,6 @@ pkgs.R pkgs.gimp pkgs.gajim + pkgs.vlc ]; } From fad91cfe25f2a9a0da198fdfe0e3a77eff8c0065 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:25:51 +0000 Subject: [PATCH 06/31] change semita worker to use oci containers --- .../semitamaps-tileserver.nix | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix index c8c1d8f..5cd6e50 100644 --- a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix +++ b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix @@ -1,31 +1,25 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.semitamaps-tileserver = { - settings = { - services.tileserver-gl.service = { - ports = [ - "8080:8080" - ]; - container_name = "tileserver-gl"; - image = "maptiler/tileserver-gl"; - restart = "always"; - volumes = [ - "/data/semitamaps-data/tileserver-gl/data:/data" - ]; - command = "-c /data/config.json --public_url https://tiles.semitamaps.com/"; - }; - - services.mbgl-renderer.service = { - ports = [ - "8081:80" - ]; - container_name = "mbgl-renderer"; - image = "mbgl-renderer"; - restart = "always"; - }; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + tileserver-gl = { + image = "maptiler/tileserver-gl"; + ports = [ + "8080:8080" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; + }; + mbgl-renderer = { + image = "mbgl-renderer"; + ports = [ + "8081:80" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; }; }; }; } - From c73aa9933ac5561f8a72c5ad1d01057a179c0a9d Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 30 Jan 2025 15:06:26 +0000 Subject: [PATCH 07/31] remove lnd and rtl from nix-bitcoin --- .../optional/nixos-containers/nix-bitcoin.nix | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 0bfd532..2788a60 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -62,7 +62,7 @@ in { }: { imports = [ inputs.nix-bitcoin.nixosModules.default - inputs.lnbits.nixosModules.default + # inputs.lnbits.nixosModules.default ]; environment.systemPackages = with pkgs; [ vim @@ -80,14 +80,13 @@ in { firewall = { enable = true; allowedTCPPorts = [ - 80 - 443 - 22 + # 80 + # 443 config.containers.bitcoin-node.config.services.bitcoind.rpc.port config.containers.bitcoin-node.config.services.mempool.frontend.port config.containers.bitcoin-node.config.services.electrs.port - config.containers.bitcoin-node.config.services.rtl.port - config.containers.bitcoin-node.config.services.lnd.port + # config.containers.bitcoin-node.config.services.rtl.port + # config.containers.bitcoin-node.config.services.lnd.port ]; }; useHostResolvConf = lib.mkForce false; @@ -144,7 +143,7 @@ in { }; }; lnd = { - enable = true; + enable = false; lndconnect = { enable = true; onion = true; @@ -156,34 +155,34 @@ in { ''; }; rtl = { - enable = true; + enable = false; nodes.lnd.enable = true; address = "0.0.0.0"; }; - lnbits = { - enable = true; - openFirewall = true; - host = "0.0.0.0"; - port = 8231; - env = { - LNBITS_ADMIN_UI = "true"; - LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; - LND_REST_ENDPOINT = "https://127.0.0.1:8080"; - LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; - LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; - AUTH_ALLOWED_METHODS = "user-id-only, username-password"; - }; - }; + # lnbits = { + # enable = false; + # openFirewall = true; + # host = "0.0.0.0"; + # port = 8231; + # env = { + # LNBITS_ADMIN_UI = "true"; + # LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; + # LND_REST_ENDPOINT = "https://127.0.0.1:8080"; + # LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; + # LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; + # AUTH_ALLOWED_METHODS = "user-id-only, username-password"; + # }; + # }; }; # Add custom systemd overrides for above services - systemd.services.lnbits.after = ["lnd.service"]; + # systemd.services.lnbits.after = ["lnd.service"]; nix-bitcoin.onionServices = { bitcoind.enable = true; electrs.enable = true; mempool-frontend.enable = true; - lnd.public = true; + # lnd.public = true; }; services.prometheus = { From 37c50ce7b5f7517d56cb952ff6a751942eae1d91 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 31 Jan 2025 21:17:49 +0000 Subject: [PATCH 08/31] add restic-exporter to prometheus metrics --- flake.lock | 8 ++-- .../nixos-containers/metrics-server.nix | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 550380e..0b5877e 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737899664, - "narHash": "sha256-iZpzTSERNQ5UvFfEzrBLuEmcRUGjBSal7ShtXurYq8Q=", + "lastModified": 1738356588, + "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", "ref": "refs/heads/master", - "rev": "a9844a78dcbdc8a84679835112970d80822b113c", - "revCount": 257, + "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", + "revCount": 267, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 50417f4..148fbac 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "metrics-server"; @@ -15,6 +16,7 @@ bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; postres = configVars.networking.addresses.postgres.ip; backupServer = configVars.networking.addresses.backup-server.ip; + sops-nix = inputs.sops-nix; http_endpoints = configVars.metrics-server.blackbox.http_endpoints; @@ -52,6 +54,10 @@ in { hostPath = metricsServerContainerData; isReadOnly = false; }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; }; config = { @@ -59,7 +65,10 @@ in { lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -79,9 +88,23 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/restic-passphrase" = {}; + "software/restic-exporter-credentials" = {}; + }; + }; + services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -110,6 +133,16 @@ in { } ]; } + { + job_name = "restic-exporter"; + static_configs = [ + { + targets = [ + "0.0.0.0:8001" + ]; + } + ]; + } { job_name = "blackbox"; @@ -175,6 +208,14 @@ in { enabledCollectors = ["systemd"]; port = 9002; }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; + }; }; }; From f7876d08f6f99ece6320626186b38db9568a2307 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 13:15:18 +0000 Subject: [PATCH 09/31] add baikal oci container to docker --- flake.lock | 8 ++++---- .../common/optional/arion-containers/baikal.nix | 17 +++++++++++++++++ .../common/optional/nixos-containers/docker.nix | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 hosts/common/optional/arion-containers/baikal.nix diff --git a/flake.lock b/flake.lock index 0b5877e..9d5a28c 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738356588, - "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", + "lastModified": 1738358831, + "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", "ref": "refs/heads/master", - "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", - "revCount": 267, + "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", + "revCount": 268, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/arion-containers/baikal.nix new file mode 100644 index 0000000..5e41de5 --- /dev/null +++ b/hosts/common/optional/arion-containers/baikal.nix @@ -0,0 +1,17 @@ +{ + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + baikal = { + image = "ckulka/baikal:nginx"; + ports = [ + "6734:80" + ]; + volumes = [ + "/srv/docker/baikal/config:/var/www/baikal/config" + "/srv/docker/baikal/data:/var/www/baikal/Specific" + ]; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 6b1f7b2..12659aa 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -182,6 +182,7 @@ in { ../arion-containers/jellyfin.nix ../arion-containers/photoprism.nix ../arion-containers/syncthing.nix + ../arion-containers/baikal.nix (import ../arion-containers/searxng.nix {configVars = configVars;}) ]; From 996e51f56e4aa5609d33482718340ca7460065e8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:28:59 +0000 Subject: [PATCH 10/31] setup prometheus alertmanager --- flake.lock | 8 +- .../nixos-containers/metrics-server.nix | 149 +++++++++++++----- vars/default.nix | 1 + 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/flake.lock b/flake.lock index 9d5a28c..70e52a9 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738358831, - "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", + "lastModified": 1738685297, + "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", "ref": "refs/heads/master", - "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", - "revCount": 268, + "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", + "revCount": 269, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 148fbac..dd7c746 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -8,6 +8,9 @@ containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + notifybotJid = configVars.xmpp.notifybotJid; + receiverJid = configVars.xmpp.personalAccount; + dockerContainerIp = configVars.networking.addresses.docker.ip; smWorkerIp = configVars.networking.addresses.sm-worker.ip; merlinIp = configVars.networking.addresses.merlin.ip; @@ -83,6 +86,7 @@ in { config.services.prometheus.port config.services.grafana.port config.services.prometheus.exporters.blackbox.port + 9199 #xmpp listen port ]; }; useHostResolvConf = lib.mkForce false; @@ -98,6 +102,9 @@ in { secrets = { "software/restic-passphrase" = {}; "software/restic-exporter-credentials" = {}; + "comms/xmpp/notifybot/password" = { + mode = "0644"; + }; }; }; @@ -112,9 +119,48 @@ in { pkgs.git ]; + services.grafana = { + enable = true; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; + }; + + # main prometheus service services.prometheus = { enable = true; port = 9001; + alertmanagers = [ + { + scheme = "http"; + path_prefix = "/"; + static_configs = [ + { + targets = [ + "0.0.0.0:9093" + ]; + } + ]; + } + ]; + ruleFiles = [ + "${pkgs.writeText + "alert_rule.yml" + '' + groups: + - name: blackbox_alert + rules: + - alert: EndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Endpoint {{ $labels.instance }} down" + description: "An endpoint has been down for more than 1 minute." + ''}" + ]; scrapeConfigs = [ { job_name = "node_exporter"; @@ -171,51 +217,74 @@ in { ]; }; - services.grafana = { + # setup alertmanager + services.prometheus.xmpp-alerts = { enable = true; - settings.server = { - http_port = 2342; - http_addr = "0.0.0.0"; + settings = { + jid = notifybotJid; + password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}"; + to_jid = receiverJid; + listen_address = "0.0.0.0"; + listen_port = 9199; }; }; + services.prometheus.alertmanager = { + webExternalUrl = containerIp; + enable = true; + openFirewall = true; + port = 9093; + configText = '' + global: + resolve_timeout: 1m - services.prometheus = { - exporters = { - blackbox = { - enable = true; - configFile = pkgs.writeText "blackbox-conf.yaml" '' - modules: - http_basic: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - valid_http_versions: ["HTTP/1.1", "HTTP/2"] - method: GET - # fail_if_ssl: false - # fail_if_not_ssl: true - # tls_config: - # insecure_skip_verify: true - tcp_connect: - prober: tcp - tcp: - preferred_ip_protocol: ip4 + route: + group_by: ['...'] + repeat_interval: 1h + receiver: 'xmpp-alerts' - ''; - }; - node = { - enable = true; - enabledCollectors = ["systemd"]; - port = 9002; - }; - restic = { - enable = true; - repository = ""; - environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; - passwordFile = config.sops.secrets."software/restic-passphrase".path; - refreshInterval = 10800; # refresh every 3 hours - port = 8001; - }; + receivers: + - name: 'xmpp-alerts' + webhook_configs: + - url: 'http://0.0.0.0:9199/alert' + ''; + }; + + # prometheus exporters + services.prometheus.exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + # fail_if_ssl: false + # fail_if_not_ssl: true + # tls_config: + # insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; }; }; diff --git a/vars/default.nix b/vars/default.nix index f6973cc..82b6969 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -4,6 +4,7 @@ networking email metrics-server + xmpp ; locations = { mediaDataMountPoint = "/media/media"; From bb5ccb0efc6df9e6f1260db353c4375c7a6c3edc Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:47:32 +0000 Subject: [PATCH 11/31] metrics-server add externalURL to prometheus --- hosts/common/optional/nixos-containers/metrics-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index dd7c746..a880109 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -130,6 +130,7 @@ in { # main prometheus service services.prometheus = { enable = true; + webExternalUrl = "http://${containerIp}:9001"; port = 9001; alertmanagers = [ { @@ -229,7 +230,7 @@ in { }; }; services.prometheus.alertmanager = { - webExternalUrl = containerIp; + webExternalUrl = "http://${containerIp}:9093"; enable = true; openFirewall = true; port = 9093; From 43943485ee5250a4e45bdb9581c54d82b83a72d4 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 13:37:52 +0000 Subject: [PATCH 12/31] change jellyfin to oci container --- .../optional/arion-containers/jellyfin.nix | 5 ++++- .../common/optional/nixos-containers/docker.nix | 16 ++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 0792ea3..600eefe 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -20,8 +20,11 @@ PUID = "1000"; PGID = "1000"; DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; + NVIDIA_VISIBLE_DEVICES = "all"; }; - extraOptions = ["--gpus=all"]; + extraOptions = [ + "--device=nvidia.com/gpu=all" + ]; }; }; }; diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index cc74b46..6b1f7b2 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -53,6 +53,10 @@ in { node = "/dev/nvidiactl"; modifier = "rwm"; } + { + node = "/dev/nvidia-uvm"; + modifier = "rwm"; + } { node = "/dev/fuse"; modifier = "rwm"; @@ -204,18 +208,6 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; - systemd.services.podman-autostart = { - enable = true; - after = ["podman.service"]; - wantedBy = ["multi-user.target"]; - description = "Automatically start containers with --restart=always tag"; - serviceConfig = { - Type = "idle"; - ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; - ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; - }; - }; - services.prometheus = { exporters = { node = { From 7416197e54a39869b81ce573820a5ed9c6e67db8 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:09:34 +0000 Subject: [PATCH 13/31] add vlc --- home/common/optional/desktop/common/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/home/common/optional/desktop/common/default.nix b/home/common/optional/desktop/common/default.nix index 1c0bbad..79ccb19 100644 --- a/home/common/optional/desktop/common/default.nix +++ b/home/common/optional/desktop/common/default.nix @@ -29,5 +29,6 @@ pkgs.R pkgs.gimp pkgs.gajim + pkgs.vlc ]; } From 12aa157690e454466364833341c42a4d2d069c35 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:25:51 +0000 Subject: [PATCH 14/31] change semita worker to use oci containers --- .../semitamaps-tileserver.nix | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix index c8c1d8f..5cd6e50 100644 --- a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix +++ b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix @@ -1,31 +1,25 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.semitamaps-tileserver = { - settings = { - services.tileserver-gl.service = { - ports = [ - "8080:8080" - ]; - container_name = "tileserver-gl"; - image = "maptiler/tileserver-gl"; - restart = "always"; - volumes = [ - "/data/semitamaps-data/tileserver-gl/data:/data" - ]; - command = "-c /data/config.json --public_url https://tiles.semitamaps.com/"; - }; - - services.mbgl-renderer.service = { - ports = [ - "8081:80" - ]; - container_name = "mbgl-renderer"; - image = "mbgl-renderer"; - restart = "always"; - }; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + tileserver-gl = { + image = "maptiler/tileserver-gl"; + ports = [ + "8080:8080" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; + }; + mbgl-renderer = { + image = "mbgl-renderer"; + ports = [ + "8081:80" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; }; }; }; } - From 9abf175b5c376af2d549a15440e1e7010029c68b Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 30 Jan 2025 15:06:26 +0000 Subject: [PATCH 15/31] remove lnd and rtl from nix-bitcoin --- .../optional/nixos-containers/nix-bitcoin.nix | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 0bfd532..2788a60 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -62,7 +62,7 @@ in { }: { imports = [ inputs.nix-bitcoin.nixosModules.default - inputs.lnbits.nixosModules.default + # inputs.lnbits.nixosModules.default ]; environment.systemPackages = with pkgs; [ vim @@ -80,14 +80,13 @@ in { firewall = { enable = true; allowedTCPPorts = [ - 80 - 443 - 22 + # 80 + # 443 config.containers.bitcoin-node.config.services.bitcoind.rpc.port config.containers.bitcoin-node.config.services.mempool.frontend.port config.containers.bitcoin-node.config.services.electrs.port - config.containers.bitcoin-node.config.services.rtl.port - config.containers.bitcoin-node.config.services.lnd.port + # config.containers.bitcoin-node.config.services.rtl.port + # config.containers.bitcoin-node.config.services.lnd.port ]; }; useHostResolvConf = lib.mkForce false; @@ -144,7 +143,7 @@ in { }; }; lnd = { - enable = true; + enable = false; lndconnect = { enable = true; onion = true; @@ -156,34 +155,34 @@ in { ''; }; rtl = { - enable = true; + enable = false; nodes.lnd.enable = true; address = "0.0.0.0"; }; - lnbits = { - enable = true; - openFirewall = true; - host = "0.0.0.0"; - port = 8231; - env = { - LNBITS_ADMIN_UI = "true"; - LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; - LND_REST_ENDPOINT = "https://127.0.0.1:8080"; - LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; - LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; - AUTH_ALLOWED_METHODS = "user-id-only, username-password"; - }; - }; + # lnbits = { + # enable = false; + # openFirewall = true; + # host = "0.0.0.0"; + # port = 8231; + # env = { + # LNBITS_ADMIN_UI = "true"; + # LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; + # LND_REST_ENDPOINT = "https://127.0.0.1:8080"; + # LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; + # LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; + # AUTH_ALLOWED_METHODS = "user-id-only, username-password"; + # }; + # }; }; # Add custom systemd overrides for above services - systemd.services.lnbits.after = ["lnd.service"]; + # systemd.services.lnbits.after = ["lnd.service"]; nix-bitcoin.onionServices = { bitcoind.enable = true; electrs.enable = true; mempool-frontend.enable = true; - lnd.public = true; + # lnd.public = true; }; services.prometheus = { From 24e1bc12ea1ec74c7b119426d319feac083105e2 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 31 Jan 2025 21:17:49 +0000 Subject: [PATCH 16/31] add restic-exporter to prometheus metrics --- flake.lock | 8 ++-- .../nixos-containers/metrics-server.nix | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 550380e..0b5877e 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737899664, - "narHash": "sha256-iZpzTSERNQ5UvFfEzrBLuEmcRUGjBSal7ShtXurYq8Q=", + "lastModified": 1738356588, + "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", "ref": "refs/heads/master", - "rev": "a9844a78dcbdc8a84679835112970d80822b113c", - "revCount": 257, + "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", + "revCount": 267, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 50417f4..148fbac 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "metrics-server"; @@ -15,6 +16,7 @@ bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; postres = configVars.networking.addresses.postgres.ip; backupServer = configVars.networking.addresses.backup-server.ip; + sops-nix = inputs.sops-nix; http_endpoints = configVars.metrics-server.blackbox.http_endpoints; @@ -52,6 +54,10 @@ in { hostPath = metricsServerContainerData; isReadOnly = false; }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; }; config = { @@ -59,7 +65,10 @@ in { lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -79,9 +88,23 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/restic-passphrase" = {}; + "software/restic-exporter-credentials" = {}; + }; + }; + services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -110,6 +133,16 @@ in { } ]; } + { + job_name = "restic-exporter"; + static_configs = [ + { + targets = [ + "0.0.0.0:8001" + ]; + } + ]; + } { job_name = "blackbox"; @@ -175,6 +208,14 @@ in { enabledCollectors = ["systemd"]; port = 9002; }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; + }; }; }; From b1e4be205318dd22a3c5da63713b749604dd448f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 13:15:18 +0000 Subject: [PATCH 17/31] add baikal oci container to docker --- flake.lock | 8 ++++---- .../common/optional/arion-containers/baikal.nix | 17 +++++++++++++++++ .../common/optional/nixos-containers/docker.nix | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 hosts/common/optional/arion-containers/baikal.nix diff --git a/flake.lock b/flake.lock index 0b5877e..9d5a28c 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738356588, - "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", + "lastModified": 1738358831, + "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", "ref": "refs/heads/master", - "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", - "revCount": 267, + "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", + "revCount": 268, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/arion-containers/baikal.nix new file mode 100644 index 0000000..5e41de5 --- /dev/null +++ b/hosts/common/optional/arion-containers/baikal.nix @@ -0,0 +1,17 @@ +{ + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + baikal = { + image = "ckulka/baikal:nginx"; + ports = [ + "6734:80" + ]; + volumes = [ + "/srv/docker/baikal/config:/var/www/baikal/config" + "/srv/docker/baikal/data:/var/www/baikal/Specific" + ]; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 6b1f7b2..12659aa 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -182,6 +182,7 @@ in { ../arion-containers/jellyfin.nix ../arion-containers/photoprism.nix ../arion-containers/syncthing.nix + ../arion-containers/baikal.nix (import ../arion-containers/searxng.nix {configVars = configVars;}) ]; From 068593c82c19ca576fe1cccd124257d3094b7f63 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:28:59 +0000 Subject: [PATCH 18/31] setup prometheus alertmanager --- flake.lock | 8 +- .../nixos-containers/metrics-server.nix | 149 +++++++++++++----- vars/default.nix | 1 + 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/flake.lock b/flake.lock index 9d5a28c..70e52a9 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738358831, - "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", + "lastModified": 1738685297, + "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", "ref": "refs/heads/master", - "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", - "revCount": 268, + "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", + "revCount": 269, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 148fbac..dd7c746 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -8,6 +8,9 @@ containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + notifybotJid = configVars.xmpp.notifybotJid; + receiverJid = configVars.xmpp.personalAccount; + dockerContainerIp = configVars.networking.addresses.docker.ip; smWorkerIp = configVars.networking.addresses.sm-worker.ip; merlinIp = configVars.networking.addresses.merlin.ip; @@ -83,6 +86,7 @@ in { config.services.prometheus.port config.services.grafana.port config.services.prometheus.exporters.blackbox.port + 9199 #xmpp listen port ]; }; useHostResolvConf = lib.mkForce false; @@ -98,6 +102,9 @@ in { secrets = { "software/restic-passphrase" = {}; "software/restic-exporter-credentials" = {}; + "comms/xmpp/notifybot/password" = { + mode = "0644"; + }; }; }; @@ -112,9 +119,48 @@ in { pkgs.git ]; + services.grafana = { + enable = true; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; + }; + + # main prometheus service services.prometheus = { enable = true; port = 9001; + alertmanagers = [ + { + scheme = "http"; + path_prefix = "/"; + static_configs = [ + { + targets = [ + "0.0.0.0:9093" + ]; + } + ]; + } + ]; + ruleFiles = [ + "${pkgs.writeText + "alert_rule.yml" + '' + groups: + - name: blackbox_alert + rules: + - alert: EndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Endpoint {{ $labels.instance }} down" + description: "An endpoint has been down for more than 1 minute." + ''}" + ]; scrapeConfigs = [ { job_name = "node_exporter"; @@ -171,51 +217,74 @@ in { ]; }; - services.grafana = { + # setup alertmanager + services.prometheus.xmpp-alerts = { enable = true; - settings.server = { - http_port = 2342; - http_addr = "0.0.0.0"; + settings = { + jid = notifybotJid; + password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}"; + to_jid = receiverJid; + listen_address = "0.0.0.0"; + listen_port = 9199; }; }; + services.prometheus.alertmanager = { + webExternalUrl = containerIp; + enable = true; + openFirewall = true; + port = 9093; + configText = '' + global: + resolve_timeout: 1m - services.prometheus = { - exporters = { - blackbox = { - enable = true; - configFile = pkgs.writeText "blackbox-conf.yaml" '' - modules: - http_basic: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - valid_http_versions: ["HTTP/1.1", "HTTP/2"] - method: GET - # fail_if_ssl: false - # fail_if_not_ssl: true - # tls_config: - # insecure_skip_verify: true - tcp_connect: - prober: tcp - tcp: - preferred_ip_protocol: ip4 + route: + group_by: ['...'] + repeat_interval: 1h + receiver: 'xmpp-alerts' - ''; - }; - node = { - enable = true; - enabledCollectors = ["systemd"]; - port = 9002; - }; - restic = { - enable = true; - repository = ""; - environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; - passwordFile = config.sops.secrets."software/restic-passphrase".path; - refreshInterval = 10800; # refresh every 3 hours - port = 8001; - }; + receivers: + - name: 'xmpp-alerts' + webhook_configs: + - url: 'http://0.0.0.0:9199/alert' + ''; + }; + + # prometheus exporters + services.prometheus.exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + # fail_if_ssl: false + # fail_if_not_ssl: true + # tls_config: + # insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; }; }; diff --git a/vars/default.nix b/vars/default.nix index f6973cc..82b6969 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -4,6 +4,7 @@ networking email metrics-server + xmpp ; locations = { mediaDataMountPoint = "/media/media"; From 595f1f92d5d7370f3a6ce3f36066135510aa83e8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:47:32 +0000 Subject: [PATCH 19/31] metrics-server add externalURL to prometheus --- hosts/common/optional/nixos-containers/metrics-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index dd7c746..a880109 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -130,6 +130,7 @@ in { # main prometheus service services.prometheus = { enable = true; + webExternalUrl = "http://${containerIp}:9001"; port = 9001; alertmanagers = [ { @@ -229,7 +230,7 @@ in { }; }; services.prometheus.alertmanager = { - webExternalUrl = containerIp; + webExternalUrl = "http://${containerIp}:9093"; enable = true; openFirewall = true; port = 9093; From 2fc2bb92d5e84b4925499e41e16507ccbdf56dfd Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 6 Feb 2025 08:34:27 +0000 Subject: [PATCH 20/31] rename arion-containers dir to docker-containers --- .../arrstack.nix | 0 .../baikal.nix | 0 .../jellyfin.nix | 0 .../photoprism.nix | 0 .../pihole.nix | 0 .../searxng.nix | 0 .../semitamaps-tileserver.nix | 0 .../syncthing.nix | 0 hosts/common/optional/nixos-containers/docker.nix | 12 ++++++------ hosts/common/optional/nixos-containers/pihole.nix | 2 +- .../optional/nixos-containers/semitamaps-worker.nix | 2 +- 11 files changed, 8 insertions(+), 8 deletions(-) rename hosts/common/optional/{arion-containers => docker-containers}/arrstack.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/baikal.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/jellyfin.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/photoprism.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/pihole.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/searxng.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/semitamaps-tileserver.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/syncthing.nix (100%) diff --git a/hosts/common/optional/arion-containers/arrstack.nix b/hosts/common/optional/docker-containers/arrstack.nix similarity index 100% rename from hosts/common/optional/arion-containers/arrstack.nix rename to hosts/common/optional/docker-containers/arrstack.nix diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/docker-containers/baikal.nix similarity index 100% rename from hosts/common/optional/arion-containers/baikal.nix rename to hosts/common/optional/docker-containers/baikal.nix diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/docker-containers/jellyfin.nix similarity index 100% rename from hosts/common/optional/arion-containers/jellyfin.nix rename to hosts/common/optional/docker-containers/jellyfin.nix diff --git a/hosts/common/optional/arion-containers/photoprism.nix b/hosts/common/optional/docker-containers/photoprism.nix similarity index 100% rename from hosts/common/optional/arion-containers/photoprism.nix rename to hosts/common/optional/docker-containers/photoprism.nix diff --git a/hosts/common/optional/arion-containers/pihole.nix b/hosts/common/optional/docker-containers/pihole.nix similarity index 100% rename from hosts/common/optional/arion-containers/pihole.nix rename to hosts/common/optional/docker-containers/pihole.nix diff --git a/hosts/common/optional/arion-containers/searxng.nix b/hosts/common/optional/docker-containers/searxng.nix similarity index 100% rename from hosts/common/optional/arion-containers/searxng.nix rename to hosts/common/optional/docker-containers/searxng.nix diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/docker-containers/semitamaps-tileserver.nix similarity index 100% rename from hosts/common/optional/arion-containers/semitamaps-tileserver.nix rename to hosts/common/optional/docker-containers/semitamaps-tileserver.nix diff --git a/hosts/common/optional/arion-containers/syncthing.nix b/hosts/common/optional/docker-containers/syncthing.nix similarity index 100% rename from hosts/common/optional/arion-containers/syncthing.nix rename to hosts/common/optional/docker-containers/syncthing.nix diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 12659aa..833ef00 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -178,12 +178,12 @@ in { imports = [ arion.nixosModules.arion sops-nix.nixosModules.sops - ../arion-containers/arrstack.nix - ../arion-containers/jellyfin.nix - ../arion-containers/photoprism.nix - ../arion-containers/syncthing.nix - ../arion-containers/baikal.nix - (import ../arion-containers/searxng.nix {configVars = configVars;}) + ../docker-containers/arrstack.nix + ../docker-containers/jellyfin.nix + ../docker-containers/photoprism.nix + ../docker-containers/syncthing.nix + ../docker-containers/baikal.nix + (import ../docker-containers/searxng.nix {configVars = configVars;}) ]; environment.systemPackages = [ diff --git a/hosts/common/optional/nixos-containers/pihole.nix b/hosts/common/optional/nixos-containers/pihole.nix index 1f648fd..31d5a13 100644 --- a/hosts/common/optional/nixos-containers/pihole.nix +++ b/hosts/common/optional/nixos-containers/pihole.nix @@ -72,7 +72,7 @@ in { imports = [ arion.nixosModules.arion - ../arion-containers/pihole.nix + ../docker-containers/pihole.nix ]; environment.systemPackages = [ diff --git a/hosts/common/optional/nixos-containers/semitamaps-worker.nix b/hosts/common/optional/nixos-containers/semitamaps-worker.nix index 9270136..c3ee543 100644 --- a/hosts/common/optional/nixos-containers/semitamaps-worker.nix +++ b/hosts/common/optional/nixos-containers/semitamaps-worker.nix @@ -101,7 +101,7 @@ in { imports = [ sops-nix.nixosModules.sops arion.nixosModules.arion - ../arion-containers/semitamaps-tileserver.nix + ../docker-containers/semitamaps-tileserver.nix ]; environment.systemPackages = [ From 7c8a6a0d091f41ce5b9c7335a12d03ead456849d Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:02:21 +0000 Subject: [PATCH 21/31] update flake secrets --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 70e52a9..5726670 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738685297, - "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", + "lastModified": 1738864447, + "narHash": "sha256-MT8kRKA8s7nNen4mJPuITfQ10kpNmGZSNMD2v3/vdz8=", "ref": "refs/heads/master", - "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", - "revCount": 269, + "rev": "b5977aae476314875fed887c9a64fc53b024df84", + "revCount": 273, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, From 27a7b0056525855669caf484fbb9a4de35eccea9 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:02:40 +0000 Subject: [PATCH 22/31] setup fail2ban --- hosts/common/optional/fail2ban.nix | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 hosts/common/optional/fail2ban.nix diff --git a/hosts/common/optional/fail2ban.nix b/hosts/common/optional/fail2ban.nix new file mode 100644 index 0000000..5811fcc --- /dev/null +++ b/hosts/common/optional/fail2ban.nix @@ -0,0 +1,34 @@ +{pkgs, ...}: { + + environment.systemPackages = [pkgs.fail2ban]; + + environment.etc = { + "fail2ban/filter.d/nginx-bruteforce.conf".text = '' + [Definition] + failregex = ^.*(GET|POST).* (404|444|403|400) .*$ + ''; + }; + + services.fail2ban = { + enable = true; + maxretry = 5; + ignoreIP = [ + ]; + bantime-increment = { + enable = true; + multipliers = "1 2 4 8 16 32 64"; + maxtime = "168h"; + }; + jails = { + nginx-spam.settings = { + filter = "nginx-bruteforce"; + action = "iptables-allports"; + logpath = "/var/log/nginx/access.log"; + backend = "auto"; + findtime = 600; + bantime = 600; + maxretry = 10; + }; + }; + }; +} From 53106e91da15cbd1181521f02fedcbfb17f291b9 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:03:01 +0000 Subject: [PATCH 23/31] nginx configs for semitamaps and vaultwarden --- hosts/common/optional/nginx/semitamaps.nix | 7 ++++++ hosts/common/optional/nginx/vaultwarden.nix | 24 +++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 hosts/common/optional/nginx/semitamaps.nix create mode 100644 hosts/common/optional/nginx/vaultwarden.nix diff --git a/hosts/common/optional/nginx/semitamaps.nix b/hosts/common/optional/nginx/semitamaps.nix new file mode 100644 index 0000000..e9a65aa --- /dev/null +++ b/hosts/common/optional/nginx/semitamaps.nix @@ -0,0 +1,7 @@ +{ + networking.firewall.allowedTCPPorts = [ 80 ]; + services.nginx.enable = true; + services.nginx.virtualHosts."samchance.xyz" = { + root = "/srv/hello/"; + }; +} diff --git a/hosts/common/optional/nginx/vaultwarden.nix b/hosts/common/optional/nginx/vaultwarden.nix new file mode 100644 index 0000000..097912f --- /dev/null +++ b/hosts/common/optional/nginx/vaultwarden.nix @@ -0,0 +1,24 @@ +{configVars, ...}: let + email = configVars.email.user; + domain = configVars.domains.vaultwarden; + vaultwardenIp = configVars.networking.addresses.vaultwarden.localAddress; + vaultwardenPort = configVars.networking.addresses.vaultwarden.port; +in { + networking.firewall.allowedTCPPorts = [80 443]; + security.acme = { + acceptTerms = true; + defaults.email = email; + }; + services.nginx = { + enable = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + virtualHosts."${domain}" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://${vaultwardenIp}:${toString vaultwardenPort}"; + }; + }; + }; +} From 2e984daca084a9af04632deaf4511795800051a0 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:07:25 +0000 Subject: [PATCH 24/31] add vaultwarden and semitamaps containers --- .../optional/nixos-containers/semitamaps.nix | 75 ++++++++++++ .../optional/nixos-containers/vaultwarden.nix | 110 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 hosts/common/optional/nixos-containers/semitamaps.nix create mode 100644 hosts/common/optional/nixos-containers/vaultwarden.nix diff --git a/hosts/common/optional/nixos-containers/semitamaps.nix b/hosts/common/optional/nixos-containers/semitamaps.nix new file mode 100644 index 0000000..993deec --- /dev/null +++ b/hosts/common/optional/nixos-containers/semitamaps.nix @@ -0,0 +1,75 @@ +{ + pkgs, + lib, + ... +}: let + containerName = "semitamaps"; + pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + hostAddress = configVars.networking.addresses.semitamaps.hostAddress; + localAddress = configVars.networking.addresses.semitamaps.localAddress; +in { + + networking = { + nat = { + enable = true; + internalInterfaces = ["ve-+"]; + externalInterface = "enp1s0"; + }; + }; + + environment.persistence."/persist" = { + hideMounts = true; + directories = [ + "/var/lib/nixos-containers/${containerName}" + ]; + }; + + containers."${containerName}" = { + autoStart = true; + privateNetwork = true; + hostAddress = hostAddress; + localAddress = localAddress; + nixpkgs = pkgs.path; + + config = { + pkgs, + lib, + ... + }: { + + networking = { + firewall = { + enable = true; + rejectPackets = true; + allowedTCPPorts = [ + 80 443 + ]; + }; + useHostResolvConf = lib.mkForce false; + }; + + services.resolved.enable = true; + + imports = [ + ]; + + environment.systemPackages = [ + pkgs.vim + pkgs.git + ]; + + services.openssh = { + enable = true; + settings.PasswordAuthentication = false; + }; + + users.users = { + root = { + openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); + }; + }; + + system.stateVersion = "24.05"; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/vaultwarden.nix b/hosts/common/optional/nixos-containers/vaultwarden.nix new file mode 100644 index 0000000..0d132d5 --- /dev/null +++ b/hosts/common/optional/nixos-containers/vaultwarden.nix @@ -0,0 +1,110 @@ +{ + pkgs, + lib, + configVars, + inputs, + ... +}: let + containerName = "vaultwarden"; + pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + hostAddress = configVars.networking.addresses.vaultwarden.hostAddress; + localAddress = configVars.networking.addresses.vaultwarden.localAddress; + vaultwardenPort = configVars.networking.addresses.vaultwarden.port; + cloudnixIp = configVars.networking.addresses.cloudnix.ip; + sops-nix = inputs.sops-nix; +in { + + networking = { + nat = { + enable = true; + internalInterfaces = ["ve-+"]; + externalInterface = "enp1s0"; + }; + }; + + environment.persistence."/persist" = { + hideMounts = true; + directories = [ + "/var/lib/nixos-containers/${containerName}" + ]; + }; + + containers."${containerName}" = { + autoStart = true; + privateNetwork = true; + hostAddress = hostAddress; + localAddress = localAddress; + nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + }; + + config = { + pkgs, + lib, + ... + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { + + networking = { + defaultGateway = cloudnixIp; + firewall = { + enable = true; + allowedTCPPorts = [ + vaultwardenPort + ]; + }; + useHostResolvConf = lib.mkForce false; + }; + + services.resolved.enable = true; + + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + }; + + imports = [ + sops-nix.nixosModules.sops + ]; + + environment.systemPackages = [ + pkgs.vim + pkgs.git + pkgs.lsof + ]; + + services.vaultwarden = { + enable = true; + dbBackend = "sqlite"; + config = { + ROCKET_ADDRESS = "0.0.0.0"; + ROCKET_PORT = vaultwardenPort; + ROCKET_LOG = "critical"; + }; + }; + + services.openssh = { + enable = true; + settings.PasswordAuthentication = false; + }; + + users.users = { + root = { + openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); + }; + }; + + system.stateVersion = "24.05"; + }; + }; +} From 27f9052eb8d42d7d156da0c29e13841488c37131 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:07:48 +0000 Subject: [PATCH 25/31] update flake secrets --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 5726670..f9bbc07 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738864447, - "narHash": "sha256-MT8kRKA8s7nNen4mJPuITfQ10kpNmGZSNMD2v3/vdz8=", + "lastModified": 1738940734, + "narHash": "sha256-7yH/LFyop6RoazqROOpQGlMRqbw80DfzUroTj9rTVro=", "ref": "refs/heads/master", - "rev": "b5977aae476314875fed887c9a64fc53b024df84", - "revCount": 273, + "rev": "031ee73ab11dd5f91f56cdcac1483b1d59788178", + "revCount": 274, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, From cfc84545407dd4cf79596adac03016d0ac1ac266 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 7 Feb 2025 15:08:19 +0000 Subject: [PATCH 26/31] add services to cloudnix and domains to vars --- hosts/cloudnix/default.nix | 28 ++++++++++++++++++++++++++++ vars/default.nix | 1 + 2 files changed, 29 insertions(+) diff --git a/hosts/cloudnix/default.nix b/hosts/cloudnix/default.nix index bddaabd..679c528 100644 --- a/hosts/cloudnix/default.nix +++ b/hosts/cloudnix/default.nix @@ -30,10 +30,28 @@ in { ../common/optional/openssh.nix ../common/optional/distributed-builds/local-machine.nix + ../common/optional/nixos-containers/semitamaps.nix + ../common/optional/nixos-containers/vaultwarden.nix + ../common/optional/fail2ban.nix + ../common/optional/restic-backup.nix + + ../common/optional/nginx/semitamaps.nix + ../common/optional/nginx/vaultwarden.nix + outputs.nixosModules.nixosAutoUpgrade ]; + services.restic.backups = { + daily = { + paths = [ + "/persist/" + ]; + exclude = [ + ]; + }; + }; + boot = { loader = { efi.canTouchEfiVariables = false; @@ -62,6 +80,16 @@ in { user = "admin"; }; + environment.persistence."/persist" = { + directories = [ + "/var/lib/tailscale" + ]; + files = [ "/etc/machine-id" ]; + }; + + # enable tailscale + services.tailscale.enable = true; + networking = { hostName = "cloudnix"; nameservers = ["8.8.8.8"]; diff --git a/vars/default.nix b/vars/default.nix index 82b6969..77172bf 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -5,6 +5,7 @@ email metrics-server xmpp + domains ; locations = { mediaDataMountPoint = "/media/media"; From 2d6b274b8aa1266b628e6e0d5cce896484098605 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 8 Feb 2025 14:15:45 +0000 Subject: [PATCH 27/31] add semitamaps webserver container --- .../optional/nixos-containers/semitamaps.nix | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/hosts/common/optional/nixos-containers/semitamaps.nix b/hosts/common/optional/nixos-containers/semitamaps.nix index 993deec..fd6dc50 100644 --- a/hosts/common/optional/nixos-containers/semitamaps.nix +++ b/hosts/common/optional/nixos-containers/semitamaps.nix @@ -1,6 +1,7 @@ { pkgs, lib, + configVars, ... }: let containerName = "semitamaps"; @@ -8,6 +9,9 @@ hostAddress = configVars.networking.addresses.semitamaps.hostAddress; localAddress = configVars.networking.addresses.semitamaps.localAddress; in { + systemd.tmpfiles.rules = [ + "d /var/run/sockets 0770 root root -" + ]; networking = { nat = { @@ -30,19 +34,29 @@ in { hostAddress = hostAddress; localAddress = localAddress; nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + "/var/run/sockets" = { + hostPath = "/var/run/sockets"; + isReadOnly = false; + }; + }; config = { pkgs, lib, ... }: { - networking = { firewall = { enable = true; rejectPackets = true; allowedTCPPorts = [ - 80 443 + 80 + 443 ]; }; useHostResolvConf = lib.mkForce false; @@ -63,6 +77,37 @@ in { settings.PasswordAuthentication = false; }; + systemd.services.semitamaps-api = { + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + description = "Deploys and serves semitamaps api"; + environment = { + }; + serviceConfig = { + ExecStartPre = pkgs.writeShellScript "semitamaps-api-prestart" '' + set -e + + GITCMD="${pkgs.openssh}/bin/ssh -i /etc/ssh/ssh_host_ed25519_key" + if [ ! -d "/srv/semitamaps" ]; then + export GIT_SSH_COMMAND=$GITCMD + ${pkgs.git}/bin/git clone git@git.bitlab21.com:sam/semitamaps.com.git /srv/semitamaps + mkdir /srv/semitamaps/.venv + fi + cd /srv/semitamaps + ${pkgs.poetry}/bin/poetry install + ''; + ExecStart = pkgs.writeShellScript "semitamaps-api-start" '' + cd /srv/semitamaps + .venv/bin/python .venv/bin/uvicorn --workers 4 --uds /var/run/sockets/baseddata.sock backend.app:app + ''; + Restart = "on-failure"; + }; + }; + + programs.ssh.knownHosts = { + "git.bitlab21.com".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIALNd2BGf64heYjWT9yt0fVmngepiHRIMsL7au/MRteg"; + }; + users.users = { root = { openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); From 312eca4835f80c8fa1626cd814b489f214c18667 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 11 Feb 2025 12:26:58 +0000 Subject: [PATCH 28/31] add prosody xmpp container --- flake.lock | 8 +- hosts/cloudnix/default.nix | 4 +- hosts/common/optional/nginx/semitamaps.nix | 23 +- hosts/common/optional/nginx/xmpp.nix | 92 ++++++++ .../common/optional/nixos-containers/xmpp.nix | 210 ++++++++++++++++++ 5 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 hosts/common/optional/nginx/xmpp.nix create mode 100644 hosts/common/optional/nixos-containers/xmpp.nix diff --git a/flake.lock b/flake.lock index f9bbc07..30cea79 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738940734, - "narHash": "sha256-7yH/LFyop6RoazqROOpQGlMRqbw80DfzUroTj9rTVro=", + "lastModified": 1739193599, + "narHash": "sha256-oJBav9MiFmhZxQWt6si1T5QQuhxWqGOOQNekeJABaXU=", "ref": "refs/heads/master", - "rev": "031ee73ab11dd5f91f56cdcac1483b1d59788178", - "revCount": 274, + "rev": "0d69dc15bea7b1a99fce08ea8517f392cbc253ee", + "revCount": 278, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/cloudnix/default.nix b/hosts/cloudnix/default.nix index 679c528..8e7c97f 100644 --- a/hosts/cloudnix/default.nix +++ b/hosts/cloudnix/default.nix @@ -32,11 +32,13 @@ in { ../common/optional/distributed-builds/local-machine.nix ../common/optional/nixos-containers/semitamaps.nix ../common/optional/nixos-containers/vaultwarden.nix + ../common/optional/nixos-containers/xmpp.nix + ../common/optional/fail2ban.nix ../common/optional/restic-backup.nix - ../common/optional/nginx/semitamaps.nix ../common/optional/nginx/vaultwarden.nix + ../common/optional/nginx/xmpp.nix outputs.nixosModules.nixosAutoUpgrade diff --git a/hosts/common/optional/nginx/semitamaps.nix b/hosts/common/optional/nginx/semitamaps.nix index e9a65aa..9e1deef 100644 --- a/hosts/common/optional/nginx/semitamaps.nix +++ b/hosts/common/optional/nginx/semitamaps.nix @@ -1,7 +1,20 @@ -{ - networking.firewall.allowedTCPPorts = [ 80 ]; - services.nginx.enable = true; - services.nginx.virtualHosts."samchance.xyz" = { - root = "/srv/hello/"; +{configVars, ...}: let + email = configVars.email.user; + domain = configVars.domains.xmpp; +in { + security.acme = { + acceptTerms = true; + defaults.email = email; + }; + networking.firewall.allowedTCPPorts = [80 443]; + services.nginx = { + enable = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + virtualHosts."${domain}" = { + enableACME = true; + forceSSL = true; + root = "/srv/hello/"; + }; }; } diff --git a/hosts/common/optional/nginx/xmpp.nix b/hosts/common/optional/nginx/xmpp.nix new file mode 100644 index 0000000..dc4d7a8 --- /dev/null +++ b/hosts/common/optional/nginx/xmpp.nix @@ -0,0 +1,92 @@ +{configVars, ...}: let + email = configVars.email.user; + xmppDomain = configVars.domains.xmpp; + xmppIp = configVars.networking.addresses.xmpp.localAddress; + xmppPort = configVars.networking.addresses.xmpp.port; +in { + networking.firewall.allowedTCPPorts = [80 443]; + users.groups.www-data = { + gid = 33; + }; + + users.users.nginx = { + isSystemUser = true; + uid = 60; + extraGroups = ["www-data"]; + }; + + systemd.tmpfiles.rules = [ + "d /var/www/${xmppDomain} 0777 root root" + ]; + + services.httpd.virtualHosts."root" = { + hostName = "${xmppDomain}"; + documentRoot = "/var/www/${xmppDomain}"; + }; + + security.acme = { + acceptTerms = true; + defaults.email = email; + certs = { + "${xmppDomain}" = { + webroot = "/var/www/${xmppDomain}"; + email = email; + extraDomainNames = [ + "chat.${xmppDomain}" + ]; + group = "www-data"; + }; + }; + }; + + services.nginx = { + enable = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + virtualHosts."chat.${xmppDomain}" = { + # enableACME = true; + forceSSL = true; + extraConfig = '' + client_max_body_size 10G; + ''; + sslCertificate = "/var/lib/acme/${xmppDomain}/fullchain.pem"; + sslCertificateKey = "/var/lib/acme/${xmppDomain}/key.pem"; + locations = { + "/" = { + proxyPass = "http://${xmppIp}:${toString xmppPort}"; + extraConfig = '' + proxy_set_header Host "${xmppDomain}"; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + tcp_nodelay on; + ''; + }; + "/xmpp-websocket" = { + proxyPass = "http://${xmppIp}:${toString xmppPort}/xmpp-websocket"; + extraConfig = '' + proxy_http_version 1.1; + proxy_set_header Connection "Upgrade"; + proxy_set_header Upgrade $http_upgrade; + + proxy_set_header Host "${xmppDomain}"; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900s; + ''; + }; + "/upload/" = { + extraConfig = '' + proxy_buffering off; + proxy_set_header Host $host; + # pass PUT requests to mod_http_upload for processing + if ($request_method = PUT) { + proxy_pass http://${xmppIp}:${toString xmppPort}; + } + alias /var/lib/prosody/http_upload; # storage path of mod_http_upload. NGINX will serve these files to the clients. + ''; + }; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/xmpp.nix b/hosts/common/optional/nixos-containers/xmpp.nix new file mode 100644 index 0000000..72743f0 --- /dev/null +++ b/hosts/common/optional/nixos-containers/xmpp.nix @@ -0,0 +1,210 @@ +{ + pkgs, + lib, + configVars, + inputs, + ... +}: let + containerName = "xmpp"; + xmppDomain = configVars.domains.xmpp; + pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + hostAddress = configVars.networking.addresses.xmpp.hostAddress; + localAddress = configVars.networking.addresses.xmpp.localAddress; + sops-nix = inputs.sops-nix; + xmppPorts = [3478 5281 5280 5269 5222 5223]; +in { + networking = { + nat = { + enable = true; + internalInterfaces = ["ve-+"]; + externalInterface = "enp1s0"; + }; + firewall = { + enable = true; + allowedTCPPorts = xmppPorts; + }; + }; + + environment.persistence."/persist" = { + hideMounts = true; + directories = [ + "/var/lib/nixos-containers/${containerName}" + ]; + }; + + systemd.tmpfiles.rules = [ + "d /var/lib/prosody 0750" + ]; + + containers."${containerName}" = { + autoStart = true; + privateNetwork = true; + hostAddress = hostAddress; + localAddress = localAddress; + nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + "/var/lib/prosody" = { + hostPath = "/var/lib/prosody"; + isReadOnly = false; + }; + "/var/lib/acme/${xmppDomain}/" = { + hostPath = "/var/lib/acme/${xmppDomain}/"; + isReadOnly = false; + }; + }; + forwardPorts = + lib.map (port: { + containerPort = port; + hostPort = port; + }) + xmppPorts; + config = { + pkgs, + lib, + config, + ... + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { + users.groups.www-data = { + gid = 33; + }; + + users.users.prosody = { + isSystemUser = true; + uid = 149; + extraGroups = ["www-data"]; + }; + + networking = { + firewall = { + enable = true; + rejectPackets = true; + allowedTCPPorts = xmppPorts ++ [80 443]; + }; + useHostResolvConf = lib.mkForce false; + }; + + services.resolved.enable = true; + + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/coturn/static-auth-secret" = { + mode = "0644"; + }; + }; + }; + + imports = [ + sops-nix.nixosModules.sops + ]; + + environment.systemPackages = [ + pkgs.vim + pkgs.git + pkgs.prosody + ]; + + sops.templates."prosody_secrets.lua" = { + mode = "644"; + content = '' + turn_external_secret = ${config.sops.placeholder."software/coturn/static-auth-secret"}; + ''; + }; + + services.prosody = { + enable = true; + package = pkgs.prosody.override { + withCommunityModules = [ + "turn_external" + "conversejs" + "admin_web" + ]; + }; + extraModules = ["turn_external" "conversejs" "admin_web" "http" "websocket"]; + allowRegistration = true; + extraConfig = '' + include "${config.sops.templates."prosody_secrets.lua".path}" + registration_invite_only = true; + allow_user_invites = true; + cross_domain_bosh = true; + turn_external_host = "turn.${xmppDomain}"; + turn_external_port = 3478; + http_default_host = "${xmppDomain}"; + certificates = "certs" + cross_domain_websocket = { "https://${xmppDomain}" } + consider_websocket_secure = true + + legacy_ssl_ports = { 5223 } + legacy_ssl_ssl = { + certificate = "/var/lib/acme/${xmppDomain}/cert.pem"; + key = "/var/lib/acme/${xmppDomain}/key.pem"; + } + + ''; + modules.bosh = true; + s2sRequireEncryption = true; + c2sRequireEncryption = true; + s2sSecureAuth = false; + admins = ["root@${xmppDomain}"]; + ssl.cert = "/var/lib/acme/${xmppDomain}/fullchain.pem"; + ssl.key = "/var/lib/acme/${xmppDomain}/key.pem"; + virtualHosts."${xmppDomain}" = { + enabled = true; + ssl.cert = "/var/lib/acme/${xmppDomain}/fullchain.pem"; + ssl.key = "/var/lib/acme/${xmppDomain}/key.pem"; + extraConfig = '' + http_external_url = "https://chat.${xmppDomain}/" + invites_page = "https://chat.${xmppDomain}/register?t={invite.token}" + http_paths = { + invites_page = "/invite"; + invites_register_web = "/register"; + } + ''; + domain = "${xmppDomain}"; + }; + muc = [ + { + domain = "conference.${xmppDomain}"; + } + ]; + uploadHttp = { + domain = "https://upload.${xmppDomain}"; + uploadFileSizeLimit = "1000000000"; # 1 gb file-size limit + uploadExpireAfter = "31557600"; # files deleted after 1 year + }; + }; + + services.coturn = { + enable = true; + realm = "turn.${xmppDomain}"; + use-auth-secret = true; + static-auth-secret-file = config.sops.secrets."software/coturn/static-auth-secret".path; + }; + + services.openssh = { + enable = true; + settings.PasswordAuthentication = false; + }; + + users.users = { + root = { + openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); + }; + }; + + system.stateVersion = "24.05"; + }; + }; +} From eeb4639bdbc22bf443c797d0d04866b5866fc584 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 12 Feb 2025 20:10:27 +0000 Subject: [PATCH 29/31] fix coturn and modularise ports --- flake.lock | 8 +- hosts/common/optional/nginx/xmpp.nix | 2 +- .../common/optional/nixos-containers/xmpp.nix | 115 ++++++++++++++---- 3 files changed, 96 insertions(+), 29 deletions(-) diff --git a/flake.lock b/flake.lock index 30cea79..97665ed 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1739193599, - "narHash": "sha256-oJBav9MiFmhZxQWt6si1T5QQuhxWqGOOQNekeJABaXU=", + "lastModified": 1739387047, + "narHash": "sha256-KpogJP00vwuMIKkGJff3zp0YfV9GfOG//UzMK4nWWUw=", "ref": "refs/heads/master", - "rev": "0d69dc15bea7b1a99fce08ea8517f392cbc253ee", - "revCount": 278, + "rev": "be51e237b5b3d441a194f3e516175f6a543aee35", + "revCount": 280, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nginx/xmpp.nix b/hosts/common/optional/nginx/xmpp.nix index dc4d7a8..568a5ff 100644 --- a/hosts/common/optional/nginx/xmpp.nix +++ b/hosts/common/optional/nginx/xmpp.nix @@ -2,7 +2,7 @@ email = configVars.email.user; xmppDomain = configVars.domains.xmpp; xmppIp = configVars.networking.addresses.xmpp.localAddress; - xmppPort = configVars.networking.addresses.xmpp.port; + xmppPort = configVars.networking.addresses.xmpp.ports.xmpp-c2s; in { networking.firewall.allowedTCPPorts = [80 443]; users.groups.www-data = { diff --git a/hosts/common/optional/nixos-containers/xmpp.nix b/hosts/common/optional/nixos-containers/xmpp.nix index 72743f0..ce1719d 100644 --- a/hosts/common/optional/nixos-containers/xmpp.nix +++ b/hosts/common/optional/nixos-containers/xmpp.nix @@ -9,9 +9,26 @@ xmppDomain = configVars.domains.xmpp; pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; hostAddress = configVars.networking.addresses.xmpp.hostAddress; + externalIp = configVars.networking.addresses.cloudnix.ip; localAddress = configVars.networking.addresses.xmpp.localAddress; sops-nix = inputs.sops-nix; - xmppPorts = [3478 5281 5280 5269 5222 5223]; + xmppPorts = configVars.networking.addresses.xmpp.ports; + xmppUDPPorts = + [ + xmppPorts.coturn + xmppPorts.coturn-tls + ] + ++ lib.range xmppPorts.coturn-min-udp xmppPorts.coturn-max-udp; + xmppTCPPorts = [ + xmppPorts.coturn + xmppPorts.coturn-tls + xmppPorts.xmpp-https + xmppPorts.xmpp-http + xmppPorts.xmpp-s2s + xmppPorts.xmpp-c2s + xmppPorts.xmpp-c2s-legacy-tls + xmppPorts.xmpp-s2s-tls + ]; in { networking = { nat = { @@ -21,7 +38,8 @@ in { }; firewall = { enable = true; - allowedTCPPorts = xmppPorts; + allowedTCPPorts = xmppTCPPorts; + allowedUDPPorts = xmppUDPPorts; }; }; @@ -58,10 +76,17 @@ in { }; forwardPorts = lib.map (port: { + protocol = "tcp"; containerPort = port; hostPort = port; }) - xmppPorts; + xmppTCPPorts + ++ lib.map (port: { + protocol = "udp"; + containerPort = port; + hostPort = port; + }) + xmppUDPPorts; config = { pkgs, lib, @@ -81,11 +106,18 @@ in { extraGroups = ["www-data"]; }; + users.users.turnserver = { + isSystemUser = true; + uid = 249; + extraGroups = ["www-data"]; + }; + networking = { firewall = { enable = true; rejectPackets = true; - allowedTCPPorts = xmppPorts ++ [80 443]; + allowedTCPPorts = xmppTCPPorts ++ [80 443]; + allowedUDPPorts = xmppUDPPorts; }; useHostResolvConf = lib.mkForce false; }; @@ -114,12 +146,13 @@ in { pkgs.vim pkgs.git pkgs.prosody + pkgs.coturn ]; sops.templates."prosody_secrets.lua" = { - mode = "644"; + mode = "444"; content = '' - turn_external_secret = ${config.sops.placeholder."software/coturn/static-auth-secret"}; + turn_external_secret = "${config.sops.placeholder."software/coturn/static-auth-secret"}"; ''; }; @@ -130,28 +163,56 @@ in { "turn_external" "conversejs" "admin_web" + "external_services" + "http_altconnect" ]; }; - extraModules = ["turn_external" "conversejs" "admin_web" "http" "websocket"]; + extraModules = [ + "server_contact_info" + "http_file_share" + "external_services" + "turn_external" + "conversejs" + "admin_web" + "http" + "websocket" + "http_altconnect" + ]; allowRegistration = true; extraConfig = '' - include "${config.sops.templates."prosody_secrets.lua".path}" + Include "${config.sops.templates."prosody_secrets.lua".path}" registration_invite_only = true; allow_user_invites = true; cross_domain_bosh = true; + cross_domain_websocket = true; turn_external_host = "turn.${xmppDomain}"; - turn_external_port = 3478; + turn_external_port = ${toString xmppPorts.coturn}; http_default_host = "${xmppDomain}"; certificates = "certs" - cross_domain_websocket = { "https://${xmppDomain}" } consider_websocket_secure = true - - legacy_ssl_ports = { 5223 } + external_services = { + { + port="${toString xmppPorts.coturn}"; + transport="tcp"; + type="stun"; + host="turn.${xmppDomain}" + }; + { + port="${toString xmppPorts.coturn}"; + transport="udp"; + type="turn"; + host="turn.${xmppDomain}" + }; + } + s2s_direct_tls_ports = { ${toString xmppPorts.xmpp-s2s-tls} } + legacy_ssl_ports = { ${toString xmppPorts.xmpp-c2s-legacy-tls} } legacy_ssl_ssl = { certificate = "/var/lib/acme/${xmppDomain}/cert.pem"; key = "/var/lib/acme/${xmppDomain}/key.pem"; } - + contact_info = { + admin = { "mailto:admin@${xmppDomain}", "xmpp:admin@${xmppDomain}" }; + } ''; modules.bosh = true; s2sRequireEncryption = true; @@ -160,6 +221,7 @@ in { admins = ["root@${xmppDomain}"]; ssl.cert = "/var/lib/acme/${xmppDomain}/fullchain.pem"; ssl.key = "/var/lib/acme/${xmppDomain}/key.pem"; + httpFileShare.domain = "upload.${xmppDomain}"; virtualHosts."${xmppDomain}" = { enabled = true; ssl.cert = "/var/lib/acme/${xmppDomain}/fullchain.pem"; @@ -171,6 +233,12 @@ in { invites_page = "/invite"; invites_register_web = "/register"; } + disco_items = { + { "upload.${xmppDomain}.com" }, + { "rooms.${xmppDomain}.com" }, + { "turn.${xmppDomain}.com" }, + } + ''; domain = "${xmppDomain}"; }; @@ -191,17 +259,16 @@ in { realm = "turn.${xmppDomain}"; use-auth-secret = true; static-auth-secret-file = config.sops.secrets."software/coturn/static-auth-secret".path; - }; - - services.openssh = { - enable = true; - settings.PasswordAuthentication = false; - }; - - users.users = { - root = { - openssh.authorizedKeys.keys = lib.lists.forEach pubKeys (key: builtins.readFile key); - }; + tls-listening-port = xmppPorts.coturn-tls; + cert = "/var/lib/acme/${xmppDomain}/cert.pem"; + pkey = "/var/lib/acme/${xmppDomain}/key.pem"; + min-port = xmppPorts.coturn-min-udp; + max-port = xmppPorts.coturn-max-udp; + extraConfig = '' + external-ip = ${externalIp}/${localAddress} + log = /var/log/turnserver.log + verbose + ''; }; system.stateVersion = "24.05"; From c2b13f6908144373b6c6312f31c2fba383b331ca Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 13 Feb 2025 11:59:52 +0000 Subject: [PATCH 30/31] correct xmpp port --- hosts/common/optional/nginx/xmpp.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/common/optional/nginx/xmpp.nix b/hosts/common/optional/nginx/xmpp.nix index 568a5ff..beefc8f 100644 --- a/hosts/common/optional/nginx/xmpp.nix +++ b/hosts/common/optional/nginx/xmpp.nix @@ -2,7 +2,7 @@ email = configVars.email.user; xmppDomain = configVars.domains.xmpp; xmppIp = configVars.networking.addresses.xmpp.localAddress; - xmppPort = configVars.networking.addresses.xmpp.ports.xmpp-c2s; + xmppPort = configVars.networking.addresses.xmpp.ports.xmpp-http; in { networking.firewall.allowedTCPPorts = [80 443]; users.groups.www-data = { From 4552297c29fe2b8f96ab127c0c6ba813861247a7 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 14 Feb 2025 19:14:32 +0000 Subject: [PATCH 31/31] make changes to semitamaps container --- .../optional/nixos-containers/semitamaps.nix | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/hosts/common/optional/nixos-containers/semitamaps.nix b/hosts/common/optional/nixos-containers/semitamaps.nix index fd6dc50..7113dfe 100644 --- a/hosts/common/optional/nixos-containers/semitamaps.nix +++ b/hosts/common/optional/nixos-containers/semitamaps.nix @@ -8,9 +8,10 @@ pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; hostAddress = configVars.networking.addresses.semitamaps.hostAddress; localAddress = configVars.networking.addresses.semitamaps.localAddress; + workingDirectory = "/var/www/semitamaps"; in { systemd.tmpfiles.rules = [ - "d /var/run/sockets 0770 root root -" + "d /var/run/sockets 0660 www-data www-data -" ]; networking = { @@ -62,6 +63,12 @@ in { useHostResolvConf = lib.mkForce false; }; + systemd.tmpfiles.rules = [ + "d ${workingDirectory} 0750 www-data www-data" + "d ${workingDirectory}/.venv 0750 www-data www-data" + "d ${workingDirectory}/public/uploads 0775 www-data www-data" + ]; + services.resolved.enable = true; imports = [ @@ -77,28 +84,26 @@ in { settings.PasswordAuthentication = false; }; - systemd.services.semitamaps-api = { + systemd.services.semitamaps = { wantedBy = ["multi-user.target"]; after = ["network.target"]; - description = "Deploys and serves semitamaps api"; + description = "Deploys and serves semitamaps"; environment = { }; serviceConfig = { - ExecStartPre = pkgs.writeShellScript "semitamaps-api-prestart" '' + WorkingDirectory = "${workingDirectory}"; + ExecStartPre = pkgs.writeShellScript "semitamaps-prestart" '' set -e GITCMD="${pkgs.openssh}/bin/ssh -i /etc/ssh/ssh_host_ed25519_key" - if [ ! -d "/srv/semitamaps" ]; then + if [ ! -d ${workingDirectory}/.git ]; then export GIT_SSH_COMMAND=$GITCMD - ${pkgs.git}/bin/git clone git@git.bitlab21.com:sam/semitamaps.com.git /srv/semitamaps - mkdir /srv/semitamaps/.venv + ${pkgs.git}/bin/git clone git@git.bitlab21.com:sam/semitamaps.com.git ${workingDirectory} fi - cd /srv/semitamaps ${pkgs.poetry}/bin/poetry install ''; - ExecStart = pkgs.writeShellScript "semitamaps-api-start" '' - cd /srv/semitamaps - .venv/bin/python .venv/bin/uvicorn --workers 4 --uds /var/run/sockets/baseddata.sock backend.app:app + ExecStart = pkgs.writeShellScript "semitamaps-start" '' + .venv/bin/python .venv/bin/uvicorn --workers 4 --uds /var/run/sockets/semitamaps.sock app:app ''; Restart = "on-failure"; };