From 3c0baaed18810faddaddc0854759af965f6c2206 Mon Sep 17 00:00:00 2001 From: Sam Date: Sun, 26 Jan 2025 23:59:29 +0000 Subject: [PATCH 01/22] add ssl certs to reverse proxy --- .../nixos-containers/reverse-proxy.nix | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/reverse-proxy.nix b/hosts/common/optional/nixos-containers/reverse-proxy.nix index bbb57d2..ea82408 100644 --- a/hosts/common/optional/nixos-containers/reverse-proxy.nix +++ b/hosts/common/optional/nixos-containers/reverse-proxy.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "reverse-proxy"; @@ -10,6 +11,7 @@ gatewayIp = configVars.networking.addresses.gateway.ip; pubKeys = lib.filesystem.listFilesRecursive ../../users/keys; + sops-nix = inputs.sops-nix; dockerContainerIp = configVars.networking.addresses.docker.ip; bdWorker = configVars.networking.addresses.bd-worker.ip; pihole = configVars.networking.addresses.pihole.ip; @@ -33,13 +35,40 @@ in { privateNetwork = true; hostBridge = "br0"; nixpkgs = pkgs.path; + bindMounts = { + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; + }; config = { pkgs, lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + }; + + sops.secrets = { + "ssl_keys/lan-selfsigned.crt" = { + mode = "0644"; + }; + "ssl_keys/lan-selfsigned.key" = { + mode = "0644"; + }; + }; + networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -52,6 +81,7 @@ in { enable = true; allowedTCPPorts = [ 80 + 443 ]; }; useHostResolvConf = lib.mkForce false; @@ -60,6 +90,7 @@ in { services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -72,27 +103,51 @@ in { enable = true; virtualHosts = { "jellyfin.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${dockerContainerIp}:8096"; }; "mempool.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${bitcoinNode}:4080"; + extraConfig = '' + proxy_set_header Host mempool.lan; + ''; }; "grafana.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${metricsServer}:2342"; extraConfig = '' proxy_set_header Host grafana.lan; ''; }; "metrics.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${metricsServer}:9001"; }; "searx.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${dockerContainerIp}:8855"; }; "dns.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${pihole}:80"; }; "prefect.lan" = { + forceSSL = true; + sslCertificate = "${config.sops.secrets."ssl_keys/lan-selfsigned.crt".path}"; + sslCertificateKey = "${config.sops.secrets."ssl_keys/lan-selfsigned.key".path}"; locations."/".proxyPass = "http://${bdWorker}:4200"; }; }; From 05012c2056a9b70faf8e8108031904b999f16e80 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 10:41:43 +0000 Subject: [PATCH 02/22] change jellyfin from arion to oci container --- .../optional/arion-containers/jellyfin.nix | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 65a075c..0792ea3 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -1,31 +1,27 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.jellyfin = { - settings = { - services.jellyfin.service = { - ports = [ - "8096:8096" - ]; - container_name = "jellyfin"; - image = "lscr.io/linuxserver/jellyfin:latest"; - restart = "always"; - volumes = [ - "/srv/docker/media-server/jellyfin/config:/config" - "/media/media/tv:/data/tvshows:ro" - "/media/media/movies:/data/movies:ro" - "/media/media/music/music_data:/data/music:ro" - "/media/media/youtube:/data/youtube:ro" - "/media/media/podcasts:/data/podcasts:ro" - "/srv/docker/media-server/jellyfin/config/custom-cont-init.d:/custom-cont-init.d:ro" - ]; - environment = { - PUID = "1000"; - PGID = "1000"; - DOCKER_MODS="linuxserver/mods:jellyfin-opencl-intel"; - }; - devices = ["/dev/dri:/dev/dri"]; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + jellyfin = { + image = "lscr.io/linuxserver/jellyfin:latest"; + ports = [ + "8096:8096" + ]; + volumes = [ + "/srv/docker/media-server/jellyfin/config:/config" + "/media/media/tv:/data/tvshows:ro" + "/media/media/movies:/data/movies:ro" + "/media/media/music/music_data:/data/music:ro" + "/media/media/youtube:/data/youtube:ro" + "/media/media/podcasts:/data/podcasts:ro" + "/srv/docker/media-server/jellyfin/config/custom-cont-init.d:/custom-cont-init.d:ro" + ]; + environment = { + PUID = "1000"; + PGID = "1000"; + DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; }; + extraOptions = ["--gpus=all"]; }; }; }; From 2ad550029420d0c7dedaa7d279f30a1665489542 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 13:37:52 +0000 Subject: [PATCH 03/22] change jellyfin to oci container --- .../optional/arion-containers/jellyfin.nix | 5 ++++- .../common/optional/nixos-containers/docker.nix | 16 ++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 0792ea3..600eefe 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -20,8 +20,11 @@ PUID = "1000"; PGID = "1000"; DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; + NVIDIA_VISIBLE_DEVICES = "all"; }; - extraOptions = ["--gpus=all"]; + extraOptions = [ + "--device=nvidia.com/gpu=all" + ]; }; }; }; diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index cc74b46..6b1f7b2 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -53,6 +53,10 @@ in { node = "/dev/nvidiactl"; modifier = "rwm"; } + { + node = "/dev/nvidia-uvm"; + modifier = "rwm"; + } { node = "/dev/fuse"; modifier = "rwm"; @@ -204,18 +208,6 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; - systemd.services.podman-autostart = { - enable = true; - after = ["podman.service"]; - wantedBy = ["multi-user.target"]; - description = "Automatically start containers with --restart=always tag"; - serviceConfig = { - Type = "idle"; - ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; - ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; - }; - }; - services.prometheus = { exporters = { node = { From 3f28197cafb19ee3fef4f3f1db354f0a58182a79 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:07:48 +0000 Subject: [PATCH 04/22] rm nvim add ccrypt --- hosts/iso/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/iso/default.nix b/hosts/iso/default.nix index 4e8fd4c..7d9736b 100644 --- a/hosts/iso/default.nix +++ b/hosts/iso/default.nix @@ -3,7 +3,7 @@ in { imports = [(modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix")]; environment.systemPackages = [ pkgs.openssl - pkgs.nvim + pkgs.ccrypt ]; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; } From 31e98b52591e45513f46bd30a3ef3b7cb5ffb55d Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:09:34 +0000 Subject: [PATCH 05/22] add vlc --- home/common/optional/desktop/common/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/home/common/optional/desktop/common/default.nix b/home/common/optional/desktop/common/default.nix index 1c0bbad..79ccb19 100644 --- a/home/common/optional/desktop/common/default.nix +++ b/home/common/optional/desktop/common/default.nix @@ -29,5 +29,6 @@ pkgs.R pkgs.gimp pkgs.gajim + pkgs.vlc ]; } From fad91cfe25f2a9a0da198fdfe0e3a77eff8c0065 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:25:51 +0000 Subject: [PATCH 06/22] change semita worker to use oci containers --- .../semitamaps-tileserver.nix | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix index c8c1d8f..5cd6e50 100644 --- a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix +++ b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix @@ -1,31 +1,25 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.semitamaps-tileserver = { - settings = { - services.tileserver-gl.service = { - ports = [ - "8080:8080" - ]; - container_name = "tileserver-gl"; - image = "maptiler/tileserver-gl"; - restart = "always"; - volumes = [ - "/data/semitamaps-data/tileserver-gl/data:/data" - ]; - command = "-c /data/config.json --public_url https://tiles.semitamaps.com/"; - }; - - services.mbgl-renderer.service = { - ports = [ - "8081:80" - ]; - container_name = "mbgl-renderer"; - image = "mbgl-renderer"; - restart = "always"; - }; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + tileserver-gl = { + image = "maptiler/tileserver-gl"; + ports = [ + "8080:8080" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; + }; + mbgl-renderer = { + image = "mbgl-renderer"; + ports = [ + "8081:80" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; }; }; }; } - From c73aa9933ac5561f8a72c5ad1d01057a179c0a9d Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 30 Jan 2025 15:06:26 +0000 Subject: [PATCH 07/22] remove lnd and rtl from nix-bitcoin --- .../optional/nixos-containers/nix-bitcoin.nix | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 0bfd532..2788a60 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -62,7 +62,7 @@ in { }: { imports = [ inputs.nix-bitcoin.nixosModules.default - inputs.lnbits.nixosModules.default + # inputs.lnbits.nixosModules.default ]; environment.systemPackages = with pkgs; [ vim @@ -80,14 +80,13 @@ in { firewall = { enable = true; allowedTCPPorts = [ - 80 - 443 - 22 + # 80 + # 443 config.containers.bitcoin-node.config.services.bitcoind.rpc.port config.containers.bitcoin-node.config.services.mempool.frontend.port config.containers.bitcoin-node.config.services.electrs.port - config.containers.bitcoin-node.config.services.rtl.port - config.containers.bitcoin-node.config.services.lnd.port + # config.containers.bitcoin-node.config.services.rtl.port + # config.containers.bitcoin-node.config.services.lnd.port ]; }; useHostResolvConf = lib.mkForce false; @@ -144,7 +143,7 @@ in { }; }; lnd = { - enable = true; + enable = false; lndconnect = { enable = true; onion = true; @@ -156,34 +155,34 @@ in { ''; }; rtl = { - enable = true; + enable = false; nodes.lnd.enable = true; address = "0.0.0.0"; }; - lnbits = { - enable = true; - openFirewall = true; - host = "0.0.0.0"; - port = 8231; - env = { - LNBITS_ADMIN_UI = "true"; - LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; - LND_REST_ENDPOINT = "https://127.0.0.1:8080"; - LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; - LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; - AUTH_ALLOWED_METHODS = "user-id-only, username-password"; - }; - }; + # lnbits = { + # enable = false; + # openFirewall = true; + # host = "0.0.0.0"; + # port = 8231; + # env = { + # LNBITS_ADMIN_UI = "true"; + # LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; + # LND_REST_ENDPOINT = "https://127.0.0.1:8080"; + # LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; + # LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; + # AUTH_ALLOWED_METHODS = "user-id-only, username-password"; + # }; + # }; }; # Add custom systemd overrides for above services - systemd.services.lnbits.after = ["lnd.service"]; + # systemd.services.lnbits.after = ["lnd.service"]; nix-bitcoin.onionServices = { bitcoind.enable = true; electrs.enable = true; mempool-frontend.enable = true; - lnd.public = true; + # lnd.public = true; }; services.prometheus = { From 37c50ce7b5f7517d56cb952ff6a751942eae1d91 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 31 Jan 2025 21:17:49 +0000 Subject: [PATCH 08/22] add restic-exporter to prometheus metrics --- flake.lock | 8 ++-- .../nixos-containers/metrics-server.nix | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 550380e..0b5877e 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737899664, - "narHash": "sha256-iZpzTSERNQ5UvFfEzrBLuEmcRUGjBSal7ShtXurYq8Q=", + "lastModified": 1738356588, + "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", "ref": "refs/heads/master", - "rev": "a9844a78dcbdc8a84679835112970d80822b113c", - "revCount": 257, + "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", + "revCount": 267, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 50417f4..148fbac 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "metrics-server"; @@ -15,6 +16,7 @@ bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; postres = configVars.networking.addresses.postgres.ip; backupServer = configVars.networking.addresses.backup-server.ip; + sops-nix = inputs.sops-nix; http_endpoints = configVars.metrics-server.blackbox.http_endpoints; @@ -52,6 +54,10 @@ in { hostPath = metricsServerContainerData; isReadOnly = false; }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; }; config = { @@ -59,7 +65,10 @@ in { lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -79,9 +88,23 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/restic-passphrase" = {}; + "software/restic-exporter-credentials" = {}; + }; + }; + services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -110,6 +133,16 @@ in { } ]; } + { + job_name = "restic-exporter"; + static_configs = [ + { + targets = [ + "0.0.0.0:8001" + ]; + } + ]; + } { job_name = "blackbox"; @@ -175,6 +208,14 @@ in { enabledCollectors = ["systemd"]; port = 9002; }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; + }; }; }; From f7876d08f6f99ece6320626186b38db9568a2307 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 13:15:18 +0000 Subject: [PATCH 09/22] add baikal oci container to docker --- flake.lock | 8 ++++---- .../common/optional/arion-containers/baikal.nix | 17 +++++++++++++++++ .../common/optional/nixos-containers/docker.nix | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 hosts/common/optional/arion-containers/baikal.nix diff --git a/flake.lock b/flake.lock index 0b5877e..9d5a28c 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738356588, - "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", + "lastModified": 1738358831, + "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", "ref": "refs/heads/master", - "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", - "revCount": 267, + "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", + "revCount": 268, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/arion-containers/baikal.nix new file mode 100644 index 0000000..5e41de5 --- /dev/null +++ b/hosts/common/optional/arion-containers/baikal.nix @@ -0,0 +1,17 @@ +{ + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + baikal = { + image = "ckulka/baikal:nginx"; + ports = [ + "6734:80" + ]; + volumes = [ + "/srv/docker/baikal/config:/var/www/baikal/config" + "/srv/docker/baikal/data:/var/www/baikal/Specific" + ]; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 6b1f7b2..12659aa 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -182,6 +182,7 @@ in { ../arion-containers/jellyfin.nix ../arion-containers/photoprism.nix ../arion-containers/syncthing.nix + ../arion-containers/baikal.nix (import ../arion-containers/searxng.nix {configVars = configVars;}) ]; From 996e51f56e4aa5609d33482718340ca7460065e8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:28:59 +0000 Subject: [PATCH 10/22] setup prometheus alertmanager --- flake.lock | 8 +- .../nixos-containers/metrics-server.nix | 149 +++++++++++++----- vars/default.nix | 1 + 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/flake.lock b/flake.lock index 9d5a28c..70e52a9 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738358831, - "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", + "lastModified": 1738685297, + "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", "ref": "refs/heads/master", - "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", - "revCount": 268, + "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", + "revCount": 269, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 148fbac..dd7c746 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -8,6 +8,9 @@ containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + notifybotJid = configVars.xmpp.notifybotJid; + receiverJid = configVars.xmpp.personalAccount; + dockerContainerIp = configVars.networking.addresses.docker.ip; smWorkerIp = configVars.networking.addresses.sm-worker.ip; merlinIp = configVars.networking.addresses.merlin.ip; @@ -83,6 +86,7 @@ in { config.services.prometheus.port config.services.grafana.port config.services.prometheus.exporters.blackbox.port + 9199 #xmpp listen port ]; }; useHostResolvConf = lib.mkForce false; @@ -98,6 +102,9 @@ in { secrets = { "software/restic-passphrase" = {}; "software/restic-exporter-credentials" = {}; + "comms/xmpp/notifybot/password" = { + mode = "0644"; + }; }; }; @@ -112,9 +119,48 @@ in { pkgs.git ]; + services.grafana = { + enable = true; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; + }; + + # main prometheus service services.prometheus = { enable = true; port = 9001; + alertmanagers = [ + { + scheme = "http"; + path_prefix = "/"; + static_configs = [ + { + targets = [ + "0.0.0.0:9093" + ]; + } + ]; + } + ]; + ruleFiles = [ + "${pkgs.writeText + "alert_rule.yml" + '' + groups: + - name: blackbox_alert + rules: + - alert: EndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Endpoint {{ $labels.instance }} down" + description: "An endpoint has been down for more than 1 minute." + ''}" + ]; scrapeConfigs = [ { job_name = "node_exporter"; @@ -171,51 +217,74 @@ in { ]; }; - services.grafana = { + # setup alertmanager + services.prometheus.xmpp-alerts = { enable = true; - settings.server = { - http_port = 2342; - http_addr = "0.0.0.0"; + settings = { + jid = notifybotJid; + password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}"; + to_jid = receiverJid; + listen_address = "0.0.0.0"; + listen_port = 9199; }; }; + services.prometheus.alertmanager = { + webExternalUrl = containerIp; + enable = true; + openFirewall = true; + port = 9093; + configText = '' + global: + resolve_timeout: 1m - services.prometheus = { - exporters = { - blackbox = { - enable = true; - configFile = pkgs.writeText "blackbox-conf.yaml" '' - modules: - http_basic: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - valid_http_versions: ["HTTP/1.1", "HTTP/2"] - method: GET - # fail_if_ssl: false - # fail_if_not_ssl: true - # tls_config: - # insecure_skip_verify: true - tcp_connect: - prober: tcp - tcp: - preferred_ip_protocol: ip4 + route: + group_by: ['...'] + repeat_interval: 1h + receiver: 'xmpp-alerts' - ''; - }; - node = { - enable = true; - enabledCollectors = ["systemd"]; - port = 9002; - }; - restic = { - enable = true; - repository = ""; - environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; - passwordFile = config.sops.secrets."software/restic-passphrase".path; - refreshInterval = 10800; # refresh every 3 hours - port = 8001; - }; + receivers: + - name: 'xmpp-alerts' + webhook_configs: + - url: 'http://0.0.0.0:9199/alert' + ''; + }; + + # prometheus exporters + services.prometheus.exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + # fail_if_ssl: false + # fail_if_not_ssl: true + # tls_config: + # insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; }; }; diff --git a/vars/default.nix b/vars/default.nix index f6973cc..82b6969 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -4,6 +4,7 @@ networking email metrics-server + xmpp ; locations = { mediaDataMountPoint = "/media/media"; From bb5ccb0efc6df9e6f1260db353c4375c7a6c3edc Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:47:32 +0000 Subject: [PATCH 11/22] metrics-server add externalURL to prometheus --- hosts/common/optional/nixos-containers/metrics-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index dd7c746..a880109 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -130,6 +130,7 @@ in { # main prometheus service services.prometheus = { enable = true; + webExternalUrl = "http://${containerIp}:9001"; port = 9001; alertmanagers = [ { @@ -229,7 +230,7 @@ in { }; }; services.prometheus.alertmanager = { - webExternalUrl = containerIp; + webExternalUrl = "http://${containerIp}:9093"; enable = true; openFirewall = true; port = 9093; From 43943485ee5250a4e45bdb9581c54d82b83a72d4 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 13:37:52 +0000 Subject: [PATCH 12/22] change jellyfin to oci container --- .../optional/arion-containers/jellyfin.nix | 5 ++++- .../common/optional/nixos-containers/docker.nix | 16 ++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 0792ea3..600eefe 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -20,8 +20,11 @@ PUID = "1000"; PGID = "1000"; DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; + NVIDIA_VISIBLE_DEVICES = "all"; }; - extraOptions = ["--gpus=all"]; + extraOptions = [ + "--device=nvidia.com/gpu=all" + ]; }; }; }; diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index cc74b46..6b1f7b2 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -53,6 +53,10 @@ in { node = "/dev/nvidiactl"; modifier = "rwm"; } + { + node = "/dev/nvidia-uvm"; + modifier = "rwm"; + } { node = "/dev/fuse"; modifier = "rwm"; @@ -204,18 +208,6 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; - systemd.services.podman-autostart = { - enable = true; - after = ["podman.service"]; - wantedBy = ["multi-user.target"]; - description = "Automatically start containers with --restart=always tag"; - serviceConfig = { - Type = "idle"; - ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; - ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; - }; - }; - services.prometheus = { exporters = { node = { From 7416197e54a39869b81ce573820a5ed9c6e67db8 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:09:34 +0000 Subject: [PATCH 13/22] add vlc --- home/common/optional/desktop/common/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/home/common/optional/desktop/common/default.nix b/home/common/optional/desktop/common/default.nix index 1c0bbad..79ccb19 100644 --- a/home/common/optional/desktop/common/default.nix +++ b/home/common/optional/desktop/common/default.nix @@ -29,5 +29,6 @@ pkgs.R pkgs.gimp pkgs.gajim + pkgs.vlc ]; } From 12aa157690e454466364833341c42a4d2d069c35 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:25:51 +0000 Subject: [PATCH 14/22] change semita worker to use oci containers --- .../semitamaps-tileserver.nix | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix index c8c1d8f..5cd6e50 100644 --- a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix +++ b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix @@ -1,31 +1,25 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.semitamaps-tileserver = { - settings = { - services.tileserver-gl.service = { - ports = [ - "8080:8080" - ]; - container_name = "tileserver-gl"; - image = "maptiler/tileserver-gl"; - restart = "always"; - volumes = [ - "/data/semitamaps-data/tileserver-gl/data:/data" - ]; - command = "-c /data/config.json --public_url https://tiles.semitamaps.com/"; - }; - - services.mbgl-renderer.service = { - ports = [ - "8081:80" - ]; - container_name = "mbgl-renderer"; - image = "mbgl-renderer"; - restart = "always"; - }; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + tileserver-gl = { + image = "maptiler/tileserver-gl"; + ports = [ + "8080:8080" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; + }; + mbgl-renderer = { + image = "mbgl-renderer"; + ports = [ + "8081:80" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; }; }; }; } - From 9abf175b5c376af2d549a15440e1e7010029c68b Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 30 Jan 2025 15:06:26 +0000 Subject: [PATCH 15/22] remove lnd and rtl from nix-bitcoin --- .../optional/nixos-containers/nix-bitcoin.nix | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 0bfd532..2788a60 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -62,7 +62,7 @@ in { }: { imports = [ inputs.nix-bitcoin.nixosModules.default - inputs.lnbits.nixosModules.default + # inputs.lnbits.nixosModules.default ]; environment.systemPackages = with pkgs; [ vim @@ -80,14 +80,13 @@ in { firewall = { enable = true; allowedTCPPorts = [ - 80 - 443 - 22 + # 80 + # 443 config.containers.bitcoin-node.config.services.bitcoind.rpc.port config.containers.bitcoin-node.config.services.mempool.frontend.port config.containers.bitcoin-node.config.services.electrs.port - config.containers.bitcoin-node.config.services.rtl.port - config.containers.bitcoin-node.config.services.lnd.port + # config.containers.bitcoin-node.config.services.rtl.port + # config.containers.bitcoin-node.config.services.lnd.port ]; }; useHostResolvConf = lib.mkForce false; @@ -144,7 +143,7 @@ in { }; }; lnd = { - enable = true; + enable = false; lndconnect = { enable = true; onion = true; @@ -156,34 +155,34 @@ in { ''; }; rtl = { - enable = true; + enable = false; nodes.lnd.enable = true; address = "0.0.0.0"; }; - lnbits = { - enable = true; - openFirewall = true; - host = "0.0.0.0"; - port = 8231; - env = { - LNBITS_ADMIN_UI = "true"; - LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; - LND_REST_ENDPOINT = "https://127.0.0.1:8080"; - LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; - LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; - AUTH_ALLOWED_METHODS = "user-id-only, username-password"; - }; - }; + # lnbits = { + # enable = false; + # openFirewall = true; + # host = "0.0.0.0"; + # port = 8231; + # env = { + # LNBITS_ADMIN_UI = "true"; + # LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; + # LND_REST_ENDPOINT = "https://127.0.0.1:8080"; + # LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; + # LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; + # AUTH_ALLOWED_METHODS = "user-id-only, username-password"; + # }; + # }; }; # Add custom systemd overrides for above services - systemd.services.lnbits.after = ["lnd.service"]; + # systemd.services.lnbits.after = ["lnd.service"]; nix-bitcoin.onionServices = { bitcoind.enable = true; electrs.enable = true; mempool-frontend.enable = true; - lnd.public = true; + # lnd.public = true; }; services.prometheus = { From 24e1bc12ea1ec74c7b119426d319feac083105e2 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 31 Jan 2025 21:17:49 +0000 Subject: [PATCH 16/22] add restic-exporter to prometheus metrics --- flake.lock | 8 ++-- .../nixos-containers/metrics-server.nix | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 550380e..0b5877e 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737899664, - "narHash": "sha256-iZpzTSERNQ5UvFfEzrBLuEmcRUGjBSal7ShtXurYq8Q=", + "lastModified": 1738356588, + "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", "ref": "refs/heads/master", - "rev": "a9844a78dcbdc8a84679835112970d80822b113c", - "revCount": 257, + "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", + "revCount": 267, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 50417f4..148fbac 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "metrics-server"; @@ -15,6 +16,7 @@ bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; postres = configVars.networking.addresses.postgres.ip; backupServer = configVars.networking.addresses.backup-server.ip; + sops-nix = inputs.sops-nix; http_endpoints = configVars.metrics-server.blackbox.http_endpoints; @@ -52,6 +54,10 @@ in { hostPath = metricsServerContainerData; isReadOnly = false; }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; }; config = { @@ -59,7 +65,10 @@ in { lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -79,9 +88,23 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/restic-passphrase" = {}; + "software/restic-exporter-credentials" = {}; + }; + }; + services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -110,6 +133,16 @@ in { } ]; } + { + job_name = "restic-exporter"; + static_configs = [ + { + targets = [ + "0.0.0.0:8001" + ]; + } + ]; + } { job_name = "blackbox"; @@ -175,6 +208,14 @@ in { enabledCollectors = ["systemd"]; port = 9002; }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; + }; }; }; From b1e4be205318dd22a3c5da63713b749604dd448f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 13:15:18 +0000 Subject: [PATCH 17/22] add baikal oci container to docker --- flake.lock | 8 ++++---- .../common/optional/arion-containers/baikal.nix | 17 +++++++++++++++++ .../common/optional/nixos-containers/docker.nix | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 hosts/common/optional/arion-containers/baikal.nix diff --git a/flake.lock b/flake.lock index 0b5877e..9d5a28c 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738356588, - "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", + "lastModified": 1738358831, + "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", "ref": "refs/heads/master", - "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", - "revCount": 267, + "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", + "revCount": 268, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/arion-containers/baikal.nix new file mode 100644 index 0000000..5e41de5 --- /dev/null +++ b/hosts/common/optional/arion-containers/baikal.nix @@ -0,0 +1,17 @@ +{ + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + baikal = { + image = "ckulka/baikal:nginx"; + ports = [ + "6734:80" + ]; + volumes = [ + "/srv/docker/baikal/config:/var/www/baikal/config" + "/srv/docker/baikal/data:/var/www/baikal/Specific" + ]; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 6b1f7b2..12659aa 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -182,6 +182,7 @@ in { ../arion-containers/jellyfin.nix ../arion-containers/photoprism.nix ../arion-containers/syncthing.nix + ../arion-containers/baikal.nix (import ../arion-containers/searxng.nix {configVars = configVars;}) ]; From 068593c82c19ca576fe1cccd124257d3094b7f63 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:28:59 +0000 Subject: [PATCH 18/22] setup prometheus alertmanager --- flake.lock | 8 +- .../nixos-containers/metrics-server.nix | 149 +++++++++++++----- vars/default.nix | 1 + 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/flake.lock b/flake.lock index 9d5a28c..70e52a9 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738358831, - "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", + "lastModified": 1738685297, + "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", "ref": "refs/heads/master", - "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", - "revCount": 268, + "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", + "revCount": 269, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 148fbac..dd7c746 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -8,6 +8,9 @@ containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + notifybotJid = configVars.xmpp.notifybotJid; + receiverJid = configVars.xmpp.personalAccount; + dockerContainerIp = configVars.networking.addresses.docker.ip; smWorkerIp = configVars.networking.addresses.sm-worker.ip; merlinIp = configVars.networking.addresses.merlin.ip; @@ -83,6 +86,7 @@ in { config.services.prometheus.port config.services.grafana.port config.services.prometheus.exporters.blackbox.port + 9199 #xmpp listen port ]; }; useHostResolvConf = lib.mkForce false; @@ -98,6 +102,9 @@ in { secrets = { "software/restic-passphrase" = {}; "software/restic-exporter-credentials" = {}; + "comms/xmpp/notifybot/password" = { + mode = "0644"; + }; }; }; @@ -112,9 +119,48 @@ in { pkgs.git ]; + services.grafana = { + enable = true; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; + }; + + # main prometheus service services.prometheus = { enable = true; port = 9001; + alertmanagers = [ + { + scheme = "http"; + path_prefix = "/"; + static_configs = [ + { + targets = [ + "0.0.0.0:9093" + ]; + } + ]; + } + ]; + ruleFiles = [ + "${pkgs.writeText + "alert_rule.yml" + '' + groups: + - name: blackbox_alert + rules: + - alert: EndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Endpoint {{ $labels.instance }} down" + description: "An endpoint has been down for more than 1 minute." + ''}" + ]; scrapeConfigs = [ { job_name = "node_exporter"; @@ -171,51 +217,74 @@ in { ]; }; - services.grafana = { + # setup alertmanager + services.prometheus.xmpp-alerts = { enable = true; - settings.server = { - http_port = 2342; - http_addr = "0.0.0.0"; + settings = { + jid = notifybotJid; + password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}"; + to_jid = receiverJid; + listen_address = "0.0.0.0"; + listen_port = 9199; }; }; + services.prometheus.alertmanager = { + webExternalUrl = containerIp; + enable = true; + openFirewall = true; + port = 9093; + configText = '' + global: + resolve_timeout: 1m - services.prometheus = { - exporters = { - blackbox = { - enable = true; - configFile = pkgs.writeText "blackbox-conf.yaml" '' - modules: - http_basic: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - valid_http_versions: ["HTTP/1.1", "HTTP/2"] - method: GET - # fail_if_ssl: false - # fail_if_not_ssl: true - # tls_config: - # insecure_skip_verify: true - tcp_connect: - prober: tcp - tcp: - preferred_ip_protocol: ip4 + route: + group_by: ['...'] + repeat_interval: 1h + receiver: 'xmpp-alerts' - ''; - }; - node = { - enable = true; - enabledCollectors = ["systemd"]; - port = 9002; - }; - restic = { - enable = true; - repository = ""; - environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; - passwordFile = config.sops.secrets."software/restic-passphrase".path; - refreshInterval = 10800; # refresh every 3 hours - port = 8001; - }; + receivers: + - name: 'xmpp-alerts' + webhook_configs: + - url: 'http://0.0.0.0:9199/alert' + ''; + }; + + # prometheus exporters + services.prometheus.exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + # fail_if_ssl: false + # fail_if_not_ssl: true + # tls_config: + # insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; }; }; diff --git a/vars/default.nix b/vars/default.nix index f6973cc..82b6969 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -4,6 +4,7 @@ networking email metrics-server + xmpp ; locations = { mediaDataMountPoint = "/media/media"; From 595f1f92d5d7370f3a6ce3f36066135510aa83e8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:47:32 +0000 Subject: [PATCH 19/22] metrics-server add externalURL to prometheus --- hosts/common/optional/nixos-containers/metrics-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index dd7c746..a880109 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -130,6 +130,7 @@ in { # main prometheus service services.prometheus = { enable = true; + webExternalUrl = "http://${containerIp}:9001"; port = 9001; alertmanagers = [ { @@ -229,7 +230,7 @@ in { }; }; services.prometheus.alertmanager = { - webExternalUrl = containerIp; + webExternalUrl = "http://${containerIp}:9093"; enable = true; openFirewall = true; port = 9093; From 2fc2bb92d5e84b4925499e41e16507ccbdf56dfd Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 6 Feb 2025 08:34:27 +0000 Subject: [PATCH 20/22] rename arion-containers dir to docker-containers --- .../arrstack.nix | 0 .../baikal.nix | 0 .../jellyfin.nix | 0 .../photoprism.nix | 0 .../pihole.nix | 0 .../searxng.nix | 0 .../semitamaps-tileserver.nix | 0 .../syncthing.nix | 0 hosts/common/optional/nixos-containers/docker.nix | 12 ++++++------ hosts/common/optional/nixos-containers/pihole.nix | 2 +- .../optional/nixos-containers/semitamaps-worker.nix | 2 +- 11 files changed, 8 insertions(+), 8 deletions(-) rename hosts/common/optional/{arion-containers => docker-containers}/arrstack.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/baikal.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/jellyfin.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/photoprism.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/pihole.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/searxng.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/semitamaps-tileserver.nix (100%) rename hosts/common/optional/{arion-containers => docker-containers}/syncthing.nix (100%) diff --git a/hosts/common/optional/arion-containers/arrstack.nix b/hosts/common/optional/docker-containers/arrstack.nix similarity index 100% rename from hosts/common/optional/arion-containers/arrstack.nix rename to hosts/common/optional/docker-containers/arrstack.nix diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/docker-containers/baikal.nix similarity index 100% rename from hosts/common/optional/arion-containers/baikal.nix rename to hosts/common/optional/docker-containers/baikal.nix diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/docker-containers/jellyfin.nix similarity index 100% rename from hosts/common/optional/arion-containers/jellyfin.nix rename to hosts/common/optional/docker-containers/jellyfin.nix diff --git a/hosts/common/optional/arion-containers/photoprism.nix b/hosts/common/optional/docker-containers/photoprism.nix similarity index 100% rename from hosts/common/optional/arion-containers/photoprism.nix rename to hosts/common/optional/docker-containers/photoprism.nix diff --git a/hosts/common/optional/arion-containers/pihole.nix b/hosts/common/optional/docker-containers/pihole.nix similarity index 100% rename from hosts/common/optional/arion-containers/pihole.nix rename to hosts/common/optional/docker-containers/pihole.nix diff --git a/hosts/common/optional/arion-containers/searxng.nix b/hosts/common/optional/docker-containers/searxng.nix similarity index 100% rename from hosts/common/optional/arion-containers/searxng.nix rename to hosts/common/optional/docker-containers/searxng.nix diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/docker-containers/semitamaps-tileserver.nix similarity index 100% rename from hosts/common/optional/arion-containers/semitamaps-tileserver.nix rename to hosts/common/optional/docker-containers/semitamaps-tileserver.nix diff --git a/hosts/common/optional/arion-containers/syncthing.nix b/hosts/common/optional/docker-containers/syncthing.nix similarity index 100% rename from hosts/common/optional/arion-containers/syncthing.nix rename to hosts/common/optional/docker-containers/syncthing.nix diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 12659aa..833ef00 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -178,12 +178,12 @@ in { imports = [ arion.nixosModules.arion sops-nix.nixosModules.sops - ../arion-containers/arrstack.nix - ../arion-containers/jellyfin.nix - ../arion-containers/photoprism.nix - ../arion-containers/syncthing.nix - ../arion-containers/baikal.nix - (import ../arion-containers/searxng.nix {configVars = configVars;}) + ../docker-containers/arrstack.nix + ../docker-containers/jellyfin.nix + ../docker-containers/photoprism.nix + ../docker-containers/syncthing.nix + ../docker-containers/baikal.nix + (import ../docker-containers/searxng.nix {configVars = configVars;}) ]; environment.systemPackages = [ diff --git a/hosts/common/optional/nixos-containers/pihole.nix b/hosts/common/optional/nixos-containers/pihole.nix index 1f648fd..31d5a13 100644 --- a/hosts/common/optional/nixos-containers/pihole.nix +++ b/hosts/common/optional/nixos-containers/pihole.nix @@ -72,7 +72,7 @@ in { imports = [ arion.nixosModules.arion - ../arion-containers/pihole.nix + ../docker-containers/pihole.nix ]; environment.systemPackages = [ diff --git a/hosts/common/optional/nixos-containers/semitamaps-worker.nix b/hosts/common/optional/nixos-containers/semitamaps-worker.nix index 9270136..c3ee543 100644 --- a/hosts/common/optional/nixos-containers/semitamaps-worker.nix +++ b/hosts/common/optional/nixos-containers/semitamaps-worker.nix @@ -101,7 +101,7 @@ in { imports = [ sops-nix.nixosModules.sops arion.nixosModules.arion - ../arion-containers/semitamaps-tileserver.nix + ../docker-containers/semitamaps-tileserver.nix ]; environment.systemPackages = [ From 61c40e0e9c1605970374d115dc793852e1f7ebac Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 13 Feb 2025 14:39:18 +0000 Subject: [PATCH 21/22] persist machineid --- hosts/common/optional/persistence.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/common/optional/persistence.nix b/hosts/common/optional/persistence.nix index 1f2f8c4..260686a 100644 --- a/hosts/common/optional/persistence.nix +++ b/hosts/common/optional/persistence.nix @@ -12,6 +12,7 @@ files = [ "/etc/ssh/ssh_host_ed25519_key" "/etc/ssh/ssh_host_ed25519_key.pub" + "/etc/machine-id" ]; }; } From 1dfe204376d9343bd958020ac70089e4522a689f Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 13 Feb 2025 14:42:20 +0000 Subject: [PATCH 22/22] update flake --- flake.lock | 62 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/flake.lock b/flake.lock index 70e52a9..54d52f6 100644 --- a/flake.lock +++ b/flake.lock @@ -170,11 +170,11 @@ ] }, "locked": { - "lastModified": 1736143030, - "narHash": "sha256-+hu54pAoLDEZT9pjHlqL9DNzWz0NbUn8NEAHP7PQPzU=", + "lastModified": 1738453229, + "narHash": "sha256-7H9XgNiGLKN1G1CgRh0vUL4AheZSYzPm+zmZ7vxbJdo=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "b905f6fc23a9051a6e1b741e1438dbfc0634c6de", + "rev": "32ea77a06711b758da0ad9bd6a844c5740a87abd", "type": "github" }, "original": { @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738685297, - "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", + "lastModified": 1739387047, + "narHash": "sha256-KpogJP00vwuMIKkGJff3zp0YfV9GfOG//UzMK4nWWUw=", "ref": "refs/heads/master", - "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", - "revCount": 269, + "rev": "be51e237b5b3d441a194f3e516175f6a543aee35", + "revCount": 280, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, @@ -601,11 +601,11 @@ }, "nixpkgs-unstable_2": { "locked": { - "lastModified": 1737746512, - "narHash": "sha256-nU6AezEX4EuahTO1YopzueAXfjFfmCHylYEFCagduHU=", + "lastModified": 1739214665, + "narHash": "sha256-26L8VAu3/1YRxS8MHgBOyOM8xALdo6N0I04PgorE7UM=", "owner": "nixos", "repo": "nixpkgs", - "rev": "825479c345a7f806485b7f00dbe3abb50641b083", + "rev": "64e75cd44acf21c7933d61d7721e812eac1b5a0a", "type": "github" }, "original": { @@ -633,11 +633,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1737893696, - "narHash": "sha256-FV16HsdY2KY/6ixpizYH+SVuLdDCa5WvJVHglMtvQ74=", + "lastModified": 1739440421, + "narHash": "sha256-SIHcTBrgenH2Vswi0Lro98Ep+cdeLdjjUSSnhiUQEkc=", "owner": "nixos", "repo": "nixpkgs", - "rev": "7db27bab4129d44b0433dd45fa9a30f7a0795db3", + "rev": "d72b0e85ee3a63c468217903707bcc654997bf39", "type": "github" }, "original": { @@ -649,11 +649,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1737746512, - "narHash": "sha256-nU6AezEX4EuahTO1YopzueAXfjFfmCHylYEFCagduHU=", + "lastModified": 1739214665, + "narHash": "sha256-26L8VAu3/1YRxS8MHgBOyOM8xALdo6N0I04PgorE7UM=", "owner": "nixos", "repo": "nixpkgs", - "rev": "825479c345a7f806485b7f00dbe3abb50641b083", + "rev": "64e75cd44acf21c7933d61d7721e812eac1b5a0a", "type": "github" }, "original": { @@ -678,11 +678,11 @@ "treefmt-nix": "treefmt-nix_2" }, "locked": { - "lastModified": 1737731711, - "narHash": "sha256-6ubhKkCkBMuqFMjzeg+/2L5dNipKKf1KE9i8r8inyEg=", + "lastModified": 1739018339, + "narHash": "sha256-/J2H70Gsi13nh8vGEV6aBr+e5avFaGOSr2TF4VjnVAk=", "owner": "nix-community", "repo": "nixvim", - "rev": "841155edf9c4578f2f9a7bd6993e1da2ce73b35c", + "rev": "9f0cb35c26a7dbc0acce0658f73b08b2198487c6", "type": "github" }, "original": { @@ -699,11 +699,11 @@ "treefmt-nix": "treefmt-nix_3" }, "locked": { - "lastModified": 1737896356, - "narHash": "sha256-JVZPYiBjllZqFjb6mmmdEQDmAkZCbXSNjOGRJWO52PA=", + "lastModified": 1739457502, + "narHash": "sha256-65vfR43/AvvGa3puZG+e6VraepB66zVty22xW2FLX0M=", "owner": "nix-community", "repo": "NUR", - "rev": "5ee047c2c9ca4400689d913843351c2c7e4f56fb", + "rev": "b890157155fb46d4aebeca5bad62fd494588ddd5", "type": "github" }, "original": { @@ -722,11 +722,11 @@ ] }, "locked": { - "lastModified": 1737372689, - "narHash": "sha256-nH3zK2ki0fd5o5qvbGHxukE4qnOLJa1uCzoDObG5vrE=", + "lastModified": 1738508923, + "narHash": "sha256-4DaDrQDAIxlWhTjH6h/+xfG05jt3qDZrZE/7zDLQaS4=", "owner": "NuschtOS", "repo": "search", - "rev": "570cc17bbc25650eb7d69e4fcda8cfd2f1656922", + "rev": "86e2038290859006e05ca7201425ea5b5de4aecb", "type": "github" }, "original": { @@ -784,11 +784,11 @@ ] }, "locked": { - "lastModified": 1737411508, - "narHash": "sha256-j9IdflJwRtqo9WpM0OfAZml47eBblUHGNQTe62OUqTw=", + "lastModified": 1739262228, + "narHash": "sha256-7JAGezJ0Dn5qIyA2+T4Dt/xQgAbhCglh6lzCekTVMeU=", "owner": "mic92", "repo": "sops-nix", - "rev": "015d461c16678fc02a2f405eb453abb509d4e1d4", + "rev": "07af005bb7d60c7f118d9d9f5530485da5d1e975", "type": "github" }, "original": { @@ -886,11 +886,11 @@ ] }, "locked": { - "lastModified": 1737483750, - "narHash": "sha256-5An1wq5U8sNycOBBg3nsDDgpwBmR9liOpDGlhliA6Xo=", + "lastModified": 1738953846, + "narHash": "sha256-yrK3Hjcr8F7qS/j2F+r7C7o010eVWWlm4T1PrbKBOxQ=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "f2cc121df15418d028a59c9737d38e3a90fbaf8f", + "rev": "4f09b473c936d41582dd744e19f34ec27592c5fd", "type": "github" }, "original": {