From 43943485ee5250a4e45bdb9581c54d82b83a72d4 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 13:37:52 +0000 Subject: [PATCH 1/8] change jellyfin to oci container --- .../optional/arion-containers/jellyfin.nix | 5 ++++- .../common/optional/nixos-containers/docker.nix | 16 ++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/hosts/common/optional/arion-containers/jellyfin.nix b/hosts/common/optional/arion-containers/jellyfin.nix index 0792ea3..600eefe 100644 --- a/hosts/common/optional/arion-containers/jellyfin.nix +++ b/hosts/common/optional/arion-containers/jellyfin.nix @@ -20,8 +20,11 @@ PUID = "1000"; PGID = "1000"; DOCKER_MODS = "linuxserver/mods:jellyfin-opencl-intel"; + NVIDIA_VISIBLE_DEVICES = "all"; }; - extraOptions = ["--gpus=all"]; + extraOptions = [ + "--device=nvidia.com/gpu=all" + ]; }; }; }; diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index cc74b46..6b1f7b2 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -53,6 +53,10 @@ in { node = "/dev/nvidiactl"; modifier = "rwm"; } + { + node = "/dev/nvidia-uvm"; + modifier = "rwm"; + } { node = "/dev/fuse"; modifier = "rwm"; @@ -204,18 +208,6 @@ in { networking.firewall.interfaces."podman+".allowedUDPPorts = [53]; - systemd.services.podman-autostart = { - enable = true; - after = ["podman.service"]; - wantedBy = ["multi-user.target"]; - description = "Automatically start containers with --restart=always tag"; - serviceConfig = { - Type = "idle"; - ExecStartPre = ''${pkgs.coreutils}/bin/sleep 1''; - ExecStart = ''/run/current-system/sw/bin/podman start --all --filter restart-policy=always''; - }; - }; - services.prometheus = { exporters = { node = { From 7416197e54a39869b81ce573820a5ed9c6e67db8 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:09:34 +0000 Subject: [PATCH 2/8] add vlc --- home/common/optional/desktop/common/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/home/common/optional/desktop/common/default.nix b/home/common/optional/desktop/common/default.nix index 1c0bbad..79ccb19 100644 --- a/home/common/optional/desktop/common/default.nix +++ b/home/common/optional/desktop/common/default.nix @@ -29,5 +29,6 @@ pkgs.R pkgs.gimp pkgs.gajim + pkgs.vlc ]; } From 12aa157690e454466364833341c42a4d2d069c35 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 27 Jan 2025 20:25:51 +0000 Subject: [PATCH 3/8] change semita worker to use oci containers --- .../semitamaps-tileserver.nix | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix index c8c1d8f..5cd6e50 100644 --- a/hosts/common/optional/arion-containers/semitamaps-tileserver.nix +++ b/hosts/common/optional/arion-containers/semitamaps-tileserver.nix @@ -1,31 +1,25 @@ { - virtualisation.arion = { - backend = "podman-socket"; - projects.semitamaps-tileserver = { - settings = { - services.tileserver-gl.service = { - ports = [ - "8080:8080" - ]; - container_name = "tileserver-gl"; - image = "maptiler/tileserver-gl"; - restart = "always"; - volumes = [ - "/data/semitamaps-data/tileserver-gl/data:/data" - ]; - command = "-c /data/config.json --public_url https://tiles.semitamaps.com/"; - }; - - services.mbgl-renderer.service = { - ports = [ - "8081:80" - ]; - container_name = "mbgl-renderer"; - image = "mbgl-renderer"; - restart = "always"; - }; + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + tileserver-gl = { + image = "maptiler/tileserver-gl"; + ports = [ + "8080:8080" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; + }; + mbgl-renderer = { + image = "mbgl-renderer"; + ports = [ + "8081:80" + ]; + volumes = [ + "/data/semitamaps-data/tileserver-gl/data:/data" + ]; }; }; }; } - From 9abf175b5c376af2d549a15440e1e7010029c68b Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 30 Jan 2025 15:06:26 +0000 Subject: [PATCH 4/8] remove lnd and rtl from nix-bitcoin --- .../optional/nixos-containers/nix-bitcoin.nix | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/hosts/common/optional/nixos-containers/nix-bitcoin.nix b/hosts/common/optional/nixos-containers/nix-bitcoin.nix index 0bfd532..2788a60 100644 --- a/hosts/common/optional/nixos-containers/nix-bitcoin.nix +++ b/hosts/common/optional/nixos-containers/nix-bitcoin.nix @@ -62,7 +62,7 @@ in { }: { imports = [ inputs.nix-bitcoin.nixosModules.default - inputs.lnbits.nixosModules.default + # inputs.lnbits.nixosModules.default ]; environment.systemPackages = with pkgs; [ vim @@ -80,14 +80,13 @@ in { firewall = { enable = true; allowedTCPPorts = [ - 80 - 443 - 22 + # 80 + # 443 config.containers.bitcoin-node.config.services.bitcoind.rpc.port config.containers.bitcoin-node.config.services.mempool.frontend.port config.containers.bitcoin-node.config.services.electrs.port - config.containers.bitcoin-node.config.services.rtl.port - config.containers.bitcoin-node.config.services.lnd.port + # config.containers.bitcoin-node.config.services.rtl.port + # config.containers.bitcoin-node.config.services.lnd.port ]; }; useHostResolvConf = lib.mkForce false; @@ -144,7 +143,7 @@ in { }; }; lnd = { - enable = true; + enable = false; lndconnect = { enable = true; onion = true; @@ -156,34 +155,34 @@ in { ''; }; rtl = { - enable = true; + enable = false; nodes.lnd.enable = true; address = "0.0.0.0"; }; - lnbits = { - enable = true; - openFirewall = true; - host = "0.0.0.0"; - port = 8231; - env = { - LNBITS_ADMIN_UI = "true"; - LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; - LND_REST_ENDPOINT = "https://127.0.0.1:8080"; - LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; - LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; - AUTH_ALLOWED_METHODS = "user-id-only, username-password"; - }; - }; + # lnbits = { + # enable = false; + # openFirewall = true; + # host = "0.0.0.0"; + # port = 8231; + # env = { + # LNBITS_ADMIN_UI = "true"; + # LNBITS_BACKEND_WALLET_CLASS = "LndRestWallet"; + # LND_REST_ENDPOINT = "https://127.0.0.1:8080"; + # LND_REST_CERT = "/etc/nix-bitcoin-secrets/lnd-cert"; + # LND_REST_MACAROON = "/var/lib/lnbits/admin.macaroon"; + # AUTH_ALLOWED_METHODS = "user-id-only, username-password"; + # }; + # }; }; # Add custom systemd overrides for above services - systemd.services.lnbits.after = ["lnd.service"]; + # systemd.services.lnbits.after = ["lnd.service"]; nix-bitcoin.onionServices = { bitcoind.enable = true; electrs.enable = true; mempool-frontend.enable = true; - lnd.public = true; + # lnd.public = true; }; services.prometheus = { From 24e1bc12ea1ec74c7b119426d319feac083105e2 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 31 Jan 2025 21:17:49 +0000 Subject: [PATCH 5/8] add restic-exporter to prometheus metrics --- flake.lock | 8 ++-- .../nixos-containers/metrics-server.nix | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 550380e..0b5877e 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1737899664, - "narHash": "sha256-iZpzTSERNQ5UvFfEzrBLuEmcRUGjBSal7ShtXurYq8Q=", + "lastModified": 1738356588, + "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", "ref": "refs/heads/master", - "rev": "a9844a78dcbdc8a84679835112970d80822b113c", - "revCount": 257, + "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", + "revCount": 267, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 50417f4..148fbac 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -2,6 +2,7 @@ pkgs, lib, configVars, + inputs, ... }: let containerName = "metrics-server"; @@ -15,6 +16,7 @@ bitcoinNode = configVars.networking.addresses.bitcoin-node.ip; postres = configVars.networking.addresses.postgres.ip; backupServer = configVars.networking.addresses.backup-server.ip; + sops-nix = inputs.sops-nix; http_endpoints = configVars.metrics-server.blackbox.http_endpoints; @@ -52,6 +54,10 @@ in { hostPath = metricsServerContainerData; isReadOnly = false; }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "/etc/ssh/ssh_host_ed25519_key"; + isReadOnly = true; + }; }; config = { @@ -59,7 +65,10 @@ in { lib, config, ... - }: { + }: let + secretsDirectory = builtins.toString inputs.nix-secrets; + secretsFile = "${secretsDirectory}/secrets.yaml"; + in { networking = { defaultGateway = "${gatewayIp}"; interfaces.eth0.ipv4.addresses = [ @@ -79,9 +88,23 @@ in { useHostResolvConf = lib.mkForce false; }; + sops = { + defaultSopsFile = "${secretsFile}"; + validateSopsFiles = false; + + age = { + sshKeyPaths = ["/etc/ssh/ssh_host_ed25519_key"]; + }; + secrets = { + "software/restic-passphrase" = {}; + "software/restic-exporter-credentials" = {}; + }; + }; + services.resolved.enable = true; imports = [ + sops-nix.nixosModules.sops ]; environment.systemPackages = [ @@ -110,6 +133,16 @@ in { } ]; } + { + job_name = "restic-exporter"; + static_configs = [ + { + targets = [ + "0.0.0.0:8001" + ]; + } + ]; + } { job_name = "blackbox"; @@ -175,6 +208,14 @@ in { enabledCollectors = ["systemd"]; port = 9002; }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; + }; }; }; From b1e4be205318dd22a3c5da63713b749604dd448f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 13:15:18 +0000 Subject: [PATCH 6/8] add baikal oci container to docker --- flake.lock | 8 ++++---- .../common/optional/arion-containers/baikal.nix | 17 +++++++++++++++++ .../common/optional/nixos-containers/docker.nix | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 hosts/common/optional/arion-containers/baikal.nix diff --git a/flake.lock b/flake.lock index 0b5877e..9d5a28c 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738356588, - "narHash": "sha256-mb3P2bNaZuCz1is4NR05r2xm66n6ABQAkYLP5U5/eCY=", + "lastModified": 1738358831, + "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", "ref": "refs/heads/master", - "rev": "3ae59d3cfe419e10087da719129cca5c01b8cbcd", - "revCount": 267, + "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", + "revCount": 268, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/arion-containers/baikal.nix b/hosts/common/optional/arion-containers/baikal.nix new file mode 100644 index 0000000..5e41de5 --- /dev/null +++ b/hosts/common/optional/arion-containers/baikal.nix @@ -0,0 +1,17 @@ +{ + config.virtualisation.oci-containers = { + backend = "podman"; + containers = { + baikal = { + image = "ckulka/baikal:nginx"; + ports = [ + "6734:80" + ]; + volumes = [ + "/srv/docker/baikal/config:/var/www/baikal/config" + "/srv/docker/baikal/data:/var/www/baikal/Specific" + ]; + }; + }; + }; +} diff --git a/hosts/common/optional/nixos-containers/docker.nix b/hosts/common/optional/nixos-containers/docker.nix index 6b1f7b2..12659aa 100644 --- a/hosts/common/optional/nixos-containers/docker.nix +++ b/hosts/common/optional/nixos-containers/docker.nix @@ -182,6 +182,7 @@ in { ../arion-containers/jellyfin.nix ../arion-containers/photoprism.nix ../arion-containers/syncthing.nix + ../arion-containers/baikal.nix (import ../arion-containers/searxng.nix {configVars = configVars;}) ]; From 068593c82c19ca576fe1cccd124257d3094b7f63 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:28:59 +0000 Subject: [PATCH 7/8] setup prometheus alertmanager --- flake.lock | 8 +- .../nixos-containers/metrics-server.nix | 149 +++++++++++++----- vars/default.nix | 1 + 3 files changed, 114 insertions(+), 44 deletions(-) diff --git a/flake.lock b/flake.lock index 9d5a28c..70e52a9 100644 --- a/flake.lock +++ b/flake.lock @@ -539,11 +539,11 @@ }, "nix-secrets": { "locked": { - "lastModified": 1738358831, - "narHash": "sha256-BFkqC7xQwGpA7mYYGDBkzw9iehWao+BkR5Bp/dFicWY=", + "lastModified": 1738685297, + "narHash": "sha256-JOv3+toYlftzBm47QF5tzaBhTbQIm1IBq1tKeQrQLyM=", "ref": "refs/heads/master", - "rev": "e7311c8f523ad3ffe187efe63f6438140fa0cf45", - "revCount": 268, + "rev": "3be1d509f9823292dd9ca6b396743fbf722bd8b9", + "revCount": 269, "type": "git", "url": "ssh://git@git.bitlab21.com/sam/nix-secrets.git" }, diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index 148fbac..dd7c746 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -8,6 +8,9 @@ containerName = "metrics-server"; containerIp = configVars.networking.addresses.metrics-server.ip; + notifybotJid = configVars.xmpp.notifybotJid; + receiverJid = configVars.xmpp.personalAccount; + dockerContainerIp = configVars.networking.addresses.docker.ip; smWorkerIp = configVars.networking.addresses.sm-worker.ip; merlinIp = configVars.networking.addresses.merlin.ip; @@ -83,6 +86,7 @@ in { config.services.prometheus.port config.services.grafana.port config.services.prometheus.exporters.blackbox.port + 9199 #xmpp listen port ]; }; useHostResolvConf = lib.mkForce false; @@ -98,6 +102,9 @@ in { secrets = { "software/restic-passphrase" = {}; "software/restic-exporter-credentials" = {}; + "comms/xmpp/notifybot/password" = { + mode = "0644"; + }; }; }; @@ -112,9 +119,48 @@ in { pkgs.git ]; + services.grafana = { + enable = true; + settings.server = { + http_port = 2342; + http_addr = "0.0.0.0"; + }; + }; + + # main prometheus service services.prometheus = { enable = true; port = 9001; + alertmanagers = [ + { + scheme = "http"; + path_prefix = "/"; + static_configs = [ + { + targets = [ + "0.0.0.0:9093" + ]; + } + ]; + } + ]; + ruleFiles = [ + "${pkgs.writeText + "alert_rule.yml" + '' + groups: + - name: blackbox_alert + rules: + - alert: EndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Endpoint {{ $labels.instance }} down" + description: "An endpoint has been down for more than 1 minute." + ''}" + ]; scrapeConfigs = [ { job_name = "node_exporter"; @@ -171,51 +217,74 @@ in { ]; }; - services.grafana = { + # setup alertmanager + services.prometheus.xmpp-alerts = { enable = true; - settings.server = { - http_port = 2342; - http_addr = "0.0.0.0"; + settings = { + jid = notifybotJid; + password_command = "cat ${config.sops.secrets."comms/xmpp/notifybot/password".path}"; + to_jid = receiverJid; + listen_address = "0.0.0.0"; + listen_port = 9199; }; }; + services.prometheus.alertmanager = { + webExternalUrl = containerIp; + enable = true; + openFirewall = true; + port = 9093; + configText = '' + global: + resolve_timeout: 1m - services.prometheus = { - exporters = { - blackbox = { - enable = true; - configFile = pkgs.writeText "blackbox-conf.yaml" '' - modules: - http_basic: - prober: http - timeout: 5s - http: - preferred_ip_protocol: ip4 - valid_http_versions: ["HTTP/1.1", "HTTP/2"] - method: GET - # fail_if_ssl: false - # fail_if_not_ssl: true - # tls_config: - # insecure_skip_verify: true - tcp_connect: - prober: tcp - tcp: - preferred_ip_protocol: ip4 + route: + group_by: ['...'] + repeat_interval: 1h + receiver: 'xmpp-alerts' - ''; - }; - node = { - enable = true; - enabledCollectors = ["systemd"]; - port = 9002; - }; - restic = { - enable = true; - repository = ""; - environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; - passwordFile = config.sops.secrets."software/restic-passphrase".path; - refreshInterval = 10800; # refresh every 3 hours - port = 8001; - }; + receivers: + - name: 'xmpp-alerts' + webhook_configs: + - url: 'http://0.0.0.0:9199/alert' + ''; + }; + + # prometheus exporters + services.prometheus.exporters = { + blackbox = { + enable = true; + configFile = pkgs.writeText "blackbox-conf.yaml" '' + modules: + http_basic: + prober: http + timeout: 5s + http: + preferred_ip_protocol: ip4 + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + method: GET + # fail_if_ssl: false + # fail_if_not_ssl: true + # tls_config: + # insecure_skip_verify: true + tcp_connect: + prober: tcp + tcp: + preferred_ip_protocol: ip4 + + ''; + }; + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + restic = { + enable = true; + repository = ""; + environmentFile = config.sops.secrets."software/restic-exporter-credentials".path; + passwordFile = config.sops.secrets."software/restic-passphrase".path; + refreshInterval = 10800; # refresh every 3 hours + port = 8001; }; }; diff --git a/vars/default.nix b/vars/default.nix index f6973cc..82b6969 100644 --- a/vars/default.nix +++ b/vars/default.nix @@ -4,6 +4,7 @@ networking email metrics-server + xmpp ; locations = { mediaDataMountPoint = "/media/media"; From 595f1f92d5d7370f3a6ce3f36066135510aa83e8 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 4 Feb 2025 16:47:32 +0000 Subject: [PATCH 8/8] metrics-server add externalURL to prometheus --- hosts/common/optional/nixos-containers/metrics-server.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hosts/common/optional/nixos-containers/metrics-server.nix b/hosts/common/optional/nixos-containers/metrics-server.nix index dd7c746..a880109 100644 --- a/hosts/common/optional/nixos-containers/metrics-server.nix +++ b/hosts/common/optional/nixos-containers/metrics-server.nix @@ -130,6 +130,7 @@ in { # main prometheus service services.prometheus = { enable = true; + webExternalUrl = "http://${containerIp}:9001"; port = 9001; alertmanagers = [ { @@ -229,7 +230,7 @@ in { }; }; services.prometheus.alertmanager = { - webExternalUrl = containerIp; + webExternalUrl = "http://${containerIp}:9093"; enable = true; openFirewall = true; port = 9093;