Add ntfy failure alerts for all systemd services

common/default.nix

@@ -6,6 +6,7 @@
     ./binary-cache.nix
     ./flakes.nix
     ./auto-update.nix
+    ./ntfy-alerts.nix
     ./shell.nix
     ./network
     ./boot

common/ntfy-alerts.nix

@@ -0,0 +1,57 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.ntfy-alerts;
+in
+{
+  options.ntfy-alerts = {
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      default = "https://ntfy.neet.dev";
+      description = "Base URL of the ntfy server.";
+    };
+
+    topic = lib.mkOption {
+      type = lib.types.str;
+      default = "service-failures";
+      description = "ntfy topic to publish alerts to.";
+    };
+  };
+
+  config = lib.mkIf (config.thisMachine.hasRole."server" || config.thisMachine.hasRole."personal") {
+    age.secrets.ntfy-token.file = ../secrets/ntfy-token.age;
+
+    systemd.services."ntfy-failure@" = {
+      description = "Send ntfy alert for failed unit %i";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        EnvironmentFile = "/run/agenix/ntfy-token";
+        ExecStart = "${pkgs.writeShellScript "ntfy-failure-notify" ''
+          unit="$1"
+          ${lib.getExe pkgs.curl} \
+            --fail --silent --show-error \
+            --max-time 30 --retry 3 \
+            -H "Authorization: Bearer $NTFY_TOKEN" \
+            -H "Title: Service failure on ${config.networking.hostName}" \
+            -H "Priority: high" \
+            -H "Tags: rotating_light" \
+            -d "Unit $unit failed at $(date +%c)" \
+            "${cfg.serverUrl}/${cfg.topic}"
+        ''} %i";
+      };
+    };
+
+    # Apply OnFailure to all services via a systemd drop-in
+    systemd.packages = [
+      (pkgs.runCommand "ntfy-on-failure-dropin" { } ''
+        mkdir -p $out/lib/systemd/system/service.d
+        cat > $out/lib/systemd/system/service.d/ntfy-on-failure.conf <<'EOF'
+        [Unit]
+        OnFailure=ntfy-failure@%p.service
+        EOF
+      '')
+    ];
+  };
+}

secrets/ntfy-token.age

@@ -0,0 +1,19 @@
+age-encryption.org/v1
+-> ssh-ed25519 qEbiMg 5JtpNApPNiFqAB/gQcAsE1gz0Fg/uHW92f6Kx1J2ggQ
+RzC1MQxyDYW1IuMo+OtSgcsND4v7XIRn0rCSkKCFA3A
+-> ssh-ed25519 N7drjg mn6LWo+2zWEtUavbFQar966+j+g5su+lcBfWYz1aZDQ
+EmKpdfkCSQao1+O/HJdOiam7UvBnDYcEEkgH6KrudQI
+-> ssh-ed25519 jQaHAA an3Ukqz3BVoz0FEAA6/Lw1XKOkQWHwmTut+XD4E4vS8
+9N2ePtXG2FPJSmOwcAO9p92MJKJJpTlEhKSmgMiinB0
+-> ssh-ed25519 ZDy34A v98HzmBgwgOpUk2WrRuFsCdNR+nF2veVLzyT2pU2ZXY
+o2pO5JbVEeaOFQ3beBvej6qgDdT9mgPCVHxmw2umhA0
+-> ssh-ed25519 w3nu8g Uba2LWQueJ50Ds1/RjvkXI+VH7calMiM7dbL02sRJ3U
+mFj5skmDXhJV9lK5iwUpebqxqVPAexdUntrbWJEix+Q
+-> ssh-ed25519 evqvfg wLEEdTdmDRiFGYDrYFQjvRzc3zmGXgvztIy3UuFXnWg
+CtCV/PpaxBmtDV+6InmcbKxNDmbPUTzyCm8tCf1Qw/4
+-> ssh-ed25519 6AT2/g NaA1AhOdb+FGOCoWFEX0QN4cXS1CxlpFwpsH1L6vBA0
+Z9aMYAofQ4Ath0zsg0YdZG3GTnCN2uQW7EG02bMZRsc
+-> ssh-ed25519 hPp1nw +shAZrydcbjXfYxm1UW1YosKg5ZwBBKO6cct4HdotBo
+GxnopKlmZQ/I6kMZPNurLgqwwkFHpUradaNYTPnlMFU
+--- W6DrhCmU08IEIyPpHDiRV21xVeALNk1bDHrLYc2YcC4
+'+Æ©<C386>C
Ël”i.Z£t;ýL²1Eãk#5ŸŠÑ£38‰?± šFWhö6?±_Ã=Âæ<C382>ãj/æÌRFÆáãåiÿóãÌ
Z{‘

secrets/secrets.nix

@@ -9,7 +9,7 @@ let
   nobody = sshKeys.userKeys;
 
   # For secrets that all machines need to know
-  everyone = roles.personal ++ roles.server;
+  everyone = lib.unique (roles.personal ++ roles.server);
 in
 
 with roles;
@@ -43,8 +43,11 @@ with roles;
   "linkwarden-environment.age".publicKeys = linkwarden;
 
   # backups
-  "backblaze-s3-backups.age".publicKeys = personal ++ server;
+  "backblaze-s3-backups.age".publicKeys = everyone;
-  "restic-password.age".publicKeys = personal ++ server;
+  "restic-password.age".publicKeys = everyone;
+
+  # ntfy alerts
+  "ntfy-token.age".publicKeys = everyone;
 
   # gitea actions runner
   "gitea-actions-runner-token.age".publicKeys = gitea-actions-runner;