Add ntfy ssh login alerts. Include systemd service logs with service errors

common/default.nix

@@ -6,8 +6,7 @@
     ./binary-cache.nix
     ./flakes.nix
     ./auto-update.nix
-    ./ntfy-alerts.nix
+    ./ntfy
-    ./zfs-alerts.nix
     ./shell.nix
     ./network
     ./boot

common/ntfy/default.nix

@@ -0,0 +1,27 @@
+{ config, lib, ... }:
+
+{
+  imports = [
+    ./service-failure.nix
+    ./ssh-login.nix
+    ./zfs.nix
+  ];
+
+  options.ntfy-alerts = {
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      default = "https://ntfy.neet.dev";
+      description = "Base URL of the ntfy server.";
+    };
+
+    curlExtraArgs = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "Extra arguments to pass to curl (e.g. --proxy http://host:port).";
+    };
+  };
+
+  config = lib.mkIf config.thisMachine.hasRole."ntfy" {
+    age.secrets.ntfy-token.file = ../../secrets/ntfy-token.age;
+  };
+}

common/ntfy/service-failure.nix

@@ -4,29 +4,7 @@ let
   cfg = config.ntfy-alerts;
 in
 {
-  options.ntfy-alerts = {
-    serverUrl = lib.mkOption {
-      type = lib.types.str;
-      default = "https://ntfy.neet.dev";
-      description = "Base URL of the ntfy server.";
-    };
-
-    topic = lib.mkOption {
-      type = lib.types.str;
-      default = "service-failures";
-      description = "ntfy topic to publish alerts to.";
-    };
-
-    curlExtraArgs = lib.mkOption {
-      type = lib.types.str;
-      default = "";
-      description = "Extra arguments to pass to curl (e.g. --proxy http://host:port).";
-    };
-  };
-
   config = lib.mkIf config.thisMachine.hasRole."ntfy" {
-    age.secrets.ntfy-token.file = ../secrets/ntfy-token.age;
-
     systemd.services."ntfy-failure@" = {
       description = "Send ntfy alert for failed unit %i";
       wants = [ "network-online.target" ];
@@ -36,7 +14,12 @@ in
         EnvironmentFile = "/run/agenix/ntfy-token";
         ExecStart = "${pkgs.writeShellScript "ntfy-failure-notify" ''
           unit="$1"
+          logfile=$(mktemp)
+          trap 'rm -f "$logfile"' EXIT
+          ${pkgs.systemd}/bin/journalctl -u "$unit" -n 50 --no-pager -o short > "$logfile" 2>/dev/null \
+            || echo "(no logs available)" > "$logfile"
           ${lib.getExe pkgs.curl} \
+            -T "$logfile" \
             --fail --silent --show-error \
             --max-time 30 --retry 3 \
             ${cfg.curlExtraArgs} \
@@ -44,8 +27,9 @@ in
             -H "Title: Service failure on ${config.networking.hostName}" \
             -H "Priority: high" \
             -H "Tags: rotating_light" \
-            -d "Unit $unit failed at $(date +%c)" \
+            -H "Message: Unit $unit failed at $(date +%c)" \
-            "${cfg.serverUrl}/${cfg.topic}"
+            -H "Filename: $unit.log" \
+            "${cfg.serverUrl}/service-failures"
         ''} %i";
       };
     };

common/ntfy/ssh-login.nix

@@ -0,0 +1,36 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.ntfy-alerts;
+
+  notifyScript = pkgs.writeShellScript "ssh-login-notify" ''
+    # Only notify on session open, not close
+    [ "$PAM_TYPE" = "open_session" ] || exit 0
+
+    . /run/agenix/ntfy-token
+
+    # Send notification in background so login isn't delayed
+    ${lib.getExe pkgs.curl} \
+      --fail --silent --show-error \
+      --max-time 10 --retry 1 \
+      ${cfg.curlExtraArgs} \
+      -H "Authorization: Bearer $NTFY_TOKEN" \
+      -H "Title: SSH login on ${config.networking.hostName}" \
+      -H "Tags: key" \
+      -d "$PAM_USER from $PAM_RHOST at $(date +%c)" \
+      "${cfg.serverUrl}/ssh-logins" &
+  '';
+in
+{
+  config = lib.mkIf config.thisMachine.hasRole."ntfy" {
+    security.pam.services.sshd.rules.session.ntfy-login = {
+      order = 99999;
+      control = "optional";
+      modulePath = "${pkgs.pam}/lib/security/pam_exec.so";
+      args = [
+        "quiet"
+        (toString notifyScript)
+      ];
+    };
+  };
+}

common/ntfy/zfs.nix

@@ -53,7 +53,7 @@ let
         -H "Priority: urgent" \
         -H "Tags: warning" \
         -d "$message" \
-        "${cfg.serverUrl}/${cfg.topic}"
+        "${cfg.serverUrl}/service-failures"
 
       echo "$message" >&2
     fi