diff --git a/common/default.nix b/common/default.nix index 68361fb..33a30f7 100644 --- a/common/default.nix +++ b/common/default.nix @@ -6,6 +6,7 @@ ./binary-cache.nix ./flakes.nix ./auto-update.nix + ./ntfy-alerts.nix ./shell.nix ./network ./boot diff --git a/common/ntfy-alerts.nix b/common/ntfy-alerts.nix new file mode 100644 index 0000000..e5a3206 --- /dev/null +++ b/common/ntfy-alerts.nix @@ -0,0 +1,57 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.ntfy-alerts; +in +{ + options.ntfy-alerts = { + serverUrl = lib.mkOption { + type = lib.types.str; + default = "https://ntfy.neet.dev"; + description = "Base URL of the ntfy server."; + }; + + topic = lib.mkOption { + type = lib.types.str; + default = "service-failures"; + description = "ntfy topic to publish alerts to."; + }; + }; + + config = lib.mkIf (config.thisMachine.hasRole."server" || config.thisMachine.hasRole."personal") { + age.secrets.ntfy-token.file = ../secrets/ntfy-token.age; + + systemd.services."ntfy-failure@" = { + description = "Send ntfy alert for failed unit %i"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + EnvironmentFile = "/run/agenix/ntfy-token"; + ExecStart = "${pkgs.writeShellScript "ntfy-failure-notify" '' + unit="$1" + ${lib.getExe pkgs.curl} \ + --fail --silent --show-error \ + --max-time 30 --retry 3 \ + -H "Authorization: Bearer $NTFY_TOKEN" \ + -H "Title: Service failure on ${config.networking.hostName}" \ + -H "Priority: high" \ + -H "Tags: rotating_light" \ + -d "Unit $unit failed at $(date +%c)" \ + "${cfg.serverUrl}/${cfg.topic}" + ''} %i"; + }; + }; + + # Apply OnFailure to all services via a systemd drop-in + systemd.packages = [ + (pkgs.runCommand "ntfy-on-failure-dropin" { } '' + mkdir -p $out/lib/systemd/system/service.d + cat > $out/lib/systemd/system/service.d/ntfy-on-failure.conf <<'EOF' + [Unit] + OnFailure=ntfy-failure@%p.service + EOF + '') + ]; + }; +} diff --git a/secrets/ntfy-token.age b/secrets/ntfy-token.age new file mode 100644 index 0000000..eaafeec --- /dev/null +++ b/secrets/ntfy-token.age @@ -0,0 +1,19 @@ +age-encryption.org/v1 +-> ssh-ed25519 qEbiMg 5JtpNApPNiFqAB/gQcAsE1gz0Fg/uHW92f6Kx1J2ggQ +RzC1MQxyDYW1IuMo+OtSgcsND4v7XIRn0rCSkKCFA3A +-> ssh-ed25519 N7drjg mn6LWo+2zWEtUavbFQar966+j+g5su+lcBfWYz1aZDQ +EmKpdfkCSQao1+O/HJdOiam7UvBnDYcEEkgH6KrudQI +-> ssh-ed25519 jQaHAA an3Ukqz3BVoz0FEAA6/Lw1XKOkQWHwmTut+XD4E4vS8 +9N2ePtXG2FPJSmOwcAO9p92MJKJJpTlEhKSmgMiinB0 +-> ssh-ed25519 ZDy34A v98HzmBgwgOpUk2WrRuFsCdNR+nF2veVLzyT2pU2ZXY +o2pO5JbVEeaOFQ3beBvej6qgDdT9mgPCVHxmw2umhA0 +-> ssh-ed25519 w3nu8g Uba2LWQueJ50Ds1/RjvkXI+VH7calMiM7dbL02sRJ3U +mFj5skmDXhJV9lK5iwUpebqxqVPAexdUntrbWJEix+Q +-> ssh-ed25519 evqvfg wLEEdTdmDRiFGYDrYFQjvRzc3zmGXgvztIy3UuFXnWg +CtCV/PpaxBmtDV+6InmcbKxNDmbPUTzyCm8tCf1Qw/4 +-> ssh-ed25519 6AT2/g NaA1AhOdb+FGOCoWFEX0QN4cXS1CxlpFwpsH1L6vBA0 +Z9aMYAofQ4Ath0zsg0YdZG3GTnCN2uQW7EG02bMZRsc +-> ssh-ed25519 hPp1nw +shAZrydcbjXfYxm1UW1YosKg5ZwBBKO6cct4HdotBo +GxnopKlmZQ/I6kMZPNurLgqwwkFHpUradaNYTPnlMFU +--- W6DrhCmU08IEIyPpHDiRV21xVeALNk1bDHrLYc2YcC4 +'+ƩCli.Zt;L1Ek#5ѣ38?FWh6?_=j/RFiZ{ \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index f9d042e..47602ac 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -9,7 +9,7 @@ let nobody = sshKeys.userKeys; # For secrets that all machines need to know - everyone = roles.personal ++ roles.server; + everyone = lib.unique (roles.personal ++ roles.server); in with roles; @@ -43,8 +43,11 @@ with roles; "linkwarden-environment.age".publicKeys = linkwarden; # backups - "backblaze-s3-backups.age".publicKeys = personal ++ server; - "restic-password.age".publicKeys = personal ++ server; + "backblaze-s3-backups.age".publicKeys = everyone; + "restic-password.age".publicKeys = everyone; + + # ntfy alerts + "ntfy-token.age".publicKeys = everyone; # gitea actions runner "gitea-actions-runner-token.age".publicKeys = gitea-actions-runner;