Replace Uptime Kuma with Gatus for declarative uptime monitoring

Gatus is configured entirely via YAML (mapped from Nix attrsets),
making nix-config the single source of truth for all monitoring
config instead of Uptime Kuma's web UI/SQLite database.

.gitignore

@@ -1 +1,2 @@
 result
+.claude/worktrees

CLAUDE.md

@@ -67,6 +67,12 @@ IP allocation convention: VMs `.10-.49`, containers `.50-.89`, incus `.90-.129`
 
 `flake.nix` applies patches from `/patches/` to nixpkgs before building (workaround for nix#3920).
 
+### Service Dashboard & Monitoring
+
+When adding or removing a web-facing service, update both:
+- **Gatus** (`common/server/gatus.nix`) — add/remove the endpoint monitor
+- **Dashy** — add/remove the service entry from the dashboard config
+
 ### Key Conventions
 
 - Uses `doas` instead of `sudo` everywhere
@@ -79,3 +85,9 @@ IP allocation convention: VMs `.10-.49`, containers `.50-.89`, incus `.90-.129`
 - Always use `--no-link` when running `nix build`
 - Don't use `nix build --dry-run` unless you only need evaluation — it skips the actual build
 - Avoid `2>&1` on nix commands — it can cause error output to be missed
+
+## Git Worktree Requirement
+
+When instructed to work in a git worktree (e.g., via `isolation: "worktree"` or told to use a worktree), you **MUST** do so. If you are unable to create or use a git worktree, you **MUST** stop work immediately and report the failure to the user. Do not fall back to working in the main working tree.
+
+When applying work from a git worktree back to the main branch, commit in the worktree first, then use `git cherry-pick` from the main working tree to bring the commit over. Do not use `git checkout` or `git apply` to copy files directly. Do **not** automatically apply worktree work to the main branch — always ask the user for approval first.

common/binary-cache.nix

@@ -6,7 +6,7 @@
       substituters = [
         "https://cache.nixos.org/"
         "https://nix-community.cachix.org"
-        "http://s0.koi-bebop.ts.net:28338/nixos"
+        "http://s0.neet.dev:28338/nixos"
       ];
       trusted-public-keys = [
         "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="

common/default.nix

@@ -7,6 +7,7 @@
     ./flakes.nix
     ./auto-update.nix
     ./ntfy-alerts.nix
+    ./zfs-alerts.nix
     ./shell.nix
     ./network
     ./boot

common/ntfy-alerts.nix

@@ -18,7 +18,7 @@ in
     };
   };
 
-  config = lib.mkIf (config.thisMachine.hasRole."server" || config.thisMachine.hasRole."personal") {
+  config = lib.mkIf config.thisMachine.hasRole."ntfy" {
     age.secrets.ntfy-token.file = ../secrets/ntfy-token.age;
 
     systemd.services."ntfy-failure@" = {

common/sandboxed-workspace/base.nix

@@ -27,6 +27,7 @@ in
     ../shell.nix
     hostConfig.inputs.home-manager.nixosModules.home-manager
     hostConfig.inputs.nix-index-database.nixosModules.default
+    hostConfig.inputs.agenix.nixosModules.default
   ];
 
   nixpkgs.overlays = [
@@ -116,6 +117,13 @@ in
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
   nix.settings.trusted-users = [ "googlebot" ];
 
+  # Binary cache configuration (inherited from host's common/binary-cache.nix)
+  nix.settings.substituters = hostConfig.nix.settings.substituters;
+  nix.settings.trusted-public-keys = hostConfig.nix.settings.trusted-public-keys;
+  nix.settings.fallback = true;
+  nix.settings.netrc-file = config.age.secrets.attic-netrc.path;
+  age.secrets.attic-netrc.file = ../../secrets/attic-netrc.age;
+
   # Make nixpkgs available in NIX_PATH and registry (like the NixOS ISO)
   # This allows `nix-shell -p`, `nix repl '<nixpkgs>'`, etc. to work
   nix.nixPath = [ "nixpkgs=${hostConfig.inputs.nixpkgs}" ];

common/server/default.nix

@@ -17,6 +17,6 @@
     ./actualbudget.nix
     ./unifi.nix
     ./ntfy.nix
-    ./uptime-kuma.nix
+    ./gatus.nix
   ];
 }

common/server/gatus.nix

@@ -0,0 +1,146 @@
+{ lib, config, ... }:
+
+let
+  cfg = config.services.gatus;
+  port = 31103;
+in
+{
+  options.services.gatus = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      example = "status.example.com";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.gatus = {
+      environmentFile = "/run/agenix/ntfy-token";
+      settings = {
+        storage = {
+          type = "sqlite";
+          path = "/var/lib/gatus/data.db";
+        };
+
+        web = {
+          address = "127.0.0.1";
+          port = port;
+        };
+
+        alerting.ntfy = {
+          url = "https://ntfy.neet.dev";
+          topic = "service-failures";
+          priority = 4;
+          default-alert = {
+            enabled = true;
+            failure-threshold = 3;
+            success-threshold = 2;
+            send-on-resolved = true;
+          };
+          token = "$NTFY_TOKEN";
+        };
+
+        endpoints = [
+          {
+            name = "Gitea";
+            group = "services";
+            url = "https://git.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "The Lounge";
+            group = "services";
+            url = "https://irc.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "ntfy";
+            group = "services";
+            url = "https://ntfy.neet.dev/v1/health";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Librechat";
+            group = "services";
+            url = "https://chat.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Owncast";
+            group = "services";
+            url = "https://live.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Nextcloud";
+            group = "services";
+            url = "https://neet.cloud";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == any(200, 302)"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Element Web";
+            group = "services";
+            url = "https://chat.neet.space";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Mumble";
+            group = "services";
+            url = "tcp://voice.neet.space:23563";
+            interval = "5m";
+            conditions = [
+              "[CONNECTED] == true"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Navidrome";
+            group = "services";
+            url = "https://navidrome.neet.cloud";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+        ];
+      };
+    };
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.${cfg.hostname} = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString port}";
+        proxyWebsockets = true;
+      };
+    };
+  };
+}

common/server/uptime-kuma.nix

@@ -1,36 +0,0 @@
-{ lib, config, ... }:
-
-let
-  cfg = config.services.uptime-kuma;
-  port = 3001;
-in
-{
-  options.services.uptime-kuma = {
-    hostname = lib.mkOption {
-      type = lib.types.str;
-      example = "status.example.com";
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.uptime-kuma.settings = {
-      HOST = "127.0.0.1";
-      PORT = toString port;
-    };
-
-    # backups
-    backup.group."uptime-kuma".paths = [
-      "/var/lib/uptime-kuma"
-    ];
-
-    services.nginx.enable = true;
-    services.nginx.virtualHosts.${cfg.hostname} = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/" = {
-        proxyPass = "http://127.0.0.1:${toString port}";
-        proxyWebsockets = true;
-      };
-    };
-  };
-}

common/zfs-alerts.nix

@@ -0,0 +1,87 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.ntfy-alerts;
+  hasZfs = config.boot.supportedFilesystems.zfs or false;
+  hasNtfy = config.thisMachine.hasRole."ntfy";
+
+  checkScript = pkgs.writeShellScript "zfs-health-check" ''
+    PATH="${lib.makeBinPath [ pkgs.zfs pkgs.coreutils pkgs.gawk pkgs.curl ]}"
+
+    unhealthy=""
+
+    # Check pool health status
+    while IFS=$'\t' read -r pool state; do
+      if [ "$state" != "ONLINE" ]; then
+        unhealthy="$unhealthy"$'\n'"Pool '$pool' is $state"
+      fi
+    done < <(zpool list -H -o name,health)
+
+    # Check for errors (read, write, checksum) on any vdev
+    while IFS=$'\t' read -r pool errors; do
+      if [ "$errors" != "No known data errors" ] && [ -n "$errors" ]; then
+        unhealthy="$unhealthy"$'\n'"Pool '$pool' has errors: $errors"
+      fi
+    done < <(zpool status -x 2>/dev/null | awk '
+      /pool:/ { pool=$2 }
+      /errors:/ { sub(/^[[:space:]]*errors: /, ""); print pool "\t" $0 }
+    ')
+
+    # Check for any drives with non-zero error counts
+    drive_errors=$(zpool status 2>/dev/null | awk '
+      /DEGRADED|FAULTED|OFFLINE|UNAVAIL|REMOVED/ && !/pool:/ && !/state:/ {
+        print "  " $0
+      }
+      /[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+/ {
+        if ($3 > 0 || $4 > 0 || $5 > 0) {
+          print "  " $1 " (read:" $3 " write:" $4 " cksum:" $5 ")"
+        }
+      }
+    ')
+    if [ -n "$drive_errors" ]; then
+      unhealthy="$unhealthy"$'\n'"Device errors:"$'\n'"$drive_errors"
+    fi
+
+    if [ -n "$unhealthy" ]; then
+      message="ZFS health check failed on ${config.networking.hostName}:$unhealthy"
+
+      curl \
+        --fail --silent --show-error \
+        --max-time 30 --retry 3 \
+        -H "Authorization: Bearer $NTFY_TOKEN" \
+        -H "Title: ZFS issue on ${config.networking.hostName}" \
+        -H "Priority: urgent" \
+        -H "Tags: warning" \
+        -d "$message" \
+        "${cfg.serverUrl}/${cfg.topic}"
+
+      echo "$message" >&2
+    fi
+
+    echo "All ZFS pools healthy"
+  '';
+in
+{
+  config = lib.mkIf (hasZfs && hasNtfy) {
+    systemd.services.zfs-health-check = {
+      description = "Check ZFS pool health and alert on issues";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" "zfs.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        EnvironmentFile = "/run/agenix/ntfy-token";
+        ExecStart = checkScript;
+      };
+    };
+
+    systemd.timers.zfs-health-check = {
+      description = "Periodic ZFS health check";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "daily";
+        Persistent = true;
+        RandomizedDelaySec = "1h";
+      };
+    };
+  };
+}

machines/fry/properties.nix

@@ -8,6 +8,7 @@
   systemRoles = [
     "personal"
     "dns-challenge"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID/Df5lG07Il7fizEgZR/T9bMlR0joESRJ7cqM9BkOyP";

machines/howl/properties.nix

@@ -7,6 +7,7 @@
 
   systemRoles = [
     "personal"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQi3q8jU6vRruExAL60J7GFO1gS8HsmXVJuKRT4ljrG";

machines/ponyo/default.nix

@@ -114,6 +114,6 @@
   services.ntfy-sh.hostname = "ntfy.neet.dev";
 
   # uptime monitoring
-  services.uptime-kuma.enable = true;
+  services.gatus.enable = true;
-  services.uptime-kuma.hostname = "status.neet.dev";
+  services.gatus.hostname = "status.neet.dev";
 }

machines/ponyo/properties.nix

@@ -15,6 +15,7 @@
     "dailybot"
     "gitea"
     "librechat"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";

machines/storage/s0/properties.nix

@@ -18,6 +18,7 @@
     "linkwarden"
     "outline"
     "dns-challenge"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";

machines/zoidberg/properties.nix

@@ -8,6 +8,7 @@
   systemRoles = [
     "personal"
     "media-center"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHvdC1EiLqSNVmk5L1p7cWRIrrlelbK+NMj6tEBrwqIq";