Compare commits
6 Commits
master
...
9bfc71f9ac
| Author | SHA1 | Date | |
|---|---|---|---|
| 9bfc71f9ac | |||
| d61d56bf0f | |||
| 592d0a8f8d | |||
| 04fe2d36a9 | |||
| 382b2c374d | |||
| 58d54d1f17 |
@@ -20,41 +20,23 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
ref: master
|
ref: master
|
||||||
token: ${{ secrets.PUSH_TOKEN }}
|
|
||||||
|
|
||||||
- name: Configure git identity
|
|
||||||
run: |
|
|
||||||
git config user.name "gitea-runner"
|
|
||||||
git config user.email "gitea-runner@neet.dev"
|
|
||||||
|
|
||||||
- name: Update flake inputs
|
- name: Update flake inputs
|
||||||
id: update
|
run: nix flake update --commit-lock-file
|
||||||
run: |
|
|
||||||
nix flake update
|
|
||||||
if git diff --quiet flake.lock; then
|
|
||||||
echo "No changes to flake.lock, nothing to do"
|
|
||||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
git add flake.lock
|
|
||||||
git commit -m "flake.lock: update inputs"
|
|
||||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Build and cache
|
- name: Build and cache
|
||||||
if: steps.update.outputs.changed == 'true'
|
|
||||||
run: bash .gitea/scripts/build-and-cache.sh
|
run: bash .gitea/scripts/build-and-cache.sh
|
||||||
|
|
||||||
- name: Push updated lockfile
|
- name: Push updated lockfile
|
||||||
if: steps.update.outputs.changed == 'true'
|
|
||||||
run: git push
|
run: git push
|
||||||
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure() && steps.update.outputs.changed == 'true'
|
if: failure()
|
||||||
run: |
|
run: |
|
||||||
curl -s \
|
curl -s \
|
||||||
-H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" \
|
-H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" \
|
||||||
-H "Title: Flake auto-update failed" \
|
-H "Title: Flake auto-update failed" \
|
||||||
-H "Priority: high" \
|
-H "Priority: high" \
|
||||||
-H "Tags: warning" \
|
-H "Tags: warning" \
|
||||||
-d "Auto-update workflow failed. Check: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_number }}" \
|
-d "Auto-update workflow failed. Check: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}" \
|
||||||
https://ntfy.neet.dev/nix-flake-updates
|
ntfy.neet.dev/nix-flake-updates
|
||||||
|
|||||||
@@ -20,14 +20,3 @@ jobs:
|
|||||||
|
|
||||||
- name: Build and cache
|
- name: Build and cache
|
||||||
run: bash .gitea/scripts/build-and-cache.sh
|
run: bash .gitea/scripts/build-and-cache.sh
|
||||||
|
|
||||||
- name: Notify on failure
|
|
||||||
if: failure()
|
|
||||||
run: |
|
|
||||||
curl -s \
|
|
||||||
-H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" \
|
|
||||||
-H "Title: Flake check failed" \
|
|
||||||
-H "Priority: high" \
|
|
||||||
-H "Tags: warning" \
|
|
||||||
-d "Check failed for ${{ gitea.ref_name }}. Check: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_number }}" \
|
|
||||||
https://ntfy.neet.dev/nix-flake-updates
|
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1 @@
|
|||||||
result
|
result
|
||||||
.claude/worktrees
|
|
||||||
|
|||||||
12
CLAUDE.md
12
CLAUDE.md
@@ -67,12 +67,6 @@ IP allocation convention: VMs `.10-.49`, containers `.50-.89`, incus `.90-.129`
|
|||||||
|
|
||||||
`flake.nix` applies patches from `/patches/` to nixpkgs before building (workaround for nix#3920).
|
`flake.nix` applies patches from `/patches/` to nixpkgs before building (workaround for nix#3920).
|
||||||
|
|
||||||
### Service Dashboard & Monitoring
|
|
||||||
|
|
||||||
When adding or removing a web-facing service, update both:
|
|
||||||
- **Gatus** (`common/server/gatus.nix`) — add/remove the endpoint monitor
|
|
||||||
- **Dashy** — add/remove the service entry from the dashboard config
|
|
||||||
|
|
||||||
### Key Conventions
|
### Key Conventions
|
||||||
|
|
||||||
- Uses `doas` instead of `sudo` everywhere
|
- Uses `doas` instead of `sudo` everywhere
|
||||||
@@ -85,9 +79,3 @@ When adding or removing a web-facing service, update both:
|
|||||||
- Always use `--no-link` when running `nix build`
|
- Always use `--no-link` when running `nix build`
|
||||||
- Don't use `nix build --dry-run` unless you only need evaluation — it skips the actual build
|
- Don't use `nix build --dry-run` unless you only need evaluation — it skips the actual build
|
||||||
- Avoid `2>&1` on nix commands — it can cause error output to be missed
|
- Avoid `2>&1` on nix commands — it can cause error output to be missed
|
||||||
|
|
||||||
## Git Worktree Requirement
|
|
||||||
|
|
||||||
When instructed to work in a git worktree (e.g., via `isolation: "worktree"` or told to use a worktree), you **MUST** do so. If you are unable to create or use a git worktree, you **MUST** stop work immediately and report the failure to the user. Do not fall back to working in the main working tree.
|
|
||||||
|
|
||||||
When applying work from a git worktree back to the main branch, commit in the worktree first, then use `git cherry-pick` from the main working tree to bring the commit over. Do not use `git checkout` or `git apply` to copy files directly. Do **not** automatically apply worktree work to the main branch — always ask the user for approval first.
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
substituters = [
|
substituters = [
|
||||||
"https://cache.nixos.org/"
|
"https://cache.nixos.org/"
|
||||||
"https://nix-community.cachix.org"
|
"https://nix-community.cachix.org"
|
||||||
"http://s0.neet.dev:28338/nixos"
|
"http://s0.koi-bebop.ts.net:28338/nixos"
|
||||||
];
|
];
|
||||||
trusted-public-keys = [
|
trusted-public-keys = [
|
||||||
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
|
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
|
||||||
|
|||||||
@@ -6,8 +6,6 @@
|
|||||||
./binary-cache.nix
|
./binary-cache.nix
|
||||||
./flakes.nix
|
./flakes.nix
|
||||||
./auto-update.nix
|
./auto-update.nix
|
||||||
./ntfy-alerts.nix
|
|
||||||
./zfs-alerts.nix
|
|
||||||
./shell.nix
|
./shell.nix
|
||||||
./network
|
./network
|
||||||
./boot
|
./boot
|
||||||
@@ -102,5 +100,5 @@
|
|||||||
security.acme.defaults.email = "zuckerberg@neet.dev";
|
security.acme.defaults.email = "zuckerberg@neet.dev";
|
||||||
|
|
||||||
# Enable Desktop Environment if this is a PC (machine role is "personal")
|
# Enable Desktop Environment if this is a PC (machine role is "personal")
|
||||||
de.enable = lib.mkDefault (config.thisMachine.hasRole."personal" && !config.boot.isContainer);
|
de.enable = lib.mkDefault (config.thisMachine.hasRole."personal");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ let
|
|||||||
in
|
in
|
||||||
lib.mkMerge [
|
lib.mkMerge [
|
||||||
# configure builder
|
# configure builder
|
||||||
(lib.mkIf (thisMachineIsABuilder && !config.boot.isContainer) {
|
(lib.mkIf thisMachineIsABuilder {
|
||||||
users.users.${builderUserName} = {
|
users.users.${builderUserName} = {
|
||||||
description = "Distributed Nix Build User";
|
description = "Distributed Nix Build User";
|
||||||
group = builderUserName;
|
group = builderUserName;
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
{ config, lib, pkgs, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
cfg = config.ntfy-alerts;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.ntfy-alerts = {
|
|
||||||
serverUrl = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "https://ntfy.neet.dev";
|
|
||||||
description = "Base URL of the ntfy server.";
|
|
||||||
};
|
|
||||||
|
|
||||||
topic = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "service-failures";
|
|
||||||
description = "ntfy topic to publish alerts to.";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf config.thisMachine.hasRole."ntfy" {
|
|
||||||
age.secrets.ntfy-token.file = ../secrets/ntfy-token.age;
|
|
||||||
|
|
||||||
systemd.services."ntfy-failure@" = {
|
|
||||||
description = "Send ntfy alert for failed unit %i";
|
|
||||||
wants = [ "network-online.target" ];
|
|
||||||
after = [ "network-online.target" ];
|
|
||||||
serviceConfig = {
|
|
||||||
Type = "oneshot";
|
|
||||||
EnvironmentFile = "/run/agenix/ntfy-token";
|
|
||||||
ExecStart = "${pkgs.writeShellScript "ntfy-failure-notify" ''
|
|
||||||
unit="$1"
|
|
||||||
${lib.getExe pkgs.curl} \
|
|
||||||
--fail --silent --show-error \
|
|
||||||
--max-time 30 --retry 3 \
|
|
||||||
-H "Authorization: Bearer $NTFY_TOKEN" \
|
|
||||||
-H "Title: Service failure on ${config.networking.hostName}" \
|
|
||||||
-H "Priority: high" \
|
|
||||||
-H "Tags: rotating_light" \
|
|
||||||
-d "Unit $unit failed at $(date +%c)" \
|
|
||||||
"${cfg.serverUrl}/${cfg.topic}"
|
|
||||||
''} %i";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
# Apply OnFailure to all services via a systemd drop-in
|
|
||||||
systemd.packages = [
|
|
||||||
(pkgs.runCommand "ntfy-on-failure-dropin" { } ''
|
|
||||||
mkdir -p $out/lib/systemd/system/service.d
|
|
||||||
cat > $out/lib/systemd/system/service.d/ntfy-on-failure.conf <<'EOF'
|
|
||||||
[Unit]
|
|
||||||
OnFailure=ntfy-failure@%p.service
|
|
||||||
EOF
|
|
||||||
'')
|
|
||||||
];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -27,7 +27,6 @@ in
|
|||||||
../shell.nix
|
../shell.nix
|
||||||
hostConfig.inputs.home-manager.nixosModules.home-manager
|
hostConfig.inputs.home-manager.nixosModules.home-manager
|
||||||
hostConfig.inputs.nix-index-database.nixosModules.default
|
hostConfig.inputs.nix-index-database.nixosModules.default
|
||||||
hostConfig.inputs.agenix.nixosModules.default
|
|
||||||
];
|
];
|
||||||
|
|
||||||
nixpkgs.overlays = [
|
nixpkgs.overlays = [
|
||||||
@@ -117,13 +116,6 @@ in
|
|||||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||||
nix.settings.trusted-users = [ "googlebot" ];
|
nix.settings.trusted-users = [ "googlebot" ];
|
||||||
|
|
||||||
# Binary cache configuration (inherited from host's common/binary-cache.nix)
|
|
||||||
nix.settings.substituters = hostConfig.nix.settings.substituters;
|
|
||||||
nix.settings.trusted-public-keys = hostConfig.nix.settings.trusted-public-keys;
|
|
||||||
nix.settings.fallback = true;
|
|
||||||
nix.settings.netrc-file = config.age.secrets.attic-netrc.path;
|
|
||||||
age.secrets.attic-netrc.file = ../../secrets/attic-netrc.age;
|
|
||||||
|
|
||||||
# Make nixpkgs available in NIX_PATH and registry (like the NixOS ISO)
|
# Make nixpkgs available in NIX_PATH and registry (like the NixOS ISO)
|
||||||
# This allows `nix-shell -p`, `nix repl '<nixpkgs>'`, etc. to work
|
# This allows `nix-shell -p`, `nix repl '<nixpkgs>'`, etc. to work
|
||||||
nix.nixPath = [ "nixpkgs=${hostConfig.inputs.nixpkgs}" ];
|
nix.nixPath = [ "nixpkgs=${hostConfig.inputs.nixpkgs}" ];
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{ config, lib, ... }:
|
{ config, lib, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
config = lib.mkIf (config.thisMachine.hasRole."binary-cache" && !config.boot.isContainer) {
|
config = lib.mkIf (config.thisMachine.hasRole."binary-cache") {
|
||||||
services.atticd = {
|
services.atticd = {
|
||||||
enable = true;
|
enable = true;
|
||||||
environmentFile = config.age.secrets.atticd-credentials.path;
|
environmentFile = config.age.secrets.atticd-credentials.path;
|
||||||
@@ -49,7 +49,6 @@
|
|||||||
systemd.services.atticd = {
|
systemd.services.atticd = {
|
||||||
after = [ "postgresql.service" ];
|
after = [ "postgresql.service" ];
|
||||||
requires = [ "postgresql.service" ];
|
requires = [ "postgresql.service" ];
|
||||||
partOf = [ "postgresql.service" ];
|
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
DynamicUser = lib.mkForce false;
|
DynamicUser = lib.mkForce false;
|
||||||
User = "atticd";
|
User = "atticd";
|
||||||
|
|||||||
@@ -17,6 +17,5 @@
|
|||||||
./actualbudget.nix
|
./actualbudget.nix
|
||||||
./unifi.nix
|
./unifi.nix
|
||||||
./ntfy.nix
|
./ntfy.nix
|
||||||
./gatus.nix
|
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,146 +0,0 @@
|
|||||||
{ lib, config, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
cfg = config.services.gatus;
|
|
||||||
port = 31103;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.services.gatus = {
|
|
||||||
hostname = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
example = "status.example.com";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
|
||||||
services.gatus = {
|
|
||||||
environmentFile = "/run/agenix/ntfy-token";
|
|
||||||
settings = {
|
|
||||||
storage = {
|
|
||||||
type = "sqlite";
|
|
||||||
path = "/var/lib/gatus/data.db";
|
|
||||||
};
|
|
||||||
|
|
||||||
web = {
|
|
||||||
address = "127.0.0.1";
|
|
||||||
port = port;
|
|
||||||
};
|
|
||||||
|
|
||||||
alerting.ntfy = {
|
|
||||||
url = "https://ntfy.neet.dev";
|
|
||||||
topic = "service-failures";
|
|
||||||
priority = 4;
|
|
||||||
default-alert = {
|
|
||||||
enabled = true;
|
|
||||||
failure-threshold = 3;
|
|
||||||
success-threshold = 2;
|
|
||||||
send-on-resolved = true;
|
|
||||||
};
|
|
||||||
token = "$NTFY_TOKEN";
|
|
||||||
};
|
|
||||||
|
|
||||||
endpoints = [
|
|
||||||
{
|
|
||||||
name = "Gitea";
|
|
||||||
group = "services";
|
|
||||||
url = "https://git.neet.dev";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "The Lounge";
|
|
||||||
group = "services";
|
|
||||||
url = "https://irc.neet.dev";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "ntfy";
|
|
||||||
group = "services";
|
|
||||||
url = "https://ntfy.neet.dev/v1/health";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Librechat";
|
|
||||||
group = "services";
|
|
||||||
url = "https://chat.neet.dev";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Owncast";
|
|
||||||
group = "services";
|
|
||||||
url = "https://live.neet.dev";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Nextcloud";
|
|
||||||
group = "services";
|
|
||||||
url = "https://neet.cloud";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == any(200, 302)"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Element Web";
|
|
||||||
group = "services";
|
|
||||||
url = "https://chat.neet.space";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Mumble";
|
|
||||||
group = "services";
|
|
||||||
url = "tcp://voice.neet.space:23563";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[CONNECTED] == true"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "Navidrome";
|
|
||||||
group = "services";
|
|
||||||
url = "https://navidrome.neet.cloud";
|
|
||||||
interval = "5m";
|
|
||||||
conditions = [
|
|
||||||
"[STATUS] == 200"
|
|
||||||
];
|
|
||||||
alerts = [{ type = "ntfy"; }];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
services.nginx.enable = true;
|
|
||||||
services.nginx.virtualHosts.${cfg.hostname} = {
|
|
||||||
enableACME = true;
|
|
||||||
forceSSL = true;
|
|
||||||
locations."/" = {
|
|
||||||
proxyPass = "http://127.0.0.1:${toString port}";
|
|
||||||
proxyWebsockets = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -16,8 +16,10 @@ in
|
|||||||
base-url = "https://${cfg.hostname}";
|
base-url = "https://${cfg.hostname}";
|
||||||
listen-http = "127.0.0.1:2586";
|
listen-http = "127.0.0.1:2586";
|
||||||
auth-default-access = "deny-all";
|
auth-default-access = "deny-all";
|
||||||
|
auth-file = "/var/lib/ntfy-sh/user.db";
|
||||||
behind-proxy = true;
|
behind-proxy = true;
|
||||||
enable-login = true;
|
enable-login = true;
|
||||||
|
attachment-cache-dir = "/var/lib/ntfy-sh/attachments";
|
||||||
};
|
};
|
||||||
|
|
||||||
# backups
|
# backups
|
||||||
|
|||||||
@@ -1,87 +0,0 @@
|
|||||||
{ config, lib, pkgs, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
cfg = config.ntfy-alerts;
|
|
||||||
hasZfs = config.boot.supportedFilesystems.zfs or false;
|
|
||||||
hasNtfy = config.thisMachine.hasRole."ntfy";
|
|
||||||
|
|
||||||
checkScript = pkgs.writeShellScript "zfs-health-check" ''
|
|
||||||
PATH="${lib.makeBinPath [ pkgs.zfs pkgs.coreutils pkgs.gawk pkgs.curl ]}"
|
|
||||||
|
|
||||||
unhealthy=""
|
|
||||||
|
|
||||||
# Check pool health status
|
|
||||||
while IFS=$'\t' read -r pool state; do
|
|
||||||
if [ "$state" != "ONLINE" ]; then
|
|
||||||
unhealthy="$unhealthy"$'\n'"Pool '$pool' is $state"
|
|
||||||
fi
|
|
||||||
done < <(zpool list -H -o name,health)
|
|
||||||
|
|
||||||
# Check for errors (read, write, checksum) on any vdev
|
|
||||||
while IFS=$'\t' read -r pool errors; do
|
|
||||||
if [ "$errors" != "No known data errors" ] && [ -n "$errors" ]; then
|
|
||||||
unhealthy="$unhealthy"$'\n'"Pool '$pool' has errors: $errors"
|
|
||||||
fi
|
|
||||||
done < <(zpool status -x 2>/dev/null | awk '
|
|
||||||
/pool:/ { pool=$2 }
|
|
||||||
/errors:/ { sub(/^[[:space:]]*errors: /, ""); print pool "\t" $0 }
|
|
||||||
')
|
|
||||||
|
|
||||||
# Check for any drives with non-zero error counts
|
|
||||||
drive_errors=$(zpool status 2>/dev/null | awk '
|
|
||||||
/DEGRADED|FAULTED|OFFLINE|UNAVAIL|REMOVED/ && !/pool:/ && !/state:/ {
|
|
||||||
print " " $0
|
|
||||||
}
|
|
||||||
/[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+/ {
|
|
||||||
if ($3 > 0 || $4 > 0 || $5 > 0) {
|
|
||||||
print " " $1 " (read:" $3 " write:" $4 " cksum:" $5 ")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
')
|
|
||||||
if [ -n "$drive_errors" ]; then
|
|
||||||
unhealthy="$unhealthy"$'\n'"Device errors:"$'\n'"$drive_errors"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$unhealthy" ]; then
|
|
||||||
message="ZFS health check failed on ${config.networking.hostName}:$unhealthy"
|
|
||||||
|
|
||||||
curl \
|
|
||||||
--fail --silent --show-error \
|
|
||||||
--max-time 30 --retry 3 \
|
|
||||||
-H "Authorization: Bearer $NTFY_TOKEN" \
|
|
||||||
-H "Title: ZFS issue on ${config.networking.hostName}" \
|
|
||||||
-H "Priority: urgent" \
|
|
||||||
-H "Tags: warning" \
|
|
||||||
-d "$message" \
|
|
||||||
"${cfg.serverUrl}/${cfg.topic}"
|
|
||||||
|
|
||||||
echo "$message" >&2
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "All ZFS pools healthy"
|
|
||||||
'';
|
|
||||||
in
|
|
||||||
{
|
|
||||||
config = lib.mkIf (hasZfs && hasNtfy) {
|
|
||||||
systemd.services.zfs-health-check = {
|
|
||||||
description = "Check ZFS pool health and alert on issues";
|
|
||||||
wants = [ "network-online.target" ];
|
|
||||||
after = [ "network-online.target" "zfs.target" ];
|
|
||||||
serviceConfig = {
|
|
||||||
Type = "oneshot";
|
|
||||||
EnvironmentFile = "/run/agenix/ntfy-token";
|
|
||||||
ExecStart = checkScript;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.timers.zfs-health-check = {
|
|
||||||
description = "Periodic ZFS health check";
|
|
||||||
wantedBy = [ "timers.target" ];
|
|
||||||
timerConfig = {
|
|
||||||
OnCalendar = "daily";
|
|
||||||
Persistent = true;
|
|
||||||
RandomizedDelaySec = "1h";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
18
flake.lock
generated
18
flake.lock
generated
@@ -228,11 +228,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1771756436,
|
"lastModified": 1771683283,
|
||||||
"narHash": "sha256-Tl2I0YXdhSTufGqAaD1ySh8x+cvVsEI1mJyJg12lxhI=",
|
"narHash": "sha256-WxAEkAbo8dP7qiyPM6VN4ZGAxfuBVlNBNPkrqkrXVEc=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "home-manager",
|
"repo": "home-manager",
|
||||||
"rev": "5bd3589390b431a63072868a90c0f24771ff4cbb",
|
"rev": "c6ed3eab64d23520bcbb858aa53fe2b533725d4a",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -250,11 +250,11 @@
|
|||||||
"spectrum": "spectrum"
|
"spectrum": "spectrum"
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1771802632,
|
"lastModified": 1771365290,
|
||||||
"narHash": "sha256-UAH8YfrHRvXAMeFxUzJ4h4B1loz1K1wiNUNI8KiPqOg=",
|
"narHash": "sha256-1XJOslVyF7yzf6yd/yl1VjGLywsbtwmQh3X1LuJcLI4=",
|
||||||
"owner": "astro",
|
"owner": "astro",
|
||||||
"repo": "microvm.nix",
|
"repo": "microvm.nix",
|
||||||
"rev": "b67e3d80df3ec35bdfd3a00ad64ee437ef4fcded",
|
"rev": "789c90b164b55b4379e7a94af8b9c01489024c18",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@@ -270,11 +270,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1771734689,
|
"lastModified": 1771130777,
|
||||||
"narHash": "sha256-/phvMgr1yutyAMjKnZlxkVplzxHiz60i4rc+gKzpwhg=",
|
"narHash": "sha256-UIKOwG0D9XVIJfNWg6+gENAvQP+7LO46eO0Jpe+ItJ0=",
|
||||||
"owner": "Mic92",
|
"owner": "Mic92",
|
||||||
"repo": "nix-index-database",
|
"repo": "nix-index-database",
|
||||||
"rev": "8f590b832326ab9699444f3a48240595954a4b10",
|
"rev": "efec7aaad8d43f8e5194df46a007456093c40f88",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|||||||
@@ -8,7 +8,6 @@
|
|||||||
systemRoles = [
|
systemRoles = [
|
||||||
"personal"
|
"personal"
|
||||||
"dns-challenge"
|
"dns-challenge"
|
||||||
"ntfy"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID/Df5lG07Il7fizEgZR/T9bMlR0joESRJ7cqM9BkOyP";
|
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID/Df5lG07Il7fizEgZR/T9bMlR0joESRJ7cqM9BkOyP";
|
||||||
|
|||||||
@@ -7,7 +7,6 @@
|
|||||||
|
|
||||||
systemRoles = [
|
systemRoles = [
|
||||||
"personal"
|
"personal"
|
||||||
"ntfy"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQi3q8jU6vRruExAL60J7GFO1gS8HsmXVJuKRT4ljrG";
|
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQi3q8jU6vRruExAL60J7GFO1gS8HsmXVJuKRT4ljrG";
|
||||||
|
|||||||
@@ -112,8 +112,4 @@
|
|||||||
# push notifications
|
# push notifications
|
||||||
services.ntfy-sh.enable = true;
|
services.ntfy-sh.enable = true;
|
||||||
services.ntfy-sh.hostname = "ntfy.neet.dev";
|
services.ntfy-sh.hostname = "ntfy.neet.dev";
|
||||||
|
|
||||||
# uptime monitoring
|
|
||||||
services.gatus.enable = true;
|
|
||||||
services.gatus.hostname = "status.neet.dev";
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,6 @@
|
|||||||
"dailybot"
|
"dailybot"
|
||||||
"gitea"
|
"gitea"
|
||||||
"librechat"
|
"librechat"
|
||||||
"ntfy"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";
|
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";
|
||||||
|
|||||||
@@ -45,6 +45,12 @@
|
|||||||
fsType = "zfs";
|
fsType = "zfs";
|
||||||
options = [ "zfsutil" "X-mount.mkdir" ];
|
options = [ "zfsutil" "X-mount.mkdir" ];
|
||||||
};
|
};
|
||||||
|
fileSystems."/var/lib/atticd" =
|
||||||
|
{
|
||||||
|
device = "rpool/nixos/var/lib/atticd";
|
||||||
|
fsType = "zfs";
|
||||||
|
options = [ "zfsutil" "X-mount.mkdir" ];
|
||||||
|
};
|
||||||
fileSystems."/var/log" =
|
fileSystems."/var/log" =
|
||||||
{
|
{
|
||||||
device = "rpool/nixos/var/log";
|
device = "rpool/nixos/var/log";
|
||||||
|
|||||||
@@ -18,7 +18,6 @@
|
|||||||
"linkwarden"
|
"linkwarden"
|
||||||
"outline"
|
"outline"
|
||||||
"dns-challenge"
|
"dns-challenge"
|
||||||
"ntfy"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";
|
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";
|
||||||
|
|||||||
@@ -8,7 +8,6 @@
|
|||||||
systemRoles = [
|
systemRoles = [
|
||||||
"personal"
|
"personal"
|
||||||
"media-center"
|
"media-center"
|
||||||
"ntfy"
|
|
||||||
];
|
];
|
||||||
|
|
||||||
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHvdC1EiLqSNVmk5L1p7cWRIrrlelbK+NMj6tEBrwqIq";
|
hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHvdC1EiLqSNVmk5L1p7cWRIrrlelbK+NMj6tEBrwqIq";
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
age-encryption.org/v1
|
|
||||||
-> ssh-ed25519 qEbiMg 5JtpNApPNiFqAB/gQcAsE1gz0Fg/uHW92f6Kx1J2ggQ
|
|
||||||
RzC1MQxyDYW1IuMo+OtSgcsND4v7XIRn0rCSkKCFA3A
|
|
||||||
-> ssh-ed25519 N7drjg mn6LWo+2zWEtUavbFQar966+j+g5su+lcBfWYz1aZDQ
|
|
||||||
EmKpdfkCSQao1+O/HJdOiam7UvBnDYcEEkgH6KrudQI
|
|
||||||
-> ssh-ed25519 jQaHAA an3Ukqz3BVoz0FEAA6/Lw1XKOkQWHwmTut+XD4E4vS8
|
|
||||||
9N2ePtXG2FPJSmOwcAO9p92MJKJJpTlEhKSmgMiinB0
|
|
||||||
-> ssh-ed25519 ZDy34A v98HzmBgwgOpUk2WrRuFsCdNR+nF2veVLzyT2pU2ZXY
|
|
||||||
o2pO5JbVEeaOFQ3beBvej6qgDdT9mgPCVHxmw2umhA0
|
|
||||||
-> ssh-ed25519 w3nu8g Uba2LWQueJ50Ds1/RjvkXI+VH7calMiM7dbL02sRJ3U
|
|
||||||
mFj5skmDXhJV9lK5iwUpebqxqVPAexdUntrbWJEix+Q
|
|
||||||
-> ssh-ed25519 evqvfg wLEEdTdmDRiFGYDrYFQjvRzc3zmGXgvztIy3UuFXnWg
|
|
||||||
CtCV/PpaxBmtDV+6InmcbKxNDmbPUTzyCm8tCf1Qw/4
|
|
||||||
-> ssh-ed25519 6AT2/g NaA1AhOdb+FGOCoWFEX0QN4cXS1CxlpFwpsH1L6vBA0
|
|
||||||
Z9aMYAofQ4Ath0zsg0YdZG3GTnCN2uQW7EG02bMZRsc
|
|
||||||
-> ssh-ed25519 hPp1nw +shAZrydcbjXfYxm1UW1YosKg5ZwBBKO6cct4HdotBo
|
|
||||||
GxnopKlmZQ/I6kMZPNurLgqwwkFHpUradaNYTPnlMFU
|
|
||||||
--- W6DrhCmU08IEIyPpHDiRV21xVeALNk1bDHrLYc2YcC4
|
|
||||||
'+Æ©<C386>CËl”i.Z£t;ýL²1Eãk#5ŸŠÑ£38‰?± šFWhö6?±_Ã=Âæ<C382>ãj/æÌRFÆáãåiÿóãÌZ{‘
|
|
||||||
@@ -9,7 +9,7 @@ let
|
|||||||
nobody = sshKeys.userKeys;
|
nobody = sshKeys.userKeys;
|
||||||
|
|
||||||
# For secrets that all machines need to know
|
# For secrets that all machines need to know
|
||||||
everyone = lib.unique (roles.personal ++ roles.server);
|
everyone = roles.personal ++ roles.server;
|
||||||
in
|
in
|
||||||
|
|
||||||
with roles;
|
with roles;
|
||||||
@@ -43,11 +43,8 @@ with roles;
|
|||||||
"linkwarden-environment.age".publicKeys = linkwarden;
|
"linkwarden-environment.age".publicKeys = linkwarden;
|
||||||
|
|
||||||
# backups
|
# backups
|
||||||
"backblaze-s3-backups.age".publicKeys = everyone;
|
"backblaze-s3-backups.age".publicKeys = personal ++ server;
|
||||||
"restic-password.age".publicKeys = everyone;
|
"restic-password.age".publicKeys = personal ++ server;
|
||||||
|
|
||||||
# ntfy alerts
|
|
||||||
"ntfy-token.age".publicKeys = everyone;
|
|
||||||
|
|
||||||
# gitea actions runner
|
# gitea actions runner
|
||||||
"gitea-actions-runner-token.age".publicKeys = gitea-actions-runner;
|
"gitea-actions-runner-token.age".publicKeys = gitea-actions-runner;
|
||||||
|
|||||||
Reference in New Issue
Block a user