dedupe

Replace the single VPN container (veth pair, host-side auth scripts) with a
multi-container setup on a shared bridge network:

- Dedicated VPN container handles all PIA auth, WireGuard config, NAT, and
  optional port forwarding DNAT
- Service containers default-route through VPN container (leak-proof by topology)
- Host runs tinyproxy on bridge for PIA API bootstrap before WG is up
- WG interface is still created in host netns and moved into VPN container
  namespace
- Monthly renewal to ensure that connection stays up (PIA allows connections to
  last up to 2 months)
- Drop OpenVPN support entirely

.gitea/scripts/build-and-cache.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Configure Attic cache
+attic login local "$ATTIC_ENDPOINT" "$ATTIC_TOKEN"
+attic use local:nixos
+
+# Check flake
+nix flake check --all-systems --print-build-logs --log-format raw --show-trace
+
+# Build all systems
+nix eval .#nixosConfigurations --apply 'cs: builtins.attrNames cs' --json \
+  | jq -r '.[]' \
+  | xargs -I{} nix build ".#nixosConfigurations.{}.config.system.build.toplevel" \
+      --no-link --print-build-logs --log-format raw
+
+# Push to cache (only locally-built paths >= 0.5MB)
+toplevels=$(nix eval .#nixosConfigurations \
+  --apply 'cs: map (n: "${cs.${n}.config.system.build.toplevel}") (builtins.attrNames cs)' \
+  --json | jq -r '.[]')
+echo "Found $(echo "$toplevels" | wc -l) system toplevels"
+paths=$(echo "$toplevels" \
+  | xargs nix path-info -r --json \
+  | jq -r '[to_entries[] | select(
+      (.value.signatures | all(startswith("cache.nixos.org") | not))
+      and .value.narSize >= 524288
+    ) | .key] | unique[]')
+echo "Pushing $(echo "$paths" | wc -l) unique paths to cache"
+echo "$paths" | xargs attic push local:nixos

.gitea/workflows/auto-update.yaml

@@ -0,0 +1,60 @@
+name: Auto Update Flake
+on:
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch: {}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+  PATH: /run/current-system/sw/bin/
+  XDG_CONFIG_HOME: ${{ runner.temp }}/.config
+  ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }}
+  ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+
+jobs:
+  auto-update:
+    runs-on: nixos
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: master
+          token: ${{ secrets.PUSH_TOKEN }}
+
+      - name: Configure git identity
+        run: |
+          git config user.name "gitea-runner"
+          git config user.email "gitea-runner@neet.dev"
+
+      - name: Update flake inputs
+        id: update
+        run: |
+          nix flake update
+          if git diff --quiet flake.lock; then
+            echo "No changes to flake.lock, nothing to do"
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            git add flake.lock
+            git commit -m "flake.lock: update inputs"
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build and cache
+        if: steps.update.outputs.changed == 'true'
+        run: bash .gitea/scripts/build-and-cache.sh
+
+      - name: Push updated lockfile
+        if: steps.update.outputs.changed == 'true'
+        run: git push
+
+      - name: Notify on failure
+        if: failure() && steps.update.outputs.changed == 'true'
+        run: |
+          curl -s \
+            -H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" \
+            -H "Title: Flake auto-update failed" \
+            -H "Priority: high" \
+            -H "Tags: warning" \
+            -d "Auto-update workflow failed. Check: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_number }}" \
+            https://ntfy.neet.dev/nix-flake-updates

.gitea/workflows/check-flake.yaml

@@ -5,6 +5,9 @@ on: [push]
 env:
   DEBIAN_FRONTEND: noninteractive
   PATH: /run/current-system/sw/bin/
+  XDG_CONFIG_HOME: ${{ runner.temp }}/.config
+  ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }}
+  ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
 
 jobs:
   check-flake:
@@ -15,5 +18,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Check Flake
-        run: nix flake check --all-systems --print-build-logs --log-format raw --show-trace
+      - name: Build and cache
+        run: bash .gitea/scripts/build-and-cache.sh
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          curl -s \
+            -H "Authorization: Bearer ${{ secrets.NTFY_TOKEN }}" \
+            -H "Title: Flake check failed" \
+            -H "Priority: high" \
+            -H "Tags: warning" \
+            -d "Check failed for ${{ gitea.ref_name }}. Check: ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_number }}" \
+            https://ntfy.neet.dev/nix-flake-updates

.gitignore

@@ -1 +1,2 @@
 result
+.claude/worktrees

CLAUDE.md

@@ -67,6 +67,12 @@ IP allocation convention: VMs `.10-.49`, containers `.50-.89`, incus `.90-.129`
 
 `flake.nix` applies patches from `/patches/` to nixpkgs before building (workaround for nix#3920).
 
+### Service Dashboard & Monitoring
+
+When adding or removing a web-facing service, update both:
+- **Gatus** (`common/server/gatus.nix`) — add/remove the endpoint monitor
+- **Dashy** — add/remove the service entry from the dashboard config
+
 ### Key Conventions
 
 - Uses `doas` instead of `sudo` everywhere
@@ -79,3 +85,17 @@ IP allocation convention: VMs `.10-.49`, containers `.50-.89`, incus `.90-.129`
 - Always use `--no-link` when running `nix build`
 - Don't use `nix build --dry-run` unless you only need evaluation — it skips the actual build
 - Avoid `2>&1` on nix commands — it can cause error output to be missed
+
+## Git Worktrees
+
+When the user asks you to "start a worktree" or work in a worktree, **do not create one manually** with `git worktree add`. Instead, tell the user to start a new session with:
+
+```bash
+claude --worktree <name>
+```
+
+This is the built-in Claude Code worktree workflow. It creates the worktree at `.claude/worktrees/<name>/` with a branch `worktree-<name>` and starts a new Claude session inside it. Cleanup is handled automatically on exit.
+
+When instructed to work in a git worktree (e.g., via `isolation: "worktree"` on a subagent), you **MUST** do so. If you are unable to create or use a git worktree, you **MUST** stop work immediately and report the failure to the user. Do not fall back to working in the main working tree.
+
+When applying work from a git worktree back to the main branch, commit in the worktree first, then use `git cherry-pick` from the main working tree to bring the commit over. Do not use `git checkout` or `git apply` to copy files directly. Do **not** automatically apply worktree work to the main branch — always ask the user for approval first.

README.md

@@ -1,11 +1,32 @@
-# My NixOS configurations
+# NixOS Configuration
 
-### Source Layout
-- `/common` - common configuration imported into all `/machines`
-    - `/boot` - config related to bootloaders, cpu microcode, and unlocking LUKS root disks over tor
-    - `/network` - config for tailscale, and NixOS container with automatic vpn tunneling via PIA
-    - `/pc` - config that a graphical PC should have. Have the `personal` role set in the machine's `properties.nix` to enable everthing.
-    - `/server` - config that creates new nixos services or extends existing ones to meet my needs
-- `/machines` - all my NixOS machines along with their machine unique configuration for hardware and services
-    - `/kexec` - a special machine for generating minimal kexec images. Does not import `/common`
-- `/secrets` - encrypted shared secrets unlocked through `/machines` ssh host keys
+A NixOS flake managing multiple machines with role-based configuration, agenix secrets, and sandboxed dev workspaces.
+
+## Layout
+
+- `/common` - shared configuration imported by all machines
+  - `/boot` - bootloaders, CPU microcode, remote LUKS unlock over Tor
+  - `/network` - Tailscale, VPN tunneling via PIA
+  - `/pc` - desktop/graphical config (enabled by the `personal` role)
+  - `/server` - service definitions and extensions
+  - `/sandboxed-workspace` - isolated dev environments (VM, container, or Incus)
+- `/machines` - per-machine config (`default.nix`, `hardware-configuration.nix`, `properties.nix`)
+- `/secrets` - agenix-encrypted secrets, decryptable by machines based on their roles
+- `/home` - Home Manager user config
+- `/lib` - custom library functions extending nixpkgs lib
+- `/overlays` - nixpkgs overlays applied globally
+- `/patches` - patches applied to nixpkgs at build time
+
+## Notable Features
+
+**Auto-discovery & roles** — Machines register themselves by placing a `properties.nix` under `/machines/`. No manual listing in `flake.nix`. Roles declared per-machine (`"personal"`, `"dns-challenge"`, etc.) drive feature enablement via `config.thisMachine.hasRole.<role>` and control which agenix secrets each machine can decrypt.
+
+**Machine properties module system** — `properties.nix` files form a separate lightweight module system (`machine-info`) for recording machine metadata (hostnames, architecture, roles, SSH keys). Since every machine's properties are visible to every other machine, each system can reflect on the properties of the entire fleet — enabling automatic SSH trust, role-based secret access, and cross-machine coordination without duplicating information.
+
+**Remote LUKS unlock over Tor** — Machines with encrypted root disks can be unlocked remotely via SSH. An embedded Tor hidden service starts in the initrd so the machine is reachable even without a known IP, using a separate SSH host key for the boot environment.
+
+**VPN containers** — A `vpn-container` module spins up an ephemeral NixOS container with a PIA WireGuard tunnel. The host creates the WireGuard interface and authenticates with PIA, then hands it off to the container's network namespace. This ensures that the container can **never** have direct internet access. Leakage is impossible.
+
+**Sandboxed workspaces** — Isolated dev environments backed by microVMs (cloud-hypervisor), systemd-nspawn containers, or Incus. Each workspace gets a static IP on a NAT'd bridge, auto-generated SSH host keys, shell aliases for management, and comes pre-configured with Claude Code. The sandbox network blocks access to the local LAN while allowing internet.
+
+**Snapshot-aware backups** — Restic backups to Backblaze B2 automatically create ZFS snapshots or btrfs read-only snapshots before backing up, using mount namespaces to bind-mount frozen data over the original paths so restic records correct paths. Each backup group gets a `restic_<group>` CLI wrapper. Supports `.nobackup` marker files.

common/binary-cache.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, ... }:
 
 {
   nix = {
@@ -6,11 +6,11 @@
       substituters = [
         "https://cache.nixos.org/"
         "https://nix-community.cachix.org"
-        "http://s0.koi-bebop.ts.net:5000"
+        "http://s0.neet.dev:28338/nixos"
       ];
       trusted-public-keys = [
         "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-        "s0.koi-bebop.ts.net:OjbzD86YjyJZpCp9RWaQKANaflcpKhtzBMNP8I2aPUU="
+        "nixos:e5AMCUWWEX9MESWAAMjBkZdGUpl588NhgsUO3HsdhFw="
       ];
 
       # Allow substituters to be offline
@@ -19,6 +19,11 @@
       # and use this flag as intended for deciding if it should build missing
       # derivations locally. See https://github.com/NixOS/nix/issues/6901
       fallback = true;
+
+      # Authenticate to private nixos cache
+      netrc-file = config.age.secrets.attic-netrc.path;
     };
   };
+
+  age.secrets.attic-netrc.file = ../secrets/attic-netrc.age;
 }

common/boot/default.nix

@@ -1,4 +1,4 @@
-{ lib, config, pkgs, ... }:
+{ ... }:
 
 {
   imports = [

common/boot/firmware.nix

@@ -1,4 +1,4 @@
-{ lib, config, pkgs, ... }:
+{ lib, config, ... }:
 
 with lib;
 let

common/default.nix

@@ -6,6 +6,8 @@
     ./binary-cache.nix
     ./flakes.nix
     ./auto-update.nix
+    ./ntfy-alerts.nix
+    ./zfs-alerts.nix
     ./shell.nix
     ./network
     ./boot
@@ -56,7 +58,6 @@
     pciutils
     usbutils
     killall
-    screen
     micro
     helix
     lm_sensors
@@ -95,11 +96,11 @@
     { groups = [ "wheel" ]; persist = true; }
   ];
 
-  nix.gc.automatic = true;
+  nix.gc.automatic = !config.boot.isContainer;
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "zuckerberg@neet.dev";
 
   # Enable Desktop Environment if this is a PC (machine role is "personal")
-  de.enable = lib.mkDefault (config.thisMachine.hasRole."personal");
+  de.enable = lib.mkDefault (config.thisMachine.hasRole."personal" && !config.boot.isContainer);
 }

common/flakes.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, config, ... }:
+{ lib, config, ... }:
 with lib;
 let
   cfg = config.nix.flakes;

common/machine-info/default.nix

@@ -1,7 +1,7 @@
 # Gathers info about each machine to constuct overall configuration
 # Ex: Each machine already trusts each others SSH fingerprint already
 
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 
 let
   machines = config.machines.hosts;

common/network/default.nix

@@ -7,11 +7,8 @@ let
 in
 {
   imports = [
-    ./pia-openvpn.nix
-    ./pia-wireguard.nix
-    ./ping.nix
+    ./pia-vpn
     ./tailscale.nix
-    ./vpn.nix
     ./sandbox.nix
   ];
 

common/network/pia-openvpn.nix

@@ -1,113 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.pia.openvpn;
-  vpnfailsafe = pkgs.stdenv.mkDerivation {
-    pname = "vpnfailsafe";
-    version = "0.0.1";
-    src = ./.;
-    installPhase = ''
-      mkdir -p $out
-      cp vpnfailsafe.sh $out/vpnfailsafe.sh
-      sed -i 's|getent|${pkgs.getent}/bin/getent|' $out/vpnfailsafe.sh
-    '';
-  };
-in
-{
-  options.pia.openvpn = {
-    enable = lib.mkEnableOption "Enable private internet access";
-    server = lib.mkOption {
-      type = lib.types.str;
-      default = "us-washingtondc.privacy.network";
-      example = "swiss.privacy.network";
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.openvpn = {
-      servers = {
-        pia = {
-          config = ''
-            client
-            dev tun
-            proto udp
-            remote ${cfg.server} 1198
-            resolv-retry infinite
-            nobind
-            persist-key
-            persist-tun
-            cipher aes-128-cbc
-            auth sha1
-            tls-client
-            remote-cert-tls server
-
-            auth-user-pass
-            compress
-            verb 1
-            reneg-sec 0
-            <crl-verify>
-            -----BEGIN X509 CRL-----
-            MIICWDCCAUAwDQYJKoZIhvcNAQENBQAwgegxCzAJBgNVBAYTAlVTMQswCQYDVQQI
-            EwJDQTETMBEGA1UEBxMKTG9zQW5nZWxlczEgMB4GA1UEChMXUHJpdmF0ZSBJbnRl
-            cm5ldCBBY2Nlc3MxIDAeBgNVBAsTF1ByaXZhdGUgSW50ZXJuZXQgQWNjZXNzMSAw
-            HgYDVQQDExdQcml2YXRlIEludGVybmV0IEFjY2VzczEgMB4GA1UEKRMXUHJpdmF0
-            ZSBJbnRlcm5ldCBBY2Nlc3MxLzAtBgkqhkiG9w0BCQEWIHNlY3VyZUBwcml2YXRl
-            aW50ZXJuZXRhY2Nlc3MuY29tFw0xNjA3MDgxOTAwNDZaFw0zNjA3MDMxOTAwNDZa
-            MCYwEQIBARcMMTYwNzA4MTkwMDQ2MBECAQYXDDE2MDcwODE5MDA0NjANBgkqhkiG
-            9w0BAQ0FAAOCAQEAQZo9X97ci8EcPYu/uK2HB152OZbeZCINmYyluLDOdcSvg6B5
-            jI+ffKN3laDvczsG6CxmY3jNyc79XVpEYUnq4rT3FfveW1+Ralf+Vf38HdpwB8EW
-            B4hZlQ205+21CALLvZvR8HcPxC9KEnev1mU46wkTiov0EKc+EdRxkj5yMgv0V2Re
-            ze7AP+NQ9ykvDScH4eYCsmufNpIjBLhpLE2cuZZXBLcPhuRzVoU3l7A9lvzG9mjA
-            5YijHJGHNjlWFqyrn1CfYS6koa4TGEPngBoAziWRbDGdhEgJABHrpoaFYaL61zqy
-            MR6jC0K2ps9qyZAN74LEBedEfK7tBOzWMwr58A==
-            -----END X509 CRL-----
-            </crl-verify>
-
-            <ca>
-            -----BEGIN CERTIFICATE-----
-            MIIFqzCCBJOgAwIBAgIJAKZ7D5Yv87qDMA0GCSqGSIb3DQEBDQUAMIHoMQswCQYD
-            VQQGEwJVUzELMAkGA1UECBMCQ0ExEzARBgNVBAcTCkxvc0FuZ2VsZXMxIDAeBgNV
-            BAoTF1ByaXZhdGUgSW50ZXJuZXQgQWNjZXNzMSAwHgYDVQQLExdQcml2YXRlIElu
-            dGVybmV0IEFjY2VzczEgMB4GA1UEAxMXUHJpdmF0ZSBJbnRlcm5ldCBBY2Nlc3Mx
-            IDAeBgNVBCkTF1ByaXZhdGUgSW50ZXJuZXQgQWNjZXNzMS8wLQYJKoZIhvcNAQkB
-            FiBzZWN1cmVAcHJpdmF0ZWludGVybmV0YWNjZXNzLmNvbTAeFw0xNDA0MTcxNzM1
-            MThaFw0zNDA0MTIxNzM1MThaMIHoMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0Ex
-            EzARBgNVBAcTCkxvc0FuZ2VsZXMxIDAeBgNVBAoTF1ByaXZhdGUgSW50ZXJuZXQg
-            QWNjZXNzMSAwHgYDVQQLExdQcml2YXRlIEludGVybmV0IEFjY2VzczEgMB4GA1UE
-            AxMXUHJpdmF0ZSBJbnRlcm5ldCBBY2Nlc3MxIDAeBgNVBCkTF1ByaXZhdGUgSW50
-            ZXJuZXQgQWNjZXNzMS8wLQYJKoZIhvcNAQkBFiBzZWN1cmVAcHJpdmF0ZWludGVy
-            bmV0YWNjZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAPXD
-            L1L9tX6DGf36liA7UBTy5I869z0UVo3lImfOs/GSiFKPtInlesP65577nd7UNzzX
-            lH/P/CnFPdBWlLp5ze3HRBCc/Avgr5CdMRkEsySL5GHBZsx6w2cayQ2EcRhVTwWp
-            cdldeNO+pPr9rIgPrtXqT4SWViTQRBeGM8CDxAyTopTsobjSiYZCF9Ta1gunl0G/
-            8Vfp+SXfYCC+ZzWvP+L1pFhPRqzQQ8k+wMZIovObK1s+nlwPaLyayzw9a8sUnvWB
-            /5rGPdIYnQWPgoNlLN9HpSmsAcw2z8DXI9pIxbr74cb3/HSfuYGOLkRqrOk6h4RC
-            OfuWoTrZup1uEOn+fw8CAwEAAaOCAVQwggFQMB0GA1UdDgQWBBQv63nQ/pJAt5tL
-            y8VJcbHe22ZOsjCCAR8GA1UdIwSCARYwggESgBQv63nQ/pJAt5tLy8VJcbHe22ZO
-            sqGB7qSB6zCB6DELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRMwEQYDVQQHEwpM
-            b3NBbmdlbGVzMSAwHgYDVQQKExdQcml2YXRlIEludGVybmV0IEFjY2VzczEgMB4G
-            A1UECxMXUHJpdmF0ZSBJbnRlcm5ldCBBY2Nlc3MxIDAeBgNVBAMTF1ByaXZhdGUg
-            SW50ZXJuZXQgQWNjZXNzMSAwHgYDVQQpExdQcml2YXRlIEludGVybmV0IEFjY2Vz
-            czEvMC0GCSqGSIb3DQEJARYgc2VjdXJlQHByaXZhdGVpbnRlcm5ldGFjY2Vzcy5j
-            b22CCQCmew+WL/O6gzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBDQUAA4IBAQAn
-            a5PgrtxfwTumD4+3/SYvwoD66cB8IcK//h1mCzAduU8KgUXocLx7QgJWo9lnZ8xU
-            ryXvWab2usg4fqk7FPi00bED4f4qVQFVfGfPZIH9QQ7/48bPM9RyfzImZWUCenK3
-            7pdw4Bvgoys2rHLHbGen7f28knT2j/cbMxd78tQc20TIObGjo8+ISTRclSTRBtyC
-            GohseKYpTS9himFERpUgNtefvYHbn70mIOzfOJFTVqfrptf9jXa9N8Mpy3ayfodz
-            1wiqdteqFXkTYoSDctgKMiZ6GdocK9nMroQipIQtpnwd4yBDWIyC6Bvlkrq5TQUt
-            YDQ8z9v+DMO6iwyIDRiU
-            -----END CERTIFICATE-----
-            </ca>
-
-            disable-occ
-            auth-user-pass /run/agenix/pia-login.conf
-          '';
-          autoStart = true;
-          up = "${vpnfailsafe}/vpnfailsafe.sh";
-          down = "${vpnfailsafe}/vpnfailsafe.sh";
-        };
-      };
-    };
-    age.secrets."pia-login.conf".file = ../../secrets/pia-login.age;
-  };
-}

common/network/pia-vpn/README.md

@@ -0,0 +1,89 @@
+# PIA VPN Multi-Container Module
+
+Routes service containers through a PIA WireGuard VPN using a shared bridge network.
+
+## Architecture
+
+```
+                  internet
+                     │
+              ┌──────┴──────┐
+              │    Host      │
+              │  tinyproxy   │  ← PIA API bootstrap proxy
+              │  10.100.0.1  │
+              └──────┬───────┘
+                     │ br-vpn (no IPMasquerade)
+        ┌────────────┼──────────────┐
+        │            │              │
+ ┌──────┴──────┐ ┌───┴────┐  ┌─────┴──────┐
+ │ VPN ctr     │ │ servarr│  │transmission│
+ │ 10.100.0.2  │ │  .11   │  │   .10      │
+ │ piaw (WG)   │ │        │  │            │
+ │ gateway+NAT │ └────────┘  └────────────┘
+ └─────────────┘
+```
+
+- **Host** creates the WG interface (encrypted UDP stays in host netns) and runs tinyproxy on the bridge so the VPN container can bootstrap PIA auth before WG is up.
+- **VPN container** authenticates with PIA via the proxy, configures WG, sets up NAT (masquerade bridge→WG) and optional port forwarding DNAT.
+- **Service containers** default-route through the VPN container. No WG interface = no internet if VPN is down = leak-proof by topology.
+- **Host** reaches containers directly on the bridge for nginx reverse proxying.
+
+## Key design decisions
+
+- **Bridge, not veth pairs**: All containers share one bridge (`br-vpn`), so the VPN container can act as a single gateway. The host does NOT masquerade bridge traffic — only the VPN container does (through WG).
+- **Port forwarding is implicit**: If any container sets `receiveForwardedPort`, the VPN container automatically handles PIA port forwarding and DNAT. No separate toggle needed.
+- **DNS through WG**: Service containers use the VPN container as their DNS server. The VPN container runs `systemd-resolved` listening on its bridge IP, forwarding queries through the WG tunnel.
+- **Monthly renewal**: `pia-vpn-setup` uses `Type=simple` + `Restart=always` + `RuntimeMaxSec=30d` to periodically re-authenticate with PIA and get a fresh port forwarding signature (signatures expire after ~2 months). Service containers are unaffected during renewal.
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `default.nix` | Options, bridge, tinyproxy, host firewall, WG interface creation, assertions |
+| `vpn-container.nix` | VPN container: PIA auth, WG config, NAT, DNAT, port refresh timer |
+| `service-container.nix` | Generates service containers with static IP and gateway→VPN |
+| `scripts.nix` | Bash function library for PIA API calls and WG configuration |
+| `ca.rsa.4096.crt` | PIA CA certificate for API TLS verification |
+
+## Usage
+
+```nix
+pia-vpn = {
+  enable = true;
+  serverLocation = "swiss";
+
+  containers.my-service = {
+    ip = "10.100.0.10";
+    mounts."/data".hostPath = "/data";
+    config = { services.my-app.enable = true; };
+
+    # Optional: receive PIA's forwarded port (at most one container)
+    receiveForwardedPort = { port = 8080; protocol = "both"; };
+    onPortForwarded = ''
+      echo "PIA assigned port $PORT, forwarding to $TARGET_IP:8080"
+    '';
+  };
+};
+```
+
+## Debugging
+
+```bash
+# Check VPN container status
+machinectl shell pia-vpn
+systemctl status pia-vpn-setup
+journalctl -u pia-vpn-setup
+
+# Verify WG tunnel
+wg show
+
+# Check NAT/DNAT rules
+iptables -t nat -L -v
+iptables -L FORWARD -v
+
+# From a service container — verify VPN routing
+curl ifconfig.me
+
+# Port refresh logs
+journalctl -u pia-vpn-port-refresh
+```

common/network/pia-vpn/default.nix

@@ -0,0 +1,279 @@
+{ config, lib, pkgs, ... }:
+
+# PIA VPN multi-container module.
+#
+# Architecture:
+#   Host creates WG interface, runs tinyproxy on bridge for PIA API bootstrap.
+#   VPN container does all PIA logic via proxy, configures WG, masquerades bridge→piaw.
+#   Service containers default route → VPN container (leak-proof by topology).
+#
+# Reference: https://www.wireguard.com/netns/#ordinary-containerization
+
+with lib;
+
+let
+  cfg = config.pia-vpn;
+
+  # Derive prefix length from subnet CIDR (e.g. "10.100.0.0/24" → "24")
+  subnetPrefixLen = last (splitString "/" cfg.subnet);
+
+  containerSubmodule = types.submodule ({ name, ... }: {
+    options = {
+      ip = mkOption {
+        type = types.str;
+        description = "Static IP address for this container on the VPN bridge";
+      };
+
+      config = mkOption {
+        type = types.anything;
+        default = { };
+        description = "NixOS configuration for this container";
+      };
+
+      mounts = mkOption {
+        type = types.attrsOf (types.submodule {
+          options = {
+            hostPath = mkOption {
+              type = types.str;
+              description = "Path on the host to bind mount";
+            };
+            isReadOnly = mkOption {
+              type = types.bool;
+              default = false;
+              description = "Whether the mount is read-only";
+            };
+          };
+        });
+        default = { };
+        description = "Bind mounts for the container";
+      };
+
+      receiveForwardedPort = mkOption {
+        type = types.nullOr (types.submodule {
+          options = {
+            port = mkOption {
+              type = types.nullOr types.port;
+              default = null;
+              description = ''
+                Target port to forward to. If null, forwards to the same PIA-assigned port.
+                PIA-assigned ports below 10000 are rejected to avoid accidentally
+                forwarding traffic to other services.
+              '';
+            };
+            protocol = mkOption {
+              type = types.enum [ "tcp" "udp" "both" ];
+              default = "both";
+              description = "Protocol(s) to forward";
+            };
+          };
+        });
+        default = null;
+        description = "Port forwarding configuration. At most one container may set this.";
+      };
+
+      onPortForwarded = mkOption {
+        type = types.nullOr types.lines;
+        default = null;
+        description = ''
+          Optional script run in the VPN container after port forwarding is established.
+          Available environment variables: $PORT (PIA-assigned port), $TARGET_IP (this container's IP).
+        '';
+      };
+    };
+  });
+
+  # NOTE: All derivations of cfg.containers are kept INSIDE config = mkIf ... { }
+  # to avoid infinite recursion. The module system's pushDownProperties eagerly
+  # evaluates let bindings and mkMerge contents, so any top-level let binding
+  # that touches cfg.containers would force config evaluation during structure
+  # discovery, creating a cycle.
+in
+{
+  imports = [
+    ./vpn-container.nix
+    ./service-container.nix
+  ];
+
+  options.pia-vpn = {
+    enable = mkEnableOption "PIA VPN multi-container setup";
+
+    serverLocation = mkOption {
+      type = types.str;
+      default = "swiss";
+      description = "PIA server region ID";
+    };
+
+    interfaceName = mkOption {
+      type = types.str;
+      default = "piaw";
+      description = "WireGuard interface name";
+    };
+
+    wireguardListenPort = mkOption {
+      type = types.port;
+      default = 51820;
+      description = "WireGuard listen port";
+    };
+
+    bridgeName = mkOption {
+      type = types.str;
+      default = "br-vpn";
+      description = "Bridge interface name for VPN containers";
+    };
+
+    subnet = mkOption {
+      type = types.str;
+      default = "10.100.0.0/24";
+      description = "Subnet CIDR for VPN bridge network";
+    };
+
+    hostAddress = mkOption {
+      type = types.str;
+      default = "10.100.0.1";
+      description = "Host IP on the VPN bridge";
+    };
+
+    vpnAddress = mkOption {
+      type = types.str;
+      default = "10.100.0.2";
+      description = "VPN container IP on the bridge";
+    };
+
+    proxyPort = mkOption {
+      type = types.port;
+      default = 8888;
+      description = "Tinyproxy port for PIA API bootstrap";
+    };
+
+    containers = mkOption {
+      type = types.attrsOf containerSubmodule;
+      default = { };
+      description = "Service containers that route through the VPN";
+    };
+
+    # Subnet prefix length derived from cfg.subnet (exposed for other submodules)
+    subnetPrefixLen = mkOption {
+      type = types.str;
+      default = subnetPrefixLen;
+      description = "Prefix length derived from subnet CIDR";
+      readOnly = true;
+    };
+
+    # Shared host entries for all containers (host + VPN + service containers)
+    containerHosts = mkOption {
+      type = types.attrsOf (types.listOf types.str);
+      internal = true;
+      readOnly = true;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions =
+      let
+        forwardingContainers = filterAttrs (_: c: c.receiveForwardedPort != null) cfg.containers;
+        containerIPs = mapAttrsToList (_: c: c.ip) cfg.containers;
+      in
+      [
+        {
+          assertion = length (attrNames forwardingContainers) <= 1;
+          message = "At most one pia-vpn container may set receiveForwardedPort";
+        }
+        {
+          assertion = length containerIPs == length (unique containerIPs);
+          message = "pia-vpn container IPs must be unique";
+        }
+      ];
+
+    # Enable systemd-networkd for bridge management
+    systemd.network.enable = true;
+
+    # TODO: re-enable once primary networking uses networkd
+    systemd.network.wait-online.enable = false;
+
+    # Tell NetworkManager to ignore VPN bridge and container interfaces
+    networking.networkmanager.unmanaged = mkIf config.networking.networkmanager.enable [
+      "interface-name:${cfg.bridgeName}"
+      "interface-name:ve-*"
+    ];
+
+    # Bridge network device
+    systemd.network.netdevs."20-${cfg.bridgeName}".netdevConfig = {
+      Kind = "bridge";
+      Name = cfg.bridgeName;
+    };
+
+    # Bridge network configuration — NO IPMasquerade (host must NOT be gateway)
+    systemd.network.networks."20-${cfg.bridgeName}" = {
+      matchConfig.Name = cfg.bridgeName;
+      networkConfig = {
+        Address = "${cfg.hostAddress}/${cfg.subnetPrefixLen}";
+        DHCPServer = false;
+      };
+      linkConfig.RequiredForOnline = "no";
+    };
+
+    # Allow wireguard traffic through rpfilter
+    networking.firewall.checkReversePath = "loose";
+
+    # Block bridge → outside forwarding (prevents host from being a gateway for containers)
+    networking.firewall.extraForwardRules = ''
+      iifname "${cfg.bridgeName}" oifname != "${cfg.bridgeName}" drop
+    '';
+
+    # Allow tinyproxy from bridge (tinyproxy itself restricts to VPN container IP)
+    networking.firewall.interfaces.${cfg.bridgeName}.allowedTCPPorts = [ cfg.proxyPort ];
+
+    # Tinyproxy — runs on bridge IP so VPN container can bootstrap PIA auth
+    services.tinyproxy = {
+      enable = true;
+      settings = {
+        Listen = cfg.hostAddress;
+        Port = cfg.proxyPort;
+      };
+    };
+    systemd.services.tinyproxy.before = [ "container@pia-vpn.service" ];
+
+    # WireGuard interface creation (host-side oneshot)
+    # Creates the interface in the host namespace so encrypted UDP stays in host netns.
+    # The container takes ownership of the interface on startup via `interfaces = [ ... ]`.
+    systemd.services.pia-vpn-wg-create = {
+      description = "Create PIA VPN WireGuard interface";
+
+      before = [ "container@pia-vpn.service" ];
+      requiredBy = [ "container@pia-vpn.service" ];
+      partOf = [ "container@pia-vpn.service" ];
+      wantedBy = [ "multi-user.target" ];
+
+      path = with pkgs; [ iproute2 ];
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      script = ''
+        [[ -z $(ip link show dev ${cfg.interfaceName} 2>/dev/null) ]] || exit 0
+        ip link add ${cfg.interfaceName} type wireguard
+      '';
+
+      preStop = ''
+        ip link del ${cfg.interfaceName} 2>/dev/null || true
+      '';
+    };
+
+    # Host entries for container hostnames — NixOS only auto-creates these for
+    # hostAddress/localAddress containers, not hostBridge. Use the standard
+    # {name}.containers convention.
+    pia-vpn.containerHosts =
+      { ${cfg.vpnAddress} = [ "pia-vpn.containers" ]; }
+      // mapAttrs' (name: ctr: nameValuePair ctr.ip [ "${name}.containers" ]) cfg.containers;
+
+    networking.hosts = cfg.containerHosts;
+
+    # PIA login secret
+    age.secrets."pia-login.conf".file = ../../../secrets/pia-login.age;
+
+    # IP forwarding needed for bridge traffic between containers
+    networking.ip_forward = true;
+  };
+}

common/network/pia-vpn/scripts.nix

@@ -0,0 +1,209 @@
+let
+  caPath = ./ca.rsa.4096.crt;
+in
+
+# Bash function library for PIA VPN WireGuard operations.
+  # All PIA API calls accept an optional $proxy variable:
+  #   proxy="http://10.100.0.1:8888" fetchPIAToken
+  # When $proxy is set, curl uses --proxy "$proxy"; otherwise direct connection.
+
+  # Reference materials:
+  #   https://serverlist.piaservers.net/vpninfo/servers/v6
+  #   https://github.com/pia-foss/manual-connections
+  #   https://github.com/thrnz/docker-wireguard-pia/blob/master/extra/wg-gen.sh
+  #   https://www.wireguard.com/netns/#ordinary-containerization
+
+{
+  scriptCommon = ''
+    proxy_args() {
+      if [[ -n "''${proxy:-}" ]]; then
+        echo "--proxy $proxy"
+      fi
+    }
+
+    fetchPIAToken() {
+      local PIA_USER PIA_PASS resp
+      echo "Reading PIA credentials..."
+      PIA_USER=$(sed '1q;d' /run/agenix/pia-login.conf)
+      PIA_PASS=$(sed '2q;d' /run/agenix/pia-login.conf)
+      echo "Requesting PIA authentication token..."
+      resp=$(curl -s $(proxy_args) -u "$PIA_USER:$PIA_PASS" \
+        "https://www.privateinternetaccess.com/gtoken/generateToken")
+      PIA_TOKEN=$(echo "$resp" | jq -r '.token')
+      if [[ -z "$PIA_TOKEN" || "$PIA_TOKEN" == "null" ]]; then
+        echo "ERROR: Failed to fetch PIA token: $resp" >&2
+        return 1
+      fi
+      echo "PIA token acquired"
+    }
+
+    choosePIAServer() {
+      local serverLocation=$1
+      local servers servers_json totalservers serverindex
+      servers=$(mktemp)
+      servers_json=$(mktemp)
+      echo "Fetching PIA server list..."
+      curl -s $(proxy_args) \
+        "https://serverlist.piaservers.net/vpninfo/servers/v6" > "$servers"
+      head -n 1 "$servers" | tr -d '\n' > "$servers_json"
+
+      totalservers=$(jq -r \
+        '.regions | .[] | select(.id=="'"$serverLocation"'") | .servers.wg | length' \
+        "$servers_json")
+      if ! [[ "$totalservers" =~ ^[0-9]+$ ]] || [ "$totalservers" -eq 0 ] 2>/dev/null; then
+        echo "ERROR: Location \"$serverLocation\" not found." >&2
+        rm -f "$servers_json" "$servers"
+        return 1
+      fi
+      echo "Found $totalservers WireGuard servers in region '$serverLocation'"
+      serverindex=$(( RANDOM % totalservers ))
+
+      WG_HOSTNAME=$(jq -r \
+        '.regions | .[] | select(.id=="'"$serverLocation"'") | .servers.wg | .['"$serverindex"'].cn' \
+        "$servers_json")
+      WG_SERVER_IP=$(jq -r \
+        '.regions | .[] | select(.id=="'"$serverLocation"'") | .servers.wg | .['"$serverindex"'].ip' \
+        "$servers_json")
+      WG_SERVER_PORT=$(jq -r '.groups.wg | .[0] | .ports | .[0]' "$servers_json")
+
+      rm -f "$servers_json" "$servers"
+      echo "Selected server $serverindex/$totalservers: $WG_HOSTNAME ($WG_SERVER_IP:$WG_SERVER_PORT)"
+    }
+
+    generateWireguardKey() {
+      PRIVATE_KEY=$(wg genkey)
+      PUBLIC_KEY=$(echo "$PRIVATE_KEY" | wg pubkey)
+      echo "Generated WireGuard keypair"
+    }
+
+    authorizeKeyWithPIAServer() {
+      local addKeyResponse
+      echo "Sending addKey request to $WG_HOSTNAME ($WG_SERVER_IP:$WG_SERVER_PORT)..."
+      addKeyResponse=$(curl -s -G $(proxy_args) \
+        --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" \
+        --cacert "${caPath}" \
+        --data-urlencode "pt=$PIA_TOKEN" \
+        --data-urlencode "pubkey=$PUBLIC_KEY" \
+        "https://$WG_HOSTNAME:$WG_SERVER_PORT/addKey")
+      local status
+      status=$(echo "$addKeyResponse" | jq -r '.status')
+      if [[ "$status" != "OK" ]]; then
+        echo "ERROR: addKey failed: $addKeyResponse" >&2
+        return 1
+      fi
+      MY_IP=$(echo "$addKeyResponse" | jq -r '.peer_ip')
+      WG_SERVER_PUBLIC_KEY=$(echo "$addKeyResponse" | jq -r '.server_key')
+      WG_SERVER_PORT=$(echo "$addKeyResponse" | jq -r '.server_port')
+      echo "Key authorized — assigned VPN IP: $MY_IP, server port: $WG_SERVER_PORT"
+    }
+
+    writeWireguardQuickFile() {
+      local wgFile=$1
+      local listenPort=$2
+      rm -f "$wgFile"
+      touch "$wgFile"
+      chmod 700 "$wgFile"
+      cat > "$wgFile" <<WGEOF
+    [Interface]
+    PrivateKey = $PRIVATE_KEY
+    ListenPort = $listenPort
+    [Peer]
+    PersistentKeepalive = 25
+    PublicKey = $WG_SERVER_PUBLIC_KEY
+    AllowedIPs = 0.0.0.0/0
+    Endpoint = $WG_SERVER_IP:$WG_SERVER_PORT
+    WGEOF
+      echo "Wrote WireGuard config to $wgFile (listen=$listenPort)"
+    }
+
+    writeChosenServerToFile() {
+      local serverFile=$1
+      jq -n \
+        --arg hostname "$WG_HOSTNAME" \
+        --arg ip "$WG_SERVER_IP" \
+        --arg port "$WG_SERVER_PORT" \
+        '{hostname: $hostname, ip: $ip, port: $port}' > "$serverFile"
+      chmod 700 "$serverFile"
+      echo "Wrote server info to $serverFile"
+    }
+
+    loadChosenServerFromFile() {
+      local serverFile=$1
+      WG_HOSTNAME=$(jq -r '.hostname' "$serverFile")
+      WG_SERVER_IP=$(jq -r '.ip' "$serverFile")
+      WG_SERVER_PORT=$(jq -r '.port' "$serverFile")
+      echo "Loaded server info from $serverFile: $WG_HOSTNAME ($WG_SERVER_IP:$WG_SERVER_PORT)"
+    }
+
+    # Reset WG interface and tear down NAT/forwarding rules.
+    # Called on startup (clear stale state) and on exit via trap.
+    cleanupVpn() {
+      local interfaceName=$1
+      wg set "$interfaceName" listen-port 0 2>/dev/null || true
+      ip -4 address flush dev "$interfaceName" 2>/dev/null || true
+      ip route del default dev "$interfaceName" 2>/dev/null || true
+      iptables -t nat -F 2>/dev/null || true
+      iptables -F FORWARD 2>/dev/null || true
+    }
+
+    connectToServer() {
+      local wgFile=$1
+      local interfaceName=$2
+
+      echo "Applying WireGuard config to $interfaceName..."
+      wg setconf "$interfaceName" "$wgFile"
+      ip -4 address add "$MY_IP" dev "$interfaceName"
+      ip link set mtu 1420 up dev "$interfaceName"
+      echo "WireGuard interface $interfaceName is up with IP $MY_IP"
+    }
+
+    reservePortForward() {
+      local payload_and_signature
+      echo "Requesting port forward signature from $WG_HOSTNAME..."
+      payload_and_signature=$(curl -s -m 5 \
+        --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" \
+        --cacert "${caPath}" \
+        -G --data-urlencode "token=$PIA_TOKEN" \
+        "https://$WG_HOSTNAME:19999/getSignature")
+      local status
+      status=$(echo "$payload_and_signature" | jq -r '.status')
+      if [[ "$status" != "OK" ]]; then
+        echo "ERROR: getSignature failed: $payload_and_signature" >&2
+        return 1
+      fi
+      PORT_SIGNATURE=$(echo "$payload_and_signature" | jq -r '.signature')
+      PORT_PAYLOAD=$(echo "$payload_and_signature" | jq -r '.payload')
+      PORT=$(echo "$PORT_PAYLOAD" | base64 -d | jq -r '.port')
+      echo "Port forward reserved: port $PORT"
+    }
+
+    writePortRenewalFile() {
+      local portRenewalFile=$1
+      jq -n \
+        --arg signature "$PORT_SIGNATURE" \
+        --arg payload "$PORT_PAYLOAD" \
+        '{signature: $signature, payload: $payload}' > "$portRenewalFile"
+      chmod 700 "$portRenewalFile"
+      echo "Wrote port renewal data to $portRenewalFile"
+    }
+
+    readPortRenewalFile() {
+      local portRenewalFile=$1
+      PORT_SIGNATURE=$(jq -r '.signature' "$portRenewalFile")
+      PORT_PAYLOAD=$(jq -r '.payload' "$portRenewalFile")
+      echo "Loaded port renewal data from $portRenewalFile"
+    }
+
+    refreshPIAPort() {
+      local bindPortResponse
+      echo "Refreshing port forward binding with $WG_HOSTNAME..."
+      bindPortResponse=$(curl -Gs -m 5 \
+        --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" \
+        --cacert "${caPath}" \
+        --data-urlencode "payload=$PORT_PAYLOAD" \
+        --data-urlencode "signature=$PORT_SIGNATURE" \
+        "https://$WG_HOSTNAME:19999/bindPort")
+      echo "bindPort response: $bindPortResponse"
+    }
+  '';
+}

common/network/pia-vpn/service-container.nix

@@ -0,0 +1,87 @@
+{ config, lib, allModules, ... }:
+
+# Generates service containers that route all traffic through the VPN container.
+# Each container gets a static IP on the VPN bridge with default route → VPN container.
+#
+# Uses lazy mapAttrs inside fixed config keys to avoid infinite recursion.
+# (mkMerge + mapAttrsToList at the top level forces eager evaluation of cfg.containers
+# during module structure discovery, which creates a cycle with config evaluation.)
+
+with lib;
+
+let
+  cfg = config.pia-vpn;
+
+  mkContainer = name: ctr: {
+    autoStart = true;
+    ephemeral = true;
+    privateNetwork = true;
+    hostBridge = cfg.bridgeName;
+
+    bindMounts = mapAttrs
+      (_: mount: {
+        hostPath = mount.hostPath;
+        isReadOnly = mount.isReadOnly;
+      })
+      ctr.mounts;
+
+    config = { config, pkgs, lib, ... }: {
+      imports = allModules ++ [ ctr.config ];
+
+      # Static IP with gateway pointing to VPN container
+      networking.useNetworkd = true;
+      systemd.network.enable = true;
+      networking.useDHCP = false;
+
+      systemd.network.networks."20-eth0" = {
+        matchConfig.Name = "eth0";
+        networkConfig = {
+          Address = "${ctr.ip}/${cfg.subnetPrefixLen}";
+          Gateway = cfg.vpnAddress;
+          DNS = [ cfg.vpnAddress ];
+        };
+      };
+
+      networking.hosts = cfg.containerHosts;
+
+      # DNS through VPN container (queries go through WG tunnel = no DNS leak)
+      networking.nameservers = [ cfg.vpnAddress ];
+
+      # Wait for actual VPN connectivity before network-online.target.
+      # Without this, services start before the VPN tunnel is ready and failures
+      # can't be reported to ntfy (no outbound connectivity yet).
+      systemd.services.wait-for-vpn = {
+        description = "Wait for VPN connectivity";
+        before = [ "network-online.target" ];
+        wantedBy = [ "network-online.target" ];
+        after = [ "systemd-networkd-wait-online.service" ];
+        serviceConfig.Type = "oneshot";
+        path = [ pkgs.iputils ];
+        script = ''
+          until ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; do
+            echo "Waiting for VPN connectivity..."
+            sleep 1
+          done
+        '';
+      };
+
+      # Trust the bridge interface (host reaches us directly for nginx)
+      networking.firewall.trustedInterfaces = [ "eth0" ];
+
+      # Disable host resolv.conf — we use our own networkd DNS config
+      networking.useHostResolvConf = false;
+    };
+  };
+
+  mkContainerOrdering = name: _ctr: nameValuePair "container@${name}" {
+    after = [ "container@pia-vpn.service" ];
+    requires = [ "container@pia-vpn.service" ];
+    partOf = [ "container@pia-vpn.service" ];
+  };
+in
+{
+  config = mkIf cfg.enable {
+    containers = mapAttrs mkContainer cfg.containers;
+    systemd.services = mapAttrs' mkContainerOrdering cfg.containers;
+  };
+}

common/network/pia-vpn/vpn-container.nix

@@ -0,0 +1,232 @@
+{ config, lib, allModules, ... }:
+
+# VPN container: runs all PIA logic, acts as WireGuard gateway + NAT for service containers.
+
+with lib;
+
+let
+  cfg = config.pia-vpn;
+  scripts = import ./scripts.nix;
+
+  # Port forwarding derived state
+  forwardingContainers = filterAttrs (_: c: c.receiveForwardedPort != null) cfg.containers;
+  portForwarding = forwardingContainers != { };
+  forwardingContainerName = if portForwarding then head (attrNames forwardingContainers) else null;
+  forwardingContainer = if portForwarding then forwardingContainers.${forwardingContainerName} else null;
+
+  serverFile = "/var/lib/pia-vpn/server.json";
+  wgFile = "/var/lib/pia-vpn/wg.conf";
+  portRenewalFile = "/var/lib/pia-vpn/port-renewal.json";
+  proxy = "http://${cfg.hostAddress}:${toString cfg.proxyPort}";
+
+  # DNAT/forwarding rules for port forwarding
+  dnatSetupScript = optionalString portForwarding (
+    let
+      fwd = forwardingContainer.receiveForwardedPort;
+      targetIp = forwardingContainer.ip;
+      dnatTarget = if fwd.port != null then "${targetIp}:${toString fwd.port}" else targetIp;
+      targetPort = if fwd.port != null then toString fwd.port else "$PORT";
+      tcpRules = optionalString (fwd.protocol == "tcp" || fwd.protocol == "both") ''
+        echo "Setting up TCP DNAT: port $PORT → ${targetIp}:${targetPort}"
+        iptables -t nat -A PREROUTING -i ${cfg.interfaceName} -p tcp --dport $PORT -j DNAT --to ${dnatTarget}
+        iptables -A FORWARD -i ${cfg.interfaceName} -d ${targetIp} -p tcp --dport ${targetPort} -j ACCEPT
+      '';
+      udpRules = optionalString (fwd.protocol == "udp" || fwd.protocol == "both") ''
+        echo "Setting up UDP DNAT: port $PORT → ${targetIp}:${targetPort}"
+        iptables -t nat -A PREROUTING -i ${cfg.interfaceName} -p udp --dport $PORT -j DNAT --to ${dnatTarget}
+        iptables -A FORWARD -i ${cfg.interfaceName} -d ${targetIp} -p udp --dport ${targetPort} -j ACCEPT
+      '';
+      onPortForwarded = optionalString (forwardingContainer.onPortForwarded != null) ''
+        TARGET_IP="${targetIp}"
+        export PORT TARGET_IP
+        echo "Running onPortForwarded hook for ${forwardingContainerName} (port=$PORT, target=$TARGET_IP)"
+        ${forwardingContainer.onPortForwarded}
+      '';
+    in
+    ''
+      if [ "$PORT" -lt 10000 ]; then
+        echo "ERROR: PIA assigned low port $PORT (< 10000), refusing to set up DNAT" >&2
+      else
+        ${tcpRules}
+        ${udpRules}
+        ${onPortForwarded}
+      fi
+    ''
+  );
+in
+{
+  config = mkIf cfg.enable {
+    # Give the container more time to boot (pia-vpn-setup retries can delay readiness)
+    systemd.services."container@pia-vpn".serviceConfig.TimeoutStartSec = mkForce "180s";
+
+    containers.pia-vpn = {
+      autoStart = true;
+      ephemeral = true;
+      privateNetwork = true;
+      hostBridge = cfg.bridgeName;
+      interfaces = [ cfg.interfaceName ];
+
+      bindMounts."/run/agenix" = {
+        hostPath = "/run/agenix";
+        isReadOnly = true;
+      };
+
+      config = { config, pkgs, lib, ... }:
+        let
+          scriptPkgs = with pkgs; [ wireguard-tools iproute2 curl jq iptables coreutils ];
+        in
+        {
+          imports = allModules;
+
+          networking.hosts = cfg.containerHosts;
+
+          # Static IP on bridge — no gateway (VPN container routes via WG only)
+          networking.useNetworkd = true;
+          systemd.network.enable = true;
+          networking.useDHCP = false;
+
+          systemd.network.networks."20-eth0" = {
+            matchConfig.Name = "eth0";
+            networkConfig = {
+              Address = "${cfg.vpnAddress}/${cfg.subnetPrefixLen}";
+              DHCPServer = false;
+            };
+          };
+
+          # Ignore WG interface for wait-online (it's configured manually, not by networkd)
+          systemd.network.wait-online.ignoredInterfaces = [ cfg.interfaceName ];
+
+          # Route ntfy alerts through the host proxy (VPN container has no gateway on eth0)
+          ntfy-alerts.curlExtraArgs = "--proxy http://${cfg.hostAddress}:${toString cfg.proxyPort}";
+
+          # Enable forwarding so bridge traffic can go through WG
+          boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
+
+          # Trust bridge interface
+          networking.firewall.trustedInterfaces = [ "eth0" ];
+
+          # DNS: use systemd-resolved listening on bridge IP so service containers
+          # can use VPN container as DNS server (queries go through WG tunnel = no DNS leak)
+          services.resolved = {
+            enable = true;
+            settings.Resolve.DNSStubListenerExtra = cfg.vpnAddress;
+          };
+
+          # Don't use host resolv.conf — resolved manages DNS
+          networking.useHostResolvConf = false;
+
+          # State directory for PIA config files
+          systemd.tmpfiles.rules = [
+            "d /var/lib/pia-vpn 0700 root root -"
+          ];
+
+          # PIA VPN setup service — does all the PIA auth, WG config, and NAT setup
+          systemd.services.pia-vpn-setup = {
+            description = "PIA VPN WireGuard Setup";
+
+            wants = [ "network-online.target" ];
+            after = [ "network.target" "network-online.target" "systemd-networkd.service" ];
+            wantedBy = [ "multi-user.target" ];
+
+            path = scriptPkgs;
+
+            serviceConfig = {
+              Type = "simple";
+              Restart = "always";
+              RestartSec = "10s";
+              RuntimeMaxSec = "30d";
+            };
+
+            script = ''
+              set -euo pipefail
+              ${scripts.scriptCommon}
+
+              trap 'cleanupVpn ${cfg.interfaceName}' EXIT
+              cleanupVpn ${cfg.interfaceName}
+
+              proxy="${proxy}"
+
+              # 1. Authenticate with PIA via proxy (VPN container has no internet yet)
+              echo "Choosing PIA server in region '${cfg.serverLocation}'..."
+              choosePIAServer '${cfg.serverLocation}'
+
+              echo "Fetching PIA authentication token..."
+              fetchPIAToken
+
+              # 2. Generate WG keys and authorize with PIA server
+              echo "Generating WireGuard keypair..."
+              generateWireguardKey
+
+              echo "Authorizing key with PIA server $WG_HOSTNAME..."
+              authorizeKeyWithPIAServer
+
+              # 3. Configure WG interface (already created by host and moved into our namespace)
+              echo "Configuring WireGuard interface ${cfg.interfaceName}..."
+              writeWireguardQuickFile '${wgFile}' ${toString cfg.wireguardListenPort}
+              writeChosenServerToFile '${serverFile}'
+              connectToServer '${wgFile}' '${cfg.interfaceName}'
+
+              # 4. Default route through WG
+              ip route replace default dev ${cfg.interfaceName}
+              echo "Default route set through ${cfg.interfaceName}"
+
+              # 5. NAT: masquerade bridge → WG (so service containers' traffic appears to come from VPN IP)
+              echo "Setting up NAT masquerade..."
+              iptables -t nat -A POSTROUTING -o ${cfg.interfaceName} -j MASQUERADE
+              iptables -A FORWARD -i eth0 -o ${cfg.interfaceName} -j ACCEPT
+              iptables -A FORWARD -i ${cfg.interfaceName} -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT
+
+              ${optionalString portForwarding ''
+                # 6. Port forwarding setup
+                echo "Reserving port forward..."
+                reservePortForward
+                writePortRenewalFile '${portRenewalFile}'
+
+                # First bindPort triggers actual port allocation
+                echo "Binding port $PORT..."
+                refreshPIAPort
+
+                echo "PIA assigned port: $PORT"
+
+                # DNAT rules to forward PIA port to target container
+                ${dnatSetupScript}
+              ''}
+
+              echo "PIA VPN setup complete"
+              exec sleep infinity
+            '';
+
+          };
+
+          # Port refresh timer (every 10 min) — keeps PIA port forwarding alive
+          systemd.services.pia-vpn-port-refresh = mkIf portForwarding {
+            description = "PIA VPN Port Forward Refresh";
+            after = [ "pia-vpn-setup.service" ];
+            requires = [ "pia-vpn-setup.service" ];
+
+            path = scriptPkgs;
+
+            serviceConfig.Type = "oneshot";
+
+            script = ''
+              set -euo pipefail
+              ${scripts.scriptCommon}
+              loadChosenServerFromFile '${serverFile}'
+              readPortRenewalFile '${portRenewalFile}'
+              echo "Refreshing PIA port forward..."
+              refreshPIAPort
+            '';
+          };
+
+          systemd.timers.pia-vpn-port-refresh = mkIf portForwarding {
+            partOf = [ "pia-vpn-port-refresh.service" ];
+            wantedBy = [ "timers.target" ];
+            timerConfig = {
+              OnCalendar = "*:0/10";
+              RandomizedDelaySec = "1m";
+            };
+          };
+        };
+    };
+  };
+}

common/network/pia-wireguard.nix

@@ -1,363 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-# Server list:
-#   https://serverlist.piaservers.net/vpninfo/servers/v6
-# Reference materials:
-#   https://github.com/pia-foss/manual-connections
-#   https://github.com/thrnz/docker-wireguard-pia/blob/master/extra/wg-gen.sh
-
-# TODO handle potential errors (or at least print status, success, and failures to the console)
-# TODO parameterize names of systemd services so that multiple wg VPNs could coexist in theory easier
-# TODO implement this module such that the wireguard VPN doesn't have to live in a container
-# TODO don't add forward rules if the PIA port is the same as cfg.forwardedPort
-# TODO verify signatures of PIA responses
-# TODO `RuntimeMaxSec = "30d";` for pia-vpn-wireguard-init isn't allowed per the systemd logs. Find alternative.
-
-with builtins;
-with lib;
-
-let
-  cfg = config.pia.wireguard;
-
-  getPIAToken = ''
-    PIA_USER=`sed '1q;d' /run/agenix/pia-login.conf`
-    PIA_PASS=`sed '2q;d' /run/agenix/pia-login.conf`
-    # PIA_TOKEN only lasts 24hrs
-    PIA_TOKEN=`curl -s -u "$PIA_USER:$PIA_PASS" https://www.privateinternetaccess.com/gtoken/generateToken | jq -r '.token'`
-  '';
-
-  chooseWireguardServer = ''
-    servers=$(mktemp)
-    servers_json=$(mktemp)
-    curl -s "https://serverlist.piaservers.net/vpninfo/servers/v6" > "$servers"
-    # extract json part only
-    head -n 1 "$servers" | tr -d '\n' > "$servers_json"
-
-    echo "Available location ids:" && jq '.regions | .[] | {name, id, port_forward}' "$servers_json"
-
-    # Some locations have multiple servers available. Pick a random one.
-    totalservers=$(jq -r '.regions | .[] | select(.id=="'${cfg.serverLocation}'") | .servers.wg | length' "$servers_json")
-    if ! [[ "$totalservers" =~ ^[0-9]+$ ]] || [ "$totalservers" -eq 0 ] 2>/dev/null; then
-      echo "Location \"${cfg.serverLocation}\" not found."
-      exit 1
-    fi
-    serverindex=$(( RANDOM % totalservers))
-    WG_HOSTNAME=$(jq -r '.regions | .[] | select(.id=="'${cfg.serverLocation}'") | .servers.wg | .['$serverindex'].cn' "$servers_json")
-    WG_SERVER_IP=$(jq -r '.regions | .[] | select(.id=="'${cfg.serverLocation}'") | .servers.wg | .['$serverindex'].ip' "$servers_json")
-    WG_SERVER_PORT=$(jq -r '.groups.wg | .[0] | .ports | .[0]' "$servers_json")
-
-    # write chosen server
-    rm -f /tmp/${cfg.interfaceName}-server.conf
-    touch /tmp/${cfg.interfaceName}-server.conf
-    chmod 700 /tmp/${cfg.interfaceName}-server.conf
-    echo "$WG_HOSTNAME" >> /tmp/${cfg.interfaceName}-server.conf
-    echo "$WG_SERVER_IP" >> /tmp/${cfg.interfaceName}-server.conf
-    echo "$WG_SERVER_PORT" >> /tmp/${cfg.interfaceName}-server.conf
-
-    rm $servers_json $servers
-  '';
-
-  getChosenWireguardServer = ''
-    WG_HOSTNAME=`sed '1q;d' /tmp/${cfg.interfaceName}-server.conf`
-    WG_SERVER_IP=`sed '2q;d' /tmp/${cfg.interfaceName}-server.conf`
-    WG_SERVER_PORT=`sed '3q;d' /tmp/${cfg.interfaceName}-server.conf`
-  '';
-
-  refreshPIAPort = ''
-    ${getChosenWireguardServer}
-    signature=`sed '1q;d' /tmp/${cfg.interfaceName}-port-renewal`
-    payload=`sed '2q;d' /tmp/${cfg.interfaceName}-port-renewal`
-    bind_port_response=`curl -Gs -m 5 --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" --cacert "${./ca.rsa.4096.crt}" --data-urlencode "payload=$payload" --data-urlencode "signature=$signature" "https://$WG_HOSTNAME:19999/bindPort"`
-  '';
-
-  portForwarding = cfg.forwardPortForTransmission || cfg.forwardedPort != null;
-
-  containerServiceName = "container@${config.vpn-container.containerName}.service";
-in
-{
-  options.pia.wireguard = {
-    enable = mkEnableOption "Enable private internet access";
-    badPortForwardPorts = mkOption {
-      type = types.listOf types.port;
-      description = ''
-        Ports that will not be accepted from PIA.
-        If PIA assigns a port from this list, the connection is aborted since we cannot ask for a different port.
-        This is used to guarantee we are not assigned a port that is used by a service we do not want exposed.
-      '';
-    };
-    wireguardListenPort = mkOption {
-      type = types.port;
-      description = "The port wireguard listens on for this VPN connection";
-      default = 51820;
-    };
-    serverLocation = mkOption {
-      type = types.str;
-      default = "swiss";
-    };
-    interfaceName = mkOption {
-      type = types.str;
-      default = "piaw";
-    };
-    forwardedPort = mkOption {
-      type = types.nullOr types.port;
-      description = "The port to redirect port forwarded TCP VPN traffic too";
-      default = null;
-    };
-    forwardPortForTransmission = mkEnableOption "PIA port forwarding for transmission should be performed.";
-  };
-
-  config = mkIf cfg.enable {
-    assertions = [
-      {
-        assertion = cfg.forwardPortForTransmission != (cfg.forwardedPort != null);
-        message = ''
-          The PIA forwarded port cannot simultaneously be used by transmission and redirected to another port.
-        '';
-      }
-    ];
-
-    # mounts used to pass the connection parameters to the container
-    # the container doesn't have internet until it uses these parameters so it cannot fetch them itself
-    vpn-container.mounts = [
-      "/tmp/${cfg.interfaceName}.conf"
-      "/tmp/${cfg.interfaceName}-server.conf"
-      "/tmp/${cfg.interfaceName}-address.conf"
-    ];
-
-    # The container takes ownership of the wireguard interface on its startup
-    containers.vpn.interfaces = [ cfg.interfaceName ];
-
-    # TODO: while this is much better than "loose" networking, it seems to have issues with firewall restarts
-    # allow traffic for wireguard interface to pass since wireguard trips up rpfilter
-    # networking.firewall = {
-    #   extraCommands = ''
-    #     ip46tables -t raw -I nixos-fw-rpfilter -p udp -m udp --sport ${toString cfg.wireguardListenPort} -j RETURN
-    #     ip46tables -t raw -I nixos-fw-rpfilter -p udp -m udp --dport ${toString cfg.wireguardListenPort} -j RETURN
-    #   '';
-    #   extraStopCommands = ''
-    #     ip46tables -t raw -D nixos-fw-rpfilter -p udp -m udp --sport ${toString cfg.wireguardListenPort} -j RETURN || true
-    #     ip46tables -t raw -D nixos-fw-rpfilter -p udp -m udp --dport ${toString cfg.wireguardListenPort} -j RETURN || true
-    #   '';
-    # };
-    networking.firewall.checkReversePath = "loose";
-
-    systemd.services.pia-vpn-wireguard-init = {
-      description = "Creates PIA VPN Wireguard Interface";
-
-      wants = [ "network-online.target" ];
-      after = [ "network.target" "network-online.target" ];
-      before = [ containerServiceName ];
-      requiredBy = [ containerServiceName ];
-      partOf = [ containerServiceName ];
-      wantedBy = [ "multi-user.target" ];
-
-      path = with pkgs; [ wireguard-tools jq curl iproute2 iputils ];
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-
-        # restart once a month; PIA forwarded port expires after two months
-        # because the container is "PartOf" this unit, it gets restarted too
-        RuntimeMaxSec = "30d";
-      };
-
-      script = ''
-        echo Waiting for internet...
-        while ! ping -c 1 -W 1 1.1.1.1; do
-            sleep 1
-        done
-
-        # Prepare to connect by generating wg secrets and auth'ing with PIA since the container
-        # cannot do without internet to start with. NAT'ing the host's internet would address this
-        # issue but is not ideal because then leaking network outside of the VPN is more likely.
-
-        ${chooseWireguardServer}
-
-        ${getPIAToken}
-
-        # generate wireguard keys
-        privKey=$(wg genkey)
-        pubKey=$(echo "$privKey" | wg pubkey)
-
-        # authorize our WG keys with the PIA server we are about to connect to
-        wireguard_json=`curl -s -G --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" --cacert "${./ca.rsa.4096.crt}" --data-urlencode "pt=$PIA_TOKEN" --data-urlencode "pubkey=$pubKey" https://$WG_HOSTNAME:$WG_SERVER_PORT/addKey`
-
-        # create wg-quick config file
-        rm -f /tmp/${cfg.interfaceName}.conf /tmp/${cfg.interfaceName}-address.conf
-        touch /tmp/${cfg.interfaceName}.conf /tmp/${cfg.interfaceName}-address.conf
-        chmod 700 /tmp/${cfg.interfaceName}.conf /tmp/${cfg.interfaceName}-address.conf
-        echo "
-        [Interface]
-        # Address = $(echo "$wireguard_json" | jq -r '.peer_ip')
-        PrivateKey = $privKey
-        ListenPort = ${toString cfg.wireguardListenPort}
-        [Peer]
-        PersistentKeepalive = 25
-        PublicKey = $(echo "$wireguard_json" | jq -r '.server_key')
-        AllowedIPs = 0.0.0.0/0
-        Endpoint = $WG_SERVER_IP:$(echo "$wireguard_json" | jq -r '.server_port')
-        " >> /tmp/${cfg.interfaceName}.conf
-
-        # create file storing the VPN ip address PIA assigned to us
-        echo "$wireguard_json" | jq -r '.peer_ip' >> /tmp/${cfg.interfaceName}-address.conf
-
-        # Create wg interface now so it inherits from the namespace with internet access
-        # the container will handle actually connecting the interface since that info is
-        # not preserved upon moving into the container's networking namespace
-        # Roughly following this guide https://www.wireguard.com/netns/#ordinary-containerization
-        [[ -z $(ip link show dev ${cfg.interfaceName} 2>/dev/null) ]] || exit
-        ip link add ${cfg.interfaceName} type wireguard
-      '';
-
-      preStop = ''
-        # cleanup wireguard interface
-        ip link del ${cfg.interfaceName}
-        rm -f /tmp/${cfg.interfaceName}.conf /tmp/${cfg.interfaceName}-address.conf
-      '';
-    };
-
-    vpn-container.config.systemd.services.pia-vpn-wireguard = {
-      description = "Initializes the PIA VPN WireGuard Tunnel";
-
-      wants = [ "network-online.target" ];
-      after = [ "network.target" "network-online.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      path = with pkgs; [ wireguard-tools iproute2 curl jq iptables ];
-
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-
-      script = ''
-        # pseudo calls wg-quick
-        # Near equivalent of "wg-quick up /tmp/${cfg.interfaceName}.conf"
-        # cannot actually call wg-quick because the interface has to be already
-        # created before the container taken ownership of the interface
-        # Thus, assumes wg interface was already created:
-        #   ip link add ${cfg.interfaceName} type wireguard
-
-        ${getChosenWireguardServer}
-
-        myaddress=`cat /tmp/${cfg.interfaceName}-address.conf`
-
-        wg setconf ${cfg.interfaceName} /tmp/${cfg.interfaceName}.conf
-        ip -4 address add $myaddress dev ${cfg.interfaceName}
-        ip link set mtu 1420 up dev ${cfg.interfaceName}
-        wg set ${cfg.interfaceName} fwmark ${toString cfg.wireguardListenPort}
-        ip -4 route add 0.0.0.0/0 dev ${cfg.interfaceName} table ${toString cfg.wireguardListenPort}
-
-        # TODO is this needed?
-        ip -4 rule add not fwmark ${toString cfg.wireguardListenPort} table ${toString cfg.wireguardListenPort}
-        ip -4 rule add table main suppress_prefixlength 0
-
-        # The rest of the script is only for only for port forwarding skip if not needed
-        if [ ${boolToString portForwarding} == false ]; then exit 0; fi
-
-        # Reserve port
-        ${getPIAToken}
-        payload_and_signature=`curl -s -m 5 --connect-to "$WG_HOSTNAME::$WG_SERVER_IP:" --cacert "${./ca.rsa.4096.crt}" -G --data-urlencode "token=$PIA_TOKEN" "https://$WG_HOSTNAME:19999/getSignature"`
-        signature=$(echo "$payload_and_signature" | jq -r '.signature')
-        payload=$(echo "$payload_and_signature" | jq -r '.payload')
-        port=$(echo "$payload" | base64 -d | jq -r '.port')
-
-        # Check if the port is acceptable
-        notallowed=(${concatStringsSep " " (map toString cfg.badPortForwardPorts)})
-        if [[ " ''${notallowed[*]} " =~ " $port " ]]; then
-          # the port PIA assigned is not allowed, kill the connection
-          wg-quick down /tmp/${cfg.interfaceName}.conf
-          exit 1
-        fi
-
-        # write reserved port to file readable for all users
-        echo $port > /tmp/${cfg.interfaceName}-port
-        chmod 644 /tmp/${cfg.interfaceName}-port
-
-        # write payload and signature info needed to allow refreshing allocated forwarded port
-        rm -f /tmp/${cfg.interfaceName}-port-renewal
-        touch /tmp/${cfg.interfaceName}-port-renewal
-        chmod 700 /tmp/${cfg.interfaceName}-port-renewal
-        echo $signature >> /tmp/${cfg.interfaceName}-port-renewal
-        echo $payload >> /tmp/${cfg.interfaceName}-port-renewal
-
-        # Block all traffic from VPN interface except for traffic that is from the forwarded port
-        iptables -I nixos-fw -p tcp --dport $port -j nixos-fw-accept -i ${cfg.interfaceName}
-        iptables -I nixos-fw -p udp --dport $port -j nixos-fw-accept -i ${cfg.interfaceName}
-
-        # The first port refresh triggers the port to be actually allocated
-        ${refreshPIAPort}
-
-        ${optionalString (cfg.forwardedPort != null) ''
-          # redirect the fowarded port
-          iptables -A INPUT -i ${cfg.interfaceName} -p tcp --dport $port -j ACCEPT
-          iptables -A INPUT -i ${cfg.interfaceName} -p udp --dport $port -j ACCEPT
-          iptables -A INPUT -i ${cfg.interfaceName} -p tcp --dport ${toString cfg.forwardedPort} -j ACCEPT
-          iptables -A INPUT -i ${cfg.interfaceName} -p udp --dport ${toString cfg.forwardedPort} -j ACCEPT
-          iptables -A PREROUTING -t nat -i ${cfg.interfaceName} -p tcp --dport $port -j REDIRECT --to-port ${toString cfg.forwardedPort}
-          iptables -A PREROUTING -t nat -i ${cfg.interfaceName} -p udp --dport $port -j REDIRECT --to-port ${toString cfg.forwardedPort}
-        ''}
-
-        ${optionalString cfg.forwardPortForTransmission ''
-          # assumes no auth needed for transmission
-          curlout=$(curl localhost:9091/transmission/rpc 2>/dev/null)
-          regex='X-Transmission-Session-Id\: (\w*)'
-          if [[ $curlout =~ $regex ]]; then
-              sessionId=''${BASH_REMATCH[1]}
-          else
-              exit 1
-          fi
-
-          # set the port in transmission
-          data='{"method": "session-set", "arguments": { "peer-port" :'$port' } }'
-          curl http://localhost:9091/transmission/rpc -d "$data" -H "X-Transmission-Session-Id: $sessionId"
-        ''}
-      '';
-
-      preStop = ''
-        wg-quick down /tmp/${cfg.interfaceName}.conf
-
-        # The rest of the script is only for only for port forwarding skip if not needed
-        if [ ${boolToString portForwarding} == false ]; then exit 0; fi
-
-        ${optionalString (cfg.forwardedPort != null) ''
-          # stop redirecting the forwarded port
-          iptables -D INPUT -i ${cfg.interfaceName} -p tcp --dport $port -j ACCEPT
-          iptables -D INPUT -i ${cfg.interfaceName} -p udp --dport $port -j ACCEPT
-          iptables -D INPUT -i ${cfg.interfaceName} -p tcp --dport ${toString cfg.forwardedPort} -j ACCEPT
-          iptables -D INPUT -i ${cfg.interfaceName} -p udp --dport ${toString cfg.forwardedPort} -j ACCEPT
-          iptables -D PREROUTING -t nat -i ${cfg.interfaceName} -p tcp --dport $port -j REDIRECT --to-port ${toString cfg.forwardedPort}
-          iptables -D PREROUTING -t nat -i ${cfg.interfaceName} -p udp --dport $port -j REDIRECT --to-port ${toString cfg.forwardedPort}
-        ''}
-      '';
-    };
-
-    vpn-container.config.systemd.services.pia-vpn-wireguard-forward-port = {
-      enable = portForwarding;
-      description = "PIA VPN WireGuard Tunnel Port Forwarding";
-      after = [ "pia-vpn-wireguard.service" ];
-      requires = [ "pia-vpn-wireguard.service" ];
-
-      path = with pkgs; [ curl ];
-
-      serviceConfig = {
-        Type = "oneshot";
-      };
-
-      script = refreshPIAPort;
-    };
-
-    vpn-container.config.systemd.timers.pia-vpn-wireguard-forward-port = {
-      enable = portForwarding;
-      partOf = [ "pia-vpn-wireguard-forward-port.service" ];
-      wantedBy = [ "timers.target" ];
-      timerConfig = {
-        OnCalendar = "*:0/10"; # 10 minutes
-        RandomizedDelaySec = "1m"; # vary by 1 min to give PIA servers some relief
-      };
-    };
-
-    age.secrets."pia-login.conf".file = ../../secrets/pia-login.age;
-  };
-}

common/network/ping.nix

@@ -1,59 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-# keeps peer to peer connections alive with a periodic ping
-
-with lib;
-with builtins;
-
-# todo auto restart
-
-let
-  cfg = config.keepalive-ping;
-
-  serviceTemplate = host:
-    {
-      "keepalive-ping@${host}" = {
-        description = "Periodic ping keep alive for ${host} connection";
-
-        requires = [ "network-online.target" ];
-        after = [ "network.target" "network-online.target" ];
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig.Restart = "always";
-
-        path = with pkgs; [ iputils ];
-
-        script = ''
-          ping -i ${cfg.delay} ${host} &>/dev/null
-        '';
-      };
-    };
-
-  combineAttrs = foldl recursiveUpdate { };
-
-  serviceList = map serviceTemplate cfg.hosts;
-
-  services = combineAttrs serviceList;
-in
-{
-  options.keepalive-ping = {
-    enable = mkEnableOption "Enable keep alive ping task";
-    hosts = mkOption {
-      type = types.listOf types.str;
-      default = [ ];
-      description = ''
-        Hosts to ping periodically
-      '';
-    };
-    delay = mkOption {
-      type = types.str;
-      default = "60";
-      description = ''
-        Ping interval in seconds of periodic ping per host being pinged
-      '';
-    };
-  };
-
-  config = mkIf cfg.enable {
-    systemd.services = services;
-  };
-}

common/network/tailscale.nix

@@ -10,6 +10,10 @@ in
 
   config.services.tailscale.enable = mkDefault (!config.boot.isContainer);
 
+  # Trust Tailscale interface - access control is handled by Tailscale ACLs.
+  # Required because nftables (used by Incus) breaks Tailscale's automatic iptables rules.
+  config.networking.firewall.trustedInterfaces = mkIf cfg.enable [ "tailscale0" ];
+
   # MagicDNS
   config.networking.nameservers = mkIf cfg.enable [ "1.1.1.1" "8.8.8.8" ];
   config.networking.search = mkIf cfg.enable [ "koi-bebop.ts.net" ];

common/network/vpn.nix

@@ -1,106 +0,0 @@
-{ config, pkgs, lib, allModules, ... }:
-
-with lib;
-
-let
-  cfg = config.vpn-container;
-in
-{
-  options.vpn-container = {
-    enable = mkEnableOption "Enable VPN container";
-
-    containerName = mkOption {
-      type = types.str;
-      default = "vpn";
-      description = ''
-        Name of the VPN container.
-      '';
-    };
-
-    mounts = mkOption {
-      type = types.listOf types.str;
-      default = [ "/var/lib" ];
-      example = "/home/example";
-      description = ''
-        List of mounts on the host to bind to the vpn container.
-      '';
-    };
-
-    useOpenVPN = mkEnableOption "Uses OpenVPN instead of wireguard for PIA VPN connection";
-
-    config = mkOption {
-      type = types.anything;
-      default = { };
-      example = ''
-        {
-          services.nginx.enable = true;
-        }
-      '';
-      description = ''
-        NixOS config for the vpn container.
-      '';
-    };
-  };
-
-  config = mkIf cfg.enable {
-    pia.wireguard.enable = !cfg.useOpenVPN;
-    pia.wireguard.forwardPortForTransmission = !cfg.useOpenVPN;
-
-    containers.${cfg.containerName} = {
-      ephemeral = true;
-      autoStart = true;
-
-      bindMounts = mkMerge ([{
-        "/run/agenix" = {
-          hostPath = "/run/agenix";
-          isReadOnly = true;
-        };
-      }] ++ (lists.forEach cfg.mounts (mount:
-        {
-          "${mount}" = {
-            hostPath = mount;
-            isReadOnly = false;
-          };
-        }
-      )));
-
-      enableTun = cfg.useOpenVPN;
-      privateNetwork = true;
-      hostAddress = "172.16.100.1";
-      localAddress = "172.16.100.2";
-
-      config = {
-        imports = allModules ++ [ cfg.config ];
-
-        # networking.firewall.enable = mkForce false;
-        networking.firewall.trustedInterfaces = [
-          # completely trust internal interface to host
-          "eth0"
-        ];
-
-        pia.openvpn.enable = cfg.useOpenVPN;
-        pia.openvpn.server = "swiss.privacy.network"; # swiss vpn
-
-        # TODO fix so it does run it's own resolver again
-        # run it's own DNS resolver
-        networking.useHostResolvConf = false;
-        # services.resolved.enable = true;
-        networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
-      };
-    };
-
-    # load secrets the container needs
-    age.secrets = config.containers.${cfg.containerName}.config.age.secrets;
-
-    # forwarding for vpn container (only for OpenVPN)
-    networking.nat.enable = mkIf cfg.useOpenVPN true;
-    networking.nat.internalInterfaces = mkIf cfg.useOpenVPN [
-      "ve-${cfg.containerName}"
-    ];
-    networking.ip_forward = mkIf cfg.useOpenVPN true;
-
-    # assumes only one potential interface
-    networking.usePredictableInterfaceNames = false;
-    networking.nat.externalInterface = "eth0";
-  };
-}

common/network/vpnfailsafe.sh

@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-
-set -eEo pipefail
-
-# $@ := ""
-set_route_vars() {
-    local network_var
-    local -a network_vars; read -ra network_vars <<<"${!route_network_*}"
-    for network_var in "${network_vars[@]}"; do
-        local -i i="${network_var#route_network_}"
-        local -a vars=("route_network_$i" "route_netmask_$i" "route_gateway_$i" "route_metric_$i")
-        route_networks[i]="${!vars[0]}"
-        route_netmasks[i]="${!vars[1]:-255.255.255.255}"
-        route_gateways[i]="${!vars[2]:-$route_vpn_gateway}"
-        route_metrics[i]="${!vars[3]:-0}"
-    done
-}
-
-# Configuration.
-readonly prog="$(basename "$0")"
-readonly private_nets="127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
-declare -a remotes cnf_remote_domains cnf_remote_ips route_networks route_netmasks route_gateways route_metrics
-read -ra remotes <<<"$(env|grep -oP '^remote_[0-9]+=.*'|sort -n|cut -d= -f2|tr '\n' '\t')"
-read -ra cnf_remote_domains <<<"$(printf '%s\n' "${remotes[@]%%*[0-9]}"|sort -u|tr '\n' '\t')"
-read -ra cnf_remote_ips <<<"$(printf '%s\n' "${remotes[@]##*[!0-9.]*}"|sort -u|tr '\n' '\t')"
-set_route_vars
-read -ra numbered_vars <<<"${!foreign_option_*} ${!proto_*} ${!remote_*} ${!remote_port_*} \
-                           ${!route_network_*} ${!route_netmask_*} ${!route_gateway_*} ${!route_metric_*}"
-readonly numbered_vars "${numbered_vars[@]}" dev ifconfig_local ifconfig_netmask ifconfig_remote \
-         route_net_gateway route_vpn_gateway script_type trusted_ip trusted_port untrusted_ip untrusted_port \
-         remotes cnf_remote_domains cnf_remote_ips route_networks route_netmasks route_gateways route_metrics
-readonly cur_remote_ip="${trusted_ip:-$untrusted_ip}"
-readonly cur_port="${trusted_port:-$untrusted_port}"
-
-# $@ := ""
-update_hosts() {
-    if remote_entries="$(getent -s dns hosts "${cnf_remote_domains[@]}"|grep -v :)"; then
-        local -r beg="# VPNFAILSAFE BEGIN" end="# VPNFAILSAFE END"
-        {
-            sed -e "/^$beg/,/^$end/d" /etc/hosts
-            echo -e "$beg\\n$remote_entries\\n$end"
-        } >/etc/hosts.vpnfailsafe
-        chmod --reference=/etc/hosts /etc/hosts.vpnfailsafe
-        mv /etc/hosts.vpnfailsafe /etc/hosts
-    fi
-}
-
-# $@ := "up" | "down"
-update_routes() {
-    local -a resolved_ips
-    read -ra resolved_ips <<<"$(getent -s files hosts "${cnf_remote_domains[@]:-ENOENT}"|cut -d' ' -f1|tr '\n' '\t' || true)"
-    local -ar remote_ips=("$cur_remote_ip" "${resolved_ips[@]}" "${cnf_remote_ips[@]}")
-    if [[ "$*" == up ]]; then
-        for remote_ip in "${remote_ips[@]}"; do
-            if [[ -n "$remote_ip" && -z "$(ip route show "$remote_ip")" ]]; then
-                ip route add "$remote_ip" via "$route_net_gateway"
-            fi
-        done
-        for net in 0.0.0.0/1 128.0.0.0/1; do
-            if [[ -z "$(ip route show "$net")" ]]; then
-                ip route add "$net" via "$route_vpn_gateway"
-            fi
-        done
-        for i in $(seq 1 "${#route_networks[@]}"); do
-            if [[ -z "$(ip route show "${route_networks[i]}/${route_netmasks[i]}")" ]]; then
-                ip route add "${route_networks[i]}/${route_netmasks[i]}" \
-                  via "${route_gateways[i]}" metric "${route_metrics[i]}" dev "$dev"
-            fi
-        done
-    elif [[ "$*" == down ]]; then
-        for route in "${remote_ips[@]}" 0.0.0.0/1 128.0.0.0/1; do
-            if [[ -n "$route" && -n "$(ip route show "$route")" ]]; then
-                ip route del "$route"
-            fi
-        done
-        for i in $(seq 1 "${#route_networks[@]}"); do
-            if [[ -n "$(ip route show "${route_networks[i]}/${route_netmasks[i]}")" ]]; then
-                ip route del "${route_networks[i]}/${route_netmasks[i]}"
-            fi
-        done
-    fi
-}
-
-# $@ := ""
-update_firewall() {
-    # $@ := "INPUT" | "OUTPUT" | "FORWARD"
-    insert_chain() {
-        if iptables -C "$*" -j "VPNFAILSAFE_$*" 2>/dev/null; then
-            iptables -D "$*" -j "VPNFAILSAFE_$*"
-            for opt in F X; do
-                iptables -"$opt" "VPNFAILSAFE_$*"
-            done
-        fi
-        iptables -N "VPNFAILSAFE_$*"
-        iptables -I "$*" -j "VPNFAILSAFE_$*"
-    }
-
-    # $@ := "INPUT" | "OUTPUT"
-    accept_remotes() {
-        case "$@" in
-            INPUT)  local -r icmp_type=reply   io=i sd=s states="";;
-            OUTPUT) local -r icmp_type=request io=o sd=d states=NEW,;;
-        esac
-        local -r public_nic="$(ip route show "$cur_remote_ip"|cut -d' ' -f5)"
-        local -ar suf=(-m conntrack --ctstate "$states"RELATED,ESTABLISHED -"$io" "${public_nic:?}" -j ACCEPT)
-        icmp_rule() {
-            iptables "$1" "$2" -p icmp --icmp-type "echo-$icmp_type" -"$sd" "$3" "${suf[@]/%ACCEPT/RETURN}"
-        }
-        for ((i=1; i <= ${#remotes[*]}; ++i)); do
-            local port="remote_port_$i"
-            local proto="proto_$i"
-            iptables -A "VPNFAILSAFE_$*" -p "${!proto%-client}" -"$sd" "${remotes[i-1]}" --"$sd"port "${!port}" "${suf[@]}"
-            if ! icmp_rule -C "VPNFAILSAFE_$*" "${remotes[i-1]}" 2>/dev/null; then
-                icmp_rule -A "VPNFAILSAFE_$*" "${remotes[i-1]}"
-            fi
-        done
-        if ! iptables -S|grep -q "^-A VPNFAILSAFE_$* .*-$sd $cur_remote_ip/32 .*-j ACCEPT$"; then
-            for p in tcp udp; do
-                iptables -A "VPNFAILSAFE_$*" -p "$p" -"$sd" "$cur_remote_ip" --"$sd"port "${cur_port}" "${suf[@]}"
-            done
-            icmp_rule -A "VPNFAILSAFE_$*" "$cur_remote_ip"
-        fi
-    }
-
-    # $@ := "OUTPUT" | "FORWARD"
-    reject_dns() {
-        for proto in udp tcp; do
-            iptables -A "VPNFAILSAFE_$*" -p "$proto" --dport 53 ! -o "$dev" -j REJECT
-        done
-    }
-
-    # $@ := "INPUT" | "OUTPUT" | "FORWARD"
-    pass_private_nets() { 
-        case "$@" in
-            INPUT) local -r io=i sd=s;;&
-            OUTPUT|FORWARD) local -r io=o sd=d;;&
-            INPUT) local -r vpn="${ifconfig_remote:-$ifconfig_local}/${ifconfig_netmask:-32}"
-               iptables -A "VPNFAILSAFE_$*" -"$sd" "$vpn" -"$io" "$dev" -j RETURN
-               for i in $(seq 1 "${#route_networks[@]}"); do
-                   iptables -A "VPNFAILSAFE_$*" -"$sd" "${route_networks[i]}/${route_netmasks[i]}" -"$io" "$dev" -j RETURN
-               done;;&
-            *) iptables -A "VPNFAILSAFE_$*" -"$sd" "$private_nets" ! -"$io" "$dev" -j RETURN;;&
-            INPUT) iptables -A "VPNFAILSAFE_$*" -s "$private_nets" -i "$dev" -j DROP;;&
-            *) for iface in "$dev" lo+; do
-                   iptables -A "VPNFAILSAFE_$*" -"$io" "$iface" -j RETURN
-               done;;
-        esac
-    }
-
-    # $@ := "INPUT" | "OUTPUT" | "FORWARD"
-    drop_other() {
-        iptables -A "VPNFAILSAFE_$*" -j DROP
-    }
-
-    for chain in INPUT OUTPUT FORWARD; do
-        insert_chain "$chain"
-        [[ $chain == FORWARD ]] || accept_remotes "$chain"
-        [[ $chain == INPUT ]] || reject_dns "$chain"
-        pass_private_nets "$chain"
-        drop_other "$chain"
-    done
-}
-
-# $@ := ""
-cleanup() {
-    update_resolv down
-    update_routes down
-}
-trap cleanup INT TERM
-
-# $@ := line_number exit_code
-err_msg() {
-    echo "$0:$1: \`$(sed -n "$1,+0{s/^\\s*//;p}" "$0")' returned $2" >&2
-    cleanup
-}
-trap 'err_msg "$LINENO" "$?"' ERR
-
-# $@ := ""
-main() {
-    case "${script_type:-down}" in
-        up) for f in hosts routes firewall; do "update_$f" up; done;;
-        down) update_routes down
-              update_resolv down;;
-    esac
-}
-
-main

common/nix-builder.nix

@@ -4,15 +4,15 @@ let
   builderUserName = "nix-builder";
 
   builderRole = "nix-builder";
-  builders = config.machines.withRole.${builderRole};
-  thisMachineIsABuilder = config.thisMachine.hasRole.${builderRole};
+  builders = config.machines.withRole.${builderRole} or [];
+  thisMachineIsABuilder = config.thisMachine.hasRole.${builderRole} or false;
 
   # builders don't include themselves as a remote builder
   otherBuilders = lib.filter (hostname: hostname != config.networking.hostName) builders;
 in
 lib.mkMerge [
   # configure builder
-  (lib.mkIf thisMachineIsABuilder {
+  (lib.mkIf (thisMachineIsABuilder && !config.boot.isContainer) {
     users.users.${builderUserName} = {
       description = "Distributed Nix Build User";
       group = builderUserName;

common/ntfy-alerts.nix

@@ -0,0 +1,64 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.ntfy-alerts;
+in
+{
+  options.ntfy-alerts = {
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      default = "https://ntfy.neet.dev";
+      description = "Base URL of the ntfy server.";
+    };
+
+    topic = lib.mkOption {
+      type = lib.types.str;
+      default = "service-failures";
+      description = "ntfy topic to publish alerts to.";
+    };
+
+    curlExtraArgs = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "Extra arguments to pass to curl (e.g. --proxy http://host:port).";
+    };
+  };
+
+  config = lib.mkIf config.thisMachine.hasRole."ntfy" {
+    age.secrets.ntfy-token.file = ../secrets/ntfy-token.age;
+
+    systemd.services."ntfy-failure@" = {
+      description = "Send ntfy alert for failed unit %i";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        EnvironmentFile = "/run/agenix/ntfy-token";
+        ExecStart = "${pkgs.writeShellScript "ntfy-failure-notify" ''
+          unit="$1"
+          ${lib.getExe pkgs.curl} \
+            --fail --silent --show-error \
+            --max-time 30 --retry 3 \
+            ${cfg.curlExtraArgs} \
+            -H "Authorization: Bearer $NTFY_TOKEN" \
+            -H "Title: Service failure on ${config.networking.hostName}" \
+            -H "Priority: high" \
+            -H "Tags: rotating_light" \
+            -d "Unit $unit failed at $(date +%c)" \
+            "${cfg.serverUrl}/${cfg.topic}"
+        ''} %i";
+      };
+    };
+
+    # Apply OnFailure to all services via a systemd drop-in
+    systemd.packages = [
+      (pkgs.runCommand "ntfy-on-failure-dropin" { } ''
+        mkdir -p $out/lib/systemd/system/service.d
+        cat > $out/lib/systemd/system/service.d/ntfy-on-failure.conf <<'EOF'
+        [Unit]
+        OnFailure=ntfy-failure@%p.service
+        EOF
+      '')
+    ];
+  };
+}

common/pc/audio.nix

@@ -1,4 +1,4 @@
-{ lib, config, pkgs, ... }:
+{ lib, config, ... }:
 
 let
   cfg = config.de;

common/pc/touchpad.nix

@@ -1,4 +1,4 @@
-{ lib, config, pkgs, ... }:
+{ lib, config, ... }:
 
 let
   cfg = config.de;

common/sandboxed-workspace/base.nix

@@ -27,6 +27,7 @@ in
     ../shell.nix
     hostConfig.inputs.home-manager.nixosModules.home-manager
     hostConfig.inputs.nix-index-database.nixosModules.default
+    hostConfig.inputs.agenix.nixosModules.default
   ];
 
   nixpkgs.overlays = [
@@ -114,6 +115,14 @@ in
 
   # Enable flakes
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "googlebot" ];
+
+  # Binary cache configuration (inherited from host's common/binary-cache.nix)
+  nix.settings.substituters = hostConfig.nix.settings.substituters;
+  nix.settings.trusted-public-keys = hostConfig.nix.settings.trusted-public-keys;
+  nix.settings.fallback = true;
+  nix.settings.netrc-file = config.age.secrets.attic-netrc.path;
+  age.secrets.attic-netrc.file = ../../secrets/attic-netrc.age;
 
   # Make nixpkgs available in NIX_PATH and registry (like the NixOS ISO)
   # This allows `nix-shell -p`, `nix repl '<nixpkgs>'`, etc. to work

common/sandboxed-workspace/home.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ lib, pkgs, ... }:
 
 # Home Manager configuration for sandboxed workspace user environment
 # This sets up the shell and tools inside VMs and containers

common/sandboxed-workspace/incus.nix

@@ -32,6 +32,9 @@ let
             networking.useHostResolvConf = false;
             nixpkgs.config.allowUnfree = true;
 
+            # Incus containers don't support the kernel features nix sandbox requires
+            nix.settings.sandbox = false;
+
             environment.systemPackages = [
               (lib.hiPrio (pkgs.writeShellScriptBin "claude" ''
                 exec ${pkgs.claude-code}/bin/claude --dangerously-skip-permissions "$@"

common/sandboxed-workspace/vm.nix

@@ -133,8 +133,15 @@ let
   };
 in
 {
-  config = mkIf (cfg.enable && vmWorkspaces != { }) {
-    # Convert VM workspace configs to microvm.nix format
-    microvm.vms = mapAttrs mkVmConfig vmWorkspaces;
-  };
+  config = mkMerge [
+    (mkIf (cfg.enable && vmWorkspaces != { }) {
+      # Convert VM workspace configs to microvm.nix format
+      microvm.vms = mapAttrs mkVmConfig vmWorkspaces;
+    })
+
+    # microvm.nixosModules.host enables KSM, but /sys is read-only in containers
+    (mkIf config.boot.isContainer {
+      hardware.ksm.enable = false;
+    })
+  ];
 }

common/server/actualbudget.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, ... }:
 
 let
   cfg = config.services.actual;

common/server/atticd.nix

@@ -0,0 +1,62 @@
+{ config, lib, ... }:
+
+{
+  config = lib.mkIf (config.thisMachine.hasRole."binary-cache" && !config.boot.isContainer) {
+    services.atticd = {
+      enable = true;
+      environmentFile = config.age.secrets.atticd-credentials.path;
+      settings = {
+        listen = "[::]:28338";
+        database.url = "postgresql:///atticd?host=/run/postgresql";
+        require-proof-of-possession = false;
+
+        # Disable chunking — the dedup savings don't justify the CPU/IO
+        # overhead for local storage, especially on ZFS which already
+        # does block-level compression.
+        chunking = {
+          nar-size-threshold = 0;
+          min-size = 16 * 1024;
+          avg-size = 64 * 1024;
+          max-size = 256 * 1024;
+        };
+
+        # Let ZFS handle compression instead of double-compressing.
+        compression.type = "none";
+
+        garbage-collection.default-retention-period = "6 months";
+      };
+    };
+
+    # PostgreSQL for atticd
+    services.postgresql = {
+      enable = true;
+      ensureDatabases = [ "atticd" ];
+      ensureUsers = [{
+        name = "atticd";
+        ensureDBOwnership = true;
+      }];
+    };
+
+    # Use a static user so the ZFS mountpoint at /var/lib/atticd works
+    # (DynamicUser conflicts with ZFS mountpoints)
+    users.users.atticd = {
+      isSystemUser = true;
+      group = "atticd";
+      home = "/var/lib/atticd";
+    };
+    users.groups.atticd = { };
+
+    systemd.services.atticd = {
+      after = [ "postgresql.service" ];
+      requires = [ "postgresql.service" ];
+      partOf = [ "postgresql.service" ];
+      serviceConfig = {
+        DynamicUser = lib.mkForce false;
+        User = "atticd";
+        Group = "atticd";
+      };
+    };
+
+    age.secrets.atticd-credentials.file = ../../secrets/atticd-credentials.age;
+  };
+}

common/server/ceph.nix

@@ -1,43 +0,0 @@
-{ config, lib, ... }:
-
-with lib;
-let
-  cfg = config.ceph;
-in
-{
-  options.ceph = { };
-
-  config = mkIf cfg.enable {
-    # ceph.enable = true;
-
-    ## S3 Object gateway
-    #ceph.rgw.enable = true;
-    #ceph.rgw.daemons = [
-    #];
-
-    # https://docs.ceph.com/en/latest/start/intro/
-
-    # meta object storage daemon
-    ceph.osd.enable = true;
-    ceph.osd.daemons = [
-
-    ];
-    # monitor's ceph state
-    ceph.mon.enable = true;
-    ceph.mon.daemons = [
-
-    ];
-    # manage ceph
-    ceph.mgr.enable = true;
-    ceph.mgr.daemons = [
-
-    ];
-    # metadata server
-    ceph.mds.enable = true;
-    ceph.mds.daemons = [
-
-    ];
-    ceph.global.fsid = "925773DC-D95F-476C-BBCD-08E01BF0865F";
-
-  };
-}

common/server/default.nix

@@ -1,24 +1,22 @@
-{ config, pkgs, ... }:
+{ ... }:
 
 {
   imports = [
     ./nginx.nix
     ./thelounge.nix
     ./mumble.nix
-    ./icecast.nix
-    ./nginx-stream.nix
     ./matrix.nix
-    ./zerobin.nix
     ./gitea.nix
     ./samba.nix
     ./owncast.nix
     ./mailserver.nix
     ./nextcloud.nix
-    ./iodine.nix
-    ./searx.nix
     ./gitea-actions-runner.nix
+    ./atticd.nix
     ./librechat.nix
     ./actualbudget.nix
     ./unifi.nix
+    ./ntfy.nix
+    ./gatus.nix
   ];
 }

common/server/gatus.nix

@@ -0,0 +1,146 @@
+{ lib, config, ... }:
+
+let
+  cfg = config.services.gatus;
+  port = 31103;
+in
+{
+  options.services.gatus = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      example = "status.example.com";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.gatus = {
+      environmentFile = "/run/agenix/ntfy-token";
+      settings = {
+        storage = {
+          type = "sqlite";
+          path = "/var/lib/gatus/data.db";
+        };
+
+        web = {
+          address = "127.0.0.1";
+          port = port;
+        };
+
+        alerting.ntfy = {
+          url = "https://ntfy.neet.dev";
+          topic = "service-failures";
+          priority = 4;
+          default-alert = {
+            enabled = true;
+            failure-threshold = 3;
+            success-threshold = 2;
+            send-on-resolved = true;
+          };
+          token = "$NTFY_TOKEN";
+        };
+
+        endpoints = [
+          {
+            name = "Gitea";
+            group = "services";
+            url = "https://git.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "The Lounge";
+            group = "services";
+            url = "https://irc.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "ntfy";
+            group = "services";
+            url = "https://ntfy.neet.dev/v1/health";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Librechat";
+            group = "services";
+            url = "https://chat.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Owncast";
+            group = "services";
+            url = "https://live.neet.dev";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Nextcloud";
+            group = "services";
+            url = "https://neet.cloud";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == any(200, 302)"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Element Web";
+            group = "services";
+            url = "https://chat.neet.space";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Mumble";
+            group = "services";
+            url = "tcp://voice.neet.space:23563";
+            interval = "5m";
+            conditions = [
+              "[CONNECTED] == true"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+          {
+            name = "Navidrome";
+            group = "services";
+            url = "https://navidrome.neet.cloud";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+            ];
+            alerts = [{ type = "ntfy"; }];
+          }
+        ];
+      };
+    };
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.${cfg.hostname} = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString port}";
+        proxyWebsockets = true;
+      };
+    };
+  };
+}

common/server/gitea-actions-runner.nix

@@ -1,132 +1,85 @@
-{ config, pkgs, lib, allModules, ... }:
+{ config, lib, ... }:
 
-# Gitea Actions Runner. Starts 'host' runner that runs directly on the host inside of a nixos container
-# This is useful for providing a real Nix/OS builder to gitea.
-# Warning, NixOS containers are not secure. For example, the container shares the /nix/store
-# Therefore, this should not be used to run untrusted code.
-# To enable, assign a machine the 'gitea-actions-runner' system role
-
-# TODO: skipping running inside of nixos container for now because of issues getting docker/podman running
+# Gitea Actions Runner inside a NixOS container.
+# The container shares the host's /nix/store (read-only) and nix-daemon socket,
+# so builds go through the host daemon and outputs land in the host store.
+# Warning: NixOS containers are not fully secure — do not run untrusted code.
+# To enable, assign a machine the 'gitea-actions-runner' system role.
 
 let
   thisMachineIsARunner = config.thisMachine.hasRole."gitea-actions-runner";
   containerName = "gitea-runner";
+  giteaRunnerUid = 991;
+  giteaRunnerGid = 989;
 in
 {
   config = lib.mkIf (thisMachineIsARunner && !config.boot.isContainer) {
-    # containers.${containerName} = {
-    #   ephemeral = true;
-    #   autoStart = true;
 
-    #   # for podman
-    #   enableTun = true;
+    containers.${containerName} = {
+      autoStart = true;
+      ephemeral = true;
 
-    #   # privateNetwork = true;
-    #   # hostAddress = "172.16.101.1";
-    #   # localAddress = "172.16.101.2";
+      bindMounts = {
+        "/run/agenix/gitea-actions-runner-token" = {
+          hostPath = "/run/agenix/gitea-actions-runner-token";
+          isReadOnly = true;
+        };
+        "/var/lib/gitea-runner" = {
+          hostPath = "/var/lib/gitea-runner";
+          isReadOnly = false;
+        };
+      };
 
-    #   bindMounts =
-    #     {
-    #       "/run/agenix/gitea-actions-runner-token" = {
-    #         hostPath = "/run/agenix/gitea-actions-runner-token";
-    #         isReadOnly = true;
-    #       };
-    #       "/var/lib/gitea-runner" = {
-    #         hostPath = "/var/lib/gitea-runner";
-    #         isReadOnly = false;
-    #       };
-    #     };
+      config = { config, lib, pkgs, ... }: {
+        system.stateVersion = "25.11";
 
-    #   extraFlags = [
-    #     # Allow podman
-    #     ''--system-call-filter=thisystemcalldoesnotexistforsure''
-    #   ];
+        services.gitea-actions-runner.instances.inst = {
+          enable = true;
+          name = containerName;
+          url = "https://git.neet.dev/";
+          tokenFile = "/run/agenix/gitea-actions-runner-token";
+          labels = [ "nixos:host" ];
+        };
 
-    #   additionalCapabilities = [
-    #     "CAP_SYS_ADMIN"
-    #   ];
+        # Disable dynamic user so runner state persists via bind mount
+        assertions = [{
+          assertion = config.systemd.services.gitea-runner-inst.enable;
+          message = "Expected systemd service 'gitea-runner-inst' is not enabled — the gitea-actions-runner module may have changed its naming scheme.";
+        }];
+        systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
+        users.users.gitea-runner = {
+          uid = giteaRunnerUid;
+          home = "/var/lib/gitea-runner";
+          group = "gitea-runner";
+          isSystemUser = true;
+          createHome = true;
+        };
+        users.groups.gitea-runner.gid = giteaRunnerGid;
 
-    #   config = {
-    #     imports = allModules;
+        nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
-    #     # speeds up evaluation
-    #     nixpkgs.pkgs = pkgs;
-
-    #     networking.hostName = lib.mkForce containerName;
-
-    #     # don't use remote builders
-    #     nix.distributedBuilds = lib.mkForce false;
-
-    #     environment.systemPackages = with pkgs; [
-    #       git
-    #       # Gitea Actions rely heavily on node. Include it because it would be installed anyway.
-    #       nodejs
-    #     ];
-
-    #     services.gitea-actions-runner.instances.inst = {
-    #       enable = true;
-    #       name = config.networking.hostName;
-    #       url = "https://git.neet.dev/";
-    #       tokenFile = "/run/agenix/gitea-actions-runner-token";
-    #       labels = [
-    #         "ubuntu-latest:docker://node:18-bullseye"
-    #         "nixos:host"
-    #       ];
-    #     };
-
-    #     # To allow building on the host, must override the the service's config so it doesn't use a dynamic user
-    #     systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
-    #     users.users.gitea-runner = {
-    #       home = "/var/lib/gitea-runner";
-    #       group = "gitea-runner";
-    #       isSystemUser = true;
-    #       createHome = true;
-    #     };
-    #     users.groups.gitea-runner = { };
-
-    #     virtualisation.podman.enable = true;
-    #     boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
-    #   };
-    # };
-
-    # networking.nat.enable = true;
-    # networking.nat.internalInterfaces = [
-    #   "ve-${containerName}"
-    # ];
-    # networking.ip_forward = true;
-
-    # don't use remote builders
-    nix.distributedBuilds = lib.mkForce false;
-
-    services.gitea-actions-runner.instances.inst = {
-      enable = true;
-      name = config.networking.hostName;
-      url = "https://git.neet.dev/";
-      tokenFile = "/run/agenix/gitea-actions-runner-token";
-      labels = [
-        "ubuntu-latest:docker://node:18-bullseye"
-        "nixos:host"
-      ];
+        environment.systemPackages = with pkgs; [
+          git
+          nodejs
+          jq
+          attic-client
+        ];
+      };
     };
 
-    environment.systemPackages = with pkgs; [
-      git
-      # Gitea Actions rely heavily on node. Include it because it would be installed anyway.
-      nodejs
-    ];
+    # Needs to be outside of the container because container uses's the host's nix-daemon
+    nix.settings.trusted-users = [ "gitea-runner" ];
 
-    # To allow building on the host, must override the the service's config so it doesn't use a dynamic user
-    systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
+    # Matching user on host — the container's gitea-runner UID must be
+    # recognized by the host's nix-daemon as trusted (shared UID namespace)
     users.users.gitea-runner = {
+      uid = giteaRunnerUid;
       home = "/var/lib/gitea-runner";
       group = "gitea-runner";
       isSystemUser = true;
       createHome = true;
     };
-    users.groups.gitea-runner = { };
-
-    virtualisation.podman.enable = true;
-    boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
+    users.groups.gitea-runner.gid = giteaRunnerGid;
 
     age.secrets.gitea-actions-runner-token.file = ../../secrets/gitea-actions-runner-token.age;
   };

common/server/gitea.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, config, ... }:
+{ lib, config, ... }:
 
 let
   cfg = config.services.gitea;

common/server/gitlab.nix

@@ -1,42 +0,0 @@
-{ config, pkgs, ... }:
-
-{
-  services.gitlab = {
-    enable = true;
-    databasePasswordFile = "/var/keys/gitlab/db_password";
-    initialRootPasswordFile = "/var/keys/gitlab/root_password";
-    https = true;
-    host = "git.neet.dev";
-    port = 443;
-    user = "git";
-    group = "git";
-    databaseUsername = "git";
-    smtp = {
-      enable = true;
-      address = "localhost";
-      port = 25;
-    };
-    secrets = {
-      dbFile = "/var/keys/gitlab/db";
-      secretFile = "/var/keys/gitlab/secret";
-      otpFile = "/var/keys/gitlab/otp";
-      jwsFile = "/var/keys/gitlab/jws";
-    };
-    extraConfig = {
-      gitlab = {
-        email_from = "gitlab-no-reply@neet.dev";
-        email_display_name = "neet.dev GitLab";
-        email_reply_to = "gitlab-no-reply@neet.dev";
-      };
-    };
-    pagesExtraArgs = [ "-listen-proxy" "127.0.0.1:8090" ];
-  };
-
-  services.nginx.virtualHosts = {
-    "git.neet.dev" = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/".proxyPass = "http://unix:/run/gitlab/gitlab-workhorse.socket";
-    };
-  };
-}

common/server/hydra.nix

@@ -1,25 +0,0 @@
-{ config, pkgs, ... }:
-
-let
-  domain = "hydra.neet.dev";
-  port = 3000;
-  notifyEmail = "hydra@neet.dev";
-in
-{
-  services.nginx.virtualHosts."${domain}" = {
-    enableACME = true;
-    forceSSL = true;
-    locations."/" = {
-      proxyPass = "http://localhost:${toString port}";
-    };
-  };
-
-  services.hydra = {
-    enable = true;
-    inherit port;
-    hydraURL = "https://${domain}";
-    useSubstitutes = true;
-    notificationSender = notifyEmail;
-    buildMachinesFiles = [ ];
-  };
-}

common/server/icecast.nix

@@ -1,65 +0,0 @@
-{ lib, config, ... }:
-
-# configures icecast to only accept source from localhost
-# to a audio optimized stream on services.icecast.mount
-# made available via nginx for http access on
-# https://host/mount
-
-let
-  cfg = config.services.icecast;
-in
-{
-  options.services.icecast = {
-    mount = lib.mkOption {
-      type = lib.types.str;
-      example = "stream.mp3";
-    };
-    fallback = lib.mkOption {
-      type = lib.types.str;
-      example = "fallback.mp3";
-    };
-    nginx = lib.mkEnableOption "enable nginx";
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.icecast = {
-      listen.address = "0.0.0.0";
-      listen.port = 8001;
-      admin.password = "hackme";
-      extraConf = ''
-        <authentication>
-          <source-password>hackme</source-password>
-        </authentication>
-        <http-headers>
-          <header type="cors" name="Access-Control-Allow-Origin" />
-        </http-headers>
-        <mount type="normal">
-          <mount-name>/${cfg.mount}</mount-name>
-          <max-listeners>30</max-listeners>
-          <bitrate>64000</bitrate>
-          <hidden>false</hidden>
-          <public>false</public>
-          <fallback-mount>/${cfg.fallback}</fallback-mount>
-          <fallback-override>1</fallback-override>
-        </mount>
-        <mount type="normal">
-          <mount-name>/${cfg.fallback}</mount-name>
-          <max-listeners>30</max-listeners>
-          <bitrate>64000</bitrate>
-          <hidden>false</hidden>
-          <public>false</public>
-        </mount>
-      '';
-    };
-    services.nginx.virtualHosts.${cfg.hostname} = lib.mkIf cfg.nginx {
-      enableACME = true;
-      forceSSL = true;
-      locations."/${cfg.mount}" = {
-        proxyPass = "http://localhost:${toString cfg.listen.port}/${cfg.mount}";
-        extraConfig = ''
-          add_header Access-Control-Allow-Origin *;
-        '';
-      };
-    };
-  };
-}

common/server/iodine.nix

@@ -1,21 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.services.iodine.server;
-in
-{
-  config = lib.mkIf cfg.enable {
-    # iodine DNS-based vpn
-    services.iodine.server = {
-      ip = "192.168.99.1";
-      domain = "tun.neet.dev";
-      passwordFile = "/run/agenix/iodine";
-    };
-    age.secrets.iodine.file = ../../secrets/iodine.age;
-    networking.firewall.allowedUDPPorts = [ 53 ];
-
-    networking.nat.internalInterfaces = [
-      "dns0" # iodine
-    ];
-  };
-}

common/server/librechat.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 
 with lib;
 

common/server/nginx-stream.nix

@@ -1,76 +0,0 @@
-{ lib, config, pkgs, ... }:
-
-let
-  cfg = config.services.nginx.stream;
-  nginxWithRTMP = pkgs.nginx.override {
-    modules = [ pkgs.nginxModules.rtmp ];
-  };
-in
-{
-  options.services.nginx.stream = {
-    enable = lib.mkEnableOption "enable nginx rtmp/hls/dash video streaming";
-    port = lib.mkOption {
-      type = lib.types.int;
-      default = 1935;
-      description = "rtmp injest/serve port";
-    };
-    rtmpName = lib.mkOption {
-      type = lib.types.str;
-      default = "live";
-      description = "the name of the rtmp application";
-    };
-    hostname = lib.mkOption {
-      type = lib.types.str;
-      description = "the http host to serve hls";
-    };
-    httpLocation = lib.mkOption {
-      type = lib.types.str;
-      default = "/tmp";
-      description = "the path of the tmp http files";
-    };
-  };
-  config = lib.mkIf cfg.enable {
-    services.nginx = {
-      enable = true;
-
-      package = nginxWithRTMP;
-
-      virtualHosts.${cfg.hostname} = {
-        enableACME = true;
-        forceSSL = true;
-        locations = {
-          "/stream/hls".root = "${cfg.httpLocation}/hls";
-          "/stream/dash".root = "${cfg.httpLocation}/dash";
-        };
-        extraConfig = ''
-          location /stat {
-            rtmp_stat all;
-          }
-        '';
-      };
-
-      appendConfig = ''
-        rtmp {
-          server {
-            listen ${toString cfg.port};
-            chunk_size 4096;
-            application ${cfg.rtmpName} {
-              allow publish all;
-              allow publish all;
-              live on;
-              record off;
-              hls on;
-              hls_path ${cfg.httpLocation}/hls;
-              dash on;
-              dash_path ${cfg.httpLocation}/dash;
-            }
-          }
-        }
-      '';
-    };
-
-    networking.firewall.allowedTCPPorts = [
-      cfg.port
-    ];
-  };
-}

common/server/nginx.nix

@@ -1,4 +1,4 @@
-{ lib, config, pkgs, ... }:
+{ lib, config, ... }:
 
 let
   cfg = config.services.nginx;

common/server/ntfy.nix

@@ -0,0 +1,38 @@
+{ lib, config, ... }:
+
+let
+  cfg = config.services.ntfy-sh;
+in
+{
+  options.services.ntfy-sh = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      example = "ntfy.example.com";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.ntfy-sh.settings = {
+      base-url = "https://${cfg.hostname}";
+      listen-http = "127.0.0.1:2586";
+      auth-default-access = "deny-all";
+      behind-proxy = true;
+      enable-login = true;
+    };
+
+    # backups
+    backup.group."ntfy".paths = [
+      "/var/lib/ntfy-sh"
+    ];
+
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.${cfg.hostname} = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:2586";
+        proxyWebsockets = true;
+      };
+    };
+  };
+}

common/server/samba.nix

@@ -26,6 +26,16 @@
           "printcap name" = "cups";
 
           "hide files" = "/.nobackup/.DS_Store/._.DS_Store/";
+
+          # Samba 4.22+ enables SMB3 directory leases by default, allowing clients
+          # to cache directory listings locally. When files are created locally on
+          # the server (bypassing Samba), these cached listings go stale because
+          # kernel oplocks — the mechanism that would break leases on local
+          # changes — is incompatible with smb2 leases. Enabling kernel oplocks
+          # would fix this but forces Samba to disable smb2 leases, durable
+          # handles, and level2 oplocks, losing handle caching performance.
+          # https://wiki.samba.org/index.php/Editing_files_locally_on_server:_interoperability
+          "smb3 directory leases" = "no";
         };
         public = {
           path = "/data/samba/Public";

common/server/searx.nix

@@ -1,30 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.services.searx;
-in
-{
-  config = lib.mkIf cfg.enable {
-    services.searx = {
-      environmentFile = "/run/agenix/searx";
-      settings = {
-        server.port = 43254;
-        server.secret_key = "@SEARX_SECRET_KEY@";
-        engines = [{
-          name = "wolframalpha";
-          shortcut = "wa";
-          api_key = "@WOLFRAM_API_KEY@";
-          engine = "wolframalpha_api";
-        }];
-      };
-    };
-    services.nginx.virtualHosts."search.neet.space" = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/" = {
-        proxyPass = "http://localhost:${toString config.services.searx.settings.server.port}";
-      };
-    };
-    age.secrets.searx.file = ../../secrets/searx.age;
-  };
-}

common/server/video-stream.nix

@@ -1,97 +0,0 @@
-{ config, pkgs, ... }:
-
-let
-  # external
-  rtp-port = 8083;
-  webrtc-peer-lower-port = 20000;
-  webrtc-peer-upper-port = 20100;
-  domain = "live.neet.space";
-
-  # internal
-  ingest-port = 8084;
-  web-port = 8085;
-  webrtc-port = 8086;
-  toStr = builtins.toString;
-in
-{
-  networking.firewall.allowedUDPPorts = [ rtp-port ];
-  networking.firewall.allowedTCPPortRanges = [{
-    from = webrtc-peer-lower-port;
-    to = webrtc-peer-upper-port;
-  }];
-  networking.firewall.allowedUDPPortRanges = [{
-    from = webrtc-peer-lower-port;
-    to = webrtc-peer-upper-port;
-  }];
-
-  virtualisation.docker.enable = true;
-
-  services.nginx.virtualHosts.${domain} = {
-    enableACME = true;
-    forceSSL = true;
-    locations = {
-      "/" = {
-        proxyPass = "http://localhost:${toStr web-port}";
-      };
-      "websocket" = {
-        proxyPass = "http://localhost:${toStr webrtc-port}/websocket";
-        proxyWebsockets = true;
-      };
-    };
-  };
-
-  virtualisation.oci-containers = {
-    backend = "docker";
-    containers = {
-      "lightspeed-ingest" = {
-        workdir = "/var/lib/lightspeed-ingest";
-        image = "projectlightspeed/ingest";
-        ports = [
-          "${toStr ingest-port}:8084"
-        ];
-        #        imageFile = pkgs.dockerTools.pullImage {
-        #          imageName = "projectlightspeed/ingest";
-        #          finalImageTag = "version-0.1.4";
-        #          imageDigest = "sha256:9fc51833b7c27a76d26e40f092b9cec1ac1c4bfebe452e94ad3269f1f73ff2fc";
-        #          sha256 = "19kxl02x0a3i6hlnsfcm49hl6qxnq2f3hfmyv1v8qdaz58f35kd5";
-        #        };
-      };
-      "lightspeed-react" = {
-        workdir = "/var/lib/lightspeed-react";
-        image = "projectlightspeed/react";
-        ports = [
-          "${toStr web-port}:80"
-        ];
-        #        imageFile = pkgs.dockerTools.pullImage {
-        #          imageName = "projectlightspeed/react";
-        #          finalImageTag = "version-0.1.3";
-        #          imageDigest = "sha256:b7c58425f1593f7b4304726b57aa399b6e216e55af9c0962c5c19333fae638b6";
-        #          sha256 = "0d2jh7mr20h7dxgsp7ml7cw2qd4m8ja9rj75dpy59zyb6v0bn7js";
-        #        };
-      };
-      "lightspeed-webrtc" = {
-        workdir = "/var/lib/lightspeed-webrtc";
-        image = "projectlightspeed/webrtc";
-        ports = [
-          "${toStr webrtc-port}:8080"
-          "${toStr rtp-port}:65535/udp"
-          "${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}:${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}/tcp"
-          "${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}:${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}/udp"
-        ];
-        cmd = [
-          "lightspeed-webrtc"
-          "--addr=0.0.0.0"
-          "--ip=${domain}"
-          "--ports=${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}"
-          "run"
-        ];
-        #        imageFile = pkgs.dockerTools.pullImage {
-        #          imageName = "projectlightspeed/webrtc";
-        #          finalImageTag = "version-0.1.2";
-        #          imageDigest = "sha256:ddf8b3dd294485529ec11d1234a3fc38e365a53c4738998c6bc2c6930be45ecf";
-        #          sha256 = "1bdy4ak99fjdphj5bsk8rp13xxmbqdhfyfab14drbyffivg9ad2i";
-        #        };
-      };
-    };
-  };
-}

common/server/vscode/LICENSE

@@ -1,7 +0,0 @@
-Copyright 2020 Matthijs Steen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

common/server/vscode/README.md

@@ -1,37 +0,0 @@
-# Visual Studio Code Server support in NixOS
-
-Experimental support for VS Code Server in NixOS. The NodeJS by default supplied by VS Code cannot be used within NixOS due to missing hardcoded paths, so it is automatically replaced by a symlink to a compatible version of NodeJS that does work under NixOS.
-
-## Installation
-
-```nix
-{
-  imports = [
-    (fetchTarball "https://github.com/msteen/nixos-vscode-server/tarball/master")
-  ];
-
-  services.vscode-server.enable = true;
-}
-```
-
-And then enable them for the relevant users:
-
-```
-systemctl --user enable auto-fix-vscode-server.service
-```
-
-### Home Manager
-
-```nix
-{
-  imports = [
-    "${fetchTarball "https://github.com/msteen/nixos-vscode-server/tarball/master"}/modules/vscode-server/home.nix"
-  ];
-
-  services.vscode-server.enable = true;
-}
-```
-
-## Usage
-
-When the service is enabled and running it should simply work, there is nothing for you to do.

common/server/vscode/default.nix

@@ -1 +0,0 @@
-import ./modules/vscode-server

common/server/vscode/modules/vscode-server/default.nix

@@ -1,8 +0,0 @@
-import ./module.nix ({ name, description, serviceConfig }:
-
-  {
-    systemd.user.services.${name} = {
-      inherit description serviceConfig;
-      wantedBy = [ "default.target" ];
-    };
-  })

common/server/vscode/modules/vscode-server/home.nix

@@ -1,15 +0,0 @@
-import ./module.nix ({ name, description, serviceConfig }:
-
-  {
-    systemd.user.services.${name} = {
-      Unit = {
-        Description = description;
-      };
-
-      Service = serviceConfig;
-
-      Install = {
-        WantedBy = [ "default.target" ];
-      };
-    };
-  })

common/server/vscode/modules/vscode-server/module.nix

@@ -1,42 +0,0 @@
-moduleConfig:
-{ lib, pkgs, ... }:
-
-with lib;
-
-{
-  options.services.vscode-server.enable = with types; mkEnableOption "VS Code Server";
-
-  config = moduleConfig rec {
-    name = "auto-fix-vscode-server";
-    description = "Automatically fix the VS Code server used by the remote SSH extension";
-    serviceConfig = {
-      # When a monitored directory is deleted, it will stop being monitored.
-      # Even if it is later recreated it will not restart monitoring it.
-      # Unfortunately the monitor does not kill itself when it stops monitoring,
-      # so rather than creating our own restart mechanism, we leverage systemd to do this for us.
-      Restart = "always";
-      RestartSec = 0;
-      ExecStart = pkgs.writeShellScript "${name}.sh" ''
-        set -euo pipefail
-        PATH=${makeBinPath (with pkgs; [ coreutils inotify-tools ])}
-        bin_dir=~/.vscode-server/bin
-        [[ -e $bin_dir ]] &&
-        find "$bin_dir" -mindepth 2 -maxdepth 2 -name node -type f -exec ln -sfT ${pkgs.nodejs-12_x}/bin/node {} \; ||
-        mkdir -p "$bin_dir"
-        while IFS=: read -r bin_dir event; do
-          # A new version of the VS Code Server is being created.
-          if [[ $event == 'CREATE,ISDIR' ]]; then
-            # Create a trigger to know when their node is being created and replace it for our symlink.
-            touch "$bin_dir/node"
-            inotifywait -qq -e DELETE_SELF "$bin_dir/node"
-            ln -sfT ${pkgs.nodejs-12_x}/bin/node "$bin_dir/node"
-          # The monitored directory is deleted, e.g. when "Uninstall VS Code Server from Host" has been run.
-          elif [[ $event == DELETE_SELF ]]; then
-            # See the comments above Restart in the service config.
-            exit 0
-          fi
-        done < <(inotifywait -q -m -e CREATE,ISDIR -e DELETE_SELF --format '%w%f:%e' "$bin_dir")
-      '';
-    };
-  };
-}

common/server/zerobin.nix

@@ -1,34 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.services.zerobin;
-in
-{
-  options.services.zerobin = {
-    host = lib.mkOption {
-      type = lib.types.str;
-      example = "example.com";
-    };
-    port = lib.mkOption {
-      type = lib.types.int;
-      default = 33422;
-    };
-  };
-  config = lib.mkIf cfg.enable {
-    services.zerobin.listenPort = cfg.port;
-    services.zerobin.listenAddress = "localhost";
-
-    services.nginx.virtualHosts.${cfg.host} = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/" = {
-        proxyPass = "http://localhost:${toString cfg.port}";
-        proxyWebsockets = true;
-      };
-    };
-
-    # zerobin service is broken in nixpkgs currently
-    systemd.services.zerobin.serviceConfig.ExecStart = lib.mkForce
-      "${pkgs.zerobin}/bin/zerobin --host=${cfg.listenAddress} --port=${toString cfg.listenPort} --data-dir=${cfg.dataDir}";
-  };
-}

common/shell.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ pkgs, ... }:
 
 # Improvements to the default shell
 # - use nix-index for command-not-found

common/ssh.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 
 {
   programs.ssh.knownHosts = lib.filterAttrs (n: v: v != null) (lib.concatMapAttrs

common/zfs-alerts.nix

@@ -0,0 +1,87 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.ntfy-alerts;
+  hasZfs = config.boot.supportedFilesystems.zfs or false;
+  hasNtfy = config.thisMachine.hasRole."ntfy";
+
+  checkScript = pkgs.writeShellScript "zfs-health-check" ''
+    PATH="${lib.makeBinPath [ pkgs.zfs pkgs.coreutils pkgs.gawk pkgs.curl ]}"
+
+    unhealthy=""
+
+    # Check pool health status
+    while IFS=$'\t' read -r pool state; do
+      if [ "$state" != "ONLINE" ]; then
+        unhealthy="$unhealthy"$'\n'"Pool '$pool' is $state"
+      fi
+    done < <(zpool list -H -o name,health)
+
+    # Check for errors (read, write, checksum) on any vdev
+    while IFS=$'\t' read -r pool errors; do
+      if [ "$errors" != "No known data errors" ] && [ -n "$errors" ]; then
+        unhealthy="$unhealthy"$'\n'"Pool '$pool' has errors: $errors"
+      fi
+    done < <(zpool status -x 2>/dev/null | awk '
+      /pool:/ { pool=$2 }
+      /errors:/ { sub(/^[[:space:]]*errors: /, ""); print pool "\t" $0 }
+    ')
+
+    # Check for any drives with non-zero error counts
+    drive_errors=$(zpool status 2>/dev/null | awk '
+      /DEGRADED|FAULTED|OFFLINE|UNAVAIL|REMOVED/ && !/pool:/ && !/state:/ {
+        print "  " $0
+      }
+      /[0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+/ {
+        if ($3 > 0 || $4 > 0 || $5 > 0) {
+          print "  " $1 " (read:" $3 " write:" $4 " cksum:" $5 ")"
+        }
+      }
+    ')
+    if [ -n "$drive_errors" ]; then
+      unhealthy="$unhealthy"$'\n'"Device errors:"$'\n'"$drive_errors"
+    fi
+
+    if [ -n "$unhealthy" ]; then
+      message="ZFS health check failed on ${config.networking.hostName}:$unhealthy"
+
+      curl \
+        --fail --silent --show-error \
+        --max-time 30 --retry 3 \
+        -H "Authorization: Bearer $NTFY_TOKEN" \
+        -H "Title: ZFS issue on ${config.networking.hostName}" \
+        -H "Priority: urgent" \
+        -H "Tags: warning" \
+        -d "$message" \
+        "${cfg.serverUrl}/${cfg.topic}"
+
+      echo "$message" >&2
+    fi
+
+    echo "All ZFS pools healthy"
+  '';
+in
+{
+  config = lib.mkIf (hasZfs && hasNtfy) {
+    systemd.services.zfs-health-check = {
+      description = "Check ZFS pool health and alert on issues";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" "zfs.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        EnvironmentFile = "/run/agenix/ntfy-token";
+        ExecStart = checkScript;
+      };
+    };
+
+    systemd.timers.zfs-health-check = {
+      description = "Periodic ZFS health check";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "daily";
+        Persistent = true;
+        RandomizedDelaySec = "1h";
+      };
+    };
+  };
+}

flake.lock

@@ -14,11 +14,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1762618334,
-        "narHash": "sha256-wyT7Pl6tMFbFrs8Lk/TlEs81N6L+VSybPfiIgzU8lbQ=",
+        "lastModified": 1770165109,
+        "narHash": "sha256-9VnK6Oqai65puVJ4WYtCTvlJeXxMzAp/69HhQuTdl/I=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "fcdea223397448d35d9b31f798479227e80183f6",
+        "rev": "b027ee29d959fda4b60b57566d64c98a202e0feb",
         "type": "github"
       },
       "original": {
@@ -53,11 +53,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770491193,
-        "narHash": "sha256-zdnWeXmPZT8BpBo52s4oansT1Rq0SNzksXKpEcMc5lE=",
+        "lastModified": 1771632347,
+        "narHash": "sha256-kNm0YX9RUwf7GZaWQu2F71ccm4OUMz0xFkXn6mGPfps=",
         "owner": "sadjow",
         "repo": "claude-code-nix",
-        "rev": "f68a2683e812d1e4f9a022ff3e0206d46347d019",
+        "rev": "ec90f84b2ea21f6d2272e00d1becbc13030d1895",
         "type": "github"
       },
       "original": {
@@ -124,11 +124,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1766051518,
-        "narHash": "sha256-znKOwPXQnt3o7lDb3hdf19oDo0BLP4MfBOYiWkEHoik=",
+        "lastModified": 1770019181,
+        "narHash": "sha256-hwsYgDnby50JNVpTRYlF3UR/Rrpt01OrxVuryF40CFY=",
         "owner": "serokell",
         "repo": "deploy-rs",
-        "rev": "d5eff7f948535b9c723d60cd8239f8f11ddc90fa",
+        "rev": "77c906c0ba56aabdbc72041bf9111b565cdd6171",
         "type": "github"
       },
       "original": {
@@ -186,11 +186,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1763988335,
-        "narHash": "sha256-QlcnByMc8KBjpU37rbq5iP7Cp97HvjRP0ucfdh+M4Qc=",
+        "lastModified": 1769939035,
+        "narHash": "sha256-Fok2AmefgVA0+eprw2NDwqKkPGEI5wvR+twiZagBvrg=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "50b9238891e388c9fdc6a5c49e49c42533a1b5ce",
+        "rev": "a8ca480175326551d6c4121498316261cbb5b260",
         "type": "github"
       },
       "original": {
@@ -228,11 +228,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1768068402,
-        "narHash": "sha256-bAXnnJZKJiF7Xr6eNW6+PhBf1lg2P1aFUO9+xgWkXfA=",
+        "lastModified": 1771756436,
+        "narHash": "sha256-Tl2I0YXdhSTufGqAaD1ySh8x+cvVsEI1mJyJg12lxhI=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "8bc5473b6bc2b6e1529a9c4040411e1199c43b4c",
+        "rev": "5bd3589390b431a63072868a90c0f24771ff4cbb",
         "type": "github"
       },
       "original": {
@@ -250,11 +250,11 @@
         "spectrum": "spectrum"
       },
       "locked": {
-        "lastModified": 1770310890,
-        "narHash": "sha256-lyWAs4XKg3kLYaf4gm5qc5WJrDkYy3/qeV5G733fJww=",
+        "lastModified": 1771712688,
+        "narHash": "sha256-Pf4CaRoOLQV02m2POPA+0EWvb3gVdpaiS0hNNVZhO3c=",
         "owner": "astro",
         "repo": "microvm.nix",
-        "rev": "68c9f9c6ca91841f04f726a298c385411b7bfcd5",
+        "rev": "a3abc020a3d8e624e145f4144ed40702f788ea32",
         "type": "github"
       },
       "original": {
@@ -270,11 +270,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1765267181,
-        "narHash": "sha256-d3NBA9zEtBu2JFMnTBqWj7Tmi7R5OikoU2ycrdhQEws=",
+        "lastModified": 1771734689,
+        "narHash": "sha256-/phvMgr1yutyAMjKnZlxkVplzxHiz60i4rc+gKzpwhg=",
         "owner": "Mic92",
         "repo": "nix-index-database",
-        "rev": "82befcf7dc77c909b0f2a09f5da910ec95c5b78f",
+        "rev": "8f590b832326ab9699444f3a48240595954a4b10",
         "type": "github"
       },
       "original": {
@@ -285,11 +285,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1767185284,
-        "narHash": "sha256-ljDBUDpD1Cg5n3mJI81Hz5qeZAwCGxon4kQW3Ho3+6Q=",
+        "lastModified": 1771423359,
+        "narHash": "sha256-yRKJ7gpVmXbX2ZcA8nFi6CMPkJXZGjie2unsiMzj3Ig=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "40b1a28dce561bea34858287fbb23052c3ee63fe",
+        "rev": "740a22363033e9f1bb6270fbfb5a9574067af15b",
         "type": "github"
       },
       "original": {
@@ -301,16 +301,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1768250893,
-        "narHash": "sha256-fWNJYFx0QvnlGlcw54EoOYs/wv2icINHUz0FVdh9RIo=",
+        "lastModified": 1771369470,
+        "narHash": "sha256-0NBlEBKkN3lufyvFegY4TYv5mCNHbi5OmBDrzihbBMQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3971af1a8fc3646b1d554cb1269b26c84539c22e",
+        "rev": "0182a361324364ae3f436a63005877674cf45efb",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "master",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -344,11 +344,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1766321686,
-        "narHash": "sha256-icOWbnD977HXhveirqA10zoqvErczVs3NKx8Bj+ikHY=",
+        "lastModified": 1770659507,
+        "narHash": "sha256-RVZno9CypFN3eHxfULKN1K7mb/Cq0HkznnWqnshxpWY=",
         "owner": "simple-nixos-mailserver",
         "repo": "nixos-mailserver",
-        "rev": "7d433bf89882f61621f95082e90a4ab91eb0bdd3",
+        "rev": "781e833633ebc0873d251772a74e4400a73f5d78",
         "type": "gitlab"
       },
       "original": {

flake.nix

@@ -1,7 +1,7 @@
 {
   inputs = {
     # nixpkgs
-    nixpkgs.url = "github:NixOS/nixpkgs/master";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
 
     # Common Utils Among flake inputs
     systems.url = "github:nix-systems/default";
@@ -156,36 +156,30 @@
         nixpkgs.lib.mapAttrs
           (hostname: cfg:
             mkSystem cfg.arch nixpkgs cfg.configurationPath hostname)
-          machineHosts
-        //
-        (
-          let
-            mkEphemeral = system: nixpkgs.lib.nixosSystem {
-              inherit system;
-              modules = [
-                ./machines/ephemeral/minimal.nix
-                inputs.nix-index-database.nixosModules.default
-              ];
-            };
-          in
-          {
-            ephemeral-x86_64 = mkEphemeral "x86_64-linux";
-            ephemeral-aarch64 = mkEphemeral "aarch64-linux";
-          }
-        );
+          machineHosts;
 
       # kexec produces a tarball; for a self-extracting bundle see:
       # https://github.com/nix-community/nixos-generators/blob/master/formats/kexec.nix#L60
-      packages = {
-        "x86_64-linux" = {
-          kexec = self.nixosConfigurations.ephemeral-x86_64.config.system.build.images.kexec;
-          iso = self.nixosConfigurations.ephemeral-x86_64.config.system.build.images.iso;
+      packages =
+        let
+          mkEphemeral = system: nixpkgs.lib.nixosSystem {
+            inherit system;
+            modules = [
+              ./machines/ephemeral/minimal.nix
+              inputs.nix-index-database.nixosModules.default
+            ];
+          };
+        in
+        {
+          "x86_64-linux" = {
+            kexec = (mkEphemeral "x86_64-linux").config.system.build.images.kexec;
+            iso = (mkEphemeral "x86_64-linux").config.system.build.images.iso;
+          };
+          # "aarch64-linux" = {
+          #   kexec = (mkEphemeral "aarch64-linux").config.system.build.images.kexec;
+          #   iso = (mkEphemeral "aarch64-linux").config.system.build.images.iso;
+          # };
         };
-        "aarch64-linux" = {
-          kexec = self.nixosConfigurations.ephemeral-aarch64.config.system.build.images.kexec;
-          iso = self.nixosConfigurations.ephemeral-aarch64.config.system.build.images.iso;
-        };
-      };
 
       overlays.default = import ./overlays { inherit inputs; };
       nixosModules.kernel-modules = import ./overlays/kernel-modules;

home/googlebot.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, osConfig, ... }:
+{ lib, pkgs, osConfig, ... }:
 
 # https://home-manager-options.extranix.com/
 # https://nix-community.github.io/home-manager/options.xhtml
@@ -61,7 +61,8 @@ in
 
   programs.vscode = {
     enable = thisMachineIsPersonal;
-    package = pkgs.vscodium;
+    # Must use fhs version for vscode-lldb
+    package = pkgs.vscodium-fhs;
     profiles.default = {
       userSettings = {
         editor.formatOnSave = true;
@@ -78,6 +79,15 @@ in
           lsp.serverPort = 6005; # port needs to match Godot configuration
           editorPath.godot4 = "godot-mono";
         };
+        rust-analyzer = {
+          restartServerOnConfigChange = true;
+          testExplorer = true;
+          server.path = "rust-analyzer"; # Use the rust-analyzer from PATH (which is set by nixEnvSelector from the project's flake)
+        };
+        nixEnvSelector = {
+          useFlakes = true; # This hasn't ever worked for me and I have to use shell.nix... but maybe someday
+          suggestion = false; # Stop really annoy nagging
+        };
       };
       extensions = with pkgs.vscode-extensions; [
         bbenoist.nix # nix syntax support

machines/fry/default.nix

@@ -13,6 +13,18 @@
   # Upstream interface for sandbox networking (NAT)
   networking.sandbox.upstreamInterface = lib.mkDefault "enp191s0";
 
+  # Enable sandboxed workspace
+  sandboxed-workspace = {
+    enable = true;
+    workspaces.test-incus = {
+      type = "incus";
+      autoStart = true;
+      config = ./workspaces/test-container.nix;
+      ip = "192.168.83.90";
+      hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL0SNSy/MdW38NqKzLr1SG8WKrs8XkrqibacaJtJPzgW";
+    };
+  };
+
   environment.systemPackages = with pkgs; [
     system76-keyboard-configurator
   ];

machines/fry/properties.nix

@@ -8,6 +8,7 @@
   systemRoles = [
     "personal"
     "dns-challenge"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID/Df5lG07Il7fizEgZR/T9bMlR0joESRJ7cqM9BkOyP";

machines/fry/workspaces/test-container.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ pkgs, ... }:
 
 # Test container workspace configuration
 #

machines/fry/workspaces/test-vm.nix

@@ -1,23 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-# Example VM workspace configuration
-#
-# Add to sandboxed-workspace.workspaces in machines/fry/default.nix:
-#   sandboxed-workspace.workspaces.example = {
-#     type = "vm";
-#     config = ./workspaces/example.nix;
-#     ip = "192.168.83.10";
-#   };
-#
-# The workspace name ("example") becomes the hostname automatically.
-# The IP is configured in default.nix, not here.
-
-{
-  # Install packages as needed
-  environment.systemPackages = with pkgs; [
-    # Add packages here
-  ];
-
-  # Additional shares beyond the standard ones (workspace, ssh-host-keys, claude-config):
-  # microvm.shares = [ ... ];
-}

machines/howl/default.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ lib, ... }:
 
 {
   imports = [

machines/howl/properties.nix

@@ -7,6 +7,7 @@
 
   systemRoles = [
     "personal"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQi3q8jU6vRruExAL60J7GFO1gS8HsmXVJuKRT4ljrG";

machines/nat/configuration.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, fetchurl, lib, ... }:
+{ ... }:
 
 {
   imports = [

machines/nat/hardware-configuration.nix

@@ -1,7 +1,7 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{ ... }:
 
 {
   imports = [ ];

machines/phil/default.nix

@@ -1,9 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-{
-  imports = [
-    ./hardware-configuration.nix
-  ];
-
-  networking.hostName = "phil";
-}

machines/phil/hardware-configuration.nix

@@ -1,46 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [
-      (modulesPath + "/profiles/qemu-guest.nix")
-    ];
-
-  # because grub just doesn't work for some reason
-  boot.loader.systemd-boot.enable = true;
-
-  remoteLuksUnlock.enable = true;
-  remoteLuksUnlock.enableTorUnlock = false;
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" ];
-  boot.initrd.kernelModules = [ "dm-snapshot" ];
-  boot.kernelModules = [ ];
-  boot.extraModulePackages = [ ];
-
-  boot.initrd.luks.devices."enc-pv" = {
-    device = "/dev/disk/by-uuid/d26c1820-4c39-4615-98c2-51442504e194";
-    allowDiscards = true;
-  };
-
-  fileSystems."/" =
-    {
-      device = "/dev/disk/by-uuid/851bfde6-93cd-439e-9380-de28aa87eda9";
-      fsType = "btrfs";
-    };
-
-  fileSystems."/boot" =
-    {
-      device = "/dev/disk/by-uuid/F185-C4E5";
-      fsType = "vfat";
-    };
-
-  swapDevices =
-    [{ device = "/dev/disk/by-uuid/d809e3a1-3915-405a-a200-4429c5efdf87"; }];
-
-  networking.interfaces.enp0s6.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
-}

machines/phil/properties.nix

@@ -1,20 +0,0 @@
-{
-  hostNames = [
-    "phil"
-    "phil.neet.dev"
-  ];
-
-  arch = "aarch64-linux";
-
-  systemRoles = [
-    "server"
-    "nix-builder"
-  ];
-
-  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBlgRPpuUkZqe8/lHugRPm/m2vcN9psYhh5tENHZt9I2";
-
-  remoteUnlock = {
-    hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0RodotOXLMy/w70aa096gaNqPBnfgiXR5ZAH4+wGzd";
-    clearnetHost = "unlock.phil.neet.dev";
-  };
-}

machines/ponyo/default.nix

@@ -77,9 +77,6 @@
   # pin postgresql for matrix (will need to migrate eventually)
   services.postgresql.package = pkgs.postgresql_15;
 
-  # iodine DNS-based vpn
-  # services.iodine.server.enable = true;
-
   # proxied web services
   services.nginx.enable = true;
   services.nginx.virtualHosts."navidrome.neet.cloud" = {
@@ -111,4 +108,12 @@
   # librechat
   services.librechat-container.enable = true;
   services.librechat-container.host = "chat.neet.dev";
+
+  # push notifications
+  services.ntfy-sh.enable = true;
+  services.ntfy-sh.hostname = "ntfy.neet.dev";
+
+  # uptime monitoring
+  services.gatus.enable = true;
+  services.gatus.hostname = "status.neet.dev";
 }

machines/ponyo/hardware-configuration.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, modulesPath, ... }:
+{ lib, modulesPath, ... }:
 
 {
   imports =

machines/ponyo/properties.nix

@@ -10,12 +10,12 @@
   systemRoles = [
     "server"
     "email-server"
-    "iodine"
     "pia"
     "nextcloud"
     "dailybot"
     "gitea"
     "librechat"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";

machines/router/default.nix

@@ -1,37 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-  imports = [
-    ./hardware-configuration.nix
-    ./router.nix
-  ];
-
-  # https://dataswamp.org/~solene/2022-08-03-nixos-with-live-usb-router.html
-  # https://github.com/mdlayher/homelab/blob/391cfc0de06434e4dee0abe2bec7a2f0637345ac/nixos/routnerr-2/configuration.nix
-  # https://github.com/skogsbrus/os/blob/master/sys/router.nix
-  # http://trac.gateworks.com/wiki/wireless/wifi 
-
-  system.autoUpgrade.enable = true;
-
-  services.tailscale.exitNode = true;
-
-  router.enable = true;
-  router.privateSubnet = "192.168.3";
-
-  services.iperf3.enable = true;
-
-  # networking.useDHCP = lib.mkForce true;
-
-  networking.usePredictableInterfaceNames = false;
-
-  powerManagement.cpuFreqGovernor = "ondemand";
-
-
-  services.irqbalance.enable = true;
-
-  #   services.miniupnpd = {
-  #     enable = true;
-  #     externalInterface = "eth0";
-  #     internalIPs = [ "br0" ];
-  # };
-}

machines/router/generate_hostapd_config.sh

@@ -1,223 +0,0 @@
-#!/bin/sh
-
-# TODO allow adding custom parameters to ht_capab, vht_capab
-# TODO detect bad channel numbers (preferably not at runtime)
-# TODO error if 160mhz is not supported
-# TODO 'b' only goes up to 40mhz
-
-# gets the phy number using the input interface
-# Ex: get_phy_number("wlan0") -> "1"
-get_phy_number() {
-  local interface=$1
-  phy=$(iw dev "$interface" info | awk '/phy/ {gsub(/#/,"");print $2}')
-  if [[ -z "$phy" ]]; then
-    echo "Error: interface not found" >&2
-    exit 1
-  fi
-  phy=phy$phy
-}
-
-get_ht_cap_mask() {
-  ht_cap_mask=0
-
-  for cap in $(iw phy "$phy" info | grep 'Capabilities:' | cut -d: -f2); do
-    ht_cap_mask="$(($ht_cap_mask | $cap))"
-  done
-
-  local cap_rx_stbc
-  cap_rx_stbc=$((($ht_cap_mask >> 8) & 3))
-  ht_cap_mask="$(( ($ht_cap_mask & ~(0x300)) | ($cap_rx_stbc << 8) ))"
-}
-
-get_vht_cap_mask() {
-  vht_cap_mask=0
-  for cap in $(iw phy "$phy" info | awk -F "[()]" '/VHT Capabilities/ { print $2 }'); do
-    vht_cap_mask="$(($vht_cap_mask | $cap))"
-  done
-
-  local cap_rx_stbc
-  cap_rx_stbc=$((($vht_cap_mask >> 8) & 7))
-  vht_cap_mask="$(( ($vht_cap_mask & ~(0x700)) | ($cap_rx_stbc << 8) ))"
-}
-
-mac80211_add_capabilities() {
-  local __var="$1"; shift
-  local __mask="$1"; shift
-  local __out= oifs
-
-  oifs="$IFS"
-  IFS=:
-  for capab in "$@"; do
-    set -- $capab
-    [ "$(($4))" -gt 0 ] || continue
-    [ "$(($__mask & $2))" -eq "$((${3:-$2}))" ] || continue
-    __out="$__out[$1]"
-  done
-  IFS="$oifs"
-
-  export -n -- "$__var=$__out"
-}
-
-add_special_ht_capabilities() {
-  case "$hwmode" in
-    a)
-      case "$(( ($channel / 4) % 2 ))" in
-        1) ht_capab="$ht_capab[HT40+]";;
-        0) ht_capab="$ht_capab[HT40-]";;
-      esac
-    ;;
-    *)
-      if [ "$channel" -lt 7 ]; then
-        ht_capab="$ht_capab[HT40+]"
-      else
-        ht_capab="$ht_capab[HT40-]"
-      fi
-    ;;
-  esac
-}
-
-add_special_vht_capabilities() {
-  local cap_ant
-  [ "$(($vht_cap_mask & 0x800))" -gt 0 ] && {
-    cap_ant="$(( ( ($vht_cap_mask >> 16) & 3 ) + 1 ))"
-    [ "$cap_ant" -gt 1 ] && vht_capab="$vht_capab[SOUNDING-DIMENSION-$cap_ant]"
-  }
-
-  [ "$(($vht_cap_mask & 0x1000))" -gt 0 ] && {
-    cap_ant="$(( ( ($vht_cap_mask >> 13) & 3 ) + 1 ))"
-    [ "$cap_ant" -gt 1 ] && vht_capab="$vht_capab[BF-ANTENNA-$cap_ant]"
-  }
-
-  if [ "$(($vht_cap_mask & 12))" -eq 4 ]; then
-    vht_capab="$vht_capab[VHT160]"
-  fi
-
-  local vht_max_mpdu_hw=3895
-  [ "$(($vht_cap_mask & 3))" -ge 1 ] && \
-    vht_max_mpdu_hw=7991
-  [ "$(($vht_cap_mask & 3))" -ge 2 ] && \
-    vht_max_mpdu_hw=11454
-  [ "$vht_max_mpdu_hw" != 3895 ] && \
-    vht_capab="$vht_capab[MAX-MPDU-$vht_max_mpdu_hw]"
-
-  # maximum A-MPDU length exponent
-  local vht_max_a_mpdu_len_exp_hw=0
-  [ "$(($vht_cap_mask & 58720256))" -ge 8388608 ] && \
-    vht_max_a_mpdu_len_exp_hw=1
-  [ "$(($vht_cap_mask & 58720256))" -ge 16777216 ] && \
-    vht_max_a_mpdu_len_exp_hw=2
-  [ "$(($vht_cap_mask & 58720256))" -ge 25165824 ] && \
-    vht_max_a_mpdu_len_exp_hw=3
-  [ "$(($vht_cap_mask & 58720256))" -ge 33554432 ] && \
-    vht_max_a_mpdu_len_exp_hw=4
-  [ "$(($vht_cap_mask & 58720256))" -ge 41943040 ] && \
-    vht_max_a_mpdu_len_exp_hw=5
-  [ "$(($vht_cap_mask & 58720256))" -ge 50331648 ] && \
-    vht_max_a_mpdu_len_exp_hw=6
-  [ "$(($vht_cap_mask & 58720256))" -ge 58720256 ] && \
-    vht_max_a_mpdu_len_exp_hw=7
-  vht_capab="$vht_capab[MAX-A-MPDU-LEN-EXP$vht_max_a_mpdu_len_exp_hw]"
-
-  local vht_link_adapt_hw=0
-  [ "$(($vht_cap_mask & 201326592))" -ge 134217728 ] && \
-    vht_link_adapt_hw=2
-  [ "$(($vht_cap_mask & 201326592))" -ge 201326592 ] && \
-    vht_link_adapt_hw=3
-  [ "$vht_link_adapt_hw" != 0 ] && \
-    vht_capab="$vht_capab[VHT-LINK-ADAPT-$vht_link_adapt_hw]"
-}
-
-calculate_channel_offsets() {
-  vht_oper_chwidth=0
-  vht_oper_centr_freq_seg0_idx=
-
-  local idx="$channel"
-  case "$channelWidth" in
-    40)
-      case "$(( ($channel / 4) % 2 ))" in
-        1) idx=$(($channel + 2));;
-        0) idx=$(($channel - 2));;
-      esac
-      vht_oper_centr_freq_seg0_idx=$idx
-    ;;
-    80)
-      case "$(( ($channel / 4) % 4 ))" in
-        1) idx=$(($channel + 6));;
-        2) idx=$(($channel + 2));;
-        3) idx=$(($channel - 2));;
-        0) idx=$(($channel - 6));;
-      esac
-      vht_oper_chwidth=1
-      vht_oper_centr_freq_seg0_idx=$idx
-    ;;
-    160)
-      case "$channel" in
-        36|40|44|48|52|56|60|64) idx=50;;
-        100|104|108|112|116|120|124|128) idx=114;;
-      esac
-      vht_oper_chwidth=2
-      vht_oper_centr_freq_seg0_idx=$idx
-    ;;
-  esac
-
-  he_oper_chwidth=$vht_oper_chwidth
-  he_oper_centr_freq_seg0_idx=$vht_oper_centr_freq_seg0_idx
-}
-
-interface=$1
-channel=$2
-hwmode=$3
-channelWidth=$4
-
-get_phy_number $interface
-get_ht_cap_mask
-get_vht_cap_mask
-
-mac80211_add_capabilities vht_capab $vht_cap_mask \
-  RXLDPC:0x10::1 \
-  SHORT-GI-80:0x20::1 \
-  SHORT-GI-160:0x40::1 \
-  TX-STBC-2BY1:0x80::1 \
-  SU-BEAMFORMER:0x800::1 \
-  SU-BEAMFORMEE:0x1000::1 \
-  MU-BEAMFORMER:0x80000::1 \
-  MU-BEAMFORMEE:0x100000::1 \
-  VHT-TXOP-PS:0x200000::1 \
-  HTC-VHT:0x400000::1 \
-  RX-ANTENNA-PATTERN:0x10000000::1 \
-  TX-ANTENNA-PATTERN:0x20000000::1 \
-  RX-STBC-1:0x700:0x100:1 \
-  RX-STBC-12:0x700:0x200:1 \
-  RX-STBC-123:0x700:0x300:1 \
-  RX-STBC-1234:0x700:0x400:1 \
-
-mac80211_add_capabilities ht_capab $ht_cap_mask \
-  LDPC:0x1::1 \
-  GF:0x10::1 \
-  SHORT-GI-20:0x20::1 \
-  SHORT-GI-40:0x40::1 \
-  TX-STBC:0x80::1 \
-  RX-STBC1:0x300::1 \
-  MAX-AMSDU-7935:0x800::1 \
-
-  # TODO this is active when the driver doesn't support it?
-  # DSSS_CCK-40:0x1000::1 \
-
-  # TODO these are active when the driver doesn't support them?
-  # RX-STBC1:0x300:0x100:1 \
-  # RX-STBC12:0x300:0x200:1 \
-  # RX-STBC123:0x300:0x300:1 \
-
-add_special_ht_capabilities
-add_special_vht_capabilities
-
-echo ht_capab=$ht_capab
-echo vht_capab=$vht_capab
-
-if [ "$channelWidth" != "20" ]; then
-  calculate_channel_offsets
-  echo he_oper_chwidth=$he_oper_chwidth
-  echo vht_oper_chwidth=$vht_oper_chwidth
-  echo he_oper_centr_freq_seg0_idx=$he_oper_centr_freq_seg0_idx
-  echo vht_oper_centr_freq_seg0_idx=$vht_oper_centr_freq_seg0_idx
-fi

machines/router/hardware-configuration.nix

@@ -1,48 +0,0 @@
-{ config, pkgs, ... }:
-
-{
-  # kernel
-  boot.kernelPackages = pkgs.linuxPackages_latest;
-  boot.initrd.availableKernelModules = [ "igb" "mt7915e" "xhci_pci" "ahci" "ehci_pci" "usb_storage" "sd_mod" "sdhci_pci" ];
-  boot.initrd.kernelModules = [ "dm-snapshot" ];
-  boot.kernelModules = [ "kvm-amd" ];
-  boot.extraModulePackages = [ ];
-
-  # Enable serial output
-  boot.kernelParams = [
-    "console=ttyS0,115200n8" # enable serial console
-  ];
-  boot.loader.grub.extraConfig = "
-    serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
-    terminal_input serial
-    terminal_output serial
-  ";
-
-  # firmware
-  firmware.x86_64.enable = true;
-  nixpkgs.config.allowUnfree = true;
-  hardware.enableRedistributableFirmware = true;
-  hardware.enableAllFirmware = true;
-
-  # boot
-  bios = {
-    enable = true;
-    device = "/dev/sda";
-  };
-
-  # disks
-  fileSystems."/" =
-    {
-      device = "/dev/disk/by-uuid/6aa7f79e-bef8-4b0f-b22c-9d1b3e8ac94b";
-      fsType = "ext4";
-    };
-  fileSystems."/boot" =
-    {
-      device = "/dev/disk/by-uuid/14dfc562-0333-4ddd-b10c-4eeefe1cd05f";
-      fsType = "ext3";
-    };
-  swapDevices =
-    [{ device = "/dev/disk/by-uuid/adf37c64-3b54-480c-a9a7-099d61c6eac7"; }];
-
-  nixpkgs.hostPlatform = "x86_64-linux";
-}

machines/router/properties.nix

@@ -1,17 +0,0 @@
-{
-  hostNames = [
-    "router"
-    "192.168.6.159"
-    "192.168.3.1"
-  ];
-
-  arch = "x86_64-linux";
-
-  systemRoles = [
-    "server"
-    "wireless"
-    "router"
-  ];
-
-  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKDCMhEvWJxFBNyvpyuljv5Uun8AdXCxBK9HvPBRe5x6";
-}

machines/router/router.nix

@@ -1,238 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.router;
-  inherit (lib) mapAttrs' genAttrs nameValuePair mkOption types mkIf mkEnableOption;
-in
-{
-  options.router = {
-    enable = mkEnableOption "router";
-
-    privateSubnet = mkOption {
-      type = types.str;
-      default = "192.168.1";
-      description = "IP block (/24) to use for the private subnet";
-    };
-  };
-
-  config = mkIf cfg.enable {
-    networking.ip_forward = true;
-
-    networking.interfaces.enp1s0.useDHCP = true;
-
-    networking.nat = {
-      enable = true;
-      internalInterfaces = [
-        "br0"
-      ];
-      externalInterface = "enp1s0";
-    };
-
-    networking.bridges = {
-      br0 = {
-        interfaces = [
-          "eth2"
-          # "wlp4s0"
-          # "wlan1"
-          "wlan0"
-          "wlan1"
-        ];
-      };
-    };
-
-    networking.interfaces = {
-      br0 = {
-        useDHCP = false;
-        ipv4.addresses = [
-          {
-            address = "${cfg.privateSubnet}.1";
-            prefixLength = 24;
-          }
-        ];
-      };
-    };
-
-    networking.firewall = {
-      enable = true;
-      trustedInterfaces = [ "br0" "tailscale0" ];
-
-      interfaces = {
-        enp1s0 = {
-          allowedTCPPorts = [ ];
-          allowedUDPPorts = [ ];
-        };
-      };
-    };
-
-    services.dnsmasq = {
-      enable = true;
-      settings = {
-        # sensible behaviours
-        domain-needed = true;
-        bogus-priv = true;
-        no-resolv = true;
-
-        # upstream name servers
-        server = [
-          "1.1.1.1"
-          "8.8.8.8"
-        ];
-
-        # local domains
-        expand-hosts = true;
-        domain = "home";
-        local = "/home/";
-
-        # Interfaces to use DNS on
-        interface = "br0";
-
-        # subnet IP blocks to use DHCP on
-        dhcp-range = "${cfg.privateSubnet}.10,${cfg.privateSubnet}.254,24h";
-      };
-    };
-
-    services.hostapd = {
-      enable = true;
-      radios = {
-        # Simple 2.4GHz AP
-        wlan0 = {
-          countryCode = "US";
-          networks.wlan0 = {
-            ssid = "CXNK00BF9176-1";
-            authentication.saePasswords = [{ passwordFile = "/run/agenix/hostapd-pw-CXNK00BF9176"; }];
-          };
-        };
-
-        # WiFi 5 (5GHz) with two advertised networks
-        wlan1 = {
-          band = "5g";
-          channel = 0;
-          countryCode = "US";
-          networks.wlan1 = {
-            ssid = "CXNK00BF9176-1";
-            authentication.saePasswords = [{ passwordFile = "/run/agenix/hostapd-pw-CXNK00BF9176"; }];
-          };
-        };
-      };
-    };
-    age.secrets.hostapd-pw-CXNK00BF9176.file = ../../secrets/hostapd-pw-CXNK00BF9176.age;
-
-    # wlan0 5Ghz 00:0a:52:08:38:32
-    # wlp4s0 2.4Ghz 00:0a:52:08:38:33
-
-    # services.hostapd = {
-    #   enable = true;
-    #   radios = {
-    #     # 2.4GHz
-    #     wlp4s0 = {
-    #       band = "2g";
-    #       noScan = true;
-    #       channel = 6;
-    #       countryCode = "US";
-    #       wifi4 = {
-    #         capabilities = [ "LDPC" "GF" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1" "MAX-AMSDU-7935" "HT40+" ];
-    #       };
-    #       wifi5 = {
-    #         operatingChannelWidth = "20or40";
-    #         capabilities = [ "MAX-A-MPDU-LEN-EXP0" ];
-    #       };
-    #       wifi6 = {
-    #         enable = true;
-    #         singleUserBeamformer = true;
-    #         singleUserBeamformee = true;
-    #         multiUserBeamformer = true;
-    #         operatingChannelWidth = "20or40";
-    #       };
-    #       networks = {
-    #         wlp4s0 = {
-    #           ssid = "CXNK00BF9176";
-    #           authentication.saePasswordsFile = "/run/agenix/hostapd-pw-CXNK00BF9176";
-    #         };
-    #         # wlp4s0-1 = {
-    #         #   ssid = "- Experimental 5G Tower by AT&T";
-    #         #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
-    #         # };
-    #         # wlp4s0-2 = {
-    #         #   ssid = "FBI Surveillance Van 2";
-    #         #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
-    #         # };
-    #       };
-    #       settings = {
-    #         he_oper_centr_freq_seg0_idx = 8;
-    #         vht_oper_centr_freq_seg0_idx = 8;
-    #       };
-    #     };
-
-    #     # 5GHz
-    #     wlan1 = {
-    #       band = "5g";
-    #       noScan = true;
-    #       channel = 128;
-    #       countryCode = "US";
-    #       wifi4 = {
-    #         capabilities = [ "LDPC" "GF" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1" "MAX-AMSDU-7935" "HT40-" ];
-    #       };
-    #       wifi5 = {
-    #         operatingChannelWidth = "160";
-    #         capabilities = [ "RXLDPC" "SHORT-GI-80" "SHORT-GI-160" "TX-STBC-2BY1" "SU-BEAMFORMER" "SU-BEAMFORMEE" "MU-BEAMFORMER" "MU-BEAMFORMEE" "RX-ANTENNA-PATTERN" "TX-ANTENNA-PATTERN" "RX-STBC-1" "SOUNDING-DIMENSION-3" "BF-ANTENNA-3" "VHT160" "MAX-MPDU-11454" "MAX-A-MPDU-LEN-EXP7" ];
-    #       };
-    #       wifi6 = {
-    #         enable = true;
-    #         singleUserBeamformer = true;
-    #         singleUserBeamformee = true;
-    #         multiUserBeamformer = true;
-    #         operatingChannelWidth = "160";
-    #       };
-    #       networks = {
-    #         wlan1 = {
-    #           ssid = "CXNK00BF9176";
-    #           authentication.saePasswordsFile = "/run/agenix/hostapd-pw-CXNK00BF9176";
-    #         };
-    #         # wlan1-1 = {
-    #         #   ssid = "- Experimental 5G Tower by AT&T";
-    #         #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
-    #         # };
-    #         # wlan1-2 = {
-    #         #   ssid = "FBI Surveillance Van 5";
-    #         #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
-    #         # };
-    #       };
-    #       settings = {
-    #         vht_oper_centr_freq_seg0_idx = 114;
-    #         he_oper_centr_freq_seg0_idx = 114;
-    #       };
-    #     };
-    #   };
-    # };
-    # age.secrets.hostapd-pw-experimental-tower.file = ../../secrets/hostapd-pw-experimental-tower.age;
-    # age.secrets.hostapd-pw-CXNK00BF9176.file = ../../secrets/hostapd-pw-CXNK00BF9176.age;
-
-    # hardware.firmware = [
-    #   pkgs.mt7916-firmware
-    # ];
-
-    # nixpkgs.overlays = [
-    #   (self: super: {
-    #     mt7916-firmware = pkgs.stdenvNoCC.mkDerivation {
-    #       pname = "mt7916-firmware";
-    #       version = "custom-feb-02-23";
-    #       src = ./firmware/mediatek; # from here https://github.com/openwrt/mt76/issues/720#issuecomment-1413537674
-    #       dontBuild = true;
-    #       installPhase = ''
-    #         for i in \
-    #           mt7916_eeprom.bin \
-    #           mt7916_rom_patch.bin \
-    #           mt7916_wa.bin \
-    #           mt7916_wm.bin;
-    #         do
-    #           install -D -pm644 $i $out/lib/firmware/mediatek/$i
-    #         done
-    #       '';
-    #       meta = with lib; {
-    #         license = licenses.unfreeRedistributableFirmware;
-    #       };
-    #     };
-    #   })
-    # ];
-  };
-}

machines/storage/s0/default.nix

@@ -55,123 +55,135 @@
   users.users.googlebot.extraGroups = [ "transmission" ];
   users.groups.transmission.gid = config.ids.gids.transmission;
 
-  vpn-container.enable = true;
-  vpn-container.mounts = [
-    "/var/lib"
-    "/data/samba/Public"
-  ];
-  vpn-container.config = {
-    # servarr services
-    services.prowlarr.enable = true;
-    services.sonarr.enable = true;
-    services.sonarr.user = "public_data";
-    services.sonarr.group = "public_data";
-    services.bazarr.enable = true;
-    services.bazarr.user = "public_data";
-    services.bazarr.group = "public_data";
-    services.radarr.enable = true;
-    services.radarr.user = "public_data";
-    services.radarr.group = "public_data";
-    services.lidarr.enable = true;
-    services.lidarr.user = "public_data";
-    services.lidarr.group = "public_data";
-    services.recyclarr = {
-      enable = true;
-      configuration = {
-        radarr.radarr_main = {
-          api_key = {
-            _secret = "/run/credentials/recyclarr.service/radarr-api-key";
-          };
-          base_url = "http://localhost:7878";
+  pia-vpn = {
+    enable = true;
+    serverLocation = "swiss";
 
-          quality_definition.type = "movie";
+    containers.transmission = {
+      ip = "10.100.0.10";
+      mounts."/var/lib".hostPath = "/var/lib";
+      mounts."/data/samba/Public".hostPath = "/data/samba/Public";
+      receiveForwardedPort = { protocol = "both"; };
+      onPortForwarded = ''
+        # Notify Transmission of the PIA-assigned peer port via RPC
+        for i in $(seq 1 30); do
+          curlout=$(curl -s "http://transmission.containers:8080/transmission/rpc" 2>/dev/null) && break
+          sleep 2
+        done
+        regex='X-Transmission-Session-Id: (\w*)'
+        if [[ $curlout =~ $regex ]]; then
+          sessionId=''${BASH_REMATCH[1]}
+          curl -s "http://transmission.containers:8080/transmission/rpc" \
+            -d "{\"method\":\"session-set\",\"arguments\":{\"peer-port\":$PORT}}" \
+            -H "X-Transmission-Session-Id: $sessionId"
+        fi
+      '';
+      config = {
+        services.transmission = {
+          enable = true;
+          package = pkgs.transmission_4;
+          performanceNetParameters = true;
+          user = "public_data";
+          group = "public_data";
+          settings = {
+            "download-dir" = "/data/samba/Public/Media/Transmission";
+            "incomplete-dir" = "/var/lib/transmission/.incomplete";
+            "incomplete-dir-enabled" = true;
+
+            "rpc-enabled" = true;
+            "rpc-port" = 8080;
+            "rpc-bind-address" = "0.0.0.0";
+            "rpc-whitelist" = "127.0.0.1,10.100.*.*,192.168.*.*";
+            "rpc-host-whitelist-enabled" = false;
+
+            "port-forwarding-enabled" = true;
+            "peer-port" = 51413;
+            "peer-port-random-on-start" = false;
+
+            "encryption" = 1;
+            "lpd-enabled" = true;
+            "dht-enabled" = true;
+            "pex-enabled" = true;
+
+            "blocklist-enabled" = true;
+            "blocklist-updates-enabled" = true;
+            "blocklist-url" = "https://github.com/Naunter/BT_BlockLists/raw/master/bt_blocklists.gz";
+
+            "ratio-limit" = 3;
+            "ratio-limit-enabled" = true;
+
+            "download-queue-enabled" = true;
+            "download-queue-size" = 20;
+          };
+        };
+        # https://github.com/NixOS/nixpkgs/issues/258793
+        systemd.services.transmission.serviceConfig = {
+          RootDirectoryStartOnly = lib.mkForce (lib.mkForce false);
+          RootDirectory = lib.mkForce (lib.mkForce "");
         };
-        sonarr.sonarr_main = {
-          api_key = {
-            _secret = "/run/credentials/recyclarr.service/sonarr-api-key";
-          };
-          base_url = "http://localhost:8989";
 
-          quality_definition.type = "series";
+        users.groups.public_data.gid = 994;
+        users.users.public_data = {
+          isSystemUser = true;
+          group = "public_data";
+          uid = 994;
         };
       };
     };
 
-    systemd.services.recyclarr.serviceConfig.LoadCredential = [
-      "radarr-api-key:/run/agenix/radarr-api-key"
-      "sonarr-api-key:/run/agenix/sonarr-api-key"
-    ];
+    containers.servarr = {
+      ip = "10.100.0.11";
+      mounts."/var/lib".hostPath = "/var/lib";
+      mounts."/data/samba/Public".hostPath = "/data/samba/Public";
+      mounts."/run/agenix" = { hostPath = "/run/agenix"; isReadOnly = true; };
+      config = {
+        services.prowlarr.enable = true;
+        services.sonarr.enable = true;
+        services.sonarr.user = "public_data";
+        services.sonarr.group = "public_data";
+        services.bazarr.enable = true;
+        services.bazarr.user = "public_data";
+        services.bazarr.group = "public_data";
+        services.radarr.enable = true;
+        services.radarr.user = "public_data";
+        services.radarr.group = "public_data";
+        services.lidarr.enable = true;
+        services.lidarr.user = "public_data";
+        services.lidarr.group = "public_data";
+        services.recyclarr = {
+          enable = true;
+          configuration = {
+            radarr.radarr_main = {
+              api_key = {
+                _secret = "/run/credentials/recyclarr.service/radarr-api-key";
+              };
+              base_url = "http://localhost:7878";
+              quality_definition.type = "movie";
+            };
+            sonarr.sonarr_main = {
+              api_key = {
+                _secret = "/run/credentials/recyclarr.service/sonarr-api-key";
+              };
+              base_url = "http://localhost:8989";
+              quality_definition.type = "series";
+            };
+          };
+        };
 
-    services.transmission = {
-      enable = true;
-      package = pkgs.transmission_4;
-      performanceNetParameters = true;
-      user = "public_data";
-      group = "public_data";
-      settings = {
-        /* directory settings */
-        # "watch-dir" = "/srv/storage/Transmission/To-Download";
-        # "watch-dir-enabled" = true;
-        "download-dir" = "/data/samba/Public/Media/Transmission";
-        "incomplete-dir" = "/var/lib/transmission/.incomplete";
-        "incomplete-dir-enabled" = true;
+        systemd.services.recyclarr.serviceConfig.LoadCredential = [
+          "radarr-api-key:/run/agenix/radarr-api-key"
+          "sonarr-api-key:/run/agenix/sonarr-api-key"
+        ];
 
-        /* web interface, accessible from local network */
-        "rpc-enabled" = true;
-        "rpc-bind-address" = "0.0.0.0";
-        "rpc-whitelist" = "127.0.0.1,192.168.*.*,172.16.*.*";
-        "rpc-host-whitelist" = "void,192.168.*.*,172.16.*.*";
-        "rpc-host-whitelist-enabled" = false;
-
-        "port-forwarding-enabled" = true;
-        "peer-port" = 50023;
-        "peer-port-random-on-start" = false;
-
-        "encryption" = 1;
-        "lpd-enabled" = true; /* local peer discovery */
-        "dht-enabled" = true; /* dht peer discovery in swarm */
-        "pex-enabled" = true; /* peer exchange */
-
-        /* ip blocklist */
-        "blocklist-enabled" = true;
-        "blocklist-updates-enabled" = true;
-        "blocklist-url" = "https://github.com/Naunter/BT_BlockLists/raw/master/bt_blocklists.gz";
-
-        /* download speed settings */
-        # "speed-limit-down" = 1200;
-        # "speed-limit-down-enabled" = false;
-        # "speed-limit-up" = 500;
-        # "speed-limit-up-enabled" = true;
-
-        /* seeding limit */
-        "ratio-limit" = 3;
-        "ratio-limit-enabled" = true;
-
-        "download-queue-enabled" = true;
-        "download-queue-size" = 20; # gotta go fast
+        users.groups.public_data.gid = 994;
+        users.users.public_data = {
+          isSystemUser = true;
+          group = "public_data";
+          uid = 994;
+        };
       };
     };
-    # https://github.com/NixOS/nixpkgs/issues/258793
-    systemd.services.transmission.serviceConfig = {
-      RootDirectoryStartOnly = lib.mkForce (lib.mkForce false);
-      RootDirectory = lib.mkForce (lib.mkForce "");
-    };
-
-    users.groups.public_data.gid = 994;
-    users.users.public_data = {
-      isSystemUser = true;
-      group = "public_data";
-      uid = 994;
-    };
   };
-  pia.wireguard.badPortForwardPorts = [
-    9696 # prowlarr
-    8989 # sonarr
-    6767 # bazarr
-    7878 # radarr
-    8686 # lidarr
-    9091 # transmission web
-  ];
   age.secrets.radarr-api-key.file = ../../../secrets/radarr-api-key.age;
   age.secrets.sonarr-api-key.file = ../../../secrets/sonarr-api-key.age;
 
@@ -215,12 +227,12 @@
           };
       in
       lib.mkMerge [
-        (mkVirtualHost "bazarr.s0.neet.dev" "http://vpn.containers:6767")
-        (mkVirtualHost "radarr.s0.neet.dev" "http://vpn.containers:7878")
-        (mkVirtualHost "lidarr.s0.neet.dev" "http://vpn.containers:8686")
-        (mkVirtualHost "sonarr.s0.neet.dev" "http://vpn.containers:8989")
-        (mkVirtualHost "prowlarr.s0.neet.dev" "http://vpn.containers:9696")
-        (mkVirtualHost "transmission.s0.neet.dev" "http://vpn.containers:9091")
+        (mkVirtualHost "bazarr.s0.neet.dev" "http://servarr.containers:6767")
+        (mkVirtualHost "radarr.s0.neet.dev" "http://servarr.containers:7878")
+        (mkVirtualHost "lidarr.s0.neet.dev" "http://servarr.containers:8686")
+        (mkVirtualHost "sonarr.s0.neet.dev" "http://servarr.containers:8989")
+        (mkVirtualHost "prowlarr.s0.neet.dev" "http://servarr.containers:9696")
+        (mkVirtualHost "transmission.s0.neet.dev" "http://transmission.containers:8080")
         (mkVirtualHost "unifi.s0.neet.dev" "https://localhost:8443")
         (mkVirtualHost "music.s0.neet.dev" "http://localhost:4533")
         (mkVirtualHost "jellyfin.s0.neet.dev" "http://localhost:8096")

machines/storage/s0/frigate.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, ... }:
 
 let
   frigateHostname = "frigate.s0.neet.dev";

machines/storage/s0/hardware-configuration.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, modulesPath, ... }:
+{ lib, pkgs, modulesPath, ... }:
 
 {
   imports =
@@ -72,5 +72,5 @@
     };
   };
 
-  powerManagement.cpuFreqGovernor = "powersave";
+  powerManagement.cpuFreqGovernor = "schedutil";
 }

machines/storage/s0/home-automation.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, ... }:
 
 {
   services.esphome.enable = true;

machines/storage/s0/properties.nix

@@ -18,6 +18,7 @@
     "linkwarden"
     "outline"
     "dns-challenge"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";

machines/zoidberg/default.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, ... }:
 
 {
   imports = [

machines/zoidberg/properties.nix

@@ -8,6 +8,7 @@
   systemRoles = [
     "personal"
     "media-center"
+    "ntfy"
   ];
 
   hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHvdC1EiLqSNVmk5L1p7cWRIrrlelbK+NMj6tEBrwqIq";

overlays/kernel-modules/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, ... }:
 
 # Adds additional kernel modules to the nixos system
 # Not actually an overlay but a module. Has to be this way because kernel