prototype

Flake lock file updates:

• Updated input 'agenix':
    'github:ryantm/agenix/b7ffcfe77f817d9ee992640ba1f270718d197f28' (2023-01-31)
  → 'github:ryantm/agenix/2994d002dcff5353ca1ac48ec584c7f6589fe447' (2023-04-21)
• Updated input 'deploy-rs':
    'github:serokell/deploy-rs/8c9ea9605eed20528bf60fae35a2b613b901fd77' (2023-01-19)
  → 'github:serokell/deploy-rs/c2ea4e642dc50fd44b537e9860ec95867af30d39' (2023-04-21)
• Updated input 'flake-utils':
    'github:numtide/flake-utils/5aed5285a952e0b949eb3ba02c12fa4fcfef535f' (2022-11-02)
  → 'github:numtide/flake-utils/cfacdce06f30d2b68473a46042957675eebb3401' (2023-04-11)
• Added input 'flake-utils/systems':
    'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e' (2023-04-09)
• Updated input 'nix-index-database':
    'github:Mic92/nix-index-database/4306fa7c12e098360439faac1a2e6b8e509ec97c' (2023-02-26)
  → 'github:Mic92/nix-index-database/68ec961c51f48768f72d2bbdb396ce65a316677e' (2023-04-15)
• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/78c4d33c16092e535bc4ba1284ba49e3e138483a' (2023-03-03)
  → 'github:NixOS/nixpkgs/8dafae7c03d6aa8c2ae0a0612fbcb47e994e3fb8' (2023-04-22)

README.md

@@ -6,7 +6,6 @@
     - `/network` - config for tailscale, and NixOS container with automatic vpn tunneling via PIA
     - `/pc` - config that a graphical desktop computer should have. Use `de.enable = true;` to enable everthing.
     - `/server` - config that creates new nixos services or extends existing ones to meet my needs
-    - `/ssh.nix` - all ssh public host and user keys for all `/machines`
 - `/machines` - all my NixOS machines along with their machine unique configuration for hardware and services
     - `/kexec` - a special machine for generating minimal kexec images. Does not import `/common`
 - `/secrets` - encrypted shared secrets unlocked through `/machines` ssh host keys

TODO.md

@@ -10,24 +10,12 @@
 - https://nixos.wiki/wiki/Comparison_of_NixOS_setups
 
 ### Housekeeping
-- Format everything here using nixfmt
 - Cleanup the line between hardware-configuration.nix and configuration.nix in machine config
-- CI https://gvolpe.com/blog/nixos-binary-cache-ci/
 - remove `options.currentSystem`
 - allow `hostname` option for webservices to be null to disable configuring nginx
 
 ### NAS
-- helios64 extra led lights
 - safely turn off NAS on power disconnect
-- hardware de/encoding for rk3399 helios64 https://forum.pine64.org/showthread.php?tid=14018
-- tor unlock
-
-### bcachefs
-- bcachefs health alerts via email
-- bcachefs periodic snapshotting
-- use mount.bcachefs command for mounting
-- bcachefs native encryption
-    - just need a kernel module? https://github.com/firestack/bcachefs-tools-flake/blob/kf/dev/mvp/nixos/module/bcachefs.nix#L40
 
 ### Shell Comands
 - tailexitnode = `sudo tailscale up --exit-node=<exit-node-ip> --exit-node-allow-lan-access=true`
@@ -53,7 +41,6 @@
 - replace nextcloud with seafile
 
 ### Archive
-- https://www.backblaze.com/b2/cloud-storage.html
 - email
     - https://github.com/Disassembler0/dovecot-archive/blob/main/src/dovecot_archive.py
     - http://kb.unixservertech.com/software/dovecot/archiveserver
@@ -62,7 +49,32 @@
 - https://christine.website/blog/paranoid-nixos-2021-07-18
 - https://nixos.wiki/wiki/Impermanence
 
+# Setup CI
+- CI
+    - hydra
+    - https://docs.cachix.org/continuous-integration-setup/
+- Binary Cache
+    - Maybe use cachix https://gvolpe.com/blog/nixos-binary-cache-ci/
+    - Self hosted binary cache? https://www.tweag.io/blog/2019-11-21-untrusted-ci/
+    - https://github.com/edolstra/nix-serve
+    - https://nixos.wiki/wiki/Binary_Cache
+    - https://discourse.nixos.org/t/introducing-attic-a-self-hostable-nix-binary-cache-server/24343
+- Both
+    - https://garnix.io/
+    - https://nixbuild.net
+
+
+# Secrets
+- consider using headscale
+- Replace luks over tor for remote unlock with luks over tailscale using ephemeral keys
+- Rollover luks FDE passwords
+- /secrets on personal computers should only be readable using a trusted ssh key, preferably requiring a yubikey
+- Rollover shared yubikey secrets
+- offsite backup yubikey, pw db, and ssh key with /secrets access
+
 ### Misc
+- for automated kernel upgrades on luks systems, need to kexec with initrd that contains luks key
+  - https://github.com/flowztul/keyexec/blob/master/etc/default/kexec-cryptroot
 - https://github.com/pop-os/system76-scheduler
 - improve email a little bit https://helloinbox.email
 - remap razer keys https://github.com/sezanzeb/input-remapper

common/auto-update.nix

@@ -1,14 +1,25 @@
-{ config, lib, ... }:
+{ config, pkgs, lib, ... }:
 
 # Modify auto-update so that it pulls a flake
 
 let
   cfg = config.system.autoUpgrade;
-in {
-  config = lib.mkIf cfg.enable {
+in
+{
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
       system.autoUpgrade = {
         flake = "git+https://git.neet.dev/zuckerberg/nix-config.git";
-      flags = [ "--recreate-lock-file" ]; # ignore lock file, just pull the latest
-    };
+        flags = [ "--recreate-lock-file" "--no-write-lock-file" ]; # ignore lock file, just pull the latest
+
+        # dates = "03:40";
+        # kexecWindow = lib.mkDefault { lower = "01:00"; upper = "05:00"; };
+        # randomizedDelaySec = "45min";
       };
+
+      system.autoUpgrade.allowKexec = lib.mkDefault true;
+
+      luks.enableKexec = cfg.allowKexec && builtins.length config.luks.devices > 0;
+    }
+  ]);
 }

common/backups.nix

@@ -0,0 +1,78 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.backup;
+  hostname = config.networking.hostName;
+
+  mkRespository = group: "s3:s3.us-west-004.backblazeb2.com/D22TgIt0-main-backup/${group}";
+
+  mkBackup = group: paths: {
+    repository = mkRespository group;
+    inherit paths;
+
+    initialize = true;
+
+    timerConfig = {
+      OnCalendar = "daily";
+      RandomizedDelaySec = "1h";
+    };
+
+    extraBackupArgs = [
+      ''--exclude-if-present ".nobackup"''
+    ];
+
+    pruneOpts = [
+      "--keep-daily 7" # one backup for each of the last n days
+      "--keep-weekly 5" # one backup for each of the last n weeks
+      "--keep-monthly 12" # one backup for each of the last n months
+      "--keep-yearly 75" # one backup for each of the last n years
+    ];
+
+    environmentFile = "/run/agenix/backblaze-s3-backups";
+    passwordFile = "/run/agenix/restic-password";
+  };
+
+  # example usage: "sudo restic_samba unlock" (removes lockfile)
+  mkResticGroupCmd = group: pkgs.writeShellScriptBin "restic_${group}" ''
+    if [ "$EUID" -ne 0 ]
+      then echo "Run as root"
+      exit
+    fi
+    . /run/agenix/backblaze-s3-backups
+    export AWS_SECRET_ACCESS_KEY
+    export AWS_ACCESS_KEY_ID
+    export RESTIC_PASSWORD_FILE=/run/agenix/restic-password
+    export RESTIC_REPOSITORY="${mkRespository group}"
+    exec ${pkgs.restic}/bin/restic "$@"
+  '';
+in
+{
+  options.backup = {
+    group = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr (lib.types.attrsOf (lib.types.submodule {
+        options = {
+          paths = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = ''
+              Paths to backup
+            '';
+          };
+        };
+      }));
+    };
+  };
+
+  config = lib.mkIf (cfg.group != null) {
+    services.restic.backups = lib.concatMapAttrs
+      (group: groupCfg: {
+        ${group} = mkBackup group groupCfg.paths;
+      })
+      cfg.group;
+
+    age.secrets.backblaze-s3-backups.file = ../secrets/backblaze-s3-backups.age;
+    age.secrets.restic-password.file = ../secrets/restic-password.age;
+
+    environment.systemPackages = map mkResticGroupCmd (builtins.attrNames cfg.group);
+  };
+}

common/boot/bios.nix

@@ -3,7 +3,8 @@
 with lib;
 let
   cfg = config.bios;
-in {
+in
+{
   options.bios = {
     enable = mkEnableOption "enable bios boot";
     device = mkOption {

common/boot/default.nix

@@ -5,6 +5,8 @@
     ./firmware.nix
     ./efi.nix
     ./bios.nix
+    ./kexec-luks.nix
+    ./luks.nix
     ./remote-luks-unlock.nix
   ];
 }

common/boot/efi.nix

@@ -3,7 +3,8 @@
 with lib;
 let
   cfg = config.efi;
-in {
+in
+{
   options.efi = {
     enable = mkEnableOption "enable efi boot";
   };
@@ -19,7 +20,7 @@ in {
         version = 2;
         efiSupport = true;
         useOSProber = true;
-#       memtest86.enable = true;
+        #       memtest86.enable = true;
         configurationLimit = 20;
         theme = pkgs.nixos-grub2-theme;
       };

common/boot/firmware.nix

@@ -3,7 +3,8 @@
 with lib;
 let
   cfg = config.firmware;
-in {
+in
+{
   options.firmware.x86_64 = {
     enable = mkEnableOption "enable x86_64 firmware";
   };

common/boot/kexec-luks.nix

@@ -0,0 +1,121 @@
+# Allows kexec'ing as an alternative to rebooting for machines that
+# have luks encrypted partitions that need to be mounted at boot.
+# These luks partitions will be automatically unlocked, no password,
+# or any interaction needed whatsoever.
+
+# This is accomplished by fetching the luks key(s) while the system is running,
+# then building a temporary initrd that contains the luks key(s), and kexec'ing.
+
+{ config, lib, pkgs, ... }:
+
+{
+  options.luks = {
+    enableKexec = lib.mkEnableOption "Enable support for transparent passwordless kexec while using luks";
+  };
+
+  config = lib.mkIf config.luks.enableKexec {
+    luks.fallbackToPassword = true;
+    luks.disableKeyring = true;
+
+    boot.initrd.luks.devices = lib.listToAttrs
+      (builtins.map
+        (item:
+          {
+            name = item;
+            value = {
+              masterKeyFile = "/etc/${item}.key";
+            };
+          })
+        config.luks.deviceNames);
+
+    systemd.services.prepare-luks-kexec-image = {
+      description = "Prepare kexec automatic LUKS unlock on kexec reboot without a password";
+
+      wantedBy = [ "kexec.target" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+
+      path = with pkgs; [ file kexec-tools coreutils-full cpio findutils gzip xz zstd lvm2 xxd gawk ];
+
+      # based on https://github.com/flowztul/keyexec
+      script = ''
+        system=/nix/var/nix/profiles/system
+        old_initrd=$(readlink -f "$system/initrd")
+
+        umask 0077
+        CRYPTROOT_TMPDIR="$(mktemp -d --tmpdir=/dev/shm)"
+
+        cleanup() {
+          shred -fu "$CRYPTROOT_TMPDIR/initrd_contents/etc/"*.key || true
+          shred -fu "$CRYPTROOT_TMPDIR/new_initrd" || true
+          shred -fu "$CRYPTROOT_TMPDIR/secret/"* || true
+          rm -rf "$CRYPTROOT_TMPDIR"
+        }
+        # trap cleanup INT TERM EXIT
+
+        mkdir -p "$CRYPTROOT_TMPDIR"
+        cd "$CRYPTROOT_TMPDIR"
+
+        # Determine the compression type of the initrd image
+        compression=$(file -b --mime-type "$old_initrd" | awk -F'/' '{print $2}')
+
+        # Decompress the initrd image based on its compression type
+        case "$compression" in
+          gzip)
+            gunzip -c "$old_initrd" > initrd.cpio
+            ;;
+          xz)
+            unxz -c "$old_initrd" > initrd.cpio
+            ;;
+          zstd)
+            zstd -d -c "$old_initrd" > initrd.cpio
+            ;;
+          *)
+            echo "Unsupported compression type: $compression"
+            exit 1
+            ;;
+        esac
+
+        # Extract the contents of the cpio archive
+        mkdir -p initrd_contents
+        cd initrd_contents
+        cpio -idv < ../initrd.cpio
+
+        # Generate keys and add them to the extracted initrd filesystem
+        luksDeviceNames=(${builtins.concatStringsSep " " config.luks.deviceNames})
+        for item in "''${luksDeviceNames[@]}"; do
+          dmsetup --showkeys table "$item" | cut -d ' ' -f5 | xxd -ps -g1 -r > "./etc/$item.key"
+        done
+
+        # Add normal initrd secrets too
+        ${lib.concatStringsSep "\n" (lib.mapAttrsToList (dest: source:
+            let source' = if source == null then dest else builtins.toString source; in
+              ''
+                mkdir -p $(dirname "./${dest}")
+                cp -a ${source'} "./${dest}"
+              ''
+          ) config.boot.initrd.secrets)
+        }
+
+        # Create a new cpio archive with the modified contents
+        find . | cpio -o -H newc -v > ../new_initrd.cpio
+
+        # Compress the new cpio archive using the original compression type
+        cd ..
+        case "$compression" in
+          gzip)
+            gunzip -c new_initrd.cpio > new_initrd
+            ;;
+          xz)
+            unxz -c new_initrd.cpio > new_initrd
+            ;;
+          zstd)
+            zstd -c new_initrd.cpio > new_initrd
+            ;;
+        esac
+
+        kexec --load "$system/kernel" --append "init=$system/init ${builtins.concatStringsSep " " config.boot.kernelParams}" --initrd "$CRYPTROOT_TMPDIR/new_initrd"
+      '';
+    };
+  };
+}

common/boot/luks.nix

@@ -0,0 +1,74 @@
+# Makes it a little easier to configure luks partitions for boot
+# Additionally, this solves a circular dependency between kexec luks
+# and NixOS's luks module.
+
+{ config, lib, ... }:
+
+let
+  cfg = config.luks;
+
+  deviceCount = builtins.length cfg.devices;
+
+  deviceMap = lib.imap
+    (i: item: {
+      device = item;
+      name =
+        if deviceCount == 1 then "enc-pv"
+        else "enc-pv${builtins.toString (i + 1)}";
+    })
+    cfg.devices;
+in
+{
+  options.luks = {
+    devices = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+    };
+
+    allowDiscards = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+    };
+
+    fallbackToPassword = lib.mkEnableOption
+      "Fallback to interactive passphrase prompt if the cannot be found.";
+
+    disableKeyring = lib.mkEnableOption
+      "When opening LUKS2 devices, don't use the kernel keyring";
+
+    # set automatically, don't touch
+    deviceNames = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+    };
+  };
+
+  config = lib.mkMerge [
+    {
+      assertions = [
+        {
+          assertion = deviceCount == builtins.length (builtins.attrNames config.boot.initrd.luks.devices);
+          message = ''
+            All luks devices must be specified using `luks.devices` not `boot.initrd.luks.devices`.
+          '';
+        }
+      ];
+    }
+    (lib.mkIf (deviceCount != 0) {
+      luks.deviceNames = builtins.map (device: device.name) deviceMap;
+
+      boot.initrd.luks.devices = lib.listToAttrs (
+        builtins.map
+          (item:
+            {
+              name = item.name;
+              value = {
+                device = item.device;
+                allowDiscards = cfg.allowDiscards;
+                fallbackToPassword = cfg.fallbackToPassword;
+                disableKeyring = cfg.disableKeyring;
+              };
+            })
+          deviceMap);
+    })
+  ];
+}

common/boot/remote-luks-unlock.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.remoteLuksUnlock;
-in {
+in
+{
   options.remoteLuksUnlock = {
     enable = lib.mkEnableOption "enable luks root remote decrypt over ssh/tor";
     enableTorUnlock = lib.mkOption {
@@ -32,11 +33,6 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
-    # boot.initrd.luks.devices.${cfg.device.name} = {
-    #   device = cfg.device.path;
-    #   allowDiscards = cfg.device.allowDiscards;
-    # };
-
     # Unlock LUKS disk over ssh
     boot.initrd.network.enable = true;
     boot.initrd.kernelModules = cfg.kernelModules;
@@ -61,18 +57,22 @@ in {
       copy_bin_and_libs ${pkgs.haveged}/bin/haveged
     '';
     boot.initrd.network.postCommands = lib.mkMerge [
-      (''
+      (
+        ''
           # Add nice prompt for giving LUKS passphrase over ssh
           echo 'read -s -p "Unlock Passphrase: " passphrase && echo $passphrase > /crypt-ramfs/passphrase && exit' >> /root/.profile
-      '')
+        ''
+      )
 
-      (let torRc = (pkgs.writeText "tor.rc" ''
+      (
+        let torRc = (pkgs.writeText "tor.rc" ''
           DataDirectory /etc/tor
           SOCKSPort 127.0.0.1:9050 IsolateDestAddr
           SOCKSPort 127.0.0.1:9063
           HiddenServiceDir /etc/tor/onion/bootup
           HiddenServicePort 22 127.0.0.1:22
-      ''); in lib.mkIf cfg.enableTorUnlock ''
+        ''); in
+        lib.mkIf cfg.enableTorUnlock ''
           echo "tor: preparing onion folder"
           # have to do this otherwise tor does not want to start
           chmod -R 700 /etc/tor
@@ -87,7 +87,8 @@ in {
           echo "tor: starting tor"
           tor -f ${torRc} --verify-config
           tor -f ${torRc} &
-      '')
+        ''
+      )
     ];
   };
 }

common/default.nix

@@ -1,12 +1,8 @@
 { config, pkgs, ... }:
 
-let
-  ssh = import ./ssh.nix;
-  sshUserKeys = ssh.users;
-  sshHigherTrustKeys = ssh.higherTrustUserKeys;
-in
 {
   imports = [
+    ./backups.nix
     ./flakes.nix
     ./auto-update.nix
     ./shell.nix
@@ -14,6 +10,8 @@ in
     ./boot
     ./server
     ./pc
+    ./machine-info
+    ./ssh.nix
   ];
 
   nix.flakes.enable = true;
@@ -40,7 +38,8 @@ in
     wget
     kakoune
     htop
-    git git-lfs
+    git
+    git-lfs
     dnsutils
     tmux
     nethogs
@@ -53,6 +52,7 @@ in
     helix
     lm_sensors
     picocom
+    lf
   ];
 
   nixpkgs.config.allowUnfree = true;
@@ -65,17 +65,25 @@ in
       "dialout" # serial
     ];
     shell = pkgs.fish;
-    openssh.authorizedKeys.keys = sshUserKeys;
+    openssh.authorizedKeys.keys = config.machines.ssh.userKeys;
     hashedPassword = "$6$TuDO46rILr$gkPUuLKZe3psexhs8WFZMpzgEBGksE.c3Tjh1f8sD0KMC4oV89K2pqAABfl.Lpxu2jVdr5bgvR5cWnZRnji/r/";
     uid = 1000;
   };
   users.users.root = {
-    openssh.authorizedKeys.keys = sshHigherTrustKeys;
+    openssh.authorizedKeys.keys = config.machines.ssh.deployKeys;
   };
   nix.settings = {
     trusted-users = [ "root" "googlebot" ];
   };
 
+  # don't use sudo
+  security.doas.enable = true;
+  security.sudo.enable = false;
+  security.doas.extraRules = [
+    # don't ask for password every time
+    { groups = [ "wheel" ]; persist = true; }
+  ];
+
   nix.gc.automatic = true;
 
   security.acme.acceptTerms = true;

common/flakes.nix

@@ -2,7 +2,8 @@
 with lib;
 let
   cfg = config.nix.flakes;
-in {
+in
+{
   options.nix.flakes = {
     enable = mkEnableOption "use nix flakes";
   };

common/machine-info/default.nix

@@ -0,0 +1,200 @@
+# Gathers info about each machine to constuct overall configuration
+# Ex: Each machine already trusts each others SSH fingerprint already
+
+{ config, lib, pkgs, ... }:
+
+let
+  machines = config.machines.hosts;
+in
+{
+  imports = [
+    ./ssh.nix
+    ./roles.nix
+  ];
+
+  options.machines = {
+
+    hosts = lib.mkOption {
+      type = lib.types.attrsOf
+        (lib.types.submodule {
+          options = {
+
+            hostNames = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                List of hostnames for this machine. The first one is the default so it is the target of deployments.
+                Used for automatically trusting hosts for ssh connections.
+              '';
+            };
+
+            arch = lib.mkOption {
+              type = lib.types.enum [ "x86_64-linux" "aarch64-linux" ];
+              description = ''
+                The architecture of this machine.
+              '';
+            };
+
+            systemRoles = lib.mkOption {
+              type = lib.types.listOf lib.types.str; # TODO: maybe use an enum?
+              description = ''
+                The set of roles this machine holds. Affects secrets available. (TODO add service config as well using this info)
+              '';
+            };
+
+            hostKey = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The system ssh host key of this machine. Used for automatically trusting hosts for ssh connections
+                and for decrypting secrets with agenix.
+              '';
+            };
+
+            remoteUnlock = lib.mkOption {
+              default = null;
+              type = lib.types.nullOr (lib.types.submodule {
+                options = {
+
+                  hostKey = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      The system ssh host key of this machine used for luks boot unlocking only.
+                    '';
+                  };
+
+                  clearnetHost = lib.mkOption {
+                    default = null;
+                    type = lib.types.nullOr lib.types.str;
+                    description = ''
+                      The hostname resolvable over clearnet used to luks boot unlock this machine
+                    '';
+                  };
+
+                  onionHost = lib.mkOption {
+                    default = null;
+                    type = lib.types.nullOr lib.types.str;
+                    description = ''
+                      The hostname resolvable over tor used to luks boot unlock this machine
+                    '';
+                  };
+
+                };
+              });
+            };
+
+            userKeys = lib.mkOption {
+              default = [ ];
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                The list of user keys. Each key here can be used to log into all other systems as `googlebot`.
+
+                TODO: consider auto populating other programs that use ssh keys such as gitea
+              '';
+            };
+
+            deployKeys = lib.mkOption {
+              default = [ ];
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                The list of deployment keys. Each key here can be used to log into all other systems as `root`.
+              '';
+            };
+
+            configurationPath = lib.mkOption {
+              type = lib.types.path;
+              description = ''
+                The path to this machine's configuration directory.
+              '';
+            };
+
+          };
+        });
+    };
+  };
+
+  config = {
+    assertions = (lib.concatLists (lib.mapAttrsToList
+      (
+        name: cfg: [
+          {
+            assertion = builtins.length cfg.hostNames > 0;
+            message = ''
+              Error with config for ${name}
+              There must be at least one hostname.
+            '';
+          }
+          {
+            assertion = builtins.length cfg.systemRoles > 0;
+            message = ''
+              Error with config for ${name}
+              There must be at least one system role.
+            '';
+          }
+          {
+            assertion = cfg.remoteUnlock == null || cfg.remoteUnlock.hostKey != cfg.hostKey;
+            message = ''
+              Error with config for ${name}
+              Unlock hostkey and hostkey cannot be the same because unlock hostkey is in /boot, unencrypted.
+            '';
+          }
+          {
+            assertion = cfg.remoteUnlock == null || (cfg.remoteUnlock.clearnetHost != null || cfg.remoteUnlock.onionHost != null);
+            message = ''
+              Error with config for ${name}
+              At least one of clearnet host or onion host must be defined.
+            '';
+          }
+          {
+            assertion = cfg.remoteUnlock == null || cfg.remoteUnlock.clearnetHost == null || builtins.elem cfg.remoteUnlock.clearnetHost cfg.hostNames == false;
+            message = ''
+              Error with config for ${name}
+              Clearnet unlock hostname cannot be in the list of hostnames for security reasons.
+            '';
+          }
+          {
+            assertion = cfg.remoteUnlock == null || cfg.remoteUnlock.onionHost == null || lib.strings.hasSuffix ".onion" cfg.remoteUnlock.onionHost;
+            message = ''
+              Error with config for ${name}
+              Tor unlock hostname must be an onion address.
+            '';
+          }
+          {
+            assertion = builtins.elem "personal" cfg.systemRoles || builtins.length cfg.userKeys == 0;
+            message = ''
+              Error with config for ${name}
+              There must be at least one userkey defined for personal machines.
+            '';
+          }
+          {
+            assertion = builtins.elem "deploy" cfg.systemRoles || builtins.length cfg.deployKeys == 0;
+            message = ''
+              Error with config for ${name}
+              Only deploy machines are allowed to have deploy keys for security reasons.
+            '';
+          }
+        ]
+      )
+      machines));
+
+    # Set per machine properties automatically using each of their `properties.nix` files respectively
+    machines.hosts =
+      let
+        properties = dir: lib.concatMapAttrs
+          (name: path: {
+            ${name} =
+              import path
+              //
+              { configurationPath = builtins.dirOf path; };
+          })
+          (propertiesFiles dir);
+        propertiesFiles = dir:
+          lib.foldl (lib.mergeAttrs) { } (propertiesFiles' dir);
+        propertiesFiles' = dir:
+          let
+            propFiles = lib.filter (p: baseNameOf p == "properties.nix") (lib.filesystem.listFilesRecursive dir);
+            dirName = path: builtins.baseNameOf (builtins.dirOf path);
+          in
+          builtins.map (p: { "${dirName p}" = p; }) propFiles;
+      in
+      properties ../../machines;
+  };
+}

common/machine-info/moduleless.nix

@@ -0,0 +1,15 @@
+# Allows getting machine-info outside the scope of nixos configuration
+
+{ nixpkgs ? import <nixpkgs> { }
+, assertionsModule ? <nixpkgs/nixos/modules/misc/assertions.nix>
+}:
+
+{
+  machines =
+    (nixpkgs.lib.evalModules {
+      modules = [
+        ./default.nix
+        assertionsModule
+      ];
+    }).config.machines;
+}

common/machine-info/roles.nix

@@ -0,0 +1,19 @@
+{ config, lib, ... }:
+
+# Maps roles to their hosts
+
+{
+  options.machines.roles = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.listOf lib.types.str);
+  };
+
+  config = {
+    machines.roles = lib.zipAttrs
+      (lib.mapAttrsToList
+        (host: cfg:
+          lib.foldl (lib.mergeAttrs) { }
+            (builtins.map (role: { ${role} = host; })
+              cfg.systemRoles))
+        config.machines.hosts);
+  };
+}

common/machine-info/ssh.nix

@@ -0,0 +1,44 @@
+{ config, lib, ... }:
+
+let
+  machines = config.machines;
+
+  sshkeys = keyType: lib.foldl (l: cfg: l ++ cfg.${keyType}) [ ] (builtins.attrValues machines.hosts);
+in
+{
+  options.machines.ssh = {
+    userKeys = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        List of user keys aggregated from all machines.
+      '';
+    };
+
+    deployKeys = lib.mkOption {
+      default = [ ];
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        List of deploy keys aggregated from all machines.
+      '';
+    };
+
+    hostKeysByRole = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.listOf lib.types.str);
+      description = ''
+        Machine host keys divided into their roles.
+      '';
+    };
+  };
+
+  config = {
+    machines.ssh.userKeys = sshkeys "userKeys";
+    machines.ssh.deployKeys = sshkeys "deployKeys";
+
+    machines.ssh.hostKeysByRole = lib.mapAttrs
+      (role: hosts:
+        builtins.map
+          (host: machines.hosts.${host}.hostKey)
+          hosts)
+      machines.roles;
+  };
+}

common/network/default.nix

@@ -7,7 +7,6 @@ let
 in
 {
   imports = [
-    ./hosts.nix
     ./pia-openvpn.nix
     ./pia-wireguard.nix
     ./ping.nix

common/network/hosts.nix

@@ -1,62 +0,0 @@
-{ config, lib, ... }:
-
-with builtins;
-
-let
-  # TODO: remove when all systems are updated to new enough nixpkgs
-  concatMapAttrs =
-    f: with lib; flip pipe [ (mapAttrs f) attrValues (foldl' mergeAttrs { }) ];
-
-  system = (import ../ssh.nix).system;
-
-  # hostnames that resolve on clearnet for LUKS unlocking
-  unlock-clearnet-hosts = {
-    ponyo = "unlock.ponyo.neet.dev";
-    s0 = "s0";
-  };
-
-  # hostnames that resolve on tor for LUKS unlocking
-  unlock-onion-hosts = {
-    liza = "5synsrjgvfzywruomjsfvfwhhlgxqhyofkzeqt2eisyijvjvebnu2xyd.onion";
-    router = "jxx2exuihlls2t6ncs7rvrjh2dssubjmjtclwr2ysvxtr4t7jv55xmqd.onion";
-    ponyo = "cfamr6artx75qvt7ho3rrbsc7mkucmv5aawebwflsfuorusayacffryd.onion";
-    s0 = "r3zvf7f2ppaeithzswigma46pajt3hqytmkg3rshgknbl3jbni455fqd.onion";
-  };
-in {
-  programs.ssh.knownHosts = {
-    ponyo = {
-      hostNames = [ "ponyo" "ponyo.neet.dev" "git.neet.dev" ];
-      publicKey = system.ponyo;
-    };
-    ponyo-unlock = {
-      hostNames = [ unlock-clearnet-hosts.ponyo unlock-onion-hosts.ponyo ];
-      publicKey = system.ponyo-unlock;
-    };
-    router = {
-      hostNames = [ "router" "192.168.1.228" ];
-      publicKey = system.router;
-    };
-    router-unlock = {
-      hostNames = [ unlock-onion-hosts.router ];
-      publicKey = system.router-unlock;
-    };
-    ray = {
-      hostNames = [ "ray" ];
-      publicKey = system.ray;
-    };
-    s0 = {
-      hostNames = [ "s0" ];
-      publicKey = system.s0;
-    };
-    s0-unlock = {
-      hostNames = [ unlock-onion-hosts.s0 ];
-      publicKey = system.s0-unlock;
-    };
-  };
-
-  # prebuilt cmds for easy ssh LUKS unlock
-  environment.shellAliases =
-    concatMapAttrs (host: addr: {"unlock-over-tor_${host}" = "torsocks ssh root@${addr}";}) unlock-onion-hosts
-      //
-    concatMapAttrs (host: addr: {"unlock_${host}" = "ssh root@${addr}";}) unlock-clearnet-hosts;
-}

common/network/pia-openvpn.nix

@@ -108,6 +108,6 @@ in
         };
       };
     };
-    age.secrets."pia-login.conf".file = ../../secrets/pia-login.conf;
+    age.secrets."pia-login.conf".file = ../../secrets/pia-login.age;
   };
 }

common/network/pia-wireguard.nix

@@ -72,7 +72,8 @@ let
   portForwarding = cfg.forwardPortForTransmission || cfg.forwardedPort != null;
 
   containerServiceName = "container@${config.vpn-container.containerName}.service";
-in {
+in
+{
   options.pia.wireguard = {
     enable = mkEnableOption "Enable private internet access";
     badPortForwardPorts = mkOption {
@@ -157,7 +158,7 @@ in {
 
         # restart once a month; PIA forwarded port expires after two months
         # because the container is "PartOf" this unit, it gets restarted too
-        RuntimeMaxSec="30d";
+        RuntimeMaxSec = "30d";
       };
 
       script = ''
@@ -351,6 +352,6 @@ in {
       };
     };
 
-    age.secrets."pia-login.conf".file = ../../secrets/pia-login.conf;
+    age.secrets."pia-login.conf".file = ../../secrets/pia-login.age;
   };
 }

common/network/ping.nix

@@ -18,7 +18,7 @@ let
         requires = [ "network-online.target" ];
         after = [ "network.target" "network-online.target" ];
         wantedBy = [ "multi-user.target" ];
-      serviceConfig.Restart="always";
+        serviceConfig.Restart = "always";
 
         path = with pkgs; [ iputils ];
 
@@ -28,17 +28,18 @@ let
       };
     };
 
-  combineAttrs = foldl recursiveUpdate {};
+  combineAttrs = foldl recursiveUpdate { };
 
   serviceList = map serviceTemplate cfg.hosts;
 
   services = combineAttrs serviceList;
-in {
+in
+{
   options.keepalive-ping = {
     enable = mkEnableOption "Enable keep alive ping task";
     hosts = mkOption {
       type = types.listOf types.str;
-      default = [];
+      default = [ ];
       description = ''
         Hosts to ping periodically
       '';

common/network/tailscale.nix

@@ -11,7 +11,7 @@ in
   config.services.tailscale.enable = mkDefault (!config.boot.isContainer);
 
   # MagicDNS
-  config.networking.nameservers = mkIf cfg.enable [ "1.1.1.1" "8.8.8.8" "100.100.100.100" ];
+  config.networking.nameservers = mkIf cfg.enable [ "1.1.1.1" "8.8.8.8" ];
   config.networking.search = mkIf cfg.enable [ "koi-bebop.ts.net" ];
 
   # exit node

common/network/vpn.nix

@@ -30,7 +30,7 @@ in
 
     config = mkOption {
       type = types.anything;
-      default = {};
+      default = { };
       example = ''
         {
           services.nginx.enable = true;
@@ -70,7 +70,7 @@ in
       localAddress = "172.16.100.2";
 
       config = {
-        imports = allModules ++ [cfg.config];
+        imports = allModules ++ [ cfg.config ];
 
         # speeds up evaluation
         nixpkgs.pkgs = pkgs;

common/pc/audio.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # enable pulseaudio support for packages
     nixpkgs.config.pulseaudio = true;
@@ -16,45 +17,6 @@ in {
       alsa.support32Bit = true;
       pulse.enable = true;
       jack.enable = true;
-
-      # use the example session manager (no others are packaged yet so this is enabled by default,
-      # no need to redefine it in your config for now)
-      #media-session.enable = true;
-
-      config.pipewire = {
-        "context.objects" = [
-          {
-            # A default dummy driver. This handles nodes marked with the "node.always-driver"
-            # properyty when no other driver is currently active. JACK clients need this.
-            factory = "spa-node-factory";
-            args = {
-              "factory.name"     = "support.node.driver";
-              "node.name"        = "Dummy-Driver";
-              "priority.driver"  = 8000;
-            };
-          }
-          {
-            factory = "adapter";
-            args = {
-              "factory.name"     = "support.null-audio-sink";
-              "node.name"        = "Microphone-Proxy";
-              "node.description" = "Microphone";
-              "media.class"      = "Audio/Source/Virtual";
-              "audio.position"   = "MONO";
-            };
-          }
-          {
-            factory = "adapter";
-            args = {
-              "factory.name"     = "support.null-audio-sink";
-              "node.name"        = "Main-Output-Proxy";
-              "node.description" = "Main Output";
-              "media.class"      = "Audio/Sink";
-              "audio.position"   = "FL,FR";
-            };
-          }
-        ];
-      };
     };
 
     users.users.googlebot.extraGroups = [ "audio" ];

common/pc/chromium.nix

@@ -49,7 +49,8 @@ let
     ];
   };
 
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # chromium with specific extensions + settings
     programs.chromium = {

common/pc/default.nix

@@ -2,15 +2,16 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   imports = [
     ./kde.nix
     ./xfce.nix
     ./yubikey.nix
     ./chromium.nix
-#    ./firefox.nix
+    #    ./firefox.nix
     ./audio.nix
-#    ./torbrowser.nix
+    #    ./torbrowser.nix
     ./pithos.nix
     ./spotify.nix
     ./vscodium.nix
@@ -52,6 +53,10 @@ in {
       jellyfin-media-player
       joplin-desktop
       config.inputs.deploy-rs.packages.${config.currentSystem}.deploy-rs
+
+      # For Nix IDE
+      nixpkgs-fmt
+      rnix-lsp
     ];
 
     # Networking

common/pc/discord.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     users.users.googlebot.packages = [
       pkgs.discord

common/pc/kde.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # kde plasma
     services.xserver = {

common/pc/mount-samba.nix

@@ -5,15 +5,17 @@
 let
   cfg = config.services.mount-samba;
 
-  # prevents hanging on network split
-  network_opts = "x-systemd.automount,noauto,x-systemd.idle-timeout=60,x-systemd.device-timeout=5s,x-systemd.mount-timeout=5s,nostrictsync,cache=loose,handlecache,handletimeout=30000,rwpidforward,mapposix,soft,resilienthandles,echo_interval=10,noblocksend";
+  # prevents hanging on network split and other similar niceties to ensure a stable connection
+  network_opts = "nostrictsync,cache=strict,handlecache,handletimeout=30000,rwpidforward,mapposix,soft,resilienthandles,echo_interval=10,noblocksend,fsc";
 
+  systemd_opts = "x-systemd.automount,noauto,x-systemd.idle-timeout=60,x-systemd.device-timeout=5s,x-systemd.mount-timeout=5s";
   user_opts = "uid=${toString config.users.users.googlebot.uid},file_mode=0660,dir_mode=0770,user";
-  auth_opts = "credentials=/run/agenix/smb-secrets";
-  version_opts = "vers=2.1";
+  auth_opts = "sec=ntlmv2i,credentials=/run/agenix/smb-secrets";
+  version_opts = "vers=3.1.1";
 
-  opts = "${network_opts},${user_opts},${version_opts},${auth_opts}";
-in {
+  opts = "${systemd_opts},${network_opts},${user_opts},${version_opts},${auth_opts}";
+in
+{
   options.services.mount-samba = {
     enable = lib.mkEnableOption "enable mounting samba shares";
   };
@@ -33,8 +35,12 @@ in {
 
     age.secrets.smb-secrets.file = ../../secrets/smb-secrets.age;
 
-    # Encrypted Vault
     environment.shellAliases = {
+      # remount storage
+      remount_public = "sudo systemctl restart mnt-public.mount";
+      remount_private = "sudo systemctl restart mnt-private.mount";
+
+      # Encrypted Vault
       vault_unlock = "${pkgs.gocryptfs}/bin/gocryptfs /mnt/private/.vault/ /mnt/vault/";
       vault_lock = "umount /mnt/vault/";
     };

common/pc/pithos.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     nixpkgs.overlays = [
       (self: super: {

common/pc/spotify.nix

@@ -4,7 +4,7 @@ with lib;
 
 let
   cfg = config.services.spotifyd;
-  toml = pkgs.formats.toml {};
+  toml = pkgs.formats.toml { };
   spotifydConf = toml.generate "spotify.conf" cfg.settings;
 in
 {
@@ -17,7 +17,7 @@ in
       enable = mkEnableOption "spotifyd, a Spotify playing daemon";
 
       settings = mkOption {
-        default = {};
+        default = { };
         type = toml.type;
         example = { global.bitrate = 320; };
         description = ''
@@ -28,7 +28,7 @@ in
 
       users = mkOption {
         type = with types; listOf str;
-        default = [];
+        default = [ ];
         description = ''
           Usernames to be added to the "spotifyd" group, so that they
           can start and interact with the userspace daemon.

common/pc/steam.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     programs.steam.enable = true;
     hardware.steam-hardware.enable = true; # steam controller

common/pc/torbrowser.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     nixpkgs.overlays = [
       (self: super: {

common/pc/touchpad.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de.touchpad;
-in {
+in
+{
   options.de.touchpad = {
     enable = lib.mkEnableOption "enable touchpad";
   };

common/pc/vscodium.nix

@@ -4,8 +4,8 @@ let
   cfg = config.de;
 
   extensions = with pkgs.vscode-extensions; [
-#    bbenoist.Nix # nix syntax support
-#    arrterian.nix-env-selector  # nix dev envs
+    #    bbenoist.Nix # nix syntax support
+    #    arrterian.nix-env-selector  # nix dev envs
   ];
 
   vscodium-with-extensions = pkgs.vscode-with-extensions.override {

common/pc/xfce.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     services.xserver = {
       enable = true;

common/pc/yubikey.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.de;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # yubikey
     services.pcscd.enable = true;

common/server/ceph.nix

@@ -3,9 +3,9 @@
 with lib;
 let
   cfg = config.ceph;
-in {
-  options.ceph = {
-  };
+in
+{
+  options.ceph = { };
 
   config = mkIf cfg.enable {
     # ceph.enable = true;

common/server/default.nix

@@ -10,6 +10,7 @@
     ./matrix.nix
     ./zerobin.nix
     ./gitea.nix
+    ./gitea-runner.nix
     ./privatebin/privatebin.nix
     ./radio.nix
     ./samba.nix

common/server/gitea-runner.nix

@@ -0,0 +1,98 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.services.gitea-runner;
+in
+{
+  options.services.gitea-runner = {
+    enable = lib.mkEnableOption "Enables gitea runner";
+    dataDir = lib.mkOption {
+      default = "/var/lib/gitea-runner";
+      type = lib.types.str;
+      description = lib.mdDoc "gitea runner data directory.";
+    };
+    instanceUrl = lib.mkOption {
+      type = lib.types.str;
+    };
+    registrationTokenFile = lib.mkOption {
+      type = lib.types.path;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    virtualisation.docker.enable = true;
+
+    users.users.gitea-runner = {
+      description = "Gitea Runner Service";
+      home = cfg.dataDir;
+      useDefaultShell = true;
+      group = "gitea-runner";
+      isSystemUser = true;
+      createHome = true;
+      extraGroups = [
+        "docker" # allow creating docker containers
+      ];
+    };
+    users.groups.gitea-runner = { };
+
+    # registration token
+    services.gitea-runner.registrationTokenFile = "/run/agenix/gitea-runner-registration-token";
+    age.secrets.gitea-runner-registration-token = {
+      file = ../../secrets/gitea-runner-registration-token.age;
+      owner = "gitea-runner";
+    };
+
+    systemd.services.gitea-runner = {
+      description = "Gitea Runner";
+
+      serviceConfig = {
+        WorkingDirectory = cfg.dataDir;
+        User = "gitea-runner";
+        Group = "gitea-runner";
+      };
+
+      requires = [ "network-online.target" ];
+      after = [ "network.target" "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      path = with pkgs; [ gitea-actions-runner ];
+
+      # based on https://gitea.com/gitea/act_runner/src/branch/main/run.sh
+      script = ''
+        . ${cfg.registrationTokenFile}
+
+        if [[ ! -s .runner ]]; then
+          try=$((try + 1))
+          success=0
+
+          LOGFILE="$(mktemp)"
+
+          # The point of this loop is to make it simple, when running both act_runner and gitea in docker,
+          # for the act_runner to wait a moment for gitea to become available before erroring out.  Within
+          # the context of a single docker-compose, something similar could be done via healthchecks, but
+          # this is more flexible.
+          while [[ $success -eq 0 ]] && [[ $try -lt ''${10:-10} ]]; do
+            act_runner register \
+              --instance "${cfg.instanceUrl}" \
+              --token    "$GITEA_RUNNER_REGISTRATION_TOKEN" \
+              --name     "${config.networking.hostName}" \
+              --no-interactive > $LOGFILE 2>&1
+
+            cat $LOGFILE
+
+            cat $LOGFILE | grep 'Runner registered successfully' > /dev/null
+            if [[ $? -eq 0 ]]; then
+              echo "SUCCESS"
+              success=1
+            else
+              echo "Waiting to retry ..."
+              sleep 5
+            fi
+          done
+        fi
+
+        exec act_runner daemon
+      '';
+    };
+  };
+}

common/server/gitea.nix

@@ -1,8 +1,9 @@
-{ lib, config, ... }:
+{ lib, pkgs, config, ... }:
 
 let
   cfg = config.services.gitea;
-in {
+in
+{
   options.services.gitea = {
     hostname = lib.mkOption {
       type = lib.types.str;
@@ -15,7 +16,7 @@ in {
       rootUrl = "https://${cfg.hostname}/";
       appName = cfg.hostname;
       # lfs.enable = true;
-      dump.enable = true;
+      # dump.enable = true;
       settings = {
         other = {
           SHOW_FOOTER_VERSION = false;
@@ -29,8 +30,31 @@ in {
         session = {
           COOKIE_SECURE = true;
         };
+        mailer = {
+          ENABLED = true;
+          MAILER_TYPE = "smtp";
+          SMTP_ADDR = "mail.neet.dev";
+          SMTP_PORT = "465";
+          IS_TLS_ENABLED = true;
+          USER = "robot@runyan.org";
+          FROM = "no-reply@neet.dev";
+        };
+        actions = {
+          ENABLED = true;
         };
       };
+      mailerPasswordFile = "/run/agenix/robots-email-pw";
+    };
+    age.secrets.robots-email-pw = {
+      file = ../../secrets/robots-email-pw.age;
+      owner = config.services.gitea.user;
+    };
+
+    # backups
+    backup.group."gitea".paths = [
+      config.services.gitea.stateDir
+    ];
+
     services.nginx.enable = true;
     services.nginx.virtualHosts.${cfg.hostname} = {
       enableACME = true;

common/server/hydra.nix

@@ -20,6 +20,6 @@ in
     hydraURL = "https://${domain}";
     useSubstitutes = true;
     notificationSender = notifyEmail;
-    buildMachinesFiles = [];
+    buildMachinesFiles = [ ];
   };
 }

common/server/icecast.nix

@@ -7,7 +7,8 @@
 
 let
   cfg = config.services.icecast;
-in {
+in
+{
   options.services.icecast = {
     mount = lib.mkOption {
       type = lib.types.str;

common/server/iodine.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.services.iodine.server;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # iodine DNS-based vpn
     services.iodine.server = {

common/server/mailserver.nix

@@ -15,7 +15,8 @@ let
     "bsd.ninja"
     "bsd.rocks"
   ];
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     # kresd doesn't work with tailscale MagicDNS
     mailserver.localDnsResolver = false;
@@ -32,10 +33,18 @@ in {
       inherit domains;
       loginAccounts = {
         "jeremy@runyan.org" = {
-          hashedPasswordFile = "/run/agenix/email-pw";
+          hashedPasswordFile = "/run/agenix/hashed-email-pw";
           # catchall for all domains
           aliases = map (domain: "@${domain}") domains;
         };
+        "robot@runyan.org" = {
+          aliases = [
+            "no-reply@neet.dev"
+            "robot@neet.dev"
+          ];
+          sendOnly = true;
+          hashedPasswordFile = "/run/agenix/hashed-robots-email-pw";
+        };
       };
       rejectRecipients = [
         "george@runyan.org"
@@ -45,7 +54,8 @@ in {
       ];
       certificateScheme = 3; # use let's encrypt for certs
     };
-    age.secrets.email-pw.file = ../../secrets/email-pw.age;
+    age.secrets.hashed-email-pw.file = ../../secrets/hashed-email-pw.age;
+    age.secrets.hashed-robots-email-pw.file = ../../secrets/hashed-robots-email-pw.age;
 
     # sendmail to use xxx@domain instead of xxx@mail.domain
     services.postfix.origin = "$mydomain";
@@ -60,9 +70,11 @@ in {
       sender_dependent_relayhost_maps = "hash:/var/lib/postfix/conf/sender_relay";
       smtp_sender_dependent_authentication = "yes";
     };
-    services.postfix.mapFiles.sender_relay = let
+    services.postfix.mapFiles.sender_relay =
+      let
         relayHost = "[smtp.mailgun.org]:587";
-    in pkgs.writeText "sender_relay"
+      in
+      pkgs.writeText "sender_relay"
         (concatStringsSep "\n" (map (domain: "@${domain} ${relayHost}") domains));
     services.postfix.mapFiles.sasl_relay_passwd = "/run/agenix/sasl_relay_passwd";
     age.secrets.sasl_relay_passwd.file = ../../secrets/sasl_relay_passwd.age;
@@ -79,5 +91,10 @@ in {
         $config['smtp_pass'] = "%p";
       '';
     };
+
+    # backups
+    backup.group."email".paths = [
+      config.mailserver.mailDirectory
+    ];
   };
 }

common/server/matrix.nix

@@ -3,7 +3,8 @@
 let
   cfg = config.services.matrix;
   certs = config.security.acme.certs;
-in {
+in
+{
   options.services.matrix = {
     enable = lib.mkEnableOption "enable matrix";
     element-web = {
@@ -62,15 +63,15 @@ in {
       settings = {
         server_name = cfg.host;
         enable_registration = cfg.enable_registration;
-        listeners = [ {
-          bind_addresses = ["127.0.0.1"];
+        listeners = [{
+          bind_addresses = [ "127.0.0.1" ];
           port = cfg.port;
           tls = false;
-          resources = [ {
+          resources = [{
             compress = true;
             names = [ "client" "federation" ];
-          } ];
-        } ];
+          }];
+        }];
         turn_uris = [
           "turn:${cfg.turn.host}:${toString cfg.turn.port}?transport=udp"
           "turn:${cfg.turn.host}:${toString cfg.turn.port}?transport=tcp"
@@ -137,7 +138,8 @@ in {
         ];
         locations."/".proxyPass = "http://localhost:${toString cfg.port}";
       };
-      virtualHosts.${cfg.turn.host} =  { # get TLS cert for TURN server
+      virtualHosts.${cfg.turn.host} = {
+        # get TLS cert for TURN server
         enableACME = true;
         forceSSL = true;
       };

common/server/mumble.nix

@@ -3,7 +3,8 @@
 let
   cfg = config.services.murmur;
   certs = config.security.acme.certs;
-in {
+in
+{
   options.services.murmur.domain = lib.mkOption {
     type = lib.types.str;
   };

common/server/nextcloud.nix

@@ -3,7 +3,8 @@
 
 let
   cfg = config.services.nextcloud;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     services.nextcloud = {
       https = true;
@@ -19,6 +20,12 @@ in {
       file = ../../secrets/nextcloud-pw.age;
       owner = "nextcloud";
     };
+
+    # backups
+    backup.group."nextcloud".paths = [
+      config.services.nextcloud.home
+    ];
+
     services.nginx.virtualHosts.${config.services.nextcloud.hostName} = {
       enableACME = true;
       forceSSL = true;

common/server/nginx-stream.nix

@@ -5,7 +5,8 @@ let
   nginxWithRTMP = pkgs.nginx.override {
     modules = [ pkgs.nginxModules.rtmp ];
   };
-in {
+in
+{
   options.services.nginx.stream = {
     enable = lib.mkEnableOption "enable nginx rtmp/hls/dash video streaming";
     port = lib.mkOption {

common/server/nginx.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.services.nginx;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     services.nginx = {
       recommendedGzipSettings = true;

common/server/owncast.nix

@@ -4,7 +4,8 @@ with lib;
 
 let
   cfg = config.services.owncast;
-in {
+in
+{
   options.services.owncast = {
     hostname = lib.mkOption {
       type = types.str;

common/server/privatebin/privatebin.nix

@@ -14,7 +14,8 @@ let
       cp -ar $src $out
     '';
   };
-in {
+in
+{
   options.services.privatebin = {
     enable = lib.mkEnableOption "enable privatebin";
     host = lib.mkOption {
@@ -30,7 +31,7 @@ in {
       group = "privatebin";
       isSystemUser = true;
     };
-    users.groups.privatebin = {};
+    users.groups.privatebin = { };
 
     services.nginx.enable = true;
     services.nginx.virtualHosts.${cfg.host} = {

common/server/radio.nix

@@ -3,7 +3,8 @@
 let
   cfg = config.services.radio;
   radioPackage = config.inputs.radio.packages.${config.currentSystem}.radio;
-in {
+in
+{
   options.services.radio = {
     enable = lib.mkEnableOption "enable radio";
     user = lib.mkOption {
@@ -56,11 +57,11 @@ in {
       home = cfg.dataDir;
       createHome = true;
     };
-    users.groups.${cfg.group} = {};
+    users.groups.${cfg.group} = { };
     systemd.services.radio = {
       enable = true;
-      after = ["network.target"];
-      wantedBy = ["multi-user.target"];
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
       serviceConfig.ExecStart = "${radioPackage}/bin/radio ${config.services.icecast.listen.address}:${toString config.services.icecast.listen.port} ${config.services.icecast.mount} 5500";
       serviceConfig.User = cfg.user;
       serviceConfig.Group = cfg.group;

common/server/samba.nix

@@ -25,9 +25,7 @@
         printing = cups
         printcap name = cups
 
-        # horrible files
-        veto files = /._*/.DS_Store/ /._*/._.DS_Store/
-        delete veto files = yes
+        hide files = /.nobackup/.DS_Store/._.DS_Store/
       '';
 
       shares = {
@@ -77,6 +75,13 @@
       };
     };
 
+    # backups
+    backup.group."samba".paths = [
+      config.services.samba.shares.googlebot.path
+      config.services.samba.shares.cris.path
+      config.services.samba.shares.public.path
+    ];
+
     # Windows discovery of samba server
     services.samba-wsdd = {
       enable = true;
@@ -110,6 +115,6 @@
     # samba user for share
     users.users.cris.isSystemUser = true;
     users.users.cris.group = "cris";
-    users.groups.cris = {};
+    users.groups.cris = { };
   };
 }

common/server/searx.nix

@@ -2,19 +2,20 @@
 
 let
   cfg = config.services.searx;
-in {
+in
+{
   config = lib.mkIf cfg.enable {
     services.searx = {
       environmentFile = "/run/agenix/searx";
       settings = {
         server.port = 43254;
         server.secret_key = "@SEARX_SECRET_KEY@";
-        engines = [ {
+        engines = [{
           name = "wolframalpha";
           shortcut = "wa";
           api_key = "@WOLFRAM_API_KEY@";
           engine = "wolframalpha_api";
-        } ];
+        }];
       };
     };
     services.nginx.virtualHosts."search.neet.space" = {

common/server/thelounge.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.services.thelounge;
-in {
+in
+{
   options.services.thelounge = {
     fileUploadBaseUrl = lib.mkOption {
       type = lib.types.str;
@@ -42,6 +43,10 @@ in {
       };
     };
 
+    backup.group."thelounge".paths = [
+      "/var/lib/thelounge/"
+    ];
+
     # the lounge client
     services.nginx.virtualHosts.${cfg.host} = {
       enableACME = true;

common/server/video-stream.nix

@@ -15,14 +15,14 @@ let
 in
 {
   networking.firewall.allowedUDPPorts = [ rtp-port ];
-  networking.firewall.allowedTCPPortRanges = [ {
+  networking.firewall.allowedTCPPortRanges = [{
     from = webrtc-peer-lower-port;
     to = webrtc-peer-upper-port;
-  } ];
-  networking.firewall.allowedUDPPortRanges = [ { 
+  }];
+  networking.firewall.allowedUDPPortRanges = [{
     from = webrtc-peer-lower-port;
     to = webrtc-peer-upper-port;
-  } ];
+  }];
 
   virtualisation.docker.enable = true;
 
@@ -49,12 +49,12 @@ in
         ports = [
           "${toStr ingest-port}:8084"
         ];
-#        imageFile = pkgs.dockerTools.pullImage {
-#          imageName = "projectlightspeed/ingest";
-#          finalImageTag = "version-0.1.4";
-#          imageDigest = "sha256:9fc51833b7c27a76d26e40f092b9cec1ac1c4bfebe452e94ad3269f1f73ff2fc";
-#          sha256 = "19kxl02x0a3i6hlnsfcm49hl6qxnq2f3hfmyv1v8qdaz58f35kd5";
-#        };
+        #        imageFile = pkgs.dockerTools.pullImage {
+        #          imageName = "projectlightspeed/ingest";
+        #          finalImageTag = "version-0.1.4";
+        #          imageDigest = "sha256:9fc51833b7c27a76d26e40f092b9cec1ac1c4bfebe452e94ad3269f1f73ff2fc";
+        #          sha256 = "19kxl02x0a3i6hlnsfcm49hl6qxnq2f3hfmyv1v8qdaz58f35kd5";
+        #        };
       };
       "lightspeed-react" = {
         workdir = "/var/lib/lightspeed-react";
@@ -62,12 +62,12 @@ in
         ports = [
           "${toStr web-port}:80"
         ];
-#        imageFile = pkgs.dockerTools.pullImage {
-#          imageName = "projectlightspeed/react";
-#          finalImageTag = "version-0.1.3";
-#          imageDigest = "sha256:b7c58425f1593f7b4304726b57aa399b6e216e55af9c0962c5c19333fae638b6";
-#          sha256 = "0d2jh7mr20h7dxgsp7ml7cw2qd4m8ja9rj75dpy59zyb6v0bn7js";
-#        };
+        #        imageFile = pkgs.dockerTools.pullImage {
+        #          imageName = "projectlightspeed/react";
+        #          finalImageTag = "version-0.1.3";
+        #          imageDigest = "sha256:b7c58425f1593f7b4304726b57aa399b6e216e55af9c0962c5c19333fae638b6";
+        #          sha256 = "0d2jh7mr20h7dxgsp7ml7cw2qd4m8ja9rj75dpy59zyb6v0bn7js";
+        #        };
       };
       "lightspeed-webrtc" = {
         workdir = "/var/lib/lightspeed-webrtc";
@@ -79,15 +79,18 @@ in
           "${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}:${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}/udp"
         ];
         cmd = [
-          "lightspeed-webrtc" "--addr=0.0.0.0" "--ip=${domain}"
-          "--ports=${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}" "run"
+          "lightspeed-webrtc"
+          "--addr=0.0.0.0"
+          "--ip=${domain}"
+          "--ports=${toStr webrtc-peer-lower-port}-${toStr webrtc-peer-upper-port}"
+          "run"
         ];
-#        imageFile = pkgs.dockerTools.pullImage {
-#          imageName = "projectlightspeed/webrtc";
-#          finalImageTag = "version-0.1.2";
-#          imageDigest = "sha256:ddf8b3dd294485529ec11d1234a3fc38e365a53c4738998c6bc2c6930be45ecf";
-#          sha256 = "1bdy4ak99fjdphj5bsk8rp13xxmbqdhfyfab14drbyffivg9ad2i";
-#        };
+        #        imageFile = pkgs.dockerTools.pullImage {
+        #          imageName = "projectlightspeed/webrtc";
+        #          finalImageTag = "version-0.1.2";
+        #          imageDigest = "sha256:ddf8b3dd294485529ec11d1234a3fc38e365a53c4738998c6bc2c6930be45ecf";
+        #          sha256 = "1bdy4ak99fjdphj5bsk8rp13xxmbqdhfyfab14drbyffivg9ad2i";
+        #        };
       };
     };
   };

common/server/vscode/modules/vscode-server/default.nix

@@ -1,8 +1,8 @@
 import ./module.nix ({ name, description, serviceConfig }:
 
-{
+  {
     systemd.user.services.${name} = {
       inherit description serviceConfig;
       wantedBy = [ "default.target" ];
     };
-})
+  })

common/server/vscode/modules/vscode-server/home.nix

@@ -1,6 +1,6 @@
 import ./module.nix ({ name, description, serviceConfig }:
 
-{
+  {
     systemd.user.services.${name} = {
       Unit = {
         Description = description;
@@ -12,4 +12,4 @@ import ./module.nix ({ name, description, serviceConfig }:
         WantedBy = [ "default.target" ];
       };
     };
-})
+  })

common/server/zerobin.nix

@@ -2,7 +2,8 @@
 
 let
   cfg = config.services.zerobin;
-in {
+in
+{
   options.services.zerobin = {
     host = lib.mkOption {
       type = lib.types.str;

common/shell.nix

@@ -21,6 +21,8 @@
     shellInit = ''
       # disable annoying fish shell greeting
       set fish_greeting
+
+      alias sudo="doas"
     '';
   };
 

common/ssh.nix

@@ -1,40 +1,38 @@
-rec {
-  users = [
+{ config, lib, pkgs, ... }:
+
+{
+  programs.ssh.knownHosts = lib.filterAttrs (n: v: v != null) (lib.concatMapAttrs
+    (host: cfg: {
+      ${host} = {
+        hostNames = cfg.hostNames;
+        publicKey = cfg.hostKey;
+      };
+      "${host}-remote-unlock" =
+        if cfg.remoteUnlock != null then {
+          hostNames = builtins.filter (h: h != null) [ cfg.remoteUnlock.clearnetHost cfg.remoteUnlock.onionHost ];
+          publicKey = cfg.remoteUnlock.hostKey;
+        } else null;
+    })
+    config.machines.hosts);
+
+  # prebuilt cmds for easy ssh LUKS unlock
+  environment.shellAliases =
+    let
+      unlockHosts = unlockType: lib.concatMapAttrs
+        (host: cfg:
+          if cfg.remoteUnlock != null && cfg.remoteUnlock.${unlockType} != null then {
+            ${host} = cfg.remoteUnlock.${unlockType};
+          } else { })
+        config.machines.hosts;
+    in
+    lib.concatMapAttrs (host: addr: { "unlock-over-tor_${host}" = "torsocks ssh root@${addr}"; }) (unlockHosts "onionHost")
+    //
+    lib.concatMapAttrs (host: addr: { "unlock_${host}" = "ssh root@${addr}"; }) (unlockHosts "clearnetHost");
+
+  # TODO: Old ssh keys I will remove some day...
+  machines.ssh.userKeys = [
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVR/R3ZOsv7TZbICGBCHdjh1NDT8SnswUyINeJOC7QG"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0dcqL/FhHmv+a1iz3f9LJ48xubO7MZHy35rW9SZOYM"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHSkKiRUUmnErOKGx81nyge/9KqjkPh8BfDk0D3oP586" # nat
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeTK1iARlNIKP/DS8/ObBm9yUM/3L1Ub4XI5A2r9OzP" # ray
-  ] ++ higherTrustUserKeys;
-  system = {
-    ponyo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";
-    ponyo-unlock = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC9LQuuImgWlkjDhEEIbM1wOd+HqRv1RxvYZuLXPSdRi";
-    ray = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDQM8hwKRgl8cZj7UVYATSLYu4LhG7I0WFJ9m2iWowiB";
-    router = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFr2IHmWFlaLaLp5dGoSmFEYKA/eg2SwGXAogaOmLsHL";
-    router-unlock = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJOw5dTPmtKqiPBH6VKyz5MYBubn8leAh5Eaw7s/O85c";
-    s0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";
-    s0-unlock = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFNiceeFMos5ZXcYem4yFxh8PiZNNnuvhlyLbQLrgIZH";
-  };
-
-  higherTrustUserKeys = [
-    "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEaGIwLiUa6wQLlEF+keQOIYy/tCmJvV6eENzUQjSqW2AAAABHNzaDo=" # ray fido
-  ];
-
-  # groups
-  systems = with system; [
-    ponyo
-    ray
-    router
-    s0
-  ];
-  personal = with system; [
-    ray
-  ];
-  servers = with system; [
-    ponyo
-    router
-    s0
-  ];
-  storage = with system; [
-    s0
   ];
 }

flake.lock

@@ -8,11 +8,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1675176355,
-        "narHash": "sha256-Qjxh5cmN56siY97mzmBLI1+cdjXSPqmfPVsKxBvHmwI=",
+        "lastModified": 1682101079,
+        "narHash": "sha256-MdAhtjrLKnk2uiqun1FWABbKpLH090oeqCSiWemtuck=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "b7ffcfe77f817d9ee992640ba1f270718d197f28",
+        "rev": "2994d002dcff5353ca1ac48ec584c7f6589fe447",
         "type": "github"
       },
       "original": {
@@ -117,11 +117,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1674127017,
-        "narHash": "sha256-QO1xF7stu5ZMDLbHN30LFolMAwY6TVlzYvQoUs1RD68=",
+        "lastModified": 1682063650,
+        "narHash": "sha256-VaDHh2z6xlnTHaONlNVHP7qEMcK5rZ8Js3sT6mKb2XY=",
         "owner": "serokell",
         "repo": "deploy-rs",
-        "rev": "8c9ea9605eed20528bf60fae35a2b613b901fd77",
+        "rev": "c2ea4e642dc50fd44b537e9860ec95867af30d39",
         "type": "github"
       },
       "original": {
@@ -147,12 +147,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
         "type": "github"
       },
       "original": {
@@ -168,11 +171,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1677382901,
-        "narHash": "sha256-2idFWlTVG+qUZkU2/W50amGSIxmN56igIkMAXKbv4S4=",
+        "lastModified": 1681591833,
+        "narHash": "sha256-lW+xOELafAs29yw56FG4MzNOFkh8VHC/X/tRs1wsGn8=",
         "owner": "Mic92",
         "repo": "nix-index-database",
-        "rev": "4306fa7c12e098360439faac1a2e6b8e509ec97c",
+        "rev": "68ec961c51f48768f72d2bbdb396ce65a316677e",
         "type": "github"
       },
       "original": {
@@ -183,11 +186,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1677823547,
-        "narHash": "sha256-xD2qco8Pw8HAXgjf9OSi2H2N20WaTrtvgcl21525kVE=",
+        "lastModified": 1682133240,
+        "narHash": "sha256-s6yRsI/7V+k/+rckp0+/2cs/UXnea3SEfMpy95QiGcc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "78c4d33c16092e535bc4ba1284ba49e3e138483a",
+        "rev": "8dafae7c03d6aa8c2ae0a0612fbcb47e994e3fb8",
         "type": "github"
       },
       "original": {
@@ -212,6 +215,18 @@
         "type": "indirect"
       }
     },
+    "nixpkgs-hostapd-pr": {
+      "flake": false,
+      "locked": {
+        "narHash": "sha256-1rGQKcB1jeRPc1n021ulyOVkA6L6xmNYKmeqQ94+iRc=",
+        "type": "file",
+        "url": "https://github.com/NixOS/nixpkgs/pull/222536.patch"
+      },
+      "original": {
+        "type": "file",
+        "url": "https://github.com/NixOS/nixpkgs/pull/222536.patch"
+      }
+    },
     "radio": {
       "inputs": {
         "flake-utils": [
@@ -262,6 +277,7 @@
         "flake-utils": "flake-utils",
         "nix-index-database": "nix-index-database",
         "nixpkgs": "nixpkgs",
+        "nixpkgs-hostapd-pr": "nixpkgs-hostapd-pr",
         "radio": "radio",
         "radio-web": "radio-web",
         "simple-nixos-mailserver": "simple-nixos-mailserver"
@@ -291,6 +307,21 @@
         "type": "gitlab"
       }
     },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "utils": {
       "locked": {
         "lastModified": 1605370193,

flake.nix

@@ -39,13 +39,23 @@
     # prebuilt nix-index database
     nix-index-database.url = "github:Mic92/nix-index-database";
     nix-index-database.inputs.nixpkgs.follows = "nixpkgs";
+
+    nixpkgs-hostapd-pr.url = "https://github.com/NixOS/nixpkgs/pull/222536.patch";
+    nixpkgs-hostapd-pr.flake = false;
   };
 
-  outputs = { self, nixpkgs, ... }@inputs: {
-
+  outputs = { self, nixpkgs, ... }@inputs:
+    let
+      machines = (import ./common/machine-info/moduleless.nix
+        {
+          inherit nixpkgs;
+          assertionsModule = "${nixpkgs}/nixos/modules/misc/assertions.nix";
+        }).machines.hosts;
+    in
+    {
       nixosConfigurations =
         let
-      modules = system: with inputs; [
+          modules = system: hostname: with inputs; [
             ./common
             simple-nixos-mailserver.nixosModule
             agenix.nixosModules.default
@@ -53,48 +63,52 @@
             archivebox.nixosModule
             nix-index-database.nixosModules.nix-index
             ({ lib, ... }: {
-          config.environment.systemPackages = [
+              config = {
+                environment.systemPackages = [
                   agenix.packages.${system}.agenix
                 ];
 
+                networking.hostName = hostname;
+              };
+
               # because nixos specialArgs doesn't work for containers... need to pass in inputs a different way
               options.inputs = lib.mkOption { default = inputs; };
               options.currentSystem = lib.mkOption { default = system; };
             })
           ];
 
-      mkSystem = system: nixpkgs: path:
+          mkSystem = system: nixpkgs: path: hostname:
             let
-          allModules = modules system;
+              allModules = modules system hostname;
 
               # allow patching nixpkgs, remove this hack once this is solved: https://github.com/NixOS/nix/issues/3920
               patchedNixpkgsSrc = nixpkgs.legacyPackages.${system}.applyPatches {
                 name = "nixpkgs-patched";
                 src = nixpkgs;
                 patches = [
-              # inputs.nixpkgs-patch-howdy
+                  inputs.nixpkgs-hostapd-pr
+                  ./patches/kexec-luks.patch
                 ];
               };
-          patchedNixpkgs = nixpkgs.lib.fix (self: (import "${patchedNixpkgsSrc}/flake.nix").outputs { self=nixpkgs; });
+              patchedNixpkgs = nixpkgs.lib.fix (self: (import "${patchedNixpkgsSrc}/flake.nix").outputs { self = nixpkgs; });
 
-        in patchedNixpkgs.lib.nixosSystem {
+            in
+            patchedNixpkgs.lib.nixosSystem {
               inherit system;
-          modules = allModules ++ [path];
+              modules = allModules ++ [ path ];
 
               specialArgs = {
                 inherit allModules;
               };
             };
         in
-    {
-      "ray" = mkSystem "x86_64-linux" nixpkgs ./machines/ray/configuration.nix;
-      # "nat" = mkSystem "aarch64-linux" nixpkgs ./machines/nat/configuration.nix;
-      "ponyo" = mkSystem "x86_64-linux" nixpkgs ./machines/ponyo/configuration.nix;
-      "router" = mkSystem "x86_64-linux" nixpkgs ./machines/router/configuration.nix;
-      "s0" = mkSystem "x86_64-linux" nixpkgs ./machines/storage/s0/configuration.nix;
-    };
+        nixpkgs.lib.mapAttrs
+          (hostname: cfg:
+            mkSystem cfg.arch nixpkgs cfg.configurationPath hostname)
+          machines;
 
-    packages = let
+      packages =
+        let
           mkKexec = system:
             (nixpkgs.lib.nixosSystem {
               inherit system;
@@ -105,7 +119,8 @@
               inherit system;
               modules = [ ./machines/ephemeral/iso.nix ];
             }).config.system.build.isoImage;
-    in {
+        in
+        {
           "x86_64-linux"."kexec" = mkKexec "x86_64-linux";
           "x86_64-linux"."iso" = mkIso "x86_64-linux";
           "aarch64-linux"."kexec" = mkKexec "aarch64-linux";
@@ -114,18 +129,17 @@
 
       deploy.nodes =
         let
-        mkDeploy = configName: hostname: {
+          mkDeploy = configName: arch: hostname: {
             inherit hostname;
             magicRollback = false;
             sshUser = "root";
-          profiles.system.path = inputs.deploy-rs.lib.x86_64-linux.activate.nixos self.nixosConfigurations.${configName};
-        };
-
-      in {
-        s0 = mkDeploy "s0" "s0";
-        router = mkDeploy "router" "192.168.1.228";
-        ponyo = mkDeploy "ponyo" "ponyo.neet.dev";
+            profiles.system.path = inputs.deploy-rs.lib.${arch}.activate.nixos self.nixosConfigurations.${configName};
           };
+        in
+        nixpkgs.lib.mapAttrs
+          (hostname: cfg:
+            mkDeploy hostname cfg.arch (builtins.head cfg.hostNames))
+          machines;
 
       checks = builtins.mapAttrs (system: deployLib: deployLib.deployChecks self.deploy) inputs.deploy-rs.lib;
     };

machines/ephemeral/minimal.nix

@@ -1,13 +1,16 @@
-{ pkgs, modulesPath, ... }:
+{ config, pkgs, modulesPath, ... }:
 
 {
   imports = [
     (modulesPath + "/installer/cd-dvd/channel.nix")
+    ../../common/machine-info
+    ../../common/ssh.nix
   ];
 
   boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "e1000" "e1000e" "virtio_pci" "r8169" ];
   boot.kernelParams = [
-    "panic=30" "boot.panic_on_fail" # reboot the machine upon fatal boot issues
+    "panic=30"
+    "boot.panic_on_fail" # reboot the machine upon fatal boot issues
     "console=ttyS0,115200" # enable serial console
     "console=tty1"
   ];
@@ -15,13 +18,16 @@
 
   boot.kernelPackages = pkgs.linuxPackages_latest;
 
+  system.stateVersion = "21.11";
+
   # hardware.enableAllFirmware = true;
   # nixpkgs.config.allowUnfree = true;
 
   environment.systemPackages = with pkgs; [
     cryptsetup
     btrfs-progs
-    git git-lfs
+    git
+    git-lfs
     wget
     htop
     dnsutils
@@ -36,10 +42,12 @@
 
   services.openssh = {
     enable = true;
-    challengeResponseAuthentication = false;
-    passwordAuthentication = false;
+    settings = {
+      KbdInteractiveAuthentication = false;
+      PasswordAuthentication = false;
+    };
   };
 
   services.getty.autologinUser = "root";
-  users.users.root.openssh.authorizedKeys.keys = (import ../../common/ssh.nix).users;
+  users.users.root.openssh.authorizedKeys.keys = config.machines.ssh.userKeys;
 }

machines/nat/hardware-configuration.nix

@@ -12,12 +12,14 @@
   boot.extraModulePackages = [ ];
 
   fileSystems."/" =
-    { device = "/dev/disk/by-uuid/02a8c0c7-fd4e-4443-a83c-2d0b63848779";
+    {
+      device = "/dev/disk/by-uuid/02a8c0c7-fd4e-4443-a83c-2d0b63848779";
       fsType = "btrfs";
     };
 
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/0C95-1290";
+    {
+      device = "/dev/disk/by-uuid/0C95-1290";
       fsType = "vfat";
     };
 

machines/phil/default.nix

@@ -0,0 +1,14 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+  ];
+
+  services.gitea-runner = {
+    enable = true;
+    instanceUrl = "https://git.neet.dev";
+  };
+
+  system.autoUpgrade.enable = true;
+}

machines/phil/hardware-configuration.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [
+      (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  # because grub just doesn't work for some reason
+  boot.loader.systemd-boot.enable = true;
+
+  remoteLuksUnlock.enable = true;
+  remoteLuksUnlock.enableTorUnlock = false;
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  luks.devices = [ "/dev/disk/by-uuid/d26c1820-4c39-4615-98c2-51442504e194" ];
+
+  fileSystems."/" =
+    {
+      device = "/dev/disk/by-uuid/851bfde6-93cd-439e-9380-de28aa87eda9";
+      fsType = "btrfs";
+    };
+
+  fileSystems."/boot" =
+    {
+      device = "/dev/disk/by-uuid/F185-C4E5";
+      fsType = "vfat";
+    };
+
+  swapDevices =
+    [{ device = "/dev/disk/by-uuid/d809e3a1-3915-405a-a200-4429c5efdf87"; }];
+
+  networking.interfaces.enp0s6.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
+}

machines/phil/properties.nix

@@ -0,0 +1,20 @@
+{
+  hostNames = [
+    "phil"
+    "phil.neet.dev"
+  ];
+
+  arch = "aarch64-linux";
+
+  systemRoles = [
+    "server"
+    "gitea-runner"
+  ];
+
+  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBlgRPpuUkZqe8/lHugRPm/m2vcN9psYhh5tENHZt9I2";
+
+  remoteUnlock = {
+    hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0RodotOXLMy/w70aa096gaNqPBnfgiXR5ZAH4+wGzd";
+    clearnetHost = "unlock.phil.neet.dev";
+  };
+}

machines/ponyo/default.nix

@@ -1,13 +1,14 @@
 { config, pkgs, lib, ... }:
 
 {
-  imports =[
+  imports = [
     ./hardware-configuration.nix
   ];
 
-  networking.hostName = "ponyo";
-
   system.autoUpgrade.enable = true;
+  # I want to manually trigger kexec updates for now on ponyo
+  system.autoUpgrade.allowKexec = false;
+  luks.enableKexec = true;
 
   # p2p mesh network
   services.tailscale.exitNode = true;
@@ -52,6 +53,9 @@
     file = ../../secrets/wolframalpha.age;
     owner = config.services.drastikbot.user;
   };
+  backup.group."dailybot".paths = [
+    config.services.drastikbot.dataDir
+  ];
 
   # music radio
   vpn-container.enable = true;
@@ -61,7 +65,7 @@
       host = "radio.runyan.org";
     };
   };
-  pia.wireguard.badPortForwardPorts = [];
+  pia.wireguard.badPortForwardPorts = [ ];
   services.nginx.virtualHosts."radio.runyan.org" = {
     enableACME = true;
     forceSSL = true;

machines/ponyo/hardware-configuration.nix

@@ -2,7 +2,8 @@
 
 {
   imports =
-    [ (modulesPath + "/profiles/qemu-guest.nix")
+    [
+      (modulesPath + "/profiles/qemu-guest.nix")
     ];
 
   boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" ];
@@ -18,16 +19,21 @@
   };
 
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv".device = "/dev/disk/by-uuid/4cc36be4-dbff-4afe-927d-69bf4637bae2";
-  boot.initrd.luks.devices."enc-pv2".device = "/dev/disk/by-uuid/e52b01b3-81c8-4bb2-ae7e-a3d9c793cb00"; # expanded disk
+
+  luks.devices = [
+    "/dev/disk/by-uuid/4cc36be4-dbff-4afe-927d-69bf4637bae2"
+    "/dev/disk/by-uuid/e52b01b3-81c8-4bb2-ae7e-a3d9c793cb00"
+  ];
 
   fileSystems."/" =
-    { device = "/dev/mapper/enc-pv";
+    {
+      device = "/dev/mapper/enc-pv1";
       fsType = "btrfs";
     };
 
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/d3a3777d-1e70-47fa-a274-804dc70ee7fd";
+    {
+      device = "/dev/disk/by-uuid/d3a3777d-1e70-47fa-a274-804dc70ee7fd";
       fsType = "ext4";
     };
 

machines/ponyo/properties.nix

@@ -0,0 +1,28 @@
+{
+  hostNames = [
+    "ponyo"
+    "ponyo.neet.dev"
+    "git.neet.dev"
+  ];
+
+  arch = "x86_64-linux";
+
+  systemRoles = [
+    "server"
+    "email-server"
+    "iodine"
+    "pia"
+    "nextcloud"
+    "dailybot"
+    "gitea"
+  ];
+
+  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBBlTAIp38RhErU1wNNV5MBeb+WGH0mhF/dxh5RsAXN";
+
+  remoteUnlock = {
+    hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC9LQuuImgWlkjDhEEIbM1wOd+HqRv1RxvYZuLXPSdRi";
+
+    clearnetHost = "unlock.ponyo.neet.dev";
+    onionHost = "cfamr6artx75qvt7ho3rrbsc7mkucmv5aawebwflsfuorusayacffryd.onion";
+  };
+}

machines/ray/default.nix

@@ -5,8 +5,6 @@
     ./hardware-configuration.nix
   ];
 
-  networking.hostName = "ray";
-
   # for luks onlock over tor
   services.tor.enable = true;
   services.tor.client.enable = true;
@@ -18,10 +16,20 @@
   hardware.openrazer.devicesOffOnScreensaver = false;
   users.users.googlebot.packages = [ pkgs.polychromatic ];
 
-  # depthai
   services.udev.extraRules = ''
+    # depthai
     SUBSYSTEM=="usb", ATTRS{idVendor}=="03e7", MODE="0666"
+
+    # Moonlander
+    # Rules for Oryx web flashing and live training
+    KERNEL=="hidraw*", ATTRS{idVendor}=="16c0", MODE="0664", GROUP="plugdev"
+    KERNEL=="hidraw*", ATTRS{idVendor}=="3297", MODE="0664", GROUP="plugdev"
+    # Wally Flashing rules for the Moonlander and Planck EZ
+    SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="df11", MODE:="0666", SYMLINK+="stm32_dfu"
   '';
+  users.groups.plugdev = {
+    members = [ "googlebot" ];
+  };
 
   # virt-manager
   virtualisation.libvirtd.enable = true;
@@ -37,6 +45,9 @@
 
   virtualisation.docker.enable = true;
 
+  virtualisation.appvm.enable = true;
+  virtualisation.appvm.user = "googlebot";
+
   services.mount-samba.enable = true;
 
   de.enable = true;

machines/ray/hardware-configuration.nix

@@ -5,7 +5,8 @@
 
 {
   imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
+    [
+      (modulesPath + "/installer/scan/not-detected.nix")
     ];
 
   # boot
@@ -35,28 +36,24 @@
 
   # disks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv" = {
-    device = "/dev/disk/by-uuid/c1822e5f-4137-44e1-885f-954e926583ce";
-    allowDiscards = true;
-  };
+  luks.devices = [ "/dev/disk/by-uuid/c1822e5f-4137-44e1-885f-954e926583ce" ];
   fileSystems."/" =
-    { device = "/dev/vg/root";
+    {
+      device = "/dev/vg/root";
       fsType = "btrfs";
       options = [ "subvol=root" ];
     };
   fileSystems."/home" =
-    { device = "/dev/vg/root";
+    {
+      device = "/dev/vg/root";
       fsType = "btrfs";
       options = [ "subvol=home" ];
     };
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/2C85-2B59";
+    {
+      device = "/dev/disk/by-uuid/2C85-2B59";
       fsType = "vfat";
     };
   swapDevices =
-    [ { device = "/dev/vg/swap"; }
-    ];
-
-  # high-resolution display
-  hardware.video.hidpi.enable = lib.mkDefault true;
+    [{ device = "/dev/vg/swap"; }];
 }

machines/ray/properties.nix

@@ -0,0 +1,22 @@
+{
+  hostNames = [
+    "ray"
+  ];
+
+  arch = "x86_64-linux";
+
+  systemRoles = [
+    "personal"
+    "deploy"
+  ];
+
+  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDQM8hwKRgl8cZj7UVYATSLYu4LhG7I0WFJ9m2iWowiB";
+
+  userKeys = [
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeTK1iARlNIKP/DS8/ObBm9yUM/3L1Ub4XI5A2r9OzP"
+  ];
+
+  deployKeys = [
+    "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEaGIwLiUa6wQLlEF+keQOIYy/tCmJvV6eENzUQjSqW2AAAABHNzaDo="
+  ];
+}

machines/router/configuration.nix

@@ -1,15 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-  imports = [
-    ./hardware-configuration.nix
-  ];
-
-  networking.hostName = "router";
-
-  system.autoUpgrade.enable = true;
-
-  services.tailscale.exitNode = true;
-
-  networking.useDHCP = lib.mkForce true;
-}

machines/router/default.nix

@@ -0,0 +1,38 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./router.nix
+  ];
+
+  # https://dataswamp.org/~solene/2022-08-03-nixos-with-live-usb-router.html
+  # https://github.com/mdlayher/homelab/blob/391cfc0de06434e4dee0abe2bec7a2f0637345ac/nixos/routnerr-2/configuration.nix
+  # https://github.com/skogsbrus/os/blob/master/sys/router.nix
+  # http://trac.gateworks.com/wiki/wireless/wifi 
+
+  system.autoUpgrade.enable = true;
+
+  services.tailscale.exitNode = true;
+
+  router.enable = true;
+  router.privateSubnet = "192.168.3";
+
+  services.iperf3.enable = true;
+
+  # networking.useDHCP = lib.mkForce true;
+
+  # TODO
+  # networking.usePredictableInterfaceNames = true;
+
+  powerManagement.cpuFreqGovernor = "ondemand";
+
+
+  services.irqbalance.enable = true;
+
+  #   services.miniupnpd = {
+  #     enable = true;
+  #     externalInterface = "eth0";
+  #     internalIPs = [ "br0" ];
+  # };
+}

machines/router/generate_hostapd_config.sh

@@ -0,0 +1,223 @@
+#!/bin/sh
+
+# TODO allow adding custom parameters to ht_capab, vht_capab
+# TODO detect bad channel numbers (preferably not at runtime)
+# TODO error if 160mhz is not supported
+# TODO 'b' only goes up to 40mhz
+
+# gets the phy number using the input interface
+# Ex: get_phy_number("wlan0") -> "1"
+get_phy_number() {
+  local interface=$1
+  phy=$(iw dev "$interface" info | awk '/phy/ {gsub(/#/,"");print $2}')
+  if [[ -z "$phy" ]]; then
+    echo "Error: interface not found" >&2
+    exit 1
+  fi
+  phy=phy$phy
+}
+
+get_ht_cap_mask() {
+  ht_cap_mask=0
+
+  for cap in $(iw phy "$phy" info | grep 'Capabilities:' | cut -d: -f2); do
+    ht_cap_mask="$(($ht_cap_mask | $cap))"
+  done
+
+  local cap_rx_stbc
+  cap_rx_stbc=$((($ht_cap_mask >> 8) & 3))
+  ht_cap_mask="$(( ($ht_cap_mask & ~(0x300)) | ($cap_rx_stbc << 8) ))"
+}
+
+get_vht_cap_mask() {
+  vht_cap_mask=0
+  for cap in $(iw phy "$phy" info | awk -F "[()]" '/VHT Capabilities/ { print $2 }'); do
+    vht_cap_mask="$(($vht_cap_mask | $cap))"
+  done
+
+  local cap_rx_stbc
+  cap_rx_stbc=$((($vht_cap_mask >> 8) & 7))
+  vht_cap_mask="$(( ($vht_cap_mask & ~(0x700)) | ($cap_rx_stbc << 8) ))"
+}
+
+mac80211_add_capabilities() {
+  local __var="$1"; shift
+  local __mask="$1"; shift
+  local __out= oifs
+
+  oifs="$IFS"
+  IFS=:
+  for capab in "$@"; do
+    set -- $capab
+    [ "$(($4))" -gt 0 ] || continue
+    [ "$(($__mask & $2))" -eq "$((${3:-$2}))" ] || continue
+    __out="$__out[$1]"
+  done
+  IFS="$oifs"
+
+  export -n -- "$__var=$__out"
+}
+
+add_special_ht_capabilities() {
+  case "$hwmode" in
+    a)
+      case "$(( ($channel / 4) % 2 ))" in
+        1) ht_capab="$ht_capab[HT40+]";;
+        0) ht_capab="$ht_capab[HT40-]";;
+      esac
+    ;;
+    *)
+      if [ "$channel" -lt 7 ]; then
+        ht_capab="$ht_capab[HT40+]"
+      else
+        ht_capab="$ht_capab[HT40-]"
+      fi
+    ;;
+  esac
+}
+
+add_special_vht_capabilities() {
+  local cap_ant
+  [ "$(($vht_cap_mask & 0x800))" -gt 0 ] && {
+    cap_ant="$(( ( ($vht_cap_mask >> 16) & 3 ) + 1 ))"
+    [ "$cap_ant" -gt 1 ] && vht_capab="$vht_capab[SOUNDING-DIMENSION-$cap_ant]"
+  }
+
+  [ "$(($vht_cap_mask & 0x1000))" -gt 0 ] && {
+    cap_ant="$(( ( ($vht_cap_mask >> 13) & 3 ) + 1 ))"
+    [ "$cap_ant" -gt 1 ] && vht_capab="$vht_capab[BF-ANTENNA-$cap_ant]"
+  }
+
+  if [ "$(($vht_cap_mask & 12))" -eq 4 ]; then
+    vht_capab="$vht_capab[VHT160]"
+  fi
+
+  local vht_max_mpdu_hw=3895
+  [ "$(($vht_cap_mask & 3))" -ge 1 ] && \
+    vht_max_mpdu_hw=7991
+  [ "$(($vht_cap_mask & 3))" -ge 2 ] && \
+    vht_max_mpdu_hw=11454
+  [ "$vht_max_mpdu_hw" != 3895 ] && \
+    vht_capab="$vht_capab[MAX-MPDU-$vht_max_mpdu_hw]"
+
+  # maximum A-MPDU length exponent
+  local vht_max_a_mpdu_len_exp_hw=0
+  [ "$(($vht_cap_mask & 58720256))" -ge 8388608 ] && \
+    vht_max_a_mpdu_len_exp_hw=1
+  [ "$(($vht_cap_mask & 58720256))" -ge 16777216 ] && \
+    vht_max_a_mpdu_len_exp_hw=2
+  [ "$(($vht_cap_mask & 58720256))" -ge 25165824 ] && \
+    vht_max_a_mpdu_len_exp_hw=3
+  [ "$(($vht_cap_mask & 58720256))" -ge 33554432 ] && \
+    vht_max_a_mpdu_len_exp_hw=4
+  [ "$(($vht_cap_mask & 58720256))" -ge 41943040 ] && \
+    vht_max_a_mpdu_len_exp_hw=5
+  [ "$(($vht_cap_mask & 58720256))" -ge 50331648 ] && \
+    vht_max_a_mpdu_len_exp_hw=6
+  [ "$(($vht_cap_mask & 58720256))" -ge 58720256 ] && \
+    vht_max_a_mpdu_len_exp_hw=7
+  vht_capab="$vht_capab[MAX-A-MPDU-LEN-EXP$vht_max_a_mpdu_len_exp_hw]"
+
+  local vht_link_adapt_hw=0
+  [ "$(($vht_cap_mask & 201326592))" -ge 134217728 ] && \
+    vht_link_adapt_hw=2
+  [ "$(($vht_cap_mask & 201326592))" -ge 201326592 ] && \
+    vht_link_adapt_hw=3
+  [ "$vht_link_adapt_hw" != 0 ] && \
+    vht_capab="$vht_capab[VHT-LINK-ADAPT-$vht_link_adapt_hw]"
+}
+
+calculate_channel_offsets() {
+  vht_oper_chwidth=0
+  vht_oper_centr_freq_seg0_idx=
+
+  local idx="$channel"
+  case "$channelWidth" in
+    40)
+      case "$(( ($channel / 4) % 2 ))" in
+        1) idx=$(($channel + 2));;
+        0) idx=$(($channel - 2));;
+      esac
+      vht_oper_centr_freq_seg0_idx=$idx
+    ;;
+    80)
+      case "$(( ($channel / 4) % 4 ))" in
+        1) idx=$(($channel + 6));;
+        2) idx=$(($channel + 2));;
+        3) idx=$(($channel - 2));;
+        0) idx=$(($channel - 6));;
+      esac
+      vht_oper_chwidth=1
+      vht_oper_centr_freq_seg0_idx=$idx
+    ;;
+    160)
+      case "$channel" in
+        36|40|44|48|52|56|60|64) idx=50;;
+        100|104|108|112|116|120|124|128) idx=114;;
+      esac
+      vht_oper_chwidth=2
+      vht_oper_centr_freq_seg0_idx=$idx
+    ;;
+  esac
+
+  he_oper_chwidth=$vht_oper_chwidth
+  he_oper_centr_freq_seg0_idx=$vht_oper_centr_freq_seg0_idx
+}
+
+interface=$1
+channel=$2
+hwmode=$3
+channelWidth=$4
+
+get_phy_number $interface
+get_ht_cap_mask
+get_vht_cap_mask
+
+mac80211_add_capabilities vht_capab $vht_cap_mask \
+  RXLDPC:0x10::1 \
+  SHORT-GI-80:0x20::1 \
+  SHORT-GI-160:0x40::1 \
+  TX-STBC-2BY1:0x80::1 \
+  SU-BEAMFORMER:0x800::1 \
+  SU-BEAMFORMEE:0x1000::1 \
+  MU-BEAMFORMER:0x80000::1 \
+  MU-BEAMFORMEE:0x100000::1 \
+  VHT-TXOP-PS:0x200000::1 \
+  HTC-VHT:0x400000::1 \
+  RX-ANTENNA-PATTERN:0x10000000::1 \
+  TX-ANTENNA-PATTERN:0x20000000::1 \
+  RX-STBC-1:0x700:0x100:1 \
+  RX-STBC-12:0x700:0x200:1 \
+  RX-STBC-123:0x700:0x300:1 \
+  RX-STBC-1234:0x700:0x400:1 \
+
+mac80211_add_capabilities ht_capab $ht_cap_mask \
+  LDPC:0x1::1 \
+  GF:0x10::1 \
+  SHORT-GI-20:0x20::1 \
+  SHORT-GI-40:0x40::1 \
+  TX-STBC:0x80::1 \
+  RX-STBC1:0x300::1 \
+  MAX-AMSDU-7935:0x800::1 \
+
+  # TODO this is active when the driver doesn't support it?
+  # DSSS_CCK-40:0x1000::1 \
+
+  # TODO these are active when the driver doesn't support them?
+  # RX-STBC1:0x300:0x100:1 \
+  # RX-STBC12:0x300:0x200:1 \
+  # RX-STBC123:0x300:0x300:1 \
+
+add_special_ht_capabilities
+add_special_vht_capabilities
+
+echo ht_capab=$ht_capab
+echo vht_capab=$vht_capab
+
+if [ "$channelWidth" != "20" ]; then
+  calculate_channel_offsets
+  echo he_oper_chwidth=$he_oper_chwidth
+  echo vht_oper_chwidth=$vht_oper_chwidth
+  echo he_oper_centr_freq_seg0_idx=$he_oper_centr_freq_seg0_idx
+  echo vht_oper_centr_freq_seg0_idx=$vht_oper_centr_freq_seg0_idx
+fi

machines/router/hardware-configuration.nix

@@ -10,7 +10,8 @@
 
   # Enable serial output
   boot.kernelParams = [
-    "panic=30" "boot.panic_on_fail" # reboot the machine upon fatal boot issues
+    "panic=30"
+    "boot.panic_on_fail" # reboot the machine upon fatal boot issues
     "console=ttyS0,115200n8" # enable serial console
   ];
   boot.loader.grub.extraConfig = "
@@ -21,7 +22,6 @@
 
   # firmware
   firmware.x86_64.enable = true;
-  hardware.enableAllFirmware = true;
   nixpkgs.config.allowUnfree = true;
 
   # boot
@@ -32,18 +32,19 @@
 
   # disks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv".device = "/dev/disk/by-uuid/9b090551-f78e-45ca-8570-196ed6a4af0c";
+  luks.devices = [ "/dev/disk/by-uuid/9b090551-f78e-45ca-8570-196ed6a4af0c" ];
   fileSystems."/" =
-    { device = "/dev/disk/by-uuid/421c82b9-d67c-4811-8824-8bb57cb10fce";
+    {
+      device = "/dev/disk/by-uuid/421c82b9-d67c-4811-8824-8bb57cb10fce";
       fsType = "btrfs";
     };
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/d97f324f-3a2e-4b84-ae2a-4b3d1209c689";
+    {
+      device = "/dev/disk/by-uuid/d97f324f-3a2e-4b84-ae2a-4b3d1209c689";
       fsType = "ext3";
     };
   swapDevices =
-    [ { device = "/dev/disk/by-uuid/45bf58dd-67eb-45e4-9a98-246e23fa7abd"; }
-    ];
+    [{ device = "/dev/disk/by-uuid/45bf58dd-67eb-45e4-9a98-246e23fa7abd"; }];
 
   nixpkgs.hostPlatform = "x86_64-linux";
 }

machines/router/properties.nix

@@ -0,0 +1,21 @@
+{
+  hostNames = [
+    "router"
+    "192.168.1.228"
+  ];
+
+  arch = "x86_64-linux";
+
+  systemRoles = [
+    "server"
+    "wireless"
+    "router"
+  ];
+
+  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFr2IHmWFlaLaLp5dGoSmFEYKA/eg2SwGXAogaOmLsHL";
+
+  remoteUnlock = {
+    hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJOw5dTPmtKqiPBH6VKyz5MYBubn8leAh5Eaw7s/O85c";
+    onionHost = "jxx2exuihlls2t6ncs7rvrjh2dssubjmjtclwr2ysvxtr4t7jv55xmqd.onion";
+  };
+}

machines/router/router.nix

@@ -0,0 +1,205 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.router;
+  inherit (lib) mapAttrs' genAttrs nameValuePair mkOption types mkIf mkEnableOption;
+in
+{
+  options.router = {
+    enable = mkEnableOption "router";
+
+    privateSubnet = mkOption {
+      type = types.str;
+      default = "192.168.1";
+      description = "IP block (/24) to use for the private subnet";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.ip_forward = true;
+
+    networking.interfaces.enp1s0.useDHCP = true;
+
+    networking.nat = {
+      enable = true;
+      internalInterfaces = [
+        "br0"
+      ];
+      externalInterface = "enp1s0";
+    };
+
+    networking.bridges = {
+      br0 = {
+        interfaces = [
+          "enp2s0"
+          "wlp4s0"
+          "wlan1"
+        ];
+      };
+    };
+
+    networking.interfaces = {
+      br0 = {
+        useDHCP = false;
+        ipv4.addresses = [
+          {
+            address = "${cfg.privateSubnet}.1";
+            prefixLength = 24;
+          }
+        ];
+      };
+    };
+
+    networking.firewall = {
+      enable = true;
+      trustedInterfaces = [ "br0" "tailscale0" ];
+
+      interfaces = {
+        enp1s0 = {
+          allowedTCPPorts = [ ];
+          allowedUDPPorts = [ ];
+        };
+      };
+    };
+
+    services.dnsmasq = {
+      enable = true;
+      extraConfig = ''
+        # sensible behaviours
+        domain-needed
+        bogus-priv
+        no-resolv
+
+        # upstream name servers
+        server=1.1.1.1
+        server=8.8.8.8
+
+        # local domains
+        expand-hosts
+        domain=home
+        local=/home/
+
+        # Interfaces to use DNS on
+        interface=br0
+
+        # subnet IP blocks to use DHCP on
+        dhcp-range=${cfg.privateSubnet}.10,${cfg.privateSubnet}.254,24h
+      '';
+    };
+
+    services.hostapd = {
+      enable = true;
+      radios = {
+        # 2.4GHz
+        wlp4s0 = {
+          hwMode = "g";
+          noScan = true;
+          channel = 6;
+          countryCode = "US";
+          wifi4 = {
+            capabilities = [ "LDPC" "GF" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1" "MAX-AMSDU-7935" "HT40+" ];
+          };
+          wifi5 = {
+            operatingChannelWidth = "20or40";
+            capabilities = [ "MAX-A-MPDU-LEN-EXP0" ];
+          };
+          wifi6 = {
+            enable = true;
+            singleUserBeamformer = true;
+            singleUserBeamformee = true;
+            multiUserBeamformer = true;
+            operatingChannelWidth = "20or40";
+          };
+          networks = {
+            wlp4s0 = {
+              ssid = "CXNK00BF9176";
+              authentication.saePasswordsFile = "/run/agenix/hostapd-pw-CXNK00BF9176";
+            };
+            # wlp4s0-1 = {
+            #   ssid = "- Experimental 5G Tower by AT&T";
+            #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
+            # };
+            # wlp4s0-2 = {
+            #   ssid = "FBI Surveillance Van 2";
+            #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
+            # };
+          };
+          extraConfig = ''
+            he_oper_centr_freq_seg0_idx=8
+            vht_oper_centr_freq_seg0_idx=8
+          '';
+        };
+
+        # 5GHz
+        wlan1 = {
+          hwMode = "a";
+          noScan = true;
+          channel = 128;
+          countryCode = "US";
+          wifi4 = {
+            capabilities = [ "LDPC" "GF" "SHORT-GI-20" "SHORT-GI-40" "TX-STBC" "RX-STBC1" "MAX-AMSDU-7935" "HT40-" ];
+          };
+          wifi5 = {
+            operatingChannelWidth = "160";
+            capabilities = [ "RXLDPC" "SHORT-GI-80" "SHORT-GI-160" "TX-STBC-2BY1" "SU-BEAMFORMER" "SU-BEAMFORMEE" "MU-BEAMFORMER" "MU-BEAMFORMEE" "RX-ANTENNA-PATTERN" "TX-ANTENNA-PATTERN" "RX-STBC-1" "SOUNDING-DIMENSION-3" "BF-ANTENNA-3" "VHT160" "MAX-MPDU-11454" "MAX-A-MPDU-LEN-EXP7" ];
+          };
+          wifi6 = {
+            enable = true;
+            singleUserBeamformer = true;
+            singleUserBeamformee = true;
+            multiUserBeamformer = true;
+            operatingChannelWidth = "160";
+          };
+          networks = {
+            wlan1 = {
+              ssid = "CXNK00BF9176";
+              authentication.saePasswordsFile = "/run/agenix/hostapd-pw-CXNK00BF9176";
+            };
+            # wlan1-1 = {
+            #   ssid = "- Experimental 5G Tower by AT&T";
+            #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
+            # };
+            # wlan1-2 = {
+            #   ssid = "FBI Surveillance Van 5";
+            #   authentication.saePasswordsFile = "/run/agenix/hostapd-pw-experimental-tower";
+            # };
+          };
+          extraConfig = ''
+            vht_oper_centr_freq_seg0_idx=114
+            he_oper_centr_freq_seg0_idx=114
+          '';
+        };
+      };
+    };
+    age.secrets.hostapd-pw-experimental-tower.file = ../../secrets/hostapd-pw-experimental-tower.age;
+    age.secrets.hostapd-pw-CXNK00BF9176.file = ../../secrets/hostapd-pw-CXNK00BF9176.age;
+
+    hardware.firmware = [
+      pkgs.mt7916-firmware
+    ];
+
+    nixpkgs.overlays = [
+      (self: super: {
+        mt7916-firmware = pkgs.stdenvNoCC.mkDerivation {
+          pname = "mt7916-firmware";
+          version = "custom-feb-02-23";
+          src = ./firmware/mediatek; # from here https://github.com/openwrt/mt76/issues/720#issuecomment-1413537674
+          dontBuild = true;
+          installPhase = ''
+            for i in \
+              mt7916_eeprom.bin \
+              mt7916_rom_patch.bin \
+              mt7916_wa.bin \
+              mt7916_wm.bin;
+            do
+              install -D -pm644 $i $out/lib/firmware/mediatek/$i
+            done
+          '';
+          meta = with lib; {
+            license = licenses.unfreeRedistributableFirmware;
+          };
+        };
+      })
+    ];
+  };
+}

machines/storage/s0/default.nix

@@ -1,12 +1,10 @@
 { config, pkgs, lib, ... }:
 
 {
-  imports =[
+  imports = [
     ./hardware-configuration.nix
   ];
 
-  networking.hostName = "s0";
-
   system.autoUpgrade.enable = true;
 
   services.iperf3.enable = true;

machines/storage/s0/hardware-configuration.nix

@@ -2,11 +2,12 @@
 
 {
   imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
+    [
+      (modulesPath + "/installer/scan/not-detected.nix")
     ];
 
   # boot
-  efi.enable = true;
+  boot.loader.systemd-boot.enable = true;
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "uas" "sd_mod" "rtsx_pci_sdmmc" ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
@@ -24,35 +25,48 @@
 
   # luks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv1".device = "/dev/disk/by-uuid/d52e99a9-8825-4d0a-afc1-8edbef7e0a86";
-  boot.initrd.luks.devices."enc-pv2".device = "/dev/disk/by-uuid/f7275585-7760-4230-97de-36704b9a2aa3";
-  boot.initrd.luks.devices."enc-pv3".device = "/dev/disk/by-uuid/5d1002b8-a0ed-4a1c-99f5-24b8816d9e38";
-  boot.initrd.luks.devices."enc-pv4".device = "/dev/disk/by-uuid/e2c7402a-e72c-4c4a-998f-82e4c10187bc";
+  luks.devices = [
+    "/dev/disk/by-uuid/d52e99a9-8825-4d0a-afc1-8edbef7e0a86"
+    "/dev/disk/by-uuid/f7275585-7760-4230-97de-36704b9a2aa3"
+    "/dev/disk/by-uuid/5d1002b8-a0ed-4a1c-99f5-24b8816d9e38"
+    "/dev/disk/by-uuid/e2c7402a-e72c-4c4a-998f-82e4c10187bc"
+  ];
 
   # mounts
   fileSystems."/" =
-    { device = "rpool/nixos/root";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    {
+      device = "rpool/nixos/root";
+      fsType = "zfs";
+      options = [ "zfsutil" "X-mount.mkdir" ];
     };
   fileSystems."/home" =
-    { device = "rpool/nixos/home";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    {
+      device = "rpool/nixos/home";
+      fsType = "zfs";
+      options = [ "zfsutil" "X-mount.mkdir" ];
     };
   fileSystems."/var/lib" =
-    { device = "rpool/nixos/var/lib";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    {
+      device = "rpool/nixos/var/lib";
+      fsType = "zfs";
+      options = [ "zfsutil" "X-mount.mkdir" ];
     };
   fileSystems."/var/log" =
-    { device = "rpool/nixos/var/log";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    {
+      device = "rpool/nixos/var/log";
+      fsType = "zfs";
+      options = [ "zfsutil" "X-mount.mkdir" ];
     };
 
   fileSystems."/data" =
-    { device = "rpool/nixos/data";
-      fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];
+    {
+      device = "rpool/nixos/data";
+      fsType = "zfs";
+      options = [ "zfsutil" "X-mount.mkdir" ];
     };
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/4FB4-738E";
+    {
+      device = "/dev/disk/by-uuid/4FB4-738E";
       fsType = "vfat";
     };
   swapDevices = [ ];

machines/storage/s0/properties.nix

@@ -0,0 +1,20 @@
+{
+  hostNames = [
+    "s0"
+  ];
+
+  arch = "x86_64-linux";
+
+  systemRoles = [
+    "storage"
+    "server"
+    "pia"
+  ];
+
+  hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwiXcUFtAvZCayhu4+AIcF+Ktrdgv9ee/mXSIhJbp4q";
+
+  remoteUnlock = {
+    hostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFNiceeFMos5ZXcYem4yFxh8PiZNNnuvhlyLbQLrgIZH";
+    onionHost = "r3zvf7f2ppaeithzswigma46pajt3hqytmkg3rshgknbl3jbni455fqd.onion";
+  };
+}

new_machine.md

@@ -1,27 +1,34 @@
+# New Machine Setup
+
+### Prepare Shell If Needed
+
+```sh
 nix-shell -p nixFlakes git
+```
+
+### Disk Setup
+```sh
 cfdisk
-mkfs.ext3 boot
 cryptsetup luksFormat /dev/vda2
 cryptsetup luksOpen /dev/vda2 enc-pv
 pvcreate /dev/mapper/enc-pv
+vgcreate vg /dev/mapper/enc-pv
 lvcreate -L 4G -n swap vg
 lvcreate -l '100%FREE' -n root vg
 mkswap -L swap /dev/vg/swap
 swapon /dev/vg/swap
 mkfs.btrfs /dev/vg/root
 mount /dev/vg/root /mnt
-cd /mnt
-btrfs subvolume create root
-btrfs subvolume create home
-cd
-mount -o subvol=root /dev/vg/root /mnt
-mkdir /mnt/home
-mount -o subvol=home /dev/vg/root /mnt/home
-mkdir /mnt/boot
+mkfs.ext3 boot
 mount /dev/vda1 /mnt/boot
-mkdir /mnt/secret
+```
 
-/tmp/tor.rc
+### Generate Secrets
+```sh
+mkdir /mnt/secret
+```
+
+In `/tmp/tor.rc`
 ```
 DataDirectory /tmp/my-dummy.tor/
 SOCKSPort 127.0.0.1:10050 IsolateDestAddr
@@ -30,8 +37,23 @@ HiddenServiceDir /mnt/secret/onion
 HiddenServicePort 1234 127.0.0.1:1234
 ```
 
+```sh
 nix-shell -p tor --run "tor -f /tmp/tor.rc"
 ssh-keygen -q -N "" -t rsa -b 4096 -f /mnt/secret/ssh_host_rsa_key
 ssh-keygen -q -N "" -t ed25519 -f /mnt/secret/ssh_host_ed25519_key
-nixos-generate-config --root /mnt # copy hardware config
+```
+
+### Generate Hardware Config
+```sh
+nixos-generate-config --root /mnt
+```
+
+### Install
+```sh
 nixos-install --flake "git+https://git.neet.dev/zuckerberg/nix-config.git#MACHINE_NAME"
+```
+
+### Post Install Tasks
+- Add to DNS
+- Add ssh host keys (unlock key + host key)
+- Add to tailnet

patches/kexec-luks.patch

@@ -0,0 +1,264 @@
+diff --git a/nixos/modules/system/boot/luksroot.nix b/nixos/modules/system/boot/luksroot.nix
+index b8f36538e70..cc320a04c70 100644
+--- a/nixos/modules/system/boot/luksroot.nix
++++ b/nixos/modules/system/boot/luksroot.nix
+@@ -146,6 +146,7 @@ let
+     csopen = "cryptsetup luksOpen ${dev.device} ${dev.name}"
+            + optionalString dev.allowDiscards " --allow-discards"
+            + optionalString dev.bypassWorkqueues " --perf-no_read_workqueue --perf-no_write_workqueue"
++           + optionalString dev.disableKeyring " --disable-keyring"
+            + optionalString (dev.header != null) " --header=${dev.header}";
+     cschange = "cryptsetup luksChangeKey ${dev.device} ${optionalString (dev.header != null) "--header=${dev.header}"}";
+     fido2luksCredentials = dev.fido2.credentials ++ optional (dev.fido2.credential != null) dev.fido2.credential;
+@@ -243,6 +244,26 @@ let
+                do_open_passphrase
+             fi
+         fi
++        '' else if (dev.masterKeyFile != null) then ''
++        if wait_target "key file" ${dev.masterKeyFile}; then
++            ${csopen} --master-key-file=${dev.masterKeyFile}
++            cs_status=$?
++            if [ $cs_status -ne 0 ]; then
++              echo "Key File ${dev.masterKeyFile} failed!"
++              if ! try_empty_passphrase; then
++                ${if dev.fallbackToPassword then "echo" else "die"} "${dev.masterKeyFile} is unavailable"
++                echo " - failing back to interactive password prompt"
++                do_open_passphrase
++              fi
++            fi
++        else
++            # If the key file never shows up we should also try the empty passphrase
++            if ! try_empty_passphrase; then
++               ${if dev.fallbackToPassword then "echo" else "die"} "${dev.masterKeyFile} is unavailable"
++               echo " - failing back to interactive password prompt"
++               do_open_passphrase
++            fi
++        fi
+         '' else ''
+            if ! try_empty_passphrase; then
+               do_open_passphrase
+@@ -625,6 +646,15 @@ in
+             '';
+           };
+ 
++          masterKeyFile = mkOption {
++            default = null;
++            type = types.nullOr types.str;
++            description = lib.mdDoc ''
++              The name of the file (can be a raw device or a partition) that
++              should be used as the master decryption key for the encrypted device.
++            '';
++          };
++
+           tryEmptyPassphrase = mkOption {
+             default = false;
+             type = types.bool;
+@@ -700,6 +730,15 @@ in
+             '';
+           };
+ 
++          disableKeyring = mkOption {
++            default = false;
++            type = types.bool;
++            description = lib.mdDoc ''
++              Disables using the kernel keyring for LUKS2 disks.
++              This is already the default behavior for LUKS1
++            '';
++          };
++
+           fallbackToPassword = mkOption {
+             default = false;
+             type = types.bool;
+diff --git a/nixos/modules/tasks/auto-upgrade.nix b/nixos/modules/tasks/auto-upgrade.nix
+index 29e3e313336..050bf09c208 100644
+--- a/nixos/modules/tasks/auto-upgrade.nix
++++ b/nixos/modules/tasks/auto-upgrade.nix
+@@ -4,7 +4,8 @@ with lib;
+ 
+ let cfg = config.system.autoUpgrade;
+ 
+-in {
++in
++{
+ 
+   options = {
+ 
+@@ -22,7 +23,7 @@ in {
+       };
+ 
+       operation = mkOption {
+-        type = types.enum ["switch" "boot"];
++        type = types.enum [ "switch" "boot" ];
+         default = "switch";
+         example = "boot";
+         description = lib.mdDoc ''
+@@ -86,14 +87,14 @@ in {
+         '';
+       };
+ 
+-      allowReboot = mkOption {
++      allowKexec = mkOption {
+         default = false;
+         type = types.bool;
+         description = lib.mdDoc ''
+-          Reboot the system into the new generation instead of a switch
++          kexec the system into the new generation instead of a switch
+           if the new generation uses a different kernel, kernel modules
+           or initrd than the booted system.
+-          See {option}`rebootWindow` for configuring the times at which a reboot is allowed.
++          See {option}`kexecWindow` for configuring the times at which a kexec is allowed.
+         '';
+       };
+ 
+@@ -109,25 +110,25 @@ in {
+         '';
+       };
+ 
+-      rebootWindow = mkOption {
++      kexecWindow = mkOption {
+         description = lib.mdDoc ''
+           Define a lower and upper time value (in HH:MM format) which
+-          constitute a time window during which reboots are allowed after an upgrade.
+-          This option only has an effect when {option}`allowReboot` is enabled.
+-          The default value of `null` means that reboots are allowed at any time.
++          constitute a time window during which kexecs are allowed after an upgrade.
++          This option only has an effect when {option}`allowKexec` is enabled.
++          The default value of `null` means that kexecs are allowed at any time.
+         '';
+         default = null;
+         example = { lower = "01:00"; upper = "05:00"; };
+         type = with types; nullOr (submodule {
+           options = {
+             lower = mkOption {
+-              description = lib.mdDoc "Lower limit of the reboot window";
++              description = lib.mdDoc "Lower limit of the kexec window";
+               type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+               example = "01:00";
+             };
+ 
+             upper = mkOption {
+-              description = lib.mdDoc "Upper limit of the reboot window";
++              description = lib.mdDoc "Upper limit of the kexec window";
+               type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+               example = "05:00";
+             };
+@@ -165,12 +166,12 @@ in {
+     }];
+ 
+     system.autoUpgrade.flags = (if cfg.flake == null then
+-        [ "--no-build-output" ] ++ optionals (cfg.channel != null) [
+-          "-I"
+-          "nixpkgs=${cfg.channel}/nixexprs.tar.xz"
+-        ]
+-      else
+-        [ "--flake ${cfg.flake}" ]);
++      [ "--no-build-output" ] ++ optionals (cfg.channel != null) [
++        "-I"
++        "nixpkgs=${cfg.channel}/nixexprs.tar.xz"
++      ]
++    else
++      [ "--flake ${cfg.flake}" ]);
+ 
+     systemd.services.nixos-upgrade = {
+       description = "NixOS Upgrade";
+@@ -195,54 +196,56 @@ in {
+         config.programs.ssh.package
+       ];
+ 
+-      script = let
+-        nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+-        date     = "${pkgs.coreutils}/bin/date";
+-        readlink = "${pkgs.coreutils}/bin/readlink";
+-        shutdown = "${config.systemd.package}/bin/shutdown";
+-        upgradeFlag = optional (cfg.channel == null) "--upgrade";
+-      in if cfg.allowReboot then ''
+-        ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)}
+-        booted="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
+-        built="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
+-
+-        ${optionalString (cfg.rebootWindow != null) ''
+-          current_time="$(${date} +%H:%M)"
+-
+-          lower="${cfg.rebootWindow.lower}"
+-          upper="${cfg.rebootWindow.upper}"
+-
+-          if [[ "''${lower}" < "''${upper}" ]]; then
+-            if [[ "''${current_time}" > "''${lower}" ]] && \
+-               [[ "''${current_time}" < "''${upper}" ]]; then
+-              do_reboot="true"
++      script =
++        let
++          nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
++          date = "${pkgs.coreutils}/bin/date";
++          readlink = "${pkgs.coreutils}/bin/readlink";
++          systemctl_kexec = "${config.systemd.package}/bin/systemctl kexec";
++          upgradeFlag = optional (cfg.channel == null) "--upgrade";
++        in
++        if cfg.allowKexec then ''
++          ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)}
++          booted="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
++          built="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
++
++          ${optionalString (cfg.kexecWindow != null) ''
++            current_time="$(${date} +%H:%M)"
++
++            lower="${cfg.kexecWindow.lower}"
++            upper="${cfg.kexecWindow.upper}"
++
++            if [[ "''${lower}" < "''${upper}" ]]; then
++              if [[ "''${current_time}" > "''${lower}" ]] && \
++                 [[ "''${current_time}" < "''${upper}" ]]; then
++                do_kexec="true"
++              else
++                do_kexec="false"
++              fi
+             else
+-              do_reboot="false"
++              # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
++              # we want to reboot if cur > 23h or cur < 6h
++              if [[ "''${current_time}" < "''${upper}" ]] || \
++                 [[ "''${current_time}" > "''${lower}" ]]; then
++                do_kexec="true"
++              else
++                do_kexec="false"
++              fi
+             fi
++          ''}
++
++          if [ "''${booted}" = "''${built}" ]; then
++            ${nixos-rebuild} ${cfg.operation} ${toString cfg.flags}
++          ${optionalString (cfg.kexecWindow != null) ''
++            elif [ "''${do_kexec}" != true ]; then
++              echo "Outside of configured kexec window, skipping."
++          ''}
+           else
+-            # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
+-            # we want to reboot if cur > 23h or cur < 6h
+-            if [[ "''${current_time}" < "''${upper}" ]] || \
+-               [[ "''${current_time}" > "''${lower}" ]]; then
+-              do_reboot="true"
+-            else
+-              do_reboot="false"
+-            fi
++            ${systemctl_kexec}
+           fi
+-        ''}
+-
+-        if [ "''${booted}" = "''${built}" ]; then
+-          ${nixos-rebuild} ${cfg.operation} ${toString cfg.flags}
+-        ${optionalString (cfg.rebootWindow != null) ''
+-          elif [ "''${do_reboot}" != true ]; then
+-            echo "Outside of configured reboot window, skipping."
+-        ''}
+-        else
+-          ${shutdown} -r +1
+-        fi
+-      '' else ''
+-        ${nixos-rebuild} ${cfg.operation} ${toString (cfg.flags ++ upgradeFlag)}
+-      '';
++        '' else ''
++          ${nixos-rebuild} ${cfg.operation} ${toString (cfg.flags ++ upgradeFlag)}
++        '';
+ 
+       startAt = cfg.dates;
+ 

secrets/email-pw.age

@@ -1,38 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 xoAm7w H4pJd+lWWypSgHCz6JQ2I2Yqhtabmts6LMpHq/MXwwA
-izP31JcNATRlNq6m8FYC7uis+RrZo88ckluXUCqtDkU
--> ssh-ed25519 mbw8xA llqgasmhheEDBVbl0sU4O269iS3DY+ExSpUfockvBmE
-4b+10lAkTlsIIIbqafjqgPYVNOs78dqw3mhGNt/Kr+U
--> ssh-ed25519 2a2Yhw qPg4uxTQhi/pBSbiZ1tDjmbuLFqrmVck5s52Seu9WyI
-NRJnXzFPdf9m/eL2zoS8yeaz9OqMdHbbX8F9FOVedmM
--> ssh-ed25519 dMQYog l68vM+Hj4TdBVWXG8DH5wz1U2ZEuL9sz04B5ykY0GTE
-QKDaeV8CUxf34zS6FJneaP49UCeXondKMPURipqR3AQ
--> ssh-ed25519 G2eSCQ 9Sz80YngHh8GimDhEeMtqrOk3hai+YBUUt71mdmjs20
-Sc3Tit/RZ9xlrvVK92RJS9dhyPIJ/mZ1NsZo4dmiRnI
--> ssh-ed25519 6AT2/g aQOztqgJ+lmACELQnewjsKhpwRoAc4Xu6j4L0DI2+n8
-TX37F7ZCQfFOddgdllnxqKk85PBY04O48VJZvovpk1k
--> ssh-ed25519 yHDAQw /WlCVe+BEy43M37s89SahbmmLzT1jTGYEliZ5EEW1Vo
-FGwQj00KhuyAj5cYJhuiR/uZY03GQwKGm8iOSm7VGNo
--> ssh-ed25519 VyYH/Q AWfQDDmiGyVM+r7uIHE+OqhwMPSclTfSJiYjimIl4TA
-2STrZzCDqprYylCIB0b6FJK+ipMZx+08RN6ifsOqIyw
--> ssh-ed25519 hPp1nw QEaJAqCwiz0IZMbZ8AaoHi4LNj3MDGARsC4r8bLQhyY
-SxjnCzanVkfyF4moOkYyxYKgY+TOSHdX7PnesmkP42A
--> ssh-ed25519 CRfjsA GTOd0wL35nL91iLdJKzNTQ3+e3gRqEO+y3ppLjWzuQM
-QiOdvtSkwrCzdMOTpxQW29J49T6JIU/lwdy/4YoFZ6E
--> ssh-ed25519 vwVIvQ jdSld1nAFuUn14W6gOzg1k32JrZEuKFBV4KAP8/Ilhs
-4H7jlaf6wPG6xOLGSz28asD/GirT52LVfELviEFHsiY
--> ssh-ed25519 fBrw3g vu/yjmGNPBUJR/qU/tBTSz4QhTb8NR41lG55NZl+YGg
-YqF18PP5zdgRExyszHSs35212EOVLXTtN9oREduuI5Q
--> ssh-ed25519 S5xQfg L3uspJyJKVdChakXgXTcmIzDXeoXK5GVmU+DszgS1z4
-0kXsfDav+LEL6aYgpWzrD5cpb0zhjdT2v8/p3Q/Y1rI
--> ssh-ed25519 XPxfUQ 1D6dQ8Pbn5EKtdLzw1VMe28jmd4PjzAaxtoDQJ3qPjY
-u6Y3NrdowSWLw0YyRXYYEIQUTcb9NGn0UhSzK4HeGlI
--> ssh-ed25519 SpD5mg M66nCAQ05Iam/GAjfcvBvV1FwSsm0MjUsD6awuidsVY
-DZWfw5iqujq/ZU3wVuL1kvC7l5NdPSTi2uq48Oc03zA
--> ssh-ed25519 Kk8sng fsfCIcdBVu3xAreLxvq9zHu0MdL77D2cd6PxJAui804
-EqNbLicyIkWjmCABaShEmcxEftmjnvQYFFKd1AFMQs8
--> z;-grease !G/_ ;L|^3hU }
-bCokcBX9NxBIh97Cf+qsjRq92nbhFTG1E/ygPscvCeexFIgvBkBATLFlxlFZbfHo
-jU1hBN7/lA
---- Gf9q3ird84+YdfWY/WJXiO3nz5T4IHa/geMyfSTmzC0
-C+ÖÒSI=5Ñ4c²š>^ ÒI„ÍK+šK<C5A1>hŸÕ6J·É§w†íE‘à펚üy4iˆ8Ù[ÅšOþûBÅn<C385>;ö%³®	ÎÏ)²ïцœa5ßšÛ%Iô[

secrets/hashed-email-pw.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 6AT2/g 93Az2iuqeWL6H/S3XDPXFoEPcrY/n/z9mlSNb5wABkU
+LpMPjpDtBrY2aHpqHwT5AY7vtsYHNcOjpz+LFY4TGCg
+-> ssh-ed25519 dMQYog 4qT0aF1IHsTtN1avMPWYG5Az2xmEZhVUhqcwyNFdfU4
++wD0hE035JqYdDgJmkvNXwJyMzXrquA+RsD8QdK3xP8
+-> !vfM7-grease
+7nQGFFUWY9UIjfrb+/VfaG0zJ21zmDnDh5khs/0tioJevrrrlhub9Bz8iM/Jsfxy
+KUhwV8O8tL/5+30RFSlFRaAB6xPCGg24Yq6E
+--- jVsDtz2xpvK/XCHcdN5JVZx5zSxyEAM6D/xJIgN4YfY
+Ñì°ßév.rK,Æ$ 

secrets/hashed-robots-email-pw.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 6AT2/g O0XP3tiD5bv5aFK54eTpo4I6oXHk+P0/zyy5A1GQlwg
+0D5I+dRmW6Ak7VvQrNDa4NiHgBtD4sAS87U9iXKHrRs
+-> ssh-ed25519 dMQYog pofEwx9ktCWDTYGp/rXwxq1ZkMLaR4q4JTWJxHl6rRw
+CoUEZHQaQd4wFi6pcZPZfhPACXI9qgrB1xAuSilGJpI
+-> /5^-grease e yP^gopW&
+8qWOBZeGzhSSfGdjHDOGhs2MoZEQneLFMj8DsBqTrnttzgjtg8VSwuMD2JA3yiA5
+43u5T24PCzhKor8puT830nMU3HfQ3FA1RtiUpu1FWPA
+--- gJaY0whK+GQ+F6m3jCfeGkPbJoIkGxmcJ++XVseDeWI
+'½<14>¤	kª†süVûÊ
+ƒ$÷"Ãý|ÕÞÆ^n“ÑàÆÜ hï~2Tô±Ì<>6/ ˆjÈO<C388>þõvMŠôé/e¯XÆr~GÚ&oèn;=,îÂÿô½ï2]?j

secrets/hostapd-pw-CXNK00BF9176.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 VyYH/Q X+fXLJz227KkBLu45rb9mUkkIpENSMtZeEJjl6qj5Xw
+AFAFnvsiogoMMwsAJO0DDoaizL9lmCLsF4QHDjmubr0
+-> ssh-ed25519 dMQYog P84+7TBcMFSALTn6FR/aXyqFE9DfOzp38ImkdWj7nE0
+PqOn1OL9Zt0x1pBIYOSKkkS//mbk1OX5pnDGp+OLYeI
+-> @?-grease
+3JvpmcTxdTgvv6vPL8dXEwjR+g
+--- aMYF1SbC+p01YWmg24+Ih78VPQcwzGU/P1cEfgRvXV8
+Ÿ	@$™sžQ¼z<15><>®xkÊNfÛuÕ;§¿ ÎvI0•ªÇÎ^4.?, 8…çî

secrets/hostapd-pw-experimental-tower.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 VyYH/Q biHhTmstx4FJmstCt7pYUjQlwZpgbFkZVlpzQu2Lf1w
+K2lHbhdETesM86K6fKk91/2Z0LRQcwVtnXxYOtVCCaE
+-> ssh-ed25519 dMQYog cyeabgOaV3nLPdLYs4G1g41bMAEBnHlEXoeW+A7NLFw
+FRGdlHVQ81JjjYQ/PRoAq+JRMZ28rSBEUko8VHKNeDM
+-> cyfV-grease +X51me hk_`YB> 6DFo`? V
+/EoPdwBCc9S2GafKdhDnPZE06kxZBlzAob0Eb2qEMuwzVJF+7LesiK0ilpSrzdPU
+cVN1ka2+k4pH7os/QUuu6oW9r3SxCw
+--- FJOYm8/TVXlxV+9Ih376xWyJlKz2uk8KzPoY5FpIJjo
+C`Ñ” -“Ýþo’YIEœ_.y˜XãÝÈË›iþëµFØW±…Š¨Im5ŽA“õ*÷}Àßêkî”þ9ûzë

secrets/iodine.age

@@ -1,39 +1,10 @@
 age-encryption.org/v1
--> ssh-ed25519 xoAm7w ubom+8Q47IB+J+Oq2engREnGa0DE+H2wmG2fz2mjxi8
-wpYS6ylA2PFgcsvClvv1nZBgG+Cy4RD4sVpqRcdW820
--> ssh-ed25519 mbw8xA lh99YoeAM6ktJXa5z46A0JVuUcfLcwvFj3ncaf4gmxU
-WCOvhJJU+A6Slju2HN9kAQsGRm/0aE/rhprkMG+17BU
--> ssh-ed25519 2a2Yhw PVqMJAGJZOb62UAfQF5shDnGUu5VPUHFZjLOCOPfIgg
-+fRdzAagc+MmbgPdx6sOp3Vna97gaxnc4L5KZuXciRU
--> ssh-ed25519 dMQYog f0o48KwYRsZ9w1WowdZtH8aag/TgrV0w9r4DwJ4SdBo
-VGtlt0K/IpbJhOnqc/IEdcqDMt90pjUR+FwRco7Ca0U
--> ssh-ed25519 G2eSCQ vtbTZPGjRxUmWiE/V41pwPYw0y8rDSqSlK9g8FF8rmc
-iHQjLPdVYWCnuHEcQiOPwG/ymWS873dmEQ3YBeypdlE
--> ssh-ed25519 6AT2/g CTGtBM6qUCPKIp6SsipQtGIM4K2gKYlcKm1lZ7oU5Gg
-2toLIu43LGBsK4zEzPe3xu1sGdf0TiH61TuvaI1uu7Q
--> ssh-ed25519 yHDAQw mgKQj9uQitIfCzLATi0QeDjj/ZNdfmLN3mDMq4lhBxo
-HWt4Q0ZjKeoxkT4XB/doa6YhgF5OgrJEVe0NRPPpT4Y
--> ssh-ed25519 VyYH/Q zrxZoR6lY4yTOYvqxLVnJ1B/tLJqP3JiuhkFVRaUmF8
-rHBLx0H/70YLpdHfYzlqbRjFgCso3voEu0Nht7UlZz0
--> ssh-ed25519 hPp1nw 0zPkmUgLopoLsvG2pk3hXLFZ4y1U2zLYAnw6EnrsEis
-bcbjMtvpDs+aKUf5GUW2x8mXMm5xBZ2S/aS6ePYoZOM
--> ssh-ed25519 CRfjsA x5fkEqFAn2gyPjOL91H+TU0t8fGbelyYNF3Gkm2PM2I
-CCS0OxGVfRKCXzIEGIGGtVbEmzzY/uE30snLB9kRaK8
--> ssh-ed25519 vwVIvQ 1WBqhGRWns+zq69pFhRGPKWE2UwXmwAp7XPGgU8MG1o
-TAEdHH7heHvG+i/jOTAxUbtosLIJ4TL012eoivpYbF0
--> ssh-ed25519 fBrw3g rnGGZG45LWTLl9rMUtusEADK//cskSRDbVNHqymaRX0
-eQyOhsDPF3o1hlfrVJqHd873wPtMkWW1QhslCY9qyL4
--> ssh-ed25519 S5xQfg /SYkIjBEuboiTRBN3/Hy54WCm30rPeTb7hyW8Ne8XQo
-Zixg9/lPh8sEylM0k4UZMbKiM7G07C1s8z0U+RWGxb0
--> ssh-ed25519 XPxfUQ YMnUrS3KvUfwX+QXvHMDWj/dF58e8CpaFMMuQlHaBCc
-qtOCQrVvAzaG3TlG2EUHwMHXC6Q1STC5ZnEGZ/yh6U4
--> ssh-ed25519 SpD5mg e+roG6erU7LFz/JG081vYo2QG5/72pGhxrrmJfNTLVM
-taKovBNCmrYb5712fnZ2Gq1GLQfog9IyezIhyN2hYkA
--> ssh-ed25519 Kk8sng T+387c8n+DojonjyQnB19JJifJWHufU+/oTLrvjzW2I
-nhjogEUDM7m6jTLiY3qZibKfe/8SPDkoz4v5TqY0Or0
--> Nd\-#+C1-grease ~l H Gz#
-52s5cUftipJSF08pl6py9q7wRH03s++6wqZZwNXTWfTyfVRpiv+LT+1vG8G40ByB
-XZHLe7A6CJuPj6FEabA5Yz5Q9IcY5KklKoCSzlJPaWHUySon
---- VqPdz+C6jxXzt5awQag/7jMrGk8dkpe4JSPBB91fKUo
-l ˆÎyÌn‡	Œéë„<C3AB>üFŸÕ‘_Ÿ1„%~²×‰9ÌÿîŠ%üÍp^ocý<63>k
-¥:tcƒéþ<>
+-> ssh-ed25519 6AT2/g yTW46JmDIftcOqogIDjheXJf2sw/dG2WEJxfCXU/LDk
+0Co5/Rn22kmdcPr61ZOrmZJbPFHx2wJ8/YkbDjcjqKo
+-> ssh-ed25519 dMQYog RtZT0PwVL4kxUHilOhH2GBp8Z9WfyBkaxB62pjKpHA4
+muMlIt8VYQftMYacfdnQFeejfWpKTEG5gxbFNy97GTc
+-> 4|)`7yq-grease P#\5k8 +f
+jMegn6ATsj2Ai9B5Xmy+tay1nppwxvF1IGJH+hLNanYMsTIDZypM6UsNdzYQ/3mw
+VZ9ooy8TKUgAJ7jsd6IrKw
+--- tLaPQWJA0Hh5MrxfhaySURgY02K16IlzvsxKpOWGva0
+5?lﾇ'ｼ!ｹｺ<EFBDB9><EFBDBA>ﾜｷ匪Nxｽ+<2B>A9ﾟﾑﾘl/ｸﾞ諟ﾎ|旙<>Sｵ&ｺｻ､繃<>Q;_<>K

secrets/nextcloud-pw.age

@@ -1,38 +1,10 @@
 age-encryption.org/v1
--> ssh-ed25519 xoAm7w TBgxtsm66foJGt2nHrXtgzpohAgCjlFZKJJ92wVRwHE
-3v4D4OIsYkS/f1cFzjkqsZWRY+06H/hK5BRUlK0rezE
--> ssh-ed25519 mbw8xA DGk5uuy20YbA60OCsRRUpDPDn8/34kXB6fLPM5EaM1A
-gX+54o4ZGMepfR+IgWtnfRrQyJI7cIaVfrB6F0bhJpM
--> ssh-ed25519 2a2Yhw 4duxFtOvot0/yg/mat9/mq7e+3FyBRVZbDxab1kUIz0
-eYGM5bSmdkepVFtC/hu9QryA7bAmyZ1aGk+Gz11IUeI
--> ssh-ed25519 dMQYog j1ltBo3adLkVjynrfftU3SUuXbtkPJwqvLScqV3Egi0
-8MPpwdCulNGlw261+weh0Lg5Nn7nztUrMHyai9WFV5Q
--> ssh-ed25519 G2eSCQ /IbtFM08snzoYcw59E0VDWE3Y8GUSqJDZ5X6i6l5ghU
-x0TAzntr4kHb9iZVqsSYAdS0mIBjdZjG6muniNz/R1A
--> ssh-ed25519 6AT2/g htY9fgi/p9aGNEGsUdcVJ7tQ5cEXDxFXAXNRvf2aAlU
-gxhbdfZuyPVHYLl9lsfSe1Hu/cNwgFNbT8BzScL23AM
--> ssh-ed25519 yHDAQw r/Q4+U6D9kBM+2Tks18bVh3ketV6XPsPooq+mD84WhQ
-Mpy9wGJRPhBvpnI9Nl38BkRdXgezKg8aOdXZIzuttds
--> ssh-ed25519 VyYH/Q bj6YIiIkAWr9f7jagbx57jRvSDcnmbqtx6Q1ijcOTDk
-IWktummkI2rkjpxG1wP7VG5NAZKixC/qEi//fBfPWCs
--> ssh-ed25519 hPp1nw Q4EhATKtqAuvaaSZ6mvdHYxLtEUIiq6vXCt2TIW3ujw
-XzrrxplNBzH84lzWheGA7GFtw6KAhD/Fn1dUh/IC0RQ
--> ssh-ed25519 CRfjsA tJMmvRH7ycObgJ5ZjVnIsC1NuqmUwpcAfIl6zxRfdTg
-eDxKa8mqdbZbmbv+WmhbQWO+0KG7vsu3i1V+Fj1/1IU
--> ssh-ed25519 vwVIvQ mAizKoyN4QkTztzMBiz53+H7pJunPT6mZmXW75hcMh8
-/T03UWjaTw9TcJg9wrFJINs02PG11vabRgptlY/QyTk
--> ssh-ed25519 fBrw3g wTn+uGVURTaLh2uuy+hOMXADU8Y5X5rjt/I5hlgkiEU
-djupGCAIbmou/kaeS8bvCHFU4h/tDJUPj1D7Ad3uGOg
--> ssh-ed25519 S5xQfg ndVRf5DZJG6ppRG4BLadwFdHcJBO7YsLamdY8h6M4GU
-GdYFdMZtd7zRAejs0CoGkKEg3LquMD3rXqbXIakfv2g
--> ssh-ed25519 XPxfUQ OzGJy6jOdsX2FOaqyPaGfGVvnmMessbQSFaaRSiJrkA
-/F4Rb1DdKrq8cXW1wJZQME3cjdvCg8uGnORzJH4apIA
--> ssh-ed25519 SpD5mg rwLBuiTLSo3Sg1pfrKX72AtOYylGVsUqVvpPjfWy+n8
-Ly2MAPdLr5T91QEaDK/SzBHa6eRXM4qnl6AUzc1LkMk
--> ssh-ed25519 Kk8sng MPQqZeCMmHpseUkC1ysBlqhfOFZBWvb0rII9vLj2fG4
-bx/POtqvkke0V8mtug14PzjekwvKu8lqY1C3BBHdrq0
--> H;;#1-grease NIQ|aNi" ^ m ub=UpUr`
-62l691hcYEUai/MhlIgoxGmJwGl1NA6CgWl8LSGbHHfAwtCa6pO7kORspVcI1nBY
-fm6hfhPep5Djdinrz3ti+fprTjOpJafJoAsvxE14
---- fxAevZGLf/QUXm/kEserDh7XPyy0lfDaUI+NnCfrZbA
-^ŜˡÁ2Í?éÈüÇ‚‚tve<76>
+-> ssh-ed25519 6AT2/g hXS7zxzYhlu5GrUAEAnaO+CizpbifjDxIwoAK55cjV0
+xU7Z52cjARU8tmd1AJ9v8+QTQzfL/mNxP/f/bJAzYvo
+-> ssh-ed25519 dMQYog 8PEp5TmEOumhWUZvko42sOKpkqOCW9/zCrMqn+fJ2ws
+wJo8x6+hyU8iJkTqGVecZ88hG661F3ZvEvVqpJzox5w
+-> x-grease tdW'\ +(>9 da%@^H6
+q04xwjRaNOBfNhAvik762vJHio/qTfR6qQW4QsD+wzEidRYRggNdQwTl+G4jkWAu
+fx0xZeiI5qVm6WG8lg
+--- pHx5BdqI3HubR9wAtPyfMaYbr8uqRwOS1qFJhtC4wuM
+Èv°ºg9sÉÉ¡§6:`Nlëªø`.‘•ÓÍPebÅSNn<>å­Ä8C<<3C>¥a=-¨Gh¤ò.ªfHm<48>»æUëç
PGpS}µxýùã#ÎT