prototype

common/auto-update.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, pkgs, lib, ... }:
 
 # Modify auto-update so that it pulls a flake
 
@@ -6,10 +6,20 @@ let
   cfg = config.system.autoUpgrade;
 in
 {
-  config = lib.mkIf cfg.enable {
-    system.autoUpgrade = {
-      flake = "git+https://git.neet.dev/zuckerberg/nix-config.git";
-      flags = [ "--recreate-lock-file" "--no-write-lock-file" ]; # ignore lock file, just pull the latest
-    };
-  };
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    {
+      system.autoUpgrade = {
+        flake = "git+https://git.neet.dev/zuckerberg/nix-config.git";
+        flags = [ "--recreate-lock-file" "--no-write-lock-file" ]; # ignore lock file, just pull the latest
+
+        # dates = "03:40";
+        # kexecWindow = lib.mkDefault { lower = "01:00"; upper = "05:00"; };
+        # randomizedDelaySec = "45min";
+      };
+
+      system.autoUpgrade.allowKexec = lib.mkDefault true;
+
+      luks.enableKexec = cfg.allowKexec && builtins.length config.luks.devices > 0;
+    }
+  ]);
 }

common/boot/default.nix

@@ -5,6 +5,8 @@
     ./firmware.nix
     ./efi.nix
     ./bios.nix
+    ./kexec-luks.nix
+    ./luks.nix
     ./remote-luks-unlock.nix
   ];
 }

common/boot/kexec-luks.nix

@@ -0,0 +1,121 @@
+# Allows kexec'ing as an alternative to rebooting for machines that
+# have luks encrypted partitions that need to be mounted at boot.
+# These luks partitions will be automatically unlocked, no password,
+# or any interaction needed whatsoever.
+
+# This is accomplished by fetching the luks key(s) while the system is running,
+# then building a temporary initrd that contains the luks key(s), and kexec'ing.
+
+{ config, lib, pkgs, ... }:
+
+{
+  options.luks = {
+    enableKexec = lib.mkEnableOption "Enable support for transparent passwordless kexec while using luks";
+  };
+
+  config = lib.mkIf config.luks.enableKexec {
+    luks.fallbackToPassword = true;
+    luks.disableKeyring = true;
+
+    boot.initrd.luks.devices = lib.listToAttrs
+      (builtins.map
+        (item:
+          {
+            name = item;
+            value = {
+              masterKeyFile = "/etc/${item}.key";
+            };
+          })
+        config.luks.deviceNames);
+
+    systemd.services.prepare-luks-kexec-image = {
+      description = "Prepare kexec automatic LUKS unlock on kexec reboot without a password";
+
+      wantedBy = [ "kexec.target" ];
+      unitConfig.DefaultDependencies = false;
+      serviceConfig.Type = "oneshot";
+
+      path = with pkgs; [ file kexec-tools coreutils-full cpio findutils gzip xz zstd lvm2 xxd gawk ];
+
+      # based on https://github.com/flowztul/keyexec
+      script = ''
+        system=/nix/var/nix/profiles/system
+        old_initrd=$(readlink -f "$system/initrd")
+
+        umask 0077
+        CRYPTROOT_TMPDIR="$(mktemp -d --tmpdir=/dev/shm)"
+
+        cleanup() {
+          shred -fu "$CRYPTROOT_TMPDIR/initrd_contents/etc/"*.key || true
+          shred -fu "$CRYPTROOT_TMPDIR/new_initrd" || true
+          shred -fu "$CRYPTROOT_TMPDIR/secret/"* || true
+          rm -rf "$CRYPTROOT_TMPDIR"
+        }
+        # trap cleanup INT TERM EXIT
+
+        mkdir -p "$CRYPTROOT_TMPDIR"
+        cd "$CRYPTROOT_TMPDIR"
+
+        # Determine the compression type of the initrd image
+        compression=$(file -b --mime-type "$old_initrd" | awk -F'/' '{print $2}')
+
+        # Decompress the initrd image based on its compression type
+        case "$compression" in
+          gzip)
+            gunzip -c "$old_initrd" > initrd.cpio
+            ;;
+          xz)
+            unxz -c "$old_initrd" > initrd.cpio
+            ;;
+          zstd)
+            zstd -d -c "$old_initrd" > initrd.cpio
+            ;;
+          *)
+            echo "Unsupported compression type: $compression"
+            exit 1
+            ;;
+        esac
+
+        # Extract the contents of the cpio archive
+        mkdir -p initrd_contents
+        cd initrd_contents
+        cpio -idv < ../initrd.cpio
+
+        # Generate keys and add them to the extracted initrd filesystem
+        luksDeviceNames=(${builtins.concatStringsSep " " config.luks.deviceNames})
+        for item in "''${luksDeviceNames[@]}"; do
+          dmsetup --showkeys table "$item" | cut -d ' ' -f5 | xxd -ps -g1 -r > "./etc/$item.key"
+        done
+
+        # Add normal initrd secrets too
+        ${lib.concatStringsSep "\n" (lib.mapAttrsToList (dest: source:
+            let source' = if source == null then dest else builtins.toString source; in
+              ''
+                mkdir -p $(dirname "./${dest}")
+                cp -a ${source'} "./${dest}"
+              ''
+          ) config.boot.initrd.secrets)
+        }
+
+        # Create a new cpio archive with the modified contents
+        find . | cpio -o -H newc -v > ../new_initrd.cpio
+
+        # Compress the new cpio archive using the original compression type
+        cd ..
+        case "$compression" in
+          gzip)
+            gunzip -c new_initrd.cpio > new_initrd
+            ;;
+          xz)
+            unxz -c new_initrd.cpio > new_initrd
+            ;;
+          zstd)
+            zstd -c new_initrd.cpio > new_initrd
+            ;;
+        esac
+
+        kexec --load "$system/kernel" --append "init=$system/init ${builtins.concatStringsSep " " config.boot.kernelParams}" --initrd "$CRYPTROOT_TMPDIR/new_initrd"
+      '';
+    };
+  };
+}

common/boot/luks.nix

@@ -0,0 +1,74 @@
+# Makes it a little easier to configure luks partitions for boot
+# Additionally, this solves a circular dependency between kexec luks
+# and NixOS's luks module.
+
+{ config, lib, ... }:
+
+let
+  cfg = config.luks;
+
+  deviceCount = builtins.length cfg.devices;
+
+  deviceMap = lib.imap
+    (i: item: {
+      device = item;
+      name =
+        if deviceCount == 1 then "enc-pv"
+        else "enc-pv${builtins.toString (i + 1)}";
+    })
+    cfg.devices;
+in
+{
+  options.luks = {
+    devices = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+    };
+
+    allowDiscards = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+    };
+
+    fallbackToPassword = lib.mkEnableOption
+      "Fallback to interactive passphrase prompt if the cannot be found.";
+
+    disableKeyring = lib.mkEnableOption
+      "When opening LUKS2 devices, don't use the kernel keyring";
+
+    # set automatically, don't touch
+    deviceNames = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+    };
+  };
+
+  config = lib.mkMerge [
+    {
+      assertions = [
+        {
+          assertion = deviceCount == builtins.length (builtins.attrNames config.boot.initrd.luks.devices);
+          message = ''
+            All luks devices must be specified using `luks.devices` not `boot.initrd.luks.devices`.
+          '';
+        }
+      ];
+    }
+    (lib.mkIf (deviceCount != 0) {
+      luks.deviceNames = builtins.map (device: device.name) deviceMap;
+
+      boot.initrd.luks.devices = lib.listToAttrs (
+        builtins.map
+          (item:
+            {
+              name = item.name;
+              value = {
+                device = item.device;
+                allowDiscards = cfg.allowDiscards;
+                fallbackToPassword = cfg.fallbackToPassword;
+                disableKeyring = cfg.disableKeyring;
+              };
+            })
+          deviceMap);
+    })
+  ];
+}

common/boot/remote-luks-unlock.nix

@@ -33,11 +33,6 @@ in
   };
 
   config = lib.mkIf cfg.enable {
-    # boot.initrd.luks.devices.${cfg.device.name} = {
-    #   device = cfg.device.path;
-    #   allowDiscards = cfg.device.allowDiscards;
-    # };
-
     # Unlock LUKS disk over ssh
     boot.initrd.network.enable = true;
     boot.initrd.kernelModules = cfg.kernelModules;

flake.nix

@@ -87,6 +87,7 @@
                 src = nixpkgs;
                 patches = [
                   inputs.nixpkgs-hostapd-pr
+                  ./patches/kexec-luks.patch
                 ];
               };
               patchedNixpkgs = nixpkgs.lib.fix (self: (import "${patchedNixpkgsSrc}/flake.nix").outputs { self = nixpkgs; });

machines/phil/default.nix

@@ -9,4 +9,6 @@
     enable = true;
     instanceUrl = "https://git.neet.dev";
   };
+
+  system.autoUpgrade.enable = true;
 }

machines/phil/hardware-configuration.nix

@@ -20,10 +20,7 @@
   boot.kernelModules = [ ];
   boot.extraModulePackages = [ ];
 
-  boot.initrd.luks.devices."enc-pv" = {
-    device = "/dev/disk/by-uuid/d26c1820-4c39-4615-98c2-51442504e194";
-    allowDiscards = true;
-  };
+  luks.devices = [ "/dev/disk/by-uuid/d26c1820-4c39-4615-98c2-51442504e194" ];
 
   fileSystems."/" =
     {

machines/ponyo/default.nix

@@ -6,6 +6,9 @@
   ];
 
   system.autoUpgrade.enable = true;
+  # I want to manually trigger kexec updates for now on ponyo
+  system.autoUpgrade.allowKexec = false;
+  luks.enableKexec = true;
 
   # p2p mesh network
   services.tailscale.exitNode = true;

machines/ponyo/hardware-configuration.nix

@@ -19,12 +19,15 @@
   };
 
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv".device = "/dev/disk/by-uuid/4cc36be4-dbff-4afe-927d-69bf4637bae2";
-  boot.initrd.luks.devices."enc-pv2".device = "/dev/disk/by-uuid/e52b01b3-81c8-4bb2-ae7e-a3d9c793cb00"; # expanded disk
+
+  luks.devices = [
+    "/dev/disk/by-uuid/4cc36be4-dbff-4afe-927d-69bf4637bae2"
+    "/dev/disk/by-uuid/e52b01b3-81c8-4bb2-ae7e-a3d9c793cb00"
+  ];
 
   fileSystems."/" =
     {
-      device = "/dev/mapper/enc-pv";
+      device = "/dev/mapper/enc-pv1";
       fsType = "btrfs";
     };
 

machines/ray/hardware-configuration.nix

@@ -36,10 +36,7 @@
 
   # disks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv" = {
-    device = "/dev/disk/by-uuid/c1822e5f-4137-44e1-885f-954e926583ce";
-    allowDiscards = true;
-  };
+  luks.devices = [ "/dev/disk/by-uuid/c1822e5f-4137-44e1-885f-954e926583ce" ];
   fileSystems."/" =
     {
       device = "/dev/vg/root";

machines/router/hardware-configuration.nix

@@ -32,7 +32,7 @@
 
   # disks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv".device = "/dev/disk/by-uuid/9b090551-f78e-45ca-8570-196ed6a4af0c";
+  luks.devices = [ "/dev/disk/by-uuid/9b090551-f78e-45ca-8570-196ed6a4af0c" ];
   fileSystems."/" =
     {
       device = "/dev/disk/by-uuid/421c82b9-d67c-4811-8824-8bb57cb10fce";

machines/storage/s0/hardware-configuration.nix

@@ -25,10 +25,12 @@
 
   # luks
   remoteLuksUnlock.enable = true;
-  boot.initrd.luks.devices."enc-pv1".device = "/dev/disk/by-uuid/d52e99a9-8825-4d0a-afc1-8edbef7e0a86";
-  boot.initrd.luks.devices."enc-pv2".device = "/dev/disk/by-uuid/f7275585-7760-4230-97de-36704b9a2aa3";
-  boot.initrd.luks.devices."enc-pv3".device = "/dev/disk/by-uuid/5d1002b8-a0ed-4a1c-99f5-24b8816d9e38";
-  boot.initrd.luks.devices."enc-pv4".device = "/dev/disk/by-uuid/e2c7402a-e72c-4c4a-998f-82e4c10187bc";
+  luks.devices = [
+    "/dev/disk/by-uuid/d52e99a9-8825-4d0a-afc1-8edbef7e0a86"
+    "/dev/disk/by-uuid/f7275585-7760-4230-97de-36704b9a2aa3"
+    "/dev/disk/by-uuid/5d1002b8-a0ed-4a1c-99f5-24b8816d9e38"
+    "/dev/disk/by-uuid/e2c7402a-e72c-4c4a-998f-82e4c10187bc"
+  ];
 
   # mounts
   fileSystems."/" =

patches/kexec-luks.patch

@@ -0,0 +1,264 @@
+diff --git a/nixos/modules/system/boot/luksroot.nix b/nixos/modules/system/boot/luksroot.nix
+index b8f36538e70..cc320a04c70 100644
+--- a/nixos/modules/system/boot/luksroot.nix
++++ b/nixos/modules/system/boot/luksroot.nix
+@@ -146,6 +146,7 @@ let
+     csopen = "cryptsetup luksOpen ${dev.device} ${dev.name}"
+            + optionalString dev.allowDiscards " --allow-discards"
+            + optionalString dev.bypassWorkqueues " --perf-no_read_workqueue --perf-no_write_workqueue"
++           + optionalString dev.disableKeyring " --disable-keyring"
+            + optionalString (dev.header != null) " --header=${dev.header}";
+     cschange = "cryptsetup luksChangeKey ${dev.device} ${optionalString (dev.header != null) "--header=${dev.header}"}";
+     fido2luksCredentials = dev.fido2.credentials ++ optional (dev.fido2.credential != null) dev.fido2.credential;
+@@ -243,6 +244,26 @@ let
+                do_open_passphrase
+             fi
+         fi
++        '' else if (dev.masterKeyFile != null) then ''
++        if wait_target "key file" ${dev.masterKeyFile}; then
++            ${csopen} --master-key-file=${dev.masterKeyFile}
++            cs_status=$?
++            if [ $cs_status -ne 0 ]; then
++              echo "Key File ${dev.masterKeyFile} failed!"
++              if ! try_empty_passphrase; then
++                ${if dev.fallbackToPassword then "echo" else "die"} "${dev.masterKeyFile} is unavailable"
++                echo " - failing back to interactive password prompt"
++                do_open_passphrase
++              fi
++            fi
++        else
++            # If the key file never shows up we should also try the empty passphrase
++            if ! try_empty_passphrase; then
++               ${if dev.fallbackToPassword then "echo" else "die"} "${dev.masterKeyFile} is unavailable"
++               echo " - failing back to interactive password prompt"
++               do_open_passphrase
++            fi
++        fi
+         '' else ''
+            if ! try_empty_passphrase; then
+               do_open_passphrase
+@@ -625,6 +646,15 @@ in
+             '';
+           };
+ 
++          masterKeyFile = mkOption {
++            default = null;
++            type = types.nullOr types.str;
++            description = lib.mdDoc ''
++              The name of the file (can be a raw device or a partition) that
++              should be used as the master decryption key for the encrypted device.
++            '';
++          };
++
+           tryEmptyPassphrase = mkOption {
+             default = false;
+             type = types.bool;
+@@ -700,6 +730,15 @@ in
+             '';
+           };
+ 
++          disableKeyring = mkOption {
++            default = false;
++            type = types.bool;
++            description = lib.mdDoc ''
++              Disables using the kernel keyring for LUKS2 disks.
++              This is already the default behavior for LUKS1
++            '';
++          };
++
+           fallbackToPassword = mkOption {
+             default = false;
+             type = types.bool;
+diff --git a/nixos/modules/tasks/auto-upgrade.nix b/nixos/modules/tasks/auto-upgrade.nix
+index 29e3e313336..050bf09c208 100644
+--- a/nixos/modules/tasks/auto-upgrade.nix
++++ b/nixos/modules/tasks/auto-upgrade.nix
+@@ -4,7 +4,8 @@ with lib;
+ 
+ let cfg = config.system.autoUpgrade;
+ 
+-in {
++in
++{
+ 
+   options = {
+ 
+@@ -22,7 +23,7 @@ in {
+       };
+ 
+       operation = mkOption {
+-        type = types.enum ["switch" "boot"];
++        type = types.enum [ "switch" "boot" ];
+         default = "switch";
+         example = "boot";
+         description = lib.mdDoc ''
+@@ -86,14 +87,14 @@ in {
+         '';
+       };
+ 
+-      allowReboot = mkOption {
++      allowKexec = mkOption {
+         default = false;
+         type = types.bool;
+         description = lib.mdDoc ''
+-          Reboot the system into the new generation instead of a switch
++          kexec the system into the new generation instead of a switch
+           if the new generation uses a different kernel, kernel modules
+           or initrd than the booted system.
+-          See {option}`rebootWindow` for configuring the times at which a reboot is allowed.
++          See {option}`kexecWindow` for configuring the times at which a kexec is allowed.
+         '';
+       };
+ 
+@@ -109,25 +110,25 @@ in {
+         '';
+       };
+ 
+-      rebootWindow = mkOption {
++      kexecWindow = mkOption {
+         description = lib.mdDoc ''
+           Define a lower and upper time value (in HH:MM format) which
+-          constitute a time window during which reboots are allowed after an upgrade.
+-          This option only has an effect when {option}`allowReboot` is enabled.
+-          The default value of `null` means that reboots are allowed at any time.
++          constitute a time window during which kexecs are allowed after an upgrade.
++          This option only has an effect when {option}`allowKexec` is enabled.
++          The default value of `null` means that kexecs are allowed at any time.
+         '';
+         default = null;
+         example = { lower = "01:00"; upper = "05:00"; };
+         type = with types; nullOr (submodule {
+           options = {
+             lower = mkOption {
+-              description = lib.mdDoc "Lower limit of the reboot window";
++              description = lib.mdDoc "Lower limit of the kexec window";
+               type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+               example = "01:00";
+             };
+ 
+             upper = mkOption {
+-              description = lib.mdDoc "Upper limit of the reboot window";
++              description = lib.mdDoc "Upper limit of the kexec window";
+               type = types.strMatching "[[:digit:]]{2}:[[:digit:]]{2}";
+               example = "05:00";
+             };
+@@ -165,12 +166,12 @@ in {
+     }];
+ 
+     system.autoUpgrade.flags = (if cfg.flake == null then
+-        [ "--no-build-output" ] ++ optionals (cfg.channel != null) [
+-          "-I"
+-          "nixpkgs=${cfg.channel}/nixexprs.tar.xz"
+-        ]
+-      else
+-        [ "--flake ${cfg.flake}" ]);
++      [ "--no-build-output" ] ++ optionals (cfg.channel != null) [
++        "-I"
++        "nixpkgs=${cfg.channel}/nixexprs.tar.xz"
++      ]
++    else
++      [ "--flake ${cfg.flake}" ]);
+ 
+     systemd.services.nixos-upgrade = {
+       description = "NixOS Upgrade";
+@@ -195,54 +196,56 @@ in {
+         config.programs.ssh.package
+       ];
+ 
+-      script = let
+-        nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+-        date     = "${pkgs.coreutils}/bin/date";
+-        readlink = "${pkgs.coreutils}/bin/readlink";
+-        shutdown = "${config.systemd.package}/bin/shutdown";
+-        upgradeFlag = optional (cfg.channel == null) "--upgrade";
+-      in if cfg.allowReboot then ''
+-        ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)}
+-        booted="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
+-        built="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
+-
+-        ${optionalString (cfg.rebootWindow != null) ''
+-          current_time="$(${date} +%H:%M)"
+-
+-          lower="${cfg.rebootWindow.lower}"
+-          upper="${cfg.rebootWindow.upper}"
+-
+-          if [[ "''${lower}" < "''${upper}" ]]; then
+-            if [[ "''${current_time}" > "''${lower}" ]] && \
+-               [[ "''${current_time}" < "''${upper}" ]]; then
+-              do_reboot="true"
++      script =
++        let
++          nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
++          date = "${pkgs.coreutils}/bin/date";
++          readlink = "${pkgs.coreutils}/bin/readlink";
++          systemctl_kexec = "${config.systemd.package}/bin/systemctl kexec";
++          upgradeFlag = optional (cfg.channel == null) "--upgrade";
++        in
++        if cfg.allowKexec then ''
++          ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)}
++          booted="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
++          built="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
++
++          ${optionalString (cfg.kexecWindow != null) ''
++            current_time="$(${date} +%H:%M)"
++
++            lower="${cfg.kexecWindow.lower}"
++            upper="${cfg.kexecWindow.upper}"
++
++            if [[ "''${lower}" < "''${upper}" ]]; then
++              if [[ "''${current_time}" > "''${lower}" ]] && \
++                 [[ "''${current_time}" < "''${upper}" ]]; then
++                do_kexec="true"
++              else
++                do_kexec="false"
++              fi
+             else
+-              do_reboot="false"
++              # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
++              # we want to reboot if cur > 23h or cur < 6h
++              if [[ "''${current_time}" < "''${upper}" ]] || \
++                 [[ "''${current_time}" > "''${lower}" ]]; then
++                do_kexec="true"
++              else
++                do_kexec="false"
++              fi
+             fi
++          ''}
++
++          if [ "''${booted}" = "''${built}" ]; then
++            ${nixos-rebuild} ${cfg.operation} ${toString cfg.flags}
++          ${optionalString (cfg.kexecWindow != null) ''
++            elif [ "''${do_kexec}" != true ]; then
++              echo "Outside of configured kexec window, skipping."
++          ''}
+           else
+-            # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
+-            # we want to reboot if cur > 23h or cur < 6h
+-            if [[ "''${current_time}" < "''${upper}" ]] || \
+-               [[ "''${current_time}" > "''${lower}" ]]; then
+-              do_reboot="true"
+-            else
+-              do_reboot="false"
+-            fi
++            ${systemctl_kexec}
+           fi
+-        ''}
+-
+-        if [ "''${booted}" = "''${built}" ]; then
+-          ${nixos-rebuild} ${cfg.operation} ${toString cfg.flags}
+-        ${optionalString (cfg.rebootWindow != null) ''
+-          elif [ "''${do_reboot}" != true ]; then
+-            echo "Outside of configured reboot window, skipping."
+-        ''}
+-        else
+-          ${shutdown} -r +1
+-        fi
+-      '' else ''
+-        ${nixos-rebuild} ${cfg.operation} ${toString (cfg.flags ++ upgradeFlag)}
+-      '';
++        '' else ''
++          ${nixos-rebuild} ${cfg.operation} ${toString (cfg.flags ++ upgradeFlag)}
++        '';
+ 
+       startAt = cfg.dates;
+