Improve NVIDIA

machines/ray/configuration.nix

@@ -1,14 +1,5 @@
-{ config, pkgs, fetchurl, lib, ... }:
+{ config, pkgs, lib, ... }:
 
-let
-  nvidia-offload = pkgs.writeShellScriptBin "nvidia-offload" ''
-    export __NV_PRIME_RENDER_OFFLOAD=1
-    export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0
-    export __GLX_VENDOR_LIBRARY_NAME=nvidia
-    export __VK_LAYER_NV_optimus=NVIDIA_only
-    exec -a "$0" "$@"
-  '';
-in
 {
   disabledModules = [
     "hardware/video/nvidia.nix"
@@ -33,31 +24,37 @@ in
   hardware.enableAllFirmware = true;
 
   # newer kernel for wifi
-  boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_15;
+  boot.kernelPackages = pkgs.linuxPackages_latest;
 
   # gpu
   services.xserver.videoDrivers = [ "nvidia" ];
   services.xserver.logFile = "/var/log/Xorg.0.log";
-  hardware.nvidia.modesetting.enable = true; # for nvidia-vaapi-driver
+  hardware.nvidia = {
-  hardware.nvidia.prime = {
+    modesetting.enable = true; # for nvidia-vaapi-driver
-    # reverse_sync.enable = true;
+    prime = {
-    # offload.enable = true;
+      #reverse_sync.enable = true;
-    sync.enable = true;
+      offload.enable = true;
-    nvidiaBusId = "PCI:1:0:0";
+      offload.enableOffloadCmd = true;
-    amdgpuBusId = "PCI:4:0:0";
+      #sync.enable = true;
+      nvidiaBusId = "PCI:1:0:0";
+      amdgpuBusId = "PCI:4:0:0";
+    };
+    powerManagement = {
+#      enable = true;
+#      finegrained = true;
+      coarsegrained = true;
+    };
   };
 
 
   services.archivebox = {
-    enable = true;
+    enable = false;
     hostname = "localhost";
     publicCreateSnapshots = true;
   };
 
   virtualisation.docker.enable = true;
 
-  services.spotifyd.enable = true;
-
   services.zerotierone.enable = true;
 
   de.enable = true;

machines/ray/nvidia.nix

@@ -56,6 +56,15 @@ in
       '';
     };
 
+    hardware.nvidia.powerManagement.coarsegrained = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Experimental power management of PRIME offload. For more information, see
+        the NVIDIA docs, chapter 22. PCI-Express runtime power management.
+      '';
+    };
+
     hardware.nvidia.modesetting.enable = mkOption {
       type = types.bool;
       default = false;
@@ -131,7 +140,7 @@ in
       type = types.bool;
       default = false;
       description = ''
-        Configure X to allow external NVIDIA GPUs when using optimus.
+        Configure X to allow external NVIDIA GPUs when using Prime [Reverse] Sync.
       '';
     };
 
@@ -148,10 +157,26 @@ in
       '';
     };
 
+    hardware.nvidia.prime.offload.enableOffloadCmd = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Adds a `nvidia-offload` convenience script to <option>environment.systemPackages</option>
+        for offloading programs to an nvidia device. To work, should have also enabled
+        <option>hardware.nvidia.prime.offload.enable</option> or <option>hardware.nvidia.prime.reverse_sync.enable</option>
+
+        Example usage `nvidia-offload sauerbraten_client`
+      '';
+    };
+
     hardware.nvidia.prime.reverse_sync.enable = mkOption {
       type = types.bool;
       default = false;
       description = ''
+        Warning: This feature is relatively new, depending on your system this might
+        work poorly. AMD support, especially so.
+        See: https://forums.developer.nvidia.com/t/the-all-new-outputsink-feature-aka-reverse-prime/129828
+
         Enable NVIDIA Optimus support using the NVIDIA proprietary driver via reverse
         PRIME. If enabled, the Intel/AMD GPU will be used for all rendering, while
         enabling output to displays attached only to the NVIDIA GPU without a
@@ -210,11 +235,6 @@ in
       igpuBusId = if pCfg.intelBusId != "" then pCfg.intelBusId else pCfg.amdgpuBusId;
   in mkIf enabled {
     assertions = [
-      {
-        assertion = with config.services.xserver.displayManager; (gdm.enable && gdm.nvidiaWayland) -> cfg.modesetting.enable;
-        message = "You cannot use wayland with GDM without modesetting enabled for NVIDIA drivers, set `hardware.nvidia.modesetting.enable = true`";
-      }
-
       {
         assertion = primeEnabled -> pCfg.intelBusId == "" || pCfg.amdgpuBusId == "";
         message = ''
@@ -222,6 +242,13 @@ in
         '';
       }
 
+      {
+        assertion = offloadCfg.enableOffloadCmd -> offloadCfg.enable || reverseSyncCfg.enable;
+        message = ''
+          Offload command requires offloading or reverse prime sync to be enabled.
+        '';
+      }
+
       {
         assertion = primeEnabled -> pCfg.nvidiaBusId != "" && (pCfg.intelBusId != "" || pCfg.amdgpuBusId != "");
         message = ''
@@ -234,6 +261,11 @@ in
         message = "NVIDIA PRIME render offload is currently only supported on versions >= 435.21.";
       }
 
+      {
+        assertion = (reverseSyncCfg.enable && pCfg.amdgpuBusId != "") -> versionAtLeast nvidia_x11.version "470.0";
+        message = "NVIDIA PRIME render offload for AMD APUs is currently only supported on versions >= 470 beta.";
+      }
+
       {
         assertion = !(syncCfg.enable && offloadCfg.enable);
         message = "PRIME Sync and Offload cannot be both enabled";
@@ -245,7 +277,7 @@ in
       }
 
       {
-        assertion = !(syncCfg.enable && cfg.powerManagement.finegrained);
+        assertion = !(syncCfg.enable && cfg.powerManagement.finegrained && cfg.powerManagement.coarsegrained);
         message = "Sync precludes powering down the NVIDIA GPU.";
       }
 
@@ -254,6 +286,11 @@ in
         message = "Fine-grained power management requires offload to be enabled.";
       }
 
+      {
+        assertion = cfg.powerManagement.coarsegrained -> offloadCfg.enable;
+        message = "Coarse-grained power management requires offload to be enabled.";
+      }
+
       {
         assertion = cfg.powerManagement.enable -> (
           builtins.pathExists (cfg.package.out + "/bin/nvidia-sleep.sh") &&
@@ -329,12 +366,27 @@ in
     environment.etc."egl/egl_external_platform.d".source =
       "/run/opengl-driver/share/egl/egl_external_platform.d/";
 
-    hardware.opengl.extraPackages = [ nvidia_x11.out ];
+    hardware.opengl.extraPackages = [
-    hardware.opengl.extraPackages32 = [ nvidia_x11.lib32 ];
+      nvidia_x11.out
+      # pkgs.nvidia-vaapi-driver
+    ];
+    hardware.opengl.extraPackages32 = [
+      nvidia_x11.lib32
+      # pkgs.pkgsi686Linux.nvidia-vaapi-driver
+    ];
 
     environment.systemPackages = [ nvidia_x11.bin ]
       ++ optionals cfg.nvidiaSettings [ nvidia_x11.settings ]
-      ++ optionals nvidiaPersistencedEnabled [ nvidia_x11.persistenced ];
+      ++ optionals nvidiaPersistencedEnabled [ nvidia_x11.persistenced ]
+      ++ optionals offloadCfg.enableOffloadCmd [ 
+        (pkgs.writeShellScriptBin "nvidia-offload" ''
+          export __NV_PRIME_RENDER_OFFLOAD=1
+          export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0
+          export __GLX_VENDOR_LIBRARY_NAME=nvidia
+          export __VK_LAYER_NV_optimus=NVIDIA_only
+          exec -a "$0" "$@"
+        '') 
+      ];
 
     systemd.packages = optional cfg.powerManagement.enable nvidia_x11.out;
 
@@ -399,7 +451,7 @@ in
         KERNEL=="card*", SUBSYSTEM=="drm", DRIVERS=="nvidia", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia%n c $$(grep nvidia-frontend /proc/devices | cut -d \  -f 1) %n'"
         KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm c $$(grep nvidia-uvm /proc/devices | cut -d \  -f 1) 0'"
         KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm-tools c $$(grep nvidia-uvm /proc/devices | cut -d \  -f 1) 0'"
-      '' + optionalString cfg.powerManagement.finegrained ''
+      '' + optionalString (cfg.powerManagement.finegrained || cfg.powerManagement.coarsegrained) ''
         # Remove NVIDIA USB xHCI Host Controller devices, if present
         ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{remove}="1"
 
@@ -418,8 +470,10 @@ in
         ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="on"
       '';
 
-    boot.extraModprobeConfig = mkIf cfg.powerManagement.finegrained ''
+    boot.extraModprobeConfig = optionalString cfg.powerManagement.finegrained ''
       options nvidia "NVreg_DynamicPowerManagement=0x02"
+    '' + optionalString cfg.powerManagement.coarsegrained ''
+      options nvidia "NVreg_DynamicPowerManagement=0x01"
     '';
 
     boot.blacklistedKernelModules = [ "nouveau" "nvidiafb" ];