diff --git a/machines/ray/configuration.nix b/machines/ray/configuration.nix index 68fed8f..a5b143d 100644 --- a/machines/ray/configuration.nix +++ b/machines/ray/configuration.nix @@ -1,14 +1,5 @@ -{ config, pkgs, fetchurl, lib, ... }: +{ config, pkgs, lib, ... }: -let - nvidia-offload = pkgs.writeShellScriptBin "nvidia-offload" '' - export __NV_PRIME_RENDER_OFFLOAD=1 - export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0 - export __GLX_VENDOR_LIBRARY_NAME=nvidia - export __VK_LAYER_NV_optimus=NVIDIA_only - exec -a "$0" "$@" - ''; -in { disabledModules = [ "hardware/video/nvidia.nix" @@ -33,31 +24,37 @@ in hardware.enableAllFirmware = true; # newer kernel for wifi - boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_15; + boot.kernelPackages = pkgs.linuxPackages_latest; # gpu services.xserver.videoDrivers = [ "nvidia" ]; services.xserver.logFile = "/var/log/Xorg.0.log"; - hardware.nvidia.modesetting.enable = true; # for nvidia-vaapi-driver - hardware.nvidia.prime = { - # reverse_sync.enable = true; - # offload.enable = true; - sync.enable = true; - nvidiaBusId = "PCI:1:0:0"; - amdgpuBusId = "PCI:4:0:0"; + hardware.nvidia = { + modesetting.enable = true; # for nvidia-vaapi-driver + prime = { + #reverse_sync.enable = true; + offload.enable = true; + offload.enableOffloadCmd = true; + #sync.enable = true; + nvidiaBusId = "PCI:1:0:0"; + amdgpuBusId = "PCI:4:0:0"; + }; + powerManagement = { +# enable = true; +# finegrained = true; + coarsegrained = true; + }; }; services.archivebox = { - enable = true; + enable = false; hostname = "localhost"; publicCreateSnapshots = true; }; virtualisation.docker.enable = true; - services.spotifyd.enable = true; - services.zerotierone.enable = true; de.enable = true; diff --git a/machines/ray/nvidia.nix b/machines/ray/nvidia.nix index a994238..92b676f 100644 --- a/machines/ray/nvidia.nix +++ b/machines/ray/nvidia.nix @@ -56,6 +56,15 @@ in ''; }; + hardware.nvidia.powerManagement.coarsegrained = mkOption { + type = types.bool; + default = false; + description = '' + Experimental power management of PRIME offload. For more information, see + the NVIDIA docs, chapter 22. PCI-Express runtime power management. + ''; + }; + hardware.nvidia.modesetting.enable = mkOption { type = types.bool; default = false; @@ -131,7 +140,7 @@ in type = types.bool; default = false; description = '' - Configure X to allow external NVIDIA GPUs when using optimus. + Configure X to allow external NVIDIA GPUs when using Prime [Reverse] Sync. ''; }; @@ -148,10 +157,26 @@ in ''; }; + hardware.nvidia.prime.offload.enableOffloadCmd = mkOption { + type = types.bool; + default = false; + description = '' + Adds a `nvidia-offload` convenience script to + for offloading programs to an nvidia device. To work, should have also enabled + or + + Example usage `nvidia-offload sauerbraten_client` + ''; + }; + hardware.nvidia.prime.reverse_sync.enable = mkOption { type = types.bool; default = false; description = '' + Warning: This feature is relatively new, depending on your system this might + work poorly. AMD support, especially so. + See: https://forums.developer.nvidia.com/t/the-all-new-outputsink-feature-aka-reverse-prime/129828 + Enable NVIDIA Optimus support using the NVIDIA proprietary driver via reverse PRIME. If enabled, the Intel/AMD GPU will be used for all rendering, while enabling output to displays attached only to the NVIDIA GPU without a @@ -210,11 +235,6 @@ in igpuBusId = if pCfg.intelBusId != "" then pCfg.intelBusId else pCfg.amdgpuBusId; in mkIf enabled { assertions = [ - { - assertion = with config.services.xserver.displayManager; (gdm.enable && gdm.nvidiaWayland) -> cfg.modesetting.enable; - message = "You cannot use wayland with GDM without modesetting enabled for NVIDIA drivers, set `hardware.nvidia.modesetting.enable = true`"; - } - { assertion = primeEnabled -> pCfg.intelBusId == "" || pCfg.amdgpuBusId == ""; message = '' @@ -222,6 +242,13 @@ in ''; } + { + assertion = offloadCfg.enableOffloadCmd -> offloadCfg.enable || reverseSyncCfg.enable; + message = '' + Offload command requires offloading or reverse prime sync to be enabled. + ''; + } + { assertion = primeEnabled -> pCfg.nvidiaBusId != "" && (pCfg.intelBusId != "" || pCfg.amdgpuBusId != ""); message = '' @@ -234,6 +261,11 @@ in message = "NVIDIA PRIME render offload is currently only supported on versions >= 435.21."; } + { + assertion = (reverseSyncCfg.enable && pCfg.amdgpuBusId != "") -> versionAtLeast nvidia_x11.version "470.0"; + message = "NVIDIA PRIME render offload for AMD APUs is currently only supported on versions >= 470 beta."; + } + { assertion = !(syncCfg.enable && offloadCfg.enable); message = "PRIME Sync and Offload cannot be both enabled"; @@ -245,7 +277,7 @@ in } { - assertion = !(syncCfg.enable && cfg.powerManagement.finegrained); + assertion = !(syncCfg.enable && cfg.powerManagement.finegrained && cfg.powerManagement.coarsegrained); message = "Sync precludes powering down the NVIDIA GPU."; } @@ -254,6 +286,11 @@ in message = "Fine-grained power management requires offload to be enabled."; } + { + assertion = cfg.powerManagement.coarsegrained -> offloadCfg.enable; + message = "Coarse-grained power management requires offload to be enabled."; + } + { assertion = cfg.powerManagement.enable -> ( builtins.pathExists (cfg.package.out + "/bin/nvidia-sleep.sh") && @@ -329,12 +366,27 @@ in environment.etc."egl/egl_external_platform.d".source = "/run/opengl-driver/share/egl/egl_external_platform.d/"; - hardware.opengl.extraPackages = [ nvidia_x11.out ]; - hardware.opengl.extraPackages32 = [ nvidia_x11.lib32 ]; + hardware.opengl.extraPackages = [ + nvidia_x11.out + # pkgs.nvidia-vaapi-driver + ]; + hardware.opengl.extraPackages32 = [ + nvidia_x11.lib32 + # pkgs.pkgsi686Linux.nvidia-vaapi-driver + ]; environment.systemPackages = [ nvidia_x11.bin ] ++ optionals cfg.nvidiaSettings [ nvidia_x11.settings ] - ++ optionals nvidiaPersistencedEnabled [ nvidia_x11.persistenced ]; + ++ optionals nvidiaPersistencedEnabled [ nvidia_x11.persistenced ] + ++ optionals offloadCfg.enableOffloadCmd [ + (pkgs.writeShellScriptBin "nvidia-offload" '' + export __NV_PRIME_RENDER_OFFLOAD=1 + export __NV_PRIME_RENDER_OFFLOAD_PROVIDER=NVIDIA-G0 + export __GLX_VENDOR_LIBRARY_NAME=nvidia + export __VK_LAYER_NV_optimus=NVIDIA_only + exec -a "$0" "$@" + '') + ]; systemd.packages = optional cfg.powerManagement.enable nvidia_x11.out; @@ -399,7 +451,7 @@ in KERNEL=="card*", SUBSYSTEM=="drm", DRIVERS=="nvidia", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia%n c $$(grep nvidia-frontend /proc/devices | cut -d \ -f 1) %n'" KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 0'" KERNEL=="nvidia_uvm", RUN+="${pkgs.runtimeShell} -c 'mknod -m 666 /dev/nvidia-uvm-tools c $$(grep nvidia-uvm /proc/devices | cut -d \ -f 1) 0'" - '' + optionalString cfg.powerManagement.finegrained '' + '' + optionalString (cfg.powerManagement.finegrained || cfg.powerManagement.coarsegrained) '' # Remove NVIDIA USB xHCI Host Controller devices, if present ACTION=="add", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x0c0330", ATTR{remove}="1" @@ -418,8 +470,10 @@ in ACTION=="unbind", SUBSYSTEM=="pci", ATTR{vendor}=="0x10de", ATTR{class}=="0x030200", TEST=="power/control", ATTR{power/control}="on" ''; - boot.extraModprobeConfig = mkIf cfg.powerManagement.finegrained '' + boot.extraModprobeConfig = optionalString cfg.powerManagement.finegrained '' options nvidia "NVreg_DynamicPowerManagement=0x02" + '' + optionalString cfg.powerManagement.coarsegrained '' + options nvidia "NVreg_DynamicPowerManagement=0x01" ''; boot.blacklistedKernelModules = [ "nouveau" "nvidiafb" ];