hosts(pve-epyc): auto restart vGPU manager if mdev is not registered

xddxdd · Jan 8, 2025 · 3488698 · 3488698
1 parent e4fc2d7
commit 3488698
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 3 deletions.
diff --git a/flake.lock b/flake.lock
diff --git a/nixos/hardware/nvidia/vgpu-extension.nix b/nixos/hardware/nvidia/vgpu-extension.nix
@@ -58,6 +58,36 @@ in
     };
   };
 
+  systemd.services.nvidia-vgpu-watchdog = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "nvidia-vgpu-mgr.service" ];
+    before = [
+      "pvedaemon.service"
+      "pve-guests.service"
+    ];
+    requiredBy = [
+      "pvedaemon.service"
+      "pve-guests.service"
+    ];
+
+    path = with pkgs; [
+      util-linux
+      systemd
+    ];
+
+    script = ''
+      while true; do
+        (dmesg | grep nvidia | grep "MDEV: Registered") && break
+        echo "Restarting VGPU manager"
+        systemctl restart nvidia-vgpu-mgr
+        sleep 5
+      done
+      exit 0
+    '';
+
+    serviceConfig.Type = "oneshot";
+  };
+
   systemd.tmpfiles.rules = [
     "L /usr/share/nvidia/vgpu/vgpuConfig.xml - - - - ${nvidia_x11.bin}/share/nvidia/vgpu/vgpuConfig.xml"
   ];