├── doc ├── qemu_boot.png ├── uefi_shell.png └── wininst_virtio.png ├── vm.sh └── README.md /doc/qemu_boot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gmasse/gpu-pci-passthrough/HEAD/doc/qemu_boot.png -------------------------------------------------------------------------------- /doc/uefi_shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gmasse/gpu-pci-passthrough/HEAD/doc/uefi_shell.png -------------------------------------------------------------------------------- /doc/wininst_virtio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gmasse/gpu-pci-passthrough/HEAD/doc/wininst_virtio.png -------------------------------------------------------------------------------- /vm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PCIID_GPU_LIST=(02:00.0 03:00.0 81:00.0) 4 | PCIID_SND_LIST=(02:00.1 03:00.1 81:00.1) 5 | 6 | usage() { 7 | echo "$0 VM_ID" 8 | exit 1; 9 | } 10 | 11 | VM_ID=$1 12 | if ! [[ $VM_ID =~ ^[0-9]+$ ]] 13 | then 14 | usage 15 | fi 16 | 17 | 18 | 19 | RDP_PORT=$(( 4000 + $VM_ID )) 20 | DIR="/home/vm/$VM_ID" 21 | 22 | PCIID_GPU=${PCIID_GPU_LIST[$(($VM_ID-1))]} 23 | PCIID_SND=${PCIID_SND_LIST[$(($VM_ID-1))]} 24 | 25 | 26 | 27 | VM="/home/vm/$VM_ID/vm.qcow2" 28 | EFI="/home/vm/$VM_ID/OVMF_VARS.fd" 29 | 30 | # https://www.microsoft.com/fr-fr/software-download/windows10ISO 31 | ISO_WIN="/home/vm/Win10_1607_x64.iso" 32 | BOOT_ON_CD=0 33 | # https://fedoraproject.org/wiki/Windows_Virtio_Drivers 34 | ISO_VIRTIO="/home/vm/virtio-win.iso" 35 | 36 | 37 | #apt-get install ovmf 38 | #OVMF="/usr/share/OVMF/OVMF_VARS.fd" 39 | 40 | #apt-get install git build-essential uuid-dev nasm acpica-tools 41 | #git clone https://github.com/tianocore/edk2.git 42 | #cd edk2 43 | #make -C BaseTools/Source/C 44 | #nice OvmfPkg/build.sh -a X64 -n $(getconf _NPROCESSORS_ONLN) 45 | OVMF="/home/vm/OVMF.fd" 46 | 47 | #apt-get install rpm2cpio 48 | #wget https://www.kraxel.org/repos/jenkins/edk2/edk2.git-ovmf-x64-0-20161104.b2256.g3b25ca8.noarch.rpm 49 | #rpm2cpio edk2.git-ovmf-x64-*.rpm | (cd /; sudo cpio -i --make-directories) 50 | #OVMF="/usr/share/edk2.git/ovmf-x64/OVMF_VARS-pure-efi.fd" 51 | 52 | mkdir -p $DIR 53 | if [ ! -e $VM ]; then 54 | qemu-img create -f qcow2 $VM 60G 55 | BOOT_ON_CD=1 56 | fi 57 | if [ ! -e $EFI ]; then 58 | cp $OVMF $EFI 59 | fi 60 | 61 | 62 | vfiobind() { 63 | dev="$1" 64 | vendor=$(cat /sys/bus/pci/devices/$dev/vendor) 65 | device=$(cat /sys/bus/pci/devices/$dev/device) 66 | if [ -e /sys/bus/pci/devices/$dev/driver ]; then 67 | sudo sh -c "echo $dev > /sys/bus/pci/devices/$dev/driver/unbind" 68 | fi 69 | sudo sh -c "echo $vendor $device > /sys/bus/pci/drivers/vfio-pci/new_id" 70 | } 71 | 72 | sudo modprobe vfio-pci 73 | vfiobind "0000:$PCIID_GPU" 74 | vfiobind "0000:$PCIID_SND" 75 | NUMA_MODE=$( cat /sys/bus/pci/devices/0000:${PCIID_GPU}/numa_node ) 76 | 77 | 78 | OPTS="" 79 | 80 | # Basic CPU settings. 81 | OPTS="$OPTS -cpu host,kvm=off" 82 | OPTS="$OPTS -smp 4,sockets=1,cores=2,threads=2" 83 | 84 | # ICH9 emulation for better support of PCI-E passthrough 85 | #OPTS="$OPTS -machine type=q35,accel=kvm" 86 | #OPTS="$OPTS -device ioh3420,bus=pcie.0,addr=1c.0,multifunction=on,port=1,chassis=1,id=root.1" 87 | 88 | # Enable KVM full virtualization support. 89 | OPTS="$OPTS -enable-kvm" 90 | 91 | # Assign memory to the vm. 92 | OPTS="$OPTS -m 8G" 93 | 94 | # VFIO GPU and GPU sound passthrough. 95 | #OPTS="$OPTS -device vfio-pci,host=81:00.0,bus=root.1,addr=00.0,multifunction=on" 96 | #OPTS="$OPTS -device vfio-pci,host=81:00.1,bus=root.1,addr=00.1" 97 | OPTS="$OPTS -device vfio-pci,host=$PCIID_GPU,multifunction=on" 98 | OPTS="$OPTS -device vfio-pci,host=$PCIID_SND" 99 | 100 | # Supply OVMF (general UEFI bios, needed for EFI boot support with GPT disks). 101 | #OPTS="$OPTS -drive if=pflash,format=raw,readonly,file=$OVMF" 102 | #OPTS="$OPTS -drive if=pflash,format=raw,file=$EFI" 103 | OPTS="$OPTS -drive if=pflash,format=raw,file=$EFI" 104 | 105 | # Load our created VM image as a harddrive. 106 | OPTS="$OPTS -drive if=virtio,file=$VM" 107 | 108 | # Load our OS setup image e.g. ISO file. 109 | OPTS="$OPTS -drive media=cdrom,file=$ISO_WIN" 110 | OPTS="$OPTS -drive media=cdrom,file=$ISO_VIRTIO" 111 | 112 | # Use the following emulated video device (use none for disabled). 113 | OPTS="$OPTS -vga std" 114 | OPTS="$OPTS -vnc :$VM_ID,password -usbdevice tablet" 115 | 116 | # User mode network, with VirtIO NIC 117 | OPTS="$OPTS -device virtio-net,netdev=vmnic" 118 | OPTS="$OPTS -netdev user,id=vmnic" 119 | #OPTS="$OPTS -netdev user,id=vmnic,hostfwd=tcp::$RDP_PORT-:3389" 120 | OPTS="$OPTS -redir tcp:$RDP_PORT::3389" 121 | 122 | # Redirect QEMU's console input and output. 123 | OPTS="$OPTS -monitor stdio" 124 | 125 | # Boot order 126 | if [ $BOOT_ON_CD -ne 0 ] 127 | then 128 | OPTS="$OPTS -boot once=d" 129 | fi 130 | 131 | 132 | echo numactl --cpunodebind=$NUMA_MODE screen -S "vm-$VM_ID" -d -m qemu-system-x86_64 $OPTS 133 | sudo numactl --cpunodebind=$NUMA_MODE screen -S "vm-$VM_ID" -d -m qemu-system-x86_64 $OPTS 134 | 135 | LOCAL_IP=$(LC_ALL=C /sbin/ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p') 136 | VNC_PORT=$(( 5900 + $VM_ID )) 137 | echo 138 | echo "vnc://$LOCAL_IP:$VNC_PORT (change vnc password to enable vnc)" 139 | echo "rdp://$LOCAL_IP:$RDP_PORT" 140 | echo 141 | 142 | 143 | exit 0 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nvidia GTX GPU Passthrough with QEMU 2 | 3 | ## I. Installation 4 | 5 | #### 1. Host preparation 6 | 7 | Install your server with Ubuntu 16.04 LTS (with Ubuntu default Kernel) 8 | 9 | 10 | ##### i. Update system 11 | ``` 12 | # sudo apt-get update 13 | # sudo apt-get dist-upgrade 14 | # sudo apt-get install iptables 15 | ``` 16 | 17 | ##### ii. Enable firewall 18 | ``` 19 | sudo iptables -A INPUT -i lo -j ACCEPT 20 | sudo iptables -A INPUT -p icmp -j ACCEPT 21 | sudo iptables -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 22 | sudo iptables -A INPUT -s MY.OFFICE.IP.ADDR -j ACCEPT 23 | sudo iptables -P INPUT DROP 24 | sudo iptables -P FORWARD DROP 25 | sudo ip6tables -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 26 | sudo ip6tables -P INPUT DROP 27 | sudo ip6tables -P FORWARD DROP 28 | 29 | sudo apt-get install iptables-persistent 30 | ``` 31 | 32 | ##### iii. intel_iommu 33 | `# sudo vi /etc/default/grub` 34 | 35 | Append `intel_iommu=on` to `GRUB_CMDLINE_LINUX_DEFAULT` variable: 36 | 37 | `GRUB_CMDLINE_LINUX_DEFAULT="[...] intel_iommu=on"` 38 | 39 | ##### iv. loading modules at boot 40 | `# sudo vi /etc/modules` 41 | ``` 42 | vfio 43 | vfio_iommu_type1 44 | vfio_pci 45 | vfio_virqfd 46 | kvm 47 | kvm_intel 48 | ``` 49 | 50 | `# sudo sh -c 'echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf'` 51 | 52 | 53 | ##### v. disabling nvidia driver 54 | `# sudo sh -c 'echo -e "\nblacklist nouveau" >> /etc/modprobe.d/blacklist.conf'` 55 | 56 | 57 | #### 2. GPU cards 58 | 59 | ##### i. Finding NVIDIA devices ids 60 | `# sudo lspci -nn -d 10de:` 61 | 62 | ``` 63 | 02:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 64 | 02:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 65 | 03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 66 | 03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 67 | 81:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 68 | 81:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 69 | 82:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 70 | 82:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 71 | ``` 72 | In this example, GTX 1080 id is `10de:1b80` and its Audio Controller is `10de:10f0` 73 | 74 | `# sudo sh -c 'echo "options vfio-pci ids=10de:1b80,10de:10f0 disable_vga=1" > /etc/modprobe.d/vfio-pci.conf'` 75 | 76 | 77 | #### 3. Finish preparation 78 | 79 | ``` 80 | # sudo update-grub 81 | # sudo update-initramfs -u 82 | # sudo reboot 83 | ``` 84 | 85 | 86 | #### 4. Verification 87 | 88 | 89 | ##### i. Is vfio enable? 90 | 91 | `# sudo lspci -nnk | grep -i nvidia -A2` 92 | ``` 93 | 02:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 94 | Subsystem: Device [196e:119e] 95 | Kernel driver in use: vfio-pci 96 | Kernel modules: nvidiafb, nouveau 97 | 02:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 98 | Subsystem: Device [196e:119e] 99 | Kernel driver in use: vfio-pci 100 | -- 101 | 03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 102 | Subsystem: NVIDIA Corporation Device [10de:119e] 103 | Kernel driver in use: vfio-pci 104 | Kernel modules: nvidiafb, nouveau 105 | 03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 106 | Subsystem: NVIDIA Corporation Device [10de:119e] 107 | Kernel driver in use: vfio-pci 108 | Kernel modules: snd_hda_intel 109 | -- 110 | 81:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 111 | Subsystem: NVIDIA Corporation Device [10de:119e] 112 | Kernel driver in use: vfio-pci 113 | Kernel modules: nvidiafb, nouveau 114 | 81:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 115 | Subsystem: NVIDIA Corporation Device [10de:119e] 116 | Kernel driver in use: vfio-pci 117 | Kernel modules: snd_hda_intel 118 | -- 119 | 82:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104 [GeForce GTX 1080] [10de:1b80] (rev a1) 120 | Subsystem: NVIDIA Corporation Device [10de:119e] 121 | Kernel driver in use: vfio-pci 122 | Kernel modules: nvidiafb, nouveau 123 | 82:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1) 124 | Subsystem: NVIDIA Corporation Device [10de:119e] 125 | Kernel driver in use: vfio-pci 126 | Kernel modules: snd_hda_intel 127 | ``` 128 | 129 | 130 | #### 5. QEMU 131 | 132 | ``` 133 | sudo apt-get install qemu-system-x86-64 numactl 134 | mkdir /home/vm 135 | cd /home/vm 136 | wget 'https://software-download.microsoft.com/pr/Win10_1607_..._x64.iso?t=...' -O Win10_1607_x64.iso 137 | wget https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso 138 | git clone https://github.com/gmasse/gpu-pci-passthrough.git 139 | ``` 140 | 141 | ##### EFI 142 | ``` 143 | sudo apt-get install git build-essential uuid-dev nasm acpica-tools iasl 144 | mkdir src 145 | cd src 146 | git clone https://github.com/tianocore/edk2.git 147 | cd edk2 148 | make -C BaseTools/Source/C 149 | nice OvmfPkg/build.sh -a X64 -n $(getconf _NPROCESSORS_ONLN) 150 | cp Build/OvmfX64/DEBUG_GCC*/FV/OVMF.fd /home/vm/ 151 | ``` 152 | 153 | 154 | 155 | ## II. First VM boot 156 | 157 | Edit vm.sh according to your settings 158 | 159 | `# vi /home/vm/gpu-pci-passthrough/vm.sh ` 160 | 161 | 162 | Start the VM number 1 163 | 164 | `# sh vm.sh 1` 165 | 166 | `# sudo screen -r vm-1` 167 | ``` 168 | QEMU 2.5.0 monitor - type 'help' for more information 169 | (qemu) change vnc password 170 | Password: ****** 171 | (qemu) 172 | ``` 173 | Press CTRL+A CTRL+D to quit screen 174 | 175 | Start a VNC client (port 5901). On OS X, you can directly type in Safari "vnc://SERVERIP:5901". It will launch the Screen Sharing application. 176 | 177 | (Optional) If the UEFI shell appears at first boot. Just type exit and select continue. 178 | Then press any key to boot from the CD. 179 | ![Image](doc/uefi_shell.png?raw=true)![Image](doc/qemu_boot.png?raw=true) 180 | 181 | During Windows installation, load viostor and NetKVM drivers from virtio-win CD drive. 182 | ![Image](doc/wininst_virtio.png?raw=true) 183 | 184 | After windows restart, you can activate RDP and connect via rdp://SERVERIP:4001 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | Source: 193 | - https://www.evonide.com/non-root-gpu-passthrough-setup/ 194 | - https://www.pugetsystems.com/labs/articles/Multiheaded-NVIDIA-Gaming-using-Ubuntu-14-04-KVM-585/ 195 | - http://vfio.blogspot.fr/2015/05/vfio-gpu-how-to-series-part-3-host.html 196 | - https://wiki.ubuntu.com/UEFI/EDK2 197 | - https://www.microsoft.com/fr-fr/software-download/windows10ISO 198 | - https://phocean.net/tools/french-apple-macbook-keyboard-layout-for-windows 199 | - https://fedoraproject.org/wiki/Windows_Virtio_Drivers 200 | --------------------------------------------------------------------------------