├── README.md ├── passthrough-start.sh ├── qemu_fifo.sh └── win-working.xml /README.md: -------------------------------------------------------------------------------- 1 | # Win10-VFIO 2 | Collection of scripts and tweaks for making a Windows 10 virtual machine run with QEMU/KVM/libvirt with GPU passthrough. 3 | 4 | System details at time of writing: 5 | * Intel i7 8700k @ 4.8GHz 6 | * 32 GB RAM 7 | 8 | Runs fast enough for demanding VR titles :) 9 | 10 | # Additional information 11 | * Q35, PCIe configuration, physical SATA SSD given to the guest via SCSI passthrough (virtio) 12 | * Network "winbr0" created using nmcli on the host 13 | * Kernel parameters: `intel_iommu=on iommu=pt transparent_hugepage=never nmi_watchdog=0 intremap=no_x2apic_optout rcu_nocbs=1-5,7-11 nohz_full=1-5,7-11 clocksource=tsc clock=tsc force_tsc_stable=1 vfio-pci.ids=10de:1b06,10de:10ef,8086:3e92 vfio-pci.disable_vga=1 module_blacklist=i915,nouveau pcie_acs_override=downstream efifb=off video=efifb:off` 14 | * Using a custom-built mainline linux (latest rc, if available) with `CONFIG_PREEMPT_VOLUNTARY=y` (fixes long boot time with UEFI guests), default tickrate, ZFS, WireGuard and some ClearLinux patches 15 | * Latest Arch Linux qemu-headless 16 | * Host is using an AMD RX 550, the guest an NVIDIA 2080 Ti 17 | * USB via passed through USB3 on-board controller, the alternative "3.1" controller on my mainboard is left for the host (switching mouse/keyboard via physical USB switch, although I've had successes with evdev as well) 18 | * Audio works via [Scream](https://github.com/duncanthrax/scream) (using IVSHMEM for best latency) 19 | -------------------------------------------------------------------------------- /passthrough-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ $UID -ne 0 ]]; then 4 | sudo -E bash $0 "$@" 5 | exit $? 6 | fi 7 | 8 | # Necessary for parameter usage in cleanup() 9 | export TMP_PARAMS="$*" 10 | 11 | # Perform cleanup after shutdown 12 | cleanup () { 13 | 14 | # Restore screen config (needed to reset main monitor which turns off for whatever reason) 15 | #ddcutil --display 1 setvcp 60 3 16 | sudo -u pi -E /usr/share/xrandr-config.sh 17 | 18 | # Return CPU power management to default 19 | pstate-frequency --set -p auto -n 50 20 | 21 | if [[ $TMP_PARAMS == *'--keep-hugepages'* ]] 22 | then 23 | echo "Skipping hugepage deletion..." 24 | else 25 | echo "Deleting hugepages..." 26 | echo "0" > /proc/sys/vm/nr_hugepages 27 | fi 28 | 29 | # Kill all background processes 30 | killall scream-pulse || true 31 | killall synergyc || true 32 | 33 | # Restart polybar on main monitor 34 | #pkill polybar 35 | 36 | # Reset cset 37 | echo "Resetting cset groups..." 38 | cset shield --reset 39 | 40 | echo "Removing libvirt cgroup slice..." 41 | sleep 2 42 | rmdir /sys/fs/cgroup/cpuset/machine.slice 43 | 44 | echo "Undoing kernel optimizations..." 45 | echo fff > /sys/devices/virtual/workqueue/cpumask 46 | echo fff > /sys/devices/virtual/workqueue/writeback/cpumask 47 | echo 950000 > /proc/sys/kernel/sched_rt_runtime_us 48 | sysctl vm.stat_interval=1 49 | sysctl -w kernel.watchdog=1 50 | 51 | #killall polybar 52 | sudo -u pi -E sh -c "/home/pi/.config/polybar/bar_launch.sh > /dev/null 2>&1 &disown" 53 | sleep 2 54 | } 55 | 56 | 57 | if [[ $TMP_PARAMS == *'--cleanup'* ]] 58 | then 59 | echo "Cleanup only requested!" 60 | cleanup 61 | exit 0 62 | fi 63 | 64 | 65 | # CPU governor settings (keep CPU frequency up, might not work on older CPUs - use cpupower for those) 66 | pstate-frequency --set -p max 67 | 68 | 69 | # Hugepages config 70 | export HUGEPAGES=11000 71 | 72 | # Note that allocating hugepages after boot has a chance to fail. If continuous memory 73 | # cannot be allocated, a reboot will be required. 74 | export STARTING_HUGEPAGES="$(cat /proc/sys/vm/nr_hugepages)" 75 | if [[ "$STARTING_HUGEPAGES" -lt "$HUGEPAGES" ]]; then 76 | # Drop caches and compact memory to free up continuous memory for huge pages 77 | echo 3 > /proc/sys/vm/drop_caches 78 | echo 1 > /proc/sys/vm/compact_memory 79 | 80 | 81 | echo "Allocating $HUGEPAGES hugepages..." 82 | echo "$HUGEPAGES" > /proc/sys/vm/nr_hugepages 83 | ALLOC_PAGES="$(cat /proc/sys/vm/nr_hugepages)" 84 | 85 | if [[ "$ALLOC_PAGES" -lt "$HUGEPAGES" ]]; then 86 | echo 87 | echo 'Not able to allocate hugepages' 88 | echo "Current max: $ALLOC_PAGES" 89 | echo 90 | 91 | cleanup 92 | exit 1 93 | fi 94 | else 95 | echo "Hugepages already found, let's use those!" 96 | fi 97 | 98 | # Move polybar to secondary screen 99 | killall polybar 100 | 101 | # Monitor config 102 | if [[ $TMP_PARAMS == *'--display'* ]] 103 | then 104 | ddcutil --display 1 setvcp 60 1 105 | xrandr --output HDMI-A-0 --off 106 | fi 107 | 108 | # Start synergy for mouse and keyboard sharing 109 | echo "Starting SynergyC..." 110 | synergyc 10.0.0.251 111 | setxkbmap de # Required because synergy doesn't load default for whatever reason. Not needed on US keyboards. 112 | 113 | # Start Scream-Pulse receiver for audio over the NAT network 114 | echo "Starting sound receiver..." 115 | sudo -u pi -E sh -c "scream-pulse -i winbr0" & 116 | 117 | sudo -u pi -E sh -c "env SYSTRAY_MON=\"DisplayPort-0\" /home/pi/.config/polybar/bar_launch.sh > /dev/null 2>&1 &disown" 118 | 119 | # Taskset (Move all current processes to unused cores) 120 | # Done last, so it can move synergy and scream-pulse as well 121 | cset shield --kthread on --cpu 1-5,7-11 122 | #echo "Setting cset groups correctly..." 123 | echo 0 > /sys/fs/cgroup/cpuset/system/cpuset.cpu_exclusive 124 | echo 0 > /sys/fs/cgroup/cpuset/user/cpuset.cpu_exclusive 125 | 126 | #echo 0 > /sys/fs/cgroup/cpuset/machine.slice/cpuset.cpu_exclusive 127 | #echo 0-11 > /sys/fs/cgroup/cpuset/machine.slice/cpuset.cpus 128 | 129 | echo "Performing minor optimizations prior to launch..." 130 | echo 041 > /sys/devices/virtual/workqueue/cpumask 131 | echo -1 > /proc/sys/kernel/sched_rt_runtime_us 132 | for i in $(pgrep '^rcuo'); do taskset -pc 0,6 $i > /dev/null; done; 133 | for i in /sys/devices/virtual/workqueue/*/cpumask; do echo 041 > $i; done; 134 | sysctl vm.stat_interval=120 135 | sysctl -w kernel.watchdog=0 136 | 137 | ./qemu_fifo.sh & 138 | 139 | # Start VM via virt-manager 140 | echo "VM starting..." 141 | virsh start win 142 | echo 143 | 144 | # Print status and wait for exit 145 | while [[ $(virsh list --all | grep running) ]]; do 146 | echo -en "\e[1A\rVM running - " # Weirdness is for formatting 147 | date 148 | sleep 1 149 | done 150 | 151 | sleep 1 152 | echo "VM has exited" 153 | 154 | cleanup 155 | -------------------------------------------------------------------------------- /qemu_fifo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sleep 45 4 | 5 | if [[ $(sudo virsh list --all | grep running) ]]; then 6 | echo "VM running, performing action" 7 | else 8 | echo "VM no longer running, aborting" 9 | exit 1 10 | fi 11 | 12 | chrt -a -f -p 99 $(pidof qemu-system-x86_64) 13 | echo "Set QEMU execution policy!" 14 | chrt -p $(pidof qemu-system-x86_64) 15 | 16 | echo "Setting IRQ affinities..." 17 | bash -c "for i in $(sed -n -e 's/ \([0-9]\+\):.*/\1/p' /proc/interrupts); do echo '0,6' > /proc/irq/$i/smp_affinity_list; done;" > /dev/null 2>&1 18 | 19 | echo 20 | echo 21 | -------------------------------------------------------------------------------- /win-working.xml: -------------------------------------------------------------------------------- 1 | 2 | win 3 | 8da2cec8-b698-40dc-b38c-efa2b6e78cf3 4 | Windows 5 | 22000000 6 | 22000000 7 | 8 | 9 | 10 | 11 | 12 | 10 13 | 14 | -1 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | hvm 29 | /usr/share/ovmf/x64/OVMF_CODE.fd 30 | /var/lib/libvirt/qemu/nvram/win_VARS.fd 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | destroy 67 | restart 68 | destroy 69 | 70 | 71 | 72 | 73 | 74 | /usr/bin/qemu-system-x86_64 75 | 76 | 77 | 78 | 79 |
80 | 81 | 82 | 83 |
84 | 85 | 86 | 87 | 88 | 89 |
90 | 91 | 92 | 93 | 94 |
95 | 96 | 97 | 98 | 99 |
100 | 101 | 102 | 103 | 104 |
105 | 106 | 107 | 108 | 109 |
110 | 111 | 112 | 113 | 114 |
115 | 116 | 117 | 118 | 119 |
120 | 121 | 122 | 123 | 124 |
125 | 126 | 127 | 128 | 129 |
130 | 131 | 132 | 133 | 134 |
135 | 136 | 137 | 138 | 139 |
140 | 141 | 142 | 143 | 144 |
145 | 146 | 147 | 148 | 149 |
150 | 151 | 152 | 153 | 154 |
155 | 156 | 157 | 158 | 159 |
160 | 161 | 162 | 163 | 164 |
165 | 166 | 167 | 168 |
169 | 170 | 171 |
172 | 173 | 174 |
175 | 176 | 177 | 178 | 179 | 180 |
181 | 182 | 183 | 184 | 185 | 186 |
187 | 188 |
189 | 190 | 191 | 192 |
193 | 194 |
195 | 196 | 197 | 198 |
199 | 200 |
201 | 202 | 203 | 204 | 205 | 206 | --------------------------------------------------------------------------------