├── asl ├── .gitignore ├── Makefile └── ssdt1.asl ├── .gitmodules ├── split-rom.js ├── nvidia-firmware-hack.patch ├── NOTES.md ├── README.md ├── gentoo-vm.xml └── glxinfo-nvidia-guest /asl/.gitignore: -------------------------------------------------------------------------------- 1 | *.aml 2 | -------------------------------------------------------------------------------- /asl/Makefile: -------------------------------------------------------------------------------- 1 | default: ssdt1.aml 2 | 3 | clean: 4 | rm -f *.aml 5 | 6 | %.aml: %.asl 7 | iasl $< 8 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "qemu"] 2 | path = qemu 3 | url = git@github.com:jscinoz/qemu.git 4 | branch = acpi-rom-method 5 | -------------------------------------------------------------------------------- /split-rom.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const fs = require("fs"); 4 | 5 | const buf = fs.readFileSync("960m.bin"); 6 | 7 | const slices = []; 8 | 9 | slices[0] = buf.slice(0, 2**15); 10 | slices[1] = buf.slice(2**15, 2**16); 11 | slices[2] = buf.slice(2**16, 2**15 + 2**16); 12 | slices[3] = buf.slice(2**15 + 2**16, buf.length); 13 | 14 | const ITEMS_PER_LINE = 10; 15 | 16 | slices.forEach((slice, sliceIdx) => { 17 | const lines = slice.reduce((acc, x, i) => { 18 | const lineIdx = Math.floor(i / ITEMS_PER_LINE); 19 | 20 | let line = acc[lineIdx]; 21 | 22 | if (!line) { 23 | line = []; 24 | 25 | acc[lineIdx] = line; 26 | } 27 | 28 | line.push(`0x${x.toString(16).padStart(2, "0")}`); 29 | 30 | return acc; 31 | }, []); 32 | 33 | const str = lines.map(line => line.join(", ")).join(",\n"); 34 | 35 | fs.writeFileSync(`slice-${sliceIdx}`, str); 36 | }); 37 | -------------------------------------------------------------------------------- /nvidia-firmware-hack.patch: -------------------------------------------------------------------------------- 1 | --- a/kernel/nvidia/nv-acpi.c 2017-12-22 11:13:03.551686050 +1100 2 | +++ b/kernel/nvidia/nv-acpi.c 2017-12-22 12:45:44.799378440 +1100 3 | @@ -15,6 +15,14 @@ 4 | #include "nv-linux.h" 5 | #include "nv-reg.h" 6 | 7 | +// Generated with xxdi 8 | +const unsigned char IMG_960m_bin[] = { 9 | + // VBIOS data - redacted here as I'm probably not allowed to redistribute it 10 | +}; 11 | + 12 | +// Generated with xxdi, length of array above 13 | +const unsigned int IMG_960m_bin_len = 103936; 14 | + 15 | #if defined(NV_LINUX_ACPI_EVENTS_SUPPORTED) 16 | static NV_STATUS nv_acpi_extract_integer (const union acpi_object *, void *, NvU32, NvU32 *); 17 | static NV_STATUS nv_acpi_extract_buffer (const union acpi_object *, void *, NvU32, NvU32 *); 18 | @@ -1305,14 +1313,17 @@ 19 | NvU32 *pOutData 20 | ) 21 | { 22 | + /* 23 | acpi_status status; 24 | struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; 25 | union acpi_object *rom; 26 | union acpi_object rom_arg[2]; 27 | struct acpi_object_list input = { 2, rom_arg }; 28 | acpi_handle dev_handle = NULL; 29 | + */ 30 | uint32_t offset, length; 31 | 32 | + /* 33 | if (!nv_acpi_get_device_handle(nv, &dev_handle)) 34 | return NV_ERR_NOT_SUPPORTED; 35 | 36 | @@ -1321,16 +1332,28 @@ 37 | 38 | if (!NV_MAY_SLEEP()) 39 | { 40 | -#if defined(DEBUG) 41 | +#if defined(DEBUG 42 | nv_printf(NV_DBG_ERRORS, 43 | "NVRM: %s: invalid context!\n", __FUNCTION__); 44 | #endif 45 | return NV_ERR_NOT_SUPPORTED; 46 | } 47 | + */ 48 | 49 | offset = pInData[0]; 50 | length = pInData[1]; 51 | 52 | + // XXX: It is important to note, that I have absolutely no idea what I'm 53 | + // doing here. 54 | + 55 | + // XXX: Assuming our caller only calls us until it has the full image? 56 | + // Probably should check that offset + len is in bounds 57 | + memcpy(pOutData, IMG_960m_bin + offset, length); 58 | + 59 | + // return NV_ERR_NOT_SUPPORTED; 60 | + return NV_OK; 61 | + 62 | + /* 63 | rom_arg[0].type = ACPI_TYPE_INTEGER; 64 | rom_arg[0].integer.value = offset; 65 | rom_arg[1].type = ACPI_TYPE_INTEGER; 66 | @@ -1363,6 +1386,7 @@ 67 | 68 | kfree(output.pointer); 69 | return NV_OK; 70 | + */ 71 | } 72 | 73 | /* 74 | -------------------------------------------------------------------------------- /NOTES.md: -------------------------------------------------------------------------------- 1 | These are my raw notes taken as I've been experimenting with this, information 2 | in this file could be outdated / wrong - see [README.md](README.md) for 3 | information that should be current (and hopefully accurate) 4 | 5 | * For nouveau to find firmware provided by option, the kernel 6 | needs to be directly booted via the platform (i.e. EFISTUB), or via EFI 7 | handover protocol. Fedora and others patch grub to add a 'linuxefi' command 8 | that does this, but this patch does not appear mainline yet 9 | * Nouveau seems to create a render node, whether anything actually works is TBD 10 | 11 | 12 | # Current state of ACPI shenanigans 13 | * ASL works under seabios and nvidia driver loads vbios from ACPI successfully 14 | * But, nvidia drivers has errors once X is started that didn't happen under 15 | UEFI (with vbios loading faked from inline blob) 16 | * Windows BSODs with: 17 | ``` 18 | ACPI_BIOS_ERROR: 19 | 20 | Arg1: 0000000000001000, ACPI_BIOS_USING_OS_MEMORY 21 | ACPI had a fatal error when processing a memory operation region. 22 | The memory operation region tried to map memory that has been 23 | allocated for OS usage. 24 | ``` 25 | Seems it is not happy with me reading the option from from bios-assigned 26 | XROMBAR from within ASL :( 27 | * Looks like windows may be re-initialising the XROMBAR - the address my ASL 28 | tries to read when it causes the crash looks nonsense (0xFFFFF800) 29 | * Have not found a way to get the vbios from ASL under OVMF. It seems that once 30 | OVMF itself has read out option ROMs, they are no longer accessible. I think 31 | we might need to patch OVMF to retain these maybe generate another SSDT that 32 | contains OperationRegion for each PCI device with an option rom 33 | * If we define resources in ACPI for the device (such as a memory region for 34 | the oprom), will OVMF use these instead? (See: 35 | PciHostBridgeResourceAllocator, specifically usage of GetResourceBase and 36 | ProcessOptionRom), ACPI `_PRS`? 37 | * No, I don't think so, after further analysis of OVMF code 38 | * Qemu investigation 39 | * romfile is provided and rombar=0, qemu will put the rom in fw_cfg :D 40 | * Need to see if we can read this directly from ASL, or failing that, patch 41 | QEMU to generate the `_ROM` method for us 42 | 43 | ## To try next 44 | * Try reading under efi shell with mm again, but this time, set the memory space 45 | bit too! 46 | * Figure out GVT issues - only works intermittently 47 | * BIOS 48 | * Gvt with qxl as primary 49 | * GVT with qxl as secondary 50 | * gvt + nvidia, no QXL 51 | * Windows guest 52 | * GVT primary, nvidia secondary (matches real hardware) 53 | * GVT + QXL primary 54 | * GVT primary + QXL 55 | * QXL + Nvidia 56 | * QXL + GVT + Nvidia 57 | * EFI 58 | * OVMF hack to not clear XROMBAR - see if we can then get to it from ASL 59 | 60 | TODO: 61 | * Try in BIOS VM 62 | * Nouveau can load from pci rom fine, try ACPI 63 | * Seem to be able to get it from ASL too :O 64 | * Try reading ROM from qemu fw_cfg - might be exposed to ACPI 65 | * Patch OVMF to to have ACPI \_ROM method 66 | * Tried doing this through ASL, but couldn't find a way to get the ROM data 67 | from PCI XROMBAR 68 | * Just have it expose ROMs, still do the rest in ASL 69 | * Try binary nvidia + QXL (no gvt) 70 | * Upload various xorg.confs 71 | * Try with git kernel 72 | * Try with gvt bleeding edge kernel 73 | * Try using qxl + gvt (without nvidia) to debug qxl reverse prime 74 | * Experiment with qemu emulated intel_iommu to see if that resolves the swiotlb 75 | issue 76 | * See if we can debug PRIME issues further 77 | * So far, setting nvidia as output source for qxl results in BadValue from 78 | xrandr :( 79 | * WORKS IF we make gvt device primary gpu! 80 | * Need to see if we can do reverse prime then mirroring with qxl 81 | * Running into odd memory issues with qxl reverse prime: 82 | qxl 0000:00:03.0: swiotlb buffer is full (sz: 299008 bytes) 83 | qxl 0000:00:03.0: DMA: Out of SW-IOMMU space for 299008 bytes 84 | * Test with nvidia as primary VGA 85 | * Ideally without GVT 86 | * Can we have reverse prime with non-primary QXL for output? 87 | * Seems xorg requires a GPU with actual outputs as the primary one. 88 | * Could try with xf86-video-dummy, but i doubt it has prime? 89 | * custom ACPI table with GPU firmware in it 90 | * If we still have issues, try adding optimus strings from host ssdt2 91 | * Try with virtio-gpu too 92 | * More testing with GVT 93 | * Can only set nvidia offload to primary device, assinging to non primary will 94 | crash xorg with the following assertion failure 95 | xorg-server-1.19.5/dix/dispatch.c:4035: AttachOutputGPU: Assertion `new->current_master == pScreen` failed. 96 | * Test with just QXL and gvt for shiggles 97 | * Doesn't work, QXL has only "Sink Output", nothing else 98 | * Does bumblebee even work on the host? 99 | 100 | 101 | DONE 102 | * Figure out why we need to set up vfio-pci manually and libvirt fails 103 | * Seems it's setting up VFIO groups wrong? 104 | * Test using nouveau render node directly 105 | * Tentatively done, but need to actually validate our test methodology 106 | * Instead of custom DSDT, for linux nvidia driver, hack nvidia-acpi.c to just 107 | load hard-coded vbios 108 | * Works 109 | * Try with bumblebee 110 | * Didn't work with QXL without patching out check for intel GPU 111 | * Fails to start second X display on nvidia card due to lack of ouputs 112 | * Test if bumblebee even works on the host 113 | * Try with wayland 114 | * So far, wayland just flashes for a bit then freezes 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | All that follows is relating to muxless/non-MXM Optimus cards, i.e. those that 2 | have no display outputs and show as `3D Controller` in `lspci` output. If you 3 | have a MXM / output-providing card (shows as `VGA Controller` in `lspci`), then 4 | this is not for you. For such devices, see 5 | [@Misairu-G's guide](https://gist.github.com/Misairu-G/616f7b2756c488148b7309addc940b28) 6 | 7 | # What works 8 | * Compute workloads (via render nodes) on GVT device 9 | * You can find a sample program that dispatches a compute shader via render 10 | nodes [here](https://github.com/elima/gpu-playground/tree/master/render-nodes-minimal) 11 | * Compute workloads (via render nodes) on nvidia card (nouveau) 12 | * Render workloads on the nvidia card with the nouveau driver, with a GVT device 13 | as the guest's primary GPU 14 | * I have only tested this with q35. i440fx is untested 15 | * Subsystem vendor & device Id's must be set with x-pci-sub-vendor-id and 16 | x-pci-sub-device-id on the qemu vfio-pci device for the nvidia card 17 | * Nvidia VBIOS must be provided to the guest via libvirt's `` / 18 | qemu's `romfile` option on the device 19 | * VBIOS can be obtained under linux by extracting a BIOS update using [coderobe/VBiosFinder](https://github.com/coderobe/VBiosFinder) 20 | * Alternatively, when using Windows the VBIOS can be obtained by dumping the system firmware with 21 | [Universal Bios Backup Toolkit](https://forums.mydigitallife.net/threads/universal-bios-backup-toolkit.9856/) 22 | and using MMTool to extract the VBIOS fromthe "Option ROMs" section 23 | * This needs to be done under bare-metal Windows. I use a Windows-To-Go 24 | install of Server 2016 on a USB (UAS) SSD for this, to leave my 25 | machine's internal storage alone. 26 | * I've only tested this with OVMF/UEFI. It may or may not work with seabios 27 | VMs 28 | * My dumped VBIOS did **not** have EFI support, but I don't think that is 29 | necessary when the card has no outputs. rom-parser output for reference: 30 | ``` 31 | Valid ROM signature found @0h, PCIR offset 190h 32 | PCIR: type 0, vendor: 10de, device: 139b, class: 030200 33 | PCIR: revision 3, vendor revision: 1 34 | Last image 35 | ``` 36 | * Guest kernel must booted via OVMF directly, or via a bootloader that 37 | supports the EFI handover protocol 38 | * Failure to do this will result in nouveau failing to load vbios via the 39 | `FIRMWARE` interface 40 | * Mainline GRUB **does not** have EFI handover support. Fedora (and maybe 41 | other distros) have their own patches adding support for this in the form 42 | of the `linuxefi` command 43 | * No local display output at present; remote display must be used. 44 | * Render workloads on the nvidia card with the nvidia driver, with the GVT 45 | device as secondary, linked with PRIME 46 | * Requires a patch to the nvidia driver to fake loading VBIOS from ACPI. See 47 | below 48 | * Render workloads on the GVT device, with the GVT device as the guest's primary 49 | GPU 50 | * Only tested on q35 + OVMF 51 | * No VBIOS needed, everything should Just Work™ 52 | * No local display output at present; remote display must be used. 53 | 54 | # What works, with patching 55 | * Binary nvidia driver (Linux) 56 | * For Optimus cards, it will only attempt to load VBIOS via ACPI \_ROM method, 57 | which won't exist in the guest. 58 | * We can probably patch `nv_acpi_rom_method` in `kernel/nvidia/nv-acpi.c` to 59 | simply return our VBIOS from a hard-coded buffer, just for testing 60 | * **[UPDATE]** I tested this and it worked, the binary driver booted and the 61 | NVIDIA card can be set up with PRIME. [glxinfo](glxinfo-nvidia-guest), 62 | [patch](nvidia-firmware-hack.patch). Note you'll need to dump your own VBIOS and 63 | inline it in the patch. 64 | * Long term we need to build a custom ACPI table (provided to qemu with the 65 | `-acpitable` option) that has `_ROM` implemented at the correct path. The 66 | `_ROM` implementationm would need to seek over a hard-coded buffer stored 67 | elsewhere and return the VBIOS in 4kb chunks as expected by nvidia driver 68 | 69 | # What doesn't work (yet) 70 | * Windows guest 71 | * Will need custom ACPI table to get VBIOS, as detailed above 72 | * Reverse PRIME to mirror GVT display to QXL device to use Qemu's built in spice 73 | server 74 | * The `modesetting` DDX must be used with QXL, as the `qxl` DDX lacks PRIME 75 | support 76 | * The QXL-backed modesetting instance can be set up as a PRIME output provider 77 | successfully, and modes will show as available on the newly created 78 | Virtual-X-Y output (where X and Y will vary depending on your xorg server's 79 | layout) 80 | * Attempting to set any mode on the Virtual-X-Y output will fail with the 81 | following errors from the kernel: 82 | ``` 83 | qxl 0000:00:03.0: swiotlb buffer is full (sz: 299008 bytes) 84 | qxl 0000:00:03.0: DMA: Out of SW-IOMMU space for 299008 bytes 85 | ``` 86 | * Next step may be to try with Qemu's emulated `intel-iommu` device and see 87 | if this helps 88 | * QXL as primary xorg GPU with GVT/nvidia as secondary 89 | * QXL-backed modesetting provides only the `Sink Output` capability. `Sink 90 | Offload` is required for render offloading to a device with `Source Offload` 91 | * Maybe QXL could be improved to have the `Sink Offload` capability? 92 | * Attempting to set up nouveau as the `Source Offload` for the GVT card (which 93 | has `Sink Offload`) will crash Xorg with the following assertion failure: 94 | `xorg-server-1.19.5/dix/dispatch.c:4035: AttachOutputGPU: Assertion 'new->current_master == pScreen' failed.` 95 | This suggests that PRIME render offload requires the destination device (the 96 | one being used as a `Sink Offload` to be the primary Xorg GPU 97 | * Bumblebee 98 | * Attempting to use bumblebee without a GVT device will fail as `bumblebeed` 99 | will bail out at startup upon not finding an Intel card 100 | * Patching out this check will allow `bumblebeed` to start. 101 | * Once `bumblebeed` has started, it is unable to successfully start the 102 | secondary X server, as the nvidia card has no outputs and Xorg bails out due 103 | to no outputs existing. 104 | * Nouveau does **not** support/have the `AllowEmptyInitialConfiguration` 105 | option that the proprietary nvidia driver has. 106 | 107 | # What will (probably) never work 108 | * Using the nvidia card in the guest as the sole GPU 109 | * As non-MXM Optimus cards lack any outputs, xorg will not start as there are 110 | no outputs available 111 | * xf86-video-dummy cannot be used as the primary GPU as it lacks PRIME support 112 | 113 | # What's untested 114 | * Wayland 115 | 116 | # What's needed 117 | * A way to obtain a dump of nvidia VBIOS under Linux 118 | * Try [coderobe/VBiosFinder](https://github.com/coderobe/VBiosFinder) 119 | * Custom ACPI tables embedding nvidia VBIOS and exposing it via the `_ROM` 120 | method at the appropriate path 121 | -------------------------------------------------------------------------------- /gentoo-vm.xml: -------------------------------------------------------------------------------- 1 | 2 | gentoo-vm 3 | 8eb70501-6c70-475d-a948-d064bcbe22a1 4 | 8388608 5 | 8388608 6 | 8 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | /machine 19 | 20 | 21 | hvm 22 | /home/jack/src/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_CODE.fd 23 | /var/lib/libvirt/qemu/nvram/gentoo-vm_VARS.fd 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | destroy 50 | restart 51 | destroy 52 | 53 | 54 | 55 | 56 | 57 | /home/jack/optimus-vfio/qemu/x86_64-softmmu/qemu-system-x86_64 58 | 59 | 60 | 61 | 62 | 63 | 64 |
65 | 66 | 67 | 68 | 69 | 70 | 71 |
72 | 73 | 74 | 75 | 76 | 77 |
78 | 79 | 80 |
81 | 82 | 83 |
84 | 85 | 86 | 87 | 88 | 89 |
90 | 91 | 92 | 93 | 94 |
95 | 96 | 97 | 98 | 99 |
100 | 101 | 102 | 103 | 104 |
105 | 106 | 107 | 108 | 109 |
110 | 111 | 112 | 113 | 114 |
115 | 116 | 117 | 118 | 119 |
120 | 121 | 122 | 123 | 124 |
125 | 126 | 127 | 128 | 129 |
130 | 131 | 132 |
133 | 134 | 135 | 136 | 137 | 138 |
139 | 140 | 141 | 142 |
143 | 144 | 145 | 146 |
147 | 148 | 149 | 150 | 151 |
152 | 153 | 154 |
155 | 156 | 157 |
158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 |