├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __asm.h ├── aehd.rc ├── aehd ├── aehd.sln └── aehd.vcxproj ├── aehd_main.c ├── aehd_main.h ├── aehd_types.h ├── aehd_ver.h ├── arch └── x86 │ ├── include │ ├── asm │ │ ├── apicdef.h │ │ ├── cpufeatures.h │ │ ├── fpu │ │ │ └── types.h │ │ ├── kvm_emulate.h │ │ ├── kvm_host.h │ │ ├── kvm_page_track.h │ │ ├── msidef.h │ │ ├── msr-index.h │ │ ├── svm.h │ │ └── vmx.h │ └── uapi │ │ └── asm │ │ ├── debugreg.h │ │ ├── kvm.h │ │ ├── processor-flags.h │ │ ├── svm.h │ │ └── vmx.h │ └── kvm │ ├── cpuid.c │ ├── cpuid.h │ ├── emulate.c │ ├── i8259.c │ ├── ioapic.c │ ├── ioapic.h │ ├── irq.c │ ├── irq.h │ ├── irq_comm.c │ ├── kvm_cache_regs.h │ ├── lapic.c │ ├── lapic.h │ ├── mmu.c │ ├── mmu.h │ ├── mmu_audit.c │ ├── mtrr.c │ ├── page_track.c │ ├── paging_tmpl.h │ ├── pmu.c │ ├── pmu.h │ ├── pmu_amd.c │ ├── pmu_intel.c │ ├── svm.c │ ├── svm_def.h │ ├── tss.h │ ├── vmx.c │ ├── vmx_def.h │ ├── x86.c │ └── x86.h ├── asmgen ├── asmgen.c ├── asmgen.vcxproj └── asmgen.vcxproj.user ├── assembly └── x64 │ └── assembly.asm ├── include ├── kvm │ └── iodev.h ├── linux │ ├── kvm_host.h │ ├── kvm_types.h │ └── list.h └── uapi │ └── linux │ └── kvm.h ├── ntkrutils.c ├── ntkrutils.h ├── package └── make_package.sh ├── sign └── aehd │ ├── .gitignore │ ├── aehd.ddf │ └── aehd.inf └── virt └── kvm ├── irqchip.c └── kvm_main.c /.gitignore: -------------------------------------------------------------------------------- 1 | __asm.inc 2 | aehd/.vs/* 3 | aehd/DriverTest/* 4 | **/x64/* 5 | cscope* 6 | Release 7 | aehd/aehd.vcxproj.user 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Android Emulator hypervisor driver 2 | 3 | Android Emulator hypervisor driver is a hypervisor to accelerate 4 | [Android Emulator][android-studio]. It is made by porting KVM to Windows 5 | (Windows 8.1 or later, 64bit). 6 | 7 | Android Emulator hypervisor driver runs as a Windows driver. User space 8 | support for Android Emulator hypervisor driver is available from Android 9 | Emulator. 10 | 11 | ## Notice of the repository name change 12 | Android Emulator hypervisor driver for AMD Processors has been renamed to 13 | Android Emulator hypervisor driver to reflect the fact that it supports both 14 | Intel and AMD Processors. In fact, it supports Intel from version 1.0. The 15 | old name was chosen because Intel users were expected to continue using Intel 16 | HAXM. 17 | 18 | ## Download and Install 19 | Android Emulator hypervisor driver is released through [android-studio]. 20 | However, only Android Studio with version 4.0 canary 5 or above can both 21 | download and install/update the driver. Otherwise, the Android 22 | Studio will only download the driver package without performing installation. 23 | In the latter case, users are required to install the driver manually. 24 | 25 | 26 | Prerequisite: 27 | 1. CPU has virtualization extension and BIOS has NOT disabled the extension. 28 | 2. Hyper-V must be disabled. Refer to [this 29 | page](https://github.com/google/android-emulator-hypervisor-driver-for-amd-processors/wiki/Is-Hyper-V-really-disabled%3F) 30 | for more information. 31 | 32 | Install Instruction: 33 | 34 | Use an administrator command console to execute "silent_install.bat" inside 35 | the driver package. Make sure you see the desired output from the installer: 36 | STATE: 4 RUNNING 37 | 38 | ## For Windows 7 users 39 | According to Microsoft, SHA1 driver signing is deprecated (Read more 40 | [here](https://docs.microsoft.com/en-us/windows-hardware/drivers/install/deprecation-of-software-publisher-certificates-and-commercial-release-certificates) 41 | ). Version 1.8 to 2.1 cannot be loaded on Windows 7 by default. Please 42 | use version 1.7 instead. Users may disable driver signature enforcement in 43 | order to use version 1.8 to 2.1. 44 | 45 | Starting from version 2.2, Windows 7 is not supported any more. 46 | 47 | ## Contributing 48 | If you would like to contribute a patch to the code base, please read 49 | [these guidelines](CONTRIBUTING.md). 50 | 51 | ## Reporting an Issue 52 | You are welcome to file an issue at [Issuetracker]. Please remember to supply 53 | your OS information, CPU model in addition to details on the issue. 54 | 55 | ## Notes 56 | A patched QEMU can be found here at [github]. However, there is no support for 57 | it. Use at your own risk. 58 | 59 | [android-studio]: https://developer.android.com/studio/index.html 60 | [github]: https://github.com/qemu-gvm/qemu-gvm 61 | [Issuetracker]: https://issuetracker.google.com/issues?q=componentid:192727 62 | -------------------------------------------------------------------------------- /__asm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * version 2 as published by the Free Software Foundation. 7 | 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | */ 13 | 14 | #pragma once 15 | // assembly function declaration 16 | #include 17 | 18 | extern u16 aehd_read_ldt(void); 19 | extern void aehd_load_ldt(u16 sel); 20 | extern void load_TR_desc(void); 21 | extern u16 aehd_read_tr(void); 22 | extern void aehd_load_tr(u16 sel); 23 | 24 | #pragma warning(disable : 4210) 25 | #define savesegment(seg, value) \ 26 | extern u16 save_##seg ##_segment(void); \ 27 | value = save_##seg ##_segment() 28 | 29 | #define loadsegment(seg, value) \ 30 | extern u16 load_##seg ##_segment(u16 sel); \ 31 | load_##seg ##_segment(value) 32 | 33 | extern void load_gs_index(u16 value); 34 | extern void __asm_vmx_vcpu_run(void *vmx); 35 | extern void __asm_vmx_handle_external_intr(size_t entry); 36 | 37 | extern void __asm_svm_vcpu_run(void *svm); 38 | 39 | extern void __int2(void); 40 | extern void __int12(void); 41 | 42 | //debug register 43 | extern u64 __read_dr0(); 44 | extern u64 __read_dr1(); 45 | extern u64 __read_dr2(); 46 | extern u64 __read_dr3(); 47 | extern u64 __read_dr6(); 48 | extern u64 __read_dr7(); 49 | extern void __write_dr0(u64 val); 50 | extern void __write_dr1(u64 val); 51 | extern void __write_dr2(u64 val); 52 | extern void __write_dr3(u64 val); 53 | extern void __write_dr6(u64 val); 54 | extern void __write_dr7(u64 val); 55 | 56 | #define dr_read_case(regno) \ 57 | case regno: \ 58 | val = __read_dr##regno(); \ 59 | break 60 | 61 | static __forceinline u64 __get_debugreg(int regno) 62 | { 63 | u64 val = 0; 64 | 65 | switch (regno) { 66 | dr_read_case(0); 67 | dr_read_case(1); 68 | dr_read_case(2); 69 | dr_read_case(3); 70 | dr_read_case(6); 71 | dr_read_case(7); 72 | default: 73 | BUG(); 74 | } 75 | return val; 76 | } 77 | #define get_debugreg(a, b) a = __get_debugreg(b) 78 | 79 | #define dr_write_case(regno) \ 80 | case regno: \ 81 | __write_dr##regno(val); \ 82 | break 83 | 84 | static __forceinline void set_debugreg(u64 val, int regno) 85 | { 86 | switch (regno) { 87 | dr_write_case(0); 88 | dr_write_case(1); 89 | dr_write_case(2); 90 | dr_write_case(3); 91 | dr_write_case(6); 92 | dr_write_case(7); 93 | default: 94 | BUG(); 95 | } 96 | } 97 | 98 | //mmx 99 | extern void __asm_save_mm0(u64 *data); 100 | extern void __asm_save_mm1(u64 *data); 101 | extern void __asm_save_mm2(u64 *data); 102 | extern void __asm_save_mm3(u64 *data); 103 | extern void __asm_save_mm4(u64 *data); 104 | extern void __asm_save_mm5(u64 *data); 105 | extern void __asm_save_mm6(u64 *data); 106 | extern void __asm_save_mm7(u64 *data); 107 | extern void __asm_store_mm0(u64 *data); 108 | extern void __asm_store_mm1(u64 *data); 109 | extern void __asm_store_mm2(u64 *data); 110 | extern void __asm_store_mm3(u64 *data); 111 | extern void __asm_store_mm4(u64 *data); 112 | extern void __asm_store_mm5(u64 *data); 113 | extern void __asm_store_mm6(u64 *data); 114 | extern void __asm_store_mm7(u64 *data); 115 | 116 | //fpu 117 | extern void __fninit(void); 118 | extern void __fnstcw(u16 *fcw); 119 | extern void __fnstsw(u16 *fcw); 120 | extern void __fwait(void); 121 | extern void __clts(void); 122 | 123 | //bswap 124 | extern void __bswap64(u64 *val); 125 | extern void __bswap32(u32 *val); 126 | 127 | #define read_cr0 __readcr0 128 | #define read_cr3 __readcr3 129 | #define read_cr4 __readcr4 130 | #define write_cr4 __writecr4 131 | 132 | #define stts() __writecr0(__readcr0() | X86_CR0_TS) 133 | 134 | #define load_gdt(pdesc) _lgdt((void *)pdesc) 135 | #define load_idt(pdesc) __lidt((void *)pdesc) 136 | 137 | static __forceinline void cr4_set_bits(size_t mask) 138 | { 139 | size_t cr4 = __readcr4(); 140 | 141 | if ((cr4 | mask) != cr4) 142 | { 143 | cr4 |= mask; 144 | __writecr4(cr4); 145 | } 146 | } 147 | 148 | static __forceinline void cr4_clear_bits(size_t mask) 149 | { 150 | size_t cr4 = __readcr4(); 151 | 152 | if ((cr4 & ~mask) != cr4) 153 | { 154 | cr4 &= ~mask; 155 | __writecr4(cr4); 156 | } 157 | } 158 | 159 | static __forceinline void native_store_gdt(void *gdt) 160 | { 161 | _sgdt(gdt); 162 | } 163 | 164 | static __forceinline void native_store_idt(void *idt) 165 | { 166 | __sidt(idt); 167 | } 168 | 169 | extern void __asm_invvpid(int ext, void *op); 170 | extern void __asm_invept(int ext, void *op); 171 | 172 | -------------------------------------------------------------------------------- /aehd.rc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * version 2 as published by the Free Software Foundation. 7 | 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | */ 13 | 14 | #include 15 | #include 16 | 17 | #define VER_DEBUG 2 18 | #define VER_PRERELEASE 0 19 | #define VER_FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 20 | #define VER_FILEOS VOS_NT_WINDOWS32 21 | #define VER_FILEFLAGS (VER_PRERELEASE|VER_DEBUG) 22 | 23 | #define VER_FILETYPE VFT_DRV 24 | #define VER_FILESUBTYPE VFT2_DRV_SYSTEM 25 | 26 | #define VER_COMPANYNAME_STR "Google LLC" 27 | #define VER_PRODUCTNAME_STR "Android Emulator hypervisor driver" 28 | #define VER_LEGALCOPYRIGHT_YEARS "2019" 29 | #define VER_LEGALCOPYRIGHT_STR "Copyright (c) " VER_LEGALCOPYRIGHT_YEARS " " VER_COMPANYNAME_STR 30 | #define VER_LEGALTRADEMARKS_STR VER_LEGALCOPYRIGHT_STR 31 | 32 | #define VER_PRODUCTVERSION AEHD_RC_VERSION 33 | #define VER_PRODUCTVERSION_STR AEHD_RC_VERSION_STR 34 | #define VER_PRODUCTVERSION_W (0x0200) 35 | #define VER_PRODUCTVERSION_DW (0x0200) 36 | #define VER_FILEDESCRIPTION_STR "Android Emulator hypervisor driver" 37 | #define VER_INTERNALNAME_STR "Android Emulator hypervisor driver" 38 | #define VER_ORIGINALFILENAME_STR "aehd.sys" 39 | 40 | #include "common.ver" 41 | 42 | -------------------------------------------------------------------------------- /aehd/aehd.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26228.57 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "aehd", "aehd.vcxproj", "{9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}" 7 | ProjectSection(ProjectDependencies) = postProject 8 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE} = {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE} 9 | EndProjectSection 10 | EndProject 11 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "asmgen", "..\asmgen\asmgen.vcxproj", "{07877F58-4EE6-4C6E-A6AA-AF42B477A5BE}" 12 | EndProject 13 | Global 14 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 15 | Debug|x64 = Debug|x64 16 | Release|x64 = Release|x64 17 | EndGlobalSection 18 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 19 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Debug|x64.ActiveCfg = Debug|x64 20 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Debug|x64.Build.0 = Debug|x64 21 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Debug|x64.Deploy.0 = Debug|x64 22 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Release|x64.ActiveCfg = Release|x64 23 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Release|x64.Build.0 = Release|x64 24 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E}.Release|x64.Deploy.0 = Release|x64 25 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE}.Debug|x64.ActiveCfg = Debug|x64 26 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE}.Debug|x64.Build.0 = Debug|x64 27 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE}.Release|x64.ActiveCfg = Release|x64 28 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE}.Release|x64.Build.0 = Release|x64 29 | EndGlobalSection 30 | GlobalSection(SolutionProperties) = preSolution 31 | HideSolutionNode = FALSE 32 | EndGlobalSection 33 | GlobalSection(ExtensibilityGlobals) = postSolution 34 | SolutionGuid = {47E32EEA-C78F-41B3-9BCA-D6354BE920E8} 35 | EndGlobalSection 36 | EndGlobal 37 | -------------------------------------------------------------------------------- /aehd/aehd.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | x64 7 | 8 | 9 | Release 10 | x64 11 | 12 | 13 | 14 | {9CDEE243-5FEC-44CD-9C26-A6B8AE76245E} 15 | {dd38f7fc-d7bd-488b-9242-7d8754cde80d} 16 | v4.6.1 17 | 12.0 18 | Release 19 | x64 20 | aehd 21 | $(LatestTargetPlatformVersion) 22 | 23 | 24 | 25 | WindowsV6.3 26 | true 27 | WindowsKernelModeDriver10.0 28 | Driver 29 | WDM 30 | false 31 | Desktop 32 | <_NT_TARGET_VERSION>0x0603 33 | 34 | 35 | WindowsV6.3 36 | false 37 | WindowsKernelModeDriver10.0 38 | Driver 39 | WDM 40 | false 41 | Desktop 42 | <_NT_TARGET_VERSION>0x0603 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | DbgengKernelDebugger 52 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 53 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 54 | 55 | 56 | DbgengKernelDebugger 57 | false 58 | ..\..\..\Program Files (x86)\Windows Kits\10\CodeAnalysis\DriverMinimumRules.ruleset 59 | $(SolutionDir)..\$(ConfigurationName)\ 60 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 61 | 62 | 63 | 64 | $(ProjectDir)..\arch\x86\include;$(ProjectDir)..\include;$(ProjectDir)..;%(AdditionalIncludeDirectories) 65 | CONFIG_X86_64;CONFIG_X86_LOCAL_APIC;%(PreprocessorDefinitions) 66 | false 67 | false 68 | false 69 | 70 | 71 | X64;%(PreprocessorDefinitions) 72 | $(ProjectDir)..\;%(IncludePaths) 73 | 74 | 75 | $(SolutionDir)\..\build\asmgen\x64\$(Configuration)\asmgen.exe > $(ProjectDir)..\__asm.inc 76 | 77 | 78 | 79 | sha256 80 | 81 | 82 | wdmsec.lib;%(AdditionalDependencies) 83 | 84 | 85 | 86 | 87 | $(ProjectDir)..\arch\x86\include;$(ProjectDir)..\include;$(ProjectDir)..;%(AdditionalIncludeDirectories) 88 | CONFIG_X86_64;CONFIG_X86_LOCAL_APIC;%(PreprocessorDefinitions) 89 | false 90 | 91 | 92 | $(SolutionDir)\..\build\asmgen\x64\$(Configuration)\asmgen.exe > $(ProjectDir)..\__asm.inc 93 | 94 | 95 | $(ProjectDir)..\;%(IncludePaths) 96 | 97 | 98 | 99 | sha256 100 | 101 | 102 | wdmsec.lib;%(AdditionalDependencies) 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /aehd_main.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * version 2 as published by the Free Software Foundation. 7 | 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | */ 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define AEHD_DEVICE_TOP 0 22 | #define AEHD_DEVICE_VM 1 23 | #define AEHD_DEVICE_VCPU 2 24 | struct aehd_device_extension { 25 | UINT32 DevType; 26 | PVOID PrivData; 27 | }; 28 | 29 | extern PVOID pZeroPage; 30 | 31 | extern int aehdUpdateReturnBuffer(PIRP pIrp, u32 start, void *src, u32 size); 32 | extern int aehdCopyInputBuffer(PIRP pIrp, u32 start, void* dst, u32 size); 33 | extern void* aehdMemdupUser(PIRP pIrp, u32 start, u32 size); 34 | 35 | extern void aehdWaitSuspend( 36 | _In_ PKAPC Apc, 37 | _Inout_ PKNORMAL_ROUTINE* NormalRoutine, 38 | _Inout_ PVOID* NormalContext, 39 | _Inout_ PVOID* SystemArgument1, 40 | _Inout_ PVOID* SystemArgument2) ; 41 | extern long kvm_dev_ioctl(PDEVICE_OBJECT pDevObj, PIRP pIrp, unsigned int ioctl); 42 | extern long kvm_vm_ioctl(PDEVICE_OBJECT pDevObj, PIRP pIrp, unsigned int ioctl); 43 | extern long kvm_vcpu_ioctl(PDEVICE_OBJECT pDevObj, PIRP pIrp, unsigned int ioctl); 44 | extern long kvm_vcpu_fast_ioctl_run(PDEVICE_OBJECT pDevObj); 45 | extern NTSTATUS aehdCreateVMDevice(PHANDLE pHandle, UINT32 vmNumber, INT32 vcpuNumber, 46 | PVOID PrivData); 47 | extern NTSTATUS aehdDeleteVMDevice(PDEVICE_OBJECT pDevObj, UINT32 vmNumber, INT32 vcpuNumber); 48 | -------------------------------------------------------------------------------- /aehd_ver.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * version 2 as published by the Free Software Foundation. 7 | 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | */ 13 | 14 | #pragma once 15 | 16 | #define _STR(str) #str 17 | #define _XSTR(str) _STR(str) 18 | 19 | #define AEHD_MAJOR_VERSION 2 20 | #define AEHD_MINOR_VERSION 2 21 | 22 | #define AEHD_VERSION ((AEHD_MAJOR_VERSION << 16) | AEHD_MINOR_VERSION) 23 | 24 | #define AEHD_RC_VERSION AEHD_MAJOR_VERSION,AEHD_MINOR_VERSION 25 | #define AEHD_RC_VERSION_STR _XSTR(AEHD_MAJOR_VERSION) "." _XSTR(AEHD_MINOR_VERSION) "\0" 26 | -------------------------------------------------------------------------------- /arch/x86/include/asm/apicdef.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef _ASM_X86_APICDEF_H 6 | #define _ASM_X86_APICDEF_H 7 | 8 | #include 9 | 10 | /* 11 | * Constants for various Intel APICs. (local APIC, IOAPIC, etc.) 12 | * 13 | * Alan Cox , 1995. 14 | * Ingo Molnar , 1999, 2000 15 | */ 16 | 17 | #define IO_APIC_DEFAULT_PHYS_BASE 0xfec00000 18 | #define APIC_DEFAULT_PHYS_BASE 0xfee00000 19 | 20 | /* 21 | * This is the IO-APIC register space as specified 22 | * by Intel docs: 23 | */ 24 | #define IO_APIC_SLOT_SIZE 1024 25 | 26 | #define APIC_ID 0x20 27 | 28 | #define APIC_LVR 0x30 29 | #define APIC_LVR_MASK 0xFF00FF 30 | #define APIC_LVR_DIRECTED_EOI (1 << 24) 31 | #define GET_APIC_VERSION(x) ((x) & 0xFFu) 32 | #define GET_APIC_MAXLVT(x) (((x) >> 16) & 0xFFu) 33 | #ifdef CONFIG_X86_32 34 | # define APIC_INTEGRATED(x) ((x) & 0xF0u) 35 | #else 36 | # define APIC_INTEGRATED(x) (1) 37 | #endif 38 | #define APIC_XAPIC(x) ((x) >= 0x14) 39 | #define APIC_EXT_SPACE(x) ((x) & 0x80000000) 40 | #define APIC_TASKPRI 0x80 41 | #define APIC_TPRI_MASK 0xFFu 42 | #define APIC_ARBPRI 0x90 43 | #define APIC_ARBPRI_MASK 0xFFu 44 | #define APIC_PROCPRI 0xA0 45 | #define APIC_EOI 0xB0 46 | #define APIC_EOI_ACK 0x0 /* Docs say 0 for future compat. */ 47 | #define APIC_RRR 0xC0 48 | #define APIC_LDR 0xD0 49 | #define APIC_LDR_MASK (0xFFu << 24) 50 | #define GET_APIC_LOGICAL_ID(x) (((x) >> 24) & 0xFFu) 51 | #define SET_APIC_LOGICAL_ID(x) (((x) << 24)) 52 | #define APIC_ALL_CPUS 0xFFu 53 | #define APIC_DFR 0xE0 54 | #define APIC_DFR_CLUSTER 0x0FFFFFFFul 55 | #define APIC_DFR_FLAT 0xFFFFFFFFul 56 | #define APIC_SPIV 0xF0 57 | #define APIC_SPIV_DIRECTED_EOI (1 << 12) 58 | #define APIC_SPIV_FOCUS_DISABLED (1 << 9) 59 | #define APIC_SPIV_APIC_ENABLED (1 << 8) 60 | #define APIC_ISR 0x100 61 | #define APIC_ISR_NR 0x8 /* Number of 32 bit ISR registers. */ 62 | #define APIC_TMR 0x180 63 | #define APIC_IRR 0x200 64 | #define APIC_ESR 0x280 65 | #define APIC_ESR_SEND_CS 0x00001 66 | #define APIC_ESR_RECV_CS 0x00002 67 | #define APIC_ESR_SEND_ACC 0x00004 68 | #define APIC_ESR_RECV_ACC 0x00008 69 | #define APIC_ESR_SENDILL 0x00020 70 | #define APIC_ESR_RECVILL 0x00040 71 | #define APIC_ESR_ILLREGA 0x00080 72 | #define APIC_LVTCMCI 0x2f0 73 | #define APIC_ICR 0x300 74 | #define APIC_DEST_SELF 0x40000 75 | #define APIC_DEST_ALLINC 0x80000 76 | #define APIC_DEST_ALLBUT 0xC0000 77 | #define APIC_ICR_RR_MASK 0x30000 78 | #define APIC_ICR_RR_INVALID 0x00000 79 | #define APIC_ICR_RR_INPROG 0x10000 80 | #define APIC_ICR_RR_VALID 0x20000 81 | #define APIC_INT_LEVELTRIG 0x08000 82 | #define APIC_INT_ASSERT 0x04000 83 | #define APIC_ICR_BUSY 0x01000 84 | #define APIC_DEST_LOGICAL 0x00800 85 | #define APIC_DEST_PHYSICAL 0x00000 86 | #define APIC_DM_FIXED 0x00000 87 | #define APIC_DM_FIXED_MASK 0x00700 88 | #define APIC_DM_LOWEST 0x00100 89 | #define APIC_DM_SMI 0x00200 90 | #define APIC_DM_NMI 0x00400 91 | #define APIC_DM_INIT 0x00500 92 | #define APIC_DM_STARTUP 0x00600 93 | #define APIC_DM_EXTINT 0x00700 94 | #define APIC_VECTOR_MASK 0x000FF 95 | #define APIC_ICR2 0x310 96 | #define GET_APIC_DEST_FIELD(x) (((x) >> 24) & 0xFF) 97 | #define SET_APIC_DEST_FIELD(x) ((x) << 24) 98 | #define APIC_LVTT 0x320 99 | #define APIC_LVTTHMR 0x330 100 | #define APIC_LVTPC 0x340 101 | #define APIC_LVT0 0x350 102 | #define APIC_LVT_TIMER_BASE_MASK (0x3 << 18) 103 | #define GET_APIC_TIMER_BASE(x) (((x) >> 18) & 0x3) 104 | #define SET_APIC_TIMER_BASE(x) (((x) << 18)) 105 | #define APIC_TIMER_BASE_CLKIN 0x0 106 | #define APIC_TIMER_BASE_TMBASE 0x1 107 | #define APIC_TIMER_BASE_DIV 0x2 108 | #define APIC_LVT_TIMER_ONESHOT (0 << 17) 109 | #define APIC_LVT_TIMER_PERIODIC (1 << 17) 110 | #define APIC_LVT_TIMER_TSCDEADLINE (2 << 17) 111 | #define APIC_LVT_MASKED (1 << 16) 112 | #define APIC_LVT_LEVEL_TRIGGER (1 << 15) 113 | #define APIC_LVT_REMOTE_IRR (1 << 14) 114 | #define APIC_INPUT_POLARITY (1 << 13) 115 | #define APIC_SEND_PENDING (1 << 12) 116 | #define APIC_MODE_MASK 0x700 117 | #define GET_APIC_DELIVERY_MODE(x) (((x) >> 8) & 0x7) 118 | #define SET_APIC_DELIVERY_MODE(x, y) (((x) & ~0x700) | ((y) << 8)) 119 | #define APIC_MODE_FIXED 0x0 120 | #define APIC_MODE_NMI 0x4 121 | #define APIC_MODE_EXTINT 0x7 122 | #define APIC_LVT1 0x360 123 | #define APIC_LVTERR 0x370 124 | #define APIC_TMICT 0x380 125 | #define APIC_TMCCT 0x390 126 | #define APIC_TDCR 0x3E0 127 | #define APIC_SELF_IPI 0x3F0 128 | #define APIC_TDR_DIV_TMBASE (1 << 2) 129 | #define APIC_TDR_DIV_1 0xB 130 | #define APIC_TDR_DIV_2 0x0 131 | #define APIC_TDR_DIV_4 0x1 132 | #define APIC_TDR_DIV_8 0x2 133 | #define APIC_TDR_DIV_16 0x3 134 | #define APIC_TDR_DIV_32 0x8 135 | #define APIC_TDR_DIV_64 0x9 136 | #define APIC_TDR_DIV_128 0xA 137 | #define APIC_EFEAT 0x400 138 | #define APIC_ECTRL 0x410 139 | #define APIC_EILVTn(n) (0x500 + 0x10 * n) 140 | #define APIC_EILVT_NR_AMD_K8 1 /* # of extended interrupts */ 141 | #define APIC_EILVT_NR_AMD_10H 4 142 | #define APIC_EILVT_NR_MAX APIC_EILVT_NR_AMD_10H 143 | #define APIC_EILVT_LVTOFF(x) (((x) >> 4) & 0xF) 144 | #define APIC_EILVT_MSG_FIX 0x0 145 | #define APIC_EILVT_MSG_SMI 0x2 146 | #define APIC_EILVT_MSG_NMI 0x4 147 | #define APIC_EILVT_MSG_EXT 0x7 148 | #define APIC_EILVT_MASKED (1 << 16) 149 | 150 | #define APIC_BASE (fix_to_virt(FIX_APIC_BASE)) 151 | #define APIC_BASE_MSR 0x800 152 | #define XAPIC_ENABLE (1ULL << 11) 153 | #define X2APIC_ENABLE (1ULL << 10) 154 | 155 | #ifdef CONFIG_X86_32 156 | # define MAX_IO_APICS 64 157 | # define MAX_LOCAL_APIC 256 158 | #else 159 | # define MAX_IO_APICS 128 160 | # define MAX_LOCAL_APIC 32768 161 | #endif 162 | 163 | /* 164 | * All x86-64 systems are xAPIC compatible. 165 | * In the following, "apicid" is a physical APIC ID. 166 | */ 167 | #define XAPIC_DEST_CPUS_SHIFT 4 168 | #define XAPIC_DEST_CPUS_MASK ((1u << XAPIC_DEST_CPUS_SHIFT) - 1) 169 | #define XAPIC_DEST_CLUSTER_MASK (XAPIC_DEST_CPUS_MASK << XAPIC_DEST_CPUS_SHIFT) 170 | #define APIC_CLUSTER(apicid) ((apicid) & XAPIC_DEST_CLUSTER_MASK) 171 | #define APIC_CLUSTERID(apicid) (APIC_CLUSTER(apicid) >> XAPIC_DEST_CPUS_SHIFT) 172 | #define APIC_CPUID(apicid) ((apicid) & XAPIC_DEST_CPUS_MASK) 173 | #define NUM_APIC_CLUSTERS ((BAD_APICID + 1) >> XAPIC_DEST_CPUS_SHIFT) 174 | 175 | /* 176 | * the local APIC register structure, memory mapped. Not terribly well 177 | * tested, but we might eventually use this one in the future - the 178 | * problem why we cannot use it right now is the P5 APIC, it has an 179 | * errata which cannot take 8-bit reads and writes, only 32-bit ones ... 180 | */ 181 | #define u32 unsigned int 182 | 183 | #pragma pack(push, 1) 184 | // It seems Windows SDK/WDK defines __reserved which causes conflict here 185 | #undef __reserved 186 | struct local_apic { 187 | 188 | /*000*/ struct { u32 __reserved[4]; } __reserved_01; 189 | 190 | /*010*/ struct { u32 __reserved[4]; } __reserved_02; 191 | 192 | /*020*/ struct { /* APIC ID Register */ 193 | u32 __reserved_1 : 24, 194 | phys_apic_id : 4, 195 | __reserved_2 : 4; 196 | u32 __reserved[3]; 197 | } id; 198 | 199 | /*030*/ const 200 | struct { /* APIC Version Register */ 201 | u32 version : 8, 202 | __reserved_1 : 8, 203 | max_lvt : 8, 204 | __reserved_2 : 8; 205 | u32 __reserved[3]; 206 | } version; 207 | 208 | /*040*/ struct { u32 __reserved[4]; } __reserved_03; 209 | 210 | /*050*/ struct { u32 __reserved[4]; } __reserved_04; 211 | 212 | /*060*/ struct { u32 __reserved[4]; } __reserved_05; 213 | 214 | /*070*/ struct { u32 __reserved[4]; } __reserved_06; 215 | 216 | /*080*/ struct { /* Task Priority Register */ 217 | u32 priority : 8, 218 | __reserved_1 : 24; 219 | u32 __reserved_2[3]; 220 | } tpr; 221 | 222 | /*090*/ const 223 | struct { /* Arbitration Priority Register */ 224 | u32 priority : 8, 225 | __reserved_1 : 24; 226 | u32 __reserved_2[3]; 227 | } apr; 228 | 229 | /*0A0*/ const 230 | struct { /* Processor Priority Register */ 231 | u32 priority : 8, 232 | __reserved_1 : 24; 233 | u32 __reserved_2[3]; 234 | } ppr; 235 | 236 | /*0B0*/ struct { /* End Of Interrupt Register */ 237 | u32 eoi; 238 | u32 __reserved[3]; 239 | } eoi; 240 | 241 | /*0C0*/ struct { u32 __reserved[4]; } __reserved_07; 242 | 243 | /*0D0*/ struct { /* Logical Destination Register */ 244 | u32 __reserved_1 : 24, 245 | logical_dest : 8; 246 | u32 __reserved_2[3]; 247 | } ldr; 248 | 249 | /*0E0*/ struct { /* Destination Format Register */ 250 | u32 __reserved_1 : 28, 251 | model : 4; 252 | u32 __reserved_2[3]; 253 | } dfr; 254 | 255 | /*0F0*/ struct { /* Spurious Interrupt Vector Register */ 256 | u32 spurious_vector : 8, 257 | apic_enabled : 1, 258 | focus_cpu : 1, 259 | __reserved_2 : 22; 260 | u32 __reserved_3[3]; 261 | } svr; 262 | 263 | /*100*/ struct { /* In Service Register */ 264 | /*170*/ u32 bitfield; 265 | u32 __reserved[3]; 266 | } isr[8]; 267 | 268 | /*180*/ struct { /* Trigger Mode Register */ 269 | /*1F0*/ u32 bitfield; 270 | u32 __reserved[3]; 271 | } tmr[8]; 272 | 273 | /*200*/ struct { /* Interrupt Request Register */ 274 | /*270*/ u32 bitfield; 275 | u32 __reserved[3]; 276 | } irr[8]; 277 | 278 | /*280*/ union { /* Error Status Register */ 279 | struct { 280 | u32 send_cs_error : 1, 281 | receive_cs_error : 1, 282 | send_accept_error : 1, 283 | receive_accept_error : 1, 284 | __reserved_1 : 1, 285 | send_illegal_vector : 1, 286 | receive_illegal_vector : 1, 287 | illegal_register_address : 1, 288 | __reserved_2 : 24; 289 | u32 __reserved_3[3]; 290 | } error_bits; 291 | struct { 292 | u32 errors; 293 | u32 __reserved_3[3]; 294 | } all_errors; 295 | } esr; 296 | 297 | /*290*/ struct { u32 __reserved[4]; } __reserved_08; 298 | 299 | /*2A0*/ struct { u32 __reserved[4]; } __reserved_09; 300 | 301 | /*2B0*/ struct { u32 __reserved[4]; } __reserved_10; 302 | 303 | /*2C0*/ struct { u32 __reserved[4]; } __reserved_11; 304 | 305 | /*2D0*/ struct { u32 __reserved[4]; } __reserved_12; 306 | 307 | /*2E0*/ struct { u32 __reserved[4]; } __reserved_13; 308 | 309 | /*2F0*/ struct { u32 __reserved[4]; } __reserved_14; 310 | 311 | /*300*/ struct { /* Interrupt Command Register 1 */ 312 | u32 vector : 8, 313 | delivery_mode : 3, 314 | destination_mode : 1, 315 | delivery_status : 1, 316 | __reserved_1 : 1, 317 | level : 1, 318 | trigger : 1, 319 | __reserved_2 : 2, 320 | shorthand : 2, 321 | __reserved_3 : 12; 322 | u32 __reserved_4[3]; 323 | } icr1; 324 | 325 | /*310*/ struct { /* Interrupt Command Register 2 */ 326 | union { 327 | u32 __reserved_1 : 24, 328 | phys_dest : 4, 329 | __reserved_2 : 4; 330 | u32 __reserved_3 : 24, 331 | logical_dest : 8; 332 | } dest; 333 | u32 __reserved_4[3]; 334 | } icr2; 335 | 336 | /*320*/ struct { /* LVT - Timer */ 337 | u32 vector : 8, 338 | __reserved_1 : 4, 339 | delivery_status : 1, 340 | __reserved_2 : 3, 341 | mask : 1, 342 | timer_mode : 1, 343 | __reserved_3 : 14; 344 | u32 __reserved_4[3]; 345 | } lvt_timer; 346 | 347 | /*330*/ struct { /* LVT - Thermal Sensor */ 348 | u32 vector : 8, 349 | delivery_mode : 3, 350 | __reserved_1 : 1, 351 | delivery_status : 1, 352 | __reserved_2 : 3, 353 | mask : 1, 354 | __reserved_3 : 15; 355 | u32 __reserved_4[3]; 356 | } lvt_thermal; 357 | 358 | /*340*/ struct { /* LVT - Performance Counter */ 359 | u32 vector : 8, 360 | delivery_mode : 3, 361 | __reserved_1 : 1, 362 | delivery_status : 1, 363 | __reserved_2 : 3, 364 | mask : 1, 365 | __reserved_3 : 15; 366 | u32 __reserved_4[3]; 367 | } lvt_pc; 368 | 369 | /*350*/ struct { /* LVT - LINT0 */ 370 | u32 vector : 8, 371 | delivery_mode : 3, 372 | __reserved_1 : 1, 373 | delivery_status : 1, 374 | polarity : 1, 375 | remote_irr : 1, 376 | trigger : 1, 377 | mask : 1, 378 | __reserved_2 : 15; 379 | u32 __reserved_3[3]; 380 | } lvt_lint0; 381 | 382 | /*360*/ struct { /* LVT - LINT1 */ 383 | u32 vector : 8, 384 | delivery_mode : 3, 385 | __reserved_1 : 1, 386 | delivery_status : 1, 387 | polarity : 1, 388 | remote_irr : 1, 389 | trigger : 1, 390 | mask : 1, 391 | __reserved_2 : 15; 392 | u32 __reserved_3[3]; 393 | } lvt_lint1; 394 | 395 | /*370*/ struct { /* LVT - Error */ 396 | u32 vector : 8, 397 | __reserved_1 : 4, 398 | delivery_status : 1, 399 | __reserved_2 : 3, 400 | mask : 1, 401 | __reserved_3 : 15; 402 | u32 __reserved_4[3]; 403 | } lvt_error; 404 | 405 | /*380*/ struct { /* Timer Initial Count Register */ 406 | u32 initial_count; 407 | u32 __reserved_2[3]; 408 | } timer_icr; 409 | 410 | /*390*/ const 411 | struct { /* Timer Current Count Register */ 412 | u32 curr_count; 413 | u32 __reserved_2[3]; 414 | } timer_ccr; 415 | 416 | /*3A0*/ struct { u32 __reserved[4]; } __reserved_16; 417 | 418 | /*3B0*/ struct { u32 __reserved[4]; } __reserved_17; 419 | 420 | /*3C0*/ struct { u32 __reserved[4]; } __reserved_18; 421 | 422 | /*3D0*/ struct { u32 __reserved[4]; } __reserved_19; 423 | 424 | /*3E0*/ struct { /* Timer Divide Configuration Register */ 425 | u32 divisor : 4, 426 | __reserved_1 : 28; 427 | u32 __reserved_2[3]; 428 | } timer_dcr; 429 | 430 | /*3F0*/ struct { u32 __reserved[4]; } __reserved_20; 431 | 432 | }; 433 | #pragma pack(pop) 434 | 435 | #undef u32 436 | 437 | #ifdef CONFIG_X86_32 438 | #define BAD_APICID 0xFFu 439 | #else 440 | #define BAD_APICID 0xFFFFu 441 | #endif 442 | 443 | enum ioapic_irq_destination_types { 444 | dest_Fixed = 0, 445 | dest_LowestPrio = 1, 446 | dest_SMI = 2, 447 | dest__reserved_1 = 3, 448 | dest_NMI = 4, 449 | dest_INIT = 5, 450 | dest__reserved_2 = 6, 451 | dest_ExtINT = 7 452 | }; 453 | 454 | #endif /* _ASM_X86_APICDEF_H */ 455 | -------------------------------------------------------------------------------- /arch/x86/include/asm/fpu/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | /* 6 | * FPU data structures: 7 | */ 8 | #ifndef _ASM_X86_FPU_H 9 | #define _ASM_X86_FPU_H 10 | 11 | #include 12 | 13 | /* 14 | * The legacy x87 FPU state format, as saved by FSAVE and 15 | * restored by the FRSTOR instructions: 16 | */ 17 | struct fregs_state { 18 | u32 cwd; /* FPU Control Word */ 19 | u32 swd; /* FPU Status Word */ 20 | u32 twd; /* FPU Tag Word */ 21 | u32 fip; /* FPU IP Offset */ 22 | u32 fcs; /* FPU IP Selector */ 23 | u32 foo; /* FPU Operand Pointer Offset */ 24 | u32 fos; /* FPU Operand Pointer Selector */ 25 | 26 | /* 8*10 bytes for each FP-reg = 80 bytes: */ 27 | u32 st_space[20]; 28 | 29 | /* Software status information [not touched by FSAVE]: */ 30 | u32 status; 31 | }; 32 | 33 | /* 34 | * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and 35 | * restored by the FXRSTOR instructions. It's similar to the FSAVE 36 | * format, but differs in some areas, plus has extensions at 37 | * the end for the XMM registers. 38 | */ 39 | __align(16) 40 | struct fxregs_state { 41 | u16 cwd; /* Control Word */ 42 | u16 swd; /* Status Word */ 43 | u16 twd; /* Tag Word */ 44 | u16 fop; /* Last Instruction Opcode */ 45 | union { 46 | struct { 47 | u64 rip; /* Instruction Pointer */ 48 | u64 rdp; /* Data Pointer */ 49 | }; 50 | struct { 51 | u32 fip; /* FPU IP Offset */ 52 | u32 fcs; /* FPU IP Selector */ 53 | u32 foo; /* FPU Operand Offset */ 54 | u32 fos; /* FPU Operand Selector */ 55 | }; 56 | }; 57 | u32 mxcsr; /* MXCSR Register State */ 58 | u32 mxcsr_mask; /* MXCSR Mask */ 59 | 60 | /* 8*16 bytes for each FP-reg = 128 bytes: */ 61 | u32 st_space[32]; 62 | 63 | /* 16*16 bytes for each XMM-reg = 256 bytes: */ 64 | u32 xmm_space[64]; 65 | 66 | u32 padding[12]; 67 | 68 | union { 69 | u32 padding1[12]; 70 | u32 sw_reserved[12]; 71 | }; 72 | 73 | }; 74 | 75 | /* Default value for fxregs_state.mxcsr: */ 76 | #define MXCSR_DEFAULT 0x1f80 77 | 78 | /* 79 | * Software based FPU emulation state. This is arbitrary really, 80 | * it matches the x87 format to make it easier to understand: 81 | */ 82 | struct swregs_state { 83 | u32 cwd; 84 | u32 swd; 85 | u32 twd; 86 | u32 fip; 87 | u32 fcs; 88 | u32 foo; 89 | u32 fos; 90 | /* 8*10 bytes for each FP-reg = 80 bytes: */ 91 | u32 st_space[20]; 92 | u8 ftop; 93 | u8 changed; 94 | u8 lookahead; 95 | u8 no_update; 96 | u8 rm; 97 | u8 alimit; 98 | struct math_emu_info *info; 99 | u32 entry_eip; 100 | }; 101 | 102 | /* 103 | * List of XSAVE features Linux knows about: 104 | */ 105 | enum xfeature { 106 | XFEATURE_FP, 107 | XFEATURE_SSE, 108 | /* 109 | * Values above here are "legacy states". 110 | * Those below are "extended states". 111 | */ 112 | XFEATURE_YMM, 113 | XFEATURE_BNDREGS, 114 | XFEATURE_BNDCSR, 115 | XFEATURE_OPMASK, 116 | XFEATURE_ZMM_Hi256, 117 | XFEATURE_Hi16_ZMM, 118 | XFEATURE_PT_UNIMPLEMENTED_SO_FAR, 119 | XFEATURE_PKRU, 120 | 121 | XFEATURE_MAX, 122 | }; 123 | 124 | #define XFEATURE_MASK_FP (1 << XFEATURE_FP) 125 | #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) 126 | #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) 127 | #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) 128 | #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) 129 | #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) 130 | #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) 131 | #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) 132 | #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) 133 | #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) 134 | 135 | #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) 136 | #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ 137 | | XFEATURE_MASK_ZMM_Hi256 \ 138 | | XFEATURE_MASK_Hi16_ZMM) 139 | 140 | #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM 141 | 142 | struct reg_128_bit { 143 | u8 regbytes[128/8]; 144 | }; 145 | struct reg_256_bit { 146 | u8 regbytes[256/8]; 147 | }; 148 | struct reg_512_bit { 149 | u8 regbytes[512/8]; 150 | }; 151 | 152 | #undef __packed 153 | #define __packed 154 | #pragma pack(push, 1) 155 | /* 156 | * State component 2: 157 | * 158 | * There are 16x 256-bit AVX registers named YMM0-YMM15. 159 | * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) 160 | * and are stored in 'struct fxregs_state::xmm_space[]' in the 161 | * "legacy" area. 162 | * 163 | * The high 128 bits are stored here. 164 | */ 165 | struct ymmh_struct { 166 | struct reg_128_bit hi_ymm[16]; 167 | } __packed; 168 | 169 | /* Intel MPX support: */ 170 | 171 | struct mpx_bndreg { 172 | u64 lower_bound; 173 | u64 upper_bound; 174 | } __packed; 175 | /* 176 | * State component 3 is used for the 4 128-bit bounds registers 177 | */ 178 | struct mpx_bndreg_state { 179 | struct mpx_bndreg bndreg[4]; 180 | } __packed; 181 | 182 | /* 183 | * State component 4 is used for the 64-bit user-mode MPX 184 | * configuration register BNDCFGU and the 64-bit MPX status 185 | * register BNDSTATUS. We call the pair "BNDCSR". 186 | */ 187 | struct mpx_bndcsr { 188 | u64 bndcfgu; 189 | u64 bndstatus; 190 | } __packed; 191 | 192 | /* 193 | * The BNDCSR state is padded out to be 64-bytes in size. 194 | */ 195 | struct mpx_bndcsr_state { 196 | union { 197 | struct mpx_bndcsr bndcsr; 198 | u8 pad_to_64_bytes[64]; 199 | }; 200 | } __packed; 201 | 202 | /* AVX-512 Components: */ 203 | 204 | /* 205 | * State component 5 is used for the 8 64-bit opmask registers 206 | * k0-k7 (opmask state). 207 | */ 208 | struct avx_512_opmask_state { 209 | u64 opmask_reg[8]; 210 | } __packed; 211 | 212 | /* 213 | * State component 6 is used for the upper 256 bits of the 214 | * registers ZMM0-ZMM15. These 16 256-bit values are denoted 215 | * ZMM0_H-ZMM15_H (ZMM_Hi256 state). 216 | */ 217 | struct avx_512_zmm_uppers_state { 218 | struct reg_256_bit zmm_upper[16]; 219 | } __packed; 220 | 221 | /* 222 | * State component 7 is used for the 16 512-bit registers 223 | * ZMM16-ZMM31 (Hi16_ZMM state). 224 | */ 225 | struct avx_512_hi16_state { 226 | struct reg_512_bit hi16_zmm[16]; 227 | } __packed; 228 | 229 | /* 230 | * State component 9: 32-bit PKRU register. The state is 231 | * 8 bytes long but only 4 bytes is used currently. 232 | */ 233 | struct pkru_state { 234 | u32 pkru; 235 | u32 pad; 236 | } __packed; 237 | 238 | struct xstate_header { 239 | u64 xfeatures; 240 | u64 xcomp_bv; 241 | u64 reserved[6]; 242 | }; 243 | #pragma pack(pop) 244 | 245 | /* 246 | * xstate_header.xcomp_bv[63] indicates that the extended_state_area 247 | * is in compacted format. 248 | */ 249 | #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) 250 | 251 | /* 252 | * This is our most modern FPU state format, as saved by the XSAVE 253 | * and restored by the XRSTOR instructions. 254 | * 255 | * It consists of a legacy fxregs portion, an xstate header and 256 | * subsequent areas as defined by the xstate header. Not all CPUs 257 | * support all the extensions, so the size of the extended area 258 | * can vary quite a bit between CPUs. 259 | */ 260 | #pragma pack(push, 16) 261 | struct xregs_state { 262 | struct fxregs_state i387; 263 | struct xstate_header header; 264 | u8 extended_state_area[0]; 265 | }; 266 | #pragma pack(pop) 267 | 268 | /* 269 | * This is a union of all the possible FPU state formats 270 | * put together, so that we can pick the right one runtime. 271 | * 272 | * The size of the structure is determined by the largest 273 | * member - which is the xsave area. The padding is there 274 | * to ensure that statically-allocated task_structs (just 275 | * the init_task today) have enough space. 276 | */ 277 | union fpu_state { 278 | struct fxregs_state fxsave; 279 | struct xregs_state xsave; 280 | u8 __padding[PAGE_SIZE]; 281 | }; 282 | 283 | #endif /* _ASM_X86_FPU_H */ 284 | -------------------------------------------------------------------------------- /arch/x86/include/asm/kvm_page_track.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef _ASM_X86_KVM_PAGE_TRACK_H 6 | #define _ASM_X86_KVM_PAGE_TRACK_H 7 | 8 | #include 9 | 10 | enum kvm_page_track_mode { 11 | KVM_PAGE_TRACK_WRITE, 12 | KVM_PAGE_TRACK_MAX, 13 | }; 14 | 15 | /* 16 | * The notifier represented by @kvm_page_track_notifier_node is linked into 17 | * the head which will be notified when guest is triggering the track event. 18 | * 19 | * Write access on the head is protected by kvm->mmu_lock, read access 20 | * is protected by track_srcu. 21 | */ 22 | struct kvm_page_track_notifier_head { 23 | struct srcu_struct track_srcu; 24 | struct hlist_head track_notifier_list; 25 | }; 26 | 27 | struct kvm_page_track_notifier_node { 28 | struct hlist_node node; 29 | 30 | /* 31 | * It is called when guest is writing the write-tracked page 32 | * and write emulation is finished at that time. 33 | * 34 | * @vcpu: the vcpu where the write access happened. 35 | * @gpa: the physical address written by guest. 36 | * @new: the data was written to the address. 37 | * @bytes: the written length. 38 | */ 39 | void (*track_write)(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 40 | int bytes); 41 | }; 42 | 43 | void kvm_page_track_init(struct kvm *kvm); 44 | void kvm_page_track_destroy(struct kvm *kvm); 45 | 46 | void kvm_page_track_free_memslot(struct kvm_memory_slot *free, 47 | struct kvm_memory_slot *dont); 48 | int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, 49 | size_t npages); 50 | 51 | void kvm_slot_page_track_add_page(struct kvm *kvm, 52 | struct kvm_memory_slot *slot, gfn_t gfn, 53 | enum kvm_page_track_mode mode); 54 | void kvm_slot_page_track_remove_page(struct kvm *kvm, 55 | struct kvm_memory_slot *slot, gfn_t gfn, 56 | enum kvm_page_track_mode mode); 57 | bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, 58 | enum kvm_page_track_mode mode); 59 | 60 | void 61 | kvm_page_track_register_notifier(struct kvm *kvm, 62 | struct kvm_page_track_notifier_node *n); 63 | void 64 | kvm_page_track_unregister_notifier(struct kvm *kvm, 65 | struct kvm_page_track_notifier_node *n); 66 | void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 67 | int bytes); 68 | #endif 69 | -------------------------------------------------------------------------------- /arch/x86/include/asm/msidef.h: -------------------------------------------------------------------------------- 1 | #ifndef _ASM_X86_MSIDEF_H 2 | #define _ASM_X86_MSIDEF_H 3 | 4 | /* 5 | * Constants for Intel APIC based MSI messages. 6 | */ 7 | 8 | /* 9 | * Shifts for MSI data 10 | */ 11 | 12 | #define MSI_DATA_VECTOR_SHIFT 0 13 | #define MSI_DATA_VECTOR_MASK 0x000000ff 14 | #define MSI_DATA_VECTOR(v) (((v) << MSI_DATA_VECTOR_SHIFT) & \ 15 | MSI_DATA_VECTOR_MASK) 16 | 17 | #define MSI_DATA_DELIVERY_MODE_SHIFT 8 18 | #define MSI_DATA_DELIVERY_FIXED (0 << MSI_DATA_DELIVERY_MODE_SHIFT) 19 | #define MSI_DATA_DELIVERY_LOWPRI (1 << MSI_DATA_DELIVERY_MODE_SHIFT) 20 | 21 | #define MSI_DATA_LEVEL_SHIFT 14 22 | #define MSI_DATA_LEVEL_DEASSERT (0 << MSI_DATA_LEVEL_SHIFT) 23 | #define MSI_DATA_LEVEL_ASSERT (1 << MSI_DATA_LEVEL_SHIFT) 24 | 25 | #define MSI_DATA_TRIGGER_SHIFT 15 26 | #define MSI_DATA_TRIGGER_EDGE (0 << MSI_DATA_TRIGGER_SHIFT) 27 | #define MSI_DATA_TRIGGER_LEVEL (1 << MSI_DATA_TRIGGER_SHIFT) 28 | 29 | /* 30 | * Shift/mask fields for msi address 31 | */ 32 | 33 | #define MSI_ADDR_BASE_HI 0 34 | #define MSI_ADDR_BASE_LO 0xfee00000 35 | 36 | #define MSI_ADDR_DEST_MODE_SHIFT 2 37 | #define MSI_ADDR_DEST_MODE_PHYSICAL (0 << MSI_ADDR_DEST_MODE_SHIFT) 38 | #define MSI_ADDR_DEST_MODE_LOGICAL (1 << MSI_ADDR_DEST_MODE_SHIFT) 39 | 40 | #define MSI_ADDR_REDIRECTION_SHIFT 3 41 | #define MSI_ADDR_REDIRECTION_CPU (0 << MSI_ADDR_REDIRECTION_SHIFT) 42 | /* dedicated cpu */ 43 | #define MSI_ADDR_REDIRECTION_LOWPRI (1 << MSI_ADDR_REDIRECTION_SHIFT) 44 | /* lowest priority */ 45 | 46 | #define MSI_ADDR_DEST_ID_SHIFT 12 47 | #define MSI_ADDR_DEST_ID_MASK 0x00ffff0 48 | #define MSI_ADDR_DEST_ID(dest) (((dest) << MSI_ADDR_DEST_ID_SHIFT) & \ 49 | MSI_ADDR_DEST_ID_MASK) 50 | #define MSI_ADDR_EXT_DEST_ID(dest) ((dest) & 0xffffff00) 51 | 52 | #define MSI_ADDR_IR_EXT_INT (1 << 4) 53 | #define MSI_ADDR_IR_SHV (1 << 3) 54 | #define MSI_ADDR_IR_INDEX1(index) ((index & 0x8000) >> 13) 55 | #define MSI_ADDR_IR_INDEX2(index) ((index & 0x7fff) << 5) 56 | #endif /* _ASM_X86_MSIDEF_H */ 57 | -------------------------------------------------------------------------------- /arch/x86/include/asm/svm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | /* SPDX-License-Identifier: GPL-2.0 */ 6 | #ifndef __SVM_H 7 | #define __SVM_H 8 | 9 | #include 10 | 11 | 12 | enum { 13 | INTERCEPT_INTR, 14 | INTERCEPT_NMI, 15 | INTERCEPT_SMI, 16 | INTERCEPT_INIT, 17 | INTERCEPT_VINTR, 18 | INTERCEPT_SELECTIVE_CR0, 19 | INTERCEPT_STORE_IDTR, 20 | INTERCEPT_STORE_GDTR, 21 | INTERCEPT_STORE_LDTR, 22 | INTERCEPT_STORE_TR, 23 | INTERCEPT_LOAD_IDTR, 24 | INTERCEPT_LOAD_GDTR, 25 | INTERCEPT_LOAD_LDTR, 26 | INTERCEPT_LOAD_TR, 27 | INTERCEPT_RDTSC, 28 | INTERCEPT_RDPMC, 29 | INTERCEPT_PUSHF, 30 | INTERCEPT_POPF, 31 | INTERCEPT_CPUID, 32 | INTERCEPT_RSM, 33 | INTERCEPT_IRET, 34 | INTERCEPT_INTn, 35 | INTERCEPT_INVD, 36 | INTERCEPT_PAUSE, 37 | INTERCEPT_HLT, 38 | INTERCEPT_INVLPG, 39 | INTERCEPT_INVLPGA, 40 | INTERCEPT_IOIO_PROT, 41 | INTERCEPT_MSR_PROT, 42 | INTERCEPT_TASK_SWITCH, 43 | INTERCEPT_FERR_FREEZE, 44 | INTERCEPT_SHUTDOWN, 45 | INTERCEPT_VMRUN, 46 | INTERCEPT_VMMCALL, 47 | INTERCEPT_VMLOAD, 48 | INTERCEPT_VMSAVE, 49 | INTERCEPT_STGI, 50 | INTERCEPT_CLGI, 51 | INTERCEPT_SKINIT, 52 | INTERCEPT_RDTSCP, 53 | INTERCEPT_ICEBP, 54 | INTERCEPT_WBINVD, 55 | INTERCEPT_MONITOR, 56 | INTERCEPT_MWAIT, 57 | INTERCEPT_MWAIT_COND, 58 | INTERCEPT_XSETBV, 59 | }; 60 | 61 | #pragma pack(push, 1) 62 | struct vmcb_control_area { 63 | u32 intercept_cr; 64 | u32 intercept_dr; 65 | u32 intercept_exceptions; 66 | u64 intercept; 67 | u8 reserved_1[40]; 68 | u16 pause_filter_thresh; 69 | u16 pause_filter_count; 70 | u64 iopm_base_pa; 71 | u64 msrpm_base_pa; 72 | u64 tsc_offset; 73 | u32 asid; 74 | u8 tlb_ctl; 75 | u8 reserved_2[3]; 76 | u32 int_ctl; 77 | u32 int_vector; 78 | u32 int_state; 79 | u8 reserved_3[4]; 80 | u32 exit_code; 81 | u32 exit_code_hi; 82 | u64 exit_info_1; 83 | u64 exit_info_2; 84 | u32 exit_int_info; 85 | u32 exit_int_info_err; 86 | u64 nested_ctl; 87 | u64 avic_vapic_bar; 88 | u8 reserved_4[8]; 89 | u32 event_inj; 90 | u32 event_inj_err; 91 | u64 nested_cr3; 92 | u64 lbr_ctl; 93 | u32 clean; 94 | u32 reserved_5; 95 | u64 next_rip; 96 | u8 insn_len; 97 | u8 insn_bytes[15]; 98 | u64 avic_backing_page; /* Offset 0xe0 */ 99 | u8 reserved_6[8]; /* Offset 0xe8 */ 100 | u64 avic_logical_id; /* Offset 0xf0 */ 101 | u64 avic_physical_id; /* Offset 0xf8 */ 102 | u8 reserved_7[768]; 103 | }; 104 | #pragma pack(pop) 105 | 106 | #define TLB_CONTROL_DO_NOTHING 0 107 | #define TLB_CONTROL_FLUSH_ALL_ASID 1 108 | #define TLB_CONTROL_FLUSH_ASID 3 109 | #define TLB_CONTROL_FLUSH_ASID_LOCAL 7 110 | 111 | #define V_TPR_MASK 0x0f 112 | 113 | #define V_IRQ_SHIFT 8 114 | #define V_IRQ_MASK (1 << V_IRQ_SHIFT) 115 | 116 | #define V_GIF_SHIFT 9 117 | #define V_GIF_MASK (1 << V_GIF_SHIFT) 118 | 119 | #define V_INTR_PRIO_SHIFT 16 120 | #define V_INTR_PRIO_MASK (0x0f << V_INTR_PRIO_SHIFT) 121 | 122 | #define V_IGN_TPR_SHIFT 20 123 | #define V_IGN_TPR_MASK (1 << V_IGN_TPR_SHIFT) 124 | 125 | #define V_INTR_MASKING_SHIFT 24 126 | #define V_INTR_MASKING_MASK (1 << V_INTR_MASKING_SHIFT) 127 | 128 | #define V_GIF_ENABLE_SHIFT 25 129 | #define V_GIF_ENABLE_MASK (1 << V_GIF_ENABLE_SHIFT) 130 | 131 | #define AVIC_ENABLE_SHIFT 31 132 | #define AVIC_ENABLE_MASK (1 << AVIC_ENABLE_SHIFT) 133 | 134 | #define LBR_CTL_ENABLE_MASK BIT_ULL(0) 135 | #define VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK BIT_ULL(1) 136 | 137 | #define SVM_INTERRUPT_SHADOW_MASK 1 138 | 139 | #define SVM_IOIO_STR_SHIFT 2 140 | #define SVM_IOIO_REP_SHIFT 3 141 | #define SVM_IOIO_SIZE_SHIFT 4 142 | #define SVM_IOIO_ASIZE_SHIFT 7 143 | 144 | #define SVM_IOIO_TYPE_MASK 1 145 | #define SVM_IOIO_STR_MASK (1 << SVM_IOIO_STR_SHIFT) 146 | #define SVM_IOIO_REP_MASK (1 << SVM_IOIO_REP_SHIFT) 147 | #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) 148 | #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) 149 | 150 | #define SVM_VM_CR_VALID_MASK 0x001fULL 151 | #define SVM_VM_CR_SVM_LOCK_MASK 0x0008ULL 152 | #define SVM_VM_CR_SVM_DIS_MASK 0x0010ULL 153 | 154 | #define SVM_NESTED_CTL_NP_ENABLE BIT(0) 155 | #define SVM_NESTED_CTL_SEV_ENABLE BIT(1) 156 | 157 | #pragma pack(push, 1) 158 | struct vmcb_seg { 159 | u16 selector; 160 | u16 attrib; 161 | u32 limit; 162 | u64 base; 163 | }; 164 | 165 | struct vmcb_save_area { 166 | struct vmcb_seg es; 167 | struct vmcb_seg cs; 168 | struct vmcb_seg ss; 169 | struct vmcb_seg ds; 170 | struct vmcb_seg fs; 171 | struct vmcb_seg gs; 172 | struct vmcb_seg gdtr; 173 | struct vmcb_seg ldtr; 174 | struct vmcb_seg idtr; 175 | struct vmcb_seg tr; 176 | u8 reserved_1[43]; 177 | u8 cpl; 178 | u8 reserved_2[4]; 179 | u64 efer; 180 | u8 reserved_3[112]; 181 | u64 cr4; 182 | u64 cr3; 183 | u64 cr0; 184 | u64 dr7; 185 | u64 dr6; 186 | u64 rflags; 187 | u64 rip; 188 | u8 reserved_4[88]; 189 | u64 rsp; 190 | u8 reserved_5[24]; 191 | u64 rax; 192 | u64 star; 193 | u64 lstar; 194 | u64 cstar; 195 | u64 sfmask; 196 | u64 kernel_gs_base; 197 | u64 sysenter_cs; 198 | u64 sysenter_esp; 199 | u64 sysenter_eip; 200 | u64 cr2; 201 | u8 reserved_6[32]; 202 | u64 g_pat; 203 | u64 dbgctl; 204 | u64 br_from; 205 | u64 br_to; 206 | u64 last_excp_from; 207 | u64 last_excp_to; 208 | }; 209 | 210 | struct vmcb { 211 | struct vmcb_control_area control; 212 | struct vmcb_save_area save; 213 | }; 214 | #pragma pack(pop) 215 | 216 | #define SVM_CPUID_FUNC 0x8000000a 217 | 218 | #define SVM_VM_CR_SVM_DISABLE 4 219 | 220 | #define SVM_SELECTOR_S_SHIFT 4 221 | #define SVM_SELECTOR_DPL_SHIFT 5 222 | #define SVM_SELECTOR_P_SHIFT 7 223 | #define SVM_SELECTOR_AVL_SHIFT 8 224 | #define SVM_SELECTOR_L_SHIFT 9 225 | #define SVM_SELECTOR_DB_SHIFT 10 226 | #define SVM_SELECTOR_G_SHIFT 11 227 | 228 | #define SVM_SELECTOR_TYPE_MASK (0xf) 229 | #define SVM_SELECTOR_S_MASK (1 << SVM_SELECTOR_S_SHIFT) 230 | #define SVM_SELECTOR_DPL_MASK (3 << SVM_SELECTOR_DPL_SHIFT) 231 | #define SVM_SELECTOR_P_MASK (1 << SVM_SELECTOR_P_SHIFT) 232 | #define SVM_SELECTOR_AVL_MASK (1 << SVM_SELECTOR_AVL_SHIFT) 233 | #define SVM_SELECTOR_L_MASK (1 << SVM_SELECTOR_L_SHIFT) 234 | #define SVM_SELECTOR_DB_MASK (1 << SVM_SELECTOR_DB_SHIFT) 235 | #define SVM_SELECTOR_G_MASK (1 << SVM_SELECTOR_G_SHIFT) 236 | 237 | #define SVM_SELECTOR_WRITE_MASK (1 << 1) 238 | #define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK 239 | #define SVM_SELECTOR_CODE_MASK (1 << 3) 240 | 241 | #define INTERCEPT_CR0_READ 0 242 | #define INTERCEPT_CR3_READ 3 243 | #define INTERCEPT_CR4_READ 4 244 | #define INTERCEPT_CR8_READ 8 245 | #define INTERCEPT_CR0_WRITE (16 + 0) 246 | #define INTERCEPT_CR3_WRITE (16 + 3) 247 | #define INTERCEPT_CR4_WRITE (16 + 4) 248 | #define INTERCEPT_CR8_WRITE (16 + 8) 249 | 250 | #define INTERCEPT_DR0_READ 0 251 | #define INTERCEPT_DR1_READ 1 252 | #define INTERCEPT_DR2_READ 2 253 | #define INTERCEPT_DR3_READ 3 254 | #define INTERCEPT_DR4_READ 4 255 | #define INTERCEPT_DR5_READ 5 256 | #define INTERCEPT_DR6_READ 6 257 | #define INTERCEPT_DR7_READ 7 258 | #define INTERCEPT_DR0_WRITE (16 + 0) 259 | #define INTERCEPT_DR1_WRITE (16 + 1) 260 | #define INTERCEPT_DR2_WRITE (16 + 2) 261 | #define INTERCEPT_DR3_WRITE (16 + 3) 262 | #define INTERCEPT_DR4_WRITE (16 + 4) 263 | #define INTERCEPT_DR5_WRITE (16 + 5) 264 | #define INTERCEPT_DR6_WRITE (16 + 6) 265 | #define INTERCEPT_DR7_WRITE (16 + 7) 266 | 267 | #define SVM_EVTINJ_VEC_MASK 0xff 268 | 269 | #define SVM_EVTINJ_TYPE_SHIFT 8 270 | #define SVM_EVTINJ_TYPE_MASK (7 << SVM_EVTINJ_TYPE_SHIFT) 271 | 272 | #define SVM_EVTINJ_TYPE_INTR (0 << SVM_EVTINJ_TYPE_SHIFT) 273 | #define SVM_EVTINJ_TYPE_NMI (2 << SVM_EVTINJ_TYPE_SHIFT) 274 | #define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) 275 | #define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) 276 | 277 | #define SVM_EVTINJ_VALID (1 << 31) 278 | #define SVM_EVTINJ_VALID_ERR (1 << 11) 279 | 280 | #define SVM_EXITINTINFO_VEC_MASK SVM_EVTINJ_VEC_MASK 281 | #define SVM_EXITINTINFO_TYPE_MASK SVM_EVTINJ_TYPE_MASK 282 | 283 | #define SVM_EXITINTINFO_TYPE_INTR SVM_EVTINJ_TYPE_INTR 284 | #define SVM_EXITINTINFO_TYPE_NMI SVM_EVTINJ_TYPE_NMI 285 | #define SVM_EXITINTINFO_TYPE_EXEPT SVM_EVTINJ_TYPE_EXEPT 286 | #define SVM_EXITINTINFO_TYPE_SOFT SVM_EVTINJ_TYPE_SOFT 287 | 288 | #define SVM_EXITINTINFO_VALID SVM_EVTINJ_VALID 289 | #define SVM_EXITINTINFO_VALID_ERR SVM_EVTINJ_VALID_ERR 290 | 291 | #define SVM_EXITINFOSHIFT_TS_REASON_IRET 36 292 | #define SVM_EXITINFOSHIFT_TS_REASON_JMP 38 293 | #define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44 294 | 295 | #define SVM_EXITINFO_REG_MASK 0x0F 296 | 297 | #define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP) 298 | 299 | #define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" 300 | #define SVM_VMRUN ".byte 0x0f, 0x01, 0xd8" 301 | #define SVM_VMSAVE ".byte 0x0f, 0x01, 0xdb" 302 | #define SVM_CLGI ".byte 0x0f, 0x01, 0xdd" 303 | #define SVM_STGI ".byte 0x0f, 0x01, 0xdc" 304 | #define SVM_INVLPGA ".byte 0x0f, 0x01, 0xdf" 305 | 306 | #endif 307 | -------------------------------------------------------------------------------- /arch/x86/include/uapi/asm/debugreg.h: -------------------------------------------------------------------------------- 1 | #ifndef _UAPI_ASM_X86_DEBUGREG_H 2 | #define _UAPI_ASM_X86_DEBUGREG_H 3 | 4 | 5 | /* Indicate the register numbers for a number of the specific 6 | debug registers. Registers 0-3 contain the addresses we wish to trap on */ 7 | #define DR_FIRSTADDR 0 /* u_debugreg[DR_FIRSTADDR] */ 8 | #define DR_LASTADDR 3 /* u_debugreg[DR_LASTADDR] */ 9 | 10 | #define DR_STATUS 6 /* u_debugreg[DR_STATUS] */ 11 | #define DR_CONTROL 7 /* u_debugreg[DR_CONTROL] */ 12 | 13 | /* Define a few things for the status register. We can use this to determine 14 | which debugging register was responsible for the trap. The other bits 15 | are either reserved or not of interest to us. */ 16 | 17 | /* Define reserved bits in DR6 which are always set to 1 */ 18 | #define DR6_RESERVED (0xFFFF0FF0) 19 | 20 | #define DR_TRAP0 (0x1) /* db0 */ 21 | #define DR_TRAP1 (0x2) /* db1 */ 22 | #define DR_TRAP2 (0x4) /* db2 */ 23 | #define DR_TRAP3 (0x8) /* db3 */ 24 | #define DR_TRAP_BITS (DR_TRAP0|DR_TRAP1|DR_TRAP2|DR_TRAP3) 25 | 26 | #define DR_STEP (0x4000) /* single-step */ 27 | #define DR_SWITCH (0x8000) /* task switch */ 28 | 29 | /* Now define a bunch of things for manipulating the control register. 30 | The top two bytes of the control register consist of 4 fields of 4 31 | bits - each field corresponds to one of the four debug registers, 32 | and indicates what types of access we trap on, and how large the data 33 | field is that we are looking at */ 34 | 35 | #define DR_CONTROL_SHIFT 16 /* Skip this many bits in ctl register */ 36 | #define DR_CONTROL_SIZE 4 /* 4 control bits per register */ 37 | 38 | #define DR_RW_EXECUTE (0x0) /* Settings for the access types to trap on */ 39 | #define DR_RW_WRITE (0x1) 40 | #define DR_RW_READ (0x3) 41 | 42 | #define DR_LEN_1 (0x0) /* Settings for data length to trap on */ 43 | #define DR_LEN_2 (0x4) 44 | #define DR_LEN_4 (0xC) 45 | #define DR_LEN_8 (0x8) 46 | 47 | /* The low byte to the control register determine which registers are 48 | enabled. There are 4 fields of two bits. One bit is "local", meaning 49 | that the processor will reset the bit after a task switch and the other 50 | is global meaning that we have to explicitly reset the bit. With linux, 51 | you can use either one, since we explicitly zero the register when we enter 52 | kernel mode. */ 53 | 54 | #define DR_LOCAL_ENABLE_SHIFT 0 /* Extra shift to the local enable bit */ 55 | #define DR_GLOBAL_ENABLE_SHIFT 1 /* Extra shift to the global enable bit */ 56 | #define DR_LOCAL_ENABLE (0x1) /* Local enable for reg 0 */ 57 | #define DR_GLOBAL_ENABLE (0x2) /* Global enable for reg 0 */ 58 | #define DR_ENABLE_SIZE 2 /* 2 enable bits per register */ 59 | 60 | #define DR_LOCAL_ENABLE_MASK (0x55) /* Set local bits for all 4 regs */ 61 | #define DR_GLOBAL_ENABLE_MASK (0xAA) /* Set global bits for all 4 regs */ 62 | 63 | /* The second byte to the control register has a few special things. 64 | We can slow the instruction pipeline for instructions coming via the 65 | gdt or the ldt if we want to. I am not sure why this is an advantage */ 66 | 67 | #ifdef __i386__ 68 | #define DR_CONTROL_RESERVED (0xFC00) /* Reserved by Intel */ 69 | #else 70 | #define DR_CONTROL_RESERVED (0xFFFFFFFF0000FC00UL) /* Reserved */ 71 | #endif 72 | 73 | #define DR_LOCAL_SLOWDOWN (0x100) /* Local slow the pipeline */ 74 | #define DR_GLOBAL_SLOWDOWN (0x200) /* Global slow the pipeline */ 75 | 76 | /* 77 | * HW breakpoint additions 78 | */ 79 | 80 | #endif /* _UAPI_ASM_X86_DEBUGREG_H */ 81 | -------------------------------------------------------------------------------- /arch/x86/include/uapi/asm/kvm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef _ASM_X86_KVM_H 6 | #define _ASM_X86_KVM_H 7 | 8 | /* 9 | * kvm x86 specific structures and definitions 10 | * 11 | */ 12 | 13 | #include 14 | 15 | #define DE_VECTOR 0 16 | #define DB_VECTOR 1 17 | #define BP_VECTOR 3 18 | #define OF_VECTOR 4 19 | #define BR_VECTOR 5 20 | #define UD_VECTOR 6 21 | #define NM_VECTOR 7 22 | #define DF_VECTOR 8 23 | #define TS_VECTOR 10 24 | #define NP_VECTOR 11 25 | #define SS_VECTOR 12 26 | #define GP_VECTOR 13 27 | #define PF_VECTOR 14 28 | #define MF_VECTOR 16 29 | #define AC_VECTOR 17 30 | #define MC_VECTOR 18 31 | #define XM_VECTOR 19 32 | #define VE_VECTOR 20 33 | 34 | /* Select x86 specific features in */ 35 | #define __AEHD_HAVE_MSI 36 | #define __AEHD_HAVE_USER_NMI 37 | #define __AEHD_HAVE_GUEST_DEBUG 38 | #define __AEHD_HAVE_MSIX 39 | #define __AEHD_HAVE_VCPU_EVENTS 40 | #define __AEHD_HAVE_DEBUGREGS 41 | #define __AEHD_HAVE_XSAVE 42 | #define __AEHD_HAVE_XCRS 43 | #define __AEHD_HAVE_READONLY_MEM 44 | 45 | /* Architectural interrupt line count. */ 46 | #define AEHD_NR_INTERRUPTS 256 47 | 48 | struct kvm_memory_alias { 49 | __u32 slot; /* this has a different namespace than memory slots */ 50 | __u32 flags; 51 | __u64 guest_phys_addr; 52 | __u64 memory_size; 53 | __u64 target_phys_addr; 54 | }; 55 | 56 | /* for AEHD_GET_IRQCHIP and AEHD_SET_IRQCHIP */ 57 | struct kvm_pic_state { 58 | __u8 last_irr; /* edge detection */ 59 | __u8 irr; /* interrupt request register */ 60 | __u8 imr; /* interrupt mask register */ 61 | __u8 isr; /* interrupt service register */ 62 | __u8 priority_add; /* highest irq priority */ 63 | __u8 irq_base; 64 | __u8 read_reg_select; 65 | __u8 poll; 66 | __u8 special_mask; 67 | __u8 init_state; 68 | __u8 auto_eoi; 69 | __u8 rotate_on_auto_eoi; 70 | __u8 special_fully_nested_mode; 71 | __u8 init4; /* true if 4 byte init */ 72 | __u8 elcr; /* PIIX edge/trigger selection */ 73 | __u8 elcr_mask; 74 | }; 75 | 76 | #define AEHD_IOAPIC_NUM_PINS 24 77 | struct kvm_ioapic_state { 78 | __u64 base_address; 79 | __u32 ioregsel; 80 | __u32 id; 81 | __u32 irr; 82 | __u32 pad; 83 | union { 84 | __u64 bits; 85 | struct { 86 | __u8 vector; 87 | __u8 delivery_mode:3; 88 | __u8 dest_mode:1; 89 | __u8 delivery_status:1; 90 | __u8 polarity:1; 91 | __u8 remote_irr:1; 92 | __u8 trig_mode:1; 93 | __u8 mask:1; 94 | __u8 reserve:7; 95 | __u8 reserved[4]; 96 | __u8 dest_id; 97 | } fields; 98 | } redirtbl[AEHD_IOAPIC_NUM_PINS]; 99 | }; 100 | 101 | #define AEHD_IRQCHIP_PIC_MASTER 0 102 | #define AEHD_IRQCHIP_PIC_SLAVE 1 103 | #define AEHD_IRQCHIP_IOAPIC 2 104 | #define AEHD_NR_IRQCHIPS 3 105 | 106 | #define AEHD_RUN_X86_SMM (1 << 0) 107 | 108 | /* for AEHD_GET_REGS and AEHD_SET_REGS */ 109 | struct kvm_regs { 110 | /* out (AEHD_GET_REGS) / in (AEHD_SET_REGS) */ 111 | __u64 rax, rbx, rcx, rdx; 112 | __u64 rsi, rdi, rsp, rbp; 113 | __u64 r8, r9, r10, r11; 114 | __u64 r12, r13, r14, r15; 115 | __u64 rip, rflags; 116 | }; 117 | 118 | /* for AEHD_GET_LAPIC and AEHD_SET_LAPIC */ 119 | #define AEHD_APIC_REG_SIZE 0x400 120 | struct kvm_lapic_state { 121 | char regs[AEHD_APIC_REG_SIZE]; 122 | }; 123 | 124 | struct kvm_segment { 125 | __u64 base; 126 | __u32 limit; 127 | __u16 selector; 128 | __u8 type; 129 | __u8 present, dpl, db, s, l, g, avl; 130 | __u8 unusable; 131 | __u8 padding; 132 | }; 133 | 134 | struct kvm_dtable { 135 | __u64 base; 136 | __u16 limit; 137 | __u16 padding[3]; 138 | }; 139 | 140 | 141 | /* for AEHD_GET_SREGS and AEHD_SET_SREGS */ 142 | struct kvm_sregs { 143 | /* out (AEHD_GET_SREGS) / in (AEHD_SET_SREGS) */ 144 | struct kvm_segment cs, ds, es, fs, gs, ss; 145 | struct kvm_segment tr, ldt; 146 | struct kvm_dtable gdt, idt; 147 | __u64 cr0, cr2, cr3, cr4, cr8; 148 | __u64 efer; 149 | __u64 apic_base; 150 | __u64 interrupt_bitmap[(AEHD_NR_INTERRUPTS + 63) / 64]; 151 | }; 152 | 153 | /* for AEHD_GET_FPU and AEHD_SET_FPU */ 154 | struct kvm_fpu { 155 | __u8 fpr[8][16]; 156 | __u16 fcw; 157 | __u16 fsw; 158 | __u8 ftwx; /* in fxsave format */ 159 | __u8 pad1; 160 | __u16 last_opcode; 161 | __u64 last_ip; 162 | __u64 last_dp; 163 | __u8 xmm[16][16]; 164 | __u32 mxcsr; 165 | __u32 pad2; 166 | }; 167 | 168 | struct kvm_msr_entry { 169 | __u32 index; 170 | __u32 reserved; 171 | __u64 data; 172 | }; 173 | 174 | #pragma warning(disable : 4200) 175 | /* for AEHD_GET_MSRS and AEHD_SET_MSRS */ 176 | struct kvm_msrs { 177 | __u32 nmsrs; /* number of msrs in entries */ 178 | __u32 pad; 179 | 180 | struct kvm_msr_entry entries[0]; 181 | }; 182 | 183 | /* for AEHD_GET_MSR_INDEX_LIST */ 184 | struct kvm_msr_list { 185 | __u32 nmsrs; /* number of msrs in entries */ 186 | __u32 indices[0]; 187 | }; 188 | 189 | struct kvm_cpuid_entry { 190 | __u32 function; 191 | __u32 index; 192 | __u32 flags; 193 | __u32 eax; 194 | __u32 ebx; 195 | __u32 ecx; 196 | __u32 edx; 197 | __u32 padding[3]; 198 | }; 199 | 200 | #define AEHD_CPUID_FLAG_SIGNIFCANT_INDEX (1 << 0) 201 | #define AEHD_CPUID_FLAG_STATEFUL_FUNC (1 << 1) 202 | #define AEHD_CPUID_FLAG_STATE_READ_NEXT (1 << 2) 203 | 204 | /* for AEHD_SET_CPUID */ 205 | struct kvm_cpuid { 206 | __u32 nent; 207 | __u32 padding; 208 | struct kvm_cpuid_entry entries[0]; 209 | }; 210 | 211 | /* for AEHD_GET_PIT and AEHD_SET_PIT */ 212 | struct kvm_pit_channel_state { 213 | __u32 count; /* can be 65536 */ 214 | __u16 latched_count; 215 | __u8 count_latched; 216 | __u8 status_latched; 217 | __u8 status; 218 | __u8 read_state; 219 | __u8 write_state; 220 | __u8 write_latch; 221 | __u8 rw_mode; 222 | __u8 mode; 223 | __u8 bcd; 224 | __u8 gate; 225 | __s64 count_load_time; 226 | }; 227 | 228 | struct kvm_debug_exit_arch { 229 | __u32 exception; 230 | __u32 pad; 231 | __u64 pc; 232 | __u64 dr6; 233 | __u64 dr7; 234 | }; 235 | 236 | #define AEHD_GUESTDBG_USE_SW_BP 0x00010000 237 | #define AEHD_GUESTDBG_USE_HW_BP 0x00020000 238 | #define AEHD_GUESTDBG_INJECT_DB 0x00040000 239 | #define AEHD_GUESTDBG_INJECT_BP 0x00080000 240 | 241 | /* for AEHD_SET_GUEST_DEBUG */ 242 | struct kvm_guest_debug_arch { 243 | __u64 debugreg[8]; 244 | }; 245 | 246 | struct kvm_reinject_control { 247 | __u8 pit_reinject; 248 | __u8 reserved[31]; 249 | }; 250 | 251 | /* When set in flags, include corresponding fields on AEHD_SET_VCPU_EVENTS */ 252 | #define AEHD_VCPUEVENT_VALID_NMI_PENDING 0x00000001 253 | #define AEHD_VCPUEVENT_VALID_SIPI_VECTOR 0x00000002 254 | #define AEHD_VCPUEVENT_VALID_SHADOW 0x00000004 255 | #define AEHD_VCPUEVENT_VALID_SMM 0x00000008 256 | 257 | /* Interrupt shadow states */ 258 | #define AEHD_X86_SHADOW_INT_MOV_SS 0x01 259 | #define AEHD_X86_SHADOW_INT_STI 0x02 260 | 261 | /* for AEHD_GET/SET_VCPU_EVENTS */ 262 | struct kvm_vcpu_events { 263 | struct { 264 | __u8 injected; 265 | __u8 nr; 266 | __u8 has_error_code; 267 | __u8 pad; 268 | __u32 error_code; 269 | } exception; 270 | struct { 271 | __u8 injected; 272 | __u8 nr; 273 | __u8 soft; 274 | __u8 shadow; 275 | } interrupt; 276 | struct { 277 | __u8 injected; 278 | __u8 pending; 279 | __u8 masked; 280 | __u8 pad; 281 | } nmi; 282 | __u32 sipi_vector; 283 | __u32 flags; 284 | struct { 285 | __u8 smm; 286 | __u8 pending; 287 | __u8 smm_inside_nmi; 288 | __u8 latched_init; 289 | } smi; 290 | __u32 reserved[9]; 291 | }; 292 | 293 | /* for AEHD_GET/SET_DEBUGREGS */ 294 | struct kvm_debugregs { 295 | __u64 db[4]; 296 | __u64 dr6; 297 | __u64 dr7; 298 | __u64 flags; 299 | __u64 reserved[9]; 300 | }; 301 | 302 | /* for AEHD_CAP_XSAVE */ 303 | struct kvm_xsave { 304 | __u32 region[1024]; 305 | }; 306 | 307 | #define AEHD_MAX_XCRS 16 308 | 309 | struct kvm_xcr { 310 | __u32 xcr; 311 | __u32 reserved; 312 | __u64 value; 313 | }; 314 | 315 | struct kvm_xcrs { 316 | __u32 nr_xcrs; 317 | __u32 flags; 318 | struct kvm_xcr xcrs[AEHD_MAX_XCRS]; 319 | __u64 padding[16]; 320 | }; 321 | 322 | /* definition of registers in kvm_run */ 323 | struct kvm_sync_regs { 324 | u64 reg; 325 | }; 326 | 327 | #define AEHD_X86_QUIRK_LINT0_REENABLED (1 << 0) 328 | #define AEHD_X86_QUIRK_CD_NW_CLEARED (1 << 1) 329 | 330 | #endif /* _ASM_X86_KVM_H */ 331 | -------------------------------------------------------------------------------- /arch/x86/include/uapi/asm/processor-flags.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef _UAPI_ASM_X86_PROCESSOR_FLAGS_H 6 | #define _UAPI_ASM_X86_PROCESSOR_FLAGS_H 7 | /* Various flags defined: can be included from assembler. */ 8 | 9 | #ifdef CONFIG_X86_64 10 | #define _BITUL(a) ((1ULL) << a) 11 | #else 12 | #define _BITUL(a) ((1UL) << a) 13 | #endif 14 | #define _AC(X, Y) X##Y 15 | 16 | /* 17 | * EFLAGS bits 18 | */ 19 | #define X86_EFLAGS_CF_BIT 0 /* Carry Flag */ 20 | #define X86_EFLAGS_CF _BITUL(X86_EFLAGS_CF_BIT) 21 | #define X86_EFLAGS_FIXED_BIT 1 /* Bit 1 - always on */ 22 | #define X86_EFLAGS_FIXED _BITUL(X86_EFLAGS_FIXED_BIT) 23 | #define X86_EFLAGS_PF_BIT 2 /* Parity Flag */ 24 | #define X86_EFLAGS_PF _BITUL(X86_EFLAGS_PF_BIT) 25 | #define X86_EFLAGS_AF_BIT 4 /* Auxiliary carry Flag */ 26 | #define X86_EFLAGS_AF _BITUL(X86_EFLAGS_AF_BIT) 27 | #define X86_EFLAGS_ZF_BIT 6 /* Zero Flag */ 28 | #define X86_EFLAGS_ZF _BITUL(X86_EFLAGS_ZF_BIT) 29 | #define X86_EFLAGS_SF_BIT 7 /* Sign Flag */ 30 | #define X86_EFLAGS_SF _BITUL(X86_EFLAGS_SF_BIT) 31 | #define X86_EFLAGS_TF_BIT 8 /* Trap Flag */ 32 | #define X86_EFLAGS_TF _BITUL(X86_EFLAGS_TF_BIT) 33 | #define X86_EFLAGS_IF_BIT 9 /* Interrupt Flag */ 34 | #define X86_EFLAGS_IF _BITUL(X86_EFLAGS_IF_BIT) 35 | #define X86_EFLAGS_DF_BIT 10 /* Direction Flag */ 36 | #define X86_EFLAGS_DF _BITUL(X86_EFLAGS_DF_BIT) 37 | #define X86_EFLAGS_OF_BIT 11 /* Overflow Flag */ 38 | #define X86_EFLAGS_OF _BITUL(X86_EFLAGS_OF_BIT) 39 | #define X86_EFLAGS_IOPL_BIT 12 /* I/O Privilege Level (2 bits) */ 40 | #define X86_EFLAGS_IOPL (_AC(3,UL) << X86_EFLAGS_IOPL_BIT) 41 | #define X86_EFLAGS_NT_BIT 14 /* Nested Task */ 42 | #define X86_EFLAGS_NT _BITUL(X86_EFLAGS_NT_BIT) 43 | #define X86_EFLAGS_RF_BIT 16 /* Resume Flag */ 44 | #define X86_EFLAGS_RF _BITUL(X86_EFLAGS_RF_BIT) 45 | #define X86_EFLAGS_VM_BIT 17 /* Virtual Mode */ 46 | #define X86_EFLAGS_VM _BITUL(X86_EFLAGS_VM_BIT) 47 | #define X86_EFLAGS_AC_BIT 18 /* Alignment Check/Access Control */ 48 | #define X86_EFLAGS_AC _BITUL(X86_EFLAGS_AC_BIT) 49 | #define X86_EFLAGS_VIF_BIT 19 /* Virtual Interrupt Flag */ 50 | #define X86_EFLAGS_VIF _BITUL(X86_EFLAGS_VIF_BIT) 51 | #define X86_EFLAGS_VIP_BIT 20 /* Virtual Interrupt Pending */ 52 | #define X86_EFLAGS_VIP _BITUL(X86_EFLAGS_VIP_BIT) 53 | #define X86_EFLAGS_ID_BIT 21 /* CPUID detection */ 54 | #define X86_EFLAGS_ID _BITUL(X86_EFLAGS_ID_BIT) 55 | 56 | /* 57 | * Basic CPU control in CR0 58 | */ 59 | #define X86_CR0_PE_BIT 0 /* Protection Enable */ 60 | #define X86_CR0_PE _BITUL(X86_CR0_PE_BIT) 61 | #define X86_CR0_MP_BIT 1 /* Monitor Coprocessor */ 62 | #define X86_CR0_MP _BITUL(X86_CR0_MP_BIT) 63 | #define X86_CR0_EM_BIT 2 /* Emulation */ 64 | #define X86_CR0_EM _BITUL(X86_CR0_EM_BIT) 65 | #define X86_CR0_TS_BIT 3 /* Task Switched */ 66 | #define X86_CR0_TS _BITUL(X86_CR0_TS_BIT) 67 | #define X86_CR0_ET_BIT 4 /* Extension Type */ 68 | #define X86_CR0_ET _BITUL(X86_CR0_ET_BIT) 69 | #define X86_CR0_NE_BIT 5 /* Numeric Error */ 70 | #define X86_CR0_NE _BITUL(X86_CR0_NE_BIT) 71 | #define X86_CR0_WP_BIT 16 /* Write Protect */ 72 | #define X86_CR0_WP _BITUL(X86_CR0_WP_BIT) 73 | #define X86_CR0_AM_BIT 18 /* Alignment Mask */ 74 | #define X86_CR0_AM _BITUL(X86_CR0_AM_BIT) 75 | #define X86_CR0_NW_BIT 29 /* Not Write-through */ 76 | #define X86_CR0_NW _BITUL(X86_CR0_NW_BIT) 77 | #define X86_CR0_CD_BIT 30 /* Cache Disable */ 78 | #define X86_CR0_CD _BITUL(X86_CR0_CD_BIT) 79 | #define X86_CR0_PG_BIT 31 /* Paging */ 80 | #define X86_CR0_PG _BITUL(X86_CR0_PG_BIT) 81 | 82 | /* 83 | * Paging options in CR3 84 | */ 85 | #define X86_CR3_PWT_BIT 3 /* Page Write Through */ 86 | #define X86_CR3_PWT _BITUL(X86_CR3_PWT_BIT) 87 | #define X86_CR3_PCD_BIT 4 /* Page Cache Disable */ 88 | #define X86_CR3_PCD _BITUL(X86_CR3_PCD_BIT) 89 | #define X86_CR3_PCID_MASK _AC(0x00000fff,UL) /* PCID Mask */ 90 | 91 | /* 92 | * Intel CPU features in CR4 93 | */ 94 | #define X86_CR4_VME_BIT 0 /* enable vm86 extensions */ 95 | #define X86_CR4_VME _BITUL(X86_CR4_VME_BIT) 96 | #define X86_CR4_PVI_BIT 1 /* virtual interrupts flag enable */ 97 | #define X86_CR4_PVI _BITUL(X86_CR4_PVI_BIT) 98 | #define X86_CR4_TSD_BIT 2 /* disable time stamp at ipl 3 */ 99 | #define X86_CR4_TSD _BITUL(X86_CR4_TSD_BIT) 100 | #define X86_CR4_DE_BIT 3 /* enable debugging extensions */ 101 | #define X86_CR4_DE _BITUL(X86_CR4_DE_BIT) 102 | #define X86_CR4_PSE_BIT 4 /* enable page size extensions */ 103 | #define X86_CR4_PSE _BITUL(X86_CR4_PSE_BIT) 104 | #define X86_CR4_PAE_BIT 5 /* enable physical address extensions */ 105 | #define X86_CR4_PAE _BITUL(X86_CR4_PAE_BIT) 106 | #define X86_CR4_MCE_BIT 6 /* Machine check enable */ 107 | #define X86_CR4_MCE _BITUL(X86_CR4_MCE_BIT) 108 | #define X86_CR4_PGE_BIT 7 /* enable global pages */ 109 | #define X86_CR4_PGE _BITUL(X86_CR4_PGE_BIT) 110 | #define X86_CR4_PCE_BIT 8 /* enable performance counters at ipl 3 */ 111 | #define X86_CR4_PCE _BITUL(X86_CR4_PCE_BIT) 112 | #define X86_CR4_OSFXSR_BIT 9 /* enable fast FPU save and restore */ 113 | #define X86_CR4_OSFXSR _BITUL(X86_CR4_OSFXSR_BIT) 114 | #define X86_CR4_OSXMMEXCPT_BIT 10 /* enable unmasked SSE exceptions */ 115 | #define X86_CR4_OSXMMEXCPT _BITUL(X86_CR4_OSXMMEXCPT_BIT) 116 | #define X86_CR4_VMXE_BIT 13 /* enable VMX virtualization */ 117 | #define X86_CR4_VMXE _BITUL(X86_CR4_VMXE_BIT) 118 | #define X86_CR4_SMXE_BIT 14 /* enable safer mode (TXT) */ 119 | #define X86_CR4_SMXE _BITUL(X86_CR4_SMXE_BIT) 120 | #define X86_CR4_FSGSBASE_BIT 16 /* enable RDWRFSGS support */ 121 | #define X86_CR4_FSGSBASE _BITUL(X86_CR4_FSGSBASE_BIT) 122 | #define X86_CR4_PCIDE_BIT 17 /* enable PCID support */ 123 | #define X86_CR4_PCIDE _BITUL(X86_CR4_PCIDE_BIT) 124 | #define X86_CR4_OSXSAVE_BIT 18 /* enable xsave and xrestore */ 125 | #define X86_CR4_OSXSAVE _BITUL(X86_CR4_OSXSAVE_BIT) 126 | #define X86_CR4_SMEP_BIT 20 /* enable SMEP support */ 127 | #define X86_CR4_SMEP _BITUL(X86_CR4_SMEP_BIT) 128 | #define X86_CR4_SMAP_BIT 21 /* enable SMAP support */ 129 | #define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT) 130 | #define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */ 131 | #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) 132 | 133 | /* 134 | * x86-64 Task Priority Register, CR8 135 | */ 136 | #define X86_CR8_TPR _AC(0x0000000f,UL) /* task priority register */ 137 | 138 | /* 139 | * AMD and Transmeta use MSRs for configuration; see 140 | */ 141 | 142 | /* 143 | * NSC/Cyrix CPU configuration register indexes 144 | */ 145 | #define CX86_PCR0 0x20 146 | #define CX86_GCR 0xb8 147 | #define CX86_CCR0 0xc0 148 | #define CX86_CCR1 0xc1 149 | #define CX86_CCR2 0xc2 150 | #define CX86_CCR3 0xc3 151 | #define CX86_CCR4 0xe8 152 | #define CX86_CCR5 0xe9 153 | #define CX86_CCR6 0xea 154 | #define CX86_CCR7 0xeb 155 | #define CX86_PCR1 0xf0 156 | #define CX86_DIR0 0xfe 157 | #define CX86_DIR1 0xff 158 | #define CX86_ARR_BASE 0xc4 159 | #define CX86_RCR_BASE 0xdc 160 | 161 | 162 | #endif /* _UAPI_ASM_X86_PROCESSOR_FLAGS_H */ 163 | -------------------------------------------------------------------------------- /arch/x86/include/uapi/asm/svm.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ 2 | #ifndef _UAPI__SVM_H 3 | #define _UAPI__SVM_H 4 | 5 | #define SVM_EXIT_READ_CR0 0x000 6 | #define SVM_EXIT_READ_CR2 0x002 7 | #define SVM_EXIT_READ_CR3 0x003 8 | #define SVM_EXIT_READ_CR4 0x004 9 | #define SVM_EXIT_READ_CR8 0x008 10 | #define SVM_EXIT_WRITE_CR0 0x010 11 | #define SVM_EXIT_WRITE_CR2 0x012 12 | #define SVM_EXIT_WRITE_CR3 0x013 13 | #define SVM_EXIT_WRITE_CR4 0x014 14 | #define SVM_EXIT_WRITE_CR8 0x018 15 | #define SVM_EXIT_READ_DR0 0x020 16 | #define SVM_EXIT_READ_DR1 0x021 17 | #define SVM_EXIT_READ_DR2 0x022 18 | #define SVM_EXIT_READ_DR3 0x023 19 | #define SVM_EXIT_READ_DR4 0x024 20 | #define SVM_EXIT_READ_DR5 0x025 21 | #define SVM_EXIT_READ_DR6 0x026 22 | #define SVM_EXIT_READ_DR7 0x027 23 | #define SVM_EXIT_WRITE_DR0 0x030 24 | #define SVM_EXIT_WRITE_DR1 0x031 25 | #define SVM_EXIT_WRITE_DR2 0x032 26 | #define SVM_EXIT_WRITE_DR3 0x033 27 | #define SVM_EXIT_WRITE_DR4 0x034 28 | #define SVM_EXIT_WRITE_DR5 0x035 29 | #define SVM_EXIT_WRITE_DR6 0x036 30 | #define SVM_EXIT_WRITE_DR7 0x037 31 | #define SVM_EXIT_EXCP_BASE 0x040 32 | #define SVM_EXIT_INTR 0x060 33 | #define SVM_EXIT_NMI 0x061 34 | #define SVM_EXIT_SMI 0x062 35 | #define SVM_EXIT_INIT 0x063 36 | #define SVM_EXIT_VINTR 0x064 37 | #define SVM_EXIT_CR0_SEL_WRITE 0x065 38 | #define SVM_EXIT_IDTR_READ 0x066 39 | #define SVM_EXIT_GDTR_READ 0x067 40 | #define SVM_EXIT_LDTR_READ 0x068 41 | #define SVM_EXIT_TR_READ 0x069 42 | #define SVM_EXIT_IDTR_WRITE 0x06a 43 | #define SVM_EXIT_GDTR_WRITE 0x06b 44 | #define SVM_EXIT_LDTR_WRITE 0x06c 45 | #define SVM_EXIT_TR_WRITE 0x06d 46 | #define SVM_EXIT_RDTSC 0x06e 47 | #define SVM_EXIT_RDPMC 0x06f 48 | #define SVM_EXIT_PUSHF 0x070 49 | #define SVM_EXIT_POPF 0x071 50 | #define SVM_EXIT_CPUID 0x072 51 | #define SVM_EXIT_RSM 0x073 52 | #define SVM_EXIT_IRET 0x074 53 | #define SVM_EXIT_SWINT 0x075 54 | #define SVM_EXIT_INVD 0x076 55 | #define SVM_EXIT_PAUSE 0x077 56 | #define SVM_EXIT_HLT 0x078 57 | #define SVM_EXIT_INVLPG 0x079 58 | #define SVM_EXIT_INVLPGA 0x07a 59 | #define SVM_EXIT_IOIO 0x07b 60 | #define SVM_EXIT_MSR 0x07c 61 | #define SVM_EXIT_TASK_SWITCH 0x07d 62 | #define SVM_EXIT_FERR_FREEZE 0x07e 63 | #define SVM_EXIT_SHUTDOWN 0x07f 64 | #define SVM_EXIT_VMRUN 0x080 65 | #define SVM_EXIT_VMMCALL 0x081 66 | #define SVM_EXIT_VMLOAD 0x082 67 | #define SVM_EXIT_VMSAVE 0x083 68 | #define SVM_EXIT_STGI 0x084 69 | #define SVM_EXIT_CLGI 0x085 70 | #define SVM_EXIT_SKINIT 0x086 71 | #define SVM_EXIT_RDTSCP 0x087 72 | #define SVM_EXIT_ICEBP 0x088 73 | #define SVM_EXIT_WBINVD 0x089 74 | #define SVM_EXIT_MONITOR 0x08a 75 | #define SVM_EXIT_MWAIT 0x08b 76 | #define SVM_EXIT_MWAIT_COND 0x08c 77 | #define SVM_EXIT_XSETBV 0x08d 78 | #define SVM_EXIT_NPF 0x400 79 | #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 80 | #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 81 | 82 | #define SVM_EXIT_ERR -1 83 | 84 | #define SVM_EXIT_REASONS \ 85 | { SVM_EXIT_READ_CR0, "read_cr0" }, \ 86 | { SVM_EXIT_READ_CR2, "read_cr2" }, \ 87 | { SVM_EXIT_READ_CR3, "read_cr3" }, \ 88 | { SVM_EXIT_READ_CR4, "read_cr4" }, \ 89 | { SVM_EXIT_READ_CR8, "read_cr8" }, \ 90 | { SVM_EXIT_WRITE_CR0, "write_cr0" }, \ 91 | { SVM_EXIT_WRITE_CR2, "write_cr2" }, \ 92 | { SVM_EXIT_WRITE_CR3, "write_cr3" }, \ 93 | { SVM_EXIT_WRITE_CR4, "write_cr4" }, \ 94 | { SVM_EXIT_WRITE_CR8, "write_cr8" }, \ 95 | { SVM_EXIT_READ_DR0, "read_dr0" }, \ 96 | { SVM_EXIT_READ_DR1, "read_dr1" }, \ 97 | { SVM_EXIT_READ_DR2, "read_dr2" }, \ 98 | { SVM_EXIT_READ_DR3, "read_dr3" }, \ 99 | { SVM_EXIT_READ_DR4, "read_dr4" }, \ 100 | { SVM_EXIT_READ_DR5, "read_dr5" }, \ 101 | { SVM_EXIT_READ_DR6, "read_dr6" }, \ 102 | { SVM_EXIT_READ_DR7, "read_dr7" }, \ 103 | { SVM_EXIT_WRITE_DR0, "write_dr0" }, \ 104 | { SVM_EXIT_WRITE_DR1, "write_dr1" }, \ 105 | { SVM_EXIT_WRITE_DR2, "write_dr2" }, \ 106 | { SVM_EXIT_WRITE_DR3, "write_dr3" }, \ 107 | { SVM_EXIT_WRITE_DR4, "write_dr4" }, \ 108 | { SVM_EXIT_WRITE_DR5, "write_dr5" }, \ 109 | { SVM_EXIT_WRITE_DR6, "write_dr6" }, \ 110 | { SVM_EXIT_WRITE_DR7, "write_dr7" }, \ 111 | { SVM_EXIT_EXCP_BASE + DE_VECTOR, "DE excp" }, \ 112 | { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" }, \ 113 | { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" }, \ 114 | { SVM_EXIT_EXCP_BASE + OF_VECTOR, "OF excp" }, \ 115 | { SVM_EXIT_EXCP_BASE + BR_VECTOR, "BR excp" }, \ 116 | { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" }, \ 117 | { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" }, \ 118 | { SVM_EXIT_EXCP_BASE + DF_VECTOR, "DF excp" }, \ 119 | { SVM_EXIT_EXCP_BASE + TS_VECTOR, "TS excp" }, \ 120 | { SVM_EXIT_EXCP_BASE + NP_VECTOR, "NP excp" }, \ 121 | { SVM_EXIT_EXCP_BASE + SS_VECTOR, "SS excp" }, \ 122 | { SVM_EXIT_EXCP_BASE + GP_VECTOR, "GP excp" }, \ 123 | { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" }, \ 124 | { SVM_EXIT_EXCP_BASE + MF_VECTOR, "MF excp" }, \ 125 | { SVM_EXIT_EXCP_BASE + AC_VECTOR, "AC excp" }, \ 126 | { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" }, \ 127 | { SVM_EXIT_EXCP_BASE + XM_VECTOR, "XF excp" }, \ 128 | { SVM_EXIT_INTR, "interrupt" }, \ 129 | { SVM_EXIT_NMI, "nmi" }, \ 130 | { SVM_EXIT_SMI, "smi" }, \ 131 | { SVM_EXIT_INIT, "init" }, \ 132 | { SVM_EXIT_VINTR, "vintr" }, \ 133 | { SVM_EXIT_CR0_SEL_WRITE, "cr0_sel_write" }, \ 134 | { SVM_EXIT_IDTR_READ, "read_idtr" }, \ 135 | { SVM_EXIT_GDTR_READ, "read_gdtr" }, \ 136 | { SVM_EXIT_LDTR_READ, "read_ldtr" }, \ 137 | { SVM_EXIT_TR_READ, "read_rt" }, \ 138 | { SVM_EXIT_IDTR_WRITE, "write_idtr" }, \ 139 | { SVM_EXIT_GDTR_WRITE, "write_gdtr" }, \ 140 | { SVM_EXIT_LDTR_WRITE, "write_ldtr" }, \ 141 | { SVM_EXIT_TR_WRITE, "write_rt" }, \ 142 | { SVM_EXIT_RDTSC, "rdtsc" }, \ 143 | { SVM_EXIT_RDPMC, "rdpmc" }, \ 144 | { SVM_EXIT_PUSHF, "pushf" }, \ 145 | { SVM_EXIT_POPF, "popf" }, \ 146 | { SVM_EXIT_CPUID, "cpuid" }, \ 147 | { SVM_EXIT_RSM, "rsm" }, \ 148 | { SVM_EXIT_IRET, "iret" }, \ 149 | { SVM_EXIT_SWINT, "swint" }, \ 150 | { SVM_EXIT_INVD, "invd" }, \ 151 | { SVM_EXIT_PAUSE, "pause" }, \ 152 | { SVM_EXIT_HLT, "hlt" }, \ 153 | { SVM_EXIT_INVLPG, "invlpg" }, \ 154 | { SVM_EXIT_INVLPGA, "invlpga" }, \ 155 | { SVM_EXIT_IOIO, "io" }, \ 156 | { SVM_EXIT_MSR, "msr" }, \ 157 | { SVM_EXIT_TASK_SWITCH, "task_switch" }, \ 158 | { SVM_EXIT_FERR_FREEZE, "ferr_freeze" }, \ 159 | { SVM_EXIT_SHUTDOWN, "shutdown" }, \ 160 | { SVM_EXIT_VMRUN, "vmrun" }, \ 161 | { SVM_EXIT_VMMCALL, "hypercall" }, \ 162 | { SVM_EXIT_VMLOAD, "vmload" }, \ 163 | { SVM_EXIT_VMSAVE, "vmsave" }, \ 164 | { SVM_EXIT_STGI, "stgi" }, \ 165 | { SVM_EXIT_CLGI, "clgi" }, \ 166 | { SVM_EXIT_SKINIT, "skinit" }, \ 167 | { SVM_EXIT_RDTSCP, "rdtscp" }, \ 168 | { SVM_EXIT_ICEBP, "icebp" }, \ 169 | { SVM_EXIT_WBINVD, "wbinvd" }, \ 170 | { SVM_EXIT_MONITOR, "monitor" }, \ 171 | { SVM_EXIT_MWAIT, "mwait" }, \ 172 | { SVM_EXIT_XSETBV, "xsetbv" }, \ 173 | { SVM_EXIT_NPF, "npf" }, \ 174 | { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ 175 | { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ 176 | { SVM_EXIT_ERR, "invalid_guest_state" } 177 | 178 | 179 | #endif /* _UAPI__SVM_H */ 180 | -------------------------------------------------------------------------------- /arch/x86/include/uapi/asm/vmx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * vmx.h: VMX Architecture related definitions 3 | * Copyright (c) 2004, Intel Corporation. 4 | * 5 | * This program is free software; you can redistribute it and/or modify it 6 | * under the terms and conditions of the GNU General Public License, 7 | * version 2, as published by the Free Software Foundation. 8 | * 9 | * This program is distributed in the hope it will be useful, but WITHOUT 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 | * more details. 13 | * 14 | * You should have received a copy of the GNU General Public License along with 15 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 16 | * Place - Suite 330, Boston, MA 02111-1307 USA. 17 | * 18 | * A few random additions are: 19 | * Copyright (C) 2006 Qumranet 20 | * Avi Kivity 21 | * Yaniv Kamay 22 | * 23 | */ 24 | #ifndef _UAPIVMX_H 25 | #define _UAPIVMX_H 26 | 27 | 28 | #define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000 29 | 30 | #define EXIT_REASON_EXCEPTION_NMI 0 31 | #define EXIT_REASON_EXTERNAL_INTERRUPT 1 32 | #define EXIT_REASON_TRIPLE_FAULT 2 33 | 34 | #define EXIT_REASON_PENDING_INTERRUPT 7 35 | #define EXIT_REASON_NMI_WINDOW 8 36 | #define EXIT_REASON_TASK_SWITCH 9 37 | #define EXIT_REASON_CPUID 10 38 | #define EXIT_REASON_HLT 12 39 | #define EXIT_REASON_INVD 13 40 | #define EXIT_REASON_INVLPG 14 41 | #define EXIT_REASON_RDPMC 15 42 | #define EXIT_REASON_RDTSC 16 43 | #define EXIT_REASON_VMCALL 18 44 | #define EXIT_REASON_VMCLEAR 19 45 | #define EXIT_REASON_VMLAUNCH 20 46 | #define EXIT_REASON_VMPTRLD 21 47 | #define EXIT_REASON_VMPTRST 22 48 | #define EXIT_REASON_VMREAD 23 49 | #define EXIT_REASON_VMRESUME 24 50 | #define EXIT_REASON_VMWRITE 25 51 | #define EXIT_REASON_VMOFF 26 52 | #define EXIT_REASON_VMON 27 53 | #define EXIT_REASON_CR_ACCESS 28 54 | #define EXIT_REASON_DR_ACCESS 29 55 | #define EXIT_REASON_IO_INSTRUCTION 30 56 | #define EXIT_REASON_MSR_READ 31 57 | #define EXIT_REASON_MSR_WRITE 32 58 | #define EXIT_REASON_INVALID_STATE 33 59 | #define EXIT_REASON_MSR_LOAD_FAIL 34 60 | #define EXIT_REASON_MWAIT_INSTRUCTION 36 61 | #define EXIT_REASON_MONITOR_TRAP_FLAG 37 62 | #define EXIT_REASON_MONITOR_INSTRUCTION 39 63 | #define EXIT_REASON_PAUSE_INSTRUCTION 40 64 | #define EXIT_REASON_MCE_DURING_VMENTRY 41 65 | #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 66 | #define EXIT_REASON_APIC_ACCESS 44 67 | #define EXIT_REASON_EOI_INDUCED 45 68 | #define EXIT_REASON_EPT_VIOLATION 48 69 | #define EXIT_REASON_EPT_MISCONFIG 49 70 | #define EXIT_REASON_INVEPT 50 71 | #define EXIT_REASON_RDTSCP 51 72 | #define EXIT_REASON_INVVPID 53 73 | #define EXIT_REASON_WBINVD 54 74 | #define EXIT_REASON_XSETBV 55 75 | #define EXIT_REASON_APIC_WRITE 56 76 | #define EXIT_REASON_INVPCID 58 77 | #define EXIT_REASON_PML_FULL 62 78 | #define EXIT_REASON_XSAVES 63 79 | #define EXIT_REASON_XRSTORS 64 80 | 81 | #define VMX_EXIT_REASONS \ 82 | { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \ 83 | { EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \ 84 | { EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \ 85 | { EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \ 86 | { EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \ 87 | { EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \ 88 | { EXIT_REASON_CPUID, "CPUID" }, \ 89 | { EXIT_REASON_HLT, "HLT" }, \ 90 | { EXIT_REASON_INVLPG, "INVLPG" }, \ 91 | { EXIT_REASON_RDPMC, "RDPMC" }, \ 92 | { EXIT_REASON_RDTSC, "RDTSC" }, \ 93 | { EXIT_REASON_VMCALL, "VMCALL" }, \ 94 | { EXIT_REASON_VMCLEAR, "VMCLEAR" }, \ 95 | { EXIT_REASON_VMLAUNCH, "VMLAUNCH" }, \ 96 | { EXIT_REASON_VMPTRLD, "VMPTRLD" }, \ 97 | { EXIT_REASON_VMPTRST, "VMPTRST" }, \ 98 | { EXIT_REASON_VMREAD, "VMREAD" }, \ 99 | { EXIT_REASON_VMRESUME, "VMRESUME" }, \ 100 | { EXIT_REASON_VMWRITE, "VMWRITE" }, \ 101 | { EXIT_REASON_VMOFF, "VMOFF" }, \ 102 | { EXIT_REASON_VMON, "VMON" }, \ 103 | { EXIT_REASON_CR_ACCESS, "CR_ACCESS" }, \ 104 | { EXIT_REASON_DR_ACCESS, "DR_ACCESS" }, \ 105 | { EXIT_REASON_IO_INSTRUCTION, "IO_INSTRUCTION" }, \ 106 | { EXIT_REASON_MSR_READ, "MSR_READ" }, \ 107 | { EXIT_REASON_MSR_WRITE, "MSR_WRITE" }, \ 108 | { EXIT_REASON_MWAIT_INSTRUCTION, "MWAIT_INSTRUCTION" }, \ 109 | { EXIT_REASON_MONITOR_TRAP_FLAG, "MONITOR_TRAP_FLAG" }, \ 110 | { EXIT_REASON_MONITOR_INSTRUCTION, "MONITOR_INSTRUCTION" }, \ 111 | { EXIT_REASON_PAUSE_INSTRUCTION, "PAUSE_INSTRUCTION" }, \ 112 | { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \ 113 | { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \ 114 | { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \ 115 | { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \ 116 | { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \ 117 | { EXIT_REASON_INVEPT, "INVEPT" }, \ 118 | { EXIT_REASON_PREEMPTION_TIMER, "PREEMPTION_TIMER" }, \ 119 | { EXIT_REASON_WBINVD, "WBINVD" }, \ 120 | { EXIT_REASON_APIC_WRITE, "APIC_WRITE" }, \ 121 | { EXIT_REASON_EOI_INDUCED, "EOI_INDUCED" }, \ 122 | { EXIT_REASON_INVALID_STATE, "INVALID_STATE" }, \ 123 | { EXIT_REASON_MSR_LOAD_FAIL, "MSR_LOAD_FAIL" }, \ 124 | { EXIT_REASON_INVD, "INVD" }, \ 125 | { EXIT_REASON_INVVPID, "INVVPID" }, \ 126 | { EXIT_REASON_INVPCID, "INVPCID" }, \ 127 | { EXIT_REASON_XSAVES, "XSAVES" }, \ 128 | { EXIT_REASON_XRSTORS, "XRSTORS" } 129 | 130 | #define VMX_ABORT_SAVE_GUEST_MSR_FAIL 1 131 | #define VMX_ABORT_LOAD_HOST_MSR_FAIL 4 132 | 133 | #endif /* _UAPIVMX_H */ 134 | -------------------------------------------------------------------------------- /arch/x86/kvm/cpuid.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef ARCH_X86_KVM_CPUID_H 6 | #define ARCH_X86_KVM_CPUID_H 7 | 8 | #include "x86.h" 9 | #include 10 | #include 11 | #include 12 | 13 | int kvm_update_cpuid(struct kvm_vcpu *vcpu); 14 | bool kvm_mpx_supported(void); 15 | struct kvm_cpuid_entry *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, 16 | u32 function, u32 index); 17 | int kvm_dev_ioctl_get_cpuid(PIRP pIrp, struct kvm_cpuid *cpuid, 18 | struct kvm_cpuid_entry __user *entries, 19 | unsigned int type); 20 | int kvm_vcpu_ioctl_set_cpuid(PIRP Irp, struct kvm_vcpu *vcpu, 21 | struct kvm_cpuid *cpuid, 22 | struct kvm_cpuid_entry __user *entries); 23 | int kvm_vcpu_ioctl_get_cpuid(struct kvm_vcpu *vcpu, 24 | struct kvm_cpuid *cpuid, 25 | struct kvm_cpuid_entry __user *entries); 26 | void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx); 27 | 28 | int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); 29 | 30 | static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) 31 | { 32 | return vcpu->arch.maxphyaddr; 33 | } 34 | 35 | static inline bool guest_cpuid_has_xsave(struct kvm_vcpu *vcpu) 36 | { 37 | struct kvm_cpuid_entry *best; 38 | 39 | if (!static_cpu_has(X86_FEATURE_XSAVE)) 40 | return false; 41 | 42 | best = kvm_find_cpuid_entry(vcpu, 1, 0); 43 | return best && (best->ecx & bit(X86_FEATURE_XSAVE)); 44 | } 45 | 46 | static inline bool guest_cpuid_has_mtrr(struct kvm_vcpu *vcpu) 47 | { 48 | struct kvm_cpuid_entry *best; 49 | 50 | best = kvm_find_cpuid_entry(vcpu, 1, 0); 51 | return best && (best->edx & bit(X86_FEATURE_MTRR)); 52 | } 53 | 54 | static inline bool guest_cpuid_has_tsc_adjust(struct kvm_vcpu *vcpu) 55 | { 56 | struct kvm_cpuid_entry *best; 57 | 58 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 59 | return best && (best->ebx & bit(X86_FEATURE_TSC_ADJUST)); 60 | } 61 | 62 | static inline bool guest_cpuid_has_smep(struct kvm_vcpu *vcpu) 63 | { 64 | struct kvm_cpuid_entry *best; 65 | 66 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 67 | return best && (best->ebx & bit(X86_FEATURE_SMEP)); 68 | } 69 | 70 | static inline bool guest_cpuid_has_smap(struct kvm_vcpu *vcpu) 71 | { 72 | struct kvm_cpuid_entry *best; 73 | 74 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 75 | return best && (best->ebx & bit(X86_FEATURE_SMAP)); 76 | } 77 | 78 | static inline bool guest_cpuid_has_fsgsbase(struct kvm_vcpu *vcpu) 79 | { 80 | struct kvm_cpuid_entry *best; 81 | 82 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 83 | return best && (best->ebx & bit(X86_FEATURE_FSGSBASE)); 84 | } 85 | 86 | static inline bool guest_cpuid_has_pku(struct kvm_vcpu *vcpu) 87 | { 88 | struct kvm_cpuid_entry *best; 89 | 90 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 91 | return best && (best->ecx & bit(X86_FEATURE_PKU)); 92 | } 93 | 94 | static inline bool guest_cpuid_has_longmode(struct kvm_vcpu *vcpu) 95 | { 96 | struct kvm_cpuid_entry *best; 97 | 98 | best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 99 | return best && (best->edx & bit(X86_FEATURE_LM)); 100 | } 101 | 102 | static inline bool guest_cpuid_has_osvw(struct kvm_vcpu *vcpu) 103 | { 104 | struct kvm_cpuid_entry *best; 105 | 106 | best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 107 | return best && (best->ecx & bit(X86_FEATURE_OSVW)); 108 | } 109 | 110 | static inline bool guest_cpuid_has_pcid(struct kvm_vcpu *vcpu) 111 | { 112 | struct kvm_cpuid_entry *best; 113 | 114 | best = kvm_find_cpuid_entry(vcpu, 1, 0); 115 | return best && (best->ecx & bit(X86_FEATURE_PCID)); 116 | } 117 | 118 | static inline bool guest_cpuid_has_x2apic(struct kvm_vcpu *vcpu) 119 | { 120 | struct kvm_cpuid_entry *best; 121 | 122 | best = kvm_find_cpuid_entry(vcpu, 1, 0); 123 | return best && (best->ecx & bit(X86_FEATURE_X2APIC)); 124 | } 125 | 126 | static inline bool guest_cpuid_is_amd(struct kvm_vcpu *vcpu) 127 | { 128 | struct kvm_cpuid_entry *best; 129 | 130 | best = kvm_find_cpuid_entry(vcpu, 0, 0); 131 | return best && best->ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx; 132 | } 133 | 134 | static inline bool guest_cpuid_has_gbpages(struct kvm_vcpu *vcpu) 135 | { 136 | struct kvm_cpuid_entry *best; 137 | 138 | best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 139 | return best && (best->edx & bit(X86_FEATURE_GBPAGES)); 140 | } 141 | 142 | static inline bool guest_cpuid_has_rtm(struct kvm_vcpu *vcpu) 143 | { 144 | struct kvm_cpuid_entry *best; 145 | 146 | best = kvm_find_cpuid_entry(vcpu, 7, 0); 147 | return best && (best->ebx & bit(X86_FEATURE_RTM)); 148 | } 149 | 150 | static inline bool guest_cpuid_has_rdtscp(struct kvm_vcpu *vcpu) 151 | { 152 | struct kvm_cpuid_entry *best; 153 | 154 | best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 155 | return best && (best->edx & bit(X86_FEATURE_RDTSCP)); 156 | } 157 | 158 | /* 159 | * NRIPS is provided through cpuidfn 0x8000000a.edx bit 3 160 | */ 161 | #define BIT_NRIPS 3 162 | 163 | static inline bool guest_cpuid_has_nrips(struct kvm_vcpu *vcpu) 164 | { 165 | struct kvm_cpuid_entry *best; 166 | 167 | best = kvm_find_cpuid_entry(vcpu, 0x8000000a, 0); 168 | 169 | /* 170 | * NRIPS is a scattered cpuid feature, so we can't use 171 | * X86_FEATURE_NRIPS here (X86_FEATURE_NRIPS would be bit 172 | * position 8, not 3). 173 | */ 174 | return best && (best->edx & bit(BIT_NRIPS)); 175 | } 176 | #undef BIT_NRIPS 177 | 178 | static inline int guest_cpuid_family(struct kvm_vcpu *vcpu) 179 | { 180 | struct kvm_cpuid_entry *best; 181 | 182 | best = kvm_find_cpuid_entry(vcpu, 0x1, 0); 183 | if (!best) 184 | return -1; 185 | 186 | return x86_family(best->eax); 187 | } 188 | 189 | static inline int guest_cpuid_model(struct kvm_vcpu *vcpu) 190 | { 191 | struct kvm_cpuid_entry *best; 192 | 193 | best = kvm_find_cpuid_entry(vcpu, 0x1, 0); 194 | if (!best) 195 | return -1; 196 | 197 | return x86_model(best->eax); 198 | } 199 | 200 | static inline int guest_cpuid_stepping(struct kvm_vcpu *vcpu) 201 | { 202 | struct kvm_cpuid_entry *best; 203 | 204 | best = kvm_find_cpuid_entry(vcpu, 0x1, 0); 205 | if (!best) 206 | return -1; 207 | 208 | return x86_stepping(best->eax); 209 | } 210 | 211 | #endif 212 | -------------------------------------------------------------------------------- /arch/x86/kvm/ioapic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef __KVM_IO_APIC_H 6 | #define __KVM_IO_APIC_H 7 | 8 | #include 9 | 10 | #include 11 | 12 | #include 13 | 14 | struct kvm; 15 | struct kvm_vcpu; 16 | 17 | #define IOAPIC_NUM_PINS AEHD_IOAPIC_NUM_PINS 18 | #define MAX_NR_RESERVED_IOAPIC_PINS AEHD_MAX_IRQ_ROUTES 19 | #define IOAPIC_VERSION_ID 0x11 /* IOAPIC version */ 20 | #define IOAPIC_EDGE_TRIG 0 21 | #define IOAPIC_LEVEL_TRIG 1 22 | 23 | #define IOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000 24 | #define IOAPIC_MEM_LENGTH 0x100 25 | 26 | /* Direct registers. */ 27 | #define IOAPIC_REG_SELECT 0x00 28 | #define IOAPIC_REG_WINDOW 0x10 29 | 30 | /* Indirect registers. */ 31 | #define IOAPIC_REG_APIC_ID 0x00 /* x86 IOAPIC only */ 32 | #define IOAPIC_REG_VERSION 0x01 33 | #define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ 34 | 35 | /*ioapic delivery mode*/ 36 | #define IOAPIC_FIXED 0x0 37 | #define IOAPIC_LOWEST_PRIORITY 0x1 38 | #define IOAPIC_PMI 0x2 39 | #define IOAPIC_NMI 0x4 40 | #define IOAPIC_INIT 0x5 41 | #define IOAPIC_EXTINT 0x7 42 | 43 | #define RTC_GSI 8 44 | 45 | struct dest_map { 46 | /* vcpu bitmap where IRQ has been sent */ 47 | DECLARE_BITMAP(map, AEHD_MAX_VCPU_ID); 48 | 49 | /* 50 | * Vector sent to a given vcpu, only valid when 51 | * the vcpu's bit in map is set 52 | */ 53 | u8 vectors[AEHD_MAX_VCPU_ID]; 54 | }; 55 | 56 | 57 | struct rtc_status { 58 | int pending_eoi; 59 | struct dest_map dest_map; 60 | }; 61 | 62 | union kvm_ioapic_redirect_entry { 63 | u64 bits; 64 | struct { 65 | u8 vector; 66 | u8 delivery_mode:3; 67 | u8 dest_mode:1; 68 | u8 delivery_status:1; 69 | u8 polarity:1; 70 | u8 remote_irr:1; 71 | u8 trig_mode:1; 72 | u8 mask:1; 73 | u8 reserve:7; 74 | u8 reserved[4]; 75 | u8 dest_id; 76 | } fields; 77 | }; 78 | 79 | struct kvm_ioapic { 80 | u64 base_address; 81 | u32 ioregsel; 82 | u32 id; 83 | u32 irr; 84 | u32 pad; 85 | union kvm_ioapic_redirect_entry redirtbl[IOAPIC_NUM_PINS]; 86 | size_t irq_states[IOAPIC_NUM_PINS]; 87 | struct kvm_io_device dev; 88 | struct kvm *kvm; 89 | void (*ack_notifier)(void *opaque, int irq); 90 | spinlock_t lock; 91 | struct rtc_status rtc_status; 92 | u32 irq_eoi[IOAPIC_NUM_PINS]; 93 | u32 irr_delivered; 94 | }; 95 | 96 | static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) 97 | { 98 | return kvm->arch.vioapic; 99 | } 100 | 101 | static inline int ioapic_in_kernel(struct kvm *kvm) 102 | { 103 | int ret; 104 | 105 | ret = (ioapic_irqchip(kvm) != NULL); 106 | return ret; 107 | } 108 | 109 | void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu); 110 | bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 111 | int short_hand, unsigned int dest, int dest_mode); 112 | int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2); 113 | void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, 114 | int trigger_mode); 115 | int kvm_ioapic_init(struct kvm *kvm); 116 | void kvm_ioapic_destroy(struct kvm *kvm); 117 | int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id, 118 | int level, bool line_status); 119 | void kvm_ioapic_clear_all(struct kvm_ioapic *ioapic, int irq_source_id); 120 | int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, 121 | struct kvm_lapic_irq *irq, 122 | struct dest_map *dest_map); 123 | int kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state); 124 | int kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state); 125 | void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu, 126 | ulong *ioapic_handled_vectors); 127 | void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, 128 | ulong *ioapic_handled_vectors); 129 | #endif 130 | -------------------------------------------------------------------------------- /arch/x86/kvm/irq.c: -------------------------------------------------------------------------------- 1 | /* 2 | * irq.c: API for in kernel interrupt controller 3 | * Copyright (c) 2007, Intel Corporation. 4 | * Copyright 2009 Red Hat, Inc. and/or its affiliates. 5 | * Copyright 2019 Google LLC 6 | * 7 | * This program is free software; you can redistribute it and/or modify it 8 | * under the terms and conditions of the GNU General Public License, 9 | * version 2, as published by the Free Software Foundation. 10 | * 11 | * This program is distributed in the hope it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 | * more details. 15 | * 16 | * You should have received a copy of the GNU General Public License along with 17 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 18 | * Place - Suite 330, Boston, MA 02111-1307 USA. 19 | * Authors: 20 | * Yaozu (Eddie) Dong 21 | * 22 | */ 23 | 24 | #include 25 | 26 | #include "irq.h" 27 | #include "x86.h" 28 | 29 | /* 30 | * check if there are pending timer events 31 | * to be processed. 32 | */ 33 | int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 34 | { 35 | if (lapic_in_kernel(vcpu)) 36 | return apic_has_pending_timer(vcpu); 37 | 38 | return 0; 39 | } 40 | 41 | /* 42 | * check if there is a pending userspace external interrupt 43 | */ 44 | static int pending_userspace_extint(struct kvm_vcpu *v) 45 | { 46 | return v->arch.pending_external_vector != -1; 47 | } 48 | 49 | /* 50 | * check if there is pending interrupt from 51 | * non-APIC source without intack. 52 | */ 53 | static int kvm_cpu_has_extint(struct kvm_vcpu *v) 54 | { 55 | u8 accept = kvm_apic_accept_pic_intr(v); 56 | 57 | if (accept) { 58 | return pic_irqchip(v->kvm)->output; 59 | } else 60 | return 0; 61 | } 62 | 63 | /* 64 | * check if there is injectable interrupt: 65 | * when virtual interrupt delivery enabled, 66 | * interrupt from apic will handled by hardware, 67 | * we don't need to check it here. 68 | */ 69 | int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v) 70 | { 71 | if (!lapic_in_kernel(v)) 72 | return v->arch.interrupt.pending; 73 | 74 | if (kvm_cpu_has_extint(v)) 75 | return 1; 76 | 77 | if (kvm_vcpu_apicv_active(v)) 78 | return 0; 79 | 80 | return kvm_apic_has_interrupt(v) != -1; /* LAPIC */ 81 | } 82 | 83 | /* 84 | * check if there is pending interrupt without 85 | * intack. 86 | */ 87 | int kvm_cpu_has_interrupt(struct kvm_vcpu *v) 88 | { 89 | if (!lapic_in_kernel(v)) 90 | return v->arch.interrupt.pending; 91 | 92 | if (kvm_cpu_has_extint(v)) 93 | return 1; 94 | 95 | return kvm_apic_has_interrupt(v) != -1; /* LAPIC */ 96 | } 97 | 98 | /* 99 | * Read pending interrupt(from non-APIC source) 100 | * vector and intack. 101 | */ 102 | static int kvm_cpu_get_extint(struct kvm_vcpu *v) 103 | { 104 | if (kvm_cpu_has_extint(v)) { 105 | return kvm_pic_read_irq(v->kvm); /* PIC */ 106 | } else 107 | return -1; 108 | } 109 | 110 | /* 111 | * Read pending interrupt vector and intack. 112 | */ 113 | int kvm_cpu_get_interrupt(struct kvm_vcpu *v) 114 | { 115 | int vector; 116 | 117 | if (!lapic_in_kernel(v)) 118 | return v->arch.interrupt.nr; 119 | 120 | vector = kvm_cpu_get_extint(v); 121 | 122 | if (vector != -1) 123 | return vector; /* PIC */ 124 | 125 | return kvm_get_apic_interrupt(v); /* APIC */ 126 | } 127 | 128 | void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu) 129 | { 130 | if (lapic_in_kernel(vcpu)) 131 | kvm_inject_apic_timer_irqs(vcpu); 132 | } 133 | -------------------------------------------------------------------------------- /arch/x86/kvm/irq.h: -------------------------------------------------------------------------------- 1 | /* 2 | * irq.h: in kernel interrupt controller related definitions 3 | * Copyright (c) 2007, Intel Corporation. 4 | * Copyright 2019 Google LLC 5 | * 6 | * This program is free software; you can redistribute it and/or modify it 7 | * under the terms and conditions of the GNU General Public License, 8 | * version 2, as published by the Free Software Foundation. 9 | * 10 | * This program is distributed in the hope it will be useful, but WITHOUT 11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 | * more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along with 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 17 | * Place - Suite 330, Boston, MA 02111-1307 USA. 18 | * Authors: 19 | * Yaozu (Eddie) Dong 20 | * 21 | */ 22 | 23 | #ifndef __IRQ_H 24 | #define __IRQ_H 25 | 26 | #include 27 | 28 | #include 29 | #include "ioapic.h" 30 | #include "lapic.h" 31 | 32 | #define PIC_NUM_PINS 16 33 | #define SELECT_PIC(irq) \ 34 | ((irq) < 8 ? AEHD_IRQCHIP_PIC_MASTER : AEHD_IRQCHIP_PIC_SLAVE) 35 | 36 | struct kvm; 37 | struct kvm_vcpu; 38 | 39 | struct kvm_kpic_state { 40 | u8 last_irr; /* edge detection */ 41 | u8 irr; /* interrupt request register */ 42 | u8 imr; /* interrupt mask register */ 43 | u8 isr; /* interrupt service register */ 44 | u8 priority_add; /* highest irq priority */ 45 | u8 irq_base; 46 | u8 read_reg_select; 47 | u8 poll; 48 | u8 special_mask; 49 | u8 init_state; 50 | u8 auto_eoi; 51 | u8 rotate_on_auto_eoi; 52 | u8 special_fully_nested_mode; 53 | u8 init4; /* true if 4 byte init */ 54 | u8 elcr; /* PIIX edge/trigger selection */ 55 | u8 elcr_mask; 56 | u8 isr_ack; /* interrupt ack detection */ 57 | struct kvm_pic *pics_state; 58 | }; 59 | 60 | struct kvm_pic { 61 | spinlock_t lock; 62 | bool wakeup_needed; 63 | unsigned pending_acks; 64 | struct kvm *kvm; 65 | struct kvm_kpic_state pics[2]; /* 0 is master pic, 1 is slave pic */ 66 | int output; /* intr from master PIC */ 67 | struct kvm_io_device dev_master; 68 | struct kvm_io_device dev_slave; 69 | struct kvm_io_device dev_eclr; 70 | void (*ack_notifier)(void *opaque, int irq); 71 | size_t irq_states[PIC_NUM_PINS]; 72 | }; 73 | 74 | struct kvm_pic *kvm_create_pic(struct kvm *kvm); 75 | void kvm_destroy_pic(struct kvm_pic *vpic); 76 | int kvm_pic_read_irq(struct kvm *kvm); 77 | void kvm_pic_update_irq(struct kvm_pic *s); 78 | 79 | static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) 80 | { 81 | return kvm->arch.vpic; 82 | } 83 | 84 | static inline int pic_in_kernel(struct kvm *kvm) 85 | { 86 | int ret; 87 | 88 | ret = (pic_irqchip(kvm) != NULL); 89 | return ret; 90 | } 91 | 92 | static inline int irqchip_in_kernel(struct kvm *kvm) 93 | { 94 | struct kvm_pic *vpic = pic_irqchip(kvm); 95 | bool ret; 96 | 97 | ret = (vpic != NULL); 98 | 99 | /* Read vpic before kvm->irq_routing. */ 100 | smp_rmb(); 101 | return ret; 102 | } 103 | 104 | void kvm_pic_reset(struct kvm_kpic_state *s); 105 | 106 | void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); 107 | void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); 108 | void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu); 109 | 110 | int apic_has_pending_timer(struct kvm_vcpu *vcpu); 111 | 112 | int kvm_setup_default_irq_routing(struct kvm *kvm); 113 | int kvm_setup_empty_irq_routing(struct kvm *kvm); 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /arch/x86/kvm/irq_comm.c: -------------------------------------------------------------------------------- 1 | /* 2 | * irq_comm.c: Common API for in kernel interrupt controller 3 | * Copyright (c) 2007, Intel Corporation. 4 | * Copyright 2019 Google LLC 5 | * 6 | * This program is free software; you can redistribute it and/or modify it 7 | * under the terms and conditions of the GNU General Public License, 8 | * version 2, as published by the Free Software Foundation. 9 | * 10 | * This program is distributed in the hope it will be useful, but WITHOUT 11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 | * more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along with 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 17 | * Place - Suite 330, Boston, MA 02111-1307 USA. 18 | * Authors: 19 | * Yaozu (Eddie) Dong 20 | * 21 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. 22 | */ 23 | 24 | #include 25 | #include 26 | #include "irq.h" 27 | 28 | #include "ioapic.h" 29 | #include "lapic.h" 30 | #include "x86.h" 31 | 32 | #include 33 | 34 | static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, 35 | struct kvm *kvm, int irq_source_id, int level, 36 | bool line_status) 37 | { 38 | struct kvm_pic *pic = pic_irqchip(kvm); 39 | 40 | /* 41 | * XXX: rejecting pic routes when pic isn't in use would be better, 42 | * but the default routing table is installed while kvm->arch.vpic is 43 | * NULL and AEHD_CREATE_IRQCHIP can race with AEHD_IRQ_LINE. 44 | */ 45 | if (!pic) 46 | return -1; 47 | 48 | return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); 49 | } 50 | 51 | static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, 52 | struct kvm *kvm, int irq_source_id, int level, 53 | bool line_status) 54 | { 55 | struct kvm_ioapic *ioapic = kvm->arch.vioapic; 56 | 57 | if (!ioapic) 58 | return -1; 59 | 60 | return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, 61 | line_status); 62 | } 63 | 64 | int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, 65 | struct kvm_lapic_irq *irq, struct dest_map *dest_map) 66 | { 67 | int i, r = -1; 68 | struct kvm_vcpu *vcpu, *lowest = NULL; 69 | size_t dest_vcpu_bitmap[BITS_TO_LONGS(AEHD_MAX_VCPUS)]; 70 | unsigned int dest_vcpus = 0; 71 | 72 | if (irq->dest_mode == 0 && irq->dest_id == 0xff && 73 | kvm_lowest_prio_delivery(irq)) { 74 | printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n"); 75 | irq->delivery_mode = APIC_DM_FIXED; 76 | } 77 | 78 | if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) 79 | return r; 80 | 81 | memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap)); 82 | 83 | kvm_for_each_vcpu(i, vcpu, kvm) { 84 | if (!kvm_apic_present(vcpu)) 85 | continue; 86 | 87 | if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, 88 | irq->dest_id, irq->dest_mode)) 89 | continue; 90 | 91 | if (!kvm_lowest_prio_delivery(irq)) { 92 | if (r < 0) 93 | r = 0; 94 | r += kvm_apic_set_irq(vcpu, irq, dest_map); 95 | } else if (kvm_lapic_enabled(vcpu)) { 96 | if (!kvm_vector_hashing_enabled()) { 97 | if (!lowest) 98 | lowest = vcpu; 99 | else if (kvm_apic_compare_prio(vcpu, lowest) < 0) 100 | lowest = vcpu; 101 | } else { 102 | __set_bit(i, dest_vcpu_bitmap); 103 | dest_vcpus++; 104 | } 105 | } 106 | } 107 | 108 | if (dest_vcpus != 0) { 109 | int idx = kvm_vector_to_index(irq->vector, dest_vcpus, 110 | dest_vcpu_bitmap, AEHD_MAX_VCPUS); 111 | 112 | lowest = kvm_get_vcpu(kvm, idx); 113 | } 114 | 115 | if (lowest) 116 | r = kvm_apic_set_irq(lowest, irq, dest_map); 117 | 118 | return r; 119 | } 120 | 121 | void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 122 | struct kvm_lapic_irq *irq) 123 | { 124 | irq->dest_id = (e->msi.address_lo & 125 | MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT; 126 | if (kvm->arch.x2apic_format) 127 | irq->dest_id |= MSI_ADDR_EXT_DEST_ID(e->msi.address_hi); 128 | irq->vector = (e->msi.data & 129 | MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT; 130 | irq->dest_mode = (1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo; 131 | irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data; 132 | irq->delivery_mode = e->msi.data & 0x700; 133 | irq->msi_redir_hint = ((e->msi.address_lo 134 | & MSI_ADDR_REDIRECTION_LOWPRI) > 0); 135 | irq->level = 1; 136 | irq->shorthand = 0; 137 | } 138 | 139 | static inline bool kvm_msi_route_invalid(struct kvm *kvm, 140 | struct kvm_kernel_irq_routing_entry *e) 141 | { 142 | return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff); 143 | } 144 | 145 | int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, 146 | struct kvm *kvm, int irq_source_id, int level, bool line_status) 147 | { 148 | struct kvm_lapic_irq irq; 149 | 150 | if (kvm_msi_route_invalid(kvm, e)) 151 | return -EINVAL; 152 | 153 | if (!level) 154 | return -1; 155 | 156 | kvm_set_msi_irq(kvm, e, &irq); 157 | 158 | return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); 159 | } 160 | 161 | 162 | int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 163 | struct kvm *kvm, int irq_source_id, int level, 164 | bool line_status) 165 | { 166 | struct kvm_lapic_irq irq; 167 | int r; 168 | 169 | switch (e->type) { 170 | case AEHD_IRQ_ROUTING_MSI: 171 | if (kvm_msi_route_invalid(kvm, e)) 172 | return -EINVAL; 173 | 174 | kvm_set_msi_irq(kvm, e, &irq); 175 | 176 | if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL)) 177 | return r; 178 | break; 179 | 180 | default: 181 | break; 182 | } 183 | 184 | return -EWOULDBLOCK; 185 | } 186 | 187 | int kvm_request_irq_source_id(struct kvm *kvm) 188 | { 189 | size_t *bitmap = &kvm->arch.irq_sources_bitmap; 190 | int irq_source_id; 191 | 192 | mutex_lock(&kvm->irq_lock); 193 | irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); 194 | 195 | if (irq_source_id >= BITS_PER_LONG) { 196 | printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n"); 197 | irq_source_id = -EFAULT; 198 | goto unlock; 199 | } 200 | 201 | ASSERT(irq_source_id != AEHD_USERSPACE_IRQ_SOURCE_ID); 202 | set_bit(irq_source_id, bitmap); 203 | unlock: 204 | mutex_unlock(&kvm->irq_lock); 205 | 206 | return irq_source_id; 207 | } 208 | 209 | void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) 210 | { 211 | ASSERT(irq_source_id != AEHD_USERSPACE_IRQ_SOURCE_ID); 212 | 213 | mutex_lock(&kvm->irq_lock); 214 | if (irq_source_id < 0 || 215 | irq_source_id >= BITS_PER_LONG) { 216 | printk(KERN_ERR "kvm: IRQ source ID out of range!\n"); 217 | goto unlock; 218 | } 219 | clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); 220 | if (!ioapic_in_kernel(kvm)) 221 | goto unlock; 222 | 223 | kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id); 224 | kvm_pic_clear_all(pic_irqchip(kvm), irq_source_id); 225 | unlock: 226 | mutex_unlock(&kvm->irq_lock); 227 | } 228 | 229 | void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, 230 | struct kvm_irq_mask_notifier *kimn) 231 | { 232 | mutex_lock(&kvm->irq_lock); 233 | kimn->irq = irq; 234 | hlist_add_head(&kimn->link, &kvm->arch.mask_notifier_list); 235 | mutex_unlock(&kvm->irq_lock); 236 | } 237 | 238 | void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, 239 | struct kvm_irq_mask_notifier *kimn) 240 | { 241 | mutex_lock(&kvm->irq_lock); 242 | hlist_del(&kimn->link); 243 | mutex_unlock(&kvm->irq_lock); 244 | } 245 | 246 | void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, 247 | bool mask) 248 | { 249 | struct kvm_irq_mask_notifier *kimn; 250 | int gsi; 251 | 252 | mutex_lock(&kvm->irq_lock); 253 | gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); 254 | if (gsi != -1) 255 | #define LIST_ENTRY_TYPE_INFO struct kvm_irq_mask_notifier 256 | hlist_for_each_entry(kimn, &kvm->arch.mask_notifier_list, link) 257 | if (kimn->irq == gsi) 258 | kimn->func(kimn, mask); 259 | #undef LIST_ENTRY_TYPE_INFO 260 | mutex_unlock(&kvm->irq_lock); 261 | } 262 | 263 | int kvm_set_routing_entry(struct kvm *kvm, 264 | struct kvm_kernel_irq_routing_entry *e, 265 | const struct kvm_irq_routing_entry *ue) 266 | { 267 | int r = -EINVAL; 268 | int delta; 269 | unsigned max_pin; 270 | 271 | switch (ue->type) { 272 | case AEHD_IRQ_ROUTING_IRQCHIP: 273 | delta = 0; 274 | switch (ue->u.irqchip.irqchip) { 275 | case AEHD_IRQCHIP_PIC_MASTER: 276 | e->set = kvm_set_pic_irq; 277 | max_pin = PIC_NUM_PINS; 278 | break; 279 | case AEHD_IRQCHIP_PIC_SLAVE: 280 | e->set = kvm_set_pic_irq; 281 | max_pin = PIC_NUM_PINS; 282 | delta = 8; 283 | break; 284 | case AEHD_IRQCHIP_IOAPIC: 285 | max_pin = AEHD_IOAPIC_NUM_PINS; 286 | e->set = kvm_set_ioapic_irq; 287 | break; 288 | default: 289 | goto out; 290 | } 291 | e->irqchip.irqchip = ue->u.irqchip.irqchip; 292 | e->irqchip.pin = ue->u.irqchip.pin + delta; 293 | if (e->irqchip.pin >= max_pin) 294 | goto out; 295 | break; 296 | case AEHD_IRQ_ROUTING_MSI: 297 | e->set = kvm_set_msi; 298 | e->msi.address_lo = ue->u.msi.address_lo; 299 | e->msi.address_hi = ue->u.msi.address_hi; 300 | e->msi.data = ue->u.msi.data; 301 | 302 | if (kvm_msi_route_invalid(kvm, e)) 303 | goto out; 304 | break; 305 | default: 306 | goto out; 307 | } 308 | 309 | r = 0; 310 | out: 311 | return r; 312 | } 313 | 314 | bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, 315 | struct kvm_vcpu **dest_vcpu) 316 | { 317 | int i, r = 0; 318 | struct kvm_vcpu *vcpu; 319 | 320 | if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) 321 | return true; 322 | 323 | kvm_for_each_vcpu(i, vcpu, kvm) { 324 | if (!kvm_apic_present(vcpu)) 325 | continue; 326 | 327 | if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, 328 | irq->dest_id, irq->dest_mode)) 329 | continue; 330 | 331 | if (++r == 2) 332 | return false; 333 | 334 | *dest_vcpu = vcpu; 335 | } 336 | 337 | return r == 1; 338 | } 339 | 340 | #define IOAPIC_ROUTING_ENTRY(irq) \ 341 | { .gsi = irq, .type = AEHD_IRQ_ROUTING_IRQCHIP, \ 342 | .u.irqchip = { .irqchip = AEHD_IRQCHIP_IOAPIC, .pin = (irq) } } 343 | #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq) 344 | 345 | #define PIC_ROUTING_ENTRY(irq) \ 346 | { .gsi = irq, .type = AEHD_IRQ_ROUTING_IRQCHIP, \ 347 | .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } } 348 | #define ROUTING_ENTRY2(irq) \ 349 | IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq) 350 | 351 | static const struct kvm_irq_routing_entry default_routing[] = { 352 | ROUTING_ENTRY2(0), ROUTING_ENTRY2(1), 353 | ROUTING_ENTRY2(2), ROUTING_ENTRY2(3), 354 | ROUTING_ENTRY2(4), ROUTING_ENTRY2(5), 355 | ROUTING_ENTRY2(6), ROUTING_ENTRY2(7), 356 | ROUTING_ENTRY2(8), ROUTING_ENTRY2(9), 357 | ROUTING_ENTRY2(10), ROUTING_ENTRY2(11), 358 | ROUTING_ENTRY2(12), ROUTING_ENTRY2(13), 359 | ROUTING_ENTRY2(14), ROUTING_ENTRY2(15), 360 | ROUTING_ENTRY1(16), ROUTING_ENTRY1(17), 361 | ROUTING_ENTRY1(18), ROUTING_ENTRY1(19), 362 | ROUTING_ENTRY1(20), ROUTING_ENTRY1(21), 363 | ROUTING_ENTRY1(22), ROUTING_ENTRY1(23), 364 | }; 365 | 366 | int kvm_setup_default_irq_routing(struct kvm *kvm) 367 | { 368 | return kvm_set_irq_routing(kvm, default_routing, 369 | ARRAY_SIZE(default_routing), 0); 370 | } 371 | 372 | void kvm_arch_post_irq_routing_update(struct kvm *kvm) 373 | { 374 | if (ioapic_in_kernel(kvm) || !irqchip_in_kernel(kvm)) 375 | return; 376 | kvm_make_scan_ioapic_request(kvm); 377 | } 378 | 379 | -------------------------------------------------------------------------------- /arch/x86/kvm/kvm_cache_regs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef ASM_KVM_CACHE_REGS_H 6 | #define ASM_KVM_CACHE_REGS_H 7 | 8 | #include 9 | 10 | #define KVM_POSSIBLE_CR0_GUEST_BITS X86_CR0_TS 11 | #define KVM_POSSIBLE_CR4_GUEST_BITS \ 12 | (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \ 13 | | X86_CR4_OSXMMEXCPT | X86_CR4_PGE) 14 | 15 | static inline size_t kvm_register_read(struct kvm_vcpu *vcpu, 16 | enum kvm_reg reg) 17 | { 18 | if (!test_bit(reg, (size_t *)&vcpu->arch.regs_avail)) 19 | kvm_x86_ops->cache_reg(vcpu, reg); 20 | 21 | return vcpu->arch.regs[reg]; 22 | } 23 | 24 | static inline void kvm_register_write(struct kvm_vcpu *vcpu, 25 | enum kvm_reg reg, 26 | size_t val) 27 | { 28 | vcpu->arch.regs[reg] = val; 29 | __set_bit(reg, (size_t *)&vcpu->arch.regs_dirty); 30 | __set_bit(reg, (size_t *)&vcpu->arch.regs_avail); 31 | } 32 | 33 | static inline size_t kvm_rip_read(struct kvm_vcpu *vcpu) 34 | { 35 | return kvm_register_read(vcpu, VCPU_REGS_RIP); 36 | } 37 | 38 | static inline void kvm_rip_write(struct kvm_vcpu *vcpu, size_t val) 39 | { 40 | kvm_register_write(vcpu, VCPU_REGS_RIP, val); 41 | } 42 | 43 | static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) 44 | { 45 | might_sleep(); /* on svm */ 46 | 47 | if (!test_bit(VCPU_EXREG_PDPTR, 48 | (size_t *)&vcpu->arch.regs_avail)) 49 | kvm_x86_ops->cache_reg(vcpu, VCPU_EXREG_PDPTR); 50 | 51 | return vcpu->arch.walk_mmu->pdptrs[index]; 52 | } 53 | 54 | static inline size_t kvm_read_cr0_bits(struct kvm_vcpu *vcpu, size_t mask) 55 | { 56 | size_t tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS; 57 | if (tmask & vcpu->arch.cr0_guest_owned_bits) 58 | kvm_x86_ops->decache_cr0_guest_bits(vcpu); 59 | return vcpu->arch.cr0 & mask; 60 | } 61 | 62 | static inline size_t kvm_read_cr0(struct kvm_vcpu *vcpu) 63 | { 64 | return kvm_read_cr0_bits(vcpu, ~(size_t)0); 65 | } 66 | 67 | static inline size_t kvm_read_cr4_bits(struct kvm_vcpu *vcpu, size_t mask) 68 | { 69 | size_t tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS; 70 | if (tmask & vcpu->arch.cr4_guest_owned_bits) 71 | kvm_x86_ops->decache_cr4_guest_bits(vcpu); 72 | return vcpu->arch.cr4 & mask; 73 | } 74 | 75 | static inline size_t kvm_read_cr3(struct kvm_vcpu *vcpu) 76 | { 77 | if (!test_bit(VCPU_EXREG_CR3, (size_t *)&vcpu->arch.regs_avail)) 78 | kvm_x86_ops->decache_cr3(vcpu); 79 | return vcpu->arch.cr3; 80 | } 81 | 82 | static inline size_t kvm_read_cr4(struct kvm_vcpu *vcpu) 83 | { 84 | return kvm_read_cr4_bits(vcpu, ~(size_t)0); 85 | } 86 | 87 | static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) 88 | { 89 | return (kvm_register_read(vcpu, VCPU_REGS_RAX) & (unsigned)-1) 90 | | ((u64)(kvm_register_read(vcpu, VCPU_REGS_RDX) & (unsigned)-1) << 32); 91 | } 92 | 93 | static inline void enter_guest_mode(struct kvm_vcpu *vcpu) 94 | { 95 | vcpu->arch.hflags |= HF_GUEST_MASK; 96 | } 97 | 98 | static inline void leave_guest_mode(struct kvm_vcpu *vcpu) 99 | { 100 | vcpu->arch.hflags &= ~HF_GUEST_MASK; 101 | } 102 | 103 | static inline bool is_guest_mode(struct kvm_vcpu *vcpu) 104 | { 105 | return vcpu->arch.hflags & HF_GUEST_MASK; 106 | } 107 | 108 | static inline bool is_smm(struct kvm_vcpu *vcpu) 109 | { 110 | return vcpu->arch.hflags & HF_SMM_MASK; 111 | } 112 | 113 | #endif 114 | -------------------------------------------------------------------------------- /arch/x86/kvm/lapic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef __KVM_X86_LAPIC_H 6 | #define __KVM_X86_LAPIC_H 7 | 8 | #include 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define AEHD_APIC_INIT 0 19 | #define AEHD_APIC_SIPI 1 20 | #define AEHD_APIC_LVT_NUM 6 21 | 22 | #define AEHD_APIC_SHORT_MASK 0xc0000 23 | #define AEHD_APIC_DEST_MASK 0x800 24 | 25 | #define u32 unsigned int 26 | 27 | struct kvm_timer { 28 | struct hrtimer timer; 29 | s64 period; /* unit: ns */ 30 | u32 timer_mode; 31 | u32 timer_mode_mask; 32 | atomic_t pending; /* accumulated triggered timers */ 33 | }; 34 | 35 | struct kvm_lapic { 36 | size_t base_address; 37 | struct kvm_io_device dev; 38 | struct kvm_timer lapic_timer; 39 | u32 divide_count; 40 | struct kvm_vcpu *vcpu; 41 | bool sw_enabled; 42 | bool irr_pending; 43 | bool lvt0_in_nmi_mode; 44 | /* Number of bits set in ISR. */ 45 | s16 isr_count; 46 | /* The highest vector set in ISR; if -1 - invalid, must scan ISR. */ 47 | int highest_isr_cache; 48 | /** 49 | * APIC register page. The layout matches the register layout seen by 50 | * the guest 1:1, because it is accessed by the vmx microcode. 51 | * Note: Only one register, the TPR, is used by the microcode. 52 | */ 53 | u8 *regs; 54 | gpa_t vapic_addr; 55 | struct gfn_to_hva_cache vapic_cache; 56 | size_t pending_events; 57 | unsigned int sipi_vector; 58 | }; 59 | 60 | struct dest_map; 61 | 62 | int kvm_create_lapic(struct kvm_vcpu *vcpu); 63 | void kvm_free_lapic(struct kvm_vcpu *vcpu); 64 | 65 | int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); 66 | int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); 67 | int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); 68 | void kvm_apic_accept_events(struct kvm_vcpu *vcpu); 69 | void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event); 70 | u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); 71 | void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, size_t cr8); 72 | void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu); 73 | void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); 74 | u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu); 75 | void kvm_apic_set_version(struct kvm_vcpu *vcpu); 76 | int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val); 77 | int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 78 | void *data); 79 | bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 80 | int short_hand, unsigned int dest, int dest_mode); 81 | 82 | void __kvm_apic_update_irr(u32 *pir, void *regs); 83 | void kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir); 84 | int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 85 | struct dest_map *dest_map); 86 | int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type); 87 | 88 | bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 89 | struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map); 90 | 91 | u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); 92 | int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info); 93 | int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); 94 | int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); 95 | int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); 96 | 97 | void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); 98 | void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); 99 | 100 | int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); 101 | void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); 102 | void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu); 103 | 104 | int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data); 105 | int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); 106 | 107 | void kvm_lapic_init(void); 108 | 109 | #define VEC_POS(v) ((v) & (32 - 1)) 110 | #define REG_POS(v) (((v) >> 5) << 4) 111 | 112 | static inline void kvm_lapic_set_vector(int vec, void *bitmap) 113 | { 114 | set_bit(VEC_POS(vec), (size_t *)((u8 *)(bitmap) + REG_POS(vec))); 115 | } 116 | 117 | static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) 118 | { 119 | kvm_lapic_set_vector(vec, (unsigned char *)apic->regs + APIC_IRR); 120 | /* 121 | * irr_pending must be true if any interrupt is pending; set it after 122 | * APIC_IRR to avoid race with apic_clear_irr 123 | */ 124 | apic->irr_pending = true; 125 | } 126 | 127 | static inline u32 kvm_lapic_get_reg(struct kvm_lapic *apic, int reg_off) 128 | { 129 | return *((u32 *) ((unsigned char *)apic->regs + reg_off)); 130 | } 131 | 132 | static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 133 | { 134 | *((u32 *) ((unsigned char *)apic->regs + reg_off)) = val; 135 | } 136 | 137 | static inline bool lapic_in_kernel(struct kvm_vcpu *vcpu) 138 | { 139 | return vcpu->arch.apic; 140 | } 141 | 142 | static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic) 143 | { 144 | return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; 145 | } 146 | 147 | static inline bool kvm_apic_sw_enabled(struct kvm_lapic *apic) 148 | { 149 | return apic->sw_enabled; 150 | } 151 | 152 | static inline bool kvm_apic_present(struct kvm_vcpu *vcpu) 153 | { 154 | return lapic_in_kernel(vcpu) && kvm_apic_hw_enabled(vcpu->arch.apic); 155 | } 156 | 157 | static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu) 158 | { 159 | return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic); 160 | } 161 | 162 | static inline int apic_x2apic_mode(struct kvm_lapic *apic) 163 | { 164 | return apic->vcpu->arch.apic_base & X2APIC_ENABLE; 165 | } 166 | 167 | static inline bool kvm_vcpu_apicv_active(struct kvm_vcpu *vcpu) 168 | { 169 | return vcpu->arch.apic && vcpu->arch.apicv_active; 170 | } 171 | 172 | static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu) 173 | { 174 | return lapic_in_kernel(vcpu) && vcpu->arch.apic->pending_events; 175 | } 176 | 177 | static inline bool kvm_lowest_prio_delivery(struct kvm_lapic_irq *irq) 178 | { 179 | return (irq->delivery_mode == APIC_DM_LOWEST || 180 | irq->msi_redir_hint); 181 | } 182 | 183 | static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) 184 | { 185 | return lapic_in_kernel(vcpu) && test_bit(AEHD_APIC_INIT, &vcpu->arch.apic->pending_events); 186 | } 187 | 188 | static inline u32 kvm_apic_id(struct kvm_lapic *apic) 189 | { 190 | /* To avoid a race between apic_base and following APIC_ID update when 191 | * switching to x2apic_mode, the x2apic mode returns initial x2apic id. 192 | */ 193 | if (apic_x2apic_mode(apic)) 194 | return apic->vcpu->vcpu_id; 195 | 196 | return kvm_lapic_get_reg(apic, APIC_ID) >> 24; 197 | } 198 | 199 | bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); 200 | 201 | bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 202 | struct kvm_vcpu **dest_vcpu); 203 | int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 204 | const size_t *bitmap, u32 bitmap_size); 205 | #endif 206 | -------------------------------------------------------------------------------- /arch/x86/kvm/mmu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef __KVM_X86_MMU_H 6 | #define __KVM_X86_MMU_H 7 | 8 | #include 9 | #include "kvm_cache_regs.h" 10 | 11 | #define PT64_PT_BITS 9 12 | #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS) 13 | #define PT32_PT_BITS 10 14 | #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS) 15 | 16 | #define PT_WRITABLE_SHIFT 1 17 | #define PT_USER_SHIFT 2 18 | 19 | #define PT_PRESENT_MASK (1ULL << 0) 20 | #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT) 21 | #define PT_USER_MASK (1ULL << PT_USER_SHIFT) 22 | #define PT_PWT_MASK (1ULL << 3) 23 | #define PT_PCD_MASK (1ULL << 4) 24 | #define PT_ACCESSED_SHIFT 5 25 | #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT) 26 | #define PT_DIRTY_SHIFT 6 27 | #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT) 28 | #define PT_PAGE_SIZE_SHIFT 7 29 | #define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT) 30 | #define PT_PAT_MASK (1ULL << 7) 31 | #define PT_GLOBAL_MASK (1ULL << 8) 32 | #define PT64_NX_SHIFT 63 33 | #define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) 34 | 35 | #define PT_PAT_SHIFT 7 36 | #define PT_DIR_PAT_SHIFT 12 37 | #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT) 38 | 39 | #define PT32_DIR_PSE36_SIZE 4 40 | #define PT32_DIR_PSE36_SHIFT 13 41 | #define PT32_DIR_PSE36_MASK \ 42 | (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) 43 | 44 | #define PT64_ROOT_LEVEL 4 45 | #define PT32_ROOT_LEVEL 2 46 | #define PT32E_ROOT_LEVEL 3 47 | 48 | #define PT_PDPE_LEVEL 3 49 | #define PT_DIRECTORY_LEVEL 2 50 | #define PT_PAGE_TABLE_LEVEL 1 51 | #define PT_MAX_HUGEPAGE_LEVEL (PT_PAGE_TABLE_LEVEL + AEHD_NR_PAGE_SIZES - 1) 52 | 53 | static inline u64 rsvd_bits(int s, int e) 54 | { 55 | return ((1ULL << (e - s + 1)) - 1) << s; 56 | } 57 | 58 | void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask); 59 | 60 | void 61 | reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context); 62 | 63 | /* 64 | * Return values of handle_mmio_page_fault: 65 | * RET_MMIO_PF_EMULATE: it is a real mmio page fault, emulate the instruction 66 | * directly. 67 | * RET_MMIO_PF_INVALID: invalid spte is detected then let the real page 68 | * fault path update the mmio spte. 69 | * RET_MMIO_PF_RETRY: let CPU fault again on the address. 70 | * RET_MMIO_PF_BUG: a bug was detected (and a WARN was printed). 71 | */ 72 | enum { 73 | RET_MMIO_PF_EMULATE = 1, 74 | RET_MMIO_PF_INVALID = 2, 75 | RET_MMIO_PF_RETRY = 0, 76 | RET_MMIO_PF_BUG = -1 77 | }; 78 | 79 | int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct); 80 | void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu); 81 | void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly); 82 | 83 | static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm) 84 | { 85 | if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages) 86 | return kvm->arch.n_max_mmu_pages - 87 | kvm->arch.n_used_mmu_pages; 88 | 89 | return 0; 90 | } 91 | 92 | static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) 93 | { 94 | if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) 95 | return 0; 96 | 97 | return kvm_mmu_load(vcpu); 98 | } 99 | 100 | /* 101 | * Currently, we have two sorts of write-protection, a) the first one 102 | * write-protects guest page to sync the guest modification, b) another one is 103 | * used to sync dirty bitmap when we do AEHD_GET_DIRTY_LOG. The differences 104 | * between these two sorts are: 105 | * 1) the first case clears SPTE_MMU_WRITEABLE bit. 106 | * 2) the first case requires flushing tlb immediately avoiding corrupting 107 | * shadow page table between all vcpus so it should be in the protection of 108 | * mmu-lock. And the another case does not need to flush tlb until returning 109 | * the dirty bitmap to userspace since it only write-protects the page 110 | * logged in the bitmap, that means the page in the dirty bitmap is not 111 | * missed, so it can flush tlb out of mmu-lock. 112 | * 113 | * So, there is the problem: the first case can meet the corrupted tlb caused 114 | * by another case which write-protects pages but without flush tlb 115 | * immediately. In order to making the first case be aware this problem we let 116 | * it flush tlb if we try to write-protect a spte whose SPTE_MMU_WRITEABLE bit 117 | * is set, it works since another case never touches SPTE_MMU_WRITEABLE bit. 118 | * 119 | * Anyway, whenever a spte is updated (only permission and status bits are 120 | * changed) we need to check whether the spte with SPTE_MMU_WRITEABLE becomes 121 | * readonly, if that happens, we need to flush tlb. Fortunately, 122 | * mmu_spte_update() has already handled it perfectly. 123 | * 124 | * The rules to use SPTE_MMU_WRITEABLE and PT_WRITABLE_MASK: 125 | * - if we want to see if it has writable tlb entry or if the spte can be 126 | * writable on the mmu mapping, check SPTE_MMU_WRITEABLE, this is the most 127 | * case, otherwise 128 | * - if we fix page fault on the spte or do write-protection by dirty logging, 129 | * check PT_WRITABLE_MASK. 130 | * 131 | * TODO: introduce APIs to split these two cases. 132 | */ 133 | static inline int is_writable_pte(size_t pte) 134 | { 135 | return pte & PT_WRITABLE_MASK; 136 | } 137 | 138 | static inline bool is_write_protection(struct kvm_vcpu *vcpu) 139 | { 140 | return kvm_read_cr0_bits(vcpu, X86_CR0_WP); 141 | } 142 | 143 | /* 144 | * Check if a given access (described through the I/D, W/R and U/S bits of a 145 | * page fault error code pfec) causes a permission fault with the given PTE 146 | * access rights (in ACC_* format). 147 | * 148 | * Return zero if the access does not fault; return the page fault error code 149 | * if the access faults. 150 | */ 151 | static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 152 | unsigned pte_access, unsigned pte_pkey, 153 | unsigned pfec) 154 | { 155 | int cpl = kvm_x86_ops->get_cpl(vcpu); 156 | size_t rflags = kvm_x86_ops->get_rflags(vcpu); 157 | 158 | /* 159 | * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1. 160 | * 161 | * If CPL = 3, SMAP applies to all supervisor-mode data accesses 162 | * (these are implicit supervisor accesses) regardless of the value 163 | * of EFLAGS.AC. 164 | * 165 | * This computes (cpl < 3) && (rflags & X86_EFLAGS_AC), leaving 166 | * the result in X86_EFLAGS_AC. We then insert it in place of 167 | * the PFERR_RSVD_MASK bit; this bit will always be zero in pfec, 168 | * but it will be one in index if SMAP checks are being overridden. 169 | * It is important to keep this branchless. 170 | */ 171 | size_t smap = (cpl - 3) & (rflags & X86_EFLAGS_AC); 172 | int index = (pfec >> 1) + 173 | (smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1)); 174 | bool fault = (mmu->permissions[index] >> pte_access) & 1; 175 | u32 errcode = PFERR_PRESENT_MASK; 176 | 177 | WARN_ON(pfec & (PFERR_PK_MASK | PFERR_RSVD_MASK)); 178 | 179 | return -(s32)fault & errcode; 180 | } 181 | 182 | void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm); 183 | void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end); 184 | 185 | bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, 186 | struct kvm_memory_slot *slot, u64 gfn); 187 | #endif 188 | -------------------------------------------------------------------------------- /arch/x86/kvm/mmu_audit.c: -------------------------------------------------------------------------------- 1 | /* 2 | * mmu_audit.c: 3 | * 4 | * Audit code for KVM MMU 5 | * 6 | * Copyright (C) 2006 Qumranet, Inc. 7 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. 8 | * Copyright 2019 Google LLC 9 | * 10 | * Authors: 11 | * Yaniv Kamay 12 | * Avi Kivity 13 | * Marcelo Tosatti 14 | * Xiao Guangrong 15 | * 16 | * This work is licensed under the terms of the GNU GPL, version 2. See 17 | * the COPYING file in the top-level directory. 18 | * 19 | */ 20 | 21 | #if 0 22 | #include 23 | 24 | char const *audit_point_name[] = { 25 | "pre page fault", 26 | "post page fault", 27 | "pre pte write", 28 | "post pte write", 29 | "pre sync", 30 | "post sync" 31 | }; 32 | 33 | #define audit_printk(kvm, fmt, args...) \ 34 | printk(KERN_ERR "audit: (%s) error: " \ 35 | fmt, audit_point_name[kvm->arch.audit_point], ##args) 36 | 37 | typedef void (*inspect_spte_fn) (struct kvm_vcpu *vcpu, u64 *sptep, int level); 38 | 39 | static void __mmu_spte_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, 40 | inspect_spte_fn fn, int level) 41 | { 42 | int i; 43 | 44 | for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { 45 | u64 *ent = sp->spt; 46 | 47 | fn(vcpu, ent + i, level); 48 | 49 | if (is_shadow_present_pte(ent[i]) && 50 | !is_last_spte(ent[i], level)) { 51 | struct kvm_mmu_page *child; 52 | 53 | child = page_header(ent[i] & PT64_BASE_ADDR_MASK); 54 | __mmu_spte_walk(vcpu, child, fn, level - 1); 55 | } 56 | } 57 | } 58 | 59 | static void mmu_spte_walk(struct kvm_vcpu *vcpu, inspect_spte_fn fn) 60 | { 61 | int i; 62 | struct kvm_mmu_page *sp; 63 | 64 | if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) 65 | return; 66 | 67 | if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) { 68 | hpa_t root = vcpu->arch.mmu.root_hpa; 69 | 70 | sp = page_header(root); 71 | __mmu_spte_walk(vcpu, sp, fn, PT64_ROOT_LEVEL); 72 | return; 73 | } 74 | 75 | for (i = 0; i < 4; ++i) { 76 | hpa_t root = vcpu->arch.mmu.pae_root[i]; 77 | 78 | if (root && VALID_PAGE(root)) { 79 | root &= PT64_BASE_ADDR_MASK; 80 | sp = page_header(root); 81 | __mmu_spte_walk(vcpu, sp, fn, 2); 82 | } 83 | } 84 | 85 | return; 86 | } 87 | 88 | typedef void (*sp_handler) (struct kvm *kvm, struct kvm_mmu_page *sp); 89 | 90 | static void walk_all_active_sps(struct kvm *kvm, sp_handler fn) 91 | { 92 | struct kvm_mmu_page *sp; 93 | 94 | list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) 95 | fn(kvm, sp); 96 | } 97 | 98 | static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) 99 | { 100 | struct kvm_mmu_page *sp; 101 | gfn_t gfn; 102 | kvm_pfn_t pfn; 103 | hpa_t hpa; 104 | 105 | sp = page_header(__pa(sptep)); 106 | 107 | if (sp->unsync) { 108 | if (level != PT_PAGE_TABLE_LEVEL) { 109 | audit_printk(vcpu->kvm, "unsync sp: %p " 110 | "level = %d\n", sp, level); 111 | return; 112 | } 113 | } 114 | 115 | if (!is_shadow_present_pte(*sptep) || !is_last_spte(*sptep, level)) 116 | return; 117 | 118 | gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); 119 | pfn = kvm_vcpu_gfn_to_pfn_atomic(vcpu, gfn); 120 | 121 | if (is_error_pfn(pfn)) 122 | return; 123 | 124 | hpa = pfn << PAGE_SHIFT; 125 | if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) 126 | audit_printk(vcpu->kvm, "levels %d pfn %llx hpa %llx " 127 | "ent %llxn", vcpu->arch.mmu.root_level, pfn, 128 | hpa, *sptep); 129 | } 130 | 131 | static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep) 132 | { 133 | static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); 134 | struct kvm_rmap_head *rmap_head; 135 | struct kvm_mmu_page *rev_sp; 136 | struct kvm_memslots *slots; 137 | struct kvm_memory_slot *slot; 138 | gfn_t gfn; 139 | 140 | rev_sp = page_header(__pa(sptep)); 141 | gfn = kvm_mmu_page_get_gfn(rev_sp, sptep - rev_sp->spt); 142 | 143 | slots = kvm_memslots_for_spte_role(kvm, rev_sp->role); 144 | slot = __gfn_to_memslot(slots, gfn); 145 | if (!slot) { 146 | if (!__ratelimit(&ratelimit_state)) 147 | return; 148 | audit_printk(kvm, "no memslot for gfn %llx\n", gfn); 149 | audit_printk(kvm, "index %ld of sp (gfn=%llx)\n", 150 | (long int)(sptep - rev_sp->spt), rev_sp->gfn); 151 | dump_stack(); 152 | return; 153 | } 154 | 155 | rmap_head = __gfn_to_rmap(gfn, rev_sp->role.level, slot); 156 | if (!rmap_head->val) { 157 | if (!__ratelimit(&ratelimit_state)) 158 | return; 159 | audit_printk(kvm, "no rmap for writable spte %llx\n", 160 | *sptep); 161 | dump_stack(); 162 | } 163 | } 164 | 165 | static void audit_sptes_have_rmaps(struct kvm_vcpu *vcpu, u64 *sptep, int level) 166 | { 167 | if (is_shadow_present_pte(*sptep) && is_last_spte(*sptep, level)) 168 | inspect_spte_has_rmap(vcpu->kvm, sptep); 169 | } 170 | 171 | static void audit_spte_after_sync(struct kvm_vcpu *vcpu, u64 *sptep, int level) 172 | { 173 | struct kvm_mmu_page *sp = page_header(__pa(sptep)); 174 | 175 | if (vcpu->kvm->arch.audit_point == AUDIT_POST_SYNC && sp->unsync) 176 | audit_printk(vcpu->kvm, "meet unsync sp(%p) after sync " 177 | "root.\n", sp); 178 | } 179 | 180 | static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp) 181 | { 182 | int i; 183 | 184 | if (sp->role.level != PT_PAGE_TABLE_LEVEL) 185 | return; 186 | 187 | for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { 188 | if (!is_shadow_present_pte(sp->spt[i])) 189 | continue; 190 | 191 | inspect_spte_has_rmap(kvm, sp->spt + i); 192 | } 193 | } 194 | 195 | static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) 196 | { 197 | struct kvm_rmap_head *rmap_head; 198 | u64 *sptep; 199 | struct rmap_iterator iter; 200 | struct kvm_memslots *slots; 201 | struct kvm_memory_slot *slot; 202 | 203 | if (sp->role.direct || sp->unsync || sp->role.invalid) 204 | return; 205 | 206 | slots = kvm_memslots_for_spte_role(kvm, sp->role); 207 | slot = __gfn_to_memslot(slots, sp->gfn); 208 | rmap_head = __gfn_to_rmap(sp->gfn, PT_PAGE_TABLE_LEVEL, slot); 209 | 210 | for_each_rmap_spte(rmap_head, &iter, sptep) { 211 | if (is_writable_pte(*sptep)) 212 | audit_printk(kvm, "shadow page has writable " 213 | "mappings: gfn %llx role %x\n", 214 | sp->gfn, sp->role.word); 215 | } 216 | } 217 | 218 | static void audit_sp(struct kvm *kvm, struct kvm_mmu_page *sp) 219 | { 220 | check_mappings_rmap(kvm, sp); 221 | audit_write_protection(kvm, sp); 222 | } 223 | 224 | static void audit_all_active_sps(struct kvm *kvm) 225 | { 226 | walk_all_active_sps(kvm, audit_sp); 227 | } 228 | 229 | static void audit_spte(struct kvm_vcpu *vcpu, u64 *sptep, int level) 230 | { 231 | audit_sptes_have_rmaps(vcpu, sptep, level); 232 | audit_mappings(vcpu, sptep, level); 233 | audit_spte_after_sync(vcpu, sptep, level); 234 | } 235 | 236 | static void audit_vcpu_spte(struct kvm_vcpu *vcpu) 237 | { 238 | mmu_spte_walk(vcpu, audit_spte); 239 | } 240 | 241 | static bool mmu_audit; 242 | static struct static_key mmu_audit_key; 243 | 244 | static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) 245 | { 246 | static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); 247 | 248 | if (!__ratelimit(&ratelimit_state)) 249 | return; 250 | 251 | vcpu->kvm->arch.audit_point = point; 252 | audit_all_active_sps(vcpu->kvm); 253 | audit_vcpu_spte(vcpu); 254 | } 255 | 256 | static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) 257 | { 258 | if (static_key_false((&mmu_audit_key))) 259 | __kvm_mmu_audit(vcpu, point); 260 | } 261 | 262 | static void mmu_audit_enable(void) 263 | { 264 | if (mmu_audit) 265 | return; 266 | 267 | static_key_slow_inc(&mmu_audit_key); 268 | mmu_audit = true; 269 | } 270 | 271 | static void mmu_audit_disable(void) 272 | { 273 | if (!mmu_audit) 274 | return; 275 | 276 | static_key_slow_dec(&mmu_audit_key); 277 | mmu_audit = false; 278 | } 279 | 280 | static int mmu_audit_set(const char *val, const struct kernel_param *kp) 281 | { 282 | int ret; 283 | size_t enable; 284 | 285 | ret = kstrtoul(val, 10, &enable); 286 | if (ret < 0) 287 | return -EINVAL; 288 | 289 | switch (enable) { 290 | case 0: 291 | mmu_audit_disable(); 292 | break; 293 | case 1: 294 | mmu_audit_enable(); 295 | break; 296 | default: 297 | return -EINVAL; 298 | } 299 | 300 | return 0; 301 | } 302 | 303 | static const struct kernel_param_ops audit_param_ops = { 304 | .set = mmu_audit_set, 305 | .get = param_get_bool, 306 | }; 307 | 308 | arch_param_cb(mmu_audit, &audit_param_ops, &mmu_audit, 0644); 309 | #endif 310 | -------------------------------------------------------------------------------- /arch/x86/kvm/page_track.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Support KVM gust page tracking 3 | * 4 | * This feature allows us to track page access in guest. Currently, only 5 | * write access is tracked. 6 | * 7 | * Copyright(C) 2015 Intel Corporation. 8 | * Copyright 2019 Google LLC 9 | * 10 | * Author: 11 | * Xiao Guangrong 12 | * 13 | * This work is licensed under the terms of the GNU GPL, version 2. See 14 | * the COPYING file in the top-level directory. 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "mmu.h" 22 | 23 | void kvm_page_track_free_memslot(struct kvm_memory_slot *free, 24 | struct kvm_memory_slot *dont) 25 | { 26 | int i; 27 | 28 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) 29 | if (!dont || free->arch.gfn_track[i] != 30 | dont->arch.gfn_track[i]) { 31 | kvfree(free->arch.gfn_track[i]); 32 | free->arch.gfn_track[i] = NULL; 33 | } 34 | } 35 | 36 | int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, 37 | size_t npages) 38 | { 39 | int i; 40 | 41 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 42 | slot->arch.gfn_track[i] = kvm_kvzalloc(npages * 43 | sizeof(*slot->arch.gfn_track[i])); 44 | if (!slot->arch.gfn_track[i]) 45 | goto track_free; 46 | } 47 | 48 | return 0; 49 | 50 | track_free: 51 | kvm_page_track_free_memslot(slot, NULL); 52 | return -ENOMEM; 53 | } 54 | 55 | static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) 56 | { 57 | if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) 58 | return false; 59 | 60 | return true; 61 | } 62 | 63 | static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, 64 | enum kvm_page_track_mode mode, short count) 65 | { 66 | int index, val; 67 | 68 | index = gfn - slot->base_gfn; 69 | 70 | val = slot->arch.gfn_track[mode][index]; 71 | 72 | if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) 73 | return; 74 | 75 | slot->arch.gfn_track[mode][index] += count; 76 | } 77 | 78 | /* 79 | * add guest page to the tracking pool so that corresponding access on that 80 | * page will be intercepted. 81 | * 82 | * It should be called under the protection both of mmu-lock and kvm->srcu 83 | * or kvm->slots_lock. 84 | * 85 | * @kvm: the guest instance we are interested in. 86 | * @slot: the @gfn belongs to. 87 | * @gfn: the guest page. 88 | * @mode: tracking mode, currently only write track is supported. 89 | */ 90 | void kvm_slot_page_track_add_page(struct kvm *kvm, 91 | struct kvm_memory_slot *slot, gfn_t gfn, 92 | enum kvm_page_track_mode mode) 93 | { 94 | 95 | if (WARN_ON(!page_track_mode_is_valid(mode))) 96 | return; 97 | 98 | update_gfn_track(slot, gfn, mode, 1); 99 | 100 | if (mode == KVM_PAGE_TRACK_WRITE) 101 | if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) 102 | kvm_flush_remote_tlbs(kvm); 103 | } 104 | 105 | /* 106 | * remove the guest page from the tracking pool which stops the interception 107 | * of corresponding access on that page. It is the opposed operation of 108 | * kvm_slot_page_track_add_page(). 109 | * 110 | * It should be called under the protection both of mmu-lock and kvm->srcu 111 | * or kvm->slots_lock. 112 | * 113 | * @kvm: the guest instance we are interested in. 114 | * @slot: the @gfn belongs to. 115 | * @gfn: the guest page. 116 | * @mode: tracking mode, currently only write track is supported. 117 | */ 118 | void kvm_slot_page_track_remove_page(struct kvm *kvm, 119 | struct kvm_memory_slot *slot, gfn_t gfn, 120 | enum kvm_page_track_mode mode) 121 | { 122 | if (WARN_ON(!page_track_mode_is_valid(mode))) 123 | return; 124 | 125 | update_gfn_track(slot, gfn, mode, -1); 126 | } 127 | 128 | /* 129 | * check if the corresponding access on the specified guest page is tracked. 130 | */ 131 | bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, 132 | enum kvm_page_track_mode mode) 133 | { 134 | struct kvm_memory_slot *slot; 135 | int index; 136 | unsigned short temp; 137 | 138 | if (WARN_ON(!page_track_mode_is_valid(mode))) 139 | return false; 140 | 141 | slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 142 | if (!slot) 143 | return false; 144 | 145 | index = gfn - slot->base_gfn; 146 | ACCESS_ONCE(slot->arch.gfn_track[mode][index], temp); 147 | return !!temp; 148 | } 149 | 150 | void kvm_page_track_init(struct kvm *kvm) 151 | { 152 | struct kvm_page_track_notifier_head *head; 153 | 154 | head = &kvm->arch.track_notifier_head; 155 | init_srcu_struct(&head->track_srcu); 156 | INIT_HLIST_HEAD(&head->track_notifier_list); 157 | } 158 | 159 | void kvm_page_track_destroy(struct kvm *kvm) 160 | { 161 | struct kvm_page_track_notifier_head *head; 162 | 163 | head = &kvm->arch.track_notifier_head; 164 | cleanup_srcu_struct(&head->track_srcu); 165 | } 166 | 167 | /* 168 | * register the notifier so that event interception for the tracked guest 169 | * pages can be received. 170 | */ 171 | void 172 | kvm_page_track_register_notifier(struct kvm *kvm, 173 | struct kvm_page_track_notifier_node *n) 174 | { 175 | struct kvm_page_track_notifier_head *head; 176 | 177 | head = &kvm->arch.track_notifier_head; 178 | 179 | spin_lock(&kvm->mmu_lock); 180 | hlist_add_head_rcu(&n->node, &head->track_notifier_list); 181 | spin_unlock(&kvm->mmu_lock); 182 | } 183 | 184 | /* 185 | * stop receiving the event interception. It is the opposed operation of 186 | * kvm_page_track_register_notifier(). 187 | */ 188 | void 189 | kvm_page_track_unregister_notifier(struct kvm *kvm, 190 | struct kvm_page_track_notifier_node *n) 191 | { 192 | struct kvm_page_track_notifier_head *head; 193 | 194 | head = &kvm->arch.track_notifier_head; 195 | 196 | spin_lock(&kvm->mmu_lock); 197 | hlist_del_rcu(&n->node); 198 | spin_unlock(&kvm->mmu_lock); 199 | synchronize_srcu(&head->track_srcu); 200 | } 201 | 202 | /* 203 | * Notify the node that write access is intercepted and write emulation is 204 | * finished at this time. 205 | * 206 | * The node should figure out if the written page is the one that node is 207 | * interested in by itself. 208 | */ 209 | void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 210 | int bytes) 211 | { 212 | struct kvm_page_track_notifier_head *head; 213 | struct kvm_page_track_notifier_node *n; 214 | int idx; 215 | 216 | head = &vcpu->kvm->arch.track_notifier_head; 217 | 218 | if (hlist_empty(&head->track_notifier_list)) 219 | return; 220 | 221 | idx = srcu_read_lock(&head->track_srcu); 222 | #define LIST_ENTRY_TYPE_INFO struct kvm_page_track_notifier_node 223 | hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) 224 | if (n->track_write) 225 | n->track_write(vcpu, gpa, new, bytes); 226 | #undef LIST_ENTRY_TYPE_INFO 227 | srcu_read_unlock(&head->track_srcu, idx); 228 | } 229 | -------------------------------------------------------------------------------- /arch/x86/kvm/pmu.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Kernel-based Virtual Machine -- Performance Monitoring Unit support 3 | * 4 | * Copyright 2015 Red Hat, Inc. and/or its affiliates. 5 | * Copyright 2019 Google LLC 6 | * 7 | * Authors: 8 | * Avi Kivity 9 | * Gleb Natapov 10 | * Wei Huang 11 | * 12 | * This work is licensed under the terms of the GNU GPL, version 2. See 13 | * the COPYING file in the top-level directory. 14 | * 15 | */ 16 | 17 | #if 0 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include "x86.h" 23 | #include "cpuid.h" 24 | #include "lapic.h" 25 | #include "pmu.h" 26 | 27 | /* NOTE: 28 | * - Each perf counter is defined as "struct kvm_pmc"; 29 | * - There are two types of perf counters: general purpose (gp) and fixed. 30 | * gp counters are stored in gp_counters[] and fixed counters are stored 31 | * in fixed_counters[] respectively. Both of them are part of "struct 32 | * kvm_pmu"; 33 | * - pmu.c understands the difference between gp counters and fixed counters. 34 | * However AMD doesn't support fixed-counters; 35 | * - There are three types of index to access perf counters (PMC): 36 | * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 37 | * has MSR_K7_PERFCTRn. 38 | * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 39 | * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 40 | * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 41 | * that it also supports fixed counters. idx can be used to as index to 42 | * gp and fixed counters. 43 | * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 44 | * code. Each pmc, stored in kvm_pmc.idx field, is unique across 45 | * all perf counters (both gp and fixed). The mapping relationship 46 | * between pmc and perf counters is as the following: 47 | * * Intel: [0 .. INTEL_PMC_MAX_GENERIC-1] <=> gp counters 48 | * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed 49 | * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters 50 | */ 51 | 52 | static void kvm_pmi_trigger_fn(struct irq_work *irq_work) 53 | { 54 | struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work); 55 | struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); 56 | 57 | kvm_pmu_deliver_pmi(vcpu); 58 | } 59 | 60 | static void kvm_perf_overflow(struct perf_event *perf_event, 61 | struct perf_sample_data *data, 62 | struct pt_regs *regs) 63 | { 64 | struct kvm_pmc *pmc = perf_event->overflow_handler_context; 65 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); 66 | 67 | if (!test_and_set_bit(pmc->idx, 68 | (size_t *)&pmu->reprogram_pmi)) { 69 | __set_bit(pmc->idx, (size_t *)&pmu->global_status); 70 | kvm_make_request(AEHD_REQ_PMU, pmc->vcpu); 71 | } 72 | } 73 | 74 | static void kvm_perf_overflow_intr(struct perf_event *perf_event, 75 | struct perf_sample_data *data, 76 | struct pt_regs *regs) 77 | { 78 | struct kvm_pmc *pmc = perf_event->overflow_handler_context; 79 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); 80 | 81 | if (!test_and_set_bit(pmc->idx, 82 | (size_t *)&pmu->reprogram_pmi)) { 83 | __set_bit(pmc->idx, (size_t *)&pmu->global_status); 84 | kvm_make_request(AEHD_REQ_PMU, pmc->vcpu); 85 | 86 | /* 87 | * Inject PMI. If vcpu was in a guest mode during NMI PMI 88 | * can be ejected on a guest mode re-entry. Otherwise we can't 89 | * be sure that vcpu wasn't executing hlt instruction at the 90 | * time of vmexit and is not going to re-enter guest mode until 91 | * woken up. So we should wake it, but this is impossible from 92 | * NMI context. Do it from irq work instead. 93 | */ 94 | if (!kvm_is_in_guest()) 95 | irq_work_queue(&pmc_to_pmu(pmc)->irq_work); 96 | else 97 | kvm_make_request(AEHD_REQ_PMI, pmc->vcpu); 98 | } 99 | } 100 | 101 | static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, 102 | unsigned config, bool exclude_user, 103 | bool exclude_kernel, bool intr, 104 | bool in_tx, bool in_tx_cp) 105 | { 106 | struct perf_event *event; 107 | struct perf_event_attr attr = { 108 | .type = type, 109 | .size = sizeof(attr), 110 | .pinned = true, 111 | .exclude_idle = true, 112 | .exclude_host = 1, 113 | .exclude_user = exclude_user, 114 | .exclude_kernel = exclude_kernel, 115 | .config = config, 116 | }; 117 | 118 | if (in_tx) 119 | attr.config |= HSW_IN_TX; 120 | if (in_tx_cp) 121 | attr.config |= HSW_IN_TX_CHECKPOINTED; 122 | 123 | attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc); 124 | 125 | event = perf_event_create_kernel_counter(&attr, -1, current, 126 | intr ? kvm_perf_overflow_intr : 127 | kvm_perf_overflow, pmc); 128 | if (IS_ERR(event)) { 129 | printk_once("kvm_pmu: event creation failed %ld\n", 130 | PTR_ERR(event)); 131 | return; 132 | } 133 | 134 | pmc->perf_event = event; 135 | clear_bit(pmc->idx, (size_t*)&pmc_to_pmu(pmc)->reprogram_pmi); 136 | } 137 | 138 | void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) 139 | { 140 | unsigned config, type = PERF_TYPE_RAW; 141 | u8 event_select, unit_mask; 142 | 143 | if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 144 | printk_once("kvm pmu: pin control bit is ignored\n"); 145 | 146 | pmc->eventsel = eventsel; 147 | 148 | pmc_stop_counter(pmc); 149 | 150 | if (!(eventsel & ARCH_PERFMON_EVENTSEL_ENABLE) || !pmc_is_enabled(pmc)) 151 | return; 152 | 153 | event_select = eventsel & ARCH_PERFMON_EVENTSEL_EVENT; 154 | unit_mask = (eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 155 | 156 | if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE | 157 | ARCH_PERFMON_EVENTSEL_INV | 158 | ARCH_PERFMON_EVENTSEL_CMASK | 159 | HSW_IN_TX | 160 | HSW_IN_TX_CHECKPOINTED))) { 161 | config = kvm_x86_ops->pmu_ops->find_arch_event(pmc_to_pmu(pmc), 162 | event_select, 163 | unit_mask); 164 | if (config != PERF_COUNT_HW_MAX) 165 | type = PERF_TYPE_HARDWARE; 166 | } 167 | 168 | if (type == PERF_TYPE_RAW) 169 | config = eventsel & X86_RAW_EVENT_MASK; 170 | 171 | pmc_reprogram_counter(pmc, type, config, 172 | !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 173 | !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 174 | eventsel & ARCH_PERFMON_EVENTSEL_INT, 175 | (eventsel & HSW_IN_TX), 176 | (eventsel & HSW_IN_TX_CHECKPOINTED)); 177 | } 178 | 179 | void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) 180 | { 181 | unsigned en_field = ctrl & 0x3; 182 | bool pmi = ctrl & 0x8; 183 | 184 | pmc_stop_counter(pmc); 185 | 186 | if (!en_field || !pmc_is_enabled(pmc)) 187 | return; 188 | 189 | pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, 190 | kvm_x86_ops->pmu_ops->find_fixed_event(idx), 191 | !(en_field & 0x2), /* exclude user */ 192 | !(en_field & 0x1), /* exclude kernel */ 193 | pmi, false, false); 194 | } 195 | 196 | void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) 197 | { 198 | struct kvm_pmc *pmc = kvm_x86_ops->pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx); 199 | 200 | if (!pmc) 201 | return; 202 | 203 | if (pmc_is_gp(pmc)) 204 | reprogram_gp_counter(pmc, pmc->eventsel); 205 | else { 206 | int idx = pmc_idx - INTEL_PMC_IDX_FIXED; 207 | u8 ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, idx); 208 | 209 | reprogram_fixed_counter(pmc, ctrl, idx); 210 | } 211 | } 212 | 213 | void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 214 | { 215 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 216 | u64 bitmask; 217 | int bit; 218 | 219 | bitmask = pmu->reprogram_pmi; 220 | 221 | for_each_set_bit(bit, (size_t *)&bitmask, X86_PMC_IDX_MAX) { 222 | struct kvm_pmc *pmc = kvm_x86_ops->pmu_ops->pmc_idx_to_pmc(pmu, bit); 223 | 224 | if (unlikely(!pmc || !pmc->perf_event)) { 225 | clear_bit(bit, (size_t *)&pmu->reprogram_pmi); 226 | continue; 227 | } 228 | 229 | reprogram_counter(pmu, bit); 230 | } 231 | } 232 | 233 | /* check if idx is a valid index to access PMU */ 234 | int kvm_pmu_is_valid_msr_idx(struct kvm_vcpu *vcpu, unsigned idx) 235 | { 236 | return kvm_x86_ops->pmu_ops->is_valid_msr_idx(vcpu, idx); 237 | } 238 | 239 | int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 240 | { 241 | bool fast_mode = idx & (1u << 31); 242 | struct kvm_pmc *pmc; 243 | u64 ctr_val; 244 | 245 | pmc = kvm_x86_ops->pmu_ops->msr_idx_to_pmc(vcpu, idx); 246 | if (!pmc) 247 | return 1; 248 | 249 | ctr_val = pmc_read_counter(pmc); 250 | if (fast_mode) 251 | ctr_val = (u32)ctr_val; 252 | 253 | *data = ctr_val; 254 | return 0; 255 | } 256 | 257 | void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 258 | { 259 | if (lapic_in_kernel(vcpu)) 260 | kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 261 | } 262 | 263 | bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 264 | { 265 | return kvm_x86_ops->pmu_ops->is_valid_msr(vcpu, msr); 266 | } 267 | 268 | int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 269 | { 270 | return kvm_x86_ops->pmu_ops->get_msr(vcpu, msr, data); 271 | } 272 | 273 | int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 274 | { 275 | return kvm_x86_ops->pmu_ops->set_msr(vcpu, msr_info); 276 | } 277 | 278 | /* refresh PMU settings. This function generally is called when underlying 279 | * settings are changed (such as changes of PMU CPUID by guest VMs), which 280 | * should rarely happen. 281 | */ 282 | void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 283 | { 284 | kvm_x86_ops->pmu_ops->refresh(vcpu); 285 | } 286 | 287 | void kvm_pmu_reset(struct kvm_vcpu *vcpu) 288 | { 289 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 290 | 291 | irq_work_sync(&pmu->irq_work); 292 | kvm_x86_ops->pmu_ops->reset(vcpu); 293 | } 294 | 295 | void kvm_pmu_init(struct kvm_vcpu *vcpu) 296 | { 297 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 298 | 299 | memset(pmu, 0, sizeof(*pmu)); 300 | kvm_x86_ops->pmu_ops->init(vcpu); 301 | init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); 302 | kvm_pmu_refresh(vcpu); 303 | } 304 | 305 | void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 306 | { 307 | kvm_pmu_reset(vcpu); 308 | } 309 | #endif 310 | -------------------------------------------------------------------------------- /arch/x86/kvm/pmu.h: -------------------------------------------------------------------------------- 1 | #ifndef __KVM_X86_PMU_H 2 | #define __KVM_X86_PMU_H 3 | 4 | #if 0 5 | 6 | #define vcpu_to_pmu(vcpu) (&(vcpu)->arch.pmu) 7 | #define pmu_to_vcpu(pmu) (container_of((pmu), struct kvm_vcpu, arch.pmu)) 8 | #define pmc_to_pmu(pmc) (&(pmc)->vcpu->arch.pmu) 9 | 10 | /* retrieve the 4 bits for EN and PMI out of IA32_FIXED_CTR_CTRL */ 11 | #define fixed_ctrl_field(ctrl_reg, idx) (((ctrl_reg) >> ((idx)*4)) & 0xf) 12 | 13 | struct kvm_event_hw_type_mapping { 14 | u8 eventsel; 15 | u8 unit_mask; 16 | unsigned event_type; 17 | }; 18 | 19 | struct kvm_pmu_ops { 20 | unsigned (*find_arch_event)(struct kvm_pmu *pmu, u8 event_select, 21 | u8 unit_mask); 22 | unsigned (*find_fixed_event)(int idx); 23 | bool (*pmc_is_enabled)(struct kvm_pmc *pmc); 24 | struct kvm_pmc *(*pmc_idx_to_pmc)(struct kvm_pmu *pmu, int pmc_idx); 25 | struct kvm_pmc *(*msr_idx_to_pmc)(struct kvm_vcpu *vcpu, unsigned idx); 26 | int (*is_valid_msr_idx)(struct kvm_vcpu *vcpu, unsigned idx); 27 | bool (*is_valid_msr)(struct kvm_vcpu *vcpu, u32 msr); 28 | int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr, u64 *data); 29 | int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr_info); 30 | void (*refresh)(struct kvm_vcpu *vcpu); 31 | void (*init)(struct kvm_vcpu *vcpu); 32 | void (*reset)(struct kvm_vcpu *vcpu); 33 | }; 34 | 35 | static inline u64 pmc_bitmask(struct kvm_pmc *pmc) 36 | { 37 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); 38 | 39 | return pmu->counter_bitmask[pmc->type]; 40 | } 41 | 42 | static inline u64 pmc_read_counter(struct kvm_pmc *pmc) 43 | { 44 | u64 counter, enabled, running; 45 | 46 | counter = pmc->counter; 47 | if (pmc->perf_event) 48 | counter += perf_event_read_value(pmc->perf_event, 49 | &enabled, &running); 50 | /* FIXME: Scaling needed? */ 51 | return counter & pmc_bitmask(pmc); 52 | } 53 | 54 | static inline void pmc_stop_counter(struct kvm_pmc *pmc) 55 | { 56 | if (pmc->perf_event) { 57 | pmc->counter = pmc_read_counter(pmc); 58 | perf_event_release_kernel(pmc->perf_event); 59 | pmc->perf_event = NULL; 60 | } 61 | } 62 | 63 | static inline bool pmc_is_gp(struct kvm_pmc *pmc) 64 | { 65 | return pmc->type == KVM_PMC_GP; 66 | } 67 | 68 | static inline bool pmc_is_fixed(struct kvm_pmc *pmc) 69 | { 70 | return pmc->type == KVM_PMC_FIXED; 71 | } 72 | 73 | static inline bool pmc_is_enabled(struct kvm_pmc *pmc) 74 | { 75 | return kvm_x86_ops->pmu_ops->pmc_is_enabled(pmc); 76 | } 77 | 78 | /* returns general purpose PMC with the specified MSR. Note that it can be 79 | * used for both PERFCTRn and EVNTSELn; that is why it accepts base as a 80 | * paramenter to tell them apart. 81 | */ 82 | static inline struct kvm_pmc *get_gp_pmc(struct kvm_pmu *pmu, u32 msr, 83 | u32 base) 84 | { 85 | if (msr >= base && msr < base + pmu->nr_arch_gp_counters) 86 | return &pmu->gp_counters[msr - base]; 87 | 88 | return NULL; 89 | } 90 | 91 | /* returns fixed PMC with the specified MSR */ 92 | static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr) 93 | { 94 | int base = MSR_CORE_PERF_FIXED_CTR0; 95 | 96 | if (msr >= base && msr < base + pmu->nr_arch_fixed_counters) 97 | return &pmu->fixed_counters[msr - base]; 98 | 99 | return NULL; 100 | } 101 | 102 | void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel); 103 | void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int fixed_idx); 104 | void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx); 105 | 106 | void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu); 107 | void kvm_pmu_handle_event(struct kvm_vcpu *vcpu); 108 | int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); 109 | int kvm_pmu_is_valid_msr_idx(struct kvm_vcpu *vcpu, unsigned idx); 110 | bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr); 111 | int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data); 112 | int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); 113 | void kvm_pmu_refresh(struct kvm_vcpu *vcpu); 114 | void kvm_pmu_reset(struct kvm_vcpu *vcpu); 115 | void kvm_pmu_init(struct kvm_vcpu *vcpu); 116 | void kvm_pmu_destroy(struct kvm_vcpu *vcpu); 117 | 118 | extern struct kvm_pmu_ops intel_pmu_ops; 119 | extern struct kvm_pmu_ops amd_pmu_ops; 120 | #endif 121 | 122 | #endif /* __KVM_X86_PMU_H */ 123 | -------------------------------------------------------------------------------- /arch/x86/kvm/pmu_amd.c: -------------------------------------------------------------------------------- 1 | /* 2 | * KVM PMU support for AMD 3 | * 4 | * Copyright 2015, Red Hat, Inc. and/or its affiliates. 5 | * 6 | * Author: 7 | * Wei Huang 8 | * 9 | * This work is licensed under the terms of the GNU GPL, version 2. See 10 | * the COPYING file in the top-level directory. 11 | * 12 | * Implementation is based on pmu_intel.c file 13 | */ 14 | #if 0 15 | #include 16 | #include 17 | #include 18 | #include "x86.h" 19 | #include "cpuid.h" 20 | #include "lapic.h" 21 | #include "pmu.h" 22 | 23 | /* duplicated from amd_perfmon_event_map, K7 and above should work. */ 24 | static struct kvm_event_hw_type_mapping amd_event_mapping[] = { 25 | [0] = { 0x76, 0x00, PERF_COUNT_HW_CPU_CYCLES }, 26 | [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS }, 27 | [2] = { 0x7d, 0x07, PERF_COUNT_HW_CACHE_REFERENCES }, 28 | [3] = { 0x7e, 0x07, PERF_COUNT_HW_CACHE_MISSES }, 29 | [4] = { 0xc2, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, 30 | [5] = { 0xc3, 0x00, PERF_COUNT_HW_BRANCH_MISSES }, 31 | [6] = { 0xd0, 0x00, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND }, 32 | [7] = { 0xd1, 0x00, PERF_COUNT_HW_STALLED_CYCLES_BACKEND }, 33 | }; 34 | 35 | static unsigned amd_find_arch_event(struct kvm_pmu *pmu, 36 | u8 event_select, 37 | u8 unit_mask) 38 | { 39 | int i; 40 | 41 | for (i = 0; i < ARRAY_SIZE(amd_event_mapping); i++) 42 | if (amd_event_mapping[i].eventsel == event_select 43 | && amd_event_mapping[i].unit_mask == unit_mask) 44 | break; 45 | 46 | if (i == ARRAY_SIZE(amd_event_mapping)) 47 | return PERF_COUNT_HW_MAX; 48 | 49 | return amd_event_mapping[i].event_type; 50 | } 51 | 52 | /* return PERF_COUNT_HW_MAX as AMD doesn't have fixed events */ 53 | static unsigned amd_find_fixed_event(int idx) 54 | { 55 | return PERF_COUNT_HW_MAX; 56 | } 57 | 58 | /* check if a PMC is enabled by comparing it against global_ctrl bits. Because 59 | * AMD CPU doesn't have global_ctrl MSR, all PMCs are enabled (return TRUE). 60 | */ 61 | static bool amd_pmc_is_enabled(struct kvm_pmc *pmc) 62 | { 63 | return true; 64 | } 65 | 66 | static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) 67 | { 68 | return get_gp_pmc(pmu, MSR_K7_EVNTSEL0 + pmc_idx, MSR_K7_EVNTSEL0); 69 | } 70 | 71 | /* returns 0 if idx's corresponding MSR exists; otherwise returns 1. */ 72 | static int amd_is_valid_msr_idx(struct kvm_vcpu *vcpu, unsigned idx) 73 | { 74 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 75 | 76 | idx &= ~(3u << 30); 77 | 78 | return (idx >= pmu->nr_arch_gp_counters); 79 | } 80 | 81 | /* idx is the ECX register of RDPMC instruction */ 82 | static struct kvm_pmc *amd_msr_idx_to_pmc(struct kvm_vcpu *vcpu, unsigned idx) 83 | { 84 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 85 | struct kvm_pmc *counters; 86 | 87 | idx &= ~(3u << 30); 88 | if (idx >= pmu->nr_arch_gp_counters) 89 | return NULL; 90 | counters = pmu->gp_counters; 91 | 92 | return &counters[idx]; 93 | } 94 | 95 | static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 96 | { 97 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 98 | int ret = false; 99 | 100 | ret = get_gp_pmc(pmu, msr, MSR_K7_PERFCTR0) || 101 | get_gp_pmc(pmu, msr, MSR_K7_EVNTSEL0); 102 | 103 | return ret; 104 | } 105 | 106 | static int amd_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 107 | { 108 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 109 | struct kvm_pmc *pmc; 110 | 111 | /* MSR_K7_PERFCTRn */ 112 | pmc = get_gp_pmc(pmu, msr, MSR_K7_PERFCTR0); 113 | if (pmc) { 114 | *data = pmc_read_counter(pmc); 115 | return 0; 116 | } 117 | /* MSR_K7_EVNTSELn */ 118 | pmc = get_gp_pmc(pmu, msr, MSR_K7_EVNTSEL0); 119 | if (pmc) { 120 | *data = pmc->eventsel; 121 | return 0; 122 | } 123 | 124 | return 1; 125 | } 126 | 127 | static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 128 | { 129 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 130 | struct kvm_pmc *pmc; 131 | u32 msr = msr_info->index; 132 | u64 data = msr_info->data; 133 | 134 | /* MSR_K7_PERFCTRn */ 135 | pmc = get_gp_pmc(pmu, msr, MSR_K7_PERFCTR0); 136 | if (pmc) { 137 | pmc->counter += data - pmc_read_counter(pmc); 138 | return 0; 139 | } 140 | /* MSR_K7_EVNTSELn */ 141 | pmc = get_gp_pmc(pmu, msr, MSR_K7_EVNTSEL0); 142 | if (pmc) { 143 | if (data == pmc->eventsel) 144 | return 0; 145 | if (!(data & pmu->reserved_bits)) { 146 | reprogram_gp_counter(pmc, data); 147 | return 0; 148 | } 149 | } 150 | 151 | return 1; 152 | } 153 | 154 | static void amd_pmu_refresh(struct kvm_vcpu *vcpu) 155 | { 156 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 157 | 158 | pmu->nr_arch_gp_counters = AMD64_NUM_COUNTERS; 159 | pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << 48) - 1; 160 | pmu->reserved_bits = 0xffffffff00200000ull; 161 | /* not applicable to AMD; but clean them to prevent any fall out */ 162 | pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 163 | pmu->nr_arch_fixed_counters = 0; 164 | pmu->version = 0; 165 | pmu->global_status = 0; 166 | } 167 | 168 | static void amd_pmu_init(struct kvm_vcpu *vcpu) 169 | { 170 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 171 | int i; 172 | 173 | for (i = 0; i < AMD64_NUM_COUNTERS ; i++) { 174 | pmu->gp_counters[i].type = KVM_PMC_GP; 175 | pmu->gp_counters[i].vcpu = vcpu; 176 | pmu->gp_counters[i].idx = i; 177 | } 178 | } 179 | 180 | static void amd_pmu_reset(struct kvm_vcpu *vcpu) 181 | { 182 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 183 | int i; 184 | 185 | for (i = 0; i < AMD64_NUM_COUNTERS; i++) { 186 | struct kvm_pmc *pmc = &pmu->gp_counters[i]; 187 | 188 | pmc_stop_counter(pmc); 189 | pmc->counter = pmc->eventsel = 0; 190 | } 191 | } 192 | 193 | struct kvm_pmu_ops amd_pmu_ops = { 194 | .find_arch_event = amd_find_arch_event, 195 | .find_fixed_event = amd_find_fixed_event, 196 | .pmc_is_enabled = amd_pmc_is_enabled, 197 | .pmc_idx_to_pmc = amd_pmc_idx_to_pmc, 198 | .msr_idx_to_pmc = amd_msr_idx_to_pmc, 199 | .is_valid_msr_idx = amd_is_valid_msr_idx, 200 | .is_valid_msr = amd_is_valid_msr, 201 | .get_msr = amd_pmu_get_msr, 202 | .set_msr = amd_pmu_set_msr, 203 | .refresh = amd_pmu_refresh, 204 | .init = amd_pmu_init, 205 | .reset = amd_pmu_reset, 206 | }; 207 | #endif 208 | -------------------------------------------------------------------------------- /arch/x86/kvm/pmu_intel.c: -------------------------------------------------------------------------------- 1 | /* 2 | * KVM PMU support for Intel CPUs 3 | * 4 | * Copyright 2011 Red Hat, Inc. and/or its affiliates. 5 | * Copyright 2019 Google LLC 6 | * 7 | * Authors: 8 | * Avi Kivity 9 | * Gleb Natapov 10 | * 11 | * This work is licensed under the terms of the GNU GPL, version 2. See 12 | * the COPYING file in the top-level directory. 13 | * 14 | */ 15 | #if 0 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include "x86.h" 21 | #include "cpuid.h" 22 | #include "lapic.h" 23 | #include "pmu.h" 24 | 25 | static struct kvm_event_hw_type_mapping intel_arch_events[] = { 26 | /* Index must match CPUID 0x0A.EBX bit vector */ 27 | [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES }, 28 | [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS }, 29 | [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES }, 30 | [3] = { 0x2e, 0x4f, PERF_COUNT_HW_CACHE_REFERENCES }, 31 | [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES }, 32 | [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, 33 | [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES }, 34 | [7] = { 0x00, 0x30, PERF_COUNT_HW_REF_CPU_CYCLES }, 35 | }; 36 | 37 | /* mapping between fixed pmc index and intel_arch_events array */ 38 | static int fixed_pmc_events[] = {1, 0, 7}; 39 | 40 | static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) 41 | { 42 | int i; 43 | 44 | for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 45 | u8 new_ctrl = fixed_ctrl_field(data, i); 46 | u8 old_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, i); 47 | struct kvm_pmc *pmc; 48 | 49 | pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i); 50 | 51 | if (old_ctrl == new_ctrl) 52 | continue; 53 | 54 | reprogram_fixed_counter(pmc, new_ctrl, i); 55 | } 56 | 57 | pmu->fixed_ctr_ctrl = data; 58 | } 59 | 60 | /* function is called when global control register has been updated. */ 61 | static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data) 62 | { 63 | int bit; 64 | u64 diff = pmu->global_ctrl ^ data; 65 | 66 | pmu->global_ctrl = data; 67 | 68 | for_each_set_bit(bit, (size_t *)&diff, X86_PMC_IDX_MAX) 69 | reprogram_counter(pmu, bit); 70 | } 71 | 72 | static unsigned intel_find_arch_event(struct kvm_pmu *pmu, 73 | u8 event_select, 74 | u8 unit_mask) 75 | { 76 | int i; 77 | 78 | for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) 79 | if (intel_arch_events[i].eventsel == event_select 80 | && intel_arch_events[i].unit_mask == unit_mask 81 | && (pmu->available_event_types & (1 << i))) 82 | break; 83 | 84 | if (i == ARRAY_SIZE(intel_arch_events)) 85 | return PERF_COUNT_HW_MAX; 86 | 87 | return intel_arch_events[i].event_type; 88 | } 89 | 90 | static unsigned intel_find_fixed_event(int idx) 91 | { 92 | if (idx >= ARRAY_SIZE(fixed_pmc_events)) 93 | return PERF_COUNT_HW_MAX; 94 | 95 | return intel_arch_events[fixed_pmc_events[idx]].event_type; 96 | } 97 | 98 | /* check if a PMC is enabled by comparing it with globl_ctrl bits. */ 99 | static bool intel_pmc_is_enabled(struct kvm_pmc *pmc) 100 | { 101 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); 102 | 103 | return test_bit(pmc->idx, (size_t *)&pmu->global_ctrl); 104 | } 105 | 106 | static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) 107 | { 108 | if (pmc_idx < INTEL_PMC_IDX_FIXED) 109 | return get_gp_pmc(pmu, MSR_P6_EVNTSEL0 + pmc_idx, 110 | MSR_P6_EVNTSEL0); 111 | else { 112 | u32 idx = pmc_idx - INTEL_PMC_IDX_FIXED; 113 | 114 | return get_fixed_pmc(pmu, idx + MSR_CORE_PERF_FIXED_CTR0); 115 | } 116 | } 117 | 118 | /* returns 0 if idx's corresponding MSR exists; otherwise returns 1. */ 119 | static int intel_is_valid_msr_idx(struct kvm_vcpu *vcpu, unsigned idx) 120 | { 121 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 122 | bool fixed = idx & (1u << 30); 123 | 124 | idx &= ~(3u << 30); 125 | 126 | return (!fixed && idx >= pmu->nr_arch_gp_counters) || 127 | (fixed && idx >= pmu->nr_arch_fixed_counters); 128 | } 129 | 130 | static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, 131 | unsigned idx) 132 | { 133 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 134 | bool fixed = idx & (1u << 30); 135 | struct kvm_pmc *counters; 136 | 137 | idx &= ~(3u << 30); 138 | if (!fixed && idx >= pmu->nr_arch_gp_counters) 139 | return NULL; 140 | if (fixed && idx >= pmu->nr_arch_fixed_counters) 141 | return NULL; 142 | counters = fixed ? pmu->fixed_counters : pmu->gp_counters; 143 | 144 | return &counters[idx]; 145 | } 146 | 147 | static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 148 | { 149 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 150 | int ret; 151 | 152 | switch (msr) { 153 | case MSR_CORE_PERF_FIXED_CTR_CTRL: 154 | case MSR_CORE_PERF_GLOBAL_STATUS: 155 | case MSR_CORE_PERF_GLOBAL_CTRL: 156 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 157 | ret = pmu->version > 1; 158 | break; 159 | default: 160 | ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || 161 | get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) || 162 | get_fixed_pmc(pmu, msr); 163 | break; 164 | } 165 | 166 | return ret; 167 | } 168 | 169 | static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 170 | { 171 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 172 | struct kvm_pmc *pmc; 173 | 174 | switch (msr) { 175 | case MSR_CORE_PERF_FIXED_CTR_CTRL: 176 | *data = pmu->fixed_ctr_ctrl; 177 | return 0; 178 | case MSR_CORE_PERF_GLOBAL_STATUS: 179 | *data = pmu->global_status; 180 | return 0; 181 | case MSR_CORE_PERF_GLOBAL_CTRL: 182 | *data = pmu->global_ctrl; 183 | return 0; 184 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 185 | *data = pmu->global_ovf_ctrl; 186 | return 0; 187 | default: 188 | if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 189 | (pmc = get_fixed_pmc(pmu, msr))) { 190 | *data = pmc_read_counter(pmc); 191 | return 0; 192 | } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 193 | *data = pmc->eventsel; 194 | return 0; 195 | } 196 | } 197 | 198 | return 1; 199 | } 200 | 201 | static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 202 | { 203 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 204 | struct kvm_pmc *pmc; 205 | u32 msr = msr_info->index; 206 | u64 data = msr_info->data; 207 | 208 | switch (msr) { 209 | case MSR_CORE_PERF_FIXED_CTR_CTRL: 210 | if (pmu->fixed_ctr_ctrl == data) 211 | return 0; 212 | if (!(data & 0xfffffffffffff444ull)) { 213 | reprogram_fixed_counters(pmu, data); 214 | return 0; 215 | } 216 | break; 217 | case MSR_CORE_PERF_GLOBAL_STATUS: 218 | if (msr_info->host_initiated) { 219 | pmu->global_status = data; 220 | return 0; 221 | } 222 | break; /* RO MSR */ 223 | case MSR_CORE_PERF_GLOBAL_CTRL: 224 | if (pmu->global_ctrl == data) 225 | return 0; 226 | if (!(data & pmu->global_ctrl_mask)) { 227 | global_ctrl_changed(pmu, data); 228 | return 0; 229 | } 230 | break; 231 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 232 | if (!(data & (pmu->global_ctrl_mask & ~(3ull<<62)))) { 233 | if (!msr_info->host_initiated) 234 | pmu->global_status &= ~data; 235 | pmu->global_ovf_ctrl = data; 236 | return 0; 237 | } 238 | break; 239 | default: 240 | if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 241 | (pmc = get_fixed_pmc(pmu, msr))) { 242 | if (!msr_info->host_initiated) 243 | data = (s64)(s32)data; 244 | pmc->counter += data - pmc_read_counter(pmc); 245 | return 0; 246 | } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 247 | if (data == pmc->eventsel) 248 | return 0; 249 | if (!(data & pmu->reserved_bits)) { 250 | reprogram_gp_counter(pmc, data); 251 | return 0; 252 | } 253 | } 254 | } 255 | 256 | return 1; 257 | } 258 | 259 | static void intel_pmu_refresh(struct kvm_vcpu *vcpu) 260 | { 261 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 262 | struct kvm_cpuid_entry2 *entry; 263 | union cpuid10_eax eax; 264 | union cpuid10_edx edx; 265 | 266 | pmu->nr_arch_gp_counters = 0; 267 | pmu->nr_arch_fixed_counters = 0; 268 | pmu->counter_bitmask[KVM_PMC_GP] = 0; 269 | pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 270 | pmu->version = 0; 271 | pmu->reserved_bits = 0xffffffff00200000ull; 272 | 273 | entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); 274 | if (!entry) 275 | return; 276 | eax.full = entry->eax; 277 | edx.full = entry->edx; 278 | 279 | pmu->version = eax.split.version_id; 280 | if (!pmu->version) 281 | return; 282 | 283 | pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, 284 | INTEL_PMC_MAX_GENERIC); 285 | pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1; 286 | pmu->available_event_types = ~entry->ebx & 287 | ((1ull << eax.split.mask_length) - 1); 288 | 289 | if (pmu->version == 1) { 290 | pmu->nr_arch_fixed_counters = 0; 291 | } else { 292 | pmu->nr_arch_fixed_counters = 293 | min_t(int, edx.split.num_counters_fixed, 294 | INTEL_PMC_MAX_FIXED); 295 | pmu->counter_bitmask[KVM_PMC_FIXED] = 296 | ((u64)1 << edx.split.bit_width_fixed) - 1; 297 | } 298 | 299 | pmu->global_ctrl = ((1 << pmu->nr_arch_gp_counters) - 1) | 300 | (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED); 301 | pmu->global_ctrl_mask = ~pmu->global_ctrl; 302 | 303 | entry = kvm_find_cpuid_entry(vcpu, 7, 0); 304 | if (entry && 305 | (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && 306 | (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) 307 | pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED; 308 | } 309 | 310 | static void intel_pmu_init(struct kvm_vcpu *vcpu) 311 | { 312 | int i; 313 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 314 | 315 | for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { 316 | pmu->gp_counters[i].type = KVM_PMC_GP; 317 | pmu->gp_counters[i].vcpu = vcpu; 318 | pmu->gp_counters[i].idx = i; 319 | } 320 | 321 | for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { 322 | pmu->fixed_counters[i].type = KVM_PMC_FIXED; 323 | pmu->fixed_counters[i].vcpu = vcpu; 324 | pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED; 325 | } 326 | } 327 | 328 | static void intel_pmu_reset(struct kvm_vcpu *vcpu) 329 | { 330 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 331 | int i; 332 | 333 | for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { 334 | struct kvm_pmc *pmc = &pmu->gp_counters[i]; 335 | 336 | pmc_stop_counter(pmc); 337 | pmc->counter = pmc->eventsel = 0; 338 | } 339 | 340 | for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) 341 | pmc_stop_counter(&pmu->fixed_counters[i]); 342 | 343 | pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 344 | pmu->global_ovf_ctrl = 0; 345 | } 346 | 347 | struct kvm_pmu_ops intel_pmu_ops = { 348 | .find_arch_event = intel_find_arch_event, 349 | .find_fixed_event = intel_find_fixed_event, 350 | .pmc_is_enabled = intel_pmc_is_enabled, 351 | .pmc_idx_to_pmc = intel_pmc_idx_to_pmc, 352 | .msr_idx_to_pmc = intel_msr_idx_to_pmc, 353 | .is_valid_msr_idx = intel_is_valid_msr_idx, 354 | .is_valid_msr = intel_is_valid_msr, 355 | .get_msr = intel_pmu_get_msr, 356 | .set_msr = intel_pmu_set_msr, 357 | .refresh = intel_pmu_refresh, 358 | .init = intel_pmu_init, 359 | .reset = intel_pmu_reset, 360 | }; 361 | #endif 362 | -------------------------------------------------------------------------------- /arch/x86/kvm/svm_def.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Kernel-based Virtual Machine driver for Linux 3 | * 4 | * AMD SVM support 5 | * 6 | * Copyright (C) 2006 Qumranet, Inc. 7 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. 8 | * Copyright 2019 Google LLC 9 | * 10 | * Authors: 11 | * Yaniv Kamay 12 | * Avi Kivity 13 | * 14 | * This work is licensed under the terms of the GNU GPL, version 2. See 15 | * the COPYING file in the top-level directory. 16 | * 17 | */ 18 | 19 | #define pr_fmt(fmt) "SVM: " fmt 20 | 21 | #include 22 | 23 | #include "irq.h" 24 | #include "mmu.h" 25 | #include "kvm_cache_regs.h" 26 | #include "x86.h" 27 | #include "cpuid.h" 28 | #include "pmu.h" 29 | 30 | #include 31 | #include 32 | 33 | #include <__asm.h> 34 | 35 | #define IOPM_ALLOC_ORDER 2 36 | #define MSRPM_ALLOC_ORDER 1 37 | 38 | #define SEG_TYPE_LDT 2 39 | #define SEG_TYPE_BUSY_TSS16 3 40 | 41 | #define SVM_FEATURE_NPT (1 << 0) 42 | #define SVM_FEATURE_LBRV (1 << 1) 43 | #define SVM_FEATURE_SVML (1 << 2) 44 | #define SVM_FEATURE_NRIP (1 << 3) 45 | #define SVM_FEATURE_TSC_RATE (1 << 4) 46 | #define SVM_FEATURE_VMCB_CLEAN (1 << 5) 47 | #define SVM_FEATURE_FLUSH_ASID (1 << 6) 48 | #define SVM_FEATURE_DECODE_ASSIST (1 << 7) 49 | #define SVM_FEATURE_PAUSE_FILTER (1 << 10) 50 | 51 | #define SVM_AVIC_DOORBELL 0xc001011b 52 | 53 | #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ 54 | #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ 55 | #define NESTED_EXIT_CONTINUE 2 /* Further checks needed */ 56 | 57 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) 58 | 59 | #define TSC_RATIO_RSVD 0xffffff0000000000ULL 60 | #define TSC_RATIO_MIN 0x0000000000000001ULL 61 | #define TSC_RATIO_MAX 0x000000ffffffffffULL 62 | 63 | #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) 64 | 65 | /* 66 | * 0xff is broadcast, so the max index allowed for physical APIC ID 67 | * table is 0xfe. APIC IDs above 0xff are reserved. 68 | */ 69 | #define AVIC_MAX_PHYSICAL_ID_COUNT 255 70 | 71 | #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 72 | #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 73 | #define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF 74 | 75 | /* AVIC GATAG is encoded using VM and VCPU IDs */ 76 | #define AVIC_VCPU_ID_BITS 8 77 | #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 78 | 79 | #define AVIC_VM_ID_BITS 24 80 | #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 81 | #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 82 | 83 | #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 84 | (y & AVIC_VCPU_ID_MASK)) 85 | #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 86 | #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 87 | 88 | static bool erratum_383_found __read_mostly; 89 | 90 | static const u32 host_save_user_msrs[] = { 91 | #ifdef CONFIG_X86_64 92 | MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, 93 | MSR_FS_BASE, 94 | #endif 95 | MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, 96 | MSR_TSC_AUX, 97 | }; 98 | 99 | #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) 100 | 101 | struct kvm_vcpu; 102 | 103 | struct nested_state { 104 | struct vmcb *hsave; 105 | u64 hsave_msr; 106 | u64 vm_cr_msr; 107 | u64 vmcb; 108 | 109 | /* These are the merged vectors */ 110 | u32 *msrpm; 111 | 112 | /* gpa pointers to the real vectors */ 113 | u64 vmcb_msrpm; 114 | u64 vmcb_iopm; 115 | 116 | /* A VMEXIT is required but not yet emulated */ 117 | bool exit_required; 118 | 119 | /* cache for intercepts of the guest */ 120 | u32 intercept_cr; 121 | u32 intercept_dr; 122 | u32 intercept_exceptions; 123 | u64 intercept; 124 | 125 | /* Nested Paging related state */ 126 | u64 nested_cr3; 127 | }; 128 | 129 | #define MSRPM_OFFSETS 16 130 | static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; 131 | 132 | /* 133 | * Set osvw_len to higher value when updated Revision Guides 134 | * are published and we know what the new status bits are 135 | */ 136 | static uint64_t osvw_len = 4, osvw_status; 137 | 138 | struct vcpu_svm { 139 | struct kvm_vcpu vcpu; 140 | struct vmcb *vmcb; 141 | size_t vmcb_pa; 142 | struct svm_cpu_data *svm_data; 143 | uint64_t asid_generation; 144 | uint64_t sysenter_esp; 145 | uint64_t sysenter_eip; 146 | uint64_t tsc_aux; 147 | 148 | u64 next_rip; 149 | 150 | u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; 151 | struct { 152 | u16 fs; 153 | u16 gs; 154 | u16 ldt; 155 | u64 gs_base; 156 | } host; 157 | 158 | u32 *msrpm; 159 | 160 | ulong nmi_iret_rip; 161 | 162 | struct nested_state nested; 163 | 164 | bool nmi_singlestep; 165 | 166 | unsigned int3_injected; 167 | size_t int3_rip; 168 | 169 | /* cached guest cpuid flags for faster access */ 170 | bool nrips_enabled : 1; 171 | 172 | u32 ldr_reg; 173 | struct page *avic_backing_page; 174 | u64 *avic_physical_id_cache; 175 | bool avic_is_running; 176 | }; 177 | -------------------------------------------------------------------------------- /arch/x86/kvm/tss.h: -------------------------------------------------------------------------------- 1 | #ifndef __TSS_SEGMENT_H 2 | #define __TSS_SEGMENT_H 3 | 4 | struct tss_segment_32 { 5 | u32 prev_task_link; 6 | u32 esp0; 7 | u32 ss0; 8 | u32 esp1; 9 | u32 ss1; 10 | u32 esp2; 11 | u32 ss2; 12 | u32 cr3; 13 | u32 eip; 14 | u32 eflags; 15 | u32 eax; 16 | u32 ecx; 17 | u32 edx; 18 | u32 ebx; 19 | u32 esp; 20 | u32 ebp; 21 | u32 esi; 22 | u32 edi; 23 | u32 es; 24 | u32 cs; 25 | u32 ss; 26 | u32 ds; 27 | u32 fs; 28 | u32 gs; 29 | u32 ldt_selector; 30 | u16 t; 31 | u16 io_map; 32 | }; 33 | 34 | struct tss_segment_16 { 35 | u16 prev_task_link; 36 | u16 sp0; 37 | u16 ss0; 38 | u16 sp1; 39 | u16 ss1; 40 | u16 sp2; 41 | u16 ss2; 42 | u16 ip; 43 | u16 flag; 44 | u16 ax; 45 | u16 cx; 46 | u16 dx; 47 | u16 bx; 48 | u16 sp; 49 | u16 bp; 50 | u16 si; 51 | u16 di; 52 | u16 es; 53 | u16 cs; 54 | u16 ss; 55 | u16 ds; 56 | u16 ldt; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /arch/x86/kvm/x86.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | */ 4 | 5 | #ifndef ARCH_X86_AEHD_X86_H 6 | #define ARCH_X86_AEHD_X86_H 7 | 8 | #include 9 | #include 10 | #include "kvm_cache_regs.h" 11 | #include 12 | 13 | #define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL 14 | 15 | static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu) 16 | { 17 | vcpu->arch.exception.pending = false; 18 | } 19 | 20 | static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector, 21 | bool soft) 22 | { 23 | vcpu->arch.interrupt.pending = true; 24 | vcpu->arch.interrupt.soft = soft; 25 | vcpu->arch.interrupt.nr = vector; 26 | } 27 | 28 | static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu) 29 | { 30 | vcpu->arch.interrupt.pending = false; 31 | } 32 | 33 | static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu) 34 | { 35 | return vcpu->arch.exception.pending || vcpu->arch.interrupt.pending || 36 | vcpu->arch.nmi_injected; 37 | } 38 | 39 | static inline bool kvm_exception_is_soft(unsigned int nr) 40 | { 41 | return (nr == BP_VECTOR) || (nr == OF_VECTOR); 42 | } 43 | 44 | static inline bool is_protmode(struct kvm_vcpu *vcpu) 45 | { 46 | return kvm_read_cr0_bits(vcpu, X86_CR0_PE); 47 | } 48 | 49 | static inline int is_long_mode(struct kvm_vcpu *vcpu) 50 | { 51 | #ifdef CONFIG_X86_64 52 | return vcpu->arch.efer & EFER_LMA; 53 | #else 54 | return 0; 55 | #endif 56 | } 57 | 58 | static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) 59 | { 60 | int cs_db, cs_l; 61 | 62 | if (!is_long_mode(vcpu)) 63 | return false; 64 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 65 | return cs_l; 66 | } 67 | 68 | static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) 69 | { 70 | return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu; 71 | } 72 | 73 | static inline int is_pae(struct kvm_vcpu *vcpu) 74 | { 75 | return (int)kvm_read_cr4_bits(vcpu, X86_CR4_PAE); 76 | } 77 | 78 | static inline int is_pse(struct kvm_vcpu *vcpu) 79 | { 80 | return (int)kvm_read_cr4_bits(vcpu, X86_CR4_PSE); 81 | } 82 | 83 | static inline int is_paging(struct kvm_vcpu *vcpu) 84 | { 85 | return likely((int)kvm_read_cr0_bits(vcpu, X86_CR0_PG)); 86 | } 87 | 88 | static inline u32 bit(int bitno) 89 | { 90 | return 1 << (bitno & 31); 91 | } 92 | 93 | static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, 94 | gva_t gva, gfn_t gfn, unsigned access) 95 | { 96 | vcpu->arch.mmio_gva = gva & PAGE_MASK; 97 | vcpu->arch.access = access; 98 | vcpu->arch.mmio_gfn = gfn; 99 | vcpu->arch.mmio_gen = kvm_memslots(vcpu->kvm)->generation; 100 | } 101 | 102 | static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu) 103 | { 104 | return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation; 105 | } 106 | 107 | /* 108 | * Clear the mmio cache info for the given gva. If gva is MMIO_GVA_ANY, we 109 | * clear all mmio cache info. 110 | */ 111 | #define MMIO_GVA_ANY (~(gva_t)0) 112 | 113 | static inline void vcpu_clear_mmio_info(struct kvm_vcpu *vcpu, gva_t gva) 114 | { 115 | if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK)) 116 | return; 117 | 118 | vcpu->arch.mmio_gva = 0; 119 | } 120 | 121 | static inline bool vcpu_match_mmio_gva(struct kvm_vcpu *vcpu, size_t gva) 122 | { 123 | if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva && 124 | vcpu->arch.mmio_gva == (gva & PAGE_MASK)) 125 | return true; 126 | 127 | return false; 128 | } 129 | 130 | static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) 131 | { 132 | if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn && 133 | vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT) 134 | return true; 135 | 136 | return false; 137 | } 138 | 139 | static inline size_t kvm_register_readl(struct kvm_vcpu *vcpu, 140 | enum kvm_reg reg) 141 | { 142 | size_t val = kvm_register_read(vcpu, reg); 143 | 144 | return is_64_bit_mode(vcpu) ? val : (u32)val; 145 | } 146 | 147 | static inline void kvm_register_writel(struct kvm_vcpu *vcpu, 148 | enum kvm_reg reg, 149 | size_t val) 150 | { 151 | if (!is_64_bit_mode(vcpu)) 152 | val = (u32)val; 153 | kvm_register_write(vcpu, reg, val); 154 | } 155 | 156 | static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) 157 | { 158 | return !(kvm->arch.disabled_quirks & quirk); 159 | } 160 | 161 | void kvm_before_handle_nmi(struct kvm_vcpu *vcpu); 162 | void kvm_after_handle_nmi(struct kvm_vcpu *vcpu); 163 | void kvm_set_pending_timer(struct kvm_vcpu *vcpu); 164 | int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); 165 | 166 | void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr); 167 | u64 get_kvmclock_ns(struct kvm *kvm); 168 | 169 | int kvm_read_guest_virt(struct kvm_vcpu *vcpu, 170 | gva_t addr, void *val, unsigned int bytes, 171 | struct x86_exception *exception); 172 | 173 | int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, 174 | gva_t addr, void *val, unsigned int bytes, 175 | struct x86_exception *exception); 176 | 177 | void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu); 178 | u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn); 179 | bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data); 180 | int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); 181 | int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); 182 | bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, 183 | int page_num); 184 | bool kvm_vector_hashing_enabled(void); 185 | 186 | #define AEHD_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 187 | | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 188 | | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ 189 | | XFEATURE_MASK_PKRU) 190 | extern u64 host_xcr0; 191 | 192 | extern u64 kvm_supported_xcr0(void); 193 | 194 | extern unsigned int min_timer_period_us; 195 | 196 | extern unsigned int lapic_timer_advance_ns; 197 | 198 | extern int kvm_no_apic_vcpu; 199 | 200 | /* Same "calling convention" as do_div: 201 | * - divide (n << 32) by base 202 | * - put result in n 203 | * - return remainder 204 | */ 205 | #define do_shl32_div32(n, base) \ 206 | ({ \ 207 | u32 __quot, __rem; \ 208 | asm("divl %2" : "=a" (__quot), "=d" (__rem) \ 209 | : "rm" (base), "0" (0), "1" ((u32) n)); \ 210 | n = __quot; \ 211 | __rem; \ 212 | }) 213 | 214 | #endif 215 | -------------------------------------------------------------------------------- /asmgen/asmgen.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | 4 | * This program is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License 6 | * version 2 as published by the Free Software Foundation. 7 | 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | */ 13 | 14 | /* 15 | * This program prepares data definitions needed by assembly code 16 | * in driver. 17 | */ 18 | #pragma warning(disable:4146) 19 | #pragma warning(disable:4013) 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #define ASM_GEN_OFFSET(name, type, field) \ 29 | printf("\t" #name "\tEQU 0%zxh\n", offsetof(type, field)) 30 | 31 | int main(void) 32 | { 33 | _ReadWriteBarrier(); 34 | printf("; This is generated by asmgen\n"); 35 | printf("; Please make sure to rerun asmgen after updating\n"); 36 | printf("; key data structures used by both assembly and C.\n\n"); 37 | 38 | //struct vcpu_vmx 39 | printf("\n"); 40 | ASM_GEN_OFFSET(VMX_TO_LAUNCHED, struct vcpu_vmx, __launched); 41 | ASM_GEN_OFFSET(VMX_TO_FAIL, struct vcpu_vmx, fail); 42 | ASM_GEN_OFFSET(VMX_TO_RSP, struct vcpu_vmx, host_rsp); 43 | ASM_GEN_OFFSET(VMX_TO_RAX, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX]); 44 | ASM_GEN_OFFSET(VMX_TO_RBX, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX]); 45 | ASM_GEN_OFFSET(VMX_TO_RCX, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX]); 46 | ASM_GEN_OFFSET(VMX_TO_RDX, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX]); 47 | ASM_GEN_OFFSET(VMX_TO_RSI, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI]); 48 | ASM_GEN_OFFSET(VMX_TO_RDI, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI]); 49 | ASM_GEN_OFFSET(VMX_TO_RBP, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP]); 50 | ASM_GEN_OFFSET(VMX_TO_R8, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8]); 51 | ASM_GEN_OFFSET(VMX_TO_R9, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9]); 52 | ASM_GEN_OFFSET(VMX_TO_R10, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10]); 53 | ASM_GEN_OFFSET(VMX_TO_R11, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11]); 54 | ASM_GEN_OFFSET(VMX_TO_R12, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12]); 55 | ASM_GEN_OFFSET(VMX_TO_R13, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13]); 56 | ASM_GEN_OFFSET(VMX_TO_R14, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14]); 57 | ASM_GEN_OFFSET(VMX_TO_R15, struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15]); 58 | ASM_GEN_OFFSET(VMX_TO_CR2, struct vcpu_vmx, vcpu.arch.cr2); 59 | 60 | //struct vcpu_svm 61 | ASM_GEN_OFFSET(SVM_TO_VMCB_PA, struct vcpu_svm, vmcb_pa); 62 | ASM_GEN_OFFSET(SVM_TO_RBX, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX]); 63 | ASM_GEN_OFFSET(SVM_TO_RCX, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX]); 64 | ASM_GEN_OFFSET(SVM_TO_RDX, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX]); 65 | ASM_GEN_OFFSET(SVM_TO_RSI, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI]); 66 | ASM_GEN_OFFSET(SVM_TO_RDI, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI]); 67 | ASM_GEN_OFFSET(SVM_TO_RBP, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]); 68 | ASM_GEN_OFFSET(SVM_TO_R8, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8]); 69 | ASM_GEN_OFFSET(SVM_TO_R9, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9]); 70 | ASM_GEN_OFFSET(SVM_TO_R10, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10]); 71 | ASM_GEN_OFFSET(SVM_TO_R11, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11]); 72 | ASM_GEN_OFFSET(SVM_TO_R12, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12]); 73 | ASM_GEN_OFFSET(SVM_TO_R13, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13]); 74 | ASM_GEN_OFFSET(SVM_TO_R14, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14]); 75 | ASM_GEN_OFFSET(SVM_TO_R15, struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]); 76 | 77 | ASM_GEN_OFFSET(CXT_TO_DST, struct x86_emulate_ctxt, dst.val); 78 | ASM_GEN_OFFSET(CXT_TO_SRC, struct x86_emulate_ctxt, src.val); 79 | ASM_GEN_OFFSET(CXT_TO_SRC2, struct x86_emulate_ctxt, src2.val); 80 | } 81 | -------------------------------------------------------------------------------- /asmgen/asmgen.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | x64 7 | 8 | 9 | Release 10 | x64 11 | 12 | 13 | 14 | {07877F58-4EE6-4C6E-A6AA-AF42B477A5BE} 15 | asmgen 16 | 10.0 17 | 18 | 19 | 20 | Application 21 | true 22 | v142 23 | MultiByte 24 | false 25 | 26 | 27 | Application 28 | false 29 | v142 30 | true 31 | MultiByte 32 | false 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 48 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 49 | 50 | 51 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 52 | $(ProjectDir)..\build\$(ProjectName)\$(Platform)\$(Configuration)\ 53 | 54 | 55 | 56 | Level3 57 | Disabled 58 | true 59 | $(KIT_SHARED_IncludePath)\..\km;$(ProjectDIr)..\arch\x86\include;$(ProjectDIr)..\include;$(ProjectDIr)..;%(AdditionalIncludeDirectories) 60 | CONFIG_X86_64;CONFIG_X86_LOCAL_APIC;WINNT=1;_AMD64_;%(PreprocessorDefinitions) 61 | 62 | 63 | 64 | 65 | Level3 66 | MaxSpeed 67 | true 68 | true 69 | true 70 | $(KIT_SHARED_IncludePath)\..\km;$(ProjectDIr)..\arch\x86\include;$(ProjectDIr)..\include;$(ProjectDIr)..;%(AdditionalIncludeDirectories) 71 | CONFIG_X86_64;CONFIG_X86_LOCAL_APIC;WINNT=1;_AMD64_;%(PreprocessorDefinitions) 72 | 73 | 74 | true 75 | true 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /asmgen/asmgen.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /include/kvm/iodev.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation; either version 2 of the License. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | #ifndef __KVM_IODEV_H__ 18 | #define __KVM_IODEV_H__ 19 | 20 | #include 21 | 22 | struct kvm_io_device; 23 | struct kvm_vcpu; 24 | 25 | /** 26 | * kvm_io_device_ops are called under kvm slots_lock. 27 | * read and write handlers return 0 if the transaction has been handled, 28 | * or non-zero to have it passed to the next device. 29 | **/ 30 | struct kvm_io_device_ops { 31 | int (*read)(struct kvm_vcpu *vcpu, 32 | struct kvm_io_device *this, 33 | gpa_t addr, 34 | int len, 35 | void *val); 36 | int (*write)(struct kvm_vcpu *vcpu, 37 | struct kvm_io_device *this, 38 | gpa_t addr, 39 | int len, 40 | const void *val); 41 | void (*destructor)(struct kvm_io_device *this); 42 | }; 43 | 44 | 45 | struct kvm_io_device { 46 | const struct kvm_io_device_ops *ops; 47 | }; 48 | 49 | static inline void kvm_iodevice_init(struct kvm_io_device *dev, 50 | const struct kvm_io_device_ops *ops) 51 | { 52 | dev->ops = ops; 53 | } 54 | 55 | static inline int kvm_iodevice_read(struct kvm_vcpu *vcpu, 56 | struct kvm_io_device *dev, gpa_t addr, 57 | int l, void *v) 58 | { 59 | return dev->ops->read ? dev->ops->read(vcpu, dev, addr, l, v) 60 | : -EOPNOTSUPP; 61 | } 62 | 63 | static inline int kvm_iodevice_write(struct kvm_vcpu *vcpu, 64 | struct kvm_io_device *dev, gpa_t addr, 65 | int l, const void *v) 66 | { 67 | return dev->ops->write ? dev->ops->write(vcpu, dev, addr, l, v) 68 | : -EOPNOTSUPP; 69 | } 70 | 71 | static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) 72 | { 73 | if (dev->ops->destructor) 74 | dev->ops->destructor(dev); 75 | } 76 | 77 | #endif /* __KVM_IODEV_H__ */ 78 | -------------------------------------------------------------------------------- /include/linux/kvm_types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation; either version 2 of the License. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program; if not, write to the Free Software 15 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 16 | * 17 | */ 18 | 19 | #ifndef __KVM_TYPES_H__ 20 | #define __KVM_TYPES_H__ 21 | 22 | struct kvm; 23 | struct kvm_interrupt; 24 | struct kvm_irq_routing_table; 25 | struct kvm_memory_slot; 26 | struct kvm_one_reg; 27 | struct kvm_run; 28 | struct kvm_userspace_memory_region; 29 | struct kvm_vcpu; 30 | struct kvm_memslots; 31 | 32 | enum kvm_mr_change; 33 | 34 | #include 35 | 36 | /* 37 | * Address types: 38 | * 39 | * gva - guest virtual address 40 | * gpa - guest physical address 41 | * gfn - guest frame number 42 | * hva - host virtual address 43 | * hpa - host physical address 44 | * hfn - host frame number 45 | */ 46 | 47 | typedef size_t gva_t; 48 | typedef u64 gpa_t; 49 | typedef u64 gfn_t; 50 | 51 | typedef size_t hva_t; 52 | typedef u64 hpa_t; 53 | typedef u64 hfn_t; 54 | 55 | typedef hfn_t kvm_pfn_t; 56 | 57 | struct gfn_to_hva_cache { 58 | u64 generation; 59 | gpa_t gpa; 60 | size_t hva; 61 | size_t len; 62 | struct kvm_memory_slot *memslot; 63 | }; 64 | 65 | #endif /* __KVM_TYPES_H__ */ 66 | -------------------------------------------------------------------------------- /package/make_package.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | 3 | # This program is free software; you can redistribute it and/or 4 | # modify it under the terms of the GNU General Public License 5 | # version 2 as published by the Free Software Foundation. 6 | 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | 12 | # Package source codes together with release binary and symbols 13 | # Run under WSL on Windows 10. 14 | 15 | #!/bin/bash 16 | cd .. 17 | zip -r package/aehd-`date +%Y%m%d-%H%M`.zip Release/ arch/ asmgen/ aehd* virt/ __asm.* ntkrutils.* include/ assembly/ 18 | -------------------------------------------------------------------------------- /sign/aehd/.gitignore: -------------------------------------------------------------------------------- 1 | setup.* 2 | disk1/ 3 | aehd.sys 4 | -------------------------------------------------------------------------------- /sign/aehd/aehd.ddf: -------------------------------------------------------------------------------- 1 | ; Copyright 2019 Google LLC 2 | 3 | ; This program is free software; you can redistribute it and/or 4 | ; modify it under the terms of the GNU General Public License 5 | ; version 2 as published by the Free Software Foundation. 6 | 7 | ; This program is distributed in the hope that it will be useful, 8 | ; but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | ; GNU General Public License for more details. 11 | 12 | .OPTION EXPLICIT ; Generate errors 13 | .Set CabinetFileCountThreshold=0 14 | .Set FolderFileCountThreshold=0 15 | .Set FolderSizeThreshold=0 16 | .Set MaxCabinetSize=0 17 | .Set MaxDiskFileCount=0 18 | .Set MaxDiskSize=0 19 | .Set CompressionType=MSZIP 20 | .Set Cabinet=on 21 | .Set Compress=on 22 | .Set CabinetNameTemplate=aehd.cab 23 | .Set DestinationDir=aehd 24 | ;Specify files to be included in cab file 25 | aehd.Inf 26 | aehd.Sys 27 | -------------------------------------------------------------------------------- /sign/aehd/aehd.inf: -------------------------------------------------------------------------------- 1 | ; Copyright 2019 Google LLC 2 | 3 | ; This program is free software; you can redistribute it and/or 4 | ; modify it under the terms of the GNU General Public License 5 | ; version 2 as published by the Free Software Foundation. 6 | 7 | ; This program is distributed in the hope that it will be useful, 8 | ; but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | ; GNU General Public License for more details. 11 | 12 | [Version] 13 | Signature = "$Windows NT$" 14 | Provider = Google 15 | Class = System 16 | ClassGuid = {4d36e97d-e325-11ce-bfc1-08002be10318} 17 | DriverVer = 02/10/2023,2.1.0.0 18 | DriverPackageType = KernelService 19 | CatalogFile = aehd.cat 20 | 21 | [DestinationDirs] 22 | DefaultDestDir = 12 23 | 24 | [DefaultInstall.NTamd64] 25 | CopyFiles = aehd.DriverFiles 26 | 27 | [DefaultInstall.NTamd64.Services] 28 | AddService = aehd,0x00000002,aehd.Service 29 | 30 | [DefaultUninstall.NTamd64] 31 | LegacyUninstall=1 32 | DelFiles = aehd.DriverFiles 33 | 34 | [DefaultUninstall.NTamd64.Services] 35 | DelService = aehd,0x200 36 | 37 | [SourceDisksFiles] 38 | aehd.sys = 1 39 | 40 | [SourceDisksNames] 41 | 1 = %DISK_NAME%, 42 | 43 | [aehd.DriverFiles] 44 | aehd.sys 45 | 46 | [aehd.Service] 47 | DisplayName = Android Emulator hypervisor driver Service 48 | ServiceType = 1 ; SERVICE_KERNEL_DRIVER 49 | StartType = 1 ; SERVICE_SYSTEM_START 50 | ErrorControl = 1 ; SERVICE_ERROR_NORMAL 51 | ServiceBinary = %12%\aehd.sys 52 | 53 | [Strings] 54 | DISK_NAME = "Android Emulator hypervisor driver Installation Media" 55 | -------------------------------------------------------------------------------- /virt/kvm/irqchip.c: -------------------------------------------------------------------------------- 1 | /* 2 | * irqchip.c: Common API for in kernel interrupt controllers 3 | * Copyright (c) 2007, Intel Corporation. 4 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. 5 | * Copyright (c) 2013, Alexander Graf 6 | * Copyright 2019 Google LLC 7 | * 8 | * This program is free software; you can redistribute it and/or modify it 9 | * under the terms and conditions of the GNU General Public License, 10 | * version 2, as published by the Free Software Foundation. 11 | * 12 | * This program is distributed in the hope it will be useful, but WITHOUT 13 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 15 | * more details. 16 | * 17 | * You should have received a copy of the GNU General Public License along with 18 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 19 | * Place - Suite 330, Boston, MA 02111-1307 USA. 20 | * 21 | * This file is derived from virt/kvm/irq_comm.c. 22 | * 23 | * Authors: 24 | * Yaozu (Eddie) Dong 25 | * Alexander Graf 26 | */ 27 | 28 | #include 29 | #include "arch\x86\kvm\irq.h" 30 | 31 | int kvm_irq_map_gsi(struct kvm *kvm, 32 | struct kvm_kernel_irq_routing_entry *entries, int gsi) 33 | { 34 | struct kvm_irq_routing_table *irq_rt; 35 | struct kvm_kernel_irq_routing_entry *e; 36 | int n = 0; 37 | 38 | irq_rt = kvm->irq_routing; 39 | 40 | if (irq_rt && gsi < irq_rt->nr_rt_entries) { 41 | #define LIST_ENTRY_TYPE_INFO struct kvm_kernel_irq_routing_entry 42 | hlist_for_each_entry(e, &irq_rt->map[gsi], link) { 43 | entries[n] = *e; 44 | ++n; 45 | } 46 | #undef LIST_ENTRY_TYPE_INFO 47 | } 48 | 49 | return n; 50 | } 51 | 52 | int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin) 53 | { 54 | struct kvm_irq_routing_table *irq_rt; 55 | 56 | irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 57 | return irq_rt->chip[irqchip][pin]; 58 | } 59 | 60 | int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi) 61 | { 62 | struct kvm_kernel_irq_routing_entry route; 63 | 64 | if (!irqchip_in_kernel(kvm) || (msi->flags & ~AEHD_MSI_VALID_DEVID)) 65 | return -EINVAL; 66 | 67 | route.msi.address_lo = msi->address_lo; 68 | route.msi.address_hi = msi->address_hi; 69 | route.msi.data = msi->data; 70 | route.msi.flags = msi->flags; 71 | route.msi.devid = msi->devid; 72 | 73 | return kvm_set_msi(&route, kvm, AEHD_USERSPACE_IRQ_SOURCE_ID, 1, false); 74 | } 75 | 76 | /* 77 | * Return value: 78 | * < 0 Interrupt was ignored (masked or not delivered for other reasons) 79 | * = 0 Interrupt was coalesced (previous irq is still pending) 80 | * > 0 Number of CPUs interrupt was delivered to 81 | */ 82 | int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 83 | bool line_status) 84 | { 85 | struct kvm_kernel_irq_routing_entry irq_set[AEHD_NR_IRQCHIPS]; 86 | int ret = -1, i, idx; 87 | 88 | /* Not possible to detect if the guest uses the PIC or the 89 | * IOAPIC. So set the bit in both. The guest will ignore 90 | * writes to the unused one. 91 | */ 92 | idx = srcu_read_lock(&kvm->irq_srcu); 93 | i = kvm_irq_map_gsi(kvm, irq_set, irq); 94 | srcu_read_unlock(&kvm->irq_srcu, idx); 95 | 96 | while (i--) { 97 | int r; 98 | r = irq_set[i].set(&irq_set[i], kvm, irq_source_id, level, 99 | line_status); 100 | if (r < 0) 101 | continue; 102 | 103 | ret = r + ((ret < 0) ? 0 : ret); 104 | } 105 | 106 | return ret; 107 | } 108 | 109 | static void free_irq_routing_table(struct kvm_irq_routing_table *rt) 110 | { 111 | int i; 112 | 113 | if (!rt) 114 | return; 115 | 116 | for (i = 0; i < rt->nr_rt_entries; ++i) { 117 | struct kvm_kernel_irq_routing_entry *e; 118 | struct hlist_node *n; 119 | 120 | #define LIST_ENTRY_TYPE_INFO struct kvm_kernel_irq_routing_entry 121 | hlist_for_each_entry_safe(e, n, &rt->map[i], link) { 122 | n = e->link.next; 123 | hlist_del(&e->link); 124 | kfree(e); 125 | } 126 | #undef LIST_ENTRY_TYPE_INFO 127 | } 128 | 129 | kfree(rt); 130 | } 131 | 132 | void kvm_free_irq_routing(struct kvm *kvm) 133 | { 134 | /* Called only during vm destruction. Nobody can use the pointer 135 | at this stage */ 136 | struct kvm_irq_routing_table *rt = kvm->irq_routing; 137 | free_irq_routing_table(rt); 138 | } 139 | 140 | static int setup_routing_entry(struct kvm *kvm, 141 | struct kvm_irq_routing_table *rt, 142 | struct kvm_kernel_irq_routing_entry *e, 143 | const struct kvm_irq_routing_entry *ue) 144 | { 145 | int r = -EINVAL; 146 | struct kvm_kernel_irq_routing_entry *ei; 147 | 148 | /* 149 | * Do not allow GSI to be mapped to the same irqchip more than once. 150 | * Allow only one to one mapping between GSI and non-irqchip routing. 151 | */ 152 | #define LIST_ENTRY_TYPE_INFO struct kvm_kernel_irq_routing_entry 153 | hlist_for_each_entry(ei, &rt->map[ue->gsi], link) 154 | if (ei->type != AEHD_IRQ_ROUTING_IRQCHIP || 155 | ue->type != AEHD_IRQ_ROUTING_IRQCHIP || 156 | ue->u.irqchip.irqchip == ei->irqchip.irqchip) 157 | return r; 158 | #undef LIST_ENTRY_TYPE_INFO 159 | 160 | e->gsi = ue->gsi; 161 | e->type = ue->type; 162 | r = kvm_set_routing_entry(kvm, e, ue); 163 | if (r) 164 | goto out; 165 | if (e->type == AEHD_IRQ_ROUTING_IRQCHIP) 166 | rt->chip[e->irqchip.irqchip][e->irqchip.pin] = e->gsi; 167 | 168 | hlist_add_head(&e->link, &rt->map[e->gsi]); 169 | r = 0; 170 | out: 171 | return r; 172 | } 173 | 174 | void kvm_arch_irq_routing_update_default(struct kvm *kvm) 175 | { 176 | } 177 | #pragma comment(linker, "/alternatename:kvm_arch_irq_routing_update=kvm_arch_irq_routing_update_default") 178 | 179 | int kvm_set_irq_routing(struct kvm *kvm, 180 | const struct kvm_irq_routing_entry *ue, 181 | unsigned nr, 182 | unsigned flags) 183 | { 184 | struct kvm_irq_routing_table *new, *old; 185 | struct kvm_kernel_irq_routing_entry *e; 186 | u32 i, j, nr_rt_entries = 0; 187 | int r; 188 | 189 | for (i = 0; i < nr; ++i) { 190 | if (ue[i].gsi >= AEHD_MAX_IRQ_ROUTES) 191 | return -EINVAL; 192 | nr_rt_entries = max(nr_rt_entries, ue[i].gsi); 193 | } 194 | 195 | nr_rt_entries += 1; 196 | 197 | new = kzalloc(sizeof(*new) + (nr_rt_entries * sizeof(struct hlist_head)), 198 | GFP_KERNEL); 199 | 200 | if (!new) 201 | return -ENOMEM; 202 | 203 | new->nr_rt_entries = nr_rt_entries; 204 | for (i = 0; i < AEHD_NR_IRQCHIPS; i++) 205 | for (j = 0; j < AEHD_IRQCHIP_NUM_PINS; j++) 206 | new->chip[i][j] = -1; 207 | 208 | for (i = 0; i < nr; ++i) { 209 | r = -ENOMEM; 210 | e = kzalloc(sizeof(*e), GFP_KERNEL); 211 | if (!e) 212 | goto out; 213 | 214 | r = -EINVAL; 215 | switch (ue->type) { 216 | case AEHD_IRQ_ROUTING_MSI: 217 | if (ue->flags & ~AEHD_MSI_VALID_DEVID) 218 | goto free_entry; 219 | break; 220 | default: 221 | if (ue->flags) 222 | goto free_entry; 223 | break; 224 | } 225 | r = setup_routing_entry(kvm, new, e, ue); 226 | if (r) 227 | goto free_entry; 228 | ++ue; 229 | } 230 | 231 | mutex_lock(&kvm->irq_lock); 232 | old = kvm->irq_routing; 233 | kvm->irq_routing = new; 234 | mutex_unlock(&kvm->irq_lock); 235 | 236 | kvm_arch_post_irq_routing_update(kvm); 237 | 238 | synchronize_srcu_expedited(&kvm->irq_srcu); 239 | 240 | new = old; 241 | r = 0; 242 | goto out; 243 | 244 | free_entry: 245 | kfree(e); 246 | out: 247 | free_irq_routing_table(new); 248 | 249 | return r; 250 | } 251 | --------------------------------------------------------------------------------