├── CMakeLists.txt ├── LIBS2EConfig.cmake.in ├── LICENSE ├── Makefile ├── Makefile.target ├── README.md ├── configure └── src ├── CMakeLists.txt ├── FileDescriptorManager.h ├── crashdump.cpp ├── libs2e.cpp ├── libs2e.h ├── mapfile ├── s2e-kvm-io.cpp ├── s2e-kvm-state.cpp ├── s2e-kvm-trace.cpp ├── s2e-kvm-trace.h ├── s2e-kvm-vcpu.cpp ├── s2e-kvm-vcpu.h ├── s2e-kvm-vm.cpp ├── s2e-kvm-vm.h ├── s2e-kvm.cpp ├── s2e-kvm.h ├── s2e-libcpu-interface.cpp ├── syscalls.h └── test.cpp /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Cyberhaven 2 | # All rights reserved. 3 | # 4 | # Licensed under the Cyberhaven Research License Agreement. 5 | 6 | 7 | cmake_minimum_required(VERSION 3.4.3) 8 | 9 | project(LIBS2E) 10 | set(LIBS2E_VERSION_MAJOR 2) 11 | set(LIBS2E_VERSION_MINOR 0) 12 | set(LIBS2E_VERSION_PATCH 0) 13 | set(LIBS2E_PACKAGE_VERSION 14 | "${LIBS2E_VERSION_MAJOR}.${LIBS2E_VERSION_MINOR}.${LIBS2E_VERSION_PATCH}") 15 | 16 | include(CMakePackageConfigHelpers) 17 | set(CMAKE_VERSION_FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Version.cmake") 18 | write_basic_package_version_file(${CMAKE_VERSION_FILE} 19 | VERSION ${LIBS2E_PACKAGE_VERSION} 20 | COMPATIBILITY AnyNewerVersion) 21 | 22 | set(CMAKE_CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake") 23 | set(LIBS2E_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") 24 | set(LIBS2E_LIBRARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/src") 25 | configure_file(LIBS2EConfig.cmake.in ${CMAKE_CONFIG_FILE} @ONLY) 26 | 27 | find_package(PkgConfig REQUIRED) 28 | 29 | pkg_check_modules(GLIB_PKG glib-2.0) 30 | include_directories(${GLIB_PKG_INCLUDE_DIRS}) 31 | 32 | set(Protobuf_USE_STATIC_LIBS ON) 33 | include(FindProtobuf) 34 | find_package(Protobuf REQUIRED) 35 | 36 | # Old CMake does not support static libraries, so we have to patch the library manually 37 | string(REPLACE .so .a PROTOBUF_LIBRARIES "${PROTOBUF_LIBRARIES}") 38 | 39 | 40 | ##### LLVM ##### 41 | find_package(LLVM REQUIRED) 42 | message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") 43 | 44 | add_definitions(${LLVM_DEFINITIONS}) 45 | 46 | include_directories("include" ${LLVM_INCLUDE_DIRS}) 47 | 48 | llvm_map_components_to_libnames(LLVM_LIBS all) 49 | 50 | set(LLVM_CONFIG "${LLVM_TOOLS_BINARY_DIR}/llvm-config" 51 | CACHE PATH "Path to llvm-config") 52 | execute_process(COMMAND ${LLVM_CONFIG} "--cxxflags" 53 | RESULT_VARIABLE LLVM_CONFIG_RESULT 54 | OUTPUT_VARIABLE LLVM_CXXFLAGS 55 | OUTPUT_STRIP_TRAILING_WHITESPACE) 56 | 57 | ################## 58 | 59 | find_package(KLEE REQUIRED) 60 | message(STATUS "Found klee ${KLEE_PACKAGE_VERSION}") 61 | 62 | find_package(LIBQ REQUIRED) 63 | message(STATUS "Found libq ${LIBQ_PACKAGE_VERSION}") 64 | 65 | find_package(LIBCPU REQUIRED) 66 | message(STATUS "Found libcpu ${LIBCPU_PACKAGE_VERSION}") 67 | 68 | find_package(LIBTCG REQUIRED) 69 | message(STATUS "Found libtcg ${LIBTCG_PACKAGE_VERSION}") 70 | 71 | find_package(FSIGCXX REQUIRED) 72 | message(STATUS "Found fsigc++ ${FSIGCXX_PACKAGE_VERSION}") 73 | 74 | find_package(VMI REQUIRED) 75 | message(STATUS "Found libvmi ${VMI_PACKAGE_VERSION}") 76 | 77 | find_package(LIBCOROUTINE REQUIRED) 78 | message(STATUS "Found libcoroutine ${LIBCOROUTINE_PACKAGE_VERSION}") 79 | 80 | if(WITH_TARGET MATCHES "s2e") 81 | # TODO: look at libcpu compile options to figure this out§ 82 | find_package(LIBS2ECORE REQUIRED) 83 | message(STATUS "Found s2e core ${LIBS2ECORE_PACKAGE_VERSION}") 84 | 85 | find_package(LIBS2EPLUGINS REQUIRED) 86 | message(STATUS "Found s2e plugins ${LIBS2EPLUGINS_PACKAGE_VERSION}") 87 | 88 | include_directories(${LIBS2ECORE_INCLUDE_DIR} ${LIBS2EPLUGINS_INCLUDE_DIR}) 89 | endif() 90 | 91 | # We want to keep NDEBUG in all builds 92 | foreach (flags_var_to_scrub 93 | CMAKE_CXX_FLAGS_RELEASE 94 | CMAKE_CXX_FLAGS_RELWITHDEBINFO 95 | CMAKE_CXX_FLAGS_MINSIZEREL 96 | CMAKE_C_FLAGS_RELEASE 97 | CMAKE_C_FLAGS_RELWITHDEBINFO 98 | CMAKE_C_FLAGS_MINSIZEREL) 99 | string (REGEX REPLACE "(^| )[/-]D *NDEBUG($| )" " " 100 | "${flags_var_to_scrub}" "${${flags_var_to_scrub}}") 101 | endforeach() 102 | 103 | 104 | 105 | include_directories(${GLIB_PKG_INCLUDE_DIRS} 106 | ${LIBTCG_INCLUDE_DIR} 107 | ${LIBCPU_INCLUDE_DIR} 108 | ${FSIGCXX_INCLUDE_DIR} 109 | ${LIBQ_INCLUDE_DIR} 110 | ${LIBCOROUTINE_INCLUDE_DIR} 111 | ${VMI_INCLUDE_DIR}) 112 | 113 | link_directories(${GLIB_PKG_LIBRARY_DIRS} 114 | ${LIBTCG_LIBRARY_DIR} 115 | ${LIBCPU_LIBRARY_DIR} 116 | ${LIBQ_LIBRARY_DIR} 117 | ${KLEE_LIBRARY_DIR} 118 | ${LUA_DIR} 119 | ${Z3_DIR} 120 | ${FSIGCXX_LIBRARY_DIR} 121 | ${LIBCOROUTINE_LIBRARY_DIR} 122 | ${LIBS2ECORE_LIBRARY_DIR} 123 | ${LIBS2EPLUGINS_LIBRARY_DIR} 124 | ${VMI_LIBRARY_DIR}) 125 | 126 | add_subdirectory(src) 127 | -------------------------------------------------------------------------------- /LIBS2EConfig.cmake.in: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Cyberhaven 2 | # All rights reserved. 3 | # 4 | # Licensed under the Cyberhaven Research License Agreement. 5 | 6 | set(LIBS2E_VERSION_MAJOR @LIBS2E_VERSION_MAJOR@) 7 | set(LIBS2E_VERSION_MINOR @LIBS2E_VERSION_MINOR@) 8 | set(LIBS2E_VERSION_PATCH @LIBS2E_VERSION_PATCH@) 9 | set(LIBS2E_PACKAGE_VERSION @LIBS2E_PACKAGE_VERSION@) 10 | 11 | set(LIBS2E_INCLUDE_DIR "@LIBS2E_INCLUDE_DIR@") 12 | set(LIBS2E_LIBRARY_DIR "@LIBS2E_LIBRARY_DIR@") 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Cyberhaven Research License Agreement 2 | Non-Commercial Use Only 3 | _____________________________________________________________________ 4 | 5 | This Cyberhaven Research License Agreement, including all exhibits (“Agreement”) is a legal agreement between you and Cyberhaven, Inc. (“Company” or “we”) for the software and data included with this Agreement, which may include associated materials, text or speech files, associated media and “online” or electronic documentation and any updates we provide in our discretion (together, the “Software”). 6 | 7 | By installing, copying, or otherwise using this Software, you agree to be bound by the terms of this Agreement. If you do not agree, do not install copy or use the Software. The Software is protected by copyright and other intellectual property laws and is licensed, not sold. 8 | 9 | SCOPE OF RIGHTS: 10 | You may use, copy, reproduce, and distribute this Software for any non-commercial purpose, subject to the restrictions in this Agreement. Some purposes which can be non-commercial are teaching, academic research, public demonstrations and personal experimentation. You may also distribute this Software with books or other teaching materials, or publish the Software on websites, that are intended to teach the use of the Software for academic or other non-commercial purposes. 11 | 12 | You may not use or distribute this Software or any derivative works in any form for commercial purposes. Examples of commercial purposes would be running business operations, licensing, leasing, or selling the Software, distributing the Software for use with commercial products, using the Software in the creation or use of commercial products or any other activity which purpose is to procure a commercial gain to you or others. 13 | 14 | You may create derivative works of the Software source code and distribute the modified Software solely for non-commercial academic purposes, as provided herein. If you distribute the Software or any derivative works of the Software, you will distribute them under the same terms and conditions as in this license, and you will not grant other rights to the Software or derivative works that are different from those provided by this Agreement. 15 | 16 | If you have created derivative works of the Software, and distribute such derivative works, you will cause the modified files to carry prominent notices so that recipients know that they are not receiving the original Software. Such notices must state: (i) that you have changed the Software; and (ii) the date of any changes. 17 | 18 | In return, you agree as follows: 19 | 20 | 1. That you will not remove any copyright or other notices from the Software. 21 | 22 | 2. That if any of the Software is in binary format, you will not attempt to modify such portions of the Software, or to reverse engineer or decompile them, except and only to the extent authorized by applicable law. 23 | 24 | 3. That Company is granted back, without any restrictions or limitations, a non-exclusive, perpetual, irrevocable, royalty-free, assignable and sub-licensable license, to reproduce, publicly perform or display, install, use, modify, post, distribute, make and have made, sell and transfer your modifications to and/or derivative works of the Software source code or data, for any purpose. 25 | 26 | 4. That any feedback about the Software provided by you to us is voluntarily given, and Company shall be free to use the feedback as it sees fit without obligation or restriction of any kind, even if the feedback is designated by you as confidential. 27 | 28 | 5. THAT THE SOFTWARE COMES “AS IS”, WITH NO WARRANTIES. THIS MEANS NO EXPRESS, IMPLIED OR STATUTORY WARRANTY, INCLUDING WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, ANY WARRANTY AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE SOFTWARE OR ANY WARRANTY OF TITLE OR NON-INFRINGEMENT. THERE IS NO WARRANTY THAT THIS SOFTWARE WILL FULFILL ANY OF YOUR PARTICULAR PURPOSES OR NEEDS. ALSO, YOU MUST PASS THIS DISCLAIMER ON WHENEVER YOU DISTRIBUTE THE SOFTWARE OR DERIVATIVE WORKS. 29 | 30 | 6. THAT NEITHER COMPANY NOR ANY CONTRIBUTOR TO THE SOFTWARE WILL BE LIABLE FOR ANY DAMAGES RELATED TO THE SOFTWARE OR THIS AGREEMENT, INCLUDING DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL OR INCIDENTAL DAMAGES, TO THE MAXIMUM EXTENT THE LAW PERMITS, NO MATTER WHAT LEGAL THEORY IT IS BASED ON. ALSO, YOU MUST PASS THIS LIMITATION OF LIABILITY ON WHENEVER YOU DISTRIBUTE THE SOFTWARE OR DERIVATIVE WORKS. 31 | 32 | 7. That we have no duty of reasonable care or lack of negligence, and we are not obligated to (and will not) provide technical support for the Software. 33 | 34 | 8. That if you breach this Agreement or if you sue anyone over patents that you think may apply to or read on the Software or anyone’s use of the Software, this Agreement (and your license and rights obtained herein) terminate automatically. Upon any such termination, you shall destroy all of your copies of the Software immediately. Sections 3, 4, 5, 6, 7, 8, 11 and 12 of this Agreement shall survive any termination of this Agreement. 35 | 36 | 9. That the patent rights, if any, granted to you in this Agreement only apply to the Software, not to any derivative works you make. 37 | 38 | 10. That the Software may be subject to export or import laws in various jurisdictions. You agree to comply with all such laws and regulations that may apply to the Software after delivery of the Software to you. 39 | 40 | 11. That all rights not expressly granted to you in this Agreement are reserved. 41 | 42 | 12. That this Agreement shall be construed and controlled by the laws of the State of California, USA, without regard to conflicts of law. If any provision of this Agreement shall be deemed unenforceable or contrary to law, the rest of this Agreement shall remain in full effect and interpreted in an enforceable manner that most nearly captures the intent of the original language. 43 | 44 | 45 | 46 | =========== 47 | 48 | libs2ecore uses 3rd-party libraries that have their own licenses: 49 | 50 | libq: LGPL v2.1 (https://github.com/S2E/libq) 51 | libcpu: LGPL v2.1 (https://github.com/S2E/libcpu) 52 | libtcg: BSD/MIT (https://github.com/S2E/libtcg) 53 | libcoroutine: LGPL v2.1 (https://github.com/S2E/libcoroutine) 54 | klee: University of Illinois/NCSA (https://github.com/S2E/klee) 55 | llvm: University of Illinois/NCSA (http://llvm.org) 56 | lua: MIT (https://www.lua.org) 57 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Cyberhaven 2 | # All rights reserved. 3 | # 4 | # Licensed under the Cyberhaven Research License Agreement. 5 | 6 | TARGETS := $(wildcard *-softmmu) 7 | 8 | all: $(TARGETS) 9 | .PHONY: $(TARGETS) 10 | 11 | $(TARGETS): 12 | $(MAKE) -C $@ 13 | 14 | clean: 15 | for f in $(TARGETS); do $(MAKE) -C $$f clean; done 16 | -------------------------------------------------------------------------------- /Makefile.target: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Cyberhaven 2 | # All rights reserved. 3 | # 4 | # Licensed under the Cyberhaven Research License Agreement. 5 | 6 | include config.mak 7 | 8 | ifeq ($(CONFIG_SYMBEX), 1) 9 | ifeq ($(CONFIG_SYMBEX_MP), 1) 10 | BC_TARGETS := op_helper.bc.$(TARGET_ARCH) 11 | else 12 | BC_TARGETS := op_helper_sp.bc.$(TARGET_ARCH) 13 | endif 14 | endif 15 | 16 | 17 | TARGETS := libs2e.so $(BC_TARGETS) 18 | 19 | .PHONY: libs2e/src/libs2e.so libtcg/src/libtcg.a libcpu/src/libcpu.a libs2ecore/src/libs2ecore.a libs2eplugins/src/libs2eplugins.a 20 | 21 | all: $(TARGETS) 22 | 23 | libtcg/src/libtcg.a: 24 | $(MAKE) -C libtcg 25 | 26 | libcpu/src/libcpu.a: 27 | $(MAKE) -C libcpu 28 | 29 | libcpu/src/op_helper.bc: libcpu/src/libcpu.a 30 | @echo $@ 31 | 32 | libs2ecore/src/libs2ecore.a: libcpu/src/libcpu.a libtcg/src/libtcg.a 33 | $(MAKE) -C libs2ecore 34 | 35 | libs2eplugins/src/libs2eplugins.a: libs2ecore/src/libs2ecore.a 36 | $(MAKE) -C libs2eplugins 37 | 38 | 39 | ifeq ($(CONFIG_SYMBEX), 1) 40 | libs2e/src/libs2e.so: libcpu/src/libcpu.a libtcg/src/libtcg.a libs2eplugins/src/libs2eplugins.a 41 | $(MAKE) -C libs2e 42 | else 43 | libs2e/src/libs2e.so: libcpu/src/libcpu.a libtcg/src/libtcg.a 44 | $(MAKE) -C libs2e 45 | endif 46 | 47 | op_helper_sp.bc.$(TARGET_ARCH) op_helper.bc.$(TARGET_ARCH): libcpu/src/op_helper.bc 48 | cp -v $^ $@ 49 | 50 | libs2e.so: libs2e/src/libs2e.so 51 | cp -v $^ $@ 52 | 53 | clean: 54 | rm -f *.a *.so *.o *.bc* 55 | for d in libtcg libcpu libs2ecore libs2eplugins libs2e; do cd "$$d" && make clean && cd ..; done 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Building and Running libs2e 2 | =========================== 3 | 4 | 1. Build S2E as usual 5 | 6 | ``` 7 | make -f ../s2e/Makefile all-release all-debug 8 | ``` 9 | 10 | 2. Build qemu from https://github.com/S2E/qemu (You will need to use this qemu further) 11 | 12 | ``` 13 | git clone https://github.com/S2E/qemu 14 | mkdir s2e-qemu-build 15 | cd s2e-qemu-build 16 | ../s2e-qemu/configure --enable-debug --disable-werror --target-list="i386-softmmu x86_64-softmmu" --disable-docs 17 | make 18 | export QEMU_BUILD=/path/to/qemu-build 19 | ``` 20 | 21 | 3. Run libs2e in non-S2E mode 22 | 23 | ``` 24 | LD_PRELOAD=${S2E_BUILD}/libs2e-release/x86_64-softmmu/libs2e.so \ 25 | ${QEMU_BUILD}/x86_64-softmmu/qemu-system-x86_64 -drive file=windows7.raw.s2e,cache=writeback,format=s2e -m 2G -enable-kvm 26 | ``` 27 | 28 | 4. Run libs2e in S2E mode 29 | 30 | ``` 31 | export S2E_CONFIG=s2e-config.lua 32 | export S2E_SHARED_DIR=${S2E_BUILD}/libs2e-release/x86_64-s2e-softmmu/ 33 | 34 | LD_PRELOAD=${S2E_BUILD}/libs2e-release/x86_64-s2e-softmmu/libs2e.so \ 35 | ${QEMU_BUILD}/x86_64-softmmu/qemu-system-x86_64 -drive file=windows7.raw.s2e,cache=writeback,format=s2e -m 2G -enable-kvm -net none -loadvm ready 36 | ``` 37 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (C) 2017, Cyberhaven 4 | # All rights reserved. 5 | # 6 | # Licensed under the Cyberhaven Research License Agreement. 7 | 8 | 9 | print_usage() { 10 | echo "Usage: configure options..." 11 | echo 12 | echo " --prefix=PATH Installation prefix (e.g., /opt)" 13 | echo " --with-cc=PATH Path to clang" 14 | echo " --with-cxx=PATH Path to clang++" 15 | echo " --with-cflags=FLAGS C compiler flags" 16 | echo " --with-cxxflags=FLAGS C++ compiler flags" 17 | echo " --with-llvm=PATH LLVM binaries path (PATH/bin/llvm-config must exist)" 18 | echo " --with-klee=PATH KLEE binaries path" 19 | echo " --with-libvmi=PATH libvmi build directory" 20 | echo " --with-fsigc++=PATH fsigc++ build directory" 21 | echo " --with-libq=PATH libq build directory" 22 | echo " --with-liblua=PATH Path to lua binaries" 23 | echo " --with-libcoroutine=PATH libcoroutine build directory" 24 | echo " --with-libtcg-src=PATH Path to libtcg source" 25 | echo " --with-libcpu-src=PATH Path to libcpu source" 26 | echo " --with-libs2ecore-src=PATH Path to libs2ecore source" 27 | echo " --with-libs2eplugins-src=PATH Path to libs2eplugins source" 28 | echo " --with-z3-incdir=PATH Z3 include directory" 29 | echo " --with-z3-libdir=PATH Z3 library directory" 30 | echo " --with-capstone-incdir=PATH Capstone include directory" 31 | echo " --with-capstone-libdir=PATH Capstone lib directory" 32 | echo " --enable-debug Compile library with debug information" 33 | echo " --with-s2e-guest-incdir=PATH Path to S2E guest headers" 34 | } 35 | 36 | LLVM_ROOT= 37 | DEBUG=n 38 | 39 | for opt do 40 | optarg=`expr "x$opt" : 'x[^=]*=\(.*\)'` 41 | case "$opt" in 42 | --prefix=*) 43 | PREFIX="$optarg" 44 | ;; 45 | 46 | --with-cc=*) 47 | CC="$optarg" 48 | ;; 49 | 50 | --with-cxx=*) 51 | CXX="$optarg" 52 | ;; 53 | 54 | --with-cflags=*) 55 | CFLAGS="$optarg" 56 | ;; 57 | 58 | --with-cxxflags=*) 59 | CXXFLAGS="$optarg" 60 | ;; 61 | 62 | --with-llvm=*) 63 | LLVM_ROOT="$optarg" 64 | ;; 65 | 66 | --with-klee=*) 67 | KLEE_ROOT="$optarg" 68 | ;; 69 | 70 | --with-libvmi=*) 71 | LIBVMI_DIR="$optarg" 72 | ;; 73 | 74 | --with-liblua=*) 75 | LIBLUA_LIB="$optarg" 76 | ;; 77 | 78 | --with-z3-incdir=*) 79 | Z3_INCDIR="$optarg" 80 | ;; 81 | 82 | --with-z3-libdir=*) 83 | Z3_LIBDIR="$optarg" 84 | ;; 85 | 86 | --with-capstone-incdir=*) 87 | CAPSTONE_INCDIR="$optarg" 88 | ;; 89 | 90 | --with-capstone-libdir=*) 91 | CAPSTONE_LIBDIR="$optarg" 92 | ;; 93 | 94 | --with-fsigc++*) 95 | FSIGCXX_DIR="$optarg" 96 | ;; 97 | 98 | --with-libq=*) 99 | LIBQ_DIR="$optarg" 100 | ;; 101 | 102 | --with-libcoroutine=*) 103 | LIBCOROUTINE_DIR="$optarg" 104 | ;; 105 | 106 | --with-libtcg-src=*) 107 | LIBTCG_SRC="$optarg" 108 | ;; 109 | 110 | --with-libcpu-src=*) 111 | LIBCPU_SRC="$optarg" 112 | ;; 113 | 114 | --with-libs2ecore-src=*) 115 | LIBS2ECORE_SRC="$optarg" 116 | ;; 117 | 118 | --with-libs2eplugins-src=*) 119 | LIBS2EPLUGINS_SRC="$optarg" 120 | ;; 121 | 122 | --enable-debug) 123 | DEBUG=y 124 | ;; 125 | 126 | --with-s2e-guest-incdir=*) 127 | S2EGUEST_INCDIR="$optarg" 128 | ;; 129 | 130 | *) echo "Unknown option $opt" 131 | ;; 132 | esac 133 | ARGCOUNT=$(expr $ARGCOUNT + 1) 134 | done 135 | 136 | if [ ! -x "$LLVM_ROOT/bin/llvm-config" ]; then 137 | echo "$LLVM_ROOT/bin/llvm-config does not exist" 138 | print_usage 139 | exit 1 140 | fi 141 | 142 | if ! echo "$CC" | grep -q clang; then 143 | echo "You must use clang to compile libs2e" 144 | exit 1 145 | fi 146 | 147 | if ! echo "$CXX" | grep -q clang++; then 148 | echo "You must use clang++ to compile libs2e" 149 | exit 1 150 | fi 151 | 152 | if [ ! -e "$Z3_INCDIR/z3.h" ]; then 153 | echo "Cannot find Z3 header files in $Z3_INCIDR" 154 | print_usage 155 | exit 1 156 | fi 157 | 158 | if [ ! -e "$Z3_LIBDIR/libz3.a" ]; then 159 | echo "Cannot find Z3 library files in $Z3_LIBDIR" 160 | print_usage 161 | exit 1 162 | fi 163 | 164 | if [ ! -e "$CAPSTONE_INCDIR/capstone/xcore.h" ]; then 165 | echo "Cannot find Capstone header files in $CAPSTONE_INCDIR" 166 | print_usage 167 | exit 1 168 | fi 169 | 170 | if [ ! -e "$CAPSTONE_LIBDIR/libcapstone.a" ]; then 171 | echo "Cannot find Capstone library files in $CAPSTONE_LIBDIR" 172 | print_usage 173 | exit 1 174 | fi 175 | 176 | rm -f config.log 177 | echo "# libs2e configured with:" >> config.log 178 | printf " '%s'" "$0" "$@" >> config.log 179 | 180 | BUILD_DIR="$(pwd)" 181 | cd "$(dirname $0)" 182 | SRC_DIR="$(pwd)" 183 | 184 | cd "$BUILD_DIR" 185 | ln -sf "$SRC_DIR/Makefile" Makefile 186 | 187 | get_git_revision() { 188 | CURDIR="$(pwd)" 189 | cd $(dirname "$0") 190 | git --no-pager log --pretty=format:'%H' -n 1 191 | cd "$CURDIR" 192 | } 193 | 194 | LIBCPU_REVISION="$(get_git_revision)" 195 | 196 | ################################################################# 197 | # Configuring each target 198 | ################################################################# 199 | 200 | for target in i386-softmmu i386-s2e-softmmu i386-s2e_sp-softmmu x86_64-softmmu x86_64-s2e-softmmu x86_64-s2e_sp-softmmu; do 201 | echo "============== Configuring $target ============== " 202 | echo 203 | 204 | cd "$BUILD_DIR" 205 | mkdir -p "$target" 206 | cd "$target" 207 | 208 | ln -sf "$SRC_DIR/Makefile.target" Makefile 209 | 210 | ARCH="$(echo $target | cut -d '-' -f 1)" 211 | 212 | #Filter debug flags 213 | if [ $DEBUG = "y" ]; then 214 | BUILD_TYPE=Debug 215 | else 216 | BUILD_TYPE=RelWithDebInfo 217 | fi 218 | 219 | 220 | echo > config.mak 221 | 222 | if echo $target | grep -q s2e; then 223 | echo "S2E mode activated for $target" 224 | 225 | if ! echo $target | grep -q s2e_sp; then 226 | echo "CONFIG_SYMBEX_MP := 1" >> config.mak 227 | fi 228 | 229 | echo "CONFIG_SYMBEX = 1" >> config.mak 230 | echo "TARGET_ARCH := $ARCH" >> config.mak 231 | fi 232 | 233 | echo "CC := $CC" >> config.mak 234 | echo "CXX := $CXX" >> config.mak 235 | echo "CFLAGS := $CFLAGS" >> config.mak 236 | echo "CXXFLAGS := $CXXFLAGS" >> config.mak 237 | 238 | echo "=== Configuring libtcg... ===" 239 | rm -rf libtcg && mkdir -p libtcg && cd libtcg 240 | LLVM_DIR="$LLVM_ROOT/lib/cmake/llvm" 241 | cmake -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ 242 | -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ 243 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DWITH_GUEST="$target" \ 244 | -DLIBQ_DIR=$LIBQ_DIR \ 245 | -DLLVM_DIR="$LLVM_DIR" "$LIBTCG_SRC" 246 | if [ $? -ne 0 ]; then 247 | exit 1 248 | fi 249 | cd .. 250 | 251 | echo "=== Configuring libcpu... ===" 252 | rm -rf libcpu && mkdir -p libcpu && cd libcpu 253 | 254 | cmake -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ 255 | -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ 256 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ 257 | -DLIBTCG_DIR="$(pwd)/../libtcg" \ 258 | -DHOST_INCLUDE_DIR="$(cd $(pwd)/../../ && echo $(pwd))" \ 259 | -DTARGET_INCLUDE_DIR="$(cd $(pwd)/../ && echo $(pwd))" \ 260 | -DS2EGUEST_INCLUDE_DIR=$S2EGUEST_INCDIR \ 261 | -DLIBQ_DIR=$LIBQ_DIR \ 262 | -DCAPSTONE_INCLUDE_DIR=$CAPSTONE_INCDIR \ 263 | -DWITH_TARGET="$target" \ 264 | -DCONFIG_DATE="$(date)" \ 265 | -DCONFIG_LIBCPU_DATADIR="$BUILD_DIR/$target" \ 266 | -DLIBCPU_REVISION="$LIBCPU_REVISION" \ 267 | "$LIBCPU_SRC" 268 | 269 | if [ $? -ne 0 ]; then 270 | exit 1 271 | fi 272 | cd .. 273 | 274 | if echo $target | grep -q s2e; then 275 | echo "=== Configuring libs2ecore... ===" 276 | rm -rf libs2ecore && mkdir -p libs2ecore && cd libs2ecore 277 | 278 | cmake -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ 279 | -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ 280 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ 281 | -DCMAKE_PREFIX_PATH="$PREFIX" \ 282 | -DLIBTCG_DIR="$(pwd)/../libtcg" \ 283 | -DLIBCPU_DIR="$(pwd)/../libcpu" \ 284 | -DLLVM_DIR="$LLVM_DIR" \ 285 | -DKLEE_DIR="$KLEE_ROOT" \ 286 | -DVMI_DIR="$LIBVMI_DIR" \ 287 | -DFSIGCXX_DIR="$FSIGCXX_DIR" \ 288 | -DLIBQ_DIR="$LIBQ_DIR" \ 289 | -DLUA_DIR="$LIBLUA_LIB" \ 290 | "$LIBS2ECORE_SRC" 291 | 292 | if [ $? -ne 0 ]; then 293 | exit 1 294 | fi 295 | cd .. 296 | 297 | 298 | echo "=== Configuring libs2eplugins... ===" 299 | rm -rf libs2eplugins && mkdir -p libs2eplugins && cd libs2eplugins 300 | 301 | cmake -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ 302 | -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ 303 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ 304 | -DCMAKE_PREFIX_PATH="$PREFIX" \ 305 | -DLIBTCG_DIR="$(pwd)/../libtcg" \ 306 | -DLIBCPU_DIR="$(pwd)/../libcpu" \ 307 | -DLIBS2ECORE_DIR="$(pwd)/../libs2ecore" \ 308 | -DLLVM_DIR="$LLVM_DIR" \ 309 | -DKLEE_DIR="$KLEE_ROOT" \ 310 | -DVMI_DIR="$LIBVMI_DIR" \ 311 | -DFSIGCXX_DIR="$FSIGCXX_DIR" \ 312 | -DLIBQ_DIR=$LIBQ_DIR \ 313 | -DLUA_DIR=$LIBLUA_LIB \ 314 | -DS2EGUEST_INCLUDE_DIR=$S2EGUEST_INCDIR \ 315 | "$LIBS2EPLUGINS_SRC" 316 | 317 | if [ $? -ne 0 ]; then 318 | exit 1 319 | fi 320 | cd .. 321 | 322 | fi 323 | 324 | echo "=== Configuring libs2e... ===" 325 | rm -rf libs2e && mkdir -p libs2e && cd libs2e 326 | 327 | cmake -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ 328 | -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ 329 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ 330 | -DCMAKE_PREFIX_PATH="$PREFIX" \ 331 | -DLIBTCG_DIR="$(pwd)/../libtcg" \ 332 | -DLIBCPU_DIR="$(pwd)/../libcpu" \ 333 | -DLIBS2ECORE_DIR="$(pwd)/../libs2ecore" \ 334 | -DLIBS2EPLUGINS_DIR="$(pwd)/../libs2eplugins" \ 335 | -DLIBQ_DIR="$LIBQ_DIR" \ 336 | -DLLVM_DIR="$LLVM_DIR" \ 337 | -DLIBCOROUTINE_DIR="$LIBCOROUTINE_DIR" \ 338 | -DKLEE_DIR="$KLEE_ROOT" \ 339 | -DVMI_DIR="$LIBVMI_DIR" \ 340 | -DLUA_DIR="$LIBLUA_LIB" \ 341 | -DFSIGCXX_DIR="$FSIGCXX_DIR" \ 342 | -DZ3_DIR="$Z3_LIBDIR" \ 343 | -DCAPSTONE_LIB_DIR=$CAPSTONE_LIBDIR \ 344 | -DWITH_TARGET="$target" \ 345 | "$SRC_DIR" 346 | 347 | if [ $? -ne 0 ]; then 348 | exit 1 349 | fi 350 | cd .. 351 | 352 | 353 | echo "============== Configuring $target done ============== " 354 | 355 | done 356 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Cyberhaven 2 | # All rights reserved. 3 | # 4 | # Licensed under the Cyberhaven Research License Agreement. 5 | 6 | add_library( 7 | s2e SHARED 8 | libs2e.cpp 9 | s2e-kvm-io.cpp 10 | s2e-kvm-state.cpp 11 | s2e-kvm-trace.cpp 12 | s2e-kvm.cpp 13 | s2e-kvm-vm.cpp 14 | s2e-kvm-vcpu.cpp 15 | s2e-libcpu-interface.cpp 16 | crashdump.cpp 17 | ) 18 | 19 | add_executable( 20 | tests2e 21 | test.cpp 22 | ) 23 | 24 | # Specify whole path for libraries so that libs2e can be relinked if any of them changes. 25 | # Just specifying the short library name prevents cmake from detecting the dependency. 26 | if(WITH_TARGET MATCHES "s2e") 27 | set(LIBS -Wl,--whole-archive ${LIBS2EPLUGINS_LIBRARY_DIR}/libs2eplugins.a ${LIBS2ECORE_LIBRARY_DIR}/libs2ecore.a -Wl,--no-whole-archive ${LIBS}) 28 | endif() 29 | 30 | set(LIBS ${LIBS} ${LIBCPU_LIBRARY_DIR}/libcpu.a ${LIBTCG_LIBRARY_DIR}/libtcg.a) 31 | 32 | if(WITH_TARGET MATCHES "s2e") 33 | set(LIBS ${LIBS} ${KLEE_LIBRARY_DIR}/libkleeCore.a 34 | ${KLEE_LIBRARY_DIR}/libkleeModule.a 35 | ${KLEE_LIBRARY_DIR}/libkleaverSolver.a 36 | ${KLEE_LIBRARY_DIR}/libkleaverExpr.a 37 | ${KLEE_LIBRARY_DIR}/libkleeSupport.a 38 | ${KLEE_LIBRARY_DIR}/libkleeBasic.a) 39 | set(LIBS ${LIBS} ${VMI_LIBRARY_DIR}/libvmi.a 40 | elf) 41 | set(LIBS ${LIBS} memcached 42 | lua 43 | ${LLVM_LIBS} 44 | z3 45 | ${LIBQ_LIBRARY_DIR}/libq.a 46 | ${PROTOBUF_LIBRARIES}) 47 | set(LIBS ${LIBS} boost_serialization 48 | boost_system 49 | boost_regex) 50 | endif() 51 | 52 | set(LIBS ${LIBS} ${LIBCOROUTINE_LIBRARY_DIR}/libcoroutine.a ${FSIGCXX_LIBRARY_DIR}/libfsigc++.a ${CAPSTONE_LIB_DIR}/libcapstone.a pthread glib-2.0 bsd) 53 | 54 | target_link_libraries(tests2e PUBLIC ${LIBCOROUTINE_LIBRARY_DIR}/libcoroutine.a pthread glib-2.0) 55 | target_link_libraries(s2e ${LIBS}) 56 | 57 | 58 | set(WERROR_FLAGS "-Werror -Wno-zero-length-array -Wno-c99-extensions \ 59 | -Wno-gnu-anonymous-struct -Wno-nested-anon-types \ 60 | -Wno-extended-offsetof -Wno-gnu-statement-expression \ 61 | -Wno-gnu-zero-variadic-macro-arguments -Wno-vla-extension \ 62 | -Wno-covered-switch-default -Wno-shift-negative-value \ 63 | -Wno-deprecated-register -Wno-sign-compare \ 64 | -Wno-missing-field-initializers -Wno-mismatched-tags \ 65 | -Wno-deprecated-declarations -Wno-initializer-overrides \ 66 | -Wno-zero-length-array") 67 | 68 | set(COMMON_FLAGS "-D__STDC_FORMAT_MACROS -D_GNU_SOURCE -DNEED_CPU_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DTARGET_PHYS_ADDR_BITS=64") 69 | set(COMMON_FLAGS "${COMMON_FLAGS} -Wall -fPIC -fno-strict-aliasing -fexceptions") 70 | 71 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WERROR_FLAGS} ${COMMON_FLAGS} -std=c++11") 72 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WERROR_FLAGS} ${COMMON_FLAGS}") 73 | set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/mapfile") 74 | -------------------------------------------------------------------------------- /src/FileDescriptorManager.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_FDMGR_H 9 | 10 | #define S2E_KVM_FDMGR_H 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace s2e { 22 | namespace kvm { 23 | 24 | class IFile { 25 | public: 26 | virtual int sys_ioctl(int fd, int request, uint64_t arg1) { 27 | return -1; 28 | } 29 | 30 | virtual int close(int fd) { 31 | return -1; 32 | } 33 | 34 | virtual void *sys_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) { 35 | errno = ENOSYS; 36 | return MAP_FAILED; 37 | } 38 | 39 | // KVM interface does not support writes 40 | virtual ssize_t sys_write(int fd, const void *buf, size_t count) { 41 | printf("write %d count=%ld\n", fd, count); 42 | exit(-1); 43 | return -1; 44 | } 45 | 46 | virtual int sys_dup(int fd) { 47 | return -1; 48 | } 49 | }; 50 | 51 | typedef std::shared_ptr IFilePtr; 52 | 53 | class FileDescriptorManager { 54 | private: 55 | std::unordered_map m_map; 56 | 57 | public: 58 | IFilePtr get(int fd) const { 59 | auto it = m_map.find(fd); 60 | if (it == m_map.end()) { 61 | return nullptr; 62 | } 63 | return (*it).second; 64 | } 65 | 66 | IFilePtr get(IFile *ptr) const { 67 | for (const auto &it : m_map) { 68 | if (it.second.get() == ptr) { 69 | return it.second; 70 | } 71 | } 72 | return nullptr; 73 | } 74 | 75 | int registerInterface(IFilePtr iface) { 76 | /* Reserve a dummy file descriptor */ 77 | int fd = open("/dev/null", O_RDWR | O_CREAT | O_TRUNC, 0700); 78 | if (fd < 0) { 79 | return -1; 80 | } 81 | 82 | m_map[fd] = iface; 83 | return fd; 84 | } 85 | 86 | bool close(int fd) { 87 | auto ret = m_map.erase(fd) > 0; 88 | if (ret) { 89 | close(fd); 90 | } 91 | return ret; 92 | } 93 | }; 94 | 95 | typedef std::shared_ptr FileDescriptorManagerPtr; 96 | } 97 | } 98 | 99 | #endif 100 | -------------------------------------------------------------------------------- /src/crashdump.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | 11 | extern "C" { 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #ifdef CONFIG_SYMBEX 19 | #include 20 | #endif 21 | 22 | extern struct CPUX86State *env; 23 | uintptr_t s2e_get_host_address(target_phys_addr_t paddr); 24 | void generate_crashdump(void); 25 | } 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | using namespace vmi; 34 | using namespace vmi::windows; 35 | 36 | bool readX86Register(void *opaque, unsigned regIndex, void *buffer, unsigned size) { 37 | if (regIndex <= X86_GS) { 38 | switch (regIndex) { 39 | case X86_EAX: 40 | memcpy(buffer, &env->regs[R_EAX], size); 41 | break; 42 | case X86_EBX: 43 | memcpy(buffer, &env->regs[R_EBX], size); 44 | break; 45 | case X86_ECX: 46 | memcpy(buffer, &env->regs[R_ECX], size); 47 | break; 48 | case X86_EDX: 49 | memcpy(buffer, &env->regs[R_EDX], size); 50 | break; 51 | case X86_ESI: 52 | memcpy(buffer, &env->regs[R_ESI], size); 53 | break; 54 | case X86_EDI: 55 | memcpy(buffer, &env->regs[R_EDI], size); 56 | break; 57 | case X86_ESP: 58 | memcpy(buffer, &env->regs[R_ESP], size); 59 | break; 60 | case X86_EBP: 61 | memcpy(buffer, &env->regs[R_EBP], size); 62 | break; 63 | 64 | case X86_CS: 65 | memcpy(buffer, &env->segs[R_CS], size); 66 | break; 67 | case X86_DS: 68 | memcpy(buffer, &env->segs[R_DS], size); 69 | break; 70 | case X86_ES: 71 | memcpy(buffer, &env->segs[R_ES], size); 72 | break; 73 | case X86_SS: 74 | memcpy(buffer, &env->segs[R_SS], size); 75 | break; 76 | case X86_FS: 77 | memcpy(buffer, &env->segs[R_FS], size); 78 | break; 79 | case X86_GS: 80 | memcpy(buffer, &env->segs[R_GS], size); 81 | break; 82 | default: 83 | assert(false); 84 | } 85 | return true; 86 | } else if (regIndex <= X86_CR4) { 87 | memcpy(buffer, &env->cr[regIndex - X86_CR0], size); 88 | return true; 89 | } else if (regIndex <= X86_DR7) { 90 | memcpy(buffer, &env->cr[regIndex - X86_DR0], size); 91 | return true; 92 | } else if (regIndex == X86_EFLAGS) { 93 | uint64_t flags = cpu_get_eflags(env); 94 | memcpy(buffer, &flags, size); 95 | } else if (regIndex == X86_EIP) { 96 | memcpy(buffer, &env->eip, size); 97 | } else { 98 | return false; 99 | } 100 | 101 | return true; 102 | } 103 | 104 | static uint64_t getPhysicalAddress(uint64_t virtualAddress) { 105 | target_phys_addr_t physicalAddress = cpu_get_phys_page_debug(env, virtualAddress & TARGET_PAGE_MASK); 106 | if (physicalAddress == (target_phys_addr_t) -1) 107 | return (uint64_t) -1; 108 | 109 | return physicalAddress | (virtualAddress & ~TARGET_PAGE_MASK); 110 | } 111 | 112 | static uint64_t getHostAddress(uint64_t address) { 113 | uint64_t phys_addr = getPhysicalAddress(address); 114 | if (phys_addr == (uint64_t) -1) 115 | return (uint64_t) -1; 116 | 117 | const MemoryDesc *desc = mem_desc_find(phys_addr); 118 | if (!desc) { 119 | return -1; 120 | } 121 | 122 | uint64_t offset = mem_desc_get_offset(desc, phys_addr); 123 | 124 | return desc->kvm.userspace_addr + offset; 125 | } 126 | 127 | bool rwGuestVirtual(void *opaque, uint64_t address, void *buf, unsigned size, bool is_write) { 128 | while (size > 0) { 129 | uint64_t hostAddress = getHostAddress(address); 130 | if (hostAddress == (uint64_t) -1) { 131 | return false; 132 | } 133 | 134 | uint64_t hostPage = hostAddress & TARGET_PAGE_MASK; 135 | uint64_t length = (hostPage + TARGET_PAGE_SIZE) - hostAddress; 136 | if (length > size) { 137 | length = size; 138 | } 139 | 140 | #ifdef CONFIG_SYMBEX 141 | if (is_write) { 142 | g_sqi.mem.write_ram_concrete(hostAddress, (const uint8_t *) buf, length); 143 | } else { 144 | g_sqi.mem.read_ram_concrete(hostAddress, buf, length); 145 | } 146 | #else 147 | if (is_write) { 148 | memcpy((void *) hostAddress, buf, length); 149 | } else { 150 | memcpy(buf, (void *) hostAddress, length); 151 | } 152 | #endif 153 | 154 | buf = (uint8_t *) buf + length; 155 | address += length; 156 | size -= length; 157 | } 158 | 159 | return true; 160 | } 161 | 162 | bool readGuestVirtual(void *opaque, uint64_t address, void *buf, unsigned size) { 163 | return rwGuestVirtual(opaque, address, buf, size, false); 164 | } 165 | 166 | bool writeGuestVirtual(void *opaque, uint64_t address, const void *buf, unsigned size) { 167 | return rwGuestVirtual(opaque, address, (void *) buf, size, true); 168 | } 169 | 170 | bool readGuestPhysical(void *opaque, uint64_t address, void *dest, unsigned size) { 171 | cpu_physical_memory_rw(address, (uint8_t *) dest, size, 0); 172 | return true; 173 | } 174 | 175 | void generate_crashdump(void) { 176 | extern CPUX86State *env; 177 | uint64_t KernelNativeBase = 0x400000; 178 | 179 | auto vp = GuestMemoryFileProvider::get(env, readGuestVirtual, writeGuestVirtual, "virt"); 180 | auto pp = GuestMemoryFileProvider::get(env, readGuestPhysical, NULL, "phys"); 181 | X86RegisterProvider rp(env, readX86Register, NULL); 182 | 183 | std::stringstream ss; 184 | ss << "crash-" << llvm::sys::TimeValue::now().seconds() << ".dmp"; 185 | 186 | auto fp = FileSystemFileProvider::get(ss.str(), true); 187 | if (!fp) { 188 | return; 189 | } 190 | 191 | auto cgen = WindowsCrashDumpGenerator::get(vp, pp, &rp, fp); 192 | 193 | uint64_t KPCR = 0xffdff000; 194 | uint64_t KPCRB = KPCR + 0x120; 195 | 196 | uint64_t pKdVersionBlock = KPCR + 0x34; 197 | 198 | uint32_t KdVersionBlock; 199 | if (vp->read(&KdVersionBlock, sizeof(KdVersionBlock), pKdVersionBlock) != sizeof(KdVersionBlock)) { 200 | return; 201 | } 202 | 203 | DBGKD_GET_VERSION64 kdVersion; 204 | if (vp->read(&kdVersion, sizeof(kdVersion), KdVersionBlock) != sizeof(kdVersion)) { 205 | return; 206 | } 207 | 208 | // TODO: auto detect build type 209 | // WinXP SP3 210 | // - free build base is at 0x804d7000 211 | // - checked build base is at 0x80a02000 212 | 213 | // ntoskrnl.exe 214 | // uint64_t KdDebuggerDataBlock = 0x475de0 - KernelNativeBase + kdVersion.KernBase; 215 | 216 | // ntkrnlpa.exe free build 217 | uint64_t KdDebuggerDataBlock = 0x46eae0 - KernelNativeBase + kdVersion.KernBase; 218 | 219 | // ntkrnlpa.exe checked build 220 | // uint64_t KdDebuggerDataBlock = 0x004ec3f0 - KernelNativeBase + kdVersion.KernBase; 221 | 222 | CONTEXT32 context; 223 | 224 | context.ContextFlags = CONTEXT_FULL; 225 | context.Dr0 = env->dr[0]; 226 | context.Dr1 = env->dr[1]; 227 | context.Dr2 = env->dr[2]; 228 | context.Dr3 = env->dr[3]; 229 | context.Dr6 = env->dr[6]; 230 | context.Dr7 = env->dr[7]; 231 | 232 | context.SegGs = env->segs[R_GS].selector; 233 | context.SegFs = env->segs[R_FS].selector; 234 | context.SegEs = env->segs[R_ES].selector; 235 | context.SegDs = env->segs[R_DS].selector; 236 | context.SegCs = env->segs[R_CS].selector; 237 | context.SegSs = env->segs[R_SS].selector; 238 | 239 | context.Eax = env->regs[R_EAX]; 240 | context.Ebx = env->regs[R_EBX]; 241 | context.Ecx = env->regs[R_ECX]; 242 | context.Edx = env->regs[R_EDX]; 243 | context.Edi = env->regs[R_EDI]; 244 | context.Esi = env->regs[R_ESI]; 245 | context.Esp = env->regs[R_ESP]; 246 | context.Ebp = env->regs[R_EBP]; 247 | context.Eip = env->eip; 248 | context.EFlags = env->mflags; 249 | 250 | BugCheckDescription bugDesc; 251 | vp->read(&bugDesc.code, sizeof(uint32_t), env->regs[R_ESP] + 4); 252 | for (int i = 0; i < 4; ++i) { 253 | vp->read(&bugDesc.parameters[i], sizeof(uint32_t), env->regs[R_ESP] + 4 + (i + 1) * 4); 254 | } 255 | 256 | cgen->generate(KdDebuggerDataBlock, KPCRB, kdVersion, context, bugDesc); 257 | } 258 | -------------------------------------------------------------------------------- /src/libs2e.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #undef __REDIRECT_NTH 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | #include "s2e-kvm-trace.h" 27 | #include "s2e-kvm.h" 28 | 29 | #ifdef CONFIG_SYMBEX 30 | #include 31 | #endif 32 | 33 | #include "FileDescriptorManager.h" 34 | #include "libs2e.h" 35 | #include "syscalls.h" 36 | 37 | namespace s2e { 38 | SyscallEvents g_syscalls; 39 | } 40 | 41 | // TODO: make this an env variable 42 | int g_trace = 0; 43 | 44 | s2e::kvm::FileDescriptorManagerPtr g_fdm = std::make_shared(); 45 | 46 | extern "C" { 47 | 48 | open_t g_original_open; 49 | int open64(const char *pathname, int flags, ...) { 50 | va_list list; 51 | va_start(list, flags); 52 | mode_t mode = va_arg(list, mode_t); 53 | va_end(list); 54 | 55 | if (!strcmp(pathname, "/dev/kvm")) { 56 | printf("Opening %s\n", pathname); 57 | 58 | s2e::kvm::IFilePtr kvm; 59 | 60 | if (g_trace) { 61 | kvm = s2e::kvm::KVMTrace::create(); 62 | } else { 63 | kvm = s2e::kvm::S2EKVM::create(); 64 | } 65 | 66 | if (!kvm) { 67 | return -1; 68 | } 69 | 70 | int fd = g_fdm->registerInterface(kvm); 71 | if (fd < 0) { 72 | printf("Could not register fake kvm file descriptor\n"); 73 | exit(-1); 74 | } 75 | 76 | return fd; 77 | } else { 78 | return g_original_open(pathname, flags, mode); 79 | } 80 | } 81 | 82 | static close_t s_original_close; 83 | int close64(int fd) { 84 | if (g_fdm->close(fd)) { 85 | return 0; 86 | } else { 87 | return s_original_close(fd); 88 | } 89 | } 90 | 91 | static write_t s_original_write; 92 | ssize_t write(int fd, const void *buf, size_t count) { 93 | auto ifp = g_fdm->get(fd); 94 | if (ifp) { 95 | return ifp->sys_write(fd, buf, count); 96 | } else { 97 | return s_original_write(fd, buf, count); 98 | } 99 | } 100 | 101 | ioctl_t g_original_ioctl; 102 | int ioctl(int fd, int request, uint64_t arg1) { 103 | auto ifp = g_fdm->get(fd); 104 | if (ifp) { 105 | return ifp->sys_ioctl(fd, request, arg1); 106 | } else { 107 | return g_original_ioctl(fd, request, arg1); 108 | } 109 | } 110 | 111 | static poll_t s_original_poll; 112 | int poll(struct pollfd *fds, nfds_t nfds, int timeout) { 113 | // TODO: do we actually have to request exit from here? 114 | return s_original_poll(fds, nfds, timeout); 115 | } 116 | 117 | static select_t s_original_select; 118 | int select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout) { 119 | int ret = s_original_select(nfds, readfds, writefds, exceptfds, timeout); 120 | s2e::g_syscalls.onSelect.emit(); 121 | return ret; 122 | } 123 | 124 | exit_t g_original_exit; 125 | void exit(int code) { 126 | s2e::g_syscalls.onExit.emit(code); 127 | g_original_exit(code); 128 | } 129 | 130 | #undef mmap 131 | 132 | mmap_t g_original_mmap; 133 | void *mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) { 134 | auto ifp = g_fdm->get(fd); 135 | if (ifp) { 136 | return ifp->sys_mmap(addr, len, prot, flags, fd, offset); 137 | } else { 138 | return g_original_mmap(addr, len, prot, flags, fd, offset); 139 | } 140 | } 141 | 142 | mmap64_t g_original_mmap64; 143 | void *mmap64(void *addr, size_t len, int prot, int flags, int fd, off64_t offset) { 144 | auto ifp = g_fdm->get(fd); 145 | if (ifp) { 146 | return ifp->sys_mmap(addr, len, prot, flags, fd, offset); 147 | } else { 148 | return g_original_mmap64(addr, len, prot, flags, fd, offset); 149 | } 150 | } 151 | 152 | static dup_t s_original_dup; 153 | int dup(int fd) { 154 | auto ifp = g_fdm->get(fd); 155 | if (ifp) { 156 | return g_fdm->registerInterface(ifp); 157 | } else { 158 | return s_original_dup(fd); 159 | } 160 | } 161 | 162 | static madvise_t s_original_madvise; 163 | int madvise(void *addr, size_t len, int advice) { 164 | if (advice & MADV_DONTFORK) { 165 | // We must fork all memory for multi-core mode 166 | advice &= ~MADV_DONTFORK; 167 | } 168 | 169 | if (!advice) { 170 | return 0; 171 | } 172 | 173 | return s_original_madvise(addr, len, advice); 174 | } 175 | 176 | ////////////////////////////////////////////////// 177 | // Intercept process's print functions to redirect 178 | // them to S2E's debug logs. 179 | 180 | #ifdef CONFIG_SYMBEX 181 | static printf_t s_original_printf; 182 | int printf(const char *fmt, ...) { 183 | va_list vl; 184 | va_start(vl, fmt); 185 | int ret = vprintf(fmt, vl); 186 | va_end(vl); 187 | 188 | va_start(vl, fmt); 189 | s2e_vprintf(fmt, false, vl); 190 | va_end(vl); 191 | 192 | return ret; 193 | } 194 | 195 | static fprintf_t s_original_fprintf; 196 | int fprintf(FILE *fp, const char *fmt, ...) { 197 | va_list vl; 198 | va_start(vl, fmt); 199 | int ret = vfprintf(fp, fmt, vl); 200 | va_end(vl); 201 | 202 | if (fp == stdout || fp == stderr) { 203 | va_start(vl, fmt); 204 | s2e_vprintf(fmt, false, vl); 205 | va_end(vl); 206 | } 207 | 208 | return ret; 209 | } 210 | #endif 211 | 212 | /// 213 | /// \brief check_kvm_switch verifies that KVM mode is enabled. 214 | /// 215 | /// It's a common mistake to preload libs2e.so but forget the --enable-kvm switch 216 | /// 217 | /// \param argc command line arg count 218 | /// \param argv command line arguments 219 | /// \return true if kvm switch is found 220 | /// 221 | static bool check_kvm_switch(int argc, char **argv) { 222 | for (int i = 0; i < argc; ++i) { 223 | if (strstr(argv[i], "-enable-kvm")) { 224 | return true; 225 | } 226 | } 227 | 228 | return false; 229 | } 230 | 231 | void libs2e_init_syscalls(void) { 232 | static bool inited = false; 233 | 234 | if (inited) { 235 | return; 236 | } 237 | 238 | g_original_open = (open_t) dlsym(RTLD_NEXT, "open64"); 239 | s_original_close = (close_t) dlsym(RTLD_NEXT, "close64"); 240 | g_original_ioctl = (ioctl_t) dlsym(RTLD_NEXT, "ioctl"); 241 | s_original_write = (write_t) dlsym(RTLD_NEXT, "write"); 242 | s_original_select = (select_t) dlsym(RTLD_NEXT, "select"); 243 | s_original_poll = (poll_t) dlsym(RTLD_NEXT, "poll"); 244 | g_original_exit = (exit_t) dlsym(RTLD_NEXT, "exit"); 245 | g_original_mmap = (mmap_t) dlsym(RTLD_NEXT, "mmap"); 246 | g_original_mmap64 = (mmap64_t) dlsym(RTLD_NEXT, "mmap64"); 247 | s_original_madvise = (madvise_t) dlsym(RTLD_NEXT, "madvise"); 248 | s_original_dup = (dup_t) dlsym(RTLD_NEXT, "dup"); 249 | 250 | #ifdef CONFIG_SYMBEX 251 | s_original_printf = (printf_t) dlsym(RTLD_NEXT, "printf"); 252 | s_original_fprintf = (fprintf_t) dlsym(RTLD_NEXT, "fprintf"); 253 | #endif 254 | 255 | inited = true; 256 | } 257 | 258 | // **************************** 259 | // Overriding __llibc_start_main 260 | // **************************** 261 | 262 | // The type of __libc_start_main 263 | typedef int (*T_libc_start_main)(int *(main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), 264 | void (*fini)(void), void (*rtld_fini)(void), void(*stack_end)); 265 | 266 | int __libc_start_main(int *(main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), 267 | void (*fini)(void), void (*rtld_fini)(void), void *stack_end) __attribute__((noreturn)); 268 | 269 | int __libc_start_main(int *(main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), 270 | void (*fini)(void), void (*rtld_fini)(void), void *stack_end) { 271 | 272 | T_libc_start_main orig_libc_start_main = (T_libc_start_main) dlsym(RTLD_NEXT, "__libc_start_main"); 273 | 274 | libs2e_init_syscalls(); 275 | 276 | // Hack when we are called from gdb or through a shell command 277 | if (strstr(ubp_av[0], "bash")) { 278 | (*orig_libc_start_main)(main, argc, ubp_av, init, fini, rtld_fini, stack_end); 279 | exit(-1); 280 | } 281 | 282 | printf("Starting libs2e...\n"); 283 | 284 | // libs2e might spawn other processes (e.g., from plugin code). 285 | // This will fail if we preload libs2e.so for these processes, 286 | // so we must remove this environment variable. 287 | unsetenv("LD_PRELOAD"); 288 | 289 | // When libs2e is used with qemu, verify that enable-kvm switch 290 | // has been specified. 291 | if (strstr(ubp_av[0], "qemu") && !check_kvm_switch(argc, ubp_av)) { 292 | fprintf(stderr, "Please use -enable-kvm switch before starting QEMU\n"); 293 | exit(-1); 294 | } 295 | 296 | if (!init_ram_size(argc, ubp_av)) { 297 | exit(-1); 298 | } 299 | 300 | (*orig_libc_start_main)(main, argc, ubp_av, init, fini, rtld_fini, stack_end); 301 | 302 | exit(1); // This is never reached 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /src/libs2e.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef LIBS2E_H 9 | 10 | #define LIBS2E_H 11 | 12 | #include "FileDescriptorManager.h" 13 | 14 | extern s2e::kvm::FileDescriptorManagerPtr g_fdm; 15 | 16 | struct se_libcpu_interface_t; 17 | void init_s2e_libcpu_interface(struct se_libcpu_interface_t *sqi); 18 | 19 | namespace s2e { 20 | namespace kvm { 21 | extern struct cpu_io_funcs_t g_io; 22 | } 23 | } 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /src/mapfile: -------------------------------------------------------------------------------- 1 | { 2 | global: 3 | open64; 4 | close64; 5 | write; 6 | sigaction; 7 | ioctl; 8 | select; 9 | poll; 10 | mmap; 11 | mmap64; 12 | madvise; 13 | printf; 14 | fprintf; 15 | dup; 16 | __libc_start_main; 17 | local: 18 | *; 19 | }; 20 | -------------------------------------------------------------------------------- /src/s2e-kvm-io.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include "s2e-kvm-vcpu.h" 19 | #include "s2e-kvm.h" 20 | 21 | extern CPUX86State *env; 22 | 23 | namespace s2e { 24 | namespace kvm { 25 | 26 | // This is an experimental feature 27 | // #define ENABLE_RETRANSLATE 28 | 29 | // This function aborts the execution of the current translation block. 30 | // It is useful when the KVM client modifies the program counter during 31 | // an I/O operation (e.g., VAPIC emulation). 32 | static void abort_and_retranslate_if_needed() { 33 | #ifdef ENABLE_RETRANSLATE 34 | if (env->se_current_tb->icount == 1) { 35 | return; 36 | } 37 | 38 | if (cpu_restore_state(env->se_current_tb, env, env->mem_io_pc)) { 39 | abort(); 40 | } 41 | 42 | libcpu_log("Aborting and retranslating at eip=%#lx\n", (uint64_t) env->eip); 43 | 44 | env->translate_single_instruction = 1; 45 | env->exception_index = -1; 46 | tb_phys_invalidate(env->se_current_tb, -1); 47 | cpu_loop_exit(env); 48 | #endif 49 | } 50 | 51 | uint64_t s2e_kvm_mmio_read(target_phys_addr_t addr, unsigned size) { 52 | int is_apic_tpr_access = 0; 53 | 54 | ++g_stats.mmio_reads; 55 | 56 | if ((addr >> TARGET_PAGE_BITS) == (env->v_apic_base >> TARGET_PAGE_BITS)) { 57 | if ((addr & 0xfff) == 0x80) { 58 | is_apic_tpr_access = 1; 59 | } 60 | } 61 | 62 | if (is_apic_tpr_access) { 63 | abort_and_retranslate_if_needed(); 64 | } 65 | 66 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_MMIO; 67 | g_kvm_vcpu_buffer->mmio.is_write = 0; 68 | g_kvm_vcpu_buffer->mmio.phys_addr = addr; 69 | g_kvm_vcpu_buffer->mmio.len = size; 70 | 71 | uint8_t *dataptr = g_kvm_vcpu_buffer->mmio.data; 72 | 73 | coroutine_yield(); 74 | 75 | uint64_t ret; 76 | switch (size) { 77 | case 1: 78 | ret = *(uint8_t *) dataptr; 79 | break; 80 | case 2: 81 | ret = *(uint16_t *) dataptr; 82 | break; 83 | case 4: 84 | ret = *(uint32_t *) dataptr; 85 | break; 86 | default: 87 | assert(false && "Can't get here"); 88 | } 89 | 90 | // This is a fix for 32-bits guests that access apic directly 91 | // and don't use cr8. Writing to cr8 clears the low four bits 92 | // of the TPR, which may confuse the guest. 93 | // Note that in 64-bit mode, guests should either use cr8 or 94 | // MMIO, but not both, so we should still be consistent. 95 | if (is_apic_tpr_access) { 96 | if (!(env->hflags & HF_LMA_MASK)) { 97 | assert((env->v_apic_tpr & 0xf0) == (ret & 0xf0)); 98 | ret |= env->v_apic_tpr & 0x3; 99 | } 100 | } 101 | 102 | #ifdef SE_KVM_DEBUG_MMIO 103 | unsigned print_addr = 0; 104 | #ifdef SE_KVM_DEBUG_APIC 105 | if (addr >= 0xf0000000) 106 | print_addr = 1; 107 | #endif 108 | if (print_addr) { 109 | printf("mmior%d[%" PRIx64 "]=%" PRIx64 "\n", size, (uint64_t) addr, ret); 110 | // printf("env->mflags=%x hflags=%x hflags2=%x\n", 111 | // env->mflags, env->hflags, env->hflags2); 112 | } 113 | #endif 114 | return ret; 115 | } 116 | 117 | void s2e_kvm_mmio_write(target_phys_addr_t addr, uint64_t data, unsigned size) { 118 | ++g_stats.mmio_writes; 119 | 120 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_MMIO; 121 | g_kvm_vcpu_buffer->mmio.is_write = 1; 122 | g_kvm_vcpu_buffer->mmio.phys_addr = addr; 123 | g_kvm_vcpu_buffer->mmio.len = size; 124 | 125 | uint8_t *dataptr = g_kvm_vcpu_buffer->mmio.data; 126 | 127 | #ifdef SE_KVM_DEBUG_MMIO 128 | unsigned print_addr = 0; 129 | #ifdef SE_KVM_DEBUG_APIC 130 | if (addr >= 0xf0000000) 131 | print_addr = 1; 132 | #endif 133 | 134 | if (print_addr) { 135 | printf("mmiow%d[%" PRIx64 "]=%" PRIx64 "\n", size, (uint64_t) addr, data); 136 | // printf("env->mflags=%x hflags=%x hflags2=%x\n", 137 | // env->mflags, env->hflags, env->hflags2); 138 | } 139 | #endif 140 | 141 | switch (size) { 142 | case 1: 143 | *(uint8_t *) dataptr = data; 144 | break; 145 | case 2: 146 | *(uint16_t *) dataptr = data; 147 | break; 148 | case 4: 149 | *(uint32_t *) dataptr = data; 150 | break; 151 | default: 152 | assert(false && "Can't get here"); 153 | } 154 | 155 | bool is_apic_tpr_access = false; 156 | if ((addr >> TARGET_PAGE_BITS) == (env->v_apic_base >> TARGET_PAGE_BITS)) { 157 | if ((addr & 0xfff) == 0x80) { 158 | abort_and_retranslate_if_needed(); 159 | env->v_apic_tpr = (uint8_t) data; 160 | env->v_tpr = env->v_apic_tpr >> 4; 161 | is_apic_tpr_access = true; 162 | } 163 | } 164 | 165 | coroutine_yield(); 166 | 167 | // A write to the task priority register may umask hardware interrupts. 168 | // A real KVM implementation would handle them ASAP on the next instruction. 169 | // We try to do it as best as we can here by requesting an exit from the CPU loop. 170 | // Some buggy guests may crash if we exit too late (e.g., winxp). 171 | // This mechanism is complementary to s2e_kvm_request_exit(). 172 | if (is_apic_tpr_access) { 173 | cpu_exit(env); 174 | } 175 | } 176 | 177 | uint64_t s2e_kvm_ioport_read(pio_addr_t addr, unsigned size) { 178 | ++g_stats.io_reads; 179 | 180 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_IO; 181 | g_kvm_vcpu_buffer->io.direction = KVM_EXIT_IO_IN; 182 | g_kvm_vcpu_buffer->io.size = size; 183 | g_kvm_vcpu_buffer->io.port = addr; 184 | g_kvm_vcpu_buffer->io.count = 1; 185 | 186 | unsigned offs = sizeof(struct kvm_run); 187 | uint8_t *dataptr = (uint8_t *) g_kvm_vcpu_buffer; 188 | dataptr += offs; 189 | 190 | g_kvm_vcpu_buffer->io.data_offset = offs; 191 | 192 | coroutine_yield(); 193 | 194 | uint64_t ret; 195 | switch (size) { 196 | case 1: 197 | ret = *(uint8_t *) dataptr; 198 | break; 199 | case 2: 200 | ret = *(uint16_t *) dataptr; 201 | break; 202 | case 4: 203 | ret = *(uint32_t *) dataptr; 204 | break; 205 | default: 206 | assert(false && "Can't get here"); 207 | } 208 | 209 | #ifdef SE_KVM_DEBUG_IO 210 | printf("ior%d[%x]=%" PRIx64 "\n", size, addr, ret); 211 | // printf("env->mflags=%x hflags=%x hflags2=%x\n", 212 | // env->mflags, env->hflags, env->hflags2); 213 | #endif 214 | 215 | return ret; 216 | } 217 | 218 | void s2e_kvm_ioport_write(pio_addr_t addr, uint64_t data, unsigned size) { 219 | ++g_stats.io_writes; 220 | 221 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_IO; 222 | g_kvm_vcpu_buffer->io.direction = KVM_EXIT_IO_OUT; 223 | g_kvm_vcpu_buffer->io.size = size; 224 | g_kvm_vcpu_buffer->io.port = addr; 225 | g_kvm_vcpu_buffer->io.count = 1; 226 | 227 | unsigned offs = sizeof(struct kvm_run); 228 | uint8_t *dataptr = (uint8_t *) g_kvm_vcpu_buffer; 229 | dataptr += offs; 230 | 231 | g_kvm_vcpu_buffer->io.data_offset = offs; 232 | 233 | switch (size) { 234 | case 1: 235 | *(uint8_t *) dataptr = data; 236 | break; 237 | case 2: 238 | *(uint16_t *) dataptr = data; 239 | break; 240 | case 4: 241 | *(uint32_t *) dataptr = data; 242 | break; 243 | default: 244 | assert(false && "Can't get here"); 245 | } 246 | 247 | #ifdef SE_KVM_DEBUG_IO 248 | printf("iow%d[%x]=%" PRIx64 "\n", size, addr, data); 249 | // printf("env->mflags=%x hflags=%x hflags2=%x\n", 250 | // env->mflags, env->hflags, env->hflags2); 251 | #endif 252 | 253 | coroutine_yield(); 254 | } 255 | 256 | struct cpu_io_funcs_t g_io = { 257 | .io_read = s2e_kvm_ioport_read, 258 | .io_write = s2e_kvm_ioport_write, 259 | .mmio_read = s2e_kvm_mmio_read, 260 | .mmio_write = s2e_kvm_mmio_write, 261 | }; 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /src/s2e-kvm-state.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "s2e-kvm-vcpu.h" 21 | 22 | #define WR_cpu(cpu, reg, value) \ 23 | g_sqi.regs.write_concrete(offsetof(CPUX86State, reg), (uint8_t *) &value, sizeof(target_ulong)) 24 | #define RR_cpu(cpu, reg, value) \ 25 | g_sqi.regs.read_concrete(offsetof(CPUX86State, reg), (uint8_t *) &value, sizeof(target_ulong)) 26 | 27 | extern "C" { 28 | // XXX: fix this declaration 29 | void helper_wrmsr_v(target_ulong index, uint64_t val); 30 | uint64_t helper_rdmsr_v(uint64_t index); 31 | } 32 | 33 | namespace s2e { 34 | namespace kvm { 35 | 36 | int VCPU::setRegisters(kvm_regs *regs) { 37 | #ifdef CONFIG_SYMBEX 38 | WR_cpu(m_env, regs[R_EAX], regs->rax); 39 | WR_cpu(m_env, regs[R_EBX], regs->rbx); 40 | WR_cpu(m_env, regs[R_ECX], regs->rcx); 41 | WR_cpu(m_env, regs[R_EDX], regs->rdx); 42 | WR_cpu(m_env, regs[R_ESI], regs->rsi); 43 | WR_cpu(m_env, regs[R_EDI], regs->rdi); 44 | WR_cpu(m_env, regs[R_ESP], regs->rsp); 45 | WR_cpu(m_env, regs[R_EBP], regs->rbp); 46 | 47 | #ifdef TARGET_X86_64 48 | WR_cpu(m_env, regs[8], regs->r8); 49 | WR_cpu(m_env, regs[9], regs->r9); 50 | WR_cpu(m_env, regs[10], regs->r10); 51 | WR_cpu(m_env, regs[11], regs->r11); 52 | WR_cpu(m_env, regs[12], regs->r12); 53 | WR_cpu(m_env, regs[13], regs->r13); 54 | WR_cpu(m_env, regs[14], regs->r14); 55 | WR_cpu(m_env, regs[15], regs->r15); 56 | #endif 57 | #else 58 | m_env->regs[R_EAX] = regs->rax; 59 | m_env->regs[R_EBX] = regs->rbx; 60 | m_env->regs[R_ECX] = regs->rcx; 61 | m_env->regs[R_EDX] = regs->rdx; 62 | m_env->regs[R_ESI] = regs->rsi; 63 | m_env->regs[R_EDI] = regs->rdi; 64 | m_env->regs[R_ESP] = regs->rsp; 65 | m_env->regs[R_EBP] = regs->rbp; 66 | 67 | #ifdef TARGET_X86_64 68 | m_env->regs[8] = regs->r8; 69 | m_env->regs[9] = regs->r9; 70 | m_env->regs[10] = regs->r10; 71 | m_env->regs[11] = regs->r11; 72 | m_env->regs[12] = regs->r12; 73 | m_env->regs[13] = regs->r13; 74 | m_env->regs[14] = regs->r14; 75 | m_env->regs[15] = regs->r15; 76 | #endif 77 | #endif 78 | 79 | if (regs->rip != m_env->eip) { 80 | if (m_handlingKvmCallback || !m_cpuStateIsPrecise) { 81 | // We don't support this at all, it's better to crash than to risk 82 | // guest corruption. 83 | abort(); 84 | } 85 | } 86 | 87 | m_env->eip = regs->rip; 88 | 89 | if (m_handlingKvmCallback) { 90 | fprintf(stderr, "warning: kvm setting cpu state while handling io\n"); 91 | // TODO: try to set the system part of the flags register. 92 | // It should be OK to skip these because the KVM client usually writes 93 | // back the value it has just read when KVM_RUN exits. That value 94 | // is already stored in the CPU state of the symbex engine. 95 | assert(regs->rflags == m_env->mflags); 96 | } else { 97 | cpu_set_eflags(m_env, regs->rflags); 98 | } 99 | 100 | return 0; 101 | } 102 | 103 | int VCPU::setFPU(kvm_fpu *fpu) { 104 | m_env->fpstt = (fpu->fsw >> 11) & 7; 105 | m_env->fpus = fpu->fsw; 106 | m_env->fpuc = fpu->fcw; 107 | m_env->fpop = fpu->last_opcode; 108 | m_env->fpip = fpu->last_ip; 109 | m_env->fpdp = fpu->last_dp; 110 | for (unsigned i = 0; i < 8; ++i) { 111 | m_env->fptags[i] = !((fpu->ftwx >> i) & 1); 112 | } 113 | memcpy(m_env->fpregs, fpu->fpr, sizeof m_env->fpregs); 114 | memcpy(m_env->xmm_regs, fpu->xmm, sizeof m_env->xmm_regs); 115 | m_env->mxcsr = fpu->mxcsr; 116 | return 0; 117 | } 118 | 119 | void VCPU::setCpuSegment(SegmentCache *libcpu_seg, const kvm_segment *kvm_seg) { 120 | libcpu_seg->selector = kvm_seg->selector; 121 | libcpu_seg->base = kvm_seg->base; 122 | libcpu_seg->limit = kvm_seg->limit; 123 | libcpu_seg->flags = (kvm_seg->type << DESC_TYPE_SHIFT) | (kvm_seg->present * DESC_P_MASK) | 124 | (kvm_seg->dpl << DESC_DPL_SHIFT) | (kvm_seg->db << DESC_B_SHIFT) | (kvm_seg->s * DESC_S_MASK) | 125 | (kvm_seg->l << DESC_L_SHIFT) | (kvm_seg->g * DESC_G_MASK) | (kvm_seg->avl * DESC_AVL_MASK); 126 | 127 | if (libcpu_seg->flags & DESC_G_MASK) { 128 | libcpu_seg->flags |= (libcpu_seg->limit >> 12) & 0x000f0000; 129 | } 130 | 131 | libcpu_seg->flags |= libcpu_seg->base & 0xff000000; 132 | libcpu_seg->flags |= (libcpu_seg->base & 0x00ff0000) >> 16; 133 | } 134 | 135 | int VCPU::setSystemRegisters(kvm_sregs *sregs) { 136 | // XXX: what about the interrupt bitmap? 137 | setCpuSegment(&m_env->segs[R_CS], &sregs->cs); 138 | setCpuSegment(&m_env->segs[R_DS], &sregs->ds); 139 | setCpuSegment(&m_env->segs[R_ES], &sregs->es); 140 | setCpuSegment(&m_env->segs[R_FS], &sregs->fs); 141 | setCpuSegment(&m_env->segs[R_GS], &sregs->gs); 142 | setCpuSegment(&m_env->segs[R_SS], &sregs->ss); 143 | 144 | setCpuSegment(&m_env->tr, &sregs->tr); 145 | setCpuSegment(&m_env->ldt, &sregs->ldt); 146 | 147 | m_env->idt.limit = sregs->idt.limit; 148 | m_env->idt.base = sregs->idt.base; 149 | m_env->gdt.limit = sregs->gdt.limit; 150 | m_env->gdt.base = sregs->gdt.base; 151 | 152 | m_env->cr[0] = sregs->cr0; 153 | m_env->cr[2] = sregs->cr2; 154 | m_env->cr[3] = sregs->cr3; 155 | m_env->cr[4] = sregs->cr4; 156 | m_env->v_tpr = sregs->cr8; 157 | m_env->v_apic_tpr = sregs->cr8 << 4; 158 | 159 | if (sregs->apic_base) { 160 | m_env->v_apic_base = sregs->apic_base; 161 | } 162 | 163 | m_env->efer = sregs->efer; 164 | m_env->hflags = cpu_compute_hflags(m_env); 165 | 166 | return 0; 167 | } 168 | 169 | int VCPU::setMSRs(kvm_msrs *msrs) { 170 | for (unsigned i = 0; i < msrs->nmsrs; ++i) { 171 | helper_wrmsr_v(msrs->entries[i].index, msrs->entries[i].data); 172 | } 173 | return msrs->nmsrs; 174 | } 175 | 176 | int VCPU::setMPState(kvm_mp_state *mp) { 177 | /* Only needed when using an irq chip */ 178 | return 0; 179 | } 180 | 181 | int VCPU::getRegisters(kvm_regs *regs) { 182 | if (!m_cpuStateIsPrecise) { 183 | // Probably OK to let execution continue 184 | fprintf(stderr, "Getting register state in the middle of a translation block, eip/flags may be imprecise\n"); 185 | } 186 | 187 | #ifdef CONFIG_SYMBEX 188 | RR_cpu(m_env, regs[R_EAX], regs->rax); 189 | RR_cpu(m_env, regs[R_EBX], regs->rbx); 190 | RR_cpu(m_env, regs[R_ECX], regs->rcx); 191 | RR_cpu(m_env, regs[R_EDX], regs->rdx); 192 | RR_cpu(m_env, regs[R_ESI], regs->rsi); 193 | RR_cpu(m_env, regs[R_EDI], regs->rdi); 194 | RR_cpu(m_env, regs[R_ESP], regs->rsp); 195 | RR_cpu(m_env, regs[R_EBP], regs->rbp); 196 | 197 | #ifdef TARGET_X86_64 198 | RR_cpu(m_env, regs[8], regs->r8); 199 | RR_cpu(m_env, regs[9], regs->r9); 200 | RR_cpu(m_env, regs[10], regs->r10); 201 | RR_cpu(m_env, regs[11], regs->r11); 202 | RR_cpu(m_env, regs[12], regs->r12); 203 | RR_cpu(m_env, regs[13], regs->r13); 204 | RR_cpu(m_env, regs[14], regs->r14); 205 | RR_cpu(m_env, regs[15], regs->r15); 206 | #endif 207 | #else 208 | regs->rax = m_env->regs[R_EAX]; 209 | regs->rbx = m_env->regs[R_EBX]; 210 | regs->rcx = m_env->regs[R_ECX]; 211 | regs->rdx = m_env->regs[R_EDX]; 212 | regs->rsi = m_env->regs[R_ESI]; 213 | regs->rdi = m_env->regs[R_EDI]; 214 | regs->rsp = m_env->regs[R_ESP]; 215 | regs->rbp = m_env->regs[R_EBP]; 216 | 217 | #ifdef TARGET_X86_64 218 | regs->r8 = m_env->regs[8]; 219 | regs->r9 = m_env->regs[9]; 220 | regs->r10 = m_env->regs[10]; 221 | regs->r11 = m_env->regs[11]; 222 | regs->r12 = m_env->regs[12]; 223 | regs->r13 = m_env->regs[13]; 224 | regs->r14 = m_env->regs[14]; 225 | regs->r15 = m_env->regs[15]; 226 | #endif 227 | #endif 228 | 229 | regs->rip = m_env->eip; 230 | 231 | if (!m_handlingKvmCallback) { 232 | regs->rflags = cpu_get_eflags(m_env); 233 | } else { 234 | fprintf(stderr, "warning: kvm asking cpu state while handling io\n"); 235 | // We must at least give the system flags to the KVM client, which 236 | // may use them to compute the segment registers. 237 | regs->rflags = m_env->mflags; 238 | } 239 | 240 | return 0; 241 | } 242 | 243 | int VCPU::getFPU(kvm_fpu *fpu) { 244 | int i; 245 | 246 | fpu->fsw = m_env->fpus & ~(7 << 11); 247 | fpu->fsw |= (m_env->fpstt & 7) << 11; 248 | fpu->fcw = m_env->fpuc; 249 | fpu->last_opcode = m_env->fpop; 250 | fpu->last_ip = m_env->fpip; 251 | fpu->last_dp = m_env->fpdp; 252 | for (i = 0; i < 8; ++i) { 253 | fpu->ftwx |= (!m_env->fptags[i]) << i; 254 | } 255 | memcpy(fpu->fpr, m_env->fpregs, sizeof m_env->fpregs); 256 | memcpy(fpu->xmm, m_env->xmm_regs, sizeof m_env->xmm_regs); 257 | fpu->mxcsr = m_env->mxcsr; 258 | 259 | return 0; 260 | } 261 | 262 | void VCPU::getCpuSegment(kvm_segment *kvm_seg, const SegmentCache *libcpu_seg) { 263 | unsigned flags = libcpu_seg->flags; 264 | kvm_seg->selector = libcpu_seg->selector; 265 | kvm_seg->base = libcpu_seg->base; 266 | kvm_seg->limit = libcpu_seg->limit; 267 | kvm_seg->type = (flags >> DESC_TYPE_SHIFT) & 15; 268 | kvm_seg->present = (flags & DESC_P_MASK) != 0; 269 | kvm_seg->dpl = (flags >> DESC_DPL_SHIFT) & 3; 270 | kvm_seg->db = (flags >> DESC_B_SHIFT) & 1; 271 | kvm_seg->s = (flags & DESC_S_MASK) != 0; 272 | kvm_seg->l = (flags >> DESC_L_SHIFT) & 1; 273 | kvm_seg->g = (flags & DESC_G_MASK) != 0; 274 | kvm_seg->avl = (flags & DESC_AVL_MASK) != 0; 275 | kvm_seg->unusable = 0; 276 | kvm_seg->padding = 0; 277 | } 278 | 279 | void VCPU::get8086Segment(kvm_segment *kvm_seg, const SegmentCache *libcpu_seg) { 280 | kvm_seg->selector = libcpu_seg->selector; 281 | kvm_seg->base = libcpu_seg->base; 282 | kvm_seg->limit = libcpu_seg->limit; 283 | kvm_seg->type = 3; 284 | kvm_seg->present = 1; 285 | kvm_seg->dpl = 3; 286 | kvm_seg->db = 0; 287 | kvm_seg->s = 1; 288 | kvm_seg->l = 0; 289 | kvm_seg->g = 0; 290 | kvm_seg->avl = 0; 291 | kvm_seg->unusable = 0; 292 | } 293 | 294 | int VCPU::getSystemRegisters(kvm_sregs *sregs) { 295 | // XXX: what about the interrupt bitmap? 296 | 297 | if (m_env->mflags & VM_MASK) { 298 | get8086Segment(&sregs->cs, &m_env->segs[R_CS]); 299 | get8086Segment(&sregs->ds, &m_env->segs[R_DS]); 300 | get8086Segment(&sregs->es, &m_env->segs[R_ES]); 301 | get8086Segment(&sregs->fs, &m_env->segs[R_FS]); 302 | get8086Segment(&sregs->gs, &m_env->segs[R_GS]); 303 | get8086Segment(&sregs->ss, &m_env->segs[R_SS]); 304 | } else { 305 | getCpuSegment(&sregs->cs, &m_env->segs[R_CS]); 306 | getCpuSegment(&sregs->ds, &m_env->segs[R_DS]); 307 | getCpuSegment(&sregs->es, &m_env->segs[R_ES]); 308 | getCpuSegment(&sregs->fs, &m_env->segs[R_FS]); 309 | getCpuSegment(&sregs->gs, &m_env->segs[R_GS]); 310 | getCpuSegment(&sregs->ss, &m_env->segs[R_SS]); 311 | } 312 | 313 | getCpuSegment(&sregs->tr, &m_env->tr); 314 | getCpuSegment(&sregs->ldt, &m_env->ldt); 315 | 316 | sregs->idt.limit = m_env->idt.limit; 317 | sregs->idt.base = m_env->idt.base; 318 | memset(sregs->idt.padding, 0, sizeof sregs->idt.padding); 319 | sregs->gdt.limit = m_env->gdt.limit; 320 | sregs->gdt.base = m_env->gdt.base; 321 | memset(sregs->gdt.padding, 0, sizeof sregs->gdt.padding); 322 | 323 | sregs->cr0 = m_env->cr[0]; 324 | sregs->cr2 = m_env->cr[2]; 325 | sregs->cr3 = m_env->cr[3]; 326 | sregs->cr4 = m_env->cr[4]; 327 | sregs->cr8 = m_env->v_tpr; 328 | 329 | sregs->apic_base = m_env->v_apic_base; 330 | sregs->cr8 = m_env->v_tpr; 331 | 332 | sregs->efer = m_env->efer; 333 | return 0; 334 | } 335 | 336 | int VCPU::getMSRs(kvm_msrs *msrs) { 337 | for (unsigned i = 0; i < msrs->nmsrs; ++i) { 338 | msrs->entries[i].data = helper_rdmsr_v(msrs->entries[i].index); 339 | } 340 | return msrs->nmsrs; 341 | } 342 | 343 | int VCPU::getMPState(kvm_mp_state *mp) { 344 | // Not needed without IRQ chip? 345 | mp->mp_state = KVM_MP_STATE_RUNNABLE; 346 | return 0; 347 | } 348 | } 349 | } 350 | -------------------------------------------------------------------------------- /src/s2e-kvm-trace.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include "libs2e.h" 25 | #include "s2e-kvm-trace.h" 26 | 27 | namespace s2e { 28 | namespace kvm { 29 | 30 | static struct kvm_run *s_kvm_run; 31 | static int s_kvm_vcpu_size; 32 | 33 | static void trace_s2e_kvm_run(struct kvm_run *run, int ret) { 34 | printf("KVM_RUN EXIT ret=%#x reason=%#x apic_base=%llx cr8=%llx if_flag=%d req_int=%d rdy=%d\n", ret, 35 | run->exit_reason, run->apic_base, run->cr8, run->if_flag, run->request_interrupt_window, 36 | run->ready_for_interrupt_injection); 37 | switch (run->exit_reason) { 38 | case KVM_EXIT_IO: { 39 | printf("KVM_EXIT_IO dir=%d size=%d port=%#" PRIx16 " count=%d\n", run->io.direction, run->io.size, 40 | run->io.port, run->io.count); 41 | } break; 42 | case KVM_EXIT_MMIO: { 43 | printf("KVM_EXIT_MMIO is_write=%d physaddr=%#llx len=%d\n", run->mmio.is_write, run->mmio.phys_addr, 44 | run->mmio.len); 45 | } break; 46 | } 47 | } 48 | 49 | static void trace_s2e_kvm_set_user_memory_region(struct kvm_userspace_memory_region *region) { 50 | printf("%s %s slot=%d guest_phys=%llx size=%llx vaddr=%llx flags=%x\n", __FUNCTION__, "KVM_SET_USER_MEMORY_REGION", 51 | region->slot, region->guest_phys_addr, region->memory_size, region->userspace_addr, region->flags); 52 | } 53 | 54 | IFilePtr KVMTrace::create() { 55 | auto fd = g_original_open("/dev/kvm", O_RDWR, 0); 56 | if (fd < 0) { 57 | return nullptr; 58 | } 59 | 60 | return std::shared_ptr(new KVMTrace(fd)); 61 | } 62 | 63 | int KVMTrace::sys_ioctl(int fd, int request, uint64_t arg1) { 64 | int ret = -1; 65 | 66 | switch ((uint32_t) request) { 67 | case KVM_GET_API_VERSION: 68 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 69 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_API_VERSION", request, 70 | arg1, ret); 71 | break; 72 | 73 | case KVM_CHECK_EXTENSION: 74 | switch (arg1) { 75 | case KVM_CAP_NR_MEMSLOTS: { 76 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 77 | } break; 78 | 79 | case KVM_CAP_MP_STATE: 80 | case KVM_CAP_EXT_CPUID: 81 | case KVM_CAP_SET_TSS_ADDR: 82 | case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 83 | case KVM_CAP_USER_MEMORY: 84 | case KVM_CAP_NR_VCPUS: 85 | case KVM_CAP_MAX_VCPUS: 86 | case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 87 | // case KVM_CAP_READONLY_MEM: 88 | ret = 1; 89 | break; 90 | 91 | default: 92 | // return s_original_ioctl(fd, request, arg1); 93 | printf("Unsupported cap %#" PRIx64 "\n", arg1); 94 | ret = -1; 95 | break; 96 | } 97 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_CHECK_EXTENSION", request, 98 | arg1, ret); 99 | break; 100 | 101 | case KVM_CREATE_VM: { 102 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 103 | if (ret < 0) { 104 | printf("Could not create vm fd (errno=%d %s)\n", errno, strerror(errno)); 105 | exit(-1); 106 | } 107 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_CREATE_VM", request, arg1, 108 | ret); 109 | 110 | auto vm = KVMTraceVM::create(ret); 111 | ret = g_fdm->registerInterface(vm); 112 | 113 | } break; 114 | 115 | case KVM_GET_VCPU_MMAP_SIZE: { 116 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 117 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_VCPU_MMAP_SIZE", 118 | request, arg1, ret); 119 | s_kvm_vcpu_size = ret; 120 | } break; 121 | 122 | case KVM_GET_MSR_INDEX_LIST: { 123 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 124 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_MSR_INDEX_LIST", 125 | request, arg1, ret); 126 | } break; 127 | 128 | case KVM_GET_SUPPORTED_CPUID: { 129 | ret = g_original_ioctl(m_kvm_fd, request, arg1); 130 | printf("KVMTrace::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_SUPPORTED_CPUID", 131 | request, arg1, ret); 132 | } break; 133 | 134 | default: { 135 | printf("KVMTrace::%s Unsupported request %x\n", __FUNCTION__, request); 136 | // return s_original_ioctl(fd, request, arg1); 137 | exit(-1); 138 | } 139 | } 140 | 141 | return ret; 142 | } 143 | 144 | IFilePtr KVMTraceVM::create(int vm_fd) { 145 | return std::shared_ptr(new KVMTraceVM(vm_fd)); 146 | } 147 | 148 | int KVMTraceVM::sys_ioctl(int fd, int request, uint64_t arg1) { 149 | int ret = -1; 150 | switch ((unsigned) request) { 151 | case KVM_SET_TSS_ADDR: { 152 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 153 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_TSS_ADDR", request, 154 | arg1, ret); 155 | } break; 156 | 157 | case KVM_CREATE_VCPU: { 158 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 159 | printf("KVMTraceVM: created vcpu %d\n", ret); 160 | 161 | auto vm = KVMTraceVCPU::create(ret); 162 | ret = g_fdm->registerInterface(vm); 163 | 164 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_CREATE_VCPU", request, 165 | arg1, ret); 166 | } break; 167 | 168 | case KVM_SET_USER_MEMORY_REGION: { 169 | trace_s2e_kvm_set_user_memory_region((struct kvm_userspace_memory_region *) arg1); 170 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 171 | } break; 172 | 173 | case KVM_GET_CLOCK: { 174 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 175 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_CLOCK", request, arg1, 176 | ret); 177 | } break; 178 | 179 | case KVM_SET_CLOCK: { 180 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 181 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_CLOCK", request, arg1, 182 | ret); 183 | } break; 184 | 185 | case KVM_ENABLE_CAP: { 186 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 187 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_ENABLE_CAP", request, 188 | arg1, ret); 189 | } break; 190 | 191 | case KVM_IOEVENTFD: { 192 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 193 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_IOEVENTFD", request, arg1, 194 | ret); 195 | struct kvm_ioeventfd *event = (struct kvm_ioeventfd *) arg1; 196 | printf(" >kvm_ioeventd datamatch=%#llx addr=%llx len=%d fd=%d flags=%#" PRIx32 "\n", event->datamatch, 197 | event->addr, event->len, event->fd, event->flags); 198 | 199 | } break; 200 | 201 | case KVM_SET_IDENTITY_MAP_ADDR: { 202 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 203 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_IDENTITY_MAP_ADDR", 204 | request, arg1, ret); 205 | } break; 206 | 207 | case KVM_GET_DIRTY_LOG: { 208 | ret = g_original_ioctl(m_kvm_vm_fd, request, arg1); 209 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_DIRTY_LOG", request, 210 | arg1, ret); 211 | } break; 212 | 213 | default: { 214 | printf("KVMTraceVM::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "UNSUPPORTED_API", request, 215 | arg1, ret); 216 | } break; 217 | } 218 | 219 | return ret; 220 | } 221 | 222 | IFilePtr KVMTraceVCPU::create(int vcpu_fd) { 223 | return std::shared_ptr(new KVMTraceVCPU(vcpu_fd)); 224 | } 225 | 226 | void *KVMTraceVCPU::sys_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) { 227 | return g_original_mmap64(addr, len, prot, flags, m_kvm_vcpu_fd, offset); 228 | } 229 | 230 | int KVMTraceVCPU::sys_ioctl(int fd, int request, uint64_t arg1) { 231 | int ret = -1; 232 | switch ((uint32_t) request) { 233 | case KVM_GET_CLOCK: { 234 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 235 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_CLOCK", request, 236 | arg1, ret); 237 | } break; 238 | 239 | case KVM_SET_CPUID2: { 240 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 241 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_CPUID2", request, 242 | arg1, ret); 243 | } break; 244 | 245 | case KVM_SET_SIGNAL_MASK: { 246 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 247 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_SIGNAL_MASK", 248 | request, arg1, ret); 249 | } break; 250 | 251 | /***********************************************/ 252 | case KVM_SET_REGS: { 253 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 254 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_REGS", request, 255 | arg1, ret); 256 | } break; 257 | 258 | case KVM_SET_FPU: { 259 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 260 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_FPU", request, arg1, 261 | ret); 262 | } break; 263 | 264 | case KVM_SET_SREGS: { 265 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 266 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_SREGS", request, 267 | arg1, ret); 268 | } break; 269 | 270 | case KVM_SET_MSRS: { 271 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 272 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_MSRS", request, 273 | arg1, ret); 274 | } break; 275 | 276 | case KVM_SET_MP_STATE: { 277 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 278 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_SET_MP_STATE", request, 279 | arg1, ret); 280 | } break; 281 | /***********************************************/ 282 | case KVM_GET_REGS: { 283 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 284 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_REGS", request, 285 | arg1, ret); 286 | } break; 287 | 288 | case KVM_GET_FPU: { 289 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 290 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_FPU", request, arg1, 291 | ret); 292 | } break; 293 | 294 | case KVM_GET_SREGS: { 295 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 296 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_SREGS", request, 297 | arg1, ret); 298 | } break; 299 | 300 | case KVM_GET_MSRS: { 301 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 302 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_MSRS", request, 303 | arg1, ret); 304 | } break; 305 | 306 | case KVM_GET_MP_STATE: { 307 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 308 | printf("KVMTraceVCPU::%s %s req=%#x arg=%#" PRIx64 " ret=%#x\n", __FUNCTION__, "KVM_GET_MP_STATE", request, 309 | arg1, ret); 310 | } break; 311 | 312 | /***********************************************/ 313 | case KVM_RUN: { 314 | if (!s_kvm_run) { 315 | s_kvm_run = (kvm_run *) g_original_mmap(NULL, s_kvm_vcpu_size, PROT_READ | PROT_WRITE, MAP_SHARED, 316 | m_kvm_vcpu_fd, 0); 317 | if (s_kvm_run == MAP_FAILED) { 318 | printf("Could not map the cpu struct errno=%s\n", strerror(errno)); 319 | exit(-1); 320 | } 321 | } 322 | 323 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 324 | trace_s2e_kvm_run(s_kvm_run, ret); 325 | } break; 326 | 327 | case KVM_INTERRUPT: { 328 | struct kvm_interrupt *interrupt = (struct kvm_interrupt *) arg1; 329 | ret = g_original_ioctl(m_kvm_vcpu_fd, request, arg1); 330 | printf("KVMTraceVCPU::%s %s req=%#x irq=%#" PRIx32 " ret=%#x\n", __FUNCTION__, "KVM_INTERRUPT", request, 331 | interrupt->irq, ret); 332 | } break; 333 | 334 | default: { 335 | printf("KVMTraceVCPU::ioctl vcpu %d request=%#x arg=%#" PRIx64 " ret=%#x\n", fd, request, arg1, ret); 336 | exit(-1); 337 | } 338 | } 339 | 340 | return ret; 341 | } 342 | } 343 | } 344 | -------------------------------------------------------------------------------- /src/s2e-kvm-trace.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_TRACE_H 9 | 10 | #define S2E_KVM_TRACE_H 11 | 12 | #include 13 | #include 14 | 15 | #include "FileDescriptorManager.h" 16 | #include "syscalls.h" 17 | 18 | namespace s2e { 19 | namespace kvm { 20 | 21 | class KVMTrace : public IFile { 22 | private: 23 | int m_kvm_fd; 24 | 25 | KVMTrace(int fd) : m_kvm_fd(fd) { 26 | } 27 | 28 | public: 29 | static IFilePtr create(); 30 | 31 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 32 | }; 33 | 34 | class KVMTraceVM : public IFile { 35 | private: 36 | int m_kvm_vm_fd; 37 | 38 | KVMTraceVM(int vm_fd) : m_kvm_vm_fd(vm_fd) { 39 | } 40 | 41 | public: 42 | static IFilePtr create(int vm_fd); 43 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 44 | }; 45 | 46 | class KVMTraceVCPU : public IFile { 47 | private: 48 | int m_kvm_vcpu_fd; 49 | 50 | KVMTraceVCPU(int vcpu_fd) : m_kvm_vcpu_fd(vcpu_fd) { 51 | } 52 | 53 | public: 54 | static IFilePtr create(int vcpu_fd); 55 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 56 | virtual void *sys_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset); 57 | }; 58 | } 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/s2e-kvm-vcpu.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #ifdef CONFIG_SYMBEX 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #endif 25 | 26 | extern "C" { 27 | void tcg_register_thread(void); 28 | } 29 | 30 | #include "s2e-kvm-vcpu.h" 31 | #include "syscalls.h" 32 | 33 | extern "C" { 34 | // Convenience variable to help debugging in gdb. 35 | // env is present in both inside qemu and libs2e, which 36 | // causes confusion. 37 | CPUX86State *g_cpu_env; 38 | } 39 | 40 | // TODO: remove this global var from libcpu 41 | extern CPUX86State *env; 42 | 43 | namespace s2e { 44 | namespace kvm { 45 | 46 | kvm_run *g_kvm_vcpu_buffer; 47 | static VCPU *s_vcpu; 48 | 49 | // We may need a very large stack in case of deep expressions. 50 | // Default stack is a few megabytes, it's not enough. 51 | static const uint64_t S2E_STACK_SIZE = 1024 * 1024 * 1024; 52 | 53 | static const int CPU_EXIT_SIGNAL = SIGUSR2; 54 | 55 | VCPU::VCPU(std::shared_ptr &kvm, std::shared_ptr &vm, kvm_run *buffer) { 56 | s_vcpu = this; 57 | m_kvm = kvm; 58 | m_vm = vm; 59 | m_cpuBuffer = buffer; 60 | assert(!g_kvm_vcpu_buffer); 61 | g_kvm_vcpu_buffer = buffer; 62 | 63 | tcg_register_thread(); 64 | 65 | m_onExit = g_syscalls.onExit.connect(sigc::mem_fun(*this, &VCPU::requestProcessExit)); 66 | m_onSelect = g_syscalls.onSelect.connect(sigc::mem_fun(*this, &VCPU::requestExit)); 67 | 68 | if (initCpuLock() < 0) { 69 | exit(-1); 70 | } 71 | 72 | /* We want the default libcpu CPU, not the KVM one. */ 73 | m_env = g_cpu_env = env = cpu_x86_init(&m_kvm->getCpuid()); 74 | 75 | if (!m_env) { 76 | printf("Could not create cpu\n"); 77 | exit(-1); 78 | } 79 | 80 | m_env->v_apic_base = 0xfee00000; 81 | m_env->size = sizeof(*m_env); 82 | 83 | #ifdef CONFIG_SYMBEX 84 | s2e_register_cpu(m_env); 85 | #endif 86 | 87 | do_cpu_init(m_env); 88 | 89 | cpu_exec_init_all(); 90 | } 91 | 92 | VCPU::~VCPU() { 93 | m_onExit.disconnect(); 94 | m_onSelect.disconnect(); 95 | } 96 | 97 | std::shared_ptr VCPU::create(std::shared_ptr &kvm, std::shared_ptr &vm) { 98 | size_t size = S2EKVM::getVCPUMemoryMapSize(); 99 | auto buffer = (kvm_run *) ::mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); 100 | if (!buffer) { 101 | return nullptr; 102 | } 103 | 104 | return std::shared_ptr(new VCPU(kvm, vm, buffer)); 105 | } 106 | 107 | int VCPU::initCpuLock(void) { 108 | int ret = pthread_mutex_init(&m_cpuLock, nullptr); 109 | if (ret < 0) { 110 | fprintf(stderr, "Could not init cpu lock\n"); 111 | } 112 | 113 | return ret; 114 | } 115 | 116 | void VCPU::lock() { 117 | pthread_mutex_lock(&m_cpuLock); 118 | } 119 | 120 | void VCPU::tryLock() { 121 | pthread_mutex_trylock(&m_cpuLock); 122 | } 123 | 124 | void VCPU::unlock() { 125 | pthread_mutex_unlock(&m_cpuLock); 126 | } 127 | 128 | #ifdef SE_KVM_DEBUG_CPUID 129 | static void print_cpuid2(struct kvm_cpuid_entry2 *e) { 130 | printf("cpuid function=%#010" PRIx32 " index=%#010" PRIx32 " flags=%#010" PRIx32 " eax=%#010" PRIx32 131 | " ebx=%#010" PRIx32 " ecx=%#010" PRIx32 " edx=%#010" PRIx32 "\n", 132 | e->function, e->index, e->flags, e->eax, e->ebx, e->ecx, e->edx); 133 | } 134 | #endif 135 | 136 | int VCPU::getClock(kvm_clock_data *clock) { 137 | assert(false && "Not implemented"); 138 | } 139 | 140 | int VCPU::setCPUID2(kvm_cpuid2 *cpuid) { 141 | /** 142 | * QEMU insists on using host cpuid flags when running in KVM mode. 143 | * We want to use those set in DBT mode instead. 144 | * TODO: for now, we have no way to configure custom flags. 145 | * Snapshots will not work if using anything other that defaults. 146 | */ 147 | 148 | /// This check ensures that users don't mistakenly use the wrong build of libs2e. 149 | #if defined(TARGET_X86_64) 150 | if (cpuid->nent == 15) { 151 | fprintf(stderr, "libs2e for 64-bit guests is used but the KVM client requested 32-bit features\n"); 152 | exit(1); 153 | } 154 | #elif defined(TARGET_I386) 155 | if (cpuid->nent == 21) { 156 | fprintf(stderr, "libs2e for 32-bit guests is used but the KVM client requested 64-bit features\n"); 157 | exit(1); 158 | } 159 | #else 160 | #error unknown architecture 161 | #endif 162 | 163 | for (unsigned i = 0; i < cpuid->nent; ++i) { 164 | const struct kvm_cpuid_entry2 *e = &cpuid->entries[i]; 165 | if (e->function == 1) { 166 | // Allow the KVM client to disable MMX/SSE features. 167 | // E.g., in QEMU, one could do -cpu pentium,-mmx. 168 | // We don't let control all CPUID features yet. 169 | uint32_t allowed_bits = CPUID_MMX | CPUID_SSE | CPUID_SSE2; 170 | uint32_t mask = e->edx & allowed_bits; 171 | m_env->cpuid.cpuid_features &= ~allowed_bits; 172 | m_env->cpuid.cpuid_features |= mask; 173 | } 174 | } 175 | 176 | return 0; 177 | } 178 | 179 | void VCPU::cpuExitSignal(int signum) { 180 | s_vcpu->m_env->kvm_request_interrupt_window = 1; 181 | cpu_exit(s_vcpu->m_env); 182 | } 183 | 184 | void VCPU::initializeCpuExitSignal() { 185 | struct sigaction act; 186 | memset(&act, 0, sizeof(act)); 187 | sigfillset(&act.sa_mask); 188 | act.sa_flags = 0; 189 | act.sa_handler = cpuExitSignal; 190 | 191 | if (sigaction(CPU_EXIT_SIGNAL, &act, NULL) < 0) { 192 | perror("Could not initialize cpu exit signal"); 193 | exit(-1); 194 | } 195 | 196 | // The KVM client usually blocks all signals on the CPU thread. 197 | // This interferes with our ability to exit the CPU loop, so we must unblock it. 198 | union s2e_kvm_sigmask_t mask = m_sigmask; 199 | sigaddset(&mask.sigset, CPU_EXIT_SIGNAL); 200 | if (pthread_sigmask(SIG_UNBLOCK, &mask.sigset, NULL) < 0) { 201 | abort(); 202 | } 203 | } 204 | 205 | // Defines which signals are blocked during execution of kvm. 206 | int VCPU::setSignalMask(kvm_signal_mask *mask) { 207 | // XXX: doesn't seem to matter for typical kvm clients, 208 | // not sure what the implications of spurious signals are. 209 | m_sigmask_size = mask->len; 210 | for (unsigned i = 0; i < mask->len; ++i) { 211 | #ifdef SE_KVM_DEBUG_INTERFACE 212 | printf(" signals %#04x\n", mask->sigset[i]); 213 | #endif 214 | m_sigmask.bytes[i] = mask->sigset[i]; 215 | } 216 | return 0; 217 | } 218 | 219 | void VCPU::coroutineFcn(void *opaque) { 220 | VCPU *vcpu = reinterpret_cast(opaque); 221 | CPUX86State *env = vcpu->m_env; 222 | auto buffer = vcpu->m_cpuBuffer; 223 | 224 | #ifdef SE_KVM_DEBUG_IRQ 225 | static uint64_t prev_mflags = 0; 226 | #endif 227 | 228 | while (1) { 229 | libcpu_run_all_timers(); 230 | 231 | assert(env->current_tb == NULL); 232 | 233 | // XXX: need to save irq state on state switches 234 | if (env->kvm_irq != -1) { 235 | if (env->interrupt_request == 0) { 236 | printf("Forcing IRQ\n"); 237 | } 238 | env->interrupt_request |= CPU_INTERRUPT_HARD; 239 | } 240 | 241 | #ifdef SE_KVM_DEBUG_IRQ 242 | if (env->interrupt_request & CPU_INTERRUPT_HARD) { 243 | printf("Handling IRQ %d req=%#x hflags=%x hflags2=%#x mflags=%#lx tpr=%#x esp=%#lx signal=%d\n", 244 | env->kvm_irq, env->interrupt_request, env->hflags, env->hflags2, (uint64_t) env->mflags, env->v_tpr, 245 | (uint64_t) env->regs[R_ESP], g_signal_pending); 246 | } 247 | #endif 248 | 249 | env->kvm_request_interrupt_window |= buffer->request_interrupt_window; 250 | 251 | #ifdef SE_KVM_DEBUG_IRQ 252 | prev_mflags = env->mflags; 253 | uint64_t prev_eip = env->eip; 254 | #endif 255 | 256 | vcpu->m_cpuStateIsPrecise = false; 257 | env->exit_request = 0; 258 | cpu_x86_exec(env); 259 | vcpu->m_cpuStateIsPrecise = true; 260 | // printf("cpu_exec return %#x\n", ret); 261 | 262 | #ifdef SE_KVM_DEBUG_IRQ 263 | bool mflags_changed = (prev_mflags != env->mflags); 264 | if (mflags_changed) { 265 | printf("mflags changed: %lx old=%lx new=%lx reqwnd=%d peip=%lx, eip=%lx\n", (uint64_t) mflags_changed, 266 | (uint64_t) prev_mflags, (uint64_t) env->mflags, g_kvm_vcpu_buffer->request_interrupt_window, 267 | (uint64_t) prev_eip, (uint64_t) env->eip); 268 | } 269 | prev_mflags = env->mflags; 270 | #endif 271 | 272 | assert(env->current_tb == NULL); 273 | 274 | env->exception_index = 0; 275 | coroutine_yield(); 276 | } 277 | } 278 | 279 | int VCPU::run(int vcpu_fd) { 280 | int ret = 0; 281 | 282 | ++g_stats.kvm_runs; 283 | 284 | if (!m_coroutine) { 285 | m_coroutine = coroutine_create(coroutineFcn, S2E_STACK_SIZE); 286 | if (!m_coroutine) { 287 | fprintf(stderr, "Could not create cpu coroutine\n"); 288 | exit(-1); 289 | } 290 | } 291 | 292 | if (!m_cpuThreadInited) { 293 | initializeCpuExitSignal(); 294 | m_cpuThread = pthread_self(); 295 | m_cpuThreadInited = true; 296 | } 297 | 298 | if (m_kvm->exiting()) { 299 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_INTR; 300 | kill(getpid(), SIGTERM); 301 | errno = EINTR; 302 | return -1; 303 | } 304 | 305 | /* Return asap if interrupts can be injected */ 306 | m_cpuBuffer->if_flag = (m_env->mflags & IF_MASK) != 0; 307 | m_cpuBuffer->apic_base = m_env->v_apic_base; 308 | m_cpuBuffer->cr8 = m_env->v_tpr; 309 | 310 | m_cpuBuffer->ready_for_interrupt_injection = !m_handlingKvmCallback && m_cpuBuffer->request_interrupt_window && 311 | m_cpuBuffer->if_flag && (m_env->kvm_irq == -1); 312 | 313 | if (m_cpuBuffer->ready_for_interrupt_injection) { 314 | #ifdef SE_KVM_DEBUG_IRQ 315 | printf("%s early ret for ints\n", __FUNCTION__); 316 | #endif 317 | m_cpuBuffer->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 318 | return 0; 319 | } 320 | 321 | lock(); 322 | 323 | m_inKvmRun = true; 324 | 325 | #ifdef SE_KVM_DEBUG_RUN 326 | if (!m_handlingKvmCallback) { 327 | printf("%s riw=%d cr8=%#x\n", __FUNCTION__, g_kvm_vcpu_buffer->request_interrupt_window, 328 | (unsigned) g_kvm_vcpu_buffer->cr8); 329 | } 330 | #endif 331 | 332 | m_cpuBuffer->exit_reason = -1; 333 | 334 | /** 335 | * Some KVM clients do not set this when calling kvm_run, although the KVM 336 | * spec says they should. For now, we patch the clients to pass the right value. 337 | * Eventually, we'll need to figure out how KVM handles it. 338 | * Having an incorrect (null) APIC base will cause the APIC to get stuck. 339 | */ 340 | m_env->v_apic_base = m_cpuBuffer->apic_base; 341 | m_env->v_tpr = m_cpuBuffer->cr8; 342 | 343 | m_handlingKvmCallback = false; 344 | m_handlingDeviceState = false; 345 | 346 | coroutine_enter(m_coroutine, this); 347 | 348 | if (m_kvm->exiting()) { 349 | unlock(); 350 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_INTR; 351 | kill(getpid(), SIGTERM); 352 | errno = EINTR; 353 | return -1; 354 | } 355 | 356 | m_handlingKvmCallback = 357 | m_cpuBuffer->exit_reason == KVM_EXIT_IO || m_cpuBuffer->exit_reason == KVM_EXIT_MMIO || 358 | m_cpuBuffer->exit_reason == KVM_EXIT_FLUSH_DISK || m_cpuBuffer->exit_reason == KVM_EXIT_SAVE_DEV_STATE || 359 | m_cpuBuffer->exit_reason == KVM_EXIT_RESTORE_DEV_STATE || m_cpuBuffer->exit_reason == KVM_EXIT_CLONE_PROCESS; 360 | 361 | // Might not be NULL if resuming from an interrupted I/O 362 | // assert(env->current_tb == NULL); 363 | 364 | m_cpuBuffer->if_flag = (m_env->mflags & IF_MASK) != 0; 365 | m_cpuBuffer->apic_base = m_env->v_apic_base; 366 | m_cpuBuffer->cr8 = m_env->v_tpr; 367 | 368 | // KVM specs says that we should also check for request for interrupt window, 369 | // but that causes missed interrupts. 370 | m_cpuBuffer->ready_for_interrupt_injection = !m_handlingKvmCallback && m_cpuBuffer->request_interrupt_window && 371 | m_cpuBuffer->if_flag && (m_env->kvm_irq == -1); 372 | 373 | if (m_cpuBuffer->exit_reason == -1) { 374 | if (m_env->halted) { 375 | m_cpuBuffer->exit_reason = KVM_EXIT_HLT; 376 | } else if (m_cpuBuffer->ready_for_interrupt_injection) { 377 | m_cpuBuffer->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 378 | } else { 379 | m_cpuBuffer->exit_reason = KVM_EXIT_INTR; 380 | m_signalPending = false; 381 | } 382 | } 383 | 384 | #if defined(SE_KVM_DEBUG_HLT) 385 | if (g_kvm_vcpu_buffer->exit_reason == KVM_EXIT_HLT) { 386 | trace_s2e_kvm_run(g_kvm_vcpu_buffer, ret); 387 | } 388 | #endif 389 | 390 | assert(m_cpuBuffer->exit_reason != 1); 391 | 392 | #ifdef SE_KVM_DEBUG_RUN 393 | if (!m_handlingKvmCallback) { 394 | printf("%s riw=%d rii=%d er=%#x cr8=%#x\n", __FUNCTION__, g_kvm_vcpu_buffer->request_interrupt_window, 395 | g_kvm_vcpu_buffer->ready_for_interrupt_injection, g_kvm_vcpu_buffer->exit_reason, 396 | (unsigned) g_kvm_vcpu_buffer->cr8); 397 | } 398 | #endif 399 | 400 | if (m_cpuBuffer->exit_reason == KVM_EXIT_INTR) { 401 | // This must be set at the very end, because syscalls might 402 | // overwrite errno. 403 | errno = EINTR; 404 | ret = -1; 405 | } 406 | 407 | assert(ret >= 0 || errno == EINTR); 408 | assert(m_cpuBuffer->exit_reason != -1); 409 | 410 | m_inKvmRun = false; 411 | 412 | unlock(); 413 | 414 | return ret; 415 | } 416 | 417 | int VCPU::interrupt(kvm_interrupt *interrupt) { 418 | #ifdef SE_KVM_DEBUG_IRQ 419 | printf("IRQ %d env->mflags=%lx hflags=%x hflags2=%x ptr=%#x\n", interrupt->irq, (uint64_t) env->mflags, env->hflags, 420 | env->hflags2, env->v_tpr); 421 | fflush(stdout); 422 | #endif 423 | 424 | if (m_env->cr[0] & CR0_PE_MASK) { 425 | assert(interrupt->irq > (m_env->v_tpr << 4)); 426 | } 427 | assert(!m_handlingKvmCallback); 428 | assert(!m_inKvmRun); 429 | assert(m_env->mflags & IF_MASK); 430 | assert(!(m_env->interrupt_request & CPU_INTERRUPT_HARD)); 431 | m_env->interrupt_request |= CPU_INTERRUPT_HARD; 432 | m_env->kvm_irq = interrupt->irq; 433 | 434 | return 0; 435 | } 436 | 437 | int VCPU::nmi() { 438 | m_env->interrupt_request |= CPU_INTERRUPT_NMI; 439 | return 0; 440 | } 441 | 442 | void VCPU::flushDisk(void) { 443 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_FLUSH_DISK; 444 | m_handlingDeviceState = true; 445 | coroutine_yield(); 446 | } 447 | 448 | void VCPU::saveDeviceState(void) { 449 | #ifdef SE_KVM_DEBUG_DEV_STATE 450 | libcpu_log("Saving device state\n"); 451 | log_cpu_state(g_cpu_env, 0); 452 | #endif 453 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_SAVE_DEV_STATE; 454 | m_handlingDeviceState = true; 455 | coroutine_yield(); 456 | } 457 | 458 | void VCPU::restoreDeviceState(void) { 459 | #ifdef SE_KVM_DEBUG_DEV_STATE 460 | libcpu_log("Restoring device state\n"); 461 | log_cpu_state(g_cpu_env, 0); 462 | #endif 463 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_RESTORE_DEV_STATE; 464 | m_handlingDeviceState = 1; 465 | coroutine_yield(); 466 | } 467 | 468 | void VCPU::cloneProcess(void) { 469 | g_kvm_vcpu_buffer->exit_reason = KVM_EXIT_CLONE_PROCESS; 470 | 471 | coroutine_yield(); 472 | 473 | m_cpuThread = pthread_self(); 474 | 475 | if (m_kvm->initTimerThread() < 0) { 476 | exit(-1); 477 | } 478 | } 479 | 480 | /// 481 | /// \brief s2e_kvm_send_cpu_exit_signal sends a signal 482 | /// to the cpu loop thread in order to exit the cpu loop. 483 | /// 484 | /// It is important to use a signal that executes on the 485 | /// same thread as the cpu loop in order to avoid race conditions 486 | /// and complex locking. 487 | /// 488 | void VCPU::sendExitSignal() { 489 | if (!m_cpuThreadInited) { 490 | return; 491 | } 492 | 493 | if (pthread_kill(m_cpuThread, CPU_EXIT_SIGNAL) < 0) { 494 | abort(); 495 | } 496 | } 497 | 498 | /// 499 | /// \brief s2e_kvm_request_exit triggers an exit from the cpu loop 500 | /// 501 | /// In vanilla KVM, the CPU stops executing guest code when there is 502 | /// an external event pending. Execution can stop at any instruction. 503 | /// 504 | /// In our emulated KVM, stopping at any instruction is not possible 505 | /// because of TB chaining, threading, etc. 506 | /// 507 | /// This may cause missed interrupts. The KVM client is ready to inject an interrupt, 508 | /// but cannot do so because kvm_run has not exited yet. While it is running, 509 | /// several interrupts of different priorities may be queued up. When kvm_run 510 | /// eventually returns, the highest priority interrupt is injected first. 511 | /// Because DBT is much slower than native execution, it often happens 512 | /// that lower priority don't get to run at all, and higher 513 | /// priority ones are missed. 514 | /// 515 | /// Since we can't easily replicate KVM's behavior, we resort to doing 516 | /// what vanilla QEMU in DBT mode would do: interrupt the CPU loop when an interrupt 517 | /// is raised so that the interrupt is scheduled asap. 518 | /// 519 | /// This requires adding an extra API to KVM. Things that have been tried 520 | /// to avoid adding the extra API, but did not work properly: 521 | /// - Intercept pthread_kill. KVM client may kick the CPU when an interrupt is ready. 522 | /// This is still too slow. 523 | /// - Intercept eventfd. KVM clients call poll eventfds instead of using signals. 524 | /// Polling for them from a separate thread didn't work either. 525 | /// 526 | void VCPU::requestExit(void) { 527 | if (!m_env) { 528 | return; 529 | } 530 | 531 | #ifdef SE_KVM_DEBUG_RUN 532 | printf("s2e_kvm_request_exit\n"); 533 | #endif 534 | 535 | sendExitSignal(); 536 | } 537 | 538 | /// 539 | /// \brief s2e_kvm_request_process_exit cleanly exits 540 | /// the process by sending it SIGTERM 541 | /// 542 | /// It is not possible to call exit() directly, as this will 543 | /// abort the process in an unclean manner, possibly causing 544 | /// crashes in other threads. 545 | /// 546 | /// Instead, we intercept the exit() call from the S2E plugin 547 | /// and transform it into a signal that the process will use 548 | /// to exit cleanly. 549 | /// 550 | /// WARNING: this call aborts the cpu loop without cleaning the 551 | /// stack. Any allocated objects there will leak. 552 | /// 553 | /// \param original_exit the original exit() syscall function 554 | /// \param code the exit code 555 | /// 556 | void VCPU::requestProcessExit(int code) { 557 | m_kvm->setExiting(); 558 | 559 | if (!m_coroutine) { 560 | g_original_exit(code); 561 | } 562 | 563 | if (pthread_self() == m_cpuThread) { 564 | coroutine_yield(); 565 | abort(); 566 | } 567 | 568 | g_original_exit(code); 569 | } 570 | 571 | int VCPU::sys_ioctl(int fd, int request, uint64_t arg1) { 572 | int ret = -1; 573 | switch ((uint32_t) request) { 574 | case KVM_GET_CLOCK: { 575 | ret = getClock((kvm_clock_data *) arg1); 576 | } break; 577 | 578 | case KVM_SET_CPUID2: { 579 | ret = setCPUID2((kvm_cpuid2 *) arg1); 580 | } break; 581 | 582 | case KVM_SET_SIGNAL_MASK: { 583 | ret = setSignalMask((kvm_signal_mask *) arg1); 584 | } break; 585 | 586 | /***********************************************/ 587 | // When the symbolic execution engine needs to take a system snapshot, 588 | // it must rely on the KVM client to save the device state. That client 589 | // will typically also save/restore the CPU state. We don't want the client 590 | // to do that, so in order to not modify the client too much, we ignore 591 | // the calls to register setters when they are done in the context of 592 | // device state snapshotting. 593 | case KVM_SET_REGS: { 594 | if (m_handlingDeviceState) { 595 | ret = 0; 596 | } else { 597 | ret = setRegisters((kvm_regs *) arg1); 598 | } 599 | } break; 600 | 601 | case KVM_SET_FPU: { 602 | if (m_handlingDeviceState) { 603 | ret = 0; 604 | } else { 605 | ret = setFPU((kvm_fpu *) arg1); 606 | } 607 | } break; 608 | 609 | case KVM_SET_SREGS: { 610 | if (m_handlingDeviceState) { 611 | ret = 0; 612 | } else { 613 | ret = setSystemRegisters((kvm_sregs *) arg1); 614 | } 615 | } break; 616 | 617 | case KVM_SET_MSRS: { 618 | if (m_handlingDeviceState) { 619 | ret = ((kvm_msrs *) arg1)->nmsrs; 620 | } else { 621 | ret = setMSRs((kvm_msrs *) arg1); 622 | } 623 | } break; 624 | 625 | case KVM_SET_MP_STATE: { 626 | if (m_handlingDeviceState) { 627 | ret = 0; 628 | } else { 629 | ret = setMPState((kvm_mp_state *) arg1); 630 | } 631 | } break; 632 | /***********************************************/ 633 | case KVM_GET_REGS: { 634 | if (m_handlingDeviceState) { 635 | // Poison the returned registers to make sure we don't use 636 | // it again by accident. We can't just fail the call because 637 | // the client needs it to save the cpu state (that we ignore). 638 | memset((void *) arg1, 0xff, sizeof(kvm_regs)); 639 | ret = 0; 640 | } else { 641 | ret = getRegisters((kvm_regs *) arg1); 642 | } 643 | } break; 644 | 645 | case KVM_GET_FPU: { 646 | ret = getFPU((kvm_fpu *) arg1); 647 | } break; 648 | 649 | case KVM_GET_SREGS: { 650 | ret = getSystemRegisters((kvm_sregs *) arg1); 651 | } break; 652 | 653 | case KVM_GET_MSRS: { 654 | ret = getMSRs((kvm_msrs *) arg1); 655 | } break; 656 | 657 | case KVM_GET_MP_STATE: { 658 | ret = getMPState((kvm_mp_state *) arg1); 659 | } break; 660 | 661 | /***********************************************/ 662 | case KVM_RUN: { 663 | return run(fd); 664 | } break; 665 | 666 | case KVM_INTERRUPT: { 667 | ret = interrupt((kvm_interrupt *) arg1); 668 | } break; 669 | 670 | case KVM_NMI: { 671 | ret = nmi(); 672 | } break; 673 | 674 | default: { 675 | fprintf(stderr, "libs2e: unknown KVM VCPU IOCTL vcpu %d request=%#x arg=%#" PRIx64 " ret=%#x\n", fd, 676 | request, arg1, ret); 677 | exit(-1); 678 | } 679 | } 680 | 681 | return ret; 682 | } 683 | 684 | void *VCPU::sys_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) { 685 | int real_size = S2EKVM::getVCPUMemoryMapSize(); 686 | assert(real_size == len); 687 | assert(m_cpuBuffer); 688 | 689 | return m_cpuBuffer; 690 | } 691 | 692 | void VCPU::flushTlb() { 693 | tlb_flush(m_env, 1); 694 | } 695 | } 696 | } 697 | 698 | extern "C" { 699 | 700 | // TODO: pass an interface to S2E instead of having these here 701 | void s2e_kvm_flush_disk(void) { 702 | s2e::kvm::s_vcpu->flushDisk(); 703 | } 704 | 705 | void s2e_kvm_save_device_state(void) { 706 | s2e::kvm::s_vcpu->saveDeviceState(); 707 | } 708 | 709 | void s2e_kvm_restore_device_state(void) { 710 | s2e::kvm::s_vcpu->restoreDeviceState(); 711 | } 712 | 713 | void s2e_kvm_clone_process(void) { 714 | s2e::kvm::s_vcpu->cloneProcess(); 715 | } 716 | } 717 | -------------------------------------------------------------------------------- /src/s2e-kvm-vcpu.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_VCPU_H 9 | 10 | #define S2E_KVM_VCPU_H 11 | 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "s2e-kvm-vm.h" 20 | #include "s2e-kvm.h" 21 | 22 | #include "FileDescriptorManager.h" 23 | #include "syscalls.h" 24 | 25 | namespace s2e { 26 | namespace kvm { 27 | 28 | // TODO: remove this global var 29 | extern kvm_run *g_kvm_vcpu_buffer; 30 | 31 | class VCPU : public IFile { 32 | private: 33 | std::shared_ptr m_kvm; 34 | std::shared_ptr m_vm; 35 | 36 | int m_fd = -1; 37 | 38 | CPUX86State *m_env = nullptr; 39 | 40 | unsigned m_sigmask_size = 0; 41 | 42 | union s2e_kvm_sigmask_t { 43 | sigset_t sigset; 44 | uint8_t bytes[32]; 45 | } m_sigmask = {}; 46 | 47 | pthread_mutex_t m_cpuLock; 48 | pthread_t m_cpuThread; 49 | bool m_cpuThreadInited = false; 50 | 51 | kvm_run *m_cpuBuffer = nullptr; 52 | Coroutine *m_coroutine = nullptr; 53 | 54 | // Indicates that the cpu loop returned with a coroutine switch. 55 | // This happens when an instruction had to suspend its execution 56 | // to let the kvm client handle the operation (e.g., mmio, snapshot, etc.). 57 | bool m_handlingKvmCallback = false; 58 | 59 | // Indicates that the cpu loop is handling a device state snaphsot load/save. 60 | // This implies that g_handling_kvm_cb is 1. 61 | bool m_handlingDeviceState = false; 62 | 63 | bool m_cpuStateIsPrecise = true; 64 | 65 | // XXX: do we need this? Looks unused 66 | bool m_signalPending = false; 67 | 68 | volatile bool m_inKvmRun = false; 69 | 70 | fsigc::connection m_onExit; 71 | fsigc::connection m_onSelect; 72 | 73 | static void cpuExitSignal(int signum); 74 | void initializeCpuExitSignal(); 75 | 76 | static void setCpuSegment(SegmentCache *libcpu_seg, const kvm_segment *kvm_seg); 77 | static void getCpuSegment(kvm_segment *kvm_seg, const SegmentCache *libcpu_seg); 78 | static void get8086Segment(kvm_segment *kvm_seg, const SegmentCache *libcpu_seg); 79 | static void coroutineFcn(void *opaque); 80 | 81 | int initCpuLock(void); 82 | 83 | VCPU(std::shared_ptr &kvm, std::shared_ptr &vm, kvm_run *buffer); 84 | 85 | int getClock(kvm_clock_data *clock); 86 | int setCPUID2(kvm_cpuid2 *cpuid); 87 | 88 | // Defines which signals are blocked during execution of kvm. 89 | int setSignalMask(kvm_signal_mask *mask); 90 | 91 | /// 92 | /// \brief s2e_kvm_vcpu_set_regs set the general purpose registers of the CPU 93 | /// 94 | /// libcpu does not track register the program counter and eflags state precisely, 95 | /// in order to speed up execution. More precisely, it will not update these registers 96 | /// after each instruction is executed. This has important implications for KVM clients. 97 | /// When guest code executes an instruction that causes a VM exit (e.g., memory access 98 | /// to a device), the following happens: 99 | /// 100 | /// 1. libcpu suspends the current translation block and calls the I/O handler in libs2e 101 | /// 2. Functions in s2e-kvm-io.c trigger a coroutine switch to s2e_kvm_vcpu_run, 102 | /// which returns to the KVM client 103 | /// 3. The KVM client handles the I/O emulation 104 | /// 4. The KVM client re-enters s2e_kvm_vcpu_run, which switches back to the coroutine 105 | /// interrupted in step 2. 106 | /// 5. Execution of the translation block resumes 107 | /// 108 | /// During step 3, I/O emulation may want to access the guest cpu register state using 109 | /// the corresponding KVM APIs. In vanilla KVM, these APIs expect the CPU state to be 110 | /// fully consistent. However, this consistency is broken in libs2e because of how CPU 111 | /// emulation works (see explanation above). Luckily, this situation does not usually 112 | /// happen in practice, as the KVM client reads the CPU state when it is in sync. 113 | /// This function nevertheless checks for this and prints a warning. 114 | /// 115 | /// Same remarks apply for register setters, which may corrupt CPU state if called 116 | /// at a time where the CPU state is not properly committed. 117 | /// 118 | /// In principle, fixing this issue would require calling cpu_restore_state at every 119 | /// exit point. 120 | /// 121 | int setRegisters(kvm_regs *regs); 122 | 123 | int setFPU(kvm_fpu *fpu); 124 | int setSystemRegisters(kvm_sregs *sregs); 125 | int setMSRs(kvm_msrs *msrs); 126 | int setMPState(kvm_mp_state *mp); 127 | 128 | int getRegisters(kvm_regs *regs); 129 | int getFPU(kvm_fpu *fpu); 130 | int getSystemRegisters(kvm_sregs *sregs); 131 | int getMSRs(kvm_msrs *msrs); 132 | int getMPState(kvm_mp_state *mp); 133 | 134 | int run(int vcpu_fd); 135 | int interrupt(kvm_interrupt *interrupt); 136 | int nmi(); 137 | 138 | void requestProcessExit(int code); 139 | 140 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 141 | virtual void *sys_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset); 142 | 143 | public: 144 | virtual ~VCPU(); 145 | 146 | static std::shared_ptr create(std::shared_ptr &kvm, std::shared_ptr &vm); 147 | 148 | void lock(); 149 | void tryLock(); 150 | void unlock(); 151 | 152 | void sendExitSignal(); 153 | void requestExit(void); 154 | 155 | void flushDisk(void); 156 | void saveDeviceState(void); 157 | void restoreDeviceState(void); 158 | void cloneProcess(void); 159 | 160 | inline bool inKvmRun() const { 161 | return m_inKvmRun; 162 | } 163 | 164 | void flushTlb(); 165 | }; 166 | } 167 | } 168 | 169 | #endif 170 | -------------------------------------------------------------------------------- /src/s2e-kvm-vm.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #ifdef CONFIG_SYMBEX 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #endif 28 | 29 | #include "libs2e.h" 30 | #include "s2e-kvm-vcpu.h" 31 | #include "s2e-kvm-vm.h" 32 | #include "s2e-kvm.h" 33 | 34 | #ifdef CONFIG_SYMBEX 35 | bool g_execute_always_klee = false; 36 | #endif 37 | 38 | namespace s2e { 39 | namespace kvm { 40 | 41 | std::shared_ptr VM::create(std::shared_ptr &kvm) { 42 | auto ret = std::shared_ptr(new VM(kvm)); 43 | 44 | cpu_register_io(&g_io); 45 | tcg_exec_init(0); 46 | 47 | init_clocks(); 48 | 49 | #ifdef CONFIG_SYMBEX 50 | s2e_init_device_state(); 51 | s2e_init_timers(); 52 | 53 | s2e_initialize_execution(g_execute_always_klee); 54 | s2e_register_dirty_mask((uint64_t) get_ram_list_phys_dirty(), get_ram_list_phys_dirty_size() >> TARGET_PAGE_BITS); 55 | s2e_on_initialization_complete(); 56 | #endif 57 | 58 | return ret; 59 | } 60 | 61 | void VM::sendCpuExitSignal() { 62 | if (m_cpu) { 63 | m_cpu->sendExitSignal(); 64 | } 65 | } 66 | 67 | int VM::enableCapability(kvm_enable_cap *cap) { 68 | printf("Enable capability not supported %d\n", cap->cap); 69 | errno = 1; 70 | return -1; 71 | } 72 | 73 | int VM::createVirtualCPU() { 74 | if (m_cpu) { 75 | return -1; 76 | } 77 | 78 | auto vm = std::dynamic_pointer_cast(g_fdm->get(this)); 79 | assert(vm && vm.get() == this); 80 | 81 | // TODO: implement this 82 | auto vcpu = VCPU::create(m_kvm, vm); 83 | if (!vcpu) { 84 | return -1; 85 | } 86 | 87 | m_cpu = vcpu; 88 | 89 | return g_fdm->registerInterface(vcpu); 90 | } 91 | 92 | int VM::setTSSAddress(uint64_t tss_addr) { 93 | #ifdef SE_KVM_DEBUG_INTERFACE 94 | printf("Setting tss addr %#" PRIx64 " not implemented yet\n", tss_addr); 95 | #endif 96 | return 0; 97 | } 98 | 99 | int VM::setUserMemoryRegion(kvm_userspace_memory_region *region) { 100 | m_cpu->requestExit(); 101 | 102 | m_cpu->lock(); 103 | 104 | assert(!m_cpu->inKvmRun()); 105 | m_cpu->flushTlb(); 106 | mem_desc_unregister(region->slot); 107 | mem_desc_register(region); 108 | 109 | m_cpu->unlock(); 110 | 111 | return 0; 112 | } 113 | 114 | int VM::memoryReadWrite(kvm_mem_rw *mem) { 115 | #if !defined(CONFIG_SYMBEX_MP) 116 | if (!mem->is_write) { 117 | // Fast path for reads 118 | // TODO: also do it for writes 119 | memcpy((void *) mem->dest, (void *) mem->source, mem->length); 120 | return 0; 121 | } 122 | #endif 123 | 124 | m_cpu->requestExit(); 125 | m_cpu->lock(); 126 | cpu_host_memory_rw(mem->source, mem->dest, mem->length, mem->is_write); 127 | m_cpu->unlock(); 128 | return 0; 129 | } 130 | 131 | int VM::registerFixedRegion(kvm_fixed_region *region) { 132 | #ifdef CONFIG_SYMBEX_MP 133 | s2e_register_ram2(region->name, region->host_address, region->size, region->flags & KVM_MEM_SHARED_CONCRETE); 134 | #endif 135 | return 0; 136 | } 137 | 138 | int VM::getDirtyLog(kvm_dirty_log *log) { 139 | m_cpu->requestExit(); 140 | 141 | const MemoryDesc *r = mem_desc_get_slot(log->slot); 142 | 143 | if (m_kvm->exiting()) { 144 | // This may happen if we are called from an exit handler, e.g., if 145 | // plugin code called exit() from the cpu loop. We don't want 146 | // to deadlock in this case, so return conservatively all dirty. 147 | memset(log->dirty_bitmap, 0xff, (r->kvm.memory_size >> TARGET_PAGE_BITS) / 8); 148 | return 0; 149 | } 150 | 151 | m_cpu->tryLock(); 152 | 153 | cpu_physical_memory_get_dirty_bitmap((uint8_t *) log->dirty_bitmap, r->ram_addr, r->kvm.memory_size, 154 | VGA_DIRTY_FLAG); 155 | 156 | cpu_physical_memory_reset_dirty(r->ram_addr, r->ram_addr + r->kvm.memory_size - 1, VGA_DIRTY_FLAG); 157 | 158 | m_cpu->unlock(); 159 | return 0; 160 | } 161 | 162 | int VM::setIdentityMapAddress(uint64_t addr) { 163 | assert(false && "Not implemented"); 164 | } 165 | 166 | int VM::setClock(kvm_clock_data *clock) { 167 | g_clock_start = clock->clock; 168 | g_clock_offset = cpu_get_real_ticks(); 169 | return 0; 170 | } 171 | 172 | int VM::getClock(kvm_clock_data *clock) { 173 | clock->clock = cpu_get_real_ticks() - g_clock_offset + g_clock_start; 174 | clock->flags = 0; 175 | return 0; 176 | } 177 | 178 | int VM::ioEventFD(kvm_ioeventfd *event) { 179 | #ifdef SE_KVM_DEBUG_INTERFACE 180 | printf("kvm_ioeventd datamatch=%#llx addr=%#llx len=%d fd=%d flags=%#" PRIx32 "\n", event->datamatch, event->addr, 181 | event->len, event->fd, event->flags); 182 | #endif 183 | return -1; 184 | } 185 | 186 | int VM::diskReadWrite(kvm_disk_rw *d) { 187 | #ifdef CONFIG_SYMBEX 188 | if (d->is_write) { 189 | d->count = s2e_bdrv_write(nullptr, d->sector, (uint8_t *) d->host_address, d->count); 190 | } else { 191 | d->count = s2e_bdrv_read(nullptr, d->sector, (uint8_t *) d->host_address, d->count); 192 | } 193 | return 0; 194 | #else 195 | return -1; 196 | #endif 197 | } 198 | 199 | int VM::deviceSnapshot(kvm_dev_snapshot *s) { 200 | #ifdef CONFIG_SYMBEX_MP 201 | if (s->is_write) { 202 | return s2e_dev_save((void *) s->buffer, s->size); 203 | } else { 204 | return s2e_dev_restore((void *) s->buffer, s->pos, s->size); 205 | } 206 | #else 207 | return -1; 208 | #endif 209 | } 210 | 211 | int VM::setClockScalePointer(unsigned *scale) { 212 | #ifdef CONFIG_SYMBEX 213 | if (!scale) { 214 | return -1; 215 | } 216 | 217 | g_sqi.exec.clock_scaling_factor = scale; 218 | return 0; 219 | #else 220 | return -1; 221 | #endif 222 | } 223 | 224 | int VM::sys_ioctl(int fd, int request, uint64_t arg1) { 225 | int ret = -1; 226 | switch ((uint32_t) request) { 227 | case KVM_CHECK_EXTENSION: 228 | ret = m_kvm->checkExtension(arg1); 229 | if (ret < 0) { 230 | errno = 1; 231 | } 232 | break; 233 | 234 | case KVM_SET_TSS_ADDR: { 235 | ret = setTSSAddress(arg1); 236 | } break; 237 | 238 | case KVM_CREATE_VCPU: { 239 | ret = createVirtualCPU(); 240 | } break; 241 | 242 | case KVM_SET_USER_MEMORY_REGION: { 243 | ret = setUserMemoryRegion((kvm_userspace_memory_region *) arg1); 244 | } break; 245 | 246 | case KVM_SET_CLOCK: { 247 | ret = setClock((kvm_clock_data *) arg1); 248 | } break; 249 | 250 | case KVM_GET_CLOCK: { 251 | ret = getClock((kvm_clock_data *) arg1); 252 | } break; 253 | 254 | case KVM_ENABLE_CAP: { 255 | ret = enableCapability((kvm_enable_cap *) arg1); 256 | } break; 257 | 258 | case KVM_IOEVENTFD: { 259 | ret = ioEventFD((kvm_ioeventfd *) arg1); 260 | } break; 261 | 262 | case KVM_SET_IDENTITY_MAP_ADDR: { 263 | ret = setIdentityMapAddress(arg1); 264 | } break; 265 | 266 | case KVM_GET_DIRTY_LOG: { 267 | ret = getDirtyLog((kvm_dirty_log *) arg1); 268 | } break; 269 | 270 | case KVM_MEM_RW: { 271 | ret = memoryReadWrite((kvm_mem_rw *) arg1); 272 | } break; 273 | 274 | case KVM_FORCE_EXIT: { 275 | m_cpu->requestExit(); 276 | ret = 0; 277 | } break; 278 | 279 | case KVM_MEM_REGISTER_FIXED_REGION: { 280 | ret = registerFixedRegion((kvm_fixed_region *) arg1); 281 | } break; 282 | 283 | case KVM_DISK_RW: { 284 | ret = diskReadWrite((kvm_disk_rw *) arg1); 285 | } break; 286 | 287 | case KVM_DEV_SNAPSHOT: { 288 | ret = deviceSnapshot((kvm_dev_snapshot *) arg1); 289 | } break; 290 | 291 | case KVM_SET_CLOCK_SCALE: { 292 | ret = setClockScalePointer((unsigned *) arg1); 293 | } break; 294 | 295 | default: { 296 | fprintf(stderr, "libs2e: unknown KVM VM IOCTL %x\n", request); 297 | exit(-1); 298 | } 299 | } 300 | 301 | return ret; 302 | } 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /src/s2e-kvm-vm.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_VM_H 9 | 10 | #define S2E_KVM_VM_H 11 | 12 | #include 13 | #include 14 | 15 | #include "FileDescriptorManager.h" 16 | #include "s2e-kvm.h" 17 | 18 | namespace s2e { 19 | namespace kvm { 20 | 21 | class VCPU; 22 | 23 | class VM : public IFile { 24 | private: 25 | std::shared_ptr m_kvm; 26 | std::shared_ptr m_cpu; 27 | 28 | VM(std::shared_ptr &kvm) : m_kvm(kvm) { 29 | } 30 | 31 | int enableCapability(kvm_enable_cap *cap); 32 | 33 | int createVirtualCPU(); 34 | 35 | int setTSSAddress(uint64_t tss_addr); 36 | 37 | int setUserMemoryRegion(kvm_userspace_memory_region *region); 38 | 39 | /// 40 | /// \brief memoryReadWrite intercepts all dma read/writes from the kvm client. 41 | /// 42 | /// This is important in order to keep the cpu code cache consistent as well 43 | /// as to keep track of dirty page. 44 | /// 45 | /// In multi-path mode, this ensures that dma reads/writes go to the right state 46 | /// in addition to keeping track of dirty pages. 47 | /// 48 | /// \param vm_fd the vm descriptor 49 | /// \param mem the memory descriptor 50 | /// \return 51 | /// 52 | int memoryReadWrite(kvm_mem_rw *mem); 53 | 54 | int registerFixedRegion(kvm_fixed_region *region); 55 | 56 | /// 57 | /// \brief getDirtyLog returns a bitmap of dirty pages 58 | /// for the given memory buffer. 59 | /// 60 | /// This is usually used for graphics memory by kvm clients. 61 | /// 62 | /// \param vm_fd the virtual machine fd 63 | /// \param log the bitmap structure 64 | /// \return 65 | /// 66 | int getDirtyLog(kvm_dirty_log *log); 67 | 68 | int setIdentityMapAddress(uint64_t addr); 69 | 70 | int setClock(kvm_clock_data *clock); 71 | int getClock(kvm_clock_data *clock); 72 | 73 | int ioEventFD(kvm_ioeventfd *event); 74 | 75 | int diskReadWrite(kvm_disk_rw *d); 76 | int deviceSnapshot(kvm_dev_snapshot *s); 77 | int setClockScalePointer(unsigned *scale); 78 | 79 | public: 80 | virtual ~VM() { 81 | } 82 | 83 | static std::shared_ptr create(std::shared_ptr &kvm); 84 | 85 | void sendCpuExitSignal(); 86 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 87 | }; 88 | } 89 | } 90 | 91 | #endif 92 | -------------------------------------------------------------------------------- /src/s2e-kvm.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #ifdef CONFIG_SYMBEX 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #endif 30 | 31 | #include "libs2e.h" 32 | #include "s2e-kvm-vm.h" 33 | #include "s2e-kvm.h" 34 | 35 | extern void *g_s2e; 36 | 37 | extern CPUX86State *env; 38 | 39 | namespace s2e { 40 | namespace kvm { 41 | 42 | static std::shared_ptr s_s2e_kvm; 43 | struct stats_t g_stats; 44 | 45 | static const int MAX_MEMORY_SLOTS = 32; 46 | 47 | // clang-format off 48 | static uint32_t s_msr_list [] = { 49 | MSR_IA32_SYSENTER_CS, 50 | MSR_IA32_SYSENTER_ESP, 51 | MSR_IA32_SYSENTER_EIP, 52 | MSR_IA32_APICBASE, 53 | MSR_EFER, 54 | MSR_STAR, 55 | MSR_PAT, 56 | MSR_VM_HSAVE_PA, 57 | #ifdef TARGET_X86_64 58 | MSR_LSTAR, 59 | MSR_CSTAR, 60 | MSR_FMASK, 61 | MSR_FSBASE, 62 | MSR_GSBASE, 63 | MSR_KERNELGSBASE, 64 | #endif 65 | MSR_MTRRphysBase(0), 66 | MSR_MTRRphysBase(1), 67 | MSR_MTRRphysBase(2), 68 | MSR_MTRRphysBase(3), 69 | MSR_MTRRphysBase(4), 70 | MSR_MTRRphysBase(5), 71 | MSR_MTRRphysBase(6), 72 | MSR_MTRRphysBase(7), 73 | MSR_MTRRphysMask(0), 74 | MSR_MTRRphysMask(1), 75 | MSR_MTRRphysMask(2), 76 | MSR_MTRRphysMask(3), 77 | MSR_MTRRphysMask(4), 78 | MSR_MTRRphysMask(5), 79 | MSR_MTRRphysMask(6), 80 | MSR_MTRRphysMask(7), 81 | MSR_MTRRfix64K_00000, 82 | MSR_MTRRfix16K_80000, 83 | MSR_MTRRfix16K_A0000, 84 | MSR_MTRRfix4K_C0000, 85 | MSR_MTRRfix4K_C8000, 86 | MSR_MTRRfix4K_D0000, 87 | MSR_MTRRfix4K_D8000, 88 | MSR_MTRRfix4K_E0000, 89 | MSR_MTRRfix4K_E8000, 90 | MSR_MTRRfix4K_F0000, 91 | MSR_MTRRfix4K_F8000, 92 | MSR_MTRRdefType, 93 | MSR_MCG_STATUS, 94 | MSR_MCG_CTL, 95 | MSR_TSC_AUX, 96 | MSR_IA32_MISC_ENABLE, 97 | MSR_MC0_CTL, 98 | MSR_MC0_STATUS, 99 | MSR_MC0_ADDR, 100 | MSR_MC0_MISC 101 | }; 102 | 103 | #define KVM_CPUID_SIGNATURE 0x40000000 104 | #define KVM_CPUID_FEATURES 0x40000001 105 | #define KVM_FEATURE_CLOCKSOURCE 0 106 | 107 | /* Array of valid (function, index) entries */ 108 | static uint32_t s_cpuid_entries[][2] = { 109 | {0, (uint32_t) -1}, 110 | {1, (uint32_t) -1}, 111 | {2, (uint32_t) -1}, 112 | {4, 0}, 113 | {4, 1}, 114 | {4, 2}, 115 | {4, 3}, 116 | {5, (uint32_t) -1}, 117 | {6, (uint32_t) -1}, 118 | {7, (uint32_t) -1}, 119 | {9, (uint32_t) -1}, 120 | {0xa, (uint32_t) -1}, 121 | {0xd, (uint32_t) -1}, 122 | {KVM_CPUID_SIGNATURE, (uint32_t) -1}, 123 | {KVM_CPUID_FEATURES, (uint32_t) -1}, 124 | {0x80000000, (uint32_t) -1}, 125 | {0x80000001, (uint32_t) -1}, 126 | {0x80000002, (uint32_t) -1}, 127 | {0x80000003, (uint32_t) -1}, 128 | {0x80000004, (uint32_t) -1}, 129 | {0x80000005, (uint32_t) -1}, 130 | {0x80000006, (uint32_t) -1}, 131 | {0x80000008, (uint32_t) -1}, 132 | {0x8000000a, (uint32_t) -1}, 133 | {0xc0000000, (uint32_t) -1}, 134 | {0xc0000001, (uint32_t) -1}, 135 | {0xc0000002, (uint32_t) -1}, 136 | {0xc0000003, (uint32_t) -1}, 137 | {0xc0000004, (uint32_t) -1} 138 | }; 139 | // clang-format on 140 | 141 | #if defined(TARGET_X86_64) 142 | const char *S2EKVM::s_cpuModel = "qemu64-s2e"; 143 | const bool S2EKVM::s_is64 = true; 144 | #elif defined(TARGET_I386) 145 | const char *S2EKVM::s_cpuModel = "qemu32-s2e"; 146 | const bool S2EKVM::s_is64 = false; 147 | #else 148 | #error unknown architecture 149 | #endif 150 | 151 | IFilePtr S2EKVM::create() { 152 | auto ret = std::shared_ptr(new S2EKVM()); 153 | ret->init(); 154 | s_s2e_kvm = ret; 155 | return ret; 156 | } 157 | 158 | void S2EKVM::cleanup(void) { 159 | s_s2e_kvm->m_exiting = true; 160 | 161 | while (!s_s2e_kvm->m_timerExited) { 162 | } 163 | 164 | #ifdef CONFIG_SYMBEX 165 | if (g_s2e) { 166 | monitor_close(); 167 | s2e_close(); 168 | g_s2e = nullptr; 169 | s_s2e_kvm = nullptr; 170 | } 171 | #endif 172 | } 173 | 174 | #ifdef CONFIG_SYMBEX 175 | std::string S2EKVM::getBitcodeLibrary(const std::string &dir) { 176 | #ifdef CONFIG_SYMBEX_MP 177 | std::string name = "op_helper.bc." TARGET_ARCH; 178 | #else 179 | std::string name = "op_helper_sp.bc." TARGET_ARCH; 180 | #endif 181 | 182 | std::stringstream ss; 183 | ss << dir << "/" << name; 184 | auto ret = ss.str(); 185 | if (access(ret.c_str(), R_OK)) { 186 | fprintf(stderr, "Could not find %s.\n" 187 | "Make sure that the environment variable S2E_SHARED_DIR is set properly.\n", 188 | ret.c_str()); 189 | exit(-1); 190 | } 191 | 192 | return ret; 193 | } 194 | #endif 195 | 196 | void S2EKVM::init(void) { 197 | x86_cpudef_setup(); 198 | printf("Initializing %s cpu\n", s_cpuModel); 199 | if (cpu_x86_register(&m_cpuid, s_cpuModel, s_is64) < 0) { 200 | fprintf(stderr, "Could not register CPUID for model %s\n", s_cpuModel); 201 | exit(-1); 202 | } 203 | 204 | initLogLevel(); 205 | 206 | #ifdef CONFIG_SYMBEX 207 | const char *shared_dir = getenv("S2E_SHARED_DIR"); 208 | if (!shared_dir) { 209 | fprintf(stderr, "Warning: S2E_SHARED_DIR environment variable was not specified, " 210 | "using %s\n", 211 | CONFIG_LIBCPU_DATADIR); 212 | shared_dir = CONFIG_LIBCPU_DATADIR; 213 | } 214 | 215 | auto config_file = getenv("S2E_CONFIG"); 216 | 217 | if (!config_file) { 218 | fprintf(stderr, "Warning: S2E_CONFIG environment variable was not specified, " 219 | "using the default (empty) config file\n"); 220 | } 221 | 222 | auto output_dir = getenv("S2E_OUTPUT_DIR"); 223 | 224 | auto bc = getBitcodeLibrary(shared_dir); 225 | fprintf(stdout, "Using module %s\n", bc.c_str()); 226 | tcg_llvm_translator = TCGLLVMTranslator::create(bc); 227 | 228 | if (monitor_init() < 0) { 229 | exit(-1); 230 | } 231 | 232 | int unbuffered_stream = 0; 233 | const char *us = getenv("S2E_UNBUFFERED_STREAM"); 234 | if (us && us[0] == '1') { 235 | unbuffered_stream = 1; 236 | } 237 | 238 | int max_processes = 1; 239 | const char *max_processes_str = getenv("S2E_MAX_PROCESSES"); 240 | if (max_processes) { 241 | max_processes = strtol(max_processes_str, NULL, 0); 242 | } 243 | 244 | init_s2e_libcpu_interface(&g_sqi); 245 | 246 | int argc = 0; 247 | char **argv = {NULL}; 248 | 249 | s2e_initialize(argc, argv, tcg_llvm_translator, config_file, output_dir, unbuffered_stream, 0, max_processes, 250 | shared_dir); 251 | 252 | // Call it twice, because event pointers are only known 253 | // after s2e is inited. 254 | init_s2e_libcpu_interface(&g_sqi); 255 | 256 | s2e_create_initial_state(); 257 | #endif 258 | 259 | atexit(cleanup); 260 | } 261 | 262 | /// 263 | /// \brief s2e_kvm_init_log_level initializes the libcpu log level. 264 | /// 265 | /// This is the same as the -d switch from vanilla QEMU. 266 | /// 267 | void S2EKVM::initLogLevel(void) { 268 | loglevel = 0; 269 | const char *libcpu_log_level = getenv("LIBCPU_LOG_LEVEL"); 270 | if (libcpu_log_level) { 271 | loglevel = cpu_str_to_log_mask(libcpu_log_level); 272 | } 273 | 274 | const char *libcpu_log_file = getenv("LIBCPU_LOG_FILE"); 275 | if (libcpu_log_file) { 276 | logfile = fopen(libcpu_log_file, "w"); 277 | if (!logfile) { 278 | printf("Could not open log file %s\n", libcpu_log_file); 279 | exit(-1); 280 | } 281 | } else { 282 | logfile = stdout; 283 | } 284 | } 285 | 286 | int S2EKVM::getApiVersion(void) { 287 | return KVM_API_VERSION; 288 | } 289 | 290 | int S2EKVM::checkExtension(int capability) { 291 | switch (capability) { 292 | case KVM_CAP_NR_MEMSLOTS: { 293 | return MAX_MEMORY_SLOTS; 294 | } break; 295 | 296 | case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: 297 | case KVM_CAP_MP_STATE: 298 | case KVM_CAP_EXT_CPUID: 299 | case KVM_CAP_SET_TSS_ADDR: 300 | case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 301 | case KVM_CAP_USER_MEMORY: 302 | case KVM_CAP_NR_VCPUS: 303 | case KVM_CAP_MAX_VCPUS: 304 | 305 | // We don't really need to support this call, just pretend that we do. 306 | // The real exit will be done through our custom KVM_CAP_FORCE_EXIT. 307 | case KVM_CAP_IMMEDIATE_EXIT: 308 | 309 | /* libs2e-specific calls */ 310 | case KVM_CAP_DBT: 311 | case KVM_CAP_MEM_RW: 312 | case KVM_CAP_FORCE_EXIT: 313 | return 1; 314 | 315 | #ifdef CONFIG_SYMBEX 316 | case KVM_CAP_MEM_FIXED_REGION: 317 | case KVM_CAP_DISK_RW: 318 | case KVM_CAP_CPU_CLOCK_SCALE: 319 | return 1; 320 | #endif 321 | 322 | // Per-path disk state support is only available with symbex builds. 323 | // Can't write snapshot files there. 324 | #ifdef CONFIG_SYMBEX_MP 325 | case KVM_CAP_DEV_SNAPSHOT: 326 | return 1; 327 | #endif 328 | 329 | case KVM_CAP_ADJUST_CLOCK: 330 | return KVM_CLOCK_TSC_STABLE; 331 | 332 | default: 333 | #ifdef SE_KVM_DEBUG_INTERFACE 334 | printf("Unsupported cap %x\n", capability); 335 | #endif 336 | return -1; 337 | } 338 | } 339 | 340 | int S2EKVM::createVM() { 341 | if (m_vm) { 342 | // Only allow one VM for now 343 | return -1; 344 | } 345 | 346 | auto kvm = std::dynamic_pointer_cast(g_fdm->get(this)); 347 | assert(kvm && kvm.get() == this); 348 | 349 | auto vm = VM::create(kvm); 350 | if (!vm) { 351 | return -1; 352 | } 353 | 354 | m_vm = vm; 355 | 356 | if (initTimerThread() < 0) { 357 | exit(-1); 358 | } 359 | 360 | return g_fdm->registerInterface(vm); 361 | } 362 | 363 | int S2EKVM::getVCPUMemoryMapSize(void) { 364 | return 0x10000; /* Some magic value */ 365 | } 366 | 367 | int S2EKVM::getMSRIndexList(struct kvm_msr_list *list) { 368 | if (list->nmsrs == 0) { 369 | list->nmsrs = sizeof(s_msr_list) / sizeof(s_msr_list[0]); 370 | } else { 371 | for (int i = 0; i < list->nmsrs; ++i) { 372 | list->indices[i] = s_msr_list[i]; 373 | } 374 | } 375 | 376 | return 0; 377 | } 378 | 379 | int S2EKVM::getSupportedCPUID(struct kvm_cpuid2 *cpuid) { 380 | #ifdef SE_KVM_DEBUG_CPUID 381 | printf("%s\n", __FUNCTION__); 382 | #endif 383 | 384 | unsigned int nentries = sizeof(s_cpuid_entries) / sizeof(s_cpuid_entries[0]); 385 | if (cpuid->nent < nentries) { 386 | errno = E2BIG; 387 | return -1; 388 | } else if (cpuid->nent >= nentries) { 389 | cpuid->nent = nentries; 390 | } 391 | 392 | for (unsigned i = 0; i < nentries; ++i) { 393 | struct kvm_cpuid_entry2 *e = &cpuid->entries[i]; 394 | 395 | // KVM-specific CPUIDs go here rather than to cpu_x86_cpuid 396 | // because we don't want to expose them to the guest. 397 | switch (s_cpuid_entries[i][0]) { 398 | case KVM_CPUID_SIGNATURE: 399 | // This returns "KVMKVMVKM" 400 | e->eax = 0x40000001; 401 | e->ebx = 0x4b4d564b; 402 | e->ecx = 0x564b4d56; 403 | e->edx = 0x4d; 404 | break; 405 | 406 | case KVM_CPUID_FEATURES: 407 | // Unlike QEMU 1.0, QEMU 3.0 required this CPUID flag to be set 408 | // in order to use get/set clock. Not implementing this feature 409 | // may cause guests to hang on resume because the TSC is not 410 | // restored in that case. 411 | e->eax = 1 << KVM_FEATURE_CLOCKSOURCE; 412 | break; 413 | default: 414 | cpu_x86_cpuid(&m_cpuid, s_cpuid_entries[i][0], s_cpuid_entries[i][1], &e->eax, &e->ebx, &e->ecx, 415 | &e->edx); 416 | break; 417 | } 418 | 419 | e->flags = 0; 420 | e->index = 0; 421 | if (s_cpuid_entries[i][1] != -1) { 422 | e->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 423 | e->index = s_cpuid_entries[i][1]; 424 | } 425 | e->function = s_cpuid_entries[i][0]; 426 | 427 | #ifdef SE_KVM_DEBUG_CPUID 428 | print_cpuid2(e); 429 | #endif 430 | } 431 | 432 | return 0; 433 | } 434 | 435 | void S2EKVM::sendCpuExitSignal() { 436 | assert(m_vm); 437 | m_vm->sendCpuExitSignal(); 438 | } 439 | 440 | void *S2EKVM::timerCb(void *param) { 441 | auto obj = reinterpret_cast(param); 442 | 443 | while (!obj->m_exiting) { 444 | usleep(100 * 1000); 445 | 446 | // Send a signal to exit CPU loop only when no slow KLEE code 447 | // is running. Otherwise, there are too many exits and little 448 | // progress in the guest. 449 | if (timers_state.cpu_clock_scale_factor == 1) { 450 | // Required for shutdown, otherwise kvm clients may get stuck 451 | // Also required to give a chance timers to run 452 | 453 | obj->sendCpuExitSignal(); 454 | } 455 | } 456 | 457 | obj->m_timerExited = true; 458 | return nullptr; 459 | } 460 | 461 | int S2EKVM::initTimerThread(void) { 462 | int ret; 463 | pthread_attr_t attr; 464 | sigset_t signals; 465 | 466 | ret = pthread_attr_init(&attr); 467 | if (ret < 0) { 468 | fprintf(stderr, "Could not init thread attributes\n"); 469 | goto err1; 470 | } 471 | 472 | ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); 473 | if (ret < 0) { 474 | fprintf(stderr, "Could not set detached state for thread\n"); 475 | goto err1; 476 | } 477 | 478 | ret = pthread_create(&m_timerThread, &attr, timerCb, this); 479 | if (ret < 0) { 480 | fprintf(stderr, "could not create timer thread\n"); 481 | goto err1; 482 | } 483 | 484 | sigfillset(&signals); 485 | if (pthread_sigmask(SIG_BLOCK, &signals, NULL) < 0) { 486 | fprintf(stderr, "could not block signals on the timer thread\n"); 487 | goto err1; 488 | } 489 | 490 | pthread_attr_destroy(&attr); 491 | 492 | err1: 493 | return ret; 494 | } 495 | 496 | int S2EKVM::sys_ioctl(int fd, int request, uint64_t arg1) { 497 | int ret = -1; 498 | 499 | switch ((uint32_t) request) { 500 | case KVM_GET_API_VERSION: 501 | return getApiVersion(); 502 | 503 | case KVM_CHECK_EXTENSION: 504 | ret = checkExtension(arg1); 505 | if (ret < 0) { 506 | errno = 1; 507 | } 508 | break; 509 | 510 | case KVM_CREATE_VM: { 511 | ret = createVM(); 512 | if (ret < 0) { 513 | printf("Could not create vm fd (errno=%d %s)\n", errno, strerror(errno)); 514 | exit(-1); 515 | } 516 | } break; 517 | 518 | case KVM_GET_VCPU_MMAP_SIZE: { 519 | ret = getVCPUMemoryMapSize(); 520 | } break; 521 | 522 | case KVM_GET_MSR_INDEX_LIST: { 523 | ret = getMSRIndexList((kvm_msr_list *) arg1); 524 | } break; 525 | 526 | case KVM_GET_SUPPORTED_CPUID: { 527 | ret = getSupportedCPUID((kvm_cpuid2 *) arg1); 528 | } break; 529 | 530 | default: { 531 | fprintf(stderr, "libs2e: unknown KVM IOCTL %x\n", request); 532 | exit(-1); 533 | } 534 | } 535 | 536 | return ret; 537 | } 538 | } 539 | } 540 | -------------------------------------------------------------------------------- /src/s2e-kvm.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_H 9 | 10 | #define S2E_KVM_H 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "FileDescriptorManager.h" 17 | #include "syscalls.h" 18 | 19 | namespace s2e { 20 | namespace kvm { 21 | 22 | struct stats_t { 23 | uint64_t mmio_reads; 24 | uint64_t mmio_writes; 25 | uint64_t io_reads; 26 | uint64_t io_writes; 27 | 28 | uint64_t kvm_runs; 29 | uint64_t cpu_exit; 30 | }; 31 | 32 | extern struct stats_t g_stats; 33 | 34 | class VM; 35 | 36 | class S2EKVM : public IFile { 37 | private: 38 | static const char *s_cpuModel; 39 | static const bool s_is64; 40 | 41 | pthread_t m_timerThread; 42 | bool m_exiting = false; 43 | volatile bool m_timerExited = false; 44 | cpuid_t m_cpuid; 45 | 46 | std::shared_ptr m_vm; 47 | 48 | #ifdef CONFIG_SYMBEX 49 | static std::string getBitcodeLibrary(const std::string &dir); 50 | #endif 51 | 52 | static void *timerCb(void *param); 53 | void init(void); 54 | void initLogLevel(void); 55 | 56 | S2EKVM() = default; 57 | 58 | static void cleanup(); 59 | 60 | void sendCpuExitSignal(); 61 | 62 | int getApiVersion(void); 63 | int createVM(); 64 | int getMSRIndexList(kvm_msr_list *list); 65 | int getSupportedCPUID(kvm_cpuid2 *cpuid); 66 | 67 | public: 68 | virtual ~S2EKVM() { 69 | } 70 | 71 | static IFilePtr create(); 72 | static int getVCPUMemoryMapSize(void); 73 | 74 | virtual int sys_ioctl(int fd, int request, uint64_t arg1); 75 | 76 | int checkExtension(int capability); 77 | int initTimerThread(void); 78 | 79 | bool exiting() const { 80 | return m_exiting; 81 | } 82 | 83 | void setExiting() { 84 | m_exiting = true; 85 | } 86 | 87 | const cpuid_t &getCpuid() const { 88 | return m_cpuid; 89 | } 90 | }; 91 | } 92 | } 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /src/s2e-libcpu-interface.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | // TODO: fix license 9 | 10 | #include 11 | 12 | #ifdef CONFIG_SYMBEX 13 | #include 14 | #include 15 | 16 | extern "C" { 17 | // XXX: move these declarations to s2e 18 | uint8_t __ldb_mmu_trace(uint8_t *host_addr, target_ulong vaddr); 19 | uint16_t __ldw_mmu_trace(uint16_t *host_addr, target_ulong vaddr); 20 | uint32_t __ldl_mmu_trace(uint32_t *host_addr, target_ulong vaddr); 21 | uint64_t __ldq_mmu_trace(uint64_t *host_addr, target_ulong vaddr); 22 | 23 | void __stb_mmu_trace(uint8_t *host_addr, target_ulong vaddr); 24 | void __stw_mmu_trace(uint16_t *host_addr, target_ulong vaddr); 25 | void __stl_mmu_trace(uint32_t *host_addr, target_ulong vaddr); 26 | void __stq_mmu_trace(uint64_t *host_addr, target_ulong vaddr); 27 | 28 | extern se_do_interrupt_all_t g_s2e_do_interrupt_all; 29 | } 30 | #endif 31 | 32 | void init_s2e_libcpu_interface(struct se_libcpu_interface_t *sqi) { 33 | #ifdef CONFIG_SYMBEX 34 | sqi->size = sizeof(*sqi); 35 | 36 | sqi->mode.fast_concrete_invocation = &g_s2e_fast_concrete_invocation; 37 | sqi->mode.fork_on_symbolic_address = &g_s2e_fork_on_symbolic_address; 38 | sqi->mode.running_concrete = &g_s2e_running_concrete; 39 | sqi->mode.running_exception_emulation_code = &g_s2e_running_exception_emulation_code; 40 | sqi->mode.single_path_mode = &g_s2e_single_path_mode; 41 | sqi->mode.allow_custom_instructions = &g_s2e_allow_custom_instructions; 42 | sqi->mode.concretize_io_writes = &g_s2e_concretize_io_writes; 43 | sqi->mode.concretize_io_addresses = &g_s2e_concretize_io_addresses; 44 | 45 | sqi->exec.helper_register_symbol = helper_register_symbol; 46 | sqi->exec.cleanup_tb_exec = s2e_libcpu_cleanup_tb_exec; 47 | sqi->exec.finalize_tb_exec = s2e_libcpu_finalize_tb_exec; 48 | sqi->exec.is_yielded = s2e_is_yielded; 49 | sqi->exec.is_runnable = s2e_is_runnable; 50 | sqi->exec.is_running_concrete = s2e_is_running_concrete; 51 | sqi->exec.reset_state_switch_timer = s2e_reset_state_switch_timer; 52 | sqi->exec.switch_to_symbolic = s2e_switch_to_symbolic; 53 | sqi->exec.tb_exec = se_libcpu_tb_exec; 54 | sqi->exec.do_interrupt_all = g_s2e_do_interrupt_all; 55 | 56 | sqi->tb.tb_alloc = se_tb_alloc; 57 | sqi->tb.flush_tb_cache = s2e_flush_tb_cache; 58 | sqi->tb.set_tb_function = s2e_set_tb_function; 59 | sqi->tb.is_tb_instrumented = s2e_is_tb_instrumented; 60 | sqi->tb.increment_tb_stats = s2e_increment_tb_stats; 61 | 62 | sqi->tlb.flush_tlb_cache = s2e_flush_tlb_cache; 63 | sqi->tlb.flush_tlb_cache_page = se_flush_tlb_cache_page; 64 | sqi->tlb.update_tlb_entry = s2e_update_tlb_entry; 65 | 66 | sqi->regs.read_concrete = s2e_read_register_concrete; 67 | sqi->regs.write_concrete = s2e_write_register_concrete; 68 | sqi->regs.set_cc_op_eflags = s2e_set_cc_op_eflags; 69 | 70 | sqi->mem.read_dirty_mask = se_read_dirty_mask; 71 | sqi->mem.write_dirty_mask = se_write_dirty_mask; 72 | sqi->mem.dma_read = s2e_dma_read; 73 | sqi->mem.dma_write = s2e_dma_write; 74 | sqi->mem.read_ram_concrete = s2e_read_ram_concrete; 75 | sqi->mem.write_ram_concrete = s2e_write_ram_concrete; 76 | sqi->mem.read_ram_concrete_check = s2e_read_ram_concrete_check; 77 | sqi->mem.read_mem_io_vaddr = s2e_read_mem_io_vaddr; 78 | sqi->mem.is_port_symbolic = s2e_is_port_symbolic; 79 | sqi->mem.is_mmio_symbolic = s2e_is_mmio_symbolic; 80 | sqi->mem.is_vmem_symbolic = se_is_vmem_symbolic; 81 | sqi->mem.get_host_address = se_get_host_address; 82 | 83 | sqi->mem.__ldb_mmu_trace = __ldb_mmu_trace; 84 | sqi->mem.__ldw_mmu_trace = __ldw_mmu_trace; 85 | sqi->mem.__ldl_mmu_trace = __ldl_mmu_trace; 86 | sqi->mem.__ldq_mmu_trace = __ldq_mmu_trace; 87 | sqi->mem.__stb_mmu_trace = __stb_mmu_trace; 88 | sqi->mem.__stw_mmu_trace = __stw_mmu_trace; 89 | sqi->mem.__stl_mmu_trace = __stl_mmu_trace; 90 | sqi->mem.__stq_mmu_trace = __stq_mmu_trace; 91 | 92 | sqi->expr.mgr = s2e_expr_mgr; 93 | sqi->expr.clear = s2e_expr_clear; 94 | sqi->expr.mgr_clear = s2e_expr_mgr_clear; 95 | sqi->expr.andc = s2e_expr_and; 96 | sqi->expr.to_constant = s2e_expr_to_constant; 97 | sqi->expr.set = s2e_expr_set; 98 | sqi->expr.write_cpu = s2e_expr_write_cpu; 99 | sqi->expr.read_cpu = s2e_expr_read_cpu; 100 | sqi->expr.read_mem_l = s2e_expr_read_mem_l; 101 | sqi->expr.read_mem_q = s2e_expr_read_mem_q; 102 | 103 | sqi->events.before_memory_access_signals_count = g_s2e_before_memory_access_signals_count; 104 | sqi->events.after_memory_access_signals_count = g_s2e_after_memory_access_signals_count; 105 | sqi->events.on_translate_soft_interrupt_signals_count = g_s2e_on_translate_soft_interrupt_signals_count; 106 | sqi->events.on_translate_block_start_signals_count = g_s2e_on_translate_block_start_signals_count; 107 | sqi->events.on_translate_block_end_signals_count = g_s2e_on_translate_block_end_signals_count; 108 | sqi->events.on_translate_block_complete_signals_count = g_s2e_on_translate_block_complete_signals_count; 109 | sqi->events.on_translate_instruction_start_signals_count = g_s2e_on_translate_instruction_start_signals_count; 110 | sqi->events.on_translate_special_instruction_end_signals_count = 111 | g_s2e_on_translate_special_instruction_end_signals_count; 112 | sqi->events.on_translate_jump_start_signals_count = g_s2e_on_translate_jump_start_signals_count; 113 | sqi->events.on_translate_lea_rip_relative_signals_count = g_s2e_on_translate_lea_rip_relative_signals_count; 114 | sqi->events.on_translate_instruction_end_signals_count = g_s2e_on_translate_instruction_end_signals_count; 115 | sqi->events.on_translate_register_access_signals_count = g_s2e_on_translate_register_access_signals_count; 116 | sqi->events.on_exception_signals_count = g_s2e_on_exception_signals_count; 117 | sqi->events.on_page_fault_signals_count = g_s2e_on_page_fault_signals_count; 118 | sqi->events.on_tlb_miss_signals_count = g_s2e_on_tlb_miss_signals_count; 119 | sqi->events.on_port_access_signals_count = g_s2e_on_port_access_signals_count; 120 | sqi->events.on_privilege_change_signals_count = g_s2e_on_privilege_change_signals_count; 121 | sqi->events.on_page_directory_change_signals_count = g_s2e_on_page_directory_change_signals_count; 122 | sqi->events.on_call_return_signals_count = g_s2e_on_call_return_signals_count; 123 | 124 | sqi->events.on_privilege_change = s2e_on_privilege_change; 125 | sqi->events.on_page_directory_change = s2e_on_page_directory_change; 126 | sqi->events.on_page_fault = s2e_on_page_fault; 127 | sqi->events.on_tlb_miss = s2e_on_tlb_miss; 128 | sqi->events.after_memory_access = s2e_after_memory_access; 129 | sqi->events.trace_port_access = s2e_trace_port_access; 130 | sqi->events.tcg_execution_handler = s2e_tcg_execution_handler; 131 | sqi->events.tcg_custom_instruction_handler = s2e_tcg_custom_instruction_handler; 132 | sqi->events.tcg_emit_custom_instruction = s2e_tcg_emit_custom_instruction; 133 | 134 | sqi->events.on_translate_soft_interrupt_start = s2e_on_translate_soft_interrupt_start; 135 | sqi->events.on_translate_block_start = s2e_on_translate_block_start; 136 | sqi->events.on_translate_block_end = s2e_on_translate_block_end; 137 | sqi->events.on_translate_block_complete = s2e_on_translate_block_complete; 138 | sqi->events.on_translate_instruction_start = s2e_on_translate_instruction_start; 139 | sqi->events.on_translate_special_instruction_end = s2e_on_translate_special_instruction_end; 140 | sqi->events.on_translate_instruction_end = s2e_on_translate_instruction_end; 141 | sqi->events.on_translate_jump_start = s2e_on_translate_jump_start; 142 | sqi->events.on_translate_indirect_cti_start = s2e_on_translate_indirect_cti_start; 143 | sqi->events.on_translate_lea_rip_relative = s2e_on_translate_lea_rip_relative; 144 | sqi->events.on_translate_register_access = s2e_on_translate_register_access; 145 | sqi->events.on_call_return_translate = s2e_on_call_return_translate; 146 | 147 | sqi->log.debug = s2e_debug_print; 148 | #endif 149 | } 150 | -------------------------------------------------------------------------------- /src/syscalls.h: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2019, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #ifndef S2E_KVM_SYSCALLS_H 9 | #define S2E_KVM_SYSCALLS_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | typedef int (*open_t)(const char *pathname, int flags, mode_t mode); 17 | typedef int (*close_t)(int fd); 18 | typedef int (*ioctl_t)(int d, int request, ...); 19 | typedef ssize_t (*write_t)(int fd, const void *buf, size_t count); 20 | typedef int (*dup_t)(int fd); 21 | 22 | typedef int (*poll_t)(struct pollfd *fds, nfds_t nfds, int timeout); 23 | typedef int (*select_t)(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); 24 | 25 | typedef void (*exit_t)(int ret) __attribute__((__noreturn__)); 26 | 27 | typedef void *(*mmap_t)(void *addr, size_t len, int prot, int flags, int fd, off_t offset); 28 | typedef void *(*mmap64_t)(void *addr, size_t len, int prot, int flags, int fd, off64_t offset); 29 | typedef int (*madvise_t)(void *addr, size_t len, int advice); 30 | 31 | typedef int (*printf_t)(const char *fmt, ...); 32 | typedef int (*fprintf_t)(FILE *fp, const char *fmt, ...); 33 | 34 | namespace s2e { 35 | 36 | class SyscallEvents { 37 | public: 38 | // Signal emitted when exit() is invoked 39 | sigc::signal onExit; 40 | 41 | // Signal emitted when select is called 42 | sigc::signal onSelect; 43 | }; 44 | 45 | extern SyscallEvents g_syscalls; 46 | } 47 | 48 | extern "C" { 49 | extern exit_t g_original_exit; 50 | extern ioctl_t g_original_ioctl; 51 | extern open_t g_original_open; 52 | extern mmap_t g_original_mmap; 53 | extern mmap_t g_original_mmap64; 54 | } 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/test.cpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Copyright (C) 2015-2017, Cyberhaven 3 | /// All rights reserved. 4 | /// 5 | /// Licensed under the Cyberhaven Research License Agreement. 6 | /// 7 | 8 | #include 9 | 10 | #include 11 | 12 | #include "coroutine.h" 13 | 14 | static Coroutine *s_coroutine; 15 | static uint64_t s_max = 100 * 1000 * 1000; 16 | 17 | static void fcn(void *p) { 18 | while (1) { 19 | coroutine_yield(); 20 | } 21 | } 22 | 23 | // This tests the overhead of a coroutine context switch 24 | static void test_coroutine() { 25 | s_coroutine = coroutine_create(fcn, 1 << 20); 26 | 27 | for (uint64_t i = 0; i < s_max; ++i) { 28 | coroutine_enter(s_coroutine, NULL); 29 | } 30 | } 31 | 32 | int main() { 33 | // cpu_gen_init(); 34 | // tcg_prologue_init(&tcg_ctx); 35 | printf("Starting tests...\n"); 36 | test_coroutine(); 37 | 38 | return 0; 39 | } 40 | --------------------------------------------------------------------------------