├── .gitignore ├── .gitmodules ├── LICENSES ├── APACHE_LICENSE └── GPL_LICENSE ├── README.md ├── linux-donard.pdf ├── lsmod.list ├── packages.list ├── papers └── 2014_07_02_FlashMemorySummit_Donard.pdf └── perform ├── mtr_sys ├── perform └── perform.sh /.gitignore: -------------------------------------------------------------------------------- 1 | perform/*.log 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libargconfig"] 2 | path = libargconfig 3 | url = https://github.com/sbates130272/libargconfig.git 4 | [submodule "nvme_donard"] 5 | path = nvme_donard 6 | url = https://github.com/sbates130272/nvme_donard.git 7 | [submodule "libdonard"] 8 | path = libdonard 9 | url = https://github.com/sbates130272/libdonard.git 10 | [submodule "donard_rdma"] 11 | path = donard_rdma 12 | url = https://github.com/sbates130272/donard_rdma.git 13 | [submodule "mmap_dd"] 14 | path = mmap_dd 15 | url = https://github.com/sbates130272/mmap_dd.git 16 | [submodule "mmap_test"] 17 | path = mmap_test 18 | url = https://github.com/sbates130272/mmap_test.git 19 | [submodule "nvram_bench"] 20 | path = nvram_bench 21 | url = https://github.com/sbates130272/nvram_bench.git 22 | [submodule "donard_tools"] 23 | path = donard_tools 24 | url = https://github.com/sbates130272/donard_tools.git 25 | [submodule "io_peer_mem"] 26 | path = io_peer_mem 27 | url = https://github.com/sbates130272/io_peer_mem 28 | [submodule "perftest"] 29 | path = perftest 30 | url = https://github.com/lsgunth/perftest.git 31 | [submodule "likwid"] 32 | path = likwid 33 | url = https://github.com/sbates130272/likwid.git 34 | [submodule "mbw"] 35 | path = mbw 36 | url = https://github.com/sbates130272/mbw.git 37 | -------------------------------------------------------------------------------- /LICENSES/APACHE_LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSES/GPL_LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | 341 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Donard 2 | 3 | ## Summary 4 | 5 | Donard: A PCIe Peer-2-Peer Library that builds on top of NVM Express 6 | 7 | ## Code Structure 8 | 9 | The Donard repo is really just a container for a multiple of smaller 10 | git repos. See of them in turn for their README and licensing 11 | information. 12 | 13 | ## Installation 14 | 15 | 1. Start from your Linux distro of choice on a bare-metal machine. You 16 | could try running this inside a VM but since we need to access 17 | bare-metal I am not sure why you would. We used Debian Wheezy as a 18 | start point but we have made many mods from there. 19 | 20 | 2. Install the packages required to install a updated Linux kernel. At 21 | a minimum you are going to need the following 22 | packages. kernel-package, libncurses5-dev, fakeroot, bzip2 and bc. 23 | 24 | 3. Clone the Donard version of the linux kernel from the relevant 25 | GitHub repo. i.e. git clone 26 | https://github.com/sbates130272/linux-donard.git. For a view of how 27 | this kernel is constructed see linux-donard.pdf. 28 | 29 | 4. Build and install this version of the linux kernel. 30 | cd linux-donard 31 | make-kpkg clean 32 | fakeroot make-kpkg --initrd --append-to-version=-docker-donard kernel_image kernel_headers 33 | cd .. 34 | dpkg -i linux-image-3.16.3-docker-donard+_3.16.3-docker-donard+-10.00.Custom_amd64.deb 35 | dpkg -i linux-headers-3.16.3-docker-donard+_3.16.3-docker-donard+-10.00.Custom_amd64.deb 36 | 37 | 5. Now pull the rest of the donard project code: 38 | git clone https://github.com/sbates130272/donard.git 39 | 40 | 6. Install the Nvidia driver. We used the instructions at 41 | https://wiki.debian.org/NvidiaGraphicsDrivers. Note that several 42 | people have has issues with this step and tieing the Nvidia code 43 | into the nvme_donard module. So we outline a more complete 44 | procedure in the next section. 45 | 46 | ## Installation - Nvidia code and nvme_donard (thanks Jack ;-)) 47 | 48 | 1. Build ubuntu 14.04.3 server on the system 49 | 50 | 2. apt-get install git kernel-package, libncurses5-dev fakerootbzip2 bc 51 | 52 | 3. git clone https://github.com/sbates130272/linux-donard.git. 53 | 1. apt-get update followed by apt–get upgrade 54 | 2. cd to linux-donard directory 55 | 3. make-kpkg clean 56 | 4. fakeroot make-kpkg —initrd --append-to-version=-docker-donard 57 | kernel_image kernel_headers (Accept defaults from pmem and DAX). 58 | 5. cd .. 59 | 6. (As root) dpkg -I *donard*.deb 60 | 7. Reboot 61 | 8. uname -r to verify kernel loaded 62 | 63 | 4. Check soft links 64 | 1. /lib/modules/3.19.1-docker-donard+/build and 65 | /lib/modules/3.19.1-docker-donard+/source should point to 66 | /usr/src/linux-headers-3.19.1-docker-donard+ 67 | 68 | 5. Load latest cuda package from nvidia 69 | 70 | 6. As root 71 | 1. Copy .deb file from https://developer.nvidia.com/cuda-downloads 72 | 2. dpkg –I cuda*.deb 73 | 3. apt-get update 74 | 4. Apt-get install cuda 75 | 5. Export PATH and library variables 76 | 6. Driver will be installed in /usr/src/nvidia-352-352.39 for cuda 77 | 7.5 78 | 7. cd to nvidia src directory 79 | 80 | 6. As root, not sudo 81 | 1. kernelver=$(uname –r) 82 | 2. kernel_source_dir=/lib/modules/$kernelver/build 83 | 3. make module KERNDIR=/lib/modules/$kernelver \ 84 | IGNORE_XEN_PRESENCE=1 IGNORE_CC_MISMATCH=1 \ 85 | SYSSRC=$kernel_source_dir LD=/usr/bin/ld.bfd 86 | 87 | 7. Verify Module.symvers is built in nvidia src directory 88 | 89 | 8. Build Donard components 90 | 1. cd to home 91 | 2. git clone —recursive https://github.com/sbates130272/donard.git 92 | 3. cd to donard/nvme_donard directory 93 | 4. Edit Makefile to point to correct nvidia src directory 94 | 5. Run make install as root 95 | 6. lsmod |grep nvme_donard 96 | 97 | 9. If module not visible then 98 | 1. modprobe nvme_donard 99 | 100 | 10. Build and run tests 101 | 2. May have to make modules followed by make install 102 | 3. cd donard/libargconfig 103 | 4. Execute ./waf , may have to ./waf install 104 | 5. cd to donard/libdonard 105 | 6. apt-get install libfftw3-dev libmagickwand-dev 106 | 7. ./waf 107 | 8. modprobe donard_nv_pinbuf 108 | 9. mkdir /temp 109 | 10. mkfs.ext4 /dev/nvme0n1 110 | 11. mount /dev/nvme0n1 /temp 111 | 12. dd if=/dev/zero of=/temp/test1.dat bs=1K count=100K 112 | 13. cd donard/libdonard/build/speed 113 | 14. ./nvme2gpu_read -b 128M -D /temp/test1.dat 114 | 15. ./nvme2gpu_read -b 128M /temp/test1.dat 115 | 116 | ### Example output 117 | ~/donard/libdonard/build/speed# ./nvme2gpu_read -b 128M /temp/test1.dat 118 | Total CPU Time: 0.0s user, 0.9s system 119 | Page Faults: 9 120 | Copied 104.86MB in 348.6 ms 300.78MB/s 121 | 122 | ~/donard/libdonard/build/speed# ./nvme2gpu_read -b 128M -D /temp/test1.dat 123 | Total CPU Time: 0.1s user, 0.9s system 124 | Page Faults: 1607 125 | 126 | Copied 104.86MB in 80.9 ms 1.30GB/s 127 | 128 | ## Quick Start - NVMe<->GPU 129 | 130 | 1. cd /nvme_donard 131 | 2. make install (this loads the nvme_donard kernel module and blacklists the defult nvme one). 132 | 3. lsmod | grep nvme should return nvme_donard (and not nvme). 133 | 4. cd /libdonard 134 | 5. ./waf 135 | 6. cd /libdonard/build/speed 136 | 7. dd if=/dev/zero of=/\/test1.dat bs=1K count=100K (to create a test file, for now keep it at 128MB or less) 137 | 8. ./nvme2gpu_read -b 128M -D /\/test1.dat 138 | 9. ./nvme2gpu_read -b 128M =/\/test1.dat 139 | 140 | 141 | If all of this runs you should see the –D mode has WAY more page 142 | faults the without the –D (the non –D is p2p, with –D the transfer 143 | goes via DRAM). Depending on your system the non –D option may be 144 | faster too. This is nvme->gpu transfer. There is a similar executable 145 | in the same fodler to go the other way. You can use likwid-perfctr to 146 | get better memory and CPU measurements too. 147 | 148 | ## Quick Start - perform.sh 149 | 150 | A simple bash script resides in the perform folder. When run it checks 151 | for certain files on the path and then executes a client/server 152 | perftest test whilst tracking DRAM bandwidth on the server. 153 | 154 | ## Quick Start - NVDIMMs 155 | 156 | We have some code in here to test NVDIMMs and the IOMEM exposed in the 157 | PMC Flashtec NVRAM drive. You can also use it to test any memory 158 | region really. Anyway here are some steps for the NVDIMM... 159 | 160 | 1. git clone --recursive https://github.com/sbates130272/donard.git pulls the code. 161 | 2. cd donard/libargconfig 162 | 3. sudo ./waf install 163 | 4. cd ../.. 164 | 5. cd donard/nvram_bench 165 | 6. ./waf (builds the code, should be no errors). 166 | 7. Since our kernel has the PMEM+DAX patches we can setup the NVDIMM but adding the following line to /etc/modules: 167 | pmem pmem_start_gb=8 pmem_size_gb=8 168 | 8. The NVDIMM appears as a /dev/pmem\ and we can mount it using the following in /etc/fstab: 169 | /dev/pmem\ /mnt/nvdimm ext4 dax,noatime 0 0 170 | 9. You can now run the nvdimm.sh script on the nvdimm. 171 | 172 | ## Quick Start - RDMA 173 | 174 | For these tests you will need two machines (a server and a client) and 175 | the donard kernel and OFED drivers on both machines. Each machine will 176 | also need a OFED compliant NIC installed. We've done some testing on 177 | both the Chelsio T540-CR iWARP card and the Mellanox MT27600 IB card. 178 | 179 | ### Server 180 | 181 | 1. git clone --recursive https://github.com/sbates130272/donard.git pulls the code. 182 | 2. cd donard/donard_rdma 183 | 3. ./waf 184 | 4. Run the server (three modes available): 185 | i. ./build/donard_rdma_server (runs in main memory) 186 | ii. ./build/donard_rdma_server -g (runs in a bar on GPU) 187 | iii. ./build/donard_rdma_server -m (runs in a mmap of the 188 | specified file. If that file is on a NVRAM card it will use that and 189 | do p2p). 190 | 4. Run the client (see below). When the client runs you should see 191 | somethng like the following on the server: 192 | 193 | Buffer Type: CPU 194 | Listening on port 11935 195 | 196 | Buffer Created: 0x7fa8040a6010 length 1024kB 197 | Accepting Client Connection: 172.16.0.2 198 | Testing Send/Recv 199 | Send Completed Successfully. 200 | Recv Completed Succesfully. 201 | Got Seed 220246839, length 32768, use_zeros 0 202 | Buffer Matches Random Seed. 203 | Client Disconnected. 204 | 205 | 206 | ### Client 207 | 208 | 1. git clone --recursive https://github.com/sbates130272/donard.git pulls the code. 209 | 2. cd donard/donard_rdma 210 | 3. ./waf 211 | 4. ./build/donard_rdma_client -a
-w (note address will be 212 | system specific and in our case is donard-rdma) 213 | 5. If things work as expected you should get something like: 214 | 215 | batesste@cgy1-flash:~/donard/donard_rdma$ donard_rdma_client -a 216 | donard-rdma 217 | Seed: 1422379394 218 | rdma_connect: Connection refused 219 | batesste@cgy1-flash:~/donard/donard_rdma$ donard_rdma_client -a 220 | donard-rdma -w 221 | Seed: 1422379414 222 | Remote Buffer: 0x7fa8040a6010 : length 1024KiB : bs = 32768B 223 | Testing Send/Recv 224 | Recv Completed Succesfully. 225 | Send Completed Successfully. 226 | 227 | Testing Writes 228 | 229 | Wrote: 8MiB 230 | Transfered: 8.42MB in 0.0s 281.21MB/s 231 | 232 | ## References 233 | 234 | A good place to get started is the [Flash Memory 235 | Summit](www.flashmemorysummit.com) 2014 paper (which can be found in 236 | the papers subfolder) that discusses Donard. Another reference is the 237 | article on [PMC-Sierra's 238 | blog](http://blog.pmcs.com/project-donard-peer-to-peer-communication-with-nvm-express-devices-part-1/) 239 | 240 | ## Licensing 241 | 242 | This code is licensed under Apache Version 2.0 and, where required, GPL Version 2.0 243 | -------------------------------------------------------------------------------- /linux-donard.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sbates130272/donard/fae61df8a5757f9df6b71bbdad7e6fd8a99f0bdf/linux-donard.pdf -------------------------------------------------------------------------------- /lsmod.list: -------------------------------------------------------------------------------- 1 | # 14th April 2015 2 | # 3 | # Output of the command 'sudo lsmod' on the donard server. Of course 4 | #not all these pacakges are needed to run the donard peer-2-peer code 5 | # but thislist is provided to aid with server setup. 6 | Module Size Used by 7 | io_peer_mem 16384 0 8 | nvidia_uvm 36864 0 9 | ip_tables 24576 0 10 | x_tables 20480 1 ip_tables 11 | bridge 73728 0 12 | stp 16384 1 bridge 13 | llc 16384 2 stp,bridge 14 | dm_thin_pool 49152 1 15 | dm_persistent_data 49152 1 dm_thin_pool 16 | dm_bio_prison 16384 1 dm_thin_pool 17 | dm_bufio 20480 1 dm_persistent_data 18 | libcrc32c 16384 1 dm_persistent_data 19 | binfmt_misc 16384 1 20 | nfsd 200704 2 21 | auth_rpcgss 40960 1 nfsd 22 | oid_registry 16384 1 auth_rpcgss 23 | nfs_acl 16384 1 nfsd 24 | nfs 151552 0 25 | lockd 57344 2 nfs,nfsd 26 | grace 16384 2 nfsd,lockd 27 | fscache 45056 1 nfs 28 | sunrpc 192512 6 nfs,nfsd,auth_rpcgss,lockd,nfs_acl 29 | ext2 61440 1 30 | nv_peer_mem 16384 0 31 | ib_ipoib 69632 0 32 | ib_umad 20480 0 33 | mlx5_ib 77824 0 34 | mtramonb 16384 1 35 | pmem 16384 1 36 | rdma_ucm 24576 0 37 | ib_uverbs 40960 1 rdma_ucm 38 | rdma_cm 32768 1 rdma_ucm 39 | ib_cm 32768 2 rdma_cm,ib_ipoib 40 | ib_sa 28672 4 rdma_cm,ib_cm,rdma_ucm,ib_ipoib 41 | ib_mad 36864 3 ib_cm,ib_sa,ib_umad 42 | donard_nv_pinbuf 16384 0 43 | iw_cxgb4 118784 0 44 | iw_cm 28672 2 iw_cxgb4,rdma_cm 45 | ib_core 69632 13 iw_cxgb4,rdma_cm,ib_cm,ib_sa,iw_cm,nv_peer_mem,mlx5_ib,ib_mad,ib_umad,ib_uverbs,rdma_ucm,io_peer_mem,ib_ipoib 46 | ib_addr 16384 3 rdma_cm,ib_core,rdma_ucm 47 | cxgb4 122880 1 iw_cxgb4 48 | msr 16384 0 49 | loop 24576 4 50 | nvidia 10629120 3 nv_peer_mem,donard_nv_pinbuf,nvidia_uvm 51 | snd_hda_intel 28672 0 52 | snd_hda_controller 24576 1 snd_hda_intel 53 | x86_pkg_temp_thermal 16384 0 54 | snd_hda_codec 90112 2 snd_hda_intel,snd_hda_controller 55 | coretemp 16384 0 56 | snd_hwdep 16384 1 snd_hda_codec 57 | kvm_intel 131072 0 58 | snd_pcm 73728 3 snd_hda_codec,snd_hda_intel,snd_hda_controller 59 | snd_timer 24576 1 snd_pcm 60 | kvm 339968 1 kvm_intel 61 | snd 57344 5 snd_hwdep,snd_timer,snd_pcm,snd_hda_codec,snd_hda_intel 62 | ghash_clmulni_intel 16384 0 63 | aesni_intel 167936 0 64 | aes_x86_64 20480 1 aesni_intel 65 | soundcore 16384 2 snd,snd_hda_codec 66 | nvme_donard 45056 2 67 | drm 237568 2 nvidia 68 | psmouse 86016 0 69 | ablk_helper 16384 1 aesni_intel 70 | cryptd 16384 3 ghash_clmulni_intel,aesni_intel,ablk_helper 71 | lrw 16384 1 aesni_intel 72 | ioatdma 40960 0 73 | pcspkr 16384 0 74 | serio_raw 16384 0 75 | iTCO_wdt 16384 0 76 | iTCO_vendor_support 16384 1 iTCO_wdt 77 | sb_edac 24576 0 78 | gf128mul 16384 1 lrw 79 | joydev 20480 0 80 | evdev 20480 3 81 | edac_core 40960 1 sb_edac 82 | glue_helper 16384 1 aesni_intel 83 | microcode 32768 0 84 | i2c_i801 20480 0 85 | shpchp 32768 0 86 | lpc_ich 24576 0 87 | mfd_core 16384 1 lpc_ich 88 | ipmi_si 40960 0 89 | ipmi_msghandler 36864 1 ipmi_si 90 | acpi_cpufreq 16384 0 91 | processor 28672 1 acpi_cpufreq 92 | thermal_sys 28672 2 processor,x86_pkg_temp_thermal 93 | tpm_tis 20480 0 94 | wmi 16384 0 95 | tpm 28672 1 tpm_tis 96 | button 16384 0 97 | sr_mod 24576 0 98 | cdrom 28672 1 sr_mod 99 | ext4 401408 7 100 | jbd2 77824 1 ext4 101 | mbcache 16384 2 ext2,ext4 102 | dm_mod 73728 14 dm_persistent_data,dm_bufio,dm_thin_pool 103 | ohci_hcd 32768 0 104 | uhci_hcd 28672 0 105 | xhci_hcd 94208 0 106 | sg 32768 0 107 | sd_mod 40960 5 108 | usb_storage 49152 0 109 | hid_generic 16384 0 110 | usbhid 40960 0 111 | hid 94208 2 hid_generic,usbhid 112 | crc32c_intel 24576 1 113 | ahci 32768 1 114 | igb 131072 0 115 | libahci 24576 1 ahci 116 | isci 94208 2 117 | libsas 61440 1 isci 118 | ehci_pci 16384 0 119 | i2c_algo_bit 16384 1 igb 120 | ehci_hcd 45056 1 ehci_pci 121 | i2c_core 36864 5 drm,igb,i2c_i801,i2c_algo_bit,nvidia 122 | libata 147456 3 ahci,libahci,libsas 123 | dca 16384 2 igb,ioatdma 124 | usbcore 147456 7 uhci_hcd,usb_storage,ohci_hcd,ehci_hcd,ehci_pci,usbhid,xhci_hcd 125 | ptp 20480 1 igb 126 | scsi_transport_sas 32768 2 isci,libsas 127 | usb_common 16384 1 usbcore 128 | pps_core 16384 1 ptp 129 | scsi_mod 176128 8 sg,isci,scsi_transport_sas,usb_storage,libata,libsas,sd_mod,sr_mod 130 | mlx5_core 61440 1 mlx5_ib 131 | -------------------------------------------------------------------------------- /papers/2014_07_02_FlashMemorySummit_Donard.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sbates130272/donard/fae61df8a5757f9df6b71bbdad7e6fd8a99f0bdf/papers/2014_07_02_FlashMemorySummit_Donard.pdf -------------------------------------------------------------------------------- /perform/mtr_sys: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | BUS=$(lspci -d 11f8:f117 | head -n 1 | awk '{ print $1 }') 5 | echo "/sys/bus/pci/devices/0000:${BUS}/resource4_wc" 6 | -------------------------------------------------------------------------------- /perform/perform: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | ######################################################################## 3 | ## 4 | ## Copyright 2015 PMC-Sierra, Inc. 5 | ## 6 | ## Licensed under the Apache License, Version 2.0 (the "License"); you 7 | ## may not use this file except in compliance with the License. You may 8 | ## obtain a copy of the License at 9 | ## http://www.apache.org/licenses/LICENSE-2.0 Unless required by 10 | ## applicable law or agreed to in writing, software distributed under the 11 | ## License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | ## CONDITIONS OF ANY KIND, either express or implied. See the License for 13 | ## the specific language governing permissions and limitations under the 14 | ## License. 15 | ## 16 | ######################################################################## 17 | 18 | ######################################################################## 19 | ## 20 | ## Author: Logan Gunthorpe 21 | ## 22 | ## Description: 23 | ## This script runs perftest tools with mbw in the background to 24 | ## measure the memory bandwidth effects of using P2P RDMA transfers. 25 | ## 26 | ######################################################################## 27 | 28 | from __future__ import print_function 29 | 30 | import os 31 | import sys 32 | import subprocess as sp 33 | import pty 34 | import threading 35 | import re 36 | import traceback 37 | import signal 38 | import errno 39 | import time 40 | import getpass 41 | 42 | from itertools import repeat, chain 43 | 44 | suffix_re = re.compile(r"^([0-9\.]+)([KMGTP]?)", re.IGNORECASE) 45 | suffix_values = [('' , 1), 46 | ('K', 1 << 10), 47 | ('M', 1 << 20), 48 | ('G', 1 << 30), 49 | ('T', 1 << 40), 50 | ('P', 1 << 50)] 51 | suffix_dict = dict(suffix_values) 52 | 53 | def parse_suffix(value): 54 | m = suffix_re.match(value) 55 | if not m: 56 | raise ValueError("Could not parse: '{}'".format(value)) 57 | 58 | value = float(m.group(1)) 59 | value *= suffix_dict[m.group(2).upper()] 60 | return value 61 | 62 | class Suffix(object): 63 | def __init__(self, value, unit="B"): 64 | self.unit = unit 65 | 66 | for s, v in suffix_values[::-1]: 67 | if value < v: 68 | continue 69 | if s: 70 | s+= "i" 71 | 72 | self.div = v 73 | self.value = float(value) / v 74 | self.suffix = s 75 | break 76 | else: 77 | self.value = float(value) 78 | self.suffix = "" 79 | 80 | def __format__(self, format): 81 | return ("{:" + format + "} {}{}").format(self.value, self.suffix, 82 | self.unit) 83 | 84 | class ProcRunnerException(Exception): 85 | def __str__(self): 86 | pr, msg = self.args 87 | 88 | ret = "".join(pr.output_lines) + "\n\n" 89 | ret += msg + "\n" 90 | ret += " " + " ".join(pr.exe + pr.args) 91 | return ret 92 | 93 | class ProcRunner(threading.Thread): 94 | kill_sig = signal.SIGINT 95 | 96 | def __init__(self, log_file=None, print_output=False, 97 | wait_for=False, *args, **kws): 98 | self.log_file = log_file 99 | self.print_output = print_output 100 | self.kill_me = not wait_for 101 | self.started = threading.Event() 102 | self.exception = None 103 | self.output_lines = [] 104 | self.args = [] 105 | super(ProcRunner, self).__init__(*args, **kws) 106 | 107 | def process_line(self, line): 108 | self.output_lines.append(line) 109 | 110 | if self.log_file: 111 | self.log_file.write(line) 112 | if self.print_output: 113 | sys.stdout.write(line) 114 | sys.stdout.flush() 115 | 116 | def setup(self): 117 | pass 118 | 119 | def finish(self): 120 | pass 121 | 122 | def run(self): 123 | try: 124 | master, slave = pty.openpty() 125 | self.slave = slave 126 | try: 127 | self.p = sp.Popen(self.exe + self.args, 128 | stdin=sp.PIPE, 129 | stdout=slave, 130 | stderr=slave, 131 | preexec_fn=os.setsid) 132 | except OSError as e: 133 | e.filename = self.exe[0] 134 | raise 135 | 136 | mf = os.fdopen(master, "U") 137 | 138 | self.setup() 139 | 140 | while True: 141 | self.started.set() 142 | line = mf.readline() 143 | 144 | try: 145 | self.process_line(line) 146 | except: 147 | print("Exception occured while processing line:") 148 | traceback.print_exc() 149 | 150 | except IOError: 151 | self.finish() 152 | except Exception as e: 153 | self.started.set() 154 | self.exception = sys.exc_info() 155 | 156 | 157 | def check_started(self): 158 | self.started.wait(2) 159 | if self.exception: 160 | self.join(5) 161 | traceback.print_exception(*self.exception) 162 | sys.exit(1) 163 | 164 | def start(self): 165 | super(ProcRunner, self).start() 166 | self.check_started() 167 | 168 | def wait(self): 169 | if self.started.wait(2): 170 | killed = False 171 | 172 | if self.p.poll() is None: 173 | #Delay a bit and see if the process finishes on its own 174 | time.sleep(0.1) 175 | 176 | if self.p.poll() is None and self.kill_me: 177 | killed = True 178 | os.killpg(self.p.pid, self.kill_sig) 179 | 180 | try: 181 | ret = self.p.wait() 182 | except KeyboardInterrupt: 183 | killed = True 184 | os.killpg(self.p.pid, self.kill_sig) 185 | ret = self.p.wait() 186 | 187 | os.close(self.slave) 188 | 189 | else: 190 | raise ProcRunnerException(self, "Timed out waiting for process " 191 | "to start") 192 | 193 | self.join(1) 194 | if ret and not killed: 195 | raise ProcRunnerException(self, "Error occured while running " 196 | "command") 197 | 198 | return ret 199 | 200 | def __enter__(self): 201 | try: 202 | self.start() 203 | except: 204 | try: 205 | self.wait() 206 | finally: 207 | raise 208 | 209 | return self 210 | 211 | def __exit__(self, type, value, traceback): 212 | self.wait() 213 | 214 | class LikwidPerfMixin(object): 215 | likwid_re = re.compile(r"^\|\s+(?P[^\[\|]+)" + 216 | r"( \[(?P.*?)\])?\s+" + 217 | r"\|\s+(?P[0-9\.e\+]+)\s+\|") 218 | 219 | def __init__(self, group="MEM", cpu="S0:0", *args, **kwargs): 220 | super(LikwidPerfMixin, self).__init__(*args, **kwargs) 221 | self.exe = ["likwid-perfctr", "-g", group, "-C", cpu] + self.exe 222 | self.likwid_stats = {} 223 | self.likwid_units = {} 224 | 225 | def process_line(self, line): 226 | super(LikwidPerfMixin, self).process_line(line) 227 | 228 | m = self.likwid_re.match(line) 229 | if not m: return 230 | self.likwid_stats[m.group("name")] = float(m.group("value")) 231 | self.likwid_units[m.group("name")] = m.group("units") 232 | 233 | class TimeProcessLineMixin(object): 234 | time_re = re.compile(r"^(?Preal|user|sys) (?P[0-9\.]+)$") 235 | 236 | def __init__(self, *args, **kwargs): 237 | super(TimeProcessLineMixin, self).__init__(*args, **kwargs) 238 | self.time_stats = {} 239 | 240 | def process_line(self, line): 241 | super(TimeProcessLineMixin, self).process_line(line) 242 | 243 | m = self.time_re.match(line) 244 | if not m: return 245 | 246 | name = m.group('name') 247 | value = float(m.group('value')) 248 | self.time_stats[name] = value 249 | 250 | if 'user' in self.time_stats and 'sys' in self.time_stats: 251 | self.time_stats['total'] = (self.time_stats['user'] + 252 | self.time_stats['sys']) 253 | 254 | 255 | def calculate_results(self, duration=None, *args, **kwargs): 256 | if duration is not None: 257 | self.time_stats["duration"] = duration 258 | self.time_stats["cpu_percent_duration"] = \ 259 | (100 * self.time_stats["total"] / duration) 260 | 261 | self.time_stats["cpu_percent_total"] = (self.time_stats["total"] / 262 | self.time_stats["real"]) * 100 263 | 264 | 265 | class TimeMixin(TimeProcessLineMixin): 266 | def __init__(self, group="MEM", cpu="S0:0", *args, **kwargs): 267 | super(TimeMixin, self).__init__(*args, **kwargs) 268 | self.exe = ["time", "-p"] + self.exe 269 | 270 | 271 | class MBWRunner(ProcRunner): 272 | exe = ["mbw"] 273 | mbw_re = re.compile(r"(?P[0-9]+)\s+" + 274 | r"Method: (?P[A-Z]+)\s+" + 275 | r"Elapsed: (?P[0-9\.]+)\s+" + 276 | r"MiB: (?P[0-9\.]+)\s+" + 277 | r"Copy: (?P[0-9\.]+) MiB/s+") 278 | 279 | 280 | def __init__(self, loops=10000, array_size_mb=512, tests=[0], 281 | *args, **kws): 282 | 283 | super(MBWRunner, self).__init__(*args, **kws) 284 | tests = [str(t) for t in tests] 285 | self.args = (["-n", str(loops)] + 286 | list(chain(*zip(repeat('-t'), tests))) + 287 | [str(array_size_mb)]) 288 | self.rates = [] 289 | self.volume = 0. 290 | 291 | def process_line(self, line): 292 | super(MBWRunner, self).process_line(line) 293 | 294 | m = self.mbw_re.match(line) 295 | if not m: return 296 | 297 | self.volume += float(m.group("mib")) * (1 << 20) 298 | rate = float(m.group("rate")) * (1 << 20) 299 | self.rates.append(rate) 300 | 301 | def clear(self): 302 | self.rates = [] 303 | 304 | def stats(self): 305 | r = self.rates[1:-1] 306 | if not r: return {} 307 | return {"max": max(r), 308 | "min": min(r), 309 | "avg": sum(r) / len(r), 310 | "count": len(r), 311 | "volume": (self.volume * 2)} # multiply by 2 for read and write 312 | 313 | class CPUPerfRunner(ProcRunner): 314 | exe = ["cpuperf.py"] 315 | 316 | def __init__(self, perftest, *args, **kws): 317 | super(CPUPerfRunner, self).__init__(*args, **kws) 318 | self.args = ["-s", "-C", perftest] 319 | 320 | 321 | class LikwidMBWRunner(LikwidPerfMixin, MBWRunner): 322 | pass 323 | 324 | class PerfTestOutputMixin(object): 325 | def __init__(self, mbw=None, *args, **kwargs): 326 | super(PerfTestOutputMixin, self).__init__(*args, **kwargs) 327 | self.mbw = mbw 328 | self.mbw_stats = None 329 | 330 | self.results_line = False 331 | 332 | self.units = {} 333 | self.values = {} 334 | 335 | def _process_field(self, f): 336 | f = f.strip().strip("# ") 337 | if "[" in f: 338 | name, unit = f.split("[") 339 | unit = unit.strip("[]") 340 | self.units[name] = unit 341 | return name 342 | else: 343 | return f.strip() 344 | 345 | def process_line(self, line): 346 | super(PerfTestOutputMixin, self).process_line(line) 347 | 348 | if self.results_line: 349 | self.results_line = False 350 | 351 | try: 352 | line = [float(x) for x in line.split()] 353 | self.values.update(zip(self.fields, line)) 354 | if self.mbw is not None: 355 | self.mbw_stats = self.mbw.stats() 356 | except ValueError: 357 | pass 358 | elif line.startswith(" #bytes"): 359 | self.fields = [self._process_field(f) for f in re.split(" {2,}", line)] 360 | self.results_line = True 361 | 362 | def finish(self): 363 | if self.mbw_stats is None and self.mbw is not None: 364 | self.mbw_stats = self.mbw.stats() 365 | 366 | super(PerfTestOutputMixin, self).finish() 367 | 368 | def bandwidth(self): 369 | bw = self.values.get("BW average", None) 370 | if bw is None: return None 371 | 372 | return bw * 1000**2 373 | 374 | def volume(self): 375 | bytes = self.values.get("bytes", None) 376 | its = self.values.get("iterations", None) 377 | 378 | if bytes is None or its is None: 379 | return None 380 | 381 | return bytes * its 382 | 383 | def latency(self): 384 | avg = self.values.get("t_typical", self.values.get("t_avg", None)) 385 | if avg is None: return None 386 | 387 | mn = self.values.get("t_min", 0) 388 | mx = self.values.get("t_max", 0) 389 | 390 | return {'avg': avg, 391 | 'min': mn, 392 | 'max': mx} 393 | 394 | class PerfTestServer(PerfTestOutputMixin, ProcRunner): 395 | def __init__(self, command="ib_write_bw", block_size=8388608, duration=5, 396 | mmap=None, test_args=[], *args, **kwargs): 397 | super(PerfTestServer, self).__init__(*args, **kwargs) 398 | self.exe = [command] 399 | self.args = ["-R", "-s", str(size), "-D", str(duration)] 400 | self.args += test_args 401 | 402 | if mmap: 403 | self.args += ["--mmap", mmap] 404 | 405 | self.ready = threading.Event() 406 | 407 | def process_line(self, line): 408 | super(PerfTestServer, self).process_line(line) 409 | 410 | if "Waiting for client to connect" in line: 411 | self.ready.set() 412 | 413 | if self.mbw is not None and line.startswith(" #bytes"): 414 | self.mbw.clear() 415 | 416 | def start(self): 417 | super(PerfTestServer, self).start() 418 | if not self.ready.wait(2): 419 | raise ProcRunnerException(self, "Timed out waiting for perftest " 420 | "server to start") 421 | 422 | class LikwidPerfTestServer(LikwidPerfMixin, TimeMixin, PerfTestServer): 423 | pass 424 | 425 | class PerfTestClient(PerfTestOutputMixin, TimeProcessLineMixin, ProcRunner): 426 | def __init__(self, host, command="ib_write_bw", block_size=8388608, 427 | duration=5, test_args=[], *args, **kwargs): 428 | super(PerfTestClient, self).__init__(*args, **kwargs) 429 | self.exe = ["ssh", host, "time", "-p", command] 430 | self.args = ["-R", "${SSH_CLIENT%% *}", "-s", str(size), 431 | "-D", str(duration)] 432 | self.args += test_args 433 | 434 | class FioRunner(ProcRunner): 435 | def __init__(self, test_args=[], **kwargs): 436 | self.exe = ["fio", "-"] 437 | 438 | self.extra_job_lines = "\n".join(test_args) 439 | super(FioRunner, self).__init__(**kwargs) 440 | 441 | def setup(self): 442 | opts = dict(self.__class__.__dict__) 443 | opts.update(self.__dict__) 444 | 445 | job = self.job.format(**opts) 446 | job = job + "\n" + self.extra_job_lines 447 | 448 | if self.log_file: 449 | print("\n", file=self.log_file) 450 | print("-"*40, file=self.log_file) 451 | print("Job File", file=self.log_file) 452 | print("-"*40, file=self.log_file) 453 | for l in job.split(): 454 | print(l.strip(), file=self.log_file) 455 | print("-"*40, file=self.log_file) 456 | print("", file=self.log_file) 457 | 458 | self.p.stdin.write(job) 459 | self.p.stdin.close() 460 | 461 | class FioServer(FioRunner): 462 | job = """[rdma-server] 463 | rw=read 464 | ioengine=rdma 465 | port=11692 466 | bs={block_size} 467 | size={size}""" 468 | 469 | mbw_stats = {} 470 | 471 | def __init__(self, command=None, block_size=1<<20, size=100<<40, 472 | duration=None, mbw=None, mmap=None, **kwargs): 473 | self.block_size = block_size 474 | if self.block_size < 4096: self.block_size = 4096 475 | self.size = size 476 | self.mbw = mbw 477 | self.mbw_started = False 478 | self.mmap = mmap 479 | if self.mmap is not None: 480 | self.job += "\nmem=mmapshared:{mmap}" 481 | 482 | self.ready = threading.Event() 483 | 484 | super(FioServer, self).__init__(**kwargs) 485 | 486 | def process_line(self, line): 487 | super(FioServer, self).process_line(line) 488 | 489 | if "waiting for connection" in line: 490 | self.ready.set() 491 | elif (self.mbw is not None and not self.mbw_started and 492 | line.startswith("Jobs")): 493 | self.mbw.clear() 494 | self.mbw_started = True 495 | elif self.mbw_started and line.startswith("rdma-server"): 496 | self.mbw_stats = self.mbw.stats() 497 | self.mbw_started = False 498 | 499 | def start(self): 500 | super(FioServer, self).start() 501 | if not self.ready.wait(6): 502 | raise ProcRunnerException(self, "Timed out waiting for fio " 503 | "server to start") 504 | 505 | def bandwidth(self): 506 | return None 507 | 508 | def latency(self): 509 | return None 510 | 511 | def volume(self): 512 | return None 513 | 514 | class LikwidFioServer(LikwidPerfMixin, TimeMixin, FioServer): 515 | pass 516 | 517 | 518 | class FioClient(TimeProcessLineMixin, FioRunner): 519 | job = """[rdma-client] 520 | rw=write 521 | ioengine=rdma 522 | runtime={duration} 523 | hostname=${{SERVER}} 524 | port=11692 525 | verb={verb} 526 | bs={block_size} 527 | size={size}""" 528 | 529 | def __init__(self, host, command="fio:write", duration=10, block_size=512, 530 | size=100<<40, **kwargs): 531 | self.block_size = block_size 532 | self.size = size 533 | if ":" in command: 534 | self.verb = command.split(":")[1] 535 | else: 536 | self.verb = "write" 537 | self.duration = duration 538 | self._volume = self._bandwidth = self._latency = None 539 | 540 | super(FioClient, self).__init__(**kwargs) 541 | 542 | self.args = ["SERVER=${SSH_CLIENT%% *}", "time", "-p"] + self.exe 543 | self.exe = ["ssh", host] 544 | 545 | lat_re = re.compile(r"^[^:]*: *min= *(?P[0-9.e]+), *" 546 | r"max= *(?P[0-9.e]+), *" 547 | r"avg= *(?P[0-9.e]+), *" 548 | r"stdev= *(?P[0-9.e]+)") 549 | 550 | def process_line(self, line): 551 | super(FioClient, self).process_line(line) 552 | 553 | line = line.strip() 554 | if line.startswith("write:"): 555 | line = line.split() 556 | self._volume = parse_suffix(line[1].split("=")[1]) 557 | self._bandwidth = parse_suffix(line[2].split("=")[1]) 558 | elif line.startswith("lat"): 559 | m = self.lat_re.match(line) 560 | if m: 561 | self._latency = {k: float(v) for k,v in m.groupdict().items()} 562 | 563 | def bandwidth(self): 564 | return self._bandwidth 565 | 566 | def latency(self): 567 | return self._latency 568 | 569 | def volume(self): 570 | return self._volume 571 | 572 | class DummyContext(object): 573 | def __enter__(self): 574 | return self 575 | 576 | def __exit__(self, type, value, traceback): 577 | pass 578 | 579 | def run_test(client="flash-rdma", size=8388608, duration=5, mmap=None, 580 | socket=0, log_file=None, client_log_file=None, verbose=False, 581 | perftest="ib_write_bw", cpu_perf=None, test_args=[], **kwargs): 582 | 583 | mbw = LikwidMBWRunner(cpu="S{}:1".format(socket), 584 | log_file=log_file) 585 | 586 | if cpu_perf: 587 | cpuperf = CPUPerfRunner(perftest=perftest, log_file=cpu_perf) 588 | else: 589 | cpuperf = DummyContext() 590 | 591 | 592 | Server = LikwidPerfTestServer 593 | Client = PerfTestClient 594 | 595 | if perftest.startswith("fio"): 596 | Server = LikwidFioServer 597 | Client = FioClient 598 | 599 | server = Server(print_output=verbose, 600 | cpu="S{}:0".format(socket), mbw=mbw, 601 | block_size=size, duration=duration, mmap=mmap, 602 | log_file=log_file, 603 | command=perftest, 604 | test_args=test_args) 605 | 606 | client = Client(host=client, block_size=size, duration=duration, 607 | command=perftest, wait_for=True, 608 | log_file=client_log_file, 609 | test_args=test_args) 610 | 611 | with cpuperf, mbw, server, client: 612 | pass 613 | 614 | server.calculate_results(duration=duration) 615 | client.calculate_results(duration=duration) 616 | 617 | return {"rdma_bw": server.bandwidth() or client.bandwidth(), 618 | "rdma_lat": server.latency() or client.latency(), 619 | "rdma_vol": server.volume() or client.volume(), 620 | "mbw_stats": server.mbw_stats, 621 | "likwid_stats": server.likwid_stats, 622 | "likwid_units": server.likwid_units, 623 | "server_time": server.time_stats, 624 | "client_time": client.time_stats} 625 | 626 | 627 | def print_results(results, indent=0): 628 | results["ind"] = " "*indent 629 | tmpl = "" 630 | 631 | results["rdma_bw"] = Suffix(results["rdma_bw"], unit="B/s") 632 | results["rdma_vol"] = Suffix(results["rdma_vol"], unit="B") 633 | 634 | if results["rdma_bw"]: 635 | tmpl += "{ind}RDMA Bandwidth {rdma_bw:>10.2f}\n" 636 | if results["rdma_lat"]: 637 | tmpl += "{ind}RDMA Latency {rdma_lat[avg]:>10.2f} us" 638 | if results["rdma_lat"]["max"]: 639 | tmpl += " (max: {rdma_lat[max]:.2f}, min: {rdma_lat[min]:.2f})\n" 640 | else: 641 | tmpl += "\n" 642 | 643 | tmpl += "{ind}RDMA Data Volume: {rdma_vol:>10.2f}\n" 644 | 645 | if results["server_time"]: 646 | tmpl += "{ind}Server CPU Time {server_time[total]:>10.2f} s\n" 647 | tmpl += "{ind}Server Duration {server_time[duration]:>10.2f} s" 648 | tmpl += " ({server_time[cpu_percent_duration]:5.1f}%)\n" 649 | tmpl += "{ind}Server Elapsed Time {server_time[real]:>10.2f} s" 650 | tmpl += " ({server_time[cpu_percent_total]:5.1f}%)\n" 651 | 652 | if results["client_time"]: 653 | tmpl += "{ind}Client CPU Time {client_time[total]:>10.2f} s\n" 654 | tmpl += "{ind}Client Duration {client_time[duration]:>10.2f} s" 655 | tmpl += " ({client_time[cpu_percent_duration]:5.1f}%)\n" 656 | tmpl += "{ind}Client Elapsed Time {client_time[real]:>10.2f} s" 657 | tmpl += " ({client_time[cpu_percent_total]:5.1f}%)\n" 658 | 659 | if results["mbw_stats"]: 660 | results["mbw_stats"]["avg"] = Suffix(results["mbw_stats"]["avg"], "B/s") 661 | results["mbw_stats"]["max"] /= results["mbw_stats"]["avg"].div 662 | results["mbw_stats"]["min"] /= results["mbw_stats"]["avg"].div 663 | results["mbw_stats"]["volume"] = Suffix(results["mbw_stats"]["volume"], "B") 664 | 665 | tmpl += ("{ind}MBW Background Bandwidth {mbw_stats[avg]:>10.2f} " 666 | " (max: {mbw_stats[max]:.2f}, min: {mbw_stats[min]:.2f})\n" + 667 | "{ind}MBW Data Volume {mbw_stats[volume]:>10.1f}\n") 668 | 669 | tmpl += ("{ind}LikWid Memory Data Volume {likwid_stats[Memory data volume]:>10.1f} " 670 | "{likwid_units[Memory data volume]}\n") 671 | tmpl += ("{ind}Total DDR Write BW {likwid_stats[Memory Write BW]:>10.1f} " 672 | "{likwid_units[Memory Write BW]}\n") 673 | tmpl += ("{ind}Total DDR Read BW {likwid_stats[Memory Read BW]:>10.1f} " 674 | "{likwid_units[Memory Read BW]}\n") 675 | 676 | 677 | print(tmpl.format(**results)) 678 | 679 | def check_mmap_dev(mmap): 680 | try: 681 | with open(mmap, "a"): 682 | pass 683 | except IOError as e: 684 | if e.errno != errno.EACCES: 685 | raise 686 | 687 | print("Permission denied accessing {}".format(mmap)) 688 | print() 689 | 690 | sudo_cmd = ["sudo", "chown", getpass.getuser(), mmap] 691 | print("Run this sudo command? (assuming you trust us)") 692 | print(" ".join(sudo_cmd)) 693 | res = raw_input("(Y/n) ") 694 | if res.strip() and res[0].strip().lower() == 'n': 695 | sys.exit(1) 696 | 697 | sp.check_call(sudo_cmd) 698 | print() 699 | 700 | with open(mmap, "a"): 701 | pass 702 | 703 | if __name__ == "__main__": 704 | import argparse 705 | 706 | p = argparse.ArgumentParser() 707 | p.add_argument("-c", "--client", default="flash-rdma", 708 | help="host to run the perftest client on (the server " 709 | "is run locally), default: %(default)s") 710 | p.add_argument("-C", "--cpu-perf", type=argparse.FileType('w'), 711 | help="run cpu perf in the background and save the log to " 712 | "file specified") 713 | p.add_argument("-D", "--duration", type=int, default=5, 714 | help="duration, in seconds to run the perftest for, " 715 | "default: %(default)s") 716 | p.add_argument("-s", "--size", type=parse_suffix, default=8388608, 717 | help="RDMA message size in bytes (set <2 for all sizes), " 718 | "default: %(default)s") 719 | p.add_argument("-L", "--log-file", type=argparse.FileType('w'), 720 | help="save command output to a log file") 721 | p.add_argument("--mmap", metavar="DEV", 722 | help="device to use as an RDMA target, default: %(default)s") 723 | p.add_argument("-p", "--perftest", default="ib_write_bw", 724 | help="which perftest binary to use, default: %(default)s") 725 | p.add_argument("-S", "--socket", type=int, default=0, 726 | help="cpu socket to pin the processes to (should have the same " 727 | "locality as the device specified in --mmap), " 728 | "default: %(default)s") 729 | p.add_argument("-v", "--verbose", action="count", 730 | help="print command output to stdout") 731 | p.add_argument("test_args", nargs=argparse.REMAINDER, 732 | help="extra arguments are passed to the pertfest run") 733 | options = p.parse_args() 734 | 735 | if "--" in options.test_args: 736 | options.test_args.remove("--") 737 | 738 | try: 739 | 740 | opts = options.__dict__ 741 | mmap = opts.pop('mmap') 742 | 743 | if mmap: 744 | check_mmap_dev(mmap) 745 | 746 | if opts['size']<2: 747 | sizes = (int(2**exp) for exp in range(1,24)) 748 | else: 749 | sizes = [int(opts['size'])] 750 | 751 | if opts['log_file']: 752 | fname = opts['log_file'].name 753 | fname, ext = os.path.splitext(fname) 754 | fname = fname + "_client" + ext 755 | try: 756 | opts['client_log_file'] = open(fname, "w") 757 | except IOError as e: 758 | print(e) 759 | sys.exit(1) 760 | 761 | for size in sizes: 762 | 763 | opts['size'] = size 764 | 765 | print("Running system memory test\t({}, {:.0f})." 766 | .format(opts['perftest'], Suffix(opts['size']))) 767 | mem_res = run_test(**opts) 768 | 769 | if mmap: 770 | print("Running mmap memory test\t({}, {:.0f})." 771 | .format(opts['perftest'], Suffix(opts['size']))) 772 | mmap_res = run_test(mmap=mmap, **opts) 773 | 774 | print() 775 | print() 776 | print("{} with system memory:".format(options.perftest)) 777 | print_results(mem_res, 2) 778 | 779 | if mmap: 780 | print("{} with {}:".format(options.perftest, mmap)) 781 | print_results(mmap_res, 2) 782 | 783 | except KeyboardInterrupt: 784 | print() 785 | print() 786 | pass 787 | except (OSError, ProcRunnerException) as e: 788 | print() 789 | print() 790 | print(e) 791 | -------------------------------------------------------------------------------- /perform/perform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ######################################################################### 3 | ## 4 | ## Copyright 2015 PMC-Sierra, Inc. 5 | ## 6 | ## Licensed under the Apache License, Version 2.0 (the "License"); you 7 | ## may not use this file except in compliance with the License. You may 8 | ## obtain a copy of the License at 9 | ## http://www.apache.org/licenses/LICENSE-2.0 Unless required by 10 | ## applicable law or agreed to in writing, software distributed under the 11 | ## License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | ## CONDITIONS OF ANY KIND, either express or implied. See the License for 13 | ## the specific language governing permissions and limitations under the 14 | ## License. 15 | ## 16 | ######################################################################## 17 | 18 | ######################################################################## 19 | ## 20 | ## Description: 21 | ## A simple shell script to run some RDMA performance testing on 22 | ## iomem, GPU mem and or main memory. 23 | ## 24 | ######################################################################## 25 | 26 | 27 | # Parameters for running the performance code 28 | EXE=ib_write_bw 29 | ARGS="-R -D 5 -s 8388608" 30 | SERVER=donard-rdma 31 | CLIENT=192.168.5.143 32 | BAR=/sys/bus/pci/devices/0000:00:03.0/0000:03:00.0/resource4 33 | PERF="likwid-perfctr" 34 | PARGS="-g MEM -C S0:0" 35 | LOG="perform.log" 36 | MEM=mbw 37 | MARGS="-n 0 -t 0 1024" 38 | 39 | # Pre-authenticate with sudo to prevent the background processes below 40 | # from going nuts. 41 | sudo -v 42 | 43 | # Accept some key parameter changes from the command line. 44 | while getopts "e:a:s:c:" opt; do 45 | case "$opt" in 46 | e) EXE=${OPTARG} 47 | ;; 48 | a) ARGS=${OPTARG} 49 | ;; 50 | s) SERVER=${OPTARG} 51 | ;; 52 | c) CLIENT=${OPTARG} 53 | ;; 54 | \?) 55 | echo "Invalid option: -$OPTARG" >&2 56 | exit 1 57 | ;; 58 | :) 59 | echo "Option -$OPTARG requires an argument." >&2 60 | exit 1 61 | ;; 62 | esac 63 | done 64 | 65 | # Perform some error checking 66 | TMP=$(which ${EXE}) 67 | if [ ! -x ${TMP} ] ; then 68 | echo "ERROR: Could not find ${EXE} on path. You can point to it using the -e option." 69 | exit -1 70 | fi 71 | TMP=$(which ${PERF}) 72 | if [ ! -x ${TMP} ] ; then 73 | echo "ERROR: Could not find ${PERF} on path. Please fix this and then re-run." 74 | exit -1 75 | fi 76 | TMP=$(which ${MEM}) 77 | if [ ! -x ${TMP} ] ; then 78 | echo "ERROR: Could not find ${MEM} on path. Please fix this and then re-run." 79 | exit -1 80 | fi 81 | 82 | # Run the performance test on the server and then use ssh to run the 83 | # command on the client side. 84 | 85 | cleanup() { 86 | kill ${MEM_PID} 87 | } 88 | 89 | run_test() { 90 | taskset -c 1 ${MEM} ${MARGS} &> memory.log & 91 | MEM_PID=$! 92 | trap cleanup EXIT 93 | 94 | sudo ${PERF} ${PARGS} ${EXE} ${ARGS} --mmap=${BAR} & 95 | SERVER_PID=$! 96 | sleep 2 97 | ssh ${CLIENT} ${EXE} ${ARGS} ${SERVER} &> /dev/null 98 | wait $SERVER_PID || exit 1 99 | 100 | sudo ${PERF} ${PARGS} ${EXE} ${ARGS} & 101 | SERVER_PID=$! 102 | sleep 2 103 | ssh ${CLIENT} ${EXE} ${ARGS} ${SERVER} &> /dev/null 104 | wait $SERVER_PID || exit 1 105 | 106 | } 107 | 108 | run_test | tee ${LOG} 109 | --------------------------------------------------------------------------------