├── .gitignore ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── TROUBLESHOOTING.md ├── after_run_setup.sh ├── before_run_setup.sh ├── clean_data.sh ├── clean_webapp_data.sh ├── compare.sh ├── config.php.template ├── create_site_data_file.sh ├── defaults.properties ├── delete_run.php ├── details.php ├── detect_big_differences.php ├── download_run.php ├── index.php ├── jmeter_config.properties.dist ├── lib └── lib.sh ├── logs └── empty ├── recorder.bsf ├── recorderfunctions.bsf ├── restart_services.sh ├── runs └── empty ├── runs_outputs └── empty ├── runs_samples └── empty ├── set_moodle_site.php ├── test_runner.sh ├── tests └── test.sh ├── webapp ├── DejaVuSans.license ├── DejaVuSans.ttf ├── classes │ ├── google_chart.php │ ├── google_charts_renderer.php │ ├── properties_reader.php │ ├── report.php │ ├── report_renderer.php │ └── test_plan_runs.php ├── css │ ├── redmond │ │ ├── images │ │ │ ├── animated-overlay.gif │ │ │ ├── ui-bg_flat_0_aaaaaa_40x100.png │ │ │ ├── ui-bg_flat_55_fbec88_40x100.png │ │ │ ├── ui-bg_glass_75_d0e5f5_1x400.png │ │ │ ├── ui-bg_glass_85_dfeffc_1x400.png │ │ │ ├── ui-bg_glass_95_fef1ec_1x400.png │ │ │ ├── ui-bg_gloss-wave_55_5c9ccc_500x100.png │ │ │ ├── ui-bg_inset-hard_100_f5f8f9_1x100.png │ │ │ ├── ui-bg_inset-hard_100_fcfdfd_1x100.png │ │ │ ├── ui-icons_217bc0_256x240.png │ │ │ ├── ui-icons_2e83ff_256x240.png │ │ │ ├── ui-icons_469bdd_256x240.png │ │ │ ├── ui-icons_6da8d5_256x240.png │ │ │ ├── ui-icons_cd0a0a_256x240.png │ │ │ ├── ui-icons_d8e7f3_256x240.png │ │ │ └── ui-icons_f9bd01_256x240.png │ │ ├── jquery-ui-1.10.3.custom.min.css │ │ └── jquery-ui.css │ └── styles.css ├── gradient.png ├── graph.php ├── inc.php ├── jmeter.css ├── jmeter.js ├── js │ ├── jquery-3.6.4.min.js │ ├── jquery-ui-1.13.2.custom.min.js │ ├── jquery-ui.js │ ├── jquery.js │ └── ui.js └── lib.php └── webserver_config.properties.dist /.gitignore: -------------------------------------------------------------------------------- 1 | cache/ 2 | logs/jmeter.* 3 | runs_samples/data.* 4 | runs_outputs/*.output 5 | runs/* 6 | moodle/ 7 | jmeter_config.properties 8 | webserver_config.properties 9 | test_files.properties 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Modification history: 2 | 3 | 28/03/2014 Rajesh Taneja Graph used to show total sum for each run variable 4 | (dbreads, dbwrites etc.). Now average will be shown 5 | (total sum /(users * loop count)), this will help 6 | user get correct change value, per user*loop. 7 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {{description}} 294 | Copyright (C) {{year}} {{fullname}} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Tools to compare Moodle sites and/or branches performance. 2 | 3 | 4 | ## Purpose 5 | 6 | They can be used to compare: 7 | 8 | * Performance before/after applying a patch 9 | * Two different branches performance 10 | * Different configurations and cache stores configurations 11 | * Different hardware 12 | * Web, database and other services tuning 13 | * Also works restoring your site sql dumps rather than using the fixed generated dataset, more info in [Using your own sql dump Moodle 2.5 onwards](#using-your-own-sql-dump-moodle-25-onwards) or [Using your own sql dump (before Moodle 2.5)](#using-your-own-sql-dump-before-moodle-25) 14 | 15 | 16 | ## Features 17 | 18 | * Clean site installation (from Moodle 2.5 onwards) 19 | * Fixed data set generation with courses, users, enrolments, module instances... (from Moodle 2.5 onwards) 20 | * JMeter test plan generation from course contents (from Moodle 2.5 onwards) 21 | * Web and database warm-up processes included in the test plan (results not collected) 22 | * JMeter runs gathering results about moodle performance data (database reads/writes/querytime, memory usage...) 23 | * Database query time value will differ depending on database and hardware used. 24 | * Runs results comparison 25 | 26 | There are scripts for both the web server and the JMeter server sides. 27 | 28 | * In case they are both in the same server you just need to clone the project once. 29 | * In case they are in different servers you need to clone the project in both servers 30 | + test_runner.sh along with jmeter_config.properties will be used in the server hosting JMeter 31 | + before_run_setup.sh, after_run_setup.sh and restart_services.sh will be used in the web server 32 | 33 | 34 | ## Requirements 35 | * MySQL or PostgreSQL 36 | * Git 37 | * cURL 38 | * PHP 5.3 39 | * Java 6 or later 40 | * JMeter - https://jmeter.apache.org/download_jmeter.cgi binaries (probably you will face problems using apt-get or other package management systems to download it) 41 | 42 | ## Installation 43 | 44 | The installation process differs depending whether you have both the web server and JMeter in the same computer or not. 45 | 46 | ### Web and JMeter servers in the same computer (usually a development computer) 47 | * Get the code 48 | + *cd /var/www* (or any other place, but accessible through a web server, not a public one please, read [security](#security) below) 49 | + *git clone https://github.com/moodlehq/moodle-performance-comparison.git moodle-performance-comparison* 50 | + *cd moodle-performance-comparison* 51 | * Configure the tool 52 | + *cp webserver_config.properties.dist webserver_config.properties* 53 | + Edit webserver_config.properties with your own values 54 | + *cp jmeter_config.properties.dist jmeter_config.properties* 55 | + Edit jmeter_config.properties with your own values 56 | 57 | ### Web server and JMeter running from a different server 58 | * Get the code in the web server 59 | + *cd /var/www* (or any other place, but accessible through a web server, not a public one please, read [security](#Security) below) 60 | + *git clone https://github.com/moodlehq/moodle-performance-comparison.git moodle-performance-comparison* 61 | + *cd moodle-performance-comparison* 62 | * Get the code in the JMeter server 63 | + *cd /wherever/you/want* 64 | + *git clone https://github.com/moodlehq/moodle-performance-comparison.git moodle-performance-comparison* 65 | + *cd moodle-performance-comparison* 66 | * Configure the tool in the web server 67 | + *cp webserver_config.properties.dist webserver_config.properties* 68 | + Edit webserver_config.properties with your own values 69 | * Configure the tool in the JMeter server 70 | + *cp jmeter_config.properties.dist jmeter_config.properties* 71 | + Edit jmeter_config.properties with your own values 72 | 73 | 74 | ## Usage 75 | 76 | The simplest is to just execute *compare.sh*, but it will only work in development computers where jmeter is installed in the web server and when you are testing differences between different branches. For other cases the process also differs depending whether you have both web server and JMeter in the same computer or not. Here there is another alternative, you can load your sql dump instead of having a clean brand new site with a fixed dataset, so you can run the generated test plan using real site generated data. 77 | 78 | The groupname and description arguments of test_runner.sh are useful to identify the run when comparing results, you can use it to set the branch name, the settings you used or whatever will help you identify which run is it. 79 | 80 | Note that you can run the tests as many times as you want, you just need to run after_run_setup.sh and restart_services.sh before running test_runner.sh every time to clean up the site. 81 | 82 | It is recommendable that you run all the scripts using the same user (there is no need to use a root user at all) you can use different users to run them (there are no restrictions about it) but be sure that the permissions are correct, it seems to be one of the more common issues when running this tool. 83 | 84 | ### Web and JMeter servers in the same computer, to find performance differences between different branches (usually a development computer) 85 | * Run compare.sh, the browser will be automatically opened after both runs are finished 86 | + ./compare.sh 87 | * In case the browser doesn't open properly the comparison page, browse to 88 | + http://localhost/moodle-performance-comparison/index.php (change to your URL according to your configuration) 89 | 90 | ### Web and JMeter servers in the same computer, to find performance differences changing site settings / cache stores 91 | * Generate the data and run the tests 92 | + cd /path/to/moodle-performance-comparison 93 | + *./before_run_setup.sh {XS|S|M|L|XL|XXL}* 94 | + Change site settings if necessary according to what you are comparing 95 | + *./restart_services.sh* 96 | + *./test_runner.sh* {groupname} {descriptioname} 97 | + *./after_run_setup.sh* 98 | + Change site settings if necessary according to what you are comparing 99 | + *./restart_services.sh* 100 | + *./test_runner.sh* {groupname} {descriptioname} 101 | * Check the results 102 | + http://localhost/moodle-performance-comparison/index.php (change to your URL according to your configuration) 103 | 104 | ### Web server and JMeter running from a different server 105 | * Generate the data and the test plan (web server) 106 | + *cd /path/to/moodle-performance-comparison* 107 | + *./before_run_setup.sh {XS|S|M|L|XL|XXL}* 108 | + Change site settings if necessary according to what you are comparing 109 | + *./restart_services.sh* 110 | * Get the test plan files (jmeter server) 111 | + *cd /path/to/moodle-performance-comparison* 112 | + *curl -O http://webserver/moodle/site/path/testusers.csv -O http://webserver/moodle/site/path/testplan.jmx* 113 | * Get the $beforebranch moodle version data (jmeter server) 114 | + *cd /path/to/moodle-performance-comparison* 115 | + *curl -O http://webserver/moodle/site/path/site_data.properties* 116 | * Run the before test (jmeter server) 117 | + *cd /path/to/moodle-performance-comparison* 118 | + *./test_runner.sh {groupname} {descriptioname} testplan.jmx testusers.csv site_data.properties* 119 | * Restore the base state to run the after branch (web server) 120 | + *cd /path/to/moodle-performance-comparison* 121 | + *./after_run_setup.sh* 122 | + Change site settings if necessary according to what you are comparing 123 | + *./restart_services.sh* 124 | * Get the $afterbranch moodle version data (jmeter server) 125 | + *cd /path/to/moodle-performance-comparison* 126 | + *curl -O http://webserver/moodle/site/path/site_data.properties* 127 | * Run the after test (jmeter server) 128 | + *cd /path/to/moodle-performance-comparison* 129 | + *./test_runner.sh {groupname} {descriptioname} testplan.jmx testusers.csv site_data.properties* 130 | * Check the results (web server) 131 | + http://localhost/moodle-performance-comparison/index.php (change to your URL according to your configuration) 132 | 133 | ### Using your own sql dump (Moodle 2.5 onwards) 134 | The installation and configuration is the same, it also depends on if you use the same computer for both web server and JMeter or not, but the usage changes when you want to use your own sql dump, it is not that easy to automate, as you need to specify which course do you want to use as target course and you can not use before_run_setup.sh to generate the test plan and test_files.properties. 135 | 136 | * *cd /webserver/path/to/moodle-performance-comparison* 137 | * Restore your dataroot to $dataroot 138 | * Restore your database to $dbname in $dbhost 139 | * Get the moodle code 140 | * Upgrade the site to $beforebranch 141 | + *cd moodle/* 142 | + *git checkout $beforebranch* 143 | + *php admin/cli/upgrade.php --allow-unstable --non-interactive* 144 | * Generate the test plan updating users passwords. You need to provide the shortname of the course that will be tested 145 | + *php admin/tool/generator/cli/maketestplan.php --size="THESIZEYOUWANT" --shortname="TARGETCOURSESHORTNAME" --bypasscheck --updateuserspassword* 146 | * Generate the site_data.properties file, with the current moodle version data, in the root directory of moodle-performance-comparison 147 | + *cd ..* 148 | + *./create_site_data_file.sh* 149 | * Download the test plan and the test users. The URLs are provided by maketestsite.php in the previous step, before the performance info output begins. 150 | + *cd moodle/* 151 | + *curl -o testplan.jmx http://webserver/url/provided/by/maketestsite.php/in/the/previous/step/testplan_NNNNNNNNNNNN_NNNN.jmx* 152 | + *curl -o testusers.csv http://webserver/url/provided/by/maketestsite.php/in/the/previous/step/users_NNNNNNNNNNNN_NNNN.jmx* 153 | * Backup dataroot and database (pg_dump or mysqldump), this backup will contain the updated passwords 154 | * Create moodle-performance-comparison/test_files.properties with the backups you just generated and the test plan data 155 | + *cd ../* 156 | + Create a new /path/to/moodle-performance-comparison/test_files.properties file with the following content: 157 | 158 | > testplanfile="/absolute/path/to/testplan.jmx" 159 | > 160 | > datarootbackup="/absolute/path/to/the/dataroot/backup/directory" 161 | > 162 | > testusersfile="/absolute/path/to/testusers.csv" 163 | > 164 | > databasebackup="/absolute/path/to/the/database/backup.sql" 165 | 166 | * cd */path/to/moodle-performance-comparison* and continue the normal process from restart_services.sh -> test_runner.sh -> after_run_setup.sh -> restart_services.sh -> test_runner.sh 167 | 168 | ### Using your own sql dump (before Moodle 2.5) 169 | Moodle 2.5 introduces the site and the test plan generators, so you can not use them if you are comparing previous branches. But you can: 170 | * Use the template included in Moodle 2.5 codebase and fill the placeholders with one of your site courses info and the test plan users, loops and ramp up period 171 | + The test plan template is located in *admin/tool/generator/testplan.template.jmx* 172 | * Fill a testusers.php with the target course data 173 | + You will need to check that the test data has enough users according to the data you provided in the test plan 174 | * Generate the site_data.properties file, with the current moodle version data, in the root directory of moodle-performance-comparison 175 | + *cd ..* 176 | + *./create_site_data_file.sh* 177 | * Follow [Using your own sql dump (Moodle 2.5 onwards)](#using-your-own-sql-dump-moodle-25-onwards) instructions 178 | 179 | 180 | ## Advanced usage 181 | * You can overwrite the values provided by the test plan using test_runner.sh options: 182 | + -u=[users_number] 183 | + -l=[loops_number] 184 | + -r=[rampup_period] 185 | + -t=[throughput] 186 | 187 | 188 | ## Security 189 | 190 | This tool in only intended to be used in development/testing environments inside the local network, it would be insecure to expose the project root in a public accessible web server, the same only exposing moodle/ directory: 191 | 192 | * Database connection data and other sensitive data is stored in properties files (you can change permissions) 193 | * It uses default sugar passwords (you can change the defaults in webserver_config.properties) 194 | * Stores test users credentials in Moodle's wwwroot (you can change permissions) 195 | * In general all files permissions are non secure at all (you can change permissions) 196 | * Other things I probably forgot, to resume, don't do it unless you are sure what you are doing 197 | 198 | 199 | ## Troubleshooting 200 | * You can find an extensive troubleshooting guide [here](https://github.com/moodlehq/moodle-performance-comparison/blob/main/TROUBLESHOOTING.md) 201 | * You might be interested in raising the PHP memory_limit to 512MB (apache) or something like that to 'M' or bigger when comparing results. 202 | * You can find JMeter logs in logs/ 203 | * You can find runs outputs in runs_outputs/ the results in runs_samples/ and the php arrays generated from them in runs/ 204 | * The generated .jtl files can be big. Don't hesitate to get rid of them if you don't need them for extra analytic purposes. 205 | * Same with $backupsdir/ contents, if you run before_run_setup.sh many time you will have a looot of hd space wasted. 206 | * If files with _java_pid[\d]+.hprof_ format are generated in your project root means that jmeter is running out of resource. http://wiki.apache.org/jmeter/JMeterFAQ#JMeter_keeps_getting_.22Out_of_Memory.22_errors.__What_can_I_do.3F for more info. 207 | -------------------------------------------------------------------------------- /TROUBLESHOOTING.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting guide 2 | 3 | This guide purpose, as you can guess, is to help you make this tool run, provinding info about what is happening behind the scripts and how you can backtrace aunexpected error until you find the cause and a solution. 4 | 5 | moodle-performance-comparison tool uses both bash scripts and PHP scripts, needs a web server and a database engine, it will probably run in multiple and different infrastructures, OS versions, PHP and bash interpreters, web servers, jmeter versions... So, depending on your configuration and your environment you may find issues when configuring it, downloading it's dependencies or running it that we have not detected yet; the tool provides error messages for most of the common issues you can fall into and we will be adding more of those error messages as long as you let us know the problems you are finding while trying to make it run, so don't hesitate to open an issue or comment about them and other people will not spend the time you did trying to find the solution for a problem. 6 | 7 | 8 | ## Requirements 9 | 10 | ### Java 11 | * Java 6 or later is required 12 | 13 | #### JMeter dependencies 14 | * The tool has been tested with JMeter 2.9 and 2.11, but will probably work from JMeter 2.7 onwards 15 | * You should download the binaries 16 | * You may have problems with the JMeter dependencies, ensure you downloaded the binaries from *http://jmeter.apache.org/download_jmeter.cgi* rather than using a package management system. You may find errors like the one below, stating that there are undefined classes: 17 | 18 | > 2014/01/13 00:54:50 ERROR - jmeter.util.BeanShellInterpreter: Error invoking bsh method: source Sourced file: recorder.bsf : Typed variable declaration : Typed variable declaration : Attempt to resolve method: rightPad() on undefined variable or class name: StringUtils 19 | 20 | 21 | ## Installing the tool 22 | 23 | ### Source code 24 | * Not much to say here, we recommend *git clone https://github.com/moodlehq/moodle-performance-comparison.git* otherwise you can just download the ZIP for the branch you want to use 25 | * The only changes between the project branches are the default hashes proposed in *webserver_config.properties.dist* 26 | * Remember that this tool is not intended to be used in public servers and there are serious security risks doing it 27 | 28 | 29 | ## Configuring 30 | 31 | In general, the best tip is to follow the *webserver_config.properties.dist* and *jmeter_config.properties.dist* provided values, changing them according to your system, probably you will have to change the database connection data, *$wwwroot*, *$dataroot* and *$backupsdir*. Depending on the use of the tool you will need to change *$afterbranch* and _$afterbranchrepository_ or both after and before. Would be better to avoid using values with tricky characters like quotes, double quotes, accents... 32 | 33 | ### File permissions 34 | * Ensure the user you are using has write permissions over the parent directories of *$dataroot* and *$backupsdir* 35 | 36 | ### Database connection 37 | * Only mysql and postgres are fully supported. Read README information about how to use the tool using other DB engines 38 | * Ensure you can access your database using the credentials you set in *webserver_config.properties* 39 | * If your database server is in a different server than the web server ensure they can access each other, the CLI commands used to create the databases are psql and mysql, you can also overwrite the path to the commands using *webserver_config.properties* 40 | * *$dbuser* should have permissions to create a database in *$dbhost* 41 | 42 | ### JMeter 43 | * You should provide the path to the directory where you extracted the JMeter files not the one with the jmeter sh script; the one containing bin/, lib/, README, LICENSE... 44 | 45 | ### Moodle site 46 | * Ensure your *$wwwroot* value includes *http://*, your locahost/the-host-name/ip and the path to the moodle-performance-comparison project + */moodle* as the moodle site is installed in moodle-performance-comparison/moodle 47 | 48 | 49 | ## Running the tool 50 | 51 | Here we will explain what the scripts are doing. Basically this is a resume of what are they doing when running all together (*compare.sh* follows all this process): 52 | 53 | 1. Checkout a base moodle codebase for both before and after branches 54 | 2. Installs a moodle site 55 | 3. Generates courses, users, enrolments and activities 56 | 4. Generates a JMeter test plan based on the site data 57 | 5. Backs up the database and the dataroot 58 | 6. Upgrades moodle to *$beforebranch* 59 | 7. Runs the tests 60 | 8. Restores the database and the dataroot to #5 61 | 9. Upgrades moodle to *$afterbranch* 62 | 10. Runs the tests 63 | 11. Opens a browser window to display the differences between runs 64 | 65 | The error messages the tool provides informs you about what went wrong and they are following the STDERR messages provided by the command that failed, so in most of the cases you will know what is going wrong. If you end up with one of those errors and you can not solve it you can always find the error message in the scripts and run manually the command that is failing to see the whole output. 66 | 67 | Following the scripts in the order they should be executed and the points where you can have problems: 68 | 69 | *before_run_setup.sh* 70 | 71 | 1. Creates directories to store moodle's dataroot and dirroot using provided *$dataroot* var. You might have problems if you provided a wrong *$dataroot* value. 72 | 2. Cleans previous existing JMeter test plan files 73 | 3. A clean database owned by *$dbuser* is created, droping any previous one if it existed. Here you can experience database permissions problems or you can find firewall restrictions to establish a connection between the web server and the database server 74 | 4. Checks out the base commit. Probably you have not changed that value, so it should be ok 75 | 5. Creates a moodle config file in *moodle/config.php* based on the template contents 76 | 6. Installs moodle using the default site and admin data 77 | 7. Checks that what you set as *$wwwroot* is the test site that has just been installed. Here it can fail if you provided a wrong *$wwwroot* value or the site can not be accessed by curl. You can also check that you can access *$wwwroot* manually using a browser, and logging in with admin/admin 78 | 8. Generates data to populate the site with users, courses, enrolments... All of moodle's php scripts returns != 0 exit codes so you will see an error message if something goes wrong. Same as before, you can log in to see if the courses, users, enrolments and activities are generated according to the test size you specified. 79 | 9. Generates the JMeter test plan; it creates two files the .jmx test plan file and the users file with the login details. They are stored in *moodle-performance-comparison/moodle* under the names *testplan.jmx* and *testusers.csv*, you can open them and see if you find any issue with their contents. 80 | 10. Database and dataroot are backed up. You can check *$backups* dir contents and confirm they are there, otherwise the base populated site would not be restored 81 | 11. Creates a file containing the all generated files. You can open *moodle-performance-comparison/test_files.properties* and ensure the mentioned files exists 82 | 12. Checks out *$beforebranch* and upgrades moodle to it. 83 | 13. Stores information about *$beforebranch*; the moodle version and the git commit info. You can open *moodle-performance-comparison/moodle/site_data.properties* and check if it's contents makes sense 84 | 85 | *test_runner.sh* 86 | 87 | 1. Runs JMeter using the info contained in the test plan files and the site data info (*testplan.jmx*, *testusers.csv* and *site_data.properties*) It takes a while depending on the size you specified, so if it finishes too fast, you can suspect that something went wrong. It generates a few files that you can open and check: *logs/jmeter.DATE.log* (the JMeter logs), *runs_outputs/DATE.output* (list of threads that JMeter run and it's HTTP status code), *runs_samples/data.DATE.jtl* (HTTP samples XML) and runs/DATE.php (PHP file with all the run data and results) 88 | 2. Checks *logs/jmeter.DATE.log* looking for warnings or errors to let the user know about unexpected errors. 89 | 90 | *after_run_setup.sh* 91 | 92 | 1. Restores database and dataroot removing the previous ones. Here you can face permissions problems. 93 | 2. Upgrades moodle to $afterbranch like *before_run_setup.sh* does in #12 94 | 3. Saves the site data like *before_run_setup.sh* does in #13 95 | 96 | *test_runner.sh* 97 | 98 | 1. Same as before 99 | 2. Same as before 100 | 101 | 102 | ## Viewing the results 103 | * Not much to comment about here, you can find issues when using the detailed view as it was inherited from a previous tool, but it works quite well. 104 | -------------------------------------------------------------------------------- /after_run_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Prepares the next test run after finished running the before_run_setup.sh script. 4 | # * Restores database 5 | # * Restores dataroot 6 | # * Upgrades moodle if necessary 7 | # 8 | # Auto-restore feature only available for postgres and mysql. 9 | # 10 | # Usage: cd /path/to/moodle-performance-comparison && ./after_run_setup.sh 11 | # 12 | # No arguments 13 | # 14 | ############################################## 15 | 16 | # Exit on errors. 17 | set -e 18 | 19 | # Dependencies. 20 | . ./lib/lib.sh 21 | 22 | # We need the paths. 23 | if [ ! -z "$1" ]; then 24 | output="Usage: `basename $0` 25 | 26 | Prepares the next test run after finished running the before_run_setup.sh script. 27 | * Restores database 28 | * Restores dataroot 29 | * Upgrades moodle if necessary 30 | " 31 | echo "$output" >&2 32 | exit 1 33 | fi 34 | 35 | # Checking as much as we can that before_run_setup.sh was 36 | # already executed and finished successfully. 37 | errormsg="Error: Did you run before_run_test.sh before running this one? " 38 | if [ ! -e "test_files.properties" ]; then 39 | echo $errormsg >&2 40 | exit 1 41 | fi 42 | if [ ! -d "moodle" ]; then 43 | echo $errormsg >&2 44 | exit 1 45 | fi 46 | if [ ! -d "moodle/.git" ]; then 47 | echo $errormsg >&2 48 | exit 1 49 | fi 50 | 51 | # Get user info. 52 | load_properties "defaults.properties" 53 | load_properties "webserver_config.properties" 54 | 55 | # Checks the $cmds. 56 | check_cmds 57 | 58 | # Get generated test plan values. 59 | load_properties "test_files.properties" 60 | 61 | # Move to the moodle directory. 62 | cd moodle 63 | 64 | # Remove current dataroot and restore the provided one (Better using chown...). 65 | if [ ! -d "$dataroot" ] || [ -z "$dataroot" ]; then 66 | echo "Error: Armageddon prevented just 2 lines of code above a rm -rf. 67 | Please, assign a value to \$dataroot var in webserver_config.properties" >&2 68 | exit 1 69 | fi 70 | delete_files $dataroot 1 71 | cp -r $datarootbackup $dataroot || \ 72 | throw_error "$datarootbackup can not be copied to $dataroot" 73 | 74 | chmod -R 777 $dataroot 75 | 76 | # Drop and restore the database. 77 | if [ "$dbtype" == "pgsql" ]; then 78 | echo "#######################################################################" 79 | echo "Restoring database and dataroot to Moodle ($basecommit)" 80 | export PGPASSWORD=${dbpass} 81 | 82 | # Checking that the table exists. 83 | databaseexists="$( ${pgsqlcmd} -h "$dbhost" -U "$dbuser" -l | \ 84 | grep "$dbname" | \ 85 | wc -l )" 86 | if [ "$databaseexists" != "0" ]; then 87 | ${pgsqlcmd} \ 88 | -h "$dbhost" \ 89 | -U "$dbuser" \ 90 | -d template1 \ 91 | -c "DROP DATABASE $dbname" \ 92 | --quiet 93 | fi 94 | ${pgsqlcmd} \ 95 | -h "$dbhost" \ 96 | -U "$dbuser" \ 97 | -d template1 \ 98 | -c "CREATE DATABASE $dbname WITH OWNER $dbuser ENCODING 'UTF8'" --quiet 99 | ${pgsqlcmd} \ 100 | --quiet \ 101 | -h "$dbhost" \ 102 | -U "$dbuser" \ 103 | $dbname < $databasebackup > /dev/null 104 | 105 | elif [ "$dbtype" == "mysqli" ]; then 106 | echo "#######################################################################" 107 | echo "Restoring database and dataroot to Moodle ($basecommit)" 108 | 109 | # Checking that the table exists. 110 | databaseexists="$( ${mysqlcmd} \ 111 | --host=${dbhost} \ 112 | --user=${dbuser} \ 113 | --password=${dbpass} \ 114 | -e "SHOW DATABASES LIKE '$dbname'" )" 115 | if [ ! -z "$databaseexists" ];then 116 | ${mysqlcmd} \ 117 | --host=${dbhost} \ 118 | --user=${dbuser} \ 119 | --password=${dbpass} \ 120 | -e "DROP DATABASE $dbname" \ 121 | --silent 122 | fi 123 | ${mysqlcmd} \ 124 | --host=${dbhost} \ 125 | --user=${dbuser} \ 126 | --password=${dbpass} \ 127 | -e "CREATE DATABASE $dbname DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_unicode_ci" \ 128 | --silent 129 | ${mysqlcmd} \ 130 | --silent \ 131 | --host=${dbhost} \ 132 | --user=${dbuser} \ 133 | --password=${dbpass} \ 134 | $dbname < $databasebackup > /dev/null 135 | else 136 | confirmoutput="Only postgres and mysql support: You need to manually restore your database. 137 | Press [q] to stop the script or, if you have already done it, any other key to continue. 138 | " 139 | echo "$confirmoutput" 140 | read confirmation 141 | if [ "$confirmation" == "q" ]; then 142 | exit 1 143 | fi 144 | fi 145 | 146 | # Upgrading moodle, although we are not sure that before and after branches are different. 147 | echo "Checking out Moodle from repo: $afterbranchrepository, ref: $afterbranch" 148 | checkout_branch $afterbranchrepository 'after' $afterbranch 149 | echo "Upgrading Moodle ($basecommit) to $(git rev-parse after/$afterbranch)" 150 | ${phpcmd} admin/cli/upgrade.php --non-interactive --allow-unstable > /dev/null || \ 151 | throw_error "Moodle can not be upgraded to $afterbranch" 152 | 153 | # Stores the site data in an jmeter-accessible file. 154 | save_moodle_site_data 155 | 156 | # Returning to the root. 157 | cd .. 158 | 159 | # Info, all went as expected and we are all happy. 160 | outputinfo=" 161 | ####################################################################### 162 | 'After' run setup finished successfully. 163 | 164 | Now you can: 165 | - Change the site configuration 166 | - Change the cache stores 167 | And to continue with the test you should: 168 | - Run restart_services.sh (or manually restart web and database servers 169 | if this script doesn\'t suit your system) 170 | - Run test_runner.sh 171 | " 172 | 173 | echo "$outputinfo" 174 | exit 0 175 | -------------------------------------------------------------------------------- /before_run_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Sets up a moodle site with courses and users and generates a JMeter test plan. 4 | # 5 | # Auto-backup feature only available for postgres and mysql, only available as interactive 6 | # script when running other drivers. 7 | # 8 | # Usage: cd /path/to/moodle-performance-comparison && ./before_run_setup.sh {size} 9 | # 10 | # Arguments: 11 | # * $1 => Size, one of the following options: XS, S, M, L, XL, XXL. More than 'M' is not recommended for development computers. 12 | # 13 | ############################################## 14 | 15 | # Exit on errors. 16 | set -e 17 | 18 | # Dependencies. 19 | . ./lib/lib.sh 20 | 21 | # Hardcoded values. 22 | readonly SITE_FULL_NAME="Full site name" 23 | readonly SITE_SHORT_NAME="Short site name" 24 | readonly SITE_ADMIN_USERNAME="admin" 25 | readonly SITE_ADMIN_PASSWORD="admin" 26 | readonly CURRENT_WORKING_DIRECTORY=`pwd` 27 | readonly FILE_NAME_USERS="$CURRENT_WORKING_DIRECTORY/moodle/testusers.csv" 28 | readonly FILE_NAME_TEST_PLAN="$CURRENT_WORKING_DIRECTORY/moodle/testplan.jmx" 29 | readonly PERMISSIONS=775 30 | 31 | # Validate the passed size ($1) 32 | case "$1" in 33 | 'XS') 34 | targetcourse='testcourse_3' 35 | ;; 36 | 'S') 37 | targetcourse='testcourse_12' 38 | ;; 39 | 'M') 40 | targetcourse='testcourse_73' 41 | ;; 42 | 'L') 43 | targetcourse='testcourse_277' 44 | ;; 45 | 'XL') 46 | targetcourse='testcourse_1065' 47 | ;; 48 | 'XXL') 49 | targetcourse='testcourse_4177' 50 | ;; 51 | *) 52 | echo "Usage: `basename $0` {size} 53 | 54 | Sets up a moodle site with courses and users and generates a JMeter test plan. 55 | 56 | Arguments: 57 | * $1 => Size, one of the following options: XS, S, M, L, XL, XXL. More than 58 | 'M' is not recommended in development computers. 59 | " >&2 60 | exit 1 61 | esac 62 | 63 | # Get user info. 64 | load_properties "defaults.properties" 65 | load_properties "webserver_config.properties" 66 | 67 | # Checks the $cmds. 68 | check_cmds 69 | 70 | # Creating & cleaning dirroot & dataroot (keeping .git) 71 | if [ ! -e "$dataroot" ]; then 72 | mkdir -m $PERMISSIONS $dataroot || \ 73 | throw_error "There was a problem creating $dataroot directory" 74 | else 75 | # If it already existed we clean it 76 | delete_files "$dataroot/*" 77 | fi 78 | 79 | if [ ! -e "moodle" ]; then 80 | mkdir -m $PERMISSIONS "moodle" || \ 81 | throw_error "There was a problem creating moodle/ directory" 82 | fi 83 | 84 | # Cleaning previous test plan files. 85 | if [ -e "$FILE_NAME_USERS" ]; then 86 | delete_files "$FILE_NAME_USERS" 1 87 | fi 88 | if [ -e "$FILE_NAME_TEST_PLAN" ]; then 89 | delete_files "$FILE_NAME_TEST_PLAN" 1 90 | fi 91 | 92 | 93 | # Creating new database and delete it if it already exists. 94 | if [ "$dbtype" == "pgsql" ]; then 95 | export PGPASSWORD=${dbpass} 96 | 97 | # Checking that the table exists. 98 | databaseexists="$( ${pgsqlcmd} -h "$dbhost" -U "$dbuser" -l | \ 99 | grep "$dbname" | \ 100 | wc -l )" 101 | if [ "$databaseexists" != "0" ]; then 102 | ${pgsqlcmd} \ 103 | -h "$dbhost" \ 104 | -U "$dbuser" \ 105 | -d template1 \ 106 | -c "DROP DATABASE $dbname" \ 107 | --quiet 108 | fi 109 | 110 | ${pgsqlcmd} \ 111 | -h "$dbhost" \ 112 | -U "$dbuser" \ 113 | -d template1 \ 114 | -c "CREATE DATABASE $dbname WITH OWNER $dbuser ENCODING 'UTF8'" \ 115 | --quiet 116 | 117 | elif [ "$dbtype" == "mysqli" ]; then 118 | 119 | # Checking that the table exists. 120 | databaseexists="$( ${mysqlcmd} \ 121 | --host=${dbhost} \ 122 | --user=${dbuser} \ 123 | --password=${dbpass} \ 124 | -e "SHOW DATABASES LIKE '$dbname'" )" 125 | if [ ! -z "$databaseexists" ];then 126 | ${mysqlcmd} \ 127 | --host=${dbhost} \ 128 | --user=${dbuser} \ 129 | --password=${dbpass} \ 130 | -e "DROP DATABASE $dbname" \ 131 | --silent 132 | fi 133 | 134 | ${mysqlcmd} \ 135 | --host=${dbhost} \ 136 | --user=${dbuser} \ 137 | --password=${dbpass} \ 138 | -e "CREATE DATABASE $dbname DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_unicode_ci" \ 139 | --silent 140 | 141 | else 142 | confirmoutput="Only postgres (pgsql) and mysql (mysqli) support: You \ 143 | need to manually create your database. 144 | Press [q] to stop the script or, if you have already done it, any other key to continue. 145 | " 146 | echo "$confirmoutput" 147 | read confirmation 148 | if [ "$confirmation" == "q" ]; then 149 | exit 1 150 | fi 151 | fi 152 | 153 | # Move to moodle dirroot and begin setting up everything. 154 | cd moodle 155 | 156 | checkout_branch $repository 'origin' $basecommit 157 | 158 | # Copy config.php template and set user properties. 159 | replacements="%%dbtype%%#$dbtype 160 | %%dbhost%%#$dbhost 161 | %%dbname%%#$dbname 162 | %%dbuser%%#$dbuser 163 | %%dbpass%%#$dbpass 164 | %%dbprefix%%#$dbprefix 165 | %%wwwroot%%#$wwwroot 166 | %%dataroot%%#$dataroot 167 | %%toolgeneratorpassword%%#$toolgeneratorpassword" 168 | 169 | configfilecontents="$( cat ../config.php.template )" 170 | for i in ${replacements}; do 171 | configfilecontents=$( echo "${configfilecontents}" | sed "s#${i}#g" ) 172 | done 173 | 174 | # Overwrites the previous config.php file. 175 | errorstr="Moodle's config.php can not be written, \ 176 | check $CURRENT_WORKING_DIRECTORY/moodle directory \ 177 | (and $CURRENT_WORKING_DIRECTORY/moodle/config.php if it exists) permissions." 178 | 179 | echo "${configfilecontents}" > config.php || \ 180 | throw_error "$errorstr" 181 | chmod $PERMISSIONS config.php 182 | 183 | # Install the site with user specified params. 184 | echo "#######################################################################" 185 | echo "Installing Moodle ($basecommit)" 186 | ${phpcmd} admin/cli/install_database.php \ 187 | --agree-license \ 188 | --fullname="$SITE_FULL_NAME" \ 189 | --shortname="$SITE_SHORT_NAME" \ 190 | --adminuser="$SITE_ADMIN_USERNAME" \ 191 | --adminpass="$SITE_ADMIN_PASSWORD" \ 192 | > /dev/null || \ 193 | throw_error "Moodle can not be installed, check that the database data is correctly set" 194 | 195 | # Check that the installed site is properly installed and can be accessed 196 | # using the provided wwwroot. 197 | siteindex="${wwwroot%/}/index.php" 198 | ${curlcmd} --silent "$siteindex" | \ 199 | grep --quiet "$SITE_FULL_NAME" || \ 200 | throw_error "There is a problem with your wwwroot config var or with \ 201 | the test site. Browse to $wwwroot and ensure you see a moodle site." 202 | 203 | 204 | # Generate courses. 205 | ${phpcmd} admin/tool/generator/cli/maketestsite.php \ 206 | --size=$1 \ 207 | --fixeddataset \ 208 | --bypasscheck \ 209 | --filesizelimit="1000" \ 210 | --quiet \ 211 | > /dev/null || \ 212 | throw_error "The test site can not be generated, check that the site is correctly installed" 213 | 214 | # Enable advanced settings and list courses in the frontpage. 215 | ${phpcmd} ../set_moodle_site.php || \ 216 | throw_error "The test site can not be configured, check that the site is correctly installed" 217 | 218 | # We capture the output to get the files we will need. 219 | testplancommand=${phpcmd}' admin/tool/generator/cli/maketestplan.php \ 220 | --size='$1' \ 221 | --shortname='${targetcourse}' \ 222 | --bypasscheck' \ 223 | > /dev/null || \ 224 | throw_error "Moodle's test plan generator could not finish as expected" 225 | testplanfiles="$(${testplancommand})" 226 | 227 | # We only get the first two items as there is more performance info. 228 | if [[ "$testplanfiles" == *"testplan"* ]]; then 229 | # Prepare curl arguments. 230 | files=( $testplanfiles ) 231 | if [ "${#files[*]}" -ne 2 ]; then 232 | echo "Error: There was a problem generating the test plan." >&2 233 | exit 1 234 | fi 235 | ${curlcmd} \ 236 | -o $FILE_NAME_TEST_PLAN ${files[0]} \ 237 | -o $FILE_NAME_USERS ${files[1]} \ 238 | --silent || \ 239 | throw_error "There was a problem getting the test plan files. Check your wwwroot setting." 240 | else 241 | echo "Error: There was a problem generating the test plan." >&2 242 | exit 1 243 | fi 244 | 245 | # Backups. 246 | if [ ! -e "$backupsdir" ]; then 247 | mkdir -m $PERMISSIONS $backupsdir || \ 248 | throw_error "There was a problem creating $backupsdir directory" 249 | 250 | fi 251 | datesufix=`date '+%Y%m%d%H%M'` 252 | filenamedataroot="$backupsdir/dataroot_backup_$datesufix" 253 | filenamedatabase="$backupsdir/database_backup_$datesufix.sql" 254 | 255 | # Dataroot backup. 256 | echo "Creating Moodle ($basecommit) database and dataroot backups" 257 | delete_files "$dataroot/sessions" 258 | cp -r $dataroot $filenamedataroot || \ 259 | throw_error "$dataroot can not be copied to $filenamedataroot" 260 | 261 | # Database backup. 262 | if [ "$dbtype" == "pgsql" ]; then 263 | export PGPASSWORD=${dbpass} 264 | ${pgsqldumpcmd} \ 265 | -h "$dbhost" \ 266 | -U "$dbuser" \ 267 | $dbname > $filenamedatabase 268 | elif [ "$dbtype" == "mysqli" ]; then 269 | ${mysqldumpcmd} \ 270 | --host=${dbhost} \ 271 | --user=${dbuser} \ 272 | --password=${dbpass} \ 273 | ${dbname} > $filenamedatabase 274 | else 275 | echo "Only postgres and mysql backup/restore support, you will have to backup it manually." 276 | $filenamedatabase='NOT AVAILABLE' 277 | fi 278 | 279 | # Info about what have we done, stored inside moodle's dirroot to be visible. 280 | # Overwrites the old file if it exists. 281 | errorstr="Moodle can not add the info about the generated files to \ 282 | $CURRENT_WORKING_DIRECTORY/test_files.properties, check the permissions" 283 | 284 | generatedfiles="testplanfile=$FILE_NAME_TEST_PLAN 285 | testusersfile=$FILE_NAME_USERS 286 | datarootbackup=$filenamedataroot 287 | databasebackup=$filenamedatabase" 288 | echo "$generatedfiles" > "$CURRENT_WORKING_DIRECTORY/test_files.properties" || \ 289 | throw_error "$errorstr" 290 | 291 | # Upgrading moodle, although we are not sure that base and before branch are different. 292 | echo "Checking out Moodle from repo: $beforebranchrepository, ref: $beforebranch" 293 | checkout_branch $beforebranchrepository 'before' $beforebranch 294 | echo "Upgrading Moodle ($basecommit) to $(git rev-parse before/$beforebranch)" 295 | ${phpcmd} admin/cli/upgrade.php \ 296 | --non-interactive \ 297 | --allow-unstable \ 298 | > /dev/null || \ 299 | throw_error "Moodle can not be upgraded to $beforebranch" 300 | 301 | # Stores the site data in an jmeter-accessible file. 302 | save_moodle_site_data 303 | 304 | # Returning to the root. 305 | cd .. 306 | 307 | # Also output the info. 308 | outputinfo=" 309 | ####################################################################### 310 | 'Before' run setup finished successfully. 311 | 312 | Note the following files were generated, you will need this info when running 313 | testrunner.sh in a different server, they are also saved in test_files.properties. 314 | - Test plan: $FILE_NAME_TEST_PLAN 315 | - Test users: $FILE_NAME_USERS 316 | - Dataroot backup: $filenamedataroot 317 | - Database backup: $filenamedatabase 318 | 319 | Now you can: 320 | - Change the site configuration 321 | - Change the cache stores 322 | And to continue with the test you should: 323 | - Run restart_services.sh (or manually restart web and database servers if 324 | this script doesn\'t suit your system) 325 | - Run test_runner.sh 326 | " 327 | echo "$outputinfo" 328 | exit 0 329 | -------------------------------------------------------------------------------- /clean_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Cleans all test results data 4 | # 5 | # Data generated by test runs can become huge 6 | # this script gets rid of all previous results. 7 | # 8 | ############################################## 9 | 10 | # Exit on errors. 11 | set -e 12 | 13 | # Dependencies. 14 | . ./lib/lib.sh 15 | 16 | # Get config. 17 | load_properties "defaults.properties" 18 | load_properties "webserver_config.properties" 19 | 20 | delete_files "runs/*.php" 21 | delete_files "runs_samples/*.jtl" 22 | delete_files "runs_outputs/*.output" 23 | delete_files "logs/*.log" 24 | 25 | # Also backups. 26 | delete_files "$backupsdir/*" 27 | 28 | # Delete compare files. 29 | if [ -f "moodle/site_data.properties" ]; then 30 | delete_files "moodle/site_data.properties" 1 31 | fi 32 | 33 | if [ -f "test_files.properties" ]; then 34 | delete_files "test_files.properties" 1 35 | fi 36 | 37 | outputinfo=" 38 | ####################################################################### 39 | Runs results and backups deleted successfully. 40 | " 41 | echo "$outputinfo" 42 | exit 0 43 | -------------------------------------------------------------------------------- /clean_webapp_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Cleans the web application cache. 4 | # 5 | # This script needs to be run by the user 6 | # running the web server. 7 | # 8 | # e.g. \$sudo -u www-data ./clean_webapp_data.sh 9 | # 10 | ############################################## 11 | 12 | # Exit on errors. 13 | set -e 14 | 15 | # Dependencies. 16 | . ./lib/lib.sh 17 | 18 | # Also images cache. 19 | delete_files "cache/*" 20 | 21 | outputinfo=" 22 | ####################################################################### 23 | Web application cached results deleted successfully. 24 | " 25 | echo "$outputinfo" 26 | exit 0 27 | -------------------------------------------------------------------------------- /compare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Runs the whole scripts chain and opens the browser once finished to compare results 4 | # 5 | # Note that this is only useful when running jmeter in the moodle site server 6 | # and you can only rely on results like db queries, memory or files included, 7 | # as this is no controlling the server load at all. 8 | # 9 | # Usage: cd /path/to/moodle-performance-comparison && ./compare.sh 10 | # 11 | ############################################## 12 | 13 | # Exit on errors. 14 | set -e 15 | 16 | # Dependencies. 17 | . ./lib/lib.sh 18 | 19 | # Get user info. 20 | load_properties "defaults.properties" 21 | load_properties "webserver_config.properties" 22 | 23 | # Checks the $cmds. 24 | check_cmds 25 | 26 | timestart=`date +%s` 27 | 28 | # Runs descriptions according to branches. 29 | # Group name according to date. 30 | groupname="compare_"`date '+%Y%m%d%H%M'` 31 | 32 | # Hardcoding S as the size, with 5 loops is enough to have consistent results. 33 | ./before_run_setup.sh $defaultcomparesize || \ 34 | throw_error "Before run setup didn't finish as expected" 35 | 36 | ./test_runner.sh "$groupname" "before" || \ 37 | throw_error "The before test run didn't finish as expected" 38 | 39 | # We don't restart the browser here, this is a development machine 40 | # and probably you are not staring at the CLI waiting for it to 41 | # finish. 42 | 43 | ./after_run_setup.sh || \ 44 | throw_error "After run setup didn't finish as expected" 45 | 46 | ./test_runner.sh "$groupname" "after" || \ 47 | throw_error "The after test run didn't finish as expected" 48 | 49 | timeend=`date +%s` 50 | 51 | # Output time elapsed. 52 | elapsedtime=$[$timeend - $timestart] 53 | show_elapsed_time $elapsedtime 54 | output=" 55 | ####################################################################### 56 | Comparison test finished successfully. 57 | " 58 | echo "$outputinfo" 59 | 60 | # Opens the comparison web interface in a browser. 61 | if [[ "$OSTYPE" == "darwin"* ]];then 62 | open -a $browser "$wwwroot/../" 63 | else 64 | $browser "$wwwroot/../" 65 | fi 66 | -------------------------------------------------------------------------------- /config.php.template: -------------------------------------------------------------------------------- 1 | dbtype = '%%dbtype%%'; 8 | $CFG->dblibrary = 'native'; 9 | $CFG->dbhost = '%%dbhost%%'; 10 | $CFG->dbname = '%%dbname%%'; 11 | $CFG->dbuser = '%%dbuser%%'; 12 | $CFG->dbpass = '%%dbpass%%'; 13 | $CFG->prefix = '%%dbprefix%%'; 14 | $CFG->dboptions = array ( 15 | 'dbpersist' => 0, 16 | 'dbsocket' => 0, 17 | 'fetchbuffersize' => 0, // We don't use big data and cursors lead to an excess of dbreads (AUX). Hence, disabling them. 18 | ); 19 | 20 | $CFG->wwwroot = '%%wwwroot%%'; 21 | $CFG->dataroot = '%%dataroot%%'; 22 | $CFG->admin = 'admin'; 23 | 24 | // No debug! it changes db reads and db writes values. 25 | $CFG->debug = false; 26 | $CFG->debugdisplay = false; 27 | 28 | // No cache_text to have results as stable as possible. 29 | $CFG->cachetext = 0; 30 | 31 | // Time between threads accesses are randomly generated so we can 32 | // not have stable results with the core LASTACCESS_UPDATE_SECS value. 33 | // Moodle will try to define it again, the php error will be hidden with 34 | // debug mode disabled. https://tracker.moodle.org/browse/MDL-41910 35 | define('LASTACCESS_UPDATE_SECS', 9999999999); 36 | 37 | // Some more settings towards results stability. 38 | $CFG->sessiontimeout = 172800; 39 | $CFG->session_update_timemodified_frequency = 9999999999; 40 | $CFG->messaging = 0; // Disable messaging, it's not used with current JMX plan. 41 | 42 | // Set the generated users password to avoid the default non-loggeable one. 43 | $CFG->tool_generator_users_password = '%%toolgeneratorpassword%%'; 44 | 45 | // Using file sessions. 46 | $CFG->dbsessions = false; 47 | 48 | if (!defined('CLI_SCRIPT')) { 49 | define('MDL_PERF_TEST', true); 50 | define('MDL_PERF', true); 51 | define('MDL_PERFDB', true); 52 | define('MDL_PERFTOLOG', true); 53 | define('MDL_PERFTOFOOT', true); 54 | } 55 | 56 | $CFG->directorypermissions = 0777; 57 | 58 | $CFG->defaulthomepage = 0; // Our current JMX plan expects this. 59 | 60 | require_once(dirname(__FILE__) . '/lib/setup.php'); 61 | 62 | // There is no php closing tag in this file, 63 | // it is intentional because it prevents trailing whitespace problems! 64 | -------------------------------------------------------------------------------- /create_site_data_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Creates a site_data.properties file with the moodle/ site version data. 4 | # 5 | # Useful when using your own SQL dump and you can't use before_run_setup.sh 6 | # 7 | # Usage: cd /path/to/moodle-performance-comparison && ./create_site_data_file.sh 8 | # 9 | # Arguments: 10 | # No arguments 11 | # 12 | ############################################## 13 | 14 | # Exit on errors. 15 | set -e 16 | 17 | # Dependencies. 18 | . ./lib/lib.sh 19 | 20 | # Get user info. 21 | load_properties "defaults.properties" 22 | load_properties "webserver_config.properties" 23 | 24 | # Checks the $cmds. 25 | check_cmds 26 | 27 | cd moodle 28 | 29 | # This should be enough. 30 | save_moodle_site_data 31 | 32 | # Returning home in case this script is called by others. 33 | cd .. 34 | 35 | outputinfo=" 36 | ####################################################################### 37 | Site info file created successfully 38 | 39 | " 40 | 41 | echo "$outputinfo" 42 | exit 0 43 | -------------------------------------------------------------------------------- /defaults.properties: -------------------------------------------------------------------------------- 1 | browser="firefox" 2 | phpcmd='php' 3 | mysqlcmd='mysql' 4 | pgsqlcmd='psql' 5 | mysqldumpcmd='mysqldump' 6 | pgsqldumpcmd='pg_dump' 7 | gitcmd='git' 8 | curlcmd='curl' 9 | groupedthreshold=4 10 | singlestepthreshold=2 11 | includelogs=1 12 | defaultcomparesize=S 13 | readonlyweb='' 14 | -------------------------------------------------------------------------------- /delete_run.php: -------------------------------------------------------------------------------- 1 | delete()) { 22 | echo 'Error: There was a problem deleting the file'; 23 | } else { 24 | echo '

Run deleted

'; 25 | 26 | // Remove the deleted one. 27 | $timestamps = explode('&', $returnurl); 28 | foreach ($timestamps as $key => $timestamp) { 29 | if (strstr($timestamp, $filename) != false) { 30 | unset($timestamps[$key]); 31 | } 32 | } 33 | $returnurl = implode('&', $timestamps); 34 | } 35 | 36 | // Link to return to the index. 37 | echo '

Return to the runs page'; 38 | -------------------------------------------------------------------------------- /details.php: -------------------------------------------------------------------------------- 1 | "; 54 | echo ""; 55 | $httpyuilib = 'http://yui.yahooapis.com'; 56 | if (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] != 'off') { 57 | $httpyuilib = 'https://yui-s.yahooapis.com'; 58 | } 59 | echo ''; 60 | echo ''; 61 | echo ""; 62 | echo ""; 63 | echo ""; 64 | 65 | display_run_selector($runs, $before, $after, array('w' => $width, 'h' => $height), $organiseby, $mostcommononly, $normalize); 66 | 67 | if ($before && $after) { 68 | $count = 0; 69 | echo "
"; 70 | $statsarray = array(); 71 | foreach ($pages as $key => $page) { 72 | if (!is_object($page['before']) || !is_object($page['after'])) { 73 | continue; 74 | } 75 | $count++; 76 | $class = ($count%2)?'odd':'even'; 77 | $classkey = substr($key, 0, 8); 78 | if ($mostcommononly) { 79 | $page['before']->strip_to_most_common_only($organiseby); 80 | $page['after']->strip_to_most_common_only($organiseby); 81 | } 82 | echo "
"; 83 | echo "

".$page['before']->name."

"; 84 | echo "

url."'>".$page['before']->url."

"; 85 | echo "
"; 86 | 87 | list($output, $stats) = display_results($page['before'], $page['after']); 88 | echo $stats; 89 | echo $output; 90 | $statsarray[] = $stats; 91 | display_organised_results($organiseby, $page['before'], $page['after']); 92 | 93 | echo "
"; 94 | foreach ($PROPERTIES as $PROPERTY) { 95 | if (!property_exists($page['before'], $PROPERTY)) { 96 | continue; 97 | } 98 | $graphfile = produce_page_graph($PROPERTY, $before, $page['before'], $after, $page['after'], $width, $height, array('x' => $mostcommononly, 'n' => $normalize)); 99 | echo ""; 100 | echo "$PROPERTY"; 101 | echo ""; 102 | } 103 | echo "
"; 104 | echo "
"; 105 | echo "
"; 106 | } 107 | echo "
"; 108 | echo "

Combined stats

"; 109 | $cstats = array_pop($statsarray); 110 | array_unshift($statsarray, $cstats); 111 | foreach ($statsarray as $stats) { 112 | echo $stats; 113 | } 114 | echo "
"; 115 | echo "
"; 116 | } 117 | echo "\n\n"; 118 | echo ""; 119 | echo ""; 120 | -------------------------------------------------------------------------------- /detect_big_differences.php: -------------------------------------------------------------------------------- 1 | $arg) { 27 | if ($arg == '--outliers' || substr($arg, 0, 9) == '--normali') { 28 | $normalize = true; 29 | unset($argv[$key]); 30 | } 31 | } 32 | 33 | if (empty($argv)) { 34 | echo 'Error: You need to specify the runs filenames without their .php sufix.' . PHP_EOL; 35 | exit(1); 36 | } 37 | 38 | if (count($argv) == 1) { 39 | echo 'Error: You should specify, at least, two runs to compare.' . PHP_EOL; 40 | exit(1); 41 | } 42 | 43 | // The filename without .php. 44 | $timestamps = $argv; 45 | 46 | $report = new report(); 47 | if (!$report->parse_runs($timestamps, $normalize)) { 48 | echo 'Error: The selected runs are not comparable.' . PHP_EOL; 49 | foreach ($report->get_errors() as $var => $error) { 50 | echo $var . ': ' . $error . PHP_EOL; 51 | } 52 | exit(1); 53 | } 54 | 55 | // Uses the thresholds specified in the .properties files. 56 | if (!$report->calculate_big_differences()) { 57 | echo 'Error: No way to get the default thresholds...' . PHP_EOL; 58 | exit(1); 59 | } 60 | $branches = $report->get_big_differences(); 61 | 62 | // Report changes. 63 | $exitcode = 0; 64 | if ($branches) { 65 | foreach ($branches as $branchnames => $changes) { 66 | if (!empty($changes)) { 67 | echo "$branchnames" . PHP_EOL; 68 | } 69 | foreach ($changes as $state => $data) { 70 | foreach ($data as $var => $steps) { 71 | foreach ($steps as $stepname => $info) { 72 | $normalizestr = $normalize ? '(Normalized) ' : ''; 73 | echo $normalizestr; 74 | echo "- $state: $var - $stepname -> $info" . PHP_EOL; 75 | } 76 | } 77 | } 78 | 79 | if (!empty($changes['increment'])) { 80 | $exitcode = 1; 81 | } 82 | } 83 | } 84 | 85 | exit($exitcode); 86 | -------------------------------------------------------------------------------- /download_run.php: -------------------------------------------------------------------------------- 1 | download(); 15 | -------------------------------------------------------------------------------- /index.php: -------------------------------------------------------------------------------- 1 | make($_GET['timestamps'], $normalize); 17 | 18 | } 19 | 20 | // Render it. 21 | $renderer = new report_renderer($report); 22 | $renderer->render($normalize); 23 | -------------------------------------------------------------------------------- /jmeter_config.properties.dist: -------------------------------------------------------------------------------- 1 | ######################################### 2 | # Configure with your own values 3 | ######################################### 4 | 5 | ## Jmeter ################################################## 6 | 7 | # Jmeter directory (/bin/jmeter will be added by test_runner.sh). 8 | jmeter_path=/opt/apache-jmeter-2.9 9 | 10 | 11 | ## Results ####################### 12 | 13 | # You can use these vars to specify what difference between runs 14 | # can be considered an increase or a decrease and what can 15 | # be ignored. The number represents a percentage between the first 16 | # run and the other/s one/s. 17 | # 18 | # This var value will mark the accepted difference between the 19 | # different vars of a run when comparing each of the test plan 20 | # steps. Default value as defined in defaults.properties: 21 | # 22 | # groupedthreshold=4 23 | # 24 | # This var value will mark the accepted difference between the 25 | # different vars of a run when comparing the sum of all test 26 | # plan steps. Default value as defined in defaults.properties: 27 | # 28 | # singlestepthreshold=2 29 | # 30 | # You can also specify a JSON associative array with the detailed list of thresholds which will 31 | # have preference over $groupedthreshold and $singlestepthreshold. The JSON string should be defined 32 | # all in one line, wrapped using single quotes, and following this format: 33 | # 34 | # thresholds='{"bystep":{"dbreads":2,"dbwrites":2,"dbquerytime":2,"memoryused":2,"filesincluded":1,"serverload":10,"sessionsize":2,"timeused":5},"total":{"dbreads":2,"dbwrites":2,"dbquerytime":2,"memoryused":2,"filesincluded":1,"serverload":10,"sessionsize":2,"timeused":5}}' 35 | # 36 | # Include logs table writes as db writes, this is enabled by default. 37 | # Set includelogs="" in case you don't want to include them. 38 | # 39 | # includelogs=1 40 | 41 | 42 | ## Web interface ####################### 43 | 44 | # The web interface allows users to perform both read action (compare results, 45 | # download runs results...) and write actions (delete runs) if you are planning 46 | # to publish your results but you don't want users to delete your runs set any 47 | # non-empty value to this setting. By default, the web interface is read & write. 48 | # 49 | # readonlyweb='' 50 | -------------------------------------------------------------------------------- /lib/lib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Common functions. 4 | 5 | ################################################ 6 | # Checks that last command was successfully executed 7 | # otherwise exits showing an error. 8 | # 9 | # Arguments: 10 | # * $1 => The error message 11 | # 12 | ################################################ 13 | throw_error() 14 | { 15 | local errorcode=$? 16 | if [ "$errorcode" -ne "0" ]; then 17 | 18 | # Print the provided error message. 19 | if [ ! -z "$1" ]; then 20 | echo "Error: $1" >&2 21 | fi 22 | 23 | # Exit using the last command error code. 24 | exit $errorcode 25 | fi 26 | } 27 | 28 | ################################################ 29 | # Deletes the files 30 | # 31 | # Arguments: 32 | # * $1 => The file/directories to delete 33 | # * $2 => Set $2 will make the function exit if it is an unexisting file 34 | # 35 | # Accepts dir/*.extension format like ls or rm does. 36 | # 37 | ################################################ 38 | delete_files() 39 | { 40 | # Checking that the provided value is not empty or it is a "dangerous" value. 41 | # We can not prevent anything, just a few of them. 42 | if [ -z "$1" ] || \ 43 | [ "$1" == "." ] || \ 44 | [ "$1" == ".." ] || \ 45 | [ "$1" == "/" ] || \ 46 | [ "$1" == "./" ] || \ 47 | [ "$1" == "../" ] || \ 48 | [ "$1" == "*" ] || \ 49 | [ "$1" == "./*" ] || \ 50 | [ "$1" == "../*" ]; then 51 | echo "Error: delete_files() does not accept \"$1\" as something to delete" >&2 52 | exit 1 53 | fi 54 | 55 | # Checking that the directory exists. Exiting as it is a development issue. 56 | if [ ! -z "$2" ]; then 57 | test -e "$1" || \ 58 | throw_error "The provided \"$1\" file or directory does not exist or is not valid." 59 | fi 60 | 61 | # Kill them all (ok, yes, we don't always require that options). 62 | rm -rf $1 63 | } 64 | 65 | ################################################ 66 | # Checks that the provided cmd commands are properly set. 67 | # 68 | ################################################ 69 | check_cmds() 70 | { 71 | local readonly genericstr=" has a valid value or overwrite the default one using webserver_config.properties" 72 | 73 | ${phpcmd} -v > /dev/null || \ 74 | throw_error 'Ensure $phpcmd'$genericstr 75 | 76 | # Only if mysql is being used. 77 | if [ "$dbtype" == "mysqli" ]; then 78 | ${mysqlcmd} -V > /dev/null || \ 79 | throw_error 'Ensure $mysqlcmd'$genericstr 80 | 81 | ${mysqldumpcmd} -V > /dev/null || \ 82 | throw_error 'Ensure $mysqldumpcmd'$genericstr 83 | fi 84 | 85 | # Only if pgsql is being used. 86 | if [ "$dbtype" == "pgsql" ]; then 87 | ${pgsqlcmd} --version > /dev/null || \ 88 | throw_error 'Ensure $pgsqlcmd'$genericstr 89 | 90 | ${pgsqldumpcmd} --version > /dev/null || \ 91 | throw_error 'Ensure $pgsqldumpcmd'$genericstr 92 | fi 93 | 94 | ${gitcmd} version > /dev/null || \ 95 | throw_error 'Ensure $gitcmd'$genericstr 96 | 97 | ${curlcmd} -V > /dev/null || \ 98 | throw_error 'Ensure $curlcmd'$genericstr 99 | } 100 | 101 | ################################################ 102 | # Loads configuration and static vars. 103 | # 104 | # Should be a first include before moving to other directories. 105 | # 106 | # For non-config files the caller script should check that the 107 | # file exists to provide a more acurate error message. 108 | # 109 | # Arguments: 110 | # $1 => The file to include 111 | # 112 | ################################################ 113 | load_properties() 114 | { 115 | # User configured properties. 116 | local configfile="./$1" 117 | if [ ! -r "$configfile" ]; then 118 | echo "Error: Properties file does not exist, copy $1.dist to $1 and edit the values according to your system" >&2 119 | exit 1 120 | fi 121 | . $configfile 122 | } 123 | 124 | ################################################ 125 | # Checks out the specified branch codebase for the specified repository 126 | # 127 | # Arguments: 128 | # $1 => repo 129 | # $2 => remote alias 130 | # $3 => branch 131 | # 132 | ################################################ 133 | checkout_branch() 134 | { 135 | 136 | # Getting the code. 137 | if [ ! -e ".git" ]; then 138 | ${gitcmd} init --quiet 139 | fi 140 | 141 | # Add/update the remote if necessary. 142 | local remotes="$( ${gitcmd} remote show )" 143 | if [[ "$remotes" == *$2* ]] || [ "$remotes" == "$2" ]; then 144 | 145 | # Remove the remote if it already exists and it is different. 146 | local remoteinfo="$( ${gitcmd} remote show "$2" -n | head -n 3 )" 147 | if [[ ! "$remoteinfo" == *$1* ]]; then 148 | ${gitcmd} remote rm $2 || \ 149 | throw_error "$1 remote value you provide can not be removed. Check webserver_config.properties.dist" 150 | ${gitcmd} remote add $2 $1 || \ 151 | throw_error "$1 remote value you provided can not be added as $2. Check webserver_config.properties.dist" 152 | fi 153 | # Add it if it is not there. 154 | else 155 | ${gitcmd} remote add $2 $1 || \ 156 | throw_error "$1 remote can not be added as $2. Check webserver_config.properties.dist" 157 | fi 158 | 159 | # Fetching from the repo. 160 | ${gitcmd} fetch $2 --quiet || \ 161 | throw_error "$2 remote can not be fetched. Check webserver_config.properties.dist" 162 | 163 | # Checking if it is a branch or a hash. 164 | local isareference="$( ${gitcmd} show-ref | grep "refs/remotes/$2/$3$" | wc -l )" 165 | if [ "$isareference" == "1" ]; then 166 | 167 | # Checkout the last version of the branch. 168 | # Reset to avoid conflicts if there are git history changes. 169 | ${gitcmd} checkout -B $3 $2/$3 --quiet || \ 170 | throw_error "The '$3' tag or branch you provided does not exist or it is not set. Check webserver_config.properties.dist" 171 | 172 | else 173 | # Just checkout the hash and let if fail if it is incorrect. 174 | ${gitcmd} checkout $3 --quiet || \ 175 | throw_error "The '$3' hash you provided does not exist or it is not set. Check webserver_config.properties.dist" 176 | 177 | fi 178 | 179 | } 180 | 181 | ################################################ 182 | # Shows the time elapsed in hours, mins and secs. 183 | # 184 | # Arguments: 185 | # $1 => Number of seconds 186 | # 187 | ################################################ 188 | show_elapsed_time() 189 | { 190 | local h=$((${1}/3600)) 191 | local m=$(((${1}%3600)/60)) 192 | local s=$((${1}%60)) 193 | printf "Elapsed time: %02d:%02d:%02d\n" $h $m $s 194 | } 195 | 196 | ################################################ 197 | # Creates a file with data about the site. 198 | # 199 | # Requires scripts to move to moodle/ before 200 | # calling it and returning to root if necessary. 201 | # 202 | ################################################ 203 | save_moodle_site_data() 204 | { 205 | 206 | # We should already be in moodle/. 207 | if [ ! -f "version.php" ]; then 208 | echo "Error: save_moodle_site_data() should only be called after cd to moodle/" >&2 209 | exit 1 210 | fi 211 | 212 | # Getting the current site data. 213 | local siteversion="$(cat version.php | \ 214 | grep '$version' | \ 215 | grep -o '[0-9]\+.[0-9]\+' | \ 216 | head -n 1)" 217 | local sitebranch="$(cat version.php | \ 218 | grep '$branch' | \ 219 | grep -o '[0-9]\+' | \ 220 | head -n 1)" 221 | local sitecommit="$(${gitcmd} show --oneline | \ 222 | head -n 1 | \ 223 | sed 's/\"/\\"/g')" 224 | 225 | local sitedatacontents="siteversion=\"$siteversion\" 226 | sitebranch=\"$sitebranch\" 227 | sitecommit=\"$sitecommit\"" 228 | 229 | echo "${sitedatacontents}" > site_data.properties || \ 230 | throw_error "Site data properties file can not be written, check $currentwd/moodle directory permissions." 231 | } 232 | -------------------------------------------------------------------------------- /logs/empty: -------------------------------------------------------------------------------- 1 | This directory will store jmeter log files. 2 | -------------------------------------------------------------------------------- /recorder.bsf: -------------------------------------------------------------------------------- 1 | import java.io.*; 2 | import java.util.regex.*; 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.apache.jmeter.util.JMeterUtils; // http://jakarta.apache.org/jmeter/api/org/apache/jmeter/util/JMeterUtils.html 5 | import org.apache.jmeter.threads.JMeterContext; // http://jakarta.apache.org/jmeter/api/org/apache/jmeter/threads/JMeterContext.html 6 | import org.apache.jmeter.samplers.SampleResult; // http://jakarta.apache.org/jmeter/api/org/apache/jmeter/samplers/SampleResult.html 7 | 8 | MoodleResult(JMeterContext ctx) { 9 | 10 | Integer thread = ctx.getThreadNum(); 11 | SampleResult result = ctx.getPreviousResult(); 12 | 13 | String html = result.getResponseDataAsString(); 14 | 15 | String dbreads = "0"; 16 | Pattern pdbreads = Pattern.compile(".*?DB reads/writes: (\\d+)/\\d+.*", Pattern.UNIX_LINES | Pattern.DOTALL); 17 | Matcher mdbreads = pdbreads.matcher(html); 18 | if (mdbreads.matches()) { 19 | dbreads = mdbreads.group(1); 20 | } 21 | 22 | String dbwritesstr = "0"; 23 | Pattern pdbwrites = Pattern.compile(".*?DB reads/writes: \\d+/(\\d+).*", Pattern.UNIX_LINES | Pattern.DOTALL); 24 | Matcher mdbwrites = pdbwrites.matcher(html); 25 | if (mdbwrites.matches()) { 26 | dbwritesstr = mdbwrites.group(1); 27 | } 28 | Integer dbwrites = Integer.parseInt(dbwritesstr); 29 | 30 | // Adding logs if required. 31 | if (props.get("includelogs") != null) { 32 | Pattern plogwrites = Pattern.compile(".*?Log DB writes (\\d+).*", Pattern.UNIX_LINES | Pattern.DOTALL); 33 | Matcher mlogwrites = plogwrites.matcher(html); 34 | if (mlogwrites.matches()) { 35 | dbwrites = dbwrites + Integer.parseInt(mlogwrites.group(1)); 36 | } 37 | } 38 | 39 | String dbquerytime = "0"; 40 | Pattern pdbquerytime = Pattern.compile(".*?DB queries time: (\\d+(\\.\\d+)?) secs.*", Pattern.UNIX_LINES | Pattern.DOTALL); 41 | Matcher mdbquerytime = pdbquerytime.matcher(html); 42 | if (mdbquerytime.matches()) { 43 | dbquerytime = mdbquerytime.group(1); 44 | } 45 | 46 | String memoryused = "0"; 47 | Pattern pmemoryused = Pattern.compile(".*?RAM: (\\d+(\\.\\d+)?)[^M]*MB.*", Pattern.UNIX_LINES | Pattern.DOTALL); 48 | Matcher mmemoryused = pmemoryused.matcher(html); 49 | if (mmemoryused.matches()) { 50 | memoryused = mmemoryused.group(1); 51 | } 52 | 53 | String filesincluded = "0"; 54 | Pattern pfilesincluded = Pattern.compile(".*?Included (\\d+) files.*", Pattern.UNIX_LINES | Pattern.DOTALL); 55 | Matcher mfilesincluded = pfilesincluded.matcher(html); 56 | if (mfilesincluded.matches()) { 57 | filesincluded = mfilesincluded.group(1); 58 | } 59 | 60 | String serverload = "0"; 61 | Pattern pserverload = Pattern.compile(".*?Load average: (\\d+(\\.\\d+)?).*", Pattern.UNIX_LINES | Pattern.DOTALL); 62 | Matcher mserverload = pserverload.matcher(html); 63 | if (mserverload.matches()) { 64 | serverload = mserverload.group(1); 65 | } 66 | 67 | String sessionsize = "0"; 68 | Pattern psessionsize = Pattern.compile(".*?Session[^:]*: (\\d+(\\.\\d+)? ?[a-zA-Z]*).*", Pattern.UNIX_LINES | Pattern.DOTALL); 69 | Matcher msessionsize = psessionsize.matcher(html); 70 | if (msessionsize.matches()) { 71 | sessionsize = msessionsize.group(1); 72 | } 73 | 74 | String timeused = "0"; 75 | Pattern ptimeused = Pattern.compile(".*?\"timeused[^\"]*\">(\\d+(\\.\\d+)?) secs.*", Pattern.UNIX_LINES | Pattern.DOTALL); 76 | Matcher mtimeused = ptimeused.matcher(html); 77 | if (mtimeused.matches()) { 78 | timeused = mtimeused.group(1); 79 | } 80 | 81 | // Actual information collected about the sample by jmeter 82 | String username = vars.get("username"); 83 | String name = StringUtils.rightPad(result.getSampleLabel(), 30); 84 | String url = result.getUrlAsString(); 85 | Integer bytes = result.getBytes(); 86 | Long time = result.getTime(); 87 | Long latency = result.getLatency(); 88 | Long starttime = result.getStartTime(); 89 | String status = result.getResponseCode(); 90 | 91 | headerToString() { 92 | String str = "status | thread | "; 93 | str += StringUtils.rightPad("user", 10) + " | "; 94 | str += StringUtils.rightPad("name", 30) + " | db-r | db-w | "; 95 | str += StringUtils.rightPad("dbquerytime", 8) + " | "; 96 | str += StringUtils.rightPad("memory", 8) + " | "; 97 | str += StringUtils.rightPad("files", 6) + " | "; 98 | str += StringUtils.rightPad("load", 6) + " |"; 99 | return str; 100 | } 101 | 102 | toString() { 103 | String str = StringUtils.rightPad(status, 6) + " | "; 104 | str += StringUtils.rightPad(Integer.toString(thread), 6) + " | "; 105 | str += StringUtils.rightPad(username, 10) + " | "; 106 | str += StringUtils.rightPad(name, 30) + " | "; 107 | str += StringUtils.rightPad(dbreads, 4) + " | "; 108 | str += StringUtils.rightPad(Integer.toString(dbwrites), 4) + " | "; 109 | str += StringUtils.rightPad(dbquerytime, 8) + " | "; 110 | str += StringUtils.rightPad(memoryused, 8) + " | "; 111 | str += StringUtils.rightPad(filesincluded, 6) + " | "; 112 | str += StringUtils.rightPad(serverload, 6) + " | "; 113 | str += url; 114 | return str; 115 | } 116 | 117 | toPHP() { 118 | 119 | int bytesPos = sessionsize.indexOf(" bytes"); 120 | int kbsPos = sessionsize.indexOf("KB"); 121 | // Convert the size to KB and strip out the measure. 122 | if (bytesPos != -1) { 123 | sessionsize = "0." + sessionsize.substring(0, bytesPos); 124 | } else if (kbsPos != -1) { 125 | sessionsize = sessionsize.substring(0, kbsPos); 126 | } 127 | 128 | String php = "$results["+thread+"][] = array(\n"; 129 | php += " 'thread'=>"+thread+",\n"; // Int 130 | php += " 'starttime'=>"+starttime+",\n"; // Long 131 | php += " 'dbreads'=>"+Integer.parseInt(dbreads)+",\n"; // String => Int 132 | php += " 'dbwrites'=>"+dbwrites+",\n"; 133 | php += " 'dbquerytime'=>"+dbquerytime+",\n"; 134 | php += " 'memoryused'=>'"+memoryused+"',\n"; 135 | php += " 'filesincluded'=>'"+filesincluded+"',\n"; 136 | php += " 'serverload'=>'"+serverload+"',\n"; 137 | php += " 'sessionsize'=>'"+sessionsize+"',\n"; 138 | php += " 'timeused'=>'"+timeused+"',\n"; 139 | php += " 'name'=>'"+name+"',\n"; 140 | php += " 'url'=>'"+url+"',\n"; 141 | php += " 'bytes'=>'"+bytes+"',\n"; 142 | php += " 'time'=>'"+time+"',\n"; 143 | php += " 'latency'=>'"+latency+"',\n"; 144 | php += ");\n"; 145 | return php; 146 | } 147 | 148 | return this; 149 | } 150 | 151 | EscapeQuotes(String text) { 152 | return text.replace("'", "\\'"); 153 | } 154 | 155 | Runnable mr = MoodleResult(ctx); 156 | 157 | // Get the file (it is created in testStarted). 158 | String filenamepath = "runs/tmpfilename.php"; 159 | 160 | // We add the run info when starting the first thread 161 | if (JMeterUtils.getProperty("headerprinted") == null) { 162 | 163 | // Output headers. 164 | JMeterUtils.setProperty("headerprinted", "1"); 165 | print(mr.headerToString()); 166 | 167 | FileWriter fstream = new FileWriter(filenamepath, true); 168 | BufferedWriter out = new BufferedWriter(fstream); 169 | out.write("$host = '"+vars.get("host")+"';\n"); 170 | out.write("$sitepath = '"+vars.get("sitepath")+"';\n"); 171 | out.write("$group = '"+EscapeQuotes(props.get("group"))+"';\n"); 172 | out.write("$rundesc = '"+EscapeQuotes(props.get("desc"))+"';\n"); 173 | out.write("$users = '"+vars.get("users")+"';\n"); 174 | out.write("$loopcount = '"+vars.get("loops")+"';\n"); 175 | out.write("$rampup = '"+vars.get("rampup")+"';\n"); 176 | out.write("$throughput = '"+vars.get("throughput")+"';\n"); 177 | out.write("$size = '"+vars.get("size")+"';\n"); 178 | out.write("$baseversion = '"+vars.get("moodleversion")+"';\n"); 179 | out.write("$siteversion = '"+EscapeQuotes(props.get("siteversion"))+"';\n"); 180 | out.write("$sitebranch = '"+EscapeQuotes(props.get("sitebranch"))+"';\n"); 181 | out.write("$sitecommit = '"+EscapeQuotes(props.get("sitecommit"))+"';\n"); 182 | out.close(); 183 | 184 | // Send the run timestamp to set it as run filename. 185 | props.put("filepath", "runs/" + vars.get("runtimestamp") + ".php"); 186 | } 187 | 188 | FileWriter fstream = new FileWriter(filenamepath, true); 189 | BufferedWriter out = new BufferedWriter(fstream); 190 | out.write(mr.toPHP()); 191 | out.close(); 192 | 193 | print(mr.toString()); 194 | -------------------------------------------------------------------------------- /recorderfunctions.bsf: -------------------------------------------------------------------------------- 1 | import java.io.*; 2 | import org.apache.commons.lang3.StringUtils; 3 | import org.apache.jmeter.util.JMeterUtils; // http://jakarta.apache.org/jmeter/api/org/apache/jmeter/util/JMeterUtils.html 4 | 5 | 6 | testStarted(){ 7 | String filenamepath = "runs/tmpfilename.php"; 8 | FileWriter fstream = new FileWriter(filenamepath, true); 9 | BufferedWriter out = new BufferedWriter(fstream); 10 | out.write("&2 32 | exit 1 33 | fi 34 | 35 | # Stop all the services 36 | for service in "${servicesarray[@]}"; do 37 | service $service stop 38 | done 39 | 40 | # Start them again. 41 | for service in "${servicesarray[@]}"; do 42 | service $service start 43 | done 44 | 45 | outputinfo=" 46 | ####################################################################### 47 | Services restarted successfully. 48 | 49 | Now you can begin running the tests with test_runner.sh. 50 | " 51 | echo "$outputinfo" 52 | exit 0 53 | -------------------------------------------------------------------------------- /runs/empty: -------------------------------------------------------------------------------- 1 | This directory will store the jmeter runs results. 2 | -------------------------------------------------------------------------------- /runs_outputs/empty: -------------------------------------------------------------------------------- 1 | This directory will store the jmeter runs outputs showing all the requests. 2 | -------------------------------------------------------------------------------- /runs_samples/empty: -------------------------------------------------------------------------------- 1 | This directory stores the test plan runs samples. 2 | -------------------------------------------------------------------------------- /set_moodle_site.php: -------------------------------------------------------------------------------- 1 | get_record('user', array('username' => 'admin')); 25 | $user->email = 'moodle@moodlemoodle.com'; 26 | $user->firstname = 'Admin'; 27 | $user->lastname = 'User'; 28 | $user->city = 'Perth'; 29 | $user->country = 'AU'; 30 | $DB->update_record('user', $user); 31 | 32 | // Disable email message processor. 33 | $DB->set_field('message_processors', 'enabled', '0', array('name' => 'email')); 34 | 35 | // Course list when not logged in and enrolled courses when logged as a usual configuration. 36 | $frontpage = new admin_setting_courselist_frontpage(false); 37 | $frontpage->write_setting(array(FRONTPAGEALLCOURSELIST)); 38 | $frontpagelogged = new admin_setting_courselist_frontpage(true); 39 | $frontpagelogged->write_setting(array(FRONTPAGEENROLLEDCOURSELIST)); 40 | 41 | echo "Moodle site configuration finished successfully.\n"; 42 | -------------------------------------------------------------------------------- /test_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Script to run the test plan using jmeter 4 | # 5 | # Runs will be grouped according to $1 so they 6 | # can be compared easily. The run description 7 | # will be useful to identify them. 8 | # 9 | # Usage: 10 | # cd /path/to/moodle-performance-comparison 11 | # ./test_runner.sh [OPTIONS] {run_group_name} {run_description} {test_plan_file_path} {users_file_path} 12 | # 13 | # Arguments: 14 | # * $1 => The run group name, there will be comparision graphs by this group name 15 | # * $2 => The run description, useful to identify the changes between runs. 16 | # * $3 => The test plan file path 17 | # * $4 => The path to the file with user's login data 18 | # * $5 => The path to the file with the tested site data 19 | # 20 | # Options: 21 | # * -u => Force the number of users (threads) 22 | # * -l => Force the number of loops 23 | # * -r => Force the ramp-up period 24 | # * -t => Force the throughput 25 | # 26 | ############################################## 27 | 28 | # Exit on errors. 29 | set -e 30 | 31 | # Dependencies. 32 | . ./lib/lib.sh 33 | 34 | # Load properties. 35 | load_properties "defaults.properties" 36 | load_properties "jmeter_config.properties" 37 | 38 | # Load the generated files locations 39 | # (when jmeter is running in the same server than the web server). 40 | if [ -e "test_files.properties" ]; then 41 | load_properties "test_files.properties" 42 | fi 43 | 44 | if [ "$#" -lt 2 ]; then 45 | echo "Error: Not enough arguments. Open test_runner.sh for more details." >&2 46 | exit 1 47 | fi 48 | 49 | # Getting jmeter custom options. 50 | while [ $# -gt 0 ]; do 51 | case "$1" in 52 | -u) 53 | users=" -Jusers=$2" 54 | shift 2 55 | ;; 56 | -l) 57 | loops=" -Jloops=$2" 58 | shift 2 59 | ;; 60 | -r) 61 | rampup=" -Jrampup=$2" 62 | shift 2 63 | ;; 64 | -t) 65 | throughput=" -Jthroughput=$2" 66 | shift 2 67 | ;; 68 | *) 69 | # Wrong argument; True... we don't support "-[a-zA-Z] arguments. 70 | if [ "${1:0:1}" == "-" ]; then 71 | echo "Error: Unsupported option $1" 72 | exit 1 73 | fi 74 | 75 | if [ -z "$group" ] && [ "${1:0:1}" != "-" ]; then 76 | group=$1 77 | shift 78 | fi 79 | 80 | if [ -z "$description" ] && [ "${1:0:1}" != "-" ]; then 81 | description=$1 82 | shift 83 | fi 84 | 85 | # When JMeter is in another server we need extra files. 86 | if [ ! -z "$1" ] && [ -z "$testplanarg" ] && [ "${1:0:1}" != "-" ]; then 87 | testplanarg=$1 88 | shift 89 | fi 90 | 91 | if [ ! -z "$1" ] && [ -z "$testusersfilearg" ] && [ "${1:0:1}" != "-" ]; then 92 | testusersfilearg=$1 93 | shift 94 | fi 95 | 96 | if [ ! -z "$1" ] && [ -z "$sitedatafilearg" ] && [ "${1:0:1}" != "-" ]; then 97 | sitedatafilearg=$1 98 | shift 99 | fi 100 | ;; 101 | esac 102 | done 103 | 104 | # We give priority to the ones that comes as arguments. 105 | if [ ! -z "$testplanarg" ]; then 106 | testplanfile=$testplanarg 107 | fi 108 | if [ ! -z "$testusersfilearg" ]; then 109 | testusersfile=$testusersfilearg 110 | fi 111 | 112 | # Load the site data. 113 | # Defaults when jmeter is running in the same server than the web server. 114 | sitedatafile="moodle/site_data.properties" 115 | if [ ! -z "$sitedatafilearg" ]; then 116 | sitedatafile=$sitedatafilearg 117 | fi 118 | if [ ! -e "$sitedatafile" ]; then 119 | echo "Error: The specified site data properties file does not exist." >&2 120 | exit 1 121 | fi 122 | load_properties $sitedatafile 123 | 124 | 125 | # If there is no test_files.properties and no files were provided we throw an error. 126 | if [ -z "$testplanfile" ] || [ -z "$testusersfile" ]; then 127 | echo "Usage: `basename $0` {run_group} {run_description} {test_plan_file_path} {users_file_path}" >&2 128 | exit 1 129 | fi 130 | 131 | # Creating the results cache directory for images. 132 | if [ ! -d "cache" ]; then 133 | mkdir -m 777 "cache" 134 | else 135 | chmod 777 "cache" 136 | fi 137 | 138 | # Uses the test plan specified in the CLI call. 139 | datestring=`date '+%Y%m%d%H%M'` 140 | logfile="logs/jmeter.$datestring.log" 141 | runoutput="runs_outputs/$datestring.output" 142 | 143 | # Include logs string. 144 | includelogsstr="-Jincludelogs=$includelogs" 145 | samplerinitstr="-Jbeanshell.listener.init=recorderfunctions.bsf" 146 | 147 | # Run it baby! (without GUI). 148 | echo "#######################################################################" 149 | echo "Test running... (time for a coffee?)" 150 | 151 | jmetererrormsg="Jmeter can not run, ensure that: 152 | * The test plan and the users files are ok 153 | * You provide correct arguments to the script" 154 | 155 | jmeterbin=${jmeter_path%/}/bin/jmeter 156 | $jmeterbin \ 157 | -n \ 158 | -j "$logfile" \ 159 | -t "$testplanfile" \ 160 | -Jusersfile="$testusersfile" \ 161 | -Jgroup="$group" \ 162 | -Jdesc="$description" \ 163 | -Jsiteversion="$siteversion" \ 164 | -Jsitebranch="$sitebranch" \ 165 | -Jsitecommit="$sitecommit" \ 166 | $samplerinitstr \ 167 | $includelogsstr \ 168 | $users \ 169 | $loops \ 170 | $rampup \ 171 | $throughput \ 172 | > $runoutput || \ 173 | throw_error $jmetererrormsg 174 | 175 | # Log file correctly generated. 176 | if [ ! -f $logfile ]; then 177 | echo "Error: JMeter has not generated any log file in $logfile" 178 | exit 1 179 | fi 180 | 181 | # Grep the logs looking for errors and warnings. 182 | for errorkey in ERROR WARN; do 183 | 184 | # Also checking that the errorkey is the log entry type. 185 | if grep $errorkey $logfile | awk '{print $3}' | grep -q $errorkey ; then 186 | 187 | echo "Error: \"$errorkey\" found in jmeter logs, read $logfile \ 188 | to see the full trace. If you think that this is a false failure report the \ 189 | issue in https://github.com/moodlehq/moodle-performance-comparison/issues." 190 | exit 1 191 | fi 192 | done 193 | 194 | outputinfo=" 195 | ####################################################################### 196 | Test plan completed successfully. 197 | 198 | To compare this run with others remember to execute after_run_setup.sh before 199 | it to clean the site restoring the database and the dataroot. 200 | " 201 | echo "$outputinfo" 202 | exit 0 203 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## 4 | # Tests the intial setup of the tool to avoid big problems. 5 | # 6 | # Those tests are fragile, but they are better than nothing. 7 | # 8 | # Note that this script should run just after a checkout 9 | # as it is supposed to begin from a clear base and it's 10 | # purpose is to test the tool setup. 11 | # 12 | # Usage: 13 | # cd tests/ 14 | # ./test.sh 15 | ## 16 | 17 | # Cleans the site data after running this script. 18 | clean_test() 19 | { 20 | rm webserver_config.properties 21 | rm jmeter_config.properties 22 | 23 | if [ ! -z "$cwd" ] && [ -d "$cwd/testdataroot" ]; then 24 | rm -rf "$cwd/testdataroot" 25 | fi 26 | } 27 | 28 | # Looks for the expected string in the command output. 29 | # $1 => command, $2 => expected output substring, $3 => error msg. 30 | check_output() 31 | { 32 | output=$( $1 ) 33 | if [[ "$output" != *"$2"* ]]; then 34 | echo "Test failed: "$output >&2 35 | echo $3 >&2 36 | 37 | # Clean the tool. 38 | clean_test 39 | 40 | exit 1 41 | fi 42 | } 43 | 44 | ################################################################ 45 | 46 | # Not set -e here. 47 | 48 | # Hardcoded values ######### 49 | cwd=`pwd` 50 | 51 | # Move to the tool dir. 52 | cd .. 53 | 54 | # Ensure that the tool is not set up. This should be a framework level failure. 55 | if [ -f "moodle" ] || [ -f "webserver_config.properties" ] || [ -f "jmeter_config.properties" ]; then 56 | echo "Error: The tool has been previously used or initialized, checkout a new clone and run the tests there." >&2 57 | exit 1 58 | fi 59 | 60 | 61 | # We should receive an error when trying to run the tool without config file. 62 | check_output "./compare.sh" "Properties file does not exist" "The tool should not work without configuration files" 63 | 64 | # We should receive errors when we just copy the config files. 65 | cp webserver_config.properties.dist webserver_config.properties 66 | cp jmeter_config.properties.dist jmeter_config.properties 67 | check_output "./compare.sh" "/your/dataroot/directory" "/your/dataroot/directory should not be created" 68 | 69 | clean_test 70 | 71 | # Return to tests/ just in case. 72 | cd tests 73 | 74 | echo "All tests passed :)" 75 | exit 0 76 | -------------------------------------------------------------------------------- /webapp/DejaVuSans.license: -------------------------------------------------------------------------------- 1 | Fonts are (c) Bitstream (see below). DejaVu changes are in public domain. 2 | Glyphs imported from Arev fonts are (c) Tavmjong Bah (see below) 3 | 4 | Bitstream Vera Fonts Copyright 5 | ------------------------------ 6 | 7 | Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Bitstream Vera is 8 | a trademark of Bitstream, Inc. 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of the fonts accompanying this license ("Fonts") and associated 12 | documentation files (the "Font Software"), to reproduce and distribute the 13 | Font Software, including without limitation the rights to use, copy, merge, 14 | publish, distribute, and/or sell copies of the Font Software, and to permit 15 | persons to whom the Font Software is furnished to do so, subject to the 16 | following conditions: 17 | 18 | The above copyright and trademark notices and this permission notice shall 19 | be included in all copies of one or more of the Font Software typefaces. 20 | 21 | The Font Software may be modified, altered, or added to, and in particular 22 | the designs of glyphs or characters in the Fonts may be modified and 23 | additional glyphs or characters may be added to the Fonts, only if the fonts 24 | are renamed to names not containing either the words "Bitstream" or the word 25 | "Vera". 26 | 27 | This License becomes null and void to the extent applicable to Fonts or Font 28 | Software that has been modified and is distributed under the "Bitstream 29 | Vera" names. 30 | 31 | The Font Software may be sold as part of a larger software package but no 32 | copy of one or more of the Font Software typefaces may be sold by itself. 33 | 34 | THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 35 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, 36 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, 37 | TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL BITSTREAM OR THE GNOME 38 | FOUNDATION BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, INCLUDING 39 | ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, 40 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF 41 | THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE 42 | FONT SOFTWARE. 43 | 44 | Except as contained in this notice, the names of Gnome, the Gnome 45 | Foundation, and Bitstream Inc., shall not be used in advertising or 46 | otherwise to promote the sale, use or other dealings in this Font Software 47 | without prior written authorization from the Gnome Foundation or Bitstream 48 | Inc., respectively. For further information, contact: fonts at gnome dot 49 | org. 50 | 51 | Arev Fonts Copyright 52 | ------------------------------ 53 | 54 | Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. 55 | 56 | Permission is hereby granted, free of charge, to any person obtaining 57 | a copy of the fonts accompanying this license ("Fonts") and 58 | associated documentation files (the "Font Software"), to reproduce 59 | and distribute the modifications to the Bitstream Vera Font Software, 60 | including without limitation the rights to use, copy, merge, publish, 61 | distribute, and/or sell copies of the Font Software, and to permit 62 | persons to whom the Font Software is furnished to do so, subject to 63 | the following conditions: 64 | 65 | The above copyright and trademark notices and this permission notice 66 | shall be included in all copies of one or more of the Font Software 67 | typefaces. 68 | 69 | The Font Software may be modified, altered, or added to, and in 70 | particular the designs of glyphs or characters in the Fonts may be 71 | modified and additional glyphs or characters may be added to the 72 | Fonts, only if the fonts are renamed to names not containing either 73 | the words "Tavmjong Bah" or the word "Arev". 74 | 75 | This License becomes null and void to the extent applicable to Fonts 76 | or Font Software that has been modified and is distributed under the 77 | "Tavmjong Bah Arev" names. 78 | 79 | The Font Software may be sold as part of a larger software package but 80 | no copy of one or more of the Font Software typefaces may be sold by 81 | itself. 82 | 83 | THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 84 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF 85 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 86 | OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL 87 | TAVMJONG BAH BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 88 | INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 89 | DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 90 | FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM 91 | OTHER DEALINGS IN THE FONT SOFTWARE. 92 | 93 | Except as contained in this notice, the name of Tavmjong Bah shall not 94 | be used in advertising or otherwise to promote the sale, use or other 95 | dealings in this Font Software without prior written authorization 96 | from Tavmjong Bah. For further information, contact: tavmjong @ free 97 | . fr. 98 | 99 | $Id: LICENSE 2133 2007-11-28 02:46:28Z lechimp $ 100 | -------------------------------------------------------------------------------- /webapp/DejaVuSans.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moodlehq/moodle-performance-comparison/e2a26e5829cb359752dd49b5d332dd18ef957228/webapp/DejaVuSans.ttf -------------------------------------------------------------------------------- /webapp/classes/google_chart.php: -------------------------------------------------------------------------------- 1 | chartid = $chartid; 51 | $this->charttype = $charttype; 52 | $this->dataset = $data; 53 | $this->chartoptions = $chartoptions; 54 | } 55 | 56 | /** 57 | * Returns the generated Javascript to display the chart. 58 | * 59 | * @return string The generated JS. 60 | */ 61 | public function output_js() { 62 | 63 | $output = ''; 64 | 65 | // Chart data set. 66 | $output .= 'var data = google.visualization.arrayToDataTable([' . PHP_EOL; 67 | foreach ($this->dataset as $row) { 68 | $output .= '['; 69 | 70 | // Passing to JS strings or numbers. 71 | foreach ($row as $key => $value) { 72 | if (is_int($value) || is_float($value)) { 73 | $row[$key] = $value; 74 | } else { 75 | $row[$key] = "'" . addslashes($value) . "'"; 76 | } 77 | } 78 | $output .= implode(", ", $row); 79 | $output .= '],' . PHP_EOL; 80 | } 81 | $output .= ']);' . PHP_EOL; 82 | 83 | // Chart options. 84 | if ($this->chartoptions) { 85 | $output .= "var options = " . json_encode($this->chartoptions) . ";"; 86 | } else { 87 | $output .= "var options = null;"; 88 | } 89 | $output .= PHP_EOL; 90 | 91 | // Draw the chart. 92 | $output .= "var chart = new google.visualization.{$this->charttype}(document.getElementById('{$this->chartid}'));" . PHP_EOL . 93 | "chart.draw(data, options);" . PHP_EOL . PHP_EOL; 94 | 95 | return $output; 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /webapp/classes/google_charts_renderer.php: -------------------------------------------------------------------------------- 1 | '; 38 | 39 | if (self::$charts) { 40 | echo self::load_google_chart_api(); 41 | echo self::create_onload_callback(); 42 | } 43 | 44 | echo ''; 45 | } 46 | 47 | /** 48 | * Returns the JS Google libs. 49 | * 50 | * @return string The JS includes 51 | */ 52 | protected static function load_google_chart_api() { 53 | 54 | return '' . PHP_EOL . 55 | '' . PHP_EOL; 58 | } 59 | 60 | /** 61 | * Returns the callback function code. 62 | * 63 | * @return string Callback function contents including all charts. 64 | */ 65 | protected static function create_onload_callback() { 66 | 67 | $output = '' . PHP_EOL; 77 | 78 | return $output; 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /webapp/classes/properties_reader.php: -------------------------------------------------------------------------------- 1 | chartsdeclaration = array( 56 | 'vars_bar' => array( 57 | 'id' => 'vars_bar', 58 | 'name' => 'Comparing variables step by step', 59 | 'class' => 'BarChart', 60 | 'orientation' => 'steporienteddataset', 61 | 'perrow' => 3, 62 | 'height' => 400, 63 | 'width' => 500, 64 | ), 65 | 'vars_area' => array( 66 | 'id' => 'vars_area', 67 | 'name' => 'Comparing variables step by step', 68 | 'class' => 'AreaChart', 69 | 'orientation' => 'steporienteddataset', 70 | 'perrow' => 3, 71 | 'height' => 400, 72 | 'width' => 500, 73 | ), 74 | 'grouped_steppedarea' => array( 75 | 'id' => 'grouped_steppedarea', 76 | 'name' => 'Grouped steps', 77 | 'class' => 'SteppedAreaChart', 78 | 'orientation' => 'runorienteddataset', 79 | 'perrow' => 2, 80 | 'height' => 500, 81 | 'width' => 600, 82 | ) 83 | ); 84 | 85 | // Init the containers array. 86 | foreach ($this->chartsdeclaration as $chartid => $chartdata) { 87 | $this->containers[$chartid] = array(); 88 | } 89 | 90 | } 91 | 92 | /** 93 | * Gets the runs data 94 | * 95 | * @param array $timestamps We will get the runs files from their timestamp (is part of the name). 96 | * @param bool $normalize We want to normalize clear outliers. 97 | * @return bool Whether runs are comparable or not. 98 | */ 99 | public function parse_runs(array $timestamps, bool $normalize = false) { 100 | 101 | foreach ($timestamps as $timestamp) { 102 | 103 | if (!is_numeric($timestamp)) { 104 | die('Error: Timestamps are supposed to be [0-9]' . PHP_EOL); 105 | } 106 | 107 | // Creating the run object and parsing it. 108 | $run = new test_plan_run($timestamp, $normalize); 109 | $run->parse_results(); 110 | $this->runs[] = $run; 111 | } 112 | 113 | // Stop when runs are not comparables between them. 114 | if (!$this->check_runs_are_comparable()) { 115 | return false; 116 | } 117 | 118 | return true; 119 | } 120 | 121 | /** 122 | * Generates the report 123 | * 124 | * @param array $timestamps We will get the runs files from their timestamp (is part of the name). 125 | * @param bool $normalize We want to normalize clear outliers. 126 | * @return bool False if problems were found. 127 | */ 128 | public function make(array $timestamps, $normalize = false) { 129 | 130 | // They come from the form in the opposite order. 131 | krsort($timestamps); 132 | 133 | // Gets the runs data and checks that it is comparable. 134 | if (!$this->parse_runs($timestamps, $normalize)) { 135 | // No need to parse anything if it is not comparable. 136 | return false; 137 | } 138 | // Will be used to get runs generic data like the steps names, they are supposed to be 139 | // the same in all the runs, the UI should restrict the comparisons to comparable runs. 140 | $genericrun = & $this->runs[0]; 141 | 142 | // Generating the data arrays. 143 | $vars = $this->runs[0]->get_run_var_names(); 144 | foreach ($vars as $var) { 145 | 146 | // TODO: Do something with the raw data. 147 | $sums = array(); 148 | $avg = array(); 149 | foreach ($this->runs as $runkey => $run) { 150 | list($sum, $raw, $average) = $run->get_run_dataset($var); 151 | $sums[$runkey] = $sum; 152 | $avg[$runkey] = $average; 153 | } 154 | 155 | // Getting all the data, ready for step-oriented charts and branch (run) oriented. 156 | $steporienteddataset = array(); 157 | $runorienteddataset = array(); 158 | 159 | $steporienteddataset[0] = array('Step'); 160 | $runorienteddataset[0] = array('Run'); 161 | foreach ($this->runs as $key => $run) { 162 | 163 | // Step-oriented dataset includes headers with runs info. 164 | // We init the headers here. 165 | $steporienteddataset[0][] = $run->get_run_info_string(); 166 | 167 | // The header is also there. 168 | $datasetkey = $key + 1; 169 | 170 | $runorienteddataset[$datasetkey] = array($run->get_run_info_string()); 171 | // And now we add the multiple runs data. 172 | foreach ($genericrun->get_run_steps() as $stepkey => $step) { 173 | $runorienteddataset[$datasetkey][] = $avg[$key][$step]; 174 | } 175 | 176 | } 177 | 178 | // Real runs data. 179 | foreach ($genericrun->get_run_steps() as $key => $step) { 180 | 181 | // Runs-oriented dataset includes headers with steps info 182 | // We init the headers here. 183 | $runorienteddataset[0][] = $step; 184 | 185 | // The header is also there. 186 | $datasetkey = $key + 1; 187 | 188 | $steporienteddataset[$datasetkey] = array($step); 189 | // And now we add the multiple runs data. 190 | foreach ($this->runs as $runkey => $run) { 191 | $steporienteddataset[$datasetkey][] = $avg[$runkey][$step]; 192 | } 193 | } 194 | 195 | $this->create_charts($var, $steporienteddataset, $runorienteddataset); 196 | } 197 | 198 | // We calculate differences between runs to list them. 199 | $this->calculate_big_differences(); 200 | 201 | return true; 202 | } 203 | 204 | /** 205 | * Returns the runs info. 206 | * 207 | * @return array 208 | */ 209 | public function get_run_files_info() { 210 | 211 | $runfiles = array(); 212 | $runsvalues = array(); 213 | 214 | $dir = __DIR__ . '/../../' . self::RUNS_RELATIVE_PATH; 215 | if ($dh = opendir($dir)) { 216 | while (($filename = readdir($dh)) !== false) { 217 | 218 | // We only want the run files that are ready. 219 | if ($filename != '.' && $filename != '..' && 220 | $filename != 'empty' && $filename != 'tmpfilename.php') { 221 | 222 | // Verify the file is ok. 223 | $line = fgets(fopen("$dir/$filename", 'r')); 224 | if (strpos($line, ' $name) { 233 | 234 | // In case some runs misses vars. 235 | if (!empty($runfiles[$timestamp]->get_run_info()->{$param})) { 236 | $value = $runfiles[$timestamp]->get_run_info()->{$param}; 237 | } else { 238 | $value = 'Unknown'; 239 | } 240 | 241 | if (empty($runsvalues[$param])) { 242 | $runsvalues[$param] = array(); 243 | } 244 | $runsvalues[$param][$value] = $value; 245 | } 246 | 247 | // Discard it if filters are set (once we got it's params). 248 | if (!empty($_GET['filters'])) { 249 | foreach ($_GET['filters'] as $param => $filteredvalue) { 250 | // In case some runs misses vars. 251 | if (!empty($runfiles[$timestamp]->get_run_info()->{$param})) { 252 | $runvar = $runfiles[$timestamp]->get_run_info()->{$param}; 253 | } else { 254 | $runvar = 'Unknown'; 255 | } 256 | // Ensure it still exists. 257 | if (!empty($filteredvalue) && !empty($runfiles[$timestamp]) && 258 | $filteredvalue != $runvar) { 259 | unset($runfiles[$timestamp]); 260 | break; 261 | } 262 | } 263 | } 264 | } 265 | } 266 | closedir($dh); 267 | } 268 | 269 | // Ordering them by timestamp DESC. 270 | krsort($runfiles); 271 | 272 | return array($runfiles, $runsvalues); 273 | } 274 | 275 | /** 276 | * Gets the big changes comparing the first run against the other runs results 277 | * 278 | * @param array $thresholds Format: array('bystep' => array('dbreads' => 1, 'dbwrites' => ...), 'total' => array('dbreads' => 2, 'dbwrites'...)) 279 | * @return bool Whether it finished ok. 280 | */ 281 | public function calculate_big_differences(array $thresholds = array()) { 282 | 283 | // Default values if nothing was provided. 284 | if (empty($thresholds)) { 285 | if (!$thresholds = $this->get_default_thresholds()) { 286 | return false; 287 | } 288 | } 289 | 290 | // We get the first run as a base to compare with the other runs 291 | $baserun = & $this->runs[0]; 292 | $basetocompare = $baserun->get_run_dataset(false, 'totalsums'); 293 | 294 | // Comparing each other run against the base one. 295 | $nruns = count($this->runs); 296 | // We skip the first one. 297 | for ($i = 1; $i < $nruns; $i++) { 298 | 299 | $run = & $this->runs[$i]; 300 | 301 | $varaggregates = array_fill_keys($run->get_run_var_names(), 0); 302 | 303 | $runtotals = $run->get_run_dataset(false, 'totalsums'); 304 | 305 | foreach ($runtotals as $var => $steps) { 306 | 307 | $branchnames = 'between ' . $baserun->get_run_info_string() . ' and ' . $run->get_run_info_string(); 308 | 309 | // Check differences between specific steps. 310 | foreach ($steps as $stepname => $value) { 311 | 312 | if ($changed = $this->get_value_changes($basetocompare[$var][$stepname], $value, $thresholds['bystep'][$var])) { 313 | list($state, $msg) = $changed; 314 | $this->bigdifferences[$branchnames][$state][$var][$stepname] = $msg; 315 | } 316 | 317 | // Add it to the global $var sum 318 | $varaggregates[$var] = $varaggregates[$var] + $value; 319 | } 320 | 321 | // Has the performance changed in general. 322 | if ($changed = $this->get_value_changes(array_sum($basetocompare[$var]), $varaggregates[$var], $thresholds['total'][$var])) { 323 | 324 | list($state, $msg) = $changed; 325 | 326 | $this->bigdifferences[$branchnames][$state][$var]['All steps data combined'] = $msg; 327 | } 328 | } 329 | } 330 | 331 | return true; 332 | } 333 | 334 | /** 335 | * Gets the big differences between runs. 336 | * 337 | * @return array|bool List of big differences between runs. False if there are no runs. 338 | */ 339 | public function get_big_differences() { 340 | return $this->bigdifferences; 341 | } 342 | 343 | /** 344 | * Gets the default thresholds. 345 | * 346 | * Hopefully your eyes will not burn after reading this function's code. 347 | * 348 | * Uses the .properties files looking for the threshold values. Gives preference to 349 | * $thresholds array over $groupedthreshold and $singlestepthreshold. 350 | * 351 | * @return array Format: array('bystep' => array('dbreads' => 1, 'dbwrites' => ...), 'total' => array('dbreads' => 2, 'dbwrites'...)) 352 | */ 353 | protected function get_default_thresholds() { 354 | 355 | // Read values from the properties file. 356 | $vars = array('groupedthreshold', 'singlestepthreshold', 'thresholds'); 357 | $properties = properties_reader::get($vars); 358 | 359 | // There will always be a value in defaults.properties. 360 | if (empty($properties['groupedthreshold']) || empty($properties['singlestepthreshold'])) { 361 | die('Error: defaults.properties thresholds values can not be found' . PHP_EOL); 362 | } 363 | 364 | // Preference to $thresholds. 365 | if (!empty($properties['thresholds'])) { 366 | return json_decode($properties['thresholds'], true); 367 | } 368 | 369 | // Generate the default thresholds array. 370 | $thresholds = array('bystep' => array(), 'total' => array()); 371 | foreach (test_plan_run::$runvars as $var) { 372 | $thresholds['bystep'][$var] = $properties['singlestepthreshold']; 373 | $thresholds['total'][$var] = $properties['groupedthreshold']; 374 | 375 | } 376 | 377 | return $thresholds; 378 | } 379 | 380 | /** 381 | * Describes the changes between two values using the provided threshold 382 | * 383 | * @param float $from 384 | * @param float $to 385 | * @param float $threshold 386 | * @return bool|string The string describing the changes or false if there are no changes 387 | */ 388 | protected function get_value_changes($from, $to, $threshold) { 389 | 390 | // Different treatment for near-zero values. 391 | // If there are real problems the sum of all the steps will spot them. 392 | if ($to == 0 && $from == 0) { 393 | // Skip it. 394 | return false; 395 | } else if ($to == 0) { 396 | if ($from > self::FALSE_POSITIVE_SCALAR_THRESHOLD) { 397 | // It is a real decrease if goes to 0. 398 | return array('decrease', 'from ' . $from . ' to 0'); 399 | } else { 400 | // Ignore the change. 401 | return false; 402 | } 403 | } else if ($from == 0) { 404 | if ($to > self::FALSE_POSITIVE_SCALAR_THRESHOLD) { 405 | // It is a increment if it was 0 and now is too much. 406 | return array('increment', 'from 0 to ' . $to); 407 | } else { 408 | // Ignore the change. 409 | return false; 410 | } 411 | } 412 | 413 | $difference = ($to * 100) / $from; 414 | 415 | if ($difference > 100) { 416 | $change = round($difference - 100, 2); 417 | } else { 418 | $change = round(100 - $difference, 2); 419 | } 420 | 421 | if (($difference - $threshold) > 100) { 422 | return array('increment', $change . '% worse'); 423 | } else if (($difference + $threshold) < 100) { 424 | return array('decrease', $change . '% better'); 425 | } 426 | 427 | return false; 428 | } 429 | 430 | /** 431 | * Getter for the charts declaration 432 | * 433 | * @return array 434 | */ 435 | public function get_charts_declaration() { 436 | return $this->chartsdeclaration; 437 | } 438 | 439 | /** 440 | * Returns the test_plan_run objects 441 | * 442 | * @return array 443 | */ 444 | public function get_runs() { 445 | return $this->runs; 446 | } 447 | 448 | /** 449 | * Returns the charts containers. 450 | * 451 | * @return array 452 | */ 453 | public function get_containers() { 454 | return $this->containers; 455 | } 456 | 457 | /** 458 | * Returns wheter if errors have been found. 459 | * 460 | * @return array 461 | */ 462 | public function get_errors() { 463 | return $this->errors; 464 | } 465 | 466 | /** 467 | * Returns true if the runs are comparable between them. 468 | * 469 | * @return bool True if they are comparable 470 | */ 471 | protected function check_runs_are_comparable() { 472 | 473 | $values = array(); 474 | foreach (test_plan_run::$runcomparablevars as $var) { 475 | foreach ($this->runs as $run) { 476 | 477 | $runvalue = $run->get_run_info()->$var; 478 | 479 | // All should match the fist one. 480 | if (empty($values[$var])) { 481 | $values[$var] = $runvalue; 482 | } 483 | 484 | if ($values[$var] != $runvalue) { 485 | $this->errors[$var] = "You can not compare runs with a different $var value"; 486 | } 487 | } 488 | } 489 | 490 | // Run variables can be different for each run, so only compare which has same run variables (dbread, dbwrite..). 491 | foreach ($this->runs as $run) { 492 | 493 | $runvars = $run->get_run_var_names(); 494 | 495 | // All should have this runval. 496 | if (empty($values['runvars'])) { 497 | $values['runvars'] = $runvars; 498 | } 499 | 500 | if ($values['runvars'] != $runvars) { 501 | $this->errors['runvars'] = "You can not compare runs with a different run variables."; 502 | } 503 | } 504 | 505 | if (!empty($this->errors)) { 506 | return false; 507 | } 508 | 509 | return true; 510 | } 511 | 512 | /** 513 | * Creates the charts as defined in the constructor. 514 | * 515 | * @param string $var 516 | * @param array $steporienteddataset 517 | * @param array $runorienteddataset 518 | * @return void 519 | */ 520 | protected function create_charts($var, $steporienteddataset, $runorienteddataset) { 521 | foreach ($this->chartsdeclaration as $chartid => $chartdeclaration) { 522 | $this->create_chart($var, ${$chartdeclaration['orientation']}, $chartdeclaration); 523 | } 524 | } 525 | /** 526 | * Creates a chart and adds it to the lists. 527 | * @param string $var 528 | * @param array $dataset 529 | * @param array $charttypeid 530 | * @return void 531 | */ 532 | protected function create_chart($var, $dataset, $chartdeclaration) { 533 | 534 | $chartid = $var . '_' . $chartdeclaration['id']; 535 | 536 | // TODO: Merge with the declared ones to allow specific behaviours. 537 | $options = array('title' => $var); 538 | 539 | $chart = new google_chart($chartid, $dataset, $chartdeclaration['class'], $options); 540 | google_charts_renderer::add($chart); 541 | 542 | // The DOM container. 543 | $width = $chartdeclaration['width']; 544 | $height = $chartdeclaration['height']; 545 | $this->containers[$chartdeclaration['id']][] = '
' . PHP_EOL; 546 | } 547 | 548 | } 549 | -------------------------------------------------------------------------------- /webapp/classes/report_renderer.php: -------------------------------------------------------------------------------- 1 | report = $report; 25 | } 26 | 27 | /** 28 | * Outputs wrapper. 29 | * 30 | * @param bool $normalize are we calculating normalized results 31 | * 32 | * @return void 33 | */ 34 | public function render($normalize = false) { 35 | echo $this->output_head(); 36 | 37 | echo $this->output_form(); 38 | echo $this->output_runs_info(); 39 | echo $this->output_charts_containers(); 40 | echo $this->output_differences(); 41 | 42 | // Link to Sam's tool with detailed data (just the first 2 runs). 43 | if (!empty($_GET['timestamps']) && count($_GET['timestamps']) >= 2) { 44 | $urlparams = 'before=' . $_GET['timestamps'][1] . '&after=' . $_GET['timestamps'][0]; 45 | $urlparams .= $normalize ? '&n=1' : ''; 46 | echo '
See numeric info
'; 47 | } 48 | } 49 | 50 | /** 51 | * All JS & CSS. 52 | * 53 | * @return string HTML 54 | */ 55 | protected function output_head() { 56 | 57 | google_charts_renderer::render(); 58 | 59 | $head = ''; 60 | $head .= ''; 61 | $head .= ''; 62 | $head .= ''; 63 | $head .= ''; 64 | 65 | return $head; 66 | } 67 | 68 | /** 69 | * Outputs the form. 70 | * 71 | * @return string Form HTML 72 | */ 73 | protected function output_form() { 74 | 75 | list($runs, $runsvalues) = $this->report->get_run_files_info(); 76 | 77 | // Filter runs form. 78 | $output = '
'; 79 | $fields = array(); 80 | foreach (test_plan_run::$runparams as $key => $name) { 81 | $field = $name . ': '; 98 | $fields[] = $field; 99 | } 100 | 101 | // We want a new tr for the button so we add enough empty tds to have a full row. 102 | $needsemptycells = count($fields) % self::FILTERS_PER_ROW; 103 | if ($needsemptycells) { 104 | for ($i = 0; $i < (self::FILTERS_PER_ROW - $needsemptycells); $i++) { 105 | $fields[] = ''; 106 | } 107 | } 108 | 109 | $fields[] = ''; 110 | $output .= $this->create_table('Filter runs', $fields, self::FILTERS_PER_ROW); 111 | $output .= '
'; 112 | 113 | // Select runs form. 114 | $output .= '
'; 115 | 116 | // Restrict the select size. 117 | $sizestr = ''; 118 | $sizelimit = 20; 119 | if (count($runs) > $sizelimit) { 120 | $sizestr = ' size="' . $sizelimit . '" '; 121 | } else { 122 | $sizestr = ' size="' . count($runs) . '" '; 123 | } 124 | 125 | $runsselect = '' . PHP_EOL; 136 | 137 | // Add a message if there are no runs. 138 | if (!$runs) { 139 | $link = 'https://github.com/moodlehq/moodle-performance-comparison/blob/main/README.md#usage'; 140 | $runsselect .= '

There are no runs, more info in ' . $link . ''; 141 | } 142 | 143 | // Keep the filter runs values if there is something filtered. 144 | if (!empty($_GET['filters'])) { 145 | foreach ($_GET['filters'] as $filter => $value) { 146 | if (!empty($value)) { 147 | $runsselect .= ''; 148 | } 149 | } 150 | } 151 | 152 | $runsselect .= '

'; 153 | // Allow to select normalize here. 154 | $normalize = false; 155 | if (!empty($_GET['n']) && preg_match('/^(0|1|true|false)$/', $_GET['n'])) { 156 | $normalize = (bool)$_GET['n']; 157 | } 158 | $checked = $normalize ? ' checked' : ''; 159 | $runsselect .= ' ' . 160 | ''; 161 | $output .= $this->create_table('Select runs', array($runsselect), 1); 162 | $output .= '
'; 163 | 164 | return $output; 165 | } 166 | 167 | /** 168 | * Outputs all the runs info in a one row table. 169 | * 170 | * @return string HTML 171 | */ 172 | protected function output_runs_info() { 173 | 174 | if (!$this->report->get_runs()) { 175 | return false; 176 | } 177 | 178 | // Is the web interface read only?. 179 | $properties = properties_reader::get('readonlyweb'); 180 | 181 | $runsinfo = array(); 182 | foreach ($this->report->get_runs() as $run) { 183 | 184 | // Run vars. 185 | $runinfo = $run->get_run_info(); 186 | 187 | $filenamestr = 'filename=' . $run->get_filename(false); 188 | 189 | // Link to download the run. 190 | $runinfo->downloadlink = 'Download'; 191 | 192 | // Only if the web is read/write. 193 | if (empty($properties['readonlyweb'])) { 194 | // Link to delete the run. 195 | $returnurlstr = 'returnurl=' . urlencode('timestamps[]=' . implode('×tamps[]=', $_GET['timestamps'])); 196 | $runinfo->deletelink = 'Delete'; 197 | } 198 | 199 | $runsinfo[] = $this->get_info_container($runinfo); 200 | } 201 | 202 | $output = '

Are you sure you want to delete this run?

'; 203 | $output .= $this->create_table('Runs information', $runsinfo, count($runsinfo), '', 'runs-info'); 204 | 205 | return $output; 206 | } 207 | 208 | /** 209 | * Outputs the charts containers. 210 | * 211 | * @return string HTML 212 | */ 213 | protected function output_charts_containers() { 214 | 215 | if (!$this->report->get_runs()) { 216 | return false; 217 | } 218 | 219 | $output = ''; 220 | 221 | // Stop on errors. 222 | if ($errors = $this->report->get_errors()) { 223 | $output .= '
'; 224 | foreach ($errors as $error) { 225 | $output .= '
' . $error . '
' . PHP_EOL; 226 | } 227 | $output .= '
'; 228 | return $output; 229 | } 230 | 231 | // Number of columns per row. 232 | $containers = $this->report->get_containers(); 233 | foreach ($this->report->get_charts_declaration() as $chartsdeclaration) { 234 | $output .= $this->create_table($chartsdeclaration['name'], $containers[$chartsdeclaration['id']], $chartsdeclaration['perrow']); 235 | } 236 | 237 | return $output; 238 | } 239 | 240 | /** 241 | * Outputs all the major differences between runs. 242 | * 243 | * @return string HTML 244 | */ 245 | protected function output_differences() { 246 | 247 | if (!$branches = $this->report->get_big_differences()) { 248 | return ''; 249 | } 250 | 251 | $lines = array(); 252 | foreach ($branches as $branchnames => $changes) { 253 | 254 | // Output which branches differs. 255 | $lines[] = '' . $branchnames . ''; 256 | $lines[] = ''; 257 | 258 | foreach ($changes as $state => $data) { 259 | foreach ($data as $var => $step) { 260 | foreach ($step as $stepname => $msg) { 261 | $lines[] = '
' . $var . ': ' . $state . ' - ' . $stepname . ', ' . $msg . '

'; 262 | } 263 | } 264 | } 265 | } 266 | 267 | return $this->create_table('Performance changes between runs', $lines, 1); 268 | } 269 | 270 | /** 271 | * Returns a run's info. 272 | * 273 | * @param stdClass $runinfo 274 | * @return string HTML 275 | */ 276 | protected function get_info_container(stdClass $runinfo) { 277 | 278 | $container = '

' . $runinfo->rundesc . '