├── .gitignore ├── CONTRIBUTING ├── LICENSE ├── MANIFEST.in ├── README.md ├── __init__.py ├── config.yml ├── distaf ├── __init__.py ├── client_rpyc.py ├── config_parser.py ├── create_passwdless_ssh.sh ├── main.py └── util.py ├── docs ├── HOWTO.md ├── Makefile ├── conf.py ├── images │ └── distaf_acrhitecture.jpg ├── index.rst ├── make.bat ├── source │ ├── distaf.client_rpyc.rst │ ├── distaf.config_parser.rst │ ├── distaf.main.rst │ ├── distaf.rst │ ├── distaf.util.rst │ └── modules.rst └── userguide │ ├── howto.rst │ └── overview.rst ├── main.py ├── setup.py └── tests_d ├── __init__.py └── examples ├── __init__.py ├── test_async.py ├── test_basic_gluster_tests.py ├── test_connections.py ├── test_docstring.py ├── test_passfail.py └── test_stdout_stderr.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | Developer's Certificate of Origin 1.1 2 | 3 | By making a contribution to this project, I certify that: 4 | 5 | (a) The contribution was created in whole or in part by me and I 6 | have the right to submit it under the open source license 7 | indicated in the file; or 8 | 9 | (b) The contribution is based upon previous work that, to the best 10 | of my knowledge, is covered under an appropriate open source 11 | license and I have the right under that license to submit that 12 | work with modifications, whether created in whole or in part 13 | by me, under the same open source license (unless I am 14 | permitted to submit under a different license), as indicated 15 | in the file; or 16 | 17 | (c) The contribution was provided directly to me by some other 18 | person who certified (a), (b) or (c) and I have not modified 19 | it. 20 | 21 | (d) I understand and agree that this project and the contribution 22 | are public and that a record of the contribution (including all 23 | personal information I submit with it, including my sign-off) is 24 | maintained indefinitely and may be redistributed consistent with 25 | this project or the open source license(s) involved. 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | 341 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include CONTRIBUTING 3 | include LICENSE 4 | include config.sh 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DiSTAF - Di'stributed Systems Test Automation Framework 2 | ======================================================== 3 | 4 | DiSTAF is a test automation framework for distributed systems. It is used for test automation of glusterfs and its related projects. This framework is written with modularity in mind. So many parts of it can be modified for the liking of the project, without affecting other parts. DiSTAF can be used to test projects which runs on physical machines, virtual machines or even containers. DiSTAF requires remote machines (or containers) to be reachable by IP address (or FQDN). On Linux systems it requires sshd to be running in the remote systems with bash environment as well. 5 | 6 | 7 | ## About the name 8 | DiSTAF (or distaf) is short for Di'stributed Systems Test Automation Framework. 9 | 'distaff' is a tool used in spinning, which is designed to hold unspun fibres together keeping them untangled and thus easing the process of spinning.This framework is also trying to do just that, keeping the machines untangled and easing the process of writing and executing the test cases. Hence the name DiSTAF (distaf). 10 | 11 | Architecture of the Framework 12 | ============================== 13 | 14 | > Terminologies used 15 | >* **Management node** - The node from which this test suite is executed. This node is responsible for orchestration of test automation. 16 | >* **Test machines** - These include all the systems that participate in the tests. This can be physical machines, VMs or containers. 17 | 18 | ![Architecture of distaf](docs/images/distaf_acrhitecture.jpg) 19 | 20 | To run distaf, passwordless ssh should be setup from *management node* to all the *test machines*. The *management node* connects to *test machines* using rpyc zero-deploy, which internally makes use of ssh tunneling protocol for establishing and maintaining the secure connections. The connection is kept open for the entire duration of the tests. All the synchronous commands run by the test cases, uses this connection to run them. For asynchronous calls, a new connection is opened. This connection will be closed when async command returns. 21 | 22 | DiSTAF uses `python-unittest` for running tests and generating the results. 23 | 24 | When a test run is started, DiSTAF first reads a *config file*, which is in yaml format. 25 | The *config file* will have information about servers and clients DiSTAF can connect to. 26 | DiSTAF establishes a ssh connection to each of the servers and clients, 27 | and maintains the connection until the end of the test run. 28 | All the remote commands, bash or python will go through this connection. 29 | DiSTAF provides two APIs to run commands on the servers or clients synchronously and asynchronously. 30 | For more information about distaf APIs, please refer HOWTO. 31 | 32 | ## Test case philosophy 33 | 34 | DiSTAF has two modes of running. The ***Global Mode*** and ***Non-global Mode***. There is a configuration variable in config.yml ***global_mode*** to toggle between them. The idea here is that each test case should be independent of the volume type and access protocol used to mount the volume. 35 | 36 | When the distaf is started in the *non-global mode*, 37 | it runs each test case against all the volume type and mount protocol combinations. 38 | This means a single test case will run many times and each time a different volume and mount combination is used. 39 | Each test case will have it's own metadata in yaml format in test case docstring. 40 | For more information about the fields and values of test case metadata (test case config), please refer to HOWTO. 41 | 42 | When distaf is started in *global mode*, each test case is run only once. 43 | The volume type and mount protocol specified in the config.yml is used for each test case. 44 | This is helpful if a test case needs to run against a particular type of volume, to run some checks. 45 | 46 | ### Few things to take care before running test case in DiSTAF. 47 | * Setting up and provisioning the test machines. This needs to be handled before running distaf tests. 48 | * Updating the config.yml and setting up password-less ssh from management node to test machines. 49 | * Keeping the test machines in the same state if a test case fails. Since distaf does not manage the bringing up and maintaining the test machine, this should be handled outside distaf as well. 50 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/distaf/62f5892153ec0f36283fb26e4e31549f18f87d26/__init__.py -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | log_file: /var/log/tests/distaf_test_run.log 2 | log_level: DEBUG 3 | remote_user: root 4 | servers: 5 | - host: server-vm1 6 | devices: ["/dev/vdb", "/dev/vdc", "/dev/vdd", "/dev/vde"] 7 | - host: server-vm2 8 | devices: ["/dev/vdb", "/dev/vdc", "/dev/vdd", "/dev/vde"] 9 | - host: server-vm3 10 | devices: ["/dev/vdb", "/dev/vdc", "/dev/vdd", "/dev/vde"] 11 | - host: server-vm4 12 | devices: ["/dev/vdb", "/dev/vdc", "/dev/vdd", "/dev/vde"] 13 | clients: 14 | - host: client-vm1 15 | - host: client-vm2 16 | 17 | global_mode: True 18 | -------------------------------------------------------------------------------- /distaf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/distaf/62f5892153ec0f36283fb26e4e31549f18f87d26/distaf/__init__.py -------------------------------------------------------------------------------- /distaf/client_rpyc.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | import os 20 | import time 21 | import logging 22 | 23 | from plumbum import SshMachine 24 | from rpyc.utils.zerodeploy import DeployedServer 25 | 26 | 27 | class BigBang(): 28 | """ 29 | The big bang which starts the life in distaf 30 | """ 31 | def __init__(self, configs): 32 | """ 33 | Initialises the whole environment and establishes connection 34 | """ 35 | self.global_config = configs 36 | 37 | # Initialise servers and clients 38 | self.servers = [] 39 | self.clients = [] 40 | for server in self.global_config['servers']: 41 | self.servers.append(server['host']) 42 | for client in self.global_config['clients']: 43 | self.clients.append(client['host']) 44 | self.all_nodes = list(set(self.servers + self.clients)) 45 | self.num_servers = len(self.servers) 46 | self.num_clients = len(self.clients) 47 | self.user = self.global_config['remote_user'] 48 | self.global_flag = {} 49 | 50 | # setup logging in distaf 51 | client_logfile = os.path.abspath(self.global_config['log_file']) 52 | loglevel = getattr(logging, self.global_config['log_level'].upper()) 53 | client_logdir = os.path.dirname(client_logfile) 54 | if not os.path.exists(client_logdir): 55 | os.makedirs(client_logdir) 56 | self.logger = logging.getLogger('distaf') 57 | self.lhndlr = logging.FileHandler(client_logfile) 58 | formatter = logging.Formatter('%(asctime)s %(levelname)s %(funcName)s ' 59 | '%(message)s') 60 | self.lhndlr.setFormatter(formatter) 61 | self.logger.addHandler(self.lhndlr) 62 | self.logger.setLevel(loglevel) 63 | self.skip_log_inject = self.global_config.get('skip_log_inject', False) 64 | 65 | # Determine default connection type 66 | # Default to ssh w/ control persist 67 | self.connection_engine = self.global_config.get('connection_engine', 68 | 'ssh_controlpersist') 69 | if self.connection_engine == "ssh_controlpersist": 70 | self.use_ssh = True 71 | self.use_controlpersist = True 72 | elif self.connection_engine == "ssh": 73 | self.use_ssh = True 74 | self.use_controlpersist = False 75 | else: 76 | self.use_ssh = False 77 | self.use_controlpersist = False 78 | 79 | # connection store for _get_ssh() 80 | if self.use_ssh: 81 | # using separate ssh connections at the moment 82 | # ssh connections are requested on-the-fly via run() 83 | self.sshconns = {} 84 | 85 | # rpyc connection handles (still needed for on-the-fly connections) 86 | self.connection_handles = {} 87 | self.subp_conn = {} 88 | # skipping the rpyc connections until requested to speed startup time 89 | # if zerodeploy delay interferes with a testcase, user can "prime" the 90 | # connection by calling establish_connection in the testcase.setup 91 | if not self.use_ssh: 92 | for node in self.all_nodes: 93 | self.logger.debug("Connecting to node: %s" % node) 94 | ret = self.establish_connection(node, self.user) 95 | if not ret: 96 | self.logger.warning("Unable to establish connection " 97 | "with: %s" % node) 98 | else: 99 | self.logger.debug("Connected to node: %s" % node) 100 | 101 | def establish_connection(self, node, user): 102 | """ 103 | Establishes rpyc connection from localhost to node via SshMachine 104 | and zerodeploy. The connection is authenticated and hence secure. 105 | Populates the connection in a dict called connection_handles. 106 | This function does not take care of timeouts. Timeouts need to 107 | be handled by the calling function 108 | Returns True on success and False otherwise 109 | """ 110 | keyfile = None 111 | if 'ssh_keyfile' in self.global_config: 112 | keyfile = self.global_config['ssh_keyfile'] 113 | try: 114 | self.connection_handles[node] = {} 115 | self.subp_conn[node] = {} 116 | rem = SshMachine(node, user, keyfile=keyfile) 117 | dep = DeployedServer(rem) 118 | conn = dep.classic_connect() 119 | self.connection_handles[node][user] = (rem, dep, conn) 120 | self.subp_conn[node][user] = conn.modules.subprocess 121 | except: 122 | return False 123 | return True 124 | 125 | def refresh_connection(self, node, user='', timeout=210): 126 | """ 127 | Refresh the connection to the user@node 128 | 129 | This should be called either from test script, but internally 130 | run/run_async will also call this if connection is found to be 131 | disconnected. Any reboot will make the connection go bad. 132 | """ 133 | if user == '': 134 | user = self.user 135 | try: 136 | self.connection_handles[node][user][2].close() 137 | self.connection_handles[node][user][1].close() 138 | self.connection_handles[node][user][0].close() 139 | except: 140 | pass 141 | while timeout >= 0: 142 | try: 143 | self.establish_connection(node, user) 144 | break 145 | except: 146 | self.logger.debug("Couldn't connect to %s. Retrying in 42 " 147 | "seconds" % node) 148 | time.sleep(42) 149 | timeout = timeout - 42 150 | if timeout < 0: 151 | self.logger.critical("Unable to connect to %s" % node) 152 | return False 153 | else: 154 | self.logger.debug("Connection (re-)established to %s" % node) 155 | return True 156 | 157 | def _get_ssh(self, node, user): 158 | """Setup a SshMachine connection for non-rpyc connections""" 159 | ssh_opts = () 160 | ssh_opts += ('-T', 161 | '-oPasswordAuthentication=no', 162 | '-oStrictHostKeyChecking=no', 163 | '-oPort=22', 164 | '-oConnectTimeout=10') 165 | 166 | keyfile = None 167 | if 'ssh_keyfile' in self.global_config: 168 | keyfile = self.global_config['ssh_keyfile'] 169 | ssh_opts += ('-o', 'IdentityFile=%s' % keyfile) 170 | 171 | if self.use_controlpersist: 172 | ssh_opts += ('-oControlMaster=auto', 173 | '-oControlPersist=30m', 174 | '-oControlPath=~/.ssh/distaf-ssh-%r@%h:%p') 175 | 176 | conn_name = "%s@%s" % (user, node) 177 | # if no existing connection, create one 178 | if conn_name not in self.sshconns: 179 | # we already have plumbum imported for rpyc, so let's use it 180 | ssh = SshMachine(node, user, ssh_opts=ssh_opts) 181 | self.sshconns[conn_name] = ssh 182 | else: 183 | ssh = self.sshconns[conn_name] 184 | 185 | if ssh: 186 | self.logger.debug("Have ssh for %s. Returning ssh." % conn_name) 187 | return ssh 188 | 189 | self.logger.error("oops. did not get ssh for %s", conn_name) 190 | return None 191 | 192 | def run(self, node, cmd, user='', verbose=True): 193 | """ 194 | Run the specified command in specified remote machine 195 | 196 | Returns a tuple of (retcode, stdout, stderr) of the command 197 | in remote machine 198 | """ 199 | if user == '': 200 | user = self.user 201 | self.logger.info("Executing %s on %s" % (cmd, node)) 202 | 203 | if self.use_ssh: 204 | """ 205 | Use straight ssh for all run shell command connections, 206 | and only create rpyc connections as needed. 207 | """ 208 | ctlpersist = '' 209 | if self.use_controlpersist: 210 | ctlpersist = " (cp)" 211 | 212 | # output command 213 | self.logger.debug("%s@%s%s: %s", user, node, ctlpersist, cmd) 214 | # run the command 215 | ssh = self._get_ssh(node, user) 216 | p = ssh.popen(cmd) 217 | pout, perr = p.communicate() 218 | prcode = p.returncode 219 | 220 | # output command results 221 | self.logger.info("Running %s on %s@%s RETCODE: %s", 222 | cmd, user, node, prcode) 223 | if pout != "" and verbose: 224 | self.logger.info("Running %s on %s@%s STDOUT:\n%s", 225 | cmd, user, node, pout) 226 | if perr != "" and verbose: 227 | self.logger.error("Running %s on %s@%s STDERR:\n%s", 228 | cmd, user, node, perr) 229 | 230 | return (prcode, pout, perr) 231 | else: 232 | try: 233 | subp = self.subp_conn[node][user] 234 | p = subp.Popen(cmd, shell=True, 235 | stdout=subp.PIPE, stderr=subp.PIPE) 236 | except: 237 | ret = self.refresh_connection(node, user) 238 | if not ret: 239 | self.logger.critical("Unable to connect to %s@%s" % 240 | (user, node)) 241 | return (-1, -1, -1) 242 | subp = self.subp_conn[node][user] 243 | p = subp.Popen(cmd, shell=True, 244 | stdout=subp.PIPE, stderr=subp.PIPE) 245 | pout, perr = p.communicate() 246 | ret = p.returncode 247 | self.logger.info("\"%s\" on %s: RETCODE is %d" % (cmd, node, ret)) 248 | if pout != "" and verbose: 249 | self.logger.info("\"%s\" on %s: STDOUT is \n %s" % 250 | (cmd, node, pout)) 251 | if perr != "" and verbose: 252 | self.logger.error("\"%s\" on %s: STDERR is \n %s" % 253 | (cmd, node, perr)) 254 | return (ret, pout, perr) 255 | 256 | def run_async(self, node, cmd, user='', verbose=True): 257 | """ 258 | Run the specified command in specified remote node asynchronously 259 | """ 260 | if user == '': 261 | user = self.user 262 | 263 | if self.use_ssh: 264 | ctlpersist = '' 265 | if self.use_controlpersist: 266 | ctlpersist = " (cp)" 267 | 268 | # output command 269 | self.logger.debug("%s@%s%s: %s" % (user, node, ctlpersist, cmd)) 270 | # run the command 271 | ssh = self._get_ssh(node, user) 272 | p = ssh.popen(cmd) 273 | 274 | def value(): 275 | stdout, stderr = p.communicate(input=cmd) 276 | retcode = p.returncode 277 | # output command results 278 | self.logger.info("Running %s on %s@%s RETCODE: %s", 279 | cmd, user, node, retcode) 280 | if stdout: 281 | self.logger.info("Running %s on %s@%s STDOUT...\n%s", 282 | cmd, user, node, stdout) 283 | if stderr: 284 | self.logger.info("Running %s on %s@%s STDERR...\n%s", 285 | cmd, user, node, stderr) 286 | return (retcode, stdout, stderr) 287 | 288 | p.value = value 289 | return p 290 | else: 291 | try: 292 | c = self.connection_handles[node][user][1].classic_connect() 293 | except: 294 | ret = self.refresh_connection(node, user) 295 | if not ret: 296 | self.logger.critical("Couldn't connect to %s" % node) 297 | return None 298 | c = self.connection_handles[node][user][1].classic_connect() 299 | self.logger.info("Executing %s on %s asynchronously" % (cmd, node)) 300 | p = c.modules.subprocess.Popen(cmd, shell=True, 301 | stdout=c.modules.subprocess.PIPE, 302 | stderr=c.modules.subprocess.PIPE) 303 | 304 | def value(): 305 | """ 306 | A function which returns the tuple of 307 | (retcode, stdout, stdin) 308 | """ 309 | pout, perr = p.communicate() 310 | retc = p.returncode 311 | c.close() 312 | self.logger.info("\"%s\" on \"%s\": RETCODE is %d" % 313 | (cmd, node, retc)) 314 | if pout != "" and verbose: 315 | self.logger.debug("\"%s\" on \"%s\": STDOUT is \n %s" % 316 | (cmd, node, pout)) 317 | if perr != "" and verbose: 318 | self.logger.error("\"%s\" on \"%s\": STDERR is \n %s" % 319 | (cmd, node, perr)) 320 | return (retc, pout, perr) 321 | 322 | p.value = value 323 | p.close = lambda: c.close() 324 | return p 325 | 326 | def run_servers(self, command, user='', servers='', verbose=True): 327 | """ 328 | Run the specified command in each of the server in parallel 329 | """ 330 | if user == '': 331 | user = self.user 332 | if servers == '': 333 | servers = self.servers 334 | servers = list(set(servers)) 335 | sdict = {} 336 | out_dict = {} 337 | ret = True 338 | for server in servers: 339 | sdict[server] = self.run_async(server, command, user, verbose) 340 | for server in servers: 341 | ps, _, _ = sdict[server].value() 342 | out_dict[server] = ps 343 | if 0 != ps: 344 | ret = False 345 | return (ret, out_dict) 346 | 347 | def get_connection(self, node, user=''): 348 | """ 349 | Establishes a connection to the remote node and returns 350 | the connection handle. Returns -1 if connection couldn't 351 | be established. 352 | """ 353 | if user == '': 354 | user = self.user 355 | try: 356 | conn = self.connection_handles[node][user][1].classic_connect() 357 | except: 358 | ret = self.refresh_connection(node, user) 359 | if not ret: 360 | self.logger.critical("Couldn't connect to %s" % node) 361 | return -1 362 | conn = self.connection_handles[node][user][1].classic_connect() 363 | return conn 364 | 365 | def upload(self, node, localpath, remotepath, user=''): 366 | """ 367 | Uploads the file/directory in localpath to file/directory to 368 | remotepath in node 369 | Returns None if success 370 | Raises an exception in case of failure 371 | """ 372 | if user == '': 373 | user = self.user 374 | if self.use_ssh: 375 | rem = self._get_ssh(node, user) 376 | else: 377 | rem = self.connection_handles[node][user][0] 378 | rem.upload(localpath, remotepath) 379 | return None 380 | 381 | def add_group(self, node, group): 382 | """ 383 | Adds a group in the remote node 384 | 385 | Returns True on success and False on failure 386 | """ 387 | if 'root' not in self.connection_handles[node]: 388 | self.logger.error("ssh connection to 'root' of %s is not present" % 389 | node) 390 | return False 391 | conn = self.get_connection(node, 'root') 392 | if conn == -1: 393 | self.logger.error("Unable to get connection for root@%s" % node) 394 | return False 395 | try: 396 | conn.modules.grp.getgrnam(group) 397 | self.logger.debug("Group %s already exists on %s" % (group, node)) 398 | conn.close() 399 | return True 400 | except KeyError: 401 | self.logger.debug("group %s does not exist in %s. Creating now" % 402 | (group, node)) 403 | conn.close() 404 | ret = self.run(node, "groupadd %s" % group) 405 | if ret[0] != 0: 406 | self.logger.error("Unable to add group %s to %s" % (group, node)) 407 | return False 408 | else: 409 | self.logger.debug("group %s added to %s" % (group, node)) 410 | return True 411 | 412 | def add_user(self, node, user, password='foobar', group=''): 413 | """ 414 | Add the user 'user' to the node 'node' 415 | For this to work, connection to 'root' account of remote machine 416 | should be already established. 417 | This functions also takes care creating a passwordless ssh 418 | connection from management node to remote node. 419 | And then it connects to remote user and updates the 420 | dict of connection_handles 421 | """ 422 | if 'root' not in self.connection_handles[node]: 423 | self.logger.error("ssh connection to 'root' of %s is not present" % 424 | node) 425 | return False 426 | conn = self.get_connection(node, 'root') 427 | if conn == -1: 428 | self.logger.error("Unable to get connection to 'root' of node %s" % 429 | node) 430 | return False 431 | try: 432 | conn.modules.pwd.getpwnam(user) 433 | self.logger.debug("User %s already exist in %s" % (user, node)) 434 | return True 435 | except KeyError: 436 | self.logger.debug("User %s doesn't exist in %s. Creating now" % 437 | (user, node)) 438 | grp_add_cmd = '' 439 | if group != '': 440 | ret = self.add_group(node, group) 441 | if not ret: 442 | return False 443 | grp_add_cmd = "-g %s" % group 444 | else: 445 | group = user 446 | grp_add_cmd = '-U' 447 | 448 | ret = self.run(node, "useradd -m %s -p $(perl -e'print " 449 | "crypt(%s, \"salt\")') %s" % 450 | (grp_add_cmd, password, user), user='root') 451 | if ret[0] != 0: 452 | self.logger.error("Unable to add the user %s to %s" % (user, node)) 453 | return False 454 | conn.modules.os.makedirs("/home/%s/.ssh" % user) 455 | rfh = conn.builtin.open("/home/%s/.ssh/authorized_keys" % user, "a") 456 | localhome = os.path.expanduser('~') 457 | try: 458 | with open("%s/.ssh/id_rsa.pub" % localhome, 'r') as f: 459 | for line in f: 460 | rfh.write(line) 461 | ruid = conn.modules.pwd.getpwnam(user).pw_uid 462 | rgid = conn.modules.grp.getgrnam(group).gr_gid 463 | conn.modules.os.chown("/home/%s/.ssh/authorized_keys" % 464 | user, ruid, rgid) 465 | except: 466 | self.logger.error("Unable to write the rsa pub file to %s@%s" % 467 | (user, node)) 468 | return False 469 | rfh.close() 470 | conn.close() 471 | ret = self.establish_connection(node, user) 472 | if not ret: 473 | self.logger.critical("Unable to connect to %s@%s" % (user, node)) 474 | return False 475 | return True 476 | 477 | def fini(self): 478 | """ 479 | Destroy all stored connections to user@remote-machine 480 | """ 481 | for node in self.connection_handles.keys(): 482 | for user in self.connection_handles[node].keys(): 483 | self.logger.debug("Closing all connection to %s@%s" % 484 | (user, node)) 485 | self.connection_handles[node][user][2].close() 486 | self.connection_handles[node][user][1].close() 487 | self.connection_handles[node][user][0].close() 488 | -------------------------------------------------------------------------------- /distaf/config_parser.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | import yaml 20 | 21 | 22 | def get_global_config(config_files): 23 | """ 24 | Gets all the config data from the distaf_config.yml file 25 | 26 | Returns the parsed output of config in a dictionary format 27 | """ 28 | configs = {} 29 | for config_file in config_files: 30 | configs.update(yaml.load(open(config_file, 'r'))) 31 | 32 | return configs 33 | 34 | 35 | def get_testcase_config(doc_string): 36 | """ 37 | Parses the config yaml structure from the 38 | doc string passed to the function 39 | 40 | @params: doc string with yaml structure 41 | @returns: python dict with config values on Success 42 | Upon failure python dict with defaults 43 | """ 44 | if not doc_string: 45 | config_dict = {} 46 | config_dict['runs_on_volumes'] = 'ALL' 47 | config_dict['runs_on_protocol'] = 'ALL' 48 | config_dict['reuse_setup'] = True 49 | else: 50 | if "---\n" in doc_string: 51 | config_string = doc_string.split("---\n")[1] 52 | else: 53 | config_string = doc_string 54 | 55 | try: 56 | config_dict = yaml.load(config_string) 57 | if isinstance(config_dict, str): 58 | config_dict = {} 59 | except yaml.YAMLError: 60 | config_dict = {} 61 | config_dict['runs_on_volumes'] = 'ALL' 62 | config_dict['runs_on_protocol'] = 'ALL' 63 | config_dict['reuse_setup'] = True 64 | if 'runs_on_volumes' not in config_dict: 65 | config_dict['runs_on_volumes'] = 'ALL' 66 | if config_dict['runs_on_volumes'] == 'ALL': 67 | config_dict['runs_on_volumes'] = ['distribute', 'replicate', \ 68 | 'dist_rep', 'disperse', 'dist_disperse'] 69 | if 'runs_on_protocol' not in config_dict: 70 | config_dict['runs_on_protocol'] = 'ALL' 71 | if config_dict['runs_on_protocol'] == 'ALL': 72 | config_dict['runs_on_protocol'] = ['glusterfs', 'nfs'] 73 | if 'reuse_setup' not in config_dict: 74 | config_dict['reuse_setup'] = True 75 | return config_dict 76 | -------------------------------------------------------------------------------- /distaf/create_passwdless_ssh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This file is part of DiSTAF 3 | # Copyright (C) 2015-2016 Red Hat, Inc. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | 19 | 20 | function create_passwordless_ssh() 21 | { 22 | target_node="$1" 23 | sed -i "/$target_node/d" ~/.ssh/known_hosts 24 | ssh-keyscan $target_node 1 >> ~/.ssh/known_hosts 25 | test -f ~/.ssh/id_rsa.pub || { \ 26 | expect -c " 27 | set timeout 90 28 | set env(TERM) 29 | spawn ssh-keygen -t rsa 30 | expect \"id_rsa): \" 31 | send \"\r\" 32 | expect \"passphrase): \" 33 | send \"\r\" 34 | expect \"passphrase again: \" 35 | send \"\r\" 36 | expect eof 37 | " 38 | } 39 | 40 | expect -c " 41 | set timeout 90 42 | set env(TERM) 43 | spawn ssh-copy-id root@$target_node 44 | expect \"password: \" 45 | send \"$password\r\" 46 | expect eof 47 | " 48 | } 49 | 50 | function main() 51 | { 52 | ALL_NODES="$NODES $CLIENTS $PEERS $GM_NODES $GS_NODES" 53 | echo -n "Enter the password of your test machines: " 54 | stty -echo 55 | trap 'stty echo' EXIT 56 | read password 57 | stty echo 58 | trap - EXIT 59 | for node in $ALL_NODES; do 60 | create_passwordless_ssh $node 61 | done 62 | } 63 | 64 | main"$@" 65 | -------------------------------------------------------------------------------- /distaf/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # This file is part of DiSTAF 3 | # Copyright (C) 2015-2016 Red Hat, Inc. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License along 16 | # with this program; if not, write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | 19 | 20 | import os 21 | import re 22 | import sys 23 | import unittest 24 | import argparse 25 | 26 | __version__ = '0.0.3' 27 | 28 | if os.getcwd() not in sys.path: 29 | sys.path.append(os.getcwd()) 30 | 31 | from distaf.util import testcases, test_list, distaf_init, distaf_finii, \ 32 | test_seq, test_mounts 33 | 34 | 35 | def collect_tests(_dir="tests_d"): 36 | """ 37 | Collects all the tests and populates them in a global list 'ts' 38 | Each element of the list is the tuple of name of testcase name it's 39 | function object. The function object is later used to set the tests 40 | to gluster_tests class. 41 | """ 42 | if os.path.isfile(_dir): 43 | __import__((_dir.replace(".py", "")).replace("/", ".")) 44 | else: 45 | for top, _, files in os.walk(_dir, topdown=False): 46 | for _f in files: 47 | if _f.startswith("test_") and _f.endswith(".py"): 48 | iname = top + '/' + _f.replace(".py", "") 49 | # When testcase is imported, decorator populates the 50 | # Testcase and its value in a tuple. And that will be later 51 | # added to ts list 52 | __import__(iname.replace("/", ".")) 53 | 54 | 55 | class gluster_tests(unittest.TestCase): 56 | """ 57 | Empty class. But will be populated with test cases during runtime 58 | """ 59 | pass 60 | 61 | 62 | def set_tests(tests=''): 63 | """ 64 | Sets the gluster_tests Test class with the test cases. 65 | Name of the tests will be prepended with test_ to enable 66 | unittest to recognise them as test case 67 | """ 68 | if tests == '': 69 | tests = testcases.keys() 70 | if test_list == {}: 71 | test_list[''] = testcases.keys() 72 | i = 0 73 | for voltype, vol_tests in test_list.iteritems(): 74 | for test in vol_tests: 75 | if test in tests: 76 | if test not in test_mounts: 77 | test_mounts[test] = [''] 78 | for mount in test_mounts[test]: 79 | try: 80 | setattr(gluster_tests, "test_%d_%s_%s_%s" % \ 81 | (i, voltype, mount, test), testcases[test]) 82 | i = i + 1 83 | test_seq.append((voltype, mount)) 84 | except KeyError: 85 | sys.stderr.write("Unable to find test %s." \ 86 | "Skipping...\n" % test) 87 | 88 | 89 | def main(): 90 | parser = argparse.ArgumentParser() 91 | parser.add_argument("-t", help="Test case(s) to run") 92 | parser.add_argument("-d", help="Directory to choose tests from") 93 | parser.add_argument("-f", help="Find the test cases from the file") 94 | parser.add_argument("-c", help="The (yaml) config file to use", 95 | default="config.yml") 96 | parser.add_argument("-j", help="Directory to store JUnit XML file", 97 | action="store", 98 | dest="xmldir") 99 | args = parser.parse_args() 100 | 101 | _ = distaf_init(args.c) 102 | 103 | if args.f != None: 104 | collect_tests(args.f) 105 | set_tests() 106 | elif args.d != None and args.t != None: 107 | collect_tests("tests_d/%s" % args.d) 108 | set_tests(args.t.split(' ')) 109 | elif args.t != None: 110 | collect_tests() 111 | set_tests(args.t.split(' ')) 112 | elif args.d != None: 113 | collect_tests("tests_d/%s" % args.d) 114 | set_tests() 115 | else: 116 | collect_tests() 117 | set_tests() 118 | 119 | get_num = lambda x: int(re.search(r'test_([0-9]+)_', x).group(1)) 120 | sortcmp = lambda _, x, y: cmp(get_num(x), get_num(y)) 121 | unittest.TestLoader.sortTestMethodsUsing = sortcmp 122 | 123 | if args.xmldir != None: 124 | import xmlrunner 125 | runner = xmlrunner.XMLTestRunner(output=args.xmldir) 126 | else: 127 | runner = unittest.TextTestRunner(verbosity=2) 128 | 129 | itersuite = unittest.TestLoader().loadTestsFromTestCase(gluster_tests) 130 | runner.run(itersuite) 131 | distaf_finii() 132 | 133 | if __name__ == '__main__': 134 | main() 135 | -------------------------------------------------------------------------------- /distaf/util.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from types import FunctionType 20 | from distaf.client_rpyc import BigBang 21 | from distaf.config_parser import get_global_config, get_testcase_config 22 | 23 | 24 | testcases = {} 25 | test_list = {} 26 | test_seq = [] 27 | test_mounts = {} 28 | globl_configs = {} 29 | global_mode = None 30 | tc = None 31 | 32 | 33 | def distaf_init(config_file_string="config.yml"): 34 | """ 35 | The distaf init function which calls the BigBang 36 | """ 37 | config_files = config_file_string.split() 38 | global globl_configs, global_mode, tc 39 | globl_configs = get_global_config(config_files) 40 | global_mode = globl_configs['global_mode'] 41 | tc = BigBang(globl_configs) 42 | return globl_configs 43 | 44 | 45 | def inject_gluster_logs(label, servers=''): 46 | """ 47 | Injects the label in gluster related logs 48 | 49 | This is mainly to help identifying what was going 50 | on during the test case 51 | 52 | @parameter: A label string which will be injected to gluster logs 53 | A list of servers in which this log inejection should be 54 | done 55 | 56 | @returns: None 57 | """ 58 | if servers == '': 59 | servers = tc.all_nodes 60 | cmd = ("for file in `find $(gluster --print-logdir) -type f " 61 | "-name '*.log'`; do echo \"%s\" >> $file; done" % label) 62 | tc.run_servers(cmd, servers=servers, verbose=False) 63 | return None 64 | 65 | 66 | def testcase(name): 67 | def decorator(func): 68 | tc_config = get_testcase_config(func.__doc__) 69 | 70 | def wrapper(self): 71 | tc.logger.info("Starting the test: %s" % name) 72 | voltype, mount_proto = test_seq.pop(0) 73 | 74 | if not tc.skip_log_inject: 75 | inject_gluster_logs("%s_%s" % (voltype, name)) 76 | 77 | _ret = True 78 | globl_configs['reuse_setup'] = tc_config['reuse_setup'] 79 | globl_configs.update(tc_config) 80 | globl_configs['voltype'] = voltype 81 | globl_configs['mount_proto'] = mount_proto 82 | if isinstance(func, FunctionType): 83 | _ret = func() 84 | else: 85 | try: 86 | func_obj = func(globl_configs) 87 | ret = func_obj.setup() 88 | if not ret: 89 | tc.logger.error("The setup of %s failed" % name) 90 | _ret = False 91 | if _ret: 92 | ret = func_obj.run() 93 | if not ret: 94 | tc.logger.error("The execution of testcase %s " 95 | "failed" % name) 96 | _ret = False 97 | ret = func_obj.teardown() 98 | if not ret: 99 | tc.logger.error("The teardown of %s failed" % name) 100 | _ret = False 101 | if len(test_seq) == 0 or voltype != test_seq[0][0]: 102 | tc.logger.info("Last test case to use %s volume type" 103 | % voltype) 104 | ret = func_obj.cleanup() 105 | if not ret: 106 | tc.logger.error("The cleanup of volume %s failed" 107 | % name) 108 | _ret = False 109 | except: 110 | tc.logger.exception("Exception while running %s" % name) 111 | _ret = False 112 | self.assertTrue(_ret, "Testcase %s failed" % name) 113 | if not tc.skip_log_inject: 114 | inject_gluster_logs("%s_%s" % (voltype, name)) 115 | tc.logger.info("Ending the test: %s" % name) 116 | return _ret 117 | 118 | testcases[name] = wrapper 119 | if not global_mode and tc_config is not None: 120 | for voltype in tc_config['runs_on_volumes']: 121 | if voltype not in test_list: 122 | test_list[voltype] = [] 123 | if not tc_config['reuse_setup']: 124 | test_list[voltype].insert(0, name) 125 | else: 126 | test_list[voltype].append(name) 127 | test_mounts[name] = tc_config['runs_on_protocol'] 128 | return wrapper 129 | 130 | return decorator 131 | 132 | 133 | def distaf_finii(): 134 | """ 135 | The fini() function which closes all connection to the servers 136 | """ 137 | tc.fini() 138 | -------------------------------------------------------------------------------- /docs/HOWTO.md: -------------------------------------------------------------------------------- 1 | # DiSTAF 2 | 3 | DiSTAF (or distaf) is a test framework for distributed systems like glusterfs. This file contains information about how to write test cases in distaf framework and how to execute them once they are written. For information about overview and architecture, please refer to [README.md](https://github.com/gluster/distaf/blob/master/README.md). 4 | 5 | ## Little about the APIs 6 | 7 | DiSTAF exposes few APIs to interact and control the remote test machines. That is the core part of distaf. And since most test steps of glusterfs are bash/shell commands (at least on the server side), distaf exposes two APIs to run the bash/shell commands remotely. These APIs are actually methods of connection manager object and will be accessible through the object `tc` 8 | 9 | ```python 10 | (ret, stdout, stderr) = tc.run(node, command, user='root', verbose=True) 11 | ``` 12 | 13 | So the `run` method execute the bash `command` in remote `node` as user `root` synchronously and returns a python tuple of return code, output in stdout and output in stderr. 14 | 15 | The asynchronous version of `run` method returns a `popen` object which is very similar to `subprocess.Popen`. The object has methods to wait, communicate and get the return code etc. It has has another method `value()` to get the tuple of `(ret, stdout, stderr)` 16 | 17 | ```python 18 | popen = tc.run_async(node, command, user='root', verbose=True) 19 | popen.wait() 20 | (ret, stdout, stderr) = popen.value() 21 | ``` 22 | 23 | These are most used APIs for distaf. You can also request for a connection and run python command on the remote machines. The below example should make things bit more clear. 24 | 25 | ```python 26 | conn = tc.get_connection(node, user='root') 27 | if conn == -1: 28 | tc.logger.error("Unable to establish connection to %s@%s", node, user) 29 | return False 30 | # Remote python modules are accessible through conn.modules. 31 | pwd = conn.modules.os.getcwd() 32 | conn.modules.os.chdir("/tmp") # Stateful. Remains in /tmp 33 | pwd = conn.modules.os.getcwd() 34 | r_hostname = conn.modules.socket.gethostname() 35 | ``` 36 | 37 | For more info about all the APIs, please refer "DiSTAF API Guide" 38 | 39 | ## Writing Tests in DiSTAF 40 | #### Things to mind before writing the tests 41 | * DiSTAF expects that the file in which test cases are written starts with `test_`. For example `test_basic_gluster.py`. 42 | * And each test case file needs to import connection manager object to interact with the remote test nodes. 43 | * Each of the test case should have a decorator `@testcase` with the test case name. 44 | * Each test case should have its metadata, namely the volume type it can run on, the possible mount protocols with which it can be run and if fresh setup is required to run that test. The next section will explain a bit more about these metadata. 45 | * A test case can be a class or a simple function. 46 | * If test case is a function, it is considered standalone. That is, the test case should take care of starting glusterd, creating volume, cleaning up the volume after the test etc. The framework itself will not be doing anything to assist the test case function. 47 | * If the test case is a class it should have at least three methods. 48 | 1. `setup` - To setup part of test case. Like volume create maybe. 49 | 2. `run` - This contains the test case steps. This part does all the validations and verifications. 50 | 3. `teardown` - This part contains the tearing down the setup that was done in `setup` 51 | 52 | There is one more method `cleanup`, which is hidden. This method does the hard cleanup of gluster environment. Test case writers are not advised to use this method unless its really required. This method is called internally by DiSTAF when needed. 53 | 54 | As you can see, there are obvious advantages of having test case as python class. So to further make the test case writing easier, a base class is written for gluster tests which does implement `setup` and `teardown` methods. This base class is called `GlusterBaseClass`. So all test cases should use this class as base class which creates a volume for the test case, depending on the values in configuration file. So test case class has to just implement the `run` method which has test case steps, assuming the volume is already created. Read more about these methods in the next section. 55 | 56 | Example skeleton of a test case in DiSTAF. 57 | ```python 58 | from distaf.util import tc, testcase 59 | 60 | @testcase("testcase_skeleton") 61 | class skeleton_gluster_test(GlusterBaseClass): 62 | """ 63 | runs_on_volume: [ distribute, replicate ] 64 | runs_on_protocol: [ glusterfs, nfs ] 65 | resuse_setup: True 66 | summary: This is just a skeleton of a gluster test class in distaf 67 | # The setup and teardown are already implemented in GlusterBaseClass 68 | """ 69 | def run(self): 70 | tc.logger.info("The volume name is %s", self.volname) 71 | pass # Test case steps here 72 | return True 73 | ``` 74 | 75 | #### About the test case metadata 76 | Each test case has three or four metadata about the test case. These fields explain on what conditions, the test case can be run. 77 | 78 | * `runs_on_volume: ALL` - This explains on what all volume types this test case can be run. The possible values are "distribute, replicate, dist_rep, disperse, dist_disperse". As of now DiSTAF only does string comparison, so the value should match. Alternatively you can mention ALL, which will be expanded to all possible volume types. The tiered volume type will be added soon. 79 | * `runs_on_protocol: glusterfs` - The possible mount protocols which can be used to run this test case with. The possible values are glusterfs and nfs. The samba and cifs will be added soon. 80 | * `reuse_setup: True` - If your test case requires a fresh setup of volume (e.g glusterfind), this should be set to False. If your testcase can reuse the existing setup, please set it to True. 81 | 82 | We plan to have few more metadata soon. Like `testcase_tags` and `runs_on_server_version` etc. 83 | 84 | 85 | #### About the methods of test case class 86 | As explained in above section, each test class should have at least `run` method implemented. The `setup` and `teardown` can be used from the base class. 87 | 88 | ##### The `setup` method: 89 | This method is responsible for creating the volumes (if it doesn't exist already). Only override this class with your own implementation if you don't need to create volume as part of setup. Or have some requirement to not to do so. Note that volume will not be mounted as part of this method and has to be taken care in `run` method. Also this method takes care of cleaning up the previous volume and re-creating it if `reuse_setup=False`. So if you override this method, please consider it as well. 90 | ##### The `run` method: 91 | Each test case class is supposed to implement this. This should contain the actual test case steps and should do all validations and verifications needed for the test case. This is not implemented in the base class, so this must be implemented in the test case class. 92 | ##### The `teardown` method: 93 | If should tear down any specific things you do in `run` method. Like unmounting the volume, removing the files maybe etc. 94 | ##### The `cleanup` method: 95 | This is more of a internal method used to hard cleanup while jumping from one volume type to next volume. And this will be called only if the volume type changes from one test case to next test case. 96 | 97 | Now you can start writing your test case (`run` method to be more specific). DiSTAF also has lot of gluster related library function to assist in test case writing. For more information please refer to API guide. 98 | 99 | ## Installing DiSTAF package 100 | Please note that, to install this package you need to have python-setuptools, git(Most likely will be available through yum/apt-get) and python modules like rpyc, pyyaml (will be available through pip) and should be run with root privileges. 101 | ```bash 102 | yum install python-setuptools 103 | easy_install pip 104 | yum install git 105 | pip install rpyc 106 | pip install pyyaml 107 | ``` 108 | The distaf core package can be installed through below command. It will install the distaf core package from the git HEAD available from distaf.git. 109 | ```bash 110 | 111 | pip install git+https://github.com/gluster/distaf@master 112 | ``` 113 | If you have cloned the distaf.git, please follow the below steps to install distaf package 114 | ``` 115 | cd 116 | python setup.py install 117 | ``` 118 | 119 | ## Running the tests written in DiSTAF 120 | Before running the distaf tests, please read the [README](https://github.com/gluster/distaf/blob/master/README.md). So before running, you should have a server with glusterfs installed and a client (if your test case require it). 121 | 122 | #### Updating the config.yml file 123 | DiSTAF reads the run time configuration parameters from the yaml config file. Please [take a look at the sample config file](https://github.com/gluster/distaf/blob/master/config.yml). Most of the fields explain themselves. 124 | * The `remote_user` field is the user with which distaf connects to remote test machines. It is to this user you should setup password-less ssh to. 125 | * All server related details will go to servers field. It has subsection host and devices. 126 | * All client related details will go to clients field. 127 | * You can have fields for volume types and its configurations. 128 | * When global_mode=True, all test cases will be run against the volume type and configuration which is mentioned in the config yaml file and ignores 'runs_on_volume' and 'runs_on_protocol' in testcase metadata. If global_mode=False, each test case will run against all possible types of volume and mount protocol which is mentioned in the testcase metadata. 129 | 130 | #### Starting the DiSTAF run 131 | There are few ways to run the distaf test cases. 132 | 133 | ###### Running all the tests in a directory 134 | ```bash 135 | distaf -c -d "dir_name" 136 | ``` 137 | Note that distaf tries to recursively find all the tests inside. This is helpful when all the tests of a component are together in a directory and you want to run them all. 138 | ###### Running all the tests in a file 139 | ```bash 140 | distaf -c -f 141 | ``` 142 | Make sure that is the file where test case class is implemented. 143 | ###### Running only the tests specified 144 | ```bash 145 | distaf -c -d "dir_to_look" -t "test0 test1 test2" 146 | ``` 147 | Only the tests specified from that directory is executed. If the test case is not found, it is skipped and other test cases which are found are executed. 148 | ###### Get the result in junit style 149 | ```bash 150 | distaf -c "test_dir" -t "Test0 Test1 Test2" -j "result_dir" 151 | ``` 152 | All DiSTAF results are by default text format and thrown to the console. If you rather use Jenkins friendly junit style xml output, you should pass `-j` with a dir where results will be populated. 153 | ###### Running all the tests in a directory with multiple config files 154 | ```bash 155 | distaf -c " " -d "dir_name" 156 | ``` 157 | 158 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " epub3 to make an epub3" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | @echo " dummy to check syntax errors of document sources" 51 | 52 | .PHONY: clean 53 | clean: 54 | rm -rf $(BUILDDIR)/* 55 | 56 | .PHONY: html 57 | html: 58 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 61 | 62 | .PHONY: dirhtml 63 | dirhtml: 64 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 65 | @echo 66 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 67 | 68 | .PHONY: singlehtml 69 | singlehtml: 70 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 71 | @echo 72 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 73 | 74 | .PHONY: pickle 75 | pickle: 76 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 77 | @echo 78 | @echo "Build finished; now you can process the pickle files." 79 | 80 | .PHONY: json 81 | json: 82 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 83 | @echo 84 | @echo "Build finished; now you can process the JSON files." 85 | 86 | .PHONY: htmlhelp 87 | htmlhelp: 88 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 89 | @echo 90 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 91 | ".hhp project file in $(BUILDDIR)/htmlhelp." 92 | 93 | .PHONY: qthelp 94 | qthelp: 95 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 96 | @echo 97 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 98 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 99 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DiSTAF.qhcp" 100 | @echo "To view the help file:" 101 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DiSTAF.qhc" 102 | 103 | .PHONY: applehelp 104 | applehelp: 105 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 106 | @echo 107 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 108 | @echo "N.B. You won't be able to view it unless you put it in" \ 109 | "~/Library/Documentation/Help or install it in your application" \ 110 | "bundle." 111 | 112 | .PHONY: devhelp 113 | devhelp: 114 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 115 | @echo 116 | @echo "Build finished." 117 | @echo "To view the help file:" 118 | @echo "# mkdir -p $$HOME/.local/share/devhelp/DiSTAF" 119 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DiSTAF" 120 | @echo "# devhelp" 121 | 122 | .PHONY: epub 123 | epub: 124 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 125 | @echo 126 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 127 | 128 | .PHONY: epub3 129 | epub3: 130 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 131 | @echo 132 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 133 | 134 | .PHONY: latex 135 | latex: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo 138 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 139 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 140 | "(use \`make latexpdf' here to do that automatically)." 141 | 142 | .PHONY: latexpdf 143 | latexpdf: 144 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 145 | @echo "Running LaTeX files through pdflatex..." 146 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 147 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 148 | 149 | .PHONY: latexpdfja 150 | latexpdfja: 151 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 152 | @echo "Running LaTeX files through platex and dvipdfmx..." 153 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 154 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 155 | 156 | .PHONY: text 157 | text: 158 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 159 | @echo 160 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 161 | 162 | .PHONY: man 163 | man: 164 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 165 | @echo 166 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 167 | 168 | .PHONY: texinfo 169 | texinfo: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo 172 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 173 | @echo "Run \`make' in that directory to run these through makeinfo" \ 174 | "(use \`make info' here to do that automatically)." 175 | 176 | .PHONY: info 177 | info: 178 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 179 | @echo "Running Texinfo files through makeinfo..." 180 | make -C $(BUILDDIR)/texinfo info 181 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 182 | 183 | .PHONY: gettext 184 | gettext: 185 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 186 | @echo 187 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 188 | 189 | .PHONY: changes 190 | changes: 191 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 192 | @echo 193 | @echo "The overview file is in $(BUILDDIR)/changes." 194 | 195 | .PHONY: linkcheck 196 | linkcheck: 197 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 198 | @echo 199 | @echo "Link check complete; look for any errors in the above output " \ 200 | "or in $(BUILDDIR)/linkcheck/output.txt." 201 | 202 | .PHONY: doctest 203 | doctest: 204 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 205 | @echo "Testing of doctests in the sources finished, look at the " \ 206 | "results in $(BUILDDIR)/doctest/output.txt." 207 | 208 | .PHONY: coverage 209 | coverage: 210 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 211 | @echo "Testing of coverage in the sources finished, look at the " \ 212 | "results in $(BUILDDIR)/coverage/python.txt." 213 | 214 | .PHONY: xml 215 | xml: 216 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 217 | @echo 218 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 219 | 220 | .PHONY: pseudoxml 221 | pseudoxml: 222 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 223 | @echo 224 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 225 | 226 | .PHONY: dummy 227 | dummy: 228 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 229 | @echo 230 | @echo "Build finished. Dummy builder generates no files." 231 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # DiSTAF documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Jun 1 19:57:06 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | sys.path.insert(0, os.path.abspath('..')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.coverage', 34 | 'sphinx.ext.napoleon', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'DiSTAF' 53 | copyright = u'2016, DiSTAF Dev Team' 54 | author = '' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = u'0.1' 62 | # The full version, including alpha/beta/rc tags. 63 | release = u'0.1' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # There are two options for replacing |today|: either, you set today to some 73 | # non-false value, then it is used: 74 | #today = '' 75 | # Else, today_fmt is used as the format for a strftime call. 76 | #today_fmt = '%B %d, %Y' 77 | 78 | # List of patterns, relative to source directory, that match files and 79 | # directories to ignore when looking for source files. 80 | # This patterns also effect to html_static_path and html_extra_path 81 | exclude_patterns = ['_build'] 82 | 83 | # The reST default role (used for this markup: `text`) to use for all 84 | # documents. 85 | #default_role = None 86 | 87 | # If true, '()' will be appended to :func: etc. cross-reference text. 88 | #add_function_parentheses = True 89 | 90 | # If true, the current module name will be prepended to all description 91 | # unit titles (such as .. function::). 92 | #add_module_names = True 93 | 94 | # If true, sectionauthor and moduleauthor directives will be shown in the 95 | # output. They are ignored by default. 96 | #show_authors = False 97 | 98 | # The name of the Pygments (syntax highlighting) style to use. 99 | pygments_style = 'sphinx' 100 | 101 | # A list of ignored prefixes for module index sorting. 102 | #modindex_common_prefix = [] 103 | 104 | # If true, keep warnings as "system message" paragraphs in the built documents. 105 | #keep_warnings = False 106 | 107 | # If true, `todo` and `todoList` produce output, else they produce nothing. 108 | todo_include_todos = False 109 | 110 | 111 | # -- Options for HTML output ---------------------------------------------- 112 | 113 | # The theme to use for HTML and HTML Help pages. See the documentation for 114 | # a list of builtin themes. 115 | html_theme = 'default' 116 | 117 | # Theme options are theme-specific and customize the look and feel of a theme 118 | # further. For a list of options available for each theme, see the 119 | # documentation. 120 | #html_theme_options = {} 121 | 122 | # Add any paths that contain custom themes here, relative to this directory. 123 | #html_theme_path = [] 124 | 125 | # The name for this set of Sphinx documents. 126 | # " v documentation" by default. 127 | #html_title = u'DiSTAF v0.1' 128 | 129 | # A shorter title for the navigation bar. Default is the same as html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the top 133 | # of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (relative to this directory) to use as a favicon of 137 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 138 | # pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) here, 142 | # relative to this directory. They are copied after the builtin static files, 143 | # so a file named "default.css" will overwrite the builtin "default.css". 144 | html_static_path = ['_static'] 145 | 146 | # Add any extra paths that contain custom files (such as robots.txt or 147 | # .htaccess) here, relative to this directory. These files are copied 148 | # directly to the root of the documentation. 149 | #html_extra_path = [] 150 | 151 | # If not None, a 'Last updated on:' timestamp is inserted at every page 152 | # bottom, using the given strftime format. 153 | # The empty string is equivalent to '%b %d, %Y'. 154 | #html_last_updated_fmt = None 155 | 156 | # If true, SmartyPants will be used to convert quotes and dashes to 157 | # typographically correct entities. 158 | #html_use_smartypants = True 159 | 160 | # Custom sidebar templates, maps document names to template names. 161 | #html_sidebars = {} 162 | 163 | # Additional templates that should be rendered to pages, maps page names to 164 | # template names. 165 | #html_additional_pages = {} 166 | 167 | # If false, no module index is generated. 168 | #html_domain_indices = True 169 | 170 | # If false, no index is generated. 171 | #html_use_index = True 172 | 173 | # If true, the index is split into individual pages for each letter. 174 | #html_split_index = False 175 | 176 | # If true, links to the reST sources are added to the pages. 177 | #html_show_sourcelink = True 178 | 179 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 180 | #html_show_sphinx = True 181 | 182 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 183 | #html_show_copyright = True 184 | 185 | # If true, an OpenSearch description file will be output, and all pages will 186 | # contain a tag referring to it. The value of this option must be the 187 | # base URL from which the finished HTML is served. 188 | #html_use_opensearch = '' 189 | 190 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 191 | #html_file_suffix = None 192 | 193 | # Language to be used for generating the HTML full-text search index. 194 | # Sphinx supports the following languages: 195 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 196 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' 197 | #html_search_language = 'en' 198 | 199 | # A dictionary with options for the search language support, empty by default. 200 | # 'ja' uses this config value. 201 | # 'zh' user can custom change `jieba` dictionary path. 202 | #html_search_options = {'type': 'default'} 203 | 204 | # The name of a javascript file (relative to the configuration directory) that 205 | # implements a search results scorer. If empty, the default will be used. 206 | #html_search_scorer = 'scorer.js' 207 | 208 | # Output file base name for HTML help builder. 209 | htmlhelp_basename = 'DiSTAFdoc' 210 | 211 | # -- Options for LaTeX output --------------------------------------------- 212 | 213 | latex_elements = { 214 | # The paper size ('letterpaper' or 'a4paper'). 215 | #'papersize': 'letterpaper', 216 | 217 | # The font size ('10pt', '11pt' or '12pt'). 218 | #'pointsize': '10pt', 219 | 220 | # Additional stuff for the LaTeX preamble. 221 | #'preamble': '', 222 | 223 | # Latex figure (float) alignment 224 | #'figure_align': 'htbp', 225 | } 226 | 227 | # Grouping the document tree into LaTeX files. List of tuples 228 | # (source start file, target name, title, 229 | # author, documentclass [howto, manual, or own class]). 230 | latex_documents = [ 231 | (master_doc, 'DiSTAF.tex', u'DiSTAF Documentation', 232 | u'DiSTAF Dev Team', 'manual'), 233 | ] 234 | 235 | # The name of an image file (relative to this directory) to place at the top of 236 | # the title page. 237 | #latex_logo = None 238 | 239 | # For "manual" documents, if this is true, then toplevel headings are parts, 240 | # not chapters. 241 | #latex_use_parts = False 242 | 243 | # If true, show page references after internal links. 244 | #latex_show_pagerefs = False 245 | 246 | # If true, show URL addresses after external links. 247 | #latex_show_urls = False 248 | 249 | # Documents to append as an appendix to all manuals. 250 | #latex_appendices = [] 251 | 252 | # If false, no module index is generated. 253 | #latex_domain_indices = True 254 | 255 | 256 | # -- Options for manual page output --------------------------------------- 257 | 258 | # One entry per manual page. List of tuples 259 | # (source start file, name, description, authors, manual section). 260 | man_pages = [ 261 | (master_doc, 'distaf', u'DiSTAF Documentation', 262 | [author], 1) 263 | ] 264 | 265 | # If true, show URL addresses after external links. 266 | #man_show_urls = False 267 | 268 | 269 | # -- Options for Texinfo output ------------------------------------------- 270 | 271 | # Grouping the document tree into Texinfo files. List of tuples 272 | # (source start file, target name, title, author, 273 | # dir menu entry, description, category) 274 | texinfo_documents = [ 275 | (master_doc, 'DiSTAF', u'DiSTAF Documentation', 276 | author, 'DiSTAF', 'One line description of project.', 277 | 'Miscellaneous'), 278 | ] 279 | 280 | # Documents to append as an appendix to all manuals. 281 | #texinfo_appendices = [] 282 | 283 | # If false, no module index is generated. 284 | #texinfo_domain_indices = True 285 | 286 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 287 | #texinfo_show_urls = 'footnote' 288 | 289 | # If true, do not generate a @detailmenu in the "Top" node's menu. 290 | #texinfo_no_detailmenu = False 291 | 292 | 293 | # -- Options for Epub output ---------------------------------------------- 294 | 295 | # Bibliographic Dublin Core info. 296 | epub_title = project 297 | epub_author = author 298 | epub_publisher = author 299 | epub_copyright = copyright 300 | 301 | # The basename for the epub file. It defaults to the project name. 302 | #epub_basename = project 303 | 304 | # The HTML theme for the epub output. Since the default themes are not 305 | # optimized for small screen space, using the same theme for HTML and epub 306 | # output is usually not wise. This defaults to 'epub', a theme designed to save 307 | # visual space. 308 | #epub_theme = 'epub' 309 | 310 | # The language of the text. It defaults to the language option 311 | # or 'en' if the language is not set. 312 | #epub_language = '' 313 | 314 | # The scheme of the identifier. Typical schemes are ISBN or URL. 315 | #epub_scheme = '' 316 | 317 | # The unique identifier of the text. This can be a ISBN number 318 | # or the project homepage. 319 | #epub_identifier = '' 320 | 321 | # A unique identification for the text. 322 | #epub_uid = '' 323 | 324 | # A tuple containing the cover image and cover page html template filenames. 325 | #epub_cover = () 326 | 327 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 328 | #epub_guide = () 329 | 330 | # HTML files that should be inserted before the pages created by sphinx. 331 | # The format is a list of tuples containing the path and title. 332 | #epub_pre_files = [] 333 | 334 | # HTML files that should be inserted after the pages created by sphinx. 335 | # The format is a list of tuples containing the path and title. 336 | #epub_post_files = [] 337 | 338 | # A list of files that should not be packed into the epub file. 339 | epub_exclude_files = ['search.html'] 340 | 341 | # The depth of the table of contents in toc.ncx. 342 | #epub_tocdepth = 3 343 | 344 | # Allow duplicate toc entries. 345 | #epub_tocdup = True 346 | 347 | # Choose between 'default' and 'includehidden'. 348 | #epub_tocscope = 'default' 349 | 350 | # Fix unsupported image types using the Pillow. 351 | #epub_fix_images = False 352 | 353 | # Scale large images. 354 | #epub_max_image_width = 0 355 | 356 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 357 | #epub_show_urls = 'inline' 358 | 359 | # If false, no index is generated. 360 | #epub_use_index = True 361 | 362 | autodoc_member_order = 'bysource' 363 | autoclass_content = 'both' 364 | 365 | napoleon_google_docstring = True 366 | napoleon_numpy_docstring = True 367 | napoleon_include_private_with_doc = True 368 | napoleon_include_special_with_doc = True 369 | napoleon_use_admonition_for_examples = False 370 | napoleon_use_admonition_for_notes = False 371 | napoleon_use_admonition_for_references = False 372 | napoleon_use_ivar = False 373 | napoleon_use_param = True 374 | napoleon_use_rtype = True 375 | 376 | 377 | -------------------------------------------------------------------------------- /docs/images/distaf_acrhitecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/distaf/62f5892153ec0f36283fb26e4e31549f18f87d26/docs/images/distaf_acrhitecture.jpg -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. DiSTAF documentation master file, created by 2 | sphinx-quickstart on Wed Jun 1 19:57:06 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to DiSTAF's documentation! 7 | ================================== 8 | 9 | User Guide 10 | ---------- 11 | 12 | .. toctree:: 13 | :maxdepth: 3 14 | 15 | userguide/overview 16 | userguide/howto 17 | 18 | 19 | API 20 | --- 21 | 22 | .. toctree:: 23 | :maxdepth: 2 24 | 25 | source/distaf 26 | 27 | 28 | Related Libraries 29 | ----------------- 30 | 31 | `distaflibs-gluster: Library for Red Hat Gluster related features. `_ 32 | 33 | 34 | Indices and Tables 35 | ------------------ 36 | 37 | * :ref:`genindex` 38 | * :ref:`modindex` 39 | * :ref:`search` 40 | 41 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | echo. dummy to check syntax errors of document sources 43 | goto end 44 | ) 45 | 46 | if "%1" == "clean" ( 47 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 48 | del /q /s %BUILDDIR%\* 49 | goto end 50 | ) 51 | 52 | 53 | REM Check if sphinx-build is available and fallback to Python version if any 54 | %SPHINXBUILD% 1>NUL 2>NUL 55 | if errorlevel 9009 goto sphinx_python 56 | goto sphinx_ok 57 | 58 | :sphinx_python 59 | 60 | set SPHINXBUILD=python -m sphinx.__init__ 61 | %SPHINXBUILD% 2> nul 62 | if errorlevel 9009 ( 63 | echo. 64 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 65 | echo.installed, then set the SPHINXBUILD environment variable to point 66 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 67 | echo.may add the Sphinx directory to PATH. 68 | echo. 69 | echo.If you don't have Sphinx installed, grab it from 70 | echo.http://sphinx-doc.org/ 71 | exit /b 1 72 | ) 73 | 74 | :sphinx_ok 75 | 76 | 77 | if "%1" == "html" ( 78 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 79 | if errorlevel 1 exit /b 1 80 | echo. 81 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 82 | goto end 83 | ) 84 | 85 | if "%1" == "dirhtml" ( 86 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 87 | if errorlevel 1 exit /b 1 88 | echo. 89 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 90 | goto end 91 | ) 92 | 93 | if "%1" == "singlehtml" ( 94 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 95 | if errorlevel 1 exit /b 1 96 | echo. 97 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 98 | goto end 99 | ) 100 | 101 | if "%1" == "pickle" ( 102 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 103 | if errorlevel 1 exit /b 1 104 | echo. 105 | echo.Build finished; now you can process the pickle files. 106 | goto end 107 | ) 108 | 109 | if "%1" == "json" ( 110 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 111 | if errorlevel 1 exit /b 1 112 | echo. 113 | echo.Build finished; now you can process the JSON files. 114 | goto end 115 | ) 116 | 117 | if "%1" == "htmlhelp" ( 118 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished; now you can run HTML Help Workshop with the ^ 122 | .hhp project file in %BUILDDIR%/htmlhelp. 123 | goto end 124 | ) 125 | 126 | if "%1" == "qthelp" ( 127 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 131 | .qhcp project file in %BUILDDIR%/qthelp, like this: 132 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\DiSTAF.qhcp 133 | echo.To view the help file: 134 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\DiSTAF.ghc 135 | goto end 136 | ) 137 | 138 | if "%1" == "devhelp" ( 139 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 140 | if errorlevel 1 exit /b 1 141 | echo. 142 | echo.Build finished. 143 | goto end 144 | ) 145 | 146 | if "%1" == "epub" ( 147 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 148 | if errorlevel 1 exit /b 1 149 | echo. 150 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 151 | goto end 152 | ) 153 | 154 | if "%1" == "epub3" ( 155 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 156 | if errorlevel 1 exit /b 1 157 | echo. 158 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 159 | goto end 160 | ) 161 | 162 | if "%1" == "latex" ( 163 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 164 | if errorlevel 1 exit /b 1 165 | echo. 166 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdf" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "latexpdfja" ( 181 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 182 | cd %BUILDDIR%/latex 183 | make all-pdf-ja 184 | cd %~dp0 185 | echo. 186 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 187 | goto end 188 | ) 189 | 190 | if "%1" == "text" ( 191 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Build finished. The text files are in %BUILDDIR%/text. 195 | goto end 196 | ) 197 | 198 | if "%1" == "man" ( 199 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 200 | if errorlevel 1 exit /b 1 201 | echo. 202 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 203 | goto end 204 | ) 205 | 206 | if "%1" == "texinfo" ( 207 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 208 | if errorlevel 1 exit /b 1 209 | echo. 210 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 211 | goto end 212 | ) 213 | 214 | if "%1" == "gettext" ( 215 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 216 | if errorlevel 1 exit /b 1 217 | echo. 218 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 219 | goto end 220 | ) 221 | 222 | if "%1" == "changes" ( 223 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 224 | if errorlevel 1 exit /b 1 225 | echo. 226 | echo.The overview file is in %BUILDDIR%/changes. 227 | goto end 228 | ) 229 | 230 | if "%1" == "linkcheck" ( 231 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 232 | if errorlevel 1 exit /b 1 233 | echo. 234 | echo.Link check complete; look for any errors in the above output ^ 235 | or in %BUILDDIR%/linkcheck/output.txt. 236 | goto end 237 | ) 238 | 239 | if "%1" == "doctest" ( 240 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 241 | if errorlevel 1 exit /b 1 242 | echo. 243 | echo.Testing of doctests in the sources finished, look at the ^ 244 | results in %BUILDDIR%/doctest/output.txt. 245 | goto end 246 | ) 247 | 248 | if "%1" == "coverage" ( 249 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 250 | if errorlevel 1 exit /b 1 251 | echo. 252 | echo.Testing of coverage in the sources finished, look at the ^ 253 | results in %BUILDDIR%/coverage/python.txt. 254 | goto end 255 | ) 256 | 257 | if "%1" == "xml" ( 258 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 259 | if errorlevel 1 exit /b 1 260 | echo. 261 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 262 | goto end 263 | ) 264 | 265 | if "%1" == "pseudoxml" ( 266 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 267 | if errorlevel 1 exit /b 1 268 | echo. 269 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 270 | goto end 271 | ) 272 | 273 | if "%1" == "dummy" ( 274 | %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy 275 | if errorlevel 1 exit /b 1 276 | echo. 277 | echo.Build finished. Dummy builder generates no files. 278 | goto end 279 | ) 280 | 281 | :end 282 | -------------------------------------------------------------------------------- /docs/source/distaf.client_rpyc.rst: -------------------------------------------------------------------------------- 1 | distaf.client_rpyc module 2 | ========================= 3 | 4 | .. automodule:: distaf.client_rpyc 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/distaf.config_parser.rst: -------------------------------------------------------------------------------- 1 | distaf.config_parser module 2 | =========================== 3 | 4 | .. automodule:: distaf.config_parser 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/distaf.main.rst: -------------------------------------------------------------------------------- 1 | distaf.main module 2 | ================== 3 | 4 | .. automodule:: distaf.main 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/distaf.rst: -------------------------------------------------------------------------------- 1 | distaf package 2 | ============== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | 9 | distaf.client_rpyc 10 | distaf.config_parser 11 | distaf.main 12 | distaf.util 13 | 14 | Module contents 15 | --------------- 16 | 17 | .. automodule:: distaf 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /docs/source/distaf.util.rst: -------------------------------------------------------------------------------- 1 | distaf.util module 2 | ================== 3 | 4 | .. automodule:: distaf.util 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | distaf 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | distaf 8 | -------------------------------------------------------------------------------- /docs/userguide/howto.rst: -------------------------------------------------------------------------------- 1 | How To 2 | ------ 3 | 4 | DiSTAF (or distaf) is a test framework for distributed systems like glusterfs. 5 | This file contains information about how to write test cases in distaf framework 6 | and how to execute them once they are written. For information about overview 7 | and architecture, please refer to [README.md](https://github.com/gluster/distaf/blob/master/README.md). 8 | 9 | A Little About the APIs 10 | ======================= 11 | DiSTAF exposes few APIs to interact and control the remote test machines. 12 | That is the core part of distaf. And since most test steps of glusterfs are 13 | bash/shell commands (at least on the server side), distaf exposes two APIs to 14 | run the bash/shell commands remotely. These APIs are actually methods of 15 | connection manager object and will be accessible through the object `tc` 16 | 17 | .. code-block:: python 18 | :linenos: 19 | 20 | (ret, stdout, stderr) = tc.run(node, command, user='root', verbose=True) 21 | 22 | 23 | So the `run` method execute the bash `command` in remote `node` as 24 | user `root` synchronously and returns a python tuple of return code, output 25 | in stdout and output in stderr. 26 | 27 | The asynchronous version of `run` method returns a `popen` object which is 28 | very similar to `subprocess.Popen`. The object has methods to wait, 29 | communicate and get the return code etc. It has has another 30 | method `value()` to get the tuple of `(ret, stdout, stderr)` 31 | 32 | .. code-block:: python 33 | :linenos: 34 | 35 | popen = tc.run_async(node, command, user='root', verbose=True) 36 | popen.wait() 37 | (ret, stdout, stderr) = popen.value() 38 | 39 | These are most used APIs for distaf. You can also request for a connection 40 | and run python command on the remote machines. The below example should make 41 | things bit more clear. 42 | 43 | .. code-block:: python 44 | :linenos: 45 | 46 | conn = tc.get_connection(node, user='root') 47 | if conn == -1: 48 | tc.logger.error("Unable to establish connection to %s@%s", node, user) 49 | return False 50 | # Remote python modules are accessible through conn.modules. 51 | pwd = conn.modules.os.getcwd() 52 | conn.modules.os.chdir("/tmp") # Stateful. Remains in /tmp 53 | pwd = conn.modules.os.getcwd() 54 | r_hostname = conn.modules.socket.gethostname() 55 | 56 | For more info about all the APIs, please refer "DiSTAF API Guide" 57 | 58 | Writing Tests in DiSTAF 59 | ======================= 60 | 61 | Things to mind before writing the tests 62 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 63 | 64 | * DiSTAF expects that the file in which test cases are written starts with `test_`. For example `test_basic_gluster.py`. 65 | * And each test case file needs to import connection manager object to interact with the remote test nodes. 66 | * Each of the test case should have a decorator `@testcase` with the test case name. 67 | * Each test case should have its metadata, namely the volume type it can run on, the possible mount protocols with which it can be run and if fresh setup is required to run that test. The next section will explain a bit more about these metadata. 68 | * A test case can be a class or a simple function. 69 | * If test case is a function, it is considered standalone. That is, the test case should take care of starting glusterd, creating volume, cleaning up the volume after the test etc. The framework itself will not be doing anything to assist the test case function. 70 | * If the test case is a class it should have at least three methods. 71 | 1. `setup` - To setup part of test case. Like volume create maybe. 72 | 2. `run` - This contains the test case steps. This part does all the validations and verifications. 73 | 3. `teardown` - This part contains the tearing down the setup that was done in `setup` 74 | 75 | There is one more method `cleanup`, which is hidden. This method does the hard cleanup of gluster environment. Test case writers are not advised to use this method unless its really required. This method is called internally by DiSTAF when needed. 76 | 77 | As you can see, there are obvious advantages of having test case as python class. So to further make the test case writing easier, a base class is written for gluster tests which does implement `setup` and `teardown` methods. This base class is called `GlusterBaseClass`. So all test cases should use this class as base class which creates a volume for the test case, depending on the values in configuration file. So test case class has to just implement the `run` method which has test case steps, assuming the volume is already created. Read more about these methods in the next section. 78 | 79 | Example skeleton of a test case in DiSTAF. 80 | 81 | .. code-block:: python 82 | :linenos: 83 | 84 | from distaf.util import tc, testcase 85 | 86 | @testcase("testcase_skeleton") 87 | class skeleton_gluster_test(GlusterBaseClass): 88 | """ 89 | runs_on_volume: [ distribute, replicate ] 90 | runs_on_protocol: [ glusterfs, nfs ] 91 | resuse_setup: True 92 | summary: This is just a skeleton of a gluster test class in distaf 93 | # The setup and teardown are already implemented in GlusterBaseClass 94 | """ 95 | def run(self): 96 | tc.logger.info("The volume name is %s", self.volname) 97 | pass # Test case steps here 98 | return True 99 | 100 | About the Testcase Metadata 101 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 102 | Each test case has three or four metadata about the test case. These fields 103 | explain on what conditions, the test case can be run. 104 | 105 | * `runs_on_volume: ALL` - This explains on what all volume types this test case can be run. The possible values are "distribute, replicate, dist_rep, disperse, dist_disperse". As of now DiSTAF only does string comparison, so the value should match. Alternatively you can mention ALL, which will be expanded to all possible volume types. The tiered volume type will be added soon. 106 | * `runs_on_protocol: glusterfs` - The possible mount protocols which can be used to run this test case with. The possible values are glusterfs and nfs. The samba and cifs will be added soon. 107 | * `reuse_setup: True` - If your test case requires a fresh setup of volume (e.g glusterfind), this should be set to False. If your testcase can reuse the existing setup, please set it to True. 108 | 109 | We plan to have few more metadata soon. Like `testcase_tags` and `runs_on_server_version` etc. 110 | 111 | 112 | About the Methods of Test Case Class 113 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 114 | As explained in above section, each test class should have at least `run` method implemented. The `setup` and `teardown` can be used from the base class. 115 | 116 | The `setup` method 117 | '''''''''''''''''' 118 | This method is responsible for creating the volumes (if it doesn't exist already). 119 | Only override this class with your own implementation if you don't need to create volume 120 | as part of setup. Or have some requirement to not to do so. Note that volume will not be 121 | mounted as part of this method and has to be taken care in `run` method. Also this method 122 | takes care of cleaning up the previous volume and re-creating it if `reuse_setup=False`. 123 | So if you override this method, please consider it as well. 124 | 125 | The `run` method 126 | '''''''''''''''' 127 | Each test case class is supposed to implement this. This should contain the actual test 128 | case steps and should do all validations and verifications needed for the test case. 129 | This is not implemented in the base class, so this must be implemented in the test case class. 130 | 131 | The `teardown` method 132 | ''''''''''''''''''''' 133 | If should tear down any specific things you do in `run` method. Like unmounting the volume, 134 | removing the files maybe etc. 135 | 136 | The `cleanup` method 137 | '''''''''''''''''''' 138 | This is more of a internal method used to hard cleanup while jumping from one volume type 139 | to next volume. And this will be called only if the volume type changes from one test case 140 | to next test case. 141 | 142 | Now you can start writing your test case (`run` method to be more specific). 143 | DiSTAF also has lot of gluster related library function to assist in test case writing. 144 | For more information please refer to API guide. 145 | 146 | Installing DiSTAF package 147 | ========================= 148 | 149 | .. Note:: 150 | 151 | Please note that, to install this package you need to have python-setuptools, 152 | git(Most likely will be available through yum/apt-get) and python modules like rpyc, 153 | pyyaml (will be available through pip) and should be run with root privileges. 154 | 155 | 156 | :: 157 | 158 | yum install python-setuptools 159 | easy_install pip 160 | yum install git 161 | pip install rpyc 162 | pip install pyyaml 163 | 164 | The distaf core package can be installed through below command. 165 | It will install the distaf core package from the git HEAD available from distaf.git. 166 | 167 | :: 168 | 169 | $ pip install git+https://github.com/gluster/distaf@master 170 | 171 | If you have cloned the distaf.git, please follow the below steps to install distaf package 172 | 173 | :: 174 | 175 | $ cd 176 | $ python setup.py install 177 | 178 | 179 | Running the Tests Written in DiSTAF 180 | =================================== 181 | Before running the distaf tests, please read the 182 | [README](https://github.com/gluster/distaf/blob/master/README.md). 183 | So before running, you should have a server with glusterfs installed and a client 184 | (if your test case require it). 185 | 186 | Updating the config.yml File 187 | ============================ 188 | DiSTAF reads the run time configuration parameters from the yaml config file. 189 | Please `take a look at the sample config file `__. 190 | Most of the fields explain themselves. 191 | 192 | * The `remote_user` field is the user with which distaf connects to remote test machines. It is to this user you should setup password-less ssh to. 193 | 194 | * All server related details will go to servers field. It has subsection host and devices. 195 | 196 | * All client related details will go to clients field. 197 | 198 | * You can have fields for volume types and its configurations. 199 | 200 | * When global_mode=True, all test cases will be run against the volume type and configuration which is mentioned in the config yaml file and ignores 'runs_on_volume' and 'runs_on_protocol' in testcase metadata. If global_mode=False, each test case will run against all possible types of volume and mount protocol which is mentioned in the testcase metadata. 201 | 202 | Starting the DiSTAF Run 203 | ======================= 204 | There are few ways to run the distaf test cases. 205 | 206 | **Running all the tests in a directory** 207 | 208 | :: 209 | 210 | $ distaf -c -d "dir_name" 211 | 212 | Note that distaf tries to recursively find all the tests inside. 213 | This is helpful when all the tests of a component are together in a directory and you want to run them all. 214 | 215 | **Running all the tests in a file** 216 | 217 | :: 218 | 219 | $ distaf -c -f 220 | 221 | Make sure that is the file where test case class is implemented. 222 | 223 | **Running only the tests specified** 224 | 225 | :: 226 | 227 | $ distaf -c -d "dir_to_look" -t "test0 test1 test2" 228 | 229 | Only the tests specified from that directory is executed. 230 | If the test case is not found, it is skipped and other test cases which are found are executed. 231 | 232 | 233 | **Get the result in junit style** 234 | 235 | :: 236 | 237 | $ distaf -c "test_dir" -t "Test0 Test1 Test2" -j "result_dir" 238 | 239 | All DiSTAF results are by default text format and thrown to the console. 240 | If you rather use Jenkins friendly junit style xml output, you should pass `-j` with a dir where results will be populated. 241 | 242 | **Running all the tests in a directory with multiple config files** 243 | 244 | :: 245 | 246 | $ distaf -c " " -d "dir_name" 247 | -------------------------------------------------------------------------------- /docs/userguide/overview.rst: -------------------------------------------------------------------------------- 1 | DiSTAF - Di'stributed Systems Test Automation Framework 2 | ------------------------------------------------------- 3 | 4 | DiSTAF is a test automation framework for distributed systems. 5 | It is used for test automation of glusterfs and its related projects. 6 | This framework is written with modularity in mind. So many parts of it can 7 | be modified for the liking of the project, without affecting other parts. 8 | DiSTAF can be used to test projects which runs on physical machines, virtual 9 | machines or even containers. DiSTAF requires remote machines (or containers) 10 | to be reachable by IP address (or FQDN). On Linux systems it requires sshd 11 | to be running in the remote systems with bash environment as well. 12 | 13 | 14 | About the Name 15 | ============== 16 | DiSTAF (or distaf) is short for Di'stributed Systems Test Automation Framework. 17 | 'distaff' is a tool used in spinning, which is designed to hold unspun 18 | fibres together keeping them untangled and thus easing the process of spinning. 19 | This framework is also trying to do just that, keeping the machines untangled and 20 | easing the process of writing and executing the test cases. Hence the name DiSTAF (distaf). 21 | 22 | Architecture of the Framework 23 | ============================= 24 | 25 | Terminologies used: 26 | *Management node* 27 | The node from which this test suite is executed. 28 | This node is responsible for orchestration of test automation. 29 | *Test machines* 30 | These include all the systems that participate in the tests. 31 | This can be physical machines, VMs or containers. 32 | 33 | .. figure:: ../images/distaf_acrhitecture.jpg 34 | :align: center 35 | :width: 1200 36 | 37 | Architecture of DiSTAF 38 | 39 | To run distaf, passwordless ssh should be setup from *management node* to all 40 | the *test machines*. The *management node* connects to *test machines* using 41 | rpyc zero-deploy, which internally makes use of ssh tunneling protocol for 42 | establishing and maintaining the secure connections. The connection is kept 43 | open for the entire duration of the tests. All the synchronous commands run by 44 | the test cases, uses this connection to run them. For asynchronous calls, a 45 | new connection is opened. This connection will be closed when async command returns. 46 | 47 | DiSTAF uses `python-unittest` for running tests and generating the results. 48 | 49 | When a test run is started, DiSTAF first reads a *config file*, which is in yaml format. 50 | The *config file* will have information about servers and clients DiSTAF can connect to. 51 | DiSTAF establishes a ssh connection to each of the servers and clients, 52 | and maintains the connection until the end of the test run. 53 | All the remote commands, bash or python will go through this connection. 54 | DiSTAF provides two APIs to run commands on the servers or clients synchronously and asynchronously. 55 | For more information about distaf APIs, please refer HOWTO. 56 | 57 | Testcase Philosophy 58 | =================== 59 | 60 | DiSTAF has two modes of running. The **Global Mode** and **Non-global Mode**. 61 | There is a configuration variable in config.yml **global_mode** to toggle between them. 62 | The idea here is that each test case should be independent of the volume type and access 63 | protocol used to mount the volume. 64 | 65 | When the distaf is started in the *non-global mode*, 66 | it runs each test case against all the volume type and mount protocol combinations. 67 | This means a single test case will run many times and each time a different volume and mount combination is used. 68 | Each test case will have it's own metadata in yaml format in test case docstring. 69 | For more information about the fields and values of test case metadata (test case config), please refer to HOWTO. 70 | 71 | When distaf is started in *global mode*, each test case is run only once. 72 | The volume type and mount protocol specified in the config.yml is used for each test case. 73 | This is helpful if a test case needs to run against a particular type of volume, to run some checks. 74 | 75 | A few things to take care before running test case in DiSTAF. 76 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 77 | * Setting up and provisioning the test machines. This needs to be handled before running distaf tests. 78 | * Updating the config.yml and setting up password-less ssh from management node to test machines. 79 | * Keeping the test machines in the same state if a test case fails. Since distaf does not manage the bringing up and maintaining the test machine, this should be handled outside distaf as well. 80 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | distaf/main.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2015 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | # 18 | 19 | from setuptools import setup 20 | 21 | name = 'distaf' 22 | 23 | setup( 24 | name=name, 25 | version='0.0.3', 26 | description='Di\'stributed Systems Test Automation Framework', 27 | license='GPLv2+', 28 | author='Red Hat, Inc.', 29 | author_email='gluster-devel@gluster.org', 30 | url='http://www.gluster.org', 31 | packages=['distaf'], 32 | classifiers=[ 33 | 'Development Status :: 4 - Beta' 34 | 'Environment :: Console' 35 | 'Intended Audience :: Developers' 36 | 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)' 37 | 'Operating System :: POSIX :: Linux' 38 | 'Programming Language :: Python' 39 | 'Programming Language :: Python :: 2' 40 | 'Programming Language :: Python :: 2.6' 41 | 'Programming Language :: Python :: 2.7' 42 | 'Topic :: Software Development :: Testing' 43 | ], 44 | install_requires=['plumbum', 'rpyc', 'unittest-xml-reporting'], 45 | entry_points={ 46 | 'console_scripts': [ 47 | 'distaf = distaf.main:main', 48 | ] 49 | } 50 | ) 51 | -------------------------------------------------------------------------------- /tests_d/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/distaf/62f5892153ec0f36283fb26e4e31549f18f87d26/tests_d/__init__.py -------------------------------------------------------------------------------- /tests_d/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gluster/distaf/62f5892153ec0f36283fb26e4e31549f18f87d26/tests_d/examples/__init__.py -------------------------------------------------------------------------------- /tests_d/examples/test_async.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.distaf_base_class import DistafTestClass 21 | 22 | 23 | @testcase("run_async") 24 | class RunAsync(DistafTestClass): 25 | """Run a command against all servers via run_async() 26 | 27 | Run with... 28 | time python main.py -d examples -t run_async 29 | """ 30 | def setup(self): 31 | return True 32 | 33 | def run(self): 34 | retstat = 0 35 | tc.logger.info("Run command on all servers via run_async()") 36 | 37 | rasyncs = {} 38 | results = {} 39 | command = 'ls -ail; hostname' 40 | for node in tc.nodes: 41 | rasyncs[node] = tc.run_async(node, command) 42 | if not rasyncs[node]: 43 | retstat = retstat | rcode 44 | 45 | for node, proc in rasyncs.items(): 46 | rcode, rout, rerr = proc.value() 47 | retstat = retstat | rcode 48 | 49 | if retstat == 0: 50 | return True 51 | 52 | return False 53 | 54 | def cleanup(self): 55 | return True 56 | 57 | def teardown(self): 58 | return True 59 | -------------------------------------------------------------------------------- /tests_d/examples/test_basic_gluster_tests.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.gluster_base_class import GlusterBaseClass 21 | from distaf.mount_ops import mount_volume, umount_volume 22 | from distaf.volume_ops import setup_vol, stop_volume, delete_volume 23 | 24 | 25 | @testcase("gluster_basic_test") 26 | class gluster_basic_test(GlusterBaseClass): 27 | """ 28 | runs_on_volumes: ALL 29 | runs_on_protocol: [ glusterfs, nfs ] 30 | reuse_setup: True 31 | """ 32 | def run(self): 33 | _rc = True 34 | client = self.clients[0] 35 | tc.run(self.mnode, "gluster volume status %s" % self.volname) 36 | ret, _, _ = mount_volume(self.volname, self.mount_proto, \ 37 | self.mountpoint, mclient=client) 38 | if ret != 0: 39 | tc.logger.error("Unable to mount the volume %s in %s" \ 40 | "Please check the logs" % (self.volname, client)) 41 | return False 42 | ret, _, _ = tc.run(client, "cp -r /etc %s" % self.mountpoint) 43 | if ret != 0: 44 | tc.logger.error("cp failed in %s. Please check the logs" % client) 45 | _rc = False 46 | tc.run(client, "rm -rf %s/etc" % self.mountpoint) 47 | umount_volume(client, self.mountpoint) 48 | return _rc 49 | -------------------------------------------------------------------------------- /tests_d/examples/test_connections.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.distaf_base_class import DistafTestClass 21 | 22 | 23 | @testcase("exercise_remote_commands") 24 | class ExerciseRemoteConnections(DistafTestClass): 25 | """Run some commands against remote servers to 26 | test connectivity and speed. 27 | 28 | To test speed differences, set combinations of... 29 | connection_engine : ssh_controlpersist (default) 30 | connection_engine : rpyc 31 | connection_engine : ssh 32 | 33 | skip_log_inject : True|False 34 | 35 | Run with... 36 | time python main.py -d examples -t exercise_remote_commands 37 | """ 38 | def setup(self): 39 | return True 40 | 41 | def run(self): 42 | retstat = 0 43 | for i in range(1, 10): 44 | for node in tc.all_nodes: 45 | tc.logger.info("Connection %s %i" % (node, i)) 46 | rcode, _, _ = tc.run(node, "ls -dl /etc") 47 | if rcode != 0: 48 | retstat = retstat | rcode 49 | 50 | rcode, _, _ = tc.run(node, "ls -dl /etc >&2") 51 | if rcode != 0: 52 | retstat = retstat | rcode 53 | 54 | if retstat == 0: 55 | return True 56 | 57 | return False 58 | 59 | def cleanup(self): 60 | return True 61 | 62 | def teardown(self): 63 | return True 64 | -------------------------------------------------------------------------------- /tests_d/examples/test_docstring.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.gluster_base_class import GlusterBaseClass 21 | 22 | 23 | # An example with both doc and config in docstring 24 | @testcase("doc_and_config_test") 25 | class DocAndConfig(GlusterBaseClass): 26 | """ Testing docstring configuration options 27 | This is an example of a basic distaf test with mixed comment and config 28 | Any necessary description doc string text goes here and can include any 29 | plain text normally found in a docstring. 30 | Distaf specific config yaml can be included using the yaml standard 31 | document triple-dash separator below. 32 | --- 33 | runs_on_volumes: [ distributed ] 34 | runs_on_protocol: [ glusterfs ] 35 | reuse_setup: False 36 | tags: 37 | - tag1 38 | - tag2 39 | - tag3 40 | """ 41 | def run(self): 42 | tc.logger.info("Running with doc and config in docstring") 43 | config = self.config_data 44 | tc.logger.debug("Tag 2 is %s" % config["tags"][1]) 45 | tag2 = config["tags"][1] 46 | if tag2 == "tag2": 47 | return True 48 | 49 | return False 50 | 51 | def setup(self): 52 | return True 53 | 54 | def cleanup(self): 55 | return True 56 | 57 | def teardown(self): 58 | return True 59 | 60 | 61 | # An example with only config in docstring 62 | @testcase("config_only_test") 63 | class ConfigOnly(GlusterBaseClass): 64 | """ 65 | runs_on_volumes: [ distributed ] 66 | runs_on_protocol: [ glusterfs, cifs ] 67 | reuse_setup: False 68 | tags: 69 | - tag1 70 | - tag2 71 | - tag3 72 | """ 73 | def run(self): 74 | tc.logger.info("Running with only config in docstring") 75 | config = self.config_data 76 | tc.logger.debug("Tag 2 is %s" % config["tags"][1]) 77 | tag2 = config["tags"][1] 78 | if tag2 == "tag2": 79 | return True 80 | 81 | return False 82 | 83 | def setup(self): 84 | return True 85 | 86 | def cleanup(self): 87 | return True 88 | 89 | def teardown(self): 90 | return True 91 | 92 | 93 | # An example with only doc in docstring 94 | @testcase("doc_only_test") 95 | class DocOnly(GlusterBaseClass): 96 | """ Testing docstring configuration options 97 | This is an example of a basic distaf test with mixed comment and config 98 | Any necessary description doc string text goes here and can include any 99 | plain text normally found in a docstring. 100 | """ 101 | def run(self): 102 | tc.logger.info("Running with only doc in docstring") 103 | return True 104 | 105 | def setup(self): 106 | return True 107 | 108 | def cleanup(self): 109 | return True 110 | 111 | def teardown(self): 112 | return True 113 | 114 | 115 | # An example without a docstring 116 | @testcase("no_docstring_test") 117 | class NoDocstring(GlusterBaseClass): 118 | def run(self): 119 | tc.logger.info("Running with no docstring") 120 | 121 | return True 122 | 123 | def setup(self): 124 | return True 125 | 126 | def cleanup(self): 127 | return True 128 | 129 | def teardown(self): 130 | return True 131 | -------------------------------------------------------------------------------- /tests_d/examples/test_passfail.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.gluster_base_class import GlusterBaseClass 21 | 22 | 23 | @testcase("this_should_pass") 24 | class GoingToPass(GlusterBaseClass): 25 | """ Testing connectivity and framework pass 26 | This is an example of a basic distaf test with mixed comment and config 27 | Any necessary description doc string text goes here and can include any 28 | plain text normally found in a docstring. 29 | Distaf specific config yaml can be included using the yaml standard 30 | document triple-dash separator below. 31 | --- 32 | runs_on_volumes: [ distributed ] 33 | runs_on_protocol: [ glusterfs ] 34 | reuse_setup: False 35 | tags: 36 | - tag1 37 | - tag2 38 | - tag3 39 | """ 40 | def run(self): 41 | config = self.config_data 42 | tc.logger.info("Testing connection and command exec") 43 | tc.logger.debug("Tag 1 is %s" % config["tags"][0]) 44 | ret, _, _ = tc.run(self.nodes[0], "hostname") 45 | if ret != 0: 46 | tc.logger.error("hostname command failed") 47 | return False 48 | else: 49 | return True 50 | 51 | def setup(self): 52 | return True 53 | 54 | def cleanup(self): 55 | return True 56 | 57 | def teardown(self): 58 | return True 59 | 60 | 61 | @testcase("this_should_fail") 62 | class GoingToFail(GlusterBaseClass): 63 | """ Testing connectivity and fail 64 | --- 65 | runs_on_volumes: [ distributed ] 66 | runs_on_protocol: [ glusterfs, cifs ] 67 | reuse_setup: False 68 | tags: 69 | - tag1 70 | - tag2 71 | - tag3 72 | """ 73 | def run(self): 74 | config = self.config_data 75 | tc.logger.info("Testing fail output") 76 | tc.logger.debug("Tag 1 is %s" % config["tags"][0]) 77 | ret, _, _ = tc.run(self.nodes[0], "false") 78 | if ret != 0: 79 | tc.logger.error("false command failed") 80 | return False 81 | else: 82 | return True 83 | 84 | def setup(self): 85 | return True 86 | 87 | def cleanup(self): 88 | return True 89 | 90 | def teardown(self): 91 | return True 92 | -------------------------------------------------------------------------------- /tests_d/examples/test_stdout_stderr.py: -------------------------------------------------------------------------------- 1 | # This file is part of DiSTAF 2 | # Copyright (C) 2015-2016 Red Hat, Inc. 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License along 15 | # with this program; if not, write to the Free Software Foundation, Inc., 16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | 18 | 19 | from distaf.util import tc, testcase 20 | from distaf.distaf_base_class import DistafTestClass 21 | 22 | 23 | @testcase("text_to_stdout_stderr") 24 | class TextToStdoutStderr(DistafTestClass): 25 | """Run some commands against remote servers to 26 | test sending data to stdout and stderr. 27 | Set to 1000, dump approx. 150M of text to output 28 | 29 | To test speed differences, set combinations of... 30 | connection_engine : ssh_controlpersist (default) 31 | connection_engine : rpyc 32 | connection_engine : ssh 33 | 34 | skip_log_inject : True|False 35 | 36 | Run with... 37 | time python main.py -d examples -t text_to_stdout_stderr 38 | """ 39 | def setup(self): 40 | return True 41 | 42 | def run(self): 43 | retstat = 0 44 | tc.logger.info("Send load of text output to stdout") 45 | command = '''ls -Rail /etc > /tmp/railetc 46 | for i in $(seq 1 1000) 47 | do 48 | cat /tmp/railetc 49 | done 50 | echo "Complete" 51 | ''' 52 | rcode, _, _ = tc.run(tc.nodes[0], command) 53 | if rcode != 0: 54 | retstat = retstat | rcode 55 | 56 | tc.logger.info("Send load of text output to stderr") 57 | command = '''ls -Rail /etc > /tmp/railetc 58 | for i in $(seq 1 1000) 59 | do 60 | cat /tmp/railetc >&2 61 | done 62 | echo "Complete" >&2 63 | ''' 64 | rcode, _, _ = tc.run(tc.nodes[0], command) 65 | if rcode != 0: 66 | retstat = retstat | rcode 67 | 68 | if retstat == 0: 69 | return True 70 | 71 | return False 72 | 73 | def cleanup(self): 74 | command = "rm -f /tmp/railetc" 75 | return True 76 | 77 | def teardown(self): 78 | return True 79 | --------------------------------------------------------------------------------