├── .gitignore
├── .project
├── .pydevproject
├── COPYING
├── MANIFEST.in
├── README.md
├── changelog.txt
├── examples
    ├── MegaRAID.txt
    ├── compare.txt
    ├── intel320.txt
    └── mdadmRAID.txt
├── scripts
    ├── tkperf
    └── tkperf-cmp
├── setup.py
└── src
    ├── fio
        ├── FioJob.py
        └── __init__.py
    ├── perfTest
        ├── DeviceTests.py
        ├── Devices.py
        ├── Options.py
        ├── PerfTest.py
        ├── StdyState.py
        └── __init__.py
    ├── plots
        ├── __init__.py
        ├── compPlots.py
        └── genPlots.py
    ├── reports
        ├── RstReport.py
        ├── XmlReport.py
        ├── __init__.py
        └── pics
        │   └── TKperf_logo.png
    └── system
        ├── Mail.py
        ├── OS.py
        └── __init__.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | build
3 | 


--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>TKperf</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.dltk.core.scriptbuilder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 		<buildCommand>
14 | 			<name>org.python.pydev.PyDevBuilder</name>
15 | 			<arguments>
16 | 			</arguments>
17 | 		</buildCommand>
18 | 	</buildSpec>
19 | 	<natures>
20 | 		<nature>org.python.pydev.pythonNature</nature>
21 | 		<nature>net.sourceforge.shelled.core.nature</nature>
22 | 	</natures>
23 | </projectDescription>
24 | 


--------------------------------------------------------------------------------
/.pydevproject:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8" standalone="no"?>
2 | <?eclipse-pydev version="1.0"?><pydev_project>
3 | <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
4 | <path>/TKperf/src</path>
5 | </pydev_pathproperty>
6 | <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
7 | <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
8 | </pydev_project>
9 | 


--------------------------------------------------------------------------------
/COPYING:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 |                             Preamble
  9 | 
 10 |   The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 |   The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 |   When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 |   To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 |   For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 |   Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 |   For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 |   Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 |   Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 |   The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |                        TERMS AND CONDITIONS
 72 | 
 73 |   0. Definitions.
 74 | 
 75 |   "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 |   "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 |   "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 |   To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 |   A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 |   To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 |   To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 |   An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 |   1. Source Code.
113 | 
114 |   The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 |   A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 |   The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 |   The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 |   The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 |   The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 |   2. Basic Permissions.
155 | 
156 |   All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 |   You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 |   Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 |   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 |   No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 |   When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 |   4. Conveying Verbatim Copies.
196 | 
197 |   You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 |   You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 |   5. Conveying Modified Source Versions.
209 | 
210 |   You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 |     a) The work must carry prominent notices stating that you modified
215 |     it, and giving a relevant date.
216 | 
217 |     b) The work must carry prominent notices stating that it is
218 |     released under this License and any conditions added under section
219 |     7.  This requirement modifies the requirement in section 4 to
220 |     "keep intact all notices".
221 | 
222 |     c) You must license the entire work, as a whole, under this
223 |     License to anyone who comes into possession of a copy.  This
224 |     License will therefore apply, along with any applicable section 7
225 |     additional terms, to the whole of the work, and all its parts,
226 |     regardless of how they are packaged.  This License gives no
227 |     permission to license the work in any other way, but it does not
228 |     invalidate such permission if you have separately received it.
229 | 
230 |     d) If the work has interactive user interfaces, each must display
231 |     Appropriate Legal Notices; however, if the Program has interactive
232 |     interfaces that do not display Appropriate Legal Notices, your
233 |     work need not make them do so.
234 | 
235 |   A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 |   6. Conveying Non-Source Forms.
246 | 
247 |   You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 |     a) Convey the object code in, or embodied in, a physical product
253 |     (including a physical distribution medium), accompanied by the
254 |     Corresponding Source fixed on a durable physical medium
255 |     customarily used for software interchange.
256 | 
257 |     b) Convey the object code in, or embodied in, a physical product
258 |     (including a physical distribution medium), accompanied by a
259 |     written offer, valid for at least three years and valid for as
260 |     long as you offer spare parts or customer support for that product
261 |     model, to give anyone who possesses the object code either (1) a
262 |     copy of the Corresponding Source for all the software in the
263 |     product that is covered by this License, on a durable physical
264 |     medium customarily used for software interchange, for a price no
265 |     more than your reasonable cost of physically performing this
266 |     conveying of source, or (2) access to copy the
267 |     Corresponding Source from a network server at no charge.
268 | 
269 |     c) Convey individual copies of the object code with a copy of the
270 |     written offer to provide the Corresponding Source.  This
271 |     alternative is allowed only occasionally and noncommercially, and
272 |     only if you received the object code with such an offer, in accord
273 |     with subsection 6b.
274 | 
275 |     d) Convey the object code by offering access from a designated
276 |     place (gratis or for a charge), and offer equivalent access to the
277 |     Corresponding Source in the same way through the same place at no
278 |     further charge.  You need not require recipients to copy the
279 |     Corresponding Source along with the object code.  If the place to
280 |     copy the object code is a network server, the Corresponding Source
281 |     may be on a different server (operated by you or a third party)
282 |     that supports equivalent copying facilities, provided you maintain
283 |     clear directions next to the object code saying where to find the
284 |     Corresponding Source.  Regardless of what server hosts the
285 |     Corresponding Source, you remain obligated to ensure that it is
286 |     available for as long as needed to satisfy these requirements.
287 | 
288 |     e) Convey the object code using peer-to-peer transmission, provided
289 |     you inform other peers where the object code and Corresponding
290 |     Source of the work are being offered to the general public at no
291 |     charge under subsection 6d.
292 | 
293 |   A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 |   A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 |   "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 |   If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 |   The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 |   Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 |   7. Additional Terms.
344 | 
345 |   "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 |   When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 |   Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 |     a) Disclaiming warranty or limiting liability differently from the
366 |     terms of sections 15 and 16 of this License; or
367 | 
368 |     b) Requiring preservation of specified reasonable legal notices or
369 |     author attributions in that material or in the Appropriate Legal
370 |     Notices displayed by works containing it; or
371 | 
372 |     c) Prohibiting misrepresentation of the origin of that material, or
373 |     requiring that modified versions of such material be marked in
374 |     reasonable ways as different from the original version; or
375 | 
376 |     d) Limiting the use for publicity purposes of names of licensors or
377 |     authors of the material; or
378 | 
379 |     e) Declining to grant rights under trademark law for use of some
380 |     trade names, trademarks, or service marks; or
381 | 
382 |     f) Requiring indemnification of licensors and authors of that
383 |     material by anyone who conveys the material (or modified versions of
384 |     it) with contractual assumptions of liability to the recipient, for
385 |     any liability that these contractual assumptions directly impose on
386 |     those licensors and authors.
387 | 
388 |   All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 |   If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 |   Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 |   8. Termination.
408 | 
409 |   You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 |   However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 |   Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 |   Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 |   9. Acceptance Not Required for Having Copies.
436 | 
437 |   You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 |   10. Automatic Licensing of Downstream Recipients.
447 | 
448 |   Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 |   An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 |   You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 |   11. Patents.
472 | 
473 |   A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 |   A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 |   Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 |   In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 |   If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 |   If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 |   A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 |   Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 |   12. No Surrender of Others' Freedom.
541 | 
542 |   If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 |   13. Use with the GNU Affero General Public License.
553 | 
554 |   Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 |   14. Revised Versions of this License.
564 | 
565 |   The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 |   Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 |   If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 |   Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 |   15. Disclaimer of Warranty.
590 | 
591 |   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 |   16. Limitation of Liability.
601 | 
602 |   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 |   17. Interpretation of Sections 15 and 16.
613 | 
614 |   If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |                      END OF TERMS AND CONDITIONS
622 | 
623 |             How to Apply These Terms to Your New Programs
624 | 
625 |   If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 |   To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 |     <one line to give the program's name and a brief idea of what it does.>
635 |     Copyright (C) <year>  <name of author>
636 | 
637 |     This program is free software: you can redistribute it and/or modify
638 |     it under the terms of the GNU General Public License as published by
639 |     the Free Software Foundation, either version 3 of the License, or
640 |     (at your option) any later version.
641 | 
642 |     This program is distributed in the hope that it will be useful,
643 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
644 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 |     GNU General Public License for more details.
646 | 
647 |     You should have received a copy of the GNU General Public License
648 |     along with this program.  If not, see <http://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 |   If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 |     <program>  Copyright (C) <year>  <name of author>
656 |     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 |     This is free software, and you are welcome to redistribute it
658 |     under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 |   You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <http://www.gnu.org/licenses/>.
668 | 
669 |   The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <http://www.gnu.org/philosophy/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | recursive-include examples *.txt
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # TKperf performance test tool for SSDs, HDDs and RAID devices
  2 | 
  3 | ## Disclaimer
  4 | * **ATTENTION: All data on the tested device is lost!**
  5 | * The given device becomes overwritten multiple times!
  6 | * For SSDs a secure erase is carried out, too.
  7 | * For RAID devices each device becomes overwritten or secure erased
  8 | 
  9 | ## Requirements
 10 | * The following python packages are required:
 11 |   * logging
 12 |   * json
 13 |   * lxml
 14 |   * subprocess
 15 |   * datetime
 16 |   * argparse
 17 |   * copy
 18 |   * collections
 19 |   * numpy
 20 |   * matplotlib
 21 |   * os
 22 |   * sys
 23 |   * makedirs, path, walk
 24 |   * zipfile
 25 |   * smtplib
 26 |   * email
 27 |   * ABCMeta, abstractmethod
 28 |   * string.split
 29 |   * lstat
 30 |   * stat.S_ISBLK
 31 |   * string.split
 32 |   * time.sleep
 33 |   * setuptools
 34 | * Under Ubuntu it is normally sufficient to install:
 35 |     $ sudo apt-get install python3-matplotlib \
 36 |     python3-lxml python3-numpy python3-simplejson python3-setuptools
 37 | * The following software is required:
 38 |   * FIO version 2.0.3 or higher.
 39 |     Please note that older FIO versions are not supported due to an
 40 |     incompatibility in the terse output of FIO. The tests will not produce any
 41 |     valuable information if FIO is older than 2.0.3.
 42 |   * hdparm
 43 |   * sg3-utils (for testing SAS devices)
 44 |   * nvme-cli (for testing nvme devices)
 45 |     * E.g. https://github.com/linux-nvme/nvme-cli.git
 46 |   * rst2pdf
 47 | 
 48 | ## Setup
 49 | * TKperf checks if FIO is found with:
 50 |     $ which fio
 51 |   Please ensure that FIO is in your path and executable.
 52 | * To install FIO visit
 53 |     http://freecode.com/projects/fio
 54 | * To fetch FIO per git use:
 55 |     $ git clone git://git.kernel.dk/fio.git
 56 | * An installation howto (currently only in German):
 57 |     http://www.thomas-krenn.com/de/wiki/Fio_kompilieren
 58 | 
 59 | ## Installation
 60 | * To install TKperf simply call
 61 |     $ sudo python setup.py install
 62 | * If the source is updated via git ensure to reinstall TKperf.
 63 | * An installation howto (in German):
 64 |     https://www.thomas-krenn.com/de/wiki/TKperf
 65 | 
 66 | ## Hardware Prerequisites
 67 | * To carry out a Secure Erase it is necessary that the SSD is NOT in the
 68 |   frozen state.
 69 |   (http://www.thomas-krenn.com/en/wiki/SSD_Secure_Erase#Step_1:_NOT_Frozen)
 70 |   If the SSD is frozen un- and replug the device while the host is on.
 71 | 
 72 | ## Running a test
 73 | * The main script file is called tkperf:
 74 |     $ which tkperf
 75 |     /usr/local/bin/tkperf
 76 | * Call tkperf with the command line options described below.
 77 |   To get the help output use:
 78 |     $ tkperf -h
 79 | 
 80 | ### RAID Tests
 81 | * To run a RAID test you have to create a RAID config file (cf. 'examples').
 82 |   The config file specifies RAID type, devices and level. For Avago/LSI use
 83 |   'hw_lsi' as type, for mdadm 'sw_mdadm'. hw_lsi uses enclosure IDs for the
 84 |   devices, you can get them with:
 85 |   $ sudo storcli64 /c0 /eall /sall show
 86 |   mdadm uses normal device paths like '/dev/sda', '/dev/sdb' etc.
 87 |   Example config:
 88 |   {
 89 |     "devices": [
 90 |         "252:0",
 91 |         "252:1"
 92 |     ],
 93 |     "raidlevel": 1,
 94 |     "type": "hw_lsi"
 95 |   }
 96 | 
 97 | ## Log File
 98 | * The log file is named after the given test name (e.g. 'intel320' in the
 99 |   example below). Inspect the log from time to time to ensure that no errors
100 |   occur. In the log all calls to FIO are stated out. Also the results from the
101 |   steady rounds are written to the log file. If you have any doubts that the
102 |   results in the pdf report are correct check the log what performance values
103 |   FIO returned. As the log file contains the FIO output in terse version, read
104 |   the FIO HOWTO to find out how to interpret it.
105 |   If a report is generated from the xml file a seperate log is created. This log
106 |   gets '.xml' appended.
107 | 
108 | ## Examples
109 | * To run a test on a remote machine and ensure it is not stopped if the
110 |   connection closes it is good practice to start the script in a screen session.
111 |   See 'man screen' for more information. Basically it is sufficient to call
112 |   'screen', start the script and then detach with 'Ctrl-a d'.
113 | * As root or with 'sudo':
114 |     $ sudo tkperf ssd intel320 /dev/sde -nj 2 -iod 16 -rfb
115 | * If hdparm does not output any valuable information for your device,
116 |   use a description file to provide the device informations:
117 |     $ sudo tkperf ssd intel320 /dev/sde -nj 2 -iod 16 -rfb -dsc intel320.dsc
118 | * Also nohup could be used to start a test. To disable the warning that the
119 |   device will be erased ('-ft', '--force-test'). Note that with nohup it is
120 |   difficult to use sudo and redirect stdout/stderr to output files.
121 |     $ sudo nohup tkperf ssd intel320 /dev/sde -ft -nj 2 \
122 |       -iod 16 -rfb 1>runTest.out 2>runTest.err &
123 | 
124 | ### RAID Examples
125 | * To deal with Avago RAID devices, you have to install storcli
126 | * To deal with software RAID devices, you have to install mdadm
127 | * First, create the RAID device manually, this has only to be done the first
128 |   time. Afterwards the RAID is re-created automatically:
129 |     $ sudo storcli64 /c0 add vd type=raid1 drives=252:0,252:1
130 |   Then check with e.g. 'lsblk' which device was created and start the test
131 |   with:
132 |     $ sudo tkperf raid LSI-I3500-R5-4 /dev/sdb -c raid5.cfg -nj 2 -iod 16
133 | 
134 | ## SSD Compression
135 | * If the SSD controller uses compression use the '-rfb' switch to ensure that
136 |   data buffers used by FIO are completely random. This option enables the
137 |   'refill_buffers' from FIO.
138 | 
139 | ## Description File
140 | * If 'hdparm -I' does not provide any valuable information about a drive, you
141 |   have to provide a so called 'description file'. This file just contains some
142 |   meta information for the pdf report and so it can be a simple text file
143 |   describing the tested device. When calling the script provide the path to the
144 |   description file with '-dsc PATH_TO_FILE'.
145 | 
146 | ## Generate a PDF report
147 | * To generate a pdf from the rst use rst2pdf:
148 |     $ rst2pdf test.rst test.pdf
149 | 
150 | ## Loading from XML
151 | * To load an already carried out test from the generated xml file, use
152 |   the '-xml' option:
153 |     $ sudo tkperf ssd intel520 none -xml
154 | * In this case as tested device 'none' is used, it is there just as a
155 |   placeholder.
156 | * Loading from an xml file is useful if something has changed in the plotting
157 |   methods and you want to re-plot the results. This is mainly required during a
158 |   development process or if a major update/bugfix has been made to tkperf.
159 | 
160 | ## Creating compare plots
161 | * With the help of the generated xml files multiple devices can be compared (up
162 |   to seven devices). The script 'tkperf-cmp' generates the compare plots for
163 |   write saturation, throughput, IOPS and latency. To generate the plots use:
164 |   $ tkperf-cmp ssd Samsung840PRO-256GB.xml Samsung840EVO-250GB.xml
165 | * Guide (in German)
166 |   https://www.thomas-krenn.com/de/wiki/SSD_Performance_mit_TKperf_vergleichen
167 | 
168 | ## Further information
169 | * To get more information about how the SSD tests are carried out, visit
170 |   http://www.snia.org/tech_activities/standards/curr_standards/pts for the
171 |   standard these tests are based on.
172 | 
173 | ## Help Text
174 | 
175 | ### tkperf
176 | ```
177 | usage: tkperf [-h] [-v] [-d] [-q] [-nj NUMJOBS] [-iod IODEPTH] [-rt RUNTIME]
178 |               [-i {sas,nvme,fusion}] [-xml] [-rfb] [-dsc DESC_FILE]
179 |               [-c CONFIG] [-ft] [-fm FEATURE_MATRIX] [-hddt {iops,tp}]
180 |               [-ssdt {iops,lat,tp,writesat}] [-m MAIL] [-s SMTP]
181 |               [-g GEN_REPORT]
182 |               {hdd,ssd,raid} testname device
183 | 
184 | positional arguments:
185 |   {hdd,ssd,raid}        specify the test mode for the device
186 |   testname              name of the performance tests, corresponds to the
187 |                         result output filenames
188 |   device                device to run fio test on
189 | 
190 | optional arguments:
191 |   -h, --help            show this help message and exit
192 |   -v, --version         get the version information
193 |   -d, --debug           get detailed debug information
194 |   -q, --quiet           turn off logging of info messages
195 |   -nj NUMJOBS, --numjobs NUMJOBS
196 |                         specify number of jobs for fio
197 |   -iod IODEPTH, --iodepth IODEPTH
198 |                         specify iodepth for libaio used by fio
199 |   -rt RUNTIME, --runtime RUNTIME
200 |                         specify the fio runtime of one test round, if not set
201 |                         this is 60 seconds
202 |   -i {sas,nvme,fusion}, --interface {sas,nvme,fusion}
203 |                         specify optional device interface
204 |   -xml, --fromxml       don't run tests but load test objects from xml file
205 |   -rfb, --refill_buffers
206 |                         use Fio's refill buffers option to circumvent any
207 |                         compression of devices
208 |   -dsc DESC_FILE, --desc_file DESC_FILE
209 |                         use a description file for the tested device if hdparm
210 |                         doesn't work correctly
211 |   -c CONFIG, --config CONFIG
212 |                         specify the config file for a raid device
213 |   -ft, --force_test     skip checks if the used device is mounted, don't print
214 |                         warnings and force starting the test
215 |   -fm FEATURE_MATRIX, --feature_matrix FEATURE_MATRIX
216 |                         add a feature matrix of the given device to the report
217 |   -hddt {iops,tp}, --hdd_type {iops,tp}
218 |                         choose which tests are run
219 |   -ssdt {iops,lat,tp,writesat}, --ssd_type {iops,lat,tp,writesat}
220 |                         choose which tests are run
221 |   -m MAIL, --mail MAIL  Send reports or errors to mail address, needs -s to be
222 |                         set
223 |   -s SMTP, --smtp SMTP  Use the specified smtp server to send mails, uses port
224 |                         25 to connect
225 |   -g GEN_REPORT, --gen_report GEN_REPORT
226 |                         Set and specify command to generate pdf report, e.g.
227 |                         rst2pdf
228 | ```
229 | 
230 | ### tkperf-cmp
231 | ```
232 | $ tkperf-cmp -h
233 | usage: tkperf-cmp [-h] [-v] [-d] [-q] [-f FOLDER] [-z]
234 |                   {hdd,ssd,raid} xmls [xmls ...]
235 | 
236 | positional arguments:
237 |   {hdd,ssd,raid}        specify the test mode for the device
238 |   xmls                  XML files to read from
239 | 
240 | optional arguments:
241 |   -h, --help            show this help message and exit
242 |   -v, --version         get the version information
243 |   -d, --debug           get detailed debug information
244 |   -q, --quiet           turn off logging of info messages
245 |   -f FOLDER, --folder FOLDER
246 |                         store compare plots in a subfolder, specify name of
247 |                         folder
248 |   -z, --zip             store compare plots in a zip archive, requires '-f' as
249 |                         zip is created from subfolder
250 | ```
251 | 
252 | ## Copyright (C) 2015-2018 Thomas-Krenn.AG
253 | This program is free software; you can redistribute it and/or modify it under
254 | the terms of the GNU General Public License as published by the Free Software
255 | Foundation; either version 3 of the License, or (at your option) any later
256 | version.
257 | 
258 | This program is distributed in the hope that it will be useful, but WITHOUT
259 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
260 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
261 | details.
262 | 
263 | You should have received a copy of the GNU General Public License along with
264 | this program; if not, see <http://www.gnu.org/licenses/>.
265 | 


--------------------------------------------------------------------------------
/changelog.txt:
--------------------------------------------------------------------------------
 1 | ####################################################################
 2 | Changelog for TKperf, performance tests for ssds, hdds and RAIDs
 3 | ####################################################################
 4 | 
 5 | Visit https://github.com/thomas-krenn/TKperf to get current development
 6 | versions of TKperf and detailed changes.
 7 | 
 8 | Version 2.2 20180926
 9 |   * Add timestamps to log messages
10 |   * Add read and write policies for megaraid devices
11 |   * Use markdown for README
12 |   * Set auto background initialization to off for megaraid devices
13 |   * Fix device number to 2 if megaraid RAID10 devices are created
14 |   * Add strip size for megaraid devices
15 |   * NVME: use correct lbaf for nvme format command
16 |   * Set correct StdyState if device did not reach steady state
17 |   * NVME: use correct interface with NVME RAID devices
18 | 
19 | Version 2.1 20151016
20 |   * Fix max rounds bug for SSD write saturation
21 |     * If the maximum rounds 1440 was reached for an SSD the number of
22 |       rounds has not been set correctly.
23 |   * Add option for specifying test round runtime
24 |   * Add 'fusion' to valid interface names, secure erasion FusionIO cards
25 |   * Use nvme format for devices with nvme interface
26 | 
27 | Version 2.0 20150121
28 |   * Refactor performance tests and devices with abstract classes
29 |   * Devices implementing the Device class can be used as test device
30 |   * Add a system package for interaction with OS command line tools like
31 |     storcli or mdadm
32 |   * Added new RAID tests, RAIDs are created automatically
33 |     * Avago or LSI devices can be tested with a specific RAID config file
34 |       * Preconditioning is done automatically on all devices in the RAID set
35 |     * Software RAID with mdadm can be tested with a specific RAID config file
36 |       * Preconditioning is done automatically on all devices in the RAID set
37 |       * Secure erase is done automatically on all devices in the RAID set
38 |   * SAS devices are tested automatically with sf_format and sg_info
39 |   * Added new TKperf logo
40 |   * Remove tkperf_dialog.sh as it is obsolete
41 |   * Fix error in tkperf-comp LAT plot
42 |   * Enhance tkperf-cmp
43 |     * Plots can be put into a subfolder and be compressed with zip
44 |   * If -g is used a PDF report is generated automatically
45 |   * Reports can be sent to an email automatically, use -m and -s
46 | 
47 | Version 1.3 20140804
48 |   * Support 3D plots also with matplotlib < 1.0
49 |   * Added a sleep operation after each secure erase
50 |   * Added tkperf-cmp, a script to generate compare plots
51 | 
52 | Version 1.2 20131017
53 |   * Return an error if hdparm doesn't work for HDD
54 | 
55 | Version 1.1 20130117
56 |   * Checking for correct FIO version
57 |   * Added more hdparm informations about a device (e.g. write-cache)
58 |   * Description files to provide informations for devices that
59 |     cannot be checked with hdparm
60 |   * Loading from Xml is possible if plotting or reporting functions
61 |     changed and the report should be generated again
62 |   * With setup.py an installations script is provided to ease TKperf
63 |     installation
64 |   * FIO version is included in the report
65 |   * Date of test run is included in the report
66 |   * Number of jobs and number of outstanding I/Os can be changed
67 |   * Added option "refill_buffers" to circumvent any compression of
68 |     SSD controllers
69 |   * OS distribution information is included in the report
70 |   * tkperf_dialog.sh is a shell script to start a simple test script
71 |     via answering interactive questions
72 | 
73 | Version 1.0-dev 20121123
74 |   * SSD
75 |     * Tests
76 |       * Iops
77 |       * Latency
78 |       * Write Saturation
79 |       * Throughput
80 |     * Plots and measurement tables required by SNIA spec
81 |     * Logging all FIO calls to log file
82 |     * Xml file with test results
83 |     * Rst file with restructured text
84 |     * Rst file can be converted to pdf with rst2pdf
85 |   * HDD
86 |     * Tests
87 |       * Iops
88 |       * Throughput
89 |     * Measurement plots
90 | 
91 | List of contributors to version 2.* branch:
92 |   Georg Schönberger, Thomas-Krenn.AG (author)
93 | ################################################################################
94 | 


--------------------------------------------------------------------------------
/examples/MegaRAID.txt:
--------------------------------------------------------------------------------
 1 | {
 2 |     "devices": [
 3 |         "252:0",
 4 |         "252:1"
 5 |     ],
 6 |     "raidlevel": 1,
 7 |     "type": "hw_lsi",
 8 |     "readpolicy": "nora",
 9 |     "writepolicy": "wt",
10 |     "stripsize": "256"
11 | }
12 | sudo tkperf raid MEGARAID-R1 /dev/sda -c raid1.cfg -nj 2 -iod 16
13 | 


--------------------------------------------------------------------------------
/examples/compare.txt:
--------------------------------------------------------------------------------
1 | tkperf-cmp ssd Samsung840PRO-256GB.xml Samsung840EVO-250GB.xml
2 | 


--------------------------------------------------------------------------------
/examples/intel320.txt:
--------------------------------------------------------------------------------
1 | sudo tkperf ssd intel320 /dev/sdb -nj 2 -iod 16 -rfb
2 | 


--------------------------------------------------------------------------------
/examples/mdadmRAID.txt:
--------------------------------------------------------------------------------
 1 | {
 2 |     "devices": [
 3 |         "/dev/sdb",
 4 |         "/dev/sdc",
 5 |         "/dev/sdd"
 6 |     ],
 7 |     "raidlevel": 5,
 8 |     "type": "sw_mdadm"
 9 | }
10 | sudo tkperf raid SW-I3500-R5-3 /dev/md0 -c raid.cfg -nj 2 -iod 16


--------------------------------------------------------------------------------
/scripts/tkperf:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | '''
  3 | Created on 27.06.2012
  4 | 
  5 | @author: gschoenb
  6 | @version: 2.2
  7 | '''
  8 | import argparse
  9 | import logging
 10 | import sys
 11 | 
 12 | from perfTest.Devices import SSD
 13 | from perfTest.Devices import HDD
 14 | from perfTest.Devices import RAID
 15 | from perfTest.Options import Options
 16 | from perfTest.PerfTest import SsdPerfTest
 17 | from perfTest.PerfTest import HddPerfTest
 18 | import perfTest.PerfTest as pT
 19 | from system.Mail import Mail
 20 | from email.errors import MessageError
 21 | 
 22 | if __name__ == '__main__':
 23 |     tkPerfVersion = "TKperf Version: " + pT.__version__
 24 | 
 25 |     parser = argparse.ArgumentParser()
 26 |     parser.add_argument("mode", help="specify the test mode for the device", choices=["hdd","ssd","raid"])
 27 |     parser.add_argument("testname",help="name of the performance tests, corresponds to the result output filenames")
 28 |     parser.add_argument("device",help="device to run fio test on")
 29 | 
 30 |     parser.add_argument("-v","--version", help="get the version information", action='version',version=tkPerfVersion)
 31 |     parser.add_argument("-d","--debug", help="get detailed debug information",action ='store_true')
 32 |     parser.add_argument("-q","--quiet", help="turn off logging of info messages",action ='store_true')
 33 |     parser.add_argument("-nj","--numjobs",help="specify number of jobs for fio",type=int)
 34 |     parser.add_argument("-iod","--iodepth",help="specify iodepth for libaio used by fio",type=int)
 35 |     parser.add_argument("-rt","--runtime",help="specify the fio runtime of one test round, if not set this is 60 seconds",type=int)
 36 |     parser.add_argument("-i","--interface",help="specify optional device interface",choices=["sas","nvme","fusion","usb","sdcard","compactflash"])
 37 |     parser.add_argument("-xml","--fromxml",help="don't run tests but load test objects from xml file",
 38 |                         action='store_true')
 39 |     parser.add_argument("-rfb","--refill_buffers",help="use Fio's refill buffers option to circumvent any compression of devices",
 40 |                         action='store_true')
 41 |     parser.add_argument("-dsc","--desc_file",help="use a description file for the tested device if hdparm doesn't work correctly",
 42 |                         type=argparse.FileType('r'))
 43 |     parser.add_argument("-c","--config",help="specify the config file for a raid device",
 44 |                         type=argparse.FileType('r'))
 45 |     parser.add_argument("-ft","--force_test",help="skip checks if the used device is mounted, don't print warnings and force starting the test",
 46 |                         action='store_true')
 47 |     parser.add_argument("-fm","--feature_matrix",help="add a feature matrix of the given device to the report",
 48 |                         type=argparse.FileType('r'))
 49 |     parser.add_argument("-hddt","--hdd_type",help="choose which tests are run",
 50 |                         choices=['iops','tp'],action='append',dest='hddt')
 51 |     parser.add_argument("-ssdt","--ssd_type",help="choose which tests are run",
 52 |                         choices=['iops','lat','tp','writesat'],action='append',dest='ssdt')
 53 |     parser.add_argument("-m","--mail",help="Send reports or errors to mail address, needs -s to be set")
 54 |     parser.add_argument("-s","--smtp",help="Use the specified smtp server to send mails, uses port 25 to connect")
 55 |     parser.add_argument("-g","--gen_report",help="Set and specify command to generate pdf report, e.g. rst2pdf")
 56 |     args = parser.parse_args()
 57 |     # Configure logging levels
 58 |     logformat = '%(asctime)s %(name)-8s %(levelname)-8s %(message)s'
 59 |     logdatefmt = '%Y%m%d %H:%M'
 60 |     if args.fromxml == False:
 61 |         logfile = args.testname+'.log'
 62 |     else:
 63 |         logfile = args.testname+'.xml.log'
 64 |     if args.debug == True:
 65 |         logging.basicConfig(filename=logfile,level=logging.DEBUG,format=logformat,datefmt=logdatefmt)
 66 |     if args.quiet == True:
 67 |         logging.basicConfig(filename=logfile,level=logging.WARNING,format=logformat,datefmt=logdatefmt)
 68 |     else:
 69 |         logging.basicConfig(filename=logfile,level=logging.INFO,format=logformat,datefmt=logdatefmt)
 70 |     # Create objects, a device and given options
 71 |     if args.mode == "ssd":
 72 |         devToTest = SSD(args.mode,args.device,args.testname)
 73 |     if args.mode == "hdd":
 74 |         devToTest = HDD(args.mode,args.device,args.testname)
 75 |     if args.mode == "raid":
 76 |         devToTest = RAID(args.mode,args.device,args.testname)
 77 |     if args.interface != None:
 78 |         devToTest.setInterface(args.interface)
 79 |     options = Options()
 80 |     if args.numjobs != None:
 81 |         options.setNj(args.numjobs)
 82 |     if args.iodepth != None:
 83 |         options.setIod(args.iodepth)
 84 |     if args.runtime != None:
 85 |         options.setRuntime(args.runtime)
 86 |     if args.refill_buffers == True:
 87 |         xargs = ['refill_buffers']
 88 |         options.setXargs(xargs)
 89 |     # Create performance test objects, don't yet init them
 90 |     if args.mode == "ssd" or args.mode == "raid":
 91 |         if args.ssdt != None:
 92 |             SsdPerfTest.testKeys = args.ssdt
 93 |         myTest = SsdPerfTest(args.testname, devToTest,options)
 94 |     if args.mode == "hdd":
 95 |         if args.hddt != None:
 96 |             HddPerfTest.testKeys = args.hddt
 97 |         myTest = HddPerfTest(args.testname, devToTest,options)
 98 |     # First check if we are loading values from a given xml
 99 |     if args.fromxml == True:
100 |         print("Loading from xml file...")
101 |         myTest.fromXml()
102 |         myTest.genPlots()
103 |         myTest.toRst()
104 |         exit(0)
105 |     # Start a real performance test
106 |     try:
107 |         # A raid test needs a raid config
108 |         if args.mode == "raid":
109 |             devToTest.setConfig(args.config)
110 |         myTest.initialize()
111 |     except RuntimeError:
112 |         print("### Error! ###")
113 |         print("Test initialization failed, please inspect the log file!")
114 |         exit(1)
115 |     # Keep used command line arguments
116 |     myTest.readCmdLineArgs(sys.argv)
117 |     # Check if a correct setup is given
118 |     if (not devToTest.isInitialized()) and args.desc_file == None:
119 |         print("### Error! ###")
120 |         print("Please use a description file for the current device.")
121 |         print("The information via hdparm -I is not reliable.")
122 |         print("Use -dsc DESC_FILE to provide the information")
123 |         exit(1)
124 |     if args.desc_file != None:
125 |         devToTest.readDevInfoFile(args.desc_file)
126 |     if args.feature_matrix != None:
127 |         devToTest.readFeatureFile(args.feature_matrix)
128 |     # Don't print a warning if force test is given
129 |     if args.force_test == False:
130 |         if devToTest.isMounted():
131 |             print("!!!WARNING!!!")
132 |             print("You are testing a mounted device, this is highly dangerous!")
133 |             exit(0)
134 |         if devToTest.isAvailable():
135 |             print("!!!Attention!!!")
136 |             print("All data on " + args.device + " will be lost!")
137 |             print("Are you sure you want to continue? (In case you really know what you are doing.)")
138 |             print("Press 'y' to continue, any key to stop:")
139 |             key = input()
140 |             if key != 'y':
141 |                 exit(0)
142 |         else:
143 |             print("You are not using a valid device or partition!")
144 |             exit(1)
145 |     print("Starting "+args.mode+" mode...")
146 |     print("Testing device:")
147 |     print(devToTest.getDevInfo())
148 |     try:
149 |         myTest.run()
150 |         if args.gen_report != None:
151 |             try:
152 |                 myTest.getRstReport().toPDF(args.gen_report)
153 |             except RuntimeError:
154 |                 print("### Error! ###")
155 |                 print("Generating PDF failed.")
156 |         if args.mail != None and args.smtp != None:
157 |             try:
158 |                 mail = Mail('TKperf message', 'root@tkperf.local', args.mail, args.smtp)
159 |                 mail.addMsg('Please find your TKperf report as attachment!')
160 |                 if args.gen_report != None:
161 |                     mail.addPDFAttachment(args.testname+'.pdf')
162 |                 mail.addTextAttachment(args.testname+'.rst')
163 |                 mail.addXMLAttachment(args.testname+'.xml')
164 |                 mail.addTextAttachment(logfile)
165 |                 mail.send()
166 |             except MessageError:
167 |                 print("### Error! ###")
168 |                 print("Creating and sending mail failed.")
169 |     except RuntimeError:
170 |         print("### Error! ###")
171 |         print("Running the tests failed, please inspect the log file!")
172 |         if args.mail != None and args.smtp != None:
173 |             try:
174 |                 mail = Mail('TKperf message', 'root@tkperf.local', args.mail, args.smtp)
175 |                 mail.addMsg('A TKperf error occurred, please inspect the log file!')
176 |                 mail.addTextAttachment(logfile)
177 |                 mail.send()
178 |             except MessageError:
179 |                 print("### Error! ###")
180 |                 print("Creating and sending mail failed.")
181 |         exit(1)
182 | 


--------------------------------------------------------------------------------
/scripts/tkperf-cmp:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | '''
 3 | Created on 27.06.2012
 4 | 
 5 | @author: gschoenb
 6 | '''
 7 | import argparse
 8 | import logging
 9 | from os import makedirs
10 | from os import path
11 | from os import walk
12 | import zipfile
13 | 
14 | from perfTest.Devices import SSD
15 | from perfTest.Devices import HDD
16 | from perfTest.Options import Options
17 | from perfTest.PerfTest import SsdPerfTest
18 | from perfTest.PerfTest import HddPerfTest
19 | import perfTest.PerfTest as pT
20 | import plots.compPlots as pcp
21 | 
22 | if __name__ == '__main__':
23 |     tkPerfVersion = "TKperf Version: " + pT.__version__
24 |     parser = argparse.ArgumentParser()
25 | 
26 |     parser.add_argument("mode", help="specify the test mode for the device", choices=["hdd","ssd","raid"])
27 |     parser.add_argument("xmls", help="XML files to read from", type=str, nargs='+')
28 | 
29 |     parser.add_argument("-v","--version", help="get the version information", action='version',version=tkPerfVersion)
30 |     parser.add_argument("-d","--debug", help="get detailed debug information",action ='store_true')
31 |     parser.add_argument("-q","--quiet", help="turn off logging of info messages",action ='store_true')
32 |     parser.add_argument("-f","--folder", help="store compare plots in a subfolder, specify name of folder",type=str)
33 |     parser.add_argument("-z","--zip",help="store compare plots in a zip archive, requires '-f' as zip is created from subfolder",
34 |                         action ='store_true')
35 | 
36 |     args = parser.parse_args()
37 |     if args.debug == True:
38 |         logging.basicConfig(filename='tkperf-cmp.log',level=logging.DEBUG)
39 |     if args.quiet == True:
40 |         logging.basicConfig(filename='tkperf-cmp.log',level=logging.WARNING)
41 |     else:
42 |         logging.basicConfig(filename='tkperf-cmp.log',level=logging.INFO)
43 | 
44 |     # Strip the filename suffix as it is appended automatically
45 |     for i,file in enumerate(args.xmls):
46 |         if file.endswith('.xml'):
47 |             file = file[:-4]
48 |         args.xmls[i] = file
49 |     # Check if plots go into a subfolder
50 |     if args.folder != None:
51 |         try: 
52 |             makedirs(args.folder)
53 |         except OSError:
54 |             if not path.isdir(args.folder):
55 |                 raise
56 |     # Generate objects and plots
57 |     toCompare = []
58 |     for file in args.xmls:
59 |         if args.mode == 'ssd' or args.mode == 'raid':
60 |             options = Options()
61 |             dummyDev = SSD('ssd',None,file)
62 |             myTest = SsdPerfTest(file, dummyDev, options)
63 |         if args.mode == 'hdd':
64 |             dummyDev = HDD('hdd',None,file)
65 |             myTest = HddPerfTest(file, dummyDev, options)
66 |         myTest.fromXml()
67 |         toCompare.append(myTest)
68 | 
69 |     pcp.compWriteSatIOPSPlt(toCompare, args.folder)
70 |     pcp.compILPlt(toCompare, 'IOPS', args.folder)
71 |     pcp.compILPlt(toCompare, 'LAT', args.folder)
72 |     pcp.compTPPlt(toCompare, args.folder)
73 |     # Check if a zip archive should be created
74 |     if args.zip:
75 |         if args.folder == None:
76 |             print("### Error! ###")
77 |             print("Please use Option '-f' in conjunction with '-z'.")
78 |         else:
79 |             zfile = zipfile.ZipFile(args.folder+'.zip', 'w')
80 |             for root, dirs, files in walk(args.folder):
81 |                 for file in files:  
82 |                     zfile.write(path.join(root, file))
83 |             zfile.close()
84 |     exit(0)
85 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | from distutils.core import setup
 4 | 
 5 | setup(name='TKperf',
 6 | 	version='2.2',
 7 | 	description='TKperf - Thomas Krenn IO Performance Tests',
 8 | 	author='Georg Schoenberger',
 9 | 	author_email='g.schoenberger@xortex.com',
10 | 	url='https://github.com/thomas-krenn/TKperf.git',
11 | 	package_dir = {'': 'src'},
12 | 	packages = ['fio', 'perfTest','plots','reports','system'],
13 | 	package_data  = {'reports':['pics/TKperf_logo.png']},
14 | 	scripts = ["scripts/tkperf","scripts/tkperf-cmp"],
15 | 	license = 'GPL'
16 | 	)
17 | 


--------------------------------------------------------------------------------
/src/fio/FioJob.py:
--------------------------------------------------------------------------------
  1 | ''' @package FioJob
  2 | A module realizing a fio job run.
  3 | '''
  4 | import subprocess
  5 | import logging
  6 | import re
  7 | import json
  8 | from lxml import etree
  9 | 
 10 | class FioJob(object):
 11 |     '''
 12 |     A class configuring the fio job.
 13 |     '''
 14 |     ## Position of read IOPS in the fio terse output.
 15 |     terseIOPSReadPos = 7
 16 | 
 17 |     ## Position of write IOPS in the fio terse output.
 18 |     terseIOPSWritePos = 48
 19 | 
 20 |     ## Position of write total IO in the fio terse output
 21 |     terseTotIOWritePos = 46
 22 | 
 23 |     ## Start Position of write latencies in fio terse output
 24 |     terseLatStartWritePos = 78
 25 | 
 26 |     ## Start Position of read latencies in fio terse output
 27 |     terseLatStartReadPos = 37
 28 | 
 29 |     ## Postion of total read throughput.
 30 |     terseTPReadPos = 6
 31 | 
 32 |     ## Postion of total write throughput.
 33 |     terseTPWritePos = 47
 34 | 
 35 |     def __init__(self):
 36 |         ''' The constructor '''
 37 |         ## Fio path
 38 |         self.__fioPath = None
 39 |         ## Fio version
 40 |         self.__fioVersion = None
 41 |         ## Key value arguments e.g. name="test"
 42 |         self.__fioKVArgs = {}
 43 |         ## Single arguments e.g. group_reporting
 44 |         self.__fioSglArgs = []
 45 | 
 46 |     def __str__(self):
 47 |         ''' Return a string representation of the fio executable. '''   
 48 |         res = "fio: " + self.__fioPath + ", " + self.__fioVersion
 49 |         return res
 50 | 
 51 |     def initialize(self):
 52 |         ''' Initialize Fio path and version. '''
 53 |         fio = subprocess.Popen(['which', 'fio'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
 54 |         stdout = fio.communicate()[0]
 55 |         if fio.returncode != 0:
 56 |             logging.error("# Error: command 'which fio' returned an error code.")
 57 |             raise RuntimeError("which fio command error")
 58 | 
 59 |         self.__fioPath = stdout.rstrip("\n");
 60 |         fio = subprocess.Popen(['fio','--version'],stdout=subprocess.PIPE,universal_newlines=True)
 61 |         self.__fioVersion = fio.communicate()[0]
 62 | 
 63 |     def getFioVersion(self):
 64 |         ''' Return the current Fio version string. '''
 65 |         return self.__fioVersion
 66 | 
 67 |     def setFioVersion(self,fioStr):
 68 |         ''' Set the used Fio version (useful if loading from xml). '''
 69 |         self.__fioVersion = fioStr
 70 | 
 71 |     def checkFioVersion(self):
 72 |         ''' Check if the Fio version is high enough. '''
 73 |         if self.__fioVersion != None:
 74 |             match = re.search(r'[\d\.]+',self.__fioVersion)
 75 |             if match == None:
 76 |                 logging.error("# Error: checking fio version returned a none string.")
 77 |                 raise RuntimeError("fio version string error")
 78 |             version = match.group().split('.')
 79 |             if int(version[0]) < 2:
 80 |                 logging.error("# Error: the fio version is to old, ensure to use > 2.0.3.")
 81 |                 raise RuntimeError("fio version to old error")
 82 |             if int(version[0]) >= 2:
 83 |                 if int(version[1]) == 0:
 84 |                     if int(version[2]) < 3:
 85 |                         logging.error("# Error: the fio version is to old, ensure to use > 2.0.3.")
 86 |                         raise RuntimeError("fio version to old error")
 87 | 
 88 |     def appendXml(self,root):
 89 |         '''
 90 |         Append the information about Fio to a XML node. 
 91 |         @param root The xml root tag to append the new elements to.
 92 |         ''' 
 93 |         data = json.dumps(self.__fioVersion)
 94 |         e = etree.SubElement(root,'fioversion')
 95 |         e.text = data
 96 | 
 97 |     def fromXml(self,root):
 98 |         '''
 99 |         Loads the information about Fio from XML.
100 |         @param root The given element containing the information about
101 |         the object to be initialized.
102 |         '''
103 |         if root.findtext('fioversion'):
104 |             self.__fioVersion = json.loads(root.findtext('fioversion'))
105 |         else:
106 |             self.__fioVersion = 'n.a'
107 |         logging.info("# Loading Fio version from xml")
108 | 
109 |     def getKVArgs(self):
110 |         ''' Return the current configured Fio key value arguments. '''
111 |         return self.__fioKVArgs
112 |     
113 |     def getSglArgs(self):
114 |         ''' Return the current configured Fio single key arguments. '''
115 |         return self.__fioSglArgs
116 |     
117 |     def addKVArg(self,key,value):
118 |         ''' Add a key value pair as an argument to fio.
119 |         @param key Name of the option for Fio.
120 |         @param value Value for the given Fio option.
121 |         '''
122 |         self.__fioKVArgs[key] = value
123 |         
124 |     def addSglArg(self,key):
125 |         ''' Add a single value option to fio argument list.
126 |         @param key Name of the option being added.
127 |         ''' 
128 |         self.__fioSglArgs.append(key)
129 |         
130 |     def prepKVArgs(self):
131 |         ''' Generate an argument list out of the dictionary suited for fio. '''
132 |         argList = [self.__fioPath]
133 |         for k,v in self.__fioKVArgs.items():
134 |             argList.append('--' + k + '=' + v)
135 |         return argList
136 |     
137 |     def prepSglArgs(self,argList):
138 |         ''' Generate an argument list out of the single key arguments. '''
139 |         for k in self.__fioSglArgs:
140 |             argList.append('--' + k)
141 |         return argList
142 |         
143 |     def start(self):
144 |         ''' Start a Fio job with its argument list.
145 |         The argument list defines the parameters given to Fio.
146 |         @return [True,standard output] of the Fio test or [False,0] on error.
147 |         '''
148 |         args = self.prepKVArgs()
149 |         args = self.prepSglArgs(args)
150 |         logging.info('%s',args)
151 |         if len(args) == 0:
152 |             logging.error("Error: Fio argument list is empty.")
153 |             exit(1)
154 |         out = subprocess.Popen(args,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
155 |         (stdout,stderr) = out.communicate()
156 |         if stderr != '':
157 |             logging.error("Fio encountered an error: " + stderr)
158 |             return [False,'']
159 |         else:
160 |             return [True,stdout]
161 |         
162 |     def getIOPS(self,fioOut):
163 |         '''
164 |         Parses the average IOPS out of the Fio result output.
165 |         @param fioOut The output of the Fio performance test.
166 |         @return Sum of read IOPS and write IOPS.
167 |         '''
168 |         #index 7 iops read
169 |         #index 48 iops write
170 |         fioTerse = fioOut.split(';')
171 |         return int(fioTerse[FioJob.terseIOPSReadPos]) + int(fioTerse[FioJob.terseIOPSWritePos])
172 |        
173 |     def getIOPSRead(self,fioOut):
174 |         '''
175 |         Parses the average read IOPS out of the fio result output.
176 |         @param fioOut The output of the fio performance test.
177 |         @return Read IOPS
178 |         '''
179 |         #index 7 iops read
180 |         fioTerse = fioOut.split(';')
181 |         return int(fioTerse[FioJob.terseIOPSReadPos])
182 |     
183 |     def getIOPSWrite(self,fioOut):
184 |         '''
185 |         Parses the average write IOPS out of the Fio result output.
186 |         @param fioOut The output of the Fio performance test.
187 |         @return Write IOPS
188 |         '''
189 |         #index 48 iops write
190 |         fioTerse = fioOut.split(';')
191 |         return int(fioTerse[FioJob.terseIOPSWritePos])
192 |        
193 |     def getTotIOWrite(self,fioOut):
194 |         '''
195 |         Parses the write total IO out of the Fio result output.
196 |         @param fioOut The output of the Fio performance test.
197 |         @return Write total IO in KB.
198 |         '''
199 |         #index 46 write total IO
200 |         fioTerse = fioOut.split(';')
201 |         return int(fioTerse[FioJob.terseTotIOWritePos])
202 |     
203 |     def getWriteLats(self,fioOut):
204 |         '''
205 |         Parses the write total latencies out of the Fio result output.
206 |         @param fioOut The output of the Fio performance test.
207 |         @return [min,max,mean] total write latencies in microseconds.
208 |         '''
209 |         #index 78 write total latency
210 |         fioTerse = fioOut.split(';')
211 |         return [float(fioTerse[FioJob.terseLatStartWritePos]),
212 |                 float(fioTerse[FioJob.terseLatStartWritePos + 1]),
213 |                 float(fioTerse[FioJob.terseLatStartWritePos + 2])]
214 |         
215 |     def getReadLats(self,fioOut):
216 |         '''
217 |         Parses the read total latencies out of the Fio result output.
218 |         @param fioOut The output of the Fio performance test.
219 |         @return [min,max,mean] total read latencies in microseconds.
220 |         '''
221 |         #index 78 write total latency
222 |         fioTerse = fioOut.split(';')
223 |         return [float(fioTerse[FioJob.terseLatStartReadPos]),
224 |                 float(fioTerse[FioJob.terseLatStartReadPos + 1]),
225 |                 float(fioTerse[FioJob.terseLatStartReadPos + 2])]
226 |         
227 |     def getTotLats(self,fioOut):
228 |         '''
229 |         Parses the read+write total latencies out of the Fio result output.
230 |         @param fioOut The output of the Fio performance test.
231 |         @return [min,max,mean] total latencies in microseconds.
232 |         '''
233 |         #index 78 write total latency
234 |         fioTerse = fioOut.split(';')
235 |         return [float(fioTerse[FioJob.terseLatStartReadPos]) + 
236 |                       float(fioTerse[FioJob.terseLatStartWritePos]),
237 |                 float(fioTerse[FioJob.terseLatStartReadPos + 1]) + 
238 |                       float(fioTerse[FioJob.terseLatStartWritePos + 1]),
239 |                 float(fioTerse[FioJob.terseLatStartReadPos + 2]) + 
240 |                       float(fioTerse[FioJob.terseLatStartWritePos + 2])]
241 | 
242 |     def getTPRead(self,fioOut):
243 |         '''
244 |         Parses the read bandwidth of the Fio result output.
245 |         @param fioOut The output of the Fio performance test.
246 |         @return Read total bandwidth.
247 |         '''
248 |         #index 6 write total IO
249 |         fioTerse = fioOut.split(';')
250 |         return int(fioTerse[FioJob.terseTPReadPos])
251 |         
252 |     def getTPWrite(self,fioOut):
253 |         '''
254 |         Parses the write bandwidth of the Fio result output.
255 |         @param fioOut The output of the Fio performance test.
256 |         @return Write total bandwidth.
257 |         '''
258 |         #index 47 write total IO
259 |         fioTerse = fioOut.split(';')
260 |         return int(fioTerse[FioJob.terseTPWritePos])


--------------------------------------------------------------------------------
/src/fio/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thomas-krenn/TKperf/638b0837b97b32337314993feb52b5111aae0979/src/fio/__init__.py


--------------------------------------------------------------------------------
/src/perfTest/Options.py:
--------------------------------------------------------------------------------
 1 | '''
 2 | Created on Aug 26, 2014
 3 | 
 4 | @author: gschoenb
 5 | '''
 6 | 
 7 | import logging
 8 | import json
 9 | from lxml import etree
10 | 
11 | class Options(object):
12 |     '''
13 |     A class holding user defined options on command line.
14 |     '''
15 | 
16 |     def __init__(self, nj=1, iod=1, runtime=60, xargs=None):
17 |         '''
18 |         Constructor
19 |         @param nj Number of jobs
20 |         @param iod Number for io depth
21 |         @param xargs Further argument as list for all fio jobs in tests
22 |         '''
23 |         ## Number of jobs for fio.
24 |         self.__nj = nj
25 |         ## Number of iodepth for fio.
26 |         self.__iod = iod
27 |         ## Runtime of one test round for fio.
28 |         self.__runtime = runtime
29 |         ## Further single arguments as list for fio.
30 |         self.__xargs = xargs
31 | 
32 |     def getNj(self): return self.__nj
33 |     def getIod(self): return self.__iod
34 |     def getRuntime(self): return self.__runtime
35 |     def getXargs(self): return self.__xargs
36 |     def setNj(self,nj): self.__nj = nj
37 |     def setIod(self,iod): self.__iod = iod
38 |     def setRuntime(self,rt): self.__runtime = rt
39 |     def setXargs(self,xargs): self.__xargs = xargs
40 |     
41 |     def appendXml(self,r):
42 |         '''
43 |         Append the information about options to a XML node. 
44 |         @param root The xml root tag to append the new elements to
45 |         ''' 
46 |         data = json.dumps(self.__nj)
47 |         e = etree.SubElement(r,'numjobs')
48 |         e.text = data
49 |         
50 |         data = json.dumps(self.__iod)
51 |         e = etree.SubElement(r,'iodepth')
52 |         e.text = data
53 |         
54 |         data = json.dumps(self.__runtime)
55 |         e = etree.SubElement(r,'runtime')
56 |         e.text = data
57 |         
58 |         if self.__xargs != None:
59 |             data = json.dumps(list(self.__xargs))
60 |             e = etree.SubElement(r,'xargs')
61 |             e.text = data
62 | 
63 |     def fromXml(self,root):
64 |         '''
65 |         Loads the information about options from XML.
66 |         @param root The given element containing the information about
67 |         the object to be initialized.
68 |         '''
69 |         if root.findtext('numjobs'):
70 |             self.__nj = json.loads(root.findtext('numjobs'))
71 |         if root.findtext('iodepth'):
72 |             self.__iod = json.loads(root.findtext('iodepth'))
73 |         if root.findtext('runtime'):
74 |             self.__runtime = json.loads(root.findtext('runtime'))
75 |         if root.findtext('xargs'):
76 |                 self.__xargs = json.loads(root.findtext('xargs'))
77 |         logging.info("# Loading options from xml")
78 |         logging.info("# Options nj:"+str(self.__nj))
79 |         logging.info("# Options iod: "+str(self.__iod))
80 |         if self.__xargs != None:
81 |             logging.info("# Options xargs:")
82 |             logging.info(self.__xargs)


--------------------------------------------------------------------------------
/src/perfTest/PerfTest.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on 07.08.2012
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | __version__ = '2.2'
  7 | 
  8 | from abc import ABCMeta, abstractmethod
  9 | import logging
 10 | import subprocess
 11 | from lxml import etree
 12 | import json
 13 | import datetime
 14 | import os
 15 | import time
 16 | 
 17 | import perfTest.DeviceTests as dt
 18 | from perfTest.Devices import SSD
 19 | from perfTest.Devices import HDD
 20 | from perfTest.Options import Options
 21 | from reports.XmlReport import XmlReport
 22 | from reports.RstReport import RstReport
 23 | 
 24 | class PerfTest(object):
 25 |     '''
 26 |     A performance test, consists of multiple Device Tests.
 27 |     '''
 28 | 
 29 |     def __init__(self,testname,device):
 30 |         '''
 31 |         A performance test has several reports and plots.
 32 |         @param testname Name of the performance test.
 33 |         @param device A Device object, the device to run tests on.
 34 |         '''
 35 |         ## The output file for the fio job test results.
 36 |         self.__testname = testname
 37 | 
 38 |         ## The device object to run test on.
 39 |         self.__device = device
 40 | 
 41 |         ## Xml file to write test results to
 42 |         self.__xmlReport = XmlReport(self.__testname)
 43 | 
 44 |         ## Rst file to generate pdf of
 45 |         self.__rstReport = RstReport(self.__testname)
 46 | 
 47 |         ## Dictionary of tests to carry out
 48 |         self.__tests = {}
 49 | 
 50 |         ## Date the test has been carried out
 51 |         self.__testDate = None
 52 | 
 53 |         ## Per default use the version from main module
 54 |         self.__IOPerfVersion = __version__
 55 | 
 56 |         ## Information about the used operating system
 57 |         self.__OSInfo = {}
 58 |         self.collOSInfos()
 59 | 
 60 |         ## Hold the command line used to call the test
 61 |         self.__cmdLineArgs = None
 62 | 
 63 |     def getTestname(self): return self.__testname
 64 |     def getDevice(self): return self.__device
 65 |     def getTestDate(self): return self.__testDate
 66 |     def getIOPerfVersion(self): return self.__IOPerfVersion
 67 |     def getCmdLineArgs(self): return self.__cmdLineArgs
 68 |     def getOSInfo(self): return self.__OSInfo
 69 |     def getTests(self): return self.__tests
 70 |     def getXmlReport(self): return self.__xmlReport
 71 |     def getRstReport(self): return self.__rstReport
 72 | 
 73 |     def collOSInfos(self):
 74 |         '''
 75 |         Collects some information about the current OS in use.
 76 |         @return True if all infos are present, False on error.
 77 |         '''
 78 |         out = subprocess.Popen(['uname','-r'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
 79 |         (stdout,stderr) = out.communicate()
 80 |         if stderr != '':
 81 |             logging.error("uname -r encountered an error: " + stderr)
 82 |             return False
 83 |         else:
 84 |             self.__OSInfo['kernel'] = stdout
 85 |         #Check if we are on red hat based distributions
 86 |         if os.path.isfile('/etc/redhat-release'):
 87 |             out = subprocess.Popen(['cat','/etc/redhat-release'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
 88 |         else:
 89 |             out = subprocess.Popen(['lsb_release','-d'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
 90 |         (stdout,stderr) = out.communicate()
 91 |         if stderr != '':
 92 |             logging.error("getting OS information encountered an error: " + stderr)
 93 |             return False
 94 |         else:
 95 |             self.__OSInfo['lsb'] = stdout
 96 |         return True
 97 | 
 98 |     def readCmdLineArgs(self,argv):
 99 |         '''
100 |         Reads the command line argument list argv and sets it as
101 |         __cmdLineArgs.
102 |         @param argv The command line argument list.
103 |         '''
104 |         self.__cmdLineArgs = ''
105 |         for arg in argv:
106 |             self.__cmdLineArgs += (arg + ' ')
107 |         self.__cmdLineArgs = self.__cmdLineArgs.rstrip()
108 | 
109 |     def setOSInfo(self,key,value):
110 |         '''
111 |         Sets the current OS information.
112 |         @param key A key to be used in the OS info dict.
113 |         @param value The value being assigned to the key.
114 |         '''
115 |         if value != None:
116 |             self.__OSInfo[key] = value
117 | 
118 |     def setTestDate(self,dateStr):
119 |         '''
120 |         Sets the date the test has been carried out.
121 |         @param dateStr The date string.
122 |         '''
123 |         self.__testDate = dateStr
124 | 
125 |     def setIOPerfVersion(self,verStr):
126 |         '''
127 |         Sets the current io perf version.
128 |         @param verStr The version string.
129 |         '''
130 |         self.__IOPerfVersion = verStr
131 | 
132 |     def setCmdLineArgs(self,cmdLineStr):
133 |         '''
134 |         Sets the command line arg string.
135 |         @param cmdLineStr The command line string.
136 |         '''
137 |         self.__cmdLineArgs = cmdLineStr
138 | 
139 |     def addTest(self,key,test):
140 |         '''
141 |         Add a test to the test dictionary.
142 |         @param key The key for the test in the dictionary.
143 |         @param test The test object to be added.
144 |         '''
145 |         self.__tests[key] = test
146 | 
147 |     def resetTests(self):
148 |         '''
149 |         Clear the dictionary containing the tests.
150 |         '''
151 |         self.__tests.clear()
152 | 
153 |     def initialize(self):
154 |         '''
155 |         Initialize the given tests, this sets the device and Fio
156 |         init params for all tests.
157 |         '''
158 |         sorted(self.__tests.items())
159 |         for k,v in list(self.__tests.items()):
160 |             logging.info("# Initialiazing test "+k)
161 |             v.initialize()
162 | 
163 |     def runTests(self):
164 |         '''
165 |         Call the run method of every test in the test dictionary. The run method
166 |         of a test is its core function where the performance test is carried out.
167 |         '''
168 |         #sort per key to ensure tests have the same order
169 |         sorted(self.__tests.items())
170 |         for k,v in list(self.__tests.items()):
171 |             print("Starting test: " + k)
172 |             #before each test sleep, to ensure device operations of previous
173 |             #tests are finished
174 |             logging.info("# Sleeping for 5 seconds...")
175 |             time.sleep(5)
176 |             v.run()
177 | 
178 |     def genPlots(self):
179 |         '''
180 |         Generate the plots/charts for each specific test in the dictionary.
181 |         '''
182 |         sorted(self.__tests.items())
183 |         for k,v in list(self.__tests.items()):
184 |             logging.info("# Generating plots for "+k+" test")
185 |             v.genPlots()
186 | 
187 |     def toXml(self):
188 |         '''
189 |         First the device information is written to the xml file.
190 |         Calls for every test in the test dictionary the toXMl method
191 |         and writes the results to the xml file.
192 |         '''
193 |         tests = self.getTests()
194 |         e = self.getXmlReport().getXml()
195 |         # Add the current date to the xml
196 |         if self.__testDate != None:
197 |             dev = etree.SubElement(e,'testdate')
198 |             dev.text = json.dumps(self.__testDate)
199 |         # Add the device information to the xml file
200 |         self.getDevice().toXml(e)
201 |         # Add OS information
202 |         if self.__OSInfo != None:
203 |             if 'kernel' in self.__OSInfo:
204 |                 dev = etree.SubElement(e,'kernel')
205 |                 dev.text = json.dumps(self.__OSInfo['kernel'])
206 |             if 'lsb' in self.__OSInfo:
207 |                 dev = etree.SubElement(e,'lsb')
208 |                 dev.text = json.dumps(self.__OSInfo['lsb'])
209 |         # Add the current test suite version to the xml file
210 |         dev = etree.SubElement(e,'ioperfversion')
211 |         dev.text = json.dumps(self.__IOPerfVersion)
212 |         # Add the command line to xml
213 |         if self.__cmdLineArgs != None:
214 |             dev = etree.SubElement(e,'cmdline')
215 |             dev.text = json.dumps(self.__cmdLineArgs)
216 |         # Call the xml function for every test in the dictionary
217 |         sorted(self.__tests.items())
218 |         for k,v in tests.items():
219 |             e.append(v.toXml(k))
220 |         self.getXmlReport().xmlToFile(self.getTestname())
221 | 
222 |     def fromXml(self):
223 |         '''
224 |         Reads out the xml file name 'testname.xml' and initializes the test
225 |         specified with xml. The valid tags are "iops,lat,tp,writesat" for ssd,
226 |         "iops, tp" for hdd. But there isn't always every test run, so xml can
227 |         miss a test.
228 |         '''
229 |         self.getXmlReport().fileToXml(self.getTestname())
230 |         self.resetTests()
231 |         root = self.getXmlReport().getXml()
232 | 
233 |         if(root.findtext('testdate')):
234 |             self.setTestDate(json.loads(root.findtext('testdate')))
235 |         else:
236 |             self.setTestDate('n.a.')
237 |         # Read the operating system information
238 |         if(root.findtext('kernel')):
239 |             self.setOSInfo('kernel',json.loads(root.findtext('kernel')))
240 |         else:
241 |             self.setOSInfo('kernel','n.a.')
242 |         if(root.findtext('lsb')):
243 |             self.setOSInfo('lsb',json.loads(root.findtext('lsb')))
244 |         else:
245 |             self.setOSInfo('lsb','n.a.')
246 |         # Read version and command line
247 |         if(root.findtext('ioperfversion')):
248 |             self.setIOPerfVersion(json.loads(root.findtext('ioperfversion')))
249 |         if(root.findtext('cmdline')):
250 |             self.setCmdLineArgs(json.loads(root.findtext('cmdline')))
251 |         else:
252 |             self.setCmdLineArgs('n.a.')
253 |         # Create an empty options object, it is initialized in fromXml
254 |         options = Options(None,None)
255 |         # Initialize device and performance tests
256 |         if isinstance(self, SsdPerfTest):
257 |             device = SSD('ssd',None,self.getTestname())
258 |             device.fromXml(root)
259 |             for tag in SsdPerfTest.testKeys:
260 |                 #check which test tags are in the xml file
261 |                 for elem in root.iterfind(tag):
262 |                     test = None
263 |                     if elem.tag == SsdPerfTest.iopsKey:
264 |                         test = dt.SsdIopsTest(self.getTestname(),device,options)
265 |                     if elem.tag == SsdPerfTest.latKey:
266 |                         test = dt.SsdLatencyTest(self.getTestname(),device,options)
267 |                     if elem.tag == SsdPerfTest.tpKey:
268 |                         test = dt.SsdTPTest(self.getTestname(),device,options)
269 |                     if elem.tag == SsdPerfTest.wrKey:
270 |                         test = dt.SsdWriteSatTest(self.getTestname(),device,options)
271 |                     #we found a tag in the xml file, now we can read the data from xml
272 |                     if test != None:
273 |                         test.fromXml(elem)
274 |                         self.addTest(tag, test)
275 |         elif isinstance(self, HddPerfTest):
276 |             device = HDD('hdd',None,self.getTestname())
277 |             device.fromXml(root)
278 |             for tag in HddPerfTest.testKeys:
279 |                 for elem in root.iterfind(tag):
280 |                     test = None
281 |                     if elem.tag == HddPerfTest.iopsKey:
282 |                         test = dt.HddIopsTest(self.getTestname(),device,options)
283 |                     if elem.tag == HddPerfTest.tpKey:
284 |                         test = dt.HddTPTest(self.getTestname(),device,options)
285 |                     if test != None:
286 |                         test.fromXml(elem)
287 |                         self.addTest(tag, test)
288 | 
289 |     @abstractmethod
290 |     def toRst(self):
291 |         ''' Convert tests to restructured text '''
292 | 
293 |     def run(self):
294 |         ''' The main run method, runs tests, generates plots and rst report. '''
295 |         self.runTests()
296 |         self.toXml()
297 |         self.genPlots()
298 |         self.toRst()
299 | 
300 | class SsdPerfTest(PerfTest):
301 |     '''
302 |     A performance test for ssds consists of all ssd tests.
303 |     '''
304 |     ## Keys valid for tests
305 |     iopsKey = 'iops'
306 |     latKey = 'lat'
307 |     tpKey = 'tp'
308 |     wrKey = 'writesat'
309 |     ## Keys for the tests carried out
310 |     testKeys = [iopsKey,latKey,tpKey,wrKey]
311 | 
312 |     def __init__(self,testname,device,options=None):
313 |         '''
314 |         Cf. super constructor.
315 |         '''
316 |         PerfTest.__init__(self, testname, device)
317 |         #Add current date to test
318 |         now = datetime.datetime.now()
319 |         self.setTestDate(now.strftime("%Y-%m-%d"))
320 |         #Add every test to the performance test
321 |         for testType in SsdPerfTest.testKeys:
322 |             if testType == SsdPerfTest.iopsKey:
323 |                 test = dt.SsdIopsTest(testname,device,options)
324 |             if testType == SsdPerfTest.latKey:
325 |                 test = dt.SsdLatencyTest(testname,device,options)
326 |             if testType == SsdPerfTest.tpKey:
327 |                 test = dt.SsdTPTest(testname,device,options)
328 |             if testType == SsdPerfTest.wrKey:
329 |                 test = dt.SsdWriteSatTest(testname,device,options)
330 |             #Add the test to the key/value structure
331 |             self.addTest(testType, test)
332 | 
333 |     def toRst(self):
334 |         '''
335 |         Generate the rst report file, used to convert other report
336 |         formats from. The file is a simple text file as restructured text.
337 |         '''
338 |         tests = self.getTests()
339 |         rst = self.getRstReport()
340 | 
341 |         rst.addFooter()
342 |         rst.addTitle()
343 |         #add the device information and the feature matrix for one device
344 |         for keys in tests.keys():
345 |             rst.addDevInfo(tests[keys].getDevice().getDevInfo(),tests[keys].getDevice().getFeatureMatrix())
346 |             break
347 |         rst.addCmdLine(self.getCmdLineArgs())
348 | 
349 |         #add the fio version, nj, iod and general info of one test to the report
350 |         for keys in tests.keys():
351 |             if keys != 'lat':
352 |                 rst.addSetupInfo(self.getIOPerfVersion(),tests[keys].getFioJob().getFioVersion(),
353 |                                  self.getTestDate())
354 |                 rst.addFioJobInfo(tests[keys].getOptions().getNj(), tests[keys].getOptions().getIod())
355 |                 rst.addOSInfo(self.getOSInfo())
356 |                 rst.addGeneralInfo('ssd')
357 |                 break
358 | 
359 |         if SsdPerfTest.iopsKey in tests:
360 |             rst.addChapter("IOPS")
361 |             rst.addTestInfo('ssd','iops',tests['iops'])
362 |             rst.addSection("Measurement Plots")
363 |             for i,fig in enumerate(tests['iops'].getFigures()):
364 |                 rst.addFigure(fig,'ssd','iops',i)
365 |             rst.addSection("Measurement Window Summary Table")
366 |             rst.addTable(tests['iops'].getTables()[0],tests['iops'].getBsLabels(),'iops')
367 |         if SsdPerfTest.tpKey in tests:
368 |             rst.addChapter("Throughput")
369 |             rst.addTestInfo('ssd','tp',tests['tp'])
370 |             rst.addSection("Measurement Plots")
371 |             for i,fig in enumerate(tests['tp'].getFigures()):
372 |                 rst.addFigure(fig,'ssd','tp',i)
373 |             rst.addSection("Measurement Window Summary Table")
374 |             rst.addTable(tests['tp'].getTables()[0],tests['tp'].getBsLabels(),'tp')
375 |         if SsdPerfTest.latKey in tests:
376 |             rst.addChapter("Latency")
377 |             rst.addTestInfo('ssd','lat',tests['lat'])
378 |             rst.addSection("Measurement Plots")
379 |             for i,fig in enumerate(tests['lat'].getFigures()):
380 |                 #index 2 and 3 are 2D measurement plots that are not required
381 |                 #but we need them to generate the measurement overview table
382 |                 if i == 2 or i == 3: continue
383 |                 rst.addFigure(fig,'ssd','lat',i)
384 |             rst.addSection("Measurement Window Summary Table")
385 |             rst.addTable(tests['lat'].getTables()[0],tests['lat'].getBsLabels(),'avg-lat')#avg lat
386 |             rst.addTable(tests['lat'].getTables()[1],tests['lat'].getBsLabels(),'max-lat')#max lat
387 |         if SsdPerfTest.wrKey in tests:
388 |             rst.addChapter("Write Saturation")
389 |             rst.addTestInfo('ssd','writesat',tests['writesat'])
390 |             rst.addSection("Measurement Plots")
391 |             for i,fig in enumerate(tests['writesat'].getFigures()):
392 |                 rst.addFigure(fig,'ssd','writesat',i)
393 | 
394 |         rst.toRstFile()
395 | 
396 | class HddPerfTest(PerfTest):
397 |     '''
398 |     A performance test for hdds consists of all hdd tests.
399 |     '''
400 |     ## Keys valid for tests
401 |     iopsKey = 'iops'
402 |     tpKey = 'tp'
403 |     ## Keys valid for test dictionary and xml file
404 |     testKeys = [iopsKey,tpKey]
405 | 
406 |     def __init__(self,testname,device,options=None):
407 |         '''
408 |         Cf. super constructor.
409 |         '''
410 |         PerfTest.__init__(self, testname, device)
411 |         #Add current date
412 |         now = datetime.datetime.now()
413 |         self.setTestDate(now.strftime("%Y-%m-%d"))
414 |         #Add every test to the performance test
415 |         for testType in HddPerfTest.testKeys:
416 |             if testType == HddPerfTest.iopsKey:
417 |                 test = dt.HddIopsTest(testname,device,options)
418 |             if testType == HddPerfTest.tpKey:
419 |                 test = dt.HddTPTest(testname,device,options)
420 |             #add the test to the key/value structure
421 |             self.addTest(testType, test)
422 | 
423 |     def toRst(self):
424 |         '''
425 |         Generate the rst report file, used to convert other report
426 |         formats from. The file is a simple text file as restructured text.
427 |         '''
428 |         tests = self.getTests()
429 |         rst = self.getRstReport()
430 |         rst.addFooter()
431 |         rst.addTitle()
432 |         #add the device information and the feature matrix for one device
433 |         for keys in tests.keys():
434 |             rst.addDevInfo(tests[keys].getDevice().getDevInfo(),tests[keys].getDevice().getFeatureMatrix())
435 |             break
436 |         rst.addCmdLine(self.getCmdLineArgs())
437 | 
438 |         #Setup and OS infos are the same for all tests, just take one
439 |         for keys in tests.keys():
440 |             rst.addSetupInfo(self.getIOPerfVersion(),tests[keys].getFioJob().getFioVersion(),
441 |                              self.getTestDate())
442 |             rst.addFioJobInfo(tests[keys].getOptions().getNj(), tests[keys].getOptions().getIod())
443 |             rst.addOSInfo(self.getOSInfo())
444 |             rst.addGeneralInfo('hdd')
445 |             break
446 |         if HddPerfTest.iopsKey in tests:
447 |             rst.addChapter("IOPS")
448 |             rst.addTestInfo('hdd','iops',tests['iops'])
449 |             rst.addSection("Measurement Plots")
450 |             for i,fig in enumerate(tests['iops'].getFigures()):
451 |                 rst.addFigure(fig,'hdd','iops',i)
452 |         if HddPerfTest.tpKey in tests:
453 |             rst.addChapter("Throughput")
454 |             rst.addTestInfo('hdd','tp',tests['tp'])
455 |             rst.addSection("Measurement Plots")
456 |             for i,fig in enumerate(tests['tp'].getFigures()):
457 |                 rst.addFigure(fig,'hdd','tp',i)
458 |         rst.toRstFile()
459 | 


--------------------------------------------------------------------------------
/src/perfTest/StdyState.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on Aug 26, 2014
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | 
  7 | import logging
  8 | import numpy as np
  9 | import json
 10 | from lxml import etree
 11 | 
 12 | class StdyState(object):
 13 |     '''
 14 |     Used to define a stable state of a device
 15 |     '''
 16 |     ## Max number of carried out test rounds.
 17 |     testRnds = 25
 18 |     ## Always use a sliding window of 4 to measure performance values.
 19 |     testMesWindow = 4
 20 | 
 21 |     def __init__(self):
 22 |         '''
 23 |         Constructor
 24 |         '''
 25 |         ## Number of rounds until steady state has been reached
 26 |         self.__rounds = 0
 27 |         ## Number of round where steady state has been reached.
 28 |         self.__stdyRnds = []
 29 |         ## Dependent variable to detect steady state.
 30 |         self.__stdyValues = []
 31 |         ##Measured average in measurement window
 32 |         self.__stdyAvg = 0
 33 |         ##Slope of steady regression line.
 34 |         self.__stdySlope = []
 35 |         ##States if the steady state has been reached or not
 36 |         self.__reachStdyState = None
 37 | 
 38 |     def getRnds(self): return self.__rounds
 39 |     def getStdyRnds(self): return self.__stdyRnds
 40 |     def getStdyAvg(self): return self.__stdyAvg
 41 |     def getStdyValues(self): return self.__stdyValues
 42 |     def getStdySlope(self): return self.__stdySlope
 43 | 
 44 |     def setReachStdyState(self,s): self.__reachStdyState = s
 45 | 
 46 |     def isSteady(self):
 47 |         '''
 48 |         Return if the current state is steady.
 49 |         @return: True if yes, False if no
 50 |         @exception RuntimeError if state is not set (None)
 51 |         '''
 52 |         if self.__reachStdyState == None:
 53 |             raise RuntimeError("steady state is none")
 54 |         return self.__reachStdyState
 55 | 
 56 |     def checkSteadyState(self,xs,ys,rounds):
 57 |         '''
 58 |         Checks if the steady is reached for the given values.
 59 |         The steady state is defined by the allowed data excursion from the average (+-10%), and
 60 |         the allowed slope excursion of the linear regression best fit line (+-5%).
 61 |         @param xs Values on x axis
 62 |         @param ys Corresponding values for xs on y axis
 63 |         @param rounds Number of carried out rounds
 64 |         @return True (k*x+d is slope line) if steady state is reached, False if not
 65 |         '''
 66 |         stdyState = True
 67 |         maxY = max(ys)
 68 |         minY = min(ys)
 69 |         avg = sum(ys)/len(ys)#calc average of values
 70 |         #allow max excursion of 20% of average
 71 |         avgRange = avg * 0.20
 72 |         if (maxY - minY) > avgRange:
 73 |             stdyState = False
 74 | 
 75 |         #do linear regression to calculate slope of linear best fit
 76 |         y = np.array(ys)
 77 |         x = np.array(xs)
 78 |         A = np.vstack([x, np.ones(len(x))]).T
 79 |         #calculate k*x+d
 80 |         k, d = np.linalg.lstsq(A, y, rcond=-1)[0]
 81 | 
 82 |         #as we have a measurement window of 4, we calculate
 83 |         #the slope excursion in  the window
 84 |         slopeExc = k * self.testMesWindow
 85 |         if slopeExc < 0:
 86 |             slopeExc *= -1
 87 |         maxSlopeExc = avg * 0.10 #allowed are 10% of avg
 88 |         if slopeExc > maxSlopeExc:
 89 |             stdyState = False
 90 | 
 91 |         self.__rounds = rounds
 92 |         self.__stdyRnds = xs
 93 |         self.__stdyValues = ys
 94 |         self.__stdyAvg = avg
 95 |         # Clear previous slope
 96 |         self.__stdySlope = []
 97 |         self.__stdySlope.extend([k,d])
 98 |         self.__reachStdyState = stdyState
 99 |         return stdyState
100 | 
101 |     def appendXml(self,r):
102 |         '''
103 |         Append the information about a steady state test to a XML node.
104 |         @param root The xml root tag to append the new elements to
105 |         '''
106 |         data = json.dumps(list(self.__stdyRnds))
107 |         e = etree.SubElement(r,'stdyrounds')
108 |         e.text = data
109 | 
110 |         data = json.dumps(list(self.__stdyValues))
111 |         e = etree.SubElement(r,'stdyvalues')
112 |         e.text = data
113 | 
114 |         data = json.dumps(self.__stdySlope)
115 |         e = etree.SubElement(r,'stdyslope')
116 |         e.text = data
117 | 
118 |         data = json.dumps(self.__stdyAvg)
119 |         e = etree.SubElement(r,'stdyavg')
120 |         e.text = data
121 | 
122 |         data = json.dumps(self.__reachStdyState)
123 |         e = etree.SubElement(r,'reachstdystate')
124 |         e.text = data
125 | 
126 |         data = json.dumps(self.__rounds)
127 |         e = etree.SubElement(r,'rndnr')
128 |         e.text = data
129 | 
130 |     def fromXml(self,root):
131 |         '''
132 |         Loads the information about a steady state from XML.
133 |         @param root The given element containing the information about
134 |         the object to be initialized.
135 |         '''
136 |         self.__stdyRnds = json.loads(root.findtext('stdyrounds'))
137 |         self.__stdyValues = json.loads(root.findtext('stdyvalues'))
138 |         self.__stdySlope = json.loads(root.findtext('stdyslope'))
139 |         self.__stdyAvg = json.loads(root.findtext('stdyavg'))
140 |         self.__reachStdyState = json.loads(root.findtext('reachstdystate'))
141 |         self.__rounds = json.loads(root.findtext('rndnr'))
142 |         logging.info("########### Loading steady state from xml ###########")
143 |         self.toLog()
144 | 
145 |     def toLog(self):
146 |         '''
147 |         Log information about the steady state and how it
148 |         has been reached.
149 |         '''
150 |         logging.info("Rounds of steady state:")
151 |         logging.info(self.__stdyRnds)
152 |         logging.info("Steady values:")
153 |         logging.info(self.__stdyValues)
154 |         logging.info("K and d of steady best fit slope:")
155 |         logging.info(self.__stdySlope)
156 |         logging.info("Steady average:")
157 |         logging.info(self.__stdyAvg)
158 |         logging.info("Stopped after round number:")
159 |         logging.info(self.__rounds)
160 |         logging.info("Reached steady state:")
161 |         logging.info(self.__reachStdyState)
162 | 


--------------------------------------------------------------------------------
/src/perfTest/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/src/plots/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thomas-krenn/TKperf/638b0837b97b32337314993feb52b5111aae0979/src/plots/__init__.py


--------------------------------------------------------------------------------
/src/plots/compPlots.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on Oct 3, 2013
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | 
  7 | import plots.genPlots as pgp
  8 | import matplotlib.pyplot as plt
  9 | 
 10 | __colorTable__ = ['#0000FF','#32cd32','#ffff00','#00ffff','#b22222','#9932cc','#ff4500']
 11 | 
 12 | def compWriteSatIOPSPlt(testsToPlot, subfolder=None):
 13 |     """
 14 |     Compare multiple tests and create a write saturation IOPS plot.
 15 |     All test objects in testsToPlot are plotted.
 16 |     
 17 |     Keyword arguments:
 18 |     testsToPlot -- an array of perfTest objects
 19 |     """
 20 |     plt.clf()#clear plot
 21 |     min_y = 0
 22 |     max_y = 0
 23 |     max_x = 0
 24 |     for i,tests in enumerate(testsToPlot):
 25 |         test = tests.getTests()['writesat']
 26 |         rnds = test.getRnds()
 27 |         x = list(range(rnds + 1))
 28 |         #first elem in matrix are iops
 29 |         iops_l = test.getRndMatrices()[0]
 30 |         plt.plot(x,iops_l,'-',label=test.getTestname(), color = __colorTable__[i])
 31 |         #fetch new min and max from current test values
 32 |         min_y,max_y = pgp.getMinMax(iops_l, min_y, max_y)
 33 |         if max(x) > max_x:
 34 |             max_x = max(x)
 35 |     
 36 |     plt.ylim(min_y * 0.75, max_y * 1.25)
 37 |     #every 50 rounds print the round number
 38 |     x = list(range(0, max_x, 50))
 39 |     plt.xticks(x)
 40 |     plt.suptitle("Write Saturation Test",fontweight='bold')
 41 |     plt.xlabel("Round #")
 42 |     plt.ylabel("Avg. IOPS")
 43 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
 44 |                ncol=3, fancybox=True, shadow=True,prop={'size':10})
 45 |     if subfolder == None:
 46 |         plt.savefig('compWriteSatIOPSPlt.png',dpi=300)
 47 |     else:
 48 |         plt.savefig(subfolder+'/compWriteSatIOPSPlt.png',dpi=300)
 49 | 
 50 | def compILPlt(testsToPlot, mode, subfolder=None):
 51 |     """
 52 |     Compare multiple tests and create an IOPS or Latency plot.
 53 |     All test objects in testsToPlot are plotted.
 54 |     
 55 |     Keyword arguments:
 56 |     testsToPlot -- an array of perfTest objects
 57 |     mode -- the desired test mode (IOPS or Latency)
 58 |     """
 59 |     plt.clf()#clear plot
 60 |     x = list(range(3))
 61 |     width = 1/len(testsToPlot)
 62 |     max_y = 0
 63 |     for i in range(len(x)):
 64 |         x[i] = x[i] + (i * width)
 65 |     for i,tests in enumerate(testsToPlot):
 66 |         if mode == "IOPS":
 67 |             test = tests.getTests()['iops']
 68 |             pgp.calcMsmtTable(test, 'IOPS')
 69 |         if mode == "LAT":
 70 |             test = tests.getTests()['lat']
 71 |             pgp.calcMsmtTable(test, 'avg-LAT')
 72 |         mixWLds = test.getTables()[0]
 73 |         if mode == "IOPS":
 74 |             testVal = [mixWLds[0][6],mixWLds[3][6],mixWLds[6][6]]
 75 |         if mode == "LAT":
 76 |             testVal = [mixWLds[0][1],mixWLds[1][1],mixWLds[2][1]]
 77 |         plt.bar(x, testVal, width,label=test.getTestname(),color = __colorTable__[i])
 78 |         x = [v + width for v in x]
 79 |         if max(testVal) > max_y:
 80 |             max_y = max(testVal)
 81 |     ticksx = [(len(testsToPlot)/2) * width, 1 + width + 0.5, 2 + (2 * width) + 0.5 ]
 82 |     if mode == "IOPS":
 83 |         labelsx = ['Read','50/50','Write']
 84 |     if mode == "LAT":
 85 |         labelsx = ['Read','65/35','Write']
 86 |     plt.xticks(ticksx, labelsx)
 87 |     plt.ylim(0, max_y * 1.15)
 88 |     if mode == "IOPS":
 89 |         title = "IOPS"
 90 |         plt.ylabel("Avg. " + title + " at 4k Block Size")
 91 |     if mode == "LAT":
 92 |         title = "LAT"
 93 |         plt.ylabel("Avg. "+ title + " (ms) at 4k Block Size")
 94 |     plt.suptitle(title + " Measurement Test",fontweight='bold')
 95 |     plt.xlabel("R/W Workload")
 96 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
 97 |                ncol=3,fancybox=True, shadow=True,prop={'size':10})
 98 |     if subfolder == None:
 99 |         plt.savefig('comp'+title+'Plt.png',dpi=300)
100 |     else:
101 |         plt.savefig(subfolder+'/comp'+title+'Plt.png',dpi=300)
102 | 
103 | def compTPPlt(testsToPlot, subfolder=None):
104 |     """
105 |     Compare multiple tests and create a throughput plot.
106 |     All test objects in testsToPlot are plotted.
107 |     
108 |     Keyword arguments:
109 |     testsToPlot -- an array of perfTest objects
110 |     """
111 |     plt.clf()#clear plot
112 |     height = 1/len(testsToPlot)
113 |     ticksy = [(len(testsToPlot)/2) * height, 1 + height + 0.5, 2 + (2 * height) + 0.5 ]
114 |     labelsy = ['8k','64k','1024k']
115 |     max_x = 0
116 |     fig = plt.figure()
117 |     # Plot read throughput
118 |     ax = fig.add_subplot(2, 1, 2)
119 |     y = list(range(3))
120 |     for i in range(len(y)):
121 |         y[i] = y[i] + (i * height)
122 |     for i,tests in enumerate(testsToPlot):
123 |         test = tests.getTests()['tp']
124 |         pgp.calcMsmtTPTable(test)
125 |         wlds = test.getTables()[0]
126 |         testRTP = [wlds[0][2],wlds[0][1],wlds[0][0]]
127 |         ax.barh(y, testRTP, height, label=test.getTestname(),color = __colorTable__[i])
128 |         y = [v + height for v in y]
129 |         if max(testRTP) > max_x:
130 |             max_x = max(testRTP)
131 |     plt.xlabel("Read Bandwidth (MB/s)")
132 |     plt.xlim(0,max_x*1.05)
133 |     plt.yticks(ticksy, labelsy)
134 |     plt.ylabel("Block Size (Byte)")
135 |     plt.grid()
136 |     
137 |     # Plot write throughput
138 |     ax = fig.add_subplot(2, 1, 1)
139 |     y = list(range(3))
140 |     for i in range(len(y)):
141 |         y[i] = y[i] + (i * height)
142 |     for i,tests in enumerate(testsToPlot):
143 |         test = tests.getTests()['tp']
144 |         wlds = test.getTables()[0]
145 |         testRTP = [wlds[1][2],wlds[1][1],wlds[1][0]]
146 |         ax.barh(y, testRTP, height, label=test.getTestname(),color = __colorTable__[i])
147 |         y = [v + height for v in y]
148 |     plt.xlabel("Write Bandwidth (MB/s)")
149 |     plt.xlim(0,max_x*1.05)
150 |     plt.yticks(ticksy, labelsy)
151 |     plt.ylabel("Block Size (Byte)")
152 |     plt.grid()
153 |     plt.legend()
154 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.18),
155 |        ncol=3,fancybox=True, shadow=True,prop={'size':9})
156 |     plt.suptitle("TP R/W Measurement Test",fontweight='bold')
157 |     if subfolder == None:
158 |         plt.savefig('compTPPlt.png',dpi=300)
159 |     else:
160 |         plt.savefig(subfolder+'/compTPPlt.png',dpi=300)
161 | 


--------------------------------------------------------------------------------
/src/plots/genPlots.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on 09.07.2012
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | 
  7 | import matplotlib
  8 | matplotlib.use('Agg')
  9 | import matplotlib.pyplot as plt
 10 | import matplotlib.ticker as ticker
 11 | import matplotlib.colors as pltm
 12 | from mpl_toolkits.mplot3d import Axes3D
 13 | 
 14 | import numpy as np
 15 | from copy import deepcopy
 16 | 
 17 | import perfTest.DeviceTests as dt
 18 | 
 19 | __matplotVersion__=float('.'.join(matplotlib.__version__.split('.')[0:2]))
 20 | 
 21 | def stdyStVerPlt(toPlot,mode):
 22 |     '''
 23 |     Generate a steady state verification plot.
 24 |     The plot includes:
 25 |     -Measured IOPS|Latencies|Throughput of rounds in which steady state was reached
 26 |     -Average IOPS|Latencies|Throughput in steady state rounds
 27 |     -Slope of best fit line
 28 |     -Top and Bottom limits: +-10% percent of average
 29 |     The figure is saved as SsdTest.Testname-stdyStVerPlt.png.
 30 |     @param toPlot A SsdTest object.
 31 |     @param mode A string representing the test mode (IOPS|LAT|TP)
 32 |     '''
 33 |     x = np.array(toPlot.getStdyState().getStdyRnds())
 34 |     #calculate average and its top and bottom limit
 35 |     av = []
 36 |     avT = []
 37 |     avB = []
 38 |     av.append(toPlot.getStdyState().getStdyAvg())
 39 |     avTop = toPlot.getStdyState().getStdyAvg() * 1.10
 40 |     avBot = toPlot.getStdyState().getStdyAvg() * 0.9
 41 |     avT.append(avTop)
 42 |     avB.append(avBot)
 43 |     av = av * len(x)
 44 |     avT = avT * len(x)
 45 |     avB = avB * len(x)
 46 | 
 47 |     plt.clf()#clear
 48 |     plt.plot(x,toPlot.getStdyState().getStdyValues(),'o', label=mode, markersize=10)
 49 |     plt.plot(x, toPlot.getStdyState().getStdySlope()[0]*x + toPlot.getStdyState().getStdySlope()[1], 'r', label='Slope')
 50 |     plt.plot(x, av, '-', color='black',label='Average')
 51 |     plt.plot(x, avT, '--', color='black',label='Top')
 52 |     plt.plot(x, avB, '--', color='black',label='Bottom')
 53 |     
 54 |     #set the y axes to start at 3/4 of mininum
 55 |     plt.ylim(min(toPlot.getStdyState().getStdyValues())*0.75,max(toPlot.getStdyState().getStdyValues())*1.25)
 56 |     plt.xticks(x)
 57 |     title = mode + " Steady State Verification Plot"
 58 |     plt.suptitle(title,fontweight='bold')
 59 |     plt.xlabel("Round #")
 60 |     if mode == "LAT":
 61 |         plt.ylabel("Latency (us)")
 62 |     if mode == "TP":
 63 |         plt.ylabel("Bandwidth (KB/s)")
 64 |     if mode == "IOPS":
 65 |         plt.ylabel(mode)
 66 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
 67 |                ncol=3, fancybox=True, shadow=True,prop={'size':12})
 68 |     plt.savefig(toPlot.getTestname()+'-'+mode+'-stdyStVerPlt.png',dpi=300)
 69 |     toPlot.addFigure(toPlot.getTestname()+'-'+mode+'-stdyStVerPlt.png')
 70 | 
 71 | def stdyStConvPlt(toPlot,mode):
 72 |     '''
 73 |     Generate a steady state convergence plot.
 74 |     The plot consists of:
 75 |     IOPS:
 76 |         -Measured IOPS of pure random write
 77 |     LAT:
 78 |         -Avg latency of read, mixed, write
 79 |     -All lines are the different block sizes
 80 |     -IOPS/Latencies of all the rounds are plotted
 81 |     The figure is saved as SsdTest.Testname-stdyStConvPlt.png.
 82 |     @param toPlot A SsdTest object.
 83 |     @param mode A string representing the test mode (IOPS|LAT)
 84 |     '''
 85 |     rnds = toPlot.getStdyState().getRnds()
 86 |     matrices = toPlot.getRndMatrices()
 87 |     bsLens = len(matrices[0][-1])#fetch the number of bs of the first matrix
 88 |     
 89 |     #initialize matrix for plotting
 90 |     if mode == "IOPS":
 91 |         lines = []
 92 |         for i in range(bsLens):
 93 |             lines.append([])
 94 |         for rndMat in matrices:
 95 |             row = rndMat[-1]#last row is random write
 96 |             for i in range(len(row)):
 97 |                 lines[i].append(row[i])#switch from row to column wise ordering of values
 98 |     
 99 |     if mode == "LAT":
100 |         readLines = []
101 |         writeLines = []
102 |         mixLines = []
103 |         for i in range(bsLens):
104 |             readLines.append([])
105 |             writeLines.append([])
106 |             mixLines.append([])
107 |         for rndMat in matrices:
108 |             #take read mat len, every elem has the same len
109 |             for i in range(len(rndMat[0])):
110 |                 #also convert it from us to ms
111 |                 readLines[i].append((rndMat[0][i][2]) / 1000)#mean latency
112 |                 mixLines[i].append((rndMat[1][i][2]) / 1000)#mean latency
113 |                 writeLines[i].append((rndMat[2][i][2]) / 1000)#mean latency
114 |     
115 |     plt.clf()#clear
116 | 
117 |     #fetch number of rounds, we want to include all rounds
118 |     x = list(range(rnds + 1))
119 |     max_y = 0
120 |     min_y = 0
121 |     if mode == "IOPS":
122 |         for i in range(len(lines)):
123 |             min_y,max_y = getMinMax(lines[i], min_y, max_y)
124 |             plt.plot(x,lines[i],'o-',label='bs='+toPlot.getBsLabels()[i])
125 |     if mode == "LAT":
126 |         for i in range(len(readLines)):
127 |             min_y,max_y = getMinMax(readLines[i], min_y, max_y)
128 |             plt.plot(x,readLines[i],'s-',label='bs='+toPlot.getBsLabels()[i]+' read')
129 |         for i in range(len(mixLines)):
130 |             min_y,max_y = getMinMax(mixLines[i], min_y, max_y)
131 |             plt.plot(x,mixLines[i],'^-',label='bs='+toPlot.getBsLabels()[i]+' mixed')
132 |         for i in range(len(writeLines)):
133 |             min_y,max_y = getMinMax(writeLines[i], min_y, max_y)
134 |             plt.plot(x,writeLines[i],'o-',label='bs='+toPlot.getBsLabels()[i]+' write')
135 |     
136 |     plt.xticks(x)
137 |     plt.suptitle(mode+" Steady State Convergence Plot",fontweight='bold')
138 |     plt.xlabel("Round #")
139 |     plt.ylim((min_y*0.75,max_y*1.25))
140 |     if mode == "LAT":
141 |         plt.ylabel("Latency (ms)")
142 |     if mode == "IOPS":
143 |         plt.ylabel(mode)
144 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
145 |                ncol=3, fancybox=True, shadow=True,prop={'size':12})
146 |     plt.savefig(toPlot.getTestname()+'-'+mode+'-stdyStConvPlt.png',dpi=300)
147 |     toPlot.addFigure(toPlot.getTestname()+'-'+mode+'-stdyStConvPlt.png')
148 |     
149 | def mes2DPlt(toPlot,mode):
150 |     '''
151 |     Generate a measurement 2D plot and the measurement overview table.
152 |     The plot includes:
153 |     -Lines of the workloads
154 |     -Each line consists of the average of IOPS/Latencies per round
155 |     for each block size in the measurement window!
156 |     Therefore the x axes are the block sizes, the plotted lines
157 |     are the different workloads (from 100% read to 100% write). The
158 |     y axes are the IOPS/Latencies over the measurement window for each block sizes
159 |     and workload.
160 |     The figure is saved as SsdTest.Testname-mode-mes2DPlt.png.
161 |     @param toPlot A SsdTest object.
162 |     @param mode A string representing the test mode (IOPS|max-LAT|avg-LAT)
163 |     '''
164 |     # Generate the measurement table
165 |     calcMsmtTable(toPlot, mode)
166 |     if mode == "IOPS":
167 |         wlds = dt.SsdIopsTest.mixWlds
168 |         bsLabels = toPlot.getBsLabels()
169 |         mixWLds = toPlot.getTables()[0]
170 |     if mode == "avg-LAT" or mode == "max-LAT":
171 |         wlds = dt.SsdLatencyTest.mixWlds
172 |         bsLabels = toPlot.getBsLabels()
173 |         if mode == "avg-LAT":
174 |             mixWLds = toPlot.getTables()[0]
175 |         if mode == "max-LAT":
176 |             mixWLds = toPlot.getTables()[1]
177 | 
178 |     plt.clf()#clear plot
179 |     if mode == "IOPS":
180 |         x = getBS(toPlot.getBsLabels())
181 |     if mode == "avg-LAT" or mode == "max-LAT":
182 |         x = getBS(toPlot.getBsLabels())
183 |         
184 |     max_y = 0
185 |     min_y = 0
186 |     for i in range(len(mixWLds)):
187 |         min_y,max_y = getMinMax(mixWLds[i], min_y, max_y)
188 |         #the labels are r/w percentage of mixed workload
189 |         plt.plot(x,mixWLds[i],'o-',
190 |                   label=str(wlds[i])+'/'+str(100-wlds[i]))
191 |     if mode == 'IOPS':
192 |         plt.yscale('log')
193 |         plt.xscale('log')
194 |         plt.ylabel(mode)
195 |         plt.legend(prop={'size':12})
196 |     if mode == "avg-LAT" or mode == "max-LAT":
197 |         plt.ylabel("Latency (ms)")
198 |         plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
199 |                ncol=3, fancybox=True, shadow=True,prop={'size':12})
200 |     
201 |     plt.xlabel("Block Size (Byte)")
202 |     #scale axis to min and max
203 |     plt.ylim((min_y*0.75,max_y*1.15))
204 |     plt.xticks(x,bsLabels)
205 |     plt.suptitle(mode+" Measurement Plot",fontweight='bold')
206 |     plt.savefig(toPlot.getTestname()+'-'+mode+'-mes2DPlt.png',dpi=300)
207 |     toPlot.addFigure(toPlot.getTestname()+'-'+mode+'-mes2DPlt.png')
208 | 
209 | def mes3DPlt(toPlot,mode):
210 |     '''
211 |     Generate a measurement 3D plot. This plot depends on the
212 |     mes2DPlt as there the measurement overview table is calculated.
213 |     @param toPlot A SsdTest object.
214 |     @param mode A string representing the test mode (IOPS)
215 |     '''
216 |     colorTable = ['#0000FF','#008080','#00FFFF','#FFFF00','#00FF00','#FF00FF','#800000']
217 |     if mode == 'IOPS':
218 |         #Iops have only one measurement table
219 |         matrix = deepcopy(toPlot.getTables()[0])
220 |         #reverse to start with 0/100
221 |         matrix.reverse()
222 |         #reverse the block size in each table row, to start with 512B
223 |         for row in matrix:
224 |             row.reverse()
225 |         bsLabels = list(toPlot.getBsLabels())
226 |         mixWlds = list(dt.SsdIopsTest.mixWlds)
227 |     
228 |     #define positions for bars
229 |     ypos = np.array([0.25] * len(bsLabels)) 
230 |     xpos = np.arange(0.25, len(bsLabels)+0.25, 1)
231 |     zpos = np.array([0] * len(bsLabels))
232 |     
233 |     #define widht and height (x,y) of bars
234 |     # z will be the measured values
235 |     dx = np.array([0.5] * len(bsLabels))
236 |     dy = np.array([0.5] * len(bsLabels))
237 |     
238 |     plt.clf
239 |     fig = plt.figure()
240 |     if __matplotVersion__ >= 1.0:
241 |         ax = fig.gca(projection='3d')
242 |     else:
243 |         ax = Axes3D(fig)
244 |     for j,wl in enumerate(matrix):
245 |         ax.bar3d(xpos,ypos,zpos, dx, dy, wl, color=pltm.colorConverter.to_rgba_array(colorTable[j]))
246 |         for pos in range(len(ypos)):
247 |             ypos[pos] += 1
248 |             
249 |     ticksx = np.arange(0.5, len(bsLabels), 1)
250 |     bsLabels.reverse()
251 |     ticksy = np.arange(0.5, len(mixWlds), 1)
252 |     mixWlds.reverse()
253 |     if __matplotVersion__ >= 1.0:
254 |         plt.yticks(ticksy,mixWlds)
255 |         plt.xticks(ticksx, bsLabels)
256 |     else:
257 |         ax.w_xaxis.set_major_locator(ticker.FixedLocator(ticksx))
258 |         ax.w_xaxis.set_ticklabels(bsLabels)
259 |         ax.w_yaxis.set_major_locator(ticker.FixedLocator(ticksy))
260 |         ax.w_yaxis.set_ticklabels(mixWlds)
261 |     
262 |     plt.suptitle(mode+" 3D Measurement Plot",fontweight='bold')
263 |     ax.set_xlabel('Block Size (Byte)')
264 |     ax.set_ylabel('R/W Mix%')
265 |     if mode == 'IOPS':
266 |         ax.set_zlabel('IOPS',rotation='vertical')
267 |     plt.savefig(toPlot.getTestname()+'-'+mode+'-mes3DPlt.png',dpi=300)
268 |     toPlot.addFigure(toPlot.getTestname()+'-'+mode+'-mes3DPlt.png')
269 | 
270 | def latMes3DPlt(toPlot):
271 |     '''
272 |     Generate a measurement 3D plot for latency. This plot depends on the
273 |     mes2DPlt as there the measurement overview table is calculated.
274 |     @param toPlot A SsdTest object.
275 |     '''
276 |     colorTable = ['#0000FF','#008080','#00FFFF']
277 |     mixWlds = list(dt.SsdLatencyTest.mixWlds)
278 |     bsLabels = list(toPlot.getBsLabels())
279 | 
280 |     avgMatrix = deepcopy(toPlot.getTables()[0])
281 |     maxMatrix = deepcopy(toPlot.getTables()[1])
282 |     
283 |     #define positions for bars
284 |     ypos = np.array([0.25] * len(bsLabels)) 
285 |     xpos = np.arange(0.25, len(bsLabels)+0.25, 1)
286 |     zpos = np.array([0] * len(bsLabels))
287 |     
288 |     #define widht and height (x,y) of bars
289 |     # z will be the measured values
290 |     dx = np.array([0.5] * len(bsLabels))
291 |     dy = np.array([0.5] * len(bsLabels))
292 |     
293 |     plt.clf()
294 |     fig = plt.figure()
295 |     if __matplotVersion__ >= 1.0:
296 |         ax = fig.add_subplot(2, 1, 1, projection='3d')
297 |     else:
298 |         rect = fig.add_subplot(2, 1, 1).get_position()
299 |         ax = Axes3D(fig, rect)
300 |     for j,wl in enumerate(avgMatrix):
301 |         ax.bar3d(xpos,ypos,zpos, dx, dy, wl, color=pltm.colorConverter.to_rgba_array(colorTable[j]))
302 |         for pos in range(len(ypos)):
303 |             ypos[pos] += 1
304 |     ax.xaxis.set_ticks([]) 
305 |     ax.yaxis.set_ticks([])
306 |     ax.set_zlabel('Latency (ms)',rotation='vertical')
307 |             
308 |     #Second subplot
309 |     if __matplotVersion__ >= 1.0:
310 |         ax = fig.add_subplot(2,1,2, projection='3d')
311 |     else:
312 |         rect = fig.add_subplot(2, 1, 2).get_position()
313 |         ax = Axes3D(fig, rect)
314 |     #reset ypos
315 |     ypos = np.array([0.25] * len(bsLabels)) 
316 |     for j,wl in enumerate(maxMatrix):
317 |         ax.bar3d(xpos,ypos,zpos, dx, dy, wl, color=pltm.colorConverter.to_rgba_array(colorTable[j]))
318 |         for pos in range(len(ypos)):
319 |             ypos[pos] += 1
320 |             
321 |     ticksx = np.arange(0.5, len(bsLabels), 1)
322 |     ticksy = np.arange(0.5, len(mixWlds), 1)
323 |     if __matplotVersion__ >= 1.0:
324 |         plt.xticks(ticksx, bsLabels)
325 |         plt.yticks(ticksy,mixWlds)
326 |     else:
327 |         ax.w_xaxis.set_major_locator(ticker.FixedLocator(ticksx))
328 |         ax.w_xaxis.set_ticklabels(bsLabels)
329 |         ax.w_yaxis.set_major_locator(ticker.FixedLocator(ticksy))
330 |         ax.w_yaxis.set_ticklabels(mixWlds)
331 | 
332 |     plt.suptitle("LAT 3D Measurement Plot",fontweight='bold')
333 |     #ax.set_xlabel('Block Size (Byte)')
334 |     ax.set_ylabel('R/W Mix%')
335 |     ax.set_zlabel('Latency (ms)',rotation='vertical')
336 |     plt.savefig(toPlot.getTestname()+'-LAT-mes3DPlt.png',dpi=300)
337 |     toPlot.addFigure(toPlot.getTestname()+'-LAT-mes3DPlt.png')
338 | 
339 | def writeSatIOPSPlt(toPlot):
340 |     #fetch number of rounds, we want to include all rounds
341 |     #as stdy state was reached at rnds, it must be included
342 |     rnds = toPlot.getRnds()
343 |     x = list(range(rnds + 1))
344 |     
345 |     iops_l = toPlot.getRndMatrices()[0]#first elem in matrix are iops
346 | 
347 |     plt.clf()#clear plot        
348 |     plt.plot(x,iops_l,'-',label='Avg IOPS')
349 |     plt.ylim(min(iops_l)*0.75,max(iops_l)*1.25)
350 |     #every 10 rounds print the round number
351 |     x = list(range(0,rnds + 1,50))
352 |     plt.xticks(x)
353 |     plt.suptitle("Write Saturation Test",fontweight='bold')
354 |     plt.xlabel("Round #")
355 |     plt.ylabel("IOPS")
356 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
357 |                ncol=1, fancybox=True, shadow=True,prop={'size':12})
358 |     plt.savefig(toPlot.getTestname()+'-writeSatIOPSPlt.png',dpi=300)
359 |     toPlot.addFigure(toPlot.getTestname()+'-writeSatIOPSPlt.png')
360 |     
361 | def writeSatLatPlt(toPlot):
362 |     rnds = toPlot.getRnds()
363 |     x = list(range(rnds + 1))
364 |     
365 |     lats_l = toPlot.getRndMatrices()[1]#second elem in matrix are latencies
366 |     
367 |     #get the average latencies from the lat list (last elem)
368 |     av_lats = []
369 |     for i in lats_l:
370 |         #convert from us to ms
371 |         av_lats.append((i[2]) / 1000)
372 |     
373 |     plt.clf()#clear plot
374 |     plt.plot(x,av_lats,'-',label='Avg latency')
375 |     #set the y axes to start at 3/4 of mininum
376 |     plt.ylim(min(av_lats)*0.75,max(av_lats)*1.25)
377 |     #every 50 rounds print the round number
378 |     x = list(range(0,rnds + 1,50))
379 |     plt.xticks(x)
380 |     plt.suptitle("Write Saturation Test",fontweight='bold')
381 |     plt.xlabel("Round #")
382 |     plt.ylabel("Latency (ms)")
383 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
384 |                ncol=1, fancybox=True, shadow=True,prop={'size':12})
385 |     plt.savefig(toPlot.getTestname()+'-writeSatLatPlt.png',dpi=300)
386 |     toPlot.addFigure(toPlot.getTestname()+'-writeSatLatPlt.png')
387 |     
388 | def tpRWStdyStConvPlt(toPlot):
389 |     '''
390 |     Generate one steady state convergence plot for throughput read and write measurements.
391 |     The plot consists of:
392 |     -Measured bandwidths per round per block size
393 |     -All lines are the different block sizes
394 |     -x axes is the number of all rounds
395 |     -y axes is the bw of the corresponding round
396 |     The top plot is write, below read
397 |     The figure is saved as SsdTest.Testname-TP-RW-stdyStConvPlt.png.
398 |     @param toPlot A SsdTest object.
399 |     '''
400 |     matrices = deepcopy(toPlot.getRndMatrices())
401 |     rnds = toPlot.getStdyState().getRnds()#fetch the number of total rounds
402 |     bsLens = len(matrices)#fetch the number of bs, each row is a bs in the matrix
403 |     bsLabels = toPlot.getBsLabels()
404 |     
405 |     #initialize matrix for plotting
406 |     lines = []
407 |     for i in range(bsLens):
408 |         lines.append([])
409 |     
410 |     #values for scaling the axes
411 |     max_y = 0
412 |     min_y = 0
413 |     
414 |     plt.clf()#clear
415 |     x = list(range(rnds+1))#ensure to include all rounds
416 |     
417 |     plt.clf
418 |     fig = plt.figure()
419 |     ax = fig.add_subplot(2, 1, 1)
420 |     for i,rndMat in enumerate(matrices):
421 |         row = rndMat[1]#plot the write row
422 |         #convert to MB/s
423 |         for v in range(len(row)):
424 |             row[v] = row[v] / 1024
425 |         #calc min,man to scale axes
426 |         min_y,max_y = getMinMax(row, min_y, max_y)
427 |         ax.plot(x,row,'o-',label='bs='+bsLabels[i])
428 |     plt.xticks(x)
429 |     plt.ylim((0,max_y*1.15))
430 |     plt.legend()
431 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.10),
432 |                ncol=5, fancybox=True, shadow=True,prop={'size':11})
433 |     plt.ylabel("Write TP (MB/s)")
434 |      
435 |     ax = fig.add_subplot(2, 1, 2)
436 |     for i,rndMat in enumerate(matrices):
437 |         row = rndMat[0]#plot the read row
438 |         #convert to MB/s
439 |         for v in range(len(row)):
440 |             row[v] = row[v] / 1024
441 |         #calc min,man to scale axes
442 |         min_y,max_y = getMinMax(row, min_y, max_y)
443 |         ax.plot(x,row,'o-',label='bs='+bsLabels[i])
444 |     plt.xticks(x)
445 |     plt.ylim((0,max_y*1.15))
446 |     plt.ylabel("Read TP (MB/s)")
447 | 
448 |     plt.xlabel("Round #")
449 |     plt.suptitle("TP R/W Steady State Convergence Plot",fontweight='bold')
450 | 
451 |     plt.savefig(toPlot.getTestname()+'-TP-RW-stdyStConvPlt.png',dpi=300)
452 |     toPlot.addFigure(toPlot.getTestname()+'-TP-RW-stdyStConvPlt.png')
453 | 
454 | def tpMes2DPlt(toPlot):
455 |     '''
456 |     Generate a measurement 2D plot and the measurement overview table for throughput.
457 |     The plot includes:
458 |     -One line for read, one line for write
459 |     -Each line consists of the average of IOPS per round
460 |     for each block size in the measurement window!
461 |     Therefore the x axes are the block sizes, the y axes is
462 |     the average over the measurement window for each block size.
463 |     The figure is saved as SsdTest.Testname-bw-mes2DPlt.png.
464 |     @param toPlot A SsdTest object.
465 |     '''
466 |     calcMsmtTPTable(toPlot)
467 |     wlds = toPlot.getTables()[0]
468 |     #start plotting
469 |     plt.clf()#clear
470 |     x = getBS(toPlot.getBsLabels())
471 |     for i in range(len(wlds)):
472 |         if i == 0:
473 |             label = "read"
474 |         else:
475 |             label = "write"
476 |         plt.plot(x,wlds[i],'o-',label=label)
477 | 
478 |     plt.xscale('log')
479 |     plt.suptitle("TP Measurement Plot",fontweight='bold')
480 |     plt.xlabel("Block Size (Byte)")
481 |     plt.ylabel("Bandwidth (MB/s)")
482 |     plt.xticks(x,toPlot.getBsLabels())
483 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
484 |                ncol=3, fancybox=True, shadow=True,prop={'size':12})
485 |     plt.savefig(toPlot.getTestname()+'-TP-mes2DPlt.png',dpi=300)
486 |     toPlot.addFigure(toPlot.getTestname()+'-TP-mes2DPlt.png')
487 | 
488 | ######### PLOTS FOR HDD TESTS #########
489 | def TPplot(toPlot):
490 |     '''
491 |     Generate a R/W throughput measurement plot for the hdd round results.
492 |     The plot consists of:
493 |     -Measured bandwidths per round, per block size for read and write
494 |     -x axes is the number of carried out rounds
495 |     -y axes is the bandwidth of the corresponding round
496 |     The figure is saved as TPTest.Testname-TP-RW-Plt.png.
497 |     @param toPlot A hdd TPTest object.
498 |     '''
499 |     #As the values are converted to KB, copy the matrices
500 |     matrices = deepcopy(toPlot.getRndMatrices())
501 |     rnds = dt.HddTPTest.maxRnds
502 |     bsLabels = toPlot.getBsLabels()
503 |     
504 |     #values for scaling the axes
505 |     max_y = 0
506 |     min_y = 0
507 |     
508 |     plt.clf()#clear
509 |     x = list(range(rnds))
510 |     for i,rndMat in enumerate(matrices):
511 |         #convert to MB/S
512 |         for v in range(len(rndMat[0])):
513 |             rndMat[0][v] = (rndMat[0][v]) / 1024
514 |         #plot the read row for current BS
515 |         min_y,max_y = getMinMax(rndMat[0], min_y, max_y)
516 |         plt.plot(x,rndMat[0],'o-',label='read bs='+bsLabels[i])
517 |         #plot the write row for current BS
518 |         for v in range(len(rndMat[1])):
519 |             rndMat[1][v] = (rndMat[1][v]) / 1024
520 |         min_y,max_y = getMinMax(rndMat[1], min_y, max_y)
521 |         plt.plot(x,rndMat[1],'o-',label='write bs='+bsLabels[i])
522 |     
523 |     x = list(range(0,rnds+1,16))
524 |     plt.xticks(x)
525 |     plt.suptitle("TP Measurement Plot",fontweight='bold')    
526 |     plt.xlabel("Area of Device (in rounds)")
527 |     plt.ylabel("Bandwidth (MB/s)")
528 |     #scale axis to min and max +- 15%
529 |     plt.ylim((min_y*0.6,max_y*1.15))
530 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
531 |                ncol=2, fancybox=True, shadow=True,prop={'size':12})
532 |     plt.savefig(toPlot.getTestname()+'-TP-RW-Plt.png',dpi=300)
533 |     toPlot.addFigure(toPlot.getTestname()+'-TP-RW-Plt.png')
534 | 
535 | def IOPSplot(toPlot):
536 |     '''
537 |     Generate the IOPS plot for a hdd performance test. The plot consists
538 |     of plotting the IOPS results from the 128 rounds that have been carried
539 |     out. In each round the mixed workloads and all block sizes are plotted.
540 |     @param toPlot An hdd IopsTest object.
541 |     '''
542 |     rnds = dt.HddIopsTest.maxRnds
543 |     matrices = toPlot.getRndMatrices()
544 |     
545 |     wlds = dt.HddIopsTest.mixWlds
546 |     bsLabels = toPlot.getBsLabels()
547 |     
548 |     #each row will be a workload percentage
549 |     mixWLds = []
550 |     for i in range(len(wlds)):
551 |         mixWLds.append([])
552 |         #in each row will be the different block sizes
553 |         for bs in range(len(bsLabels)):
554 |             mixWLds[i].append([])
555 |             
556 |     for rnd in matrices:
557 |         #each round has its workloads
558 |         for i,row in enumerate(rnd):
559 |             #each workload has its block sizes
560 |             for j,bs in enumerate(row):
561 |                 mixWLds[i][j].append(bs)
562 | 
563 |     plt.clf()#clear
564 |     x = list(range(rnds))
565 |     max_y = 0
566 |     min_y = 0
567 |     for i in range(len(mixWLds)):
568 |         if i == 0:
569 |             lc = 'blue'
570 |         if i == 1:
571 |             lc = 'green'
572 |         if i == 2:
573 |             lc = 'red'
574 |         for j in range(len(mixWLds[i])):
575 |             if j == 0:
576 |                 ls = 's-'
577 |             if j == 1:
578 |                 ls = 'o-'
579 |             if j == 2:
580 |                 ls = '^-'
581 |             min_y,max_y = getMinMax(mixWLds[i][j], min_y, max_y)
582 |             plt.plot(x,mixWLds[i][j],ls,color=lc,
583 |                   label=str(wlds[i])+'/bs=' + bsLabels[j])
584 |     x = list(range(0,rnds + 1,16))
585 |     plt.xticks(x)
586 |     plt.suptitle("IOPS Measurement Plot",fontweight='bold')
587 |     plt.xlabel("Area of Device (in rounds)")
588 |     plt.ylabel("IOPS")
589 |     #plt.yscale('log')
590 |     plt.ylim((min_y*0.75,max_y*1.25))
591 |     plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.07),
592 |                ncol=3, fancybox=True, shadow=True,prop={'size':12})
593 |     plt.savefig(toPlot.getTestname()+'-IOPSPlt.png',dpi=300)
594 |     toPlot.addFigure(toPlot.getTestname()+'-IOPSPlt.png')
595 |     
596 | def TPBoxPlot(toPlot):
597 |     '''
598 |     Generate a R/W throughput box plot for the hdd round results.
599 |     The plot consists of:
600 |     -Measured bandwidths per round, per block size for read and write
601 |     -x axes is the number of carried out rounds
602 |     -y axes is the bandwidth of the corresponding round
603 |     The figure is saved as TPTest.Testname-TP-RW-Plt.png.
604 |     @param toPlot A hdd TPTest object.
605 |     '''
606 |     #As the values are converted to KB, copy the matrices
607 |     matrices = deepcopy(toPlot.getRndMatrices())
608 |     bsLabels = toPlot.getBsLabels()
609 |     
610 |     plt.clf()#clear
611 |     boxes = []
612 |     min_y = 0
613 |     max_y = 0
614 |     for bsRows in matrices:
615 |         #For each BS we have read and write, both rows have equal length
616 |         for v in range(len(bsRows[0])):
617 |             bsRows[0][v] = (bsRows[0][v]) / 1024
618 |             bsRows[1][v] = (bsRows[1][v]) / 1024
619 |         boxes.append(bsRows[0])
620 |         min_y,max_y = getMinMax(bsRows[0], min_y, max_y)
621 |         boxes.append(bsRows[1])
622 |         min_y,max_y = getMinMax(bsRows[1], min_y, max_y)
623 |     #Length of BS per R/W
624 |     pos = list(range(len(bsLabels) * 2))
625 |     plt.boxplot(boxes,positions=pos)
626 |     labels = []
627 |     for l in bsLabels:
628 |         labels.append(l + ' (R)')
629 |         labels.append(l + ' (W)')
630 |     plt.xticks(pos,labels)
631 |     plt.xlabel('BS (Mode)')
632 |     plt.suptitle("TP Boxplot",fontweight='bold')    
633 |     plt.ylabel("Bandwidth (MB/s)")
634 |     #scale axis to min and max +- 15%
635 |     plt.ylim((min_y*0.7,max_y*1.10))
636 |     #Draw some fake data for legend
637 |     hB, = plt.plot([1,1],'b-')
638 |     hR, = plt.plot([1,1],'r-')
639 |     plt.legend((hB, hR),('Quartiles', 'Median'),loc='upper center', bbox_to_anchor=(0.5, 1.07),
640 |                ncol=2, fancybox=True, shadow=True,prop={'size':12})
641 |     hB.set_visible(False)
642 |     hR.set_visible(False)
643 |     plt.savefig(toPlot.getTestname()+'-TP-Boxplt.png',dpi=300)
644 |     toPlot.addFigure(toPlot.getTestname()+'-TP-Boxplt.png')
645 | 
646 | ######### HELPER FUNCTIONS TO GENERATE PLOTS #########
647 | def calcMsmtTable(toPlot,mode):
648 |     '''
649 |     Generate the measurement overview table for IOPS and Latency. The table is
650 |     an overview over the average values in the measurement window. For latency
651 |     the values are converted from us to ms also.
652 |     @param toPlot A SsdTest object.
653 |     @param mode A string representing the test mode (IOPS|max-LAT|avg-LAT)
654 |     '''
655 |     mixWLds = []
656 |     mesWin = toPlot.getStdyState().getStdyRnds() #get measurement window, only include these values
657 |     if mode == "IOPS":
658 |         wlds = dt.SsdIopsTest.mixWlds
659 |         bsLabels = toPlot.getBsLabels()
660 |     if mode == "avg-LAT" or mode == "max-LAT":
661 |         wlds = dt.SsdLatencyTest.mixWlds
662 |         bsLabels = toPlot.getBsLabels()
663 | 
664 |     #each row will be a workload percentage
665 |     for i in range(len(wlds)):
666 |         mixWLds.append([])
667 |         #in each row will be the different block sizes
668 |         for bs in range(len(bsLabels)):
669 |             mixWLds[i].append(0)
670 |     matrices = toPlot.getRndMatrices()
671 | 
672 |     #as j does not necessarily start from 0, we need k
673 |     #to calculate the average iteratively
674 |     k = 0
675 |     #limit the matrices to the measurement window
676 |     for j in mesWin:
677 |         rndMat = matrices[j]
678 |         #each row is a percentage of a workload
679 |         for i,row in enumerate(rndMat):
680 |             #in each row are the different block sizes
681 |             for bs in range(len(row)):
682 |                 #calculate average iteratively
683 |                 if mixWLds[i][bs] != 0:
684 |                     #calculate max latency or continue with average
685 |                     if mode == "max-LAT":
686 |                         if row[bs][1] > mixWLds[i][bs]:
687 |                             mixWLds[i][bs] = row[bs][1]#max latency
688 |                     else:
689 |                         mixWLds[i][bs] *= k
690 |                         if mode == "IOPS":
691 |                             mixWLds[i][bs] += row[bs]#IOPS
692 |                         if mode == "avg-LAT":
693 |                             mixWLds[i][bs] += row[bs][2]#mean latency
694 |                         mixWLds[i][bs] = (mixWLds[i][bs]) / (k+1)
695 |                 else:
696 |                     if mode == "IOPS":
697 |                         mixWLds[i][bs] = row[bs]#IOPS
698 |                     if mode == "max-LAT":
699 |                         mixWLds[i][bs] = row[bs][1]#max latency
700 |                     if mode == "avg-LAT":
701 |                         mixWLds[i][bs] = row[bs][2]#mean latency
702 |         k += 1
703 |     #for latency convert to ms
704 |     for i in range(len(mixWLds)):
705 |         if mode == "avg-LAT" or mode == "max-LAT":
706 |             for v in range(len(mixWLds[i])):
707 |                 mixWLds[i][v] = (mixWLds[i][v]) / 1000
708 |     toPlot.addTable(mixWLds)
709 | 
710 | def calcMsmtTPTable(toPlot):
711 |     '''
712 |     Generate the measurement overview table for Throughput. The table is
713 |     an overview over the average values in the measurement window. 
714 |     @param toPlot A SsdTest object.
715 |     '''
716 |     wlds = [] #read and write workload mode
717 |     #one row for read, one for write
718 |     for i in range(2):
719 |         wlds.append([])
720 |         #in each row will be the different block sizes
721 |         for bs in range(len(toPlot.getBsLabels())):
722 |             wlds[i].append(0)
723 |     matrices = deepcopy(toPlot.getRndMatrices())
724 |     #each row of the matrix is a block size
725 |     for j,bs in enumerate(matrices):
726 |         #each block size has read and write
727 |         for i,row in enumerate(bs):
728 |             #as rnd does not need to start at 0
729 |             #we need k to calculate average
730 |             k = 0
731 |             #read and write have their round values
732 |             for rnd in toPlot.getStdyState().getStdyRnds():
733 |                 #calculate average iteratively
734 |                 if wlds[i][j] != 0:
735 |                     wlds[i][j] *= k
736 |                     wlds[i][j] += row[rnd]
737 |                     wlds[i][j] = (wlds[i][j]) / (k+1)
738 |                 else:
739 |                     wlds[i][j] = row[rnd]
740 |                 k += 1
741 |     #Convert to MB/s
742 |     for i in range(len(wlds)):
743 |         for v in range(len(wlds[i])):
744 |             wlds[i][v] = (wlds[i][v]) / 1024
745 |     toPlot.addTable(wlds)
746 | 
747 | def getBS(bsLabels):
748 |     '''
749 |     Convert a list of string block size labels to a list of integers.
750 |     This can be handy if the block sizes are needed to be plotted at
751 |     the x axis.
752 |     @return A list of integer block sizes.
753 |     '''
754 |     bs = []
755 |     for b in bsLabels:
756 |         if b == "512":
757 |             bs.append(0.5)
758 |             continue
759 |         s = b[0:-1]
760 |         bs.append(int(s))
761 |     return bs
762 | 
763 | def getMinMax(values, currMin, currMax):
764 |     '''
765 |     Returns the minimum and maximum of a sequence of values.
766 |     @param values Squence to calculate min and max for
767 |     @param currMin Current minimum to compare to.
768 |     @param currMax Current maximum to compare to.
769 |     @return [newMin,newMax] if newMin is smaller than currMin,
770 |     newMax if it is greater than currMax.
771 |     '''
772 |     #TODO Testing for 0 can be a problem if minimum is really 0
773 |     #This should not happen for performance tests under normal circumstances
774 |     newMin = 0
775 |     newMax = 0
776 |     curr = max(values)
777 |     if curr > currMax:
778 |         newMax = curr
779 |     else:
780 |         newMax = currMax
781 |     curr = min(values)
782 |     if currMin == 0:
783 |         newMin = curr
784 |     else:
785 |         if curr < currMin:
786 |             newMin = curr
787 |         else:
788 |             newMin = currMin
789 |     return [newMin,newMax]
790 | 


--------------------------------------------------------------------------------
/src/reports/RstReport.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on 9 Aug 2012
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | from io import StringIO
  7 | from copy import deepcopy
  8 | import os
  9 | import inspect
 10 | import subprocess
 11 | import logging
 12 | 
 13 | import perfTest.DeviceTests as dt
 14 | from perfTest.StdyState import StdyState
 15 | 
 16 | class RstReport(object):
 17 |     '''
 18 |     A report as restructured text.
 19 |     '''
 20 | 
 21 |     def __init__(self,testname):
 22 |         '''
 23 |         @param testname Name of the test, also the filename. 
 24 |         '''
 25 |         self.__testname = testname
 26 |         self.__rst = StringIO()
 27 |     
 28 |     def getRst(self):
 29 |         return self.__rst    
 30 |     
 31 |     def addTitle(self):
 32 |         print("====================", file=self.__rst)
 33 |         print("TKperf Test Report", file=self.__rst)
 34 |         print("====================\n", file=self.__rst)
 35 |         print(".. contents::", file=self.__rst)
 36 |         print(".. sectnum::", file=self.__rst)
 37 |         print(".. include:: <isonum.txt>", file=self.__rst)
 38 |         print(".. raw:: pdf\n", file=self.__rst)
 39 |         print("\tPageBreak\n", file=self.__rst)
 40 | 
 41 |     def addFooter(self):
 42 |         print(".. |logo| image:: " + os.path.dirname(inspect.getfile(RstReport)) + "/pics/TKperf_logo.png", file=self.__rst)
 43 |         print("\t:height: 90px", file=self.__rst)
 44 |         print(".. footer::", file=self.__rst)
 45 |         print("\t |logo| A `Thomas-Krenn <http://www.thomas-krenn.com/>`_ project, Page ###Page### of ###Total###\n", file=self.__rst)
 46 | 
 47 |     def addChapter(self,chap):
 48 |         print(chap, file=self.__rst)
 49 |         line = "="
 50 |         for i in chap:
 51 |             line += "="
 52 |         print(line+'\n', file=self.__rst)
 53 | 
 54 |     def addSection(self,sec):
 55 |         print(sec, file=self.__rst)
 56 |         line = "-"
 57 |         for i in sec:
 58 |             line += "-"
 59 |         print(line+'\n', file=self.__rst)
 60 |         
 61 |     def addString(self,str):
 62 |         if str[-1] != '\n':
 63 |             str += '\n'
 64 |         print(str, file=self.__rst)
 65 |     
 66 |     def addFigure(self,filename,testtype,perftype,index):
 67 |         '''
 68 |         Adds a figure to the restructured text.
 69 |         @param filename The filename of the figure.
 70 |         @param testtype The type of the performance test (ssd,hdd)
 71 |         @param type The type of the test (iops,tp etc)
 72 |         @param index The index of the caption to insert after the figure.
 73 |         '''
 74 |         print(".. figure:: "+filename, file=self.__rst) 
 75 |         print("\t:scale: 65%", file=self.__rst)
 76 |         print("\t:figwidth: 85%\n", file=self.__rst)
 77 |         caption = ''
 78 |         if testtype == 'ssd':
 79 |             if perftype == 'iops':
 80 |                 if index == 0:
 81 |                     caption= "\tThe Steady State Convergence Plot shows the reached IOPS for "
 82 |                     caption += "all block sizes of random writes over all rounds."
 83 |                 if index == 1:
 84 |                     caption= "\tThe Steady State Verification Plot shows the measured IOPS of 4k "
 85 |                     caption += "random writes, the 20% average window and the slope of the linear best fit line "
 86 |                     caption += "in the measurement window."
 87 |                 if index == 2:
 88 |                     caption= "\tThe Measurement Plot shows the average of IOPS in the measurement window. For every "
 89 |                     caption += "workload the IOPS of all block sizes are plotted."
 90 |                 if index == 3:
 91 |                     caption= "\tThe Measurement 3D Plot shows the average of IOPS in the measurement window. For every "
 92 |                     caption += "workload the IOPS of all block sizes are plotted."
 93 |             if perftype == 'tp':
 94 |                 if index == 0:
 95 |                     caption= "\tThe Read/Write Steady State Convergence Plot shows the bandwidth for "
 96 |                     caption += "all block sizes of seq. reads over all rounds. On the top the write throughput is plotted, below "
 97 |                     caption += "the throughput for read."
 98 |                 if index == 1:
 99 |                     caption= "\tThe Steady State Verification Plot shows the bandwidth of 1024k "
100 |                     caption += "seq. writes, the 20% average window and the slope of the linear best fit line "
101 |                     caption += "in the measurement window."
102 |                 if index == 2:
103 |                     caption= "\tThe Measurement Plot shows the average bandwidth of reads and writes in the measurement window. "
104 |                     caption += "For all block sizes the seq. read and write bandwidth is plotted."
105 |             if perftype == 'lat':
106 |                 if index == 0:
107 |                     caption= "\tThe Steady State Convergence Plot shows the mean latency for "
108 |                     caption += "all block sizes of random read, mixed workload and write."
109 |                 if index == 1:
110 |                     caption= "\tThe Steady State Verification Plot shows the mean latency of 4k "
111 |                     caption += "random writes, the 20% average window and the slope of the linear best fit line "
112 |                     caption += "in the measurement window."
113 |                 if index == 4:
114 |                     caption = "\tThe Latency Measurement 3D Plot shows the average latency on top and the max latency below it. "
115 |                     caption += "For the measurement window every workload including all block sizes is plotted."
116 |             if perftype == 'writesat':
117 |                 if index == 0:
118 |                     caption= "\tThe Write Saturation IOPS Plot shows the average IOPS of 4k random "
119 |                     caption += "writes over all rounds."
120 |                 if index == 1:
121 |                     caption= "\tThe Write Saturation Latency Plot shows the mean latency of 4k random "
122 |                     caption += "writes over all rounds."
123 |         if testtype == 'hdd':
124 |             if perftype == 'iops':
125 |                 if index == 0:
126 |                     caption= "\tThe Measurement Plot shows the IOPS of each one-128th part of the disk. For every "
127 |                     caption += "workload the IOPS of all block sizes are plotted."
128 |             if perftype == 'tp':
129 |                 if index == 0:
130 |                     caption= "\tThe Measurement Plot shows the bandwidth of reads and writes in each one-128th part "
131 |                     caption += "of the disk. For all block sizes the seq. read and write bandwidth is plotted."
132 |                 if index == 1:
133 |                     caption= "\tThe Boxplot shows minimum, lower quartile, median, upper quartile and maximum. "
134 |                     caption += "For all block sizes the seq. read and write data is plotted."
135 |                     
136 |         self.addString(caption)
137 |         
138 |     def addTable(self,table,labels,perftype):
139 |         '''
140 |         Adds a table to the restructured text.
141 |         @param table The table to insert into the report.
142 |         @param type The type of performance test.
143 |         '''
144 |         #copy labels and values, don't want to change them
145 |         l = list(labels)
146 |         t = deepcopy(table)
147 |         
148 |         if perftype == 'iops':
149 |             val = StringIO()
150 |             print(".. csv-table:: Average IOPS vs. Block Size and R/W Mix %", file=self.__rst)
151 |             print("\t:header: \"Block Size\ |darr|\", \"Wld. |rarr| \" 100/0, 95/5, 65/35, 50/50, 35/65, 5/95, 0/100\n", file=self.__rst)
152 |             #reverse the block size in each table row, to start with 512B
153 |             for row in t:
154 |                 row.reverse()
155 |             #also reverse labels
156 |             l.reverse()
157 |         if perftype == 'tp':
158 |             val = StringIO()
159 |             print(".. csv-table:: Average MB/s vs. Block Size and R/W", file=self.__rst)
160 |             print("\t:header: \"Block Size\ |darr|\", \"Read\", \"Write\"\n", file=self.__rst)
161 |             
162 |         if perftype == 'avg-lat':
163 |             val = StringIO()
164 |             print(".. csv-table:: Average Latency (ms) vs. Block Size and R/W Mix %", file=self.__rst)
165 |             print("\t:header: \"Block Size\ |darr|\", \"Wld. |rarr| \" 0/100, 65/35, 100/0\n", file=self.__rst)
166 |             #reverse to start with 0/100
167 |             t.reverse()
168 |         
169 |         if perftype == 'max-lat':
170 |             val = StringIO()
171 |             print(".. csv-table:: Max Latency (ms) vs. Block Size and R/W Mix %", file=self.__rst)
172 |             print("\t:header: \"Block Size\ |darr|\", \"Wld. |rarr| \" 0/100, 65/35, 100/0\n", file=self.__rst)
173 |             #reverse to start with 0/100
174 |             t.reverse()
175 |             
176 |         for i in range(len(l)):
177 |             val.write("\t")
178 |             val.write(l[i] + ', ')
179 |             #access the matrix column wise, round the numbers to 3 after
180 |             for j,elem in enumerate(row[i] for row in t):
181 |                 if j != 0:
182 |                     val.write(", ")
183 |                 val.write(str(round(elem,3)))
184 |             val.write("\n")
185 |         self.addString(val.getvalue())
186 |         val.close()
187 |                 
188 |     
189 |     def toRstFile(self):
190 |         f = open(self.__testname+'.rst','w')
191 |         f.write(self.__rst.getvalue())
192 |         self.__rst.close()
193 |         f.close()
194 |     
195 |     def toPDF(self,pdfgen):
196 |         pdf = subprocess.Popen([pdfgen, self.__testname+'.rst'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
197 |         stderr = pdf.communicate()[1]
198 |         if pdf.returncode != 0:
199 |             logging.error("generating the PDF encountered an error: " + stderr)
200 |             raise RuntimeError("PDF gen command error")
201 |     
202 |     def addDevInfo(self,devStr,featMat):
203 |         '''
204 |         Add info about the tested device to the report.
205 |         @param devStr The device information from hdparm or the dsc file.
206 |         @param featMat The extra feature matrix given via a csv table. 
207 |         ''' 
208 |         self.addChapter("Setup Information")
209 |         print("Tested Device:", file=self.__rst)
210 |         if devStr[-1] == '\n':
211 |             devStr = devStr[:-1]
212 |         for line in devStr.split('\n'):
213 |             print(" - " + line, file=self.__rst)
214 |         print('\n', file=self.__rst)
215 |         
216 |         if featMat != None:
217 |             print("Feature Matrix:", file=self.__rst)
218 |             print(featMat + "\n", file=self.__rst)
219 |             
220 |     def addCmdLine(self,cmdLineStr):
221 |         print("Used command line:", file=self.__rst)
222 |         print(" - " + cmdLineStr, file=self.__rst)
223 |         
224 |     def addSetupInfo(self,ioVer,fioVer,dateStr):
225 |         '''
226 |         Add info about the version of Fio to the report.
227 |         @param setupStr The Fio version string, fetched via str-method of a FioJob.
228 |         @param dateStr The date string the test was carried out.
229 |         ''' 
230 |         print("Performance System:", file=self.__rst)
231 |         print(" - TKperf Version: " + ioVer, file=self.__rst)
232 |         print(" - Fio Version: " + fioVer, file=self.__rst)
233 |         print(" - Date of test run: " + dateStr, file=self.__rst)
234 |         
235 |     def addFioJobInfo(self,nj,iod):
236 |         '''
237 |         Write information about Fio number of jobs and iodepth to report.
238 |         @param nj The number of jobs.
239 |         @param iod The number of outstanding ios (iodepth).
240 |         ''' 
241 |         info = StringIO()
242 |         info.write(" - Number of jobs: " + str(nj) + "\n")
243 |         info.write(" - Number of outstanding IOs (iodepth): " + str(iod))
244 |         self.addString(info.getvalue())
245 |         info.close()
246 |         
247 |     def addOSInfo(self,OSDict):
248 |         if OSDict != None:
249 |             print("Operating System:", file=self.__rst)
250 |             if 'kernel' in OSDict:
251 |                 print(" - Kernel Version: " + OSDict['kernel'], file=self.__rst)
252 |             if 'lsb' in OSDict:
253 |                 print(" - " + OSDict['lsb'], file=self.__rst)
254 |         
255 |     def addGeneralInfo(self,testtype):
256 |         '''
257 |         Defines some general used words.
258 |         @param testtype The type of the performance test (ssd,hdd)
259 |         ''' 
260 |         info = StringIO()
261 |         self.addChapter("General Information")
262 |         info.write("- *workloads*: The percentage of read operations in the random mixed workload. In the plots the ")
263 |         info.write("term \"100/00\" means 100% read and 0% write, \"95/5\" 95% read and 5% write, and so on.\n")
264 |         info.write("- *block sizes*: The block size of Fio to be used for IO operations.\n")
265 |         if testtype == 'ssd':
266 |             info.write("- *measurement window*: Those rounds, where the dependence variable became stable.\n")
267 |             info.write("- *dependence variable*: A specific type of test variable to determine the steady state.\n")
268 |         if testtype == 'hdd':
269 |             info.write("- *round*: As an hdd's performance is different at outer and inner side, the ")
270 |             info.write("device is divided into equal size parts. In every round one part of the device is tested")
271 |         self.addString(info.getvalue())
272 |         info.close()
273 |         if testtype == 'ssd':
274 |             info = StringIO()
275 |             self.addSection("Steady State")
276 |             info.write("The Steady State is to determine if a test has reached a steady performance level. ")
277 |             info.write("Each test has a different dependence variable to check if the state has already been reached. ")
278 |             info.write("To check for the steady state the performance values of a test measurement window are taken (the last 5 rounds).\n")
279 |             info.write("The steady state is reached if:\n\n")
280 |             info.write("- The maximum data excursion is less than 20% of the average in the measurement window.\n")
281 |             info.write("- The slope of the linear best fit line is less than 10% of the average in the measurement window\n\n")
282 |             
283 |             info.write("If these two conditions are met the steady state has been reach for the specific dependence variable. ")
284 |             info.write("Therefore the test can be stopped and the performance values of the measurement window can be taken ")
285 |             info.write("for the measurement plots. If the steady state has not been reached after a maximum number of rounds the test ")
286 |             info.write("can be stopped as well. The numbers for these two variables are:\n\n")
287 |             print("- Measurement Window: " + str(StdyState.testMesWindow), file=info)
288 |             print("- Max. number of rounds: " + str(StdyState.testRnds) + '\n', file=info)
289 |             self.addString(info.getvalue())
290 |             info.close()
291 |     
292 |     def addSteadyInfo(self,test):
293 |         ''' 
294 |         Adds information about the steady state to the rst report.
295 |         @param test The corresponding test object.
296 |         '''
297 |         self.addSection("Steady State Information")
298 | 
299 |         stdyStr = StringIO()
300 |         stdyStr.write("Steady State has been reached:\n")
301 |         stdyStr.write(" - ")
302 |         print(test.getStdyState().isSteady(), file=stdyStr)
303 |         
304 |         stdyStr.write("Steady State has been reached in rounds    :\n")
305 |         stdyStr.write(" - ")
306 |         print(test.getStdyState().getStdyRnds(), file=stdyStr)
307 |         
308 |         stdyStr.write("Values in stdy measurement window:\n")
309 |         stdyStr.write(" - ")
310 |         print(test.getStdyState().getStdyValues(), file=stdyStr)
311 |         
312 |         stdyStr.write("Average in stdy measurement window:\n")
313 |         stdyStr.write(" - ")
314 |         print(test.getStdyState().getStdyAvg(), file=stdyStr)  
315 |         
316 |         self.addString(stdyStr.getvalue())
317 |         stdyStr.close()
318 |     
319 |     def addTestInfo(self,testtype,testname,test):
320 |         '''
321 |         Add information about a test to the rst report.
322 |         This part is the main information about a test, it describes how 
323 |         a test has been carried out.
324 |         @param testtype Type of performance test (hdd,ssd)
325 |         @param testname Type name of a test.
326 |         @param test The specific test object 
327 |         '''
328 |         if testtype == 'ssd':
329 |             if testname == 'iops':
330 |                 desc = StringIO()
331 |                 desc.write("The IOPS test consists of looping over the following parameters:\n")
332 |                 desc.write('\n::\n\n\t')
333 |                 print("Make Secure Erase", file=desc)
334 |                 print("\tWorkload Ind. Preconditioning", file=desc)
335 |                 print("\tWhile not Steady State", file=desc)
336 |                 print("\t\tFor workloads ", end=' ', file=desc)
337 |                 print(dt.SsdIopsTest.mixWlds, file=desc)
338 |                 desc.write('\t\t\t')
339 |                 print("For block sizes", end=' ', file=desc)
340 |                 print(test.getBsLabels(), file=desc)
341 |                 desc.write("\nEach combination of workload and block size is carried out for 60 seconds using direct IO. ")
342 |                 desc.write("The average number of read and write IOPS is measured and summed up, therefore 56 values are ")
343 |                 desc.write("the result of the two loops.\n")
344 |                 desc.write("After these loops are finished one test round has been carried out. To detect the steady state ")
345 |                 desc.write("the IOPS of 4k random write are taken.\n\n")
346 |                 print("- Dependent Variable: 4k block size, random write", file=desc)
347 |                 self.addString(desc.getvalue())
348 |                 desc.close()
349 |                 self.addSteadyInfo(test)
350 |             if testname == 'tp':  
351 |                 desc = StringIO()
352 |                 desc.write("The throughput test consists of looping over the following parameters:\n")
353 |                 desc.write('\n::\n\n\t')
354 |                 print("For block sizes ", end=' ', file=desc)
355 |                 print(test.getBsLabels(), file=desc)
356 |                 desc.write('\t\t')
357 |                 print("Make Secure Erase", file=desc)
358 |                 desc.write('\t\t')
359 |                 print("While not Steady State", file=desc)
360 |                 desc.write('\t\t\t')
361 |                 print("Sequential read", file=desc)
362 |                 desc.write('\t\t\t')
363 |                 print("Sequential write", file=desc)
364 |                 desc.write("\nFor each block size sequential read and write is carried out for 60 seconds using direct IO. ")
365 |                 desc.write("The number of kilobytes for read and write is measured, therefore 2 values are ")
366 |                 desc.write("the result of one round.\n")
367 |                 desc.write("To detect the steady state the throughput of 1024k sequential write is taken.\n\n")
368 |                 print("- Dependent Variable: 1024k block size, sequential write", file=desc)
369 |                 self.addString(desc.getvalue())
370 |                 desc.close()
371 |                 self.addSteadyInfo(test)
372 |             if testname == 'lat':  
373 |                 desc = StringIO()
374 |                 desc.write("The latency test consists of looping over the following parameters:\n")
375 |                 desc.write('\n::\n\n\t')
376 |                 print("Make Secure Erase", file=desc)
377 |                 print("\tWorkload Ind. Preconditioning", file=desc)
378 |                 print("\tWhile not Steady State", file=desc)
379 |                 print("\t\tFor workloads ", end=' ', file=desc)
380 |                 print(dt.SsdLatencyTest.mixWlds, file=desc)
381 |                 desc.write('\t\t\t')
382 |                 print("For block sizes", end=' ', file=desc)
383 |                 print(test.getBsLabels(), file=desc)
384 |                 desc.write("\nFor all block sizes random read, a 65/35 read/write mixed workload and random write is carried out for 60 ") 
385 |                 desc.write("seconds using direct IO. ")
386 |                 desc.write("For every combination the Min, Max and Mean Latency is measured. ")
387 |                 desc.write("After these loops are finished one test round has been carried out. To detect the steady state ")
388 |                 desc.write("the mean latency of 4k random write is taken.\n\n")
389 |                 print("- Dependent Variable: 4k block size, random write mean latency", file=desc)
390 |                 self.addString(desc.getvalue())
391 |                 desc.close()
392 |                 self.addSteadyInfo(test)
393 |             if testname == 'writesat':  
394 |                 desc = StringIO()
395 |                 desc.write("The write saturation test consists of looping over the following parameters:\n")
396 |                 desc.write('\n::\n\n\t')
397 |                 print("Make Secure Erase", file=desc)
398 |                 print("\tWhile not written 4x User Capacity or 24h", file=desc)
399 |                 print("\t\tCarry out random write, 4k block size for 1 minute.", file=desc)
400 |                 desc.write("\nFor 4k block size random write is carried out for 60 ") 
401 |                 desc.write("seconds using direct IO. ")
402 |                 desc.write("For each round (60 second window) the write IOPS and latencies are measured. Also the total written ")
403 |                 desc.write("IO is measured to check if 4x capacity has been written.\n\n")
404 |                 desc.write("As no steady state detection is necessary there is no dependence variable.\n\n")
405 |                 self.addString(desc.getvalue())
406 |                 desc.close()
407 |         
408 |         if testtype == 'hdd':
409 |             if testname == 'iops':
410 |                 desc = StringIO()
411 |                 desc.write("The IOPS test consists of looping over the following parameters:\n")
412 |                 desc.write('\n::\n\n\t')
413 |                 print("Divide device in " + str(dt.HddIopsTest.maxRnds) + " parts", file=desc)
414 |                 print("\tFor range(" + str(dt.HddIopsTest.maxRnds) + ")", file=desc)
415 |                 print("\t\tFor workloads ", end=' ', file=desc)
416 |                 print(dt.HddIopsTest.mixWlds, file=desc)
417 |                 desc.write('\t\t\t')
418 |                 print("For block sizes", end=' ', file=desc)
419 |                 print(test.getBsLabels(), file=desc)
420 |                 desc.write("\nEach combination of workload and block size is carried out for 60 seconds using direct IO. ")
421 |                 desc.write("The IOPS of one round are an indicator for the random performance of the corresponding area.")
422 |                 self.addString(desc.getvalue())
423 |                 desc.close()
424 |             if testname == 'tp':  
425 |                 desc = StringIO()
426 |                 desc.write("The throughput test consists of looping over the following parameters:\n")
427 |                 desc.write('\n::\n\n\t')
428 |                 print("For block sizes ", end=' ', file=desc)
429 |                 print(test.getBsLabels(), file=desc)
430 |                 desc.write('\t\t')
431 |                 print("For range(" + str(dt.HddTPTest.maxRnds) + ")", file=desc)
432 |                 desc.write('\t\t\t')
433 |                 print("Sequential read", file=desc)
434 |                 desc.write('\t\t\t')
435 |                 print("Sequential write", file=desc)
436 |                 desc.write("\nFor each block size, every area of the device (this are the rounds) is tested ")
437 |                 desc.write("with sequential read and write using direct IO. ")
438 |                 self.addString(desc.getvalue())
439 |                 desc.close()
440 | 


--------------------------------------------------------------------------------
/src/reports/XmlReport.py:
--------------------------------------------------------------------------------
 1 | '''
 2 | Created on 03.08.2012
 3 | 
 4 | @author: gschoenb
 5 | '''
 6 | from lxml import etree
 7 | 
 8 | class XmlReport(object):
 9 |     '''
10 |     Creates an xml file with the test results of a device test.
11 |     '''
12 | 
13 | 
14 |     def __init__(self,testname):
15 |         '''
16 |         Constructor
17 |         @param testname The name of the root tag, equals the name
18 |         of the performance test. 
19 |         '''
20 |         
21 |         ## The root tag of the xml file.
22 |         self.__xml = etree.Element(testname)
23 |         
24 |     def getXml(self):
25 |         return self.__xml
26 |     
27 |     def printXml(self):
28 |         print((etree.tostring(self.__xml, xml_declaration=True,pretty_print=True)))
29 |     
30 |     def xmlToFile(self,testname):
31 |         et = etree.ElementTree(self.__xml)
32 |         et.write(testname + '.xml')
33 |         
34 |     def fileToXml(self,testname):
35 |         et = etree.parse(testname + '.xml')
36 |         self.__xml = et.getroot()
37 |         


--------------------------------------------------------------------------------
/src/reports/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thomas-krenn/TKperf/638b0837b97b32337314993feb52b5111aae0979/src/reports/__init__.py


--------------------------------------------------------------------------------
/src/reports/pics/TKperf_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thomas-krenn/TKperf/638b0837b97b32337314993feb52b5111aae0979/src/reports/pics/TKperf_logo.png


--------------------------------------------------------------------------------
/src/system/Mail.py:
--------------------------------------------------------------------------------
 1 | '''
 2 | Created on Sep 26, 2014
 3 | 
 4 | @author: gschoenb
 5 | '''
 6 | 
 7 | import smtplib
 8 | from email.mime.multipart import MIMEMultipart
 9 | from email.mime.application import MIMEApplication
10 | from email.mime.text import MIMEText
11 | from os import path
12 | 
13 | class Mail(object):
14 |     '''
15 |     Sending emails via local smtp.
16 |     '''
17 | 
18 |     def __init__(self, subj, sender, rcpt, smtp):
19 |         '''
20 |         Constructor
21 |         '''
22 |         self.__sender = sender
23 |         self.__rcpt = rcpt
24 |         self.__msg = MIMEMultipart()
25 |         self.__msg['Subject'] = subj
26 |         self.__msg['From'] = sender
27 |         self.__msg['To'] = rcpt
28 |         self.__smtp = smtplib.SMTP(smtp)
29 | 
30 |     def addMsg(self, msg):
31 |         msgText = MIMEText(msg, 'plain')
32 |         self.__msg.attach(msgText)
33 | 
34 |     def addPDFAttachment(self,filename):
35 |         fp=open(filename,'rb')
36 |         att = MIMEApplication(fp.read(),_subtype="pdf")
37 |         fp.close()
38 |         name = path.basename(path.normpath(filename))
39 |         att.add_header('Content-Disposition','attachment',filename=name)
40 |         self.__msg.attach(att)
41 | 
42 |     def addXMLAttachment(self,filename):
43 |         fp=open(filename,'rb')
44 |         att = MIMEApplication(fp.read(),_subtype="xml")
45 |         fp.close()
46 |         name = path.basename(path.normpath(filename))
47 |         att.add_header('Content-Disposition','attachment',filename=name)
48 |         self.__msg.attach(att)
49 |         
50 |     def addTextAttachment(self,filename):
51 |         fp=open(filename,'rb')
52 |         att = MIMEApplication(fp.read(),_subtype="txt")
53 |         fp.close()
54 |         name = path.basename(path.normpath(filename))
55 |         att.add_header('Content-Disposition','attachment',filename=name)
56 |         self.__msg.attach(att)
57 | 
58 |     def send(self):
59 |         self.__smtp.sendmail(self.__sender, self.__rcpt, self.__msg.as_string())
60 |         self.__smtp.quit()


--------------------------------------------------------------------------------
/src/system/OS.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | Created on Sep 24, 2014
  3 | 
  4 | @author: gschoenb
  5 | '''
  6 | 
  7 | from abc import ABCMeta, abstractmethod
  8 | import subprocess
  9 | import logging
 10 | import re
 11 | from os import lstat
 12 | from stat import S_ISBLK
 13 | from time import sleep
 14 | 
 15 | class RAIDtec(object, metaclass=ABCMeta):
 16 |     '''
 17 |     Representing a RAID technology, used from the OS.
 18 |     '''
 19 | 
 20 |     def __init__(self, path, level, devices):
 21 |         ## Path of the RAID utils
 22 |         self.__util = None
 23 |         ## Path of the raid device
 24 |         self.__path = path
 25 |         ## RAID level
 26 |         self.__level = level
 27 |         ## List of devices
 28 |         self.__devices = devices
 29 |         ## List of block Devices in OS
 30 |         self.__blockdevs = None
 31 | 
 32 |     def getUtil(self): return self.__util
 33 |     def getDevPath(self): return self.__path
 34 |     def getLevel(self): return self.__level
 35 |     def getDevices(self): return self.__devices
 36 |     def getBlockDevs(self): return self.__blockdevs
 37 | 
 38 |     def setUtil(self, u):
 39 |         self.__util = u
 40 | 
 41 |     def checkBlockDevs(self):
 42 |         '''
 43 |         Checks the current available block devices.
 44 |         Sets blockdevs of OS.
 45 |         '''
 46 |         out = subprocess.Popen(['lsblk', '-l', '-n', '-e', '7', '-e', '1', '-o', 'NAME'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
 47 |         (stdout, stderr) = out.communicate()
 48 |         if stderr != '':
 49 |             logging.error("lsblk encountered an error: " + stderr)
 50 |             raise RuntimeError("lsblk command error")
 51 |         else:
 52 |             self.__blockdevs = stdout.splitlines()
 53 |             logging.info("# Got the following BDs: ")
 54 |             logging.info(self.getBlockDevs())
 55 | 
 56 |     @abstractmethod
 57 |     def initialize(self):
 58 |         ''' Initialize the specific RAID technology. '''
 59 |     @abstractmethod
 60 |     def checkRaidPath(self):
 61 |         ''' Checks if the virtual drive exists. '''
 62 |     @abstractmethod
 63 |     def checkVDs(self):
 64 |         ''' Check which virtual drives are configured. '''
 65 |     @abstractmethod
 66 |     def createVD(self):
 67 |         ''' Create a virtual drive. '''
 68 |     @abstractmethod
 69 |     def deleteVD(self):
 70 |         ''' Delete a virtual drive. '''
 71 |     @abstractmethod
 72 |     def isReady(self):
 73 |         ''' Check if a virtual drive is ready. '''
 74 | 
 75 | class Mdadm(RAIDtec):
 76 |     '''
 77 |     Represents a linux software RAID technology.
 78 |     '''
 79 | 
 80 |     def initialize(self):
 81 |         '''
 82 |         Checks for mdadm and sets the util path.
 83 |         '''
 84 |         mdadm = subprocess.Popen(['which', 'mdadm'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
 85 |         stdout = mdadm.communicate()[0]
 86 |         if mdadm.returncode != 0:
 87 |             logging.error("# Error: command 'which mdadm' returned an error code.")
 88 |             raise RuntimeError("which mdadm command error")
 89 |         else:
 90 |             self.setUtil(stdout.rstrip("\n"))
 91 | 
 92 |     def checkRaidPath(self):
 93 |         logging.info("# Checking for device "+self.getDevPath())
 94 |         try:
 95 |             mode = lstat(self.getDevPath()).st_mode
 96 |         except OSError:
 97 |             return False
 98 |         else:
 99 |             return S_ISBLK(mode)
100 | 
101 |     def checkVDs(self):
102 |         pass
103 | 
104 |     def createVD(self):
105 |         self.getDevPath()
106 |         args = [self.getUtil(), "--create", self.getDevPath(), "--quiet", "--metadata=default", str("--level=" + str(self.getLevel())), str("--raid-devices=" + str(len(self.getDevices())))]
107 |         for dev in self.getDevices():
108 |             args.append(dev)
109 |         logging.info("# Creating raid device "+self.getDevPath())
110 |         logging.info("# Command line: "+subprocess.list2cmdline(args))
111 |         mdadm = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
112 |         stderr = mdadm.communicate()[1]
113 |         if mdadm.returncode != 0:
114 |             logging.error("mdadm encountered an error: " + stderr)
115 |             raise RuntimeError("mdadm command error")
116 | 
117 |     def deleteVD(self):
118 |         logging.info("# Deleting raid device "+self.getDevPath())
119 |         mdadm = subprocess.Popen([self.getUtil(), "--stop", self.getDevPath()], stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
120 |         stderr = mdadm.communicate()[1]
121 |         if mdadm.returncode != 0:
122 |             logging.error("mdadm encountered an error: " + stderr)
123 |             raise RuntimeError("mdadm command error")
124 |         # Reset all devices in the Raid
125 |         # If the raid device was overwritten completely before (precondition), zero-superblock can fail
126 |         for dev in self.getDevices():
127 |             logging.info("# Deleting superblock for device "+dev)
128 |             mdadm = subprocess.Popen([self.getUtil(), "--zero-superblock", dev], stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
129 |             mdadm.communicate()
130 | 
131 |     def isReady(self):
132 |         logging.info("# Checking if raid device "+self.getDevPath()+" is ready...")
133 |         process = subprocess.Popen(["cat", "/proc/mdstat"], stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
134 |         (stdout, stderr) = process.communicate()
135 |         if stderr != '':
136 |             logging.error("cat mdstat encountered an error: " + stderr)
137 |             raise RuntimeError("cat mdstat command error")
138 |         else:
139 |             # Remove the Personalities line
140 |             stdout = stdout.partition("\n")[2]
141 |             # Split in single devices
142 |             mds = stdout.split("\n\n")
143 |             # Search devices for our device
144 |             match = re.search('^/dev/(.*)$', self.getDevPath())
145 |             mdName = match.group(1)
146 |             for md in mds:
147 |                 if md.startswith(mdName):
148 |                     # Check if a task is running)
149 |                     if md.find("finish") != -1:
150 |                         return False
151 |                     else:
152 |                         return True
153 | 
154 | class Storcli(RAIDtec):
155 |     '''
156 |     Represents a storcli based RAID technology.
157 |     '''
158 |     
159 |     def __init__(self, path, level, devices, readpolicy, writepolicy, stripsize):
160 |         '''
161 |         Constructor
162 |         '''
163 |         super(Storcli, self).__init__(path, level, devices)
164 |         ## The virtual drive of the raid controller
165 |         self.__vd = None
166 |         ## List of current RAID virtual drives
167 |         self.__vds = None
168 |         ## Read policy of the virtual drive
169 |         self.__readpolicy = readpolicy
170 |         ## Write policy of the virtual drive
171 |         self.__writepolicy = writepolicy
172 |         ## Strip size of the virtual drive
173 |         self.__stripsize = stripsize
174 | 
175 |     def getVD(self): return self.__vd
176 |     def getVDs(self): return self.__vds
177 |     def getREADPOLICY(self): return self.__readpolicy
178 |     def getWRITEPOLICY(self): return self.__writepolicy
179 |     def getSTRIPSIZE(self): return self.__stripsize
180 |     def setVD(self,v): self.__vd = v
181 |     def setVDs(self, v): self.__vds = v
182 | 
183 |     def initialize(self):
184 |         '''
185 |         Checks for the storcli executable and sets the path of storcli.
186 |         '''
187 |         storcli = subprocess.Popen(['which', 'storcli'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
188 |         stdout = storcli.communicate()[0]
189 |         if storcli.returncode != 0:
190 |             storcli = subprocess.Popen(['which', 'storcli64'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
191 |             stdout = storcli.communicate()[0]
192 |         if storcli.returncode != 0:
193 |             logging.error("# Error: command 'which storcli' returned an error code.")
194 |             raise RuntimeError("which storcli command error")
195 |         else:
196 |             self.setUtil(stdout.rstrip("\n"))
197 | 
198 |     def checkRaidPath(self):
199 |         '''
200 |         Checks if the virtual drive of the RAID controller is available.
201 |         @return True if yes, False if not
202 |         '''
203 |         if self.getVD() != None:
204 |             logging.info("# Checking for virtual drive "+self.getVD())
205 |             match = re.search('^[0-9]\/([0-9]+)',self.getVD())
206 |             vdNum = match.group(1)
207 |             storcli = subprocess.Popen([self.getUtil(),'/c0/v'+vdNum, 'show', 'all'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
208 |             (stdout,stderr) = storcli.communicate()
209 |             if storcli.returncode != 0:
210 |                 logging.error("storcli encountered an error: " + stderr)
211 |                 raise RuntimeError("storcli command error")
212 |             else:
213 |                 vdCheck = None
214 |                 for line in stdout.splitlines():
215 |                     match = re.search('^Description = (\w+)$',line)
216 |                     if match != None:
217 |                         if match.group(1) == 'No VDs have been configured':
218 |                             vdCheck = False
219 |                         else:
220 |                             vdCheck = True
221 |                     match = re.search('^Status = (\w+)$',line)
222 |                     if match != None:
223 |                         if match.group(1) == 'Failure':
224 |                             vdCheck = False
225 |                         else:
226 |                             vdCheck = True
227 |                 return vdCheck
228 |         else:
229 |             logging.info("# VD not set, checking for PDs: ")
230 |             logging.info(self.getDevices())
231 |             storcli = subprocess.Popen([self.getUtil(),'/c0/vall', 'show', 'all'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
232 |             (stdout,stderr) = storcli.communicate()
233 |             if storcli.returncode != 0:
234 |                 logging.error("storcli encountered an error: " + stderr)
235 |                 raise RuntimeError("storcli command error")
236 |             else:
237 |                 vdCheck = None
238 |                 for line in stdout.splitlines():
239 |                     match = re.search('^Description = (\w+)$',line)
240 |                     if match != None:
241 |                         if match.group(1) == 'No VDs have been configured':
242 |                             logging.info("# No VDs configured!")
243 |                             vdCheck = False
244 |                             break
245 |                     match = re.search('^PDs for VD ([0-9]+) \:$',line)
246 |                     if match != None:
247 |                         vdNum = match.group(1)
248 |                         PDs = self.getPDsFromVD(vdNum)
249 |                         if set(self.getDevices()) == set(PDs):
250 |                             self.setVD('0/'+vdNum)
251 |                             logging.info("# Set VD as "+self.getVD())
252 |                             vdCheck = True
253 |                             break
254 |                         else:
255 |                             vdCheck = False
256 |                 return vdCheck
257 | 
258 |     def getPDsFromVD(self,vdNum):
259 |         '''
260 |         Returns a list of PDs for a given VD.
261 |         @param vdNum Number of VD to check for
262 |         @return A list of enclosure:device IDs
263 |         '''
264 |         storcli = subprocess.Popen([self.getUtil(),'/c0/v'+vdNum, 'show', 'all'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
265 |         (stdout,stderr) = storcli.communicate()
266 |         if storcli.returncode != 0:
267 |             logging.error("storcli encountered an error: " + stderr)
268 |             raise RuntimeError("storcli command error")
269 |         else:
270 |             PDs = []
271 |             for line in stdout.splitlines():
272 |                 match = re.search('^([0-9]+\:[0-9]+).*$',line)
273 |                 if match != None:
274 |                     PDs.append(match.group(1))
275 |             logging.info("# Found PDs for VD "+ vdNum +":")
276 |             logging.info(PDs)
277 |             return PDs
278 | 
279 |     def checkVDs(self):
280 |         '''
281 |         Checks which virtual drives are configured.
282 |         Sets VDs as a list of virtual drives.
283 |         '''
284 |         process1 = subprocess.Popen([self.getUtil(), '/call', '/vall', 'show'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
285 |         process2 = subprocess.Popen(['awk', 'BEGIN{RS=ORS=\"\\n\\n\";FS=OFS=\"\\n\\n\"}/TYPE /'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=process1.stdout,universal_newlines=True)
286 |         process3 = subprocess.Popen(['awk', '/^[0-9]/{print $1}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=process2.stdout,universal_newlines=True)
287 |         process1.stdout.close()
288 |         process2.stdout.close()
289 |         (stdout, stderr) = process3.communicate()
290 |         if process3.returncode != 0:
291 |             logging.error("storcli encountered an error: " + stderr)
292 |             raise RuntimeError("storcli command error")
293 |         else:
294 |             self.setVDs(stdout.splitlines())
295 |             logging.info("# Got the following VDs: ")
296 |             logging.info(self.getVDs())
297 | 
298 |     def createVD(self):
299 |         '''
300 |         Creates a virtual drive from a given raid level and a list of
301 |         enclosure:drive IDs. self.__devices must be a list of raid devices as
302 |         strings, e.g. ['e252:1','e252:2'].
303 |         ''' 
304 |         encid = self.getDevices()[0].split(":")[0]
305 |         args = [self.getUtil(), '/c0', 'add', 'vd', str('type=r' + str(self.getLevel()))]
306 |         devicearg = "drives=" + encid + ":"
307 |         for dev in self.getDevices():
308 |             devicearg += dev.split(":")[1] + ","
309 |         args.append(devicearg.rstrip(","))
310 |         if str(self.getLevel()) == "10":
311 |             args.append(str('PDperArray=2'))
312 |         if self.getREADPOLICY():
313 |             args.append(self.getREADPOLICY())
314 |         if self.getWRITEPOLICY():
315 |             args.append(self.getWRITEPOLICY())
316 |         if self.getSTRIPSIZE():
317 |             args.append(str('strip=' + str(self.getSTRIPSIZE())))
318 |         logging.info("# Creating raid device with storcli")
319 |         logging.info("# Command line: "+subprocess.list2cmdline(args))
320 |         # Fetch VDs before creating the new one
321 |         # Wait for update of lsblk
322 |         sleep(5)
323 |         self.checkVDs()
324 |         self.checkBlockDevs()
325 |         VDsbefore = self.getVDs()
326 |         BDsbefore = self.getBlockDevs()
327 |         process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
328 |         stderr = process.communicate()[1]
329 |         if process.returncode != 0:
330 |             logging.error("storcli encountered an error: " + str(stderr))
331 |             raise RuntimeError("storcli command error")
332 |         else:
333 |             # Wait for update of lsblk
334 |             sleep(5)
335 |             # Fetch VDs after creating the new one
336 |             self.checkVDs()
337 |             self.checkBlockDevs()
338 |             VDsafter = self.getVDs()
339 |             BDsafter = self.getBlockDevs()
340 |             vd = [x for x in VDsafter if x not in VDsbefore]
341 |             if self.getVD() != None:
342 |                 if vd[0] != self.getVD():
343 |                     logging.info("# The VD changed, the new on is: " + vd[0])
344 |             self.setVD(vd[0])
345 |             bd = [x for x in BDsafter if x not in BDsbefore]
346 |             if (len(bd) != 1) or (('/dev/'+bd[0]) != self.getDevPath()):
347 |                 logging.info("Got BD: " + bd[0])
348 |                 logging.error("# Error: The new block device doesn't match the tested device path!")
349 |                 raise RuntimeError("New block dev doesn't match tested dev error")
350 |             # Set MegaRAID's automatic background initialization (autobgi) to
351 |             # off to prevent performance influences caused by autobgi
352 |             match = re.search('^[0-9]\/([0-9]+)', self.getVD())
353 |             vdNum = match.group(1)
354 |             storclibgi = subprocess.Popen([self.getUtil(),'/c0/v' + vdNum, 'set autobgi=off'], stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
355 |             stderr = storclibgi.communicate()[1]
356 |             if storclibgi.returncode != 0:
357 |                 logging.error("storcli encountered an error: " + stderr)
358 |                 raise RuntimeError("storcli command error")
359 |             else:
360 |                 logging.info("# Set autobgi=off for VD " + vdNum)
361 |             # Log information about the created VD
362 |             logging.info("# Created VD " + self.getVD())
363 |             logging.info("# Using block device " + bd[0])
364 | 
365 |     def deleteVD(self):
366 |         '''
367 |         Deletes a virtual drive, self.__vd must be a string like 0/0 specifying
368 |         the virtual drive.
369 |         '''
370 |         match = re.search('^[0-9]\/([0-9]+)',self.getVD())
371 |         vdNum = match.group(1)
372 |         storcli = subprocess.Popen([self.getUtil(),'/c0/v'+vdNum, 'del', 'force'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
373 |         stderr = storcli.communicate()[1]
374 |         if storcli.returncode != 0:
375 |             logging.error("storcli encountered an error: " + stderr)
376 |             raise RuntimeError("storcli command error")
377 |         else:
378 |             logging.info("# Deleting raid device VD "+vdNum)
379 | 
380 |     def isReady(self):
381 |         '''
382 |         Checks if a virtual device is ready, i.e. if no rebuild on any PDs is running
383 |         and if not initializarion process is going on.
384 |         @return True if VD is ready, False if not
385 |         '''
386 |         ready = None
387 |         storcli = subprocess.Popen([self.getUtil(),'/c0/eall/sall', 'show', 'rebuild'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
388 |         (stdout, stderr) = storcli.communicate()
389 |         if storcli.returncode != 0:
390 |             logging.error("storcli encountered an error: " + stderr)
391 |             raise RuntimeError("storcli command error")
392 |         else:
393 |             for line in stdout.splitlines():
394 |                 match = re.search('^\/c0\/e([0-9]+\/s[0-9]+).*$',line)
395 |                 if match != None:
396 |                     for d in self.getDevices():
397 |                         d = d.replace(':','/s')
398 |                         if d == match.group(1):
399 |                             logging.info(line)
400 |                             status = re.search('Not in progress',line)
401 |                             if status != None:
402 |                                 ready = True
403 |                             else:
404 |                                 ready = False
405 |         match = re.search('^[0-9]\/([0-9]+)',self.getVD())
406 |         vdNum = match.group(1)
407 |         storcli = subprocess.Popen([self.getUtil(),'/call', '/v'+vdNum, 'show', 'init'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True)
408 |         (stdout, stderr) = storcli.communicate()
409 |         if storcli.returncode != 0:
410 |             logging.error("storcli encountered an error: " + stderr)
411 |             raise RuntimeError("storcli command error")
412 |         else:
413 |             for line in stdout.splitlines():
414 |                 match = re.search(vdNum+' INIT',line)
415 |                 if match != None:
416 |                     logging.info(line)
417 |                     status = re.search('Not in progress',line)
418 |                     if status != None:
419 |                         ready = True
420 |                     else:
421 |                         ready = False
422 |         return ready
423 | 


--------------------------------------------------------------------------------
/src/system/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thomas-krenn/TKperf/638b0837b97b32337314993feb52b5111aae0979/src/system/__init__.py


--------------------------------------------------------------------------------