├── .gitignore ├── sample.png ├── MANIFEST.in ├── CHANGELOG ├── setup.py ├── demo.py ├── README.md ├── sar ├── __init__.py ├── multiparser.py ├── viz.py └── parser.py └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-* 2 | build/* 3 | dist/* 4 | MANIFEST 5 | *.pyc -------------------------------------------------------------------------------- /sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/milinda/python-sarviz/HEAD/sample.png -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include README 4 | include CHANGELOG 5 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 0.2.1 - 2013-11-23 2 | ------------------ 3 | FIX: minor bugfix 4 | 5 | Also, optimized values assigning for easier future expansion of included 6 | data. 7 | 8 | 9 | 0.2 - 2013-11-23 10 | ----------------- 11 | FIX: different locale times handling fixes (24hr vs 12hr AM/PM) 12 | 13 | 14 | 0.1 - 2013-11-20 15 | ----------------- 16 | Initial pre-release 17 | 18 | Known problems: 19 | - Doesn't handle SAR files with AM/PM outputs well -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup(name='sarviz', 6 | version='0.0.1', 7 | description='SAR (sysstat) output files parser and visualizer', 8 | author='Vedran Krivokuca , Milinda Pathirage ', 9 | url='https://github.com/milinda/python-sarviz', 10 | packages=['sarviz'], 11 | license='LGPL', 12 | platforms=['linux']) 13 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | Demo for SAR visualizer. 4 | ''' 5 | 6 | import os 7 | import sys 8 | 9 | from sar import parser 10 | from sar import viz 11 | 12 | 13 | def main(in_sar_log, output_path): 14 | insar = parser.Parser(in_sar_log) 15 | sar_viz = viz.Visualization(insar.get_sar_info(), paging=True, network=True, disk=True) 16 | sar_viz.save(output_path, output_type=viz.Visualization.PNG_OUTPUT) 17 | 18 | 19 | def set_include_path(): 20 | include_path = os.path.abspath("./") 21 | sys.path.append(include_path) 22 | 23 | if __name__ == "__main__": 24 | set_include_path() 25 | 26 | if not os.path.isfile(sys.argv[1]): 27 | raise 'Cannot find SAR log file {}'.format(sys.argv[1]) 28 | 29 | main(sys.argv[1], sys.argv[2]) 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-sarviz 2 | 3 | Tool for visualizing [SAR](https://en.wikipedia.org/wiki/Sar_(Unix)) logs. 4 | 5 | This is an extension of [python-sar](https://github.com/casastorta/python-sar). So this doesn't support SAR binary files, but only ASCII files. You can convert SAR binary files to ASCII files using following command. 6 | 7 | ``` 8 | sar -A -f > 9 | ``` 10 | 11 | ## How To Use 12 | 13 | ```python 14 | from sar import parser 15 | from sar import viz 16 | 17 | insar = parser.Parser('./data/sample.log') 18 | sar_viz = viz.Visualization(insar.get_sar_info(), paging=True, network=True, disk=True) 19 | sar_viz.save('sample.pdf') 20 | ``` 21 | 22 | # Example Visualization 23 | 24 | ![sarviz](https://raw.githubusercontent.com/milinda/python-sarviz/master/sample.png) 25 | -------------------------------------------------------------------------------- /sar/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | """Indicates CPU part of SAR file""" 5 | PART_CPU = 0 6 | 7 | """Indicates RAM memory usage part of SAR file""" 8 | PART_MEM = 1 9 | 10 | """Indicates swap memory usage part of SAR file""" 11 | PART_SWP = 2 12 | 13 | """I/O usage part of SAR file""" 14 | PART_IO = 3 15 | 16 | """Paging stats of SAR file""" 17 | PART_PAGING = 4 18 | 19 | """Network usage part of SAR file""" 20 | PART_NET = 5 21 | 22 | """CPU regexp pattern for detecting SAR section header""" 23 | PATTERN_CPU = ".*CPU.*(usr|user).*nice.*sys.*" 24 | 25 | """Regexp terms for finding fields in SAR parts for CPU""" 26 | FIELDS_CPU = [ 27 | '\%(usr|user)', '\%nice', '\%sys', '\%iowait', '\%idle' 28 | ] 29 | 30 | """Pair regexp terms with field names in CPU output dictionary""" 31 | FIELD_PAIRS_CPU = { 32 | 'usr': FIELDS_CPU[0], 'nice': FIELDS_CPU[1], 'sys': FIELDS_CPU[2], 33 | 'iowait': FIELDS_CPU[3], 'idle': FIELDS_CPU[4] 34 | } 35 | 36 | """Mem usage regexp pattern for detecting SAR section header""" 37 | PATTERN_MEM = ".*kbmemfree.*kbmemused.*memused.*kbbuffers.*kbcached.*" 38 | 39 | """Regexp terms for finding fields in SAR parts for memory usage""" 40 | FIELDS_MEM = [ 41 | 'kbmemfree', 'kbmemused', '\%memused', 'kbbuffers', 'kbcached' 42 | ] 43 | 44 | """Pair regexp terms with field names in memory usage output dictionary""" 45 | FIELD_PAIRS_MEM = { 46 | 'memfree': FIELDS_MEM[0], 'memused': FIELDS_MEM[1], 47 | 'memusedpercent': FIELDS_MEM[2], 'membuffer': FIELDS_MEM[3], 48 | 'memcache': FIELDS_MEM[4] 49 | } 50 | 51 | """Swap usage regexp pattern for detecting SAR section header""" 52 | PATTERN_SWP = ".*kbswpfree.*kbswpused.*swpused.*" 53 | 54 | """Regexp terms for finding fields in SAR parts for swap usage""" 55 | FIELDS_SWP = [ 56 | 'kbswpfree', 'kbswpused', '\%swpused' 57 | ] 58 | 59 | """Pair regexp terms with field names in swap usage output dictionary""" 60 | FIELD_PAIRS_SWP = { 61 | 'swapfree': FIELDS_SWP[0], 'swapused': FIELDS_SWP[1], 62 | 'swapusedpercent': FIELDS_SWP[2] 63 | } 64 | 65 | """I/O usage regexp pattern for detecting SAR section header""" 66 | PATTERN_IO = ".*tps.*rtps.*wtps.*bread\/s.*bwrtn\/s.*" 67 | 68 | """Regexp terms for finding fields in SAR parts for swap usage""" 69 | FIELDS_IO = [ 70 | '^tps', '^rtps', '^wtps', 'bread\/s', 'bwrtn\/s' 71 | ] 72 | 73 | """Pair regexp terms with field names in swap usage output dictionary""" 74 | FIELD_PAIRS_IO = { 75 | 'tps': FIELDS_IO[0], 'rtps': FIELDS_IO[1], 'wtps': FIELDS_IO[2], 76 | 'bread': FIELDS_IO[3], 'bwrite': FIELDS_IO[4], 77 | 78 | } 79 | 80 | """Paging stats regexp pattern for detecting SAR section header""" 81 | PATTERN_PAGING = ".*pgpgin\/s.*pgpgout\/s.*fault\/s.*majflt\/s.*pgfree\/s.*pgscank\/s.*pgscand\/s.*pgsteal\/s.*\%vmeff.*" 82 | 83 | """Regexp terms for finding fields in SAR parts for paging statistics""" 84 | FIELDS_PAGING = [ 85 | '^pgpgin\/s', '^pgpgout\/s', '^fault\/s', '^majflt\/s', '^pgfree\/s', 'pgscank\/s', 'pgscand\/s', 'pgsteal\/s', '\%vmeff' 86 | ] 87 | 88 | """Pair regexp terms with field names in paging status output dictionary""" 89 | FIELD_PAIRS_PAGING = { 90 | 'pgpgin': FIELDS_PAGING[0], 'pgpgout': FIELDS_PAGING[1], 'fault': FIELDS_PAGING[2], 91 | 'majflt': FIELDS_PAGING[3], 'pgfree': FIELDS_PAGING[4], 'pgscank': FIELDS_PAGING[5], 92 | 'pgscand': FIELDS_PAGING[6], 'pgsteal': FIELDS_PAGING[7], 'vmeff': FIELDS_PAGING[8] 93 | } 94 | 95 | """Network usage regexp pattern for detecting SAR section header""" 96 | PATTERN_NET = ".*IFACE.*rxpck\/s.*txpck\/s.*rxkB\/s.*txkB\/s.*rxcmp\/s.*txcmp\/s.*rxmcst\/s.*" 97 | 98 | """Regexp terms for finding fields in SAR parts for paging statistics""" 99 | FIELDS_NET = [ 100 | '^IFACE', '^rxpck\/s', '^txpck\/s', '^rxkB\/s', '^txkB\/s', '^rxcmp\/s', 'txcmp\/s', 'rxmcst\/s' 101 | ] 102 | 103 | """Pair regexp terms with field names in paging status output dictionary""" 104 | FIELD_PAIRS_NET = { 105 | 'iface': FIELDS_NET[0], 'rxpck': FIELDS_NET[1], 'txpck': FIELDS_NET[2], 106 | 'rxkB': FIELDS_NET[3], 'txkB': FIELDS_NET[4], 'rxcmp': FIELDS_NET[5], 107 | 'txcmp': FIELDS_NET[6], 'rxmcst': FIELDS_NET[7] 108 | } 109 | 110 | """Restart time regexp pattern for detecting SAR restart notices""" 111 | PATTERN_RESTART = ".*LINUX\ RESTART.*" 112 | 113 | """Pattern for splitting multiple combined SAR file""" 114 | PATTERN_MULTISPLIT = "Linux" 115 | 116 | """Split by date in multiday SAR file""" 117 | PATTERN_DATE = "[0-9][0-9][0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]" 118 | 119 | __all__ = [ 120 | "PART_CPU", "PART_MEM", "PART_SWP", "PART_IO", 121 | "PATTERN_CPU", "PATTERN_MEM", "PATTERN_SWP", "PATTERN_IO", 122 | "PATTERN_RESTART", "PATTERN_MULTISPLIT", "PATTERN_DATE" 123 | ] 124 | -------------------------------------------------------------------------------- /sar/multiparser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | :mod:`sar.multiparser` is a module containing class for parsing SAR output 5 | files where multiple files are merged into one huge file. 6 | 7 | .. WARNING:: 8 | Parses SAR ASCII output only, not binary files! 9 | 10 | .. WARNING:: 11 | Parses SAR ASCII files only, **not** ``SAR -A`` output! \ 12 | (24hr output compared to AM/PM output). 13 | Following versions might support ``SAR -A`` parsing. 14 | 15 | ''' 16 | 17 | import sar.parser as sarparse 18 | from sar import PATTERN_MULTISPLIT 19 | import mmap 20 | import os 21 | import traceback 22 | from types import StringType 23 | 24 | 25 | class Multiparser(object): 26 | ''' 27 | Multifile parser for SAR files. Derives from SAR Parser class 28 | :param filename: Name of the SAR output file, with combined data 29 | :type filename: str. 30 | ''' 31 | 32 | def __init__(self, combo_filename=''): 33 | 34 | self.__sarinfos = {} 35 | '''Dictionary for multiple dictionaries from 36 | :class:`com.nimium.sys.util.sar.parser.Parser`''' 37 | self.__splitpointers = [] 38 | '''List of pointers inside combo file where each file starts''' 39 | self.__filename = combo_filename 40 | '''SAR output filename to be parsed''' 41 | 42 | return None 43 | 44 | def load_file(self): 45 | ''' 46 | Loads combined SAR format logfile in ASCII format. 47 | :return: ``True`` if loading and parsing of file went fine, \ 48 | ``False`` if it failed (at any point) 49 | ''' 50 | daychunks = self.__split_file() 51 | 52 | if (daychunks): 53 | 54 | maxcount = len(self.__splitpointers) 55 | for i in range(maxcount): 56 | start = self.__splitpointers[i] 57 | end = None 58 | if (i < (maxcount - 1)): 59 | end = self.__splitpointers[i + 1] 60 | 61 | chunk = self.__get_chunk(start, end) 62 | 63 | parser = sarparse.Parser() 64 | cpu_usage, mem_usage, swp_usage, io_usage = \ 65 | parser._parse_file(parser._split_file(chunk)) 66 | 67 | self.__sarinfos[self.__get_part_date(chunk)] = { 68 | "cpu": cpu_usage, 69 | "mem": mem_usage, 70 | "swap": swp_usage, 71 | "io": io_usage 72 | } 73 | del(cpu_usage) 74 | del(mem_usage) 75 | del(swp_usage) 76 | del(io_usage) 77 | del(parser) 78 | 79 | return(True) 80 | 81 | def get_sar_info(self): 82 | ''' 83 | Returns parsed sar info 84 | :return: ``Dictionary``-style list of SAR data 85 | ''' 86 | return self.__sarinfos 87 | 88 | def __get_chunk(self, start=0, end=None): 89 | ''' 90 | Gets chunk from the sar combo file, from start to end 91 | :param start: where to start a pulled chunk 92 | :type start: int. 93 | :param end: where to end a pulled chunk 94 | :type end: int. 95 | :return: str. 96 | ''' 97 | piece = False 98 | 99 | if (self.__filename and os.access(self.__filename, os.R_OK)): 100 | 101 | fhandle = None 102 | 103 | try: 104 | fhandle = os.open(self.__filename, os.O_RDONLY) 105 | except OSError: 106 | print(("Couldn't open file %s" % (self.__filename))) 107 | fhandle = None 108 | 109 | if (fhandle): 110 | 111 | try: 112 | sarmap = mmap.mmap(fhandle, length=0, prot=mmap.PROT_READ) 113 | except (TypeError, IndexError): 114 | os.close(fhandle) 115 | traceback.print_exc() 116 | #sys.exit(-1) 117 | return False 118 | 119 | if (not end): 120 | end = sarmap.size() 121 | 122 | try: 123 | sarmap.seek(start) 124 | piece = sarmap.read(end - start) 125 | except: 126 | traceback.print_exc() 127 | 128 | os.close(fhandle) 129 | 130 | return(piece) 131 | 132 | def __split_file(self): 133 | ''' 134 | Splits combined SAR output file (in ASCII format) in order to 135 | extract info we need for it, in the format we want. 136 | :return: ``List``-style of SAR file sections separated by 137 | the type of info they contain (SAR file sections) without 138 | parsing what is exactly what at this point 139 | ''' 140 | # Filename passed checks through __init__ 141 | if (self.__filename and os.access(self.__filename, os.R_OK)): 142 | 143 | fhandle = None 144 | 145 | try: 146 | fhandle = os.open(self.__filename, os.O_RDONLY) 147 | except OSError: 148 | print(("Couldn't open file %s" % (self.__filename))) 149 | fhandle = None 150 | 151 | if (fhandle): 152 | 153 | try: 154 | sarmap = mmap.mmap(fhandle, length=0, prot=mmap.PROT_READ) 155 | except (TypeError, IndexError): 156 | os.close(fhandle) 157 | traceback.print_exc() 158 | #sys.exit(-1) 159 | return False 160 | 161 | sfpos = sarmap.find(PATTERN_MULTISPLIT, 0) 162 | 163 | while (sfpos > -1): 164 | 165 | '''Split by day found''' 166 | self.__splitpointers.append(sfpos) 167 | 168 | # Iterate for new position 169 | try: 170 | sfpos = sarmap.find(PATTERN_MULTISPLIT, (sfpos + 1)) 171 | except ValueError: 172 | print("ValueError on mmap.find()") 173 | return True 174 | 175 | if (self.__splitpointers): 176 | # Not sure if this will work - if empty set 177 | # goes back as True here 178 | return True 179 | 180 | return False 181 | 182 | def __get_part_date(self, part=''): 183 | ''' 184 | Retrieves date of the combo part from the file 185 | :param part: Part of the combo file (parsed out whole SAR file 186 | from the combo 187 | :type part: str. 188 | :return: string containing date in ISO format (YYY-MM-DD) 189 | ''' 190 | if (type(part) is not StringType): 191 | # We can cope with strings only 192 | return False 193 | 194 | firstline = part.split("\n")[0] 195 | 196 | info = firstline.split() 197 | datevalue = '' 198 | 199 | try: 200 | datevalue = info[3] 201 | 202 | except KeyError: 203 | datevalue = False 204 | 205 | except: 206 | traceback.print_exc() 207 | datevalue = False 208 | 209 | return(datevalue) 210 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /sar/viz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | :mod:`sar.viz` is a module containing classes for visualizing sar logs. 4 | ''' 5 | 6 | from matplotlib.backends.backend_pdf import PdfPages 7 | import matplotlib.patches as mpatches 8 | import matplotlib.pyplot as plt 9 | 10 | import numpy as np 11 | 12 | 13 | class Visualization(object): 14 | PDF_OUTPUT = 0 15 | PNG_OUTPUT = 1 16 | SAR_TYPES = ['cpu', 'mem', 'io', 'paging', 'net'] 17 | PLT_XTICK_LABEL_ROTATION = 'vertical' 18 | 19 | def __init__(self, sar_data, cpu=True, mem=True, paging=False, disk=False, 20 | network=False): 21 | """Create a sar log visualization. 22 | 23 | Only CPU and memory usage charts are enabled by default. 24 | 25 | Args: 26 | sar_data (dict): Processed sar log from Parser 27 | cpu (:obj:`bool`, optional): Enable CPU usage charts 28 | mem (:obj:`bool`, optional): Enable memory usage cgarts 29 | paging (:obj:`bool`, optional): Enable paging activity charts 30 | disk (:obj:`bool`, optional): Enable disk usage charts 31 | network (:obj:`bool`, optional): Enable network usage charts 32 | """ 33 | 34 | if not isinstance(sar_data, dict): 35 | raise 'Incompatible sar_data type: {}'.format( 36 | type(sar_data).__name__) 37 | 38 | self.sar_data = sar_data 39 | """dict: Processed sar logs""" 40 | 41 | self.enable_cpu = cpu 42 | self.enable_mem = mem 43 | self.enable_disk = disk 44 | self.enable_net = disk 45 | self.enable_paging = paging 46 | 47 | self.time_points = [] 48 | """(:obj:`list` of :obj:`str`): time points which system activity was 49 | recorded""" 50 | 51 | self.x_data = [] 52 | """(:obj:`list` of :obj:`int`): x axis data""" 53 | 54 | self.xticks = [] 55 | """(:obj:`list` of :obj:`int`): x axis ticks""" 56 | 57 | self.xtick_labels = [] 58 | """(:obj:`list` of :obj:`str`) x axis tick labels""" 59 | 60 | self.cpu_usage_usr = [] 61 | self.cpu_usage_sys = [] 62 | self.page_faults_per_sec = [] 63 | self.major_page_faults_per_sec = [] 64 | self.page_ins_per_sec = [] 65 | self.page_outs_per_sec = [] 66 | self.pct_mem_used = [] 67 | self.mem_used_mb = [] 68 | self.mem_cached_mb = [] 69 | self.mem_buffer_mb = [] 70 | self.kb_rcv_per_sec = {} 71 | self.kb_trans_per_sec = {} 72 | self.breads_per_sec = [] 73 | self.bwrites_per_sec = [] 74 | self.fig_height = 0 75 | self.num_plots = 0 76 | 77 | self._calculate_plot_height() 78 | self._preprocess_sar_data() 79 | 80 | def _calculate_plot_height(self): 81 | num_plots = 0 82 | 83 | if self.enable_cpu: 84 | num_plots += 1 85 | 86 | if self.enable_disk: 87 | num_plots += 1 88 | 89 | if self.enable_mem: 90 | num_plots += 2 91 | 92 | if self.enable_net: 93 | num_plots += 1 94 | 95 | if self.enable_paging: 96 | num_plots += 2 97 | 98 | self.num_plots = num_plots 99 | self.fig_height = num_plots * 4 100 | 101 | def _preprocess_sar_data(self): 102 | for t in Visualization.SAR_TYPES: 103 | if t in self.sar_data: 104 | time_points = self.sar_data[t].keys() 105 | time_points.sort() 106 | self.time_points = time_points 107 | break 108 | 109 | tp_count = len(time_points) 110 | xtick_label_stepsize = tp_count / 15 111 | self.x_data = range(tp_count) 112 | self.xticks = np.arange(0, tp_count, xtick_label_stepsize) 113 | self.xtick_labels = [time_points[i] for i in self.xticks] 114 | 115 | if self.enable_cpu: 116 | self.cpu_usage_sys = [self.sar_data['cpu'][tp]['all']['sys'] 117 | for tp in self.time_points] 118 | self.cpu_usage_usr = [self.sar_data['cpu'][tp]['all']['usr'] 119 | for tp in self.time_points] 120 | 121 | if self.enable_mem: 122 | self.pct_mem_used = [self.sar_data['mem'][tp]['memusedpercent'] / 1024 123 | for tp in self.time_points] 124 | self.mem_used_mb = [(self.sar_data['mem'][tp]['memused'] - ( 125 | self.sar_data['mem'][tp]['memcache'] + self.sar_data['mem'] 126 | [tp]['membuffer'])) / 1024 for tp in self.time_points] 127 | self.mem_cached_mb = [self.sar_data['mem'][tp]['memcache'] / 1024 128 | for tp in self.time_points] 129 | self.mem_buffer_mb = [self.sar_data['mem'][tp]['membuffer'] / 1024 130 | for tp in self.time_points] 131 | 132 | if self.enable_paging: 133 | self.page_faults_per_sec = [self.sar_data['paging'][tp]['fault'] 134 | for tp in self.time_points] 135 | self.major_page_faults_per_sec = [self.sar_data['paging'][tp]['majflt'] 136 | for tp in self.time_points] 137 | self.page_ins_per_sec = [self.sar_data['paging'][tp]['pgpgin'] 138 | for tp in self.time_points] 139 | self.page_outs_per_sec = [self.sar_data['paging'][tp]['pgpgout'] 140 | for tp in self.time_points] 141 | 142 | if self.enable_disk: 143 | self.breads_per_sec = [self.sar_data['io'][tp]['bread'] for tp in self.time_points] 144 | self.bwrites_per_sec = [self.sar_data['io'][tp]['bwrite'] for tp in self.time_points] 145 | 146 | if self.enable_net: 147 | net_data = self.sar_data['net'] 148 | for tp in self.time_points: 149 | dp = net_data[tp] 150 | for iface in dp.keys(): 151 | if iface not in self.kb_rcv_per_sec: 152 | self.kb_rcv_per_sec[iface] = [dp[iface]['rxkB']] 153 | else: 154 | self.kb_rcv_per_sec[iface].append(dp[iface]['rxkB']) 155 | 156 | if iface not in self.kb_trans_per_sec: 157 | self.kb_trans_per_sec[iface] = [dp[iface]['txkB']] 158 | else: 159 | self.kb_trans_per_sec[iface].append(dp[iface]['txkB']) 160 | 161 | def save(self, output_path, output_type=PDF_OUTPUT): 162 | plt_idx = 1 163 | fig = plt.figure() 164 | fig.set_figheight(self.fig_height) 165 | 166 | plt.clf() 167 | plt.subplots_adjust(wspace=1, hspace=1) 168 | 169 | if self.enable_cpu: 170 | plt.subplot(self.num_plots, 1, plt_idx) 171 | plt.xticks(self.xticks, self.xtick_labels, 172 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 173 | plt.plot(self.x_data, self.cpu_usage_usr, label='usr') 174 | plt.plot(self.x_data, self.cpu_usage_sys, label='sys') 175 | plt.xlabel('time') 176 | plt.ylabel('% usage') 177 | plt.title('CPU Usage') 178 | lg = plt.legend(frameon=False) 179 | lg_txts = lg.get_texts() 180 | plt.setp(lg_txts, fontsize=10) 181 | plt_idx += 1 182 | 183 | if self.enable_mem: 184 | plt.subplot(self.num_plots, 1, plt_idx) 185 | plt.xticks(self.xticks, self.xtick_labels, 186 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 187 | plt.plot(self.x_data, self.pct_mem_used, label='% mem used') 188 | plt.xlabel('time') 189 | plt.ylabel('% mem used') 190 | plt.title('Percentage of Memory Used') 191 | lg = plt.legend(frameon=False) 192 | lg_txts = lg.get_texts() 193 | plt.setp(lg_txts, fontsize=10) 194 | plt_idx += 1 195 | 196 | plt.subplot(self.num_plots, 1, plt_idx) 197 | plt.xticks(self.xticks, self.xtick_labels, 198 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 199 | plt.stackplot(self.x_data, self.mem_buffer_mb, self.mem_cached_mb, self.mem_used_mb, 200 | colors=['lemonchiffon', 'navajowhite', 'sandybrown']) 201 | plt.xlabel('time') 202 | plt.ylabel('Mem Usage (MB)') 203 | plt.title('Memory Usage') 204 | 205 | # lc_handle = mpatches.Patch(color='lemonchiffon', label='Buffered Memory') 206 | # nw_handle = mpatches.Patch(color='navajowhite', label='Cached Memory') 207 | # sb_handle = mpatches.Patch(color='sandybrown', label='Used Memory') 208 | 209 | lg = plt.legend([mpatches.Patch(color='lemonchiffon'), 210 | mpatches.Patch(color='navajowhite'), 211 | mpatches.Patch(color='sandybrown')], 212 | ['Buffered Memory', 'Cached Memory', 'Used Memory']) 213 | lg.get_frame().set_alpha(0.6) 214 | lg_txts = lg.get_texts() 215 | plt.setp(lg_txts, fontsize=10) 216 | plt_idx += 1 217 | 218 | if self.enable_paging: 219 | plt.subplot(self.num_plots, 1, plt_idx) 220 | plt.xticks(self.xticks, self.xtick_labels, 221 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 222 | plt.plot(self.x_data, self.page_faults_per_sec, label='faults/s') 223 | plt.plot(self.x_data, self.major_page_faults_per_sec, label='major faults/s') 224 | plt.xlabel('time') 225 | plt.ylabel('faults/s') 226 | plt.title('Page Faults') 227 | lg = plt.legend(frameon=False) 228 | lg_txts = lg.get_texts() 229 | plt.setp(lg_txts, fontsize=10) 230 | plt_idx += 1 231 | 232 | plt.subplot(self.num_plots, 1, plt_idx) 233 | plt.xticks(self.xticks, self.xtick_labels, 234 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 235 | plt.plot(self.x_data, self.page_ins_per_sec, label='page ins/s') 236 | plt.plot(self.x_data, self.page_outs_per_sec, label='page outs/s') 237 | plt.xlabel('time') 238 | plt.ylabel('KB/s') 239 | plt.title('Page Ins and Outs') 240 | lg = plt.legend(frameon=False) 241 | lg_txts = lg.get_texts() 242 | plt.setp(lg_txts, fontsize=10) 243 | plt_idx += 1 244 | 245 | if self.enable_net: 246 | plt.subplot(self.num_plots, 1, plt_idx) 247 | plt.xticks(self.xticks, self.xtick_labels, 248 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 249 | for iface in self.kb_rcv_per_sec.keys(): 250 | plt.plot(self.x_data, self.kb_rcv_per_sec[iface], label='{}-rx'.format(iface)) 251 | for iface in self.kb_trans_per_sec.keys(): 252 | plt.plot(self.x_data, self.kb_trans_per_sec[iface], label='{}-tx'.format(iface)) 253 | plt.xlabel('time') 254 | plt.ylabel('KB/s') 255 | plt.title('Network Usage') 256 | lg = plt.legend(loc=1, 257 | ncol=len(self.kb_rcv_per_sec.keys()), frameon=False) 258 | lg.get_frame().set_alpha(0) 259 | lg_txts = lg.get_texts() 260 | plt.setp(lg_txts, fontsize=10) 261 | plt_idx += 1 262 | 263 | if self.enable_disk: 264 | plt.subplot(self.num_plots, 1, plt_idx) 265 | plt.xticks(self.xticks, self.xtick_labels, 266 | rotation=Visualization.PLT_XTICK_LABEL_ROTATION) 267 | plt.plot(self.x_data, self.breads_per_sec, label='reads') 268 | plt.plot(self.x_data, self.bwrites_per_sec, label='writes') 269 | plt.xlabel('time') 270 | plt.ylabel('blocks/s') 271 | plt.title('Disk IO') 272 | lg = plt.legend(frameon=False) 273 | lg_txts = lg.get_texts() 274 | plt.setp(lg_txts, fontsize=10) 275 | 276 | fig.tight_layout() 277 | if output_type == Visualization.PDF_OUTPUT: 278 | pp = PdfPages(output_path) 279 | pp.savefig() 280 | pp.close() 281 | elif output_type == Visualization.PNG_OUTPUT: 282 | fig.savefig(output_path) 283 | plt.close(fig) 284 | -------------------------------------------------------------------------------- /sar/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | :mod:`sar.parser` is a module containing class for parsing SAR output files. 4 | 5 | .. WARNING:: 6 | Parses SAR ASCII output only, not binary files! 7 | ''' 8 | 9 | from sar import PART_CPU, PART_MEM, PART_SWP, PART_IO, PART_PAGING, PART_NET, \ 10 | PATTERN_CPU, PATTERN_MEM, PATTERN_SWP, PATTERN_IO, PATTERN_PAGING, PATTERN_NET, PATTERN_RESTART, \ 11 | FIELDS_CPU, FIELD_PAIRS_CPU, FIELDS_MEM, FIELD_PAIRS_MEM, FIELDS_SWP, \ 12 | FIELD_PAIRS_SWP, FIELDS_IO, FIELD_PAIRS_IO, FIELDS_PAGING, FIELD_PAIRS_PAGING, \ 13 | FIELDS_NET, FIELD_PAIRS_NET 14 | import mmap 15 | import os 16 | import re 17 | import traceback 18 | import logging 19 | from types import ListType 20 | import platform 21 | 22 | 23 | class Parser(object): 24 | ''' 25 | Parser for sar outputs. Uses SAR interpreter binary and parses out \ 26 | its output 27 | :param filename: Name of the SAR output file 28 | :type filename: str. 29 | ''' 30 | 31 | def __init__(self, filename=''): 32 | 33 | self._sarinfo = {} 34 | '''Hash with SAR info''' 35 | self.__file_date = '' 36 | '''String which contains date of SAR file''' 37 | self.__restart_times = [] 38 | '''List with box restart times''' 39 | self.__filename = filename 40 | '''SAR output filename to be parsed''' 41 | 42 | self.__cpu_fields = None 43 | '''CPU fields indexes''' 44 | self.__mem_fields = None 45 | '''Memory usage indexes''' 46 | self.__swp_fields = None 47 | '''Swap usage indexes''' 48 | self.__io_fields = None 49 | '''I/O usage indexes''' 50 | self.__paging_fields = None 51 | '''OS Paging indexes''' 52 | self.__net_fields = None 53 | '''Network usage indexes''' 54 | 55 | return None 56 | 57 | def load_file(self): 58 | ''' 59 | Loads SAR format logfile in ASCII format (sarXX). 60 | :return: ``True`` if loading and parsing of file went fine, \ 61 | ``False`` if it failed (at any point) 62 | ''' 63 | 64 | # We first split file into pieces 65 | searchunks = self._split_file() 66 | 67 | if (searchunks): 68 | 69 | # And then we parse pieces into meaningful data 70 | cpu_usage, mem_usage, swp_usage, io_usage, paging_stats, net_usage = \ 71 | self._parse_file(searchunks) 72 | 73 | if (cpu_usage is False): 74 | return False 75 | 76 | self._sarinfo = { 77 | "cpu": cpu_usage, 78 | "mem": mem_usage, 79 | "swap": swp_usage, 80 | "io": io_usage, 81 | "paging": paging_stats, 82 | "net": net_usage 83 | } 84 | del(cpu_usage) 85 | del(mem_usage) 86 | del(swp_usage) 87 | del(io_usage) 88 | del(paging_stats) 89 | del(net_usage) 90 | 91 | return True 92 | 93 | else: 94 | return False 95 | 96 | def get_filedate(self): 97 | ''' 98 | Returns file date of SAR file 99 | :return: ISO format (YYYY-MM-DD) of a SAR file 100 | ''' 101 | if (self.__file_date == ''): 102 | # If not already parsed out, parse it. 103 | self.__get_filedate() 104 | 105 | return self.__file_date 106 | 107 | def get_sar_info(self): 108 | ''' 109 | Returns parsed sar info 110 | :return: ``Dictionary``-style list of SAR data 111 | ''' 112 | 113 | try: 114 | test = self._sarinfo["cpu"] 115 | del(test) 116 | 117 | except KeyError: 118 | file_parsed = self.load_file() 119 | if (file_parsed): 120 | return self._sarinfo 121 | else: 122 | return False 123 | 124 | except: 125 | ### DEBUG 126 | traceback.print_exc() 127 | return False 128 | 129 | return self._sarinfo 130 | 131 | def _split_file(self, data=''): 132 | ''' 133 | Splits SAR output or SAR output file (in ASCII format) in order to 134 | extract info we need for it, in the format we want. 135 | :param data: Input data instead of file 136 | :type data: str. 137 | :return: ``List``-style of SAR file sections separated by 138 | the type of info they contain (SAR file sections) without 139 | parsing what is exactly what at this point 140 | ''' 141 | 142 | # Filename passed checks through __init__ 143 | if ((self.__filename and os.access(self.__filename, os.R_OK)) 144 | or data != ''): 145 | 146 | fhandle = None 147 | 148 | if (data == ''): 149 | try: 150 | fhandle = os.open(self.__filename, os.O_RDONLY) 151 | except OSError: 152 | print(("Couldn't open file %s" % (self.__filename))) 153 | fhandle = None 154 | 155 | if (fhandle or data != ''): 156 | 157 | datalength = 0 158 | 159 | # Dealing with mmap difference on Windows and Linux 160 | if platform.system() == 'Windows': 161 | dataprot = mmap.ACCESS_READ 162 | else: 163 | dataprot = mmap.PROT_READ 164 | 165 | if (data != ''): 166 | fhandle = -1 167 | datalength = len(data) 168 | 169 | if platform.system() == 'Windows': 170 | dataprot = mmap.ACCESS_READ | mmap.ACCESS_WRITE 171 | else: 172 | dataprot = mmap.PROT_READ | mmap.PROT_WRITE 173 | 174 | try: 175 | if platform.system() == 'Windows': 176 | sarmap = mmap.mmap( 177 | fhandle, length=datalength, access=dataprot 178 | ) 179 | else: 180 | sarmap = mmap.mmap( 181 | fhandle, length=datalength, prot=dataprot 182 | ) 183 | if (data != ''): 184 | 185 | sarmap.write(data) 186 | sarmap.flush() 187 | sarmap.seek(0, os.SEEK_SET) 188 | 189 | except (TypeError, IndexError): 190 | if (data == ''): 191 | os.close(fhandle) 192 | traceback.print_exc() 193 | #sys.exit(-1) 194 | return False 195 | 196 | # Here we'll store chunks of SAR file, unparsed 197 | searchunks = [] 198 | oldchunkpos = 0 199 | dlpos = sarmap.find("\n\n", 0) 200 | size = 0 201 | 202 | if (data == ''): 203 | # We can do mmap.size() only on read-only mmaps 204 | size = sarmap.size() 205 | else: 206 | # Otherwise, if data was passed to us, 207 | # we measure its length 208 | len(data) 209 | 210 | #oldchunkpos = dlpos 211 | 212 | while (dlpos > -1): # mmap.find() returns -1 on failure. 213 | 214 | tempchunk = sarmap.read(dlpos - oldchunkpos) 215 | searchunks.append(tempchunk.strip()) 216 | 217 | # We remember position, add 2 for 2 DD's 218 | # (newspaces in production). We have to remember 219 | # relative value 220 | oldchunkpos += (dlpos - oldchunkpos) + 2 221 | 222 | # We position to new place, to be behind \n\n 223 | # we've looked for. 224 | try: 225 | sarmap.seek(2, os.SEEK_CUR) 226 | except ValueError: 227 | print(("Out of bounds (%s)!\n" % (sarmap.tell()))) 228 | # Now we repeat find. 229 | dlpos = sarmap.find("\n\n") 230 | 231 | # If it wasn't the end of file, we want last piece of it 232 | if (oldchunkpos < size): 233 | tempchunk = sarmap[(oldchunkpos):] 234 | searchunks.append(tempchunk.strip()) 235 | 236 | sarmap.close() 237 | 238 | if (fhandle != -1): 239 | os.close(fhandle) 240 | 241 | if (searchunks): 242 | return searchunks 243 | else: 244 | return False 245 | 246 | return False 247 | 248 | def _parse_file(self, sar_parts): 249 | ''' 250 | Parses splitted file to get proper information from split parts. 251 | :param sar_parts: Array of SAR file parts 252 | :return: ``Dictionary``-style info (but still non-parsed) \ 253 | from SAR file, split into sections we want to check 254 | ''' 255 | cpu_usage = '' 256 | mem_usage = '' 257 | swp_usage = '' 258 | io_usage = '' 259 | paging_stats = '' 260 | net_usage = '' 261 | 262 | # If sar_parts is a list 263 | if (type(sar_parts) is ListType): 264 | # We will find CPU section by looking for typical line in CPU 265 | # section of SAR output 266 | cpu_pattern = re.compile(PATTERN_CPU) 267 | mem_pattern = re.compile(PATTERN_MEM) 268 | swp_pattern = re.compile(PATTERN_SWP) 269 | io_pattern = re.compile(PATTERN_IO) 270 | paging_pattern = re.compile(PATTERN_PAGING) 271 | net_pattern = re.compile(PATTERN_NET) 272 | restart_pattern = re.compile(PATTERN_RESTART) 273 | 274 | ''' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''' 275 | ''' ********** ATTENTION ******* ''' 276 | ''' THERE CAN BE MORE THAN ONE SAME SECTION IN ONE FILE ''' 277 | ''' IF SYSTEM WAS REBOOTED DURING THE DAY ''' 278 | ''' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''' 279 | 280 | for part in sar_parts: 281 | logging.debug(part) 282 | # Try to match CPU usage SAR file sections 283 | if (cpu_pattern.search(part)): 284 | if (cpu_usage == ''): 285 | cpu_usage = part 286 | try: 287 | first_line = part.split("\n")[0] 288 | except IndexError: 289 | first_line = part 290 | 291 | self.__cpu_fields = \ 292 | self.__find_column(FIELDS_CPU, first_line) 293 | 294 | else: 295 | cpu_usage += "\n" + part 296 | 297 | # Try to match memory usage SAR file sections 298 | if (mem_pattern.search(part)): 299 | if (mem_usage == ''): 300 | mem_usage = part 301 | try: 302 | first_line = part.split("\n")[0] 303 | except IndexError: 304 | first_line = part 305 | 306 | self.__mem_fields = \ 307 | self.__find_column(FIELDS_MEM, first_line) 308 | 309 | else: 310 | mem_usage += "\n" + part 311 | 312 | # Try to match swap usage SAR file sections 313 | if (swp_pattern.search(part)): 314 | if (swp_usage == ''): 315 | swp_usage = part 316 | try: 317 | first_line = part.split("\n")[0] 318 | except IndexError: 319 | first_line = part 320 | 321 | self.__swp_fields = \ 322 | self.__find_column(FIELDS_SWP, first_line) 323 | else: 324 | swp_usage += "\n" + part 325 | 326 | # Try to match IO usage SAR file sections 327 | if (io_pattern.search(part)): 328 | if (io_usage == ''): 329 | io_usage = part 330 | try: 331 | first_line = part.split("\n")[0] 332 | except IndexError: 333 | first_line = part 334 | 335 | self.__io_fields = \ 336 | self.__find_column(FIELDS_IO, first_line) 337 | else: 338 | io_usage += "\n" + part 339 | 340 | # Try to match paging stats SAR file sections 341 | if (paging_pattern.search(part)): 342 | if (paging_stats == ''): 343 | paging_stats = part 344 | try: 345 | first_line = part.split("\n")[0] 346 | except IndexError: 347 | first_line = part 348 | 349 | self.__paging_fields = \ 350 | self.__find_column(FIELDS_PAGING, first_line) 351 | else: 352 | paging_stats += "\n" + part 353 | 354 | # Try to match network usage SAR file sections 355 | if (net_pattern.search(part)): 356 | if (net_usage == ''): 357 | net_usage = part 358 | try: 359 | first_line = part.split("\n")[0] 360 | except IndexError: 361 | first_line = part 362 | 363 | self.__net_fields = \ 364 | self.__find_column(FIELDS_NET, first_line) 365 | else: 366 | net_usage += "\n" + part 367 | 368 | # Try to match restart time 369 | if (restart_pattern.search(part)): 370 | pieces = part.split() 371 | self.__restart_times.append(pieces[0]) 372 | del(pieces) 373 | 374 | del(sar_parts) 375 | 376 | # Now we have parts pulled out and combined, do further 377 | # processing. 378 | cpu_output = self.__split_info(cpu_usage, PART_CPU) 379 | mem_output = self.__split_info(mem_usage, PART_MEM) 380 | swp_output = self.__split_info(swp_usage, PART_SWP) 381 | io_output = self.__split_info(io_usage, PART_IO) 382 | paging_output = self.__split_info(paging_stats, PART_PAGING) 383 | net_output = self.__split_info(net_usage, PART_NET) 384 | 385 | del(cpu_usage) 386 | del(mem_usage) 387 | del(swp_usage) 388 | del(io_usage) 389 | del(paging_stats) 390 | del(net_usage) 391 | 392 | return (cpu_output, mem_output, swp_output, io_output, paging_output, net_output) 393 | 394 | return (False, False, False, False, False, False) 395 | 396 | def __find_column(self, column_names, part_first_line): 397 | ''' 398 | Finds the column for the column_name in sar type definition, 399 | and returns its index. 400 | :param column_name: Names of the column we look for (regex) put in 401 | the list 402 | :param part_first_line: First line of the SAR part 403 | :return: ``Dictionary`` of names => position, None for not present 404 | ''' 405 | part_parts = part_first_line.split() 406 | 407 | ### DEBUG 408 | #print("Parts: %s" % (part_parts)) 409 | 410 | return_dict = {} 411 | 412 | counter = 0 413 | for piece in part_parts: 414 | for colname in column_names: 415 | pattern_re = re.compile(colname) 416 | if (pattern_re.search(piece)): 417 | return_dict[colname] = counter 418 | break 419 | counter += 1 420 | 421 | # Verify the content of the return dictionary, fill the blanks 422 | # with -1s :-) 423 | for colver in column_names: 424 | try: 425 | tempval = return_dict[colver] 426 | del(tempval) 427 | except KeyError: 428 | return_dict[colver] = None 429 | 430 | return(return_dict) 431 | 432 | def __split_info(self, info_part, part_type=PART_CPU): 433 | ''' 434 | Splits info from SAR parts into logical stuff :-) 435 | :param info_part: Part of SAR output we want to split into usable data 436 | :param part_type: Value of a constant which tells us which SAR part \ 437 | we're parsing (because of their specifics) 438 | :return: ``List``-style info from SAR files, now finally \ 439 | completely parsed into meaningful data for further processing 440 | ''' 441 | 442 | pattern = '' 443 | 444 | if (part_type == PART_CPU): 445 | pattern = PATTERN_CPU 446 | elif (part_type == PART_MEM): 447 | pattern = PATTERN_MEM 448 | elif (part_type == PART_SWP): 449 | pattern = PATTERN_SWP 450 | elif (part_type == PART_IO): 451 | pattern = PATTERN_IO 452 | elif (part_type == PART_PAGING): 453 | pattern = PATTERN_PAGING 454 | elif (part_type == PART_NET): 455 | pattern = PATTERN_NET 456 | 457 | if (pattern == ''): 458 | return False 459 | 460 | return_dict = {} 461 | 462 | pattern_re = re.compile(pattern) 463 | 464 | for part_line in info_part.split("\n"): 465 | pattern = '' 466 | 467 | if (part_line.strip() != '') and \ 468 | not pattern_re.search(part_line): 469 | 470 | # Take care of AM/PM timestamps in SAR file 471 | is_24hr = True 472 | is_AM = False 473 | 474 | if part_line[9:11] == 'AM': 475 | is_24hr = False 476 | is_AM = True 477 | elif part_line[9:11] == 'PM': 478 | is_24hr = False 479 | is_AM = False 480 | 481 | if is_24hr is False: 482 | part_line = \ 483 | ('%s_%s XX %s' % ( 484 | part_line[:8], part_line[9:11], part_line[12:] 485 | )) 486 | 487 | # Line is not empty, nor it's header. 488 | # let's hit the road Jack! 489 | elems = part_line.split() 490 | full_time = elems[0].strip() 491 | 492 | if (full_time != "Average:"): 493 | 494 | # Convert time to 24hr format if needed 495 | if is_24hr is False: 496 | full_time = full_time[:-3] 497 | 498 | # 12 is a bitch in AM/PM notation 499 | if full_time[:2] == '12': 500 | if is_AM is True: 501 | full_time = ('%s:%s' % ('00', full_time[3:])) 502 | is_AM = not is_AM 503 | 504 | if is_AM is False and full_time[0:2] != '00': 505 | hours = int(full_time[:2]) + 12 506 | hours = ('%02d' % (hours,)) 507 | full_time = ('%s:%s' % (hours, full_time[3:])) 508 | 509 | try: 510 | blah = return_dict[full_time] 511 | del(blah) 512 | except KeyError: 513 | return_dict[full_time] = {} 514 | 515 | # Common assigner 516 | fields = None 517 | pairs = None 518 | if part_type == PART_CPU: 519 | fields = self.__cpu_fields 520 | pairs = FIELD_PAIRS_CPU 521 | elif part_type == PART_MEM: 522 | fields = self.__mem_fields 523 | pairs = FIELD_PAIRS_MEM 524 | elif part_type == PART_SWP: 525 | fields = self.__swp_fields 526 | pairs = FIELD_PAIRS_SWP 527 | elif part_type == PART_IO: 528 | fields = self.__io_fields 529 | pairs = FIELD_PAIRS_IO 530 | elif part_type == PART_PAGING: 531 | fields = self.__paging_fields 532 | pairs = FIELD_PAIRS_PAGING 533 | elif part_type == PART_NET: 534 | fields = self.__net_fields 535 | pairs = FIELD_PAIRS_NET 536 | 537 | for sectionname in pairs.iterkeys(): 538 | 539 | value = elems[fields[pairs[sectionname]]] 540 | 541 | if sectionname == 'membuffer' or \ 542 | sectionname == 'memcache' or \ 543 | sectionname == 'memfree' or \ 544 | sectionname == 'memused' or \ 545 | sectionname == 'swapfree' or \ 546 | sectionname == 'swapused': 547 | value = int(value) 548 | elif sectionname == 'iface': 549 | value = str(value) 550 | else: 551 | value = float(value) 552 | 553 | if part_type == PART_CPU: 554 | cpuid = elems[(1 if is_24hr is True else 2)] 555 | try: 556 | blah = return_dict[full_time][cpuid] 557 | del(blah) 558 | except KeyError: 559 | return_dict[full_time][cpuid] = {} 560 | return_dict[full_time][cpuid][sectionname] = \ 561 | value 562 | elif part_type == PART_NET: 563 | iface = elems[(1 if is_24hr is True else 2)] 564 | try: 565 | blah = return_dict[full_time][iface] 566 | del(blah) 567 | except KeyError: 568 | return_dict[full_time][iface] = {} 569 | return_dict[full_time][iface][sectionname] = \ 570 | value 571 | else: 572 | return_dict[full_time][sectionname] = value 573 | 574 | return (return_dict) 575 | 576 | def __get_filedate(self): 577 | ''' 578 | Parses (extracts) date of SAR data, from the SAR output file itself. 579 | :return: ISO-style (YYYY-MM-DD) date from SAR file 580 | ''' 581 | 582 | if (os.access(self.__filename, os.R_OK)): 583 | 584 | # Read first line of the file 585 | try: 586 | sar_file = open(self.__filename, "r") 587 | 588 | except OSError: 589 | ### DEBUG 590 | traceback.print_exc() 591 | return False 592 | 593 | except: 594 | ### DEBUG 595 | traceback.print_exc() 596 | return False 597 | 598 | firstline = sar_file.readline() 599 | info = firstline.split() 600 | sar_file.close() 601 | 602 | try: 603 | self.__file_date = info[3] 604 | 605 | except KeyError: 606 | self.__file_date = '' 607 | return False 608 | 609 | except: 610 | ### DEBUG 611 | traceback.print_exc() 612 | return False 613 | 614 | return True 615 | 616 | return False 617 | --------------------------------------------------------------------------------