├── graph.png ├── requirements.txt ├── scripts └── test.sh ├── .ci.yml ├── .github └── workflows │ └── test.yml ├── .gitignore ├── common.py ├── README.md ├── sargraph.py ├── LICENSE ├── watch.py └── graph.py /graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/antmicro/sargraph/HEAD/graph.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/antmicro/servis 2 | psutil 3 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | tuttest README.md | grep -v '^\$' | bash - 6 | -------------------------------------------------------------------------------- /.ci.yml: -------------------------------------------------------------------------------- 1 | image: debian:bullseye 2 | 3 | before_script: 4 | - apt-get update 5 | - apt-get install -qqy --no-install-recommends python3 python3-pip git colorized-logs 6 | - pip3 install git+https://github.com/antmicro/tuttest.git 7 | - pip3 install bokeh 8 | 9 | simple_test: 10 | script: 11 | - ./scripts/test.sh 12 | - cat plot.ascii | ansi2txt 13 | artifacts: 14 | when: always 15 | paths: 16 | - "plot.svg" 17 | - "plot.png" 18 | - "plot.ascii" 19 | - "plot.html" 20 | - "example.log" 21 | - "example.txt" 22 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Sargraph tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | readme-tests: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout sources 14 | uses: actions/checkout@v3 15 | - name: Install dependencies 16 | run: | 17 | sudo apt-get update 18 | sudo apt-get install -qqy --no-install-recommends python3 python3-pip git colorized-logs 19 | sudo pip3 install git+https://github.com/antmicro/tuttest.git 20 | - name: Run README.md snippets 21 | run: | 22 | sudo ./scripts/test.sh 23 | cat plot.ascii | ansi2txt 24 | echo -en '## Sargraph summary\n\n' >> $GITHUB_STEP_SUMMARY 25 | echo -en '```\n' >> $GITHUB_STEP_SUMMARY 26 | cat plot.ascii | ansi2txt >> $GITHUB_STEP_SUMMARY 27 | echo -en '\n```\n' >> $GITHUB_STEP_SUMMARY 28 | 29 | - name: Upload artifacts 30 | if: always() 31 | uses: actions/upload-artifact@v3 32 | with: 33 | name: sargraph-sample-graphs 34 | path: | 35 | plot.svg 36 | plot.png 37 | plot.ascii 38 | plot.html 39 | example.txt 40 | example.log 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # 4 | # (c) 2019-2023 Antmicro 5 | # License: Apache-2.0 6 | # 7 | 8 | 9 | import os 10 | import subprocess 11 | import sys 12 | import re 13 | import platform 14 | 15 | 16 | # Increase major number for general changes, middle number for smaller changes 17 | # that can cause incompatibilities and minor number for regular fixes 18 | SARGRAPH_VERSION = "2.5.0" 19 | 20 | # Define units for use with unit_str 21 | TIME_UNITS = ['seconds', 'minutes', 'hours'] 22 | DATA_UNITS = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'] 23 | SPEED_UNITS = ['Mb/s', 'Gb/s', 'Tb/s', 'Pb/s'] 24 | 25 | # Print an error message and exit with non-zero status 26 | def fail(msg): 27 | print(f"Error: {msg}", file=sys.stderr) 28 | sys.exit(1) 29 | 30 | 31 | # Run process, return subprocess object on success, exit script on failure 32 | def run_or_fail(*argv, **kwargs): 33 | try: 34 | p = subprocess.Popen(argv, **kwargs) 35 | except: 36 | fail(f"'{argv[0]}' tool not found") 37 | return p 38 | 39 | 40 | # Check if a process is running 41 | def pid_running(pid): 42 | return os.path.exists(f"/proc/{pid}") 43 | 44 | 45 | # Convert a string to float, also when the separator is a comma 46 | def stof(s): 47 | return float(s.replace(',', '.')) 48 | 49 | 50 | # Return a string without given suffix or unchange if it doesn't have it 51 | def cut_suffix(s, sfx): 52 | if s.endswith(sfx): 53 | s = s[:-len(sfx)] 54 | return s 55 | 56 | 57 | # Scale a value until it has a convenient size and unit, round the value 58 | # and return a string representation with the new value and its unit 59 | def unit_str(value, units, step=1024): 60 | value = float(value) 61 | biggest = len(units) - 1 62 | unit = 0 63 | 64 | while value >= step and unit < biggest: 65 | value /= step 66 | unit += 1 67 | return f"{round(value, 2)} {units[unit]}" 68 | 69 | 70 | # Get the first group from a given match and convert to required type 71 | def scan(regex, conv, string): 72 | match = re.search(regex, string) 73 | if not match: 74 | return None 75 | try: 76 | value = conv(match.group(1)) 77 | except ValueError: 78 | return None 79 | return value 80 | 81 | 82 | # Return True iff version string `a` is greater than or equal to `b` 83 | def is_version_ge(a, b): 84 | a = [int(n) for n in a.split('.')] 85 | b = [int(n) for n in b.split('.')] 86 | 87 | if len(a) != len(b): 88 | return len(a) > len(b) 89 | for i, _ in enumerate(a): 90 | if a[i] != b[i]: 91 | break 92 | return a[i] >= b[i] 93 | 94 | def is_darwin(): 95 | return platform.system() == 'Darwin' 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sargraph 2 | 3 | Copyright (c) 2019-2023 [Antmicro](https://www.antmicro.com) 4 | 5 | This is a simple python tool that uses "sysstat" ("sar") to save information on CPU, RAM and disk usage. 6 | The process runs in background and can be controlled with a set of sargraph sub-commands. 7 | 8 | The tool can use "gnuplot" to optionally plot the data it collected. 9 | Supported plot formats are PNG, SVG and ASCII, they are determined by filename extensions. 10 | 11 | # Install requirements 12 | 13 | The sargraph requires `gnuplot`, `sysstat` (`sar`), `python3`, `coreutils` and `screen` to operate. 14 | In Debian you can install them with: 15 | 16 | ``` 17 | # install system dependencies 18 | apt-get update 19 | apt-get install -qqy --no-install-recommends \ 20 | coreutils \ 21 | git \ 22 | gnuplot-nox \ 23 | python3 \ 24 | python3-pip \ 25 | screen \ 26 | sysstat 27 | 28 | # install Python dependencies 29 | pip3 install -r requirements.txt 30 | ``` 31 | 32 | For rendering HTML plots, you additionally have to install: 33 | 34 | ``` 35 | pip3 install git+https://github.com/antmicro/servis#egg=servis[bokeh] 36 | ``` 37 | 38 | # Example graph 39 | 40 | ![graph](graph.png) 41 | 42 | 43 | # Usage 44 | All sargraph commands use the following pattern: 45 | 46 | ``` 47 | $ ./sargraph.py [session_name] [command] [args...] 48 | ``` 49 | 50 | Let's create a sample disk on which we will run stress tests: 51 | 52 | ``` 53 | dd if=/dev/zero of=sample_disk.ext4 bs=1M count=130 54 | mkfs.ext4 sample_disk.ext4 55 | mkdir -p ./mountpoint && mount sample_disk.ext4 ./mountpoint 56 | ``` 57 | 58 | For the purpose of stress tests let's install `stress`: 59 | 60 | ``` 61 | apt-get install stress 62 | ``` 63 | 64 | ## Starting a session 65 | 66 | Start a background session and name it `example`: 67 | ``` 68 | ./sargraph.py example start -m ./mountpoint 69 | ``` 70 | 71 | The data will be saved in `example.txt`. 72 | Logs from screen will be written to `example.log`. 73 | 74 | `-m` flag allows to specify a chosen filesystem/mountpoint. 75 | 76 | ## Adding a label 77 | 78 | Add labels that will be placed as comments in the collected dataset. 79 | They will be also visible on the plots: 80 | ``` 81 | ./sargraph.py example label "Compilation start" 82 | ``` 83 | 84 | After this, let's simulate some processing: 85 | 86 | ``` 87 | pushd ./mountpoint 88 | stress -c 16 -i 1 -m 1 --vm-bytes 512M -d 1 --hdd-bytes 70M -t 160s 89 | popd 90 | ``` 91 | 92 | ## Plotting a running session 93 | 94 | It is possible to plot data collected so far in a still running session: 95 | ``` 96 | # Save plot to SVG 97 | ./sargraph.py example save plot.svg 98 | # Save plot to PNG 99 | ./sargraph.py example save plot.png 100 | # Save plot to ASCII that can be printed in terminal 101 | ./sargraph.py example save plot.ascii 102 | # Save plot to interactive HTML plot 103 | ./sargraph.py example save plot.html 104 | ``` 105 | 106 | The supported formats are: 107 | 108 | * `png` format 109 | * `svg` format 110 | * `html` format 111 | * `ascii` format - plot is rendered to text file that can be displayed in terminal 112 | 113 | ## Stopping a session 114 | 115 | Stop a session and create a final `plot.png` plot file if no other plot was created so far: 116 | ``` 117 | ./sargraph.py example stop 118 | ``` 119 | 120 | The filename of the final plot can be changed if its placed after the `stop` command. 121 | If the name is `none` then no plot will be created. 122 | 123 | ## Plotting a closed session 124 | 125 | Plot data collected in a session that is not running anymore. 126 | ``` 127 | ./sargraph.py example plot plot.ascii 128 | ``` 129 | The command requires the `example.txt` log file to be present in the working directory. 130 | -------------------------------------------------------------------------------- /sargraph.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # 4 | # (c) 2019-2023 Antmicro 5 | # License: Apache-2.0 6 | # 7 | 8 | import argparse 9 | import sys 10 | import time 11 | 12 | import graph 13 | import watch 14 | 15 | from common import * 16 | 17 | # Declare and parse command line flags 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('session', metavar='SESSION-NAME', type=str, nargs='?', default=None, help='sargraph session name') 20 | parser.add_argument('command', metavar='COMMAND', type=str, nargs='*', help='send command') 21 | parser.add_argument('-f', metavar='DEVICE-NAME', type=str, nargs='?', default=None, dest='fsdev', help='observe a chosen filesystem') 22 | parser.add_argument('-m', metavar='MOUNT-DIR', type=str, nargs='?', default=None, dest='fspath', help='observe a chosen filesystem') 23 | parser.add_argument('-n', metavar='IFACE-NAME', type=str, nargs='?', default=None, dest='iface', help='observe chosen network iface') 24 | parser.add_argument('-o', metavar='OUTPUT-NAME', type=str, nargs='?', default='data', dest='name', help='set output base names') 25 | parser.add_argument('-t', metavar='TMPFS-COLOR', type=str, nargs='?', default='#f2c71b', dest='tmpfs', help='set tmpfs plot color' ) 26 | parser.add_argument('-c', metavar='CACHE-COLOR', type=str, nargs='?', default='#ee7af0', dest='cache', help='set cache plot color' ) 27 | parser.add_argument('-u', metavar='UDP', type=str, nargs='?', default=None, dest='udp', help='set udp server address') 28 | parser.add_argument('-C', metavar='UDP_COOKIE', type=str, nargs='?', default=None, dest='udp_cookie', help='set udp message cookie') 29 | parser.add_argument('-p', action='store_true', dest='psutil', help='use psutil instead of sar') 30 | args = parser.parse_args() 31 | 32 | def send(sid, msg): 33 | p = subprocess.Popen(["screen", "-S", sid, "-X", "stuff", f"{msg}\n"]) 34 | while p.poll() is None: 35 | time.sleep(0.1) 36 | 37 | # Check if sar is available 38 | if not is_darwin(): 39 | p = run_or_fail("sar", "-V", stdout=subprocess.PIPE) 40 | 41 | # Check if screen is available 42 | p = run_or_fail("screen", "-v", stdout=subprocess.PIPE) 43 | version = scan("Screen version (\\d+)", int, p.stdout.readline().decode()) 44 | if version is None: 45 | fail("'screen' tool returned unknown output") 46 | 47 | # If the script was run with no parameters, run in background and gather data 48 | if args.session is None: 49 | # Find requested disk device 50 | if args.fspath: 51 | args.fspath = os.path.realpath(args.fspath) 52 | with open("/proc/self/mounts", "r") as f: 53 | while args.fsdev is None: 54 | args.fsdev = scan(f"^(/dev/\\S+)\\s+{re.escape(args.fspath)}\\s+", str, f.readline()) 55 | if not args.fsdev: 56 | fail(f"no device is mounted on {args.fspath}") 57 | 58 | watch.watch(args.name, args.fsdev, args.iface, args.tmpfs, args.cache, args.psutil, args.udp, args.udp_cookie) 59 | sys.exit(0) 60 | 61 | # Now handle the commands 62 | 63 | # Check if a command was provided 64 | if len(args.command) <= 0: 65 | fail("command not provided") 66 | 67 | # Get session name and command name 68 | sid = args.session 69 | cmd = args.command 70 | 71 | if cmd[0] == "start": 72 | print(f"Starting sargraph session '{sid}'") 73 | 74 | # Spawn watcher process, *sys.argv[3:] is all arguments after 'chart start' + '-o [log name]' if not given 75 | if "-o" not in sys.argv: 76 | sys.argv += ["-o", sid] 77 | p = subprocess.Popen(["screen", "-Logfile", f"{sid}.log", "-dmSL", sid, os.path.realpath(__file__), *sys.argv[3:]]) 78 | 79 | while p.poll() is None: 80 | time.sleep(0.1) 81 | gpid = 0 82 | j = 0 83 | time.sleep(1) 84 | print(f"Session '{sid}' started") 85 | elif cmd[0] == "stop": 86 | print(f"Terminating sargraph session '{sid}'") 87 | 88 | try: 89 | gpid = int(os.popen(f"screen -ls | grep '.{sid}' | tr -d ' \t' | cut -f 1 -d '.'").read()) 90 | except: 91 | print("Warning: cannot find pid.") 92 | gpid = -1 93 | if len(cmd) < 2: 94 | send(sid, "command:q:") 95 | else: 96 | send(sid, f"command:q:{cmd[1]}") 97 | if gpid == -1: 98 | print("Waiting 3 seconds.") 99 | time.sleep(3) 100 | else: 101 | while pid_running(gpid): 102 | time.sleep(0.25) 103 | elif cmd[0] == "label": 104 | # Check if the label name was provided 105 | if len(cmd) < 2: 106 | fail("label command requires an additional parameter") 107 | print(f"Adding label '{cmd[1]}' to sargraph session '{sid}'.") 108 | send(sid, f"label:{cmd[1]}") 109 | elif cmd[0] == 'save': 110 | print(f"Saving graph from session '{sid}'.") 111 | if len(cmd) < 2: 112 | send(sid, "command:s:") 113 | else: 114 | send(sid, f"command:s:{cmd[1]}") 115 | elif cmd[0] == 'plot': 116 | if len(cmd) < 2: 117 | graph.graph(sid, args.tmpfs, args.cache) 118 | else: 119 | graph.graph(sid, args.tmpfs, args.cache, cmd[1]) 120 | else: 121 | fail(f"unknown command '{cmd[0]}'") 122 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /watch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # 4 | # (c) 2019-2023 Antmicro 5 | # License: Apache-2.0 6 | # 7 | 8 | 9 | import datetime 10 | import fcntl 11 | import os 12 | import re 13 | import select 14 | import signal 15 | import subprocess 16 | import sys 17 | import time 18 | import psutil 19 | import sched 20 | import platform 21 | import logging 22 | from threading import Thread, Lock 23 | import threading 24 | from logging.handlers import DatagramHandler 25 | 26 | import graph 27 | 28 | from common import * 29 | 30 | die = 0 31 | 32 | 33 | # Initialize summary variables 34 | SAMPLE_NUMBER = 0 35 | TOTAL_RAM = 0 36 | START_DATE = "" 37 | END_DATE = "" 38 | TOTAL_LOAD = 0.0 39 | MAX_USED_RAM = 0 40 | MAX_USED_FS = 0 41 | TOTAL_FS = 0 42 | MAX_TX = 0 43 | MAX_RX = 0 44 | START_TX = 0 45 | START_RX = 0 46 | END_TX = 0 47 | END_RX = 0 48 | 49 | TOTAL_GPU_LOAD = 0.0 50 | TOTAL_GPU_RAM = 0 51 | MAX_USED_GPU_RAM = 0 52 | 53 | FS_NAME = None 54 | FS_SAR_INDEX = None 55 | 56 | IFACE_NAME = None 57 | IFACE_SAR_INDEX = None 58 | 59 | # Handle SIGTERM 60 | def kill_handler(a, b): 61 | global die 62 | die = 1 63 | 64 | class UDPHandler(DatagramHandler): 65 | def emit(self, msg): 66 | try: 67 | if self.sock is None: 68 | self.createSocket() 69 | self.sock.sendto(self.format(msg).encode(), self.address) 70 | except Exception as e: 71 | pass 72 | 73 | 74 | logger = logging.getLogger("sargraph") 75 | logger.setLevel(logging.INFO) 76 | 77 | # Read a single table from sar output 78 | def read_table(psar): 79 | # Find the header 80 | f = psar.stdout 81 | while True: 82 | header = f.readline().decode().split() 83 | if len(header) > 0: 84 | break 85 | if psar.poll() is not None: 86 | raise ValueError("The subprocess has exited") 87 | 88 | # The first columns is always just time 89 | header[0] = 'time' 90 | 91 | table = {} 92 | for title in header: 93 | table[title] = [] 94 | 95 | # Read rows 96 | while True: 97 | row = f.readline().decode().split() 98 | if len(row) <= 0: 99 | break 100 | if psar.poll() is not None: 101 | raise ValueError("The subprocess has exited") 102 | 103 | for i, value in enumerate(row): 104 | table[header[i]].append(value) 105 | 106 | return table 107 | 108 | 109 | # Read received/sent bytes from a given interface's sys stats 110 | def read_iface_stats(iface): 111 | with open(f"/sys/class/net/{iface}/statistics/rx_bytes") as f: 112 | rx = scan(r"(\d+)", int, f.readline()) 113 | with open(f"/sys/class/net/{iface}/statistics/tx_bytes") as f: 114 | tx = scan(r"(\d+)", int, f.readline()) 115 | return rx, tx 116 | 117 | 118 | # Initialize 'data.txt' where the data is dumped 119 | def initialize(session, machine): 120 | global TOTAL_RAM 121 | global TOTAL_GPU_RAM 122 | 123 | with open("/proc/meminfo") as f: 124 | TOTAL_RAM = int(scan("MemTotal:\s+(\d+)", float, f.read())) 125 | 126 | uname = machine.split(" ")[0:2] 127 | uname = f"{uname[0]} {uname[1]}" 128 | 129 | cpus = int(machine.split(" CPU)")[0].split("(")[-1]) 130 | 131 | cpu_name = "unknown" 132 | 133 | with open("/proc/cpuinfo") as f: 134 | for line in f: 135 | if "model name" in line: 136 | cpu_name = line.replace("\n", "").split(": ")[1] 137 | break 138 | header = [ 139 | f"# sargraph version: {SARGRAPH_VERSION}", 140 | f"pid: {os.getpid()}", 141 | f"machine: {uname}", 142 | f"cpu count: {cpus}", 143 | f"cpu: {cpu_name}" 144 | ] 145 | try: 146 | pgpu = subprocess.run( 147 | 'nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader,nounits'.split(' '), 148 | capture_output=True 149 | ) 150 | if pgpu.returncode == 0: 151 | gpuname, gpudriver, memory_total = pgpu.stdout.decode('utf-8').rsplit(', ', 2) 152 | header.extend([ 153 | f"gpu: {gpuname}", 154 | f"gpu driver: {gpudriver}" 155 | ]) 156 | TOTAL_GPU_RAM = int(memory_total) 157 | except Exception as e: 158 | print(e) 159 | pass 160 | 161 | logger.info(", ".join(header)) 162 | 163 | def initialize_darwin(session): 164 | global TOTAL_RAM 165 | global TOTAL_GPU_RAM 166 | 167 | TOTAL_RAM = int(psutil.virtual_memory().total / 1024) 168 | 169 | cpus = psutil.cpu_count(logical=True) 170 | 171 | cpu_name = platform.processor() or "unknown" 172 | 173 | header = [ 174 | f"# psutil version: {psutil.__version__}", 175 | f"pid: {os.getpid()}", 176 | f"machine: {platform.system()}", 177 | f"cpu count: {cpus}", 178 | f"cpu: {cpu_name}" 179 | ] 180 | logger.info(", ".join(header)) 181 | 182 | 183 | # Add a summary comment to 'data.txt' 184 | def summarize(session): 185 | # Is there anything to be summarized? 186 | if SAMPLE_NUMBER == 0: 187 | return 188 | 189 | average_load = TOTAL_LOAD / float(SAMPLE_NUMBER) 190 | max_used_ram = MAX_USED_RAM * 1024.0 191 | total_ram = TOTAL_RAM * 1024.0 192 | max_used_fs = MAX_USED_FS * 1024.0 * 1024.0 193 | total_fs = TOTAL_FS * 1024 * 1024 194 | max_tx = MAX_TX / 128 # kB/s to Mb/s 195 | max_rx = MAX_RX / 128 # kB/s to Mb/s 196 | total_tx = END_TX-START_TX 197 | total_rx = END_RX-START_RX 198 | 199 | sdt = datetime.datetime.strptime(START_DATE, '%Y-%m-%d %H:%M:%S') 200 | edt = datetime.datetime.strptime(END_DATE, '%Y-%m-%d %H:%M:%S') 201 | delta_t = (edt - sdt).total_seconds() 202 | 203 | summary = [ 204 | f"# total ram: {total_ram:.2f} B", 205 | f"total disk space: {total_fs:.2f} B", 206 | f"max ram used: {max_used_ram:.2f} B", 207 | f"max disk used: {max_used_fs:.2f} B", 208 | f"average load: {average_load:.2f} %", 209 | f"observed disk: {FS_NAME}", 210 | f"max received: {max_rx:.2f} Mb/s", 211 | f"max sent: {max_tx:.2f} Mb/s", 212 | f"observed network: {IFACE_NAME}", 213 | f"duration: {delta_t} seconds", 214 | f"total received: {total_rx} b", 215 | f"total sent: {total_tx} b" 216 | ] 217 | 218 | if TOTAL_GPU_RAM != 0: 219 | summary.extend([ 220 | f"total gpu ram: {TOTAL_GPU_RAM * 1024 * 1024:.2f} B", # default units are MiB 221 | f"max gpu ram used: {MAX_USED_GPU_RAM * 1024 * 1024:.2f} B", # default units are MiB 222 | f"average gpu load: {TOTAL_GPU_LOAD / SAMPLE_NUMBER:.2f} %" 223 | ]) 224 | 225 | logger.info(", ".join([str(i) for i in summary])) 226 | 227 | def get_meminfo(scheduler): 228 | global MAX_USED_RAM 229 | scheduler.enter(0.1, 1, get_meminfo, (scheduler,)) 230 | now = datetime.datetime.now() 231 | date = now.strftime("%Y-%m-%d") 232 | daytime = now.strftime("%H:%M:%S.%f") 233 | ram_data = psutil.virtual_memory() 234 | used = (ram_data.total - ram_data.free) 235 | if used // 1024 > MAX_USED_RAM: 236 | MAX_USED_RAM = used // 1024 237 | if is_darwin(): 238 | line = [ 239 | date + "-" + daytime, 240 | 100 * ram_data.free / ram_data.total, 241 | 0, 242 | 100 * used / ram_data.total, 243 | 0 244 | ] 245 | else: 246 | line = [ 247 | date + "-" + daytime, 248 | 100 * ram_data.free / ram_data.total, 249 | 100 * ram_data.cached / ram_data.total, 250 | 100 * ram_data.used / ram_data.total, 251 | 100 * ram_data.shared / ram_data.total 252 | ] 253 | msg = " ".join(["psu"]+[str(i) for i in line]) 254 | logger.info(msg) 255 | 256 | 257 | def watch(session, fsdev, iface, tmpfs_color, other_cache_color, use_psutil, udp=None, udp_cookie=None): 258 | file_handler = logging.FileHandler(f"{session}.txt") 259 | file_handler.setFormatter(logging.Formatter("%(message)s")) 260 | logger.addHandler(file_handler) 261 | 262 | if udp is not None: 263 | spl = udp.rsplit(':', 1) 264 | udp_handler = UDPHandler(spl[0], int(spl[1])) 265 | if udp_cookie is None: 266 | udp_handler.setFormatter(logging.Formatter("%(message)s\n")) 267 | else: 268 | udp_handler.setFormatter(logging.Formatter(f"[{udp_cookie}] %(message)s\n")) 269 | logger.addHandler(udp_handler) 270 | 271 | if is_darwin() or use_psutil: 272 | return watch_psutil(session, fsdev, iface, tmpfs_color, other_cache_color) 273 | return watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color) 274 | 275 | # Run sar and gather data from it 276 | def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color): 277 | global SAMPLE_NUMBER 278 | global START_DATE 279 | global END_DATE 280 | global TOTAL_LOAD 281 | global MAX_USED_RAM 282 | global MAX_USED_FS 283 | global MAX_RX 284 | global MAX_TX 285 | global TOTAL_FS 286 | global START_RX 287 | global START_TX 288 | global END_RX 289 | global END_TX 290 | global TOTAL_RAM 291 | global FS_SAR_INDEX 292 | global FS_NAME 293 | global IFACE_NAME 294 | global IFACE_SAR_INDEX 295 | global TOTAL_GPU_LOAD 296 | global TOTAL_GPU_RAM 297 | global MAX_USED_GPU_RAM 298 | 299 | global die 300 | 301 | # Was a graph alreay produced by save command from sargraph? 302 | dont_plot = False 303 | 304 | my_env = os.environ 305 | my_env["S_TIME_FORMAT"] = "ISO" 306 | 307 | psar = run_or_fail("sar", "-F", "-u", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env) 308 | 309 | s = sched.scheduler(time.time, time.sleep) 310 | mem_ev = s.enter(0, 1, get_meminfo, (s,)) 311 | thread = Thread(target = s.run) 312 | thread.start() 313 | 314 | # subprocess for GPU data fetching in the background 315 | try: 316 | pgpu = subprocess.Popen( 317 | 'nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader,nounits -l 1'.split(' '), 318 | stdout=subprocess.PIPE, 319 | env=my_env 320 | ) 321 | except: 322 | pgpu = None 323 | 324 | machine = psar.stdout.readline().decode() 325 | initialize(session, machine) 326 | psar.stdout.readline() 327 | 328 | signal.signal(signal.SIGTERM, kill_handler) 329 | 330 | # Make stdin nonblocking to continue working when no command is sent 331 | flags = fcntl.fcntl(sys.stdin, fcntl.F_GETFL) 332 | fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flags | os.O_NONBLOCK) 333 | 334 | # Gather data from sar output 335 | curr_gpu_util = 0 336 | curr_gpu_mem = 0 337 | 338 | while 1: 339 | # Await sar output or a command sent from command handler in sargraph.py 340 | readlist = [psar.stdout, sys.stdin] 341 | if pgpu: 342 | readlist.append(pgpu.stdout) 343 | rlist, _, _ = select.select(readlist, [], [], 0.25) 344 | now = datetime.datetime.now() 345 | if sys.stdin in rlist: 346 | if handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now): 347 | break 348 | if psar.stdout not in rlist: 349 | continue 350 | 351 | date = now.strftime("%Y-%m-%d") 352 | daytime = now.strftime("%H:%M:%S") 353 | 354 | # Read and process CPU data 355 | try: 356 | cpu_data = read_table(psar) 357 | if START_DATE == "": 358 | START_DATE = date + " " + daytime 359 | TOTAL_LOAD += stof(cpu_data["%user"][0]) 360 | SAMPLE_NUMBER += 1 361 | 362 | if TOTAL_RAM == 0: 363 | TOTAL_RAM = psutil.virtual_memory().total // 1024 364 | 365 | # Read and process network data 366 | net_data = read_table(psar) 367 | if IFACE_SAR_INDEX is None: 368 | if iface: 369 | IFACE_SAR_INDEX = net_data['IFACE'].index(iface) 370 | else: 371 | maxj, maxv = 0, 0 372 | for j, used in enumerate(net_data['IFACE']): 373 | v = stof(net_data['rxkB/s'][j]) 374 | if maxv < v: 375 | maxj, maxv = j, v 376 | IFACE_SAR_INDEX = maxj 377 | if IFACE_NAME is None: 378 | IFACE_NAME = net_data['IFACE'][IFACE_SAR_INDEX] 379 | if START_RX <= 0 or START_TX <= 0: 380 | START_RX, START_TX = read_iface_stats(IFACE_NAME) 381 | END_RX, END_TX = read_iface_stats(IFACE_NAME) 382 | if MAX_RX < stof(net_data['rxkB/s'][IFACE_SAR_INDEX]): 383 | MAX_RX = stof(net_data['rxkB/s'][IFACE_SAR_INDEX]) 384 | if MAX_TX < stof(net_data['txkB/s'][IFACE_SAR_INDEX]): 385 | MAX_TX = stof(net_data['txkB/s'][IFACE_SAR_INDEX]) 386 | 387 | # Read and process FS data 388 | fs_data = read_table(psar) 389 | if FS_SAR_INDEX is None: 390 | if fsdev: 391 | FS_SAR_INDEX = fs_data['FILESYSTEM'].index(fsdev) 392 | else: 393 | maxj, maxv = 0, 0 394 | for j, free in enumerate(fs_data['MBfsfree']): 395 | v = stof(fs_data['MBfsfree'][j]) + stof(fs_data['MBfsused'][j]) 396 | # Skip shared memory device 397 | if fs_data["FILESYSTEM"][j] == "/dev/shm": 398 | continue 399 | if maxv < v: 400 | maxj, maxv = j, v 401 | FS_SAR_INDEX = maxj 402 | if FS_NAME is None: 403 | FS_NAME = fs_data["FILESYSTEM"][FS_SAR_INDEX] 404 | if TOTAL_FS == 0: 405 | TOTAL_FS = (stof(fs_data['MBfsused'][FS_SAR_INDEX]) + stof(fs_data['MBfsfree'][FS_SAR_INDEX])) 406 | if MAX_USED_FS < int(fs_data['MBfsused'][FS_SAR_INDEX]): 407 | MAX_USED_FS = int(fs_data['MBfsused'][FS_SAR_INDEX]) 408 | 409 | END_DATE = date + " " + daytime 410 | timestamp = date + "-" + daytime 411 | except ValueError as e: 412 | print("Sar process has exited - quitting sargraph") 413 | break 414 | 415 | if pgpu and pgpu.stdout in rlist: 416 | line = pgpu.stdout.readline().decode('utf-8') 417 | if pgpu.poll() is not None: 418 | print("nvidia-smi stopped working, reason:") 419 | print(line) 420 | print(f"Error code: {pgpu.returncode}") 421 | print("Closing the GPU statistics collection") 422 | pgpu = None 423 | else: 424 | try: 425 | curr_gpu_util, curr_gpu_mem = [ 426 | int(val.strip()) for val in line.split(', ') 427 | ] 428 | if MAX_USED_GPU_RAM < curr_gpu_mem: 429 | MAX_USED_GPU_RAM = curr_gpu_mem 430 | TOTAL_GPU_LOAD += curr_gpu_util 431 | except ValueError: 432 | print(f"nvidia-smi error readout: {line}") 433 | if "Unknown Error" in line: 434 | # No valid readouts from now on, let's terminate current nvidia-smi session 435 | pgpu.terminate() 436 | pgpu = None 437 | 438 | line = [ 439 | timestamp, 440 | cpu_data['%user'][0], 441 | fs_data['%fsused'][FS_SAR_INDEX], 442 | stof(net_data['rxkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s 443 | stof(net_data['txkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s 444 | ] 445 | if pgpu and TOTAL_GPU_RAM != 0: 446 | line.extend([ 447 | f'{curr_gpu_util:.2f}', 448 | f'{curr_gpu_mem / TOTAL_GPU_RAM * 100.0:.2f}' 449 | ]) 450 | logger.info(" ".join(["sar"]+[str(i) for i in line])) 451 | 452 | if die: 453 | break 454 | 455 | list(map(s.cancel, s.queue)) 456 | thread.join() 457 | 458 | # This runs if we were stopped by SIGTERM and no plot was made so far 459 | if not dont_plot: 460 | summarize(session) 461 | graph.graph(session, tmpfs_color, other_cache_color) 462 | 463 | def watch_psutil(session, fsdev, iface, tmpfs_color, other_cache_color): 464 | # Was a graph already produced by save command from sargraph? 465 | dont_plot = False 466 | 467 | s = sched.scheduler(time.time, time.sleep) 468 | sar_ev = s.enter(0, 1, psutil_sar_simulation, (s,)) 469 | mem_ev = s.enter(0, 1, get_meminfo, (s,)) 470 | thread = Thread(target = s.run) 471 | thread.start() 472 | 473 | 474 | initialize_darwin(session) 475 | signal.signal(signal.SIGTERM, kill_handler) 476 | 477 | # Make stdin nonblocking to continue working when no command is sent 478 | flags = fcntl.fcntl(sys.stdin, fcntl.F_GETFL) 479 | fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flags | os.O_NONBLOCK) 480 | 481 | 482 | while 1: 483 | # Await sar output or a command sent from command handler in sargraph.py 484 | readlist = [sys.stdin] 485 | rlist, _, _ = select.select(readlist, [], [], 0.25) 486 | now = datetime.datetime.now() 487 | if handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now): 488 | break 489 | list(map(s.cancel, s.queue)) 490 | thread.join() 491 | 492 | # This runs if we were stopped by SIGTERM and no plot was made so far 493 | if not dont_plot: 494 | summarize(session) 495 | graph.graph(session, tmpfs_color, other_cache_color) 496 | 497 | def handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now): 498 | global die 499 | label_line = sys.stdin.readline().replace("\n", "") 500 | if label_line.startswith("command:"): 501 | label_line = label_line[len("command:"):] 502 | if label_line.startswith("q:"): 503 | label_line = label_line[len("q:"):] 504 | 505 | list(map(s.cancel, s.queue)) 506 | summarize(session) 507 | if label_line == "none": 508 | pass 509 | elif label_line: 510 | graph.graph(session, tmpfs_color, other_cache_color, label_line) 511 | elif not dont_plot: 512 | graph.graph(session, tmpfs_color, other_cache_color) 513 | dont_plot = True 514 | die = 1 515 | return True 516 | elif label_line.startswith("s:"): 517 | label_line = label_line[len("s:"):] 518 | 519 | dont_plot = True 520 | 521 | if label_line != "none": 522 | summarize(session) 523 | if not label_line: 524 | graph.graph(session, tmpfs_color, other_cache_color) 525 | else: 526 | graph.graph(session, tmpfs_color, other_cache_color, label_line) 527 | elif label_line.startswith('label:'): 528 | label_line = label_line[len('label:'):] 529 | with open(f"{session}.txt", "a") as f: 530 | timestamp = now.strftime("%Y-%m-%d-%H:%M:%S") 531 | print(f"# {timestamp} label: {label_line}", file=f) 532 | return False 533 | 534 | # sar is not available on macOS. This function creates the sar behavior, but use psutil instead. 535 | def psutil_sar_simulation(scheduler): 536 | global START_DATE 537 | global TOTAL_LOAD 538 | global SAMPLE_NUMBER 539 | global TOTAL_RAM 540 | global START_RX 541 | global START_TX 542 | global END_TX 543 | global END_RX 544 | global MAX_RX 545 | global MAX_TX 546 | global IFACE_NAME 547 | global TOTAL_FS 548 | global MAX_USED_FS 549 | global FS_NAME 550 | global END_DATE 551 | 552 | scheduler.enter(1, 1, psutil_sar_simulation, (scheduler,)) 553 | now = datetime.datetime.now() 554 | date = now.strftime("%Y-%m-%d") 555 | daytime = now.strftime("%H:%M:%S") 556 | if START_DATE == "": 557 | START_DATE = date + " " + daytime 558 | cpu_used = psutil.cpu_percent() 559 | TOTAL_LOAD += cpu_used 560 | SAMPLE_NUMBER += 1 561 | if TOTAL_RAM == 0: 562 | TOTAL_RAM = psutil.virtual_memory().total // 1024 563 | IFACE_NAME = "all" 564 | net_stats = psutil.net_io_counters() 565 | if START_RX <= 0 or START_TX <= 0: 566 | START_RX, START_TX = net_stats.bytes_recv, net_stats.bytes_sent 567 | END_RX, END_TX = net_stats.bytes_recv, net_stats.bytes_sent 568 | curr_rx, curr_tx = (net_stats.bytes_recv - END_RX) / (1024 * 8), (net_stats.bytes_sent - END_TX) / (1024 * 8) 569 | END_RX, END_TX = net_stats.bytes_recv, net_stats.bytes_sent 570 | if MAX_RX < curr_rx: 571 | MAX_RX = curr_rx 572 | if MAX_TX < curr_tx: 573 | MAX_TX = curr_tx 574 | # apfs implements lvm, so it's a better option for visualizing the place in the container (which is shared by all partitions). 575 | if is_darwin(): 576 | FS_NAME = "apfs container" 577 | disk_stats = psutil.disk_usage('/') 578 | else: 579 | largest_partition = max( 580 | psutil.disk_partitions(all=False), 581 | key=lambda p: psutil.disk_usage(p.mountpoint).total 582 | ) 583 | disk_stats = psutil.disk_usage(largest_partition.mountpoint) 584 | FS_NAME = largest_partition.device 585 | 586 | curr_used = (disk_stats.total - disk_stats.free) / (1024 * 1024) 587 | if TOTAL_FS == 0: 588 | TOTAL_FS = disk_stats.total / (1024 * 1024) 589 | if MAX_USED_FS < curr_used: 590 | MAX_USED_FS = curr_used 591 | END_DATE = date + " " + daytime 592 | timestamp = date + "-" + daytime 593 | 594 | line = [ 595 | timestamp, 596 | cpu_used, 597 | ((disk_stats.total - disk_stats.free) / disk_stats.total) * 100, 598 | curr_rx / 128, 599 | curr_tx / 128, 600 | ] 601 | 602 | logger.info(" ".join(["sar"]+[str(i) for i in line])) 603 | -------------------------------------------------------------------------------- /graph.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # 4 | # (c) 2019-2023 Antmicro 5 | # License: Apache-2.0 6 | # 7 | 8 | 9 | import datetime 10 | import os 11 | import socket 12 | import subprocess 13 | import time 14 | from common import * 15 | from pathlib import Path 16 | import tempfile 17 | 18 | global gnuplot 19 | 20 | GNUPLOT_VERSION_EXPECTED = "5.0" 21 | 22 | # Every summary variable requires a default value in case it missed in a session log 23 | START_DATE = "" 24 | END_DATE = "" 25 | AVERAGE_LOAD = 0.0 26 | MAX_USED_RAM = 0 27 | MAX_USED_FS = 0 28 | MAX_TX = 0 29 | MAX_RX = 0 30 | TOTAL_TX = 0 31 | TOTAL_RX = 0 32 | TOTAL_RAM = 0 33 | TOTAL_FS = 0 34 | NAME_FS = "unknown" 35 | NAME_IFACE = "unknown" 36 | 37 | UNAME = "unknown" 38 | CPUS = 0 39 | CPU_NAME = "unknown" 40 | DURATION = 0.0 41 | 42 | GPU_NAME = None 43 | GPU_DRIVER = None 44 | AVERAGE_GPU_LOAD = 0 45 | TOTAL_GPU_RAM = 0 46 | MAX_USED_GPU_RAM = 0 47 | 48 | HOST = socket.gethostname() 49 | 50 | # The number of plots on the graph 51 | NUMBER_OF_PLOTS = 5 52 | RAM_DATA_POSITION = 1 53 | 54 | # The default format 55 | OUTPUT_TYPE = "pngcairo" 56 | OUTPUT_EXT = "png" 57 | 58 | labels = [] 59 | 60 | 61 | # Check if the avaliable gnuplot has a required version 62 | p = run_or_fail("gnuplot", "--version", stdout=subprocess.PIPE) 63 | version = scan("gnuplot (\\S+)", str, p.stdout.readline().decode()) 64 | if not is_version_ge(version, GNUPLOT_VERSION_EXPECTED): 65 | fail( 66 | f"gnuplot version too low. Need at least {GNUPLOT_VERSION_EXPECTED} found {version}") 67 | 68 | def split_data_file(session): 69 | sar_data = [] 70 | psu_data = [] 71 | 72 | # Read the input file 73 | with open(f"{session}.txt", 'r') as file: 74 | in_summary = False 75 | for line in file: 76 | if line.startswith('#'): 77 | sar_data.append(line.strip()) 78 | else: 79 | if line.startswith('sar'): 80 | sar_data.append(line.split(' ', 1)[1].strip()) 81 | elif line.startswith('psu'): 82 | psu_data.append(line.split(' ', 1)[1].strip()) 83 | 84 | temp_dir = tempfile.mkdtemp() 85 | with open(os.path.join(temp_dir, 'sar_data.txt'), 'w') as sar_data_file: 86 | sar_data_file.write("\n".join(sar_data)) 87 | print(file=sar_data_file) 88 | 89 | with open(os.path.join(temp_dir, 'psu_data.txt'), 'w') as psu_data_file: 90 | psu_data_file.write("\n".join(psu_data)) 91 | print(file=psu_data_file) 92 | 93 | # in order: sar file, mem file 94 | return [ 95 | os.path.join(temp_dir, 'sar_data.txt'), 96 | os.path.join(temp_dir, 'psu_data.txt') 97 | ] 98 | 99 | 100 | # Run a command in a running gnuplot process 101 | def g(command): 102 | global gnuplot 103 | 104 | if not (gnuplot.poll() is None): 105 | print("Error: gnuplot not running!") 106 | return 107 | # print ("gnuplot> %s" % command) 108 | try: 109 | command = b"%s\n" % command 110 | except: 111 | command = b"%s\n" % str.encode(command) 112 | gnuplot.stdin.write(b"%s\n" % command) 113 | gnuplot.stdin.flush() 114 | 115 | if command == b"quit\n": 116 | while gnuplot.poll() is None: 117 | time.sleep(0.25) 118 | 119 | 120 | # Get gnuplot font size with respect to differences betwen SVG and PNG terminals 121 | def fix_size(size): 122 | if OUTPUT_TYPE == "svg": 123 | size = int(size*1.25) 124 | return size 125 | 126 | 127 | # Plot a single column of values from data.txt 128 | def plot(ylabel, title, sar_file, column, space=3, autoscale=None): 129 | if autoscale is None: 130 | g("set yrange [0:100]") 131 | g("set cbrange [0:100]") 132 | else: 133 | g("unset xdata") 134 | g("set yrange [0:*]") 135 | g(f"stats '{sar_file}' using {column}") 136 | g(f"set yrange [0:STATS_max*{autoscale}]") 137 | g(f"set cbrange [0:STATS_max*{autoscale}]") 138 | g("set xdata time") 139 | g(f"set ylabel '{ylabel}'") 140 | g(f"set title \"{{/:Bold {title}}}" + ("\\n" * space) + "\"") 141 | g(f"plot '{sar_file}' using 1:{column}:{column} title 'cpu' with boxes palette") 142 | 143 | def plot_stacked(ylabel, title, ram_file, column, tmpfs_color, other_cache_color, space=3, autoscale=None): 144 | if autoscale is None: 145 | g("set yrange [0:100]") 146 | g("set cbrange [0:100]") 147 | else: 148 | g("unset xdata") 149 | g("set yrange [0:*]") 150 | g(f"stats '{ram_data}' using {column}") 151 | g(f"set yrange [0:STATS_max*{autoscale}]") 152 | g(f"set cbrange [0:STATS_max*{autoscale}]") 153 | g("set xdata time") 154 | g(f"set ylabel '{ylabel}'") 155 | g(f"set title \"{{/:Bold {title}}}" + ("\\n" * space) + "\"") 156 | g('set style data histograms') 157 | g('set style histogram rowstacked') 158 | g('set key reverse below Left width -25') 159 | if is_darwin(): 160 | g(f"plot '{ram_file}' using 1:($3 + ${column}):{column} title 'RAM' with boxes palette") 161 | else: 162 | g(f"plot '{ram_file}' using 1:($3 + ${column}):{column} title 'RAM' with boxes palette, \ 163 | '' using 1:5 with boxes title 'Shared mem' lc rgb '{tmpfs_color}', \ 164 | '' using 1:($3 - $5) with boxes title 'Other cache (freed automatically)' lc rgb '{other_cache_color}'") 165 | g('unset key') 166 | 167 | # Read additional information from 'data.txt' comments 168 | def read_comments(sar_file): 169 | global START_DATE 170 | global END_DATE 171 | global AVERAGE_LOAD 172 | global MAX_USED_RAM 173 | global MAX_USED_FS 174 | global TOTAL_RAM 175 | global TOTAL_FS 176 | global NAME_FS 177 | global UNAME 178 | global CPUS 179 | global CPU_NAME 180 | global DURATION 181 | global MAX_RX 182 | global MAX_TX 183 | global TOTAL_RX 184 | global TOTAL_TX 185 | global NAME_IFACE 186 | global GPU_NAME 187 | global GPU_DRIVER 188 | global AVERAGE_GPU_LOAD 189 | global TOTAL_GPU_RAM 190 | global MAX_USED_GPU_RAM 191 | global NUMBER_OF_PLOTS 192 | 193 | data_version = None 194 | 195 | with open(sar_file, "r") as f: 196 | for line in f: 197 | value = None 198 | 199 | if len(line) <= 0: 200 | continue 201 | 202 | if line[0] != '#': 203 | if not START_DATE: 204 | START_DATE = scan("^(\\S+)", str, line) 205 | END_DATE = scan("^(\\S+)", str, line) 206 | 207 | value = scan("label: (.+)", str, line) 208 | if value is not None: 209 | key = scan("(\\S+) label:", str, line) 210 | labels.append([key, value]) 211 | 212 | # Comments are not mixed with anything else, so skip 213 | continue 214 | 215 | # Override summary variables. If they're missing, their default values are kept 216 | value = scan("sargraph version: (\\d+\\.\\d+)", str, line) 217 | if value is not None: 218 | data_version = value 219 | 220 | value = scan("psutil version: (\\d+\\.\\d+)", str, line) 221 | if value is not None: 222 | data_version = value 223 | 224 | value = scan("machine: ([^,]+)", str, line) 225 | if value is not None: 226 | UNAME = value 227 | 228 | value = scan("cpu count: ([^,]+)", int, line) 229 | if value is not None: 230 | CPUS = value 231 | 232 | value = scan("cpu: ([^,\n]+)", str, line) 233 | if value is not None: 234 | CPU_NAME = value 235 | 236 | value = scan("observed disk: ([^,]+)", str, line) 237 | if value is not None: 238 | NAME_FS = value 239 | 240 | value = scan("observed network: ([^,]+)", str, line) 241 | if value is not None: 242 | NAME_IFACE = value 243 | 244 | value = scan("total ram: (\\S+)", stof, line) 245 | if value is not None: 246 | TOTAL_RAM = value 247 | 248 | value = scan("max ram used: (\\S+)", stof, line) 249 | if value is not None: 250 | MAX_USED_RAM = value 251 | 252 | value = scan("total disk space: (\\S+)", stof, line) 253 | if value is not None: 254 | TOTAL_FS = value 255 | 256 | value = scan("max received: (\\S+)", stof, line) 257 | if value is not None: 258 | MAX_RX = value 259 | 260 | value = scan("max sent: (\\S+)", stof, line) 261 | if value is not None: 262 | MAX_TX = value 263 | 264 | value = scan("total received: (\\S+)", stof, line) 265 | if value is not None: 266 | TOTAL_RX = value 267 | 268 | value = scan("total sent: (\\S+)", stof, line) 269 | if value is not None: 270 | TOTAL_TX = value 271 | 272 | value = scan("duration: (\\S+)", stof, line) 273 | if value is not None: 274 | DURATION = value 275 | 276 | value = scan("max disk used: (\\S+)", stof, line) 277 | if value is not None: 278 | MAX_USED_FS = value 279 | 280 | value = scan("average load: (\\S+)", stof, line) 281 | if value is not None: 282 | AVERAGE_LOAD = value 283 | 284 | value = scan("total gpu ram: (\\S+)", stof, line) 285 | if value is not None: 286 | TOTAL_GPU_RAM = value 287 | 288 | value = scan("max gpu ram used: (\\S+)", stof, line) 289 | if value is not None: 290 | MAX_USED_GPU_RAM = value 291 | 292 | value = scan("gpu: ([^,\n]+)", str, line) 293 | if value is not None: 294 | GPU_NAME = value 295 | 296 | value = scan("gpu driver: ([^,\n]+)", str, line) 297 | if value is not None: 298 | GPU_DRIVER = value 299 | 300 | value = scan("average gpu load: (\\S+)", stof, line) 301 | if value is not None: 302 | AVERAGE_GPU_LOAD = value 303 | 304 | if data_version != scan("^(\\d+\\.\\d+)", str, SARGRAPH_VERSION): 305 | print("Warning: the data comes from an incompatible version of sargraph") 306 | 307 | # Translate the values to their value-unit representations 308 | TOTAL_RAM = unit_str(TOTAL_RAM, DATA_UNITS) 309 | MAX_USED_RAM = unit_str(MAX_USED_RAM, DATA_UNITS) 310 | 311 | TOTAL_FS = unit_str(TOTAL_FS, DATA_UNITS) 312 | MAX_USED_FS = unit_str(MAX_USED_FS, DATA_UNITS) 313 | 314 | MAX_RX = unit_str(MAX_RX, SPEED_UNITS) 315 | MAX_TX = unit_str(MAX_TX, SPEED_UNITS) 316 | 317 | TOTAL_RX = unit_str(TOTAL_RX, DATA_UNITS) 318 | TOTAL_TX = unit_str(TOTAL_TX, DATA_UNITS) 319 | 320 | if TOTAL_GPU_RAM: 321 | TOTAL_GPU_RAM = unit_str(TOTAL_GPU_RAM, DATA_UNITS) 322 | # Add GPU RAM utilization and GPU utilization plots 323 | NUMBER_OF_PLOTS += 2 324 | if MAX_USED_GPU_RAM: 325 | MAX_USED_GPU_RAM = unit_str(MAX_USED_GPU_RAM, DATA_UNITS) 326 | 327 | DURATION = unit_str(DURATION, TIME_UNITS, 60) 328 | 329 | 330 | def graph(session, tmpfs_color, other_cache_color, fname='plot'): 331 | global OUTPUT_TYPE 332 | global OUTPUT_EXT 333 | 334 | global labels 335 | 336 | global gnuplot 337 | 338 | labels = [] 339 | 340 | # The default format 341 | OUTPUT_TYPE = "pngcairo" 342 | OUTPUT_EXT = "png" 343 | if "SARGRAPH_OUTPUT_TYPE" in os.environ: 344 | otype = os.environ["SARGRAPH_OUTPUT_TYPE"].lower() 345 | 346 | # png is the default, so don't change anything 347 | if otype != "png": 348 | OUTPUT_TYPE = otype 349 | OUTPUT_EXT = otype 350 | elif fname.lower().endswith('.png'): 351 | # png is the default, so don't change anything 352 | pass 353 | elif fname.lower().endswith('.svg'): 354 | OUTPUT_TYPE = "svg" 355 | OUTPUT_EXT = "svg" 356 | elif fname.lower().endswith('.ascii'): 357 | OUTPUT_TYPE = "ascii" 358 | OUTPUT_EXT = "ascii" 359 | elif fname.lower().endswith('.html'): 360 | OUTPUT_TYPE = "html" 361 | OUTPUT_EXT = "html" 362 | else: 363 | pass 364 | # fail("unknown graph extension") 365 | 366 | # Leave just the base name 367 | fname = cut_suffix(fname, f".{OUTPUT_EXT}") 368 | sar_file, ram_file = split_data_file(session) 369 | 370 | # ASCII plots have their own routine 371 | if OUTPUT_TYPE == "ascii": 372 | return servis_graph(sar_file, ram_file, fname) 373 | 374 | # HTML plots have their own routine 375 | if OUTPUT_TYPE == "html": 376 | return servis_graph(sar_file, ram_file, fname, "html") 377 | 378 | read_comments(sar_file) 379 | 380 | gnuplot = run_or_fail("gnuplot", stdin=subprocess.PIPE, 381 | stdout=subprocess.PIPE) 382 | 383 | sdt = datetime.datetime.strptime(START_DATE, '%Y-%m-%d-%H:%M:%S') 384 | edt = datetime.datetime.strptime(END_DATE, '%Y-%m-%d-%H:%M:%S') 385 | 386 | seconds_between = (edt - sdt).total_seconds() 387 | if seconds_between < 100: 388 | seconds_between = 100 389 | 390 | nsdt = sdt - datetime.timedelta(seconds=(seconds_between * 0.01)) 391 | nedt = edt + datetime.timedelta(seconds=(seconds_between * 0.01)) 392 | 393 | g(f"set terminal {OUTPUT_TYPE} size 1200,1600 background '#332d37' font 'monospace,{fix_size(8)}'") 394 | 395 | g(f"set ylabel tc rgb 'white' font 'monospace,{fix_size(8)}'") 396 | 397 | g("set datafile commentschars '#'") 398 | 399 | g("set timefmt '%s'") 400 | g("set xdata time") 401 | g("set border lc rgb 'white'") 402 | g("set key tc rgb 'white'") 403 | g("set timefmt '%Y-%m-%d-%H:%M:%S'") 404 | g("set xtics format '%H:%M:%S'") 405 | g(f"set xtics font 'monospace,{fix_size(8)}' tc rgb 'white'") 406 | g(f"set ytics font 'monospace,{fix_size(8)}' tc rgb 'white'") 407 | g("set grid xtics ytics ls 12 lc rgb '#c4c2c5'") 408 | g("set style fill solid") 409 | g("set palette defined ( 0.0 '#00af91', 0.25 '#00af91', 0.75 '#d83829', 1.0 '#d83829' )") 410 | g("unset colorbox") 411 | g("unset key") 412 | g("set rmargin 6") 413 | 414 | g(f"set output '{fname}.{OUTPUT_EXT}'") 415 | 416 | title_machine = f"Running on {{/:Bold {HOST}}} \\@ {{/:Bold {UNAME}}}, {{/:Bold {CPUS}}} threads x {{/:Bold {CPU_NAME}}}" 417 | title_specs = f"Total ram: {{/:Bold {TOTAL_RAM}}}, Total disk space: {{/:Bold {TOTAL_FS}}}" 418 | if TOTAL_GPU_RAM != 0: 419 | title_gpu = f"\\nGPU: {{/:Bold {GPU_NAME}}} (driver {{/:Bold {GPU_DRIVER}}}, total ram: {{/:Bold {TOTAL_GPU_RAM}}})" 420 | else: 421 | title_gpu = "" 422 | title_times = f"Duration: {{/:Bold {START_DATE}}} .. {{/:Bold {END_DATE}}} ({DURATION})" 423 | 424 | g(f"set multiplot layout {NUMBER_OF_PLOTS},1 title \"\\n{title_machine}\\n{title_specs}{title_gpu}\\n{title_times}\" offset screen -0.475, 0 left tc rgb 'white'") 425 | 426 | g(f"set title tc rgb 'white' font 'monospace,{fix_size(11)}'") 427 | 428 | g(f"set xrange ['{nsdt.strftime('%Y-%m-%d-%H:%M:%S')}':'{nedt.strftime('%Y-%m-%d-%H:%M:%S')}']") 429 | 430 | i = 0 431 | for label in labels: 432 | if i % 2 == 0: 433 | offset = 1.08 434 | else: 435 | offset = 1.20 436 | 437 | i = i + 1 438 | 439 | content = f"{{[{i}] {label[1][0:30]}" 440 | length = len(label[1][0:30]) + len(str(i)) + 5 441 | if OUTPUT_EXT == "svg": 442 | length *= 0.75 443 | 444 | # Draw the dotted line 445 | g(f"set arrow nohead from '{label[0]}', graph 0.01 to '{label[0]}', graph {offset-0.04} front lc rgb '#e74a3c' dt 2") 446 | 447 | # Draw the small rectangle at its bottom 448 | g(f"set object rect at '{label[0]}', graph 0.0 size char 0.5, char 0.5 front lc rgb '#d83829' fc rgb '#f15f32'") 449 | 450 | # Draw the label rectangle 451 | g(f"set object rect at '{label[0]}', graph {offset} size char {length}, char 1.3 fs border lc rgb '#d83829' fc rgb '#f15f32'") 452 | 453 | # Add text to the label 454 | g(f"set label at '{label[0]}', graph {offset} '{content}' center tc rgb 'white' font 'monospace,{fix_size(7)}'") 455 | 456 | if i <= 0: 457 | space = 1 458 | elif i <= 1: 459 | space = 2 460 | else: 461 | space = 3 462 | 463 | g("set object rectangle from graph 0, graph 0 to graph 2, graph 2 behind fillcolor rgb '#000000' fillstyle solid noborder") 464 | 465 | # Set scale for plots displayed in relative units (%) 466 | plot("CPU load (%)", 467 | f"CPU load (average = {AVERAGE_LOAD:.2f} %)", sar_file, 2, space=space) 468 | plot_stacked(f"RAM usage (100% = {TOTAL_RAM})", 469 | f"RAM usage (max = {MAX_USED_RAM})", ram_file, 4, tmpfs_color, other_cache_color, space=space) 470 | plot(f"FS usage (100% = {TOTAL_FS})", f"{NAME_FS} usage (max = {MAX_USED_FS})", 471 | sar_file, 3, space=space) 472 | 473 | plot(f"{NAME_IFACE} received (Mb/s)", 474 | f"{NAME_IFACE} data received (max = {MAX_RX}, total = {TOTAL_RX})", 475 | sar_file, 4, space=space, autoscale=1.2) 476 | plot(f"{NAME_IFACE} sent (Mb/s)", 477 | f"{NAME_IFACE} data sent (max = {MAX_TX}, total = {TOTAL_TX})", 478 | sar_file, 5, space=space, autoscale=1.2) 479 | 480 | # GPU params 481 | if TOTAL_GPU_RAM != 0: 482 | plot("GPU load (%)", 483 | f"GPU load (average = {AVERAGE_GPU_LOAD} %)", sar_file, 6, space=space) 484 | plot(f"GPU RAM usage (100% = {TOTAL_GPU_RAM})", 485 | f"GPU RAM usage (max = {MAX_USED_GPU_RAM})", sar_file, 7, space=space) 486 | 487 | g("unset multiplot") 488 | g("unset output") 489 | g("quit") 490 | 491 | 492 | def read_data(sar_file, ram_file): 493 | xdata = list() 494 | xdata_ram = list() 495 | ydata = [[] for _ in range(NUMBER_OF_PLOTS)] 496 | with open(sar_file, "r") as f: 497 | for line in f: 498 | if(line[0] != '#'): 499 | line = line.split(" ") 500 | date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S') 501 | xdata.append(date) 502 | for i in range(NUMBER_OF_PLOTS): 503 | if i != RAM_DATA_POSITION: 504 | ydata[i].append(stof(line[i+1 - int(i > RAM_DATA_POSITION)])) 505 | with open(ram_file, 'r') as f: 506 | for line in f: 507 | if(line[0] != '#'): 508 | line = line.split(" ") 509 | date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S.%f') 510 | xdata_ram.append(date) 511 | ydata[RAM_DATA_POSITION].append(100-stof(line[1])) 512 | 513 | return (xdata, xdata_ram, ydata) 514 | 515 | 516 | def convert_labels_to_tags(labels): 517 | tags = [] 518 | for [label_date, label_name] in labels: 519 | label_date = datetime.datetime.strptime( 520 | label_date, '%Y-%m-%d-%H:%M:%S') 521 | label_ts = int(label_date.replace( 522 | tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 523 | tags.append({'name': label_name, 524 | 'timestamp': label_ts}) 525 | return tags 526 | 527 | 528 | def servis_graph(sar_file, ram_file, fname='plot', output_ext='ascii'): 529 | read_comments(sar_file) 530 | xdata, xdata_ram, ydata = read_data(sar_file, ram_file) 531 | titles = [f"""CPU load (average = {AVERAGE_LOAD} %)""", 532 | f"""RAM usage (max = {MAX_USED_RAM})""", 533 | f"""{NAME_FS} usage (max = {MAX_USED_FS})""", 534 | f"""{NAME_IFACE} data received (max = {MAX_RX})""", 535 | f"""{NAME_IFACE} data sent (max = {MAX_TX})"""] 536 | 537 | if TOTAL_GPU_RAM != 0: 538 | titles.extend([ 539 | f"GPU load (average = {AVERAGE_GPU_LOAD} %)", 540 | f"GPU RAM usage (max = {MAX_USED_GPU_RAM})" 541 | ]) 542 | 543 | y_titles = ["CPU load (%)", 544 | f"RAM usage (100% = {TOTAL_RAM})", 545 | f"FS usage (100% = {TOTAL_FS})", 546 | f"{NAME_IFACE} received", 547 | f"{NAME_IFACE} sent"] 548 | 549 | if TOTAL_GPU_RAM != 0: 550 | y_titles.extend([ 551 | "GPU load (%)", 552 | f"GPU RAM usage (100% = {TOTAL_GPU_RAM})" 553 | ]) 554 | 555 | xdata_to_int = [int(timestamp.replace( 556 | tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 557 | for timestamp in xdata] 558 | 559 | summary = f"Running on {UNAME}, {CPUS} threads x {CPU_NAME}\n" 560 | summary += f"Total ram: {TOTAL_RAM}, Total disk space: {TOTAL_FS}\n" 561 | if TOTAL_GPU_RAM != 0: 562 | summary += f"GPU: {GPU_NAME} (driver {GPU_DRIVER}), total ram: {TOTAL_GPU_RAM}" 563 | summary += f"Duration: {START_DATE} .. {END_DATE} ({DURATION})" 564 | 565 | y_ranges = [ 566 | (0, 100), 567 | (0, 100), 568 | (0, 100), 569 | None, 570 | None, 571 | ] 572 | 573 | if TOTAL_GPU_RAM != 0: 574 | y_ranges.extend([ 575 | (0, 100), 576 | (0, 100) 577 | ]) 578 | 579 | from servis import render_multiple_time_series_plot 580 | if output_ext == 'ascii': 581 | xdatas = [[xdata_to_int]] * (NUMBER_OF_PLOTS - 1) 582 | xdatas.insert(1, [[ 583 | int(timestamp.replace( 584 | tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 585 | for timestamp in xdata_ram 586 | ]]) 587 | 588 | render_multiple_time_series_plot( 589 | ydatas=[[yd] for yd in ydata], 590 | xdatas=xdatas, 591 | title=summary, 592 | subtitles=titles, 593 | xtitles=['time'] * NUMBER_OF_PLOTS, 594 | xunits=[None] * NUMBER_OF_PLOTS, 595 | ytitles=y_titles, 596 | yunits=[None] * NUMBER_OF_PLOTS, 597 | y_ranges=y_ranges, 598 | outpath=Path(fname), 599 | trimxvalues=False, 600 | bins=0, 601 | figsize=(900, 700) 602 | ) 603 | elif output_ext == 'html': 604 | converted_labels = convert_labels_to_tags(labels) 605 | xdatas = [ 606 | int(timestamp.replace( 607 | tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 608 | for timestamp in xdata_ram 609 | ] 610 | xdatas = [xdata_to_int] + [xdatas] + [xdata_to_int * (NUMBER_OF_PLOTS - 2)] 611 | render_multiple_time_series_plot( 612 | ydatas=ydata, 613 | xdatas=xdatas, 614 | title=summary, 615 | subtitles=titles, 616 | xtitles=['time'] * NUMBER_OF_PLOTS, 617 | xunits=[None] * NUMBER_OF_PLOTS, 618 | ytitles=y_titles, 619 | yunits=[None] * NUMBER_OF_PLOTS, 620 | y_ranges=y_ranges, 621 | outpath=Path(fname), 622 | outputext=['html'], 623 | trimxvalues=False, 624 | figsize=(1200, 1600), 625 | tags=[converted_labels] * NUMBER_OF_PLOTS, 626 | setgradientcolors=True 627 | ) 628 | --------------------------------------------------------------------------------