├── .gitignore ├── ClusterMonitor ├── __init__.py ├── cpu_ram_log.py ├── example │ ├── cpu_ram.log │ ├── cpu_ram_visualization.pdf │ └── example_image.png └── visualize_log.py ├── README.md ├── bin ├── cpu_ram_log └── visualize_log └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | ClusterMonitor.egg-info 2 | build 3 | dist 4 | -------------------------------------------------------------------------------- /ClusterMonitor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickhir/ClusterMonitor/ec9173daf9391b154b059253f02f05b59342932d/ClusterMonitor/__init__.py -------------------------------------------------------------------------------- /ClusterMonitor/cpu_ram_log.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | import math 3 | import argparse 4 | import time 5 | import csv 6 | import os 7 | import datetime 8 | import sys 9 | 10 | p = argparse.ArgumentParser() 11 | p.add_argument("-u", "--username", type=str, required=True, 12 | help="Specify the username for which we record CPU and Memory usage") 13 | p.add_argument("-o", "--output", type=str, required=True, 14 | help="Path to the file where the CPU and Memory usage gets logged (in tsv format)") 15 | p.add_argument("--interval", type=int, required=True, 16 | help="Specify the time (in seconds) over which the CPU and Memory usage gets averaged") 17 | args = p.parse_args() 18 | 19 | 20 | def bytesto(bytes, to, bsize=1024): 21 | """convert bytes to megabytes, etc. 22 | sample code: 23 | print('mb= ' + str(bytesto(314575262000000, 'm'))) 24 | sample output: 25 | mb= 300002347.946 26 | """ 27 | 28 | a = {'k': 1, 'm': 2, 'g': 3, 't': 4, 'p': 5, 'e': 6} 29 | r = float(bytes) 30 | for i in range(a[to]): 31 | r = r / bsize 32 | 33 | return (r) 34 | 35 | 36 | try: 37 | print("Start data logging") 38 | while True: 39 | # record the average over X seconds, becuase cpu usage tends to fluctuate 40 | memory_avg = [] 41 | cpu_avg = [] 42 | for i in range(args.interval): 43 | memory = [] 44 | cpu_usage = [] 45 | for proc in psutil.process_iter(): 46 | try: 47 | 48 | process_info = proc.as_dict(attrs=["pid", "name", "username", "memory_info"]) 49 | if process_info["name"] == "sleep": 50 | continue 51 | # if process occupies more than 50 MB it is like not a background process. only look at these. 52 | if process_info["username"] == args.username and proc.memory_info()[0] > 10e+2: 53 | memory.append(proc.memory_info()[0]) 54 | cpu_usage.append(proc.cpu_percent(0.2)) 55 | except: 56 | pass 57 | 58 | memory_avg.append(sum(memory)) 59 | cpu_avg.append(sum(cpu_usage)) 60 | time.sleep(1) 61 | 62 | final_cpu = round(sum(cpu_avg) / len(cpu_avg), 4) 63 | final_memory = round(bytesto(sum(memory_avg) / len(memory_avg), "g"), 4) 64 | header = ['datetime', 'CPU [%]', 'Memory [GB]'] 65 | with open(args.output, "a", newline="", encoding="UTF-8") as csv_file: 66 | writer = csv.writer(csv_file, delimiter='\t') 67 | # if file is empty create the header and add the data 68 | if os.path.getsize(args.output) == 0: 69 | writer.writerow(header) 70 | writer.writerow([datetime.datetime.now().strftime("%H:%M:%S %d/%m/%Y"), final_cpu, final_memory]) 71 | else: 72 | writer.writerow([datetime.datetime.now().strftime("%H:%M:%S %d/%m/%Y"), final_cpu, final_memory]) 73 | 74 | except KeyboardInterrupt: 75 | print(f"Logging finished and saved to file {args.output}") 76 | -------------------------------------------------------------------------------- /ClusterMonitor/example/cpu_ram.log: -------------------------------------------------------------------------------- 1 | datetime CPU [%] Memory [GB] 2 | 10:57:02 02/06/2021 1680.1 30.7936 3 | 10:57:20 02/06/2021 2249.2 32.4484 4 | 10:57:38 02/06/2021 1747.45 33.5596 5 | 10:57:56 02/06/2021 1822.35 34.591 6 | 10:58:14 02/06/2021 2067.1 35.4455 7 | 10:58:32 02/06/2021 1787.35 36.175 8 | 10:58:50 02/06/2021 2256.7 36.8172 9 | 10:59:08 02/06/2021 1857.2 37.4444 10 | 10:59:26 02/06/2021 1987.05 38.0574 11 | 10:59:44 02/06/2021 1934.6 38.6717 12 | 11:00:02 02/06/2021 1909.85 39.232 13 | 11:00:20 02/06/2021 1812.3 39.84 14 | 11:00:38 02/06/2021 1780.0 40.4068 15 | 11:00:56 02/06/2021 1822.25 40.9224 16 | 11:01:14 02/06/2021 2146.85 41.3671 17 | 11:01:32 02/06/2021 1904.75 41.8538 18 | 11:01:50 02/06/2021 1675.1 42.25 19 | 11:02:08 02/06/2021 1914.55 42.6632 20 | 11:02:26 02/06/2021 1857.35 43.0489 21 | 11:02:44 02/06/2021 1949.65 43.496 22 | 11:03:02 02/06/2021 1650.1 43.8557 23 | 11:03:21 02/06/2021 1675.0 44.2754 24 | 11:03:39 02/06/2021 1879.7 44.6685 25 | 11:03:57 02/06/2021 1889.65 45.044 26 | 11:04:15 02/06/2021 1657.65 45.4413 27 | 11:04:33 02/06/2021 1964.55 45.8126 28 | 11:04:51 02/06/2021 1792.5 46.0062 29 | 11:05:09 02/06/2021 1749.65 46.1539 30 | 11:05:27 02/06/2021 1680.0 46.3438 31 | 11:05:45 02/06/2021 1997.05 46.482 32 | 11:06:03 02/06/2021 1777.4 46.5865 33 | 11:06:21 02/06/2021 2024.55 46.7177 34 | 11:06:39 02/06/2021 1782.4 46.8469 35 | 11:06:57 02/06/2021 1897.4 46.9706 36 | 11:07:15 02/06/2021 1909.8 47.092 37 | 11:07:33 02/06/2021 1782.15 47.1976 38 | 11:07:51 02/06/2021 1695.05 47.2948 39 | 11:08:10 02/06/2021 1829.8 47.2711 40 | 11:08:28 02/06/2021 1720.1 47.2176 41 | 11:08:46 02/06/2021 1700.05 47.2449 42 | 11:09:04 02/06/2021 1971.85 47.1941 43 | 11:09:22 02/06/2021 1886.85 47.1264 44 | 11:09:39 02/06/2021 1934.75 46.9052 45 | 11:09:57 02/06/2021 1720.0 46.6676 46 | 11:10:15 02/06/2021 1799.95 46.4221 47 | 11:10:33 02/06/2021 1737.5 46.156 48 | 11:10:51 02/06/2021 1729.95 45.8705 49 | 11:11:09 02/06/2021 1937.0 45.6517 50 | 11:11:27 02/06/2021 1767.4 45.3774 51 | 11:11:45 02/06/2021 1702.6 45.1819 52 | 11:12:04 02/06/2021 1754.8 44.9538 53 | 11:12:22 02/06/2021 1817.25 44.7506 54 | 11:12:40 02/06/2021 1590.1 44.4721 55 | 11:12:58 02/06/2021 1947.2 44.2186 56 | 11:13:16 02/06/2021 1677.5 44.0175 57 | 11:13:34 02/06/2021 1762.5 43.7644 58 | 11:13:52 02/06/2021 1627.75 43.5713 59 | 11:14:10 02/06/2021 1957.3 43.3964 60 | 11:14:28 02/06/2021 1714.95 43.1987 61 | 11:14:46 02/06/2021 1789.8 42.9763 62 | 11:15:04 02/06/2021 1597.55 42.8628 63 | 11:15:22 02/06/2021 2036.95 42.6814 64 | 11:15:40 02/06/2021 1832.3 42.4837 65 | 11:15:58 02/06/2021 1807.4 42.2092 66 | 11:16:16 02/06/2021 2334.05 41.9223 67 | 11:16:34 02/06/2021 2116.85 41.7093 68 | 11:16:53 02/06/2021 1767.4 41.4615 69 | 11:17:11 02/06/2021 1784.9 41.1839 70 | 11:17:29 02/06/2021 1699.95 41.0004 71 | 11:17:47 02/06/2021 1817.4 40.755 72 | 11:18:05 02/06/2021 1919.6 40.5502 73 | 11:18:23 02/06/2021 1864.65 40.3771 74 | 11:18:41 02/06/2021 1952.15 40.2214 75 | 11:18:59 02/06/2021 1729.9 40.0337 76 | 11:19:17 02/06/2021 1792.35 39.8064 77 | 11:19:35 02/06/2021 1652.55 39.5496 78 | 11:19:53 02/06/2021 1817.35 39.3751 79 | 11:20:11 02/06/2021 1697.5 39.3138 80 | 11:20:29 02/06/2021 1764.75 39.2648 81 | 11:20:47 02/06/2021 1847.3 39.1017 82 | 11:21:05 02/06/2021 1655.05 38.902 83 | 11:21:23 02/06/2021 1814.9 38.6855 84 | 11:21:41 02/06/2021 1807.35 38.6219 85 | 11:21:59 02/06/2021 1744.95 38.6381 86 | 11:22:17 02/06/2021 2064.4 38.6998 87 | 11:22:36 02/06/2021 1690.0 38.6216 88 | 11:22:54 02/06/2021 1762.45 38.4513 89 | 11:23:12 02/06/2021 1710.1 38.2628 90 | 11:23:30 02/06/2021 1717.45 38.209 91 | 11:23:48 02/06/2021 1772.45 38.3119 92 | 11:24:06 02/06/2021 1757.45 38.2413 93 | 11:24:24 02/06/2021 1999.5 38.1153 94 | 11:24:42 02/06/2021 1966.9 38.0619 95 | 11:25:00 02/06/2021 1744.95 38.1221 96 | 11:25:18 02/06/2021 1652.6 38.2096 97 | 11:25:36 02/06/2021 1689.95 38.301 98 | 11:25:54 02/06/2021 1740.0 38.1896 99 | 11:26:12 02/06/2021 1834.85 37.9845 100 | 11:26:30 02/06/2021 1772.35 37.8568 101 | 11:26:48 02/06/2021 1834.95 37.7315 102 | 11:27:06 02/06/2021 1967.1 37.6607 103 | 11:27:24 02/06/2021 1929.7 37.5683 104 | 11:27:42 02/06/2021 2479.0 37.4643 105 | 11:28:00 02/06/2021 2074.45 37.4339 106 | 11:28:19 02/06/2021 1747.35 37.4683 107 | 11:28:37 02/06/2021 2206.75 37.5482 108 | 11:28:55 02/06/2021 2009.55 37.7657 109 | 11:29:13 02/06/2021 1899.75 37.9147 110 | 11:29:31 02/06/2021 1922.2 37.8859 111 | 11:29:49 02/06/2021 1839.8 37.9619 112 | 11:30:07 02/06/2021 1769.9 38.0365 113 | 11:30:25 02/06/2021 1914.6 37.9655 114 | 11:30:43 02/06/2021 1832.35 37.8164 115 | 11:31:01 02/06/2021 1784.9 37.8422 116 | 11:31:19 02/06/2021 1712.45 38.0743 117 | 11:31:37 02/06/2021 1814.75 38.2508 118 | 11:31:55 02/06/2021 1857.3 38.3435 119 | 11:32:13 02/06/2021 1922.2 38.3619 120 | 11:32:31 02/06/2021 1675.05 38.2396 121 | 11:32:49 02/06/2021 1827.35 37.9923 122 | 11:33:07 02/06/2021 1832.35 37.926 123 | 11:33:25 02/06/2021 1852.1 38.0968 124 | 11:33:43 02/06/2021 1832.25 38.2786 125 | 11:34:01 02/06/2021 1652.6 38.357 126 | 11:34:19 02/06/2021 1929.65 38.4426 127 | 11:34:38 02/06/2021 1822.3 38.4661 128 | 11:34:56 02/06/2021 1907.2 38.2507 129 | 11:35:14 02/06/2021 1842.2 38.0271 130 | 11:35:32 02/06/2021 1952.0 37.7933 131 | 11:35:50 02/06/2021 1779.85 37.6819 132 | 11:36:08 02/06/2021 1879.8 37.8503 133 | 11:36:26 02/06/2021 1792.4 37.9884 134 | 11:36:44 02/06/2021 1732.45 38.1901 135 | 11:37:02 02/06/2021 1949.6 38.365 136 | 11:37:20 02/06/2021 2047.05 38.3882 137 | 11:37:38 02/06/2021 2031.95 38.467 138 | 11:37:56 02/06/2021 1892.25 38.5853 139 | 11:38:14 02/06/2021 1677.6 38.7457 140 | 11:38:32 02/06/2021 1822.35 38.7632 141 | 11:38:50 02/06/2021 1932.05 38.6631 142 | 11:39:08 02/06/2021 1934.7 38.6939 143 | 11:39:26 02/06/2021 1759.95 38.7133 144 | 11:39:44 02/06/2021 1949.85 38.7807 145 | 11:40:02 02/06/2021 1797.3 38.709 146 | 11:40:20 02/06/2021 1962.05 38.6986 147 | 11:40:38 02/06/2021 1827.2 38.6716 148 | 11:40:57 02/06/2021 1717.5 38.7305 149 | 11:41:15 02/06/2021 1971.95 38.7092 150 | 11:41:33 02/06/2021 1849.8 38.9965 151 | 11:41:51 02/06/2021 1984.6 39.2506 152 | 11:42:09 02/06/2021 2084.4 39.3943 153 | 11:42:27 02/06/2021 1887.25 39.4557 154 | 11:42:45 02/06/2021 1934.75 39.3898 155 | 11:43:03 02/06/2021 1877.2 39.2111 156 | 11:43:21 02/06/2021 1712.45 39.256 157 | 11:43:39 02/06/2021 2017.1 39.4777 158 | 11:43:57 02/06/2021 1802.35 39.7147 159 | 11:44:15 02/06/2021 1857.35 40.0022 160 | 11:44:33 02/06/2021 2004.4 40.1368 161 | 11:44:51 02/06/2021 1754.95 40.1761 162 | 11:45:09 02/06/2021 1625.15 40.2626 163 | 11:45:27 02/06/2021 2064.5 40.4581 164 | 11:45:45 02/06/2021 1877.15 40.6919 165 | 11:46:03 02/06/2021 1702.6 40.8629 166 | 11:46:22 02/06/2021 1799.7 41.1047 167 | 11:46:40 02/06/2021 2104.35 41.3717 168 | 11:46:58 02/06/2021 1794.9 41.6559 169 | 11:47:16 02/06/2021 1642.6 41.9004 170 | 11:47:34 02/06/2021 1951.95 41.8855 171 | 11:47:52 02/06/2021 1792.4 41.856 172 | 11:48:10 02/06/2021 1929.65 41.7054 173 | 11:48:28 02/06/2021 1974.65 41.6919 174 | 11:48:46 02/06/2021 1939.7 41.7588 175 | 11:49:04 02/06/2021 2284.1 41.8366 176 | 11:49:22 02/06/2021 2291.6 41.8427 177 | 11:49:40 02/06/2021 1827.2 41.7868 178 | 11:49:58 02/06/2021 1722.45 41.8435 179 | 11:50:16 02/06/2021 2106.8 41.8468 180 | 11:50:34 02/06/2021 1804.85 41.7012 181 | 11:50:52 02/06/2021 1817.35 41.5621 182 | 11:51:10 02/06/2021 1909.75 41.4222 183 | 11:51:29 02/06/2021 2940.75 41.4153 184 | 11:51:47 02/06/2021 1760.0 41.4259 185 | 11:52:05 02/06/2021 1864.85 41.5618 186 | 11:52:23 02/06/2021 2149.45 41.3685 187 | 11:52:41 02/06/2021 1762.45 41.1114 188 | 11:52:59 02/06/2021 1779.8 41.0708 189 | 11:53:17 02/06/2021 1999.65 41.1969 190 | 11:53:35 02/06/2021 1814.85 41.2876 191 | 11:53:53 02/06/2021 2281.65 41.5641 192 | 11:54:11 02/06/2021 1944.6 42.0693 193 | 11:54:29 02/06/2021 1702.4 40.3678 194 | 11:54:47 02/06/2021 1844.65 40.2554 195 | 11:55:05 02/06/2021 2326.55 41.1394 196 | 11:55:23 02/06/2021 2186.7 39.6328 197 | 11:55:40 02/06/2021 1562.75 33.3637 198 | 11:55:58 02/06/2021 1884.7 32.6171 199 | 11:56:16 02/06/2021 2024.4 34.46 200 | 11:56:34 02/06/2021 2044.5 33.9584 201 | 11:56:52 02/06/2021 1799.8 35.4443 202 | 11:57:10 02/06/2021 1892.15 36.4602 203 | 11:57:28 02/06/2021 1754.95 37.095 204 | 11:57:46 02/06/2021 1969.6 37.5593 205 | 11:58:04 02/06/2021 1837.35 38.0674 206 | 11:58:22 02/06/2021 1762.45 36.9451 207 | 11:58:40 02/06/2021 1782.25 35.9304 208 | 11:58:58 02/06/2021 1904.7 36.2673 209 | 11:59:16 02/06/2021 1954.75 36.7515 210 | 11:59:34 02/06/2021 1769.9 37.9197 211 | 11:59:52 02/06/2021 1869.8 38.7703 212 | 12:00:10 02/06/2021 1814.9 39.4125 213 | 12:00:28 02/06/2021 1864.85 39.9386 214 | 12:00:46 02/06/2021 1682.45 34.8973 215 | 12:01:04 02/06/2021 2014.65 34.6517 216 | 12:01:22 02/06/2021 1892.35 35.6546 217 | 12:01:40 02/06/2021 1909.7 36.2057 218 | 12:01:58 02/06/2021 2071.9 36.5172 219 | 12:02:16 02/06/2021 1752.6 36.8001 220 | 12:02:34 02/06/2021 1837.35 37.0093 221 | 12:02:52 02/06/2021 1824.85 37.1927 222 | 12:03:10 02/06/2021 1932.25 37.352 223 | 12:03:28 02/06/2021 1874.65 37.4429 224 | 12:03:47 02/06/2021 1944.6 37.5115 225 | 12:04:05 02/06/2021 1752.4 37.5254 226 | 12:04:23 02/06/2021 1699.9 37.4973 227 | 12:04:41 02/06/2021 1807.2 37.4716 228 | 12:04:59 02/06/2021 1822.25 37.4486 229 | 12:05:17 02/06/2021 1899.8 37.3727 230 | 12:05:35 02/06/2021 1844.75 37.2242 231 | 12:05:53 02/06/2021 1789.95 37.1052 232 | 12:06:11 02/06/2021 1879.7 36.9476 233 | 12:06:29 02/06/2021 1844.8 36.7493 234 | 12:06:47 02/06/2021 1847.25 36.579 235 | 12:07:05 02/06/2021 1832.4 36.439 236 | 12:07:23 02/06/2021 1859.7 36.2053 237 | 12:07:41 02/06/2021 2259.1 35.9034 238 | 12:07:59 02/06/2021 1912.15 35.5609 239 | 12:08:17 02/06/2021 2066.7 35.2223 240 | 12:08:35 02/06/2021 1899.6 34.8239 241 | 12:08:53 02/06/2021 1959.25 34.3491 242 | 12:09:12 02/06/2021 1684.8 33.8944 243 | 12:09:30 02/06/2021 1687.6 33.4764 244 | 12:09:48 02/06/2021 1856.8 33.053 245 | 12:10:06 02/06/2021 1914.55 32.6189 246 | 12:10:24 02/06/2021 1894.4 32.2548 247 | 12:10:42 02/06/2021 1866.75 31.8751 248 | 12:11:00 02/06/2021 2129.35 31.5619 249 | 12:11:18 02/06/2021 1849.5 31.2137 250 | 12:11:36 02/06/2021 1937.05 30.8762 251 | 12:11:54 02/06/2021 1510.2 30.6108 252 | 12:12:12 02/06/2021 1879.7 30.3929 253 | 12:12:30 02/06/2021 1812.3 30.146 254 | 12:12:48 02/06/2021 1852.3 29.8102 255 | 12:13:06 02/06/2021 1952.1 29.4181 256 | 12:13:24 02/06/2021 1802.05 29.1229 257 | 12:13:42 02/06/2021 1794.75 28.9526 258 | 12:14:00 02/06/2021 2131.8 28.7657 259 | 12:14:18 02/06/2021 1879.85 28.6666 260 | 12:14:36 02/06/2021 1839.85 28.641 261 | 12:14:55 02/06/2021 2029.45 28.4222 262 | 12:15:13 02/06/2021 1817.3 28.1811 263 | 12:15:31 02/06/2021 1942.2 27.8119 264 | 12:15:49 02/06/2021 1852.25 27.4615 265 | 12:16:07 02/06/2021 1797.35 27.2616 266 | 12:16:25 02/06/2021 1655.1 27.085 267 | 12:16:43 02/06/2021 1707.35 26.7939 268 | 12:17:01 02/06/2021 1919.7 26.6341 269 | 12:17:19 02/06/2021 1937.15 26.6374 270 | 12:17:37 02/06/2021 1932.15 26.5801 271 | 12:17:55 02/06/2021 1924.65 26.5653 272 | 12:18:13 02/06/2021 1897.2 26.3548 273 | 12:18:31 02/06/2021 1814.8 26.1336 274 | 12:18:49 02/06/2021 1902.2 26.0453 275 | 12:19:07 02/06/2021 1892.25 25.9953 276 | 12:19:25 02/06/2021 1957.2 26.168 277 | 12:19:43 02/06/2021 1754.95 26.205 278 | 12:20:01 02/06/2021 1929.55 26.1309 279 | 12:20:19 02/06/2021 1884.8 26.1042 280 | 12:20:38 02/06/2021 1967.05 26.1236 281 | 12:20:56 02/06/2021 1754.9 25.9766 282 | 12:21:14 02/06/2021 1842.15 25.8464 283 | 12:21:32 02/06/2021 1774.85 25.9744 284 | 12:21:50 02/06/2021 1787.35 26.1152 285 | 12:22:08 02/06/2021 1724.95 26.0358 286 | 12:22:26 02/06/2021 1839.7 25.9338 287 | 12:22:44 02/06/2021 1597.65 25.9372 288 | 12:23:02 02/06/2021 1844.65 25.8638 289 | 12:23:20 02/06/2021 1869.65 25.7997 290 | 12:23:38 02/06/2021 1924.75 25.9451 291 | 12:23:56 02/06/2021 1904.75 26.0682 292 | 12:24:14 02/06/2021 2111.9 26.0575 293 | 12:24:32 02/06/2021 1780.0 26.0655 294 | 12:24:50 02/06/2021 1937.1 26.1189 295 | 12:25:08 02/06/2021 1774.7 26.0683 296 | 12:25:26 02/06/2021 1735.05 26.4316 297 | 12:25:44 02/06/2021 1637.5 26.7996 298 | 12:26:02 02/06/2021 1837.3 26.927 299 | 12:26:20 02/06/2021 1747.55 27.0532 300 | 12:26:39 02/06/2021 1912.25 27.235 301 | 12:26:57 02/06/2021 1957.1 27.5429 302 | 12:27:15 02/06/2021 1879.75 27.7477 303 | 12:27:33 02/06/2021 2041.95 27.7986 304 | 12:27:51 02/06/2021 2039.5 27.9938 305 | 12:28:09 02/06/2021 1772.35 28.233 306 | 12:28:27 02/06/2021 1872.3 28.3337 307 | 12:28:45 02/06/2021 1827.3 28.3793 308 | 12:29:03 02/06/2021 2059.4 28.5479 309 | 12:29:21 02/06/2021 1782.3 28.5385 310 | 12:29:39 02/06/2021 1917.1 28.4906 311 | 12:29:57 02/06/2021 1924.7 28.4753 312 | 12:30:15 02/06/2021 1889.65 28.6371 313 | 12:30:33 02/06/2021 1902.2 28.7065 314 | 12:30:51 02/06/2021 1805.0 28.5003 315 | 12:31:09 02/06/2021 1789.85 28.2984 316 | 12:31:27 02/06/2021 1932.15 28.2131 317 | 12:31:45 02/06/2021 1996.95 28.297 318 | 12:32:03 02/06/2021 1882.25 28.3973 319 | 12:32:21 02/06/2021 1727.5 28.5256 320 | 12:32:40 02/06/2021 2077.0 28.7322 321 | 12:32:58 02/06/2021 1859.7 28.9127 322 | 12:33:16 02/06/2021 1869.7 28.9242 323 | 12:33:34 02/06/2021 1789.9 29.0746 324 | 12:33:52 02/06/2021 1700.05 29.0127 325 | 12:34:10 02/06/2021 1797.4 28.827 326 | 12:34:28 02/06/2021 1909.65 28.6225 327 | 12:34:46 02/06/2021 1799.8 28.708 328 | 12:35:04 02/06/2021 1647.5 28.7712 329 | 12:35:22 02/06/2021 2042.15 28.9402 330 | 12:35:40 02/06/2021 1869.55 29.0333 331 | 12:35:58 02/06/2021 1847.25 29.1413 332 | 12:36:16 02/06/2021 1779.8 29.1936 333 | 12:36:34 02/06/2021 1987.15 29.1187 334 | 12:36:52 02/06/2021 1667.35 29.1712 335 | 12:37:10 02/06/2021 1879.65 29.2772 336 | 12:37:28 02/06/2021 1889.55 29.2586 337 | 12:37:46 02/06/2021 1667.5 29.2728 338 | 12:38:04 02/06/2021 1984.6 29.3603 339 | 12:38:22 02/06/2021 1762.45 29.4804 340 | 12:38:40 02/06/2021 1802.3 29.6153 341 | 12:38:58 02/06/2021 2049.55 29.555 342 | 12:39:16 02/06/2021 1807.35 29.4162 343 | 12:39:34 02/06/2021 1937.1 29.4699 344 | 12:39:53 02/06/2021 1842.3 29.5717 345 | 12:40:11 02/06/2021 1794.9 30.0128 346 | 12:40:29 02/06/2021 1847.3 30.472 347 | 12:40:47 02/06/2021 1802.35 30.8229 348 | 12:41:05 02/06/2021 1670.15 30.9438 349 | 12:41:23 02/06/2021 1799.85 30.8894 350 | 12:41:41 02/06/2021 1827.4 30.9686 351 | 12:41:59 02/06/2021 1759.9 31.0734 352 | 12:42:17 02/06/2021 1887.1 30.9349 353 | 12:42:35 02/06/2021 1847.15 30.8016 354 | 12:42:53 02/06/2021 2134.35 30.7176 355 | 12:43:11 02/06/2021 1829.75 30.6292 356 | 12:43:29 02/06/2021 1759.9 30.8181 357 | 12:43:47 02/06/2021 1952.1 30.9557 358 | 12:44:05 02/06/2021 1867.25 31.1014 359 | 12:44:23 02/06/2021 1827.3 31.1827 360 | 12:44:41 02/06/2021 2071.8 31.0495 361 | 12:44:59 02/06/2021 1754.8 31.0234 362 | 12:45:17 02/06/2021 2091.85 31.2303 363 | 12:45:35 02/06/2021 1924.75 31.3784 364 | 12:45:53 02/06/2021 1804.85 31.7218 365 | 12:46:11 02/06/2021 1847.3 31.8792 366 | 12:46:30 02/06/2021 1912.25 31.9212 367 | 12:46:48 02/06/2021 2311.45 31.8944 368 | 12:47:06 02/06/2021 1744.9 31.985 369 | 12:47:24 02/06/2021 1822.25 32.1295 370 | 12:47:42 02/06/2021 1702.45 32.2905 371 | 12:48:00 02/06/2021 1792.4 32.4856 372 | 12:48:18 02/06/2021 1682.6 32.6053 373 | 12:48:36 02/06/2021 1547.75 32.6976 374 | 12:48:54 02/06/2021 2104.3 32.8399 375 | 12:49:12 02/06/2021 1782.15 33.006 376 | 12:49:30 02/06/2021 1951.75 33.2131 377 | 12:49:48 02/06/2021 2004.3 33.2939 378 | 12:50:06 02/06/2021 1874.35 33.3424 379 | 12:50:24 02/06/2021 2136.9 33.2563 380 | 12:50:42 02/06/2021 1937.1 33.1446 381 | 12:51:00 02/06/2021 1899.65 33.207 382 | 12:51:18 02/06/2021 2344.0 33.4931 383 | 12:51:37 02/06/2021 1912.15 33.8254 384 | 12:51:55 02/06/2021 1690.0 34.0158 385 | 12:52:13 02/06/2021 1862.15 33.9919 386 | 12:52:31 02/06/2021 1829.7 33.99 387 | 12:52:49 02/06/2021 2069.3 33.9924 388 | 12:53:07 02/06/2021 1787.4 34.0615 389 | 12:53:24 02/06/2021 2071.95 31.8104 390 | 12:53:42 02/06/2021 1667.55 33.524 391 | 12:54:00 02/06/2021 1937.1 34.0325 392 | 12:54:18 02/06/2021 1655.05 32.3385 393 | 12:54:36 02/06/2021 1924.6 32.5512 394 | 12:54:54 02/06/2021 1869.8 31.9139 395 | 12:55:12 02/06/2021 1894.75 32.9797 396 | 12:55:30 02/06/2021 1842.3 33.553 397 | 12:55:48 02/06/2021 1792.3 33.8918 398 | 12:56:06 02/06/2021 1954.55 34.2339 399 | 12:56:24 02/06/2021 1852.15 34.6383 400 | 12:56:42 02/06/2021 1957.1 34.8954 401 | 12:57:01 02/06/2021 1852.05 35.0773 402 | 12:57:19 02/06/2021 1922.05 35.2202 403 | 12:57:37 02/06/2021 1827.25 35.3368 404 | 12:57:55 02/06/2021 1957.1 35.3673 405 | 12:58:13 02/06/2021 2014.65 34.5383 406 | 12:58:31 02/06/2021 1847.4 34.5718 407 | 12:58:49 02/06/2021 1882.3 35.3124 408 | 12:59:07 02/06/2021 2027.0 34.2583 409 | 12:59:25 02/06/2021 2004.5 35.2457 410 | 12:59:43 02/06/2021 1799.95 35.6982 411 | 13:00:01 02/06/2021 1887.2 35.9882 412 | 13:00:18 02/06/2021 1727.45 33.6464 413 | 13:00:36 02/06/2021 1914.7 35.6079 414 | 13:00:54 02/06/2021 2016.9 36.505 415 | 13:01:12 02/06/2021 1964.5 37.0718 416 | 13:01:30 02/06/2021 1807.4 37.1473 417 | 13:01:48 02/06/2021 1844.6 35.808 418 | 13:02:06 02/06/2021 1979.6 36.4349 419 | 13:02:24 02/06/2021 1909.7 36.9499 420 | 13:02:42 02/06/2021 2079.45 37.3251 421 | 13:03:00 02/06/2021 1794.8 37.631 422 | 13:03:18 02/06/2021 1719.9 34.8575 423 | 13:03:36 02/06/2021 1912.3 36.6803 424 | 13:03:54 02/06/2021 1789.95 37.3084 425 | 13:04:12 02/06/2021 1932.15 37.6042 426 | 13:04:30 02/06/2021 1859.75 37.6697 427 | 13:04:48 02/06/2021 1907.1 37.6301 428 | 13:05:06 02/06/2021 1874.8 37.6464 429 | 13:05:24 02/06/2021 2191.75 35.7988 430 | 13:05:42 02/06/2021 1864.65 36.2059 431 | 13:06:00 02/06/2021 1982.05 36.4539 432 | 13:06:18 02/06/2021 1782.4 36.4502 433 | 13:06:36 02/06/2021 2032.05 36.4216 434 | 13:06:54 02/06/2021 1877.2 36.3698 435 | 13:07:12 02/06/2021 1655.05 36.2916 436 | 13:07:30 02/06/2021 1884.75 36.273 437 | 13:07:48 02/06/2021 1912.3 36.3274 438 | 13:08:06 02/06/2021 2009.65 36.3328 439 | 13:08:24 02/06/2021 1827.15 36.2313 440 | 13:08:43 02/06/2021 1822.25 36.026 441 | 13:09:01 02/06/2021 1987.05 35.7993 442 | 13:09:19 02/06/2021 2379.0 35.7221 443 | 13:09:37 02/06/2021 1764.95 35.7071 444 | 13:09:55 02/06/2021 1852.25 35.6337 445 | 13:10:13 02/06/2021 1769.9 35.4913 446 | 13:10:31 02/06/2021 1914.7 35.1974 447 | 13:10:49 02/06/2021 1979.5 34.8789 448 | 13:11:07 02/06/2021 1807.35 34.5402 449 | 13:11:25 02/06/2021 1802.4 34.1782 450 | 13:11:42 02/06/2021 1792.45 30.8439 451 | 13:12:00 02/06/2021 1984.5 31.9756 452 | 13:12:18 02/06/2021 1979.55 32.0812 453 | 13:12:36 02/06/2021 1822.3 31.8947 454 | 13:12:54 02/06/2021 1929.6 31.6699 455 | 13:13:13 02/06/2021 1867.4 31.3802 456 | 13:13:31 02/06/2021 1867.2 31.1244 457 | 13:13:49 02/06/2021 1929.6 30.8611 458 | 13:14:07 02/06/2021 1724.85 30.6479 459 | 13:14:25 02/06/2021 1947.15 30.4496 460 | 13:14:43 02/06/2021 1887.2 30.4451 461 | 13:15:01 02/06/2021 1772.35 30.4749 462 | 13:15:19 02/06/2021 1632.5 30.4062 463 | 13:15:37 02/06/2021 1739.9 30.2618 464 | 13:15:55 02/06/2021 1744.85 30.2914 465 | 13:16:13 02/06/2021 1869.8 30.3211 466 | 13:16:31 02/06/2021 1877.15 30.2416 467 | 13:16:49 02/06/2021 1807.35 30.209 468 | 13:17:07 02/06/2021 1902.25 30.182 469 | 13:17:25 02/06/2021 1879.85 29.991 470 | 13:17:43 02/06/2021 1872.3 29.986 471 | 13:18:01 02/06/2021 1677.6 30.0412 472 | 13:18:19 02/06/2021 1937.15 30.1274 473 | 13:18:37 02/06/2021 2034.45 30.2414 474 | 13:18:55 02/06/2021 1869.7 30.3431 475 | 13:19:13 02/06/2021 2019.6 30.4722 476 | 13:19:31 02/06/2021 1874.85 30.5829 477 | 13:19:49 02/06/2021 1762.4 30.4017 478 | 13:20:07 02/06/2021 1987.0 30.3695 479 | 13:20:25 02/06/2021 1907.25 30.3434 480 | 13:20:43 02/06/2021 1927.1 30.0934 481 | 13:21:01 02/06/2021 1839.7 29.9129 482 | 13:21:19 02/06/2021 1972.05 29.9789 483 | 13:21:37 02/06/2021 1842.3 30.1083 484 | 13:21:56 02/06/2021 1862.35 30.1477 485 | 13:22:14 02/06/2021 1917.2 30.0869 486 | 13:22:32 02/06/2021 2184.25 29.8069 487 | 13:22:50 02/06/2021 1772.25 29.5557 488 | 13:23:08 02/06/2021 1962.05 29.3625 489 | 13:23:26 02/06/2021 1737.45 29.1852 490 | 13:23:44 02/06/2021 1934.65 28.907 491 | 13:24:02 02/06/2021 1969.65 28.5931 492 | 13:24:20 02/06/2021 1717.4 28.262 493 | 13:24:38 02/06/2021 1976.95 27.928 494 | 13:24:56 02/06/2021 1675.1 27.7139 495 | 13:25:14 02/06/2021 1884.7 27.6879 496 | 13:25:32 02/06/2021 2196.7 27.982 497 | 13:25:50 02/06/2021 1879.65 28.3038 498 | 13:26:08 02/06/2021 1959.65 28.556 499 | 13:26:26 02/06/2021 1769.85 28.8058 500 | 13:26:44 02/06/2021 1832.25 28.9935 501 | 13:27:02 02/06/2021 1802.35 28.9739 502 | 13:27:20 02/06/2021 1877.3 28.8418 503 | 13:27:38 02/06/2021 1699.95 28.8892 504 | 13:27:56 02/06/2021 1837.3 29.055 505 | 13:28:15 02/06/2021 2256.7 29.476 506 | 13:28:33 02/06/2021 1730.0 29.9473 507 | 13:28:51 02/06/2021 1859.75 30.4897 508 | 13:29:09 02/06/2021 1874.7 30.7983 509 | 13:29:27 02/06/2021 1784.9 30.9655 510 | 13:29:45 02/06/2021 1719.95 30.8944 511 | 13:30:03 02/06/2021 1879.7 30.5644 512 | 13:30:21 02/06/2021 1854.65 30.4252 513 | 13:30:39 02/06/2021 1909.65 30.3802 514 | 13:30:57 02/06/2021 2136.75 30.2688 515 | 13:31:15 02/06/2021 1897.2 30.1455 516 | 13:31:33 02/06/2021 1832.45 29.927 517 | 13:31:51 02/06/2021 2009.65 29.6259 518 | 13:32:09 02/06/2021 1922.2 29.4502 519 | 13:32:27 02/06/2021 1999.6 29.6126 520 | 13:32:45 02/06/2021 2116.85 29.7707 521 | 13:33:03 02/06/2021 2054.45 29.9166 522 | 13:33:21 02/06/2021 2159.3 30.0309 523 | 13:33:39 02/06/2021 1957.15 30.0522 524 | 13:33:57 02/06/2021 2066.85 30.08 525 | 13:34:15 02/06/2021 2047.0 29.9501 526 | 13:34:33 02/06/2021 2086.95 29.904 527 | 13:34:51 02/06/2021 1759.95 29.9149 528 | 13:35:10 02/06/2021 1937.15 30.0791 529 | 13:35:28 02/06/2021 1949.55 30.3695 530 | 13:35:46 02/06/2021 2007.0 30.7088 531 | 13:36:04 02/06/2021 2049.5 31.1123 532 | 13:36:22 02/06/2021 1914.7 31.3679 533 | 13:36:40 02/06/2021 1774.8 31.5118 534 | 13:36:58 02/06/2021 1749.95 31.6957 535 | 13:37:16 02/06/2021 2037.0 31.8076 536 | 13:37:34 02/06/2021 1984.55 31.7869 537 | 13:37:52 02/06/2021 2049.55 31.6185 538 | 13:38:10 02/06/2021 2019.65 31.6534 539 | 13:38:28 02/06/2021 2109.45 31.7214 540 | 13:38:46 02/06/2021 1745.0 31.7323 541 | 13:39:04 02/06/2021 3055.45 32.4128 542 | 13:39:22 02/06/2021 1882.25 32.3946 543 | 13:39:40 02/06/2021 1710.0 32.3704 544 | 13:39:58 02/06/2021 1725.0 32.5422 545 | 13:40:16 02/06/2021 1770.0 32.7917 546 | 13:40:34 02/06/2021 1864.65 32.9522 547 | 13:40:52 02/06/2021 2066.95 32.9954 548 | 13:41:10 02/06/2021 1809.8 33.0973 549 | 13:41:29 02/06/2021 1762.4 33.3387 550 | 13:41:47 02/06/2021 1737.45 33.5979 551 | 13:42:05 02/06/2021 1769.95 33.5267 552 | 13:42:23 02/06/2021 1952.1 33.2974 553 | 13:42:41 02/06/2021 1817.35 33.255 554 | 13:42:59 02/06/2021 2234.15 33.3038 555 | 13:43:17 02/06/2021 2064.4 33.59 556 | 13:43:35 02/06/2021 2254.2 33.9541 557 | 13:43:53 02/06/2021 1977.1 34.2103 558 | 13:44:11 02/06/2021 1807.5 34.4455 559 | 13:44:29 02/06/2021 1789.85 34.6811 560 | 13:44:47 02/06/2021 1907.3 34.7225 561 | 13:45:05 02/06/2021 2034.55 34.7765 562 | 13:45:23 02/06/2021 1722.45 35.0109 563 | 13:45:41 02/06/2021 1817.35 35.2506 564 | 13:45:59 02/06/2021 2229.25 35.3215 565 | 13:46:17 02/06/2021 1892.2 35.397 566 | 13:46:35 02/06/2021 1545.25 29.3859 567 | 13:46:53 02/06/2021 2109.35 31.0015 568 | 13:47:11 02/06/2021 1922.2 31.5947 569 | 13:47:29 02/06/2021 1769.9 31.8699 570 | 13:47:47 02/06/2021 2022.0 32.2574 571 | 13:48:05 02/06/2021 1692.55 32.5543 572 | 13:48:23 02/06/2021 1774.9 32.7523 573 | 13:48:41 02/06/2021 1924.7 32.8755 574 | 13:48:59 02/06/2021 1879.7 32.9911 575 | 13:49:17 02/06/2021 1779.8 33.0691 576 | 13:49:35 02/06/2021 1959.55 33.0999 577 | 13:49:53 02/06/2021 1792.45 33.0541 578 | 13:50:12 02/06/2021 1772.45 33.0895 579 | 13:50:30 02/06/2021 2022.0 32.9585 580 | 13:50:48 02/06/2021 1799.75 32.8865 581 | 13:51:06 02/06/2021 2314.1 33.0413 582 | 13:51:24 02/06/2021 1959.8 33.2516 583 | 13:51:42 02/06/2021 1864.6 33.359 584 | 13:52:00 02/06/2021 1924.7 33.5173 585 | 13:52:18 02/06/2021 1959.5 33.8219 586 | 13:52:36 02/06/2021 1879.8 34.0237 587 | 13:52:54 02/06/2021 1932.05 32.3193 588 | 13:53:12 02/06/2021 1824.85 32.9518 589 | 13:53:30 02/06/2021 2109.4 33.4239 590 | 13:53:48 02/06/2021 1660.0 31.9983 591 | 13:54:06 02/06/2021 1977.15 32.2559 592 | 13:54:24 02/06/2021 1849.7 32.9798 593 | 13:54:42 02/06/2021 1792.4 33.3766 594 | 13:55:00 02/06/2021 2101.75 33.5416 595 | 13:55:18 02/06/2021 1952.2 33.7274 596 | 13:55:36 02/06/2021 1834.75 33.7937 597 | 13:55:54 02/06/2021 1712.5 33.8407 598 | 13:56:12 02/06/2021 1690.0 32.0412 599 | 13:56:30 02/06/2021 2101.8 33.1794 600 | 13:56:48 02/06/2021 1899.65 33.8915 601 | 13:57:06 02/06/2021 2024.55 34.3194 602 | 13:57:24 02/06/2021 1924.55 34.6787 603 | 13:57:42 02/06/2021 2001.95 34.9495 604 | 13:58:00 02/06/2021 1854.75 35.0875 605 | 13:58:18 02/06/2021 1857.35 35.2029 606 | 13:58:36 02/06/2021 1692.45 35.3286 607 | 13:58:54 02/06/2021 1939.75 35.5815 608 | 13:59:13 02/06/2021 1892.15 35.7411 609 | 13:59:31 02/06/2021 1700.0 35.586 610 | 13:59:49 02/06/2021 1827.25 35.2916 611 | 14:00:06 02/06/2021 1672.55 32.7643 612 | 14:00:24 02/06/2021 1727.45 34.4821 613 | 14:00:42 02/06/2021 2224.25 35.0632 614 | 14:01:00 02/06/2021 2199.25 35.508 615 | 14:01:18 02/06/2021 1942.2 35.597 616 | 14:01:36 02/06/2021 1917.2 34.5586 617 | 14:01:54 02/06/2021 1962.15 33.9789 618 | 14:02:13 02/06/2021 1924.65 34.339 619 | 14:02:31 02/06/2021 2151.9 34.435 620 | 14:02:49 02/06/2021 1777.4 34.5929 621 | 14:03:07 02/06/2021 1996.95 34.5869 622 | 14:03:25 02/06/2021 1747.35 34.5341 623 | 14:03:43 02/06/2021 1937.15 34.4811 624 | 14:04:01 02/06/2021 1797.3 34.4663 625 | 14:04:19 02/06/2021 2042.0 34.4086 626 | 14:04:37 02/06/2021 1799.95 34.2299 627 | 14:04:55 02/06/2021 1902.05 34.1071 628 | 14:05:13 02/06/2021 1754.95 34.0614 629 | 14:05:31 02/06/2021 1894.8 33.8937 630 | 14:05:49 02/06/2021 2111.7 32.6404 631 | 14:06:07 02/06/2021 1877.2 32.1564 632 | 14:06:25 02/06/2021 1929.65 30.6826 633 | 14:06:43 02/06/2021 1852.25 31.6534 634 | 14:07:02 02/06/2021 1842.25 32.0948 635 | 14:07:20 02/06/2021 1927.15 32.3112 636 | 14:07:38 02/06/2021 1899.65 32.3906 637 | 14:07:55 02/06/2021 1602.7 29.7824 638 | 14:08:13 02/06/2021 1749.85 31.5087 639 | 14:08:31 02/06/2021 1984.5 32.1447 640 | 14:08:49 02/06/2021 1697.5 32.4548 641 | 14:09:07 02/06/2021 1874.7 32.5653 642 | 14:09:25 02/06/2021 1984.65 32.6937 643 | 14:09:43 02/06/2021 2022.1 32.6962 644 | 14:10:02 02/06/2021 1962.15 32.6726 645 | 14:10:20 02/06/2021 2067.0 32.6716 646 | 14:10:38 02/06/2021 1972.1 32.5896 647 | 14:10:56 02/06/2021 2114.4 32.6578 648 | 14:11:14 02/06/2021 1889.7 32.8076 649 | 14:11:32 02/06/2021 1957.1 32.9782 650 | 14:11:50 02/06/2021 2021.9 32.9856 651 | 14:12:08 02/06/2021 1849.8 32.9551 652 | 14:12:26 02/06/2021 2151.85 32.8629 653 | 14:12:44 02/06/2021 1852.2 32.822 654 | 14:13:02 02/06/2021 1922.15 32.8984 655 | 14:13:20 02/06/2021 1822.3 33.0974 656 | 14:13:38 02/06/2021 1722.45 33.1535 657 | 14:13:56 02/06/2021 2199.15 33.0325 658 | 14:14:14 02/06/2021 2189.15 32.8149 659 | 14:14:33 02/06/2021 1807.3 32.5841 660 | 14:14:51 02/06/2021 1867.3 32.3149 661 | 14:15:09 02/06/2021 1902.2 32.0126 662 | 14:15:27 02/06/2021 1650.1 31.8007 663 | 14:15:45 02/06/2021 1852.25 31.6299 664 | 14:16:03 02/06/2021 1919.65 31.4379 665 | 14:16:21 02/06/2021 1882.2 31.3087 666 | 14:16:39 02/06/2021 2066.95 31.2037 667 | 14:16:57 02/06/2021 1842.3 30.8109 668 | 14:17:15 02/06/2021 1999.45 30.4757 669 | 14:17:33 02/06/2021 2012.05 30.3111 670 | 14:17:51 02/06/2021 1867.2 30.1837 671 | 14:18:08 02/06/2021 1517.75 28.4969 672 | 14:18:26 02/06/2021 2161.9 28.9476 673 | 14:18:45 02/06/2021 1882.25 29.7856 674 | 14:19:03 02/06/2021 2007.15 30.3362 675 | 14:19:21 02/06/2021 1725.0 30.7588 676 | 14:19:39 02/06/2021 1717.45 30.9254 677 | 14:19:57 02/06/2021 1762.35 31.0112 678 | 14:20:15 02/06/2021 1822.25 30.8758 679 | 14:20:33 02/06/2021 1857.15 30.7399 680 | 14:20:51 02/06/2021 1842.15 30.7843 681 | 14:21:09 02/06/2021 1907.2 30.7145 682 | 14:21:27 02/06/2021 1917.1 30.4854 683 | 14:21:45 02/06/2021 1812.4 30.3348 684 | 14:22:03 02/06/2021 1725.0 30.364 685 | 14:22:21 02/06/2021 1922.2 30.4149 686 | 14:22:39 02/06/2021 1939.75 30.3683 687 | 14:22:57 02/06/2021 1864.75 30.4266 688 | 14:23:15 02/06/2021 1837.3 30.1906 689 | 14:23:33 02/06/2021 1844.85 30.1126 690 | 14:23:51 02/06/2021 1962.15 30.1194 691 | 14:24:10 02/06/2021 2077.1 30.0571 692 | 14:24:28 02/06/2021 1777.35 29.8476 693 | 14:24:46 02/06/2021 1892.45 29.5616 694 | 14:25:04 02/06/2021 1967.05 29.3302 695 | 14:25:22 02/06/2021 1622.7 29.4656 696 | 14:25:40 02/06/2021 2374.1 30.4878 697 | 14:25:58 02/06/2021 1984.55 31.1662 698 | 14:26:16 02/06/2021 1994.5 31.5931 699 | 14:26:34 02/06/2021 2032.1 31.9207 700 | 14:26:52 02/06/2021 1937.15 31.8661 701 | 14:27:10 02/06/2021 1939.6 31.7664 702 | 14:27:28 02/06/2021 2044.5 31.921 703 | 14:27:46 02/06/2021 2034.5 32.1735 704 | 14:28:04 02/06/2021 2131.75 32.1894 705 | 14:28:22 02/06/2021 1612.75 32.0509 706 | 14:28:41 02/06/2021 1849.75 31.9978 707 | 14:28:59 02/06/2021 1984.55 32.2921 708 | 14:29:17 02/06/2021 1922.2 32.552 709 | 14:29:35 02/06/2021 2047.1 32.5194 710 | 14:29:53 02/06/2021 1997.1 32.4421 711 | 14:30:11 02/06/2021 1894.7 32.4186 712 | 14:30:29 02/06/2021 1675.15 32.4684 713 | 14:30:47 02/06/2021 1924.7 32.55 714 | 14:31:05 02/06/2021 1912.25 32.6744 715 | 14:31:23 02/06/2021 1857.1 32.8618 716 | 14:31:41 02/06/2021 2019.55 33.165 717 | 14:31:59 02/06/2021 2249.15 33.2983 718 | 14:32:17 02/06/2021 1819.9 33.4536 719 | 14:32:35 02/06/2021 1827.25 33.6459 720 | 14:32:53 02/06/2021 1877.2 33.9296 721 | 14:33:11 02/06/2021 1857.2 34.0978 722 | 14:33:30 02/06/2021 1904.55 34.0486 723 | 14:33:48 02/06/2021 1894.6 34.1683 724 | 14:34:06 02/06/2021 1807.4 34.2413 725 | 14:34:24 02/06/2021 2189.2 34.1381 726 | 14:34:42 02/06/2021 2089.45 34.2623 727 | 14:35:00 02/06/2021 1762.35 34.3609 728 | 14:35:18 02/06/2021 1944.5 34.4821 729 | 14:35:36 02/06/2021 1864.65 34.4553 730 | 14:35:54 02/06/2021 1784.95 34.2412 731 | 14:36:12 02/06/2021 2056.85 34.1421 732 | 14:36:30 02/06/2021 1869.65 34.1549 733 | 14:36:48 02/06/2021 1969.55 35.0548 734 | 14:37:06 02/06/2021 2216.8 35.8202 735 | 14:37:24 02/06/2021 1794.85 36.4024 736 | 14:37:42 02/06/2021 2022.05 36.8417 737 | 14:38:00 02/06/2021 1844.75 37.1831 738 | 14:38:18 02/06/2021 1802.25 37.4898 739 | 14:38:37 02/06/2021 1764.85 37.6635 740 | 14:38:55 02/06/2021 1687.6 37.647 741 | 14:39:13 02/06/2021 2214.15 37.5891 742 | 14:39:31 02/06/2021 1949.65 38.5249 743 | 14:39:49 02/06/2021 2246.8 39.1436 744 | 14:40:07 02/06/2021 1597.55 39.4579 745 | 14:40:25 02/06/2021 2401.45 39.9309 746 | 14:40:43 02/06/2021 1839.75 40.3944 747 | 14:41:01 02/06/2021 2116.8 40.8097 748 | 14:41:18 02/06/2021 3404.7 38.5286 749 | 14:41:36 02/06/2021 1717.6 37.8873 750 | 14:41:54 02/06/2021 2116.8 38.8695 751 | 14:42:12 02/06/2021 2101.8 39.2078 752 | 14:42:31 02/06/2021 2074.4 39.4891 753 | 14:42:49 02/06/2021 1647.45 39.7414 754 | 14:43:07 02/06/2021 2006.95 39.9146 755 | 14:43:25 02/06/2021 1799.85 40.0838 756 | 14:43:43 02/06/2021 1672.45 40.2646 757 | 14:44:01 02/06/2021 2144.35 40.3331 758 | 14:44:19 02/06/2021 1884.7 40.3668 759 | 14:44:37 02/06/2021 1839.8 40.2921 760 | 14:44:55 02/06/2021 1779.95 40.1964 761 | 14:45:13 02/06/2021 2049.35 40.3075 762 | 14:45:31 02/06/2021 1847.25 40.4748 763 | 14:45:49 02/06/2021 1957.1 40.4755 764 | 14:46:07 02/06/2021 2067.15 40.3382 765 | 14:46:25 02/06/2021 1677.6 37.9389 766 | 14:46:43 02/06/2021 1897.25 39.4169 767 | 14:47:01 02/06/2021 1842.3 40.1037 768 | 14:47:19 02/06/2021 1834.7 40.4258 769 | 14:47:37 02/06/2021 1759.95 40.511 770 | 14:47:55 02/06/2021 1777.4 40.4708 771 | 14:48:13 02/06/2021 1932.2 40.5033 772 | 14:48:31 02/06/2021 2186.75 40.57 773 | 14:48:49 02/06/2021 1744.95 40.8084 774 | 14:49:07 02/06/2021 2124.4 38.5391 775 | 14:49:26 02/06/2021 1794.85 36.946 776 | 14:49:44 02/06/2021 1942.2 37.4115 777 | 14:50:02 02/06/2021 1657.55 37.6726 778 | 14:50:20 02/06/2021 1997.2 37.8021 779 | 14:50:38 02/06/2021 2374.35 37.6474 780 | 14:50:55 02/06/2021 1777.45 37.5306 781 | 14:51:13 02/06/2021 1777.6 37.6132 782 | 14:51:31 02/06/2021 1959.7 37.8098 783 | 14:51:49 02/06/2021 1772.4 36.5028 784 | 14:52:07 02/06/2021 1742.45 37.157 785 | 14:52:25 02/06/2021 1912.1 38.4688 786 | 14:52:44 02/06/2021 1899.65 39.0846 787 | 14:53:02 02/06/2021 1887.25 39.421 788 | 14:53:20 02/06/2021 1752.45 39.7674 789 | 14:53:38 02/06/2021 1959.65 40.042 790 | 14:53:56 02/06/2021 1909.85 40.2586 791 | 14:54:14 02/06/2021 1794.75 40.3544 792 | 14:54:32 02/06/2021 1992.15 40.3065 793 | 14:54:50 02/06/2021 1749.95 40.4174 794 | 14:55:08 02/06/2021 1775.0 40.4156 795 | 14:55:26 02/06/2021 1842.2 40.4204 796 | 14:55:44 02/06/2021 1872.3 40.4974 797 | 14:56:02 02/06/2021 1610.05 40.4642 798 | 14:56:20 02/06/2021 2129.3 40.4384 799 | 14:56:39 02/06/2021 1759.9 40.3955 800 | 14:56:57 02/06/2021 1799.8 40.1978 801 | 14:57:15 02/06/2021 1907.15 39.8889 802 | 14:57:33 02/06/2021 1959.75 38.5183 803 | 14:57:51 02/06/2021 2139.3 37.9115 804 | 14:58:09 02/06/2021 2084.4 38.4731 805 | 14:58:27 02/06/2021 2042.05 38.9051 806 | 14:58:45 02/06/2021 1894.75 39.0785 807 | 14:59:03 02/06/2021 2122.0 39.1519 808 | 14:59:21 02/06/2021 2291.55 39.1597 809 | 14:59:39 02/06/2021 1924.6 39.0446 810 | 14:59:57 02/06/2021 1882.25 38.8652 811 | 15:00:15 02/06/2021 1927.15 38.7654 812 | 15:00:33 02/06/2021 2394.05 38.5152 813 | 15:00:51 02/06/2021 1622.55 34.7273 814 | 15:01:09 02/06/2021 1884.65 34.0344 815 | 15:01:27 02/06/2021 2426.45 35.4571 816 | 15:01:45 02/06/2021 1937.25 36.1846 817 | 15:02:03 02/06/2021 1949.65 36.4969 818 | 15:02:21 02/06/2021 1814.8 36.6888 819 | 15:02:39 02/06/2021 2114.25 36.7231 820 | 15:02:57 02/06/2021 1937.25 36.7933 821 | 15:03:15 02/06/2021 1844.65 36.77 822 | 15:03:33 02/06/2021 2039.5 36.7381 823 | 15:03:51 02/06/2021 1877.2 36.6502 824 | 15:04:09 02/06/2021 1972.15 36.5755 825 | 15:04:27 02/06/2021 1764.85 36.4594 826 | 15:04:46 02/06/2021 1877.2 35.4119 827 | 15:05:04 02/06/2021 1799.8 35.0851 828 | 15:05:22 02/06/2021 1912.25 35.7362 829 | 15:05:40 02/06/2021 1994.5 35.981 830 | 15:05:58 02/06/2021 1777.35 35.9637 831 | 15:06:16 02/06/2021 1824.8 35.8985 832 | 15:06:34 02/06/2021 1892.15 35.8916 833 | 15:06:52 02/06/2021 2136.8 36.1068 834 | 15:07:10 02/06/2021 1742.6 36.3017 835 | 15:07:28 02/06/2021 1762.4 33.6816 836 | 15:07:46 02/06/2021 2054.35 35.2211 837 | 15:08:04 02/06/2021 1952.0 35.8538 838 | 15:08:22 02/06/2021 2129.3 36.183 839 | 15:08:40 02/06/2021 1904.7 36.3467 840 | 15:08:58 02/06/2021 1690.1 36.5505 841 | 15:09:16 02/06/2021 1872.3 36.6388 842 | 15:09:34 02/06/2021 2081.95 36.7712 843 | 15:09:52 02/06/2021 2091.9 36.9436 844 | 15:10:10 02/06/2021 2082.1 37.1409 845 | 15:10:28 02/06/2021 1822.4 37.1751 846 | 15:10:46 02/06/2021 2106.9 37.1505 847 | 15:11:04 02/06/2021 2002.2 37.3043 848 | 15:11:22 02/06/2021 1824.85 37.5523 849 | 15:11:40 02/06/2021 1989.55 35.4878 850 | 15:11:58 02/06/2021 1977.1 35.3381 851 | 15:12:16 02/06/2021 1812.25 36.1577 852 | 15:12:34 02/06/2021 2204.25 32.3433 853 | 15:12:52 02/06/2021 1974.5 33.0986 854 | 15:13:10 02/06/2021 2024.6 33.4556 855 | 15:13:28 02/06/2021 2096.9 33.5662 856 | 15:13:46 02/06/2021 1859.75 33.4824 857 | 15:14:04 02/06/2021 1747.5 33.5268 858 | 15:14:22 02/06/2021 2032.0 33.7381 859 | 15:14:40 02/06/2021 1660.0 34.0267 860 | 15:14:58 02/06/2021 1877.25 34.1607 861 | 15:15:17 02/06/2021 2129.4 34.2692 862 | 15:15:35 02/06/2021 1672.65 34.2366 863 | 15:15:53 02/06/2021 1817.45 34.1914 864 | 15:16:11 02/06/2021 2177.05 33.9887 865 | 15:16:29 02/06/2021 1847.35 33.9644 866 | 15:16:47 02/06/2021 1847.45 33.9196 867 | 15:17:05 02/06/2021 1602.8 34.6717 868 | 15:17:23 02/06/2021 1894.7 35.5834 869 | 15:17:41 02/06/2021 1837.45 35.7057 870 | 15:17:59 02/06/2021 1687.75 36.0122 871 | 15:18:17 02/06/2021 1894.75 36.2764 872 | 15:18:35 02/06/2021 1919.8 36.5425 873 | 15:18:52 02/06/2021 1702.45 30.1472 874 | 15:19:09 02/06/2021 1615.25 30.09 875 | 15:19:26 02/06/2021 1882.3 29.9909 876 | 15:19:43 02/06/2021 1495.3 29.7404 877 | 15:19:59 02/06/2021 1792.3 29.5602 878 | 15:20:16 02/06/2021 1942.2 29.5275 879 | 15:20:33 02/06/2021 1682.55 29.3292 880 | 15:20:50 02/06/2021 1722.45 29.0396 881 | 15:21:07 02/06/2021 1560.25 28.8039 882 | 15:21:24 02/06/2021 1587.7 28.7597 883 | 15:21:40 02/06/2021 1842.35 28.5977 884 | 15:21:57 02/06/2021 1697.5 28.3254 885 | 15:22:14 02/06/2021 1650.15 28.2367 886 | 15:22:31 02/06/2021 1855.0 28.3237 887 | 15:22:48 02/06/2021 1812.45 28.2839 888 | 15:23:05 02/06/2021 1665.2 28.128 889 | 15:23:21 02/06/2021 1854.7 27.8383 890 | 15:23:38 02/06/2021 1677.45 27.4886 891 | 15:23:55 02/06/2021 1642.55 27.3128 892 | 15:24:12 02/06/2021 1632.65 27.4007 893 | 15:24:29 02/06/2021 1727.5 27.581 894 | 15:24:46 02/06/2021 1700.0 27.7529 895 | 15:25:03 02/06/2021 1567.55 27.7537 896 | 15:25:19 02/06/2021 1615.2 27.7 897 | 15:25:36 02/06/2021 1784.8 27.6819 898 | 15:25:53 02/06/2021 1739.9 27.7611 899 | 15:26:10 02/06/2021 1627.55 27.9527 900 | 15:26:27 02/06/2021 1944.5 28.2621 901 | 15:26:44 02/06/2021 1772.45 28.5726 902 | 15:27:01 02/06/2021 1547.65 28.814 903 | 15:27:17 02/06/2021 1597.7 28.9952 904 | 15:27:34 02/06/2021 1627.7 29.0762 905 | 15:27:51 02/06/2021 1952.15 29.0141 906 | 15:28:08 02/06/2021 1622.55 28.8404 907 | 15:28:25 02/06/2021 1820.05 28.8511 908 | 15:28:41 02/06/2021 1590.05 28.9261 909 | 15:28:57 02/06/2021 1714.85 28.7877 910 | 15:29:14 02/06/2021 1747.4 28.5979 911 | 15:29:30 02/06/2021 1809.8 28.4732 912 | 15:29:46 02/06/2021 1784.85 28.4408 913 | 15:30:02 02/06/2021 1677.55 28.3579 914 | 15:30:18 02/06/2021 1607.6 28.2399 915 | 15:30:34 02/06/2021 1707.5 28.0274 916 | 15:30:50 02/06/2021 1552.7 28.0309 917 | 15:31:06 02/06/2021 1857.25 28.2438 918 | 15:31:22 02/06/2021 1595.15 28.4866 919 | 15:31:38 02/06/2021 1777.3 28.6647 920 | 15:31:54 02/06/2021 2014.65 28.4274 921 | 15:32:10 02/06/2021 1552.6 28.1043 922 | 15:32:26 02/06/2021 1602.7 27.7982 923 | 15:32:42 02/06/2021 1714.95 27.6857 924 | 15:32:58 02/06/2021 1367.9 27.7203 925 | 15:33:14 02/06/2021 1575.15 27.6982 926 | 15:33:30 02/06/2021 1647.55 27.7981 927 | 15:33:46 02/06/2021 1672.5 28.0329 928 | 15:34:02 02/06/2021 1592.55 28.2127 929 | 15:34:19 02/06/2021 1772.35 28.2235 930 | 15:34:35 02/06/2021 1637.6 28.1226 931 | 15:34:51 02/06/2021 1712.4 28.0892 932 | 15:35:07 02/06/2021 1892.25 28.1174 933 | 15:35:23 02/06/2021 1769.85 28.0785 934 | 15:35:39 02/06/2021 1532.75 27.9444 935 | 15:35:55 02/06/2021 1904.8 27.9268 936 | 15:36:11 02/06/2021 1794.9 28.211 937 | 15:36:27 02/06/2021 1749.9 28.3748 938 | 15:36:43 02/06/2021 1762.5 28.5152 939 | 15:36:59 02/06/2021 1587.7 28.6245 940 | 15:37:15 02/06/2021 1635.0 28.6676 941 | 15:37:31 02/06/2021 1760.05 28.7268 942 | 15:37:47 02/06/2021 1660.0 28.7732 943 | 15:38:03 02/06/2021 1669.95 28.7457 944 | 15:38:19 02/06/2021 1909.65 28.7717 945 | 15:38:35 02/06/2021 1997.0 28.7878 946 | 15:38:51 02/06/2021 1834.9 28.9027 947 | 15:39:07 02/06/2021 1602.75 29.1374 948 | 15:39:23 02/06/2021 1662.5 29.3266 949 | 15:39:39 02/06/2021 1902.2 29.333 950 | 15:39:55 02/06/2021 1512.75 29.3937 951 | 15:40:11 02/06/2021 1794.8 29.4806 952 | 15:40:27 02/06/2021 1592.65 29.6345 953 | 15:40:44 02/06/2021 1804.9 29.5641 954 | 15:41:00 02/06/2021 1974.55 29.4625 955 | 15:41:16 02/06/2021 1717.35 29.4454 956 | 15:41:32 02/06/2021 1587.65 29.2481 957 | 15:41:48 02/06/2021 1932.2 29.2694 958 | 15:42:04 02/06/2021 2084.45 29.4312 959 | 15:42:20 02/06/2021 1572.55 29.5965 960 | 15:42:36 02/06/2021 1887.2 29.7581 961 | 15:42:52 02/06/2021 1932.15 29.8101 962 | 15:43:08 02/06/2021 1829.75 29.7928 963 | 15:43:24 02/06/2021 1719.9 29.8147 964 | 15:43:40 02/06/2021 1632.6 29.7464 965 | 15:43:56 02/06/2021 1667.5 29.6496 966 | 15:44:12 02/06/2021 1927.1 29.6691 967 | 15:44:28 02/06/2021 1847.3 27.8605 968 | 15:44:43 02/06/2021 1390.35 26.175 969 | 15:44:58 02/06/2021 1428.0 26.3985 970 | 15:45:13 02/06/2021 1550.25 26.4879 971 | 15:45:27 02/06/2021 1415.45 26.3612 972 | 15:45:42 02/06/2021 1460.3 26.1939 973 | 15:45:57 02/06/2021 1755.05 26.1264 974 | 15:46:12 02/06/2021 1298.15 26.1553 975 | 15:46:27 02/06/2021 1585.0 26.2094 976 | 15:46:42 02/06/2021 1530.25 26.2239 977 | 15:46:57 02/06/2021 1520.2 26.0947 978 | 15:47:11 02/06/2021 1382.95 26.1087 979 | 15:47:26 02/06/2021 1642.6 26.0291 980 | 15:47:40 02/06/2021 1205.7 22.2234 981 | 15:47:54 02/06/2021 1412.95 22.0757 982 | 15:48:07 02/06/2021 1400.4 22.0048 983 | 15:48:21 02/06/2021 1490.3 21.9465 984 | 15:48:34 02/06/2021 1435.4 21.9518 985 | 15:48:48 02/06/2021 1572.6 22.0108 986 | 15:49:02 02/06/2021 2413.75 22.8474 987 | 15:49:15 02/06/2021 1310.5 23.4102 988 | 15:49:29 02/06/2021 1515.35 23.7095 989 | 15:49:43 02/06/2021 1388.0 23.9665 990 | 15:49:56 02/06/2021 1385.35 24.2724 991 | 15:50:09 02/06/2021 1155.8 22.8334 992 | 15:50:22 02/06/2021 1123.45 21.2938 993 | 15:50:34 02/06/2021 1193.2 21.3329 994 | 15:50:46 02/06/2021 1043.5 19.8105 995 | 15:50:57 02/06/2021 1290.6 18.2784 996 | 15:51:07 02/06/2021 886.2 12.8583 997 | 15:51:17 02/06/2021 751.35 12.9684 998 | 15:51:27 02/06/2021 716.4 13.0327 999 | 15:51:37 02/06/2021 636.5 13.2045 1000 | 15:51:47 02/06/2021 694.0 13.4139 1001 | 15:51:57 02/06/2021 669.0 10.2577 1002 | 15:52:05 02/06/2021 576.7 10.4077 1003 | 15:52:14 02/06/2021 596.55 10.6135 1004 | 15:52:23 02/06/2021 641.5 10.8102 1005 | 15:52:32 02/06/2021 664.0 10.979 1006 | 15:52:41 02/06/2021 576.65 11.0856 1007 | 15:52:49 02/06/2021 848.75 11.2241 1008 | 15:52:58 02/06/2021 586.55 11.3782 1009 | 15:53:07 02/06/2021 896.2 11.5635 1010 | 15:53:16 02/06/2021 716.45 11.7658 1011 | 15:53:25 02/06/2021 614.15 11.9699 1012 | 15:53:34 02/06/2021 599.1 12.1498 1013 | 15:53:43 02/06/2021 621.6 12.279 1014 | 15:53:51 02/06/2021 541.75 12.4055 1015 | 15:54:00 02/06/2021 611.55 12.4973 1016 | 15:54:09 02/06/2021 564.2 12.6681 1017 | 15:54:18 02/06/2021 529.25 12.8196 1018 | 15:54:27 02/06/2021 499.3 12.9489 1019 | 15:54:36 02/06/2021 661.55 13.0641 1020 | 15:54:44 02/06/2021 561.7 13.1447 1021 | 15:54:53 02/06/2021 551.7 13.2132 1022 | 15:55:02 02/06/2021 609.1 13.2766 1023 | 15:55:11 02/06/2021 566.7 13.334 1024 | 15:55:20 02/06/2021 654.0 13.3808 1025 | 15:55:28 02/06/2021 731.5 13.3766 1026 | 15:55:37 02/06/2021 559.2 13.3632 1027 | 15:55:45 02/06/2021 431.9 10.6211 1028 | 15:55:53 02/06/2021 294.55 7.8723 1029 | 15:56:01 02/06/2021 339.45 7.8451 1030 | 15:56:08 02/06/2021 344.45 7.8465 1031 | 15:56:16 02/06/2021 322.0 7.8358 1032 | 15:56:24 02/06/2021 287.05 7.8527 1033 | 15:56:31 02/06/2021 409.4 7.8771 1034 | 15:56:39 02/06/2021 506.7 7.8975 1035 | 15:56:46 02/06/2021 359.45 7.962 1036 | 15:56:54 02/06/2021 374.45 7.9975 1037 | 15:57:01 02/06/2021 396.95 8.0246 1038 | 15:57:09 02/06/2021 329.5 8.0148 1039 | 15:57:17 02/06/2021 401.9 7.9655 1040 | 15:57:24 02/06/2021 761.45 7.9339 1041 | 15:57:32 02/06/2021 496.75 7.9045 1042 | 15:57:39 02/06/2021 419.4 7.8715 1043 | 15:57:46 02/06/2021 207.2 4.1883 1044 | 15:57:52 02/06/2021 182.25 4.183 1045 | 15:57:59 02/06/2021 461.85 4.2041 1046 | 15:58:05 02/06/2021 746.4 4.2513 1047 | 15:58:11 02/06/2021 753.9 4.3081 1048 | 15:58:18 02/06/2021 431.85 4.3849 1049 | 15:58:24 02/06/2021 202.2 4.4514 1050 | 15:58:31 02/06/2021 179.7 4.498 1051 | 15:58:37 02/06/2021 479.3 4.5214 1052 | 15:58:43 02/06/2021 164.75 4.5675 1053 | 15:58:50 02/06/2021 162.3 4.617 1054 | 15:58:56 02/06/2021 197.2 4.6645 1055 | 15:59:03 02/06/2021 164.75 4.7172 1056 | 15:59:09 02/06/2021 449.3 4.7575 1057 | 15:59:15 02/06/2021 187.25 4.8019 1058 | 15:59:22 02/06/2021 122.3 4.8524 1059 | 15:59:28 02/06/2021 154.75 4.8896 1060 | 15:59:35 02/06/2021 134.8 4.9314 1061 | 15:59:41 02/06/2021 139.8 4.9721 1062 | 15:59:47 02/06/2021 182.25 4.9785 1063 | 15:59:54 02/06/2021 207.2 4.9666 1064 | 16:00:00 02/06/2021 224.65 4.9328 1065 | 16:00:06 02/06/2021 204.7 4.9003 1066 | 16:00:13 02/06/2021 172.2 4.8777 1067 | -------------------------------------------------------------------------------- /ClusterMonitor/example/cpu_ram_visualization.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickhir/ClusterMonitor/ec9173daf9391b154b059253f02f05b59342932d/ClusterMonitor/example/cpu_ram_visualization.pdf -------------------------------------------------------------------------------- /ClusterMonitor/example/example_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nickhir/ClusterMonitor/ec9173daf9391b154b059253f02f05b59342932d/ClusterMonitor/example/example_image.png -------------------------------------------------------------------------------- /ClusterMonitor/visualize_log.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import matplotlib as mpl 3 | 4 | mpl.use('Agg') 5 | import matplotlib.pyplot as plt 6 | import pandas as pd 7 | import matplotlib.dates as mdates 8 | from matplotlib.backends.backend_pdf import PdfPages 9 | 10 | p = argparse.ArgumentParser() 11 | p.add_argument("-i", "--input", type=str, help="Specify the path to the created log file") 12 | p.add_argument("-o", "--output", type=str, 13 | help="Specify path to the PDF file which will contain one plot for the CPU and and one plot for the Memory usage") 14 | args = p.parse_args() 15 | 16 | df = pd.read_csv(args.input, sep="\t") 17 | df["datetime"] = pd.to_datetime(df["datetime"], format="%H:%M:%S %d/%m/%Y") 18 | print(f"Reading in {len(df)} logged entries") 19 | with PdfPages(args.output) as pdf: 20 | fig, ax = plt.subplots(figsize=(11, 7)) 21 | ax.plot(df["datetime"], df["CPU [%]"], c="r", lw=0.5) 22 | ax.fill_between(df["datetime"], df["CPU [%]"], color="r", alpha=0.3) 23 | 24 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M %d/%m")) 25 | ax.xaxis.set_minor_formatter(mdates.DateFormatter("%H:%M %d/%m")) 26 | 27 | plt.xticks(rotation=45) 28 | plt.title("CPU Usage") 29 | plt.ylabel("CPU Usage [%]") 30 | plt.tight_layout() 31 | pdf.savefig() 32 | plt.close() 33 | 34 | fig, ax = plt.subplots(figsize=(11, 7)) 35 | ax.plot(df["datetime"], df["Memory [GB]"], c="b", lw=0.5) 36 | ax.fill_between(df["datetime"], df["Memory [GB]"], color="b", alpha=0.3) 37 | 38 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M %d/%m")) 39 | ax.xaxis.set_minor_formatter(mdates.DateFormatter("%H:%M %d/%m")) 40 | 41 | plt.xticks(rotation=45) 42 | plt.title("Memory usage") 43 | plt.ylabel("Memory [GB]") 44 | plt.tight_layout() 45 | pdf.savefig() 46 | plt.close() 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ClusterMonitor 2 | [![PyPI version](https://badge.fury.io/py/clustermonitor.svg)](https://badge.fury.io/py/clustermonitor)[![Downloads](https://static.pepy.tech/personalized-badge/clustermonitor?period=total&units=international_system&left_color=black&right_color=orange&left_text=Downloads)](https://pepy.tech/project/clustermonitor) 3 |
4 | 5 | ## What is it? 6 | A very simple python script to log the CPU and Memory consumption of submitted cluster jobs. 7 | 8 | ## Installation 9 | Simplest option is to install via [PyPi](https://pypi.org/): 10 | ```bash 11 | pip install ClusterMonitor 12 | ``` 13 | Alternatively you can also clone this repository and use the scripts inside the `ClusterMonitor` directory. 14 | ## Usage 15 | To start recording use the `cpu_ram_log.py` script. It requires 3 arguments: 16 | - `-u` which corresponds to your username. 17 | - `-o` which specifies the output file (in tsv format) 18 | - `--interval` which specifies the time (in seconds) over which the CPU and RAM usage gets averaged 19 | 20 | Example: 21 | ```python 22 | python ClusterMonitor/cpu_ram_log.py -u nickhir -o cpu_ram.log --interval 5 23 | ``` 24 | 25 | The script can simply be included in your cluster job submittion like this: 26 | ```bash 27 | #!/bin/bash 28 | #SBATCH --job-name=Example_run 29 | 30 | sambamba sort -t 3 some_bam.bam & 31 | MainScript=$! 32 | python cpu_ram_log.py -u nickhir -o cpu_ram.log --interval 5 & 33 | wait $MainScript 34 | pkill -P $$ 35 | ``` 36 | *** 37 | If you used `pip` to install the package, `cpu_ram_log` and `visualize_log` will be added to your `PATH` variable, thus you can simple use: 38 | ```bash 39 | cpu_ram_log -u nickhir --interval 5 -o cpu_ram.log 40 | ``` 41 | *** 42 | Afterwards the resulting log file can be plotted (for that `matplotlib` is needed). 43 | For that use the `visualize_log.py` script which needs 2 arguments: 44 | - `-i` which specifies the path to the log file which was created with `cpu_ram_log.py` 45 | - `-o` which specifies the path to the resulting PDF which contains a plot for the CPU and RAM usage. 46 | 47 | 48 | Example: 49 | ```python 50 | python visualize_log.py -i cpu_ram.log -o cpu_ram_visualization.pdf 51 | ``` 52 | 53 | The resulting plots will look somewhat like this: 54 | ![exampleImage](https://github.com/nickhir/ClusterMonitor/raw/main/ClusterMonitor/example/example_image.png) 55 | -------------------------------------------------------------------------------- /bin/cpu_ram_log: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import psutil 3 | import math 4 | import argparse 5 | import time 6 | import csv 7 | import os 8 | import datetime 9 | import sys 10 | 11 | p = argparse.ArgumentParser() 12 | p.add_argument("-u", "--username", type=str, required=True, 13 | help="Specify the username for which we record CPU and Memory usage") 14 | p.add_argument("-o", "--output", type=str, required=True, 15 | help="Path to the file where the CPU and Memory usage gets logged (in tsv format)") 16 | p.add_argument("--interval", type=int, required=True, 17 | help="Specify the time (in seconds) over which the CPU and Memory usage gets averaged") 18 | args = p.parse_args() 19 | 20 | 21 | def bytesto(bytes, to, bsize=1024): 22 | """convert bytes to megabytes, etc. 23 | sample code: 24 | print('mb= ' + str(bytesto(314575262000000, 'm'))) 25 | sample output: 26 | mb= 300002347.946 27 | """ 28 | 29 | a = {'k': 1, 'm': 2, 'g': 3, 't': 4, 'p': 5, 'e': 6} 30 | r = float(bytes) 31 | for i in range(a[to]): 32 | r = r / bsize 33 | 34 | return (r) 35 | 36 | 37 | try: 38 | print("Start data logging") 39 | while True: 40 | # record the average over X seconds, becuase cpu usage tends to fluctuate 41 | memory_avg = [] 42 | cpu_avg = [] 43 | for i in range(args.interval): 44 | memory = [] 45 | cpu_usage = [] 46 | for proc in psutil.process_iter(): 47 | try: 48 | 49 | process_info = proc.as_dict(attrs=["pid", "name", "username", "memory_info"]) 50 | if process_info["name"] == "sleep": 51 | continue 52 | # if process occupies more than 50 MB it is like not a background process. only look at these. 53 | if process_info["username"] == args.username and proc.memory_info()[0] > 10e+2: 54 | memory.append(proc.memory_info()[0]) 55 | cpu_usage.append(proc.cpu_percent(0.2)) 56 | except: 57 | pass 58 | 59 | memory_avg.append(sum(memory)) 60 | cpu_avg.append(sum(cpu_usage)) 61 | time.sleep(1) 62 | 63 | final_cpu = round(sum(cpu_avg) / len(cpu_avg), 4) 64 | final_memory = round(bytesto(sum(memory_avg) / len(memory_avg), "g"), 4) 65 | header = ['datetime', 'CPU [%]', 'Memory [GB]'] 66 | with open(args.output, "a", newline="", encoding="UTF-8") as csv_file: 67 | writer = csv.writer(csv_file, delimiter='\t') 68 | # if file is empty create the header and add the data 69 | if os.path.getsize(args.output) == 0: 70 | writer.writerow(header) 71 | writer.writerow([datetime.datetime.now().strftime("%H:%M:%S %d/%m/%Y"), final_cpu, final_memory]) 72 | else: 73 | writer.writerow([datetime.datetime.now().strftime("%H:%M:%S %d/%m/%Y"), final_cpu, final_memory]) 74 | 75 | except KeyboardInterrupt: 76 | print(f"Logging finished and saved to file {args.output}") 77 | -------------------------------------------------------------------------------- /bin/visualize_log: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import matplotlib as mpl 4 | 5 | mpl.use('Agg') 6 | import matplotlib.pyplot as plt 7 | import pandas as pd 8 | import matplotlib.dates as mdates 9 | from matplotlib.backends.backend_pdf import PdfPages 10 | 11 | p = argparse.ArgumentParser() 12 | p.add_argument("-i", "--input", type=str, help="Specify the path to the created log file") 13 | p.add_argument("-o", "--output", type=str, 14 | help="Specify path to the PDF file which will contain one plot for the CPU and and one plot for the Memory usage") 15 | args = p.parse_args() 16 | 17 | df = pd.read_csv(args.input, sep="\t") 18 | df["datetime"] = pd.to_datetime(df["datetime"], format="%H:%M:%S %d/%m/%Y") 19 | print(f"Reading in {len(df)} logged entries") 20 | with PdfPages(args.output) as pdf: 21 | fig, ax = plt.subplots(figsize=(11, 7)) 22 | ax.plot(df["datetime"], df["CPU [%]"], c="r", lw=0.5) 23 | ax.fill_between(df["datetime"], df["CPU [%]"], color="r", alpha=0.3) 24 | 25 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M %d/%m")) 26 | ax.xaxis.set_minor_formatter(mdates.DateFormatter("%H:%M %d/%m")) 27 | 28 | plt.xticks(rotation=45) 29 | plt.title("CPU Usage") 30 | plt.ylabel("CPU Usage [%]") 31 | plt.tight_layout() 32 | pdf.savefig() 33 | plt.close() 34 | 35 | fig, ax = plt.subplots(figsize=(11, 7)) 36 | ax.plot(df["datetime"], df["Memory [GB]"], c="b", lw=0.5) 37 | ax.fill_between(df["datetime"], df["Memory [GB]"], color="b", alpha=0.3) 38 | 39 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M %d/%m")) 40 | ax.xaxis.set_minor_formatter(mdates.DateFormatter("%H:%M %d/%m")) 41 | 42 | plt.xticks(rotation=45) 43 | plt.title("Memory usage") 44 | plt.ylabel("Memory [GB]") 45 | plt.tight_layout() 46 | pdf.savefig() 47 | plt.close() 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from pathlib import Path 3 | 4 | VERSION = '0.0.6' 5 | DESCRIPTION = 'Simple script which can be used to monitor and log CPU and RAM usage of submitted cluster jobs' 6 | this_directory = Path(__file__).parent 7 | LONG_DESCRIPTION = (this_directory / "README.md").read_text() 8 | 9 | # Setting up 10 | setup( 11 | name="ClusterMonitor", 12 | version=VERSION, 13 | author="nickhir", 14 | description=DESCRIPTION, 15 | long_description=LONG_DESCRIPTION, 16 | long_description_content_type="text/markdown", 17 | packages=find_packages(), 18 | install_requires=['matplotlib'], 19 | keywords=['python', 'cluster', 'monitor', 'CPU', 'RAM', 'usage', 'SLURM'], 20 | classifiers=["Operating System :: Unix"], 21 | scripts=['bin/cpu_ram_log', 'bin/visualize_log'] 22 | ) 23 | --------------------------------------------------------------------------------