├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cache-bench └── drop-caches /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DESTDIR ?= 2 | PREFIX ?= /usr/local 3 | BINDIR ?= $(PREFIX)/bin 4 | 5 | all: 6 | @ echo "Use: make install, make uninstall" 7 | 8 | install: 9 | install -p -d $(DESTDIR)$(BINDIR) 10 | install -p -m0755 cache-bench $(DESTDIR)$(BINDIR)/cache-bench 11 | install -p -m0755 drop-caches $(DESTDIR)$(BINDIR)/drop-caches 12 | 13 | uninstall: 14 | rm -fv $(DESTDIR)$(BINDIR)/cache-bench 15 | rm -fv $(DESTDIR)$(BINDIR)/drop-caches 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # cache-bench 3 | 4 | [![Total alerts](https://img.shields.io/lgtm/alerts/g/hakavlad/cache-bench.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/hakavlad/cache-bench/alerts/) 5 | 6 | Explore the impact of virtual memory settings on caching efficiency on Linux systems under memory pressure. 7 | 8 | The script reads the specified file in chunks of the specified size in random order (with `--read` option) and logs the result with a specified interval. 9 | 10 | Optionally read chunks are added to the list, thus increasing the memory consumption of the process for creating memory pressure and paging. 11 | 12 | The script can be used, for example, to assess the impact of virtual memory settings (`vm.swappiness`, `vm.watermark_scale_factor`, Multigenerational LRU Framework etc) on the efficiency of file caching, especially under memory pressure. The script allows you to evaluate the performance of I/O operations under memory pressure. 13 | 14 | ## Options 15 | 16 | ``` 17 | $ cache-bench -h 18 | usage: cache-bench [-h] [-f FILE] [-r READ] [-w WRITE] [-m MMAP] [-p PREREAD] [-b BLOAT] [-c CHUNK] [-i INTERVAL] [-l LOG] 19 | 20 | optional arguments: 21 | -h, --help show this help message and exit 22 | -f FILE, --file FILE path to the file to be read or written 23 | -r READ, --read READ how many mebibytes to read from the file 24 | -w WRITE, --write WRITE 25 | size of the file being written 26 | -m MMAP, --mmap MMAP mmap the file (0 | 1) 27 | -p PREREAD, --preread PREREAD 28 | preread the file (0 | 1) 29 | -b BLOAT, --bloat BLOAT 30 | bloat process memory (0 | 1) 31 | -c CHUNK, --chunk CHUNK 32 | chunk size in KiB 33 | -i INTERVAL, --interval INTERVAL 34 | output interval in seconds 35 | -l LOG, --log LOG path to the log file 36 | ``` 37 | 38 | #### -f FILE, --file FILE 39 | Path to the file to be read or written. Default value: `testfile.bench`. 40 | 41 | #### -r READ, --read READ 42 | How many mebibytes to read from the specified file. 43 | 44 | #### -w WRITE, --write WRITE 45 | The size of the file to create, in mebibytes. This is just an auxiliary option to create a file of the desired size. 46 | 47 | #### -m MMAP, --mmap MMAP 48 | mmap the file. Valid values: `0` and `1`. Default value: `0`. If set to `1`, the file will be memory-mapped and the reading will be done from the memory-mapped file object. 49 | 50 | #### -p PREREAD, --preread PREREAD 51 | Preread the file. Valid values: `0` and `1`. Default value: `0`. If set to `1`, the file will first be preread completely sequentially by mebibyte chunks. 52 | 53 | #### -b BLOAT, --bloat BLOAT 54 | Bloat process memory. Valid values: `0` and `1`. Default value: `0`. If set to `1`, the chunks will be added to the list and the memory consumed by the process will increase. This option can be used to create memory pressure during tests. If this option is enabled, the `oom_score_adj` of the process will be set to `1000`. Be prepared for possible freezing of the UI with `-b 1`. 55 | 56 | #### -c CHUNK, --chunk CHUNK 57 | Chunk size in KiB. Default value: `64`. The file will be read by chunks of a given size in random order. 58 | 59 | #### -i INTERVAL, --interval INTERVAL 60 | Output (log) interval in seconds. Default value: `2`. 61 | 62 | #### -l LOG, --log LOG 63 | Path to the log file. The output will be written with timestamps. 64 | 65 | ## Usage 66 | 67 | Select or create a file. You can use any existing file (with `--file` option), or create a test file of the desired size with the `--write` option: 68 | ``` 69 | $ cache-bench -w 300 70 | starting cache-bench 71 | file: testfile.bench 72 | file size: 300 MiB 73 | writing the file... 74 | fsync... 75 | OK 76 | ``` 77 | 78 | Optionally remove extraneous disk caches. You can use the `drop-caches` script: 79 | ``` 80 | $ drop-caches 81 | #!/bin/sh -v 82 | sudo sync 83 | [sudo] password for user: 84 | echo 3 | sudo tee /proc/sys/vm/drop_caches 85 | 3 86 | ``` 87 | 88 | Run the script with `-r` option. The default behavior is to read the data size specified by the `--read` option in 64k chunks in random order: 89 | ``` 90 | $ cache-bench -r 25000 -i 5 91 | starting cache-bench 92 | file: testfile.bench 93 | file size: 300.0 MiB 94 | log file is not set 95 | output interval: 5.0s 96 | mmap: 0, preread: 0, bloat: 0, chunk: 64 KiB 97 | reading 25000.0 MiB from the file... 98 | read 45.0M in 5.0s (9.0M/s); total 45.0M in 5.0s, avg 9.0M/s 99 | read 46.9M in 5.0s (9.4M/s); total 91.9M in 10.0s, avg 9.2M/s 100 | read 55.6M in 5.0s (11.1M/s); total 147.4M in 15.0s, avg 9.8M/s 101 | read 68.5M in 5.0s (13.7M/s); total 215.9M in 20.0s, avg 10.8M/s 102 | read 92.4M in 5.0s (18.5M/s); total 308.3M in 25.0s, avg 12.3M/s 103 | read 196.2M in 5.0s (39.2M/s); total 504.5M in 30.0s, avg 16.8M/s 104 | read 7619.3M in 5.0s (1523.9M/s); total 8123.8M in 35.0s, avg 231.9M/s 105 | read 16041.2M in 5.0s (3208.2M/s); total 24165.1M in 40.0s, avg 603.6M/s 106 | read 835.0M in 0.3s (3270.5M/s); total 25000.0M in 40.3s, avg 620.5M/s 107 | total read 25000.0 MiB in 40.3s (avg 620.5 MiB/s) 108 | ``` 109 | In the output of the script you can observe the current reading speed, the amount of data read per time interval, the average values during the reading time. 110 | 111 | With the `--bloat` option you can investigate the effectiveness of caching when available memory is low and the effect of the virtual memory settings on the result: 112 | ``` 113 | $ cache-bench -r 15000 -i 4 -b 1 -p 1 114 | starting cache-bench 115 | file: testfile.bench 116 | file size: 300.0 MiB 117 | log file is not set 118 | output interval: 4.0s 119 | mmap: 0, preread: 1, bloat: 1, chunk: 64 KiB 120 | prereading (caching) the file... 121 | preread 300.0 MiB (100.0%) in 2.6s 122 | reading 15000.0 MiB from the file... 123 | read 7042.6M in 4.0s (1760.7M/s); total 7042.6M in 4.0s, avg 1760.7M/s 124 | read 3541.1M in 4.0s (884.7M/s); total 10583.7M in 8.0s, avg 1322.5M/s 125 | read 68.6M in 4.0s (17.1M/s); total 10652.4M in 12.0s, avg 886.8M/s 126 | read 46.1M in 4.0s (11.5M/s); total 10698.4M in 16.0s, avg 667.8M/s 127 | read 51.3M in 4.0s (12.8M/s); total 10749.7M in 20.0s, avg 536.7M/s 128 | read 55.8M in 4.0s (13.9M/s); total 10805.5M in 24.0s, avg 449.6M/s 129 | read 74.3M in 4.0s (18.6M/s); total 10879.9M in 28.0s, avg 388.0M/s 130 | read 99.9M in 4.0s (24.9M/s); total 10979.7M in 32.0s, avg 342.7M/s 131 | read 169.5M in 4.0s (42.3M/s); total 11149.2M in 36.0s, avg 309.3M/s 132 | read 420.6M in 4.0s (105.1M/s); total 11569.8M in 40.1s, avg 288.9M/s 133 | read 2277.9M in 4.0s (569.5M/s); total 13847.7M in 44.1s, avg 314.4M/s 134 | read 1152.3M in 1.1s (1042.2M/s); total 15000.0M in 45.2s, avg 332.2M/s 135 | total read 15000.0 MiB in 45.2s (avg 332.2 MiB/s) 136 | ``` 137 | 138 | Optionally, you can specify the path to the log file. Log file example: 139 | ``` 140 | 2021-12-18 19:44:14,306: starting cache-bench 141 | 2021-12-18 19:44:14,307: file: testfile.bench 142 | 2021-12-18 19:44:14,307: file size: 200.0 MiB 143 | 2021-12-18 19:44:14,307: log file: /tmpfs/log 144 | 2021-12-18 19:44:14,307: output interval: 2s 145 | 2021-12-18 19:44:14,308: mmap: 1, preread: 1, bloat: 1, chunk: 32 KiB 146 | 2021-12-18 19:44:14,308: prereading (caching) the file... 147 | 2021-12-18 19:44:16,041: preread 200.0 MiB (100.0%) in 1.7s 148 | 2021-12-18 19:44:16,046: reading 8000.0 MiB from the file... 149 | 2021-12-18 19:44:18,047: read 3071.6M in 2.0s (1535.8M/s); total 3071.6M in 2.0s, avg 1535.8M/s 150 | 2021-12-18 19:44:20,047: read 3111.0M in 2.0s (1555.5M/s); total 6182.6M in 4.0s, avg 1545.6M/s 151 | 2021-12-18 19:44:21,233: read 1817.4M in 1.2s (1531.6M/s); total 8000.0M in 5.2s, avg 1542.4M/s 152 | 2021-12-18 19:44:21,233: total read 8000.0 MiB in 5.2s (avg 1542.4 MiB/s) 153 | ``` 154 | 155 | During the tests, you can check the state of the system (cache size, memory and io pressure, disk activity etc) using additional tools: `mem2log`, `psi2log`, `iostat`, `vmstat` etc. 156 | 157 | ## Requirements 158 | 159 | - Python 3.3+ 160 | 161 | ## Installation 162 | 163 | Install 164 | ```sh 165 | $ git clone https://github.com/hakavlad/cache-bench.git && cd cache-bench 166 | $ sudo make install 167 | ``` 168 | `cache-bench` and `drop-caches` scripts will be installed in `/usr/local/bin`. 169 | 170 | Uninstall 171 | ```sh 172 | $ sudo make uninstall 173 | ``` 174 | 175 | ## See also 176 | 177 | Documentation for `/proc/sys/vm/`: 178 | - https://www.kernel.org/doc/html/latest/admin-guide/sysctl/vm.html 179 | 180 | Multigenerational LRU Framework at LKML: 181 | - v1: https://lore.kernel.org/lkml/20210313075747.3781593-1-yuzhao@google.com/ 182 | - v2: https://lore.kernel.org/lkml/20210413065633.2782273-1-yuzhao@google.com/ 183 | - v3: https://lore.kernel.org/lkml/20210520065355.2736558-1-yuzhao@google.com/ 184 | - v4: https://lore.kernel.org/lkml/20210818063107.2696454-1-yuzhao@google.com/ 185 | - v5: https://lore.kernel.org/lkml/20211111041510.402534-1-yuzhao@google.com/ 186 | 187 | le9 patch can be used to protect the specified amount of cache: 188 | - https://github.com/hakavlad/le9-patch 189 | 190 | Daemons that can affect file reading performance: 191 | - [prelockd](https://github.com/hakavlad/prelockd): Lock executables and shared libraries in memory to improve system responsiveness under low-memory conditions; 192 | - [memavaild](https://github.com/hakavlad/memavaild): Keep amount of available memory by evicting memory of selected cgroups into swap space. 193 | 194 | These tools may be used to monitor memory and PSI metrics during stress tests: 195 | - [mem2log](https://github.com/hakavlad/mem2log) may be used to log memory metrics from `/proc/meminfo`; 196 | - [psi2log](https://github.com/hakavlad/nohang/blob/master/docs/psi2log.manpage.md) from [nohang](https://github.com/hakavlad/nohang) package may be used to log [PSI](https://facebookmicrosites.github.io/psi/docs/overview) metrics during tests. 197 | 198 | -------------------------------------------------------------------------------- /cache-bench: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from argparse import ArgumentParser 4 | from mmap import ACCESS_READ, mmap 5 | from os import _exit, fsync, path 6 | from random import random 7 | from signal import SIGHUP, SIGINT, SIGQUIT, SIGTERM, signal 8 | from sys import exit, stdout 9 | from time import monotonic 10 | 11 | 12 | def log(*msg): 13 | """ 14 | """ 15 | print(*msg) 16 | if separate_log: 17 | logging.info(*msg) 18 | 19 | 20 | def signal_handler(signum, frame): 21 | """ 22 | """ 23 | def signal_handler_inner(signum, frame): 24 | pass 25 | 26 | for i in sig_list: 27 | signal(i, signal_handler_inner) 28 | 29 | log('got signal {}'.format(signum)) 30 | stdout.flush() 31 | _exit(0) 32 | 33 | 34 | def writer(): 35 | """ 36 | """ 37 | STRING_MIB = '\n' * M 38 | 39 | log('starting cache-bench') 40 | log(' file: {}'.format(f_path)) 41 | log(' file size: {} MiB'.format(w_num)) 42 | log('writing the file...') 43 | 44 | with open(f_path, 'w') as f: 45 | for _ in range(w_num): 46 | f.write(STRING_MIB) 47 | log('fsync...') 48 | fsync(f.fileno()) 49 | log('OK') 50 | exit() 51 | 52 | 53 | def prereadf(f_size): 54 | """ 55 | """ 56 | log('prereading (caching) the file...') 57 | 58 | with open(f_path, 'rb') as f: 59 | if mapped: 60 | f = mmap(f.fileno(), 0, access=ACCESS_READ) 61 | 62 | r_size = 0 63 | pm_start = monotonic() 64 | pm_old = pm_start 65 | 66 | while True: 67 | r_len = len(f.read(M)) 68 | pm_new = monotonic() 69 | r_size += r_len 70 | 71 | if pm_new - pm_old > interval: 72 | pm_old = pm_new 73 | log(' preread {} MiB ({}%) in {}s'.format( 74 | round(r_size / M, 1), 75 | round(r_size / f_size * 100, 1), 76 | round(pm_new - pm_start, 1))) 77 | 78 | if r_len == 0: 79 | log(' preread {} MiB ({}%) in {}s'.format( 80 | round(r_size / M, 1), 81 | round(r_size / f_size * 100, 1), 82 | round(pm_new - pm_start, 1))) 83 | break 84 | 85 | 86 | def reader(): 87 | """ 88 | """ 89 | log('starting cache-bench') 90 | f_size = path.getsize(f_path) 91 | 92 | log(' file: {}'.format(f_path)) 93 | log(' file size: {} MiB'.format(round(f_size / M, 1))) 94 | 95 | if f_size == 0: 96 | log('the file has zero size, nothing to read!') 97 | exit(1) 98 | 99 | if separate_log: 100 | log(' log file: {}'.format(log_file)) 101 | else: 102 | log(' log file is not set') 103 | 104 | log(' output interval: {}s'.format(interval)) 105 | log(' mmap: {}, preread: {}, bloat: {}, chunk: {} KiB'.format( 106 | m_map, pre_read, bloat, chunk)) 107 | 108 | if preread: 109 | prereadf(f_size) 110 | 111 | with open(f_path, 'rb') as f: 112 | 113 | if mapped: 114 | f = mmap(f.fileno(), 0, access=ACCESS_READ) 115 | 116 | log('reading {} MiB from the file...'.format(r_num)) 117 | 118 | if bloating: 119 | with open('/proc/self/oom_score_adj', 'w') as self_oom_score_adj: 120 | self_oom_score_adj.write('1000\n') 121 | 122 | if bloating: 123 | ex_list = [] 124 | 125 | r_size = 0 126 | t0 = monotonic() 127 | t_old = t0 128 | s_old = 0 129 | 130 | while True: 131 | f.seek(int(f_size * random())) 132 | ch = f.read(CHUNK) 133 | r_size += len(ch) 134 | 135 | if bloating: 136 | ex_list.append(ch) 137 | 138 | t = monotonic() 139 | dt = t - t_old 140 | if dt >= interval: 141 | 142 | d_s = r_size - s_old 143 | t_old = t 144 | s_old = r_size 145 | dt0 = t - t0 146 | log(' read {}M in {}s ({}M/s); total {}M in {}s, avg ' 147 | '{}M/s'.format( 148 | round(d_s / M, 1), 149 | round(dt, 1), 150 | round(d_s / M / dt, 1), 151 | round(r_size / M, 1), 152 | round(dt0, 1), 153 | round(r_size / M / dt0, 1) 154 | )) 155 | 156 | if r_size >= r_bytes: 157 | 158 | d_s = r_size - s_old 159 | t_old = t 160 | s_old = r_size 161 | dt0 = t - t0 162 | log(' read {}M in {}s ({}M/s); total {}M in {}s, avg ' 163 | '{}M/s'.format( 164 | round(d_s / M, 1), 165 | round(dt, 1), 166 | round(d_s / M / dt, 1), 167 | round(r_size / M, 1), 168 | round(dt0, 1), 169 | round(r_size / M / dt0, 1) 170 | )) 171 | 172 | ddt = monotonic() - t0 173 | log('total read {} MiB in {}s (avg {} MiB/s)'.format( 174 | round(r_size / M, 1), 175 | round(ddt, 1), 176 | round(r_size / M / ddt, 1) 177 | )) 178 | stdout.flush() 179 | _exit(0) 180 | 181 | 182 | parser = ArgumentParser() 183 | 184 | parser.add_argument( 185 | '-f', 186 | '--file', 187 | help='path to the file to be read or written', 188 | default='testfile.bench', 189 | type=str 190 | ) 191 | 192 | parser.add_argument( 193 | '-r', 194 | '--read', 195 | help='how many mebibytes to read from the file', 196 | default=0, 197 | type=float 198 | ) 199 | 200 | parser.add_argument( 201 | '-w', 202 | '--write', 203 | help='size of the file being written', 204 | default=0, 205 | type=int 206 | ) 207 | 208 | parser.add_argument( 209 | '-m', 210 | '--mmap', 211 | help='mmap the file (0 | 1)', 212 | default=0, 213 | type=int 214 | ) 215 | 216 | parser.add_argument( 217 | '-p', 218 | '--preread', 219 | help='preread the file (0 | 1)', 220 | default=0, 221 | type=int 222 | ) 223 | 224 | parser.add_argument( 225 | '-b', 226 | '--bloat', 227 | help='bloat process memory (0 | 1)', 228 | default=0, 229 | type=int 230 | ) 231 | 232 | parser.add_argument( 233 | '-c', 234 | '--chunk', 235 | help='chunk size in KiB', 236 | default=64, 237 | type=int 238 | ) 239 | 240 | parser.add_argument( 241 | '-i', 242 | '--interval', 243 | help='output interval in seconds', 244 | default=2, 245 | type=float 246 | ) 247 | 248 | parser.add_argument( 249 | '-l', 250 | '--log', 251 | help='path to the log file', 252 | default=None, 253 | type=str 254 | ) 255 | 256 | args = parser.parse_args() 257 | 258 | f_path = args.file 259 | r_num = args.read 260 | w_num = args.write 261 | m_map = args.mmap 262 | pre_read = args.preread 263 | bloat = args.bloat 264 | chunk = args.chunk 265 | interval = args.interval 266 | log_file = args.log 267 | 268 | if log_file is None: 269 | separate_log = False 270 | else: 271 | separate_log = True 272 | 273 | import logging 274 | 275 | try: 276 | logging.basicConfig( 277 | filename=log_file, 278 | level=logging.INFO, 279 | format="%(asctime)s: %(message)s") 280 | except Exception as e: 281 | print(e) 282 | exit(1) 283 | 284 | if m_map == 0: 285 | mapped = False 286 | elif m_map == 1: 287 | mapped = True 288 | else: 289 | log('invalid --mmap value') 290 | exit(1) 291 | 292 | if pre_read == 0: 293 | preread = False 294 | elif pre_read == 1: 295 | preread = True 296 | else: 297 | log('invalid --preread value') 298 | exit(1) 299 | 300 | if bloat == 0: 301 | bloating = False 302 | elif bloat == 1: 303 | bloating = True 304 | else: 305 | log('invalid --bloat value') 306 | exit(1) 307 | 308 | if chunk <= 0: 309 | log('invalid --chunk value') 310 | exit(1) 311 | 312 | if interval < 0: 313 | log('invalid --interval value') 314 | exit(1) 315 | 316 | 317 | K = 1024 318 | M = K**2 319 | r_bytes = int(M * r_num) 320 | CHUNK = K * chunk 321 | 322 | sig_list = [SIGTERM, SIGINT, SIGQUIT, SIGHUP] 323 | 324 | for i in sig_list: 325 | signal(i, signal_handler) 326 | 327 | 328 | if w_num > 0: 329 | 330 | if path.exists(f_path): 331 | log('"{}" already exists'.format(f_path)) 332 | exit(1) 333 | 334 | try: 335 | with open(f_path, 'wb') as f: 336 | f.write(b'') 337 | except Exception as e: 338 | log(e) 339 | exit(1) 340 | 341 | writer() 342 | 343 | if r_num > 0: 344 | 345 | try: 346 | with open(f_path, 'rb') as f: 347 | f.read(1) 348 | except Exception as e: 349 | log(e) 350 | exit(1) 351 | 352 | reader() 353 | 354 | log('set correct --read or --write value') 355 | 356 | exit(1) 357 | -------------------------------------------------------------------------------- /drop-caches: -------------------------------------------------------------------------------- 1 | #!/bin/sh -v 2 | sudo sync 3 | echo 3 | sudo tee /proc/sys/vm/drop_caches 4 | --------------------------------------------------------------------------------