├── .gitignore └── DiskIO Benchmark.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | test.txt 3 | venv 4 | -------------------------------------------------------------------------------- /DiskIO Benchmark.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import time\n", 11 | "import random\n", 12 | "import threading\n", 13 | "import tempfile\n", 14 | "from collections import defaultdict\n", 15 | "\n", 16 | "\n", 17 | "class DiskIOBenchmark:\n", 18 | " def __init__(self):\n", 19 | " self.size = 1\n", 20 | " self.benchmarks = defaultdict(list)\n", 21 | " self.fp = os.open(\"./test.txt\", os.O_CREAT|os.O_RDWR|os.O_SYNC | os.O_TRUNC, 0o777)\n", 22 | " self.sz = os.write(self.fp, bytearray(os.urandom(1 << 30)))\n", 23 | " self._setup()\n", 24 | "\n", 25 | " def _setup(self):\n", 26 | " self.is_open = True\n", 27 | " self.io = 0\n", 28 | " self.duration = 10\n", 29 | "\n", 30 | " def compute_throughput(self):\n", 31 | " for _ in range(self.duration):\n", 32 | " time.sleep(1)\n", 33 | " self.benchmarks[self.size].append(self.io)\n", 34 | " self.io = 0\n", 35 | " self.is_open = False\n", 36 | "\n", 37 | " def random_writes(self):\n", 38 | " k = bytearray(os.urandom(self.size))\n", 39 | " while self.is_open:\n", 40 | " os.lseek(self.fp, random.randint(0, self.sz-self.size), 0)\n", 41 | " os.write(self.fp, k)\n", 42 | " os.fsync(self.fp)\n", 43 | " self.io += 1\n", 44 | " \n", 45 | " def random_reads(self):\n", 46 | " while self.is_open:\n", 47 | " os.lseek(self.fp, random.randint(0, self.sz-self.size), 0)\n", 48 | " os.read(self.fp, self.size)\n", 49 | " self.io += 1\n", 50 | " \n", 51 | " def seq_reads(self):\n", 52 | " while self.is_open:\n", 53 | " x = os.read(self.fp, self.size)\n", 54 | " if x == 0:\n", 55 | " os.lseek(self.fp, 0, 0)\n", 56 | " self.io += 1\n", 57 | " \n", 58 | " def seq_writes(self):\n", 59 | " k = bytearray(os.urandom(self.size))\n", 60 | " while self.is_open:\n", 61 | " os.write(self.fp, k)\n", 62 | " os.fsync(self.fp)\n", 63 | " self.io += 1\n", 64 | "\n", 65 | " def _benchmark(self, fn):\n", 66 | " for power in range(8, 25):\n", 67 | " size = 1 << power\n", 68 | " self.size = size\n", 69 | " self._setup()\n", 70 | " print(\"benchmarking for size\", self.size)\n", 71 | " \n", 72 | " t1 = threading.Thread(target=self.compute_throughput)\n", 73 | " t2 = threading.Thread(target=self.random_writes)\n", 74 | "\n", 75 | " t1.start()\n", 76 | " t2.start()\n", 77 | "\n", 78 | " t1.join()\n", 79 | " t2.join()\n", 80 | " \n", 81 | " print(size, sum(self.benchmarks[self.size])/ len(self.benchmarks[self.size]))\n", 82 | "\n", 83 | " def benchmark_rw(self):\n", 84 | " print(\"random_writes\")\n", 85 | " self._benchmark(self.random_writes)\n", 86 | " os.close(self.fp)\n", 87 | " \n", 88 | " def benchmark_rr(self):\n", 89 | " print(\"random_reads\")\n", 90 | " self._benchmark(self.random_reads)\n", 91 | " os.close(self.fp)\n", 92 | " \n", 93 | " def benchmark_sr(self):\n", 94 | " print(\"seq_reads\")\n", 95 | " self._benchmark(self.seq_reads)\n", 96 | " os.close(self.fp)\n", 97 | " \n", 98 | " def benchmark_sw(self):\n", 99 | " print(\"seq_writes\")\n", 100 | " self._benchmark(self.seq_writes)\n", 101 | " os.close(self.fp)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 2, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "seq_writes\n", 114 | "benchmarking for size 256\n", 115 | "256 15431.0\n", 116 | "benchmarking for size 512\n", 117 | "512 16768.4\n", 118 | "benchmarking for size 1024\n", 119 | "1024 17412.9\n", 120 | "benchmarking for size 2048\n", 121 | "2048 16426.2\n", 122 | "benchmarking for size 4096\n", 123 | "4096 15872.7\n", 124 | "benchmarking for size 8192\n", 125 | "8192 14045.2\n", 126 | "benchmarking for size 16384\n", 127 | "16384 11964.5\n", 128 | "benchmarking for size 32768\n", 129 | "32768 8139.3\n", 130 | "benchmarking for size 65536\n", 131 | "65536 3763.6\n", 132 | "benchmarking for size 131072\n", 133 | "131072 3604.9\n", 134 | "benchmarking for size 262144\n", 135 | "262144 1935.8\n", 136 | "benchmarking for size 524288\n", 137 | "524288 1323.1\n", 138 | "benchmarking for size 1048576\n", 139 | "1048576 736.1\n", 140 | "benchmarking for size 2097152\n", 141 | "2097152 353.4\n", 142 | "benchmarking for size 4194304\n", 143 | "4194304 196.7\n", 144 | "benchmarking for size 8388608\n", 145 | "8388608 99.3\n", 146 | "benchmarking for size 16777216\n", 147 | "16777216 47.8\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "bm = DiskIOBenchmark()\n", 153 | "bm.benchmark_sw()" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 3, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "data": { 163 | "text/plain": [ 164 | "defaultdict(list,\n", 165 | " {256: [11934,\n", 166 | " 16848,\n", 167 | " 13479,\n", 168 | " 15743,\n", 169 | " 17265,\n", 170 | " 16822,\n", 171 | " 15449,\n", 172 | " 17695,\n", 173 | " 12576,\n", 174 | " 16499],\n", 175 | " 512: [15390,\n", 176 | " 17437,\n", 177 | " 14902,\n", 178 | " 17110,\n", 179 | " 17613,\n", 180 | " 16871,\n", 181 | " 17668,\n", 182 | " 15127,\n", 183 | " 17881,\n", 184 | " 17685],\n", 185 | " 1024: [16821,\n", 186 | " 17815,\n", 187 | " 17614,\n", 188 | " 16020,\n", 189 | " 18426,\n", 190 | " 17249,\n", 191 | " 17570,\n", 192 | " 17444,\n", 193 | " 17361,\n", 194 | " 17809],\n", 195 | " 2048: [16723,\n", 196 | " 17174,\n", 197 | " 17086,\n", 198 | " 16167,\n", 199 | " 15393,\n", 200 | " 15188,\n", 201 | " 15938,\n", 202 | " 16208,\n", 203 | " 17410,\n", 204 | " 16975],\n", 205 | " 4096: [14965,\n", 206 | " 16799,\n", 207 | " 16970,\n", 208 | " 14970,\n", 209 | " 15697,\n", 210 | " 15768,\n", 211 | " 15541,\n", 212 | " 15066,\n", 213 | " 16209,\n", 214 | " 16742],\n", 215 | " 8192: [10344,\n", 216 | " 11763,\n", 217 | " 12499,\n", 218 | " 15331,\n", 219 | " 13909,\n", 220 | " 15894,\n", 221 | " 14775,\n", 222 | " 15776,\n", 223 | " 14986,\n", 224 | " 15175],\n", 225 | " 16384: [12087,\n", 226 | " 13703,\n", 227 | " 14474,\n", 228 | " 14339,\n", 229 | " 10946,\n", 230 | " 11356,\n", 231 | " 11002,\n", 232 | " 9553,\n", 233 | " 11958,\n", 234 | " 10227],\n", 235 | " 32768: [10852,\n", 236 | " 6128,\n", 237 | " 9157,\n", 238 | " 12396,\n", 239 | " 11892,\n", 240 | " 10370,\n", 241 | " 4398,\n", 242 | " 5947,\n", 243 | " 5694,\n", 244 | " 4559],\n", 245 | " 65536: [5309,\n", 246 | " 3033,\n", 247 | " 3104,\n", 248 | " 2056,\n", 249 | " 2240,\n", 250 | " 3200,\n", 251 | " 2883,\n", 252 | " 4549,\n", 253 | " 5313,\n", 254 | " 5949],\n", 255 | " 131072: [3945,\n", 256 | " 3124,\n", 257 | " 3123,\n", 258 | " 3796,\n", 259 | " 3191,\n", 260 | " 4224,\n", 261 | " 3132,\n", 262 | " 4073,\n", 263 | " 3611,\n", 264 | " 3830],\n", 265 | " 262144: [1773,\n", 266 | " 1791,\n", 267 | " 2332,\n", 268 | " 1857,\n", 269 | " 1532,\n", 270 | " 1663,\n", 271 | " 2009,\n", 272 | " 2207,\n", 273 | " 1904,\n", 274 | " 2290],\n", 275 | " 524288: [1334,\n", 276 | " 1281,\n", 277 | " 1235,\n", 278 | " 1230,\n", 279 | " 1303,\n", 280 | " 1300,\n", 281 | " 1265,\n", 282 | " 1415,\n", 283 | " 1440,\n", 284 | " 1428],\n", 285 | " 1048576: [754, 756, 710, 730, 760, 720, 739, 724, 747, 721],\n", 286 | " 2097152: [361, 404, 368, 365, 324, 385, 266, 277, 390, 394],\n", 287 | " 4194304: [206, 192, 190, 195, 210, 191, 203, 188, 194, 198],\n", 288 | " 8388608: [102, 97, 101, 94, 103, 96, 101, 98, 102, 99],\n", 289 | " 16777216: [38, 37, 47, 52, 51, 51, 51, 44, 54, 53]})" 290 | ] 291 | }, 292 | "execution_count": 3, 293 | "metadata": {}, 294 | "output_type": "execute_result" 295 | } 296 | ], 297 | "source": [ 298 | "bm.benchmarks" 299 | ] 300 | } 301 | ], 302 | "metadata": { 303 | "kernelspec": { 304 | "display_name": "Python 3", 305 | "language": "python", 306 | "name": "python3" 307 | }, 308 | "language_info": { 309 | "codemirror_mode": { 310 | "name": "ipython", 311 | "version": 3 312 | }, 313 | "file_extension": ".py", 314 | "mimetype": "text/x-python", 315 | "name": "python", 316 | "nbconvert_exporter": "python", 317 | "pygments_lexer": "ipython3", 318 | "version": "3.7.6" 319 | } 320 | }, 321 | "nbformat": 4, 322 | "nbformat_minor": 4 323 | } 324 | --------------------------------------------------------------------------------