├── README.md ├── benchmark.json └── benchmark.toml /README.md: -------------------------------------------------------------------------------- 1 | # Deep Learning Benchmarks 2 | 3 | ## Introduction 4 | 5 | This repository contains structured performance characteristics of the 6 | most popular Machine and Deep Learning Frameworks forward and backward 7 | performance for entire models, single layers and more. Deep Learning Benchmarks 8 | is inspired by soumith's [convnet-benchmarks][convnet-benchmarks] and zer0n's 9 | [deepframeworks][deepframeworks]. 10 | 11 | `Deep Learning Benchmarks` goal is to provide raw, structured, comparable data 12 | about the performance (operational speed, operational memory usage) of different 13 | Machine Learning Frameworks in various environments and machines. Like the other 14 | two repositories, `Deep Learning Benchmarks` highest goal is gaining objective, 15 | reproducible benchmarks, which are reviewed by the community. New benchmark data is 16 | introduced through PRs, its process is described in 17 | [Submitting Benchmark Data][submitting-benchmark-data]. 18 | 19 | [convnet-benchmarks]: https://github.com/soumith/convnet-benchmarks 20 | [deepframeworks]: https://github.com/zer0n/deepframeworks 21 | [submitting-benchmark-data]: #submitting-benchmark-data 22 | 23 | ## View Data 24 | 25 | One advantage of structured, raw benchmark data is, that it can be easily 26 | queried, visualized and compared. 27 | [Autumn's Benchmark page][autumn-benchmark-page], provides an open interface for 28 | querying, visualizing and later sharing the Deep Learning Benchmark data. 29 | 30 | 31 | ## Raw Data 32 | 33 | The repository contains this [README](README.md) and 34 | [benchmark.toml](benchmark.toml), where all the benchmark data is stored. 35 | 36 | You can consume the data like a JSON REST-API, by fetching 37 | 38 | ``` 39 | https://raw.githubusercontent.com/autumnai/deep-learning-benchmarks/master/benchmark.json 40 | ``` 41 | 42 | ```javascript 43 | fetch('https://raw.githubusercontent.com/autumnai/deep-learning-benchmarks/master/benchmark.json') 44 | .then(function(response) { 45 | return response.json() 46 | }).then(function(json) { 47 | console.log('parsed json', json) 48 | }).catch(function(ex) { 49 | console.log('parsing failed', ex) 50 | }) 51 | ``` 52 | 53 | We use TOML for the original benchmark data file format (instead of JSON), 54 | because it is more convenient for merging data. Out of convenience we convert 55 | the TOML into a JSON file locally as well before we merge a PR, this increases 56 | the usability of the data for client-side applications. 57 | 58 | ## Data Structure 59 | 60 | > WIP 61 | 62 | ## Submitting Benchmark Data 63 | 64 | > WIP 65 | -------------------------------------------------------------------------------- /benchmark.json: -------------------------------------------------------------------------------- 1 | { 2 | "nvidia-titan-x": { 3 | "devices": [ 4 | { 5 | "cores": "3072", 6 | "memory": "12GB", 7 | "memory_bandwith": "336.5GB/s", 8 | "name": "Nvidia Titan X", 9 | "quantity": 1 10 | } 11 | ], 12 | "host": { 13 | "cache": "8MB", 14 | "cores": "8", 15 | "cpu": "Intel(R) Core(TM) i7-2600K CPU @ 3.40GHz", 16 | "os": "linux/arch", 17 | "ram": "8GB" 18 | }, 19 | "models": [ 20 | { 21 | "details": "...link...", 22 | "frameworks": [ 23 | { 24 | "backward": 63, 25 | "forward": 25, 26 | "libraries": [ 27 | { 28 | "name": "cuDNN", 29 | "version": "4" 30 | } 31 | ], 32 | "name": "Leaf", 33 | "version": "0.2" 34 | }, 35 | { 36 | "backward": 71, 37 | "forward": 31, 38 | "libraries": [ 39 | { 40 | "name": "cuDNN", 41 | "version": "3" 42 | } 43 | ], 44 | "memory": "1750MB", 45 | "name": "Leaf", 46 | "version": "0.2" 47 | }, 48 | { 49 | "backward": 66, 50 | "forward": 33, 51 | "libraries": [ 52 | { 53 | "name": "cuDNN", 54 | "version": "3" 55 | } 56 | ], 57 | "memory": "1650MB", 58 | "name": "Torch", 59 | "version": "01/2016" 60 | }, 61 | { 62 | "backward": 54, 63 | "forward": 28, 64 | "libraries": [ 65 | { 66 | "name": "cuDNN", 67 | "version": "4" 68 | } 69 | ], 70 | "memory": "1650MB", 71 | "name": "Torch", 72 | "version": "07/03/2016" 73 | }, 74 | { 75 | "backward": 85, 76 | "forward": 42, 77 | "libraries": [ 78 | { 79 | "name": "cuDNN", 80 | "version": "3" 81 | } 82 | ], 83 | "memory": "1350MB", 84 | "name": "Caffe", 85 | "version": "1.0.0" 86 | }, 87 | { 88 | "backward": 202, 89 | "forward": 46, 90 | "libraries": [ 91 | { 92 | "name": "cuDNN", 93 | "version": "3" 94 | } 95 | ], 96 | "memory": "11050MB", 97 | "name": "Tensorflow", 98 | "version": "0.7" 99 | }, 100 | { 101 | "backward": 128, 102 | "forward": 40, 103 | "libraries": [ 104 | { 105 | "name": "cuDNN", 106 | "version": "4" 107 | } 108 | ], 109 | "memory": "11520MB", 110 | "name": "Tensorflow", 111 | "version": "0.7.1" 112 | } 113 | ], 114 | "name": "Alexnet", 115 | "type": "model" 116 | }, 117 | { 118 | "details": "...link...", 119 | "frameworks": [ 120 | { 121 | "backward": 180, 122 | "forward": 84, 123 | "libraries": [ 124 | { 125 | "name": "cuDNN", 126 | "version": "4" 127 | } 128 | ], 129 | "memory": "5050MB", 130 | "name": "Leaf", 131 | "version": "0.2" 132 | }, 133 | { 134 | "backward": 226, 135 | "forward": 106, 136 | "libraries": [ 137 | { 138 | "name": "cuDNN", 139 | "version": "3" 140 | } 141 | ], 142 | "memory": "5050MB", 143 | "name": "Leaf", 144 | "version": "0.2" 145 | }, 146 | { 147 | "backward": 215, 148 | "forward": 114, 149 | "libraries": [ 150 | { 151 | "name": "cuDNN", 152 | "version": "3" 153 | } 154 | ], 155 | "memory": "4950MB", 156 | "name": "Torch", 157 | "version": "01/2016" 158 | }, 159 | { 160 | "backward": 176, 161 | "forward": 95, 162 | "libraries": [ 163 | { 164 | "name": "cuDNN", 165 | "version": "4" 166 | } 167 | ], 168 | "memory": "5000MB", 169 | "name": "Torch", 170 | "version": "07/03/2016" 171 | }, 172 | { 173 | "backward": 289, 174 | "forward": 143, 175 | "libraries": [ 176 | { 177 | "name": "cuDNN", 178 | "version": "3" 179 | } 180 | ], 181 | "memory": "2400MB", 182 | "name": "Caffe", 183 | "version": "1.0.0" 184 | }, 185 | { 186 | "backward": 621, 187 | "forward": 118, 188 | "libraries": [ 189 | { 190 | "name": "cuDNN", 191 | "version": "3" 192 | } 193 | ], 194 | "memory": "11050MB", 195 | "name": "Tensorflow", 196 | "version": "0.7" 197 | }, 198 | { 199 | "backward": 300, 200 | "forward": 119, 201 | "libraries": [ 202 | { 203 | "name": "cuDNN", 204 | "version": "4" 205 | } 206 | ], 207 | "memory": "11520MB", 208 | "name": "Tensorflow", 209 | "version": "0.7.1" 210 | } 211 | ], 212 | "name": "Overfeat", 213 | "type": "model" 214 | }, 215 | { 216 | "details": "...link...", 217 | "frameworks": [ 218 | { 219 | "backward": 368, 220 | "forward": 143, 221 | "libraries": [ 222 | { 223 | "name": "cuDNN", 224 | "version": "4" 225 | } 226 | ], 227 | "memory": "8300MB", 228 | "name": "Leaf", 229 | "version": "0.2" 230 | }, 231 | { 232 | "backward": 418, 233 | "forward": 180, 234 | "libraries": [ 235 | { 236 | "name": "cuDNN", 237 | "version": "3" 238 | } 239 | ], 240 | "memory": "8300MB", 241 | "name": "Leaf", 242 | "version": "0.2" 243 | }, 244 | { 245 | "backward": 429, 246 | "forward": 201, 247 | "libraries": [ 248 | { 249 | "name": "cuDNN", 250 | "version": "3" 251 | } 252 | ], 253 | "memory": "8250MB", 254 | "name": "Torch", 255 | "version": "01/2016" 256 | }, 257 | { 258 | "backward": 367, 259 | "forward": 165, 260 | "libraries": [ 261 | { 262 | "name": "cuDNN", 263 | "version": "4" 264 | } 265 | ], 266 | "memory": "8090MB", 267 | "name": "Torch", 268 | "version": "07/03/2016" 269 | }, 270 | { 271 | "backward": 453, 272 | "forward": 225, 273 | "libraries": [ 274 | { 275 | "name": "cuDNN", 276 | "version": "3" 277 | } 278 | ], 279 | "memory": "5650MB", 280 | "name": "Caffe", 281 | "version": "1.0.0" 282 | }, 283 | { 284 | "backward": 1142, 285 | "forward": 228, 286 | "libraries": [ 287 | { 288 | "name": "cuDNN", 289 | "version": "3" 290 | } 291 | ], 292 | "memory": "11000MB", 293 | "name": "Tensorflow", 294 | "version": "0.7" 295 | }, 296 | { 297 | "backward": 840, 298 | "forward": 227, 299 | "libraries": [ 300 | { 301 | "name": "cuDNN", 302 | "version": "4" 303 | } 304 | ], 305 | "memory": "11520MB", 306 | "name": "Tensorflow", 307 | "version": "0.7.1" 308 | } 309 | ], 310 | "name": "VGG [A]", 311 | "type": "model" 312 | }, 313 | { 314 | "details": "...link...", 315 | "frameworks": [ 316 | { 317 | "backward": 361, 318 | "forward": 138, 319 | "libraries": [ 320 | { 321 | "name": "cuDNN", 322 | "version": "3" 323 | } 324 | ], 325 | "memory": "8300MB", 326 | "name": "Torch", 327 | "version": "01/2016" 328 | }, 329 | { 330 | "backward": 342, 331 | "forward": 131, 332 | "libraries": [ 333 | { 334 | "name": "cuDNN", 335 | "version": "4" 336 | } 337 | ], 338 | "memory": "8360MB", 339 | "name": "Torch", 340 | "version": "07/03/2016" 341 | }, 342 | { 343 | "backward": 345, 344 | "forward": 143, 345 | "libraries": [ 346 | { 347 | "name": "cuDNN", 348 | "version": "3" 349 | } 350 | ], 351 | "memory": "8100MB", 352 | "name": "Caffe", 353 | "version": "1.0.0" 354 | }, 355 | { 356 | "backward": 871, 357 | "forward": 197, 358 | "libraries": [ 359 | { 360 | "name": "cuDNN", 361 | "version": "3" 362 | } 363 | ], 364 | "memory": "11050MB", 365 | "name": "Tensorflow", 366 | "version": "0.7" 367 | }, 368 | { 369 | "backward": 757, 370 | "forward": 195, 371 | "libraries": [ 372 | { 373 | "name": "cuDNN", 374 | "version": "4" 375 | } 376 | ], 377 | "memory": "11520MB", 378 | "name": "Tensorflow", 379 | "version": "0.7.1" 380 | } 381 | ], 382 | "name": "GoogleNet", 383 | "type": "model" 384 | } 385 | ], 386 | "name": "Nvidia Titan X", 387 | "type": "GPU" 388 | } 389 | } -------------------------------------------------------------------------------- /benchmark.toml: -------------------------------------------------------------------------------- 1 | # This file contains the raw, structured benchmarks. More information can be 2 | # found at https://github.com/autumnai/deep-learning-benchmarks 3 | 4 | [nvidia-titan-x] 5 | name = "Nvidia Titan X" 6 | type = "GPU" 7 | 8 | [nvidia-titan-x.host] 9 | os = "linux/arch" 10 | cpu = "Intel(R) Core(TM) i7-2600K CPU @ 3.40GHz" 11 | cores = "8" 12 | cache = "8MB" 13 | ram = "8GB" 14 | 15 | [[nvidia-titan-x.devices]] 16 | name = "Nvidia Titan X" 17 | memory = "12GB" 18 | cores = "3072" 19 | memory_bandwith = "336.5GB/s" 20 | quantity = 1 21 | 22 | [[nvidia-titan-x.models]] 23 | name = "Alexnet" 24 | type = "model" 25 | details = "...link..." 26 | 27 | [[nvidia-titan-x.models.frameworks]] 28 | name = "Leaf" 29 | version = "0.2" 30 | forward = 25 31 | backward = 63 32 | 33 | [[nvidia-titan-x.models.frameworks.libraries]] 34 | name = "cuDNN" 35 | version = "4" 36 | 37 | [[nvidia-titan-x.models.frameworks]] 38 | name = "Leaf" 39 | version = "0.2" 40 | forward = 31 41 | backward = 71 42 | memory = "1750MB" 43 | 44 | [[nvidia-titan-x.models.frameworks.libraries]] 45 | name = "cuDNN" 46 | version = "3" 47 | 48 | [[nvidia-titan-x.models.frameworks]] 49 | name = "Torch" 50 | version = "01/2016" 51 | forward = 33 52 | backward = 66 53 | memory = "1650MB" 54 | 55 | [[nvidia-titan-x.models.frameworks.libraries]] 56 | name = "cuDNN" 57 | version = "3" 58 | 59 | [[nvidia-titan-x.models.frameworks]] 60 | name = "Torch" 61 | version = "07/03/2016" 62 | forward = 28 63 | backward = 54 64 | memory = "1650MB" 65 | 66 | [[nvidia-titan-x.models.frameworks.libraries]] 67 | name = "cuDNN" 68 | version = "4" 69 | 70 | [[nvidia-titan-x.models.frameworks]] 71 | name = "Caffe" 72 | version = "1.0.0" 73 | forward = 42 74 | backward = 85 75 | memory = "1350MB" 76 | 77 | [[nvidia-titan-x.models.frameworks.libraries]] 78 | name = "cuDNN" 79 | version = "3" 80 | 81 | [[nvidia-titan-x.models.frameworks]] 82 | name = "Tensorflow" 83 | version = "0.7" 84 | forward = 46 85 | backward = 202 86 | memory = "11050MB" 87 | 88 | [[nvidia-titan-x.models.frameworks.libraries]] 89 | name = "cuDNN" 90 | version = "3" 91 | 92 | [[nvidia-titan-x.models.frameworks]] 93 | name = "Tensorflow" 94 | version = "0.7.1" 95 | forward = 40 96 | backward = 128 97 | memory = "11520MB" 98 | 99 | [[nvidia-titan-x.models.frameworks.libraries]] 100 | name = "cuDNN" 101 | version = "4" 102 | 103 | [[nvidia-titan-x.models]] 104 | name = "Overfeat" 105 | type = "model" 106 | details = "...link..." 107 | 108 | [[nvidia-titan-x.models.frameworks]] 109 | name = "Leaf" 110 | version = "0.2" 111 | forward = 84 112 | backward = 180 113 | memory = "5050MB" 114 | 115 | [[nvidia-titan-x.models.frameworks.libraries]] 116 | name = "cuDNN" 117 | version = "4" 118 | 119 | [[nvidia-titan-x.models.frameworks]] 120 | name = "Leaf" 121 | version = "0.2" 122 | forward = 106 123 | backward = 226 124 | memory = "5050MB" 125 | 126 | [[nvidia-titan-x.models.frameworks.libraries]] 127 | name = "cuDNN" 128 | version = "3" 129 | 130 | [[nvidia-titan-x.models.frameworks]] 131 | name = "Torch" 132 | version = "01/2016" 133 | forward = 114 134 | backward = 215 135 | memory = "4950MB" 136 | 137 | [[nvidia-titan-x.models.frameworks.libraries]] 138 | name = "cuDNN" 139 | version = "3" 140 | 141 | [[nvidia-titan-x.models.frameworks]] 142 | name = "Torch" 143 | version = "07/03/2016" 144 | forward = 95 145 | backward = 176 146 | memory = "5000MB" 147 | 148 | [[nvidia-titan-x.models.frameworks.libraries]] 149 | name = "cuDNN" 150 | version = "4" 151 | 152 | [[nvidia-titan-x.models.frameworks]] 153 | name = "Caffe" 154 | version = "1.0.0" 155 | forward = 143 156 | backward = 289 157 | memory = "2400MB" 158 | 159 | [[nvidia-titan-x.models.frameworks.libraries]] 160 | name = "cuDNN" 161 | version = "3" 162 | 163 | [[nvidia-titan-x.models.frameworks]] 164 | name = "Tensorflow" 165 | version = "0.7" 166 | forward = 118 167 | backward = 621 168 | memory = "11050MB" 169 | 170 | [[nvidia-titan-x.models.frameworks.libraries]] 171 | name = "cuDNN" 172 | version = "3" 173 | 174 | [[nvidia-titan-x.models.frameworks]] 175 | name = "Tensorflow" 176 | version = "0.7.1" 177 | forward = 119 178 | backward = 300 179 | memory = "11520MB" 180 | 181 | [[nvidia-titan-x.models.frameworks.libraries]] 182 | name = "cuDNN" 183 | version = "4" 184 | 185 | [[nvidia-titan-x.models]] 186 | name = "VGG [A]" 187 | type = "model" 188 | details = "...link..." 189 | 190 | [[nvidia-titan-x.models.frameworks]] 191 | name = "Leaf" 192 | version = "0.2" 193 | forward = 143 194 | backward = 368 195 | memory = "8300MB" 196 | 197 | [[nvidia-titan-x.models.frameworks.libraries]] 198 | name = "cuDNN" 199 | version = "4" 200 | 201 | [[nvidia-titan-x.models.frameworks]] 202 | name = "Leaf" 203 | version = "0.2" 204 | forward = 180 205 | backward = 418 206 | memory = "8300MB" 207 | 208 | [[nvidia-titan-x.models.frameworks.libraries]] 209 | name = "cuDNN" 210 | version = "3" 211 | 212 | [[nvidia-titan-x.models.frameworks]] 213 | name = "Torch" 214 | version = "01/2016" 215 | forward = 201 216 | backward = 429 217 | memory = "8250MB" 218 | 219 | [[nvidia-titan-x.models.frameworks.libraries]] 220 | name = "cuDNN" 221 | version = "3" 222 | 223 | [[nvidia-titan-x.models.frameworks]] 224 | name = "Torch" 225 | version = "07/03/2016" 226 | forward = 165 227 | backward = 367 228 | memory = "8090MB" 229 | 230 | [[nvidia-titan-x.models.frameworks.libraries]] 231 | name = "cuDNN" 232 | version = "4" 233 | 234 | [[nvidia-titan-x.models.frameworks]] 235 | name = "Caffe" 236 | version = "1.0.0" 237 | forward = 225 238 | backward = 453 239 | memory = "5650MB" 240 | 241 | [[nvidia-titan-x.models.frameworks.libraries]] 242 | name = "cuDNN" 243 | version = "3" 244 | 245 | [[nvidia-titan-x.models.frameworks]] 246 | name = "Tensorflow" 247 | version = "0.7" 248 | forward = 228 249 | backward = 1142 250 | memory = "11000MB" 251 | 252 | [[nvidia-titan-x.models.frameworks.libraries]] 253 | name = "cuDNN" 254 | version = "3" 255 | 256 | [[nvidia-titan-x.models.frameworks]] 257 | name = "Tensorflow" 258 | version = "0.7.1" 259 | forward = 227 260 | backward = 840 261 | memory = "11520MB" 262 | 263 | [[nvidia-titan-x.models.frameworks.libraries]] 264 | name = "cuDNN" 265 | version = "4" 266 | 267 | [[nvidia-titan-x.models]] 268 | name = "GoogleNet" 269 | type = "model" 270 | details = "...link..." 271 | 272 | [[nvidia-titan-x.models.frameworks]] 273 | name = "Torch" 274 | version = "01/2016" 275 | forward = 138 276 | backward = 361 277 | memory = "8300MB" 278 | 279 | [[nvidia-titan-x.models.frameworks.libraries]] 280 | name = "cuDNN" 281 | version = "3" 282 | 283 | [[nvidia-titan-x.models.frameworks]] 284 | name = "Torch" 285 | version = "07/03/2016" 286 | forward = 131 287 | backward = 342 288 | memory = "8360MB" 289 | 290 | [[nvidia-titan-x.models.frameworks.libraries]] 291 | name = "cuDNN" 292 | version = "4" 293 | 294 | [[nvidia-titan-x.models.frameworks]] 295 | name = "Caffe" 296 | version = "1.0.0" 297 | forward = 143 298 | backward = 345 299 | memory = "8100MB" 300 | 301 | [[nvidia-titan-x.models.frameworks.libraries]] 302 | name = "cuDNN" 303 | version = "3" 304 | 305 | [[nvidia-titan-x.models.frameworks]] 306 | name = "Tensorflow" 307 | version = "0.7" 308 | forward = 197 309 | backward = 871 310 | memory = "11050MB" 311 | 312 | [[nvidia-titan-x.models.frameworks.libraries]] 313 | name = "cuDNN" 314 | version = "3" 315 | 316 | [[nvidia-titan-x.models.frameworks]] 317 | name = "Tensorflow" 318 | version = "0.7.1" 319 | forward = 195 320 | backward = 757 321 | memory = "11520MB" 322 | 323 | [[nvidia-titan-x.models.frameworks.libraries]] 324 | name = "cuDNN" 325 | version = "4" 326 | --------------------------------------------------------------------------------