├── .gitignore ├── Makefile ├── README ├── bench.py ├── charts-template.html ├── make_chart_data.py ├── make_html.py └── src ├── boost_unordered_map.cc ├── glib_hash_table.c ├── google_dense_hash_map.cc ├── google_sparse_hash_map.cc ├── python_dict.c ├── qt_qhash.cc ├── ruby_hash.c ├── stl_unordered_map.cc └── template.c /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | output 3 | charts.html 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build/glib_hash_table build/stl_unordered_map build/boost_unordered_map build/google_sparse_hash_map build/google_dense_hash_map build/qt_qhash build/python_dict build/ruby_hash 2 | 3 | build/glib_hash_table: src/glib_hash_table.c Makefile src/template.c 4 | gcc -ggdb -O2 -lm `pkg-config --cflags --libs glib-2.0` src/glib_hash_table.c -o build/glib_hash_table 5 | 6 | build/stl_unordered_map: src/stl_unordered_map.cc Makefile src/template.c 7 | g++ -O2 -lm src/stl_unordered_map.cc -o build/stl_unordered_map -std=c++0x 8 | 9 | build/boost_unordered_map: src/boost_unordered_map.cc Makefile src/template.c 10 | g++ -O2 -lm src/boost_unordered_map.cc -o build/boost_unordered_map 11 | 12 | build/google_sparse_hash_map: src/google_sparse_hash_map.cc Makefile src/template.c 13 | g++ -O2 -lm src/google_sparse_hash_map.cc -o build/google_sparse_hash_map 14 | 15 | build/google_dense_hash_map: src/google_dense_hash_map.cc Makefile src/template.c 16 | g++ -O2 -lm src/google_dense_hash_map.cc -o build/google_dense_hash_map 17 | 18 | build/qt_qhash: src/qt_qhash.cc Makefile src/template.c 19 | g++ -O2 -lm `pkg-config --cflags --libs QtCore` src/qt_qhash.cc -o build/qt_qhash 20 | 21 | build/python_dict: src/python_dict.c Makefile src/template.c 22 | gcc -O2 -lm -I/usr/include/python2.6 -lpython2.6 src/python_dict.c -o build/python_dict 23 | 24 | build/ruby_hash: src/ruby_hash.c Makefile src/template.c 25 | gcc -O2 -lm -I/usr/include/ruby-1.9.0 -I /usr/include/ruby-1.9.0/x86_64-linux -lruby1.9 src/ruby_hash.c -o build/ruby_hash 26 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | How to run these benchmarks: 2 | 3 | First, some prerequisites are: 4 | 5 | * make 6 | * gcc and recent g++ (4.3-ish?) 7 | * python (hard-coded for 2.6 -- edit Makefile to use another version) 8 | * glib 9 | * boost 10 | * google sparsehash 11 | * qt 12 | 13 | Now, run: 14 | 15 | $ make 16 | $ python bench.py 17 | $ python make_chart_data.py < output | python make_html.py 18 | 19 | Your charts are now in charts.html. 20 | 21 | You can tweak some of the values in bench.py to make it run faster at the 22 | expense of less granular data, and you might need to tweak some of the tickSize 23 | settings in charts-template.html. 24 | 25 | To run the benchmark at the highest priority possible, do this: 26 | 27 | $ sudo nice -n-20 ionice -c1 -n0 sudo -u $USER python bench.py 28 | 29 | You might also want to disable any swap files/partitions so that swapping 30 | doesn't influence performance. (The programs will just die if they try to 31 | allocate too much memory.) 32 | 33 | Copyright Information 34 | ===================== 35 | 36 | Written by Nick Welch in 2010. 37 | No copyright. This work is dedicated to the public domain. 38 | For full details, see http://creativecommons.org/publicdomain/zero/1.0/ 39 | -------------------------------------------------------------------------------- /bench.py: -------------------------------------------------------------------------------- 1 | import sys, os, subprocess, signal 2 | 3 | programs = [ 4 | 'glib_hash_table', 5 | 'stl_unordered_map', 6 | 'boost_unordered_map', 7 | 'google_sparse_hash_map', 8 | 'google_dense_hash_map', 9 | 'qt_qhash', 10 | 'python_dict', 11 | 'ruby_hash', 12 | ] 13 | 14 | minkeys = 2*1000*1000 15 | maxkeys = 40*1000*1000 16 | interval = 2*1000*1000 17 | best_out_of = 2 18 | 19 | # for the final run, use this: 20 | #minkeys = 2*1000*1000 21 | #maxkeys = 40*1000*1000 22 | #interval = 2*1000*1000 23 | #best_out_of = 3 24 | # and use nice/ionice 25 | # and shut down to the console 26 | # and swapoff any swap files/partitions 27 | 28 | outfile = open('output', 'w') 29 | 30 | if len(sys.argv) > 1: 31 | benchtypes = sys.argv[1:] 32 | else: 33 | benchtypes = ('sequential', 'random', 'delete', 'sequentialstring', 'randomstring', 'deletestring') 34 | 35 | for benchtype in benchtypes: 36 | nkeys = minkeys 37 | while nkeys <= maxkeys: 38 | for program in programs: 39 | fastest_attempt = 1000000 40 | fastest_attempt_data = '' 41 | 42 | for attempt in range(best_out_of): 43 | proc = subprocess.Popen(['./build/'+program, str(nkeys), benchtype], stdout=subprocess.PIPE) 44 | 45 | # wait for the program to fill up memory and spit out its "ready" message 46 | try: 47 | runtime = float(proc.stdout.readline().strip()) 48 | except: 49 | runtime = 0 50 | 51 | ps_proc = subprocess.Popen(['ps up %d | tail -n1' % proc.pid], shell=True, stdout=subprocess.PIPE) 52 | nbytes = int(ps_proc.stdout.read().split()[4]) * 1024 53 | ps_proc.wait() 54 | 55 | os.kill(proc.pid, signal.SIGKILL) 56 | proc.wait() 57 | 58 | if nbytes and runtime: # otherwise it crashed 59 | line = ','.join(map(str, [benchtype, nkeys, program, nbytes, "%0.6f" % runtime])) 60 | 61 | if runtime < fastest_attempt: 62 | fastest_attempt = runtime 63 | fastest_attempt_data = line 64 | 65 | if fastest_attempt != 1000000: 66 | print >> outfile, fastest_attempt_data 67 | print fastest_attempt_data 68 | 69 | nkeys += interval 70 | -------------------------------------------------------------------------------- /charts-template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 66 | 67 | 81 | 82 | 83 |

Sequential Inserts: Execution Time (integers)

84 |
85 |
number of entries in hash table
86 | 87 |

Sequential Inserts: Execution Time (strings)

88 |
89 |
number of entries in hash table
90 | 91 | 92 |

Random Inserts: Execution Time (integers)

93 |
94 |
number of entries in hash table
95 | 96 |

Random Inserts: Execution Time (strings)

97 |
98 |
number of entries in hash table
99 | 100 | 101 |

Deletes: Execution Time (integers)

102 |
103 |
number of entries in hash table
104 | 105 |

Deletes: Execution Time (strings)

106 |
107 |
number of entries in hash table
108 | 109 | 110 |

Memory Usage (integers)

111 |
112 |
number of entries in hash table
113 | 114 |

Memory Usage (strings)

115 |
116 |
number of entries in hash table
117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /make_chart_data.py: -------------------------------------------------------------------------------- 1 | # random,1310720,google_dense_hash_map,45621248,0.344362020493 2 | # random,2621440,glib_hash_table,109867008,1.01163601875 3 | # random,2621440,stl_unordered_map,130715648,1.73484396935 4 | # random,2621440,boost_unordered_map,108380160,1.11585187912 5 | # random,2621440,google_sparse_hash_map,37015552,1.76031804085 6 | # random,2621440,google_dense_hash_map,79175680,0.504401922226 7 | # random,5242880,glib_hash_table,210530304,1.86031603813 8 | # random,5242880,stl_unordered_map,250298368,3.81597208977 9 | # random,5242880,boost_unordered_map,192184320,2.63760495186 10 | # random,5242880,google_sparse_hash_map,62066688,3.93570995331 11 | # random,5242880,google_dense_hash_map,146284544,1.22620105743 12 | # random,10485760,glib_hash_table,411856896,4.16937494278 13 | # random,10485760,stl_unordered_map,490430464,7.91806197166 14 | # random,10485760,boost_unordered_map,359251968,7.52085900307 15 | # random,10485760,google_sparse_hash_map,111902720,8.11318516731 16 | # random,10485760,google_dense_hash_map,280502272,2.32930994034 17 | # random,20971520,glib_hash_table,814510080,8.32456207275 18 | # random,20971520,stl_unordered_map,971583488,16.1606841087 19 | # random,20971520,boost_unordered_map,692441088,24.5845990181 20 | # random,20971520,google_sparse_hash_map,211435520,16.2772600651 21 | # random,20971520,google_dense_hash_map,548937728,4.85360789299 22 | # random,41943040,glib_hash_table,1619816448,90.6313672066 23 | 24 | import sys, json 25 | 26 | lines = [ line.strip() for line in sys.stdin if line.strip() ] 27 | 28 | by_benchtype = {} 29 | 30 | for line in lines: 31 | benchtype, nkeys, program, nbytes, runtime = line.split(',') 32 | nkeys = int(nkeys) 33 | nbytes = int(nbytes) 34 | runtime = float(runtime) 35 | 36 | by_benchtype.setdefault("%s-runtime" % benchtype, {}).setdefault(program, []).append([nkeys, runtime]) 37 | if benchtype.startswith('sequential'): 38 | by_benchtype.setdefault("%s-memory" % benchtype, {}).setdefault(program, []).append([nkeys, nbytes]) 39 | 40 | proper_names = { 41 | 'boost_unordered_map': 'Boost 1.38 unordered_map', 42 | 'stl_unordered_map': 'GCC 4.4 std::unordered_map', 43 | 'google_sparse_hash_map': 'Google sparsehash 1.5.2 sparse_hash_map', 44 | 'google_dense_hash_map': 'Google sparsehash 1.5.2 dense_hash_map', 45 | 'glib_hash_table': 'Glib 2.22 GHashTable', 46 | 'qt_qhash': 'Qt 4.5 QHash', 47 | 'python_dict': 'Python 2.6 (C API) dict', 48 | 'ruby_hash': 'Ruby 1.9 (C API) Hash', 49 | } 50 | 51 | # do them in the desired order to make the legend not overlap the chart data 52 | # too much 53 | program_slugs = [ 54 | 'google_sparse_hash_map', 55 | 'google_dense_hash_map', 56 | 'stl_unordered_map', 57 | 'boost_unordered_map', 58 | 'python_dict', 59 | 'ruby_hash', 60 | 'glib_hash_table', 61 | 'qt_qhash', 62 | ] 63 | 64 | chart_data = {} 65 | 66 | for i, (benchtype, programs) in enumerate(by_benchtype.items()): 67 | chart_data[benchtype] = [] 68 | for j, program in enumerate(program_slugs): 69 | data = programs[program] 70 | chart_data[benchtype].append({ 71 | 'label': proper_names[program], 72 | 'data': [], 73 | }) 74 | 75 | for k, (nkeys, value) in enumerate(data): 76 | chart_data[benchtype][-1]['data'].append([nkeys, value]) 77 | 78 | print 'chart_data = ' + json.dumps(chart_data) 79 | -------------------------------------------------------------------------------- /make_html.py: -------------------------------------------------------------------------------- 1 | import sys 2 | html_template = file('charts-template.html', 'r').read() 3 | file('charts.html', 'w').write(html_template.replace('__CHART_DATA_GOES_HERE__', sys.stdin.read())) 4 | -------------------------------------------------------------------------------- /src/boost_unordered_map.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | typedef boost::unordered_map hash_t; 4 | typedef boost::unordered_map str_hash_t; 5 | #define SETUP hash_t hash; str_hash_t str_hash; 6 | #define INSERT_INT_INTO_HASH(key, value) hash.insert(hash_t::value_type(key, value)) 7 | #define DELETE_INT_FROM_HASH(key) hash.erase(key) 8 | #define INSERT_STR_INTO_HASH(key, value) str_hash.insert(str_hash_t::value_type(key, value)) 9 | #define DELETE_STR_FROM_HASH(key) str_hash.erase(key) 10 | #include "template.c" 11 | -------------------------------------------------------------------------------- /src/glib_hash_table.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define SETUP GHashTable * hash = g_hash_table_new(g_direct_hash, g_direct_equal); \ 3 | GHashTable * str_hash = g_hash_table_new(g_str_hash, g_str_equal); 4 | #define INSERT_INT_INTO_HASH(key, value) g_hash_table_insert(hash, GINT_TO_POINTER(key), &value) 5 | #define DELETE_INT_FROM_HASH(key) g_hash_table_remove(hash, GINT_TO_POINTER(key)) 6 | #define INSERT_STR_INTO_HASH(key, value) g_hash_table_insert(str_hash, key, &value) 7 | #define DELETE_STR_FROM_HASH(key) g_hash_table_remove(str_hash, key) 8 | #include "template.c" 9 | -------------------------------------------------------------------------------- /src/google_dense_hash_map.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | typedef google::dense_hash_map hash_t; 4 | typedef google::dense_hash_map str_hash_t; 5 | #define SETUP hash_t hash; hash.set_empty_key(-1); hash.set_deleted_key(-2); \ 6 | str_hash_t str_hash; str_hash.set_empty_key(""); str_hash.set_deleted_key("d"); 7 | #define INSERT_INT_INTO_HASH(key, value) hash.insert(hash_t::value_type(key, value)) 8 | #define DELETE_INT_FROM_HASH(key) hash.erase(key) 9 | #define INSERT_STR_INTO_HASH(key, value) str_hash.insert(str_hash_t::value_type(key, value)) 10 | #define DELETE_STR_FROM_HASH(key) str_hash.erase(key) 11 | #include "template.c" 12 | -------------------------------------------------------------------------------- /src/google_sparse_hash_map.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | typedef google::sparse_hash_map hash_t; 4 | typedef google::sparse_hash_map str_hash_t; 5 | #define SETUP hash_t hash; hash.set_deleted_key(-1); \ 6 | str_hash_t str_hash; str_hash.set_deleted_key(""); 7 | #define INSERT_INT_INTO_HASH(key, value) hash.insert(hash_t::value_type(key, value)) 8 | #define DELETE_INT_FROM_HASH(key) hash.erase(key) 9 | #define INSERT_STR_INTO_HASH(key, value) str_hash.insert(str_hash_t::value_type(key, value)) 10 | #define DELETE_STR_FROM_HASH(key) str_hash.erase(key) 11 | #include "template.c" 12 | -------------------------------------------------------------------------------- /src/python_dict.c: -------------------------------------------------------------------------------- 1 | #include 2 | typedef PyObject * hash_t; 3 | #define SETUP \ 4 | Py_Initialize(); \ 5 | hash_t hash = PyDict_New(); \ 6 | PyObject * py_int_value = PyInt_FromLong(0); 7 | #define INSERT_INT_INTO_HASH(key, value) do { \ 8 | PyObject * py_int_key = PyInt_FromLong(key); /* leak */ \ 9 | PyDict_SetItem(hash, py_int_key, py_int_value); \ 10 | } while(0) 11 | #define DELETE_INT_FROM_HASH(key) do { \ 12 | PyObject * py_int_key = PyInt_FromLong(key); /* leak */ \ 13 | PyDict_DelItem(hash, py_int_key); \ 14 | } while(0) 15 | #define INSERT_STR_INTO_HASH(key, value) do { \ 16 | PyDict_SetItemString(hash, key, py_int_value); \ 17 | } while(0) 18 | #define DELETE_STR_FROM_HASH(key) do { \ 19 | PyDict_DelItemString(hash, key); \ 20 | } while(0) 21 | #include "template.c" 22 | -------------------------------------------------------------------------------- /src/qt_qhash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | typedef QHash hash_t; 4 | typedef QHash str_hash_t; 5 | #define SETUP hash_t hash; str_hash_t str_hash; 6 | #define INSERT_INT_INTO_HASH(key, value) hash.insert(key, value) 7 | #define DELETE_INT_FROM_HASH(key) hash.remove(key) 8 | #define INSERT_STR_INTO_HASH(key, value) str_hash.insert(key, value) 9 | #define DELETE_STR_FROM_HASH(key) str_hash.remove(key) 10 | #include "template.c" 11 | -------------------------------------------------------------------------------- /src/ruby_hash.c: -------------------------------------------------------------------------------- 1 | #include 2 | #define SETUP \ 3 | ruby_init(); \ 4 | VALUE hash = rb_hash_new(); \ 5 | VALUE rb_int_value = INT2NUM(0); 6 | #define INSERT_INT_INTO_HASH(key, value) do { \ 7 | VALUE rb_int_key = INT2NUM(key); /* leak */ \ 8 | rb_hash_aset(hash, rb_int_key, rb_int_value); \ 9 | } while(0) 10 | #define DELETE_INT_FROM_HASH(key) do { \ 11 | VALUE rb_int_key = INT2NUM(key); /* leak */ \ 12 | rb_hash_delete(hash, rb_int_key); \ 13 | } while(0) 14 | #define INSERT_STR_INTO_HASH(key, value) do { \ 15 | VALUE rb_str_key = rb_str_new2(key); /* leak */ \ 16 | rb_hash_aset(hash, rb_str_key, rb_int_value); \ 17 | } while(0) 18 | #define DELETE_STR_FROM_HASH(key) do { \ 19 | VALUE rb_str_key = rb_str_new2(key); /* leak */ \ 20 | rb_hash_delete(hash, rb_str_key); \ 21 | } while(0) 22 | #include "template.c" 23 | -------------------------------------------------------------------------------- /src/stl_unordered_map.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | typedef std::unordered_map hash_t; 4 | typedef std::unordered_map str_hash_t; 5 | #define SETUP hash_t hash; str_hash_t str_hash; 6 | #define INSERT_INT_INTO_HASH(key, value) hash.insert(hash_t::value_type(key, value)) 7 | #define DELETE_INT_FROM_HASH(key) hash.erase(key); 8 | #define INSERT_STR_INTO_HASH(key, value) str_hash.insert(str_hash_t::value_type(key, value)) 9 | #define DELETE_STR_FROM_HASH(key) str_hash.erase(key); 10 | #include "template.c" 11 | -------------------------------------------------------------------------------- /src/template.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | double get_time(void) 11 | { 12 | struct timeval tv; 13 | gettimeofday(&tv, NULL); 14 | return tv.tv_sec + (tv.tv_usec / 1000000.0); 15 | } 16 | 17 | char * new_string_from_integer(int num) 18 | { 19 | int ndigits = num == 0 ? 1 : (int)log10(num) + 1; 20 | char * str = (char *)malloc(ndigits + 1); 21 | sprintf(str, "%d", num); 22 | return str; 23 | } 24 | 25 | int main(int argc, char ** argv) 26 | { 27 | int num_keys = atoi(argv[1]); 28 | int i, value = 0; 29 | 30 | if(argc <= 2) 31 | return 1; 32 | 33 | SETUP 34 | 35 | double before = get_time(); 36 | 37 | if(!strcmp(argv[2], "sequential")) 38 | { 39 | for(i = 0; i < num_keys; i++) 40 | INSERT_INT_INTO_HASH(i, value); 41 | } 42 | 43 | else if(!strcmp(argv[2], "random")) 44 | { 45 | srandom(1); // for a fair/deterministic comparison 46 | for(i = 0; i < num_keys; i++) 47 | INSERT_INT_INTO_HASH((int)random(), value); 48 | } 49 | 50 | else if(!strcmp(argv[2], "delete")) 51 | { 52 | for(i = 0; i < num_keys; i++) 53 | INSERT_INT_INTO_HASH(i, value); 54 | before = get_time(); 55 | for(i = 0; i < num_keys; i++) 56 | DELETE_INT_FROM_HASH(i); 57 | } 58 | 59 | else if(!strcmp(argv[2], "sequentialstring")) 60 | { 61 | for(i = 0; i < num_keys; i++) 62 | INSERT_STR_INTO_HASH(new_string_from_integer(i), value); 63 | } 64 | 65 | else if(!strcmp(argv[2], "randomstring")) 66 | { 67 | srandom(1); // for a fair/deterministic comparison 68 | for(i = 0; i < num_keys; i++) 69 | INSERT_STR_INTO_HASH(new_string_from_integer((int)random()), value); 70 | } 71 | 72 | else if(!strcmp(argv[2], "deletestring")) 73 | { 74 | for(i = 0; i < num_keys; i++) 75 | INSERT_STR_INTO_HASH(new_string_from_integer(i), value); 76 | before = get_time(); 77 | for(i = 0; i < num_keys; i++) 78 | DELETE_STR_FROM_HASH(new_string_from_integer(i)); 79 | } 80 | 81 | double after = get_time(); 82 | printf("%f\n", after-before); 83 | fflush(stdout); 84 | sleep(1000000); 85 | } 86 | --------------------------------------------------------------------------------