├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── scripts ├── all.sh ├── big.sh ├── generatebig.sh ├── hot_roaring.sh └── roaring.sh ├── src ├── benchmark.h ├── bitmagic_benchmarks.cpp ├── bitset_benchmarks.c ├── cmemcounter.h ├── concise_benchmarks.cpp ├── ewah32_benchmarks.cpp ├── ewah64_benchmarks.cpp ├── hot_roaring_benchmarks.c ├── memtrackingallocator.h ├── numbersfromtextfiles.h ├── roaring_benchmarks.c ├── stl_hashset_benchmarks.cpp ├── stl_vector_benchmarks.cpp └── wah32_benchmarks.cpp └── synthetic ├── anh_moffat_clustered.h └── gen.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "CRoaring"] 2 | path = CRoaring 3 | url = https://github.com/RoaringBitmap/CRoaring.git 4 | [submodule "EWAHBoolArray"] 5 | path = EWAHBoolArray 6 | url = https://github.com/lemire/EWAHBoolArray.git 7 | [submodule "cbitset"] 8 | path = cbitset 9 | url = https://github.com/lemire/cbitset.git 10 | [submodule "Concise"] 11 | path = Concise 12 | url = https://github.com/lemire/Concise.git 13 | [submodule "BitMagic"] 14 | path = BitMagic 15 | url = https://github.com/tlk00/BitMagic 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # minimalist makefile 2 | ###################### 3 | # To add a competitive technique, simply add your code in the src subdirectory (follow the README.md instructions) and 4 | # add your executable file name to the EXECUTABLES variable below. 5 | # along with a target to build it. 6 | ####################### 7 | .SUFFIXES: 8 | # 9 | .SUFFIXES: .cpp .o .c .h 10 | 11 | .PHONY: clean 12 | UNAME := $(shell uname) 13 | ifeq ($(UNAME), Linux) 14 | OSFLAGS= -Wl,--no-as-needed 15 | endif 16 | 17 | ####################### 18 | # SunOS gcc7.2.0 modifications QSI/Jon Strabala 19 | ######### 20 | # original CXX flag, new for FLAGS 21 | ifeq ($(UNAME), SunOS) 22 | # must be 64 bit compile, new for CFLAGS 23 | OSFLAGS= -m64 24 | # force gnu99 intead of c99 for getopt, new for CFLAGS 25 | OSCFLAGS= -std=gnu99 26 | endif 27 | 28 | 29 | 30 | ifeq ($(DEBUG),1) 31 | CFLAGS = -fuse-ld=gold -fPIC -std=c99 -ggdb -mavx2 -mbmi2 -march=native -Wall -Wextra -Wshadow -fsanitize=undefined -fno-omit-frame-pointer -fsanitize=address $(OSFLAGS) $(OSCFLAGS) -ldl 32 | CXXFLAGS = -fuse-ld=gold -fPIC -std=c++11 -ggdb -mavx2 -mbmi2 -march=native -Wall -Wextra -Wshadow -fsanitize=undefined -fno-omit-frame-pointer -fsanitize=address $(OSFLAGS) -ldl 33 | ROARFLAGS = -DCMAKE_BUILD_TYPE=Debug -DSANITIZE=ON 34 | else 35 | CFLAGS = -ggdb -fPIC -std=c99 -O3 -mavx2 -mbmi2 -march=native -Wall -Wextra -Wshadow $(OSFLAGS) -ldl 36 | CXXFLAGS = -fPIC -std=c++11 -O3 -mavx2 -mbmi2 -march=native -Wall -Wextra -Wshadow $(OSFLAGS) -ldl 37 | ROARFLAGS = -DCMAKE_BUILD_TYPE=Release 38 | endif # debug 39 | 40 | 41 | 42 | EXECUTABLES=wah32_benchmarks concise_benchmarks roaring_benchmarks slow_roaring_benchmarks bitmagic_benchmarks ewah32_benchmarks ewah64_benchmarks stl_vector_benchmarks stl_hashset_benchmarks stl_vector_benchmarks_memtracked stl_hashset_benchmarks_memtracked bitset_benchmarks malloced_roaring_benchmarks hot_roaring_benchmarks hot_slow_roaring_benchmarks gen 43 | 44 | all: $(EXECUTABLES) 45 | 46 | test: 47 | ./scripts/all.sh 48 | 49 | bigtest: 50 | ./scripts/big.sh 51 | 52 | hottest: 53 | ./scripts/hot_roaring.sh 54 | 55 | 56 | 57 | 58 | src/roaring.c : 59 | (cd src && exec ../CRoaring/amalgamation.sh && rm almagamation_demo.c && rm almagamation_demo.cpp) 60 | 61 | gen : synthetic/anh_moffat_clustered.h synthetic/gen.cpp 62 | $(CXX) $(CXXFLAGS) -o gen synthetic/gen.cpp -Isynthetic 63 | 64 | roaring_benchmarks : src/roaring.c src/roaring_benchmarks.c 65 | $(CC) $(CFLAGS) -o roaring_benchmarks src/roaring_benchmarks.c 66 | 67 | 68 | hot_roaring_benchmarks : src/roaring.c src/hot_roaring_benchmarks.c 69 | $(CC) $(CFLAGS) -ggdb -o hot_roaring_benchmarks src/hot_roaring_benchmarks.c 70 | 71 | malloced_roaring_benchmarks : src/roaring.c src/roaring_benchmarks.c 72 | $(CC) $(CFLAGS) -o malloced_roaring_benchmarks src/roaring_benchmarks.c -DRECORD_MALLOCS 73 | 74 | 75 | slow_roaring_benchmarks : src/roaring.c src/roaring_benchmarks.c 76 | $(CC) $(CFLAGS) -DDISABLE_X64 -o slow_roaring_benchmarks src/roaring_benchmarks.c 77 | 78 | hot_slow_roaring_benchmarks : src/roaring.c src/hot_roaring_benchmarks.c 79 | $(CC) $(CFLAGS) -ggdb -DDISABLE_X64 -o hot_slow_roaring_benchmarks src/hot_roaring_benchmarks.c 80 | 81 | 82 | bitmagic_benchmarks: src/bitmagic_benchmarks.cpp 83 | $(CXX) $(CXXFLAGS) -o bitmagic_benchmarks src/bitmagic_benchmarks.cpp -IBitMagic/src 84 | 85 | ewah32_benchmarks: src/ewah32_benchmarks.cpp 86 | $(CXX) $(CXXFLAGS) -o ewah32_benchmarks ./src/ewah32_benchmarks.cpp -IEWAHBoolArray/headers 87 | 88 | wah32_benchmarks: src/wah32_benchmarks.cpp 89 | $(CXX) $(CXXFLAGS) -o wah32_benchmarks ./src/wah32_benchmarks.cpp -IConcise/include 90 | 91 | concise_benchmarks: src/concise_benchmarks.cpp 92 | $(CXX) $(CXXFLAGS) -o concise_benchmarks ./src/concise_benchmarks.cpp -IConcise/include 93 | 94 | ewah64_benchmarks: src/ewah64_benchmarks.cpp 95 | $(CXX) $(CXXFLAGS) -o ewah64_benchmarks ./src/ewah64_benchmarks.cpp -IEWAHBoolArray/headers 96 | 97 | stl_vector_benchmarks: src/stl_vector_benchmarks.cpp src/memtrackingallocator.h 98 | $(CXX) $(CXXFLAGS) -o stl_vector_benchmarks ./src/stl_vector_benchmarks.cpp 99 | 100 | stl_hashset_benchmarks: src/stl_hashset_benchmarks.cpp src/memtrackingallocator.h 101 | $(CXX) $(CXXFLAGS) -o stl_hashset_benchmarks ./src/stl_hashset_benchmarks.cpp 102 | 103 | 104 | stl_vector_benchmarks_memtracked: src/stl_vector_benchmarks.cpp src/memtrackingallocator.h 105 | $(CXX) $(CXXFLAGS) -o stl_vector_benchmarks_memtracked ./src/stl_vector_benchmarks.cpp -DMEMTRACKED 106 | 107 | stl_hashset_benchmarks_memtracked: src/stl_hashset_benchmarks.cpp src/memtrackingallocator.h 108 | $(CXX) $(CXXFLAGS) -o stl_hashset_benchmarks_memtracked ./src/stl_hashset_benchmarks.cpp -DMEMTRACKED 109 | 110 | bitset_benchmarks: src/bitset_benchmarks.c cbitset/include/bitset.h cbitset/src/bitset.c 111 | $(CC) $(CFLAGS) -o bitset_benchmarks ./src/bitset_benchmarks.c cbitset/src/bitset.c -Icbitset/include 112 | 113 | clean: 114 | rm -r -f $(EXECUTABLES) src/roaring.c src/roaring.h src/roaring.hh bigtmp 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CBitmapCompetition 2 | 3 | ## Description 4 | The goal of this project is to compare on realistic data different C and C++ techniques representing sets of integers. 5 | 6 | We start from a collection of sets of sorted integers. 7 | For example, we might have 200 sets. This collection of sets is considered immutable (none of the benchmarks are allowed to modify them). 8 | For each of these collections of sets, we do the following : 9 | 10 | - We report the memory usage as best we can. Of course, data can always be compressed more or less aggressively to disk, but an important metric is how much RAM the bitmaps are using. It can be difficult to track every byte allocated, but we should do our best to get the memory usage within 1%. In C++, for STL containers, this can be done with a custom memory allocator that tracks memory allocation. 11 | - Intersections between successive bitmaps, each time generating a new bitmap. Thus if there are 200 bitmaps, we generate 199 temporary bitmaps representing the intersection. We then query and check the cardinality of the each intermediate bitmap (by summing them up). 12 | - Union between successive bitmaps, each time generating a new bitmap. We then query and check the cardinality of the result by summing them up. 13 | - We compute the union of the whole set and check the cardinality of the result. This can be done in various ways. A basic way involves doing successive in-place unions which should be the default. In this instance, it is fair to avoid creating temporary bitmaps. 14 | - Given that the maximal value stored in any bitmap is N, we check whether integers N/4, N/2 and 3N/4 belong to each bitmap (henceforth a quartile query). 15 | - Differences between successive bitmaps (as with intersections and unions above). 16 | - Symmetric differences between successive bitmaps (as with intersections and unions above). 17 | - Iteration through all set values. 18 | - Computing the intersection size between successive sets without materializing the answer. 19 | - Computing the union size between successive sets without materializing the answer. 20 | - Computing the difference size between successive sets without materializing the answer. 21 | - Computing the symmetric intersection size between successive sets without materializing the answer. 22 | 23 | Thus, we output several numbers: 24 | - The memory usage, expressed as a floating-point numbers representing the number of bits used by value. In practice, this is computed by counting the number of bits used in RAM divided by the total cardinality. The total cardinality is given by the sum of the cardinalities of all sets. Thus if you have 200 sets each containing 10 values, your total cardinality is 2000, if you use 2000 bytes of memory, then your number of bits per value will be 8.0. 25 | - The number of CPU cycles used to compute the successive intersections, normalized by the number of input values. 26 | - The number of CPU cycles used to compute the successive unions, normalized by the number of input values. 27 | - The number of CPU cycles used to compute the total union using a naive algorithm, normalized by the number of input values. 28 | - The number of CPU cycles used to compute the total union using a heap-based or sort algorithm, normalized by the number of input values. In some cases, there might not be a reasonable heap-based or sort algorithm, but that is ok. 29 | - The number of CPU cycles to for each quartile query. 30 | - The number of CPU cycles used to compute the successive differences, normalized by the number of input values. 31 | - The number of CPU cycles used to compute the successive symmetric differences, normalized by the number of input values. 32 | - The number of CPU cycles used to iterate through all values, counting them, normalized by the number of input values. 33 | - The number of CPU cycles used to compute the size of successive intersections, normalized by the number of input values. 34 | - The number of CPU cycles used to compute the size of successive unions, normalized by the number of input values. 35 | - The number of CPU cycles used to compute the size of successive differences, normalized by the number of input values. 36 | - The number of CPU cycles used to compute the size of successive symmetric differences, normalized by the number of input values. 37 | 38 | The normalization proceeds as follows. For the total union, we divide by the sum of the cardinality of all sets. For the successive intersections and unions, we divide by the number the sum of the cardinalities of the pairs of sets (so that, effectively, all but the first and last sets count twice). 39 | 40 | For each competitive technique, we apply the following recipe: 41 | 42 | * We assume a Linux-like environment. 43 | * We put any non-standard dependency as a submodule of the project. 44 | * We put the benchmarking code in a separate executable file in the ``src`` directory. Having separate files for each technique introduces redundancies and possibly errors, but it keeps the complexity of the project low. Each executable file is simple and thus easy to examine and debug. If we need to study the performance of one technique with tools like ``perf``, it is much easier of we have several independent executables. 45 | * The executable should be able to take as the sole parameter a directory name containing text files where each text file is a comma-separated list of sorted integers (e.g., one of these directories: https://github.com/RoaringBitmap/CRoaring/tree/master/benchmarks/realdata). We have a header file in the CRoaring project to help parsing such data files... https://github.com/RoaringBitmap/CRoaring/blob/master/benchmarks/numbersfromtextfiles.h Consider using it. The code should take just one such directory at a time as a parameter. 46 | * The executable should output the 4 performance numbers (memory usage, number of cycles...) on a single line, separated by spaces. If anything else is outputted, it should be prefixed by the '#' character. It is allowed for the executable to have a verbose flag (-v) that provides more insight into the results. 47 | * The executable should not try to outsmart the benchmark. Keep your code simple. 48 | * The ``Makefile`` must be such that ``make`` will build the executable. 49 | * The scripts/all.sh script should be modified so that the executable is called (this should only require adding the executable file name to the ``commands`` array). 50 | 51 | For a working example, see https://github.com/RoaringBitmap/CBitmapCompetition/blob/master/src/bitset_benchmarks.c 52 | 53 | 54 | 55 | ## Results 56 | 57 | GCC 6.2, Skylake processor (3.4 GHz), CRoaring 0.2.3 (Sept. 9 2016). 58 | 59 | For the first column of numbers, smaller is better (it is memory usage) whereas for all other columns, smaller is better (they are CPU cycles per operation). 60 | 61 | First column is memory usage per value in bits. 62 | Second column is 2-by-2 intersection time (with cardinality computation). Third column is 2-by-2 union time (with cardinality computation). 63 | Fourth and fifth columns are wide unions times (unions of many sets) using either a naive or a heap-based approach depending on the column. 64 | Last column is random access time. 65 | 66 | Where it appears, the "-r" flag means "with better compression". The "-c" flag means "copy-on-write". 67 | 68 | 69 | 70 | ```bash 71 | $ make test 72 | ./scripts/all.sh 73 | # For each data set, we print data size (in bits per value), successive intersections, successive unions and total unions [we compute the total union first naively and then (if supported) using a heap-based approach], followed by quartile point queries (in cycles per input value), successive differences, successive symmetric differences, iterations through all values, then we have pairwise count aggregates for successive intersections, successive unions, successive differences, successive symmetric differences 74 | # processing file census-income 75 | # bitset_benchmarks 76 | 5.66 0.13 0.13 0.09 0.09 3.67 0.12 0.12 7.46 0.09 0.07 0.09 0.09 77 | # stl_vector_benchmarks 78 | 0.00 7.84 7.69 43.43 16.74 420.20 8.38 8.79 0.00 4.85 4.62 5.21 4.80 79 | # stl_vector_benchmarks_memtracked 80 | 32.00 7.86 9.64 54.97 19.62 439.50 9.45 10.73 0.00 4.82 4.61 5.25 4.95 81 | # stl_hashset_benchmarks_memtracked 82 | 194.63 55.35 208.06 99.46 131.85 60.23 175.68 266.44 28.88 13.57 13.59 48.30 13.58 83 | # stl_hashset_benchmarks 84 | 0.00 55.28 209.50 98.41 131.48 60.83 176.81 266.03 28.68 13.28 13.38 47.43 13.33 85 | # bitmagic_benchmarks 86 | 7.76 0.31 0.33 0.18 0.22 15.95 0.32 0.32 21.66 0.12 0.15 0.14 0.14 87 | # bitmagic_benchmarks -r 88 | 4.46 0.38 0.35 0.16 0.20 38.36 0.34 0.35 20.73 0.14 0.37 0.28 0.39 89 | # slow_roaring_benchmarks -r 90 | 2.60 0.45 0.41 0.06 0.60 64.85 0.52 0.54 5.97 0.36 0.36 0.36 0.36 91 | # malloced_roaring_benchmarks -r 92 | 2.62 0.30 0.36 0.05 0.55 62.99 0.36 0.46 5.99 0.13 0.13 0.13 0.13 93 | # roaring_benchmarks -r 94 | 2.60 0.22 0.27 0.05 0.47 63.47 0.28 0.37 5.95 0.13 0.13 0.13 0.13 95 | # roaring_benchmarks -c -r 96 | 2.60 0.22 0.27 0.05 0.47 64.63 0.28 0.37 5.95 0.13 0.13 0.13 0.13 97 | # roaring_benchmarks 98 | 2.74 0.19 0.25 0.08 0.47 63.62 0.22 0.32 5.93 0.10 0.10 0.10 0.10 99 | # roaring_benchmarks -c 100 | 2.74 0.19 0.25 0.08 0.47 65.31 0.22 0.32 5.93 0.10 0.10 0.10 0.10 101 | # ewah32_benchmarks 102 | 3.29 1.41 2.02 0.50 1.58 3145.69 1.72 1.96 13.09 0.80 1.11 0.95 1.13 103 | # ewah64_benchmarks 104 | 3.86 0.85 1.07 0.27 1.18 1755.53 0.95 1.05 9.12 0.43 0.60 0.50 0.61 105 | # wah32_benchmarks 106 | 3.37 1.80 2.11 1.83 3.42 19323.53 1.98 2.04 9.81 0.98 0.99 1.00 0.98 107 | # concise_benchmarks 108 | 2.94 2.15 2.44 2.25 3.86 19107.68 2.34 2.38 9.45 1.25 1.32 1.26 1.30 109 | 110 | 111 | # processing file census-income_srt 112 | # bitset_benchmarks 113 | 6.01 0.15 0.14 0.10 0.10 3.65 0.12 0.13 6.72 0.09 0.07 0.10 0.10 114 | # stl_vector_benchmarks 115 | 0.00 4.52 5.45 37.25 10.89 563.99 5.54 5.99 0.00 2.08 1.87 2.28 2.10 116 | # stl_vector_benchmarks_memtracked 117 | 32.00 4.68 7.44 50.37 13.92 551.73 6.70 7.98 0.00 2.10 1.86 2.29 2.12 118 | # stl_hashset_benchmarks_memtracked 119 | 195.02 28.23 172.98 59.20 76.33 55.05 132.91 216.61 12.67 6.66 6.67 29.80 6.68 120 | # stl_hashset_benchmarks 121 | 0.00 28.50 172.71 60.18 77.28 55.39 133.72 215.42 12.69 6.76 6.76 29.54 6.77 122 | # bitmagic_benchmarks 123 | 7.99 0.32 0.36 0.19 0.23 16.19 0.33 0.36 10.70 0.11 0.15 0.14 0.14 124 | # bitmagic_benchmarks -r 125 | 1.89 0.25 0.27 0.19 0.23 61.71 0.26 0.28 6.84 0.12 0.20 0.19 0.22 126 | # slow_roaring_benchmarks -r 127 | 0.60 0.17 0.30 0.12 0.03 55.90 0.26 0.37 5.34 0.13 0.14 0.13 0.13 128 | # malloced_roaring_benchmarks -r 129 | 0.62 0.24 0.39 0.11 0.05 56.63 0.33 0.45 6.32 0.11 0.12 0.12 0.12 130 | # roaring_benchmarks -r 131 | 0.60 0.16 0.30 0.11 0.03 73.95 0.24 0.35 5.35 0.11 0.12 0.12 0.12 132 | # roaring_benchmarks -c -r 133 | 0.60 0.16 0.30 0.11 0.03 77.79 0.24 0.34 5.35 0.11 0.12 0.12 0.12 134 | # roaring_benchmarks 135 | 2.99 0.16 0.28 0.09 0.54 63.34 0.19 0.32 5.68 0.08 0.08 0.08 0.08 136 | # roaring_benchmarks -c 137 | 2.99 0.16 0.28 0.09 0.55 68.58 0.18 0.31 5.68 0.08 0.08 0.08 0.08 138 | # ewah32_benchmarks 139 | 0.64 0.28 0.56 0.16 0.13 915.91 0.42 0.62 6.11 0.19 0.29 0.23 0.31 140 | # ewah64_benchmarks 141 | 0.90 0.21 0.40 0.12 0.10 656.48 0.30 0.43 5.94 0.14 0.21 0.17 0.23 142 | # wah32_benchmarks 143 | 0.65 0.45 0.65 0.47 0.47 3698.61 0.54 0.66 7.63 0.26 0.28 0.28 0.28 144 | # concise_benchmarks 145 | 0.55 0.56 0.76 0.59 0.62 3803.17 0.65 0.79 7.84 0.34 0.38 0.35 0.38 146 | 147 | 148 | # processing file census1881 149 | # bitset_benchmarks 150 | 523.54 19.50 13.64 9.85 9.54 2.32 11.10 13.25 43.39 6.31 7.62 8.26 9.79 151 | # stl_vector_benchmarks 152 | 0.00 3.28 6.10 542.25 66.63 177.57 5.03 6.38 0.00 1.45 1.27 1.63 1.46 153 | # stl_vector_benchmarks_memtracked 154 | 32.00 3.31 8.09 722.11 75.54 171.28 6.07 8.50 0.00 1.46 1.27 1.63 1.46 155 | # stl_hashset_benchmarks_memtracked 156 | 194.74 0.49 265.16 1006.16 1232.57 40.27 235.58 369.01 67.66 0.48 0.22 44.47 0.46 157 | # stl_hashset_benchmarks 158 | 0.00 0.48 265.35 1006.96 1235.44 39.94 235.93 369.39 67.33 0.46 0.22 44.65 0.47 159 | # bitmagic_benchmarks 160 | 98.93 1.87 6.79 3.02 4.24 11.00 3.70 6.67 82.82 0.24 2.20 1.43 2.18 161 | # bitmagic_benchmarks -r 162 | 46.32 1.07 3.68 2.18 3.89 13.06 2.56 3.61 69.68 0.09 1.18 1.60 1.11 163 | # slow_roaring_benchmarks -r 164 | 15.08 0.10 1.10 4.08 10.80 16.84 0.62 1.10 6.09 0.09 0.10 0.09 0.09 165 | # malloced_roaring_benchmarks -r 166 | 15.35 0.27 2.80 2.79 10.86 18.14 1.53 2.67 5.25 0.08 0.09 0.08 0.08 167 | # roaring_benchmarks -r 168 | 15.08 0.09 1.13 2.56 8.76 18.68 0.61 1.13 6.08 0.08 0.10 0.08 0.08 169 | # roaring_benchmarks -c -r 170 | 15.08 0.09 0.52 2.55 8.76 20.20 0.25 0.37 6.11 0.08 0.10 0.08 0.09 171 | # roaring_benchmarks 172 | 15.97 0.09 1.06 2.73 6.61 18.66 0.58 1.03 6.11 0.08 0.08 0.08 0.08 173 | # roaring_benchmarks -c 174 | 15.97 0.09 0.42 2.73 6.61 20.44 0.23 0.35 6.13 0.08 0.09 0.08 0.08 175 | # ewah32_benchmarks 176 | 33.77 3.74 28.19 276.13 53.98 8533.38 15.83 27.50 29.40 3.71 12.07 8.14 12.02 177 | # ewah64_benchmarks 178 | 43.77 1.95 15.08 102.71 26.78 4864.88 8.51 14.83 26.82 1.81 6.71 4.25 6.74 179 | # wah32_benchmarks 180 | 34.32 14.53 29.27 387.67 44.85 26757.08 21.90 28.93 29.26 9.04 11.58 10.77 11.55 181 | # concise_benchmarks 182 | 25.56 20.88 33.15 401.82 57.26 30027.37 26.73 33.33 31.71 12.13 16.83 14.49 16.91 183 | 184 | 185 | # processing file census1881_srt 186 | # bitset_benchmarks 187 | 888.07 32.96 23.73 16.22 16.12 2.61 19.39 22.48 41.24 11.42 12.55 14.21 16.30 188 | # stl_vector_benchmarks 189 | 0.00 3.96 6.18 618.55 35.22 158.18 5.76 6.41 0.00 1.70 1.57 1.99 1.78 190 | # stl_vector_benchmarks_memtracked 191 | 32.00 3.95 8.39 839.73 41.27 159.29 6.80 8.74 0.00 1.77 1.56 1.94 1.77 192 | # stl_hashset_benchmarks_memtracked 193 | 194.93 1.39 200.69 357.66 400.06 39.89 171.04 243.14 20.37 1.05 0.84 21.12 0.77 194 | # stl_hashset_benchmarks 195 | 0.00 1.41 200.71 353.14 399.22 39.81 171.21 243.20 20.57 1.05 0.84 21.52 0.80 196 | # bitmagic_benchmarks 197 | 249.26 4.93 16.64 7.23 9.13 11.40 9.22 16.57 31.85 0.77 5.52 3.66 5.53 198 | # bitmagic_benchmarks -r 199 | 16.00 0.83 2.47 4.58 3.63 13.01 2.94 2.56 10.88 0.12 0.52 3.19 0.52 200 | # slow_roaring_benchmarks -r 201 | 2.16 0.14 0.93 3.36 4.29 18.15 0.57 0.93 6.35 0.06 0.15 0.09 0.12 202 | # malloced_roaring_benchmarks -r 203 | 2.77 0.53 4.98 3.81 9.14 19.85 2.73 4.87 7.25 0.06 0.14 0.10 0.13 204 | # roaring_benchmarks -r 205 | 2.16 0.14 0.95 3.26 4.22 19.63 0.57 0.95 6.40 0.06 0.14 0.09 0.13 206 | # roaring_benchmarks -c -r 207 | 2.16 0.14 0.56 3.38 4.32 20.52 0.32 0.43 6.41 0.06 0.15 0.10 0.14 208 | # roaring_benchmarks 209 | 6.09 0.14 1.18 4.03 8.79 25.15 0.69 1.12 5.81 0.08 0.09 0.08 0.08 210 | # roaring_benchmarks -c 211 | 6.09 0.15 0.51 4.23 8.82 26.81 0.30 0.39 5.83 0.08 0.10 0.08 0.09 212 | # ewah32_benchmarks 213 | 2.91 0.50 2.50 171.88 5.21 541.83 1.46 2.47 8.82 0.44 1.06 0.72 1.07 214 | # ewah64_benchmarks 215 | 4.54 0.39 2.16 129.26 4.60 446.67 1.23 2.09 8.12 0.32 0.96 0.59 0.96 216 | # wah32_benchmarks 217 | 2.95 1.62 3.03 197.52 5.72 1784.53 2.31 2.97 9.92 1.02 1.28 1.04 1.27 218 | # concise_benchmarks 219 | 2.48 1.94 3.38 225.09 6.56 1888.65 2.59 3.38 10.16 1.27 1.65 1.28 1.66 220 | 221 | 222 | # processing file weather_sept_85 223 | # bitset_benchmarks 224 | 15.26 0.42 0.40 0.35 0.37 3.53 0.39 0.40 10.04 0.27 0.23 0.28 0.29 225 | # stl_vector_benchmarks 226 | 0.00 7.36 7.70 94.08 27.33 582.88 8.32 8.72 0.00 4.54 4.36 5.11 4.51 227 | # stl_vector_benchmarks_memtracked 228 | 32.00 7.27 9.97 118.54 31.49 582.29 9.43 11.15 0.00 4.53 4.34 5.15 4.61 229 | # stl_hashset_benchmarks_memtracked 230 | 194.93 35.52 251.43 238.98 258.58 61.76 248.34 361.60 41.40 17.75 17.77 62.65 17.80 231 | # stl_hashset_benchmarks 232 | 0.00 35.46 250.10 236.64 255.70 59.45 248.38 358.76 41.10 17.52 17.58 61.38 17.57 233 | # bitmagic_benchmarks 234 | 14.80 0.59 0.67 0.37 0.44 15.85 0.61 0.67 33.14 0.24 0.28 0.27 0.28 235 | # bitmagic_benchmarks -r 236 | 8.79 0.59 0.60 0.28 0.36 31.25 0.55 0.59 31.73 0.21 0.57 0.45 0.60 237 | # slow_roaring_benchmarks -r 238 | 5.38 0.99 0.88 0.25 1.40 80.36 1.10 1.06 6.57 0.89 0.90 0.89 0.89 239 | # malloced_roaring_benchmarks -r 240 | 5.42 0.50 0.73 0.17 1.30 78.05 0.58 0.91 6.47 0.26 0.27 0.26 0.26 241 | # roaring_benchmarks -r 242 | 5.38 0.38 0.57 0.16 1.08 80.31 0.44 0.76 6.57 0.26 0.27 0.27 0.27 243 | # roaring_benchmarks -c -r 244 | 5.38 0.38 0.57 0.16 1.09 92.10 0.44 0.75 6.57 0.27 0.27 0.27 0.27 245 | # roaring_benchmarks 246 | 5.44 0.36 0.55 0.16 1.05 80.15 0.43 0.75 6.56 0.25 0.25 0.25 0.25 247 | # roaring_benchmarks -c 248 | 5.44 0.36 0.56 0.16 1.06 95.01 0.42 0.73 6.57 0.25 0.25 0.25 0.25 249 | # ewah32_benchmarks 250 | 6.67 2.64 4.46 2.35 4.64 13360.00 3.60 4.37 15.85 1.64 2.50 2.05 2.52 251 | # ewah64_benchmarks 252 | 7.87 1.56 2.26 1.29 2.32 7380.11 1.90 2.22 11.43 0.84 1.27 1.03 1.29 253 | # wah32_benchmarks 254 | 6.82 3.58 4.59 4.92 6.89 65157.34 4.28 4.50 12.41 2.09 2.10 2.14 2.09 255 | # concise_benchmarks 256 | 5.88 4.53 5.41 6.02 8.09 69144.02 5.10 5.41 12.77 2.75 2.92 2.77 2.90 257 | 258 | 259 | # processing file weather_sept_85_srt 260 | # bitset_benchmarks 261 | 11.39 0.33 0.31 0.25 0.27 3.57 0.29 0.31 6.75 0.19 0.18 0.21 0.22 262 | # stl_vector_benchmarks 263 | 0.00 4.36 6.13 69.07 14.09 679.61 5.78 6.23 0.00 1.97 1.75 2.19 1.97 264 | # stl_vector_benchmarks_memtracked 265 | 32.00 4.41 7.88 83.10 17.40 673.08 6.78 8.39 0.00 1.97 1.75 2.18 1.98 266 | # stl_hashset_benchmarks_memtracked 267 | 195.07 15.08 178.96 81.69 111.06 60.38 148.60 236.72 12.94 6.99 7.01 32.08 7.00 268 | # stl_hashset_benchmarks 269 | 0.00 15.11 180.86 82.29 111.54 61.27 149.38 237.04 12.94 7.19 7.20 32.34 7.19 270 | # bitmagic_benchmarks 271 | 9.03 0.32 0.46 0.23 0.27 14.96 0.37 0.46 10.15 0.12 0.18 0.16 0.18 272 | # bitmagic_benchmarks -r 273 | 0.96 0.16 0.20 0.21 0.33 53.37 0.21 0.22 6.84 0.09 0.14 0.17 0.14 274 | # slow_roaring_benchmarks -r 275 | 0.34 0.09 0.17 0.04 0.05 57.41 0.15 0.22 5.26 0.06 0.07 0.06 0.07 276 | # malloced_roaring_benchmarks -r 277 | 0.36 0.15 0.28 0.04 0.10 58.51 0.23 0.33 6.25 0.06 0.06 0.06 0.06 278 | # roaring_benchmarks -r 279 | 0.34 0.08 0.17 0.04 0.05 57.19 0.15 0.22 5.26 0.06 0.06 0.06 0.06 280 | # roaring_benchmarks -c -r 281 | 0.34 0.08 0.16 0.04 0.06 74.79 0.13 0.20 5.26 0.06 0.07 0.06 0.07 282 | # roaring_benchmarks 283 | 3.24 0.12 0.24 0.07 0.50 60.45 0.17 0.28 5.71 0.07 0.07 0.06 0.06 284 | # roaring_benchmarks -c 285 | 3.24 0.12 0.23 0.07 0.50 80.61 0.16 0.25 5.71 0.07 0.07 0.07 0.07 286 | # ewah32_benchmarks 287 | 0.54 0.19 0.48 0.32 0.11 2178.33 0.34 0.51 5.94 0.15 0.23 0.19 0.24 288 | # ewah64_benchmarks 289 | 0.86 0.16 0.41 0.25 0.09 1819.95 0.28 0.42 5.81 0.12 0.20 0.16 0.21 290 | # wah32_benchmarks 291 | 0.54 0.34 0.56 0.58 0.37 7482.52 0.45 0.56 7.46 0.22 0.24 0.23 0.24 292 | # concise_benchmarks 293 | 0.43 0.41 0.62 0.69 0.45 7339.01 0.51 0.63 7.68 0.26 0.30 0.27 0.30 294 | 295 | 296 | # processing file wikileaks-noquotes 297 | # bitset_benchmarks 298 | 795.50 26.43 20.61 15.39 15.86 3.80 18.36 20.01 46.69 11.69 11.77 13.53 14.85 299 | # stl_vector_benchmarks 300 | 0.00 4.15 6.95 515.14 51.62 184.54 6.05 7.41 0.00 2.12 2.37 2.53 2.23 301 | # stl_vector_benchmarks_memtracked 302 | 32.00 4.15 9.24 695.15 62.37 164.10 7.22 9.85 0.00 2.19 2.17 2.60 2.23 303 | # stl_hashset_benchmarks_memtracked 304 | 194.68 14.86 229.51 422.94 520.29 52.17 212.66 300.99 24.26 12.68 10.65 40.67 11.23 305 | # stl_hashset_benchmarks 306 | 0.00 14.86 228.66 412.53 521.99 51.60 214.63 301.39 24.56 12.76 10.61 40.31 11.26 307 | # bitmagic_benchmarks 308 | 462.53 13.84 25.68 12.14 15.11 12.77 18.10 25.28 59.19 4.57 9.50 7.59 9.46 309 | # bitmagic_benchmarks -r 310 | 29.84 3.06 5.74 11.89 16.62 21.65 6.00 5.60 12.59 1.57 2.22 5.27 2.21 311 | # slow_roaring_benchmarks -r 312 | 5.89 1.41 3.86 3.05 24.56 27.46 2.64 3.77 9.53 0.86 1.28 1.06 1.23 313 | # malloced_roaring_benchmarks -r 314 | 7.04 3.70 10.00 3.46 30.52 29.22 6.89 10.08 10.55 0.86 1.31 1.10 1.29 315 | # roaring_benchmarks -r 316 | 5.89 1.43 3.87 3.01 24.53 28.09 2.65 3.92 9.62 0.87 1.33 1.09 1.29 317 | # roaring_benchmarks -c -r 318 | 5.89 1.42 3.42 3.03 24.79 34.65 2.31 3.20 9.74 0.90 1.38 1.16 1.38 319 | # roaring_benchmarks 320 | 16.49 1.36 4.44 4.49 21.10 37.19 2.36 4.70 6.34 0.76 0.77 0.74 0.75 321 | # roaring_benchmarks -c 322 | 16.49 1.41 3.96 4.59 21.27 49.41 1.89 3.78 6.41 0.82 0.84 0.82 0.82 323 | # ewah32_benchmarks 324 | 10.83 2.86 9.23 328.40 48.87 994.66 5.98 8.96 18.36 2.46 4.08 3.28 4.09 325 | # ewah64_benchmarks 326 | 19.41 2.76 8.79 203.66 41.23 989.90 5.60 8.50 15.85 2.32 3.97 2.95 3.96 327 | # wah32_benchmarks 328 | 10.89 6.10 10.88 414.71 46.48 2402.42 8.25 10.70 16.09 3.85 4.46 4.15 4.47 329 | # concise_benchmarks 330 | 10.25 6.38 11.30 450.18 52.31 2347.83 8.44 11.36 16.66 4.11 4.89 4.29 4.93 331 | 332 | 333 | # processing file wikileaks-noquotes_srt 334 | # bitset_benchmarks 335 | 647.53 19.58 18.38 14.51 15.11 2.74 15.25 17.66 33.55 8.71 10.46 11.02 12.82 336 | # stl_vector_benchmarks 337 | 0.00 4.15 7.15 496.33 42.65 144.72 5.74 6.67 0.00 1.69 1.57 1.88 1.70 338 | # stl_vector_benchmarks_memtracked 339 | 32.00 4.28 9.51 676.85 53.54 144.41 7.57 8.99 0.00 1.70 1.54 1.88 1.71 340 | # stl_hashset_benchmarks_memtracked 341 | 195.18 26.12 179.90 273.28 359.64 49.94 159.41 229.05 14.21 10.34 9.19 25.42 8.54 342 | # stl_hashset_benchmarks 343 | 0.00 26.54 181.91 274.20 360.88 50.08 160.82 227.51 13.17 10.44 9.40 25.77 8.77 344 | # bitmagic_benchmarks 345 | 370.07 10.15 21.28 9.64 11.90 11.93 14.25 20.98 38.64 3.03 7.69 5.75 7.72 346 | # bitmagic_benchmarks -r 347 | 23.99 1.92 4.20 8.92 7.03 15.30 4.43 4.24 7.85 0.51 1.16 3.64 1.14 348 | # slow_roaring_benchmarks -r 349 | 1.63 0.59 2.13 2.06 7.67 19.26 1.27 1.87 5.96 0.25 0.45 0.35 0.44 350 | # malloced_roaring_benchmarks -r 351 | 2.58 2.15 7.35 2.31 12.66 22.79 4.64 7.15 6.86 0.26 0.50 0.37 0.47 352 | # roaring_benchmarks -r 353 | 1.63 0.58 2.16 1.94 7.60 20.64 1.27 1.95 5.89 0.26 0.49 0.37 0.47 354 | # roaring_benchmarks -c -r 355 | 1.63 0.57 1.91 1.98 7.77 23.61 1.03 1.47 6.04 0.28 0.55 0.41 0.53 356 | # roaring_benchmarks 357 | 10.67 0.70 2.96 3.15 13.64 27.80 1.51 3.08 6.06 0.35 0.36 0.33 0.34 358 | # roaring_benchmarks -c 359 | 10.67 0.69 2.30 3.20 13.79 31.16 1.04 2.15 6.11 0.37 0.40 0.37 0.38 360 | # ewah32_benchmarks 361 | 2.63 0.73 2.37 116.17 12.65 239.54 1.59 2.33 8.08 0.54 1.00 0.79 0.99 362 | # ewah64_benchmarks 363 | 4.66 0.73 2.21 92.82 11.45 233.80 1.49 2.16 7.61 0.50 0.92 0.69 0.92 364 | # wah32_benchmarks 365 | 2.67 1.28 2.57 132.60 12.38 528.32 1.93 2.51 8.91 0.76 1.02 0.95 1.02 366 | # concise_benchmarks 367 | 2.23 1.43 2.63 149.06 13.71 497.06 1.99 2.66 9.23 0.83 1.16 1.03 1.16 368 | ``` 369 | 370 | 371 | ```bash 372 | $ ./scripts/big.sh 373 | 374 | # For each data set, we print data size (in bits per value), successive intersections, successive unions and total unions [we compute the total union first na 375 | ively and then (if supported) using a heap-based approach], followed by quartile point queries (in cycles per input value), successive differences, successive 376 | symmetric differences, iterations through all values, then we have pairwise count aggregates for successive intersections, successive unions, successive differ 377 | ences, successive symmetric differences 378 | # bitset_benchmarks 379 | 100.00 25.35 25.37 5.01 5.01 4.92 25.32 380 | 25.37 28.90 2.27 2.14 2.28 2.27 381 | # stl_vector_benchmarks 382 | 0.00 11.20 14.46 457.04 134.96 1911.23 14.44 383 | 16.63 0.00 8.23 8.92 9.27 8.29 384 | # stl_vector_benchmarks_memtracked 385 | 32.00 11.67 15.67 481.40 108.32 2041.98 15.08 386 | 17.56 0.00 8.38 8.21 9.10 8.24 387 | # stl_hashset_benchmarks_memtracked 388 | # stl_hashset_benchmarks 389 | 100.05 8.97 9.21 4.80 5.37 15.33 9.17 390 | 9.20 78.84 2.61 2.95 2.92 2.95 391 | # bitmagic_benchmarks -r 392 | 38.52 9.11 9.77 3.92 5.29 82.06 8.05 9.91 46.28 5.16 8.89 7.24 9.17 393 | # slow_roaring_benchmarks -r 394 | 13.51 8.11 9.34 4.79 11.74 284.75 8.33 395 | 9.19 7.10 6.82 6.81 6.82 6.81 396 | # malloced_roaring_benchmarks -r 397 | 13.74 3.99 6.17 3.90 9.71 278.79 4.67 398 | 6.49 7.05 1.41 1.39 1.41 1.39 399 | # roaring_benchmarks -r 400 | 13.51 2.58 4.97 3.87 8.04 381.29 2.48 401 | 4.09 6.87 1.37 1.35 1.37 1.35 402 | # roaring_benchmarks -c -r 403 | 13.51 2.58 4.32 3.86 8.04 383.19 2.47 404 | 4.08 6.87 1.37 1.35 1.38 1.35 405 | # roaring_benchmarks 406 | 13.51 2.58 4.98 3.87 8.02 387.15 2.48 407 | 4.09 6.87 1.36 1.35 1.37 1.35 408 | # roaring_benchmarks -c 409 | 13.51 2.58 4.44 3.87 8.03 379.28 2.47 410 | 4.08 6.87 1.37 1.35 1.37 1.35 411 | # ewah32_benchmarks 412 | 33.42 18.04 33.54 53.03 72.10 19929633.82 26.18 32.55 25.49 13.95 17.66 16.50 17.81 413 | # ewah64_benchmarks 414 | 50.33 14.70 24.31 33.17 42.03 15554502.25 20.09 415 | 24.35 23.42 11.11 14.29 12.30 14.25 416 | # wah32_benchmarks 417 | 33.79 24.52 34.85 68.66 65.75 58645138.60 31.40 418 | 34.78 27.31 16.24 16.50 16.76 16.55 419 | # concise_benchmarks 420 | 22.56 27.79 37.54 69.49 77.28 48014150.67 32.98 421 | 37.83 27.60 18.91 19.30 18.90 19.19 422 | ``` 423 | 424 | 425 | ## Usage 426 | 427 | This project uses submodules as dependencies. 428 | 429 | To pull all the submodules: 430 | ```bash 431 | git pull && git submodule init && git submodule update && git submodule status 432 | ``` 433 | This needs to be done once at the beginning, and whenever the project has updated a submodule. 434 | 435 | ```bash 436 | make 437 | make test 438 | ``` 439 | 440 | There are additional tests. To benchmark with a large synthetic dataset, do : 441 | ```bash 442 | make bigtest 443 | ``` 444 | 445 | To find "hot" functions in roaring, do: 446 | 447 | ```bash 448 | make hottest 449 | ``` 450 | 451 | 452 | 453 | Be aware that if you call ``git pull`` while in a submodule, you are *updating* the project for everyone. Try to stick as much as possible with the ``git pull && git submodule init && git submodule update && git submodule status`` as it is safest (it only impacts your local copy). 454 | 455 | ## Usage for developers of the project 456 | 457 | If one of the submodules changes and you need to update it and update the project (for everyone, not just yourself): 458 | ```bash 459 | git submodule foreach git pull origin master 460 | make clean 461 | make 462 | make test 463 | git commit -a 464 | git push 465 | ``` 466 | 467 | Warning: issuing the command ``git submodule foreach git pull origin master`` while in the main directory or simply the command ``git pull origin master`` while in a submodule will change the repository because submodules are fixed to a given version, and you are updating the version to the latest available. 468 | 469 | 470 | ## Using perf for analysis 471 | 472 | Example: 473 | 474 | ```bash 475 | $ perf record --call-graph dwarf ./hot_roaring_benchmarks -r -m wideunion CRoaring/benchmarks/realdata/census1881_srt 476 | $ perf report --call-graph 477 | ``` 478 | 479 | 480 | ## Use Dmalloc 481 | 482 | Some users would prefer to rely on [Dmalloc](http://dmalloc.com) to monitor memory usage. You can get the desired result by first building Dmalloc: 483 | 484 | ``` 485 | wget http://dmalloc.com/releases/dmalloc-5.5.2.tgz 486 | tar xzf dmalloc-5.5.2.tgz 487 | cd ./dmalloc-5.5.2 488 | ./configure 489 | make 490 | cd .. 491 | ``` 492 | 493 | (On some older systems you might need the ``-m64`` flag. If that is needed, you can type ``export CFLAGS=-m64; export LDFLAGS=-m64;``.) 494 | 495 | Then you can copy the current ``Makefile`` to a file named ``makefile`` (it will have priority over ``Makefile``) and replace the ``malloced_roaring_benchmarks`` target 496 | by the following: 497 | 498 | ``` 499 | malloced_roaring_benchmarks : src/roaring.c src/roaring_benchmarks.c 500 | $(CC) $(CFLAGS) -I./dmalloc-5.5.2/ -o malloced_roaring_benchmarks src/roaring_benchmarks.c -DDMALLOC -DDMALLOC_FUNC_CHECK -L./dmalloc-5.5.2/ -ldmalloc ; 501 | @echo '' 502 | @echo ' Note malloced_roaring_benchmarks is linked with ./dmalloc-5.5.2/libdmalloc.a' ; 503 | @echo ' to generate a logfile when running malloced_roaring_benchmarks quixk example:' ; 504 | @echo ' setenv DMALLOC_OPTIONS log-stats,log=logfile,check-heap,log-non-free,check-fence' ; 505 | @echo ' Then inspect the new log called 'logfile' after the run (ref: http://dmalloc.com/' ; 506 | @echo '' 507 | ``` 508 | 509 | Credit: Jon Strabala 510 | -------------------------------------------------------------------------------- /scripts/all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################### 3 | # To add a technique, simply append the file name of your executable to the commands array below 4 | ####################### 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | declare -a commands=('bitset_benchmarks' 'stl_vector_benchmarks' 'stl_vector_benchmarks_memtracked' 'stl_hashset_benchmarks_memtracked' 'stl_hashset_benchmarks' 'bitmagic_benchmarks' 'bitmagic_benchmarks -r' 'slow_roaring_benchmarks -r' 'malloced_roaring_benchmarks -r' 'roaring_benchmarks -r' 'roaring_benchmarks -c -r' 'roaring_benchmarks' 'roaring_benchmarks -c' 'ewah32_benchmarks' 'ewah64_benchmarks' 'wah32_benchmarks' 'concise_benchmarks' ); 7 | echo "# For each data set, we print data size (in bits per value), successive intersections, successive unions and total unions [we compute the total union first naively and then (if supported) using a heap-based approach], followed by quartile point queries (in cycles per input value), successive differences, successive symmetric differences, iterations through all values, then we have pairwise count aggregates for successive intersections, successive unions, successive differences, successive symmetric differences " 8 | for f in census-income census-income_srt census1881 census1881_srt weather_sept_85 weather_sept_85_srt wikileaks-noquotes wikileaks-noquotes_srt ; do 9 | echo "# processing file " $f 10 | for t in "${commands[@]}"; do 11 | echo "#" $t 12 | ./$t CRoaring/benchmarks/realdata/$f; 13 | done 14 | echo 15 | echo 16 | done 17 | -------------------------------------------------------------------------------- /scripts/big.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################### 3 | # To add a technique, simply append the file name of your executable to the commands array below 4 | ####################### 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | ${DIR}/generatebig.sh 7 | declare -a commands=('bitset_benchmarks' 'stl_vector_benchmarks' 'stl_vector_benchmarks_memtracked' 'stl_hashset_benchmarks_memtracked' 'stl_hashset_benchmarks' 'bitmagic_benchmarks' 'bitmagic_benchmarks -r' 'slow_roaring_benchmarks -r' 'malloced_roaring_benchmarks -r' 'roaring_benchmarks -r' 'roaring_benchmarks -c -r' 'roaring_benchmarks' 'roaring_benchmarks -c' 'ewah32_benchmarks' 'ewah64_benchmarks' 'wah32_benchmarks' 'concise_benchmarks' ); 8 | echo "# For each data set, we print data size (in bits per value), successive intersections, successive unions and total unions [we compute the total union first naively and then (if supported) using a heap-based approach], followed by quartile point queries (in cycles per input value), successive differences, successive symmetric differences, iterations through all values, then we have pairwise count aggregates for successive intersections, successive unions, successive differences, successive symmetric differences " 9 | for t in "${commands[@]}"; do 10 | echo "#" $t 11 | ./$t bigtmp; 12 | done 13 | -------------------------------------------------------------------------------- /scripts/generatebig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################### 3 | # This will create lots of big files 4 | ####################### 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | mkdir -p ${DIR}/../bigtmp 7 | find ${DIR}/../bigtmp -size 0c -delete 8 | for (( index = 0; index < 100; index++ )) 9 | do 10 | filename=${DIR}/../bigtmp/bigtmp${index}.txt 11 | if [ ! -f ${filename} ] 12 | then 13 | echo "creating file ${index}" 14 | ${DIR}/../gen -N 10000000 -M 1000000000 > ${filename} 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /scripts/hot_roaring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################### 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | declare -a commands=( 'hot_slow_roaring_benchmarks -r' 'hot_roaring_benchmarks -r' ); 5 | for f in census-income census-income_srt census1881 census1881_srt weather_sept_85 weather_sept_85_srt wikileaks-noquotes wikileaks-noquotes_srt ; do 6 | echo "# processing file " $f 7 | for mode in intersection union difference symdifference wideunion wideunionheap intersectioncount unioncount differencecount symdifferencecount ; do 8 | for t in "${commands[@]}"; do 9 | echo "#" $t " -m " $mode "CRoaring/benchmarks/realdata/"$f 10 | perf record -q ./$t -m $mode CRoaring/benchmarks/realdata/$f; 11 | perf report -F Overhead,Symbol |cat |grep -v '^#'|head -3 12 | done 13 | done 14 | echo 15 | echo 16 | done 17 | -------------------------------------------------------------------------------- /scripts/roaring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ###################### 3 | # To add a technique, simply append the file name of your executable to the commands array below 4 | ####################### 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | declare -a commands=( 'slow_roaring_benchmarks -r' 'roaring_benchmarks -r' ); 7 | echo "# For each data set, we print data size (in bits per value), successive intersections, successive unions and total unions [we compute the total union first naively and then (if supported) using a heap-based approach], followed by quartile point queries (in cycles per input value), successive differences, successive symmetric differences, iterations through all values, then we have pairwise count aggregates for successive intersections, successive unions, successive differences, successive symmetric differences " 8 | for f in census-income census-income_srt census1881 census1881_srt weather_sept_85 weather_sept_85_srt wikileaks-noquotes wikileaks-noquotes_srt ; do 9 | echo "# processing file " $f 10 | for t in "${commands[@]}"; do 11 | echo "#" $t 12 | ./$t CRoaring/benchmarks/realdata/$f; 13 | done 14 | echo 15 | echo 16 | done 17 | -------------------------------------------------------------------------------- /src/benchmark.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BENCHMARKS_INCLUDE_BENCHMARK_H_ 3 | #define BENCHMARKS_INCLUDE_BENCHMARK_H_ 4 | 5 | const int quartile_test_repetitions = 10; 6 | 7 | 8 | #define RDTSC_START(cycles) \ 9 | do { \ 10 | register unsigned cyc_high, cyc_low; \ 11 | __asm volatile( \ 12 | "cpuid\n\t" \ 13 | "rdtsc\n\t" \ 14 | "mov %%edx, %0\n\t" \ 15 | "mov %%eax, %1\n\t" \ 16 | : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx", "%rdx"); \ 17 | (cycles) = ((uint64_t)cyc_high << 32) | cyc_low; \ 18 | } while (0) 19 | 20 | #define RDTSC_FINAL(cycles) \ 21 | do { \ 22 | register unsigned cyc_high, cyc_low; \ 23 | __asm volatile( \ 24 | "rdtscp\n\t" \ 25 | "mov %%edx, %0\n\t" \ 26 | "mov %%eax, %1\n\t" \ 27 | "cpuid\n\t" \ 28 | : "=r"(cyc_high), "=r"(cyc_low)::"%rax", "%rbx", "%rcx", "%rdx"); \ 29 | (cycles) = ((uint64_t)cyc_high << 32) | cyc_low; \ 30 | } while (0) 31 | 32 | #define STARTBEST(numberoftests) \ 33 | { \ 34 | uint64_t min_diff = -1 ; \ 35 | uint64_t boguscyclesstart = 0; \ 36 | uint64_t boguscyclesend = 0; \ 37 | for(int bogustest = 0; bogustest < numberoftests; bogustest++ ) { \ 38 | uint64_t cycles_diff = 0;\ 39 | RDTSC_START(boguscyclesstart); 40 | 41 | #define ENDBEST(outputvar) \ 42 | RDTSC_FINAL(boguscyclesend); \ 43 | cycles_diff = (boguscyclesend - boguscyclesstart); \ 44 | if (cycles_diff < min_diff) min_diff = cycles_diff; \ 45 | } \ 46 | outputvar = min_diff;\ 47 | } 48 | 49 | /* 50 | * Prints the best number of operations per cycle where 51 | * test is the function call, answer is the expected answer generated by 52 | * test, repeat is the number of times we should repeat and size is the 53 | * number of operations represented by test. 54 | */ 55 | #define BEST_TIME(test, answer, repeat, size) \ 56 | do { \ 57 | printf("%s: ", #test); \ 58 | fflush(NULL); \ 59 | uint64_t cycles_start, cycles_final, cycles_diff; \ 60 | uint64_t min_diff = (uint64_t)-1; \ 61 | int wrong_answer = 0; \ 62 | for (int i = 0; i < repeat; i++) { \ 63 | __asm volatile("" ::: /* pretend to clobber */ "memory"); \ 64 | RDTSC_START(cycles_start); \ 65 | if (test != answer) wrong_answer = 1; \ 66 | RDTSC_FINAL(cycles_final); \ 67 | cycles_diff = (cycles_final - cycles_start); \ 68 | if (cycles_diff < min_diff) min_diff = cycles_diff; \ 69 | } \ 70 | uint64_t S = (uint64_t)size; \ 71 | float cycle_per_op = (min_diff) / (float)S; \ 72 | printf(" %.2f cycles per operation", cycle_per_op); \ 73 | if (wrong_answer) printf(" [ERROR]"); \ 74 | printf("\n"); \ 75 | fflush(NULL); \ 76 | } while (0) 77 | #endif /// BENCHMARKS_INCLUDE_BENCHMARK_H_ 78 | -------------------------------------------------------------------------------- /src/bitmagic_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | #include "benchmark.h" 17 | #include "numbersfromtextfiles.h" 18 | #ifdef __cplusplus 19 | } 20 | #endif 21 | 22 | // flags recommended by BitMagic author. 23 | #define BM64OPT 24 | #define BMSSE42OPT 25 | 26 | #include "bm.h" /* bit magic */ 27 | #include "bmserial.h" /* bit magic, serialization routines */ 28 | 29 | typedef bm::bvector<> bvect; 30 | 31 | /** 32 | * Once you have collected all the integers, build the bitmaps. 33 | */ 34 | static std::vector create_all_bitmaps(size_t *howmany, 35 | uint32_t **numbers, size_t count, bool memorysavingmode) { 36 | if (numbers == NULL) return std::vector(); 37 | // we work hard to create the bitsets in memory-saving mode 38 | std::vector answer; 39 | for (size_t i = 0; i < count; i++) { 40 | bvect bm(0); 41 | if(memorysavingmode) { 42 | bm.set_new_blocks_strat(bm::BM_GAP); 43 | } 44 | uint32_t * mynumbers = numbers[i]; 45 | for(size_t j = 0; j < howmany[i] ; ++j) { 46 | bm.set(mynumbers[j]); 47 | } 48 | if(memorysavingmode) { 49 | bm.optimize();// this might be useless, redundant... 50 | } 51 | answer.push_back(bm); 52 | } 53 | return answer; 54 | } 55 | 56 | // This function has unresolved memory leaks. We don't care since we focus on performance. 57 | static bvect fast_logicalor(size_t n, bvect **inputs) { 58 | class BMVectorWrapper { 59 | public: 60 | BMVectorWrapper(bvect * p, bool o) : ptr(p), own(o) {} 61 | bvect * ptr; 62 | bool own; 63 | 64 | bool operator<(const BMVectorWrapper & o) const { 65 | return o.ptr->size() < ptr->size(); // backward on purpose 66 | } 67 | }; 68 | 69 | if (n == 0) { 70 | return bvect(); 71 | } 72 | if (n == 1) { 73 | return bvect(*inputs[0]); 74 | } 75 | std::priority_queue pq; 76 | for (size_t i = 0; i < n; i++) { 77 | // could use emplace 78 | pq.push(BMVectorWrapper(inputs[i], false)); 79 | } 80 | while (pq.size() > 1) { 81 | 82 | BMVectorWrapper x1 = pq.top(); 83 | pq.pop(); 84 | 85 | BMVectorWrapper x2 = pq.top(); 86 | pq.pop(); 87 | if(x1.own) { 88 | x1.ptr->bit_or(*x2.ptr); 89 | if(x2.own) delete x2.ptr; 90 | pq.push(x1); 91 | } else if (x2.own) { 92 | x2.ptr->bit_or(*x1.ptr); 93 | pq.push(x2); 94 | } else { 95 | bvect ans = *x1.ptr | *x2.ptr; 96 | bvect * buffer = new bvect(); 97 | buffer->swap(ans); 98 | pq.push(BMVectorWrapper(buffer, true)); 99 | } 100 | } 101 | BMVectorWrapper x1 = pq.top(); 102 | pq.pop(); 103 | 104 | return *x1.ptr; 105 | } 106 | 107 | 108 | 109 | static void printusage(char *command) { 110 | printf( 111 | " Try %s directory \n where directory could be " 112 | "benchmarks/realdata/census1881\n", 113 | command); 114 | ; 115 | printf("the -v flag turns on verbose mode"); 116 | printf("the -r flag turns on memory-saving mode"); 117 | 118 | 119 | } 120 | 121 | int main(int argc, char **argv) { 122 | int c; 123 | const char *extension = ".txt"; 124 | bool verbose = false; 125 | bool memorysavingmode = false; 126 | uint64_t data[13]; 127 | while ((c = getopt(argc, argv, "rve:h")) != -1) switch (c) { 128 | case 'e': 129 | extension = optarg; 130 | break; 131 | case 'v': 132 | verbose = true; 133 | break; 134 | case 'r': 135 | memorysavingmode = true; 136 | break; 137 | case 'h': 138 | printusage(argv[0]); 139 | return 0; 140 | default: 141 | abort(); 142 | } 143 | if (optind >= argc) { 144 | printusage(argv[0]); 145 | return -1; 146 | } 147 | if(verbose) printf("memorysavingmode=%d\n",memorysavingmode); 148 | char *dirname = argv[optind]; 149 | size_t count; 150 | 151 | size_t *howmany = NULL; 152 | uint32_t **numbers = 153 | read_all_integer_files(dirname, extension, &howmany, &count); 154 | if (numbers == NULL) { 155 | printf( 156 | "I could not find or load any data file with extension %s in " 157 | "directory %s.\n", 158 | extension, dirname); 159 | return -1; 160 | } 161 | uint32_t maxvalue = 0; 162 | for (size_t i = 0; i < count; i++) { 163 | if( howmany[i] > 0 ) { 164 | if(maxvalue < numbers[i][howmany[i]-1]) { 165 | maxvalue = numbers[i][howmany[i]-1]; 166 | } 167 | } 168 | } 169 | uint64_t totalcard = 0; 170 | for (size_t i = 0; i < count; i++) { 171 | totalcard += howmany[i]; 172 | } 173 | uint64_t successivecard = 0; 174 | for (size_t i = 1; i < count; i++) { 175 | successivecard += howmany[i-1] + howmany[i]; 176 | } 177 | uint64_t cycles_start = 0, cycles_final = 0; 178 | 179 | RDTSC_START(cycles_start); 180 | std::vector bitmaps = create_all_bitmaps(howmany, numbers, count, memorysavingmode); 181 | RDTSC_FINAL(cycles_final); 182 | if (bitmaps.empty()) return -1; 183 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 184 | uint64_t totalsize = 0; 185 | 186 | for (int i = 0; i < (int) count; ++i) { 187 | bvect & bv = bitmaps[i]; 188 | bvect::statistics st; 189 | bv.calc_stat(&st); 190 | totalsize += st.memory_used; 191 | } 192 | data[0] = totalsize; 193 | 194 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 195 | 196 | uint64_t successive_and = 0; 197 | uint64_t successive_or = 0; 198 | uint64_t total_or = 0; 199 | uint64_t total_count = 0; 200 | 201 | RDTSC_START(cycles_start); 202 | for (int i = 0; i < (int)count - 1; ++i) { 203 | bvect tempand = bitmaps[i] & bitmaps[i + 1]; 204 | successive_and += tempand.count(); 205 | } 206 | RDTSC_FINAL(cycles_final); 207 | data[1] = cycles_final - cycles_start; 208 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 209 | cycles_final - cycles_start); 210 | 211 | RDTSC_START(cycles_start); 212 | for (int i = 0; i < (int)count - 1; ++i) { 213 | bvect tempor = bitmaps[i] | bitmaps[i + 1]; 214 | successive_or += tempor.count(); 215 | } 216 | RDTSC_FINAL(cycles_final); 217 | data[2] = cycles_final - cycles_start; 218 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 219 | cycles_final - cycles_start); 220 | 221 | RDTSC_START(cycles_start); 222 | if(count>1) { 223 | bvect totalorbitmap = bitmaps[0] | bitmaps[1]; 224 | for (int i = 2; i < (int)count ; ++i) { 225 | totalorbitmap |= bitmaps[i]; 226 | } 227 | total_or = totalorbitmap.count(); 228 | } 229 | RDTSC_FINAL(cycles_final); 230 | data[3] = cycles_final - cycles_start; 231 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 232 | cycles_final - cycles_start); 233 | RDTSC_START(cycles_start); 234 | if(count>1) { 235 | bvect ** allofthem = new bvect* [count]; 236 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 237 | bvect totalorbitmap = fast_logicalor(count, allofthem); 238 | total_or = totalorbitmap.count(); 239 | delete[] allofthem; 240 | } 241 | RDTSC_FINAL(cycles_final); 242 | data[4] = cycles_final - cycles_start; 243 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 244 | cycles_final - cycles_start); 245 | 246 | uint64_t quartcount; 247 | STARTBEST(quartile_test_repetitions) 248 | quartcount = 0; 249 | for (size_t i = 0; i < count ; ++i) { 250 | quartcount += bitmaps[i].get_bit(maxvalue/4); 251 | quartcount += bitmaps[i].get_bit(maxvalue/2); 252 | quartcount += bitmaps[i].get_bit(3*maxvalue/4); 253 | } 254 | ENDBEST(data[5]) 255 | 256 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 257 | data[5]); 258 | 259 | /** 260 | * andnot and xor 261 | */ 262 | 263 | uint64_t successive_andnot = 0; 264 | uint64_t successive_xor = 0; 265 | 266 | RDTSC_START(cycles_start); 267 | for (int i = 0; i < (int)count - 1; ++i) { 268 | bvect tempandnot = bitmaps[i] - bitmaps[i + 1]; 269 | successive_andnot += tempandnot.count(); 270 | } 271 | RDTSC_FINAL(cycles_final); 272 | data[6] = cycles_final - cycles_start; 273 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 274 | cycles_final - cycles_start); 275 | 276 | RDTSC_START(cycles_start); 277 | for (int i = 0; i < (int)count - 1; ++i) { 278 | bvect tempxor = bitmaps[i] ^ bitmaps[i + 1]; 279 | successive_xor += tempxor.count(); 280 | } 281 | RDTSC_FINAL(cycles_final); 282 | data[7] = cycles_final - cycles_start; 283 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 284 | cycles_final - cycles_start); 285 | 286 | 287 | /** 288 | * end of andnot and xor 289 | */ 290 | RDTSC_START(cycles_start); 291 | for (size_t i = 0; i < count; ++i) { 292 | const bvect & b = bitmaps[i]; 293 | for(auto j = b.first(); j != b.end(); ++j) 294 | total_count ++; 295 | } 296 | RDTSC_FINAL(cycles_final); 297 | data[8] = cycles_final - cycles_start; 298 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 299 | cycles_final - cycles_start); 300 | 301 | assert(totalcard == total_count); 302 | 303 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 304 | 305 | assert(successive_xor + successive_and == successive_or); 306 | 307 | 308 | /** 309 | * and, or, andnot and xor cardinality 310 | */ 311 | uint64_t successive_andcard = 0; 312 | uint64_t successive_orcard = 0; 313 | uint64_t successive_andnotcard = 0; 314 | uint64_t successive_xorcard = 0; 315 | 316 | RDTSC_START(cycles_start); 317 | for (int i = 0; i < (int)count - 1; ++i) { 318 | successive_andcard += count_and(bitmaps[i], bitmaps[i + 1]); 319 | } 320 | RDTSC_FINAL(cycles_final); 321 | data[9] = cycles_final - cycles_start; 322 | 323 | RDTSC_START(cycles_start); 324 | for (int i = 0; i < (int)count - 1; ++i) { 325 | successive_orcard += count_or(bitmaps[i], bitmaps[i + 1]); 326 | } 327 | RDTSC_FINAL(cycles_final); 328 | data[10] = cycles_final - cycles_start; 329 | 330 | RDTSC_START(cycles_start); 331 | for (int i = 0; i < (int)count - 1; ++i) { 332 | successive_andnotcard += count_sub(bitmaps[i], bitmaps[i + 1]); 333 | } 334 | RDTSC_FINAL(cycles_final); 335 | data[11] = cycles_final - cycles_start; 336 | 337 | RDTSC_START(cycles_start); 338 | for (int i = 0; i < (int)count - 1; ++i) { 339 | successive_xorcard += count_xor(bitmaps[i], bitmaps[i + 1]); 340 | } 341 | RDTSC_FINAL(cycles_final); 342 | data[12] = cycles_final - cycles_start; 343 | 344 | assert(successive_andcard == successive_and); 345 | assert(successive_orcard == successive_or); 346 | assert(successive_xorcard == successive_xor); 347 | assert(successive_andnotcard == successive_andnot); 348 | 349 | /** 350 | * end and, or, andnot and xor cardinality 351 | */ 352 | 353 | 354 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 355 | data[0]*8.0/totalcard, 356 | data[1]*1.0/successivecard, 357 | data[2]*1.0/successivecard, 358 | data[3]*1.0/totalcard, 359 | data[4]*1.0/totalcard, 360 | data[5]*1.0/(3*count), 361 | data[6]*1.0/successivecard, 362 | data[7]*1.0/successivecard, 363 | data[8]*1.0/totalcard, 364 | data[9]*1.0/successivecard, 365 | data[10]*1.0/successivecard, 366 | data[11]*1.0/successivecard, 367 | data[12]*1.0/successivecard 368 | ); 369 | 370 | 371 | for (int i = 0; i < (int)count; ++i) { 372 | free(numbers[i]); 373 | numbers[i] = NULL; // paranoid 374 | } 375 | free(howmany); 376 | free(numbers); 377 | 378 | return 0; 379 | } 380 | -------------------------------------------------------------------------------- /src/bitset_benchmarks.c: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | 8 | #include "benchmark.h" 9 | #include "numbersfromtextfiles.h" 10 | #include "bitset.h" 11 | /** 12 | * Once you have collected all the integers, build the bitmaps. 13 | */ 14 | static bitset_t **create_all_bitmaps(size_t *howmany, 15 | uint32_t **numbers, size_t count) { 16 | if (numbers == NULL) return NULL; 17 | bitset_t **answer = (bitset_t**) malloc(sizeof(bitset_t *) * count); 18 | for (size_t i = 0; i < count; i++) { 19 | uint32_t biggest = numbers[i][howmany[i]-1]; 20 | answer[i] = bitset_create_with_capacity(biggest+1); 21 | for(size_t j = 0; j < howmany[i] ; ++j) 22 | bitset_set(answer[i],numbers[i][j]); 23 | } 24 | return answer; 25 | } 26 | 27 | static void printusage(char *command) { 28 | printf( 29 | " Try %s directory \n where directory could be " 30 | "benchmarks/realdata/census1881\n", 31 | command); 32 | ; 33 | printf("the -v flag turns on verbose mode"); 34 | 35 | } 36 | 37 | int bitset_size_compare (const void * a, const void * b) { 38 | return ( bitset_size_in_bytes(*(const bitset_t**)a) - bitset_size_in_bytes(*(const bitset_t**)b) ); 39 | } 40 | 41 | bool increment(size_t value, void *param) { 42 | uint64_t k; 43 | memcpy(&k, param, sizeof(uint64_t)); 44 | k += 1; 45 | memcpy(param, &k, sizeof(uint64_t)); 46 | return true; 47 | } 48 | 49 | int main(int argc, char **argv) { 50 | int c; 51 | bool verbose = false; 52 | char *extension = (char *) ".txt"; 53 | uint64_t data[13]; 54 | while ((c = getopt(argc, argv, "vre:h")) != -1) switch (c) { 55 | case 'e': 56 | extension = optarg; 57 | break; 58 | case 'v': 59 | verbose = true; 60 | break; 61 | case 'h': 62 | printusage(argv[0]); 63 | return 0; 64 | default: 65 | abort(); 66 | } 67 | if (optind >= argc) { 68 | printusage(argv[0]); 69 | return -1; 70 | } 71 | char *dirname = argv[optind]; 72 | size_t count; 73 | 74 | size_t *howmany = NULL; 75 | uint32_t **numbers = 76 | read_all_integer_files(dirname, extension, &howmany, &count); 77 | if (numbers == NULL) { 78 | printf( 79 | "I could not find or load any data file with extension %s in " 80 | "directory %s.\n", 81 | extension, dirname); 82 | return -1; 83 | } 84 | uint32_t maxvalue = 0; 85 | for (size_t i = 0; i < count; i++) { 86 | if( howmany[i] > 0 ) { 87 | if(maxvalue < numbers[i][howmany[i]-1]) { 88 | maxvalue = numbers[i][howmany[i]-1]; 89 | } 90 | } 91 | } 92 | uint64_t totalcard = 0; 93 | for (size_t i = 0; i < count; i++) { 94 | totalcard += howmany[i]; 95 | } 96 | uint64_t successivecard = 0; 97 | for (size_t i = 1; i < count; i++) { 98 | successivecard += howmany[i-1] + howmany[i]; 99 | } 100 | uint64_t cycles_start = 0, cycles_final = 0; 101 | 102 | RDTSC_START(cycles_start); 103 | bitset_t **bitmaps = create_all_bitmaps(howmany, numbers, count); 104 | RDTSC_FINAL(cycles_final); 105 | if (bitmaps == NULL) return -1; 106 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 107 | uint64_t totalsize = 0; 108 | for (int i = 0; i < (int) count; ++i) { 109 | totalsize += bitset_size_in_bytes(bitmaps[i]); 110 | } 111 | data[0] = totalsize; 112 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 113 | 114 | uint64_t successive_and = 0; 115 | uint64_t successive_or = 0; 116 | uint64_t successive_andnot = 0; 117 | uint64_t successive_xor = 0; 118 | uint64_t total_or = 0; 119 | uint64_t total_count = 0; 120 | 121 | RDTSC_START(cycles_start); 122 | for (int i = 0; i < (int)count - 1; ++i) { 123 | bitset_t *tempand = bitset_copy(bitmaps[i]); 124 | bitset_inplace_intersection(tempand,bitmaps[i + 1]); 125 | successive_and += bitset_count(tempand); 126 | bitset_free(tempand); 127 | } 128 | RDTSC_FINAL(cycles_final); 129 | data[1] = cycles_final - cycles_start; 130 | 131 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 132 | cycles_final - cycles_start); 133 | 134 | RDTSC_START(cycles_start); 135 | for (int i = 0; i < (int)count - 1; ++i) { 136 | bitset_t *tempor = bitset_copy(bitmaps[i]); 137 | if(!bitset_inplace_union(tempor,bitmaps[i + 1])) printf("failed to compute union"); 138 | successive_or += bitset_count(tempor); 139 | bitset_free(tempor); 140 | } 141 | RDTSC_FINAL(cycles_final); 142 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 143 | cycles_final - cycles_start); 144 | data[2] = cycles_final - cycles_start; 145 | RDTSC_START(cycles_start); 146 | if(count>1){ 147 | bitset_t * totalorbitmap = bitset_copy(bitmaps[0]); 148 | for(size_t i = 1; i < count; ++i) { 149 | if(!bitset_inplace_union(totalorbitmap,bitmaps[i])) printf("failed to compute union"); 150 | } 151 | total_or = bitset_count(totalorbitmap); 152 | bitset_free(totalorbitmap); 153 | } 154 | RDTSC_FINAL(cycles_final); 155 | if(verbose) printf("Total unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 156 | cycles_final - cycles_start); 157 | data[3] = cycles_final - cycles_start; 158 | 159 | RDTSC_START(cycles_start); 160 | if(count>1){ 161 | bitset_t **sortedbitmaps = (bitset_t**) malloc(sizeof(bitset_t *) * count); 162 | memcpy(sortedbitmaps, bitmaps, sizeof(bitset_t *) * count); 163 | qsort (sortedbitmaps, count, sizeof(bitset_t **), bitset_size_compare); 164 | bitset_t * totalorbitmap = bitset_copy(sortedbitmaps[0]); 165 | for(size_t i = 1; i < count; ++i) { 166 | if(!bitset_inplace_union(totalorbitmap,sortedbitmaps[i])) printf("failed to compute union"); 167 | } 168 | total_or = bitset_count(totalorbitmap); 169 | bitset_free(totalorbitmap); 170 | free(sortedbitmaps); 171 | } 172 | RDTSC_FINAL(cycles_final); 173 | if(verbose) printf("Total sorted unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 174 | cycles_final - cycles_start); 175 | data[4] = cycles_final - cycles_start; 176 | 177 | uint64_t quartcount; 178 | STARTBEST(quartile_test_repetitions) 179 | quartcount = 0; 180 | for (size_t i = 0; i < count ; ++i) { 181 | quartcount += bitset_get(bitmaps[i],maxvalue/4); 182 | quartcount += bitset_get(bitmaps[i],maxvalue/2); 183 | quartcount += bitset_get(bitmaps[i],3*maxvalue/4); 184 | } 185 | ENDBEST(data[5]) 186 | 187 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 188 | data[5]); 189 | 190 | RDTSC_START(cycles_start); 191 | for (int i = 0; i < (int)count - 1; ++i) { 192 | bitset_t *tempandnot = bitset_copy(bitmaps[i]); 193 | bitset_inplace_difference(tempandnot,bitmaps[i + 1]); 194 | successive_andnot += bitset_count(tempandnot); 195 | bitset_free(tempandnot); 196 | } 197 | RDTSC_FINAL(cycles_final); 198 | data[6] = cycles_final - cycles_start; 199 | 200 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 201 | cycles_final - cycles_start); 202 | 203 | RDTSC_START(cycles_start); 204 | for (int i = 0; i < (int)count - 1; ++i) { 205 | bitset_t *tempxor = bitset_copy(bitmaps[i]); 206 | bitset_inplace_symmetric_difference(tempxor,bitmaps[i + 1]); 207 | successive_xor += bitset_count(tempxor); 208 | bitset_free(tempxor); 209 | } 210 | RDTSC_FINAL(cycles_final); 211 | data[7] = cycles_final - cycles_start; 212 | 213 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 214 | cycles_final - cycles_start); 215 | 216 | RDTSC_START(cycles_start); 217 | for (size_t i = 0; i < count; ++i) { 218 | bitset_t * b = bitmaps[i]; 219 | bitset_for_each(b,increment,&total_count); 220 | //for(size_t j = 0; nextSetBit(b,&j) ; j++) { 221 | // total_count++; 222 | //} 223 | } 224 | RDTSC_FINAL(cycles_final); 225 | data[8] = cycles_final - cycles_start; 226 | assert(total_count == totalcard); 227 | 228 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 229 | cycles_final - cycles_start); 230 | 231 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,successive_andnot,successive_xor,total_or,quartcount); 232 | 233 | assert(successive_xor + successive_and == successive_or); 234 | 235 | /** 236 | * and, or, andnot and xor cardinality 237 | */ 238 | uint64_t successive_andcard = 0; 239 | uint64_t successive_orcard = 0; 240 | uint64_t successive_andnotcard = 0; 241 | uint64_t successive_xorcard = 0; 242 | 243 | RDTSC_START(cycles_start); 244 | for (int i = 0; i < (int)count - 1; ++i) { 245 | successive_andcard += bitset_intersection_count(bitmaps[i], bitmaps[i + 1]); 246 | } 247 | RDTSC_FINAL(cycles_final); 248 | data[9] = cycles_final - cycles_start; 249 | 250 | RDTSC_START(cycles_start); 251 | for (int i = 0; i < (int)count - 1; ++i) { 252 | successive_orcard += bitset_union_count(bitmaps[i], bitmaps[i + 1]); 253 | } 254 | RDTSC_FINAL(cycles_final); 255 | data[10] = cycles_final - cycles_start; 256 | 257 | RDTSC_START(cycles_start); 258 | for (int i = 0; i < (int)count - 1; ++i) { 259 | successive_andnotcard += bitset_difference_count(bitmaps[i], bitmaps[i + 1]); 260 | } 261 | RDTSC_FINAL(cycles_final); 262 | data[11] = cycles_final - cycles_start; 263 | 264 | RDTSC_START(cycles_start); 265 | for (int i = 0; i < (int)count - 1; ++i) { 266 | successive_xorcard += bitset_symmetric_difference_count(bitmaps[i], bitmaps[i + 1]); 267 | } 268 | RDTSC_FINAL(cycles_final); 269 | data[12] = cycles_final - cycles_start; 270 | 271 | assert(successive_andcard == successive_and); 272 | assert(successive_orcard == successive_or); 273 | assert(successive_xorcard == successive_xor); 274 | assert(successive_andnotcard == successive_andnot); 275 | 276 | /** 277 | * end and, or, andnot and xor cardinality 278 | */ 279 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 280 | data[0]*8.0/totalcard, 281 | data[1]*1.0/successivecard, 282 | data[2]*1.0/successivecard, 283 | data[3]*1.0/totalcard, 284 | data[4]*1.0/totalcard, 285 | data[5]*1.0/(3*count), 286 | data[6]*1.0/successivecard, 287 | data[7]*1.0/successivecard, 288 | data[8]*1.0/totalcard, 289 | data[9]*1.0/successivecard, 290 | data[10]*1.0/successivecard, 291 | data[11]*1.0/successivecard, 292 | data[12]*1.0/successivecard 293 | ); 294 | for (int i = 0; i < (int)count; ++i) { 295 | free(numbers[i]); 296 | numbers[i] = NULL; // paranoid 297 | bitset_free(bitmaps[i]); 298 | bitmaps[i] = NULL; // paranoid 299 | } 300 | free(bitmaps); 301 | free(howmany); 302 | free(numbers); 303 | 304 | return 0; 305 | } 306 | -------------------------------------------------------------------------------- /src/cmemcounter.h: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | size_t malloced_memory_usage = 0; 10 | 11 | size_t myalloc_cookie = 123456; // we can't easily prevent some free calls from coming to us from outside, mark them 12 | 13 | void* malloc(size_t sz) { 14 | void *(*libc_malloc)(size_t) = dlsym(RTLD_NEXT, "malloc"); 15 | void * answerplus = libc_malloc(sz + sizeof(size_t) + sizeof(myalloc_cookie) ); 16 | if(answerplus == NULL) return answerplus;// nothing can be done 17 | malloced_memory_usage += sz; 18 | memcpy(answerplus ,&myalloc_cookie,sizeof(myalloc_cookie)); 19 | memcpy((char *) answerplus + sizeof(myalloc_cookie),&sz,sizeof(sz)); 20 | return ((char *) answerplus) + sizeof(size_t) + sizeof(myalloc_cookie); 21 | } 22 | 23 | 24 | // can fail to produce an aligned result if alignment does not divide 2 * size_t 25 | int posix_memalign(void **memptr, size_t alignment, size_t size) { 26 | int(*libc_posix_memalign)(void **, size_t, size_t) = dlsym(RTLD_NEXT, "posix_memalign"); 27 | size_t offset = (sizeof(size_t) + sizeof(myalloc_cookie) ); 28 | void * answerplus; 29 | int ret = libc_posix_memalign(&answerplus,alignment, size + offset); 30 | if(ret) return ret;// nothing can be done 31 | malloced_memory_usage += size; 32 | memcpy(answerplus ,&myalloc_cookie,sizeof(myalloc_cookie)); 33 | memcpy((char *) answerplus + sizeof(myalloc_cookie),&size,sizeof(size)); 34 | * memptr = (char *) answerplus + offset; 35 | return ret; 36 | } 37 | 38 | void * calloc(size_t count, size_t size) { 39 | size_t sz = count * size; 40 | void *(*libc_malloc)(size_t) = dlsym(RTLD_NEXT, "malloc"); 41 | size_t volume = sz + sizeof(size_t) + sizeof(myalloc_cookie); 42 | void * answerplus = libc_malloc(volume); 43 | memset(answerplus,0,volume); 44 | if(answerplus == NULL) return answerplus;// nothing can be done 45 | malloced_memory_usage += sz; 46 | memcpy(answerplus ,&myalloc_cookie,sizeof(myalloc_cookie)); 47 | memcpy((char *) answerplus + sizeof(myalloc_cookie),&sz,sizeof(sz)); 48 | return ((char *) answerplus) + sizeof(size_t) + sizeof(myalloc_cookie); 49 | } 50 | 51 | void free(void *p) { 52 | if(p == NULL) return; // nothing to do 53 | void (*libc_free)(void*) = dlsym(RTLD_NEXT, "free"); 54 | void * truep = ((char *) p) - sizeof(size_t) - sizeof(myalloc_cookie); 55 | size_t cookie; 56 | // the cookie approach is kind of a hack, don't use in production code! 57 | memcpy(&cookie ,truep,sizeof(myalloc_cookie)); // in some case, this might read data outside of bounds 58 | if(cookie != myalloc_cookie) { 59 | libc_free(p); 60 | return; 61 | } 62 | size_t sz; 63 | memcpy(&sz,(char *) truep + sizeof(myalloc_cookie),sizeof(sz)); 64 | malloced_memory_usage -= sz; 65 | libc_free(truep); 66 | } 67 | 68 | 69 | void* realloc(void *p, size_t sz) { 70 | if(p == NULL) return malloc(sz); 71 | // implement 72 | void *(*libc_realloc)(void *m,size_t) = dlsym(RTLD_NEXT, "realloc"); 73 | void * truep = ((char *) p) - sizeof(size_t) - sizeof(myalloc_cookie); 74 | size_t cookie; 75 | // the cookie approach is kind of a hack, don't use in production code! 76 | memcpy(&cookie ,truep,sizeof(myalloc_cookie)); // in some case, this might read data outside of bounds 77 | if(cookie != myalloc_cookie) { 78 | return libc_realloc(p,sz); 79 | } 80 | size_t oldsz; 81 | memcpy(&oldsz,(char *) truep + sizeof(myalloc_cookie),sizeof(sz)); 82 | malloced_memory_usage -= oldsz; 83 | void * newp = libc_realloc(truep,sz + sizeof(size_t) + sizeof(myalloc_cookie)); 84 | if(newp == NULL) return newp;// nothing can be done? 85 | malloced_memory_usage += sz; 86 | memcpy((char *) newp + sizeof(myalloc_cookie),&sz,sizeof(sz)); 87 | return newp + sizeof(size_t) + sizeof(myalloc_cookie); 88 | } 89 | -------------------------------------------------------------------------------- /src/concise_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | #include "benchmark.h" 15 | #include "numbersfromtextfiles.h" 16 | #ifdef __cplusplus 17 | } 18 | #endif 19 | 20 | #include "concise.h" /* from Concise library */ 21 | 22 | 23 | /** 24 | * Once you have collected all the integers, build the bitmaps. 25 | */ 26 | static std::vector > create_all_bitmaps(size_t *howmany, 27 | uint32_t **numbers, size_t count) { 28 | if (numbers == NULL) return std::vector >(); 29 | std::vector > answer(count); 30 | for (size_t i = 0; i < count; i++) { 31 | ConciseSet & bm = answer[i]; 32 | uint32_t * mynumbers = numbers[i]; 33 | for(size_t j = 0; j < howmany[i] ; ++j) { 34 | bm.add(mynumbers[j]); 35 | } 36 | bm.compact(); 37 | assert(bm.size() == howmany[i]); 38 | } 39 | return answer; 40 | } 41 | 42 | static void printusage(char *command) { 43 | printf( 44 | " Try %s directory \n where directory could be " 45 | "benchmarks/realdata/census1881\n", 46 | command); 47 | ; 48 | printf("the -v flag turns on verbose mode"); 49 | 50 | } 51 | 52 | int main(int argc, char **argv) { 53 | int c; 54 | const char *extension = ".txt"; 55 | bool verbose = false; 56 | uint64_t data[13]; 57 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 58 | case 'e': 59 | extension = optarg; 60 | break; 61 | case 'v': 62 | verbose = true; 63 | break; 64 | case 'h': 65 | printusage(argv[0]); 66 | return 0; 67 | default: 68 | abort(); 69 | } 70 | if (optind >= argc) { 71 | printusage(argv[0]); 72 | return -1; 73 | } 74 | char *dirname = argv[optind]; 75 | size_t count; 76 | 77 | size_t *howmany = NULL; 78 | uint32_t **numbers = 79 | read_all_integer_files(dirname, extension, &howmany, &count); 80 | if (numbers == NULL) { 81 | printf( 82 | "I could not find or load any data file with extension %s in " 83 | "directory %s.\n", 84 | extension, dirname); 85 | return -1; 86 | } 87 | uint32_t maxvalue = 0; 88 | for (size_t i = 0; i < count; i++) { 89 | if( howmany[i] > 0 ) { 90 | if(maxvalue < numbers[i][howmany[i]-1]) { 91 | maxvalue = numbers[i][howmany[i]-1]; 92 | } 93 | } 94 | } 95 | uint64_t totalcard = 0; 96 | for (size_t i = 0; i < count; i++) { 97 | totalcard += howmany[i]; 98 | } 99 | uint64_t successivecard = 0; 100 | for (size_t i = 1; i < count; i++) { 101 | successivecard += howmany[i-1] + howmany[i]; 102 | } 103 | uint64_t cycles_start = 0, cycles_final = 0; 104 | 105 | RDTSC_START(cycles_start); 106 | std::vector > bitmaps = create_all_bitmaps(howmany, numbers, count); 107 | RDTSC_FINAL(cycles_final); 108 | if (bitmaps.empty()) return -1; 109 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 110 | uint64_t totalsize = 0; 111 | 112 | for (int i = 0; i < (int) count; ++i) { 113 | ConciseSet & bv = bitmaps[i]; 114 | totalsize += bv.sizeInBytes(); // should be close enough to memory usage 115 | } 116 | data[0] = totalsize; 117 | 118 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 119 | 120 | uint64_t successive_and = 0; 121 | uint64_t successive_or = 0; 122 | uint64_t total_or = 0; 123 | uint64_t total_count = 0; 124 | uint64_t successive_andnot = 0; 125 | uint64_t successive_xor = 0; 126 | 127 | 128 | RDTSC_START(cycles_start); 129 | for (int i = 0; i < (int)count - 1; ++i) { 130 | ConciseSet tempand = bitmaps[i].logicaland(bitmaps[i + 1]); 131 | successive_and += tempand.size(); 132 | } 133 | RDTSC_FINAL(cycles_final); 134 | data[1] = cycles_final - cycles_start; 135 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 136 | cycles_final - cycles_start); 137 | 138 | RDTSC_START(cycles_start); 139 | for (int i = 0; i < (int)count - 1; ++i) { 140 | ConciseSet tempor = bitmaps[i].logicalor(bitmaps[i + 1]); 141 | successive_or += tempor.size(); 142 | } 143 | RDTSC_FINAL(cycles_final); 144 | data[2] = cycles_final - cycles_start; 145 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 146 | cycles_final - cycles_start); 147 | 148 | RDTSC_START(cycles_start); 149 | if(count>1) { 150 | ConciseSet totalorbitmap = bitmaps[0].logicalor(bitmaps[1]); 151 | for(int i = 2 ; i < (int) count; ++i) { 152 | ConciseSet tmp = totalorbitmap.logicalor(bitmaps[i]); 153 | totalorbitmap.swap(tmp); 154 | } 155 | total_or = totalorbitmap.size(); 156 | } 157 | RDTSC_FINAL(cycles_final); 158 | data[3] = cycles_final - cycles_start; 159 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 160 | cycles_final - cycles_start); 161 | RDTSC_START(cycles_start); 162 | if(count>1) { 163 | const ConciseSet ** allofthem = new const ConciseSet* [count]; 164 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 165 | ConciseSet totalorbitmap = ConciseSet::fast_logicalor(count, allofthem); 166 | total_or = totalorbitmap.size(); 167 | delete[] allofthem; 168 | } 169 | RDTSC_FINAL(cycles_final); 170 | data[4] = cycles_final - cycles_start; 171 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 172 | cycles_final - cycles_start); 173 | 174 | uint64_t quartcount; 175 | STARTBEST(quartile_test_repetitions) 176 | quartcount = 0; 177 | for (size_t i = 0; i < count ; ++i) { 178 | quartcount += bitmaps[i].contains(maxvalue/4); 179 | quartcount += bitmaps[i].contains(maxvalue/2); 180 | quartcount += bitmaps[i].contains(3*maxvalue/4); 181 | } 182 | ENDBEST(data[5]) 183 | 184 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 185 | data[5]); 186 | 187 | RDTSC_START(cycles_start); 188 | for (int i = 0; i < (int)count - 1; ++i) { 189 | ConciseSet tempandnot = bitmaps[i].logicalandnot(bitmaps[i + 1]); 190 | successive_andnot += tempandnot.size(); 191 | } 192 | RDTSC_FINAL(cycles_final); 193 | data[6] = cycles_final - cycles_start; 194 | 195 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 196 | cycles_final - cycles_start); 197 | 198 | 199 | RDTSC_START(cycles_start); 200 | for (int i = 0; i < (int)count - 1; ++i) { 201 | ConciseSet tempxor = bitmaps[i].logicalxor(bitmaps[i + 1]); 202 | successive_xor += tempxor.size(); 203 | } 204 | RDTSC_FINAL(cycles_final); 205 | data[7] = cycles_final - cycles_start; 206 | 207 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 208 | cycles_final - cycles_start); 209 | 210 | RDTSC_START(cycles_start); 211 | for (size_t i = 0; i < count; ++i) { 212 | ConciseSet & b = bitmaps[i]; 213 | for(auto j = b.begin(); j != b.end() ; ++j) { 214 | total_count++; 215 | } 216 | } 217 | RDTSC_FINAL(cycles_final); 218 | data[8] = cycles_final - cycles_start; 219 | assert(successive_xor + successive_and == successive_or); 220 | 221 | assert(total_count == totalcard); 222 | 223 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 224 | cycles_final - cycles_start); 225 | 226 | 227 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 228 | 229 | 230 | /** 231 | * and, or, andnot and xor cardinality 232 | */ 233 | uint64_t successive_andcard = 0; 234 | uint64_t successive_orcard = 0; 235 | uint64_t successive_andnotcard = 0; 236 | uint64_t successive_xorcard = 0; 237 | 238 | RDTSC_START(cycles_start); 239 | for (int i = 0; i < (int)count - 1; ++i) { 240 | successive_andcard += bitmaps[i].logicalandCount(bitmaps[i + 1]); 241 | } 242 | RDTSC_FINAL(cycles_final); 243 | data[9] = cycles_final - cycles_start; 244 | 245 | RDTSC_START(cycles_start); 246 | for (int i = 0; i < (int)count - 1; ++i) { 247 | successive_orcard += bitmaps[i].logicalorCount(bitmaps[i + 1]); 248 | } 249 | RDTSC_FINAL(cycles_final); 250 | data[10] = cycles_final - cycles_start; 251 | 252 | RDTSC_START(cycles_start); 253 | for (int i = 0; i < (int)count - 1; ++i) { 254 | successive_andnotcard += bitmaps[i].logicalandnotCount(bitmaps[i + 1]); 255 | } 256 | RDTSC_FINAL(cycles_final); 257 | data[11] = cycles_final - cycles_start; 258 | 259 | RDTSC_START(cycles_start); 260 | for (int i = 0; i < (int)count - 1; ++i) { 261 | successive_xorcard += bitmaps[i].logicalxorCount(bitmaps[i + 1]); 262 | } 263 | RDTSC_FINAL(cycles_final); 264 | data[12] = cycles_final - cycles_start; 265 | 266 | assert(successive_andcard == successive_and); 267 | assert(successive_orcard == successive_or); 268 | assert(successive_xorcard == successive_xor); 269 | assert(successive_andnotcard == successive_andnot); 270 | 271 | /** 272 | * end and, or, andnot and xor cardinality 273 | */ 274 | 275 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 276 | data[0]*8.0/totalcard, 277 | data[1]*1.0/successivecard, 278 | data[2]*1.0/successivecard, 279 | data[3]*1.0/totalcard, 280 | data[4]*1.0/totalcard, 281 | data[5]*1.0/(3*count), 282 | data[6]*1.0/successivecard, 283 | data[7]*1.0/successivecard, 284 | data[8]*1.0/totalcard, 285 | data[9]*1.0/successivecard, 286 | data[10]*1.0/successivecard, 287 | data[11]*1.0/successivecard, 288 | data[12]*1.0/successivecard 289 | ); 290 | for (int i = 0; i < (int)count; ++i) { 291 | free(numbers[i]); 292 | numbers[i] = NULL; // paranoid 293 | } 294 | free(howmany); 295 | free(numbers); 296 | 297 | return 0; 298 | } 299 | -------------------------------------------------------------------------------- /src/ewah32_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #ifdef __cplusplus 11 | extern "C" { 12 | #endif 13 | #include "benchmark.h" 14 | #include "numbersfromtextfiles.h" 15 | #ifdef __cplusplus 16 | } 17 | #endif 18 | 19 | #include "ewah.h" /* EWAHBoolArray */ 20 | 21 | /** 22 | * Once you have collected all the integers, build the bitmaps. 23 | */ 24 | static std::vector > create_all_bitmaps(size_t *howmany, 25 | uint32_t **numbers, size_t count) { 26 | if (numbers == NULL) return std::vector >(); 27 | std::vector > answer(count); 28 | for (size_t i = 0; i < count; i++) { 29 | EWAHBoolArray & bm = answer[i]; 30 | uint32_t * mynumbers = numbers[i]; 31 | for(size_t j = 0; j < howmany[i] ; ++j) { 32 | bm.set(mynumbers[j]); 33 | } 34 | bm.trim(); 35 | } 36 | return answer; 37 | } 38 | 39 | static void printusage(char *command) { 40 | printf( 41 | " Try %s directory \n where directory could be " 42 | "benchmarks/realdata/census1881\n", 43 | command); 44 | ; 45 | printf("the -v flag turns on verbose mode"); 46 | 47 | } 48 | 49 | int main(int argc, char **argv) { 50 | int c; 51 | const char *extension = ".txt"; 52 | bool verbose = false; 53 | uint64_t data[13]; 54 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 55 | case 'e': 56 | extension = optarg; 57 | break; 58 | case 'v': 59 | verbose = true; 60 | break; 61 | case 'h': 62 | printusage(argv[0]); 63 | return 0; 64 | default: 65 | abort(); 66 | } 67 | if (optind >= argc) { 68 | printusage(argv[0]); 69 | return -1; 70 | } 71 | char *dirname = argv[optind]; 72 | size_t count; 73 | 74 | size_t *howmany = NULL; 75 | uint32_t **numbers = 76 | read_all_integer_files(dirname, extension, &howmany, &count); 77 | if (numbers == NULL) { 78 | printf( 79 | "I could not find or load any data file with extension %s in " 80 | "directory %s.\n", 81 | extension, dirname); 82 | return -1; 83 | } 84 | uint32_t maxvalue = 0; 85 | for (size_t i = 0; i < count; i++) { 86 | if( howmany[i] > 0 ) { 87 | if(maxvalue < numbers[i][howmany[i]-1]) { 88 | maxvalue = numbers[i][howmany[i]-1]; 89 | } 90 | } 91 | } 92 | uint64_t totalcard = 0; 93 | for (size_t i = 0; i < count; i++) { 94 | totalcard += howmany[i]; 95 | } 96 | uint64_t successivecard = 0; 97 | for (size_t i = 1; i < count; i++) { 98 | successivecard += howmany[i-1] + howmany[i]; 99 | } 100 | 101 | 102 | uint64_t cycles_start = 0, cycles_final = 0; 103 | 104 | RDTSC_START(cycles_start); 105 | std::vector > bitmaps = create_all_bitmaps(howmany, numbers, count); 106 | RDTSC_FINAL(cycles_final); 107 | if (bitmaps.empty()) return -1; 108 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 109 | uint64_t totalsize = 0; 110 | 111 | for (int i = 0; i < (int) count; ++i) { 112 | EWAHBoolArray & bv = bitmaps[i]; 113 | totalsize += bv.sizeInBytes(); // should be close enough to memory usage 114 | } 115 | data[0] = totalsize; 116 | 117 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 118 | 119 | uint64_t successive_and = 0; 120 | uint64_t successive_or = 0; 121 | uint64_t total_or = 0; 122 | uint64_t total_count = 0; 123 | uint64_t successive_andnot = 0; 124 | uint64_t successive_xor = 0; 125 | 126 | 127 | RDTSC_START(cycles_start); 128 | for (int i = 0; i < (int)count - 1; ++i) { 129 | EWAHBoolArray tempand; 130 | bitmaps[i].logicaland(bitmaps[i + 1],tempand); 131 | successive_and += tempand.numberOfOnes(); 132 | } 133 | RDTSC_FINAL(cycles_final); 134 | data[1] = cycles_final - cycles_start; 135 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 136 | cycles_final - cycles_start); 137 | 138 | RDTSC_START(cycles_start); 139 | for (int i = 0; i < (int)count - 1; ++i) { 140 | EWAHBoolArray tempor; 141 | bitmaps[i].logicalor(bitmaps[i + 1],tempor); 142 | successive_or += tempor.numberOfOnes(); 143 | } 144 | RDTSC_FINAL(cycles_final); 145 | data[2] = cycles_final - cycles_start; 146 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 147 | cycles_final - cycles_start); 148 | RDTSC_START(cycles_start); 149 | if(count>1) { 150 | EWAHBoolArray totalorbitmap; 151 | bitmaps[0].logicalor(bitmaps[1],totalorbitmap); 152 | for(int i = 2 ; i < (int) count; ++i) { 153 | EWAHBoolArray tmp; 154 | totalorbitmap.logicalor(bitmaps[i],tmp); 155 | tmp.swap(totalorbitmap); 156 | } 157 | total_or = totalorbitmap.numberOfOnes(); 158 | } 159 | RDTSC_FINAL(cycles_final); 160 | data[3] = cycles_final - cycles_start; 161 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 162 | cycles_final - cycles_start); 163 | 164 | RDTSC_START(cycles_start); 165 | if(count>1) { 166 | EWAHBoolArray totalorbitmap; 167 | const EWAHBoolArray ** allofthem = new const EWAHBoolArray* [count]; 168 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 169 | fast_logicalor_tocontainer(count, allofthem,totalorbitmap); 170 | total_or = totalorbitmap.numberOfOnes(); 171 | delete[] allofthem; 172 | } 173 | RDTSC_FINAL(cycles_final); 174 | data[4] = cycles_final - cycles_start; 175 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 176 | cycles_final - cycles_start); 177 | 178 | uint64_t quartcount; 179 | STARTBEST(quartile_test_repetitions) 180 | quartcount = 0; 181 | for (size_t i = 0; i < count ; ++i) { 182 | quartcount += bitmaps[i].get(maxvalue/4); 183 | quartcount += bitmaps[i].get(maxvalue/2); 184 | quartcount += bitmaps[i].get(3*maxvalue/4); 185 | } 186 | ENDBEST(data[5]) 187 | 188 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 189 | data[5]); 190 | 191 | RDTSC_START(cycles_start); 192 | for (int i = 0; i < (int)count - 1; ++i) { 193 | EWAHBoolArray tempandnot; 194 | bitmaps[i].logicalandnot(bitmaps[i + 1],tempandnot); 195 | successive_andnot += tempandnot.numberOfOnes(); 196 | } 197 | RDTSC_FINAL(cycles_final); 198 | data[6] = cycles_final - cycles_start; 199 | 200 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 201 | cycles_final - cycles_start); 202 | 203 | RDTSC_START(cycles_start); 204 | for (int i = 0; i < (int)count - 1; ++i) { 205 | EWAHBoolArray tempxor; 206 | bitmaps[i].logicalxor(bitmaps[i + 1],tempxor); 207 | successive_xor += tempxor.numberOfOnes(); 208 | } 209 | RDTSC_FINAL(cycles_final); 210 | data[7] = cycles_final - cycles_start; 211 | 212 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 213 | cycles_final - cycles_start); 214 | 215 | RDTSC_START(cycles_start); 216 | for (size_t i = 0; i < count; ++i) { 217 | EWAHBoolArray & b = bitmaps[i]; 218 | for (auto j = b.begin(); j != b.end(); ++j) { 219 | total_count++; 220 | } 221 | } 222 | RDTSC_FINAL(cycles_final); 223 | data[8] = cycles_final - cycles_start; 224 | assert(total_count == totalcard); 225 | 226 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 227 | cycles_final - cycles_start); 228 | 229 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 230 | 231 | assert(successive_xor + successive_and == successive_or); 232 | 233 | /** 234 | * and, or, andnot and xor cardinality 235 | */ 236 | uint64_t successive_andcard = 0; 237 | uint64_t successive_orcard = 0; 238 | uint64_t successive_andnotcard = 0; 239 | uint64_t successive_xorcard = 0; 240 | 241 | for (int i = 0; i < (int)count - 1; ++i) { 242 | assert(bitmaps[i].logicalandcount(bitmaps[i + 1]) == bitmaps[i].logicaland(bitmaps[i + 1]).numberOfOnes()); 243 | assert(bitmaps[i].logicalorcount(bitmaps[i + 1]) == bitmaps[i].logicalor(bitmaps[i + 1]).numberOfOnes()); 244 | assert(bitmaps[i].logicalxorcount(bitmaps[i + 1]) == bitmaps[i].logicalxor(bitmaps[i + 1]).numberOfOnes()); 245 | assert(bitmaps[i].logicalandnotcount(bitmaps[i + 1]) == bitmaps[i].logicalandnot(bitmaps[i + 1]).numberOfOnes()); 246 | } 247 | 248 | RDTSC_START(cycles_start); 249 | for (int i = 0; i < (int)count - 1; ++i) { 250 | successive_andcard += bitmaps[i].logicalandcount(bitmaps[i + 1]); 251 | } 252 | RDTSC_FINAL(cycles_final); 253 | data[9] = cycles_final - cycles_start; 254 | 255 | RDTSC_START(cycles_start); 256 | for (int i = 0; i < (int)count - 1; ++i) { 257 | successive_orcard += bitmaps[i].logicalorcount(bitmaps[i + 1]); 258 | } 259 | RDTSC_FINAL(cycles_final); 260 | data[10] = cycles_final - cycles_start; 261 | 262 | RDTSC_START(cycles_start); 263 | for (int i = 0; i < (int)count - 1; ++i) { 264 | successive_andnotcard += bitmaps[i].logicalandnotcount(bitmaps[i + 1]); 265 | } 266 | RDTSC_FINAL(cycles_final); 267 | data[11] = cycles_final - cycles_start; 268 | 269 | RDTSC_START(cycles_start); 270 | for (int i = 0; i < (int)count - 1; ++i) { 271 | successive_xorcard += bitmaps[i].logicalxorcount(bitmaps[i + 1]); 272 | } 273 | RDTSC_FINAL(cycles_final); 274 | data[12] = cycles_final - cycles_start; 275 | 276 | assert(successive_andcard == successive_and); 277 | assert(successive_xorcard == successive_xor); 278 | assert(successive_andnotcard == successive_andnot); 279 | assert(successive_orcard == successive_or); 280 | 281 | /** 282 | * end and, or, andnot and xor cardinality 283 | */ 284 | 285 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 286 | data[0]*8.0/totalcard, 287 | data[1]*1.0/successivecard, 288 | data[2]*1.0/successivecard, 289 | data[3]*1.0/totalcard, 290 | data[4]*1.0/totalcard, 291 | data[5]*1.0/(3*count), 292 | data[6]*1.0/successivecard, 293 | data[7]*1.0/successivecard, 294 | data[8]*1.0/totalcard, 295 | data[9]*1.0/successivecard, 296 | data[10]*1.0/successivecard, 297 | data[11]*1.0/successivecard, 298 | data[12]*1.0/successivecard 299 | ); 300 | for (int i = 0; i < (int)count; ++i) { 301 | free(numbers[i]); 302 | numbers[i] = NULL; // paranoid 303 | } 304 | free(howmany); 305 | free(numbers); 306 | 307 | return 0; 308 | } 309 | -------------------------------------------------------------------------------- /src/ewah64_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #ifdef __cplusplus 11 | extern "C" { 12 | #endif 13 | #include "benchmark.h" 14 | #include "numbersfromtextfiles.h" 15 | #ifdef __cplusplus 16 | } 17 | #endif 18 | 19 | #include "ewah.h" /* EWAHBoolArray */ 20 | 21 | /** 22 | * Once you have collected all the integers, build the bitmaps. 23 | */ 24 | static std::vector > create_all_bitmaps(size_t *howmany, 25 | uint32_t **numbers, size_t count) { 26 | if (numbers == NULL) return std::vector >(); 27 | std::vector > answer(count); 28 | for (size_t i = 0; i < count; i++) { 29 | EWAHBoolArray & bm = answer[i]; 30 | uint32_t * mynumbers = numbers[i]; 31 | for(size_t j = 0; j < howmany[i] ; ++j) { 32 | bm.set(mynumbers[j]); 33 | } 34 | bm.trim(); 35 | } 36 | return answer; 37 | } 38 | 39 | static void printusage(char *command) { 40 | printf( 41 | " Try %s directory \n where directory could be " 42 | "benchmarks/realdata/census1881\n", 43 | command); 44 | ; 45 | printf("the -v flag turns on verbose mode"); 46 | 47 | } 48 | 49 | int main(int argc, char **argv) { 50 | int c; 51 | const char *extension = ".txt"; 52 | bool verbose = false; 53 | uint64_t data[13]; 54 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 55 | case 'e': 56 | extension = optarg; 57 | break; 58 | case 'v': 59 | verbose = true; 60 | break; 61 | case 'h': 62 | printusage(argv[0]); 63 | return 0; 64 | default: 65 | abort(); 66 | } 67 | if (optind >= argc) { 68 | printusage(argv[0]); 69 | return -1; 70 | } 71 | char *dirname = argv[optind]; 72 | size_t count; 73 | 74 | size_t *howmany = NULL; 75 | uint32_t **numbers = 76 | read_all_integer_files(dirname, extension, &howmany, &count); 77 | if (numbers == NULL) { 78 | printf( 79 | "I could not find or load any data file with extension %s in " 80 | "directory %s.\n", 81 | extension, dirname); 82 | return -1; 83 | } 84 | uint32_t maxvalue = 0; 85 | for (size_t i = 0; i < count; i++) { 86 | if( howmany[i] > 0 ) { 87 | if(maxvalue < numbers[i][howmany[i]-1]) { 88 | maxvalue = numbers[i][howmany[i]-1]; 89 | } 90 | } 91 | } 92 | uint64_t totalcard = 0; 93 | for (size_t i = 0; i < count; i++) { 94 | totalcard += howmany[i]; 95 | } 96 | uint64_t successivecard = 0; 97 | for (size_t i = 1; i < count; i++) { 98 | successivecard += howmany[i-1] + howmany[i]; 99 | } 100 | 101 | uint64_t cycles_start = 0, cycles_final = 0; 102 | 103 | RDTSC_START(cycles_start); 104 | std::vector > bitmaps = create_all_bitmaps(howmany, numbers, count); 105 | RDTSC_FINAL(cycles_final); 106 | if (bitmaps.empty()) return -1; 107 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 108 | uint64_t totalsize = 0; 109 | 110 | for (int i = 0; i < (int) count; ++i) { 111 | EWAHBoolArray & bv = bitmaps[i]; 112 | totalsize += bv.sizeInBytes(); // should be close enough to memory usage 113 | } 114 | data[0] = totalsize; 115 | 116 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 117 | 118 | uint64_t successive_and = 0; 119 | uint64_t successive_or = 0; 120 | uint64_t total_or = 0; 121 | uint64_t total_count = 0; 122 | uint64_t successive_andnot = 0; 123 | uint64_t successive_xor = 0; 124 | 125 | 126 | RDTSC_START(cycles_start); 127 | for (int i = 0; i < (int)count - 1; ++i) { 128 | EWAHBoolArray tempand; 129 | bitmaps[i].logicaland(bitmaps[i + 1],tempand); 130 | successive_and += tempand.numberOfOnes(); 131 | } 132 | RDTSC_FINAL(cycles_final); 133 | data[1] = cycles_final - cycles_start; 134 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 135 | cycles_final - cycles_start); 136 | 137 | RDTSC_START(cycles_start); 138 | for (int i = 0; i < (int)count - 1; ++i) { 139 | EWAHBoolArray tempor; 140 | bitmaps[i].logicalor(bitmaps[i + 1],tempor); 141 | successive_or += tempor.numberOfOnes(); 142 | } 143 | RDTSC_FINAL(cycles_final); 144 | data[2] = cycles_final - cycles_start; 145 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 146 | cycles_final - cycles_start); 147 | RDTSC_START(cycles_start); 148 | if(count>1) { 149 | EWAHBoolArray totalorbitmap; 150 | bitmaps[0].logicalor(bitmaps[1],totalorbitmap); 151 | for(int i = 2 ; i < (int) count; ++i) { 152 | EWAHBoolArray tmp; 153 | totalorbitmap.logicalor(bitmaps[i],tmp); 154 | tmp.swap(totalorbitmap); 155 | } 156 | total_or = totalorbitmap.numberOfOnes(); 157 | } 158 | RDTSC_FINAL(cycles_final); 159 | data[3] = cycles_final - cycles_start; 160 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 161 | cycles_final - cycles_start); 162 | 163 | RDTSC_START(cycles_start); 164 | if(count>1) { 165 | EWAHBoolArray totalorbitmap; 166 | const EWAHBoolArray ** allofthem = new const EWAHBoolArray* [count]; 167 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 168 | fast_logicalor_tocontainer(count, allofthem,totalorbitmap); 169 | total_or = totalorbitmap.numberOfOnes(); 170 | delete[] allofthem; 171 | } 172 | RDTSC_FINAL(cycles_final); 173 | data[4] = cycles_final - cycles_start; 174 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 175 | cycles_final - cycles_start); 176 | uint64_t quartcount; 177 | STARTBEST(quartile_test_repetitions) 178 | quartcount = 0; 179 | for (size_t i = 0; i < count ; ++i) { 180 | quartcount += bitmaps[i].get(maxvalue/4); 181 | quartcount += bitmaps[i].get(maxvalue/2); 182 | quartcount += bitmaps[i].get(3*maxvalue/4); 183 | } 184 | ENDBEST(data[5]) 185 | 186 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 187 | data[5]); 188 | 189 | 190 | RDTSC_START(cycles_start); 191 | for (int i = 0; i < (int)count - 1; ++i) { 192 | EWAHBoolArray tempandnot; 193 | bitmaps[i].logicalandnot(bitmaps[i + 1],tempandnot); 194 | successive_andnot += tempandnot.numberOfOnes(); 195 | } 196 | RDTSC_FINAL(cycles_final); 197 | data[6] = cycles_final - cycles_start; 198 | 199 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 200 | cycles_final - cycles_start); 201 | 202 | RDTSC_START(cycles_start); 203 | for (int i = 0; i < (int)count - 1; ++i) { 204 | EWAHBoolArray tempxor; 205 | bitmaps[i].logicalxor(bitmaps[i + 1],tempxor); 206 | successive_xor += tempxor.numberOfOnes(); 207 | } 208 | RDTSC_FINAL(cycles_final); 209 | data[7] = cycles_final - cycles_start; 210 | 211 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 212 | cycles_final - cycles_start); 213 | 214 | RDTSC_START(cycles_start); 215 | for (size_t i = 0; i < count; ++i) { 216 | EWAHBoolArray & b = bitmaps[i]; 217 | for (auto j = b.begin(); j != b.end(); ++j) { 218 | total_count++; 219 | } 220 | } 221 | RDTSC_FINAL(cycles_final); 222 | data[8] = cycles_final - cycles_start; 223 | 224 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 225 | cycles_final - cycles_start); 226 | 227 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 228 | 229 | assert(successive_xor + successive_and == successive_or); 230 | 231 | /** 232 | * and, or, andnot and xor cardinality 233 | */ 234 | uint64_t successive_andcard = 0; 235 | uint64_t successive_orcard = 0; 236 | uint64_t successive_andnotcard = 0; 237 | uint64_t successive_xorcard = 0; 238 | for (int i = 0; i < (int)count - 1; ++i) { 239 | assert(bitmaps[i].logicalandcount(bitmaps[i + 1]) == bitmaps[i].logicaland(bitmaps[i + 1]).numberOfOnes()); 240 | assert(bitmaps[i].logicalorcount(bitmaps[i + 1]) == bitmaps[i].logicalor(bitmaps[i + 1]).numberOfOnes()); 241 | assert(bitmaps[i].logicalxorcount(bitmaps[i + 1]) == bitmaps[i].logicalxor(bitmaps[i + 1]).numberOfOnes()); 242 | assert(bitmaps[i].logicalandnotcount(bitmaps[i + 1]) == bitmaps[i].logicalandnot(bitmaps[i + 1]).numberOfOnes()); 243 | } 244 | 245 | RDTSC_START(cycles_start); 246 | for (int i = 0; i < (int)count - 1; ++i) { 247 | successive_andcard += bitmaps[i].logicalandcount(bitmaps[i + 1]); 248 | } 249 | RDTSC_FINAL(cycles_final); 250 | data[9] = cycles_final - cycles_start; 251 | 252 | RDTSC_START(cycles_start); 253 | for (int i = 0; i < (int)count - 1; ++i) { 254 | successive_orcard += bitmaps[i].logicalorcount(bitmaps[i + 1]); 255 | } 256 | RDTSC_FINAL(cycles_final); 257 | data[10] = cycles_final - cycles_start; 258 | 259 | RDTSC_START(cycles_start); 260 | for (int i = 0; i < (int)count - 1; ++i) { 261 | successive_andnotcard += bitmaps[i].logicalandnotcount(bitmaps[i + 1]); 262 | } 263 | RDTSC_FINAL(cycles_final); 264 | data[11] = cycles_final - cycles_start; 265 | 266 | RDTSC_START(cycles_start); 267 | for (int i = 0; i < (int)count - 1; ++i) { 268 | successive_xorcard += bitmaps[i].logicalxorcount(bitmaps[i + 1]); 269 | } 270 | RDTSC_FINAL(cycles_final); 271 | data[12] = cycles_final - cycles_start; 272 | 273 | assert(successive_andcard == successive_and); 274 | assert(successive_xorcard == successive_xor); 275 | assert(successive_andnotcard == successive_andnot); 276 | assert(successive_orcard == successive_or); 277 | 278 | /** 279 | * end and, or, andnot and xor cardinality 280 | */ 281 | assert(total_count == totalcard); 282 | 283 | 284 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 285 | data[0]*8.0/totalcard, 286 | data[1]*1.0/successivecard, 287 | data[2]*1.0/successivecard, 288 | data[3]*1.0/totalcard, 289 | data[4]*1.0/totalcard, 290 | data[5]*1.0/(3*count), 291 | data[6]*1.0/successivecard, 292 | data[7]*1.0/successivecard, 293 | data[8]*1.0/totalcard, 294 | data[9]*1.0/successivecard, 295 | data[10]*1.0/successivecard, 296 | data[11]*1.0/successivecard, 297 | data[12]*1.0/successivecard 298 | ); 299 | 300 | 301 | for (int i = 0; i < (int)count; ++i) { 302 | free(numbers[i]); 303 | numbers[i] = NULL; // paranoid 304 | } 305 | free(howmany); 306 | free(numbers); 307 | 308 | return 0; 309 | } 310 | -------------------------------------------------------------------------------- /src/hot_roaring_benchmarks.c: -------------------------------------------------------------------------------- 1 | #define __STDC_FORMAT_MACROS 1 2 | #define _GNU_SOURCE 3 | 4 | #include 5 | #include "benchmark.h" 6 | #include "numbersfromtextfiles.h" 7 | #include "roaring.c" 8 | 9 | bool roaring_iterator_increment(uint32_t value, void *param) { 10 | size_t count; 11 | memcpy(&count, param, sizeof(size_t)); 12 | count++; 13 | memcpy(param, &count, sizeof(size_t)); 14 | (void)value; 15 | return true; // continue till the end 16 | } 17 | 18 | /** 19 | * Once you have collected all the integers, build the bitmaps. 20 | */ 21 | static roaring_bitmap_t **create_all_bitmaps(size_t *howmany, 22 | uint32_t **numbers, size_t count, 23 | bool runoptimize, bool copyonwrite, 24 | bool verbose, 25 | uint64_t *totalsize) { 26 | *totalsize = 0; 27 | if (numbers == NULL) 28 | return NULL; 29 | size_t savedmem = 0; 30 | #ifdef RECORD_MALLOCS 31 | size_t totalmalloced = 0; 32 | #endif 33 | roaring_bitmap_t **answer = malloc(sizeof(roaring_bitmap_t *) * count); 34 | #ifdef RECORD_MALLOCS 35 | size_t bef = malloced_memory_usage; 36 | #endif 37 | for (size_t i = 0; i < count; i++) { 38 | answer[i] = roaring_bitmap_of_ptr(howmany[i], numbers[i]); 39 | answer[i]->copy_on_write = copyonwrite; 40 | if (runoptimize) 41 | roaring_bitmap_run_optimize(answer[i]); 42 | savedmem += roaring_bitmap_shrink_to_fit(answer[i]); 43 | *totalsize += roaring_bitmap_portable_size_in_bytes(answer[i]); 44 | } 45 | #ifdef RECORD_MALLOCS 46 | size_t aft = malloced_memory_usage; 47 | totalmalloced += aft - bef; 48 | if (verbose) 49 | printf("total malloc: %zu vs. reported %llu (%f %%) \n", totalmalloced, 50 | (unsigned long long)*totalsize, 51 | (totalmalloced - *totalsize) * 100.0 / *totalsize); 52 | *totalsize = totalmalloced; 53 | #endif 54 | if (verbose) 55 | printf("saved bytes by shrinking : %zu \n", savedmem); 56 | return answer; 57 | } 58 | 59 | const char *modes = 60 | "intersection, intersectioncount, union, unioncount, difference, " 61 | "differencecount, " 62 | "symdifference, symdifferencecount, wideunion, wideunionheap, access"; 63 | 64 | static void printusage(char *command) { 65 | printf(" Try %s -m intersection directory \n where directory could be " 66 | "benchmarks/realdata/census1881\n", 67 | command); 68 | ; 69 | printf("the -m flag can be followed by one of %s \n", modes); 70 | 71 | printf("the -r flag turns on run optimization\n"); 72 | printf("the -c flag turns on copy-on-write\n"); 73 | printf("the -v flag turns on verbose mode\n"); 74 | } 75 | 76 | int main(int argc, char **argv) { 77 | int c; 78 | bool runoptimize = false; 79 | bool verbose = false; 80 | bool copyonwrite = false; 81 | char *extension = ".txt"; 82 | char *mode = "none specified"; 83 | uint64_t cycles_start = 0, cycles_final = 0; 84 | 85 | RDTSC_START(cycles_start); 86 | 87 | while ((c = getopt(argc, argv, "cvrm:e:h")) != -1) 88 | switch (c) { 89 | case 'e': 90 | extension = optarg; 91 | break; 92 | case 'm': 93 | mode = optarg; 94 | if(verbose) printf("setting mode: %s \n", mode); 95 | break; 96 | case 'v': 97 | verbose = true; 98 | break; 99 | case 'r': 100 | runoptimize = true; 101 | if (verbose) 102 | printf("enabling run optimization\n"); 103 | break; 104 | case 'c': 105 | copyonwrite = true; 106 | if (verbose) 107 | printf("enabling copyonwrite\n"); 108 | break; 109 | case 'h': 110 | printusage(argv[0]); 111 | return 0; 112 | default: 113 | abort(); 114 | } 115 | if (optind >= argc) { 116 | printusage(argv[0]); 117 | return -1; 118 | } 119 | char *dirname = argv[optind]; 120 | size_t count; 121 | 122 | size_t *howmany = NULL; 123 | uint32_t **numbers = 124 | read_all_integer_files(dirname, extension, &howmany, &count); 125 | if (numbers == NULL) { 126 | printf("I could not find or load any data file with extension %s in " 127 | "directory %s.\n", 128 | extension, dirname); 129 | return -1; 130 | } 131 | uint32_t maxvalue = 0; 132 | for (size_t i = 0; i < count; i++) { 133 | if (howmany[i] > 0) { 134 | if (maxvalue < numbers[i][howmany[i] - 1]) { 135 | maxvalue = numbers[i][howmany[i] - 1]; 136 | } 137 | } 138 | } 139 | uint64_t totalcard = 0; 140 | for (size_t i = 0; i < count; i++) { 141 | totalcard += howmany[i]; 142 | } 143 | uint64_t successivecard = 0; 144 | for (size_t i = 1; i < count; i++) { 145 | successivecard += howmany[i - 1] + howmany[i]; 146 | } 147 | uint64_t totalsize = 0; 148 | roaring_bitmap_t **bitmaps = create_all_bitmaps( 149 | howmany, numbers, count, runoptimize, copyonwrite, verbose, &totalsize); 150 | if (bitmaps == NULL) 151 | return -1; 152 | if (verbose) 153 | printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 154 | if (verbose) 155 | printf("Total size in bytes = %" PRIu64 " \n", totalsize); 156 | RDTSC_FINAL(cycles_final); 157 | 158 | const size_t init_cycles = cycles_final - cycles_start; 159 | if(verbose) printf("init_cycles = %zu \n", init_cycles); 160 | size_t main_cycles = 0; 161 | size_t loops = 0; 162 | size_t bogus = 0; 163 | while (main_cycles < 100 * init_cycles) { 164 | RDTSC_START(cycles_start); 165 | if (strcmp(mode, "intersection") == 0) { 166 | for (int i = 0; i < (int)count - 1; ++i) { 167 | roaring_bitmap_t *tempand = 168 | roaring_bitmap_and(bitmaps[i], bitmaps[i + 1]); 169 | bogus += roaring_bitmap_get_cardinality(tempand); 170 | roaring_bitmap_free(tempand); 171 | } 172 | } else if (strcmp(mode, "intersectioncount") == 0) { 173 | for (int i = 0; i < (int)count - 1; ++i) { 174 | bogus += roaring_bitmap_and_cardinality(bitmaps[i], bitmaps[i + 1]); 175 | } 176 | 177 | } else if (strcmp(mode, "union") == 0) { 178 | for (int i = 0; i < (int)count - 1; ++i) { 179 | roaring_bitmap_t *tempor = 180 | roaring_bitmap_or(bitmaps[i], bitmaps[i + 1]); 181 | bogus += roaring_bitmap_get_cardinality(tempor); 182 | roaring_bitmap_free(tempor); 183 | } 184 | } else if (strcmp(mode, "unioncount") == 0) { 185 | for (int i = 0; i < (int)count - 1; ++i) { 186 | bogus += roaring_bitmap_or_cardinality(bitmaps[i], bitmaps[i + 1]); 187 | } 188 | } else if (strcmp(mode, "wideunion") == 0) { 189 | roaring_bitmap_t *totalorbitmap = 190 | roaring_bitmap_or_many(count, (const roaring_bitmap_t **)bitmaps); 191 | bogus += roaring_bitmap_get_cardinality(totalorbitmap); 192 | roaring_bitmap_free(totalorbitmap); 193 | } else if (strcmp(mode, "wideunionheap") == 0) { 194 | roaring_bitmap_t *totalorbitmapheap = roaring_bitmap_or_many_heap( 195 | count, (const roaring_bitmap_t **)bitmaps); 196 | bogus += roaring_bitmap_get_cardinality(totalorbitmapheap); 197 | roaring_bitmap_free(totalorbitmapheap); 198 | } else if (strcmp(mode, "access") == 0) { 199 | for (size_t i = 0; i < count; ++i) { 200 | bogus += roaring_bitmap_contains(bitmaps[i], maxvalue / 4); 201 | bogus += roaring_bitmap_contains(bitmaps[i], maxvalue / 2); 202 | bogus += roaring_bitmap_contains(bitmaps[i], 3 * maxvalue / 4); 203 | } 204 | } else if (strcmp(mode, "difference") == 0) { 205 | for (int i = 0; i < (int)count - 1; ++i) { 206 | roaring_bitmap_t *tempandnot = 207 | roaring_bitmap_andnot(bitmaps[i], bitmaps[i + 1]); 208 | bogus += roaring_bitmap_get_cardinality(tempandnot); 209 | roaring_bitmap_free(tempandnot); 210 | } 211 | } else if (strcmp(mode, "differencecount") == 0) { 212 | for (int i = 0; i < (int)count - 1; ++i) { 213 | bogus += roaring_bitmap_andnot_cardinality(bitmaps[i], bitmaps[i + 1]); 214 | } 215 | } else if (strcmp(mode, "symdifference") == 0) { 216 | for (int i = 0; i < (int)count - 1; ++i) { 217 | roaring_bitmap_t *tempxor = 218 | roaring_bitmap_xor(bitmaps[i], bitmaps[i + 1]); 219 | bogus += roaring_bitmap_get_cardinality(tempxor); 220 | roaring_bitmap_free(tempxor); 221 | } 222 | } else if (strcmp(mode, "symdifferencecount") == 0) { 223 | for (int i = 0; i < (int)count - 1; ++i) { 224 | bogus += roaring_bitmap_xor_cardinality(bitmaps[i], bitmaps[i + 1]); 225 | } 226 | } else if (strcmp(mode, "iterate") == 0) { 227 | for (size_t i = 0; i < count; ++i) { 228 | roaring_bitmap_t *ra = bitmaps[i]; 229 | roaring_iterate(ra, roaring_iterator_increment, &bogus); 230 | } 231 | } else /* default: */ 232 | { 233 | printf("mode: %s, try one of '%s' following the -m flag\n", mode, modes); 234 | break; 235 | } 236 | loops++; 237 | 238 | RDTSC_FINAL(cycles_final); 239 | main_cycles += cycles_final - cycles_start; 240 | 241 | } 242 | 243 | if(verbose) printf("repeated %zu times \n", loops); 244 | for (int i = 0; i < (int)count; ++i) { 245 | free(numbers[i]); 246 | numbers[i] = NULL; // paranoid 247 | roaring_bitmap_free(bitmaps[i]); 248 | bitmaps[i] = NULL; // paranoid 249 | } 250 | free(bitmaps); 251 | free(howmany); 252 | free(numbers); 253 | 254 | return (int)bogus; 255 | } 256 | -------------------------------------------------------------------------------- /src/memtrackingallocator.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_ALLOC_H 2 | #define INCLUDE_ALLOC_H 3 | 4 | #ifndef _GNU_SOURCE 5 | #define _GNU_SOURCE 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | uint64_t memory_usage = 0; 21 | 22 | // use this when calling STL object if you want 23 | // to keep track of memory usage 24 | template class MemoryCountingAllocator { 25 | public: 26 | // type definitions 27 | typedef T value_type; 28 | typedef T *pointer; 29 | typedef const T *const_pointer; 30 | typedef T &reference; 31 | typedef const T &const_reference; 32 | typedef std::size_t size_type; 33 | typedef std::ptrdiff_t difference_type; 34 | 35 | 36 | // rebind allocator to type U 37 | template struct rebind { 38 | typedef MemoryCountingAllocator other; 39 | }; 40 | 41 | pointer address(reference value) const { 42 | return &value; 43 | } 44 | const_pointer address(const_reference value) const { 45 | return &value; 46 | } 47 | 48 | MemoryCountingAllocator() : base() {} 49 | MemoryCountingAllocator(const MemoryCountingAllocator &) : base() {} 50 | template 51 | MemoryCountingAllocator(const MemoryCountingAllocator &) : base() {} 52 | ~MemoryCountingAllocator() {} 53 | 54 | // return maximum number of elements that can be allocated 55 | size_type max_size() const throw() { 56 | return base.max_size(); 57 | } 58 | 59 | pointer allocate(size_type num, const void * p = 0) { 60 | memory_usage += num * sizeof(T); 61 | return base.allocate(num,p); 62 | } 63 | 64 | void construct(pointer p, const T &value) { 65 | return base.construct(p,value); 66 | } 67 | 68 | // destroy elements of initialized storage p 69 | void destroy(pointer p) { 70 | base.destroy(p); 71 | } 72 | 73 | // deallocate storage p of deleted elements 74 | void deallocate(pointer p, size_type num ) { 75 | memory_usage -= num * sizeof(T); 76 | base.deallocate(p,num); 77 | } 78 | std::allocator base; 79 | }; 80 | 81 | // for our purposes, we don't want to distinguish between allocators. 82 | template 83 | bool operator==(const MemoryCountingAllocator &, const T2 &) throw() { 84 | return true; 85 | } 86 | 87 | template 88 | bool operator!=(const MemoryCountingAllocator &, const T2 &) throw() { 89 | return false; 90 | } 91 | 92 | #endif 93 | -------------------------------------------------------------------------------- /src/numbersfromtextfiles.h: -------------------------------------------------------------------------------- 1 | #ifndef BITMAPSFROMTEXTFILES_H_ 2 | #define BITMAPSFROMTEXTFILES_H_ 3 | #ifndef _GNU_SOURCE 4 | #define _GNU_SOURCE 5 | #endif 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | /*********************************/ 16 | /******************************** 17 | * General functions to load up bitmaps from text files. 18 | * Except format: comma-separated integers. 19 | *******************************/ 20 | /*********************************/ 21 | 22 | /** 23 | * Read the content of a file to a char array. Caller is 24 | * responsible for memory de-allocation. 25 | * Returns NULL on error. 26 | * 27 | * (If the individual files are small, this function is 28 | * a good idea.) 29 | */ 30 | static char *read_file(const char *filename) { 31 | FILE *fp = fopen(filename, "r"); 32 | if (!fp) { 33 | printf("Could not open file %s\n", filename); 34 | return NULL; 35 | } 36 | 37 | fseek(fp, 0, SEEK_END); 38 | size_t size = (size_t)ftell(fp); 39 | rewind(fp); 40 | char *answer = (char *)malloc(size + 1); 41 | if (!answer) { 42 | fclose(fp); 43 | return NULL; 44 | } 45 | if (fread(answer, size, 1, fp) != 1) { 46 | free(answer); 47 | return NULL; 48 | } 49 | answer[size] = '\0'; 50 | fclose(fp); 51 | return answer; 52 | } 53 | 54 | /** 55 | * Given a file made of comma-separated integers, 56 | * read it all and generate an array of integers. 57 | * The caller is responsible for memory de-allocation. 58 | */ 59 | static uint32_t *read_integer_file(char *filename, size_t *howmany) { 60 | char *buffer = read_file(filename); 61 | if (buffer == NULL) return NULL; 62 | 63 | size_t howmanyints = 1; 64 | size_t i1 = 0; 65 | for (; buffer[i1] != '\0'; i1++) { 66 | if (buffer[i1] == ',') ++howmanyints; 67 | } 68 | uint32_t *answer = (uint32_t *)malloc(howmanyints * sizeof(uint32_t)); 69 | if (answer == NULL) return NULL; 70 | size_t pos = 0; 71 | for (size_t i = 0; (i < i1) && (buffer[i] != '\0'); i++) { 72 | uint32_t currentint; 73 | while ((buffer[i] < '0') || (buffer[i] > '9')) { 74 | i++; 75 | if (buffer[i] == '\0') goto END; 76 | } 77 | currentint = (uint32_t)(buffer[i] - '0'); 78 | i++; 79 | for (; (buffer[i] >= '0') && (buffer[i] <= '9'); i++) 80 | currentint = currentint * 10 + (uint32_t)(buffer[i] - '0'); 81 | answer[pos++] = currentint; 82 | } 83 | END: 84 | if (pos != howmanyints) { 85 | printf("unexpected number of integers! %d %d \n", (int)pos, 86 | (int)howmanyints); 87 | } 88 | *howmany = pos; 89 | free(buffer); 90 | return answer; 91 | } 92 | 93 | /** 94 | * Does the file filename ends with the given extension. 95 | */ 96 | static bool hasExtension(const char *filename, const char *extension) { 97 | const char *ext = strrchr(filename, '.'); 98 | return (ext && !strcmp(ext, extension)); 99 | } 100 | 101 | /** 102 | * read all (count) integer files in a directory. Caller is responsible 103 | * for memory de-allocation. In case of error, a NULL is returned. 104 | */ 105 | static uint32_t **read_all_integer_files(const char *dirname, 106 | const char *extension, 107 | size_t **howmany, size_t *count) { 108 | struct dirent **entry_list; 109 | 110 | int ci = scandir(dirname, &entry_list, 0, alphasort); 111 | if (ci < 0) return NULL; 112 | size_t c = (size_t) ci; 113 | size_t truec = 0; 114 | for (size_t i = 0; i < c; i++) { 115 | if (hasExtension(entry_list[i]->d_name, extension)) ++truec; 116 | } 117 | *count = truec; 118 | *howmany = (size_t *)malloc(sizeof(size_t) * (*count)); 119 | uint32_t **answer = (uint32_t **)malloc(sizeof(uint32_t *) * (*count)); 120 | size_t dirlen = strlen(dirname); 121 | char *modifdirname = (char *)dirname; 122 | if (modifdirname[dirlen - 1] != '/') { 123 | modifdirname = (char *)malloc(dirlen + 2); 124 | strcpy(modifdirname, dirname); 125 | modifdirname[dirlen] = '/'; 126 | modifdirname[dirlen + 1] = '\0'; 127 | dirlen++; 128 | } 129 | for (size_t i = 0, pos = 0; i < (size_t)c; 130 | i++) { /* formerly looped while i < *count */ 131 | if (!hasExtension(entry_list[i]->d_name, extension)) continue; 132 | size_t filelen = strlen(entry_list[i]->d_name); 133 | char *fullpath = (char *)malloc(dirlen + filelen + 1); 134 | strcpy(fullpath, modifdirname); 135 | strcpy(fullpath + dirlen, entry_list[i]->d_name); 136 | answer[pos] = read_integer_file(fullpath, &((*howmany)[pos])); 137 | pos++; 138 | free(fullpath); 139 | } 140 | if (modifdirname != dirname) { 141 | free(modifdirname); 142 | } 143 | for (size_t i = 0; i < c; ++i) free(entry_list[i]); 144 | free(entry_list); 145 | return answer; 146 | } 147 | 148 | #endif /* BITMAPSFROMTEXTFILES_H_ */ 149 | -------------------------------------------------------------------------------- /src/roaring_benchmarks.c: -------------------------------------------------------------------------------- 1 | #define __STDC_FORMAT_MACROS 1 2 | #define _GNU_SOURCE 3 | 4 | #ifdef RECORD_MALLOCS 5 | #include "cmemcounter.h" 6 | #endif 7 | 8 | #include 9 | #include "benchmark.h" 10 | #include "numbersfromtextfiles.h" 11 | #include "roaring.c" 12 | 13 | bool roaring_iterator_increment(uint32_t value, void *param) { 14 | size_t count; 15 | memcpy(&count, param, sizeof(uint64_t)); 16 | count++; 17 | memcpy(param, &count, sizeof(uint64_t)); 18 | (void) value; 19 | return true; // continue till the end 20 | } 21 | 22 | /** 23 | * Once you have collected all the integers, build the bitmaps. 24 | */ 25 | static roaring_bitmap_t **create_all_bitmaps(size_t *howmany, 26 | uint32_t **numbers, size_t count, bool runoptimize, bool copyonwrite, bool verbose, uint64_t * totalsize) { 27 | *totalsize = 0; 28 | if (numbers == NULL) return NULL; 29 | size_t savedmem = 0; 30 | #ifdef RECORD_MALLOCS 31 | size_t totalmalloced = 0; 32 | #endif 33 | roaring_bitmap_t **answer = malloc(sizeof(roaring_bitmap_t *) * count); 34 | #ifdef RECORD_MALLOCS 35 | size_t bef = malloced_memory_usage; 36 | #endif 37 | for (size_t i = 0; i < count; i++) { 38 | answer[i] = roaring_bitmap_of_ptr(howmany[i], numbers[i]); 39 | answer[i]->copy_on_write = copyonwrite; 40 | if(runoptimize) roaring_bitmap_run_optimize(answer[i]); 41 | savedmem += roaring_bitmap_shrink_to_fit(answer[i]); 42 | *totalsize += roaring_bitmap_portable_size_in_bytes(answer[i]); 43 | } 44 | #ifdef RECORD_MALLOCS 45 | size_t aft = malloced_memory_usage; 46 | totalmalloced += aft - bef; 47 | if(verbose) printf("total malloc: %zu vs. reported %llu (%f %%) \n",totalmalloced,(unsigned long long)*totalsize,(totalmalloced-*totalsize)*100.0/ *totalsize); 48 | *totalsize = totalmalloced; 49 | #endif 50 | if(verbose) printf("saved bytes by shrinking : %zu \n",savedmem); 51 | return answer; 52 | } 53 | 54 | static void printusage(char *command) { 55 | printf( 56 | " Try %s directory \n where directory could be " 57 | "benchmarks/realdata/census1881\n", 58 | command); 59 | ; 60 | printf("the -r flag turns on run optimization"); 61 | printf("the -c flag turns on copy-on-write"); 62 | printf("the -v flag turns on verbose mode"); 63 | 64 | } 65 | 66 | 67 | int main(int argc, char **argv) { 68 | int c; 69 | bool runoptimize = false; 70 | bool verbose = false; 71 | bool copyonwrite = false; 72 | char *extension = ".txt"; 73 | uint64_t data[13]; 74 | while ((c = getopt(argc, argv, "cvre:h")) != -1) switch (c) { 75 | case 'e': 76 | extension = optarg; 77 | break; 78 | case 'v': 79 | verbose = true; 80 | break; 81 | case 'r': 82 | runoptimize = true; 83 | if(verbose) printf("enabling run optimization\n"); 84 | break; 85 | case 'c': 86 | copyonwrite = true; 87 | if(verbose) printf("enabling copyonwrite\n"); 88 | break; 89 | case 'h': 90 | printusage(argv[0]); 91 | return 0; 92 | default: 93 | abort(); 94 | } 95 | if (optind >= argc) { 96 | printusage(argv[0]); 97 | return -1; 98 | } 99 | char *dirname = argv[optind]; 100 | size_t count; 101 | 102 | size_t *howmany = NULL; 103 | uint32_t **numbers = 104 | read_all_integer_files(dirname, extension, &howmany, &count); 105 | if (numbers == NULL) { 106 | printf( 107 | "I could not find or load any data file with extension %s in " 108 | "directory %s.\n", 109 | extension, dirname); 110 | return -1; 111 | } 112 | uint32_t maxvalue = 0; 113 | for (size_t i = 0; i < count; i++) { 114 | if( howmany[i] > 0 ) { 115 | if(maxvalue < numbers[i][howmany[i]-1]) { 116 | maxvalue = numbers[i][howmany[i]-1]; 117 | } 118 | } 119 | } 120 | uint64_t totalcard = 0; 121 | for (size_t i = 0; i < count; i++) { 122 | totalcard += howmany[i]; 123 | } 124 | uint64_t successivecard = 0; 125 | for (size_t i = 1; i < count; i++) { 126 | successivecard += howmany[i-1] + howmany[i]; 127 | } 128 | uint64_t cycles_start = 0, cycles_final = 0; 129 | 130 | RDTSC_START(cycles_start); 131 | uint64_t totalsize = 0; 132 | roaring_bitmap_t **bitmaps = create_all_bitmaps(howmany, numbers, count,runoptimize,copyonwrite, verbose, &totalsize); 133 | RDTSC_FINAL(cycles_final); 134 | if (bitmaps == NULL) return -1; 135 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 136 | data[0] = totalsize; 137 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 138 | uint64_t successive_and = 0; 139 | uint64_t successive_or = 0; 140 | uint64_t total_or = 0; 141 | uint64_t total_count = 0; 142 | 143 | RDTSC_START(cycles_start); 144 | for (int i = 0; i < (int)count - 1; ++i) { 145 | roaring_bitmap_t *tempand = 146 | roaring_bitmap_and(bitmaps[i], bitmaps[i + 1]); 147 | successive_and += roaring_bitmap_get_cardinality(tempand); 148 | roaring_bitmap_free(tempand); 149 | } 150 | RDTSC_FINAL(cycles_final); 151 | data[1] = cycles_final - cycles_start; 152 | 153 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 154 | cycles_final - cycles_start); 155 | 156 | RDTSC_START(cycles_start); 157 | for (int i = 0; i < (int)count - 1; ++i) { 158 | roaring_bitmap_t *tempor = 159 | roaring_bitmap_or(bitmaps[i], bitmaps[i + 1]); 160 | successive_or += roaring_bitmap_get_cardinality(tempor); 161 | roaring_bitmap_free(tempor); 162 | } 163 | RDTSC_FINAL(cycles_final); 164 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 165 | cycles_final - cycles_start); 166 | data[2] = cycles_final - cycles_start; 167 | RDTSC_START(cycles_start); 168 | roaring_bitmap_t * totalorbitmap = roaring_bitmap_or_many(count,(const roaring_bitmap_t **)bitmaps); 169 | total_or = roaring_bitmap_get_cardinality(totalorbitmap); 170 | roaring_bitmap_free(totalorbitmap); 171 | RDTSC_FINAL(cycles_final); 172 | if(verbose) printf("Total unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 173 | cycles_final - cycles_start); 174 | data[3] = cycles_final - cycles_start; 175 | RDTSC_START(cycles_start); 176 | roaring_bitmap_t * totalorbitmapheap = roaring_bitmap_or_many_heap(count,(const roaring_bitmap_t **)bitmaps); 177 | total_or = roaring_bitmap_get_cardinality(totalorbitmapheap); 178 | roaring_bitmap_free(totalorbitmapheap); 179 | RDTSC_FINAL(cycles_final); 180 | if(verbose) printf("Total unions with heap on %zu bitmaps took %" PRIu64 " cycles\n", count, 181 | cycles_final - cycles_start); 182 | data[4] = cycles_final - cycles_start; 183 | 184 | uint64_t quartcount; 185 | STARTBEST(quartile_test_repetitions) 186 | quartcount = 0; 187 | for (size_t i = 0; i < count ; ++i) { 188 | quartcount += roaring_bitmap_contains(bitmaps[i],maxvalue/4); 189 | quartcount += roaring_bitmap_contains(bitmaps[i],maxvalue/2); 190 | quartcount += roaring_bitmap_contains(bitmaps[i],3*maxvalue/4); 191 | } 192 | ENDBEST(data[5]) 193 | 194 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 195 | data[5]); 196 | 197 | /*** 198 | * For good measure, we add ANDNOT and XOR 199 | ***/ 200 | uint64_t successive_andnot = 0; 201 | uint64_t successive_xor = 0; 202 | 203 | RDTSC_START(cycles_start); 204 | for (int i = 0; i < (int)count - 1; ++i) { 205 | roaring_bitmap_t *tempandnot = 206 | roaring_bitmap_andnot(bitmaps[i], bitmaps[i + 1]); 207 | successive_andnot += roaring_bitmap_get_cardinality(tempandnot); 208 | roaring_bitmap_free(tempandnot); 209 | } 210 | RDTSC_FINAL(cycles_final); 211 | data[6] = cycles_final - cycles_start; 212 | 213 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 214 | cycles_final - cycles_start); 215 | 216 | 217 | RDTSC_START(cycles_start); 218 | for (int i = 0; i < (int)count - 1; ++i) { 219 | roaring_bitmap_t *tempxor = 220 | roaring_bitmap_xor(bitmaps[i], bitmaps[i + 1]); 221 | successive_xor += roaring_bitmap_get_cardinality(tempxor); 222 | roaring_bitmap_free(tempxor); 223 | } 224 | RDTSC_FINAL(cycles_final); 225 | data[7] = cycles_final - cycles_start; 226 | 227 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 228 | cycles_final - cycles_start); 229 | /*** 230 | * End of ANDNOT and XOR 231 | ***/ 232 | RDTSC_START(cycles_start); 233 | for (size_t i = 0; i < count; ++i) { 234 | roaring_bitmap_t *ra = bitmaps[i]; 235 | roaring_iterate(ra, roaring_iterator_increment, &total_count); 236 | } 237 | /* 238 | for (size_t i = 0; i < count; ++i) { 239 | roaring_bitmap_t *ra = bitmaps[i]; 240 | roaring_uint32_iterator_t j; 241 | roaring_init_iterator(ra, &j); 242 | while(j.has_value) { 243 | total_count ++; 244 | roaring_advance_uint32_iterator(&j); 245 | } 246 | } 247 | */ 248 | RDTSC_FINAL(cycles_final); 249 | data[8] = cycles_final - cycles_start; 250 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 251 | cycles_final - cycles_start); 252 | 253 | assert(totalcard == total_count); 254 | 255 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 256 | 257 | /** 258 | * and, or, andnot and xor cardinality 259 | */ 260 | uint64_t successive_andcard = 0; 261 | uint64_t successive_orcard = 0; 262 | uint64_t successive_andnotcard = 0; 263 | uint64_t successive_xorcard = 0; 264 | 265 | RDTSC_START(cycles_start); 266 | for (int i = 0; i < (int)count - 1; ++i) { 267 | successive_andcard += roaring_bitmap_and_cardinality(bitmaps[i], bitmaps[i + 1]); 268 | } 269 | RDTSC_FINAL(cycles_final); 270 | data[9] = cycles_final - cycles_start; 271 | 272 | RDTSC_START(cycles_start); 273 | for (int i = 0; i < (int)count - 1; ++i) { 274 | successive_orcard += roaring_bitmap_or_cardinality(bitmaps[i], bitmaps[i + 1]); 275 | } 276 | RDTSC_FINAL(cycles_final); 277 | data[10] = cycles_final - cycles_start; 278 | 279 | RDTSC_START(cycles_start); 280 | for (int i = 0; i < (int)count - 1; ++i) { 281 | successive_andnotcard += roaring_bitmap_andnot_cardinality(bitmaps[i], bitmaps[i + 1]); 282 | } 283 | RDTSC_FINAL(cycles_final); 284 | data[11] = cycles_final - cycles_start; 285 | 286 | RDTSC_START(cycles_start); 287 | for (int i = 0; i < (int)count - 1; ++i) { 288 | successive_xorcard += roaring_bitmap_xor_cardinality(bitmaps[i], bitmaps[i + 1]); 289 | } 290 | RDTSC_FINAL(cycles_final); 291 | data[12] = cycles_final - cycles_start; 292 | 293 | assert(successive_andcard == successive_and); 294 | assert(successive_orcard == successive_or); 295 | assert(successive_xorcard == successive_xor); 296 | assert(successive_andnotcard == successive_andnot); 297 | 298 | /** 299 | * end and, or, andnot and xor cardinality 300 | */ 301 | 302 | 303 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 304 | data[0]*8.0/totalcard, 305 | data[1]*1.0/successivecard, 306 | data[2]*1.0/successivecard, 307 | data[3]*1.0/totalcard, 308 | data[4]*1.0/totalcard, 309 | data[5]*1.0/(3*count), 310 | data[6]*1.0/successivecard, 311 | data[7]*1.0/successivecard, 312 | data[8]*1.0/totalcard, 313 | data[9]*1.0/successivecard, 314 | data[10]*1.0/successivecard, 315 | data[11]*1.0/successivecard, 316 | data[12]*1.0/successivecard 317 | ); 318 | 319 | for (int i = 0; i < (int)count; ++i) { 320 | free(numbers[i]); 321 | numbers[i] = NULL; // paranoid 322 | roaring_bitmap_free(bitmaps[i]); 323 | bitmaps[i] = NULL; // paranoid 324 | } 325 | free(bitmaps); 326 | free(howmany); 327 | free(numbers); 328 | #ifdef DMALLOC // for Jon Strabala 329 | dmalloc_verify(0L); 330 | #endif 331 | return 0; 332 | } 333 | -------------------------------------------------------------------------------- /src/stl_hashset_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | #include "benchmark.h" 20 | #include "numbersfromtextfiles.h" 21 | #ifdef __cplusplus 22 | } 23 | #endif 24 | 25 | 26 | 27 | #ifdef MEMTRACKED 28 | #include "memtrackingallocator.h" 29 | #else 30 | size_t memory_usage; 31 | #endif 32 | 33 | void initializeMemUsageCounter() { 34 | memory_usage = 0; 35 | } 36 | 37 | uint64_t getMemUsageInBytes() { 38 | return memory_usage; 39 | } 40 | 41 | 42 | #ifdef MEMTRACKED 43 | typedef std::unordered_set,std::equal_to,MemoryCountingAllocator > hashset; 44 | #else 45 | typedef std::unordered_set hashset; 46 | #endif 47 | 48 | /** 49 | * Once you have collected all the integers, build the bitmaps. 50 | */ 51 | static std::vector create_all_bitmaps(size_t *howmany, 52 | uint32_t **numbers, size_t count) { 53 | if (numbers == NULL) return std::vector(); 54 | std::vector answer(count); 55 | for (size_t i = 0; i < count; i++) { 56 | hashset & bm = answer[i]; 57 | uint32_t * mynumbers = numbers[i]; 58 | for(size_t j = 0; j < howmany[i] ; ++j) { 59 | bm.insert(mynumbers[j]); 60 | } 61 | bm.rehash(howmany[i]); 62 | } 63 | return answer; 64 | } 65 | 66 | 67 | static void intersection(hashset& h1, hashset& h2, hashset& answer) { 68 | if(h1.size() > h2.size()) { 69 | intersection(h2,h1,answer); 70 | return; 71 | } 72 | answer.clear(); 73 | for(hashset::iterator i = h1.begin(); i != h1.end(); i++) { 74 | if(h2.find(*i) != h2.end()) 75 | answer.insert(*i); 76 | } 77 | } 78 | 79 | static size_t intersection_count(hashset& h1, hashset& h2) { 80 | if(h1.size() > h2.size()) { 81 | return intersection_count(h2,h1); 82 | } 83 | size_t answer = 0; 84 | for(hashset::iterator i = h1.begin(); i != h1.end(); i++) { 85 | if(h2.find(*i) != h2.end()) ++answer; 86 | } 87 | return answer; 88 | } 89 | 90 | 91 | static void difference(hashset& h1, hashset& h2, hashset& answer) { 92 | answer.clear(); 93 | for(hashset::iterator i = h1.begin(); i != h1.end(); i++) { 94 | if(h2.find(*i) == h2.end()) 95 | answer.insert(*i); 96 | } 97 | } 98 | 99 | 100 | static size_t difference_count(hashset& h1, hashset& h2) { 101 | size_t answer = 0; 102 | for(hashset::iterator i = h1.begin(); i != h1.end(); i++) { 103 | if(h2.find(*i) == h2.end()) 104 | answer++; 105 | } 106 | return answer; 107 | } 108 | 109 | static void symmetric_difference(hashset& h1, hashset& h2, hashset& answer) { 110 | answer.clear(); 111 | answer.insert(h1.begin(), h1.end()); 112 | for(hashset::iterator i = h2.begin(); i != h2.end(); i++) { 113 | auto x = answer.find(*i); 114 | if(x == answer.end()) 115 | answer.insert(*i); 116 | else 117 | answer.erase(x); 118 | } 119 | } 120 | 121 | static size_t symmetric_difference_count(hashset& h1, hashset& h2) { 122 | return h1.size() + h2.size() - 2 * intersection_count(h1,h2); 123 | } 124 | 125 | 126 | static void inplace_union(hashset& h1, hashset& h2) { 127 | h1.insert(h2.begin(), h2.end()); 128 | } 129 | 130 | static size_t union_count(hashset& h1, hashset& h2) { 131 | return h1.size() + h2.size() - intersection_count(h1,h2); 132 | } 133 | 134 | static void printusage(char *command) { 135 | printf( 136 | " Try %s directory \n where directory could be " 137 | "benchmarks/realdata/census1881\n", 138 | command); 139 | ; 140 | printf("the -v flag turns on verbose mode"); 141 | 142 | } 143 | 144 | 145 | 146 | 147 | int hashset_size_compare (const void * a, const void * b) { 148 | return ( *(const hashset**)a)->size() - (*(const hashset**)b)->size() ; 149 | } 150 | 151 | int main(int argc, char **argv) { 152 | int c; 153 | const char *extension = ".txt"; 154 | bool verbose = false; 155 | uint64_t data[13]; 156 | initializeMemUsageCounter(); 157 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 158 | case 'e': 159 | extension = optarg; 160 | break; 161 | case 'v': 162 | verbose = true; 163 | break; 164 | case 'h': 165 | printusage(argv[0]); 166 | return 0; 167 | default: 168 | abort(); 169 | } 170 | if (optind >= argc) { 171 | printusage(argv[0]); 172 | return -1; 173 | } 174 | char *dirname = argv[optind]; 175 | size_t count; 176 | 177 | 178 | size_t *howmany = NULL; 179 | uint32_t **numbers = 180 | read_all_integer_files(dirname, extension, &howmany, &count); 181 | if (numbers == NULL) { 182 | printf( 183 | "I could not find or load any data file with extension %s in " 184 | "directory %s.\n", 185 | extension, dirname); 186 | return -1; 187 | } 188 | uint32_t maxvalue = 0; 189 | for (size_t i = 0; i < count; i++) { 190 | if( howmany[i] > 0 ) { 191 | if(maxvalue < numbers[i][howmany[i]-1]) { 192 | maxvalue = numbers[i][howmany[i]-1]; 193 | } 194 | } 195 | } 196 | uint64_t totalcard = 0; 197 | for (size_t i = 0; i < count; i++) { 198 | totalcard += howmany[i]; 199 | } 200 | uint64_t successivecard = 0; 201 | for (size_t i = 1; i < count; i++) { 202 | successivecard += howmany[i-1] + howmany[i]; 203 | } 204 | uint64_t cycles_start = 0, cycles_final = 0; 205 | 206 | RDTSC_START(cycles_start); 207 | std::vector bitmaps = create_all_bitmaps(howmany, numbers, count); 208 | RDTSC_FINAL(cycles_final); 209 | if (bitmaps.empty()) return -1; 210 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 211 | uint64_t totalsize = getMemUsageInBytes(); 212 | data[0] = totalsize; 213 | 214 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 215 | 216 | uint64_t successive_and = 0; 217 | uint64_t successive_or = 0; 218 | uint64_t total_or = 0; 219 | uint64_t total_count = 0; 220 | uint64_t successive_andnot = 0; 221 | uint64_t successive_xor = 0; 222 | 223 | 224 | RDTSC_START(cycles_start); 225 | for (int i = 0; i < (int)count - 1; ++i) { 226 | hashset v; 227 | intersection(bitmaps[i], bitmaps[i + 1], v); 228 | successive_and += v.size(); 229 | } 230 | RDTSC_FINAL(cycles_final); 231 | data[1] = cycles_final - cycles_start; 232 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 233 | cycles_final - cycles_start); 234 | 235 | RDTSC_START(cycles_start); 236 | for (int i = 0; i < (int)count - 1; ++i) { 237 | hashset v (bitmaps[i]); 238 | inplace_union(v, bitmaps[i + 1]); 239 | successive_or += v.size(); 240 | } 241 | RDTSC_FINAL(cycles_final); 242 | data[2] = cycles_final - cycles_start; 243 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 244 | cycles_final - cycles_start); 245 | 246 | RDTSC_START(cycles_start); 247 | if(count>1) { 248 | hashset v (bitmaps[0]); 249 | inplace_union(v, bitmaps[1]); 250 | for (int i = 2; i < (int)count ; ++i) { 251 | inplace_union(v, bitmaps[i]); 252 | } 253 | total_or = v.size(); 254 | } 255 | RDTSC_FINAL(cycles_final); 256 | data[3] = cycles_final - cycles_start; 257 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 258 | cycles_final - cycles_start); 259 | RDTSC_START(cycles_start); 260 | if(count>1){ 261 | hashset **sortedbitmaps = (hashset**) malloc(sizeof(hashset*) * count); 262 | for (int i = 0; i < (int)count ; ++i) sortedbitmaps[i] = & bitmaps[i]; 263 | qsort (sortedbitmaps, count, sizeof(hashset *), hashset_size_compare); 264 | hashset v (*sortedbitmaps[0]); 265 | for (int i = 1; i < (int)count ; ++i) { 266 | inplace_union(v, *sortedbitmaps[i]); 267 | } 268 | total_or = v.size(); 269 | free(sortedbitmaps); 270 | } 271 | RDTSC_FINAL(cycles_final); 272 | data[4] = cycles_final - cycles_start; 273 | if(verbose) printf("Total sorted unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 274 | cycles_final - cycles_start); 275 | 276 | uint64_t quartcount; 277 | STARTBEST(quartile_test_repetitions) 278 | quartcount = 0; 279 | for (size_t i = 0; i < count ; ++i) { 280 | quartcount += (bitmaps[i].find(maxvalue/4) == bitmaps[i].end()); 281 | quartcount += (bitmaps[i].find(maxvalue/2) == bitmaps[i].end()); 282 | quartcount += (bitmaps[i].find(3*maxvalue/4) == bitmaps[i].end()); 283 | } 284 | ENDBEST(data[5]) 285 | 286 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 287 | data[5]); 288 | 289 | RDTSC_START(cycles_start); 290 | for (int i = 0; i < (int)count - 1; ++i) { 291 | hashset v; 292 | difference(bitmaps[i], bitmaps[i + 1], v); 293 | successive_andnot += v.size(); 294 | } 295 | RDTSC_FINAL(cycles_final); 296 | data[6] = cycles_final - cycles_start; 297 | 298 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 299 | cycles_final - cycles_start); 300 | 301 | RDTSC_START(cycles_start); 302 | for (int i = 0; i < (int)count - 1; ++i) { 303 | hashset v; 304 | symmetric_difference(bitmaps[i], bitmaps[i + 1], v); 305 | successive_xor += v.size(); 306 | } 307 | RDTSC_FINAL(cycles_final); 308 | data[7] = cycles_final - cycles_start; 309 | 310 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 311 | cycles_final - cycles_start); 312 | 313 | RDTSC_START(cycles_start); 314 | for (size_t i = 0; i < count; ++i) { 315 | hashset & b = bitmaps[i]; 316 | for(auto j = b.begin(); j != b.end() ; j++) { 317 | total_count++; 318 | } 319 | } 320 | RDTSC_FINAL(cycles_final); 321 | data[8] = cycles_final - cycles_start; 322 | assert(total_count == totalcard); 323 | 324 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 325 | cycles_final - cycles_start); 326 | 327 | 328 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 329 | 330 | assert(successive_xor + successive_and == successive_or); 331 | 332 | 333 | /** 334 | * and, or, andnot and xor cardinality 335 | */ 336 | uint64_t successive_andcard = 0; 337 | uint64_t successive_orcard = 0; 338 | uint64_t successive_andnotcard = 0; 339 | uint64_t successive_xorcard = 0; 340 | 341 | RDTSC_START(cycles_start); 342 | for (int i = 0; i < (int)count - 1; ++i) { 343 | successive_andcard += intersection_count(bitmaps[i], bitmaps[i + 1]); 344 | } 345 | RDTSC_FINAL(cycles_final); 346 | data[9] = cycles_final - cycles_start; 347 | 348 | RDTSC_START(cycles_start); 349 | for (int i = 0; i < (int)count - 1; ++i) { 350 | successive_orcard += union_count(bitmaps[i], bitmaps[i + 1]); 351 | } 352 | RDTSC_FINAL(cycles_final); 353 | data[10] = cycles_final - cycles_start; 354 | 355 | RDTSC_START(cycles_start); 356 | for (int i = 0; i < (int)count - 1; ++i) { 357 | successive_andnotcard += difference_count(bitmaps[i], bitmaps[i + 1]); 358 | } 359 | RDTSC_FINAL(cycles_final); 360 | data[11] = cycles_final - cycles_start; 361 | 362 | RDTSC_START(cycles_start); 363 | for (int i = 0; i < (int)count - 1; ++i) { 364 | successive_xorcard += symmetric_difference_count(bitmaps[i], bitmaps[i + 1]); 365 | } 366 | RDTSC_FINAL(cycles_final); 367 | data[12] = cycles_final - cycles_start; 368 | 369 | assert(successive_andcard == successive_and); 370 | assert(successive_orcard == successive_or); 371 | assert(successive_xorcard == successive_xor); 372 | assert(successive_andnotcard == successive_andnot); 373 | 374 | /** 375 | * end and, or, andnot and xor cardinality 376 | */ 377 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 378 | data[0]*8.0/totalcard, 379 | data[1]*1.0/successivecard, 380 | data[2]*1.0/successivecard, 381 | data[3]*1.0/totalcard, 382 | data[4]*1.0/totalcard, 383 | data[5]*1.0/(3*count), 384 | data[6]*1.0/successivecard, 385 | data[7]*1.0/successivecard, 386 | data[8]*1.0/totalcard, 387 | data[9]*1.0/successivecard, 388 | data[10]*1.0/successivecard, 389 | data[11]*1.0/successivecard, 390 | data[12]*1.0/successivecard 391 | ); 392 | 393 | for (int i = 0; i < (int)count; ++i) { 394 | free(numbers[i]); 395 | numbers[i] = NULL; // paranoid 396 | } 397 | free(howmany); 398 | free(numbers); 399 | 400 | return 0; 401 | } 402 | -------------------------------------------------------------------------------- /src/stl_vector_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | #include "benchmark.h" 18 | #include "numbersfromtextfiles.h" 19 | #ifdef __cplusplus 20 | } 21 | #endif 22 | 23 | #ifdef MEMTRACKED 24 | #include "memtrackingallocator.h" 25 | #else 26 | size_t memory_usage; 27 | #endif 28 | 29 | void initializeMemUsageCounter() { 30 | memory_usage = 0; 31 | } 32 | 33 | uint64_t getMemUsageInBytes() { 34 | return memory_usage; 35 | } 36 | 37 | // credit http://stackoverflow.com/questions/37767585/count-elements-in-union-of-two-sets-using-stl 38 | template 39 | class count_back_inserter { 40 | public: 41 | uint64_t & count; 42 | typedef void value_type; 43 | typedef void difference_type; 44 | typedef void pointer; 45 | typedef void reference; 46 | typedef std::output_iterator_tag iterator_category; 47 | count_back_inserter(uint64_t & c) : count(c) {}; 48 | void operator=(const T &){ } 49 | count_back_inserter &operator *(){ return *this; } 50 | count_back_inserter &operator++(){ count++;return *this; } 51 | 52 | }; 53 | typedef count_back_inserter inserter; 54 | 55 | #ifdef MEMTRACKED 56 | typedef std::vector > vector; 57 | #else 58 | typedef std::vector vector; 59 | #endif 60 | 61 | static vector fast_logicalor(size_t n, const vector **inputs) { 62 | class StdVectorPtr { 63 | 64 | public: 65 | StdVectorPtr(const vector *p, bool o) : ptr(p), own(o) {} 66 | const vector *ptr; 67 | bool own; // whether to clean 68 | 69 | bool operator<(const StdVectorPtr &o) const { 70 | return o.ptr->size() < ptr->size(); // backward on purpose 71 | } 72 | }; 73 | 74 | if (n == 0) { 75 | return vector(); 76 | } 77 | if (n == 1) { 78 | return vector(*inputs[0]); 79 | } 80 | std::priority_queue pq; 81 | for (size_t i = 0; i < n; i++) { 82 | // could use emplace 83 | pq.push(StdVectorPtr(inputs[i], false)); 84 | } 85 | while (pq.size() > 2) { 86 | 87 | StdVectorPtr x1 = pq.top(); 88 | pq.pop(); 89 | 90 | StdVectorPtr x2 = pq.top(); 91 | pq.pop(); 92 | vector * buffer = new vector(); 93 | std::set_union(x1.ptr->begin(), x1.ptr->end(),x2.ptr->begin(), x2.ptr->end(),std::back_inserter(*buffer)); 94 | if (x1.own) { 95 | delete x1.ptr; 96 | } 97 | if (x2.own) { 98 | delete x2.ptr; 99 | } 100 | pq.push(StdVectorPtr(buffer, true)); 101 | } 102 | StdVectorPtr x1 = pq.top(); 103 | pq.pop(); 104 | 105 | StdVectorPtr x2 = pq.top(); 106 | pq.pop(); 107 | 108 | vector container; 109 | std::set_union(x1.ptr->begin(), x1.ptr->end(),x2.ptr->begin(), x2.ptr->end(),std::back_inserter(container)); 110 | 111 | if (x1.own) { 112 | delete x1.ptr; 113 | } 114 | if (x2.own) { 115 | delete x2.ptr; 116 | } 117 | return container; 118 | } 119 | 120 | 121 | 122 | /** 123 | * Once you have collected all the integers, build the bitmaps. 124 | */ 125 | static std::vector create_all_bitmaps(size_t *howmany, 126 | uint32_t **numbers, size_t count) { 127 | if (numbers == NULL) return std::vector(); 128 | std::vector answer(count); 129 | 130 | for (size_t i = 0; i < count; i++) { 131 | vector & bm = answer[i]; 132 | uint32_t * mynumbers = numbers[i]; 133 | for(size_t j = 0; j < howmany[i] ; ++j) { 134 | bm.push_back(mynumbers[j]); 135 | } 136 | bm.shrink_to_fit(); 137 | } 138 | return answer; 139 | } 140 | 141 | static void printusage(char *command) { 142 | printf( 143 | " Try %s directory \n where directory could be " 144 | "benchmarks/realdata/census1881\n", 145 | command); 146 | ; 147 | printf("the -v flag turns on verbose mode"); 148 | 149 | } 150 | 151 | int main(int argc, char **argv) { 152 | int c; 153 | const char *extension = ".txt"; 154 | bool verbose = false; 155 | uint64_t data[13]; 156 | initializeMemUsageCounter(); 157 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 158 | case 'e': 159 | extension = optarg; 160 | break; 161 | case 'v': 162 | verbose = true; 163 | break; 164 | case 'h': 165 | printusage(argv[0]); 166 | return 0; 167 | default: 168 | abort(); 169 | } 170 | if (optind >= argc) { 171 | printusage(argv[0]); 172 | return -1; 173 | } 174 | char *dirname = argv[optind]; 175 | size_t count; 176 | 177 | 178 | size_t *howmany = NULL; 179 | uint32_t **numbers = 180 | read_all_integer_files(dirname, extension, &howmany, &count); 181 | if (numbers == NULL) { 182 | printf( 183 | "I could not find or load any data file with extension %s in " 184 | "directory %s.\n", 185 | extension, dirname); 186 | return -1; 187 | } 188 | uint32_t maxvalue = 0; 189 | for (size_t i = 0; i < count; i++) { 190 | if( howmany[i] > 0 ) { 191 | if(maxvalue < numbers[i][howmany[i]-1]) { 192 | maxvalue = numbers[i][howmany[i]-1]; 193 | } 194 | } 195 | } 196 | uint64_t totalcard = 0; 197 | for (size_t i = 0; i < count; i++) { 198 | totalcard += howmany[i]; 199 | } 200 | uint64_t successivecard = 0; 201 | for (size_t i = 1; i < count; i++) { 202 | successivecard += howmany[i-1] + howmany[i]; 203 | } 204 | uint64_t cycles_start = 0, cycles_final = 0; 205 | 206 | RDTSC_START(cycles_start); 207 | std::vector bitmaps = create_all_bitmaps(howmany, numbers, count); 208 | RDTSC_FINAL(cycles_final); 209 | if (bitmaps.empty()) return -1; 210 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 211 | uint64_t totalsize = getMemUsageInBytes(); 212 | data[0] = totalsize; 213 | 214 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 215 | 216 | uint64_t successive_and = 0; 217 | uint64_t successive_or = 0; 218 | uint64_t total_or = 0; 219 | uint64_t total_count = 0; 220 | uint64_t successive_andnot = 0; 221 | uint64_t successive_xor = 0; 222 | 223 | 224 | RDTSC_START(cycles_start); 225 | for (int i = 0; i < (int)count - 1; ++i) { 226 | vector v; 227 | std::set_intersection(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v)); 228 | successive_and += v.size(); 229 | } 230 | RDTSC_FINAL(cycles_final); 231 | data[1] = cycles_final - cycles_start; 232 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 233 | cycles_final - cycles_start); 234 | 235 | RDTSC_START(cycles_start); 236 | for (int i = 0; i < (int)count - 1; ++i) { 237 | vector v; 238 | std::set_union(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v)); 239 | successive_or += v.size(); 240 | } 241 | RDTSC_FINAL(cycles_final); 242 | data[2] = cycles_final - cycles_start; 243 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 244 | cycles_final - cycles_start); 245 | 246 | RDTSC_START(cycles_start); 247 | if(count>1) { 248 | vector v; 249 | std::set_union(bitmaps[0].begin(), bitmaps[0].end(),bitmaps[1].begin(), bitmaps[1].end(),std::back_inserter(v)); 250 | for (int i = 2; i < (int)count ; ++i) { 251 | vector newv; 252 | std::set_union(v.begin(), v.end(),bitmaps[i].begin(), bitmaps[i].end(),std::back_inserter(newv)); 253 | v.swap(newv); 254 | } 255 | total_or = v.size(); 256 | } 257 | RDTSC_FINAL(cycles_final); 258 | data[3] = cycles_final - cycles_start; 259 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 260 | cycles_final - cycles_start); 261 | RDTSC_START(cycles_start); 262 | if(count>1) { 263 | const vector ** allofthem = new const vector* [count]; 264 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 265 | vector totalorbitmap = fast_logicalor(count, allofthem); 266 | total_or = totalorbitmap.size(); 267 | delete[] allofthem; 268 | } 269 | RDTSC_FINAL(cycles_final); 270 | data[4] = cycles_final - cycles_start; 271 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 272 | cycles_final - cycles_start); 273 | 274 | RDTSC_START(cycles_start); 275 | uint64_t quartcount = 0; 276 | for (size_t i = 0; i < count ; ++i) { 277 | if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),maxvalue/4 ) ) 278 | quartcount ++; 279 | if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),maxvalue/2 ) ) 280 | quartcount ++; 281 | if ( std::binary_search(bitmaps[i].begin(),bitmaps[i].end(),3*maxvalue/4 ) ) 282 | quartcount ++; 283 | } 284 | RDTSC_FINAL(cycles_final); 285 | data[5] = cycles_final - cycles_start; 286 | 287 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 288 | cycles_final - cycles_start); 289 | 290 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 291 | 292 | RDTSC_START(cycles_start); 293 | for (int i = 0; i < (int)count - 1; ++i) { 294 | vector v; 295 | std::set_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v)); 296 | successive_andnot += v.size(); 297 | } 298 | RDTSC_FINAL(cycles_final); 299 | data[6] = cycles_final - cycles_start; 300 | 301 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 302 | cycles_final - cycles_start); 303 | 304 | RDTSC_START(cycles_start); 305 | for (int i = 0; i < (int)count - 1; ++i) { 306 | vector v; 307 | std::set_symmetric_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),std::back_inserter(v)); 308 | successive_xor += v.size(); 309 | } 310 | RDTSC_FINAL(cycles_final); 311 | data[7] = cycles_final - cycles_start; 312 | 313 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 314 | cycles_final - cycles_start); 315 | 316 | RDTSC_START(cycles_start); 317 | for (size_t i = 0; i < count; ++i) { 318 | vector & b = bitmaps[i]; 319 | for(auto j = b.begin(); j != b.end() ; j++) { 320 | total_count++; 321 | } 322 | } 323 | RDTSC_FINAL(cycles_final); 324 | data[8] = cycles_final - cycles_start; 325 | assert(total_count == totalcard); 326 | 327 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 328 | cycles_final - cycles_start); 329 | 330 | assert(successive_xor + successive_and == successive_or); 331 | 332 | /** 333 | * and, or, andnot and xor cardinality 334 | */ 335 | uint64_t successive_andcard = 0; 336 | uint64_t successive_orcard = 0; 337 | uint64_t successive_andnotcard = 0; 338 | uint64_t successive_xorcard = 0; 339 | 340 | RDTSC_START(cycles_start); 341 | for (int i = 0; i < (int)count - 1; ++i) { 342 | std::set_intersection(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_andcard)); 343 | } 344 | RDTSC_FINAL(cycles_final); 345 | data[9] = cycles_final - cycles_start; 346 | 347 | RDTSC_START(cycles_start); 348 | for (int i = 0; i < (int)count - 1; ++i) { 349 | std::set_union(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_orcard)); 350 | } 351 | RDTSC_FINAL(cycles_final); 352 | data[10] = cycles_final - cycles_start; 353 | 354 | RDTSC_START(cycles_start); 355 | for (int i = 0; i < (int)count - 1; ++i) { 356 | std::set_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_andnotcard)); 357 | } 358 | RDTSC_FINAL(cycles_final); 359 | data[11] = cycles_final - cycles_start; 360 | 361 | RDTSC_START(cycles_start); 362 | for (int i = 0; i < (int)count - 1; ++i) { 363 | std::set_symmetric_difference(bitmaps[i].begin(), bitmaps[i].end(),bitmaps[i+1].begin(), bitmaps[i+1].end(),inserter(successive_xorcard)); 364 | } 365 | RDTSC_FINAL(cycles_final); 366 | data[12] = cycles_final - cycles_start; 367 | 368 | assert(successive_andcard == successive_and); 369 | assert(successive_orcard == successive_or); 370 | assert(successive_xorcard == successive_xor); 371 | assert(successive_andnotcard == successive_andnot); 372 | 373 | /** 374 | * end and, or, andnot and xor cardinality 375 | */ 376 | 377 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 378 | data[0]*8.0/totalcard, 379 | data[1]*1.0/successivecard, 380 | data[2]*1.0/successivecard, 381 | data[3]*1.0/totalcard, 382 | data[4]*1.0/totalcard, 383 | data[5]*1.0/(3*count), 384 | data[6]*1.0/successivecard, 385 | data[7]*1.0/successivecard, 386 | data[8]*1.0/totalcard, 387 | data[9]*1.0/successivecard, 388 | data[10]*1.0/successivecard, 389 | data[11]*1.0/successivecard, 390 | data[12]*1.0/successivecard 391 | ); 392 | 393 | for (int i = 0; i < (int)count; ++i) { 394 | free(numbers[i]); 395 | numbers[i] = NULL; // paranoid 396 | } 397 | free(howmany); 398 | free(numbers); 399 | 400 | return 0; 401 | } 402 | -------------------------------------------------------------------------------- /src/wah32_benchmarks.cpp: -------------------------------------------------------------------------------- 1 | #ifndef _GNU_SOURCE 2 | #define _GNU_SOURCE 3 | #endif 4 | #define __STDC_FORMAT_MACROS 1 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | #include "benchmark.h" 15 | #include "numbersfromtextfiles.h" 16 | #ifdef __cplusplus 17 | } 18 | #endif 19 | 20 | #include "concise.h" /* from Concise library */ 21 | 22 | /** 23 | * Once you have collected all the integers, build the bitmaps. 24 | */ 25 | static std::vector > create_all_bitmaps(size_t *howmany, 26 | uint32_t **numbers, size_t count) { 27 | if (numbers == NULL) return std::vector >(); 28 | std::vector > answer(count); 29 | for (size_t i = 0; i < count; i++) { 30 | ConciseSet & bm = answer[i]; 31 | uint32_t * mynumbers = numbers[i]; 32 | for(size_t j = 0; j < howmany[i] ; ++j) { 33 | bm.add(mynumbers[j]); 34 | } 35 | bm.compact(); 36 | assert(bm.size() == howmany[i]); 37 | } 38 | return answer; 39 | } 40 | 41 | static void printusage(char *command) { 42 | printf( 43 | " Try %s directory \n where directory could be " 44 | "benchmarks/realdata/census1881\n", 45 | command); 46 | ; 47 | printf("the -v flag turns on verbose mode"); 48 | 49 | } 50 | 51 | int main(int argc, char **argv) { 52 | int c; 53 | const char *extension = ".txt"; 54 | bool verbose = false; 55 | uint64_t data[13]; 56 | while ((c = getopt(argc, argv, "ve:h")) != -1) switch (c) { 57 | case 'e': 58 | extension = optarg; 59 | break; 60 | case 'v': 61 | verbose = true; 62 | break; 63 | case 'h': 64 | printusage(argv[0]); 65 | return 0; 66 | default: 67 | abort(); 68 | } 69 | if (optind >= argc) { 70 | printusage(argv[0]); 71 | return -1; 72 | } 73 | char *dirname = argv[optind]; 74 | size_t count; 75 | 76 | size_t *howmany = NULL; 77 | uint32_t **numbers = 78 | read_all_integer_files(dirname, extension, &howmany, &count); 79 | if (numbers == NULL) { 80 | printf( 81 | "I could not find or load any data file with extension %s in " 82 | "directory %s.\n", 83 | extension, dirname); 84 | return -1; 85 | } 86 | uint32_t maxvalue = 0; 87 | for (size_t i = 0; i < count; i++) { 88 | if( howmany[i] > 0 ) { 89 | if(maxvalue < numbers[i][howmany[i]-1]) { 90 | maxvalue = numbers[i][howmany[i]-1]; 91 | } 92 | } 93 | } 94 | uint64_t totalcard = 0; 95 | for (size_t i = 0; i < count; i++) { 96 | totalcard += howmany[i]; 97 | } 98 | uint64_t successivecard = 0; 99 | for (size_t i = 1; i < count; i++) { 100 | successivecard += howmany[i-1] + howmany[i]; 101 | } 102 | uint64_t cycles_start = 0, cycles_final = 0; 103 | 104 | RDTSC_START(cycles_start); 105 | std::vector > bitmaps = create_all_bitmaps(howmany, numbers, count); 106 | RDTSC_FINAL(cycles_final); 107 | if (bitmaps.empty()) return -1; 108 | if(verbose) printf("Loaded %d bitmaps from directory %s \n", (int)count, dirname); 109 | uint64_t totalsize = 0; 110 | 111 | for (int i = 0; i < (int) count; ++i) { 112 | ConciseSet & bv = bitmaps[i]; 113 | totalsize += bv.sizeInBytes(); // should be close enough to memory usage 114 | } 115 | data[0] = totalsize; 116 | 117 | if(verbose) printf("Total size in bytes = %" PRIu64 " \n", totalsize); 118 | 119 | uint64_t successive_and = 0; 120 | uint64_t successive_or = 0; 121 | uint64_t total_or = 0; 122 | uint64_t total_count = 0; 123 | uint64_t successive_andnot = 0; 124 | uint64_t successive_xor = 0; 125 | 126 | 127 | RDTSC_START(cycles_start); 128 | for (int i = 0; i < (int)count - 1; ++i) { 129 | ConciseSet tempand = bitmaps[i].logicaland(bitmaps[i + 1]); 130 | successive_and += tempand.size(); 131 | } 132 | RDTSC_FINAL(cycles_final); 133 | data[1] = cycles_final - cycles_start; 134 | if(verbose) printf("Successive intersections on %zu bitmaps took %" PRIu64 " cycles\n", count, 135 | cycles_final - cycles_start); 136 | 137 | RDTSC_START(cycles_start); 138 | for (int i = 0; i < (int)count - 1; ++i) { 139 | ConciseSet tempor = bitmaps[i].logicalor(bitmaps[i + 1]); 140 | successive_or += tempor.size(); 141 | } 142 | RDTSC_FINAL(cycles_final); 143 | data[2] = cycles_final - cycles_start; 144 | if(verbose) printf("Successive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 145 | cycles_final - cycles_start); 146 | 147 | RDTSC_START(cycles_start); 148 | if(count>1) { 149 | ConciseSet totalorbitmap = bitmaps[0].logicalor(bitmaps[1]); 150 | for(int i = 2 ; i < (int) count; ++i) { 151 | ConciseSet tmp = totalorbitmap.logicalor(bitmaps[i]); 152 | totalorbitmap.swap(tmp); 153 | } 154 | total_or = totalorbitmap.size(); 155 | } 156 | RDTSC_FINAL(cycles_final); 157 | data[3] = cycles_final - cycles_start; 158 | if(verbose) printf("Total naive unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 159 | cycles_final - cycles_start); 160 | RDTSC_START(cycles_start); 161 | if(count>1) { 162 | const ConciseSet ** allofthem = new const ConciseSet* [count]; 163 | for(int i = 0 ; i < (int) count; ++i) allofthem[i] = & bitmaps[i]; 164 | ConciseSet totalorbitmap = ConciseSet::fast_logicalor(count, allofthem); 165 | total_or = totalorbitmap.size(); 166 | delete[] allofthem; 167 | } 168 | RDTSC_FINAL(cycles_final); 169 | data[4] = cycles_final - cycles_start; 170 | if(verbose) printf("Total heap unions on %zu bitmaps took %" PRIu64 " cycles\n", count, 171 | cycles_final - cycles_start); 172 | 173 | uint64_t quartcount; 174 | STARTBEST(quartile_test_repetitions) 175 | quartcount = 0; 176 | for (size_t i = 0; i < count ; ++i) { 177 | quartcount += bitmaps[i].contains(maxvalue/4); 178 | quartcount += bitmaps[i].contains(maxvalue/2); 179 | quartcount += bitmaps[i].contains(3*maxvalue/4); 180 | } 181 | ENDBEST(data[5]) 182 | 183 | if(verbose) printf("Quartile queries on %zu bitmaps took %" PRIu64 " cycles\n", count, 184 | data[5]); 185 | 186 | 187 | RDTSC_START(cycles_start); 188 | for (int i = 0; i < (int)count - 1; ++i) { 189 | ConciseSet tempandnot = bitmaps[i].logicalandnot(bitmaps[i + 1]); 190 | successive_andnot += tempandnot.size(); 191 | } 192 | RDTSC_FINAL(cycles_final); 193 | data[6] = cycles_final - cycles_start; 194 | 195 | if(verbose) printf("Successive differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 196 | cycles_final - cycles_start); 197 | 198 | RDTSC_START(cycles_start); 199 | for (int i = 0; i < (int)count - 1; ++i) { 200 | ConciseSet tempxor = bitmaps[i].logicalxor(bitmaps[i + 1]); 201 | successive_xor += tempxor.size(); 202 | } 203 | RDTSC_FINAL(cycles_final); 204 | data[7] = cycles_final - cycles_start; 205 | 206 | if(verbose) printf("Successive symmetric differences on %zu bitmaps took %" PRIu64 " cycles\n", count, 207 | cycles_final - cycles_start); 208 | 209 | RDTSC_START(cycles_start); 210 | for (size_t i = 0; i < count; ++i) { 211 | ConciseSet & b = bitmaps[i]; 212 | for(auto j = b.begin(); j != b.end() ; ++j) { 213 | total_count++; 214 | } 215 | } 216 | RDTSC_FINAL(cycles_final); 217 | data[8] = cycles_final - cycles_start; 218 | assert(total_count == totalcard); 219 | 220 | if(verbose) printf("Iterating over %zu bitmaps took %" PRIu64 " cycles\n", count, 221 | cycles_final - cycles_start); 222 | 223 | if(verbose) printf("Collected stats %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",successive_and,successive_or,total_or,quartcount); 224 | 225 | assert(successive_xor + successive_and == successive_or); 226 | 227 | /** 228 | * and, or, andnot and xor cardinality 229 | */ 230 | uint64_t successive_andcard = 0; 231 | uint64_t successive_orcard = 0; 232 | uint64_t successive_andnotcard = 0; 233 | uint64_t successive_xorcard = 0; 234 | 235 | RDTSC_START(cycles_start); 236 | for (int i = 0; i < (int)count - 1; ++i) { 237 | successive_andcard += bitmaps[i].logicalandCount(bitmaps[i + 1]); 238 | } 239 | RDTSC_FINAL(cycles_final); 240 | data[9] = cycles_final - cycles_start; 241 | 242 | RDTSC_START(cycles_start); 243 | for (int i = 0; i < (int)count - 1; ++i) { 244 | successive_orcard += bitmaps[i].logicalorCount(bitmaps[i + 1]); 245 | } 246 | RDTSC_FINAL(cycles_final); 247 | data[10] = cycles_final - cycles_start; 248 | 249 | RDTSC_START(cycles_start); 250 | for (int i = 0; i < (int)count - 1; ++i) { 251 | successive_andnotcard += bitmaps[i].logicalandnotCount(bitmaps[i + 1]); 252 | } 253 | RDTSC_FINAL(cycles_final); 254 | data[11] = cycles_final - cycles_start; 255 | 256 | RDTSC_START(cycles_start); 257 | for (int i = 0; i < (int)count - 1; ++i) { 258 | successive_xorcard += bitmaps[i].logicalxorCount(bitmaps[i + 1]); 259 | } 260 | RDTSC_FINAL(cycles_final); 261 | data[12] = cycles_final - cycles_start; 262 | 263 | assert(successive_andcard == successive_and); 264 | assert(successive_orcard == successive_or); 265 | assert(successive_xorcard == successive_xor); 266 | assert(successive_andnotcard == successive_andnot); 267 | 268 | /** 269 | * end and, or, andnot and xor cardinality 270 | */ 271 | 272 | printf(" %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f %20.2f\n", 273 | data[0]*8.0/totalcard, 274 | data[1]*1.0/successivecard, 275 | data[2]*1.0/successivecard, 276 | data[3]*1.0/totalcard, 277 | data[4]*1.0/totalcard, 278 | data[5]*1.0/(3*count), 279 | data[6]*1.0/successivecard, 280 | data[7]*1.0/successivecard, 281 | data[8]*1.0/totalcard, 282 | data[9]*1.0/successivecard, 283 | data[10]*1.0/successivecard, 284 | data[11]*1.0/successivecard, 285 | data[12]*1.0/successivecard 286 | ); 287 | 288 | 289 | for (int i = 0; i < (int)count; ++i) { 290 | free(numbers[i]); 291 | numbers[i] = NULL; // paranoid 292 | } 293 | free(howmany); 294 | free(numbers); 295 | 296 | return 0; 297 | } 298 | -------------------------------------------------------------------------------- /synthetic/anh_moffat_clustered.h: -------------------------------------------------------------------------------- 1 | #ifndef ANH_MOFFAT_CLUSTERED_H 2 | #define ANH_MOFFAT_CLUSTERED_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | static inline int numberOfTrailingZeros(uint64_t x) { 13 | if (x == 0) 14 | return 64; 15 | return __builtin_ctzl(x); 16 | } 17 | 18 | class BoolArray { 19 | public: 20 | vector buffer; 21 | size_t sizeinbits; 22 | BoolArray(const size_t n, const uint64_t initval = 0) 23 | : buffer(n / 64 + (n % 64 == 0 ? 0 : 1), initval), sizeinbits(n) {} 24 | 25 | BoolArray() : buffer(), sizeinbits(0) {} 26 | 27 | BoolArray(const BoolArray &ba) 28 | : buffer(ba.buffer), sizeinbits(ba.sizeinbits) {} 29 | 30 | void inplaceIntersect(const BoolArray &other) { 31 | assert(other.buffer.size() == buffer.size()); 32 | for (size_t i = 0; i < buffer.size(); ++i) 33 | buffer[i] &= other.buffer[i]; 34 | } 35 | 36 | 37 | void intersect(const BoolArray &other, BoolArray &output) { 38 | assert(other.buffer.size() == buffer.size()); 39 | output.buffer.resize(buffer.size()); 40 | for (size_t i = 0; i < buffer.size(); ++i) 41 | output.buffer[i] = buffer[i] & other.buffer[i]; 42 | } 43 | 44 | void setSizeInBits(const size_t sizeib) { sizeinbits = sizeib; } 45 | 46 | /** 47 | * Write out this bitmap to a vector as a list of integers corresponding 48 | * to set bits. The caller should have allocated enough memory. 49 | */ 50 | void toArray(vector &ans) { 51 | uint32_t pos = 0; 52 | for (uint32_t k = 0; k < buffer.size(); ++k) { 53 | uint64_t myword = buffer[k]; 54 | while (myword != 0) { 55 | int ntz = __builtin_ctzl(myword); 56 | ans[pos++] = k * 64 + ntz; 57 | myword ^= (1ll << ntz); 58 | } 59 | } 60 | ans.resize(pos); 61 | } 62 | 63 | /** 64 | * This is a version of toArray where we write to a pointer. 65 | * Returns the number of written ints. 66 | */ 67 | size_t toInts(uint32_t *out) { 68 | size_t pos = 0; 69 | for (uint32_t k = 0; k < buffer.size(); ++k) { 70 | const uint64_t myword = buffer[k]; 71 | for (int offset = 0; offset < 64; ++offset) { 72 | if ((myword >> offset) == 0) 73 | break; 74 | offset += numberOfTrailingZeros((myword >> offset)); 75 | out[pos++] = 64 * k + offset; 76 | } 77 | } 78 | return pos; 79 | } 80 | BoolArray &operator=(const BoolArray &x) { 81 | this->buffer = x.buffer; 82 | this->sizeinbits = x.sizeinbits; 83 | return *this; 84 | } 85 | 86 | /** 87 | * set to true (whether it was already set to true or not) 88 | * 89 | * This is an expensive (random access) API, you really ought to 90 | * prepare a new word and then append it. 91 | */ 92 | void set(const size_t pos) { 93 | buffer[pos / 64] |= (static_cast(1) << (pos % 64)); 94 | } 95 | 96 | /** 97 | * set to false (whether it was already set to false or not) 98 | * 99 | * This is an expensive (random access) API, you really ought to 100 | * prepare a new word and then append it. 101 | */ 102 | void unset(const size_t pos) { 103 | buffer[pos / 64] |= ~(static_cast(1) << (pos % 64)); 104 | } 105 | 106 | /** 107 | * true of false? (set or unset) 108 | */ 109 | bool get(const size_t pos) const { 110 | return (buffer[pos / 64] & (static_cast(1) << (pos % 64))) != 0; 111 | } 112 | 113 | /** 114 | * set all bits to 0 115 | */ 116 | void reset() { 117 | memset(buffer.data(), 0, 118 | sizeof(uint64_t) * 119 | buffer.size()); // memset can be slow, does it matter? 120 | sizeinbits = 0; 121 | } 122 | 123 | size_t sizeInBits() const { return sizeinbits; } 124 | 125 | size_t sizeInBytes() const { return buffer.size() * sizeof(uint64_t); } 126 | 127 | /** 128 | * Return memory usage of a bitmap spanning n bits 129 | */ 130 | static size_t sizeInBytes(size_t n) { 131 | size_t buffersize = n / 64 + (n % 64 == 0 ? 0 : 1); 132 | return buffersize * sizeof(uint64_t); 133 | } 134 | 135 | ~BoolArray() {} 136 | }; 137 | 138 | /** 139 | * Mersenne twister - random number generator. 140 | * Generate uniform distribution of 32 bit integers with the MT19937 algorithm. 141 | * source: http://bannister.us/weblog/?s=Mersenne 142 | */ 143 | class ZRandom { 144 | 145 | public: 146 | enum { N = 624, M = 397 }; 147 | unsigned int MT[N + 1]; 148 | unsigned int *map[N]; 149 | int nValues; 150 | 151 | ZRandom(unsigned int iSeed = 20070102); 152 | void seed(unsigned iSeed); 153 | unsigned int getValue(); 154 | unsigned int getValue(const uint32_t MaxValue); 155 | double getDouble(); 156 | bool test(const double p); 157 | }; 158 | 159 | ZRandom::ZRandom(unsigned iSeed) : nValues(0) { seed(iSeed); } 160 | 161 | void ZRandom::seed(unsigned iSeed) { 162 | nValues = 0; 163 | // Seed the array used in random number generation. 164 | MT[0] = iSeed; 165 | for (int i = 1; i < N; ++i) { 166 | MT[i] = 1 + (69069 * MT[i - 1]); 167 | } 168 | // Compute map once to avoid % in inner loop. 169 | for (int i = 0; i < N; ++i) { 170 | map[i] = MT + ((i + M) % N); 171 | } 172 | } 173 | 174 | inline bool ZRandom::test(const double p) { return getDouble() <= p; } 175 | inline double ZRandom::getDouble() { 176 | return double(getValue()) * (1.0 / 4294967296.0); 177 | } 178 | 179 | unsigned int ZRandom::getValue(const uint32_t MaxValue) { 180 | unsigned int used = MaxValue; 181 | used |= used >> 1; 182 | used |= used >> 2; 183 | used |= used >> 4; 184 | used |= used >> 8; 185 | used |= used >> 16; 186 | 187 | // Draw numbers until one is found in [0,n] 188 | unsigned int i; 189 | do 190 | i = getValue() & used; // toss unused bits to shorten search 191 | while (i > MaxValue); 192 | return i; 193 | } 194 | 195 | unsigned int ZRandom::getValue() { 196 | if (0 == nValues) { 197 | MT[N] = MT[0]; 198 | for (int i = 0; i < N; ++i) { 199 | unsigned y = (0x80000000 & MT[i]) | (0x7FFFFFFF & MT[i + 1]); 200 | unsigned v = *(map[i]) ^ (y >> 1); 201 | if (1 & y) 202 | v ^= 2567483615; 203 | MT[i] = v; 204 | } 205 | nValues = N; 206 | } 207 | unsigned y = MT[N - nValues--]; 208 | y ^= y >> 11; 209 | y ^= static_cast((y << 7) & 2636928640); 210 | y ^= static_cast((y << 15) & 4022730752); 211 | y ^= y >> 18; 212 | return y; 213 | } 214 | 215 | class UniformDataGenerator { 216 | public: 217 | UniformDataGenerator(uint32_t seed = static_cast(time(NULL))) 218 | : rand(seed) {} 219 | 220 | void negate(vector &in, vector &out, uint32_t Max) { 221 | out.resize(Max - in.size()); 222 | in.push_back(Max); 223 | uint32_t i = 0; 224 | size_t c = 0; 225 | for (size_t j = 0; j < in.size(); ++j) { 226 | const uint32_t v = in[j]; 227 | for (; i < v; ++i) 228 | out[c++] = i; 229 | ++i; 230 | } 231 | assert(c == out.size()); 232 | } 233 | 234 | /** 235 | * fill the vector with N numbers uniformly picked from from 0 to Max, not 236 | * including Max 237 | * if it is not possible, an exception is thrown 238 | */ 239 | vector generateUniformHash(uint32_t N, uint32_t Max, 240 | vector &ans) { 241 | if (Max < N) 242 | throw runtime_error( 243 | "can't generate enough distinct elements in small interval"); 244 | ans.clear(); 245 | if (N == 0) 246 | return ans; // nothing to do 247 | ans.reserve(N); 248 | assert(Max >= 1); 249 | unordered_set s; 250 | while (s.size() < N) 251 | s.insert(rand.getValue(Max - 1)); 252 | ans.assign(s.begin(), s.end()); 253 | sort(ans.begin(), ans.end()); 254 | assert(N == ans.size()); 255 | return ans; 256 | } 257 | 258 | void generateUniformBitmap(uint32_t N, uint32_t Max, vector &ans) { 259 | if (Max < N) 260 | throw runtime_error( 261 | "can't generate enough distinct elements in small interval"); 262 | assert(Max >= 1); 263 | BoolArray bs(Max); 264 | uint32_t card = 0; 265 | while (card < N) { 266 | uint32_t v = rand.getValue(Max - 1); 267 | if (!bs.get(v)) { 268 | bs.set(v); 269 | ++card; 270 | } 271 | } 272 | ans.resize(N); 273 | bs.toArray(ans); 274 | } 275 | 276 | void fastgenerateUniform(uint32_t N, uint32_t Max, vector &ans) { 277 | if (2 * N > Max) { 278 | vector buf(N); 279 | fastgenerateUniform(Max - N, Max, buf); 280 | negate(buf, ans, Max); 281 | return; 282 | } 283 | if (N * 1024 > Max) { 284 | generateUniformBitmap(N, Max, ans); 285 | } 286 | generateUniformHash(N, Max, ans); 287 | } 288 | 289 | // Max value is excluded from range 290 | vector generate(uint32_t N, uint32_t Max) { 291 | vector ans; 292 | ans.reserve(N); 293 | fastgenerateUniform(N, Max, ans); 294 | return ans; 295 | } 296 | ZRandom rand; 297 | }; 298 | 299 | /* 300 | * Reference: Vo Ngoc Anh and Alistair Moffat. 2010. Index compression using 301 | * 64-bit words. Softw. Pract. Exper.40, 2 (February 2010), 131-147. 302 | */ 303 | class ClusteredDataGenerator { 304 | public: 305 | vector buffer; 306 | UniformDataGenerator unidg; 307 | ClusteredDataGenerator(uint32_t seed = static_cast(time(NULL))) 308 | : buffer(), unidg(seed) {} 309 | 310 | // Max value is excluded from range 311 | template 312 | void fillUniform(iterator begin, iterator end, uint32_t Min, uint32_t Max) { 313 | unidg.fastgenerateUniform(static_cast(end - begin), Max - Min, 314 | buffer); 315 | for (size_t k = 0; k < buffer.size(); ++k) 316 | *(begin + k) = Min + buffer[k]; 317 | } 318 | 319 | // Max value is excluded from range 320 | // throws exception if impossible 321 | template 322 | void fillClustered(iterator begin, iterator end, uint32_t Min, uint32_t Max) { 323 | const uint32_t N = static_cast(end - begin); 324 | const uint32_t range = Max - Min; 325 | if (range < N) 326 | throw runtime_error("can't generate that many in small interval."); 327 | assert(range >= N); 328 | if ((range == N) or (N < 10)) { 329 | fillUniform(begin, end, Min, Max); 330 | return; 331 | } 332 | const uint32_t cut = N / 2 + unidg.rand.getValue(range - N); 333 | assert(cut >= N / 2); 334 | assert(Max - Min - cut >= N - N / 2); 335 | const double p = unidg.rand.getDouble(); 336 | assert(p <= 1); 337 | assert(p >= 0); 338 | if (p <= 0.25) { 339 | fillUniform(begin, begin + N / 2, Min, Min + cut); 340 | fillClustered(begin + N / 2, end, Min + cut, Max); 341 | } else if (p <= 0.5) { 342 | fillClustered(begin, begin + N / 2, Min, Min + cut); 343 | fillUniform(begin + N / 2, end, Min + cut, Max); 344 | } else { 345 | fillClustered(begin, begin + N / 2, Min, Min + cut); 346 | fillClustered(begin + N / 2, end, Min + cut, Max); 347 | } 348 | } 349 | 350 | // Max value is excluded from range 351 | vector generate(uint32_t N, uint32_t Max) { 352 | return generateClustered(N, Max); 353 | } 354 | 355 | // Max value is excluded from range 356 | vector generateClustered(uint32_t N, uint32_t Max) { 357 | vector ans(N); 358 | fillClustered(ans.begin(), ans.end(), 0, Max); 359 | return ans; 360 | } 361 | }; 362 | 363 | #endif 364 | -------------------------------------------------------------------------------- /synthetic/gen.cpp: -------------------------------------------------------------------------------- 1 | #define _DEFAULT_SOURCE 2 | #define _BSD_SOURCE 3 | #define _XOPEN_SOURCE 10000 4 | #define _POSIX_C_SOURCE 200809L 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "anh_moffat_clustered.h" 13 | void printusage(const char *command) { 14 | printf(" %s -N number -M maxval \n", command); 15 | } 16 | int main(int argc, char ** argv) { 17 | 18 | int c; 19 | uint32_t N = 0; 20 | uint32_t M = 0; 21 | 22 | while ((c = getopt(argc, argv, "hN:M:")) != -1) 23 | switch (c) { 24 | case 'h': 25 | printusage(argv[0]); 26 | return EXIT_SUCCESS; 27 | case 'N': 28 | N = atoi(optarg); 29 | break; 30 | case 'M': 31 | M = atoi(optarg); 32 | break; 33 | default: 34 | abort(); 35 | } 36 | if(N > M) { 37 | printf("N = %u M = %u \n",N,M); 38 | printf(" N must be smaller than M! \n"); 39 | return EXIT_FAILURE; 40 | } 41 | if(N == 0) { 42 | printf("N = %u M = %u \n",N,M); 43 | printf(" N must be greater than zero! \n"); 44 | return EXIT_FAILURE; 45 | } 46 | ClusteredDataGenerator cdg; 47 | 48 | vector v = cdg.generate(N, M); 49 | for(size_t i = 0; i < v.size(); i++) { 50 | std::cout << v[i]; 51 | if(i + 1 < v.size()) std::cout<<","; 52 | std::cout.flush(); 53 | } 54 | return EXIT_SUCCESS; 55 | } 56 | --------------------------------------------------------------------------------