├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── RoaringBitmap.Benchmark ├── App.config ├── DataSets.cs ├── MicroBenchmarks │ ├── MicroBenchmark.cs │ ├── MicroBenchmarkCensus1881.cs │ ├── MicroBenchmarkCensus1881Srt.cs │ ├── MicroBenchmarkCensusIncome.cs │ ├── MicroBenchmarkCensusIncomeSrt.cs │ ├── MicroBenchmarkDimension003.cs │ ├── MicroBenchmarkDimension008.cs │ ├── MicroBenchmarkDimension033.cs │ ├── MicroBenchmarkUsCensus2000.cs │ ├── MicroBenchmarkWeatherSept85.cs │ ├── MicroBenchmarkWeatherSept85Srt.cs │ ├── MicroBenchmarkWikileaksNoQuotes.cs │ └── MicroBenchmarkWikileaksNoQuotesSrt.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── RoaringBitmap.Benchmark.csproj ├── ZipRealDataProvider.cs └── packages.config ├── RoaringBitmap.Tests ├── BenchmarkTests.cs ├── Properties │ └── AssemblyInfo.cs ├── RoaringBitmap.Tests.csproj ├── RoaringBitmapTests.cs ├── app.config ├── bitmapwithoutruns.bin ├── bitmapwithruns.bin └── packages.config ├── RoaringBitmap.sln ├── RoaringBitmap.sln.DotSettings ├── RoaringBitmap ├── ArrayContainer.cs ├── BitmapContainer.cs ├── Container.cs ├── Properties │ └── AssemblyInfo.cs ├── RoaringArray.cs ├── RoaringBitmap.cs ├── RoaringBitmap.csproj └── Util.cs └── real-roaring-dataset ├── README ├── census-income.zip ├── census-income_srt.zip ├── census1881.zip ├── census1881_srt.zip ├── dimension_003.zip ├── dimension_008.zip ├── dimension_033.zip ├── uscensus2000.zip ├── weather_sept_85.zip ├── weather_sept_85_srt.zip ├── wikileaks-noquotes.zip └── wikileaks-noquotes_srt.zip /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | build/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studo 2015 cache/options directory 26 | .vs/ 27 | 28 | # MSTest test Results 29 | [Tt]est[Rr]esult*/ 30 | [Bb]uild[Ll]og.* 31 | 32 | # NUNIT 33 | *.VisualState.xml 34 | TestResult.xml 35 | 36 | # Build Results of an ATL Project 37 | [Dd]ebugPS/ 38 | [Rr]eleasePS/ 39 | dlldata.c 40 | 41 | *_i.c 42 | *_p.c 43 | *_i.h 44 | *.ilk 45 | *.meta 46 | *.obj 47 | *.pch 48 | *.pdb 49 | *.pgc 50 | *.pgd 51 | *.rsp 52 | *.sbr 53 | *.tlb 54 | *.tli 55 | *.tlh 56 | *.tmp 57 | *.tmp_proj 58 | *.log 59 | *.vspscc 60 | *.vssscc 61 | .builds 62 | *.pidb 63 | *.svclog 64 | *.scc 65 | 66 | # Chutzpah Test files 67 | _Chutzpah* 68 | 69 | # Visual C++ cache files 70 | ipch/ 71 | *.aps 72 | *.ncb 73 | *.opensdf 74 | *.sdf 75 | *.cachefile 76 | 77 | # Visual Studio profiler 78 | *.psess 79 | *.vsp 80 | *.vspx 81 | 82 | # TFS 2012 Local Workspace 83 | $tf/ 84 | 85 | # Guidance Automation Toolkit 86 | *.gpState 87 | 88 | # ReSharper is a .NET coding add-in 89 | _ReSharper*/ 90 | *.[Rr]e[Ss]harper 91 | *.DotSettings.user 92 | 93 | # JustCode is a .NET coding addin-in 94 | .JustCode 95 | 96 | # TeamCity is a build add-in 97 | _TeamCity* 98 | 99 | # DotCover is a Code Coverage Tool 100 | *.dotCover 101 | 102 | # NCrunch 103 | _NCrunch_* 104 | .*crunch*.local.xml 105 | 106 | # MightyMoose 107 | *.mm.* 108 | AutoTest.Net/ 109 | 110 | # Web workbench (sass) 111 | .sass-cache/ 112 | 113 | # Installshield output folder 114 | [Ee]xpress/ 115 | 116 | # DocProject is a documentation generator add-in 117 | DocProject/buildhelp/ 118 | DocProject/Help/*.HxT 119 | DocProject/Help/*.HxC 120 | DocProject/Help/*.hhc 121 | DocProject/Help/*.hhk 122 | DocProject/Help/*.hhp 123 | DocProject/Help/Html2 124 | DocProject/Help/html 125 | 126 | # Click-Once directory 127 | publish/ 128 | 129 | # Publish Web Output 130 | *.[Pp]ublish.xml 131 | *.azurePubxml 132 | # TODO: Comment the next line if you want to checkin your web deploy settings 133 | # but database connection strings (with potential passwords) will be unencrypted 134 | *.pubxml 135 | *.publishproj 136 | 137 | # NuGet Packages 138 | *.nupkg 139 | # The packages folder can be ignored because of Package Restore 140 | **/packages/* 141 | # except build/, which is used as an MSBuild target. 142 | !**/packages/build/ 143 | # Uncomment if necessary however generally it will be regenerated when needed 144 | #!**/packages/repositories.config 145 | 146 | # Windows Azure Build Output 147 | csx/ 148 | *.build.csdef 149 | 150 | # Windows Store app package directory 151 | AppPackages/ 152 | 153 | # Others 154 | *.[Cc]ache 155 | ClientBin/ 156 | [Ss]tyle[Cc]op.* 157 | ~$* 158 | *~ 159 | *.dbmdl 160 | *.dbproj.schemaview 161 | *.pfx 162 | *.publishsettings 163 | node_modules/ 164 | bower_components/ 165 | 166 | # RIA/Silverlight projects 167 | Generated_Code/ 168 | 169 | # Backup & report files from converting an old project file 170 | # to a newer Visual Studio version. Backup files are not needed, 171 | # because we have git ;-) 172 | _UpgradeReport_Files/ 173 | Backup*/ 174 | UpgradeLog*.XML 175 | UpgradeLog*.htm 176 | 177 | # SQL Server files 178 | *.mdf 179 | *.ldf 180 | 181 | # Business Intelligence projects 182 | *.rdl.data 183 | *.bim.layout 184 | *.bim_*.settings 185 | 186 | # Microsoft Fakes 187 | FakesAssemblies/ 188 | 189 | # Node.js Tools for Visual Studio 190 | .ntvs_analysis.dat 191 | 192 | # Visual Studio 6 build log 193 | *.plg 194 | 195 | # Visual Studio 6 workspace options file 196 | *.opt 197 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RoaringBitmap for .NET 2 | This is a fairly simple port of the Java bitmap library RoaringBitmap by Daniel Lemire et al. 3 | You can find the original version here: https://github.com/lemire/RoaringBitmap 4 | Most of the algorithms are ports of the original Java algorithms. 5 | This is an early version, the test coverage is ok, but edge case tests are probably missing. 6 | Target Framework is .NET 4.6 7 | 8 | # Details 9 | * Immutable data structure using readonly fields and private constructors, so it's thread-safe 10 | * Overloaded operators for AND, OR, NOT and XOR 11 | * Support for the Set Difference Operator using RoaringBitmap.AndNot 12 | 13 | # NuGet 14 | https://www.nuget.org/packages/RoaringBitmap/ 15 | 16 | # TODO 17 | * Add RunContainer support 18 | 19 | # How to use it? 20 | Compile the RoaringBitmap.sln and use 'RoaringBitmap.Create' to create your bitmap, then use bitwise operations on it. 21 | ```csharp 22 | var rb = RoaringBitmap.Create(1,2,3,4,5,7,8,9,10); 23 | var rb2 = RoaringBitmap.Create(Enumerable.Range(10000,200000)); 24 | var rb3 = rb | rb2; 25 | ``` 26 | 27 | # Performance 28 | As this is a fairly direct port of the immutable part of the Java Version of RoaringBitmap we can directly compare the Java Benchmark with this version, both benchmarks are equivalent and are also used for the Unit Testing to make sure they are actually doing the same. 29 | The CensusIncome dataset shows slightly faster performance numbers than the Java version. 30 | The Census1881 dataset is, except for AND, quite a bit faster than the Java version. 31 | 32 | ``` ini 33 | 34 | BenchmarkDotNet=v0.10.1, OS=Microsoft Windows NT 6.2.9200.0 35 | Processor=Intel(R) Core(TM) i7-4790K CPU 4.00GHz, ProcessorCount=8 36 | Frequency=3906246 Hz, Resolution=256.0003 ns, Timer=TSC 37 | [Host] : Clr 4.0.30319.42000, 64bit RyuJIT-v4.6.1586.0 38 | DefaultJob : Clr 4.0.30319.42000, 64bit RyuJIT-v4.6.1586.0 39 | 40 | Type=MicroBenchmarkCensusIncome Mode=Throughput 41 | ``` 42 | 43 | | Method | Mean | StdDev | Gen 0 | Allocated | 44 | | ------- | ---------- | --------- | -------- | --------- | 45 | | Or | 2.2688 ms | 0.0042 ms | 846.8750 | 3.89 MB | 46 | | Xor | 2.3919 ms | 0.0073 ms | 872.9167 | 3.96 MB | 47 | | And | 2.2721 ms | 0.0031 ms | 233.3333 | 1.32 MB | 48 | | AndNot | 2.3636 ms | 0.0040 ms | 517.1875 | 2.49 MB | 49 | | Iterate | 89.4243 ms | 0.0203 ms | - | 43.52 kB | 50 | 51 | ``` ini 52 | 53 | BenchmarkDotNet=v0.10.1, OS=Microsoft Windows NT 6.2.9200.0 54 | Processor=Intel(R) Core(TM) i7-4790K CPU 4.00GHz, ProcessorCount=8 55 | Frequency=3906246 Hz, Resolution=256.0003 ns, Timer=TSC 56 | [Host] : Clr 4.0.30319.42000, 64bit RyuJIT-v4.6.1586.0 57 | DefaultJob : Clr 4.0.30319.42000, 64bit RyuJIT-v4.6.1586.0 58 | 59 | Type=MicroBenchmarkCensus1881 Mode=Throughput 60 | ``` 61 | 62 | | Method | Mean | StdDev | Gen 0 | Gen 1 | Allocated | 63 | | ------- | -------------- | --------- | ------- | ------- | --------- | 64 | | Or | 234.7337 us | 0.1643 us | 82.4219 | - | 392.01 kB | 65 | | Xor | 194.7747 us | 3.3360 us | 88.0859 | - | 392.01 kB | 66 | | And | 65.8376 us | 0.3064 us | 5.7617 | - | 35.42 kB | 67 | | AndNot | 156.3676 us | 0.0581 us | 73.4375 | 10.4167 | 330.47 kB | 68 | | Iterate | 10,166.3848 us | 7.7011 us | - | - | 68.35 kB | 69 | 70 | 71 | The Java output from the recent 0.6.32-SNAPSHOT version looks like this: 72 | ``` ini 73 | # JMH 1.17 (released 47 days ago) 74 | # VM version: JDK 1.8.0_112, VM 25.112-b15 75 | # VM invoker: C:\Program Files\Java\jre1.8.0_112\bin\java.exe 76 | # VM options: -DBITMAP_TYPES=ROARING_ONLY 77 | # Warmup: 5 iterations, 1 s each 78 | # Measurement: 5 iterations, 1 s each 79 | # Timeout: 10 min per iteration 80 | # Threads: 1 thread, will synchronize iterations 81 | # Benchmark mode: Average time, time/op 82 | ``` 83 | 84 | | Method | Dataset | Score / Error | Units | 85 | | ------ | ------------- | ------------------ | ----- | 86 | | And | census-income | 2251,305 ± 34,711 | us/op | 87 | | And | census1881 | 53,618 ± 0,603 | us/op | 88 | | AndNot | census-income | 2829,971 ± 5,168 | us/op | 89 | | AndNot | census1881 | 474,517 ± 6,497 | us/op | 90 | | Or | census-income | 2916,778 ± 3,628 | us/op | 91 | | Or | census1881 | 748,054 ± 15,839 | us/op | 92 | | Xor | census-income | 2845,861 ± 18,825 | us/op | 93 | | Xor | census1881 | 811,914 ± 1,656 | us/op | 94 | -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/DataSets.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark 2 | { 3 | public class DataSets 4 | { 5 | public const string CensusIncome = "census-income.zip"; 6 | public const string Census1881 = "census1881.zip"; 7 | public const string Dimension003 = "dimension_003.zip"; 8 | public const string Dimension008 = "dimension_008.zip"; 9 | public const string Dimension033 = "dimension_033.zip"; 10 | public const string UsCensus2000 = "uscensus2000.zip"; 11 | public const string WeatherSept85 = "weather_sept_85.zip"; 12 | public const string WikileaksNoQuotes = "wikileaks-noquotes.zip"; 13 | public const string CensusIncomeSrt = "census-income_srt.zip"; 14 | public const string Census1881Srt = "census1881_srt.zip"; 15 | public const string WeatherSept85Srt = "weather_sept_85_srt.zip"; 16 | public const string WikileaksNoQuotesSrt = "wikileaks-noquotes_srt.zip"; 17 | } 18 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmark.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Linq; 3 | using BenchmarkDotNet.Attributes; 4 | 5 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 6 | { 7 | public abstract class MicroBenchmark 8 | { 9 | private readonly Collections.Special.RoaringBitmap[] m_Bitmaps; 10 | 11 | protected MicroBenchmark(string fileName) 12 | { 13 | var m_Path = @"Data"; 14 | 15 | using (var provider = new ZipRealDataProvider(Path.Combine(m_Path, fileName))) 16 | { 17 | m_Bitmaps = provider.ToArray(); 18 | } 19 | } 20 | 21 | [Benchmark] 22 | public long Or() 23 | { 24 | var total = 0L; 25 | for (var k = 0; k < m_Bitmaps.Length - 1; k++) 26 | { 27 | total += (m_Bitmaps[k] | m_Bitmaps[k + 1]).Cardinality; 28 | } 29 | return total; 30 | } 31 | 32 | [Benchmark] 33 | public long Xor() 34 | { 35 | var total = 0L; 36 | for (var k = 0; k < m_Bitmaps.Length - 1; k++) 37 | { 38 | total += (m_Bitmaps[k] ^ m_Bitmaps[k + 1]).Cardinality; 39 | } 40 | return total; 41 | } 42 | 43 | [Benchmark] 44 | public long And() 45 | { 46 | var total = 0L; 47 | for (var k = 0; k < m_Bitmaps.Length - 1; k++) 48 | { 49 | total += (m_Bitmaps[k] & m_Bitmaps[k + 1]).Cardinality; 50 | } 51 | return total; 52 | } 53 | 54 | [Benchmark] 55 | public long AndNot() 56 | { 57 | var total = 0L; 58 | for (var k = 0; k < m_Bitmaps.Length - 1; k++) 59 | { 60 | total += Collections.Special.RoaringBitmap.AndNot(m_Bitmaps[k], m_Bitmaps[k + 1]).Cardinality; 61 | } 62 | return total; 63 | } 64 | 65 | 66 | [Benchmark] 67 | public long Iterate() 68 | { 69 | var total = 0L; 70 | foreach (var roaringBitmap in m_Bitmaps) 71 | { 72 | foreach (var @int in roaringBitmap) 73 | { 74 | unchecked 75 | { 76 | total += @int; 77 | } 78 | } 79 | } 80 | return total; 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkCensus1881.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkCensus1881 : MicroBenchmark 4 | { 5 | public MicroBenchmarkCensus1881() : base(DataSets.Census1881) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkCensus1881Srt.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkCensus1881Srt : MicroBenchmark 4 | { 5 | public MicroBenchmarkCensus1881Srt() : base(DataSets.Census1881Srt) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkCensusIncome.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkCensusIncome : MicroBenchmark 4 | { 5 | public MicroBenchmarkCensusIncome() : base(DataSets.CensusIncome) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkCensusIncomeSrt.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkCensusIncomeSrt : MicroBenchmark 4 | { 5 | public MicroBenchmarkCensusIncomeSrt() : base(DataSets.CensusIncomeSrt) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkDimension003.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkDimension003 : MicroBenchmark 4 | { 5 | public MicroBenchmarkDimension003() : base(DataSets.Dimension003) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkDimension008.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkDimension008 : MicroBenchmark 4 | { 5 | public MicroBenchmarkDimension008() : base(DataSets.Dimension008) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkDimension033.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkDimension033 : MicroBenchmark 4 | { 5 | public MicroBenchmarkDimension033() : base(DataSets.Dimension033) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkUsCensus2000.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkUsCensus2000 : MicroBenchmark 4 | { 5 | public MicroBenchmarkUsCensus2000() : base(DataSets.UsCensus2000) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkWeatherSept85.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkWeatherSept85 : MicroBenchmark 4 | { 5 | public MicroBenchmarkWeatherSept85() : base(DataSets.WeatherSept85) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkWeatherSept85Srt.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkWeatherSept85Srt : MicroBenchmark 4 | { 5 | public MicroBenchmarkWeatherSept85Srt() : base(DataSets.WeatherSept85Srt) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkWikileaksNoQuotes.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkWikileaksNoQuotes : MicroBenchmark 4 | { 5 | public MicroBenchmarkWikileaksNoQuotes() : base(DataSets.WikileaksNoQuotes) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/MicroBenchmarks/MicroBenchmarkWikileaksNoQuotesSrt.cs: -------------------------------------------------------------------------------- 1 | namespace RoaringBitmap.Benchmark.MicroBenchmarks 2 | { 3 | public class MicroBenchmarkWikileaksNoQuotesSrt : MicroBenchmark 4 | { 5 | public MicroBenchmarkWikileaksNoQuotesSrt() : base(DataSets.WikileaksNoQuotesSrt) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/Program.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using BenchmarkDotNet.Running; 3 | using RoaringBitmap.Benchmark.MicroBenchmarks; 4 | 5 | namespace RoaringBitmap.Benchmark 6 | { 7 | internal class Program 8 | { 9 | private static void Main(string[] args) 10 | { 11 | var types = typeof(MicroBenchmark).Assembly.GetTypes().Where(t => !t.IsAbstract && typeof(MicroBenchmark).IsAssignableFrom(t)).ToList(); 12 | //var types = new[] { typeof(MicroBenchmarkCensusIncome) }; 13 | foreach (var type in types) 14 | { 15 | BenchmarkRunner.Run(type); 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | 8 | [assembly: AssemblyTitle("RoaringBitmap.Benchmark")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("RoaringBitmap.Benchmark")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | 21 | [assembly: ComVisible(false)] 22 | 23 | // The following GUID is for the ID of the typelib if this project is exposed to COM 24 | 25 | [assembly: Guid("e3ebddca-bb6b-48b5-b00a-cf60b5fa4bfb")] 26 | 27 | // Version information for an assembly consists of the following four values: 28 | // 29 | // Major Version 30 | // Minor Version 31 | // Build Number 32 | // Revision 33 | // 34 | // You can specify all the values or you can default the Build and Revision Numbers 35 | // by using the '*' as shown below: 36 | // [assembly: AssemblyVersion("1.0.*")] 37 | 38 | [assembly: AssemblyVersion("1.0.0.0")] 39 | [assembly: AssemblyFileVersion("1.0.0.0")] -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/RoaringBitmap.Benchmark.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB} 8 | Exe 9 | Properties 10 | RoaringBitmap.Benchmark 11 | RoaringBitmap.Benchmark 12 | v4.6 13 | 512 14 | true 15 | 16 | 17 | 18 | 19 | 20 | AnyCPU 21 | true 22 | full 23 | false 24 | bin\Debug\ 25 | DEBUG;TRACE 26 | prompt 27 | 4 28 | false 29 | 30 | 31 | AnyCPU 32 | pdbonly 33 | true 34 | bin\Release\ 35 | TRACE 36 | prompt 37 | 4 38 | false 39 | 40 | 41 | 42 | ..\packages\BenchmarkDotNet.0.10.4\lib\net46\BenchmarkDotNet.dll 43 | 44 | 45 | ..\packages\BenchmarkDotNet.Core.0.10.4\lib\net46\BenchmarkDotNet.Core.dll 46 | 47 | 48 | ..\packages\BenchmarkDotNet.Diagnostics.Windows.0.10.4\lib\net46\BenchmarkDotNet.Diagnostics.Windows.dll 49 | 50 | 51 | ..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.10.4\lib\net46\BenchmarkDotNet.Toolchains.Roslyn.dll 52 | 53 | 54 | ..\packages\Microsoft.CodeAnalysis.Common.2.1.0\lib\netstandard1.3\Microsoft.CodeAnalysis.dll 55 | 56 | 57 | ..\packages\Microsoft.CodeAnalysis.CSharp.2.1.0\lib\netstandard1.3\Microsoft.CodeAnalysis.CSharp.dll 58 | 59 | 60 | ..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.41\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll 61 | True 62 | 63 | 64 | ..\packages\Microsoft.DotNet.InternalAbstractions.1.0.0\lib\net451\Microsoft.DotNet.InternalAbstractions.dll 65 | 66 | 67 | ..\packages\Microsoft.DotNet.PlatformAbstractions.1.1.1\lib\net451\Microsoft.DotNet.PlatformAbstractions.dll 68 | 69 | 70 | 71 | ..\packages\System.AppContext.4.3.0\lib\net46\System.AppContext.dll 72 | True 73 | 74 | 75 | ..\packages\System.Collections.Immutable.1.3.1\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll 76 | True 77 | 78 | 79 | 80 | ..\packages\System.Console.4.3.0\lib\net46\System.Console.dll 81 | True 82 | 83 | 84 | 85 | ..\packages\System.Diagnostics.FileVersionInfo.4.3.0\lib\net46\System.Diagnostics.FileVersionInfo.dll 86 | True 87 | 88 | 89 | ..\packages\System.Diagnostics.StackTrace.4.3.0\lib\net46\System.Diagnostics.StackTrace.dll 90 | True 91 | 92 | 93 | ..\packages\System.IO.Compression.4.3.0\lib\net46\System.IO.Compression.dll 94 | 95 | 96 | ..\packages\System.IO.FileSystem.4.3.0\lib\net46\System.IO.FileSystem.dll 97 | True 98 | 99 | 100 | ..\packages\System.IO.FileSystem.Primitives.4.3.0\lib\net46\System.IO.FileSystem.Primitives.dll 101 | True 102 | 103 | 104 | 105 | 106 | ..\packages\System.Reflection.Metadata.1.4.2\lib\portable-net45+win8\System.Reflection.Metadata.dll 107 | 108 | 109 | ..\packages\System.Security.Cryptography.Algorithms.4.3.0\lib\net46\System.Security.Cryptography.Algorithms.dll 110 | True 111 | 112 | 113 | ..\packages\System.Security.Cryptography.Encoding.4.3.0\lib\net46\System.Security.Cryptography.Encoding.dll 114 | True 115 | 116 | 117 | ..\packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll 118 | True 119 | 120 | 121 | ..\packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net46\System.Security.Cryptography.X509Certificates.dll 122 | True 123 | 124 | 125 | ..\packages\System.Text.Encoding.CodePages.4.3.0\lib\net46\System.Text.Encoding.CodePages.dll 126 | True 127 | 128 | 129 | ..\packages\System.Threading.Tasks.Extensions.4.3.0\lib\portable-net45+win8+wp8+wpa81\System.Threading.Tasks.Extensions.dll 130 | True 131 | 132 | 133 | ..\packages\System.Threading.Thread.4.3.0\lib\net46\System.Threading.Thread.dll 134 | True 135 | 136 | 137 | ..\packages\System.ValueTuple.4.3.0\lib\netstandard1.0\System.ValueTuple.dll 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | ..\packages\System.Xml.ReaderWriter.4.3.0\lib\net46\System.Xml.ReaderWriter.dll 147 | True 148 | 149 | 150 | ..\packages\System.Xml.XmlDocument.4.3.0\lib\net46\System.Xml.XmlDocument.dll 151 | True 152 | 153 | 154 | ..\packages\System.Xml.XPath.4.3.0\lib\net46\System.Xml.XPath.dll 155 | True 156 | 157 | 158 | ..\packages\System.Xml.XPath.XDocument.4.3.0\lib\net46\System.Xml.XPath.XDocument.dll 159 | True 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | Data\census-income.zip 184 | PreserveNewest 185 | 186 | 187 | Data\census-income_srt.zip 188 | PreserveNewest 189 | 190 | 191 | Data\census1881.zip 192 | PreserveNewest 193 | 194 | 195 | Data\census1881_srt.zip 196 | PreserveNewest 197 | 198 | 199 | Data\dimension_003.zip 200 | PreserveNewest 201 | 202 | 203 | Data\dimension_008.zip 204 | PreserveNewest 205 | 206 | 207 | Data\dimension_033.zip 208 | PreserveNewest 209 | 210 | 211 | Data\uscensus2000.zip 212 | PreserveNewest 213 | 214 | 215 | Data\weather_sept_85.zip 216 | PreserveNewest 217 | 218 | 219 | Data\weather_sept_85_srt.zip 220 | PreserveNewest 221 | 222 | 223 | Data\wikileaks-noquotes.zip 224 | PreserveNewest 225 | 226 | 227 | Data\wikileaks-noquotes_srt.zip 228 | PreserveNewest 229 | 230 | 231 | Designer 232 | 233 | 234 | Designer 235 | 236 | 237 | 238 | 239 | {2ee1be04-a8f5-4358-bf08-e417d3916b49} 240 | RoaringBitmap 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 252 | 253 | 254 | 255 | 262 | -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/ZipRealDataProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.IO.Compression; 6 | using System.Linq; 7 | 8 | namespace RoaringBitmap.Benchmark 9 | { 10 | public class ZipRealDataProvider : IEnumerable, IDisposable 11 | { 12 | private readonly ZipArchive m_Archive; 13 | 14 | public ZipRealDataProvider(string path) 15 | { 16 | var fs = File.OpenRead(path); 17 | m_Archive = new ZipArchive(fs, ZipArchiveMode.Read); 18 | } 19 | 20 | public void Dispose() 21 | { 22 | Dispose(true); 23 | GC.SuppressFinalize(this); 24 | } 25 | 26 | public IEnumerator GetEnumerator() 27 | { 28 | foreach (var zipArchiveEntry in m_Archive.Entries) 29 | { 30 | using (var stream = zipArchiveEntry.Open()) 31 | { 32 | using (var stringReader = new StreamReader(stream)) 33 | { 34 | var split = stringReader.ReadLine().Split(','); 35 | var values = split.Select(int.Parse).ToList(); 36 | var bitmap = Collections.Special.RoaringBitmap.Create(values); 37 | yield return bitmap.Optimize(); 38 | } 39 | } 40 | } 41 | } 42 | 43 | IEnumerator IEnumerable.GetEnumerator() 44 | { 45 | return GetEnumerator(); 46 | } 47 | 48 | ~ZipRealDataProvider() 49 | { 50 | Dispose(false); 51 | } 52 | 53 | protected virtual void Dispose(bool disposing) 54 | { 55 | if (disposing) 56 | { 57 | m_Archive.Dispose(); 58 | } 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /RoaringBitmap.Benchmark/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /RoaringBitmap.Tests/BenchmarkTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.IO; 3 | using System.Linq; 4 | using RoaringBitmap.Benchmark; 5 | using Xunit; 6 | using Xunit.Abstractions; 7 | 8 | namespace RoaringBitmap.Tests 9 | { 10 | public class BenchmarkTests : IClassFixture 11 | { 12 | private readonly BenchmarkTestsFixture m_Fixture; 13 | private readonly ITestOutputHelper m_OutputHelper; 14 | 15 | public BenchmarkTests(BenchmarkTestsFixture fixture, ITestOutputHelper outputHelper) 16 | { 17 | m_Fixture = fixture; 18 | m_OutputHelper = outputHelper; 19 | } 20 | 21 | [Theory] 22 | [InlineData(DataSets.CensusIncome, 12487395)] 23 | [InlineData(DataSets.Census1881, 2007691)] 24 | [InlineData(DataSets.Dimension003, 7733676)] 25 | [InlineData(DataSets.Dimension008, 5555233)] 26 | [InlineData(DataSets.Dimension033, 7579526)] 27 | [InlineData(DataSets.UsCensus2000, 11954)] 28 | [InlineData(DataSets.WeatherSept85, 24729002)] 29 | [InlineData(DataSets.WikileaksNoQuotes, 541893)] 30 | [InlineData(DataSets.CensusIncomeSrt, 11257282)] 31 | [InlineData(DataSets.Census1881Srt, 1360167)] 32 | [InlineData(DataSets.WeatherSept85Srt, 30863347)] 33 | [InlineData(DataSets.WikileaksNoQuotesSrt, 574463)] 34 | public void Or(string name, int value) 35 | { 36 | var bitmaps = m_Fixture.GetBitmaps(name); 37 | Assert.NotNull(bitmaps); 38 | var total = 0L; 39 | for (var k = 0; k < bitmaps.Length - 1; k++) 40 | { 41 | total += (bitmaps[k] | bitmaps[k + 1]).Cardinality; 42 | } 43 | Assert.Equal(value, total); 44 | } 45 | 46 | [Theory] 47 | [InlineData(DataSets.CensusIncome, 11241947)] 48 | [InlineData(DataSets.Census1881, 2007668)] 49 | [InlineData(DataSets.Dimension003, 7733676)] 50 | [InlineData(DataSets.Dimension008, 5442916)] 51 | [InlineData(DataSets.Dimension033, 7579526)] 52 | [InlineData(DataSets.UsCensus2000, 11954)] 53 | [InlineData(DataSets.WeatherSept85, 24086983)] 54 | [InlineData(DataSets.WikileaksNoQuotes, 538566)] 55 | [InlineData(DataSets.CensusIncomeSrt, 10329567)] 56 | [InlineData(DataSets.Census1881Srt, 1359961)] 57 | [InlineData(DataSets.WeatherSept85Srt, 29800358)] 58 | [InlineData(DataSets.WikileaksNoQuotesSrt, 574311)] 59 | public void Xor(string name, int value) 60 | { 61 | var bitmaps = m_Fixture.GetBitmaps(name); 62 | Assert.NotNull(bitmaps); 63 | var total = 0L; 64 | for (var k = 0; k < bitmaps.Length - 1; k++) 65 | { 66 | total += (bitmaps[k] ^ bitmaps[k + 1]).Cardinality; 67 | } 68 | Assert.Equal(value, total); 69 | } 70 | 71 | [Theory] 72 | [InlineData(DataSets.CensusIncome, 1245448)] 73 | [InlineData(DataSets.Census1881, 23)] 74 | [InlineData(DataSets.Dimension003, 0)] 75 | [InlineData(DataSets.Dimension008, 112317)] 76 | [InlineData(DataSets.Dimension033, 0)] 77 | [InlineData(DataSets.UsCensus2000, 0)] 78 | [InlineData(DataSets.WeatherSept85, 642019)] 79 | [InlineData(DataSets.WikileaksNoQuotes, 3327)] 80 | [InlineData(DataSets.CensusIncomeSrt, 927715)] 81 | [InlineData(DataSets.Census1881Srt, 206)] 82 | [InlineData(DataSets.WeatherSept85Srt, 1062989)] 83 | [InlineData(DataSets.WikileaksNoQuotesSrt, 152)] 84 | public void And(string name, int value) 85 | { 86 | var bitmaps = m_Fixture.GetBitmaps(name); 87 | Assert.NotNull(bitmaps); 88 | var total = 0L; 89 | for (var k = 0; k < bitmaps.Length - 1; k++) 90 | { 91 | total += (bitmaps[k] & bitmaps[k + 1]).Cardinality; 92 | } 93 | Assert.Equal(value, total); 94 | } 95 | 96 | [Theory] 97 | [InlineData(DataSets.CensusIncome, -942184551)] 98 | [InlineData(DataSets.Census1881, 246451066)] 99 | [InlineData(DataSets.Dimension003, -1287135055)] 100 | [InlineData(DataSets.Dimension008, -423436314)] 101 | [InlineData(DataSets.Dimension033, -1287135055)] 102 | [InlineData(DataSets.UsCensus2000, -1260727955)] 103 | [InlineData(DataSets.WeatherSept85, 644036874)] 104 | [InlineData(DataSets.WikileaksNoQuotes, 413846869)] 105 | [InlineData(DataSets.CensusIncomeSrt, -679313956)] 106 | [InlineData(DataSets.Census1881Srt, 445584405)] 107 | [InlineData(DataSets.WeatherSept85Srt, 1132748056)] 108 | [InlineData(DataSets.WikileaksNoQuotesSrt, 1921022163)] 109 | public void Iterate(string name, int value) 110 | { 111 | var bitmaps = m_Fixture.GetBitmaps(name); 112 | Assert.NotNull(bitmaps); 113 | var total = 0; 114 | foreach (var roaringBitmap in bitmaps) 115 | { 116 | foreach (var @int in roaringBitmap) 117 | { 118 | unchecked 119 | { 120 | total += @int; 121 | } 122 | } 123 | } 124 | Assert.Equal(value, total); 125 | } 126 | 127 | 128 | // The Dimension data sets are simply too slow 129 | [Theory] 130 | [InlineData(DataSets.CensusIncome)] 131 | [InlineData(DataSets.Census1881)] 132 | //[InlineData(DataSets.Dimension003)] 133 | //[InlineData(DataSets.Dimension008)] 134 | //[InlineData(DataSets.Dimension033)] 135 | [InlineData(DataSets.UsCensus2000)] 136 | [InlineData(DataSets.WeatherSept85)] 137 | [InlineData(DataSets.WikileaksNoQuotes)] 138 | [InlineData(DataSets.CensusIncomeSrt)] 139 | [InlineData(DataSets.Census1881Srt)] 140 | [InlineData(DataSets.WeatherSept85Srt)] 141 | [InlineData(DataSets.WikileaksNoQuotesSrt)] 142 | public void Not(string name) 143 | { 144 | var bitmaps = m_Fixture.GetBitmaps(name); 145 | Assert.NotNull(bitmaps); 146 | foreach (var roaringBitmap in bitmaps) 147 | { 148 | var doublenegated = ~~roaringBitmap; 149 | Assert.Equal(roaringBitmap, doublenegated); 150 | } 151 | } 152 | 153 | [Theory] 154 | [InlineData(DataSets.CensusIncome, 5666586)] 155 | [InlineData(DataSets.Census1881, 1003836)] 156 | [InlineData(DataSets.Dimension003, 3866831)] 157 | [InlineData(DataSets.Dimension008, 2721459)] 158 | [InlineData(DataSets.Dimension033, 3866842)] 159 | [InlineData(DataSets.UsCensus2000, 5970)] 160 | [InlineData(DataSets.WeatherSept85, 11960876)] 161 | [InlineData(DataSets.WikileaksNoQuotes, 271605)] 162 | [InlineData(DataSets.CensusIncomeSrt, 5164671)] 163 | [InlineData(DataSets.Census1881Srt, 679375)] 164 | [InlineData(DataSets.WeatherSept85Srt, 14935706)] 165 | [InlineData(DataSets.WikileaksNoQuotesSrt, 286904)] 166 | public void AndNot(string name, int value) 167 | { 168 | var bitmaps = m_Fixture.GetBitmaps(name); 169 | Assert.NotNull(bitmaps); 170 | var total = 0L; 171 | for (var k = 0; k < bitmaps.Length - 1; k++) 172 | { 173 | total += Collections.Special.RoaringBitmap.AndNot(bitmaps[k], bitmaps[k + 1]).Cardinality; 174 | } 175 | Assert.Equal(value, total); 176 | } 177 | 178 | [Theory] 179 | [InlineData(DataSets.CensusIncome)] 180 | [InlineData(DataSets.Census1881)] 181 | [InlineData(DataSets.Dimension003)] 182 | [InlineData(DataSets.Dimension008)] 183 | [InlineData(DataSets.Dimension033)] 184 | [InlineData(DataSets.UsCensus2000)] 185 | [InlineData(DataSets.WeatherSept85)] 186 | [InlineData(DataSets.WikileaksNoQuotes)] 187 | [InlineData(DataSets.CensusIncomeSrt)] 188 | [InlineData(DataSets.Census1881Srt)] 189 | [InlineData(DataSets.WeatherSept85Srt)] 190 | [InlineData(DataSets.WikileaksNoQuotesSrt)] 191 | public void SerializeDeserialize(string name) 192 | { 193 | var bitmaps = m_Fixture.GetBitmaps(name); 194 | Assert.NotNull(bitmaps); 195 | foreach (var roaringBitmap in bitmaps) 196 | { 197 | using (var ms = new MemoryStream()) 198 | { 199 | Collections.Special.RoaringBitmap.Serialize(roaringBitmap, ms); 200 | ms.Position = 0; 201 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 202 | Assert.Equal(roaringBitmap, rb2); 203 | } 204 | } 205 | } 206 | 207 | 208 | public class BenchmarkTestsFixture 209 | { 210 | private readonly Dictionary m_BitmapDictionary = new Dictionary(); 211 | private readonly string m_Path = @"Data"; 212 | 213 | public Collections.Special.RoaringBitmap[] GetBitmaps(string name) 214 | { 215 | Collections.Special.RoaringBitmap[] bitmaps; 216 | if (!m_BitmapDictionary.TryGetValue(name, out bitmaps)) 217 | { 218 | using (var provider = new ZipRealDataProvider(Path.Combine(m_Path, name))) 219 | { 220 | bitmaps = provider.ToArray(); 221 | m_BitmapDictionary[name] = bitmaps; 222 | } 223 | } 224 | return bitmaps; 225 | } 226 | } 227 | } 228 | } -------------------------------------------------------------------------------- /RoaringBitmap.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | 8 | [assembly: AssemblyTitle("RoaringBitmap.Tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("RoaringBitmap.Tests")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | 21 | [assembly: ComVisible(false)] 22 | 23 | // The following GUID is for the ID of the typelib if this project is exposed to COM 24 | 25 | [assembly: Guid("0b057e9e-776d-42e5-9d75-99a8732fd4e3")] 26 | 27 | // Version information for an assembly consists of the following four values: 28 | // 29 | // Major Version 30 | // Minor Version 31 | // Build Number 32 | // Revision 33 | // 34 | // You can specify all the values or you can default the Build and Revision Numbers 35 | // by using the '*' as shown below: 36 | // [assembly: AssemblyVersion("1.0.*")] 37 | 38 | [assembly: AssemblyVersion("1.0.0.0")] 39 | [assembly: AssemblyFileVersion("1.0.0.0")] -------------------------------------------------------------------------------- /RoaringBitmap.Tests/RoaringBitmap.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | Debug 7 | AnyCPU 8 | {0B057E9E-776D-42E5-9D75-99A8732FD4E3} 9 | Library 10 | Properties 11 | RoaringBitmap.Tests 12 | RoaringBitmap.Tests 13 | v4.6 14 | 512 15 | 16 | 17 | 18 | 19 | 20 | true 21 | full 22 | false 23 | bin\Debug\ 24 | DEBUG;TRACE 25 | prompt 26 | 4 27 | 28 | 29 | pdbonly 30 | true 31 | bin\Release\ 32 | TRACE 33 | prompt 34 | 4 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | ..\packages\xunit.abstractions.2.0.1\lib\net35\xunit.abstractions.dll 47 | 48 | 49 | ..\packages\xunit.assert.2.2.0\lib\netstandard1.1\xunit.assert.dll 50 | 51 | 52 | ..\packages\xunit.extensibility.core.2.2.0\lib\netstandard1.1\xunit.core.dll 53 | 54 | 55 | ..\packages\xunit.extensibility.execution.2.2.0\lib\net452\xunit.execution.desktop.dll 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | {e3ebddca-bb6b-48b5-b00a-cf60b5fa4bfb} 72 | RoaringBitmap.Benchmark 73 | 74 | 75 | {2ee1be04-a8f5-4358-bf08-e417d3916b49} 76 | RoaringBitmap 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 86 | 87 | 88 | 89 | 96 | -------------------------------------------------------------------------------- /RoaringBitmap.Tests/RoaringBitmapTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using Xunit; 6 | 7 | namespace RoaringBitmap.Tests 8 | { 9 | public class RoaringBitmapTests 10 | { 11 | private static Stream GetResourceStream(string name) 12 | { 13 | return typeof(RoaringBitmapTests).Assembly.GetManifestResourceStream(typeof(RoaringBitmapTests), name); 14 | } 15 | 16 | private static List CreateMixedListOne() 17 | { 18 | var list = new List(); 19 | var baseValue = 0x10000; 20 | for (var i = 0; i < 50; i++) 21 | { 22 | list.Add(i * 62); 23 | } 24 | 25 | for (var i = baseValue; i < baseValue + 100; i++) 26 | { 27 | list.Add(i); 28 | } 29 | for (var i = 2 * baseValue; i < 3 * baseValue; i++) 30 | { 31 | list.Add(i); 32 | } 33 | return list; 34 | } 35 | 36 | private static List CreateMixedListTwo() 37 | { 38 | var list = new List(); 39 | var baseValue = 0x10000; 40 | for (var i = 1; i < 50; i++) 41 | { 42 | list.Add(i * 65); 43 | } 44 | 45 | for (var i = baseValue + 100; i < baseValue + 200; i++) 46 | { 47 | list.Add(i); 48 | } 49 | for (var i = 3 * baseValue; i < 4 * baseValue; i++) 50 | { 51 | list.Add(i); 52 | } 53 | return list; 54 | } 55 | 56 | private static List CreateRandomList(Random random, int size) 57 | { 58 | var list = new List(); 59 | var type = random.Next() % 2; 60 | if (type == 0) 61 | { 62 | for (var i = 0; i < size; i++) 63 | { 64 | list.Add(random.Next()); 65 | } 66 | } 67 | else 68 | { 69 | var start = random.Next(0, int.MaxValue - size); 70 | for (var i = start; i < start + size; i++) 71 | { 72 | list.Add(i); 73 | } 74 | } 75 | return list; 76 | } 77 | 78 | [Theory] 79 | [InlineData("bitmapwithoutruns.bin")] 80 | [InlineData("bitmapwithruns.bin")] 81 | public void DeSerializeTestContainers(string name) 82 | { 83 | using (var reference = GetResourceStream(name)) 84 | { 85 | var items = new List(); 86 | for (var k = 0; k < 100000; k += 1000) 87 | { 88 | items.Add(k); 89 | } 90 | for (var k = 100000; k < 200000; ++k) 91 | { 92 | items.Add(3 * k); 93 | } 94 | for (var k = 700000; k < 800000; ++k) 95 | { 96 | items.Add(k); 97 | } 98 | var rb = Collections.Special.RoaringBitmap.Create(items); 99 | var deserialized = Collections.Special.RoaringBitmap.Deserialize(reference); 100 | Assert.Equal(rb, deserialized); 101 | } 102 | } 103 | 104 | [Fact] 105 | public void AndDisjunct() 106 | { 107 | var firstList = CreateMixedListOne(); 108 | var secondList = CreateMixedListTwo(); 109 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 110 | var rb2 = Collections.Special.RoaringBitmap.Create(secondList); 111 | var rb3 = rb & rb2; 112 | Assert.NotNull(rb3); 113 | var rbList = rb3.ToList(); 114 | Assert.Empty(rbList); 115 | var comparison = firstList.Intersect(secondList).OrderBy(t => t).ToList(); 116 | Assert.Equal(comparison, rbList); 117 | } 118 | 119 | [Fact] 120 | public void AndNotDisjunct() 121 | { 122 | var rb = Collections.Special.RoaringBitmap.Create(CreateMixedListOne()); 123 | var rb2 = Collections.Special.RoaringBitmap.Create(CreateMixedListTwo()); 124 | 125 | var rb3 = Collections.Special.RoaringBitmap.AndNot(rb, rb2); 126 | Assert.NotNull(rb3); 127 | Assert.Equal(rb.ToList(), rb3.ToList()); 128 | } 129 | 130 | [Fact] 131 | public void AndNotPartiallyArrayContainer() 132 | { 133 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 200)); 134 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1100, 400)); 135 | var rb3 = Collections.Special.RoaringBitmap.AndNot(rb, rb2); 136 | Assert.NotNull(rb3); 137 | var rbList = rb3.ToList(); 138 | Assert.Equal(Enumerable.Range(1000, 100), rbList); 139 | } 140 | 141 | 142 | [Fact] 143 | public void AndNotPartiallyBitmapContainerArrayContainerResult() 144 | { 145 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 5000)); 146 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 5000)); 147 | var rb3 = Collections.Special.RoaringBitmap.AndNot(rb, rb2); 148 | Assert.NotNull(rb3); 149 | var rbList = rb3.ToList(); 150 | Assert.Equal(Enumerable.Range(1000, 3000), rbList); 151 | } 152 | 153 | [Fact] 154 | public void AndNotPartiallyBitmapContainerBitmapContainerResult() 155 | { 156 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 10000)); 157 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 10000)); 158 | var rb3 = Collections.Special.RoaringBitmap.AndNot(rb, rb2); 159 | Assert.NotNull(rb3); 160 | var rbList = rb3.ToList(); 161 | Assert.Equal(Enumerable.Range(1000, 3000), rbList); 162 | } 163 | 164 | [Fact] 165 | public void AndPartiallyArrayContainer() 166 | { 167 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 200)); 168 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1100, 400)); 169 | var rb3 = rb & rb2; 170 | Assert.NotNull(rb3); 171 | var rbList = rb3.ToList(); 172 | Assert.Equal(Enumerable.Range(1100, 100), rbList); 173 | } 174 | 175 | [Fact] 176 | public void AndPartiallyBitmapContainerArrayContainerResult() 177 | { 178 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 5000)); 179 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 5000)); 180 | var rb3 = rb & rb2; 181 | Assert.NotNull(rb3); 182 | var rbList = rb3.ToList(); 183 | Assert.Equal(Enumerable.Range(4000, 2000), rbList); 184 | } 185 | 186 | [Fact] 187 | public void AndPartiallyBitmapContainerBitmapContainerResult() 188 | { 189 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 10000)); 190 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 10000)); 191 | var rb3 = rb & rb2; 192 | Assert.NotNull(rb3); 193 | var rbList = rb3.ToList(); 194 | Assert.Equal(Enumerable.Range(4000, 7000), rbList); 195 | } 196 | 197 | [Fact] 198 | public void AndPartiallySameMixedContainer() 199 | { 200 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 4000)); 201 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 10000)); 202 | var rb3 = rb & rb2; 203 | Assert.NotNull(rb3); 204 | var rbList = rb3.ToList(); 205 | Assert.Equal(Enumerable.Range(4000, 1000), rbList); 206 | } 207 | 208 | 209 | [Fact] 210 | public void AndRandom() 211 | { 212 | var random = new Random(); 213 | var lists = new List>(); 214 | var firstList = CreateRandomList(random, 10000); 215 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 216 | lists.Add(firstList); 217 | for (var i = 0; i < 10; i++) 218 | { 219 | var nextList = CreateRandomList(random, 10000); 220 | lists.Add(nextList); 221 | rb &= Collections.Special.RoaringBitmap.Create(nextList); 222 | } 223 | var comparison = lists.Skip(1).Aggregate(new HashSet(lists.First()), 224 | (h, e) => 225 | { 226 | h.IntersectWith(e); 227 | return h; 228 | }); 229 | var rbList = rb.ToList(); 230 | Assert.Equal(comparison, rbList); 231 | } 232 | 233 | [Fact] 234 | public void AndSame() 235 | { 236 | var list = CreateMixedListOne(); 237 | var rb = Collections.Special.RoaringBitmap.Create(list); 238 | var rb2 = rb & rb; 239 | Assert.NotNull(rb2); 240 | var rbList = rb2.ToList(); 241 | Assert.Equal(list, rbList); 242 | } 243 | 244 | [Fact] 245 | public void BasicCreate() 246 | { 247 | var rb = Collections.Special.RoaringBitmap.Create(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); 248 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1, 10)); 249 | Assert.Equal(rb, rb2); 250 | } 251 | 252 | [Fact] 253 | public void CardinalityOfEmptySet() 254 | { 255 | var full = Collections.Special.RoaringBitmap.Create(); 256 | 257 | Assert.Equal(0, full.Cardinality); 258 | } 259 | 260 | [Fact] 261 | public void CardinalityOfFullSet() 262 | { 263 | var full = ~Collections.Special.RoaringBitmap.Create(); 264 | 265 | Assert.Equal(1L << 32, full.Cardinality); 266 | } 267 | 268 | [Fact] 269 | public void Equal() 270 | { 271 | var list = CreateMixedListOne(); 272 | var rb = Collections.Special.RoaringBitmap.Create(list); 273 | var rb2 = Collections.Special.RoaringBitmap.Create(list); 274 | Assert.Equal(rb, rb2); 275 | Assert.Equal(rb.GetHashCode(), rb2.GetHashCode()); 276 | } 277 | 278 | [Fact] 279 | public void LargeArray() 280 | { 281 | var list = CreateMixedListOne(); 282 | var rb = Collections.Special.RoaringBitmap.Create(list); 283 | var rbList = rb.ToList(); 284 | Assert.Equal(list, rbList); 285 | } 286 | 287 | [Fact] 288 | public void MaxArray() 289 | { 290 | var list = Enumerable.Range(0, 4097).ToList(); 291 | var rb = Collections.Special.RoaringBitmap.Create(list); 292 | var rbList = rb.ToList(); 293 | Assert.Equal(list, rbList); 294 | } 295 | 296 | [Fact] 297 | public void Not() 298 | { 299 | var list = CreateMixedListOne(); 300 | var rb = Collections.Special.RoaringBitmap.Create(list); 301 | var rb2 = ~rb; 302 | var rb3 = ~rb2; 303 | 304 | var values = rb3.ToList(); 305 | Assert.Equal(values, list); 306 | } 307 | 308 | [Fact] 309 | public void NotEqual() 310 | { 311 | var rb = Collections.Special.RoaringBitmap.Create(CreateMixedListOne()); 312 | var rb2 = Collections.Special.RoaringBitmap.Create(CreateMixedListTwo()); 313 | Assert.NotEqual(rb, rb2); 314 | Assert.NotEqual(rb.GetHashCode(), rb2.GetHashCode()); 315 | } 316 | 317 | [Fact] 318 | public void OptimizeFullSetArrayContainer() 319 | { 320 | var full = Collections.Special.RoaringBitmap.Create(Enumerable.Range(0, 4096)); 321 | var fullOptimized = full.Optimize(); 322 | Assert.NotNull(fullOptimized); 323 | Assert.False(ReferenceEquals(full, fullOptimized)); 324 | Assert.Equal(Enumerable.Range(0, 4096), fullOptimized.ToList()); 325 | } 326 | 327 | [Fact] 328 | public void OptimizeFullSetBitmapContainer() 329 | { 330 | var full = ~Collections.Special.RoaringBitmap.Create(); 331 | var fullOptimized = full.Optimize(); 332 | Assert.NotNull(fullOptimized); 333 | Assert.False(ReferenceEquals(full, fullOptimized)); 334 | var empty = ~fullOptimized; 335 | var emptyList = empty.ToList(); 336 | Assert.Empty(emptyList); 337 | } 338 | 339 | [Fact] 340 | public void OrDisjunct() 341 | { 342 | var firstList = CreateMixedListOne(); 343 | var secondList = CreateMixedListTwo(); 344 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 345 | var rb2 = Collections.Special.RoaringBitmap.Create(secondList); 346 | var rb3 = rb | rb2; 347 | Assert.NotNull(rb3); 348 | var rbList = rb3.ToList(); 349 | var comparison = firstList.Union(secondList).OrderBy(t => t).ToList(); 350 | Assert.Equal(comparison, rbList); 351 | } 352 | 353 | [Fact] 354 | public void OrRandom() 355 | { 356 | var random = new Random(4); 357 | var lists = new List>(); 358 | var firstList = CreateRandomList(random, 10000); 359 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 360 | lists.Add(firstList); 361 | var nextList = CreateRandomList(random, 10000); 362 | lists.Add(nextList); 363 | rb |= Collections.Special.RoaringBitmap.Create(nextList); 364 | var comparison = lists.SelectMany(t => t).Distinct().OrderBy(t => t).ToList(); 365 | var rbList = rb.ToList(); 366 | Assert.Equal(comparison, rbList); 367 | } 368 | 369 | [Fact] 370 | public void OrSame() 371 | { 372 | var list = CreateMixedListOne(); 373 | var rb = Collections.Special.RoaringBitmap.Create(list); 374 | var rb2 = rb | rb; 375 | Assert.NotNull(rb2); 376 | var rbList = rb2.ToList(); 377 | Assert.Equal(list, rbList); 378 | } 379 | 380 | 381 | [Fact] 382 | public void SerializeDeserialize() 383 | { 384 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1, 100000)); 385 | using (var ms = new MemoryStream()) 386 | { 387 | Collections.Special.RoaringBitmap.Serialize(rb, ms); 388 | ms.Position = 0; 389 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 390 | Assert.Equal(rb, rb2); 391 | } 392 | } 393 | 394 | 395 | [Fact] 396 | public void SerializeEmptySet() 397 | { 398 | var full = Collections.Special.RoaringBitmap.Create(); 399 | 400 | using (var ms = new MemoryStream()) 401 | { 402 | Collections.Special.RoaringBitmap.Serialize(full, ms); 403 | ms.Position = 0; 404 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 405 | Assert.Equal(full, rb2); 406 | } 407 | } 408 | 409 | [Fact] 410 | public void SerializeFullArrayContainer() 411 | { 412 | var full = Collections.Special.RoaringBitmap.Create(Enumerable.Range(0, 4096)); 413 | 414 | using (var ms = new MemoryStream()) 415 | { 416 | Collections.Special.RoaringBitmap.Serialize(full, ms); 417 | ms.Position = 0; 418 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 419 | Assert.Equal(full, rb2); 420 | } 421 | } 422 | 423 | [Fact] 424 | public void SerializeFullBitContainer() 425 | { 426 | var full = Collections.Special.RoaringBitmap.Create(Enumerable.Range(0, 1 << 16)); 427 | 428 | using (var ms = new MemoryStream()) 429 | { 430 | Collections.Special.RoaringBitmap.Serialize(full, ms); 431 | ms.Position = 0; 432 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 433 | Assert.Equal(full, rb2); 434 | } 435 | } 436 | 437 | [Fact] 438 | public void SerializeFullSet() 439 | { 440 | var full = ~Collections.Special.RoaringBitmap.Create(); 441 | 442 | using (var ms = new MemoryStream()) 443 | { 444 | Collections.Special.RoaringBitmap.Serialize(full, ms); 445 | ms.Position = 0; 446 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 447 | Assert.Equal(full, rb2); 448 | } 449 | } 450 | 451 | [Fact] // the testdata container https://github.com/RoaringBitmap/RoaringFormatSpec/tree/master/testdata 452 | public void SerializeTestContainer() 453 | { 454 | var items = new List(); 455 | for (var k = 0; k < 100000; k += 1000) 456 | { 457 | items.Add(k); 458 | } 459 | for (var k = 100000; k < 200000; ++k) 460 | { 461 | items.Add(3 * k); 462 | } 463 | for (var k = 700000; k < 800000; ++k) 464 | { 465 | items.Add(k); 466 | } 467 | var rb = Collections.Special.RoaringBitmap.Create(items); 468 | using (var ms = new MemoryStream()) 469 | { 470 | Collections.Special.RoaringBitmap.Serialize(rb, ms); 471 | ms.Position = 0; 472 | var rb2 = Collections.Special.RoaringBitmap.Deserialize(ms); 473 | Assert.Equal(rb, rb2); 474 | } 475 | } 476 | 477 | [Fact] 478 | public void SmallArray() 479 | { 480 | var list = Enumerable.Range(0, 100).ToList(); 481 | var rb = Collections.Special.RoaringBitmap.Create(list); 482 | var rbList = rb.ToList(); 483 | Assert.Equal(list, rbList); 484 | } 485 | 486 | [Fact] 487 | public void XorDisjunct() 488 | { 489 | var firstList = CreateMixedListOne(); 490 | var secondList = CreateMixedListTwo(); 491 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 492 | var rb2 = Collections.Special.RoaringBitmap.Create(secondList); 493 | var rb3 = rb ^ rb2; 494 | Assert.NotNull(rb3); 495 | var rbList = rb3.ToList(); 496 | Assert.NotEmpty(rbList); 497 | var comparison = firstList.Union(secondList).OrderBy(t => t).ToList(); 498 | Assert.Equal(comparison, rbList); 499 | } 500 | 501 | 502 | [Fact] 503 | public void XorPartiallyArrayContainer() 504 | { 505 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 200)); 506 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1100, 400)); 507 | var rb3 = rb ^ rb2; 508 | Assert.NotNull(rb3); 509 | var rbList = rb3.ToList(); 510 | Assert.Equal(Enumerable.Range(1000, 100).Concat(Enumerable.Range(1200, 300)), rbList); 511 | } 512 | 513 | 514 | [Fact] 515 | public void XorPartiallyBitmapContainerArrayContainerResult() 516 | { 517 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 5000)); 518 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 5000)); 519 | var rb3 = rb ^ rb2; 520 | Assert.NotNull(rb3); 521 | var rbList = rb3.ToList(); 522 | Assert.Equal(Enumerable.Range(1000, 3000).Concat(Enumerable.Range(6000, 3000)), rbList); 523 | } 524 | 525 | [Fact] 526 | public void XorPartiallyBitmapContainerBitmapContainerResult() 527 | { 528 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 10000)); 529 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 10000)); 530 | var rb3 = rb ^ rb2; 531 | Assert.NotNull(rb3); 532 | var rbList = rb3.ToList(); 533 | Assert.Equal(Enumerable.Range(1000, 3000).Concat(Enumerable.Range(11000, 3000)), rbList); 534 | } 535 | 536 | [Fact] 537 | public void XorPartiallySameMixedContainer() 538 | { 539 | var rb = Collections.Special.RoaringBitmap.Create(Enumerable.Range(1000, 4000)); 540 | var rb2 = Collections.Special.RoaringBitmap.Create(Enumerable.Range(4000, 10000)); 541 | var rb3 = rb ^ rb2; 542 | Assert.NotNull(rb3); 543 | var rbList = rb3.ToList(); 544 | Assert.Equal(Enumerable.Range(1000, 3000).Concat(Enumerable.Range(5000, 9000)), rbList); 545 | } 546 | 547 | [Fact] 548 | public void XorRandom() 549 | { 550 | var random = new Random(); 551 | var firstList = CreateRandomList(random, 10000); 552 | var rb = Collections.Special.RoaringBitmap.Create(firstList); 553 | var nextList = CreateRandomList(random, 10000); 554 | rb ^= Collections.Special.RoaringBitmap.Create(nextList); 555 | var rbList = rb.ToList(); 556 | var comparison = firstList.Union(nextList).Except(firstList.Intersect(nextList)).OrderBy(t => t).ToList(); 557 | Assert.Equal(comparison, rbList); 558 | } 559 | 560 | [Fact] 561 | public void XorSame() 562 | { 563 | var list = CreateMixedListOne(); 564 | var rb = Collections.Special.RoaringBitmap.Create(list); 565 | var rb2 = rb ^ rb; 566 | var rbList = rb2.ToList(); 567 | Assert.Empty(rbList); 568 | } 569 | } 570 | } -------------------------------------------------------------------------------- /RoaringBitmap.Tests/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /RoaringBitmap.Tests/bitmapwithoutruns.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/RoaringBitmap.Tests/bitmapwithoutruns.bin -------------------------------------------------------------------------------- /RoaringBitmap.Tests/bitmapwithruns.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/RoaringBitmap.Tests/bitmapwithruns.bin -------------------------------------------------------------------------------- /RoaringBitmap.Tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /RoaringBitmap.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26114.2 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RoaringBitmap", "RoaringBitmap\RoaringBitmap.csproj", "{2EE1BE04-A8F5-4358-BF08-E417D3916B49}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RoaringBitmap.Tests", "RoaringBitmap.Tests\RoaringBitmap.Tests.csproj", "{0B057E9E-776D-42E5-9D75-99A8732FD4E3}" 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RoaringBitmap.Benchmark", "RoaringBitmap.Benchmark\RoaringBitmap.Benchmark.csproj", "{E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Release|Any CPU = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {2EE1BE04-A8F5-4358-BF08-E417D3916B49}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {2EE1BE04-A8F5-4358-BF08-E417D3916B49}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {2EE1BE04-A8F5-4358-BF08-E417D3916B49}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {2EE1BE04-A8F5-4358-BF08-E417D3916B49}.Release|Any CPU.Build.0 = Release|Any CPU 22 | {0B057E9E-776D-42E5-9D75-99A8732FD4E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {0B057E9E-776D-42E5-9D75-99A8732FD4E3}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {0B057E9E-776D-42E5-9D75-99A8732FD4E3}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {0B057E9E-776D-42E5-9D75-99A8732FD4E3}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {E3EBDDCA-BB6B-48B5-B00A-CF60B5FA4BFB}.Release|Any CPU.Build.0 = Release|Any CPU 30 | EndGlobalSection 31 | GlobalSection(SolutionProperties) = preSolution 32 | HideSolutionNode = FALSE 33 | EndGlobalSection 34 | EndGlobal 35 | -------------------------------------------------------------------------------- /RoaringBitmap.sln.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | HINT 3 | 4 | True 5 | True 6 | True 7 | True 8 | True 9 | True 10 | True 11 | True 12 | ALWAYS_ADD 13 | ALWAYS_ADD 14 | ALWAYS_ADD 15 | ALWAYS_ADD 16 | ALWAYS_ADD 17 | ALWAYS_ADD 18 | True 19 | 200 20 | $object$_On$event$ 21 | 22 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 23 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 24 | <Policy Inspect="True" Prefix="I" Suffix="" Style="AaBb" /> 25 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 26 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 27 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 28 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 29 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 30 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 31 | <Policy Inspect="True" Prefix="m_" Suffix="" Style="AaBb" /> 32 | <Policy Inspect="True" Prefix="s_" Suffix="" Style="AaBb" /> 33 | <Policy Inspect="True" Prefix="s_" Suffix="" Style="AaBb" /> 34 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 35 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 36 | <Policy Inspect="True" Prefix="T" Suffix="" Style="AaBb" /> 37 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 38 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Public" Description="Field: static (public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 39 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private, Protected, ProtectedInternal, Internal" Description="Field: non-public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="m_" Suffix="" Style="AaBb" /></Policy> 40 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Private, Protected, ProtectedInternal, Internal" Description="Field: Static non-public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="s_" Suffix="" Style="AaBb" /></Policy> 41 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Public" Description="Field: public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 42 | $object$_On$event$ 43 | 44 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 45 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 46 | <Policy Inspect="True" Prefix="I" Suffix="" Style="AaBb" /> 47 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 48 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 49 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 50 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 51 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 52 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 53 | <Policy Inspect="False" Prefix="m_" Suffix="" Style="AaBb" /> 54 | <Policy Inspect="True" Prefix="s_" Suffix="" Style="AaBb" /> 55 | <Policy Inspect="False" Prefix="s_" Suffix="" Style="AaBb" /> 56 | <Policy Inspect="False" Prefix="" Suffix="" Style="AaBb" /> 57 | <Policy Inspect="False" Prefix="" Suffix="" Style="AaBb" /> 58 | <Policy Inspect="True" Prefix="T" Suffix="" Style="AaBb" /> 59 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 60 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Public" Description="Field: public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 61 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Private, Protected, ProtectedInternal, Internal" Description="Field: Static non-public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="s_" Suffix="" Style="AaBb" /></Policy> 62 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Public" Description="Field: static (public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 63 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private, Protected, ProtectedInternal, Internal" Description="Field: non-public (read-only)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="m_" Suffix="" Style="AaBb" /></Policy> 64 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 65 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 66 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 67 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 68 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 69 | <Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /> 70 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 71 | True 72 | True 73 | True 74 | <data /> 75 | <data><IncludeFilters /><ExcludeFilters /></data> -------------------------------------------------------------------------------- /RoaringBitmap/ArrayContainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace Collections.Special 6 | { 7 | internal class ArrayContainer : Container, IEquatable 8 | { 9 | public static readonly ArrayContainer One; 10 | private readonly int m_Cardinality; 11 | private readonly ushort[] m_Content; 12 | 13 | static ArrayContainer() 14 | { 15 | var data = new ushort[MaxSize]; 16 | for (ushort i = 0; i < MaxSize; i++) 17 | { 18 | data[i] = i; 19 | } 20 | One = new ArrayContainer(MaxSize, data); 21 | } 22 | 23 | private ArrayContainer(int cardinality, ushort[] data) 24 | { 25 | m_Content = data; 26 | m_Cardinality = cardinality; 27 | } 28 | 29 | protected internal override int Cardinality => m_Cardinality; 30 | 31 | public override int ArraySizeInBytes => m_Cardinality * sizeof(ushort); 32 | 33 | 34 | public bool Equals(ArrayContainer other) 35 | { 36 | if (ReferenceEquals(this, other)) 37 | { 38 | return true; 39 | } 40 | if (ReferenceEquals(null, other)) 41 | { 42 | return false; 43 | } 44 | if (m_Cardinality != other.m_Cardinality) 45 | { 46 | return false; 47 | } 48 | for (var i = 0; i < m_Cardinality; i++) 49 | { 50 | if (m_Content[i] != other.m_Content[i]) 51 | { 52 | return false; 53 | } 54 | } 55 | return true; 56 | } 57 | 58 | internal static ArrayContainer Create(ushort[] values) 59 | { 60 | return new ArrayContainer(values.Length, values); 61 | } 62 | 63 | internal static ArrayContainer Create(BitmapContainer bc) 64 | { 65 | var data = new ushort[bc.Cardinality]; 66 | var cardinality = bc.FillArray(data); 67 | var result = new ArrayContainer(cardinality, data); 68 | return result; 69 | } 70 | 71 | protected override bool EqualsInternal(Container other) 72 | { 73 | var ac = other as ArrayContainer; 74 | return (ac != null) && Equals(ac); 75 | } 76 | 77 | public override IEnumerator GetEnumerator() 78 | { 79 | for (var i = 0; i < m_Cardinality; i++) 80 | { 81 | yield return m_Content[i]; 82 | } 83 | } 84 | 85 | 86 | public static Container operator &(ArrayContainer x, ArrayContainer y) 87 | { 88 | var desiredCapacity = Math.Min(x.m_Cardinality, y.m_Cardinality); 89 | var data = new ushort[desiredCapacity]; 90 | var calculatedCardinality = Util.IntersectArrays(x.m_Content, x.m_Cardinality, y.m_Content, y.m_Cardinality, data); 91 | return new ArrayContainer(calculatedCardinality, data); 92 | } 93 | 94 | public static ArrayContainer operator &(ArrayContainer x, BitmapContainer y) 95 | { 96 | var data = new ushort[x.m_Content.Length]; 97 | var c = x.m_Cardinality; 98 | var pos = 0; 99 | for (var i = 0; i < c; i++) 100 | { 101 | var v = x.m_Content[i]; 102 | if (y.Contains(v)) 103 | { 104 | data[pos++] = v; 105 | } 106 | } 107 | return new ArrayContainer(pos, data); 108 | } 109 | 110 | public static Container operator |(ArrayContainer x, ArrayContainer y) 111 | { 112 | var totalCardinality = x.m_Cardinality + y.m_Cardinality; 113 | if (totalCardinality > MaxSize) 114 | { 115 | var output = new ushort[totalCardinality]; 116 | var calcCardinality = Util.UnionArrays(x.m_Content, x.m_Cardinality, y.m_Content, y.m_Cardinality, output); 117 | if (calcCardinality > MaxSize) 118 | { 119 | return BitmapContainer.Create(calcCardinality, output); 120 | } 121 | return new ArrayContainer(calcCardinality, output); 122 | } 123 | var desiredCapacity = totalCardinality; 124 | var data = new ushort[desiredCapacity]; 125 | var calculatedCardinality = Util.UnionArrays(x.m_Content, x.m_Cardinality, y.m_Content, y.m_Cardinality, data); 126 | return new ArrayContainer(calculatedCardinality, data); 127 | } 128 | 129 | public static Container operator |(ArrayContainer x, BitmapContainer y) 130 | { 131 | return y | x; 132 | } 133 | 134 | public static Container operator ~(ArrayContainer x) 135 | { 136 | return BitmapContainer.Create(x.m_Cardinality, x.m_Content, true); // an arraycontainer only contains up to 4096 values, so the negation is a bitmap container 137 | } 138 | 139 | public static Container operator ^(ArrayContainer x, ArrayContainer y) 140 | { 141 | var totalCardinality = x.m_Cardinality + y.m_Cardinality; 142 | if (totalCardinality > MaxSize) 143 | { 144 | var bc = BitmapContainer.CreateXor(x.m_Content, x.Cardinality, y.m_Content, y.Cardinality); 145 | if (bc.Cardinality <= MaxSize) 146 | { 147 | Create(bc); 148 | } 149 | } 150 | var desiredCapacity = totalCardinality; 151 | var data = new ushort[desiredCapacity]; 152 | var calculatedCardinality = Util.XorArrays(x.m_Content, x.m_Cardinality, y.m_Content, y.m_Cardinality, data); 153 | return new ArrayContainer(calculatedCardinality, data); 154 | } 155 | 156 | public static Container operator ^(ArrayContainer x, BitmapContainer y) 157 | { 158 | return y ^ x; 159 | } 160 | 161 | public static Container AndNot(ArrayContainer x, ArrayContainer y) 162 | { 163 | var desiredCapacity = x.m_Cardinality; 164 | var data = new ushort[desiredCapacity]; 165 | var calculatedCardinality = Util.DifferenceArrays(x.m_Content, x.m_Cardinality, y.m_Content, y.m_Cardinality, data); 166 | return new ArrayContainer(calculatedCardinality, data); 167 | } 168 | 169 | public static Container AndNot(ArrayContainer x, BitmapContainer y) 170 | { 171 | var data = new ushort[x.m_Content.Length]; 172 | var c = x.m_Cardinality; 173 | var pos = 0; 174 | for (var i = 0; i < c; i++) 175 | { 176 | var v = x.m_Content[i]; 177 | if (!y.Contains(v)) 178 | { 179 | data[pos++] = v; 180 | } 181 | } 182 | return new ArrayContainer(pos, data); 183 | } 184 | 185 | public int OrArray(ulong[] bitmap) 186 | { 187 | var extraCardinality = 0; 188 | var yC = m_Cardinality; 189 | for (var i = 0; i < yC; i++) 190 | { 191 | var yValue = m_Content[i]; 192 | var index = yValue >> 6; 193 | var previous = bitmap[index]; 194 | var after = previous | (1UL << yValue); 195 | bitmap[index] = after; 196 | extraCardinality += (int) ((previous - after) >> 63); 197 | } 198 | return extraCardinality; 199 | } 200 | 201 | public int XorArray(ulong[] bitmap) 202 | { 203 | var extraCardinality = 0; 204 | var yC = m_Cardinality; 205 | for (var i = 0; i < yC; i++) 206 | { 207 | var yValue = m_Content[i]; 208 | var index = yValue >> 6; 209 | var previous = bitmap[index]; 210 | var mask = 1UL << yValue; 211 | bitmap[index] = previous ^ mask; 212 | extraCardinality += (int) (1 - 2 * ((previous & mask) >> yValue)); 213 | } 214 | return extraCardinality; 215 | } 216 | 217 | 218 | public int AndNotArray(ulong[] bitmap) 219 | { 220 | var extraCardinality = 0; 221 | var yC = m_Cardinality; 222 | for (var i = 0; i < yC; i++) 223 | { 224 | var yValue = m_Content[i]; 225 | var index = yValue >> 6; 226 | var previous = bitmap[index]; 227 | var after = previous & ~(1UL << yValue); 228 | bitmap[index] = after; 229 | extraCardinality -= (int) ((previous ^ after) >> yValue); 230 | } 231 | return extraCardinality; 232 | } 233 | 234 | public override bool Equals(object obj) 235 | { 236 | var ac = obj as ArrayContainer; 237 | return (ac != null) && Equals(ac); 238 | } 239 | 240 | public override int GetHashCode() 241 | { 242 | unchecked 243 | { 244 | var code = 17; 245 | code = code * 23 + m_Cardinality; 246 | for (var i = 0; i < m_Cardinality; i++) 247 | { 248 | code = code * 23 + m_Content[i]; 249 | } 250 | return code; 251 | } 252 | } 253 | 254 | public static void Serialize(ArrayContainer ac, BinaryWriter binaryWriter) 255 | { 256 | for (var i = 0; i < ac.m_Cardinality; i++) 257 | { 258 | binaryWriter.Write(ac.m_Content[i]); 259 | } 260 | } 261 | 262 | public static ArrayContainer Deserialize(BinaryReader binaryReader, int cardinality) 263 | { 264 | var data = new ushort[cardinality]; 265 | for (var i = 0; i < cardinality; i++) 266 | { 267 | data[i] = binaryReader.ReadUInt16(); 268 | } 269 | return new ArrayContainer(cardinality, data); 270 | } 271 | } 272 | } -------------------------------------------------------------------------------- /RoaringBitmap/BitmapContainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Runtime.CompilerServices; 5 | 6 | namespace Collections.Special 7 | { 8 | internal class BitmapContainer : Container, IEquatable 9 | { 10 | private const int BitmapLength = 1024; 11 | public static readonly BitmapContainer One; 12 | private readonly ulong[] m_Bitmap; 13 | private readonly int m_Cardinality; 14 | 15 | static BitmapContainer() 16 | { 17 | var data = new ulong[BitmapLength]; 18 | for (var i = 0; i < BitmapLength; i++) 19 | { 20 | data[i] = ulong.MaxValue; 21 | } 22 | One = new BitmapContainer(1 << 16, data); 23 | } 24 | 25 | private BitmapContainer(int cardinality) 26 | { 27 | m_Bitmap = new ulong[BitmapLength]; 28 | m_Cardinality = cardinality; 29 | } 30 | 31 | private BitmapContainer(int cardinality, ulong[] data) 32 | { 33 | m_Bitmap = data; 34 | m_Cardinality = cardinality; 35 | } 36 | 37 | private BitmapContainer(int cardinality, ushort[] values, bool negated) : this(negated ? MaxCapacity - cardinality : cardinality) 38 | { 39 | if (negated) 40 | { 41 | for (var i = 0; i < BitmapLength; i++) 42 | { 43 | m_Bitmap[i] = ulong.MaxValue; 44 | } 45 | for (var i = 0; i < cardinality; i++) 46 | { 47 | var v = values[i]; 48 | m_Bitmap[v >> 6] &= ~(1UL << v); 49 | } 50 | } 51 | else 52 | { 53 | for (var i = 0; i < cardinality; i++) 54 | { 55 | var v = values[i]; 56 | m_Bitmap[v >> 6] |= 1UL << v; 57 | } 58 | } 59 | } 60 | 61 | protected internal override int Cardinality => m_Cardinality; 62 | 63 | public override int ArraySizeInBytes => MaxCapacity / 8; 64 | 65 | public bool Equals(BitmapContainer other) 66 | { 67 | if (ReferenceEquals(this, other)) 68 | { 69 | return true; 70 | } 71 | if (ReferenceEquals(null, other)) 72 | { 73 | return false; 74 | } 75 | if (m_Cardinality != other.m_Cardinality) 76 | { 77 | return false; 78 | } 79 | for (var i = 0; i < BitmapLength; i++) 80 | { 81 | if (m_Bitmap[i] != other.m_Bitmap[i]) 82 | { 83 | return false; 84 | } 85 | } 86 | return true; 87 | } 88 | 89 | 90 | internal static BitmapContainer Create(ushort[] values) 91 | { 92 | return new BitmapContainer(values.Length, values, false); 93 | } 94 | 95 | internal static BitmapContainer Create(int cardinality, ushort[] values) 96 | { 97 | return new BitmapContainer(cardinality, values, false); 98 | } 99 | 100 | internal static BitmapContainer Create(int cardinality, ushort[] values, bool negated) 101 | { 102 | return new BitmapContainer(cardinality, values, negated); 103 | } 104 | 105 | 106 | internal static BitmapContainer CreateXor(ushort[] first, int firstCardinality, ushort[] second, int secondCardinality) 107 | { 108 | var data = new ulong[BitmapLength]; 109 | for (var i = 0; i < firstCardinality; i++) 110 | { 111 | var v = first[i]; 112 | data[v >> 6] ^= 1UL << v; 113 | } 114 | 115 | for (var i = 0; i < secondCardinality; i++) 116 | { 117 | var v = second[i]; 118 | data[v >> 6] ^= 1UL << v; 119 | } 120 | var cardinality = Util.BitCount(data); 121 | return new BitmapContainer(cardinality, data); 122 | } 123 | 124 | /// 125 | /// Java version has an optimized version of this, but it's using bitcount internally which should make it slower in 126 | /// .NET 127 | /// 128 | public static Container operator &(BitmapContainer x, BitmapContainer y) 129 | { 130 | var data = Clone(x.m_Bitmap); 131 | var bc = new BitmapContainer(AndInternal(data, y.m_Bitmap), data); 132 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 133 | } 134 | 135 | private static ulong[] Clone(ulong[] data) 136 | { 137 | var result = new ulong[BitmapLength]; 138 | Buffer.BlockCopy(data, 0, result, 0, BitmapLength * sizeof(ulong)); 139 | return result; 140 | } 141 | 142 | public static ArrayContainer operator &(BitmapContainer x, ArrayContainer y) 143 | { 144 | return y & x; 145 | } 146 | 147 | public static BitmapContainer operator |(BitmapContainer x, BitmapContainer y) 148 | { 149 | var data = Clone(x.m_Bitmap); 150 | return new BitmapContainer(OrInternal(data, y.m_Bitmap), data); 151 | } 152 | 153 | public static BitmapContainer operator |(BitmapContainer x, ArrayContainer y) 154 | { 155 | var data = Clone(x.m_Bitmap); 156 | return new BitmapContainer(x.m_Cardinality + y.OrArray(data), data); 157 | } 158 | 159 | public static Container operator ~(BitmapContainer x) 160 | { 161 | var data = Clone(x.m_Bitmap); 162 | var bc = new BitmapContainer(NotInternal(data), data); 163 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 164 | } 165 | 166 | /// 167 | /// Java version has an optimized version of this, but it's using bitcount internally which should make it slower in 168 | /// .NET 169 | /// 170 | public static Container operator ^(BitmapContainer x, BitmapContainer y) 171 | { 172 | var data = Clone(x.m_Bitmap); 173 | var bc = new BitmapContainer(XorInternal(data, y.m_Bitmap), data); 174 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 175 | } 176 | 177 | 178 | public static Container operator ^(BitmapContainer x, ArrayContainer y) 179 | { 180 | var data = Clone(x.m_Bitmap); 181 | var bc = new BitmapContainer(x.m_Cardinality + y.XorArray(data), data); 182 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 183 | } 184 | 185 | public static Container AndNot(BitmapContainer x, BitmapContainer y) 186 | { 187 | var data = Clone(x.m_Bitmap); 188 | var bc = new BitmapContainer(AndNotInternal(data, y.m_Bitmap), data); 189 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 190 | } 191 | 192 | public static Container AndNot(BitmapContainer x, ArrayContainer y) 193 | { 194 | var data = Clone(x.m_Bitmap); 195 | var bc = new BitmapContainer(x.m_Cardinality + y.AndNotArray(data), data); 196 | return bc.m_Cardinality <= MaxSize ? (Container) ArrayContainer.Create(bc) : bc; 197 | } 198 | 199 | private static int XorInternal(ulong[] first, ulong[] second) 200 | { 201 | for (var k = 0; k < BitmapLength; k++) 202 | { 203 | first[k] = first[k] ^ second[k]; 204 | } 205 | var c = Util.BitCount(first); 206 | return c; 207 | } 208 | 209 | private static int AndNotInternal(ulong[] first, ulong[] second) 210 | { 211 | for (var k = 0; k < first.Length; k++) 212 | { 213 | first[k] = first[k] & ~second[k]; 214 | } 215 | var c = Util.BitCount(first); 216 | return c; 217 | } 218 | 219 | private static int NotInternal(ulong[] data) 220 | { 221 | for (var k = 0; k < BitmapLength; k++) 222 | { 223 | data[k] = ~data[k]; 224 | } 225 | var c = Util.BitCount(data); 226 | return c; 227 | } 228 | 229 | private static int OrInternal(ulong[] first, ulong[] second) 230 | { 231 | for (var k = 0; k < BitmapLength; k++) 232 | { 233 | first[k] = first[k] | second[k]; 234 | } 235 | var c = Util.BitCount(first); 236 | return c; 237 | } 238 | 239 | private static int AndInternal(ulong[] first, ulong[] second) 240 | { 241 | for (var k = 0; k < BitmapLength; k++) 242 | { 243 | first[k] = first[k] & second[k]; 244 | } 245 | var c = Util.BitCount(first); 246 | return c; 247 | } 248 | 249 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 250 | public bool Contains(ushort x) 251 | { 252 | return Contains(m_Bitmap, x); 253 | } 254 | 255 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 256 | private static bool Contains(ulong[] bitmap, ushort x) 257 | { 258 | return (bitmap[x >> 6] & (1UL << x)) != 0; 259 | } 260 | 261 | protected override bool EqualsInternal(Container other) 262 | { 263 | var bc = other as BitmapContainer; 264 | return (bc != null) && Equals(bc); 265 | } 266 | 267 | public override IEnumerator GetEnumerator() 268 | { 269 | for (var k = 0; k < BitmapLength; k++) 270 | { 271 | var bitset = m_Bitmap[k]; 272 | var shiftedK = k << 6; 273 | while (bitset != 0) 274 | { 275 | var t = bitset & (~bitset + 1); 276 | var result = (ushort) (shiftedK + Util.BitCount(t - 1)); 277 | yield return result; 278 | bitset ^= t; 279 | } 280 | } 281 | } 282 | 283 | internal int FillArray(ushort[] data) 284 | { 285 | var pos = 0; 286 | for (var k = 0; k < BitmapLength; k++) 287 | { 288 | var bitset = m_Bitmap[k]; 289 | var shiftedK = k << 6; 290 | while (bitset != 0) 291 | { 292 | var t = bitset & (~bitset + 1); 293 | data[pos++] = (ushort) (shiftedK + Util.BitCount(t - 1)); 294 | bitset ^= t; 295 | } 296 | } 297 | return m_Cardinality; 298 | } 299 | 300 | public override bool Equals(object obj) 301 | { 302 | var bc = obj as BitmapContainer; 303 | return (bc != null) && Equals(bc); 304 | } 305 | 306 | public override int GetHashCode() 307 | { 308 | unchecked 309 | { 310 | var code = 17; 311 | code = code * 23 + m_Cardinality; 312 | for (var i = 0; i < BitmapLength; i++) 313 | { 314 | code = code * 23 + m_Bitmap[i].GetHashCode(); 315 | } 316 | return code; 317 | } 318 | } 319 | 320 | public static void Serialize(BitmapContainer bc, BinaryWriter binaryWriter) 321 | { 322 | for (var i = 0; i < BitmapLength; i++) 323 | { 324 | binaryWriter.Write(bc.m_Bitmap[i]); 325 | } 326 | } 327 | 328 | public static BitmapContainer Deserialize(BinaryReader binaryReader, int cardinality) 329 | { 330 | var data = new ulong[BitmapLength]; 331 | for (var i = 0; i < BitmapLength; i++) 332 | { 333 | data[i] = binaryReader.ReadUInt64(); 334 | } 335 | return new BitmapContainer(cardinality, data); 336 | } 337 | } 338 | } -------------------------------------------------------------------------------- /RoaringBitmap/Container.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Collections.Special 5 | { 6 | internal abstract class Container : IEquatable 7 | { 8 | public const int MaxSize = 4096; // everything <= is an ArrayContainer 9 | public const int MaxCapacity = 1 << 16; 10 | 11 | protected internal abstract int Cardinality { get; } 12 | 13 | public abstract int ArraySizeInBytes { get; } 14 | 15 | public bool Equals(Container other) 16 | { 17 | if (ReferenceEquals(this, other)) 18 | { 19 | return true; 20 | } 21 | if (ReferenceEquals(null, other)) 22 | { 23 | return false; 24 | } 25 | return EqualsInternal(other); 26 | } 27 | 28 | protected abstract bool EqualsInternal(Container other); 29 | 30 | public abstract IEnumerator GetEnumerator(); 31 | 32 | public static Container operator |(Container x, Container y) 33 | { 34 | var xArrayContainer = x as ArrayContainer; 35 | var yArrayContainer = y as ArrayContainer; 36 | if ((xArrayContainer != null) && (yArrayContainer != null)) 37 | { 38 | return xArrayContainer | yArrayContainer; 39 | } 40 | if (xArrayContainer != null) 41 | { 42 | return xArrayContainer | (BitmapContainer) y; 43 | } 44 | if (yArrayContainer != null) 45 | { 46 | return (BitmapContainer) x | yArrayContainer; 47 | } 48 | return (BitmapContainer) x | (BitmapContainer) y; 49 | } 50 | 51 | public static Container operator &(Container x, Container y) 52 | { 53 | var xArrayContainer = x as ArrayContainer; 54 | var yArrayContainer = y as ArrayContainer; 55 | if ((xArrayContainer != null) && (yArrayContainer != null)) 56 | { 57 | return xArrayContainer & yArrayContainer; 58 | } 59 | if (xArrayContainer != null) 60 | { 61 | return xArrayContainer & (BitmapContainer) y; 62 | } 63 | if (yArrayContainer != null) 64 | { 65 | return (BitmapContainer) x & yArrayContainer; 66 | } 67 | return (BitmapContainer) x & (BitmapContainer) y; 68 | } 69 | 70 | public static Container operator ^(Container x, Container y) 71 | { 72 | var xArrayContainer = x as ArrayContainer; 73 | var yArrayContainer = y as ArrayContainer; 74 | if ((xArrayContainer != null) && (yArrayContainer != null)) 75 | { 76 | return xArrayContainer ^ yArrayContainer; 77 | } 78 | if (xArrayContainer != null) 79 | { 80 | return xArrayContainer ^ (BitmapContainer) y; 81 | } 82 | if (yArrayContainer != null) 83 | { 84 | return (BitmapContainer) x ^ yArrayContainer; 85 | } 86 | return (BitmapContainer) x ^ (BitmapContainer) y; 87 | } 88 | 89 | public static Container operator ~(Container x) 90 | { 91 | var xArrayContainer = x as ArrayContainer; 92 | return xArrayContainer != null ? ~xArrayContainer : ~(BitmapContainer) x; 93 | } 94 | 95 | public static Container AndNot(Container x, Container y) 96 | { 97 | var xArrayContainer = x as ArrayContainer; 98 | var yArrayContainer = y as ArrayContainer; 99 | if ((xArrayContainer != null) && (yArrayContainer != null)) 100 | { 101 | return ArrayContainer.AndNot(xArrayContainer, yArrayContainer); 102 | } 103 | if (xArrayContainer != null) 104 | { 105 | return ArrayContainer.AndNot(xArrayContainer, (BitmapContainer) y); 106 | } 107 | if (yArrayContainer != null) 108 | { 109 | return BitmapContainer.AndNot((BitmapContainer) x, yArrayContainer); 110 | } 111 | return BitmapContainer.AndNot((BitmapContainer) x, (BitmapContainer) y); 112 | } 113 | } 114 | } -------------------------------------------------------------------------------- /RoaringBitmap/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | 8 | [assembly: AssemblyCulture("")] 9 | 10 | // Setting ComVisible to false makes the types in this assembly not visible 11 | // to COM components. If you need to access a type in this assembly from 12 | // COM, set the ComVisible attribute to true on that type. 13 | 14 | [assembly: ComVisible(false)] 15 | 16 | // The following GUID is for the ID of the typelib if this project is exposed to COM 17 | 18 | [assembly: Guid("2ee1be04-a8f5-4358-bf08-e417d3916b49")] 19 | -------------------------------------------------------------------------------- /RoaringBitmap/RoaringArray.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Text; 6 | 7 | namespace Collections.Special 8 | { 9 | internal class RoaringArray : IEnumerable, IEquatable 10 | { 11 | private const int SerialCookieNoRuncontainer = 12346; 12 | private const int SerialCookie = 12347; 13 | private const int NoOffsetThreshold = 4; 14 | private readonly ushort[] m_Keys; 15 | private readonly int m_Size; 16 | private readonly Container[] m_Values; 17 | 18 | 19 | // ReSharper disable once SuggestBaseTypeForParameter 20 | /// 21 | /// Use List directly, because the enumerator is a struct 22 | /// 23 | internal RoaringArray(int size, List keys, List containers) 24 | { 25 | m_Size = size; 26 | m_Keys = new ushort[m_Size]; 27 | m_Values = new Container[m_Size]; 28 | for (var i = 0; i < m_Size; i++) 29 | { 30 | m_Keys[i] = keys[i]; 31 | m_Values[i] = containers[i]; 32 | Cardinality += m_Values[i].Cardinality; 33 | } 34 | } 35 | 36 | private RoaringArray(int size, ushort[] keys, Container[] containers) 37 | { 38 | m_Size = size; 39 | m_Keys = keys; 40 | m_Values = containers; 41 | for (var i = 0; i < containers.Length; i++) 42 | { 43 | Cardinality += containers[i].Cardinality; 44 | } 45 | } 46 | 47 | public long Cardinality { get; } 48 | 49 | public IEnumerator GetEnumerator() 50 | { 51 | for (var i = 0; i < m_Size; i++) 52 | { 53 | var key = m_Keys[i]; 54 | var shiftedKey = key << 16; 55 | var container = m_Values[i]; 56 | foreach (var @ushort in container) 57 | { 58 | yield return shiftedKey | @ushort; 59 | } 60 | } 61 | } 62 | 63 | IEnumerator IEnumerable.GetEnumerator() 64 | { 65 | return GetEnumerator(); 66 | } 67 | 68 | public bool Equals(RoaringArray other) 69 | { 70 | if (ReferenceEquals(this, other)) 71 | { 72 | return true; 73 | } 74 | if (ReferenceEquals(null, other)) 75 | { 76 | return false; 77 | } 78 | if (m_Size != other.m_Size) 79 | { 80 | return false; 81 | } 82 | for (var i = 0; i < m_Size; i++) 83 | { 84 | if ((m_Keys[i] != other.m_Keys[i]) || !m_Values[i].Equals(other.m_Values[i])) 85 | { 86 | return false; 87 | } 88 | } 89 | return true; 90 | } 91 | 92 | private int AdvanceUntil(ushort key, int index) 93 | { 94 | return Util.AdvanceUntil(m_Keys, index, m_Keys.Length, key); 95 | } 96 | 97 | public static RoaringArray operator |(RoaringArray x, RoaringArray y) 98 | { 99 | var xLength = x.m_Size; 100 | var yLength = y.m_Size; 101 | var keys = new List(xLength + yLength); 102 | var containers = new List(xLength + yLength); 103 | var size = 0; 104 | var xPos = 0; 105 | var yPos = 0; 106 | if ((xPos < xLength) && (yPos < yLength)) 107 | { 108 | var xKey = x.m_Keys[xPos]; 109 | var yKey = y.m_Keys[yPos]; 110 | while (true) 111 | { 112 | if (xKey == yKey) 113 | { 114 | keys.Add(xKey); 115 | containers.Add(x.m_Values[xPos] | y.m_Values[yPos]); 116 | size++; 117 | xPos++; 118 | yPos++; 119 | if ((xPos == xLength) || (yPos == yLength)) 120 | { 121 | break; 122 | } 123 | xKey = x.m_Keys[xPos]; 124 | yKey = y.m_Keys[yPos]; 125 | } 126 | else if (xKey < yKey) 127 | { 128 | keys.Add(xKey); 129 | containers.Add(x.m_Values[xPos]); 130 | size++; 131 | xPos++; 132 | if (xPos == xLength) 133 | { 134 | break; 135 | } 136 | xKey = x.m_Keys[xPos]; 137 | } 138 | else 139 | { 140 | keys.Add(yKey); 141 | containers.Add(y.m_Values[yPos]); 142 | size++; 143 | yPos++; 144 | if (yPos == yLength) 145 | { 146 | break; 147 | } 148 | yKey = y.m_Keys[yPos]; 149 | } 150 | } 151 | } 152 | if (xPos == xLength) 153 | { 154 | for (var i = yPos; i < yLength; i++) 155 | { 156 | keys.Add(y.m_Keys[i]); 157 | containers.Add(y.m_Values[i]); 158 | size++; 159 | } 160 | } 161 | else if (yPos == yLength) 162 | { 163 | for (var i = xPos; i < xLength; i++) 164 | { 165 | keys.Add(x.m_Keys[i]); 166 | containers.Add(x.m_Values[i]); 167 | size++; 168 | } 169 | } 170 | return new RoaringArray(size, keys, containers); 171 | } 172 | 173 | public static RoaringArray operator &(RoaringArray x, RoaringArray y) 174 | { 175 | var xLength = x.m_Size; 176 | var yLength = y.m_Size; 177 | List keys = null; 178 | List containers = null; 179 | var size = 0; 180 | var xPos = 0; 181 | var yPos = 0; 182 | while ((xPos < xLength) && (yPos < yLength)) 183 | { 184 | var xKey = x.m_Keys[xPos]; 185 | var yKey = y.m_Keys[yPos]; 186 | if (xKey == yKey) 187 | { 188 | var c = x.m_Values[xPos] & y.m_Values[yPos]; 189 | if (c.Cardinality > 0) 190 | { 191 | if (keys == null) 192 | { 193 | var length = Math.Min(xLength, yLength); 194 | keys = new List(length); 195 | containers = new List(length); 196 | } 197 | keys.Add(xKey); 198 | containers.Add(c); 199 | size++; 200 | } 201 | xPos++; 202 | yPos++; 203 | } 204 | else if (xKey < yKey) 205 | { 206 | xPos = x.AdvanceUntil(yKey, xPos); 207 | } 208 | else 209 | { 210 | yPos = y.AdvanceUntil(xKey, yPos); 211 | } 212 | } 213 | return new RoaringArray(size, keys, containers); 214 | } 215 | 216 | public static RoaringArray operator ^(RoaringArray x, RoaringArray y) 217 | { 218 | var xLength = x.m_Size; 219 | var yLength = y.m_Size; 220 | var keys = new List(xLength + yLength); 221 | var containers = new List(xLength + yLength); 222 | var size = 0; 223 | var xPos = 0; 224 | var yPos = 0; 225 | if ((xPos < xLength) && (yPos < yLength)) 226 | { 227 | var xKey = x.m_Keys[xPos]; 228 | var yKey = y.m_Keys[yPos]; 229 | while (true) 230 | { 231 | if (xKey == yKey) 232 | { 233 | keys.Add(xKey); 234 | containers.Add(x.m_Values[xPos] ^ y.m_Values[yPos]); 235 | size++; 236 | xPos++; 237 | yPos++; 238 | if ((xPos == xLength) || (yPos == yLength)) 239 | { 240 | break; 241 | } 242 | xKey = x.m_Keys[xPos]; 243 | yKey = y.m_Keys[yPos]; 244 | } 245 | else if (xKey < yKey) 246 | { 247 | keys.Add(xKey); 248 | containers.Add(x.m_Values[xPos]); 249 | size++; 250 | xPos++; 251 | if (xPos == xLength) 252 | { 253 | break; 254 | } 255 | xKey = x.m_Keys[xPos]; 256 | } 257 | else 258 | { 259 | keys.Add(yKey); 260 | containers.Add(y.m_Values[yPos]); 261 | size++; 262 | yPos++; 263 | if (yPos == yLength) 264 | { 265 | break; 266 | } 267 | yKey = y.m_Keys[yPos]; 268 | } 269 | } 270 | } 271 | if (xPos == xLength) 272 | { 273 | for (var i = yPos; i < yLength; i++) 274 | { 275 | keys.Add(y.m_Keys[i]); 276 | containers.Add(y.m_Values[i]); 277 | size++; 278 | } 279 | } 280 | else if (yPos == yLength) 281 | { 282 | for (var i = xPos; i < xLength; i++) 283 | { 284 | keys.Add(x.m_Keys[i]); 285 | containers.Add(x.m_Values[i]); 286 | size++; 287 | } 288 | } 289 | return new RoaringArray(size, keys, containers); 290 | } 291 | 292 | public static RoaringArray operator ~(RoaringArray x) 293 | { 294 | var keys = new List(Container.MaxCapacity); 295 | var size = 0; 296 | var containers = new List(Container.MaxCapacity); 297 | var oldIndex = 0; 298 | for (var i = 0; i < Container.MaxCapacity; i++) 299 | { 300 | var ushortI = (ushort) i; 301 | var index = Array.BinarySearch(x.m_Keys, oldIndex, x.m_Size - oldIndex, ushortI); 302 | if (index < 0) 303 | { 304 | keys.Add(ushortI); 305 | containers.Add(BitmapContainer.One); 306 | size++; 307 | } 308 | else 309 | { 310 | var c = x.m_Values[index]; 311 | if (!c.Equals(BitmapContainer.One)) // the bitwise negation of the one container is the zero container 312 | { 313 | var nc = ~c; 314 | if (nc.Cardinality > 0) 315 | { 316 | keys.Add(ushortI); 317 | containers.Add(nc); 318 | size++; 319 | } 320 | } 321 | oldIndex = index; 322 | } 323 | } 324 | return new RoaringArray(size, keys, containers); 325 | } 326 | 327 | public static RoaringArray AndNot(RoaringArray x, RoaringArray y) 328 | { 329 | var xLength = x.m_Size; 330 | var yLength = y.m_Size; 331 | var keys = new List(xLength); 332 | var containers = new List(xLength); 333 | var size = 0; 334 | var xPos = 0; 335 | var yPos = 0; 336 | while ((xPos < xLength) && (yPos < yLength)) 337 | { 338 | var xKey = x.m_Keys[xPos]; 339 | var yKey = y.m_Keys[yPos]; 340 | if (xKey == yKey) 341 | { 342 | var c = Container.AndNot(x.m_Values[xPos], y.m_Values[yPos]); 343 | if (c.Cardinality > 0) 344 | { 345 | keys.Add(xKey); 346 | containers.Add(c); 347 | size++; 348 | } 349 | xPos++; 350 | yPos++; 351 | } 352 | else if (xKey < yKey) 353 | { 354 | var next = x.AdvanceUntil(yKey, xPos); 355 | for (var i = xPos; i < next; i++) 356 | { 357 | keys.Add(x.m_Keys[i]); 358 | containers.Add(x.m_Values[i]); 359 | size++; 360 | } 361 | xPos = next; 362 | } 363 | else 364 | { 365 | yPos = y.AdvanceUntil(xKey, yPos); 366 | } 367 | } 368 | if (yPos == yLength) 369 | { 370 | for (var i = xPos; i < xLength; i++) 371 | { 372 | keys.Add(x.m_Keys[i]); 373 | containers.Add(x.m_Values[i]); 374 | size++; 375 | } 376 | } 377 | return new RoaringArray(size, keys, containers); 378 | } 379 | 380 | public override bool Equals(object obj) 381 | { 382 | var ra = obj as RoaringArray; 383 | return (ra != null) && Equals(ra); 384 | } 385 | 386 | public override int GetHashCode() 387 | { 388 | unchecked 389 | { 390 | var code = 17; 391 | code = code * 23 + m_Size; 392 | for (var i = 0; i < m_Size; i++) 393 | { 394 | code = code * 23 + m_Keys[i].GetHashCode(); 395 | code = code * 23 + m_Values[i].GetHashCode(); 396 | } 397 | return code; 398 | } 399 | } 400 | 401 | public static void Serialize(RoaringArray roaringArray, Stream stream) 402 | { 403 | var hasRun = HasRunContainer(roaringArray); 404 | using (var binaryWriter = new BinaryWriter(stream, Encoding.UTF8, true)) 405 | { 406 | var size = roaringArray.m_Size; 407 | var keys = roaringArray.m_Keys; 408 | var values = roaringArray.m_Values; 409 | var startOffset = 0; 410 | if (hasRun) 411 | { 412 | binaryWriter.Write(SerialCookie | ((size - 1) << 16)); 413 | var bitmapOfRunContainers = new byte[(size + 7) / 8]; 414 | for (var i = 0; i < size; ++i) 415 | { 416 | if (values[i].Equals(ArrayContainer.One) || values[i].Equals(BitmapContainer.One)) 417 | { 418 | bitmapOfRunContainers[i / 8] |= (byte) (1 << (i % 8)); 419 | } 420 | } 421 | binaryWriter.Write(bitmapOfRunContainers); 422 | } 423 | else // no run containers 424 | { 425 | binaryWriter.Write(SerialCookieNoRuncontainer); 426 | binaryWriter.Write(size); 427 | startOffset = 4 + 4 + 4 * size + 4 * size; 428 | } 429 | for (var k = 0; k < size; ++k) 430 | { 431 | binaryWriter.Write(keys[k]); 432 | binaryWriter.Write((ushort) (values[k].Cardinality - 1)); 433 | } 434 | if (!hasRun || (size >= NoOffsetThreshold)) 435 | { 436 | for (var k = 0; k < size; k++) 437 | { 438 | binaryWriter.Write(startOffset); 439 | startOffset += values[k].ArraySizeInBytes; 440 | } 441 | } 442 | for (var k = 0; k < size; ++k) 443 | { 444 | var container = values[k]; 445 | ArrayContainer ac; 446 | BitmapContainer bc; 447 | if ((ac = container as ArrayContainer) != null) 448 | { 449 | if (ac.Equals(ArrayContainer.One)) 450 | { 451 | binaryWriter.Write((ushort) 1); 452 | binaryWriter.Write((ushort) 0); 453 | binaryWriter.Write((ushort) (Container.MaxSize - 1)); 454 | } 455 | else 456 | { 457 | ArrayContainer.Serialize(ac, binaryWriter); 458 | } 459 | } 460 | else if ((bc = container as BitmapContainer) != null) 461 | { 462 | if (bc.Equals(BitmapContainer.One)) 463 | { 464 | binaryWriter.Write((ushort) 1); 465 | binaryWriter.Write((ushort) 0); 466 | binaryWriter.Write((ushort) (Container.MaxCapacity - 1)); 467 | } 468 | else 469 | { 470 | BitmapContainer.Serialize(bc, binaryWriter); 471 | } 472 | } 473 | } 474 | binaryWriter.Flush(); 475 | } 476 | } 477 | 478 | private static bool HasRunContainer(RoaringArray roaringArray) 479 | { 480 | for (var i = 0; i < roaringArray.m_Size; i++) 481 | { 482 | if (roaringArray.m_Values[i].Equals(ArrayContainer.One) || roaringArray.m_Values[i].Equals(BitmapContainer.One)) 483 | { 484 | return true; 485 | } 486 | } 487 | return false; 488 | } 489 | 490 | public static RoaringArray Deserialize(Stream stream) 491 | { 492 | using (var binaryReader = new BinaryReader(stream, Encoding.UTF8, true)) 493 | { 494 | var cookie = binaryReader.ReadUInt32(); 495 | var lbcookie = cookie & 0xFFFF; 496 | if ((lbcookie != SerialCookie) && (cookie != SerialCookieNoRuncontainer)) 497 | { 498 | throw new InvalidDataException("No RoaringBitmap file."); 499 | } 500 | var hasRun = lbcookie == SerialCookie; 501 | var size = (int) (hasRun ? (cookie >> 16) + 1 : binaryReader.ReadUInt32()); 502 | var keys = new ushort[size]; 503 | var containers = new Container[size]; 504 | var cardinalities = new int[size]; 505 | var isBitmap = new bool[size]; 506 | 507 | 508 | byte[] bitmapOfRunContainers = null; 509 | if (hasRun) 510 | { 511 | bitmapOfRunContainers = binaryReader.ReadBytes((size + 7) / 8); 512 | } 513 | for (var k = 0; k < size; ++k) 514 | { 515 | keys[k] = binaryReader.ReadUInt16(); 516 | cardinalities[k] = 1 + (0xFFFF & binaryReader.ReadUInt16()); 517 | isBitmap[k] = cardinalities[k] > Container.MaxSize; 518 | if ((bitmapOfRunContainers != null) && ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) 519 | { 520 | isBitmap[k] = false; 521 | } 522 | } 523 | if (!hasRun || (size >= NoOffsetThreshold)) 524 | { 525 | // skipping the offsets 526 | binaryReader.ReadBytes(size * 4); 527 | } 528 | for (var k = 0; k < size; ++k) 529 | { 530 | if (isBitmap[k]) 531 | { 532 | containers[k] = BitmapContainer.Deserialize(binaryReader, cardinalities[k]); 533 | } 534 | else if ((bitmapOfRunContainers != null) && ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) 535 | { 536 | var nbrruns = binaryReader.ReadUInt16(); 537 | var values = new List(nbrruns * 2); // probably more 538 | var count = 0; 539 | var specialCase = false; 540 | for (var j = 0; j < nbrruns; ++j) 541 | { 542 | var value = binaryReader.ReadUInt16(); 543 | var length = binaryReader.ReadUInt16(); 544 | 545 | if ((nbrruns == 1) && (value == 0) && (length == Container.MaxCapacity - 1)) // special one scenario 546 | { 547 | containers[k] = BitmapContainer.One; 548 | specialCase = true; 549 | break; 550 | } 551 | if ((nbrruns == 1) && (value == 0) && (length == Container.MaxSize - 1)) // special one scenario 552 | { 553 | containers[k] = ArrayContainer.One; 554 | specialCase = true; 555 | break; 556 | } 557 | for (int i = value; i < value + length + 1; i++) 558 | { 559 | values.Add((ushort) i); 560 | } 561 | count += length; 562 | } 563 | if (!specialCase) 564 | { 565 | if (count > Container.MaxSize) 566 | { 567 | containers[k] = BitmapContainer.Create(values.ToArray()); 568 | } 569 | else 570 | { 571 | containers[k] = ArrayContainer.Create(values.ToArray()); 572 | } 573 | } 574 | } 575 | else 576 | { 577 | containers[k] = ArrayContainer.Deserialize(binaryReader, cardinalities[k]); 578 | } 579 | } 580 | for (var i = 0; i < size; i++) 581 | { 582 | if (containers[i].Equals(ArrayContainer.One)) 583 | { 584 | containers[i] = ArrayContainer.One; 585 | } 586 | else if (containers[i].Equals(BitmapContainer.One)) 587 | { 588 | containers[i] = BitmapContainer.One; 589 | } 590 | } 591 | return new RoaringArray(size, keys, containers); 592 | } 593 | } 594 | 595 | public static RoaringArray Optimize(RoaringArray roaringArray) 596 | { 597 | var keys = new ushort[roaringArray.m_Size]; 598 | Array.Copy(roaringArray.m_Keys, keys, roaringArray.m_Size); 599 | var containers = new Container[roaringArray.m_Size]; 600 | for (var i = 0; i < roaringArray.m_Size; i++) 601 | { 602 | var currentContainer = roaringArray.m_Values[i]; 603 | if (currentContainer.Equals(ArrayContainer.One)) 604 | { 605 | containers[i] = ArrayContainer.One; 606 | } 607 | else if (currentContainer.Equals(BitmapContainer.One)) 608 | { 609 | containers[i] = BitmapContainer.One; 610 | } 611 | else 612 | { 613 | containers[i] = currentContainer; 614 | } 615 | } 616 | return new RoaringArray(roaringArray.m_Size, keys, containers); 617 | } 618 | } 619 | } -------------------------------------------------------------------------------- /RoaringBitmap/RoaringBitmap.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Linq; 6 | 7 | namespace Collections.Special 8 | { 9 | public class RoaringBitmap : IEnumerable, IEquatable 10 | { 11 | private readonly RoaringArray m_HighLowContainer; 12 | 13 | private RoaringBitmap(RoaringArray input) 14 | { 15 | m_HighLowContainer = input; 16 | } 17 | 18 | public long Cardinality => m_HighLowContainer.Cardinality; 19 | 20 | public IEnumerator GetEnumerator() 21 | { 22 | return m_HighLowContainer.GetEnumerator(); 23 | } 24 | 25 | IEnumerator IEnumerable.GetEnumerator() 26 | { 27 | return GetEnumerator(); 28 | } 29 | 30 | public bool Equals(RoaringBitmap other) 31 | { 32 | if (ReferenceEquals(this, other)) 33 | { 34 | return true; 35 | } 36 | if (ReferenceEquals(null, other)) 37 | { 38 | return false; 39 | } 40 | return m_HighLowContainer.Equals(other.m_HighLowContainer); 41 | } 42 | 43 | /// 44 | /// Creates a new immutable RoaringBitmap from an existing list of integers 45 | /// 46 | /// List of integers 47 | /// RoaringBitmap 48 | public static RoaringBitmap Create(params int[] values) 49 | { 50 | return Create(values.AsEnumerable()); 51 | } 52 | 53 | /// 54 | /// Optimizes a RoaringBitmap to prepare e.g. for Serialization/Deserialization 55 | /// 56 | /// RoaringBitmap 57 | public RoaringBitmap Optimize() 58 | { 59 | return new RoaringBitmap(RoaringArray.Optimize(m_HighLowContainer)); 60 | } 61 | 62 | /// 63 | /// Creates a new immutable RoaringBitmap from an existing list of integers 64 | /// 65 | /// List of integers 66 | /// RoaringBitmap 67 | public static RoaringBitmap Create(IEnumerable values) 68 | { 69 | var groupbyHb = values.Distinct().OrderBy(t => t).GroupBy(Util.HighBits).OrderBy(t => t.Key).ToList(); 70 | var keys = new List(); 71 | var containers = new List(); 72 | var size = 0; 73 | foreach (var group in groupbyHb) 74 | { 75 | keys.Add(group.Key); 76 | if (group.Count() > Container.MaxSize) 77 | { 78 | containers.Add(BitmapContainer.Create(group.Select(Util.LowBits).ToArray())); 79 | } 80 | else 81 | { 82 | containers.Add(ArrayContainer.Create(group.Select(Util.LowBits).ToArray())); 83 | } 84 | size++; 85 | } 86 | return new RoaringBitmap(new RoaringArray(size, keys, containers)); 87 | } 88 | 89 | /// 90 | /// Bitwise Or operation of two RoaringBitmaps 91 | /// 92 | /// RoaringBitmap 93 | /// RoaringBitmap 94 | /// RoaringBitmap 95 | public static RoaringBitmap operator |(RoaringBitmap x, RoaringBitmap y) 96 | { 97 | return new RoaringBitmap(x.m_HighLowContainer | y.m_HighLowContainer); 98 | } 99 | 100 | /// 101 | /// Bitwise And operation of two RoaringBitmaps 102 | /// 103 | /// RoaringBitmap 104 | /// RoaringBitmap 105 | /// RoaringBitmap 106 | public static RoaringBitmap operator &(RoaringBitmap x, RoaringBitmap y) 107 | { 108 | return new RoaringBitmap(x.m_HighLowContainer & y.m_HighLowContainer); 109 | } 110 | 111 | /// 112 | /// Bitwise Not operation of a RoaringBitmap 113 | /// 114 | /// RoaringBitmap 115 | /// RoaringBitmap 116 | public static RoaringBitmap operator ~(RoaringBitmap x) 117 | { 118 | return new RoaringBitmap(~x.m_HighLowContainer); 119 | } 120 | 121 | /// 122 | /// Bitwise Xor operation of two RoaringBitmaps 123 | /// 124 | /// RoaringBitmap 125 | /// RoaringBitmap 126 | /// RoaringBitmap 127 | public static RoaringBitmap operator ^(RoaringBitmap x, RoaringBitmap y) 128 | { 129 | return new RoaringBitmap(x.m_HighLowContainer ^ y.m_HighLowContainer); 130 | } 131 | 132 | /// 133 | /// Bitwise AndNot operation of two RoaringBitmaps 134 | /// 135 | /// RoaringBitmap 136 | /// RoaringBitmap 137 | /// RoaringBitmap 138 | public static RoaringBitmap AndNot(RoaringBitmap x, RoaringBitmap y) 139 | { 140 | return new RoaringBitmap(RoaringArray.AndNot(x.m_HighLowContainer, y.m_HighLowContainer)); 141 | } 142 | 143 | public override bool Equals(object obj) 144 | { 145 | var ra = obj as RoaringArray; 146 | return (ra != null) && Equals(ra); 147 | } 148 | 149 | public override int GetHashCode() 150 | { 151 | return (13 ^ m_HighLowContainer.GetHashCode()) << 3; 152 | } 153 | 154 | /// 155 | /// Serializes a RoaringBitmap into a stream using the 'official' RoaringBitmap file format 156 | /// 157 | /// RoaringBitmap 158 | /// Stream 159 | public static void Serialize(RoaringBitmap roaringBitmap, Stream stream) 160 | { 161 | RoaringArray.Serialize(roaringBitmap.m_HighLowContainer, stream); 162 | } 163 | 164 | /// 165 | /// Deserializes a RoaringBitmap from astream using the 'official' RoaringBitmap file format 166 | /// 167 | /// Stream 168 | public static RoaringBitmap Deserialize(Stream stream) 169 | { 170 | var ra = RoaringArray.Deserialize(stream); 171 | return new RoaringBitmap(ra); 172 | } 173 | } 174 | } -------------------------------------------------------------------------------- /RoaringBitmap/RoaringBitmap.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard1.1 5 | https://github.com/Tornhoof/RoaringBitmap 6 | https://github.com/Tornhoof/RoaringBitmap/blob/master/LICENSE 7 | RoaringBitmap 8 | True 9 | https://github.com/Tornhoof/RoaringBitmap 10 | 0.9.1 11 | RoaringBitmap for .NET 12 | RoaringBitmap Contributors 13 | Copyright © RoaringBitmap Contributors 2017 14 | Roaring Bitmap 15 | true 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /RoaringBitmap/Util.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.CompilerServices; 3 | 4 | namespace Collections.Special 5 | { 6 | /// 7 | /// Pretty much everything in here are straight conversions from the original Util class in the java Roaring Bitmap 8 | /// project. 9 | /// 10 | internal static class Util 11 | { 12 | /// 13 | /// see https://en.wikipedia.org/wiki/Hamming_weight 14 | /// Unfortunately there is no popcnt in c# 15 | /// 16 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 17 | public static int BitCount(ulong x) 18 | { 19 | x -= (x >> 1) & 0x5555555555555555UL; //put count of each 2 bits into those 2 bits 20 | x = (x & 0x3333333333333333UL) + ((x >> 2) & 0x3333333333333333UL); //put count of each 4 bits into those 4 bits 21 | x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0FUL; //put count of each 8 bits into those 8 bits 22 | return (int) ((x * 0x0101010101010101UL) >> 56); //returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24) + ... 23 | } 24 | 25 | /// 26 | /// see https://en.wikipedia.org/wiki/Hamming_weight 27 | /// Unfortunately there is no popcnt in c# 28 | /// 29 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 30 | public static int BitCount(ulong[] xArray) 31 | { 32 | var result = 0; 33 | for (var i = 0; i < xArray.Length; i++) 34 | { 35 | result += BitCount(xArray[i]); 36 | } 37 | return result; 38 | } 39 | 40 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 41 | private static void ArrayCopy(ushort[] input, int iStart, ushort[] output, int oStart, int length) 42 | { 43 | Buffer.BlockCopy(input, iStart * sizeof(ushort), output, oStart * sizeof(ushort), length * sizeof(ushort)); 44 | } 45 | 46 | public static int UnionArrays(ushort[] set1, int length1, ushort[] set2, int length2, ushort[] buffer) 47 | { 48 | var pos = 0; 49 | int k1 = 0, k2 = 0; 50 | if (0 == length2) 51 | { 52 | ArrayCopy(set1, 0, buffer, 0, length1); 53 | return length1; 54 | } 55 | if (0 == length1) 56 | { 57 | ArrayCopy(set2, 0, buffer, 0, length2); 58 | return length2; 59 | } 60 | var s1 = set1[k1]; 61 | var s2 = set2[k2]; 62 | while (true) 63 | { 64 | int v1 = s1; 65 | int v2 = s2; 66 | if (v1 < v2) 67 | { 68 | buffer[pos++] = s1; 69 | ++k1; 70 | if (k1 >= length1) 71 | { 72 | ArrayCopy(set2, k2, buffer, pos, length2 - k2); 73 | return pos + length2 - k2; 74 | } 75 | s1 = set1[k1]; 76 | } 77 | else if (v1 == v2) 78 | { 79 | buffer[pos++] = s1; 80 | ++k1; 81 | ++k2; 82 | if (k1 >= length1) 83 | { 84 | ArrayCopy(set2, k2, buffer, pos, length2 - k2); 85 | return pos + length2 - k2; 86 | } 87 | if (k2 >= length2) 88 | { 89 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 90 | return pos + length1 - k1; 91 | } 92 | s1 = set1[k1]; 93 | s2 = set2[k2]; 94 | } 95 | else // if (set1[k1]>set2[k2]) 96 | { 97 | buffer[pos++] = s2; 98 | ++k2; 99 | if (k2 >= length2) 100 | { 101 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 102 | return pos + length1 - k1; 103 | } 104 | s2 = set2[k2]; 105 | } 106 | } 107 | } 108 | 109 | public static int DifferenceArrays(ushort[] set1, int length1, ushort[] set2, int length2, ushort[] buffer) 110 | { 111 | var pos = 0; 112 | int k1 = 0, k2 = 0; 113 | if (0 == length2) 114 | { 115 | ArrayCopy(set1, 0, buffer, 0, length1); 116 | return length1; 117 | } 118 | if (0 == length1) 119 | { 120 | return 0; 121 | } 122 | var s1 = set1[k1]; 123 | var s2 = set2[k2]; 124 | while (true) 125 | { 126 | if (s1 < s2) 127 | { 128 | buffer[pos++] = s1; 129 | ++k1; 130 | if (k1 >= length1) 131 | { 132 | break; 133 | } 134 | s1 = set1[k1]; 135 | } 136 | else if (s1 == s2) 137 | { 138 | ++k1; 139 | ++k2; 140 | if (k1 >= length1) 141 | { 142 | break; 143 | } 144 | if (k2 >= length2) 145 | { 146 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 147 | return pos + length1 - k1; 148 | } 149 | s1 = set1[k1]; 150 | s2 = set2[k2]; 151 | } 152 | else // if (val1>val2) 153 | { 154 | ++k2; 155 | if (k2 >= length2) 156 | { 157 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 158 | return pos + length1 - k1; 159 | } 160 | s2 = set2[k2]; 161 | } 162 | } 163 | return pos; 164 | } 165 | 166 | public static int IntersectArrays(ushort[] set1, int length1, ushort[] set2, int length2, ushort[] buffer) 167 | { 168 | if (set1.Length << 6 < set2.Length) 169 | { 170 | return OneSidedGallopingIntersect2By2(set1, length1, set2, length2, buffer); 171 | } 172 | if (set2.Length << 6 < set1.Length) 173 | { 174 | return OneSidedGallopingIntersect2By2(set2, length2, set1, length1, buffer); 175 | } 176 | return LocalIntersect2By2(set1, length1, set2, length2, buffer); 177 | } 178 | 179 | private static int LocalIntersect2By2(ushort[] set1, int length1, ushort[] set2, int length2, ushort[] buffer) 180 | { 181 | if ((0 == length1) || (0 == length2)) 182 | { 183 | return 0; 184 | } 185 | var k1 = 0; 186 | var k2 = 0; 187 | var pos = 0; 188 | var s1 = set1[k1]; 189 | var s2 = set2[k2]; 190 | 191 | while (true) 192 | { 193 | int v1 = s1; 194 | int v2 = s2; 195 | if (v2 < v1) 196 | { 197 | do 198 | { 199 | ++k2; 200 | if (k2 == length2) 201 | { 202 | return pos; 203 | } 204 | s2 = set2[k2]; 205 | v2 = s2; 206 | } while (v2 < v1); 207 | } 208 | if (v1 < v2) 209 | { 210 | do 211 | { 212 | ++k1; 213 | if (k1 == length1) 214 | { 215 | return pos; 216 | } 217 | s1 = set1[k1]; 218 | v1 = s1; 219 | } while (v1 < v2); 220 | } 221 | else // (set2[k2] == set1[k1]) 222 | { 223 | buffer[pos++] = s1; 224 | ++k1; 225 | if (k1 == length1) 226 | { 227 | break; 228 | } 229 | ++k2; 230 | if (k2 == length2) 231 | { 232 | break; 233 | } 234 | s1 = set1[k1]; 235 | s2 = set2[k2]; 236 | } 237 | } 238 | return pos; 239 | } 240 | 241 | private static int OneSidedGallopingIntersect2By2(ushort[] smallSet, int smallLength, ushort[] largeSet, int largeLength, ushort[] buffer) 242 | { 243 | if (0 == smallLength) 244 | { 245 | return 0; 246 | } 247 | var k1 = 0; 248 | var k2 = 0; 249 | var pos = 0; 250 | var s1 = largeSet[k1]; 251 | var s2 = smallSet[k2]; 252 | while (true) 253 | { 254 | if (s1 < s2) 255 | { 256 | k1 = AdvanceUntil(largeSet, k1, largeLength, s2); 257 | if (k1 == largeLength) 258 | { 259 | break; 260 | } 261 | s1 = largeSet[k1]; 262 | } 263 | if (s2 < s1) 264 | { 265 | ++k2; 266 | if (k2 == smallLength) 267 | { 268 | break; 269 | } 270 | s2 = smallSet[k2]; 271 | } 272 | else // (set2[k2] == set1[k1]) 273 | { 274 | buffer[pos++] = s2; 275 | ++k2; 276 | if (k2 == smallLength) 277 | { 278 | break; 279 | } 280 | s2 = smallSet[k2]; 281 | k1 = AdvanceUntil(largeSet, k1, largeLength, s2); 282 | if (k1 == largeLength) 283 | { 284 | break; 285 | } 286 | s1 = largeSet[k1]; 287 | } 288 | } 289 | return pos; 290 | } 291 | 292 | /// 293 | /// Find the smallest integer larger than pos such that array[pos]>= min. 294 | /// otherwise return length 295 | /// -> The first line is BinarySearch with pos + 1, the second line is the bitwise complement if the value can't be 296 | /// found 297 | /// 298 | public static int AdvanceUntil(ushort[] array, int pos, int length, ushort min) 299 | { 300 | var start = pos + 1; // check the next one 301 | if ((start >= length) || (array[start] >= min)) // the simple cases 302 | { 303 | return start; 304 | } 305 | var result = Array.BinarySearch(array, start, length - start, min); 306 | return result < 0 ? ~result : result; 307 | } 308 | 309 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 310 | public static ushort HighBits(int value) 311 | { 312 | return (ushort) (value >> 16); 313 | } 314 | 315 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 316 | public static ushort LowBits(int value) 317 | { 318 | return (ushort) (value & 0xFFFF); 319 | } 320 | 321 | public static int XorArrays(ushort[] set1, int length1, ushort[] set2, int length2, ushort[] buffer) 322 | { 323 | var pos = 0; 324 | int k1 = 0, k2 = 0; 325 | if (0 == length2) 326 | { 327 | ArrayCopy(set1, 0, buffer, 0, length1); 328 | return length1; 329 | } 330 | if (0 == length1) 331 | { 332 | ArrayCopy(set2, 0, buffer, 0, length2); 333 | return length2; 334 | } 335 | var s1 = set1[k1]; 336 | var s2 = set2[k2]; 337 | while (true) 338 | { 339 | if (s1 < s2) 340 | { 341 | buffer[pos++] = s1; 342 | ++k1; 343 | if (k1 >= length1) 344 | { 345 | ArrayCopy(set2, k2, buffer, pos, length2 - k2); 346 | return pos + length2 - k2; 347 | } 348 | s1 = set1[k1]; 349 | } 350 | else if (s1 == s2) 351 | { 352 | ++k1; 353 | ++k2; 354 | if (k1 >= length1) 355 | { 356 | ArrayCopy(set2, k2, buffer, pos, length2 - k2); 357 | return pos + length2 - k2; 358 | } 359 | if (k2 >= length2) 360 | { 361 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 362 | return pos + length1 - k1; 363 | } 364 | s1 = set1[k1]; 365 | s2 = set2[k2]; 366 | } 367 | else // if (val1>val2) 368 | { 369 | buffer[pos++] = s2; 370 | ++k2; 371 | if (k2 >= length2) 372 | { 373 | ArrayCopy(set1, k1, buffer, pos, length1 - k1); 374 | return pos + length1 - k1; 375 | } 376 | s2 = set2[k2]; 377 | } 378 | } 379 | } 380 | } 381 | } -------------------------------------------------------------------------------- /real-roaring-dataset/README: -------------------------------------------------------------------------------- 1 | Real data sets for bitmap testing 2 | packaged by Daniel Lemire on April 3rd 2014 3 | 4 | Essentially, each file represents a set of integer values. You can create 5 | bitmaps out of these files. 6 | 7 | In many cases, the description of the data sets is provided in our paper: 8 | Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, Better bitmap performance with Roaring bitmaps, arXiv:1402.6407. 9 | http://arxiv.org/abs/1402.6407 10 | 11 | To be used with software published on http://roaringbitmap.org/ 12 | (Specifically at http://roaringbitmap.org/RoaringBitmap-0.3.4-SNAPSHOT-sources.jar ) 13 | 14 | 15 | 16 | 17 | Files starting with the prefix "dimension" were prepared by Xavier Léauté from 18 | a Druid dump. 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /real-roaring-dataset/census-income.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/census-income.zip -------------------------------------------------------------------------------- /real-roaring-dataset/census-income_srt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/census-income_srt.zip -------------------------------------------------------------------------------- /real-roaring-dataset/census1881.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/census1881.zip -------------------------------------------------------------------------------- /real-roaring-dataset/census1881_srt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/census1881_srt.zip -------------------------------------------------------------------------------- /real-roaring-dataset/dimension_003.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/dimension_003.zip -------------------------------------------------------------------------------- /real-roaring-dataset/dimension_008.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/dimension_008.zip -------------------------------------------------------------------------------- /real-roaring-dataset/dimension_033.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/dimension_033.zip -------------------------------------------------------------------------------- /real-roaring-dataset/uscensus2000.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/uscensus2000.zip -------------------------------------------------------------------------------- /real-roaring-dataset/weather_sept_85.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/weather_sept_85.zip -------------------------------------------------------------------------------- /real-roaring-dataset/weather_sept_85_srt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/weather_sept_85_srt.zip -------------------------------------------------------------------------------- /real-roaring-dataset/wikileaks-noquotes.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/wikileaks-noquotes.zip -------------------------------------------------------------------------------- /real-roaring-dataset/wikileaks-noquotes_srt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khamroevjs/RoaringBitmap/35090aa78f927482d3a4e0db4bb5b641dbee9187/real-roaring-dataset/wikileaks-noquotes_srt.zip --------------------------------------------------------------------------------