├── .gitignore ├── LICENSE.txt ├── README.md ├── pom.xml └── src ├── main └── java │ └── edu │ └── utexas │ └── ece │ └── mpc │ └── bloomier │ ├── ImmutableBloomierFilter.java │ ├── MutableBloomierFilter.java │ └── internal │ ├── BloomierHasher.java │ ├── OrderAndMatch.java │ ├── OrderAndMatchFinder.java │ └── SingletonFindingTweaker.java └── test └── java └── edu └── utexas └── ece └── mpc └── bloomier ├── ImmutableBloomierFilterTest.java └── MutableBloomierFilterTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | ### /Users/egrim/.gitignore-boilerplates/java.gitignore 2 | 3 | *.class 4 | 5 | # Package Files # 6 | *.jar 7 | *.war 8 | *.ear 9 | 10 | 11 | ### /Users/egrim/.gitignore-boilerplates/Global/eclipse.gitignore 12 | 13 | *.pydevproject 14 | #.project 15 | .metadata 16 | bin/** 17 | tmp/** 18 | tmp/**/* 19 | *.tmp 20 | *.bak 21 | *.swp 22 | *~.nib 23 | local.properties 24 | #.classpath 25 | .settings/ 26 | .loadpath 27 | 28 | # External tool builders 29 | .externalToolBuilders/ 30 | 31 | # Locally stored "Eclipse launch configurations" 32 | *.launch 33 | 34 | # CDT-specific 35 | .cproject 36 | 37 | # PDT-specific 38 | .buildpath 39 | 40 | # Maven 41 | target 42 | 43 | # more Eclipse stuff 44 | .classpath 45 | .project 46 | .settings 47 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, The University of Texas at Austin 2 | Produced in the Mobile and Pervasive Computing Lab 3 | Originally written by Evan Grim 4 | 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 11 | Neither the name of the University of Texas at Austin nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a Java implementation of the bloomier filter proposed in [Chazelle et al.][paper] (section 3: An Optimal Bloomier Filter). As far as I am aware this is the only freely available implementation online. It includes separate classes for both the immutable and mutable structures. Internally, the kryo serialization library is utilized to efficiently convert the values stored into byte arrays which can be utilized as described in the paper. Otherwise it is pretty much a straight implementation of the proposed construction and accessor algorithm. Suggestions and patches/pull requests gladly accepted to improve upon this humble first pass. 2 | 3 | [paper]: http://webee.technion.ac.il/~ayellet/Ps/nelson.pdf 4 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | 5 | com.github.egrim 6 | java-bloomier-filter 7 | jar 8 | 0.1-SNAPSHOT 9 | java-bloomier-filter 10 | A Java implementation of the bloomier filter data structure 11 | https://github.com/egrim/java-bloomier-filter 12 | 13 | 14 | Other 15 | repo 16 | https://raw.github.com/egrim/java-bloomier-filter/master/LICENSE.txt 17 | 18 | 19 | 20 | scm:git:git://github.com/hector-client/hector.git 21 | scm:git:git@github.com/egrim/java-bloomier-filter.git 22 | https://github.com/egrim/java-bloomier-filter 23 | 24 | 25 | 26 | github 27 | https://github.com/egrim/java-bloomier-filter/issues 28 | 29 | 30 | 31 | 32 | github.com 33 | gitsite:git@github.com/egrim/java-bloomier-filter.git 34 | 35 | 36 | 37 | 38 | 39 | Evan Grim 40 | egrim 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | org.apache.maven.scm 49 | maven-scm-provider-gitexe 50 | 1.3 51 | 52 | 53 | org.apache.maven.scm 54 | maven-scm-manager-plexus 55 | 1.3 56 | 57 | 58 | org.kathrynhuxtable.maven.wagon 59 | wagon-gitsite 60 | 0.3.1 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.apache.maven.plugins 68 | maven-source-plugin 69 | 2.2.1 70 | 71 | 72 | attach-sources 73 | 74 | jar 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | maven-clean-plugin 85 | 2.5 86 | 87 | 88 | maven-compiler-plugin 89 | 3.0 90 | 91 | 1.7 92 | 1.7 93 | true 94 | true 95 | true 96 | 97 | 98 | 99 | maven-deploy-plugin 100 | 2.7 101 | 102 | 103 | maven-install-plugin 104 | 2.4 105 | 106 | 107 | maven-javadoc-plugin 108 | 2.9 109 | 110 | 111 | maven-jar-plugin 112 | 2.4 113 | 114 | 115 | maven-release-plugin 116 | 2.3.2 117 | 118 | clean install 119 | 120 | 121 | 122 | maven-site-plugin 123 | 3.2 124 | 125 | 126 | maven-resources-plugin 127 | 2.6 128 | 129 | 130 | maven-surefire-plugin 131 | 2.12.4 132 | 133 | 134 | 135 | 136 | 137 | org.codehaus.mojo 138 | cobertura-maven-plugin 139 | 2.5.2 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | maven-jxr-plugin 149 | 2.3 150 | 151 | 152 | maven-surefire-report-plugin 153 | 2.12.4 154 | 155 | 156 | maven-pmd-plugin 157 | 2.7.1 158 | 159 | 160 | org.codehaus.mojo 161 | taglist-maven-plugin 162 | 2.4 163 | 164 | 165 | maven-javadoc-plugin 166 | 2.9 167 | 168 | 169 | http://java.sun.com/j2se/1.6.0/docs/api/ 170 | 171 | true 172 | 900m 173 | 1.6 174 | 175 | 176 | 177 | org.codehaus.mojo 178 | cobertura-maven-plugin 179 | 2.5.2 180 | 181 | 182 | html 183 | xml 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | UTF-8 192 | UTF-8 193 | UTF-8 194 | 195 | 196 | 197 | 198 | junit 199 | junit 200 | test 201 | 202 | 203 | com.esotericsoftware.kryo 204 | kryo 205 | 206 | 207 | 208 | 209 | 210 | 211 | junit 212 | junit 213 | 4.11 214 | test 215 | 216 | 217 | com.esotericsoftware.kryo 218 | kryo 219 | 2.24.0 220 | 221 | 222 | 223 | 224 | -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/ImmutableBloomierFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier; 13 | 14 | import java.io.ByteArrayOutputStream; 15 | import java.nio.ByteBuffer; 16 | import java.util.Arrays; 17 | import java.util.HashSet; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.Set; 21 | import java.util.concurrent.TimeoutException; 22 | 23 | import com.esotericsoftware.kryo.Kryo; 24 | import com.esotericsoftware.kryo.io.ByteBufferInput; 25 | import com.esotericsoftware.kryo.io.Input; 26 | import com.esotericsoftware.kryo.io.Output; 27 | 28 | import edu.utexas.ece.mpc.bloomier.internal.BloomierHasher; 29 | import edu.utexas.ece.mpc.bloomier.internal.OrderAndMatch; 30 | import edu.utexas.ece.mpc.bloomier.internal.OrderAndMatchFinder; 31 | 32 | public class ImmutableBloomierFilter { 33 | protected final Kryo kryo; 34 | protected final Output kryoSerializer; 35 | 36 | protected final Class valueClass; 37 | 38 | protected final int m; 39 | protected final int k; 40 | protected final int q; 41 | 42 | protected long hashSeed; 43 | protected BloomierHasher hasher; 44 | 45 | protected byte[][] table; 46 | protected int tableEntrySize; 47 | 48 | private ImmutableBloomierFilter(int m, int k, int q, Class valueClass) { 49 | kryo = new Kryo(); 50 | kryoSerializer = new Output(DEFAULT_OBJECT_BUFFER_INITIAL_SIZE, Integer.MAX_VALUE); 51 | 52 | this.m = m; 53 | this.k = k; 54 | this.q = q; 55 | 56 | this.valueClass = valueClass; 57 | 58 | // Create table with correctly sized byte arrays for encoded entries 59 | tableEntrySize = q / 8; // FIXME: why isn't this + 1?! 60 | table = new byte[m][tableEntrySize]; 61 | 62 | // The rest of the initialization will be handled by create() in public constructors 63 | } 64 | 65 | public ImmutableBloomierFilter(Map map, int m, int k, int q, Class valueClass, 66 | int timeoutMs) throws TimeoutException { 67 | this(m, k, q, valueClass); 68 | 69 | OrderAndMatchFinder oamf = new OrderAndMatchFinder(map.keySet(), m, k, q); 70 | OrderAndMatch oam = oamf.find(timeoutMs); 71 | create(map, oam); 72 | } 73 | 74 | public ImmutableBloomierFilter(Map map, int m, int k, int q, Class valueClass, 75 | long hashSeedHint) { 76 | this(m, k, q, valueClass); 77 | 78 | OrderAndMatchFinder oamf = new OrderAndMatchFinder(map.keySet(), m, k, q, 79 | hashSeedHint); 80 | OrderAndMatch oam; 81 | try { 82 | oam = oamf.find(Integer.MAX_VALUE); 83 | } catch (TimeoutException e) { 84 | throw new AssertionError("Should never be possible"); 85 | } 86 | create(map, oam); 87 | } 88 | 89 | public ImmutableBloomierFilter(Map map, int m, int k, int q, Class valueClass, 90 | int timeoutMs, long hashSeedHint) throws TimeoutException { 91 | this(m, k, q, valueClass); 92 | 93 | OrderAndMatchFinder oamf = new OrderAndMatchFinder(map.keySet(), m, k, q, 94 | hashSeedHint); 95 | OrderAndMatch oam = oamf.find(timeoutMs); 96 | create(map, oam); 97 | } 98 | 99 | // This package private constructor can be used by entities that want to supply their own OrderAndMatch 100 | ImmutableBloomierFilter(Map map, int m, int k, int q, Class valueClass, 101 | OrderAndMatch oam) { 102 | this(m, k, q, valueClass); 103 | 104 | create(map, oam); 105 | } 106 | 107 | public ImmutableBloomierFilter(int m, int k, int q, Class valueClass, long hashSeed, 108 | byte[][] table) { 109 | this(m, k, q, valueClass); 110 | 111 | this.hashSeed = hashSeed; 112 | this.table = table; 113 | 114 | hasher = new BloomierHasher(hashSeed, m, k, q); 115 | } 116 | 117 | public ImmutableBloomierFilter(ImmutableBloomierFilter orig) { 118 | this(orig.m, orig.k, orig.q, orig.valueClass, orig.hashSeed, orig.table); // TODO: it should be okay to share 119 | // the underlying table since it's 120 | // immutable, but beware this might 121 | // not be true 122 | } 123 | 124 | private void create(Map map, OrderAndMatch oam) { 125 | hashSeed = oam.getHashSeed(); 126 | hasher = new BloomierHasher(hashSeed, m, k, q); 127 | 128 | List pi = oam.getPi(); 129 | List tau = oam.getTau(); 130 | 131 | for (int i = 0; i < pi.size(); i++) { 132 | K key = pi.get(i); 133 | V value = map.get(key); 134 | byte[] encodedValue = encode(value); 135 | 136 | int[] neighborhood = hasher.getNeighborhood(key); 137 | byte[] mask = hasher.getM(key); 138 | 139 | int indexOfStorage = neighborhood[tau.get(i)]; 140 | byte[] valueToStore = new byte[tableEntrySize]; 141 | 142 | byteArrayXor(valueToStore, encodedValue); 143 | byteArrayXor(valueToStore, mask); 144 | 145 | // TODO: Duplicate neighborhood hashes cause problems - temp fix: dedup 146 | Set neighborhoodSet = new HashSet(); 147 | for (int hash: neighborhood) { 148 | neighborhoodSet.add(hash); 149 | } 150 | 151 | for (int hash: neighborhoodSet) { 152 | byteArrayXor(valueToStore, table[hash]); 153 | } 154 | 155 | table[indexOfStorage] = valueToStore; 156 | } 157 | 158 | // TODO: if hasher caches hashes, clear cache here (to reclaim memory) 159 | } 160 | 161 | public V get(K key) { 162 | int[] neighborhood = hasher.getNeighborhood(key); 163 | byte[] mask = hasher.getM(key); 164 | 165 | byte[] resultArray = new byte[tableEntrySize]; 166 | 167 | byteArrayXor(resultArray, mask); 168 | 169 | // TODO: Duplicate neighborhood hashes cause problems - temp fix: dedup 170 | Set neighborhoodSet = new HashSet(); 171 | for (int hash: neighborhood) { 172 | neighborhoodSet.add(hash); 173 | } 174 | 175 | for (int hash: neighborhoodSet) { 176 | byteArrayXor(resultArray, table[hash]); 177 | } 178 | 179 | return decode(resultArray); 180 | } 181 | 182 | public int getM() { 183 | return m; 184 | } 185 | 186 | public int getK() { 187 | return k; 188 | } 189 | 190 | public int getQ() { 191 | return q; 192 | } 193 | 194 | public long getHashSeed() { 195 | return hashSeed; 196 | } 197 | 198 | public byte[][] getTable() { 199 | return table; 200 | } 201 | 202 | private void byteArrayXor(byte[] resultArray, byte[] xorArray) { 203 | // TODO: may want to rewrite this to more intuitively handle hetero-sized arrays 204 | int length = Math.min(resultArray.length, xorArray.length); 205 | 206 | for (int i = 0; i < length; i++) { 207 | resultArray[i] ^= xorArray[i]; 208 | } 209 | } 210 | 211 | private byte[] encode(V value) { 212 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 213 | try(Output output = new Output(baos)) { 214 | kryo.writeObject(output, value); 215 | output.close(); 216 | byte[] serializedValue = baos.toByteArray(); 217 | if (serializedValue.length > tableEntrySize) { 218 | throw new IllegalArgumentException( 219 | "Encoded values are too big to fit in table (q=" + q 220 | + "; must be >= " + serializedValue.length 221 | * Byte.SIZE + ")"); 222 | } 223 | 224 | // Pad with zeros up to required tableEntrySize 225 | return Arrays.copyOf(serializedValue, tableEntrySize); 226 | } 227 | } 228 | 229 | private V decode(byte[] value) { 230 | ByteBuffer buffer = ByteBuffer.wrap(value); 231 | Input input = new ByteBufferInput(buffer); 232 | V result = kryo.readObject(input, valueClass); 233 | 234 | // Check leftovers (all must be zero of this is a detected false positive) 235 | while (buffer.hasRemaining()) { 236 | if (buffer.get() != 0) { 237 | return null; 238 | } 239 | } 240 | 241 | return result; 242 | } 243 | 244 | private static final int DEFAULT_OBJECT_BUFFER_INITIAL_SIZE = 2 * 1024; 245 | } 246 | -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/MutableBloomierFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier; 13 | 14 | import java.util.HashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.concurrent.TimeoutException; 18 | 19 | import edu.utexas.ece.mpc.bloomier.internal.BloomierHasher; 20 | import edu.utexas.ece.mpc.bloomier.internal.OrderAndMatch; 21 | import edu.utexas.ece.mpc.bloomier.internal.OrderAndMatchFinder; 22 | 23 | public class MutableBloomierFilter { 24 | private ImmutableBloomierFilter tauTable; 25 | private V[] valueTable; 26 | 27 | private long hashSeed; 28 | private BloomierHasher hasher; 29 | 30 | private List pi; 31 | private List tau; 32 | 33 | @SuppressWarnings("unchecked") 34 | public MutableBloomierFilter(Map map, int m, int k, int q, long timeoutMs) 35 | throws TimeoutException { 36 | OrderAndMatchFinder oamf = new OrderAndMatchFinder(map.keySet(), m, k, q); 37 | OrderAndMatch oam = oamf.find(timeoutMs); 38 | 39 | valueTable = (V[]) new Object[m]; 40 | 41 | hashSeed = oam.getHashSeed(); 42 | hasher = new BloomierHasher(hashSeed, m, k, q); 43 | 44 | pi = oam.getPi(); 45 | tau = oam.getTau(); 46 | 47 | Map tauMap = new HashMap(); 48 | for (int i = 0; i < pi.size(); i++) { 49 | K key = pi.get(i); 50 | V value = map.get(key); 51 | 52 | int iota = tau.get(i); 53 | tauMap.put(key, iota); 54 | 55 | int hashIndex = hasher.getNeighborhood(key)[iota]; 56 | valueTable[hashIndex] = value; 57 | } 58 | 59 | tauTable = new ImmutableBloomierFilter(tauMap, m, k, q, Integer.class, oam); 60 | } 61 | 62 | public V get(K key) { 63 | Integer iota = tauTable.get(key); 64 | if (iota == null) { 65 | return null; 66 | } 67 | 68 | Integer hashIndex = hasher.getNeighborhood(key)[iota]; 69 | return valueTable[hashIndex]; 70 | } 71 | 72 | public void set(K key, V value) { 73 | Integer iota = tauTable.get(key); 74 | 75 | if (iota == null) { 76 | throw new IllegalArgumentException("Supplied key (" + key + ") is invalid"); 77 | } 78 | 79 | Integer hashIndex = hasher.getNeighborhood(key)[iota]; 80 | valueTable[hashIndex] = value; 81 | } 82 | } -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/internal/BloomierHasher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier.internal; 13 | 14 | import java.io.DataInputStream; 15 | import java.io.IOException; 16 | import java.io.InputStream; 17 | import java.nio.ByteBuffer; 18 | import java.security.MessageDigest; 19 | import java.security.NoSuchAlgorithmException; 20 | import java.util.ArrayDeque; 21 | import java.util.Queue; 22 | 23 | public class BloomierHasher { 24 | // TODO: memoize/cache hashes 25 | private long hashSeed; 26 | 27 | private int m; 28 | private int k; 29 | private int q; 30 | 31 | public BloomierHasher(long hashSeed, int m, int k, int q) { 32 | this.hashSeed = hashSeed; 33 | this.m = m; 34 | this.k = k; 35 | this.q = q; 36 | } 37 | 38 | public int[] getNeighborhood(K key) { 39 | DataInputStream stream = null; 40 | final int[] hashes = new int[k]; 41 | try { 42 | stream = new DataInputStream(new HashInputStream(key)); 43 | 44 | for (int i = 0; i < hashes.length; i++) { 45 | try { 46 | hashes[i] = Math.abs(stream.readInt()) % m; // Massage value to be in [0,m) 47 | } catch (IOException e) { 48 | // Shouldn't be possible 49 | throw new IllegalStateException("Hash generation failed", e); 50 | } 51 | } 52 | } finally { 53 | if(stream != null) { 54 | try 55 | { 56 | stream.close(); 57 | } catch (IOException e) 58 | { 59 | // just ignore failing to close 60 | } 61 | } 62 | } 63 | 64 | return hashes; 65 | } 66 | 67 | public byte[] getM(K key) { 68 | DataInputStream stream = null; 69 | byte[] hashes = new byte[q / Byte.SIZE + 1]; 70 | 71 | try { 72 | stream = new DataInputStream(new HashInputStream(key)); 73 | 74 | for (int i = 0; i < hashes.length; i++) { 75 | try { 76 | hashes[i] = stream.readByte(); 77 | } catch (IOException e) { 78 | // Shouldn't be possible 79 | throw new IllegalStateException("Hash generation failed", e); 80 | } 81 | } 82 | } finally { 83 | if(stream != null) { 84 | try 85 | { 86 | stream.close(); 87 | } catch (IOException e) 88 | { 89 | // just ignore 90 | } 91 | } 92 | } 93 | 94 | return hashes; 95 | } 96 | 97 | private class HashInputStream extends InputStream { 98 | private byte[] data; 99 | private long salt; 100 | private MessageDigest md; 101 | 102 | private Queue buffer; 103 | 104 | public HashInputStream(K key) { 105 | this.data = ByteBuffer.allocate(Integer.SIZE / Byte.SIZE).putInt(key.hashCode()) 106 | .array(); // TODO: use .toString instead of hashCode to get a better hash range 107 | this.salt = hashSeed; 108 | 109 | try { 110 | md = MessageDigest.getInstance("MD5"); 111 | } catch (NoSuchAlgorithmException e) { 112 | throw new RuntimeException("Missing required hashing algorithm", e); 113 | } 114 | 115 | buffer = new ArrayDeque(md.getDigestLength()); 116 | topOff(); 117 | } 118 | 119 | @Override 120 | public int read() { 121 | if (buffer.isEmpty()) { 122 | topOff(); 123 | } 124 | return buffer.remove() - Byte.MIN_VALUE; 125 | } 126 | 127 | private void topOff() { 128 | md.update(ByteBuffer.allocate(Long.SIZE / Byte.SIZE).putLong(salt).array()); 129 | md.update(data); 130 | 131 | for (byte b: md.digest()) { 132 | buffer.add(b); 133 | } 134 | 135 | salt++; 136 | } 137 | } 138 | } -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/internal/OrderAndMatch.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier.internal; 13 | 14 | import java.util.List; 15 | 16 | public class OrderAndMatch { 17 | 18 | private long hashSeed; 19 | private List pi; 20 | private List tau; 21 | 22 | public OrderAndMatch(long hashSeed, List pi, List tau) { 23 | this.hashSeed = hashSeed; 24 | this.pi = pi; 25 | this.tau = tau; 26 | } 27 | 28 | public List getPi() { 29 | return pi; 30 | } 31 | 32 | public List getTau() { 33 | return tau; 34 | } 35 | 36 | public long getHashSeed() { 37 | return hashSeed; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/internal/OrderAndMatchFinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier.internal; 13 | 14 | import java.util.ArrayList; 15 | import java.util.Collection; 16 | import java.util.LinkedList; 17 | import java.util.List; 18 | import java.util.Queue; 19 | import java.util.Timer; 20 | import java.util.TimerTask; 21 | import java.util.concurrent.TimeoutException; 22 | import java.util.concurrent.atomic.AtomicBoolean; 23 | 24 | public class OrderAndMatchFinder { 25 | long hashSeed = Long.MIN_VALUE; 26 | BloomierHasher hasher; 27 | 28 | List pi; 29 | List tau; 30 | 31 | OrderAndMatch oam; 32 | 33 | Collection keys; 34 | int m; 35 | int k; 36 | int q; 37 | 38 | public OrderAndMatchFinder(Collection keys, int m, int k, int q) { 39 | this.keys = keys; 40 | this.m = m; 41 | this.k = k; 42 | this.q = q; 43 | } 44 | 45 | public OrderAndMatchFinder(Collection keys, int m, int k, int q, long hashSeedHint) { 46 | this(keys, m, k, q); 47 | 48 | hashSeed = hashSeedHint; 49 | } 50 | 51 | public OrderAndMatch find(long timeoutMs) throws TimeoutException { 52 | final AtomicBoolean hasTimedOut = new AtomicBoolean(false); 53 | Timer timer = null; 54 | try { 55 | if (timeoutMs < Long.MAX_VALUE) { 56 | timer = new Timer(true); 57 | timer.schedule(new TimerTask() { 58 | 59 | @Override 60 | public void run() { 61 | hasTimedOut.set(true); 62 | } 63 | }, timeoutMs); 64 | } 65 | 66 | for (long i = 0; i < Long.MAX_VALUE; i++) { 67 | // First check for timeout 68 | if (hasTimedOut.get()) { 69 | throw new TimeoutException( 70 | String.format("Could not find order and matching for key set in alloted time with specified parameters (m=%d;k=%d;q=%d)", 71 | m, k, q)); 72 | } 73 | 74 | hasher = new BloomierHasher(hashSeed, m, k, q); 75 | 76 | pi = new ArrayList(keys.size()); 77 | tau = new ArrayList(keys.size()); 78 | 79 | if (findMatch(new ArrayList(keys))) { // findMatch modifies key collection, so make copy 80 | oam = new OrderAndMatch(hashSeed, pi, tau); 81 | break; 82 | } 83 | 84 | hashSeed++; // will wrap around if a hashSeedHint was provided 85 | } 86 | 87 | } finally { 88 | if (timer != null) { 89 | timer.cancel(); 90 | } 91 | } 92 | 93 | return oam; 94 | } 95 | 96 | public boolean isFound() { 97 | if (oam == null) { 98 | return false; 99 | } else { 100 | return true; 101 | } 102 | } 103 | 104 | public OrderAndMatch getOrderAndMatch() { 105 | return oam; 106 | } 107 | 108 | public BloomierHasher getHasher() { 109 | return hasher; 110 | } 111 | 112 | private boolean findMatch(Collection remainingKeys) { 113 | if (remainingKeys.isEmpty()) { 114 | return true; 115 | } 116 | 117 | Queue piQueue = new LinkedList(); 118 | Queue tauQueue = new LinkedList(); 119 | 120 | SingletonFindingTweaker tweaker = new SingletonFindingTweaker(remainingKeys, hasher); 121 | 122 | for (K key: remainingKeys) { 123 | Integer iota = tweaker.tweak(key); 124 | if (iota != null) { 125 | piQueue.add(key); 126 | tauQueue.add(iota); 127 | } 128 | } 129 | 130 | if (piQueue.isEmpty()) { 131 | return false; 132 | } 133 | 134 | // Only pass along non-"easy" keys to next iteration 135 | remainingKeys.removeAll(piQueue); 136 | 137 | if (remainingKeys.isEmpty() == false) { 138 | if (findMatch(remainingKeys) == false) { 139 | return false; 140 | } 141 | } 142 | 143 | pi.addAll(piQueue); 144 | tau.addAll(tauQueue); 145 | 146 | return true; 147 | } 148 | } -------------------------------------------------------------------------------- /src/main/java/edu/utexas/ece/mpc/bloomier/internal/SingletonFindingTweaker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier.internal; 13 | 14 | import java.util.Collection; 15 | import java.util.HashSet; 16 | import java.util.Set; 17 | 18 | public class SingletonFindingTweaker { 19 | private BloomierHasher hasher; 20 | private Set nonSingletons; 21 | 22 | public SingletonFindingTweaker(Collection keys, BloomierHasher hasher) { 23 | this.hasher = hasher; 24 | 25 | Set hashesSeen = new HashSet(); 26 | nonSingletons = new HashSet(); 27 | 28 | for (K key: keys) { 29 | int[] neighborhood = hasher.getNeighborhood(key); 30 | 31 | // First pass - see if any currently qualify as singletons 32 | for (int hash: neighborhood) { 33 | if (hashesSeen.contains(hash)) { 34 | nonSingletons.add(hash); 35 | } 36 | } 37 | 38 | // Second pass - add to seen hashes 39 | for (int hash: neighborhood) { 40 | hashesSeen.add(hash); 41 | } 42 | } 43 | } 44 | 45 | public Integer tweak(K key) { 46 | int[] neighborhood = hasher.getNeighborhood(key); 47 | for (int i = 0; i < neighborhood.length; i++) { 48 | if (nonSingletons.contains(neighborhood[i]) == false) { 49 | return i; 50 | } 51 | } 52 | return null; 53 | } 54 | } -------------------------------------------------------------------------------- /src/test/java/edu/utexas/ece/mpc/bloomier/ImmutableBloomierFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier; 13 | 14 | import static org.hamcrest.CoreMatchers.equalTo; 15 | import static org.hamcrest.CoreMatchers.is; 16 | import static org.junit.Assert.*; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map.Entry; 20 | import java.util.Set; 21 | 22 | import org.junit.Assert; 23 | import org.junit.Before; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | import org.junit.rules.ErrorCollector; 27 | 28 | import edu.utexas.ece.mpc.bloomier.ImmutableBloomierFilter; 29 | 30 | public class ImmutableBloomierFilterTest { 31 | ImmutableBloomierFilter uut; 32 | 33 | @Rule 34 | public final ErrorCollector errorCollector = new ErrorCollector(); 35 | 36 | private final HashMap originalMap = new HashMap(); 37 | 38 | @Before 39 | public void setUp() throws Exception { 40 | 41 | for (int i = 0; i < 1000; i++) { 42 | originalMap.put(i, i); 43 | } 44 | 45 | uut = new ImmutableBloomierFilter(originalMap, originalMap.keySet().size() * 10, 10, 32, 46 | Integer.class, 10000); 47 | } 48 | 49 | @Test 50 | public void member() { 51 | assertEquals(Integer.valueOf(1), uut.get(1)); 52 | } 53 | 54 | @Test 55 | public void notMember() { 56 | Integer result = uut.get(2000); 57 | Assert.assertNull(result); 58 | } 59 | 60 | @Test 61 | public void testAllMembers() throws Exception { 62 | final Set> entrySet = this.originalMap.entrySet(); 63 | for(Entry entry : entrySet) { 64 | this.errorCollector.checkThat(this.uut.get(entry.getKey()), is(equalTo(entry.getValue()))); 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/edu/utexas/ece/mpc/bloomier/MutableBloomierFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, The University of Texas at Austin 3 | * Produced in the Mobile and Pervasive Computing Lab 4 | * Originally written by Evan Grim 5 | * 6 | * All rights reserved. 7 | * 8 | * See included LICENSE.txt for licensing details 9 | * 10 | */ 11 | 12 | package edu.utexas.ece.mpc.bloomier; 13 | 14 | import static org.hamcrest.CoreMatchers.equalTo; 15 | import static org.hamcrest.CoreMatchers.is; 16 | 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | import java.util.Set; 20 | import java.util.Map.Entry; 21 | 22 | import org.junit.Assert; 23 | import org.junit.Before; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | import org.junit.rules.ErrorCollector; 27 | 28 | public class MutableBloomierFilterTest { 29 | 30 | private MutableBloomierFilter uut; 31 | private final Map originalMap = new HashMap(); 32 | @Rule 33 | public final ErrorCollector errorCollector = new ErrorCollector(); 34 | 35 | @Before 36 | public void setUp() throws Exception { 37 | 38 | for (int i = 0; i < 1000; i++) { 39 | originalMap.put(i, i); 40 | } 41 | 42 | uut = new MutableBloomierFilter(originalMap, originalMap.keySet().size() * 10, 10, 32, 43 | 10000); 44 | } 45 | 46 | @Test 47 | public void member() { 48 | Assert.assertEquals(Integer.valueOf(1), uut.get(1)); 49 | } 50 | 51 | @Test 52 | public void notMember() { 53 | Integer result = uut.get(2000); 54 | Assert.assertNull(result); 55 | } 56 | 57 | @Test 58 | public void testHighBoundary() { 59 | Integer result = uut.get(1000); 60 | Assert.assertNull(result); 61 | } 62 | 63 | @Test 64 | public void testLowBoundary() { 65 | Integer result = uut.get(-1); 66 | Assert.assertNull(result); 67 | } 68 | 69 | 70 | @Test 71 | public void testNegativeNumber() { 72 | 73 | Integer result = uut.get(-1); 74 | Assert.assertNull(result); 75 | } 76 | 77 | @Test(expected=IllegalArgumentException.class) 78 | public void testNegativeNumbers() { 79 | uut.set(-1, 10); 80 | } 81 | 82 | @Test 83 | public void modify() { 84 | uut.set(500, 10); 85 | Assert.assertEquals(Integer.valueOf(10), uut.get(500)); 86 | } 87 | 88 | @Test(expected=IllegalArgumentException.class) 89 | public void modifyWithNegativeNumbers() { 90 | uut.set(-500, 10); 91 | } 92 | 93 | @Test(expected = IllegalArgumentException.class) 94 | public void illegalModify() { 95 | uut.set(2000, 10); 96 | } 97 | 98 | 99 | @Test 100 | public void testAllMembers() throws Exception { 101 | final Set> entrySet = this.originalMap.entrySet(); 102 | for(Entry entry : entrySet) { 103 | this.errorCollector.checkThat(this.uut.get(entry.getKey()), is(equalTo(entry.getValue()))); 104 | } 105 | } 106 | 107 | } 108 | --------------------------------------------------------------------------------