Cuckoo filter enables tradeoffs between its space efficiency and the false positive 29 | * probability of the membership check. 30 | * 31 | *
See the original paper https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf for more 32 | * details. 33 | * 34 | *
This class is not thread-safe.
35 | */
36 | public final class CuckooFilter Note that {@link SerializedCuckooFilterTable} does not contain any data on {@link
57 | * CuckooFilterConfig.HashFunction}, {@link CuckooFilterConfig.Strategy}, or {@link Funnel} used,
58 | * so it is up to the user to supply appropriate hash function, strategy, and funnel that were
59 | * used to generate the {@link SerializedCuckooFilterTable}.
60 | */
61 | public static By the probabilistic nature of the cuckoo filter data structure, this method may return a
92 | * false positive result. In other words, this method may incorrectly return true for an element
93 | * that was actually never inserted. This probability can depend on various factors, including the
94 | * size of the cuckoo filter and the hash function used.
95 | *
96 | * However, it is guaranteed that this method never returns a false negative result, as long as
97 | * {@code delete} method is called on an element that exists in the filter. Please see {@code
98 | * delete} method for more details.
99 | */
100 | public boolean contains(T element) {
101 | HashCode hash = config.hashFunction().hash(element, funnel);
102 | long fingerprint =
103 | config.strategy().computeFingerprint(hash, config.size().fingerprintLength());
104 | int bucketIndex = config.strategy().computeBucketIndex(hash, config.size().bucketCount());
105 | int otherBucketIndex =
106 | config
107 | .strategy()
108 | .computeOtherBucketIndex(
109 | fingerprint, bucketIndex, config.size().bucketCount(), config.hashFunction());
110 | return table.contains(bucketIndex, fingerprint)
111 | || table.contains(otherBucketIndex, fingerprint);
112 | }
113 |
114 | /**
115 | * Inserts {@code element} to the cuckoo filter, returning true if the element was inserted
116 | * successfully.
117 | *
118 | * Insertion of {@code element} will fail if there is no room for {@code element}. Note that
119 | * even when the insertion of {@code element} fails, it is possible for another element to be
120 | * inserted successfully. Even then, the insertion failure should be a good indicator that the
121 | * filter is getting close to its maximum capacity.
122 | */
123 | public boolean insert(T element) {
124 | HashCode hash = config.hashFunction().hash(element, funnel);
125 | long fingerprint =
126 | config.strategy().computeFingerprint(hash, config.size().fingerprintLength());
127 | int bucketIndex = config.strategy().computeBucketIndex(hash, config.size().bucketCount());
128 | int otherBucketIndex =
129 | config
130 | .strategy()
131 | .computeOtherBucketIndex(
132 | fingerprint, bucketIndex, config.size().bucketCount(), config.hashFunction());
133 |
134 | // First attempt to insert the fingerprint to one of the two assigned buckets.
135 | if (attemptInsertion(fingerprint, bucketIndex, otherBucketIndex)) {
136 | count++;
137 | return true;
138 | }
139 |
140 | // If both buckets are full, execute insertion with repeated replacements algorithm.
141 | int startBucketIndex = (random.nextInt(2) == 0) ? bucketIndex : otherBucketIndex;
142 | boolean inserted = insertWithRepeatedReplacements(fingerprint, startBucketIndex);
143 | if (inserted) {
144 | count++;
145 | }
146 | return inserted;
147 | }
148 |
149 | /**
150 | * Deletes {@code element} from the cuckoo filter, returning true if the element was deleted
151 | * successfully.
152 | *
153 | * It is critical for {@code delete} to be called on an already existing element. Otherwise,
154 | * the filter may incorrectly delete a wrong element. When this happens, it is possible for {@code
155 | * contains} method to return a false negative result.
156 | */
157 | public boolean delete(T element) {
158 | HashCode hash = config.hashFunction().hash(element, funnel);
159 | long fingerprint =
160 | config.strategy().computeFingerprint(hash, config.size().fingerprintLength());
161 | int bucketIndex = config.strategy().computeBucketIndex(hash, config.size().bucketCount());
162 | int otherBucketIndex =
163 | config
164 | .strategy()
165 | .computeOtherBucketIndex(
166 | fingerprint, bucketIndex, config.size().bucketCount(), config.hashFunction());
167 | boolean deleted =
168 | table.delete(bucketIndex, fingerprint) || table.delete(otherBucketIndex, fingerprint);
169 | if (deleted) {
170 | count--;
171 | }
172 | return deleted;
173 | }
174 |
175 | /** Returns the size of the cuckoo filter. */
176 | public CuckooFilterConfig.Size size() {
177 | return config.size();
178 | }
179 |
180 | /** Returns the count of the elements in the cuckoo filter. */
181 | public long count() {
182 | return count;
183 | }
184 |
185 | /**
186 | * Returns the ratio of the total number of elements in the cuckoo filter and the theoretical max
187 | * capacity.
188 | *
189 | * The returned value is in range [0, 1].
190 | */
191 | public double load() {
192 | return count / ((double) config.size().bucketCount() * config.size().bucketCapacity());
193 | }
194 |
195 | /**
196 | * Serializes the state of the cuckoo filter table.
197 | *
198 | * Note that this method does not serialize hash function, strategy, and funnel. When
199 | * instantiating a cuckoo filter from the returned {@link SerializedCuckooFilterTable}, it is up
200 | * to the user to supply appropriate hash function, strategy, and funnel that were used.
201 | */
202 | public SerializedCuckooFilterTable serializeTable() {
203 | return table.serialize();
204 | }
205 |
206 | /**
207 | * Attempts to insert {@code fingerprint} to one of the buckets with indices {@code bucketIndex}
208 | * and {@code otherBucketIndex}, returning true when successful. Returns false if both buckets are
209 | * full and the insertion failed.
210 | */
211 | private boolean attemptInsertion(long fingerprint, int bucketIndex, int otherBucketIndex) {
212 | if (!table.isFull(bucketIndex)) {
213 | table.insertWithReplacement(bucketIndex, fingerprint);
214 | return true;
215 | }
216 | if (!table.isFull(otherBucketIndex)) {
217 | table.insertWithReplacement(otherBucketIndex, fingerprint);
218 | return true;
219 | }
220 | return false;
221 | }
222 |
223 | /**
224 | * Randomly traverses the cuckoo graph to find an available bucket for insertion.
225 | *
226 | * At a high level, this algorithm starts at vertex {@code bucketIndex} and performs a random
227 | * walk of length at most {@link CuckooFilterConfig.Strategy#maxReplacementCount}. If an available
228 | * bucket is found, the algorithm "pushes" all the fingerprints (edges) that are visited (note
229 | * that in the cuckoo graph, the edges are the fingerprints) to their alternate buckets, and make
230 | * room for {@code fingerprint} to be inserted.
231 | *
232 | * If during the random walk an available bucket is not found, the insertion fails and the
233 | * method returns false.
234 | *
235 | * Note that it is possible to deterministically find an available bucket by performing breadth
236 | * first search in the cuckoo graph, but this is usually slower and the extra chance of successful
237 | * insertion is negligibly small in practice.
238 | */
239 | private boolean insertWithRepeatedReplacements(long fingerprint, int bucketIndex) {
240 | List Supports up to 64 bits per element. This will be used internally by cuckoo filter.
26 | */
27 | final class CuckooFilterArray {
28 | private final long length;
29 | private final int bitsPerElement;
30 | private final long[] bitArray;
31 |
32 | /**
33 | * Constructs a new cuckoo filter array with length {@code length}, with each element of length
34 | * {@code bitsPerElement} bits.
35 | *
36 | * @throws IllegalArgumentException if {@code length} <= 0 or {@code bitsPerElement} <= 0 or
37 | * {@code bitsPerElement} > 64.
38 | */
39 | public CuckooFilterArray(long length, int bitsPerElement) {
40 | checkLengthIsValid(length);
41 | checkBitsPerElementIsValid(bitsPerElement);
42 |
43 | this.length = length;
44 | this.bitsPerElement = bitsPerElement;
45 | long totalBits = length * bitsPerElement;
46 | // ceil(totalBits / 64) number of elements.
47 | long longArrayLength = (totalBits + Long.SIZE - 1) / Long.SIZE;
48 | checkArgument(
49 | longArrayLength < Integer.MAX_VALUE,
50 | "Too large: could not create CuckooFilterArray with length %s and bitsPerElement %s.",
51 | length,
52 | bitsPerElement);
53 | bitArray = new long[(int) longArrayLength];
54 | }
55 |
56 | /**
57 | * Constructs a cuckoo filter array with length {@code length}, with each element of length {@code
58 | * bitsPerElement}, from {@code byteArray}.
59 | */
60 | public CuckooFilterArray(long length, int bitsPerElement, byte[] byteArray) {
61 | this(length, bitsPerElement);
62 | ByteBuffer buffer = ByteBuffer.wrap(byteArray).order(ByteOrder.LITTLE_ENDIAN);
63 | for (int i = 0; i < bitArray.length; i++) {
64 | bitArray[i] = buffer.getLong();
65 | }
66 | }
67 |
68 | /** Returns the length of the array. */
69 | public long length() {
70 | return length;
71 | }
72 |
73 | /** Returns the number of bits per element. */
74 | public int bitsPerElement() {
75 | return bitsPerElement;
76 | }
77 |
78 | /**
79 | * Returns the element at the {@code index}th position as a long.
80 | *
81 | * The lowest {@code bitsPerElement} bits will correspond to the value of the element.
82 | *
83 | * @throws IllegalArgumentException if {@code index} is out of bounds.
84 | */
85 | public long getAsLong(long index) {
86 | checkIndexOutOfBounds(index);
87 | long bitStart = index * bitsPerElement;
88 | long bitEnd = bitStart + bitsPerElement;
89 | int arrayIndex1 = (int) (bitStart / Long.SIZE);
90 | int arrayIndex2 = (int) ((bitEnd - 1) / Long.SIZE);
91 |
92 | int a = (int) (bitStart % Long.SIZE);
93 | // The element intersects the two array indices.
94 | if (arrayIndex1 < arrayIndex2) {
95 | int b = a + bitsPerElement - Long.SIZE;
96 | long value1 = bitArray[arrayIndex1] >>> a;
97 | long value2 = bitArray[arrayIndex2] & mask(b);
98 | return (value1 | (value2 << (Long.SIZE - a)));
99 | }
100 | // Element is contained in one array index.
101 | return (bitArray[arrayIndex1] >>> a) & mask(bitsPerElement);
102 | }
103 |
104 | /**
105 | * Sets the element at {@code index}th position as {@code value}, using the lowest {@code
106 | * bitsPerElement} bits as the value of the element.
107 | *
108 | * @throws IllegalArgumentException if {@code index} is out of bounds.
109 | */
110 | public void set(long index, long value) {
111 | checkIndexOutOfBounds(index);
112 | long bitStart = index * bitsPerElement;
113 | long bitEnd = bitStart + bitsPerElement;
114 | int arrayIndex1 = (int) (bitStart / Long.SIZE);
115 | int arrayIndex2 = (int) ((bitEnd - 1) / Long.SIZE);
116 |
117 | // Use the lowest bitsPerElement bits and clear all other bits.
118 | value &= mask(bitsPerElement);
119 |
120 | int a = (int) (bitStart % Long.SIZE);
121 | // The element intersects the two array indices.
122 | if (arrayIndex1 < arrayIndex2) {
123 | int b = a + bitsPerElement - Long.SIZE;
124 | bitArray[arrayIndex1] &= clearMask(Long.SIZE, a, Long.SIZE);
125 | bitArray[arrayIndex1] |= (value << a);
126 | bitArray[arrayIndex2] &= clearMask(Long.SIZE, 0, b);
127 | bitArray[arrayIndex2] |= (value >>> (Long.SIZE - a));
128 | } else {
129 | // Element is contained in one array index.
130 | int b = a + bitsPerElement;
131 | bitArray[arrayIndex1] &= clearMask(Long.SIZE, a, b);
132 | bitArray[arrayIndex1] |= (value << a);
133 | }
134 | }
135 |
136 | /** Returns byte array representation of the {@link CuckooFilterArray}. */
137 | public byte[] toByteArray() {
138 | byte[] byteArray = new byte[bitArray.length * Long.BYTES];
139 | for (int i = 0; i < bitArray.length; i++) {
140 | long value = bitArray[i];
141 | for (int j = 0; j < Long.BYTES; j++) {
142 | // Explicit conversion from long to byte will truncate to lowest 8 bits.
143 | byteArray[i * Long.BYTES + j] = (byte) value;
144 | value >>>= Byte.SIZE;
145 | }
146 | }
147 | return byteArray;
148 | }
149 |
150 | // Theoretical max size of a long array is Integer.MAX_VALUE. Assuming each element is 1 bit,
151 | // we can support up to Integer.MAX_VALUE * 64 number of elements.
152 | private void checkLengthIsValid(long length) {
153 | checkArgument(
154 | 0 < length && length < (long) Integer.MAX_VALUE * Long.SIZE,
155 | "length must be in range (0, %s).",
156 | (long) Integer.MAX_VALUE * Long.SIZE);
157 | }
158 |
159 | private void checkBitsPerElementIsValid(int bitsPerElement) {
160 | checkArgument(
161 | 0 < bitsPerElement && bitsPerElement <= 64, "bitsPerElement must be in range [1, 64].");
162 | }
163 |
164 | private void checkIndexOutOfBounds(long index) {
165 | checkArgument(0 <= index && index < length, "Index is out of bounds: %s.", index);
166 | }
167 |
168 | private static long mask(int length) {
169 | if (length == Long.SIZE) {
170 | // -1 in 2s complement is 0xFFFFFFFFFFFFFFFF.
171 | return -1;
172 | }
173 | return (1L << length) - 1;
174 | }
175 |
176 | // Mask for clearing bits in range [a, b).
177 | private static long clearMask(int length, int a, int b) {
178 | long mask1 = mask(length);
179 | long mask2 = mask(b - a);
180 | return mask1 ^ (mask2 << a);
181 | }
182 | }
183 |
--------------------------------------------------------------------------------
/setfilters/src/com/google/setfilters/cuckoofilter/CuckooFilterConfig.java:
--------------------------------------------------------------------------------
1 | // Copyright 2022 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // https://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.setfilters.cuckoofilter;
16 |
17 | import static com.google.common.base.Preconditions.checkArgument;
18 |
19 | import com.google.common.collect.ImmutableMap;
20 | import com.google.common.hash.Funnel;
21 | import com.google.common.hash.HashCode;
22 | import com.google.errorprone.annotations.CanIgnoreReturnValue;
23 | import java.util.Map;
24 |
25 | /**
26 | * Specification for the cuckoo filter.
27 | *
28 | * This class is immutable.
29 | */
30 | // TODO: Handle serialization.
31 | public final class CuckooFilterConfig {
32 | private final Size size;
33 | private final HashFunction hashFunction;
34 | private final Strategy strategy;
35 | private final boolean useSpaceOptimization;
36 |
37 | private CuckooFilterConfig(
38 | Size size, HashFunction hashFunction, Strategy strategy, boolean useSpaceOptimization) {
39 | this.size = size;
40 | this.hashFunction = hashFunction;
41 | this.strategy = strategy;
42 | this.useSpaceOptimization = useSpaceOptimization;
43 | }
44 |
45 | public Size size() {
46 | return size;
47 | }
48 |
49 | public HashFunction hashFunction() {
50 | return hashFunction;
51 | }
52 |
53 | public Strategy strategy() {
54 | return strategy;
55 | }
56 |
57 | public boolean useSpaceOptimization() {
58 | return useSpaceOptimization;
59 | }
60 |
61 | public static Builder newBuilder() {
62 | return new Builder();
63 | }
64 |
65 | /** Builder for the {@link CuckooFilterConfig}. */
66 | public static class Builder {
67 | private Size size;
68 | private HashFunction hashFunction;
69 | private Strategy strategy;
70 | private boolean useSpaceOptimization;
71 |
72 | private Builder() {}
73 |
74 | @CanIgnoreReturnValue
75 | public Builder setSize(Size size) {
76 | this.size = size;
77 | return this;
78 | }
79 |
80 | @CanIgnoreReturnValue
81 | public Builder setHashFunction(HashFunction hashFunction) {
82 | this.hashFunction = hashFunction;
83 | return this;
84 | }
85 |
86 | @CanIgnoreReturnValue
87 | public Builder setStrategy(Strategy strategy) {
88 | this.strategy = strategy;
89 | return this;
90 | }
91 |
92 | /**
93 | * Whether to use space optimized filter representation (if possible).
94 | *
95 | * Setting this field to {@code true} does not guarantee the optimization algorithm to always
96 | * apply - it is best effort.
97 | *
98 | * In general, using this may result in slower filter operations, and incurs an additional
99 | * fixed space overhead. Thus, it is possible for the "optimized" version of the filter to
100 | * actually take more space than the non optimized one.
101 | */
102 | @CanIgnoreReturnValue
103 | public Builder setUseSpaceOptimization(boolean useSpaceOptimization) {
104 | this.useSpaceOptimization = useSpaceOptimization;
105 | return this;
106 | }
107 |
108 | /**
109 | * Builds {@link CuckooFilterConfig}.
110 | *
111 | * @throws IllegalArgumentException if the required parameters are not set.
112 | */
113 | public CuckooFilterConfig build() {
114 | checkArgument(size != null, "Size must be set.");
115 | checkArgument(hashFunction != null, "Hash function must be set.");
116 | checkArgument(strategy != null, "Strategy must be set.");
117 |
118 | return new CuckooFilterConfig(size, hashFunction, strategy, useSpaceOptimization);
119 | }
120 | }
121 |
122 | /**
123 | * Specification of the cuckoo filter size.
124 | *
125 | * A cuckoo filter's size can be defined as a tuple (bucketCount, bucketCapacity,
126 | * fingeprintLength); this means that there are bucketCount number of buckets, where each bucket
127 | * can store up to bucketCapacity fingerprints, and each fingerprint is of length
128 | * fingerprintLength bits.
129 | *
130 | * All fields are required and must be set explicitly.
131 | *
132 | * This class is immutable.
133 | */
134 | public static class Size {
135 | private static final int MAX_BUCKET_CAPACITY = 128;
136 | private static final int MAX_FINGERPRINT_LENGTH = 64;
137 | /** Empirical load by the bucket capacity. */
138 | private static final ImmutableMap {@code bucketCount} must be > 0.
249 | */
250 | @CanIgnoreReturnValue
251 | public Builder setBucketCount(int bucketCount) {
252 | this.bucketCount = bucketCount;
253 | return this;
254 | }
255 |
256 | /**
257 | * Sets the maximum number of fingerprints each bucket can hold.
258 | *
259 | * {@code bucketCapacity} must be in range (0, {@value #MAX_BUCKET_CAPACITY}].
260 | */
261 | @CanIgnoreReturnValue
262 | public Builder setBucketCapacity(int bucketCapacity) {
263 | this.bucketCapacity = bucketCapacity;
264 | return this;
265 | }
266 |
267 | /**
268 | * Sets the length of each fingerprint in bits.
269 | *
270 | * {@code fingerprintLength} must be in range (0, {@value #MAX_FINGERPRINT_LENGTH}].
271 | */
272 | @CanIgnoreReturnValue
273 | public Builder setFingerprintLength(int fingerprintLength) {
274 | this.fingerprintLength = fingerprintLength;
275 | return this;
276 | }
277 |
278 | /**
279 | * Builds {@link Size}.
280 | *
281 | * @throws IllegalArgumentException if the configured parameters are invalid.
282 | */
283 | public Size build() {
284 | checkArgument(bucketCount > 0, "bucketCount must be > 0: %s given instead.", bucketCount);
285 | checkArgument(
286 | 0 < bucketCapacity && bucketCapacity <= MAX_BUCKET_CAPACITY,
287 | "bucketCapacity must be in range (0, %s]: %s given instead.",
288 | MAX_BUCKET_CAPACITY,
289 | bucketCapacity);
290 | checkArgument(
291 | 0 < fingerprintLength && fingerprintLength <= MAX_FINGERPRINT_LENGTH,
292 | "fingerprintLength must be in range (0, %s]: %s given instead.",
293 | MAX_FINGERPRINT_LENGTH,
294 | fingerprintLength);
295 |
296 | return new Size(bucketCount, bucketCapacity, fingerprintLength);
297 | }
298 | }
299 |
300 | private static double log2(double x) {
301 | return Math.log(x) / Math.log(2);
302 | }
303 | }
304 |
305 | /** Hash function for transforming an arbitrary type element to a {@link HashCode}. */
306 | public interface HashFunction {
307 | /** Hashes given {@code element} to a {@link HashCode}, using the given {@code funnel}. */
308 | The returned value should be in range (0, 2^{@code fingerprintLength}). Otherwise, the
322 | * behavior of the cuckoo filter is undefined. Note that the interval is an open interval, so 0
323 | * and 2^{@code fingerprintLength} are not included.
324 | */
325 | long computeFingerprint(HashCode hash, int fingerprintLength);
326 |
327 | /**
328 | * Computes one of the bucket indices given the element's {@code hash} output from {@link
329 | * HashFunction} and {@code bucketCount} of the cuckoo filter.
330 | *
331 | * The returned value should be in range [0, {@code bucketCount}). Otherwise, the behavior of
332 | * the cuckoo filter is undefined.
333 | */
334 | int computeBucketIndex(HashCode hash, int bucketCount);
335 |
336 | /**
337 | * Computes the element's other bucket index given the element's {@code fingerprint} value and
338 | * its initial {@code bucketIndex}.
339 | *
340 | * {@code hashFunction} corresponds to the {@link HashFunction} that was supplied when the
341 | * config was constructed. Depending on the implementation, {@code hashFunction} may or may not
342 | * be used.
343 | *
344 | * The returned value should be in range [0, {@code bucketCount}), and the method needs to be
345 | * an involution with respect to {@code bucketIndex}. That is, with other parameters fixed, the
346 | * method needs to satisfy bucketIndex =
347 | * computeOtherBucketIndex(computeOtherBucketIndex(bucketIndex)) for all valid
348 | * bucketIndex. Note that other parameters are omitted for brevity. If these properties
349 | * don't hold, the behavior of the cuckoo filter is undefined.
350 | */
351 | int computeOtherBucketIndex(
352 | long fingerprint, int bucketIndex, int bucketCount, HashFunction hashFunction);
353 |
354 | /**
355 | * Maximum number of replacements to be made during insertion, before declaring that the
356 | * insertion has failed.
357 | *
358 | * If not overridden, set to 500 as a default.
359 | */
360 | default int maxReplacementCount() {
361 | return 500;
362 | }
363 | }
364 | }
365 |
--------------------------------------------------------------------------------
/setfilters/src/com/google/setfilters/cuckoofilter/CuckooFilterHashFunctions.java:
--------------------------------------------------------------------------------
1 | // Copyright 2022 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // https://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.setfilters.cuckoofilter;
16 |
17 | import com.google.common.hash.Funnel;
18 | import com.google.common.hash.HashCode;
19 | import com.google.common.hash.Hashing;
20 |
21 | /** A set of predefined {@link CuckooFilterConfig.HashFunction}s. */
22 | public enum CuckooFilterHashFunctions implements CuckooFilterConfig.HashFunction {
23 |
24 | /**
25 | * MurmurHash3 that yields 128 bit hash value.
26 | *
27 | * Behavior of MurmurHash3 is fixed and should not change in the future.
28 | */
29 | MURMUR3_128() {
30 | @Override
31 | public The {@link HashCode} generated with the hash function should be at least 64 bits. This will
27 | * achieve good false positive rate when fingerprintLength <= 32.
28 | */
29 | SIMPLE_MOD() {
30 | @Override
31 | public long computeFingerprint(HashCode hash, int fingerprintLength) {
32 | // Use the most significant fingerprintLength bits. This is needed to get rid of the
33 | // correlation with the bucket index.
34 | long fingerprint = hash.asLong() >>> (Long.SIZE - fingerprintLength);
35 | // Value 0 is reserved, so instead map to 1. This means that the generated fingerprint value
36 | // is skewed (1 is twice as more likely to be generated than any other value). Note that, we
37 | // could have taken mod (2^fingerprintLength - 1) and added 1, which would produce a more
38 | // uniform distribution. However, for performance reason, we choose to take this approach
39 | // instead.
40 | if (fingerprint == 0) {
41 | return 1L;
42 | }
43 | return fingerprint;
44 | }
45 |
46 | @Override
47 | public int computeBucketIndex(HashCode hash, int bucketCount) {
48 | return Math.floorMod(hash.asLong(), bucketCount);
49 | }
50 |
51 | @Override
52 | public int computeOtherBucketIndex(
53 | long fingerprint,
54 | int bucketIndex,
55 | int bucketCount,
56 | CuckooFilterConfig.HashFunction hashFunction) {
57 | long fingerprintHash = hashFunction.hash(fingerprint, Funnels.longFunnel()).asLong();
58 | // Use (hash(fingerprint) - bucketIndex) mod bucketCount as the involution.
59 | return Math.floorMod(fingerprintHash - bucketIndex, bucketCount);
60 | }
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/setfilters/src/com/google/setfilters/cuckoofilter/CuckooFilterTable.java:
--------------------------------------------------------------------------------
1 | // Copyright 2022 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // https://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.setfilters.cuckoofilter;
16 |
17 | import java.nio.ByteBuffer;
18 | import java.util.Optional;
19 | import java.util.Random;
20 |
21 | /** An array of buckets where each bucket can store a fixed number of fingerprints. */
22 | interface CuckooFilterTable {
23 | /** Value of the empty "slot", which is reserved as 0. */
24 | public static long EMPTY_SLOT = 0L;
25 |
26 | /**
27 | * Creates an implementation of an empty cuckoo filter based on whether space optimization should
28 | * be used.
29 | *
30 | * Space optimization is best effort, and is not guaranteed.
31 | */
32 | public static CuckooFilterTable create(
33 | CuckooFilterConfig.Size size, boolean useSpaceOptimization, Random random) {
34 | if (useSpaceOptimization && size.bucketCapacity() == 4 && size.fingerprintLength() >= 4) {
35 | return new SemiSortedCuckooFilterTable(size, random);
36 | }
37 | return new UncompressedCuckooFilterTable(size, random);
38 | }
39 |
40 | /** Creates an implementation of the cuckoo filter based on the serialization. */
41 | public static CuckooFilterTable createFromSerialization(
42 | SerializedCuckooFilterTable serializedTable, Random random) {
43 | ByteBuffer buffer = ByteBuffer.wrap(serializedTable.asByteArray());
44 |
45 | if (buffer.remaining() <= 16) {
46 | throw new IllegalArgumentException("Unable to parse the SerializedCuckooFilterTable.");
47 | }
48 |
49 | int tableType = buffer.getInt();
50 | int bucketCount = buffer.getInt();
51 | int bucketCapacity = buffer.getInt();
52 | int fingerprintLength = buffer.getInt();
53 | CuckooFilterConfig.Size size =
54 | CuckooFilterConfig.Size.newBuilder()
55 | .setBucketCount(bucketCount)
56 | .setBucketCapacity(bucketCapacity)
57 | .setFingerprintLength(fingerprintLength)
58 | .build();
59 |
60 | byte[] bitArray = new byte[buffer.remaining()];
61 | buffer.get(bitArray);
62 |
63 | if (tableType == UncompressedCuckooFilterTable.TABLE_TYPE) {
64 | return new UncompressedCuckooFilterTable(size, bitArray, random);
65 | } else if (tableType == SemiSortedCuckooFilterTable.TABLE_TYPE) {
66 | return new SemiSortedCuckooFilterTable(size, bitArray, random);
67 | } else {
68 | throw new IllegalArgumentException("Unable to parse the SerializedCuckooFilterTable.");
69 | }
70 | }
71 |
72 | /**
73 | * Inserts given {@code fingerprint} to the {@code bucketIndex}th bucket, replacing an arbitrary
74 | * fingerprint if the bucket is full.
75 | *
76 | * How this arbitrary fingerprint is chosen depends on the implementation.
77 | *
78 | * @return the value of the replaced fingerprint if the bucket is full, and an empty {@link
79 | * Optional} otherwise.
80 | */
81 | Optional If a bucket contains multiple {@code fingerprint} values, this method only deletes one.
90 | *
91 | * @return {@code true} if {@code fingerprint} is in {@code bucketIndex}th bucket and is deleted,
92 | * and {@code false} otherwise.
93 | */
94 | boolean delete(int bucketIndex, long fingerprint);
95 |
96 | /** Returns whether {@code bucketIndex}th bucket is full. */
97 | boolean isFull(int bucketIndex);
98 |
99 | /** Returns the size of {@link CuckooFilterTable}. */
100 | CuckooFilterConfig.Size size();
101 |
102 | /** Returns serialization of {@link CuckooFilterTable}. */
103 | SerializedCuckooFilterTable serialize();
104 |
105 | // TODO: Add more methods as needed.
106 | }
107 |
--------------------------------------------------------------------------------
/setfilters/src/com/google/setfilters/cuckoofilter/SemiSortedCuckooFilterTable.java:
--------------------------------------------------------------------------------
1 | // Copyright 2022 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // https://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package com.google.setfilters.cuckoofilter;
16 |
17 | import static com.google.common.base.Preconditions.checkArgument;
18 | import static java.util.Comparator.comparingInt;
19 |
20 | import com.google.common.collect.ImmutableMap;
21 | import java.nio.ByteBuffer;
22 | import java.util.Arrays;
23 | import java.util.Optional;
24 | import java.util.Random;
25 |
26 | /**
27 | * Implementation of the {@link CuckooFilterTable} using the semi-sorting bucket compression scheme
28 | * in the original paper by Fan et al (https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf) -
29 | * see section 5.2.
30 | *
31 | * The main idea behind the compression algorithm is that the order of the fingerprints in each
32 | * bucket is irrelevant - that is, the fingerprints in each bucket forms a multiset. For fingerprint
33 | * length f and bucket capacity b, the possible number of multisets of b fingerprints of f bits each
34 | * is given by C(2^f + b - 1, b), where C denotes binomial coefficient. In particular, we can encode
35 | * each bucket with ceil(log2(C(2^f + b - 1, b))) bits. On the other hand, naively encoding the
36 | * fingerprints will take b * f bits. Thus, it is theoretically possible to save b * f -
37 | * ceil(log2(C(2^f + b - 1, b))) bits per bucket (note that this is not information theoretically
38 | * tight because the distribution of the multisets is not uniform).
39 | *
40 | * For performance reason, this only supports a table with bucket capacity of size 4 and
41 | * fingerprint length >= 4 - in many cases this is not a limitation because, for many practical
42 | * applications, bucket capacity of size 4 yields the optimal cuckoo filter size and fingerprint
43 | * length < 4 will never achieve good enough false positive rate.
44 | *
45 | * Compared to the {@link UncompressedCuckooFilterTable}, this implementation can save 1 bit per
46 | * element, at the cost of slower filter operations by a constant factor (asymptotically, it is the
47 | * same as the uncompressed one). Note that for bucket capacity of size 4, saving 1 bit per element
48 | * is "optimal" up to rounding down, as the function 4 * f - ceil(log2(C(2^f + 3, 4))) < 5 for
49 | * reasonable values of f. However, this also incurs an additional fixed space overhead, so for
50 | * smaller filter the extra saving of 1 bit per element may not be worth it.
51 | */
52 | final class SemiSortedCuckooFilterTable implements CuckooFilterTable {
53 | // Implementation type of the table, to be encoded in the serialization.
54 | public static final int TABLE_TYPE = 1;
55 |
56 | // Table containing all sorted 4 bit partial fingerprints of length 4 (16 bits) by its index.
57 | private static final short[] SORTED_PARTIAL_FINGERPRINTS = computeSortedPartialFingerprints();
58 | // Inverse map of SORTED_PARTIAL_FINGERPRINTS.
59 | private static final ImmutableMap Uses the given source of {@code random} to choose the replaced fingerprint in {@code
70 | * insertWithReplacement} method.
71 | */
72 | public SemiSortedCuckooFilterTable(CuckooFilterConfig.Size size, Random random) {
73 | this.size = size;
74 | checkArgument(
75 | size.bucketCapacity() == 4,
76 | "SemiSortedCuckooFilterTable only supports bucket capacity of 4.");
77 | checkArgument(
78 | size.fingerprintLength() >= 4,
79 | "SemiSortedCuckooFilterTable only supports fingerprint length >= 4.");
80 | this.random = random;
81 | // bucketCapacity == 4 and fingerprintLength <= 64, so we can assume that it will always fit
82 | // into a long.
83 | cuckooFilterArray =
84 | new CuckooFilterArray(
85 | (long) size.bucketCount() * size.bucketCapacity(), size.fingerprintLength() - 1);
86 | }
87 |
88 | /** Creates {@link SemiSortedCuckooFilterTable} from {@link SerializedCuckooFilterTable}. */
89 | public SemiSortedCuckooFilterTable(CuckooFilterConfig.Size size, byte[] bitArray, Random random) {
90 | this.size = size;
91 | this.random = random;
92 | cuckooFilterArray =
93 | new CuckooFilterArray(
94 | (long) size.bucketCount() * size.bucketCapacity(),
95 | size.fingerprintLength() - 1,
96 | bitArray);
97 | }
98 |
99 | @Override
100 | public Optional Encoding works as follows.
202 | *
203 | * Suppose each fingerprint is logically f bits. First, sort the fingerprints by the least
204 | * significant 4 bits. Let's call the most significant f - 4 bits of the fingerprints as the
205 | * fingerprint prefixes. The least significant 4 bits of the fingerprints will be the partial
206 | * fingerprints, which will be encoded according to the SORTED_PARTIAL_FINGEPRRINTS_INDEX map as a
207 | * 12 bit value. Partition the encoded 12 bit value into four 3 bit chunks. Group each of the f -
208 | * 4 bit prefixes with each 3 bit chunk (f - 1 bits total) and insert it as a cuckoo filter array
209 | * element.
210 | */
211 | private void encodeAndPut(int bucketIndex, long[] fingerprints) {
212 | long[] fingerprintPrefixes = new long[size.bucketCapacity()];
213 | int[] partialFingerprints = new int[size.bucketCapacity()];
214 | for (int i = 0; i < size.bucketCapacity(); i++) {
215 | fingerprintPrefixes[i] = fingerprints[i] >>> 4;
216 | partialFingerprints[i] = (int) (fingerprints[i] & 0xF);
217 | }
218 | Integer[] indices = {0, 1, 2, 3};
219 | Arrays.sort(indices, comparingInt((Integer i) -> partialFingerprints[i]));
220 | short encodedSortedPartialFingerprints =
221 | (short)
222 | ((partialFingerprints[indices[0]] << 12)
223 | | (partialFingerprints[indices[1]] << 8)
224 | | (partialFingerprints[indices[2]] << 4)
225 | | partialFingerprints[indices[3]]);
226 | int encodedSortedPartialFingerprintsIndex =
227 | SORTED_PARTIAL_FINGERPRINTS_INDEX.get(encodedSortedPartialFingerprints);
228 | for (int i = size.bucketCapacity() - 1; i >= 0; i--) {
229 | long arrayIndex = toArrayIndex(bucketIndex, i);
230 | cuckooFilterArray.set(
231 | arrayIndex,
232 | (fingerprintPrefixes[indices[i]] << 3) | (encodedSortedPartialFingerprintsIndex & 0x7));
233 | encodedSortedPartialFingerprintsIndex >>>= 3;
234 | }
235 | }
236 |
237 | private static short[] computeSortedPartialFingerprints() {
238 | // (2^4 + 3 choose 4) = 3876 counts the number of multisets of size 4, with each element in
239 | // [0, 16).
240 | short[] sortedPartialFingerprints = new short[3876];
241 |
242 | final short fingerprintUpperBound = 16;
243 |
244 | int i = 0;
245 | for (short a = 0; a < fingerprintUpperBound; a++) {
246 | for (short b = a; b < fingerprintUpperBound; b++) {
247 | for (short c = b; c < fingerprintUpperBound; c++) {
248 | for (short d = c; d < fingerprintUpperBound; d++) {
249 | sortedPartialFingerprints[i] = (short) ((a << 12) | (b << 8) | (c << 4) | d);
250 | i++;
251 | }
252 | }
253 | }
254 | }
255 | return sortedPartialFingerprints;
256 | }
257 |
258 | private static ImmutableMap Thus, if a bucket can hold up to bucketCapacity number of fingerprints and each fingerprint is
28 | * of length fingerprintLength bits, it takes bucketCapacity * fingerprintLength bits to represent
29 | * each bucket.
30 | */
31 | final class UncompressedCuckooFilterTable implements CuckooFilterTable {
32 | // Implementation type of the table, to be encoded in the serialization.
33 | public static final int TABLE_TYPE = 0;
34 |
35 | private final CuckooFilterConfig.Size size;
36 | private final Random random;
37 | private final CuckooFilterArray cuckooFilterArray;
38 |
39 | /**
40 | * Creates a new uncompressed cuckoo filter table of the given size.
41 | *
42 | * Uses the given source of {@code random} to choose the replaced fingerprint in {@code
43 | * insertWithReplacement} method.
44 | */
45 | public UncompressedCuckooFilterTable(CuckooFilterConfig.Size size, Random random) {
46 | this.size = size;
47 | this.random = random;
48 | // bucketCapacity <= 128 and fingerprintLength <= 64, so we can assume that it will always fit
49 | // into a long.
50 | cuckooFilterArray =
51 | new CuckooFilterArray(
52 | (long) size.bucketCount() * size.bucketCapacity(), size.fingerprintLength());
53 | }
54 |
55 | /** Creates {@link UncompressedCuckooFilterTable} from {@link SerializedCuckooFilterTable}. */
56 | public UncompressedCuckooFilterTable(
57 | CuckooFilterConfig.Size size, byte[] bitArray, Random random) {
58 | this.size = size;
59 | this.random = random;
60 | cuckooFilterArray =
61 | new CuckooFilterArray(
62 | (long) size.bucketCount() * size.bucketCapacity(), size.fingerprintLength(), bitArray);
63 | }
64 |
65 | @Override
66 | public Optional