├── figure
    ├── DCFexample.png
    └── PKCuckooHashing.png
├── configuration
    └── config.txt
├── result
    └── result.txt
├── src_DBF
    ├── hashfunctions.h
    ├── countingbloomfilter.h
    ├── hashfunctions.cpp
    ├── dynamicbloomfilter.h
    ├── countingbloomfilter.cpp
    ├── dynamicbloomfilter.cpp
    └── test.cpp
├── src
    ├── uint.h
    ├── hashfunction.h
    ├── hashfunction.cpp
    ├── linklist.h
    ├── bithack.h
    ├── dynamiccuckoofilter.h
    ├── cuckoofilter.h
    ├── test.backup
    ├── test.cpp
    ├── dynamiccuckoofilter.cpp
    └── cuckoofilter.cpp
├── .gitignore
├── NOTICE
├── makefile
├── makefile_DBF
├── README.md
└── LICENSE


/figure/DCFexample.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CGCL-codes/DCF/HEAD/figure/DCFexample.png


--------------------------------------------------------------------------------
/figure/PKCuckooHashing.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CGCL-codes/DCF/HEAD/figure/PKCuckooHashing.png


--------------------------------------------------------------------------------
/configuration/config.txt:
--------------------------------------------------------------------------------
1 | false positive = 0.0005
2 | item number = 1000000
3 | input file path = ./input/input.txt


--------------------------------------------------------------------------------
/result/result.txt:
--------------------------------------------------------------------------------
1 |        item_num        exp_FPR     actual_FPR     actual_BBN   F_size(bits) space_cost(MB)      I_time(s)      Q_time(s)      D_time(s)    C_rate
2 |         1000000           0.02       0.007366              5             12          1.875       0.855738       0.936381        1.06508         1
3 | 


--------------------------------------------------------------------------------
/src_DBF/hashfunctions.h:
--------------------------------------------------------------------------------
 1 | #ifndef HASHFUNCTIONS_H_
 2 | #define HASHFUNCTIONS_H_
 3 | 
 4 | #include<openssl/evp.h>
 5 | 
 6 | class HashFunc{
 7 | public:
 8 | 	HashFunc();
 9 | 	~HashFunc();
10 | 	static char* sha1(const char* key);
11 | 	static char* md5(const char* key);
12 | };
13 | 
14 | #endif //HASHFUNCTIONS_H_
15 | 


--------------------------------------------------------------------------------
/src/uint.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * uint.h
 3 |  *
 4 |  *  Created on: Dec 21, 2016
 5 |  *      Author: liaoliangyi
 6 |  */
 7 | 
 8 | #ifndef UINT_H_
 9 | #define UINT_H_
10 | 
11 | typedef unsigned char uint8_t;
12 | typedef unsigned short int uint16_t;
13 | typedef unsigned long int uint32_t;
14 | typedef unsigned long long int uint64_t;
15 | 
16 | 
17 | #endif /* UINT_H_ */
18 | 


--------------------------------------------------------------------------------
/src/hashfunction.h:
--------------------------------------------------------------------------------
 1 | #ifndef HASHFUNCTION_H
 2 | #define HASHFUNCTION_H
 3 | 
 4 | #include<string>
 5 | #include<openssl/evp.h>
 6 | 
 7 | using namespace std;
 8 | 
 9 | class HashFunc{
10 | public:
11 | 	HashFunc();
12 | 	~HashFunc();
13 | 	static std::string sha1(const char* key);
14 | 	static std::string md5(const char* key);
15 | };
16 | 
17 | #endif //HASHFUNCTION_H
18 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compiled Object files
 2 | *.slo
 3 | *.lo
 4 | *.o
 5 | *.obj
 6 | 
 7 | # Precompiled Headers
 8 | *.gch
 9 | *.pch
10 | 
11 | # Compiled Dynamic libraries
12 | *.so
13 | *.dylib
14 | *.dll
15 | 
16 | # Fortran module files
17 | *.mod
18 | *.smod
19 | 
20 | # Compiled Static libraries
21 | *.lai
22 | *.la
23 | *.a
24 | *.lib
25 | 
26 | # Executables
27 | *.exe
28 | *.out
29 | *.app
30 | 


--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
 1 | The Dynamic Cuckoo Filter
 2 | 
 3 | Copyright (C) 2017, STCS & CGCL and Huazhong University of Science and Technology.
 4 | 
 5 | This product includes software developed at STCS & CGCL (http://grid.hust.edu.cn/) and Huazhong University of Science and Technology (http://www.hust.edu.cn).
 6 | 
 7 | =============================================================================================
 8 | 
 9 | This product includes software developed by Bin Fan (binfan@cs.cmu.edu), David G. Andersen (dga@cs.cmu.edu), Michael Kaminsky (michael.e.kaminsky@intel.com).
10 | 
11 | Copyright (C) 2013, Carnegie Mellon University and Intel Corporation.


--------------------------------------------------------------------------------
/makefile:
--------------------------------------------------------------------------------
 1 | objects = test.o dynamiccuckoofilter.o cuckoofilter.o hashfunction.o 
 2 | 
 3 | test: $(objects)
 4 | 	g++ -Wall -o test $(objects) -lssl -lcrypto
 5 | 
 6 | hashfunction.o: src/hashfunction.cpp src/hashfunction.h
 7 | 	g++ -Wall -c src/hashfunction.cpp -lssl -lcrypto
 8 | 
 9 | cuckoofilter.o: src/cuckoofilter.cpp src/cuckoofilter.h
10 | 	g++ -Wall -c src/cuckoofilter.cpp -lssl -lcrypto
11 | 
12 | dynamiccuckoofilter.o: src/dynamiccuckoofilter.cpp src/cuckoofilter.h
13 | 	g++ -Wall -c src/dynamiccuckoofilter.cpp -lssl -lcrypto
14 | 
15 | test.o: src/test.cpp src/dynamiccuckoofilter.h
16 | 	g++ -Wall -c src/test.cpp -lssl -lcrypto
17 | 
18 | 
19 | .PHONY : clean
20 | clean :
21 | 	rm *.o
22 | 


--------------------------------------------------------------------------------
/makefile_DBF:
--------------------------------------------------------------------------------
 1 | objects = test.o dynamicbloomfilter.o countingbloomfilter.o hashfunctions.o 
 2 | 
 3 | test: $(objects)
 4 | 	g++ -Wall -o test $(objects) -lssl -lcrypto
 5 | 
 6 | hashfunctions.o: src/hashfunctions.cpp src/hashfunctions.h
 7 | 	g++ -Wall -c src/hashfunctions.cpp -lssl -lcrypto
 8 | 
 9 | countingbloomfilter.o: src/countingbloomfilter.cpp src/countingbloomfilter.h
10 | 	g++ -Wall -c src/countingbloomfilter.cpp -lssl -lcrypto
11 | 
12 | dynamicbloomfilter.o: src/dynamicbloomfilter.cpp src/countingbloomfilter.h
13 | 	g++ -Wall -c src/dynamicbloomfilter.cpp -lssl -lcrypto
14 | 
15 | test.o: src/test.cpp src/dynamicbloomfilter.h
16 | 	g++ -Wall -c src/test.cpp -lssl -lcrypto
17 | 
18 | 
19 | .PHONY : clean
20 | clean :
21 | 	rm *.o
22 | 


--------------------------------------------------------------------------------
/src_DBF/countingbloomfilter.h:
--------------------------------------------------------------------------------
 1 | #ifndef COUNTINGBLOOMFILTER_H_
 2 | #define COUNTINGBLOOMFILTER_H_
 3 | 
 4 | #include<math.h>
 5 | 
 6 | 
 7 | class CountingBloomFilter{
 8 | private:
 9 | 	char* bits;
10 | 	int byte_num;
11 | 	int bits_num;
12 | 	int hash_num;
13 | 
14 | public:
15 | 	CountingBloomFilter* next;
16 | 	CountingBloomFilter* front;
17 | 	int capacity;
18 | 	int item_num;
19 | 
20 | 	CountingBloomFilter(int capacity, double false_positive);
21 | 	CountingBloomFilter(int n, int m);
22 | 	~CountingBloomFilter();
23 | 
24 | 	bool insertItem(unsigned long int* hash_value);
25 | 	bool queryItem(unsigned long int* hash_value);
26 | 	bool deleteItem(unsigned long int* hash_value);
27 | 
28 | 	bool write(unsigned long int hash_value, int counter);
29 | 	int read(unsigned long int hash_value);
30 | };
31 | 
32 | #endif //COUNTINGBLOOMFILTER_H_
33 | 


--------------------------------------------------------------------------------
/src_DBF/hashfunctions.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * hashfunctions.cpp
 3 |  *
 4 |  *  Created on: Nov 14, 2017
 5 |  *      Author: liaoliangyi
 6 |  */
 7 | 
 8 | #include "hashfunctions.h"
 9 | 
10 | char* HashFunc::sha1(const char* key){
11 | 	EVP_MD_CTX mdctx;
12 | 	unsigned char value[EVP_MAX_MD_SIZE];
13 | 	unsigned int md_len;
14 | 
15 | 	EVP_DigestInit(&mdctx, EVP_sha1());
16 | 	EVP_DigestUpdate(&mdctx, (const void*) key, sizeof(key));
17 | 	EVP_DigestFinal_ex(&mdctx, value, &md_len);
18 | 	EVP_MD_CTX_cleanup(&mdctx);
19 | 
20 | 	return (char*)value;
21 | }
22 | 
23 | char* HashFunc::md5(const char* key){
24 | 	EVP_MD_CTX mdctx;
25 | 	unsigned char value[EVP_MAX_MD_SIZE];
26 | 	unsigned int md_len;
27 | 
28 | 	EVP_DigestInit(&mdctx, EVP_md5());
29 | 	EVP_DigestUpdate(&mdctx, (const void*) key, sizeof(key));
30 | 	EVP_DigestFinal_ex(&mdctx, value, &md_len);
31 | 	EVP_MD_CTX_cleanup(&mdctx);
32 | 
33 | 	return (char*)value;
34 | }
35 | 
36 | 
37 | 


--------------------------------------------------------------------------------
/src/hashfunction.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * hashfunction.cpp
 3 |  *
 4 |  *  Created on: Nov 14, 2017
 5 |  *      Author: liaoliangyi
 6 |  */
 7 | 
 8 | 
 9 | 
10 | #include "hashfunction.h"
11 | 
12 | string  HashFunc::sha1(const char* key){
13 | 	EVP_MD_CTX mdctx;
14 | 	unsigned char md_value[EVP_MAX_MD_SIZE];
15 | 	unsigned int md_len;
16 | 
17 | 	EVP_DigestInit(&mdctx, EVP_sha1());
18 | 	EVP_DigestUpdate(&mdctx, (const void*) key, sizeof(key));
19 | 	EVP_DigestFinal_ex(&mdctx, md_value, &md_len);
20 | 	EVP_MD_CTX_cleanup(&mdctx);
21 | 
22 | 	return std::string((char*)md_value, (size_t)md_len);
23 | }
24 | 
25 | string HashFunc::md5(const char* key){
26 | 	EVP_MD_CTX mdctx;
27 | 	unsigned char md_value[EVP_MAX_MD_SIZE];
28 | 	unsigned int md_len;
29 | 
30 | 	EVP_DigestInit(&mdctx, EVP_md5());
31 | 	EVP_DigestUpdate(&mdctx, (const void*) key, sizeof(key));
32 | 	EVP_DigestFinal_ex(&mdctx, md_value, &md_len);
33 | 	EVP_MD_CTX_cleanup(&mdctx);
34 | 
35 | 	return std::string((char*)md_value, (size_t)md_len);
36 | }
37 | 


--------------------------------------------------------------------------------
/src/linklist.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * linklist.h
 3 |  *
 4 |  *  Created on: Jan 4, 2017
 5 |  *      Author: liaoliangyi
 6 |  */
 7 | 
 8 | #ifndef LINKLIST_H_
 9 | #define LINKLIST_H_
10 | 
11 | class LinkList{
12 | public:
13 | 	CuckooFilter* cf_pt;
14 | 	CuckooFilter* tail_pt;
15 | 	int num;
16 | 	LinkList(size_t single_table_length, size_t fingerprint_size, double single_capacity){
17 | 		cf_pt = new CuckooFilter(single_table_length, fingerprint_size, single_capacity);
18 | 		tail_pt = new CuckooFilter(single_table_length, fingerprint_size, single_capacity);
19 | 		num = 0;
20 | 	}
21 | 	~LinkList(){
22 | 		delete cf_pt;
23 | 		cf_pt = NULL;
24 | 		delete tail_pt;
25 | 		tail_pt = NULL;
26 | 	}
27 | 	bool remove(CuckooFilter* cf_remove){
28 | 		CuckooFilter* frontCF = cf_remove->front;
29 | 		if(frontCF == NULL){
30 | 			this->cf_pt = cf_remove->next;
31 | 		}else{
32 | 			frontCF->next = cf_remove->next;
33 | 		}
34 | 		cf_remove = NULL;
35 | 		num --;
36 | 		return true;
37 | 	}
38 | };
39 | 
40 | 
41 | 
42 | #endif /* LINKLIST_H_ */
43 | 


--------------------------------------------------------------------------------
/src/bithack.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * bithack.h
 3 |  *
 4 |  *  Created on: Dec 21, 2016
 5 |  *      Author: liaoliangyi
 6 |  */
 7 | #ifndef BITHACK_H_
 8 | #define BITHACK_H_
 9 | 
10 | //http://graphics.stanford.edu/~seander/bithacks.html
11 | #define haszero4(x) (((x) - 0x1111ULL) & (~(x)) & 0x8888ULL)
12 | #define hasvalue4(x,n) (haszero4((x) ^ (0x1111ULL * (n))))
13 | 
14 | #define haszero8(x) (((x) - 0x01010101ULL) & (~(x)) & 0x80808080ULL)
15 | #define hasvalue8(x,n) (haszero8((x) ^ (0x01010101ULL * (n))))
16 | 
17 | #define haszero12(x) (((x) - 0x001001001001ULL) & (~(x)) & 0x800800800800ULL)
18 | #define hasvalue12(x,n) (haszero12((x) ^ (0x001001001001ULL * (n))))
19 | 
20 | #define haszero16(x) (((x) - 0x0001000100010001ULL) & (~(x)) & 0x8000800080008000ULL)
21 | #define hasvalue16(x,n) (haszero16((x) ^ (0x0001000100010001ULL * (n))))
22 | 
23 | #define haszero24(x) (((x) - 0x000001000001000001000001ULL) & (~(x)) & 0x800000800000800000800000ULL)
24 | #define hasvalue24(x,n) (haszero24((x) ^ (0x000001000001000001000001ULL * (n))))
25 | 
26 | #define haszero32(x) (((x) - 0x00000001000000010000000100000001ULL) & (~(x)) & 0x80000000800000008000000080000000ULL)
27 | #define hasvalue32(x,n) (haszero32((x) ^ (0x00000001000000010000000100000001ULL * (n))))
28 | 
29 | 
30 | #endif //BITHACK_H_
31 | 


--------------------------------------------------------------------------------
/src_DBF/dynamicbloomfilter.h:
--------------------------------------------------------------------------------
 1 | #ifndef DYNAMICBLOOMFILTER_H_
 2 | #define DYNAMICBLOOMFILTER_H_
 3 | 
 4 | #include<string>
 5 | #include<iostream>
 6 | #include"hashfunctions.h"
 7 | #include"countingbloomfilter.h"
 8 | 
 9 | 
10 | class LinkList{
11 | public:
12 | 	CountingBloomFilter* cf_pt;
13 | 	CountingBloomFilter* tail_pt;
14 | 	static int num;
15 | 	LinkList(int single_capacity, double single_false_positive){
16 | 		cf_pt = new CountingBloomFilter(single_capacity, single_false_positive);
17 | 		tail_pt = new CountingBloomFilter(single_capacity, single_false_positive);
18 | 	}
19 | 
20 | 	LinkList(int n, int m){
21 | 		cf_pt = new CountingBloomFilter(n, m);
22 | 		tail_pt = new CountingBloomFilter(n, m);
23 | 	}
24 | 
25 | 	~LinkList(){
26 | 		delete cf_pt;
27 | 		delete tail_pt;
28 | 	}
29 | };
30 | 
31 | class DynamicBloomFilter{
32 | private:
33 | 	double false_positive;
34 | 	double single_false_positive;
35 | 	int capacity;
36 | 	int single_capacity;
37 | 	int counter;
38 | 	int bits_num;
39 | 	int hash_num;
40 | 
41 | 	CountingBloomFilter* curSBF;
42 | 	CountingBloomFilter* nextSBF;
43 | 
44 | 
45 | public:
46 | 
47 | 	// the link list of building blocks
48 | 	LinkList* sbf_list;
49 | 
50 | 	DynamicBloomFilter(const size_t capacity, const double false_positive, const size_t exp_block_num = 6);
51 | 	DynamicBloomFilter(int n,  int m);
52 | 	~DynamicBloomFilter();
53 | 
54 | 	bool insertItem(const char* item);
55 | 	CountingBloomFilter* getNextSBF(CountingBloomFilter* curSBF);
56 | 	bool queryItem(const char* item);
57 | 	bool deleteItem(const char* item);
58 | 
59 | 	unsigned long int* generateHashVal(const char* item);
60 | 
61 | 	float size_in_mb();
62 | 
63 | };
64 | 
65 | 
66 | #endif //DYNAMICBLOOMFILTER_H_
67 | 
68 | 


--------------------------------------------------------------------------------
/src/dynamiccuckoofilter.h:
--------------------------------------------------------------------------------
 1 | #ifndef DYNAMICCUCKOOFILTER_H_
 2 | #define DYNAMICCUCKOOFILTER_H_
 3 | 
 4 | 
 5 | #include"cuckoofilter.h"
 6 | #include"linklist.h"
 7 | #include<list>
 8 | #include<math.h>
 9 | #include<iostream>
10 | 
11 | 
12 | 
13 | class DynamicCuckooFilter{
14 | private:
15 | 
16 | 	int capacity;
17 | 	int single_capacity;
18 | 
19 | 	int single_table_length;
20 | 
21 | 	double false_positive;
22 | 	double single_false_positive;
23 | 
24 | 	double fingerprint_size_double;
25 | 	int fingerprint_size;
26 | 
27 | 	Victim victim;
28 | 
29 | 	CuckooFilter* curCF;
30 | 	CuckooFilter* nextCF; //it's temporary. uses getNextCF() to get the next CF;
31 | 
32 | public:
33 | 
34 | 	//record the items inside DCF
35 | 	int counter;
36 | 
37 | 	// the link list of building blocks CF1, CF2, ...
38 | 	LinkList* cf_list;
39 | 
40 | 	//construction & distruction functions
41 | 	DynamicCuckooFilter(const size_t capacity, const double false_positive, const size_t exp_block_num = 6);
42 | 	~DynamicCuckooFilter();
43 | 
44 | 	//insert & query & delete functions
45 | 	bool insertItem(const char* item);
46 | 	CuckooFilter* getNextCF(CuckooFilter* curCF);
47 | 	bool failureHandle(Victim &victim);
48 | 	bool queryItem(const char* item);
49 | 	bool deleteItem(const char* item);
50 | 
51 | 	//compaction
52 | 	bool compact();
53 | 	void sort(CuckooFilter** cfq, int queue_length);
54 | 	bool remove(CuckooFilter* cf_remove);
55 | 
56 | 	//generate 2 bucket addresses
57 | 	void generateIF(const char* item, size_t &index, uint32_t &fingerprint, int fingerprint_size, int single_table_length);
58 | 	void generateA(size_t index, uint32_t fingerprint, size_t &alt_index, int single_table_length);
59 | 
60 | 	//get info of DCF
61 | 	int getFingerprintSize();
62 | 	float size_in_mb();
63 | 
64 | 	//extra function to make sure the table length is the power of 2
65 | 	uint64_t upperpower2(uint64_t x);
66 | 
67 | 	void info();
68 | };
69 | 
70 | 
71 | 
72 | 
73 | 
74 | 
75 | 
76 | #endif //DYNAMICCUCKOOFILTER_H
77 | 


--------------------------------------------------------------------------------
/src/cuckoofilter.h:
--------------------------------------------------------------------------------
 1 | #ifndef CUCKOOFILTER_H_
 2 | #define CUCKOOFILTER_H_
 3 | 
 4 | #include<string.h>
 5 | #include<stdlib.h>
 6 | #include"hashfunction.h"
 7 | #include"bithack.h"
 8 | //#include"uint.h"
 9 | 
10 | #define MaxNumKicks 500
11 | 
12 | using namespace std;
13 | 
14 | typedef struct {
15 | 	size_t index;
16 | 	uint32_t fingerprint;
17 | } Victim;
18 | 
19 | typedef struct{
20 | 	char* bit_array;
21 | } Bucket;
22 | 
23 | 
24 | 
25 | class CuckooFilter{
26 | private:
27 | 
28 | 	int capacity;
29 | 	size_t single_table_length;
30 | 	size_t fingerprint_size;
31 | 	size_t bits_per_bucket;
32 | 	size_t bytes_per_bucket;
33 | 
34 | 	Bucket* bucket;
35 | 
36 | 	uint32_t mask;
37 | 
38 | public:
39 | 
40 | 	bool is_full;
41 | 	bool is_empty;
42 | 	int counter;
43 | 	CuckooFilter* next;
44 | 	CuckooFilter* front;
45 | 
46 | 	//construction and distruction function
47 | 	CuckooFilter(const size_t single_table_length, const size_t fingerprint_size, const int capacity);
48 | 	~CuckooFilter();
49 | 
50 | 	//insert & query & delete function
51 | 	int  insertItem(const char* item, Victim &victim);
52 | 	bool insertItem(const size_t index, const uint32_t fingerprint, bool kickout, Victim &victim);
53 | 	bool queryItem(const char* item);
54 | 	bool deleteItem(const char* item);
55 | 
56 | 	bool insertImpl(const size_t index, const uint32_t fingerprint, const bool kickout, Victim &victim);
57 | 	bool queryImpl(const size_t index, const uint32_t fingerprint);
58 | 	bool deleteImpl(const size_t index, const uint32_t fingerprint);
59 | 
60 | 	//generate two candidate bucket addresses
61 | 	void generateIF(const char* item, size_t &index, uint32_t &fingerprint, int fingerprint_size, int single_table_length);
62 | 	void generateA(size_t index, uint32_t fingerprint, size_t &alt_index, int single_table_length);
63 | 
64 | 	//read from bucket & write into bucket
65 | 	uint32_t read(const size_t index, const size_t pos);
66 | 	void write(const size_t index, const size_t pos, const uint32_t fingerprint);
67 | 
68 | 	//move corresponding fingerprints to sparser CF
69 | 	bool transfer(CuckooFilter* tarCF);
70 | 
71 | };
72 | 
73 | 
74 | 
75 | #endif //CUCKOOFILTER_H_
76 | 


--------------------------------------------------------------------------------
/src_DBF/countingbloomfilter.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * countingbloomfilter.cpp
  3 |  *
  4 |  *  Created on: Nov 14, 2017
  5 |  *      Author: liaoliangyi
  6 |  */
  7 | 
  8 | #include "countingbloomfilter.h"
  9 | 
 10 | 
 11 | using namespace std;
 12 | 
 13 | CountingBloomFilter::CountingBloomFilter(int capacity, double false_positive){
 14 | 
 15 | 	this->bits_num = (int) ceil(capacity * (1/log(2.0)) * log(1/false_positive)/log(2.0)) * 4;
 16 | 	this->byte_num = (int) ceil((double) bits_num/8);
 17 | 	this->hash_num = (int) ceil((bits_num/4)/capacity * log(2.0));
 18 | 	this->capacity = capacity;
 19 | 	this->item_num = 0;
 20 | 	this->next = 0;
 21 | 	this->front = 0;
 22 | 
 23 | 	this->bits =  new char[byte_num];
 24 | 	for(int i = 0; i<byte_num; i++){
 25 | 		bits[i] = 0;
 26 | 	}
 27 | }
 28 | 
 29 | CountingBloomFilter::CountingBloomFilter(int n, int m){
 30 | 	this->bits_num = m;
 31 | 	this->byte_num = (int) ceil((double) bits_num/8);
 32 | 	this->hash_num =7;
 33 | 	this->capacity = n;
 34 | 	this->item_num = 0;
 35 | 	this->next = 0;
 36 | 	this->front = 0;
 37 | 
 38 | 	this->bits =  new char[byte_num];
 39 | 	for(int i = 0; i<byte_num; i++){
 40 | 		bits[i] = 0;
 41 | 	}
 42 | }
 43 | 
 44 | CountingBloomFilter::~CountingBloomFilter(){}
 45 | 
 46 | bool CountingBloomFilter::write(unsigned long int hash_value, int counter){
 47 | 	if(counter > 15) {
 48 | 		counter = 15;
 49 | 	}
 50 | 	if(hash_value%2 == 0){
 51 | 		bits[hash_value>>1] &= 0x0f;
 52 | 		bits[hash_value>>1] |= (counter<<4);
 53 | 	}else{
 54 | 		bits[hash_value>>1] &= 0xf0;
 55 | 		bits[hash_value>>1] |= counter;
 56 | 	}
 57 | 
 58 | 	return true;
 59 | }
 60 | 
 61 | int CountingBloomFilter::read(unsigned long int hash_value){
 62 | 	int counter = 0;
 63 | 
 64 | 	if(hash_value%2 == 0){
 65 | 		counter = (bits[hash_value>>1] >> 4);
 66 | 	}else{
 67 | 		counter = (bits[hash_value>>1] & 0x0f);
 68 | 	}
 69 | 
 70 | 	return counter;
 71 | }
 72 | 
 73 | bool CountingBloomFilter::insertItem(unsigned long int* hash_value){
 74 | 	int counter = 0;
 75 | 	for(int i = 0; i<hash_num; i++){
 76 | 		counter = read(hash_value[i]) + 1;
 77 | 		write(hash_value[i], counter);
 78 | 	}
 79 | 
 80 | 	item_num +=1;
 81 | 
 82 | 	return true;
 83 | }
 84 | 
 85 | bool CountingBloomFilter::queryItem(unsigned long int* hash_value){
 86 | 	for(int i = 0; i<hash_num; i++){
 87 | 		if(read(hash_value[i]) == 0) {
 88 | 			return false;
 89 | 		}
 90 | 	}
 91 | 	return true;
 92 | }
 93 | 
 94 | bool CountingBloomFilter::deleteItem(unsigned long int* hash_value){
 95 | 	int counter = 0;
 96 | 
 97 | 	for(int i = 0; i<hash_num; i++){
 98 | 		counter = read(hash_value[i]);
 99 | 		if(counter != 0){
100 | 			counter -= 1;
101 | 			write(hash_value[i], counter);
102 | 		}else{
103 | 			return false;
104 | 		}
105 | 	}
106 | 	item_num -= 1;
107 | 	return true;
108 | }
109 | 
110 | 
111 | 


--------------------------------------------------------------------------------
/src_DBF/dynamicbloomfilter.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * dynamicbloomfilter.cpp
  3 |  *
  4 |  *  Created on: Nov 14, 2017
  5 |  *      Author: liaoliangyi
  6 |  */
  7 | 
  8 | 
  9 | #include "dynamicbloomfilter.h"
 10 | 
 11 | 
 12 | using namespace std;
 13 | 
 14 | int LinkList::num = 0;
 15 | 
 16 | DynamicBloomFilter::DynamicBloomFilter(const size_t item_num, const double fp, const size_t exp_block_num){
 17 | 	capacity = item_num;
 18 | 
 19 | 	single_capacity = capacity/exp_block_num; //s=6 7680 s=12 3840 s=24 1920 s=48 960 s=96 480
 20 | 
 21 | 	false_positive = fp;
 22 | 	single_false_positive = 1-pow((1-this->false_positive), (double)single_capacity/this->capacity);
 23 | 
 24 | 	counter = 0;
 25 | 	bits_num = (int) ceil(single_capacity * (1/log(2.0)) * log(1/single_false_positive)/log(2.0)) * 4;
 26 | //		this->bits_num=(int)ceil(single_capacity*log(single_false_positive)/log(0.61285))*4;
 27 | 	hash_num = (int) ceil((bits_num/4)/single_capacity * log(2.0));
 28 | 
 29 | 	curSBF = new CountingBloomFilter(single_capacity, single_false_positive);
 30 | 	nextSBF = NULL;
 31 | 	sbf_list = new LinkList(single_capacity, single_false_positive);
 32 | 	sbf_list->cf_pt = curSBF;
 33 | 	sbf_list->tail_pt = curSBF;
 34 | 	sbf_list->num = 1;
 35 | }
 36 | 
 37 | DynamicBloomFilter::DynamicBloomFilter(int n,  int m){
 38 | 	false_positive = 0; //not used
 39 | 	capacity = n;
 40 | 	single_capacity = 1024;
 41 | 	single_false_positive = 0;//not used
 42 | 	counter = 0;
 43 | 	bits_num = m;
 44 | 	hash_num = 7;
 45 | 
 46 | 	curSBF = new CountingBloomFilter(1024, m);
 47 | 	nextSBF = NULL;
 48 | 	sbf_list = new LinkList(1024, m);
 49 | 	sbf_list->cf_pt = curSBF;
 50 | 	sbf_list->tail_pt = curSBF;
 51 | 	sbf_list->num = 1;
 52 | }
 53 | 
 54 | DynamicBloomFilter::~DynamicBloomFilter(){
 55 | 	delete curSBF;
 56 | 	delete nextSBF;
 57 | 	delete sbf_list;
 58 | }
 59 | 
 60 | bool DynamicBloomFilter::insertItem(const char* item){
 61 | 	unsigned long int* hash_val = generateHashVal(item);
 62 | 
 63 | 	if(curSBF->item_num >= curSBF->capacity){
 64 | 		curSBF = getNextSBF(curSBF);
 65 | 	}
 66 | 	//duplicate filtering
 67 | //	if(!this->queryItem(item)){
 68 | //		if(curSBF->insertItem(hash_val)){
 69 | //			this->counter += 1;
 70 | //		}
 71 | //	}
 72 | 
 73 | 	if(curSBF->insertItem(hash_val)){
 74 | 		this->counter += 1;
 75 | 	}
 76 | 
 77 | 	return true;
 78 | }
 79 | 
 80 | CountingBloomFilter* DynamicBloomFilter::getNextSBF(CountingBloomFilter* curSBF){
 81 | 	if(curSBF == this->sbf_list->tail_pt){
 82 | 		nextSBF = new CountingBloomFilter(single_capacity, single_false_positive);
 83 | 		curSBF->next = nextSBF;
 84 | 		nextSBF->front = curSBF;
 85 | 		sbf_list->tail_pt = nextSBF;
 86 | 		sbf_list->num++;
 87 | 	}else{
 88 | 		nextSBF = curSBF->next;
 89 | 		if(nextSBF->item_num >= nextSBF->capacity){
 90 | 			nextSBF = getNextSBF(nextSBF);
 91 | 		}
 92 | 	}
 93 | 	return nextSBF;
 94 | }
 95 | 
 96 | bool DynamicBloomFilter::queryItem(const char* item){
 97 | 	unsigned long int* hash_val = generateHashVal(item);
 98 | 
 99 | 	CountingBloomFilter* query_pt = sbf_list->cf_pt;
100 | 	for(int i = 0; i<sbf_list->num; i++){
101 | 		if(query_pt->queryItem(hash_val) == true){
102 | 			return true;
103 | 		}
104 | 		query_pt = query_pt->next;
105 | 	}
106 | 	return false;
107 | }
108 | 
109 | bool DynamicBloomFilter::deleteItem(const char* item){
110 | 	unsigned long int* hash_val = generateHashVal(item);
111 | 	CountingBloomFilter* query_pt = sbf_list->cf_pt;
112 | 	CountingBloomFilter* delete_pt = sbf_list->cf_pt;
113 | 	int counter = 0;
114 | 
115 | 	for(int i = 0; i<sbf_list->num; i++){
116 | 		if(query_pt->queryItem(hash_val) == true){
117 | 			counter += 1;
118 | 		}
119 | 		query_pt = query_pt->next;
120 | 	}
121 | 
122 | 	if(counter == 1){
123 | 		for(int i = 0; i<sbf_list->num; i++){
124 | 			if(delete_pt->queryItem(hash_val) == true){
125 | 				delete_pt->deleteItem(hash_val);
126 | 				this->counter -= 1;
127 | 				return true;
128 | 			}
129 | 			delete_pt = delete_pt->next;
130 | 		}
131 | 	}
132 | 	return false;
133 | }
134 | 
135 | unsigned long int* DynamicBloomFilter::generateHashVal(const char* item){
136 | 	unsigned long int* hash_value = new unsigned long int[hash_num];
137 | 	unsigned long long int* value0 = (unsigned long long int*)HashFunc::sha1(item);
138 | 	unsigned long long int hv0 = *value0;
139 | 	unsigned long long int* value1 = value0;
140 | 	unsigned long long int hv1 = *value1;
141 | 
142 | 	hash_value[0] = (unsigned long int)(hv0>>32) % (bits_num/4);
143 | 	hash_value[1] = (unsigned long int)(hv1 & 0xFFFFFFFF) % (bits_num/4);
144 | 
145 | 	for(int i = 2; i<hash_num; i++) {
146 | 		hash_value[i] = (hash_value[0] + i*hash_value[1] + i*i) % (bits_num/4);
147 | 	}
148 | 
149 | 	return hash_value;
150 | }
151 | 
152 | float DynamicBloomFilter::size_in_mb(){
153 | 	return bits_num * sbf_list->num/ 8.0 / 1024 / 1024;
154 | }
155 | 
156 | 


--------------------------------------------------------------------------------
/src/test.backup:
--------------------------------------------------------------------------------
  1 | #include<stdio.h>
  2 | #include<stdlib.h>
  3 | #include<iostream>
  4 | #include<iomanip>
  5 | #include<time.h>
  6 | #include"dynamiccuckoofilter.h"
  7 | 
  8 | 
  9 | using namespace std;
 10 | 
 11 | typedef struct{
 12 | 	size_t item_num;
 13 | 	double exp_FPR;
 14 | 	int exp_BBN;
 15 | 	double actual_FPR;
 16 | 	int actual_BBN;
 17 | 	int F_size;
 18 | 	double space_cost;
 19 | 	double I_time;
 20 | 	double Q_time;
 21 | 	double D_time;
 22 | 	double C_rate;
 23 | }Metric;
 24 | 
 25 | void test(Metric &metric){
 26 | 
 27 | 	size_t item_num = metric.item_num;
 28 | 	double exp_FPR = metric.exp_FPR;
 29 | 	int exp_BBN = metric.exp_BBN;
 30 | 
 31 | 
 32 | 	DynamicCuckooFilter* dcf = new DynamicCuckooFilter(item_num, exp_FPR, exp_BBN);
 33 | 
 34 | 
 35 | 	//**********insert**********
 36 | 	long* addset = (long*) malloc(sizeof(long) * item_num);
 37 | 	metric.I_time = clock();
 38 | 	for(size_t i = 0; i<item_num; i++){
 39 | //		dcf->insertItem(item_array[i]);
 40 | 		char item[10] = {0};
 41 | 		addset[i] = rand();
 42 | 		sprintf(item, "%ld", addset[i]);
 43 | 		dcf->insertItem(item);
 44 | 	}
 45 | 	metric.I_time = clock() - metric.I_time;
 46 | 	metric.I_time = metric.I_time/CLOCKS_PER_SEC;
 47 | 
 48 | 	metric.space_cost = dcf->size_in_mb();
 49 | 
 50 | 	//**********query**********
 51 | 
 52 | 	int false_positive_count = 0;
 53 | 
 54 | 	metric.Q_time = clock();
 55 | 	for(size_t i = 0; i<item_num; i++){
 56 | //		dcf->queryItem(item_array[i]);
 57 | 		char item[10] = {0};
 58 | 		sprintf(item, "%ld", addset[i]);
 59 | 		if(dcf->queryItem(item) == false){
 60 | 			cout << "Item not found" << endl;
 61 | 		};
 62 | 	}
 63 | 	metric.Q_time = clock() - metric.Q_time;
 64 | 	metric.Q_time = metric.Q_time/CLOCKS_PER_SEC;
 65 | 
 66 | 	//calculate false
 67 | 	for(size_t i = 0; i<item_num; i++){
 68 | 		char item[10] = {0};
 69 | 		sprintf(item, "%ld", i + 1000*1000);
 70 | 		if(dcf->queryItem(item)){
 71 | 			false_positive_count++;
 72 | 		}
 73 | 	}
 74 | 
 75 | 
 76 | 	metric.actual_FPR = (double)false_positive_count/item_num;
 77 | //	printf("false positve rate: %f\n", actual_FPR);
 78 | 
 79 | 
 80 | 	//**********delete**********
 81 | 
 82 | 
 83 | 	size_t count = 0;
 84 | 	metric.D_time = clock();
 85 | 	while(count < item_num){
 86 | 		char item[10] = {0};
 87 | 		sprintf(item, "%ld", addset[count]);
 88 | 		dcf->deleteItem(item);
 89 | 		count += 1; //delete all the items
 90 | 	}
 91 | 	metric.D_time = clock() - metric.D_time;
 92 | 	metric.D_time = metric.D_time/CLOCKS_PER_SEC;
 93 | 
 94 | 
 95 | 	//**********compact**********
 96 | 
 97 | 	int size_before = dcf->cf_list->num;
 98 | 	dcf->compact();
 99 | 	int size_after = dcf->cf_list->num;
100 | 
101 | 	metric.actual_BBN = size_before;
102 | 	metric.C_rate = (double)(size_before-size_after)/size_before;
103 | 	metric.F_size = dcf->getFingerprintSize();
104 | 
105 | 
106 | }
107 | 
108 | 
109 | int main(int argc, char* argv[]){
110 | 
111 | 
112 | 	cout << setw(15) << "item_num" << setw(15) << "exp_FPR"
113 | 		 << setw(15) << "actual_FPR" << setw(15) << "actual_BBN" << setw(15) << "F_size(bits)"
114 | 		 << setw(15) << "space_cost(MB)"
115 | 		 << setw(15) << "I_time(s)" << setw(15) << "Q_time(s)" << setw(15) << "D_time(s)" << setw(10) << "C_rate"
116 | 		 << endl;
117 | 
118 | 	Metric example;
119 | 	example.item_num = 1000 * 1000;
120 | 	example.exp_FPR = 0.02;
121 | 	example.exp_BBN = 6;
122 | 
123 | 	test(example);
124 | 
125 | 
126 | 	cout << setw(15) << example.item_num << setw(15) << example.exp_FPR
127 | 		 << setw(15) << example.actual_FPR << setw(15) << example.actual_BBN << setw(15) << example.F_size
128 | 		 << setw(15) << example.space_cost
129 | 		 << setw(15) << example.I_time << setw(15) << example.Q_time << setw(15) << example.D_time << setw(10) << example.C_rate
130 | 		 << endl;
131 | 
132 | 
133 | 	example.item_num = 1000 * 1000;
134 | 	example.exp_FPR = 0.001;
135 | 	example.exp_BBN = 8;
136 | 
137 | 	test(example);
138 | 
139 | 
140 | 	cout << setw(15) << example.item_num << setw(15) << example.exp_FPR << setw(15) << example.exp_BBN
141 | 		 << setw(15) << example.actual_FPR << setw(15) << example.actual_BBN << setw(15) << example.F_size
142 | 		 << setw(15) << example.space_cost
143 | 		 << setw(15) << example.I_time << setw(15) << example.Q_time << setw(15) << example.D_time << setw(10) << example.C_rate
144 | 		 << endl;
145 | 
146 | 
147 | 
148 | 	for(double fraction = 0.2; fraction < 1.5; fraction += 0.2) {
149 | 		Metric benchmark;
150 | 		benchmark.item_num = 46080 * fraction;
151 | 		benchmark.exp_FPR = 0.012;
152 | 		benchmark.exp_BBN = 6;
153 | 		test(benchmark);
154 | 
155 | 
156 | 		cout << setw(12) << "46080 * " << fraction << setw(15) << benchmark.exp_FPR << setw(15) << benchmark.exp_BBN
157 | 			 << setw(15) << benchmark.actual_FPR << setw(15) << benchmark.actual_BBN << setw(15) << benchmark.F_size
158 | 			 << setw(15) << benchmark.space_cost
159 | 			 << setw(15) << benchmark.I_time << setw(15) << benchmark.Q_time << setw(15) << benchmark.D_time << setw(10) << benchmark.C_rate
160 | 			 << endl;
161 | 	}
162 | 
163 | 
164 | }
165 | 
166 | 


--------------------------------------------------------------------------------
/src_DBF/test.cpp:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <iostream>
  4 | #include <fstream>
  5 | #include <sstream>
  6 | #include <iomanip>
  7 | #include <time.h>
  8 | #include <vector>
  9 | 
 10 | #include "dynamicbloomfilter.h"
 11 | 
 12 | using namespace std;
 13 | 
 14 | typedef struct{
 15 | 	size_t item_num;
 16 | 	double exp_FPR;
 17 | 	string dataset_path;
 18 | }Config;
 19 | 
 20 | 
 21 | typedef struct{
 22 | //	size_t item_num;
 23 | //	double exp_FPR;
 24 | 	int exp_BBN;
 25 | 	double actual_FPR;
 26 | 	int actual_BBN;
 27 | 	int F_size;
 28 | 	double space_cost;
 29 | 	double I_time;
 30 | 	double Q_time;
 31 | 	double D_time;
 32 | 	double C_rate;
 33 | }Metric;
 34 | 
 35 | 
 36 | 
 37 | Metric test(const Config config, string *data){
 38 | 
 39 | 	Metric metric;
 40 | 	DynamicBloomFilter* dbf = new DynamicBloomFilter(config.item_num, config.exp_FPR);
 41 | 
 42 | 	//**********insert**********
 43 | 	metric.I_time = clock();
 44 | 	for(size_t i = 0; i<config.item_num; i++){
 45 | 		dbf->insertItem(data[i].c_str());
 46 | 	}
 47 | 	metric.I_time = clock() - metric.I_time;
 48 | 	metric.I_time = metric.I_time/CLOCKS_PER_SEC;
 49 | 
 50 | 	metric.space_cost = dbf->size_in_mb();
 51 | 
 52 | 	//**********query**********
 53 | 
 54 | 	int false_positive_count = 0;
 55 | 
 56 | 	metric.Q_time = clock();
 57 | 	for(size_t i = 0; i<config.item_num; i++){
 58 | 		if(dbf->queryItem(data[i].c_str()) == false){
 59 | 			cout << "Item not found" << endl;
 60 | 		};
 61 | 	}
 62 | 	metric.Q_time = clock() - metric.Q_time;
 63 | 	metric.Q_time = metric.Q_time/CLOCKS_PER_SEC;
 64 | 
 65 | 	//calculate false
 66 | 	for(size_t i = 0; i<config.item_num; i++){
 67 | 		char item[10] = {0};
 68 | 		sprintf(item, "%ld", i + 1000*1000);
 69 | 		if(dbf->queryItem(item)){
 70 | 			false_positive_count++;
 71 | 		}
 72 | 	}
 73 | 
 74 | 
 75 | 	metric.actual_FPR = (double)false_positive_count/config.item_num;
 76 | //	printf("false positve rate: %f\n", actual_FPR);
 77 | 
 78 | 
 79 | 	//**********delete**********
 80 | 
 81 | 
 82 | 	size_t count = 0;
 83 | 	metric.D_time = clock();
 84 | 	while(count < config.item_num){
 85 | 		dbf->deleteItem(data[count].c_str());
 86 | 		count += 1; //delete all the items
 87 | 	}
 88 | 	metric.D_time = clock() - metric.D_time;
 89 | 	metric.D_time = metric.D_time/CLOCKS_PER_SEC;
 90 | 
 91 | 
 92 | 	//**********compact**********
 93 | 
 94 | 	int size_before = dbf->sbf_list->num;
 95 | 
 96 | 	metric.actual_BBN = size_before;
 97 | 	metric.C_rate = 0;
 98 | 	metric.F_size = 0;
 99 | 
100 | 	return metric;
101 | 
102 | }
103 | 
104 | 
105 | string Get_Value(string config_buff){
106 | 	string value;
107 | 	int pos = config_buff.find("=", 0);
108 | 	if (pos != -1)
109 | 	{
110 | 		pos++;
111 | 		value = config_buff.substr(pos, config_buff.length());
112 | 	} else {
113 | 		exit(1);
114 | 	}
115 | 
116 | 	while(1){
117 | 		pos = value.find(" ", 0);
118 | 		if(pos >= 0){
119 | 			value = value.substr(pos+1, config_buff.length());
120 | 		}
121 | 		else{
122 | 			break;
123 | 		}
124 | 	}
125 | 
126 | 	return value;
127 | }
128 | 
129 | Config Read_Config(const string path){
130 | 	ifstream in_config(path.c_str());
131 | 	string config_buff;
132 | 	Config configuration;
133 | 	getline(in_config, config_buff);
134 | 	configuration.exp_FPR = atof(Get_Value(config_buff).c_str());
135 | 	getline(in_config, config_buff);
136 | 	configuration.item_num = atof(Get_Value(config_buff).c_str());
137 | 	getline(in_config, config_buff);
138 | 	configuration.dataset_path = Get_Value(config_buff);
139 | 
140 | 	return configuration;
141 | }
142 | 
143 | 
144 | string* Read_Dataset(const Config config, const string path){
145 | 	ifstream in_input(path.c_str());
146 | 	string input_buff;
147 | 	if(!getline(in_input, input_buff)){
148 | 		cout << "Read File Error!" << endl;
149 | 	}
150 | 	string buff;
151 | 	stringstream ss(input_buff);
152 | 
153 | 	vector<string> tokens;
154 | 	size_t item_count = 0;
155 | 	string *input_data = new string[config.item_num];
156 | 	while (item_count<config.item_num){
157 | 		ss >> buff;
158 | 		tokens.push_back(buff);
159 | 		input_data[item_count] = buff;
160 | 		item_count ++;
161 | 	}
162 | 	return input_data;
163 | }
164 | 
165 | void Print_Info(Config config, Metric metric){
166 | 
167 | 	ofstream out("./result/result.txt");
168 | 
169 | 	out << setw(15) << "item_num" << setw(15) << "exp_FPR"
170 | 		<< setw(15) << "actual_FPR" << setw(15) << "actual_BBN" << setw(15) << "F_size(bits)"
171 | 		<< setw(15) << "space_cost(MB)"
172 | 		<< setw(15) << "I_time(s)" << setw(15) << "Q_time(s)" << setw(15) << "D_time(s)" << setw(10) << "C_rate"
173 | 		<< endl;
174 | 
175 | 	out << setw(15) << config.item_num << setw(15) << config.exp_FPR
176 | 		<< setw(15) << metric.actual_FPR << setw(15) << metric.actual_BBN << setw(15) << metric.F_size
177 | 		<< setw(15) << metric.space_cost
178 | 		<< setw(15) << metric.I_time << setw(15) << metric.Q_time << setw(15) << metric.D_time << setw(10) << metric.C_rate
179 | 		<< endl;
180 | }
181 | 
182 | int main(int argc, char* argv[]){
183 | 
184 | 	string config_path = "./configuration/config.txt";
185 | 	Config config = Read_Config(config_path);
186 | 
187 | 	string dataset_path = config.dataset_path;
188 | 	string *data = Read_Dataset(config, dataset_path);
189 | 
190 | 
191 |     Metric metric = test(config, data);
192 | 
193 |     Print_Info(config, metric);
194 | 
195 | }
196 | 


--------------------------------------------------------------------------------
/src/test.cpp:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <stdlib.h>
  3 | #include <iostream>
  4 | #include <fstream>
  5 | #include <sstream>
  6 | #include <iomanip>
  7 | #include <time.h>
  8 | #include <vector>
  9 | #include "dynamiccuckoofilter.h"
 10 | 
 11 | using namespace std;
 12 | 
 13 | typedef struct{
 14 | 	size_t item_num;
 15 | 	double exp_FPR;
 16 | 	string dataset_path;
 17 | }Config;
 18 | 
 19 | 
 20 | typedef struct{
 21 | 	int exp_BBN;
 22 | 	double actual_FPR;
 23 | 	int actual_BBN;
 24 | 	int F_size;
 25 | 	double space_cost;
 26 | 	double I_time;
 27 | 	double Q_time;
 28 | 	double D_time;
 29 | 	double C_rate;
 30 | }Metric;
 31 | 
 32 | 
 33 | 
 34 | Metric test(const Config config, string *data){
 35 | 
 36 | 	Metric metric;
 37 | 	DynamicCuckooFilter* dcf = new DynamicCuckooFilter(config.item_num, config.exp_FPR);
 38 | 
 39 | 
 40 | 	//**********insert**********
 41 | 	metric.I_time = clock();
 42 | 	for(size_t i = 0; i<config.item_num; i++){
 43 | 		dcf->insertItem(data[i].c_str());
 44 | 	}
 45 | 	metric.I_time = clock() - metric.I_time;
 46 | 	metric.I_time = metric.I_time/CLOCKS_PER_SEC;
 47 | 
 48 | 	metric.space_cost = dcf->size_in_mb();
 49 | 
 50 | 	//**********query**********
 51 | 
 52 | 	int false_positive_count = 0;
 53 | 
 54 | 	metric.Q_time = clock();
 55 | 	for(size_t i = 0; i<config.item_num; i++){
 56 | 		if(dcf->queryItem(data[i].c_str()) == false){
 57 | 			cout << "Item not found" << endl;
 58 | 		};
 59 | 	}
 60 | 	metric.Q_time = clock() - metric.Q_time;
 61 | 	metric.Q_time = metric.Q_time/CLOCKS_PER_SEC;
 62 | 
 63 | 	//calculate false
 64 | 	for(size_t i = 0; i<config.item_num; i++){
 65 | 		char item[10] = {0};
 66 | 		sprintf(item, "%ld", i + 1000*1000);
 67 | 		if(dcf->queryItem(item)){
 68 | 			false_positive_count++;
 69 | 		}
 70 | 	}
 71 | 
 72 | 
 73 | 	metric.actual_FPR = (double)false_positive_count/config.item_num;
 74 | 
 75 | 
 76 | 	//**********delete**********
 77 | 
 78 | 
 79 | 	size_t count = 0;
 80 | 	metric.D_time = clock();
 81 | 	while(count < config.item_num){
 82 | 		dcf->deleteItem(data[count].c_str());
 83 | 		count += 1; //delete all the items
 84 | 	}
 85 | 	metric.D_time = clock() - metric.D_time;
 86 | 	metric.D_time = metric.D_time/CLOCKS_PER_SEC;
 87 | 
 88 | 
 89 | 	//**********compact**********
 90 | 
 91 | 	int size_before = dcf->cf_list->num;
 92 | 	dcf->compact();
 93 | 	int size_after = dcf->cf_list->num;
 94 | 
 95 | 	metric.actual_BBN = size_before;
 96 | 	metric.C_rate = (double)(size_before-size_after)/size_before;
 97 | 	metric.F_size = dcf->getFingerprintSize();
 98 | 
 99 | 	return metric;
100 | 
101 | }
102 | 
103 | 
104 | 
105 | string Get_Value(string config_buff){
106 | 	string value;
107 | 	int pos = config_buff.find("=", 0);
108 | 	if (pos != -1)
109 | 	{
110 | 		pos++;
111 | 		value = config_buff.substr(pos, config_buff.length());
112 | 	} else {
113 | 		exit(1);
114 | 	}
115 | 
116 | 	while(1){
117 | 		pos = value.find(" ", 0);
118 | 		if(pos >= 0){
119 | 			value = value.substr(pos+1, config_buff.length());
120 | 		}
121 | 		else{
122 | 			break;
123 | 		}
124 | 	}
125 | 
126 | 	return value;
127 | }
128 | 
129 | 
130 | Config Read_Config(const string path){
131 | 	ifstream in_config(path.c_str());
132 | 	string config_buff;
133 | 	Config configuration;
134 | 	getline(in_config, config_buff);
135 | 	configuration.exp_FPR = atof(Get_Value(config_buff).c_str());
136 | 	getline(in_config, config_buff);
137 | 	configuration.item_num = atof(Get_Value(config_buff).c_str());
138 | 	getline(in_config, config_buff);
139 | 	configuration.dataset_path = Get_Value(config_buff);
140 | 
141 | 	return configuration;
142 | }
143 | 
144 | 
145 | string* Read_Dataset(const Config config, const string path){
146 | 	ifstream in_input(path.c_str());
147 | 	string input_buff;
148 | 	if(!getline(in_input, input_buff)){
149 | 		cout << "Read File Error!" << endl;
150 | 	}
151 | 	string buff;
152 | 	stringstream ss(input_buff);
153 | 
154 | 	vector<string> tokens;
155 | 	size_t item_count = 0;
156 | 	string *input_data = new string[config.item_num];
157 | 	while (item_count < config.item_num){
158 | 		ss >> buff;
159 | 		tokens.push_back(buff);
160 | 		input_data[item_count] = buff;
161 | 		item_count ++;
162 | 	}
163 | 	return input_data;
164 | }
165 | 
166 | void Print_Info(Config config, Metric metric){
167 | 
168 | 	ofstream out("./result/result.txt");
169 | 
170 | 	out << setw(15) << "item_num" << setw(15) << "exp_FPR"
171 | 		<< setw(15) << "actual_FPR" << setw(15) << "actual_BBN" << setw(15) << "F_size(bits)"
172 | 		<< setw(15) << "space_cost(MB)"
173 | 		<< setw(15) << "I_time(s)" << setw(15) << "Q_time(s)" << setw(15) << "D_time(s)" << setw(10) << "C_rate"
174 | 		<< endl;
175 | 
176 | 	out << setw(15) << config.item_num << setw(15) << config.exp_FPR
177 | 		<< setw(15) << metric.actual_FPR << setw(15) << metric.actual_BBN << setw(15) << metric.F_size
178 | 		<< setw(15) << metric.space_cost
179 | 		<< setw(15) << metric.I_time << setw(15) << metric.Q_time << setw(15) << metric.D_time << setw(10) << metric.C_rate
180 | 		<< endl;
181 | }
182 | 
183 | 
184 | 
185 | 
186 | int main(int argc, char* argv[]){
187 | 
188 | 	string config_path = "./configuration/config.txt";
189 | 	Config config = Read_Config(config_path);
190 | 
191 | 	string dataset_path = config.dataset_path;
192 | 	string *data = Read_Dataset(config, dataset_path);
193 | 
194 | 
195 |     Metric metric = test(config, data);
196 | 
197 |     Print_Info(config, metric);
198 | 
199 | }
200 | 
201 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # The Dynamic Cuckoo Filter
  2 | 
  3 | ## Overview
  4 | The Dynamic Cuckoo Filter (DCF) is an efficient approximate membership test data structure. Different from the classic Bloom filter and its variants, DCF is especially designed for highly dynamic dataset and support extending and reducing its capacity. The advantages of DCF are as follows
  5 | 
  6 | * The DCF design is the first to achieve both reliable item deletion and flexibly extending/reducing for approximate set representation and membership testing
  7 | * DCF outperforms the state-of-the-art DBF design in terms of the capability of reliable item deletion
  8 | * A DCF reduces the required memory space of the DBF by 75% as well as improving the speeds of insert/query/delete operation by 30% to 80%.
  9 | 
 10 | ## Structure of DCF
 11 | 
 12 | ![DCF example](https://github.com/LiangyiLiao/TheDynamicCuckooFilter/blob/master/figure/DCFexample.png)
 13 | 
 14 | A DCF leverages CF as building block and consists of a number of s linked homogeneous CFs. The DCF leverages fingerprints
 15 | to represent items. Each fingerprints owns two candidate bucket addresses generated by partial-key cuckoo hashing. A DCF achieves dynamic capacity by appending new building blocks and removing empty building blocks generated by compact operation. 
 16 | 
 17 | ## API
 18 | Generate DCF according to the expected maximum item number, expected false positive rate.
 19 | 
 20 | ```c++
 21 | DynamicCuckooFilter* dcf = new DynamicCuckooFilter(config.item_num, config.exp_FPR);
 22 | ```
 23 | 
 24 | The expected building block number of DCF is set to 6 by default and can also be modified as follow
 25 | 
 26 | ```c++
 27 | DynamicCuckooFilter* dcf = new DynamicCuckooFilter(config.item_num, config.exp_FPR, config.exp_BBN);
 28 | ```
 29 | 
 30 | 
 31 | Four operations of DCF: Insert, Query, Delete and Compact
 32 | 
 33 | ```c++
 34 | dcf->insertItem(item) // insert item ,item is a char* format
 35 | dcf->queryItem(item)  //query item
 36 | dcf->deleteItem(item) //delete item
 37 | dcf->compact() //compact DCF
 38 | ```
 39 | 
 40 | ## How to use?
 41 | ### Environment
 42 | We implement DCF in a Linux (Ubuntu 14.04.3 LTS) with an Intel(R) Core(TM) i5-2430M CPU @2.4GHz and OpenSSL library environment. 
 43 | 
 44 | Install OpenSSL (Please refer to https://www.openssl.org to learn more).
 45 | 
 46 | ```txt
 47 | sudo apt-get install openssl
 48 | sudo apt-get install libssl-dev
 49 | ```
 50 | Build and run the example
 51 | 
 52 | ```txt
 53 | cd src/
 54 | make test
 55 | ./test
 56 | ```
 57 | 
 58 | 
 59 | ### Configurations
 60 | Configurations including false pisitive, item number and dataset path can be costomized in "configuration/config.txt". 
 61 | 
 62 | ```txt
 63 | false positive = 0.02
 64 | item number = 1000000
 65 | input file path = input/input2.txt
 66 | ```
 67 | 
 68 | ### Results
 69 | Results are shown in "output/results.txt", including false positive, fingerprint size, building block number, operation time consumed and etc. In the following is the comparison of DCF and DBF when dealing with 1,000,000 items (including insert/query/delete/compact operation).
 70 | 
 71 | Metrics:
 72 | 
 73 | item_num: total number inserted/queried/deleted
 74 | 
 75 | exp_FPR: the expected false positive rate
 76 | 
 77 | actual_FPR: the false positive rate that we measured
 78 | 
 79 | actual_BBN: the building block number that we observed
 80 | 
 81 | F_size: fingerprint size
 82 | 
 83 | space_cost: space overhead of data structure
 84 | 
 85 | I_time: insert time
 86 | 
 87 | Q_time: query time
 88 | 
 89 | D_time: delete time
 90 | 
 91 | C_rate: compact rate
 92 | 
 93 | ```txt
 94 | ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 95 | /// item_num   exp_FPR    actual_FPR    actual_BBN  F_size(bits)  space_cost(MB)   I_time(s)  Q_time(s)  D_time(s)  C_rate  ///
 96 | /// 1000000     0.0005      0.000502    5           16            2.5              1.08       1.05       1.36       1       ///
 97 | /// 1000000     0.0005      0.000505    7           0             10.8754          1.69       1.92       3.35       0       ///
 98 | ///                                                               4X               1.5X       1.8X       2.4X               ///
 99 | ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
100 | 
101 | ```
102 | After transform operation time to speed, the DCF improving the speeds of insert/query/delete by 30% to 80% and 4X space efficiency.
103 | 
104 | ## Author and Copyright
105 | 
106 | DCF is developed in Big Data Technology and System Lab, Services Computing Technology and System Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, China by Liangyi Liao (liaoliangyi@hust.edu.cn), Hanhua Chen (chen@hust.edu.cn), Hai Jin (hjin@hust.edu.cn)
107 | 
108 | Copyright (C) 2017, [STCS & CGCL](http://grid.hust.edu.cn/) and [Huazhong University of Science and Technology](http://www.hust.edu.cn).
109 | 
110 | ## Publication
111 | 
112 | If you want to know more detailed information, please refer to this paper:
113 | 
114 | Hanhua Chen, Liangyi Liao, Hai Jin, Jie Wu, "The Dynamic Cuckoo Filter," Proceedings of the 25th IEEE International Conference on Network Protocols (ICNP 2017), Toronto, Canada, Oct. 10-13, 2017.
115 | 


--------------------------------------------------------------------------------
/src/dynamiccuckoofilter.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * dynamiccuckoofilter.cpp
  3 |  *
  4 |  *  Created on: Nov 14, 2017
  5 |  *      Author: liaoliangyi
  6 |  */
  7 | 
  8 | #include "dynamiccuckoofilter.h"
  9 | 
 10 | 
 11 | using namespace std;
 12 | 
 13 | 
 14 | DynamicCuckooFilter::DynamicCuckooFilter(const size_t item_num, const double fp, const size_t exp_block_num){
 15 | 
 16 | 	capacity = item_num;
 17 | 
 18 | 	single_table_length = upperpower2(capacity/4.0/exp_block_num);//2048 1024 512 256 128 ---!!!---must be the power of 2---!!!---
 19 | 	single_capacity = single_table_length*0.9375*4;//s=6 1920 s=12 960 s=24 480 s=48 240 s=96 120
 20 | 
 21 | 	false_positive = fp;
 22 | 	single_false_positive = 1-pow(1.0-false_positive, ((double)single_capacity/capacity));
 23 | 
 24 | 	fingerprint_size_double = ceil(log(8.0/single_false_positive)/log(2));
 25 | 	if(fingerprint_size_double>0 && fingerprint_size_double<=4){
 26 | 		fingerprint_size = 4;
 27 | 	}else if(fingerprint_size_double>4 && fingerprint_size_double<=8){
 28 | 		fingerprint_size = 8;
 29 | 	}else if(fingerprint_size_double>8 && fingerprint_size_double<=12){
 30 | 		fingerprint_size = 12;
 31 | 	}else if(fingerprint_size_double>12 && fingerprint_size_double<=16){
 32 | 		fingerprint_size = 16;
 33 | 	}else if(fingerprint_size_double>16 && fingerprint_size_double<=24){
 34 | 		fingerprint_size = 16;
 35 | 	}else if(fingerprint_size_double>24 && fingerprint_size_double<=32){
 36 | 		fingerprint_size = 16;
 37 | 	}else{
 38 | 		cout<<"fingerprint out of range!!!"<<endl;
 39 | 		fingerprint_size = 16;
 40 | 	}
 41 | 
 42 | 	counter = 0;
 43 | 
 44 | 
 45 | 	curCF = new CuckooFilter(single_table_length, fingerprint_size, single_capacity);
 46 | 	nextCF = new CuckooFilter(single_table_length, fingerprint_size, single_capacity);
 47 | 
 48 | 	cf_list = new LinkList(single_table_length, fingerprint_size, single_capacity);
 49 | 	cf_list->cf_pt = curCF;
 50 | 	cf_list->tail_pt = curCF;
 51 | 	cf_list->num = 1;
 52 | }
 53 | 
 54 | DynamicCuckooFilter::~DynamicCuckooFilter(){
 55 | //	delete victim;
 56 | 	delete curCF;
 57 | 	delete nextCF;
 58 | 	delete cf_list;
 59 | }
 60 | 
 61 | 
 62 | 
 63 | bool DynamicCuckooFilter::insertItem(const char* item){
 64 | 	if(curCF->is_full == true){
 65 | 		curCF = getNextCF(curCF);
 66 | 	}
 67 | 
 68 | 	if(curCF->insertItem(item, victim)){
 69 | 		counter++;
 70 | 	}else{
 71 | 		failureHandle(victim);
 72 | 		counter++;
 73 | 	}
 74 | 
 75 | 	return true;
 76 | }
 77 | 
 78 | CuckooFilter* DynamicCuckooFilter::getNextCF(CuckooFilter* curCF){
 79 | 	if(curCF == cf_list->tail_pt){
 80 | 		nextCF = new CuckooFilter(single_table_length, fingerprint_size, single_capacity);
 81 | 		curCF->next = nextCF;
 82 | 		nextCF->front = curCF;
 83 | 		cf_list->tail_pt = nextCF;
 84 | 		cf_list->num++;
 85 | 	}else{
 86 | 		nextCF = curCF->next;
 87 | 		if(nextCF->is_full){
 88 | 			nextCF = getNextCF(nextCF);
 89 | 		}
 90 | 	}
 91 | 	return nextCF;
 92 | }
 93 | 
 94 | bool DynamicCuckooFilter::failureHandle(Victim &victim){
 95 | 	nextCF = getNextCF(curCF);
 96 | 	if(nextCF->insertItem(victim.index, victim.fingerprint,true, victim) == false){
 97 | 		nextCF = getNextCF(nextCF);
 98 | 		failureHandle(victim);
 99 | 	}
100 | 	return true;
101 | }
102 | 
103 | bool DynamicCuckooFilter::queryItem(const char* item){
104 | 	size_t index, alt_index;
105 | 	uint32_t fingerprint;
106 | 
107 | 	generateIF(item, index, fingerprint, fingerprint_size, single_table_length);
108 | 	generateA(index, fingerprint, alt_index, single_table_length);
109 | 
110 | 	CuckooFilter* query_pt = cf_list->cf_pt;
111 | 	for(int count = 0; count<cf_list->num; count++){
112 | 
113 | 		if(query_pt->queryImpl(index, fingerprint)){
114 | 			return true;
115 | 		}else if(query_pt->queryImpl(alt_index, fingerprint)){
116 | 			return true;
117 | 		}else{
118 | 			query_pt = query_pt->next;
119 | 		}
120 | 		if(query_pt == 0){
121 | 			break;
122 | 		}
123 | 
124 | //		if(query_pt->queryImpl(index, fingerprint)){
125 | //			return true;
126 | //		}
127 | //		generateA(index, fingerprint, alt_index, single_table_length);
128 | //		if(query_pt->queryImpl(alt_index, fingerprint)){
129 | //			return true;
130 | //		}else{
131 | //			query_pt = query_pt->next;
132 | //		}
133 | //		if(query_pt == 0){
134 | //			break;
135 | //		}
136 | 	}
137 | 	return false;
138 | }
139 | 
140 | bool DynamicCuckooFilter::deleteItem(const char* item){
141 | 	size_t index, alt_index;
142 | 	uint32_t fingerprint;
143 | 
144 | 	generateIF(item, index, fingerprint, fingerprint_size, single_table_length);
145 | 	generateA(index, fingerprint, alt_index, single_table_length);
146 | 	CuckooFilter* delete_pt = cf_list->cf_pt;
147 | 	for(int count = 0; count<cf_list->num; count++){
148 | 		if(delete_pt->queryImpl(index, fingerprint)){
149 | 			if(delete_pt->deleteImpl(index, fingerprint)){
150 | 				counter--;
151 | 				return true;
152 | 			}
153 | 		}else if(delete_pt->queryImpl(alt_index, fingerprint)){
154 | 			if(delete_pt->deleteImpl(alt_index ,fingerprint)){
155 | 				counter--;
156 | 				return true;
157 | 			}
158 | 		}else{
159 | 			delete_pt = delete_pt->next;
160 | 		}
161 | 	}
162 | 	return false;
163 | }
164 | 
165 | 
166 | 
167 | bool DynamicCuckooFilter::compact(){
168 | 	int queue_length = 0;
169 | 	CuckooFilter* temp = cf_list->cf_pt;
170 | 	for(int count = 0; count<cf_list->num; count++){
171 | 		if(!temp->is_full){
172 | 			queue_length++;
173 | 		}
174 | 		temp = temp->next;
175 | 	}
176 | 	if(queue_length == 0){
177 | 		return true;
178 | 	}
179 | 
180 | 	CuckooFilter** cfq  = new CuckooFilter*[queue_length];
181 | 	int pos = 0;
182 | 	temp = cf_list->cf_pt;
183 | 	for(int count = 0; count<cf_list->num; count++){
184 | 		if(!temp->is_full){
185 | 			cfq[pos] = temp;
186 | 			pos++;
187 | 		}
188 | 		temp = temp->next;
189 | 	}
190 | 
191 | 	sort(cfq, queue_length);
192 | 	for(int i = 0; i<queue_length-1; i++){
193 | 		for(int j = queue_length-1; j>i; j--){
194 | 			cfq[i]->transfer(cfq[j]);
195 | 			if(cfq[i]->is_empty == true){
196 | 				cf_list->remove(cfq[i]);
197 | 				break;
198 | 			}
199 | 		}
200 | 	}
201 | 	if(cfq[queue_length-1]->is_empty == true){
202 | 		cf_list->remove(cfq[queue_length-1]);
203 | 	}
204 | 
205 | 	return true;
206 | }
207 | 
208 | void DynamicCuckooFilter::sort(CuckooFilter** cfq, int queue_length){
209 | 	CuckooFilter* temp;
210 | 	for(int i = 0; i<queue_length-1; i++){
211 | 		for(int j = 0; j<queue_length-1-i; j++){
212 | 			if(cfq[j]->counter > cfq[j+1]->counter){
213 | 				temp = cfq[j];
214 | 				cfq[j] = cfq[j+1];
215 | 				cfq[j+1] = temp;
216 | 			}
217 | 		}
218 | 	}
219 | }
220 | 
221 | 
222 | 
223 | void DynamicCuckooFilter::generateIF(const char* item, size_t &index, uint32_t &fingerprint, int fingerprint_size, int single_table_length){
224 | 	std::string  value = HashFunc::sha1(item);
225 | 	uint64_t hv = *((uint64_t*) value.c_str());
226 | 
227 | 	index = ((uint32_t) (hv >> 32)) % single_table_length;
228 | 	fingerprint = (uint32_t) (hv & 0xFFFFFFFF);
229 | 	fingerprint &= ((0x1ULL<<fingerprint_size)-1);
230 | 	fingerprint += (fingerprint == 0);
231 | }
232 | 
233 | void DynamicCuckooFilter::generateA(size_t index, uint32_t fingerprint, size_t &alt_index, int single_table_length){
234 | 	alt_index = (index ^ (fingerprint * 0x5bd1e995)) % single_table_length;
235 | }
236 | 
237 | 
238 | 
239 | int DynamicCuckooFilter::getFingerprintSize(){
240 | 	return fingerprint_size;
241 | }
242 | 
243 | float DynamicCuckooFilter::size_in_mb(){
244 | 	return fingerprint_size * 4.0 * single_table_length * cf_list->num / 8 / 1024 / 1024;
245 | }
246 | 
247 | uint64_t DynamicCuckooFilter::upperpower2(uint64_t x) {
248 |   x--;
249 |   x |= x >> 1;
250 |   x |= x >> 2;
251 |   x |= x >> 4;
252 |   x |= x >> 8;
253 |   x |= x >> 16;
254 |   x |= x >> 32;
255 |   x++;
256 |   return x;
257 | }
258 | 


--------------------------------------------------------------------------------
/src/cuckoofilter.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * cuckoofilter.cpp
  3 |  *
  4 |  *  Created on: Nov 14, 2017
  5 |  *      Author: liaoliangyi
  6 |  */
  7 | 
  8 | #include "cuckoofilter.h"
  9 | 
 10 | CuckooFilter::CuckooFilter(const size_t table_length, const size_t fingerprint_bits, const int single_capacity){
 11 | 	fingerprint_size = fingerprint_bits;
 12 | 	bits_per_bucket = fingerprint_size*4;
 13 | 	bytes_per_bucket = (fingerprint_size*4+7)>>3;
 14 | 	single_table_length = table_length;
 15 | 	counter = 0;
 16 | 	capacity = single_capacity;
 17 | 	is_full = false;
 18 | 	is_empty = true;
 19 | 	next = NULL;
 20 | 	front = NULL;
 21 | 	mask = (1ULL << fingerprint_size) - 1;
 22 | 
 23 | 	bucket = new Bucket[single_table_length];
 24 | 	for(size_t i = 0; i<single_table_length; i++){
 25 | 		bucket[i].bit_array = new char[bytes_per_bucket];
 26 | 		memset(bucket[i].bit_array, 0, bytes_per_bucket);
 27 | 	}
 28 | 
 29 | }
 30 | 
 31 | CuckooFilter::~CuckooFilter(){
 32 | 	delete[] bucket;
 33 | 	delete next;
 34 | 	delete front;
 35 | }
 36 | 
 37 | 
 38 | 
 39 | int CuckooFilter::insertItem(const char* item, Victim &victim){
 40 | 	size_t index,alt_index;
 41 | 	uint32_t fingerprint;
 42 | 	generateIF(item, index, fingerprint, fingerprint_size, single_table_length);
 43 | 
 44 | 	//edit 3-17
 45 | 	for(size_t count = 0; count<MaxNumKicks; count++){
 46 | 		bool kickout = (count != 0);
 47 | 		if(insertImpl(index, fingerprint, kickout, victim)){
 48 | 			return true;
 49 | 		}
 50 | 
 51 | 		if (kickout){
 52 | 			index = victim.index;
 53 | 			fingerprint = victim.fingerprint;
 54 | 			generateA(index, fingerprint, alt_index, single_table_length);
 55 | 			index = alt_index;
 56 | 		}else{
 57 | 			generateA(index, fingerprint, alt_index, single_table_length);
 58 | 			index = alt_index;
 59 | 		}
 60 | 	}
 61 | 
 62 | 	return false;
 63 | }
 64 | 
 65 | bool CuckooFilter::insertItem(size_t index, uint32_t fingerprint, bool kickout, Victim &victim){
 66 | 	size_t alt_index;
 67 | 
 68 | 	for(size_t count = 0; count<MaxNumKicks; count++){
 69 | 		bool kickout = (count != 0);
 70 | 		if(insertImpl(index, fingerprint, kickout, victim)){
 71 | 			return true;
 72 | 		}
 73 | 
 74 | 		if (kickout){
 75 | 			index = victim.index;
 76 | 			fingerprint = victim.fingerprint;
 77 | 			generateA(index, fingerprint, alt_index, single_table_length);
 78 | 			index = alt_index;
 79 | 		}else{
 80 | 			generateA(index, fingerprint, alt_index, single_table_length);
 81 | 			index = alt_index;
 82 | 		}
 83 | 	}
 84 | 	return false;
 85 | }
 86 | 
 87 | bool CuckooFilter::queryItem(const char* item){
 88 | 	size_t index, alt_index;
 89 | 	uint32_t fingerprint;
 90 | 	generateIF(item, index, fingerprint, fingerprint_size, single_table_length);
 91 | 
 92 | 	if(queryImpl(index, fingerprint)){
 93 | 		return true;
 94 | 	}
 95 | 	generateA(index, fingerprint, alt_index, single_table_length);
 96 | 	if(queryImpl(alt_index, fingerprint)){
 97 | 		return true;
 98 | 	}
 99 | 	return false;
100 | }
101 | 
102 | bool CuckooFilter::deleteItem(const char* item){
103 | 	size_t index, alt_index;
104 | 	uint32_t fingerprint;
105 | 	generateIF(item, index, fingerprint, fingerprint_size, single_table_length);
106 | 
107 | 	if(deleteImpl(index, fingerprint)){
108 | 		return true;
109 | 	}
110 | 	generateA(index, fingerprint, alt_index, single_table_length);
111 | 	if(deleteImpl(alt_index, fingerprint)){
112 | 		return true;
113 | 	}
114 | 
115 | 	return false;
116 | }
117 | 
118 | 
119 | 
120 | bool CuckooFilter::insertImpl(const size_t index, const uint32_t fingerprint, const bool kickout, Victim &victim){
121 | 	for(size_t pos = 0; pos<4; pos++){
122 | 		if(read(index,pos) == 0){
123 | 			write(index,pos,fingerprint);
124 | 			counter++;
125 | 			if(this->counter == capacity){
126 | 				this->is_full = true;
127 | 			}
128 | 
129 | 			if(this->counter > 0 ){
130 | 				this->is_empty = false;
131 | 			}
132 | 			return true;
133 | 		}
134 | 	}
135 | 	if(kickout){
136 | 		int j = rand()%4;
137 | 		victim.index = index;
138 | 		victim.fingerprint = read(index,j);
139 | 		write(index,j, fingerprint);
140 | 	}
141 | 	return false;
142 | }
143 | 
144 | bool CuckooFilter::queryImpl(const size_t index, const uint32_t fingerprint){
145 | 	if(fingerprint_size == 4){
146 | 		const char* p = bucket[index].bit_array;
147 | 		uint64_t bits = *(uint64_t*)p;
148 | 		return hasvalue4(bits, fingerprint);
149 | 	}else if(fingerprint_size == 8){
150 | 		const char* p = bucket[index].bit_array;
151 | 		uint64_t bits = *(uint64_t*)p;
152 | 		return hasvalue8(bits, fingerprint);
153 | 	}else if(fingerprint_size == 12){
154 | 		const char* p = bucket[index].bit_array;
155 | 		uint64_t bits = *(uint64_t*)p;
156 | 		return hasvalue12(bits, fingerprint);
157 | 	}else if(fingerprint_size == 16){
158 | 		const char* p = bucket[index].bit_array;
159 | 		uint64_t bits = *(uint64_t*)p;
160 | 		return hasvalue16(bits, fingerprint);
161 | 	}else{
162 | 		return false;
163 | 	}
164 | }
165 | 
166 | bool CuckooFilter::deleteImpl(const size_t index, const uint32_t fingerprint){
167 | 	for(size_t pos = 0; pos<4; pos++){
168 | 		if(read(index, pos) == fingerprint){
169 | 			write(index, pos, 0);
170 | 			counter--;
171 | 			if(counter < this->capacity){
172 | 				this->is_full = false;
173 | 			}
174 | 			if(counter == 0){
175 | 				this->is_empty = true;
176 | 			}
177 | 			return true;
178 | 		}
179 | 	}
180 | 	return false;
181 | }
182 | 
183 | 
184 | 
185 | void CuckooFilter::generateIF(const char* item, size_t &index, uint32_t &fingerprint, int fingerprint_size, int single_table_length){
186 | 	std::string  value = HashFunc::sha1(item);
187 | 	uint64_t hv = *((uint64_t*) value.c_str());
188 | 
189 | 	index = ((uint32_t) (hv >> 32)) % single_table_length;
190 | 	fingerprint = (uint32_t) (hv & 0xFFFFFFFF);
191 | 	fingerprint &= ((0x1ULL<<fingerprint_size)-1);
192 | 	fingerprint += (fingerprint == 0);
193 | 
194 | }
195 | 
196 | void CuckooFilter::generateA(size_t index, uint32_t fingerprint, size_t &alt_index, int single_table_length){
197 | 	alt_index = (index ^ (fingerprint * 0x5bd1e995)) % single_table_length;
198 | }
199 | 
200 | 
201 | 
202 | uint32_t CuckooFilter::read(size_t index, size_t pos){
203 | 	const char* p = bucket[index].bit_array;
204 | 	uint32_t fingerprint;
205 | 
206 | 	if(fingerprint_size == 4){
207 | 		p += (pos >> 1);
208 | 		uint8_t bits_8 = *(uint8_t*)p;
209 | 		if((pos & 1) == 0){
210 | 			fingerprint = (bits_8>>4) & 0xf;
211 | 		}else{
212 | 			fingerprint = bits_8 & 0xf;
213 | 		}
214 | 	}else if(fingerprint_size == 8){
215 | 		p += pos;
216 | 		uint8_t bits_8 = *(uint8_t*)p;
217 | 		fingerprint = bits_8 & 0xff;
218 | 	}else if(fingerprint_size == 12){
219 | 		p += pos+(pos>>1);
220 | 		uint16_t bits_16 = *(uint16_t*)p;
221 | 		if((pos & 1) == 0){
222 | 			fingerprint = bits_16 & 0xfff;
223 | 		}else{
224 | 			fingerprint = (bits_16 >> 4) & 0xfff;
225 | 		}
226 | 	}else if(fingerprint_size == 16){
227 | 		p += (pos<<1);
228 | 		uint16_t bits_16 = *(uint16_t*)p;
229 | 		fingerprint = bits_16 & 0xffff;
230 | 	}else if(fingerprint_size == 24){
231 | 		p += pos+(pos<<1);
232 | 		uint32_t bits_32 = *(uint32_t*)p;
233 | 		fingerprint = (bits_32 >> 4);
234 | 	}else if(fingerprint_size == 32){
235 | 		p += (pos<<2);
236 | 		uint32_t bits_32 = *(uint32_t*)p;
237 | 		fingerprint = bits_32 & 0xffffffff;
238 | 	}else{
239 | 		fingerprint =0;
240 | 	}
241 | 	return fingerprint & mask;
242 | }
243 | 
244 | void CuckooFilter::write(size_t index, size_t pos, uint32_t fingerprint){
245 | 	char* p = bucket[index].bit_array;
246 | 
247 | 	if(fingerprint_size == 4){
248 | 		p += (pos>>1);
249 | 		if((pos & 1) == 0){
250 | 			*((uint8_t*)p) &= 0x0f;
251 | 			*((uint8_t*)p) |= (fingerprint<<4);
252 | 		}else{
253 | 			*((uint8_t*)p) &= 0xf0;
254 | 			*((uint8_t*)p) |= fingerprint;
255 | 		}
256 | 	}else if(fingerprint_size == 8){
257 | 		((uint8_t*)p)[pos] = fingerprint;
258 | 	}else if(fingerprint_size == 12){
259 | 		p += (pos + (pos>>1));
260 | 		if((pos & 1) == 0){
261 | 			*((uint16_t*)p) &= 0xf000; //Little-Endian
262 | 			*((uint16_t*)p) |= fingerprint;
263 | 		}else{
264 | 			*((uint16_t*)p) &= 0x000f;
265 | 			*((uint16_t*)p) |= fingerprint<<4;
266 | 		}
267 | 	}else if(fingerprint_size == 16){
268 | 		((uint16_t*) p)[pos] = fingerprint;
269 | 	}else if(fingerprint_size == 24){
270 | 		p += (pos+ (pos<<1));
271 | 		*((uint32_t*)p) &= 0xff000000; //Little-Endian
272 | 		*((uint32_t*)p) |= fingerprint;
273 | 	}else if(fingerprint_size == 32){
274 | 		((uint32_t*) p)[pos] = fingerprint;
275 | 	}
276 | }
277 | 
278 | 
279 | 
280 | bool CuckooFilter::transfer(CuckooFilter* tarCF){
281 | 	uint32_t fingerprint = 0;
282 | 
283 | 	for(size_t i = 0; i<single_table_length; i++){
284 | 		for(int j = 0; j<4; j++){
285 | 			fingerprint = read(i, j);
286 | 			if(fingerprint != 0){
287 | 				if(tarCF->is_full == true){
288 | 					return false;
289 | 				}
290 | 				if(this->is_empty == true){
291 | 					return false;
292 | 				}
293 | 				Victim victim = {0, 0};
294 | 				if(tarCF->insertImpl(i, fingerprint, false, victim)){
295 | 					this->write(i, j, 0);
296 | 					this->counter--;
297 | 
298 | 					if(this->counter < capacity){
299 | 						this->is_full = false;
300 | 					}
301 | 					if(this->counter == 0){
302 | 						this->is_empty = true;
303 | 					}
304 | 
305 | 					if(tarCF->counter == capacity){
306 | 						tarCF->is_full = true;
307 | 					}
308 | 
309 | 					if(tarCF->counter > 0 ){
310 | 						tarCF->is_empty = false;
311 | 					}
312 | 				}
313 | 			}
314 | 		}
315 | 	}
316 | 	return true;
317 | }
318 | 
319 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------