├── .gitignore ├── CPU ├── CM │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── Coco │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── Count │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── Elastic │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── HLL │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── LL │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── Locher │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp ├── README.md ├── UnivMon │ ├── CMakeLists.txt │ ├── Ideal.h │ ├── Merge.h │ ├── Ours.h │ ├── benchmark.h │ ├── config.h │ └── main.cpp └── template │ ├── Abstract.h │ ├── Ideal.h │ ├── Merge.h │ └── Ours.h ├── DPDK ├── CM │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c ├── Coco │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c ├── Count │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c ├── Elastic │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c ├── Locher │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c ├── README.md ├── UnivMon │ ├── Makefile │ ├── Merge.h │ ├── Ours.h │ ├── config.h │ └── main.c └── template │ ├── DPDK.h │ ├── Merge.h │ ├── Ours.h │ └── common.h ├── LICENSE ├── README.md ├── XDP ├── CM │ ├── XDP_CM.h │ ├── compile.sh │ ├── main.c │ └── xdp.c ├── Coco │ ├── XDP_Coco.h │ ├── compile.sh │ ├── main.c │ └── xdp.c ├── Count │ ├── XDP_Count.h │ ├── compile.sh │ ├── main.c │ └── xdp.c ├── README.md └── xdp_common │ ├── Abstract.h │ ├── hash.h │ ├── parse.h │ └── shll.h ├── common ├── bitmap.h ├── cuckooMap.h ├── hash.h ├── heap.h ├── loader.h ├── shll.h └── util.h ├── queue ├── atomicops.h └── readerwriterqueue.h └── sketch ├── CM.h ├── Coco.h ├── Count.h ├── DD.h ├── Elastic.h ├── HLL.h ├── LL.h ├── Locher.h ├── UnivMon.h └── sketch.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /CPU/CM/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(CM) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx2") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(CM main.cpp) 12 | target_link_libraries(CM) -------------------------------------------------------------------------------- /CPU/CM/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_IDEAL_H 2 | #define CM_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class CM_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyCM(); 12 | } 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /CPU/CM/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_MERGE_H 2 | #define CM_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | /** 8 | * Sketch-merge for the Count-Min sketch 9 | */ 10 | 11 | template 12 | class CM_Merge : public Merge*, thread_num>{ 13 | public: 14 | 15 | typedef ReaderWriterQueue*> myQueue; 16 | 17 | Sketch* initialize_parent(){ 18 | return new MyCM(); 19 | } 20 | 21 | Sketch* initialize_child(){ 22 | return initialize_parent(); 23 | } 24 | 25 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 26 | p->insert_one(packet); 27 | 28 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 29 | q.enqueue((MyCM*)p); 30 | return initialize_child(); 31 | } 32 | return p; 33 | } 34 | 35 | void merge(Sketch* p, MyCM* temp){ 36 | ((MyCM*)p)->Merge(temp); 37 | } 38 | 39 | }; 40 | 41 | #endif -------------------------------------------------------------------------------- /CPU/CM/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_OURS_H 2 | #define CM_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | /** 8 | * OctoSketch for the Count-Min sketch 9 | */ 10 | 11 | template 12 | class CM_Ours : public Ours, thread_num>{ 13 | public: 14 | 15 | typedef ReaderWriterQueue> myQueue; 16 | 17 | Sketch* initialize_parent(){ 18 | return new MyCM(); 19 | } 20 | 21 | Sketch* initialize_child(){ 22 | return new MyChild_CM(); 23 | } 24 | 25 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 26 | auto sketch = ((MyChild_CM*)p)->sketch; 27 | uint32_t pos[HASH_NUM]; 28 | 29 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 30 | pos[hashPos] = hash(packet, hashPos) % LENGTH; 31 | } 32 | 33 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 34 | sketch[hashPos][pos[hashPos]] += 1; 35 | if(sketch[hashPos][pos[hashPos]] >= PROMASK){ 36 | q.enqueue(CM_Entry(packet, hashPos, pos[hashPos], sketch[hashPos][pos[hashPos]])); 37 | sketch[hashPos][pos[hashPos]] = 0; 38 | } 39 | } 40 | } 41 | 42 | void merge(Sketch* p, CM_Entry temp){ 43 | ((MyCM*)p)->Merge(temp); 44 | } 45 | 46 | }; 47 | 48 | #endif -------------------------------------------------------------------------------- /CPU/CM/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | #include 5 | 6 | #include "loader.h" 7 | 8 | #include "Ideal.h" 9 | #include "Merge.h" 10 | #include "Ours.h" 11 | 12 | #define THREAD_NUM 16 // The number of workers 13 | 14 | template 15 | class Benchmark{ 16 | public: 17 | typedef std::unordered_map HashMap; 18 | 19 | /** 20 | * Load a binary file (dataset) and get the statistics 21 | */ 22 | Benchmark(const char* PATH){ 23 | result = Load(PATH); 24 | 25 | uint32_t length = result.length / sizeof(Key); 26 | Key* dataset = (Key*)result.start; 27 | 28 | for(uint32_t i = 0;i < length;++i){ 29 | mp[dataset[i]] += 1; 30 | } 31 | } 32 | 33 | ~Benchmark(){ 34 | UnLoad(result); 35 | } 36 | 37 | void Bench(){ 38 | std::vector absVec = { 39 | new CM_Ideal(), 40 | new CM_Merge(), 41 | new CM_Ours(), 42 | }; 43 | 44 | for(auto alg : absVec){ 45 | alg->update(result.start, result.length, mp); 46 | delete alg; 47 | } 48 | } 49 | 50 | private: 51 | LoadResult result; 52 | HashMap mp; 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /CPU/CM/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "CM.h" 5 | 6 | // Configuration for the sketch 7 | #define HASH_NUM 3 8 | #define LENGTH (1 << 16) 9 | #define HEAP_SIZE 0x3ff 10 | 11 | #define INTERVAL 20000 12 | 13 | template 14 | using MyCM = CM; 15 | 16 | template 17 | using MyChild_CM = Child_CM; 18 | 19 | #define THP_TIME 20 20 | 21 | #define ALPHA 0.0002 // Threshold for finding heavy hitters 22 | 23 | #define PROMASK 0x7f 24 | 25 | //#define QUEUELENGTH 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /CPU/CM/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/Coco/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Coco) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx2") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(Coco main.cpp) 12 | target_link_libraries(Coco) -------------------------------------------------------------------------------- /CPU/Coco/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_IDEAL_H 2 | #define COCO_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class Coco_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyCoco(); 12 | } 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /CPU/Coco/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_MERGE_H 2 | #define COCO_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class Coco_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyCoco(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyCoco*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyCoco* temp){ 32 | ((MyCoco*)p)->Merge(temp); 33 | } 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /CPU/Coco/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_OURS_H 2 | #define COCO_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class Coco_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyCoco(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_Coco(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto keys = ((MyChild_Coco*)p)->keys; 23 | auto counters = ((MyChild_Coco*)p)->counters; 24 | 25 | uint32_t choice; 26 | uint16_t pos[2]; 27 | 28 | *((uint32_t*)pos) = hash(packet, 0); 29 | 30 | if(keys[0][pos[0]] == packet){ 31 | counters[0][pos[0]] += 1; 32 | 33 | if(counters[0][pos[0]] >= PROMASK){ 34 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[0][pos[0]])); 35 | counters[0][pos[0]] = 0; 36 | } 37 | 38 | return; 39 | } 40 | 41 | if(keys[1][pos[1]] == packet){ 42 | counters[1][pos[1]] += 1; 43 | 44 | if(counters[1][pos[1]] >= PROMASK){ 45 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[1][pos[1]])); 46 | counters[1][pos[1]] = 0; 47 | } 48 | 49 | return; 50 | } 51 | 52 | choice = (counters[0][pos[0]] > counters[1][pos[1]]); 53 | counters[choice][pos[choice]] += 1; 54 | if(((MyChild_Coco*)p)->rng() % counters[choice][pos[choice]] == 0){ 55 | keys[choice][pos[choice]] = packet; 56 | 57 | if(counters[choice][pos[choice]] >= PROMASK){ 58 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[choice][pos[choice]])); 59 | counters[choice][pos[choice]] = 0; 60 | } 61 | } 62 | else{ 63 | if(counters[choice][pos[choice]] >= PROMASK){ 64 | q.enqueue(Coco_Entry(keys[choice][pos[choice]], pos[0], pos[1], counters[choice][pos[choice]], false)); 65 | counters[choice][pos[choice]] = 0; 66 | } 67 | } 68 | } 69 | 70 | void merge(Sketch* p, Coco_Entry temp){ 71 | ((MyCoco*)p)->Merge(temp); 72 | } 73 | 74 | }; 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /CPU/Coco/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | #include 5 | 6 | #include "loader.h" 7 | 8 | #include "Ideal.h" 9 | #include "Merge.h" 10 | #include "Ours.h" 11 | 12 | #define THREAD_NUM 16 13 | 14 | template 15 | class Benchmark{ 16 | public: 17 | typedef std::unordered_map HashMap; 18 | 19 | Benchmark(std::string PATH){ 20 | result = Load(PATH.c_str()); 21 | 22 | uint32_t length = result.length / sizeof(Key); 23 | Key* dataset = (Key*)result.start; 24 | 25 | for(uint32_t i = 0;i < length;++i){ 26 | mp[dataset[i]] += 1; 27 | } 28 | } 29 | 30 | ~Benchmark(){ 31 | UnLoad(result); 32 | } 33 | 34 | void Bench(){ 35 | std::vector absVec = { 36 | new Coco_Ideal(), 37 | new Coco_Merge(), 38 | new Coco_Ours(), 39 | }; 40 | 41 | for(auto alg : absVec){ 42 | alg->update(result.start, result.length, mp); 43 | delete alg; 44 | } 45 | } 46 | 47 | private: 48 | LoadResult result; 49 | HashMap mp; 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /CPU/Coco/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "Coco.h" 5 | 6 | #define HASH_NUM 2 7 | #define LENGTH (1 << 16) 8 | 9 | #define INTERVAL 100000 10 | 11 | template 12 | using MyCoco = Coco; 13 | 14 | template 15 | using MyChild_Coco = Child_Coco; 16 | 17 | #define THP_TIME 200 18 | 19 | #define ALPHA 0.0002 20 | 21 | #define PROMASK 0x1f 22 | 23 | //#define QUEUELENGTH 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /CPU/Coco/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/Count/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Count) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx2") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(Count main.cpp) 12 | target_link_libraries(Count) -------------------------------------------------------------------------------- /CPU/Count/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_IDEAL_H 2 | #define COUNT_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class Count_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyCount(); 12 | } 13 | }; 14 | 15 | 16 | #endif -------------------------------------------------------------------------------- /CPU/Count/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_MERGE_H 2 | #define COUNT_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class Count_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyCount(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyCount*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyCount* temp){ 32 | ((MyCount*)p)->Merge(temp); 33 | } 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /CPU/Count/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_OURS_H 2 | #define COUNT_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class Count_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyCount(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_Count(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto sketch = ((MyChild_Count*)p)->sketch; 23 | uint32_t pos[HASH_NUM]; 24 | int32_t incre[HASH_NUM]; 25 | 26 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 27 | uint32_t hashNum = hash(packet, hashPos); 28 | pos[hashPos] = (hashNum >> 1) % LENGTH; 29 | incre[hashPos] = increment[hashNum & 1]; 30 | } 31 | 32 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 33 | sketch[hashPos][pos[hashPos]] += incre[hashPos]; 34 | if(sketch[hashPos][pos[hashPos]] * incre[hashPos] >= PROMASK){ 35 | q.enqueue(Count_Entry(packet, hashPos, pos[hashPos], sketch[hashPos][pos[hashPos]])); 36 | sketch[hashPos][pos[hashPos]] = 0; 37 | } 38 | } 39 | } 40 | 41 | void merge(Sketch* p, Count_Entry temp){ 42 | ((MyCount*)p)->Merge(temp); 43 | } 44 | 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /CPU/Count/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | 5 | #include 6 | 7 | #include "loader.h" 8 | 9 | #include "Ideal.h" 10 | #include "Merge.h" 11 | #include "Ours.h" 12 | 13 | 14 | #define THREAD_NUM 16 15 | 16 | template 17 | class Benchmark{ 18 | public: 19 | typedef std::unordered_map HashMap; 20 | 21 | Benchmark(std::string PATH){ 22 | result = Load(PATH.c_str()); 23 | 24 | uint32_t length = result.length / sizeof(Key); 25 | Key* dataset = (Key*)result.start; 26 | 27 | for(uint32_t i = 0;i < length;++i){ 28 | mp[dataset[i]] += 1; 29 | } 30 | } 31 | 32 | ~Benchmark(){ 33 | UnLoad(result); 34 | } 35 | 36 | void Bench(){ 37 | std::vector absVec = { 38 | new Count_Ideal(), 39 | new Count_Merge(), 40 | new Count_Ours(), 41 | }; 42 | 43 | for(auto alg : absVec){ 44 | alg->update(result.start, result.length, mp); 45 | delete alg; 46 | } 47 | } 48 | 49 | private: 50 | LoadResult result; 51 | HashMap mp; 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /CPU/Count/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "Count.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0x3ff 9 | 10 | #define INTERVAL 20000 11 | 12 | template 13 | using MyCount = Count; 14 | 15 | template 16 | using MyChild_Count = Child_Count; 17 | 18 | #define THP_TIME 200 19 | 20 | #define ALPHA 0.0002 21 | 22 | #define PROMASK 0x7f 23 | 24 | //#define QUEUELENGTH 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /CPU/Count/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/Elastic/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Elastic) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx2") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(Elastic main.cpp) 12 | target_link_libraries(Elastic) -------------------------------------------------------------------------------- /CPU/Elastic/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_IDEAL_H 2 | #define ELASTIC_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class Elastic_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyElastic(); 12 | } 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /CPU/Elastic/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_MERGE_H 2 | #define ELASTIC_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class Elastic_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyElastic(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyElastic*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyElastic* temp){ 32 | ((MyElastic*)p)->Merge(temp); 33 | } 34 | }; 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /CPU/Elastic/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_OURS_H 2 | #define ELASTIC_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class Elastic_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyElastic(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_Elastic(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto sketch = ((MyChild_Elastic*)p)->sketch; 23 | auto buckets = ((MyChild_Elastic*)p)->buckets; 24 | 25 | uint32_t pos = hash(packet) % BUCKET_LENGTH, minPos = 0; 26 | int32_t minVal = 0x7fffffff; 27 | 28 | for (uint32_t j = 0; j < COUNTER_PER_BUCKET; j++){ 29 | if(buckets[pos].ID[j] == packet) { 30 | buckets[pos].count[j] += 1; 31 | if(buckets[pos].count[j] >= PROMASK){ 32 | q.enqueue(Elastic_Entry(buckets[pos].ID[j], pos, buckets[pos].count[j], true)); 33 | buckets[pos].count[j] = 0; 34 | } 35 | return; 36 | } 37 | 38 | if(buckets[pos].count[j] < minVal){ 39 | minPos = j; 40 | minVal = buckets[pos].count[j]; 41 | } 42 | } 43 | 44 | if((buckets[pos].vote + 1) >= minVal * 8){ 45 | buckets[pos].vote = 0; 46 | 47 | if(minVal != 0){ 48 | uint32_t position = hash(buckets[pos].ID[minPos], 101) % SKETCH_LENGTH; 49 | sketch[position] = sketch[position] + buckets[pos].count[minPos]; 50 | if(sketch[position] >= PROMASK){ 51 | q.enqueue(Elastic_Entry(buckets[pos].ID[minPos], position, sketch[position], false)); 52 | sketch[position] = 0; 53 | } 54 | } 55 | 56 | buckets[pos].ID[minPos] = packet; 57 | buckets[pos].count[minPos] = 1; 58 | } 59 | else { 60 | buckets[pos].vote += 1; 61 | uint32_t position = hash(packet, 101) % SKETCH_LENGTH; 62 | sketch[position] = sketch[position] + 1; 63 | if(sketch[position] >= PROMASK){ 64 | q.enqueue(Elastic_Entry(packet, position, sketch[position], false)); 65 | sketch[position] = 0; 66 | } 67 | } 68 | } 69 | 70 | void merge(Sketch* p, Elastic_Entry temp){ 71 | ((MyElastic*)p)->Merge(temp); 72 | } 73 | }; 74 | 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /CPU/Elastic/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | 5 | #include 6 | 7 | #include "loader.h" 8 | 9 | #include "Ideal.h" 10 | #include "Merge.h" 11 | #include "Ours.h" 12 | 13 | 14 | #define THREAD_NUM 16 15 | 16 | template 17 | class Benchmark{ 18 | public: 19 | typedef std::unordered_map HashMap; 20 | 21 | Benchmark(std::string PATH){ 22 | result = Load(PATH.c_str()); 23 | 24 | uint32_t length = result.length / sizeof(Key); 25 | Key* dataset = (Key*)result.start; 26 | 27 | for(uint32_t i = 0;i < length;++i){ 28 | mp[dataset[i]] += 1; 29 | } 30 | } 31 | 32 | ~Benchmark(){ 33 | UnLoad(result); 34 | } 35 | 36 | void Bench(){ 37 | std::vector absVec = { 38 | new Elastic_Ideal(), 39 | new Elastic_Merge(), 40 | new Elastic_Ours(), 41 | }; 42 | 43 | for(auto alg : absVec){ 44 | alg->update(result.start, result.length, mp); 45 | delete alg; 46 | } 47 | } 48 | 49 | private: 50 | LoadResult result; 51 | HashMap mp; 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /CPU/Elastic/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "Elastic.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0x3ff 9 | 10 | #define COUNTER_PER_BUCKET 4 11 | 12 | #define SKETCH_LENGTH (HASH_NUM * LENGTH) 13 | #define BUCKET_LENGTH (HEAP_SIZE * 3 / COUNTER_PER_BUCKET) 14 | 15 | #define INTERVAL 20000 16 | 17 | template 18 | using MyElastic = Elastic; 19 | 20 | template 21 | using MyChild_Elastic = Child_Elastic; 22 | 23 | #define THP_TIME 200 24 | 25 | #define ALPHA 0.0002 26 | 27 | #define PROMASK 0x1f 28 | 29 | //#define QUEUELENGTH 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /CPU/Elastic/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } -------------------------------------------------------------------------------- /CPU/HLL/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(HLL) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(HLL main.cpp) 12 | target_link_libraries(HLL) -------------------------------------------------------------------------------- /CPU/HLL/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef HLL_IDEAL_H 2 | #define HLL_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class HLL_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyHLL(); 12 | } 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /CPU/HLL/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef HLL_MERGE_H 2 | #define HLL_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class HLL_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyHLL(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyHLL*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyHLL* temp){ 32 | ((MyHLL*)p)->Merge(temp); 33 | } 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /CPU/HLL/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef HLL_OURS_H 2 | #define HLL_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class HLL_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyHLL(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_HLL(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto sketch = ((MyChild_HLL*)p)->sketch; 23 | 24 | uint32_t pos = hash(packet) % LENGTH; 25 | uint32_t temp = hash(packet, 101); 26 | uint8_t rank = MIN(31, __builtin_clz(temp) + 1); 27 | 28 | if(sketch[pos] < rank){ 29 | sketch[pos] = rank; 30 | if(rank > PROMASK){ 31 | q.enqueue(HLL_Entry(pos, sketch[pos])); 32 | } 33 | } 34 | } 35 | 36 | void merge(Sketch* p, HLL_Entry temp){ 37 | ((MyHLL*)p)->Merge(temp); 38 | } 39 | 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /CPU/HLL/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | #include 5 | 6 | #include "loader.h" 7 | 8 | #include "Ideal.h" 9 | #include "Merge.h" 10 | #include "Ours.h" 11 | 12 | #define THREAD_NUM 16 13 | 14 | template 15 | class Benchmark{ 16 | public: 17 | 18 | typedef std::unordered_map HashMap; 19 | 20 | Benchmark(std::string PATH){ 21 | result = Load(PATH.c_str()); 22 | 23 | uint32_t length = result.length / sizeof(Key); 24 | Key* dataset = (Key*)result.start; 25 | 26 | HashMap tmp; 27 | for(uint32_t i = 0;i < length;++i){ 28 | tmp[dataset[i]] += 1; 29 | } 30 | mp[0] = tmp.size(); 31 | } 32 | 33 | ~Benchmark(){ 34 | UnLoad(result); 35 | } 36 | 37 | void Bench(){ 38 | std::vector absVec = { 39 | new HLL_Ideal(), 40 | new HLL_Merge(), 41 | new HLL_Ours(), 42 | }; 43 | 44 | for(auto alg : absVec){ 45 | alg->update(result.start, result.length, mp); 46 | delete alg; 47 | } 48 | } 49 | 50 | private: 51 | LoadResult result; 52 | HashMap mp; 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /CPU/HLL/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "HLL.h" 5 | 6 | #define LENGTH (1 << 16) 7 | 8 | #define INTERVAL 10000 9 | 10 | template 11 | using MyHLL = HLL; 12 | 13 | template 14 | using MyChild_HLL = Child_HLL; 15 | 16 | #define THP_TIME 10 17 | 18 | #define ALPHA 0 19 | 20 | #define PROMASK 0x0 21 | 22 | //#define QUEUELENGTH 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /CPU/HLL/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/LL/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(LL) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(LL main.cpp) 12 | target_link_libraries(LL) -------------------------------------------------------------------------------- /CPU/LL/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef LL_IDEAL_H 2 | #define LL_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class LL_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyLL(); 12 | } 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /CPU/LL/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef LL_MERGE_H 2 | #define LL_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class LL_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyLL(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyLL*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyLL* temp){ 32 | ((MyLL*)p)->Merge(temp); 33 | } 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /CPU/LL/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef LL_OURS_H 2 | #define LL_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class LL_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyLL(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_LL(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto sketch = ((MyChild_LL*)p)->sketch; 23 | 24 | uint32_t pos = hash(packet) % LENGTH; 25 | uint32_t temp = hash(packet, 101); 26 | uint8_t rank = MIN(31, __builtin_clz(temp) + 1); 27 | 28 | if(sketch[pos] < rank){ 29 | sketch[pos] = rank; 30 | if(rank > PROMASK){ 31 | q.enqueue(LL_Entry(pos, sketch[pos])); 32 | } 33 | } 34 | } 35 | 36 | void merge(Sketch* p, LL_Entry temp){ 37 | ((MyLL*)p)->Merge(temp); 38 | } 39 | 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /CPU/LL/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | #include 5 | 6 | #include "loader.h" 7 | 8 | #include "Ideal.h" 9 | #include "Merge.h" 10 | #include "Ours.h" 11 | 12 | #define THREAD_NUM 16 13 | 14 | template 15 | class Benchmark{ 16 | public: 17 | 18 | typedef std::unordered_map HashMap; 19 | 20 | Benchmark(std::string PATH){ 21 | result = Load(PATH.c_str()); 22 | 23 | uint32_t length = result.length / sizeof(Key); 24 | Key* dataset = (Key*)result.start; 25 | 26 | HashMap tmp; 27 | for(uint32_t i = 0;i < length;++i){ 28 | tmp[dataset[i]] += 1; 29 | } 30 | mp[0] = tmp.size(); 31 | } 32 | 33 | ~Benchmark(){ 34 | UnLoad(result); 35 | } 36 | 37 | void Bench(){ 38 | std::vector absVec = { 39 | new LL_Ideal(), 40 | new LL_Merge(), 41 | new LL_Ours(), 42 | }; 43 | 44 | for(auto alg : absVec){ 45 | alg->update(result.start, result.length, mp); 46 | delete alg; 47 | } 48 | } 49 | 50 | private: 51 | LoadResult result; 52 | HashMap mp; 53 | }; 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /CPU/LL/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "LL.h" 5 | 6 | #define LENGTH (1 << 16) 7 | 8 | #define INTERVAL 10000 9 | 10 | template 11 | using MyLL = LL; 12 | 13 | template 14 | using MyChild_LL = Child_LL; 15 | 16 | #define THP_TIME 10 17 | 18 | #define ALPHA 0 19 | 20 | #define PROMASK 0x0 21 | 22 | //#define QUEUELENGTH 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /CPU/LL/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } -------------------------------------------------------------------------------- /CPU/Locher/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(Locher) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(Locher main.cpp) 12 | target_link_libraries(Locher) -------------------------------------------------------------------------------- /CPU/Locher/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_IDEAL_H 2 | #define LOCHER_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | class Locher_Ideal : public Ideal{ 8 | public: 9 | Sketch* initialize_sketch(){ 10 | return new MyLocher(); 11 | } 12 | }; 13 | 14 | #endif -------------------------------------------------------------------------------- /CPU/Locher/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_MERGE_H 2 | #define LOCHER_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class Locher_Merge : public Merge{ 9 | public: 10 | 11 | typedef ReaderWriterQueue myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyLocher(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Packet& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyLocher*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyLocher* temp){ 32 | ((MyLocher*)p)->Merge(temp); 33 | } 34 | }; 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /CPU/Locher/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_OURS_H 2 | #define LOCHER_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class Locher_Ours : public Ours{ 9 | public: 10 | 11 | typedef ReaderWriterQueue myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | MyLocher* ret = new MyLocher(); 15 | memset(ret->sketch, HARDTHRES, HASH_NUM * LENGTH * sizeof(SHLL)); 16 | return ret; 17 | } 18 | 19 | Sketch* initialize_child(){ 20 | MyChild_Locher* ret = new MyChild_Locher(); 21 | memset(ret->sketch, HARDTHRES, HASH_NUM * LENGTH * sizeof(SHLL)); 22 | return ret; 23 | } 24 | 25 | void insert_child(Sketch* p, myQueue& q, const Packet& packet){ 26 | auto sketch = ((MyChild_Locher*)p)->sketch; 27 | 28 | uint32_t pos[HASH_NUM]; 29 | uint32_t temp[HASH_NUM]; 30 | 31 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 32 | pos[hashPos] = hash(packet.src, hashPos) % LENGTH; 33 | temp[hashPos] = hash(packet.dst, hashPos); 34 | } 35 | 36 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 37 | uint32_t bucket_index = ((temp[hashPos] >> 1) & 0x7); 38 | Buckets& tempBucket = sketch[hashPos][pos[hashPos]].buckets[bucket_index]; 39 | 40 | uint32_t inbucket_index = (temp[hashPos] & 0x1); 41 | uint8_t rank = MIN(15, __builtin_clz(temp[hashPos]) + 1); 42 | 43 | switch(inbucket_index){ 44 | case 0: 45 | if(tempBucket.counter0 < rank){ 46 | tempBucket.counter0 = rank; 47 | q.enqueue(Locher_Entry(packet.src, hashPos, pos[hashPos], temp[hashPos])); 48 | } 49 | break; 50 | case 1: 51 | if(tempBucket.counter1 < rank){ 52 | tempBucket.counter1 = rank; 53 | q.enqueue(Locher_Entry(packet.src, hashPos, pos[hashPos], temp[hashPos])); 54 | } 55 | break; 56 | } 57 | } 58 | } 59 | 60 | void merge(Sketch* p, Locher_Entry temp){ 61 | ((MyLocher*)p)->Merge(temp); 62 | } 63 | 64 | }; 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /CPU/Locher/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | #include 5 | 6 | #include "loader.h" 7 | 8 | #include "Ideal.h" 9 | #include "Merge.h" 10 | #include "Ours.h" 11 | 12 | #include 13 | 14 | #define THREAD_NUM 16 15 | 16 | class Benchmark{ 17 | public: 18 | 19 | typedef std::unordered_map HashMap; 20 | 21 | Benchmark(std::string PATH){ 22 | result = Load(PATH.c_str()); 23 | 24 | uint32_t length = result.length / sizeof(Packet); 25 | Packet* dataset = (Packet*)result.start; 26 | 27 | std::unordered_set st; 28 | for(uint32_t i = 0;i < length;++i){ 29 | if(st.find(dataset[i]) == st.end()){ 30 | st.insert(dataset[i]); 31 | mp[dataset[i].src] += 1; 32 | } 33 | } 34 | //std::cout << st.size() << std::endl; 35 | } 36 | 37 | ~Benchmark(){ 38 | UnLoad(result); 39 | } 40 | 41 | void Bench(){ 42 | std::vector absVec = { 43 | new Locher_Ideal(), 44 | new Locher_Merge(), 45 | new Locher_Ours(), 46 | }; 47 | 48 | for(auto alg : absVec){ 49 | alg->update(result.start, result.length, mp); 50 | delete alg; 51 | } 52 | } 53 | 54 | private: 55 | LoadResult result; 56 | HashMap mp; 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /CPU/Locher/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "Locher.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0xff 9 | 10 | #define INTERVAL 60000 11 | 12 | using MyLocher = Locher; 13 | 14 | using MyChild_Locher = Child_Locher; 15 | 16 | #define HARDTHRES 0x11 17 | 18 | #define THP_TIME 10 19 | 20 | #define ALPHA 0.00001 21 | 22 | //#define QUEUELENGTH 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /CPU/Locher/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/README.md: -------------------------------------------------------------------------------- 1 | CPU Code 2 | ============ 3 | 4 | Repository structure 5 | -------------------- 6 | - `CM`: OctoSketch for the Count-Min Sketch 7 | - `Coco`: OctoSketch for the CocoSketch 8 | - `Count`: OctoSketch for the Count Sketch 9 | - `Elastic`: OctoSketch for the Elastic Sketch 10 | - `HLL`: OctoSketch for the HyperLogLog 11 | - `LL`: OctoSketch for the LogLog 12 | - `Locher`: OctoSketch for the Locher Sketch 13 | - `UnivMon`: OctoSketch for the UnivMon 14 | - `template`: Common functions used in OctoSketch of CPU 15 | 16 | Requirements 17 | ------- 18 | - cmake 19 | - g++ 20 | 21 | How to run (example for the Count-Min Sketch) 22 | ------- 23 | ```bash 24 | $ cd ./CM 25 | $ cmake . 26 | $ make 27 | $ ./CM your-dataset 28 | ``` -------------------------------------------------------------------------------- /CPU/UnivMon/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(UnivMon) 3 | 4 | include_directories(./ ../ ../../common/ ../../queue/ ../../sketch/) 5 | 6 | set(CMAKE_BUILD_TYPE "Release") 7 | set(CMAKE_CXX_STANDARD 14) 8 | set(CMAKE_CXX_FLAGS "-O3 -faligned-new -pthread -mavx2") 9 | #set(CMAKE_CXX_FLAGS "-O0 -Wall -g -ggdb -pthread") 10 | 11 | add_executable(UnivMon main.cpp) 12 | target_link_libraries(UnivMon) -------------------------------------------------------------------------------- /CPU/UnivMon/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIVMON_IDEAL_H 2 | #define UNIVMON_IDEAL_H 3 | 4 | #include "config.h" 5 | #include "template/Ideal.h" 6 | 7 | template 8 | class Univ_Ideal : public Ideal{ 9 | public: 10 | Sketch* initialize_sketch(){ 11 | return new MyUniv(); 12 | } 13 | }; 14 | 15 | 16 | #endif -------------------------------------------------------------------------------- /CPU/UnivMon/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIV_MERGE_H 2 | #define UNIV_MERGE_H 3 | 4 | #include "config.h" 5 | #include "template/Merge.h" 6 | 7 | template 8 | class Univ_Merge : public Merge*, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue*> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyUniv(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return initialize_parent(); 19 | } 20 | 21 | Sketch* insert_child(Sketch* p, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id){ 22 | p->insert_one(packet); 23 | 24 | if((number % (INTERVAL * thread_num)) == (INTERVAL * thread_id)){ 25 | q.enqueue((MyUniv*)p); 26 | return initialize_child(); 27 | } 28 | return p; 29 | } 30 | 31 | void merge(Sketch* p, MyUniv* temp){ 32 | ((MyUniv*)p)->Merge(temp); 33 | } 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /CPU/UnivMon/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIV_OURS_H 2 | #define UNIV_OURS_H 3 | 4 | #include "config.h" 5 | #include "template/Ours.h" 6 | 7 | template 8 | class Univ_Ours : public Ours, thread_num>{ 9 | public: 10 | 11 | typedef ReaderWriterQueue> myQueue; 12 | 13 | Sketch* initialize_parent(){ 14 | return new MyUniv(); 15 | } 16 | 17 | Sketch* initialize_child(){ 18 | return new MyChild_Univ(); 19 | } 20 | 21 | void insert_child(Sketch* p, myQueue& q, const Key& packet){ 22 | auto sketch = ((MyChild_Univ*)p)->sketch; 23 | 24 | uint32_t polar = hash(packet, 199); 25 | uint32_t max_level = MIN(MAX_LEVEL - 1, __builtin_clz(polar)); 26 | 27 | for(uint32_t level = 0; level <= max_level;++level){ 28 | uint32_t pos[HASH_NUM]; 29 | int32_t incre[HASH_NUM]; 30 | 31 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 32 | uint32_t hashNum = hash(packet, level * HASH_NUM + hashPos); 33 | pos[hashPos] = (hashNum >> 1) % LENGTH; 34 | incre[hashPos] = increment[hashNum & 1]; 35 | } 36 | 37 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 38 | sketch[level][hashPos][pos[hashPos]] += incre[hashPos]; 39 | if(sketch[level][hashPos][pos[hashPos]] * incre[hashPos] >= PROMASK){ 40 | q.enqueue(Univ_Entry(packet, level, hashPos, pos[hashPos], 41 | sketch[level][hashPos][pos[hashPos]])); 42 | sketch[level][hashPos][pos[hashPos]] = 0; 43 | } 44 | } 45 | } 46 | } 47 | 48 | void merge(Sketch* p, Univ_Entry temp){ 49 | ((MyUniv*)p)->Merge(temp); 50 | } 51 | 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /CPU/UnivMon/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef BENCHMARK_H 2 | #define BENCHMARK_H 3 | 4 | 5 | #include 6 | 7 | #include "loader.h" 8 | 9 | #include "Ideal.h" 10 | #include "Merge.h" 11 | #include "Ours.h" 12 | 13 | 14 | #define THREAD_NUM 16 15 | 16 | template 17 | class Benchmark{ 18 | public: 19 | typedef std::unordered_map HashMap; 20 | 21 | Benchmark(std::string PATH){ 22 | result = Load(PATH.c_str()); 23 | 24 | uint32_t length = result.length / sizeof(Key); 25 | Key* dataset = (Key*)result.start; 26 | 27 | for(uint32_t i = 0;i < length;++i){ 28 | mp[dataset[i]] += 1; 29 | } 30 | } 31 | 32 | ~Benchmark(){ 33 | UnLoad(result); 34 | } 35 | 36 | void Bench(){ 37 | std::vector absVec = { 38 | new Univ_Ideal(), 39 | new Univ_Merge(), 40 | new Univ_Ours(), 41 | }; 42 | 43 | for(auto alg : absVec){ 44 | alg->update(result.start, result.length, mp); 45 | delete alg; 46 | } 47 | } 48 | 49 | private: 50 | LoadResult result; 51 | HashMap mp; 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /CPU/UnivMon/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "UnivMon.h" 5 | 6 | #define MAX_LEVEL 6 7 | 8 | #define HASH_NUM 3 9 | #define LENGTH (1 << 16) 10 | #define HEAP_SIZE 0x3ff 11 | 12 | #define INTERVAL 40000 13 | 14 | template 15 | using MyUniv = UnivMon; 16 | 17 | template 18 | using MyChild_Univ = Child_UnivMon; 19 | 20 | #define THP_TIME 10 21 | 22 | #define ALPHA 0.0002 23 | 24 | #define PROMASK 0x7f 25 | 26 | //#define QUEUELENGTH 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /CPU/UnivMon/main.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark.h" 2 | 3 | int main(int argc, char *argv[]) { 4 | std::ios::sync_with_stdio(false); 5 | if(argc < 2){ 6 | std::cerr << "File name needed" << std::endl; 7 | return -1; 8 | } 9 | Benchmark bench(argv[1]); 10 | bench.Bench(); 11 | return 0; 12 | } 13 | -------------------------------------------------------------------------------- /CPU/template/Abstract.h: -------------------------------------------------------------------------------- 1 | #ifndef ABSTRACT_H 2 | #define ABSTRACT_H 3 | 4 | #include "sketch.h" 5 | #include "readerwriterqueue.h" 6 | 7 | #include 8 | #include 9 | 10 | typedef std::unordered_map HashMap; 11 | 12 | class Abstract{ 13 | public: 14 | 15 | virtual void update(void* start, uint32_t size, HashMap mp) = 0; 16 | virtual ~Abstract(){}; 17 | 18 | template 19 | static void Partition(Key* start, uint32_t size, uint32_t id, std::vector& vec){ 20 | for(uint32_t i = 0;i < size;++i){ 21 | if(hash(start[i], 101) % thread_num == id){ 22 | vec.push_back(start[i]); 23 | } 24 | } 25 | } 26 | 27 | static void HHCompare(HashMap test, HashMap real, int32_t threshold){ 28 | double estHH = 0, HH = 0, both = 0; 29 | double CR = 0, PR = 0, AAE = 0, ARE = 0; 30 | 31 | for(auto it = test.begin();it != test.end();++it){ 32 | if(it->second > threshold){ 33 | estHH += 1; 34 | if(real[it->first] > threshold){ 35 | both += 1; 36 | AAE += abs(real[it->first] - it->second); 37 | ARE += abs(real[it->first] - it->second) / (double)real[it->first]; 38 | } 39 | } 40 | } 41 | 42 | for(auto it = real.begin();it != real.end();++it){ 43 | if(it->second > threshold){ 44 | HH += 1; 45 | } 46 | } 47 | 48 | std::cout << "CR: " << both / HH << std::endl 49 | << "PR: " << both / estHH << std::endl 50 | << "AAE: " << AAE / both << std::endl 51 | << "ARE: " << ARE / both << std::endl << std::endl; 52 | } 53 | }; 54 | 55 | #endif -------------------------------------------------------------------------------- /CPU/template/Ideal.h: -------------------------------------------------------------------------------- 1 | #ifndef IDEAL_H 2 | #define IDEAL_H 3 | 4 | #include "Abstract.h" 5 | 6 | /** 7 | * Template for sketches running in a single core 8 | */ 9 | 10 | template 11 | class Ideal : public Abstract{ 12 | public: 13 | 14 | virtual Sketch* initialize_sketch() = 0; 15 | 16 | void update(void* start, uint32_t size, HashMap mp){ 17 | uint32_t length = size / sizeof(Key); 18 | Key* dataset = (Key*)start; 19 | 20 | Sketch* sketch = initialize_sketch(); 21 | 22 | for(uint32_t i = 0;i < length;++i){ 23 | sketch->insert_one(dataset[i]); 24 | } 25 | 26 | std::cout << "Baseline Accuracy:" << std::endl; 27 | 28 | HashMap ret = sketch->query_all(); 29 | HHCompare(ret, mp, length * ALPHA); 30 | 31 | delete sketch; 32 | } 33 | }; 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /CPU/template/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef MERGE_H 2 | #define MERGE_H 3 | 4 | #include "Abstract.h" 5 | 6 | /** 7 | * Template for sketch-merge running in multiple cores 8 | */ 9 | 10 | template 11 | class Merge: public Abstract{ 12 | public: 13 | 14 | typedef ReaderWriterQueue myQueue; 15 | 16 | myQueue que[thread_num]; 17 | 18 | virtual Sketch* initialize_parent() = 0; 19 | virtual Sketch* initialize_child() = 0; 20 | 21 | virtual void merge(Sketch* sketch, Entry temp) = 0; 22 | 23 | virtual Sketch* insert_child(Sketch* sketch, myQueue& q, const Key& packet, uint32_t number, uint32_t thread_id) = 0; 24 | 25 | void update(void* start, uint32_t size, HashMap mp){ 26 | std::thread parent; 27 | parent = std::thread(&Merge::ParentThread, this, &parent, start, size, &mp); 28 | parent.join(); 29 | } 30 | 31 | /** 32 | * The thread of the aggregator 33 | */ 34 | void ParentThread(std::thread* thisThd, void* start, uint32_t size, HashMap* mp){ 35 | #ifdef __linux__ 36 | if(!setaffinity(thisThd, thread_num)) 37 | return; 38 | #endif 39 | std::atomic finish(0); 40 | std::thread thd[thread_num]; 41 | 42 | Sketch* sketch = initialize_parent(); 43 | 44 | for(uint32_t i = 0;i < thread_num;++i){ 45 | thd[i] = std::thread(&Merge::ChildThread, this, 46 | &(thd[i]), i, start, size, &finish); 47 | } 48 | 49 | collect(sketch, finish); 50 | 51 | for(uint32_t i = 0;i < thread_num;++i){ 52 | thd[i].join(); 53 | } 54 | 55 | std::cout << "Merge Accuracy:" << std::endl; 56 | HashMap ret = sketch->query_all(); 57 | HHCompare(ret, (*mp), size / sizeof(Key) * ALPHA); 58 | 59 | delete sketch; 60 | } 61 | 62 | /** 63 | * The thread of each worker 64 | */ 65 | void ChildThread(std::thread* thisThd, uint32_t thread_id, void* start, uint32_t size, 66 | std::atomic* finish) 67 | { 68 | #ifdef __linux__ 69 | if(!setaffinity(thisThd, thread_id)) 70 | return; 71 | #endif 72 | Sketch* sketch = initialize_child(); 73 | 74 | std::vector dataset; 75 | 76 | Partition((Key*)start, size / sizeof(Key), thread_id, dataset); 77 | 78 | sketch = insert(dataset, sketch, que[thread_id], thread_id); 79 | 80 | (*finish)++; 81 | delete sketch; 82 | } 83 | 84 | Sketch* insert(const std::vector& dataset, Sketch* sketch, myQueue& q, uint32_t thread_id){ 85 | uint32_t number = 1, length = dataset.size(); 86 | 87 | for(uint32_t i = 0;i < length;++i){ 88 | sketch = insert_child(sketch, q, dataset[i], number, thread_id); 89 | number += 1; 90 | } 91 | return sketch; 92 | } 93 | 94 | void collect(Sketch* sketch, std::atomic& finish){ 95 | Entry temp; 96 | 97 | while(finish < thread_num){ 98 | #ifdef QUEUELENGTH 99 | uint32_t total = 0; 100 | for(uint32_t i = 0;i < thread_num;++i){ 101 | total += que[i].size_approx(); 102 | } 103 | if(total > 1){ 104 | std::cout << "Merge Delay: " << total << std::endl; 105 | } 106 | #endif 107 | for(uint32_t i = 0;i < thread_num;++i){ 108 | if(que[i].try_dequeue(temp)){ 109 | merge(sketch, temp); 110 | } 111 | } 112 | } 113 | 114 | for(uint32_t i = 0;i < thread_num;++i){ 115 | while(que[i].try_dequeue(temp)){ 116 | merge(sketch, temp); 117 | } 118 | } 119 | } 120 | 121 | }; 122 | 123 | #endif -------------------------------------------------------------------------------- /CPU/template/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef OURS_H 2 | #define OURS_H 3 | 4 | #include "Abstract.h" 5 | 6 | /** 7 | * Template for OctoSketch running in multiple cores 8 | */ 9 | 10 | template 11 | class Ours : public Abstract{ 12 | public: 13 | 14 | typedef ReaderWriterQueue myQueue; 15 | 16 | myQueue que[thread_num]; 17 | 18 | virtual Sketch* initialize_parent() = 0; 19 | virtual Sketch* initialize_child() = 0; 20 | 21 | virtual void merge(Sketch* sketch, Entry temp) = 0; 22 | 23 | virtual void insert_child(Sketch* sketch, myQueue& q, const Key& packet) = 0; 24 | 25 | void update(void* start, uint32_t size, HashMap mp){ 26 | std::thread parent; 27 | parent = std::thread(&Ours::ParentThread, this, &parent, start, size, &mp); 28 | parent.join(); 29 | } 30 | 31 | /** 32 | * The thread of the aggregator 33 | */ 34 | void ParentThread(std::thread* thisThd, void* start, uint32_t size, HashMap* mp){ 35 | #ifdef __linux__ 36 | if(!setaffinity(thisThd, thread_num)) 37 | return; 38 | #endif 39 | std::atomic finish(0); 40 | std::thread thd[thread_num]; 41 | 42 | Sketch* sketch = initialize_parent(); 43 | 44 | for(uint32_t i = 0;i < thread_num;++i){ 45 | thd[i] = std::thread(&Ours::ChildThread, this, 46 | &(thd[i]), i, start, size, &finish); 47 | } 48 | 49 | collect(sketch, finish); 50 | 51 | for(uint32_t i = 0;i < thread_num;++i){ 52 | thd[i].join(); 53 | } 54 | 55 | std::cout << "Ours Accuracy:" << std::endl; 56 | HashMap ret = sketch->query_all(); 57 | HHCompare(ret, (*mp), size / sizeof(Key) * ALPHA); 58 | 59 | delete sketch; 60 | } 61 | 62 | /** 63 | * The thread of each worker 64 | */ 65 | void ChildThread(std::thread* thisThd, uint32_t thread_id, void* start, uint32_t size, 66 | std::atomic* finish) 67 | { 68 | #ifdef __linux__ 69 | if(!setaffinity(thisThd, thread_id)) 70 | return; 71 | #endif 72 | Sketch* sketch = initialize_child(); 73 | 74 | std::vector dataset; 75 | 76 | Partition((Key*)start, size / sizeof(Key), thread_id, dataset); 77 | 78 | sketch = insert(dataset, sketch, que[thread_id]); 79 | 80 | (*finish)++; 81 | delete sketch; 82 | } 83 | 84 | Sketch* insert(const std::vector& dataset, Sketch* sketch, myQueue& q){ 85 | uint32_t length = dataset.size(); 86 | 87 | for(uint32_t i = 0;i < length;++i){ 88 | insert_child(sketch, q, dataset[i]); 89 | } 90 | 91 | return sketch; 92 | } 93 | 94 | void collect(Sketch* sketch, std::atomic& finish){ 95 | Entry temp; 96 | 97 | while(finish < thread_num){ 98 | #ifdef QUEUELENGTH 99 | uint32_t total = 0; 100 | for(uint32_t i = 0;i < thread_num;++i){ 101 | total += que[i].size_approx(); 102 | } 103 | if(total > 100 * thread_num){ 104 | std::cout << "Ours Delay: " << total << std::endl; 105 | } 106 | #endif 107 | for(uint32_t i = 0;i < thread_num;++i){ 108 | while(que[i].try_dequeue(temp)){ 109 | merge(sketch, temp); 110 | } 111 | } 112 | } 113 | 114 | for(uint32_t i = 0;i < thread_num;++i){ 115 | while(que[i].try_dequeue(temp)){ 116 | merge(sketch, temp); 117 | } 118 | } 119 | } 120 | 121 | }; 122 | 123 | #endif -------------------------------------------------------------------------------- /DPDK/CM/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/CM/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_MERGE_H 2 | #define CM_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | /** 8 | * Sketch-merge for the Count-Min sketch 9 | */ 10 | 11 | class CM_Merge : public Merge{ 12 | public: 13 | 14 | typedef ReaderWriterQueue myQueue; 15 | 16 | Sketch* initialize_parent(){ 17 | return new MyCM(); 18 | } 19 | 20 | Sketch* initialize_child(){ 21 | return initialize_parent(); 22 | } 23 | 24 | Sketch* insert_child(Sketch* p, myQueue& q, const uint64_t& packet, uint64_t number, uint32_t thread_id){ 25 | p->insert_one(packet); 26 | 27 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 28 | q.enqueue((MyCM*)p); 29 | return initialize_child(); 30 | } 31 | return p; 32 | } 33 | 34 | void merge(Sketch* p, MyCM* temp){ 35 | ((MyCM*)p)->Merge(temp); 36 | } 37 | 38 | }; 39 | 40 | #endif -------------------------------------------------------------------------------- /DPDK/CM/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_OURS_H 2 | #define CM_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | /** 8 | * OctoSketch for the Count-Min sketch 9 | */ 10 | 11 | class CM_Ours : public Ours>{ 12 | public: 13 | 14 | typedef ReaderWriterQueue> myQueue; 15 | 16 | static weak_atomic PROMASK; 17 | 18 | Sketch* initialize_parent(){ 19 | return new MyCM(); 20 | } 21 | 22 | Sketch* initialize_child(){ 23 | return new MyChild_CM(); 24 | } 25 | 26 | void insert_child(Sketch* p, myQueue& q, const uint64_t& packet){ 27 | auto sketch = ((MyChild_CM*)p)->sketch; 28 | uint32_t pos[HASH_NUM]; 29 | 30 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 31 | pos[hashPos] = hash(packet, hashPos) % LENGTH; 32 | } 33 | 34 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 35 | sketch[hashPos][pos[hashPos]] += 1; 36 | if(sketch[hashPos][pos[hashPos]] >= PROMASK){ 37 | q.enqueue(CM_Entry(packet, hashPos, pos[hashPos], sketch[hashPos][pos[hashPos]])); 38 | sketch[hashPos][pos[hashPos]] = 0; 39 | } 40 | } 41 | } 42 | 43 | void merge(Sketch* p, CM_Entry temp){ 44 | ((MyCM*)p)->Merge(temp); 45 | } 46 | 47 | int32_t old_length = 0; 48 | uint64_t old_time = -1; 49 | int32_t new_thres, tmp_mask; 50 | 51 | void modify_threshold(){ 52 | int32_t new_length = 0; 53 | for(uint32_t j = 0;j < NUM_RX_QUEUE;++j){ 54 | new_length += que[j].size_approx(); 55 | } 56 | 57 | int32_t gap = new_length - old_length; 58 | int32_t tmp_length = new_length + gap; 59 | new_thres = tmp_mask = PROMASK; 60 | 61 | if(tmp_length < LOWER_LENGTH && gap < 32){ 62 | new_thres = tmp_mask - 1; 63 | new_thres = MIN(0x3f, new_thres); 64 | new_thres = MAX(0x7, new_thres); 65 | PROMASK = new_thres; 66 | } 67 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 68 | new_thres = tmp_mask + 1; 69 | new_thres = MIN(0x3f, new_thres); 70 | new_thres = MAX(0x7, new_thres); 71 | PROMASK = new_thres; 72 | } 73 | 74 | old_length = new_length; 75 | } 76 | 77 | }; 78 | 79 | #endif -------------------------------------------------------------------------------- /DPDK/CM/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/CM.h" 5 | 6 | // Configuration for the sketch 7 | #define HASH_NUM 3 8 | #define LENGTH (1 << 16) 9 | #define HEAP_SIZE 0x3ff 10 | 11 | #define INTERVAL 20000 12 | 13 | using MyCM = CM; 14 | 15 | using MyChild_CM = Child_CM; 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /DPDK/CM/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH CM_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | weak_atomic CM_Ours::PROMASK{0x8}; 40 | 41 | int main(int argc, char **argv) 42 | { 43 | main_dpdk(argc, argv); 44 | 45 | MYSKETCH* sketch = new MYSKETCH(); 46 | 47 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 48 | 49 | return ret; 50 | } 51 | -------------------------------------------------------------------------------- /DPDK/Coco/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/Coco/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_MERGE_H 2 | #define COCO_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | class Coco_Merge : public Merge{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | return new MyCoco(); 14 | } 15 | 16 | Sketch* initialize_child(){ 17 | return initialize_parent(); 18 | } 19 | 20 | Sketch* insert_child(Sketch* p, myQueue& q, const uint64_t& packet, uint64_t number, uint32_t thread_id){ 21 | p->insert_one(packet); 22 | 23 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 24 | q.enqueue((MyCoco*)p); 25 | return initialize_child(); 26 | } 27 | return p; 28 | } 29 | 30 | void merge(Sketch* p, MyCoco* temp){ 31 | ((MyCoco*)p)->Merge(temp); 32 | } 33 | 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /DPDK/Coco/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_OURS_H 2 | #define COCO_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | class Coco_Ours : public Ours>{ 8 | public: 9 | 10 | typedef ReaderWriterQueue> myQueue; 11 | 12 | static weak_atomic PROMASK; 13 | 14 | Sketch* initialize_parent(){ 15 | return new MyCoco(); 16 | } 17 | 18 | Sketch* initialize_child(){ 19 | return new MyChild_Coco(); 20 | } 21 | 22 | void insert_child(Sketch* p, myQueue& q, const uint64_t& packet){ 23 | auto keys = ((MyChild_Coco*)p)->keys; 24 | auto counters = ((MyChild_Coco*)p)->counters; 25 | 26 | uint32_t choice; 27 | uint16_t pos[2]; 28 | 29 | *((uint32_t*)pos) = hash(packet, 0); 30 | 31 | if(keys[0][pos[0]] == packet){ 32 | counters[0][pos[0]] += 1; 33 | 34 | if(counters[0][pos[0]] >= PROMASK){ 35 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[0][pos[0]])); 36 | counters[0][pos[0]] = 0; 37 | } 38 | 39 | return; 40 | } 41 | 42 | if(keys[1][pos[1]] == packet){ 43 | counters[1][pos[1]] += 1; 44 | 45 | if(counters[1][pos[1]] >= PROMASK){ 46 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[1][pos[1]])); 47 | counters[1][pos[1]] = 0; 48 | } 49 | 50 | return; 51 | } 52 | 53 | choice = (counters[0][pos[0]] > counters[1][pos[1]]); 54 | counters[choice][pos[choice]] += 1; 55 | if(((MyChild_Coco*)p)->rng() % counters[choice][pos[choice]] == 0){ 56 | keys[choice][pos[choice]] = packet; 57 | 58 | if(counters[choice][pos[choice]] >= PROMASK){ 59 | q.enqueue(Coco_Entry(packet, pos[0], pos[1], counters[choice][pos[choice]])); 60 | counters[choice][pos[choice]] = 0; 61 | } 62 | } 63 | else{ 64 | if(counters[choice][pos[choice]] >= PROMASK){ 65 | q.enqueue(Coco_Entry(keys[choice][pos[choice]], pos[0], pos[1], counters[choice][pos[choice]], false)); 66 | counters[choice][pos[choice]] = 0; 67 | } 68 | } 69 | } 70 | 71 | void merge(Sketch* p, Coco_Entry temp){ 72 | ((MyCoco*)p)->Merge(temp); 73 | } 74 | 75 | int32_t old_length = 0; 76 | uint64_t old_time = -1; 77 | int32_t new_thres, tmp_mask; 78 | 79 | void modify_threshold(){ 80 | int32_t new_length = 0; 81 | for(uint32_t j = 0;j < NUM_RX_QUEUE;++j){ 82 | new_length += que[j].size_approx(); 83 | } 84 | 85 | int32_t gap = new_length - old_length; 86 | int32_t tmp_length = new_length + gap; 87 | new_thres = tmp_mask = PROMASK; 88 | 89 | if(tmp_length < LOWER_LENGTH && gap < 32){ 90 | new_thres = tmp_mask - 1; 91 | new_thres = MIN(0x3f, new_thres); 92 | new_thres = MAX(0x7, new_thres); 93 | PROMASK = new_thres; 94 | } 95 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 96 | new_thres = tmp_mask + 1; 97 | new_thres = MIN(0x3f, new_thres); 98 | new_thres = MAX(0x7, new_thres); 99 | PROMASK = new_thres; 100 | } 101 | 102 | old_length = new_length; 103 | } 104 | 105 | }; 106 | 107 | #endif -------------------------------------------------------------------------------- /DPDK/Coco/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/Coco.h" 5 | 6 | #define HASH_NUM 2 7 | #define LENGTH (1 << 16) 8 | 9 | #define INTERVAL 100000 10 | 11 | using MyCoco = Coco; 12 | 13 | using MyChild_Coco = Child_Coco; 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /DPDK/Coco/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH Coco_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | weak_atomic Coco_Ours::PROMASK{0x8}; 40 | 41 | int main(int argc, char **argv) 42 | { 43 | main_dpdk(argc, argv); 44 | 45 | MYSKETCH* sketch = new MYSKETCH(); 46 | 47 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 48 | 49 | return ret; 50 | } 51 | -------------------------------------------------------------------------------- /DPDK/Count/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/Count/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_MERGE_H 2 | #define COUNT_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | class Count_Merge : public Merge{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | return new MyCount(); 14 | } 15 | 16 | Sketch* initialize_child(){ 17 | return initialize_parent(); 18 | } 19 | 20 | Sketch* insert_child(Sketch* p, myQueue& q, const uint64_t& packet, uint64_t number, uint32_t thread_id){ 21 | p->insert_one(packet); 22 | 23 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 24 | q.enqueue((MyCount*)p); 25 | return initialize_child(); 26 | } 27 | return p; 28 | } 29 | 30 | void merge(Sketch* p, MyCount* temp){ 31 | ((MyCount*)p)->Merge(temp); 32 | } 33 | 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /DPDK/Count/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_OURS_H 2 | #define COUNT_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | class Count_Ours : public Ours>{ 8 | public: 9 | 10 | typedef ReaderWriterQueue> myQueue; 11 | 12 | static weak_atomic PROMASK; 13 | 14 | Sketch* initialize_parent(){ 15 | return new MyCount(); 16 | } 17 | 18 | Sketch* initialize_child(){ 19 | return new MyChild_Count(); 20 | } 21 | 22 | void insert_child(Sketch* p, myQueue& q, const uint64_t& packet){ 23 | auto sketch = ((MyChild_Count*)p)->sketch; 24 | uint32_t pos[HASH_NUM]; 25 | int32_t incre[HASH_NUM]; 26 | 27 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 28 | uint32_t hashNum = hash(packet, hashPos); 29 | pos[hashPos] = (hashNum >> 1) % LENGTH; 30 | incre[hashPos] = increment[hashNum & 1]; 31 | } 32 | 33 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 34 | sketch[hashPos][pos[hashPos]] += incre[hashPos]; 35 | if(sketch[hashPos][pos[hashPos]] * incre[hashPos] >= PROMASK){ 36 | q.enqueue(Count_Entry(packet, hashPos, pos[hashPos], sketch[hashPos][pos[hashPos]])); 37 | sketch[hashPos][pos[hashPos]] = 0; 38 | } 39 | } 40 | } 41 | 42 | void merge(Sketch* p, Count_Entry temp){ 43 | ((MyCount*)p)->Merge(temp); 44 | } 45 | 46 | int32_t old_length = 0; 47 | uint64_t old_time = -1; 48 | int32_t new_thres, tmp_mask; 49 | 50 | void modify_threshold(){ 51 | int32_t new_length = 0; 52 | for(uint32_t j = 0;j < NUM_RX_QUEUE;++j){ 53 | new_length += que[j].size_approx(); 54 | } 55 | 56 | int32_t gap = new_length - old_length; 57 | int32_t tmp_length = new_length + gap; 58 | new_thres = tmp_mask = PROMASK; 59 | 60 | if(tmp_length < LOWER_LENGTH && gap < 32){ 61 | new_thres = tmp_mask - 1; 62 | new_thres = MIN(0x3f, new_thres); 63 | new_thres = MAX(0x7, new_thres); 64 | PROMASK = new_thres; 65 | } 66 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 67 | new_thres = tmp_mask + 1; 68 | new_thres = MIN(0x3f, new_thres); 69 | new_thres = MAX(0x7, new_thres); 70 | PROMASK = new_thres; 71 | } 72 | 73 | old_length = new_length; 74 | } 75 | }; 76 | 77 | #endif -------------------------------------------------------------------------------- /DPDK/Count/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/Count.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0x3ff 9 | 10 | #define INTERVAL 20000 11 | 12 | using MyCount = Count; 13 | 14 | using MyChild_Count = Child_Count; 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /DPDK/Count/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH Count_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | weak_atomic Count_Ours::PROMASK{0x8}; 40 | 41 | int main(int argc, char **argv) 42 | { 43 | main_dpdk(argc, argv); 44 | 45 | MYSKETCH* sketch = new MYSKETCH(); 46 | 47 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 48 | 49 | return ret; 50 | } -------------------------------------------------------------------------------- /DPDK/Elastic/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/Elastic/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_MERGE_H 2 | #define ELASTIC_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | class Elastic_Merge : public Merge{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | return new MyElastic(); 14 | } 15 | 16 | Sketch* initialize_child(){ 17 | return initialize_parent(); 18 | } 19 | 20 | Sketch* insert_child(Sketch* p, myQueue& q, const uint64_t& packet, uint64_t number, uint32_t thread_id){ 21 | p->insert_one(packet); 22 | 23 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 24 | q.enqueue((MyElastic*)p); 25 | return initialize_child(); 26 | } 27 | return p; 28 | } 29 | 30 | void merge(Sketch* p, MyElastic* temp){ 31 | ((MyElastic*)p)->Merge(temp); 32 | } 33 | 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /DPDK/Elastic/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_OURS_H 2 | #define ELASTIC_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | class Elastic_Ours : public Ours>{ 8 | public: 9 | 10 | typedef ReaderWriterQueue> myQueue; 11 | 12 | static weak_atomic PROMASK; 13 | 14 | Sketch* initialize_parent(){ 15 | return new MyElastic(); 16 | } 17 | 18 | Sketch* initialize_child(){ 19 | return new MyChild_Elastic(); 20 | } 21 | 22 | void insert_child(Sketch* p, myQueue& q, const uint64_t& packet){ 23 | auto sketch = ((MyChild_Elastic*)p)->sketch; 24 | auto buckets = ((MyChild_Elastic*)p)->buckets; 25 | 26 | uint32_t pos = hash(packet) % BUCKET_LENGTH, minPos = 0; 27 | int32_t minVal = 0x7fffffff; 28 | 29 | for (uint32_t j = 0; j < COUNTER_PER_BUCKET; j++){ 30 | if(buckets[pos].ID[j] == packet) { 31 | buckets[pos].count[j] += 1; 32 | if(buckets[pos].count[j] >= PROMASK){ 33 | q.enqueue(Elastic_Entry(buckets[pos].ID[j], pos, buckets[pos].count[j], true)); 34 | buckets[pos].count[j] = 0; 35 | } 36 | return; 37 | } 38 | 39 | if(buckets[pos].count[j] < minVal){ 40 | minPos = j; 41 | minVal = buckets[pos].count[j]; 42 | } 43 | } 44 | 45 | if((buckets[pos].vote + 1) >= minVal * 8){ 46 | buckets[pos].vote = 0; 47 | 48 | if(minVal != 0){ 49 | uint32_t position = hash(buckets[pos].ID[minPos], 101) % SKETCH_LENGTH; 50 | sketch[position] = sketch[position] + buckets[pos].count[minPos]; 51 | if(sketch[position] >= PROMASK){ 52 | q.enqueue(Elastic_Entry(buckets[pos].ID[minPos], position, sketch[position], false)); 53 | sketch[position] = 0; 54 | } 55 | } 56 | 57 | buckets[pos].ID[minPos] = packet; 58 | buckets[pos].count[minPos] = 1; 59 | } 60 | else { 61 | buckets[pos].vote += 1; 62 | uint32_t position = hash(packet, 101) % SKETCH_LENGTH; 63 | sketch[position] = sketch[position] + 1; 64 | if(sketch[position] >= PROMASK){ 65 | q.enqueue(Elastic_Entry(packet, position, sketch[position], false)); 66 | sketch[position] = 0; 67 | } 68 | } 69 | } 70 | 71 | void merge(Sketch* p, Elastic_Entry temp){ 72 | ((MyElastic*)p)->Merge(temp); 73 | } 74 | 75 | int32_t old_length = 0; 76 | uint64_t old_time = -1; 77 | int32_t new_thres, tmp_mask; 78 | 79 | void modify_threshold(){ 80 | int32_t new_length = 0; 81 | for(uint32_t j = 0;j < NUM_RX_QUEUE;++j){ 82 | new_length += que[j].size_approx(); 83 | } 84 | 85 | int32_t gap = new_length - old_length; 86 | int32_t tmp_length = new_length + gap; 87 | new_thres = tmp_mask = PROMASK; 88 | 89 | if(tmp_length < LOWER_LENGTH && gap < 32){ 90 | new_thres = tmp_mask - 1; 91 | new_thres = MIN(0x3f, new_thres); 92 | new_thres = MAX(0x7, new_thres); 93 | PROMASK = new_thres; 94 | } 95 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 96 | new_thres = tmp_mask + 1; 97 | new_thres = MIN(0x3f, new_thres); 98 | new_thres = MAX(0x7, new_thres); 99 | PROMASK = new_thres; 100 | } 101 | 102 | old_length = new_length; 103 | } 104 | }; 105 | 106 | #endif -------------------------------------------------------------------------------- /DPDK/Elastic/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/Elastic.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0x3ff 9 | 10 | #define COUNTER_PER_BUCKET 4 11 | 12 | #define SKETCH_LENGTH (HASH_NUM * LENGTH) 13 | #define BUCKET_LENGTH (HEAP_SIZE * 3 / COUNTER_PER_BUCKET) 14 | 15 | #define INTERVAL 20000 16 | 17 | using MyElastic = Elastic; 18 | 19 | using MyChild_Elastic = Child_Elastic; 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /DPDK/Elastic/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH Elastic_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | weak_atomic Elastic_Ours::PROMASK{0x8}; 40 | 41 | int main(int argc, char **argv) 42 | { 43 | main_dpdk(argc, argv); 44 | 45 | MYSKETCH* sketch = new MYSKETCH(); 46 | 47 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 48 | 49 | return ret; 50 | } 51 | 52 | -------------------------------------------------------------------------------- /DPDK/Locher/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/Locher/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_MERGE_H 2 | #define LOCHER_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | class Locher_Merge : public Merge{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | return new MyLocher(); 14 | } 15 | 16 | Sketch* initialize_child(){ 17 | return initialize_parent(); 18 | } 19 | 20 | Sketch* insert_child(Sketch* p, myQueue& q, const Packet& packet, uint64_t number, uint32_t thread_id){ 21 | p->insert_one(packet); 22 | 23 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 24 | q.enqueue((MyLocher*)p); 25 | return initialize_child(); 26 | } 27 | return p; 28 | } 29 | 30 | void merge(Sketch* p, MyLocher* temp){ 31 | ((MyLocher*)p)->Merge(temp); 32 | } 33 | 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /DPDK/Locher/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_OURS_H 2 | #define LOCHER_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | class Locher_Ours : public Ours{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | MyLocher* ret = new MyLocher(); 14 | memset(ret->sketch, HARDTHRES, HASH_NUM * LENGTH * sizeof(SHLL)); 15 | return ret; 16 | } 17 | 18 | Sketch* initialize_child(){ 19 | MyChild_Locher* ret = new MyChild_Locher(); 20 | memset(ret->sketch, HARDTHRES, HASH_NUM * LENGTH * sizeof(SHLL)); 21 | return ret; 22 | } 23 | 24 | void insert_child(Sketch* p, myQueue& q, const Packet& packet){ 25 | auto sketch = ((MyChild_Locher*)p)->sketch; 26 | 27 | uint32_t pos[HASH_NUM]; 28 | uint32_t temp[HASH_NUM]; 29 | 30 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 31 | pos[hashPos] = hash(packet.src, hashPos) % LENGTH; 32 | temp[hashPos] = hash(packet.dst, hashPos); 33 | } 34 | 35 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 36 | uint32_t bucket_index = ((temp[hashPos] >> 1) & 0x7); 37 | Buckets& tempBucket = sketch[hashPos][pos[hashPos]].buckets[bucket_index]; 38 | 39 | uint32_t inbucket_index = (temp[hashPos] & 0x1); 40 | uint8_t rank = MIN(15, __builtin_clz(temp[hashPos]) + 1); 41 | 42 | switch(inbucket_index){ 43 | case 0: 44 | if(tempBucket.counter0 < rank){ 45 | tempBucket.counter0 = rank; 46 | q.enqueue(Locher_Entry(packet.src, hashPos, pos[hashPos], temp[hashPos])); 47 | } 48 | break; 49 | case 1: 50 | if(tempBucket.counter1 < rank){ 51 | tempBucket.counter1 = rank; 52 | q.enqueue(Locher_Entry(packet.src, hashPos, pos[hashPos], temp[hashPos])); 53 | } 54 | break; 55 | } 56 | } 57 | } 58 | 59 | void merge(Sketch* p, Locher_Entry temp){ 60 | ((MyLocher*)p)->Merge(temp); 61 | } 62 | 63 | void modify_threshold(){} 64 | }; 65 | 66 | #endif -------------------------------------------------------------------------------- /DPDK/Locher/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/Locher.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH (1 << 16) 8 | #define HEAP_SIZE 0xff 9 | 10 | #define INTERVAL 60000 11 | 12 | using MyLocher = Locher; 13 | 14 | using MyChild_Locher = Child_Locher; 15 | 16 | #define HARDTHRES 0x11 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /DPDK/Locher/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH Locher_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | int main(int argc, char **argv) 40 | { 41 | main_dpdk(argc, argv); 42 | 43 | MYSKETCH* sketch = new MYSKETCH(); 44 | 45 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 46 | 47 | return ret; 48 | } 49 | -------------------------------------------------------------------------------- /DPDK/README.md: -------------------------------------------------------------------------------- 1 | DPDK Integration 2 | ============ 3 | 4 | Environment 5 | -------------------- 6 | We have tested on 7 | - DPDK 21.11 8 | - Linux kernel version 5.15.0 9 | 10 | Repository structure 11 | -------------------- 12 | - `CM`: OctoSketch for the Count-Min Sketch 13 | - `Coco`: OctoSketch for the CocoSketch 14 | - `Count`: OctoSketch for the Count Sketch 15 | - `Elastic`: OctoSketch for the Elastic Sketch 16 | - `Locher`: OctoSketch for the Locher Sketch 17 | - `UnivMon`: OctoSketch for the UnivMon 18 | - `template`: Common functions used in OctoSketch of DPDK 19 | 20 | How to run (example for the Count-Min Sketch) 21 | ------- 22 | ```bash 23 | $ cd ./CM 24 | $ make 25 | $ sudo ./build/OctoSketch -l 0-12 -n 4 26 | ``` -------------------------------------------------------------------------------- /DPDK/UnivMon/Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright(c) 2010-2014 Intel Corporation 3 | 4 | # binary name 5 | APP = OctoSketch 6 | 7 | # all source are stored in SRCS-y 8 | SRCS-y := main.c 9 | 10 | PKGCONF ?= pkg-config 11 | 12 | # Build using pkg-config variables if possible 13 | ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0) 14 | $(error "no installation of DPDK found") 15 | endif 16 | 17 | all: shared 18 | .PHONY: shared static 19 | shared: build/$(APP)-shared 20 | ln -sf $(APP)-shared build/$(APP) 21 | static: build/$(APP)-static 22 | ln -sf $(APP)-static build/$(APP) 23 | 24 | PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null) 25 | CFLAGS += -O3 -std=c++11 -faligned-new -pthread -mavx2 $(shell $(PKGCONF) --cflags libdpdk) 26 | # Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API 27 | CFLAGS += -DALLOW_EXPERIMENTAL_API 28 | LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk) 29 | LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk) 30 | 31 | ifeq ($(MAKECMDGOALS),static) 32 | # check for broken pkg-config 33 | ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),) 34 | $(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.") 35 | $(error "Cannot generate statically-linked binaries with this version of pkg-config") 36 | endif 37 | endif 38 | 39 | build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build 40 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) 41 | 42 | build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build 43 | g++ $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) 44 | 45 | build: 46 | @mkdir -p $@ 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared 51 | test -d build && rmdir -p build || true 52 | -------------------------------------------------------------------------------- /DPDK/UnivMon/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIV_MERGE_H 2 | #define UNIV_MERGE_H 3 | 4 | #include "config.h" 5 | #include "../template/Merge.h" 6 | 7 | class Univ_Merge : public Merge{ 8 | public: 9 | 10 | typedef ReaderWriterQueue myQueue; 11 | 12 | Sketch* initialize_parent(){ 13 | return new MyUniv(); 14 | } 15 | 16 | Sketch* initialize_child(){ 17 | return initialize_parent(); 18 | } 19 | 20 | Sketch* insert_child(Sketch* p, myQueue& q, const uint64_t& packet, uint64_t number, uint32_t thread_id){ 21 | p->insert_one(packet); 22 | 23 | if((number % (INTERVAL * NUM_RX_QUEUE)) == (INTERVAL * thread_id)){ 24 | q.enqueue((MyUniv*)p); 25 | return initialize_child(); 26 | } 27 | return p; 28 | } 29 | 30 | void merge(Sketch* p, MyUniv* temp){ 31 | ((MyUniv*)p)->Merge(temp); 32 | } 33 | 34 | }; 35 | 36 | #endif -------------------------------------------------------------------------------- /DPDK/UnivMon/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIV_OURS_H 2 | #define UNIV_OURS_H 3 | 4 | #include "config.h" 5 | #include "../template/Ours.h" 6 | 7 | class Univ_Ours : public Ours>{ 8 | public: 9 | 10 | typedef ReaderWriterQueue> myQueue; 11 | 12 | static weak_atomic PROMASK; 13 | 14 | Sketch* initialize_parent(){ 15 | return new MyUniv(); 16 | } 17 | 18 | Sketch* initialize_child(){ 19 | return new MyChild_Univ(); 20 | } 21 | 22 | void insert_child(Sketch* p, myQueue& q, const uint64_t& packet){ 23 | auto sketch = ((MyChild_Univ*)p)->sketch; 24 | 25 | uint32_t polar = hash(packet, 199); 26 | uint32_t max_level = MIN(MAX_LEVEL - 1, __builtin_clz(polar)); 27 | 28 | for(uint32_t level = 0; level <= max_level;++level){ 29 | uint32_t pos[HASH_NUM]; 30 | int32_t incre[HASH_NUM]; 31 | 32 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 33 | uint32_t hashNum = hash(packet, level * HASH_NUM + hashPos); 34 | pos[hashPos] = (hashNum >> 1) % LENGTH; 35 | incre[hashPos] = increment[hashNum & 1]; 36 | } 37 | 38 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 39 | sketch[level][hashPos][pos[hashPos]] += incre[hashPos]; 40 | if(sketch[level][hashPos][pos[hashPos]] * incre[hashPos] >= PROMASK){ 41 | q.enqueue(Univ_Entry(packet, level, hashPos, pos[hashPos], 42 | sketch[level][hashPos][pos[hashPos]])); 43 | sketch[level][hashPos][pos[hashPos]] = 0; 44 | } 45 | } 46 | } 47 | } 48 | 49 | void merge(Sketch* p, Univ_Entry temp){ 50 | ((MyUniv*)p)->Merge(temp); 51 | } 52 | 53 | int32_t old_length = 0; 54 | uint64_t old_time = -1; 55 | int32_t new_thres, tmp_mask; 56 | 57 | void modify_threshold(){ 58 | int32_t new_length = 0; 59 | for(uint32_t j = 0;j < NUM_RX_QUEUE;++j){ 60 | new_length += que[j].size_approx(); 61 | } 62 | 63 | int32_t gap = new_length - old_length; 64 | int32_t tmp_length = new_length + gap; 65 | new_thres = tmp_mask = PROMASK; 66 | 67 | if(tmp_length < LOWER_LENGTH && gap < 32){ 68 | new_thres = tmp_mask - 1; 69 | new_thres = MIN(0x3f, new_thres); 70 | new_thres = MAX(0x7, new_thres); 71 | PROMASK = new_thres; 72 | } 73 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 74 | new_thres = tmp_mask + 1; 75 | new_thres = MIN(0x3f, new_thres); 76 | new_thres = MAX(0x7, new_thres); 77 | PROMASK = new_thres; 78 | } 79 | 80 | old_length = new_length; 81 | } 82 | 83 | }; 84 | 85 | #endif -------------------------------------------------------------------------------- /DPDK/UnivMon/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include "../../sketch/UnivMon.h" 5 | 6 | #define MAX_LEVEL 6 7 | 8 | #define HASH_NUM 3 9 | #define LENGTH (1 << 16) 10 | #define HEAP_SIZE 0x3ff 11 | 12 | #define INTERVAL 40000 13 | 14 | using MyUniv = UnivMon; 15 | 16 | using MyChild_Univ = Child_UnivMon; 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /DPDK/UnivMon/main.c: -------------------------------------------------------------------------------- 1 | #include "Merge.h" 2 | #include "Ours.h" 3 | 4 | #include "../template/DPDK.h" 5 | 6 | #define MYSKETCH Univ_Ours 7 | 8 | std::atomic is_used[NUM_RX_QUEUE + 1]; 9 | 10 | int distribute(void *arg){ 11 | MYSKETCH* sketch = (MYSKETCH*) arg; 12 | 13 | if(rte_get_main_lcore() == rte_lcore_id()){ 14 | print_stats(); 15 | return 1; 16 | } 17 | 18 | bool set = true; 19 | 20 | set = is_used[0].exchange(set); 21 | if(!set){ 22 | sketch->coordinator(NUM_RX_QUEUE); 23 | return 2; 24 | } 25 | 26 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 27 | set = is_used[i + 1].exchange(set); 28 | if(!set){ 29 | sketch->local(i); 30 | return 3; 31 | } 32 | } 33 | 34 | RTE_LOG(INFO, L2FWD, "nothing to do for lcore %u\n", rte_lcore_id()); 35 | 36 | return 0; 37 | } 38 | 39 | weak_atomic Univ_Ours::PROMASK{0x8}; 40 | 41 | int main(int argc, char **argv) 42 | { 43 | main_dpdk(argc, argv); 44 | 45 | MYSKETCH* sketch = new MYSKETCH(); 46 | 47 | int ret = rte_eal_mp_remote_launch(distribute, sketch, CALL_MAIN); 48 | 49 | return ret; 50 | } 51 | -------------------------------------------------------------------------------- /DPDK/template/DPDK.h: -------------------------------------------------------------------------------- 1 | #ifndef DPDK_H 2 | #define DPDK_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #include "common.h" 42 | 43 | /* Check the link status of all ports in up to 9s, and print them finally */ 44 | static void 45 | check_all_ports_link_status(uint32_t port_mask) 46 | { 47 | #define CHECK_INTERVAL 100 /* 100ms */ 48 | #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 49 | uint16_t portid; 50 | uint8_t count, all_ports_up, print_flag = 0; 51 | struct rte_eth_link link; 52 | int ret; 53 | char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; 54 | 55 | printf("\nChecking link status"); 56 | fflush(stdout); 57 | for (count = 0; count <= MAX_CHECK_TIME; count++) { 58 | if (force_quit) 59 | return; 60 | all_ports_up = 1; 61 | RTE_ETH_FOREACH_DEV(portid) { 62 | if (force_quit) 63 | return; 64 | if ((port_mask & (1 << portid)) == 0) 65 | continue; 66 | memset(&link, 0, sizeof(link)); 67 | ret = rte_eth_link_get_nowait(portid, &link); 68 | if (ret < 0) { 69 | all_ports_up = 0; 70 | if (print_flag == 1) 71 | printf("Port %u link get failed: %s\n", 72 | portid, rte_strerror(-ret)); 73 | continue; 74 | } 75 | /* print link status if flag set */ 76 | if (print_flag == 1) { 77 | rte_eth_link_to_str(link_status_text, 78 | sizeof(link_status_text), &link); 79 | printf("Port %d %s\n", portid, 80 | link_status_text); 81 | continue; 82 | } 83 | /* clear all_ports_up flag if any link down */ 84 | if (link.link_status == RTE_ETH_LINK_DOWN) { 85 | all_ports_up = 0; 86 | break; 87 | } 88 | } 89 | /* after finally printing all link status, get out */ 90 | if (print_flag == 1) 91 | break; 92 | 93 | if (all_ports_up == 0) { 94 | printf("."); 95 | fflush(stdout); 96 | rte_delay_ms(CHECK_INTERVAL); 97 | } 98 | 99 | /* set the print_flag if all ports up or timeout */ 100 | if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 101 | print_flag = 1; 102 | printf("done\n"); 103 | } 104 | } 105 | } 106 | 107 | static void 108 | signal_handler(int signum) 109 | { 110 | if (signum == SIGINT || signum == SIGTERM) { 111 | printf("\n\nSignal %d received, preparing to exit...\n", 112 | signum); 113 | force_quit = true; 114 | } 115 | } 116 | 117 | int 118 | main_dpdk(int argc, char **argv) 119 | { 120 | int ret, i; 121 | uint16_t nb_ports; 122 | uint16_t nb_ports_available = 1; 123 | uint16_t last_port; 124 | uint16_t portid = 0; 125 | unsigned lcore_id, rx_lcore_id; 126 | unsigned nb_ports_in_mask = 0; 127 | unsigned int nb_lcores = 1; 128 | unsigned int nb_mbufs; 129 | 130 | port_conf.rxmode.split_hdr_size = 0; 131 | port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS; 132 | port_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE; 133 | port_conf.rx_adv_conf.rss_conf.rss_key = NULL; 134 | port_conf.rx_adv_conf.rss_conf.rss_hf = (RTE_ETH_RSS_IP | RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP); 135 | 136 | /* Init EAL. 8< */ 137 | ret = rte_eal_init(argc, argv); 138 | if (ret < 0) 139 | rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); 140 | argc -= ret; 141 | argv += ret; 142 | 143 | force_quit = false; 144 | signal(SIGINT, signal_handler); 145 | signal(SIGTERM, signal_handler); 146 | 147 | /* convert to number of cycles */ 148 | timer_period *= rte_get_timer_hz(); 149 | 150 | nb_ports = rte_eth_dev_count_avail(); 151 | if (nb_ports == 0) 152 | rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); 153 | 154 | /* check port mask to possible port mask */ 155 | if (l2fwd_enabled_port_mask & ~((1 << nb_ports) - 1)) 156 | rte_exit(EXIT_FAILURE, "Invalid portmask; possible (0x%x)\n", 157 | (1 << nb_ports) - 1); 158 | 159 | /* Initialization of the driver. 8< */ 160 | 161 | last_port = 0; 162 | 163 | rx_lcore_id = 0; 164 | 165 | nb_mbufs = RTE_MAX(nb_ports * (nb_rxd * NUM_RX_QUEUE + nb_txd + MAX_PKT_BURST + 166 | nb_lcores * MEMPOOL_CACHE_SIZE), 8192U); 167 | 168 | /* Create the mbuf pool. 8< */ 169 | l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", nb_mbufs, 170 | MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, 171 | rte_socket_id()); 172 | if (l2fwd_pktmbuf_pool == NULL) 173 | rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n"); 174 | /* >8 End of create the mbuf pool. */ 175 | 176 | /* Initialise each port */ 177 | struct rte_eth_rxconf rxq_conf; 178 | struct rte_eth_txconf txq_conf; 179 | struct rte_eth_conf local_port_conf = port_conf; 180 | struct rte_eth_dev_info dev_info; 181 | 182 | /* init port */ 183 | printf("Initializing port %u... ", portid); 184 | fflush(stdout); 185 | 186 | ret = rte_eth_dev_info_get(portid, &dev_info); 187 | if (ret != 0) 188 | rte_exit(EXIT_FAILURE, 189 | "Error during getting device (port %u) info: %s\n", 190 | portid, strerror(-ret)); 191 | 192 | if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 193 | local_port_conf.txmode.offloads |= 194 | RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 195 | /* Configure the number of queues for a port. */ 196 | ret = rte_eth_dev_configure(portid, NUM_RX_QUEUE, 1, &local_port_conf); 197 | if (ret < 0) 198 | rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", 199 | ret, portid); 200 | /* >8 End of configuration of the number of queues for a port. */ 201 | 202 | ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, 203 | &nb_txd); 204 | if (ret < 0) 205 | rte_exit(EXIT_FAILURE, 206 | "Cannot adjust number of descriptors: err=%d, port=%u\n", 207 | ret, portid); 208 | 209 | ret = rte_eth_macaddr_get(portid, 210 | &l2fwd_ports_eth_addr[portid]); 211 | if (ret < 0) 212 | rte_exit(EXIT_FAILURE, 213 | "Cannot get MAC address: err=%d, port=%u\n", 214 | ret, portid); 215 | 216 | /* init one RX queue */ 217 | fflush(stdout); 218 | rxq_conf = dev_info.default_rxconf; 219 | rxq_conf.offloads = local_port_conf.rxmode.offloads; 220 | /* RX queue setup. 8< */ 221 | for(i = 0;i < NUM_RX_QUEUE;++i){ 222 | ret = rte_eth_rx_queue_setup(portid, i, nb_rxd, 223 | rte_eth_dev_socket_id(portid), 224 | &rxq_conf, 225 | l2fwd_pktmbuf_pool); 226 | if (ret < 0) 227 | rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u, queue=%d\n", 228 | ret, portid, i); 229 | } 230 | fflush(stdout); 231 | /* >8 End of RX queue setup. */ 232 | 233 | /* Start device */ 234 | ret = rte_eth_dev_start(portid); 235 | if (ret < 0) 236 | rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", 237 | ret, portid); 238 | 239 | printf("done: \n"); 240 | 241 | printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", 242 | portid, 243 | RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); 244 | 245 | fflush(stdout); 246 | 247 | /* initialize port stats */ 248 | memset(&port_statistics, 0, sizeof(port_statistics)); 249 | 250 | //check_all_ports_link_status(l2fwd_enabled_port_mask); 251 | 252 | return 0; 253 | } 254 | 255 | #endif -------------------------------------------------------------------------------- /DPDK/template/Merge.h: -------------------------------------------------------------------------------- 1 | #ifndef MERGE_H 2 | #define MERGE_H 3 | 4 | #include "DPDK.h" 5 | 6 | /** 7 | * Template for sketch-merge running in multiple cores 8 | */ 9 | 10 | template 11 | class Merge{ 12 | public: 13 | 14 | typedef ReaderWriterQueue myQueue; 15 | 16 | myQueue que[NUM_RX_QUEUE]; 17 | 18 | virtual Sketch* initialize_parent() = 0; 19 | virtual Sketch* initialize_child() = 0; 20 | 21 | virtual void merge(Sketch* sketch, Entry temp) = 0; 22 | 23 | virtual Sketch* insert_child(Sketch* sketch, myQueue& q, const Key& packet, uint64_t number, uint32_t thread_id) = 0; 24 | 25 | /** 26 | * The thread of the aggregator 27 | */ 28 | void coordinator(unsigned queue_id){ 29 | uint64_t start, end; 30 | RTE_LOG(INFO, L2FWD, "entering coordinator %u\n", queue_id); 31 | 32 | Sketch* sketch = initialize_parent(); 33 | Entry temp; 34 | 35 | while(true){ 36 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 37 | if(que[i].try_dequeue(temp)){ 38 | merge(sketch, temp); 39 | port_statistics[queue_id].rx += 1; 40 | } 41 | } 42 | } 43 | 44 | delete sketch; 45 | } 46 | 47 | /** 48 | * The thread of each worker 49 | */ 50 | void local(unsigned queue_id){ 51 | RTE_LOG(INFO, L2FWD, "entering local sketch %u\n", queue_id); 52 | 53 | uint32_t batches = 0; 54 | 55 | struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 56 | struct rte_tcp_hdr *tcp[MAX_PKT_BURST]; 57 | 58 | int sent; 59 | uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc; 60 | unsigned portid = 0; 61 | unsigned nb_rx; 62 | 63 | prev_tsc = 0; 64 | timer_tsc = 0; 65 | 66 | Key item[MAX_PKT_BURST]; 67 | Sketch* sketch = initialize_child(); 68 | 69 | uint64_t number = 0; 70 | 71 | uint64_t start, end; 72 | uint64_t one_start, one_end; 73 | 74 | while (true) { 75 | /* Read packet from RX queues. 8< */ 76 | nb_rx = rte_eth_rx_burst(portid, queue_id, 77 | pkts_burst, MAX_PKT_BURST); 78 | 79 | port_statistics[queue_id].rx += nb_rx; 80 | 81 | for(uint32_t j = 0; j < nb_rx; j++) { 82 | tcp[j] = rte_pktmbuf_mtod_offset(pkts_burst[j], struct rte_tcp_hdr *, 83 | sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 84 | rte_prefetch0(tcp[j]); 85 | } 86 | 87 | for(uint32_t j = 0; j < nb_rx; j++) { 88 | init_key(item[j], rte_be_to_cpu_16(tcp[j]->src_port), rte_be_to_cpu_16(tcp[j]->dst_port)); 89 | rte_pktmbuf_free(pkts_burst[j]); 90 | } 91 | 92 | for(uint32_t i = 0;i < nb_rx;++i){ 93 | sketch = insert_child(sketch, que[queue_id], item[i], number, queue_id); 94 | number += 1; 95 | } 96 | 97 | /* >8 End of read packet from RX queues. */ 98 | } 99 | } 100 | }; 101 | 102 | #endif -------------------------------------------------------------------------------- /DPDK/template/Ours.h: -------------------------------------------------------------------------------- 1 | #ifndef OURS_H 2 | #define OURS_H 3 | 4 | #include "DPDK.h" 5 | 6 | /** 7 | * Template for OctoSketch running in multiple cores 8 | */ 9 | 10 | template 11 | class Ours{ 12 | public: 13 | 14 | typedef ReaderWriterQueue myQueue; 15 | 16 | myQueue que[NUM_RX_QUEUE]; 17 | 18 | virtual Sketch* initialize_parent() = 0; 19 | virtual Sketch* initialize_child() = 0; 20 | 21 | virtual void merge(Sketch* sketch, Entry temp) = 0; 22 | 23 | virtual void modify_threshold() = 0; 24 | 25 | virtual void insert_child(Sketch* sketch, myQueue& q, const Key& packet) = 0; 26 | 27 | /** 28 | * The thread of the aggregator 29 | */ 30 | void coordinator(unsigned queue_id){ 31 | uint64_t start, end; 32 | uint64_t idx = 0; 33 | RTE_LOG(INFO, L2FWD, "entering coordinator %u\n", queue_id); 34 | 35 | Sketch* sketch = initialize_parent(); 36 | Entry temp; 37 | 38 | while(true){ 39 | for(uint32_t i = 0;i < NUM_RX_QUEUE;++i){ 40 | while(que[i].try_dequeue(temp)){ 41 | merge(sketch, temp); 42 | port_statistics[queue_id].rx += 1; 43 | 44 | idx += 1; 45 | if(idx > 0x3f){ 46 | idx = 0; 47 | modify_threshold(); 48 | } 49 | } 50 | } 51 | idx += NUM_RX_QUEUE; 52 | if(idx > 0x3f){ 53 | idx = 0; 54 | modify_threshold(); 55 | } 56 | } 57 | 58 | delete sketch; 59 | } 60 | 61 | /** 62 | * The thread of each worker 63 | */ 64 | void local(unsigned queue_id){ 65 | RTE_LOG(INFO, L2FWD, "%u core entering local sketch %u\n", rte_lcore_id(), queue_id); 66 | 67 | uint32_t batches = 0; 68 | 69 | struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 70 | struct rte_tcp_hdr *tcp[MAX_PKT_BURST]; 71 | 72 | int sent; 73 | uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc; 74 | unsigned portid = 0; 75 | unsigned nb_rx; 76 | 77 | prev_tsc = 0; 78 | timer_tsc = 0; 79 | 80 | Key item[MAX_PKT_BURST]; 81 | Sketch* sketch = initialize_child(); 82 | 83 | uint64_t start, end; 84 | uint64_t one_start, one_end; 85 | 86 | while(true) { 87 | /* Read packet from RX queues. 8< */ 88 | nb_rx = rte_eth_rx_burst(portid, queue_id, 89 | pkts_burst, MAX_PKT_BURST); 90 | 91 | port_statistics[queue_id].rx += nb_rx; 92 | 93 | for(uint32_t j = 0; j < nb_rx; j++) { 94 | tcp[j] = rte_pktmbuf_mtod_offset(pkts_burst[j], struct rte_tcp_hdr *, 95 | sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr)); 96 | rte_prefetch0(tcp[j]); 97 | } 98 | 99 | for(uint32_t j = 0; j < nb_rx; j++) { 100 | init_key(item[j], rte_be_to_cpu_16(tcp[j]->src_port), rte_be_to_cpu_16(tcp[j]->dst_port)); 101 | rte_pktmbuf_free(pkts_burst[j]); 102 | } 103 | 104 | for(uint32_t i = 0;i < nb_rx;++i){ 105 | insert_child(sketch, que[queue_id], item[i]); 106 | } 107 | 108 | /* >8 End of read packet from RX queues. */ 109 | } 110 | } 111 | }; 112 | 113 | #endif -------------------------------------------------------------------------------- /DPDK/template/common.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMON_H 2 | #define COMMON_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #include "../../common/util.h" 42 | #include "../../sketch/sketch.h" 43 | #include "../../queue/readerwriterqueue.h" 44 | 45 | 46 | #define NUM_RX_QUEUE 10 47 | 48 | #define UPPER_LENGTH 0xa0 49 | #define TARGET_LENGTH 0x80 50 | #define LOWER_LENGTH 0x60 51 | 52 | static weak_atomic force_quit; 53 | 54 | #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 55 | 56 | #define MAX_PKT_BURST 32 57 | #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 58 | #define MEMPOOL_CACHE_SIZE 256 59 | 60 | /* 61 | * Configurable number of RX/TX ring descriptors 62 | */ 63 | #define RTE_TEST_RX_DESC_DEFAULT 1024 64 | #define RTE_TEST_TX_DESC_DEFAULT 1024 65 | static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; 66 | static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; 67 | 68 | /* ethernet addresses of ports */ 69 | static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; 70 | 71 | /* mask of enabled ports */ 72 | static uint32_t l2fwd_enabled_port_mask = 0x1; 73 | 74 | /* list of enabled ports */ 75 | 76 | static unsigned int l2fwd_rx_queue_per_lcore = 1; 77 | 78 | #define MAX_RX_QUEUE_PER_LCORE 16 79 | #define MAX_TX_QUEUE_PER_PORT 16 80 | 81 | static struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; 82 | 83 | static struct rte_eth_conf port_conf; 84 | 85 | struct rte_mempool * l2fwd_pktmbuf_pool = NULL; 86 | 87 | /* Per-port statistics struct */ 88 | struct l2fwd_port_statistics { 89 | uint64_t rx; 90 | uint64_t poll_cycles; 91 | uint64_t sketch_cycles; 92 | } __rte_cache_aligned; 93 | struct l2fwd_port_statistics port_statistics[NUM_RX_QUEUE + 1]; 94 | 95 | #define MAX_TIMER_PERIOD 86400 /* 1 day max */ 96 | /* A tsc-based timer responsible for triggering statistics printout */ 97 | static uint64_t timer_period = 10; /* default period is 10 seconds */ 98 | 99 | inline void init_key(uint64_t& key, uint64_t src, uint64_t dst){ 100 | key = ((src << 32) | dst); 101 | } 102 | 103 | inline void init_key(Packet& packet, uint64_t src, uint64_t dst){ 104 | packet.src = src; 105 | packet.dst = dst; 106 | } 107 | 108 | /* Print out statistics */ 109 | void 110 | print_stats() 111 | { 112 | uint64_t old_total_packets_rx = 0, total_packets_rx = 0; 113 | uint64_t old_total_core_rx = 0, total_core_rx = 0; 114 | 115 | uint32_t portid = 0, i; 116 | uint64_t t, last_time = rte_get_tsc_cycles(), intervalMs; 117 | 118 | const char clr[] = { 27, '[', '2', 'J', '\0' }; 119 | const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' }; 120 | 121 | while(true){ 122 | t = rte_get_tsc_cycles(); 123 | intervalMs = 1000000 * (t - last_time) / (rte_get_tsc_hz()); 124 | 125 | if(intervalMs >= 1000000){ 126 | total_packets_rx = 0; 127 | total_core_rx = port_statistics[NUM_RX_QUEUE].rx; 128 | for(i = 0;i < NUM_RX_QUEUE;++i){ 129 | total_packets_rx += port_statistics[i].rx; 130 | } 131 | last_time = rte_get_tsc_cycles(); 132 | 133 | printf("Port statistics ====================================\n"); 134 | printf("Time: %f\n", (double)t / rte_get_tsc_hz()); 135 | 136 | printf("Packets received: %lu\n", 137 | total_packets_rx - old_total_packets_rx); 138 | printf("Core Entry received: %lu\n", 139 | total_core_rx - old_total_core_rx); 140 | 141 | printf("\n\n"); 142 | fflush(stdout); 143 | 144 | old_total_packets_rx = total_packets_rx; 145 | old_total_core_rx = total_core_rx; 146 | } 147 | } 148 | } 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OctoSketch: Enabling Real-Time, Continuous Network Monitoring over Multiple Cores 2 | 3 | ## Introduction 4 | 5 | **OctoSketch** is a software monitoring framework that can scale a wide spectrum of sketches to many cores with high online accuracy and performance. In contrast to previous systems that adopt straightforward sketch merges from individual cores to obtain the aggregated result, we devise a continuous, change-based mechanism that can generally be applied to sketches to perform the aggregation. This design ensures high online accuracy of the aggregated result at any query time and reduces computation costs to achieve high throughput. We apply OctoSketch to nine representative sketches on three software platforms (CPU, DPDK, and eBPF XDP). Our results demonstrate that OctoSketch achieves about 15.6× lower errors and up to 4.5× higher throughput than the state-of-the-art. 6 | 7 | [*Our paper*](https://yindazhang.github.io/files/OctoSketch.pdf) will appear in NSDI 2024. 8 | 9 | ## About this repo 10 | 11 | - `CPU`: OctoSketch implemented on CPU 12 | - `DPDK`: OctoSketch implemented on DPDK 13 | - `XDP`: OctoSketch implemented on eBPF XDP 14 | - `common`: Common functions used by OctoSketch 15 | - `queue`: Concurrent queues used by OctoSketch 16 | - `sketch`: Sketches applied in OctoSketch 17 | - more details can be found in folders. -------------------------------------------------------------------------------- /XDP/CM/XDP_CM.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_CM_H 2 | #define XDP_CM_H 3 | 4 | #include 5 | 6 | struct Entry{ 7 | uint64_t key; 8 | uint16_t hashPos; 9 | uint16_t pos; 10 | uint8_t value; 11 | }; 12 | 13 | #endif -------------------------------------------------------------------------------- /XDP/CM/compile.sh: -------------------------------------------------------------------------------- 1 | clang -O2 -g -Wall -target bpf -c xdp.c -o xdp.o 2 | g++ main.c -o main -lbpf -lxdp 3 | 4 | sudo /local/xdp-tools/xdp-loader/xdp-loader unload -a enp65s0f0np0 -------------------------------------------------------------------------------- /XDP/CM/main.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/Abstract.h" 2 | 3 | #include "XDP_CM.h" 4 | 5 | #include "../../sketch/CM.h" 6 | 7 | /** 8 | * The aggregtor in the user space 9 | */ 10 | 11 | #define HASH_NUM 3 12 | #define LENGTH 65536 13 | #define HEAP_SIZE 0x3ff 14 | 15 | template 16 | using MyCM = CM; 17 | 18 | static uint32_t processed = 0; 19 | static int32_t old_length = 0; 20 | static int32_t threshold = 64; 21 | 22 | static int32_t out_thd_fd, out_len_fd; 23 | 24 | static MyCM* p; 25 | 26 | 27 | // Process entries in the queue and modify the threshold 28 | int handle_entry(void *ctx, void *data, size_t data_sz){ 29 | CM_Entry *e = (CM_Entry*)data; 30 | p->Merge(*e); 31 | 32 | processed += 1; 33 | 34 | if((processed & 0x3f) == 0x3f){ 35 | struct Length len; 36 | bpf_map_lookup_elem(out_len_fd, &zero, &len); 37 | 38 | int32_t new_length = len.length; 39 | int32_t gap = new_length - old_length; 40 | int32_t tmp_length = new_length + gap; 41 | int32_t new_thres = threshold; 42 | 43 | //printf("Length: %d\n", new_length); 44 | 45 | if(tmp_length < LOWER_LENGTH && gap < 32){ 46 | new_thres = threshold - 1; 47 | new_thres = MIN(0x7e, new_thres); 48 | new_thres = MAX(0x3f, new_thres); 49 | } 50 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 51 | new_thres = threshold + 1; 52 | new_thres = MIN(0x7e, new_thres); 53 | new_thres = MAX(0x3f, new_thres); 54 | } 55 | 56 | if(new_thres != threshold){ 57 | bpf_map_update_elem(out_thd_fd, &zero, &new_thres, BPF_EXIST); 58 | } 59 | 60 | //printf("Threshold: %d\n", new_thres); 61 | 62 | threshold = new_thres; 63 | old_length = new_length; 64 | } 65 | 66 | return 0; 67 | } 68 | 69 | class Ours : public Abstract{ 70 | public: 71 | 72 | int32_t merge(){ 73 | p = new MyCM(); 74 | 75 | out_len_fd = len_fd; 76 | out_thd_fd = thd_fd; 77 | 78 | struct ring_buffer *rb = ring_buffer__new(buf_fd, handle_entry, NULL, NULL); 79 | 80 | if(!rb){ 81 | printf("Error on ring buffer\n"); 82 | return -1; 83 | } 84 | 85 | // Polling entries in the queue 86 | int32_t err = 0; 87 | while(true){ 88 | err = ring_buffer__consume(rb); 89 | if (err == -EINTR) { 90 | printf("EINTR\n"); 91 | break; 92 | } 93 | if (err < 0) { 94 | printf("Error polling ring buffer: %d\n", err); 95 | break; 96 | } 97 | } 98 | 99 | delete p; 100 | return 0; 101 | } 102 | 103 | }; 104 | 105 | int main(int argc, char *argv[]){ 106 | Abstract* abs = new Ours(); 107 | abs->update(); 108 | delete abs; 109 | return 0; 110 | } 111 | -------------------------------------------------------------------------------- /XDP/CM/xdp.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/hash.h" 2 | #include "../xdp_common/parse.h" 3 | 4 | #include "XDP_CM.h" 5 | 6 | /** 7 | * The workers in the kernel 8 | */ 9 | 10 | #define HASH_NUM 3 11 | #define LENGTH 65536 12 | 13 | struct Length{ 14 | uint64_t nanoseconds; 15 | uint64_t length; 16 | }; 17 | 18 | struct{ 19 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 20 | __type(key, __u32); 21 | __type(value, uint8_t); 22 | __uint(max_entries, HASH_NUM * LENGTH); 23 | } sketch SEC(".maps"); 24 | 25 | struct { 26 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 27 | __type(key, __u32); 28 | __type(value, uint64_t); 29 | __uint(max_entries, 1); 30 | } packets SEC(".maps"); 31 | 32 | struct { 33 | __uint(type, BPF_MAP_TYPE_ARRAY); 34 | __type(key, __u32); 35 | __type(value, int32_t); 36 | __uint(max_entries, 1); 37 | } threshold SEC(".maps"); 38 | 39 | struct { 40 | __uint(type, BPF_MAP_TYPE_RINGBUF); 41 | __uint(max_entries, 64 * 1024); 42 | } buffer SEC(".maps"); 43 | 44 | struct { 45 | __uint(type, BPF_MAP_TYPE_ARRAY); 46 | __type(key, __u32); 47 | __type(value, struct Length); 48 | __uint(max_entries, 1); 49 | } buffer_length SEC(".maps"); 50 | 51 | 52 | SEC("xdp") 53 | int xdp_prog(struct xdp_md *skb) 54 | { 55 | uint64_t key; 56 | 57 | if(parse_key(skb, (struct Packet*)&key) < 0) 58 | return XDP_DROP; 59 | 60 | uint64_t *number = bpf_map_lookup_elem(&packets, &zero); 61 | if(number){ 62 | *number += 1; 63 | if(((*number) & 0xff) == 0xff){ 64 | struct Length *len = bpf_map_lookup_elem(&buffer_length, &zero); 65 | if(len){ 66 | uint64_t ns = bpf_ktime_get_ns(); 67 | if(ns > len->nanoseconds + 100000){ 68 | len->nanoseconds = ns; 69 | len->length = bpf_ringbuf_query(&buffer, BPF_RB_AVAIL_DATA); 70 | } 71 | } 72 | } 73 | } 74 | 75 | uint32_t index[HASH_NUM]; 76 | int32_t *thd = bpf_map_lookup_elem(&threshold, &zero); 77 | 78 | if(!thd){ 79 | return XDP_DROP; 80 | } 81 | 82 | for(uint32_t i = 0;i < HASH_NUM;++i){ 83 | index[i] = hash(key, seed[i]) % (uint32_t)LENGTH + i * LENGTH; 84 | } 85 | 86 | for(uint32_t i = 0;i < HASH_NUM;++i){ 87 | uint8_t* counter = bpf_map_lookup_elem(&sketch, &index[i]); 88 | if(counter){ 89 | *counter += 1; 90 | if(*counter > *thd){ 91 | struct Entry *entry = bpf_ringbuf_reserve(&buffer, sizeof(struct Entry), 0); 92 | if(entry){ 93 | entry->key = key; 94 | entry->hashPos = i; 95 | entry->pos = index[i]; 96 | entry->value = *counter; 97 | bpf_ringbuf_submit(entry, 0); 98 | *counter = 0; 99 | } 100 | } 101 | } 102 | } 103 | 104 | return XDP_DROP; 105 | } 106 | 107 | char _license[] SEC("license") = "GPL"; 108 | -------------------------------------------------------------------------------- /XDP/Coco/XDP_Coco.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_COCO_H 2 | #define XDP_COCO_H 3 | 4 | #include 5 | #include 6 | 7 | struct Entry{ 8 | uint64_t key; 9 | uint16_t pos[2]; 10 | uint8_t value; 11 | bool pos_valid; 12 | }; 13 | 14 | #endif -------------------------------------------------------------------------------- /XDP/Coco/compile.sh: -------------------------------------------------------------------------------- 1 | clang -O2 -g -Wall -target bpf -c xdp.c -o xdp.o 2 | g++ main.c -o main -lbpf -lxdp 3 | 4 | sudo /local/xdp-tools/xdp-loader/xdp-loader unload -a enp65s0f0np0 -------------------------------------------------------------------------------- /XDP/Coco/main.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/Abstract.h" 2 | 3 | #include "XDP_Coco.h" 4 | 5 | #include "../../sketch/Coco.h" 6 | 7 | #define HASH_NUM 2 8 | #define LENGTH 65536 9 | 10 | template 11 | using MyCoco = Coco; 12 | 13 | static uint32_t processed = 0; 14 | static int32_t old_length = 0; 15 | static int32_t threshold = 64; 16 | 17 | static int32_t out_thd_fd, out_len_fd; 18 | 19 | static MyCoco* p; 20 | 21 | int handle_entry(void *ctx, void *data, size_t data_sz){ 22 | Coco_Entry *e = (Coco_Entry*)data; 23 | p->Merge(*e); 24 | 25 | processed += 1; 26 | 27 | if((processed & 0x3f) == 0x3f){ 28 | struct Length len; 29 | bpf_map_lookup_elem(out_len_fd, &zero, &len); 30 | 31 | int32_t new_length = len.length; 32 | int32_t gap = new_length - old_length; 33 | int32_t tmp_length = new_length + gap; 34 | int32_t new_thres = threshold; 35 | 36 | //printf("Length: %d\n", new_length); 37 | 38 | if(tmp_length < LOWER_LENGTH && gap < 32){ 39 | new_thres = threshold - 1; 40 | new_thres = MIN(0x7e, new_thres); 41 | new_thres = MAX(0x3f, new_thres); 42 | } 43 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 44 | new_thres = threshold + 1; 45 | new_thres = MIN(0x7e, new_thres); 46 | new_thres = MAX(0x3f, new_thres); 47 | } 48 | 49 | if(new_thres != threshold){ 50 | bpf_map_update_elem(out_thd_fd, &zero, &new_thres, BPF_EXIST); 51 | } 52 | 53 | //printf("Threshold: %d\n", new_thres); 54 | 55 | threshold = new_thres; 56 | old_length = new_length; 57 | } 58 | 59 | return 0; 60 | } 61 | 62 | class Ours : public Abstract{ 63 | public: 64 | 65 | int32_t merge(){ 66 | p = new MyCoco(); 67 | 68 | out_len_fd = len_fd; 69 | out_thd_fd = thd_fd; 70 | 71 | struct ring_buffer *rb = ring_buffer__new(buf_fd, handle_entry, NULL, NULL); 72 | 73 | if(!rb){ 74 | printf("Error on ring buffer\n"); 75 | return -1; 76 | } 77 | 78 | int32_t err = 0; 79 | while(true){ 80 | err = ring_buffer__consume(rb); 81 | if (err == -EINTR) { 82 | printf("EINTR\n"); 83 | break; 84 | } 85 | if (err < 0) { 86 | printf("Error polling ring buffer: %d\n", err); 87 | break; 88 | } 89 | } 90 | 91 | delete p; 92 | return 0; 93 | } 94 | 95 | }; 96 | 97 | int main(int argc, char *argv[]){ 98 | Abstract* abs = new Ours(); 99 | abs->update(); 100 | delete abs; 101 | return 0; 102 | } 103 | -------------------------------------------------------------------------------- /XDP/Coco/xdp.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/hash.h" 2 | #include "../xdp_common/parse.h" 3 | 4 | #include "XDP_Coco.h" 5 | 6 | #define HASH_NUM 2 7 | #define LENGTH 65536 8 | 9 | struct Length{ 10 | uint64_t nanoseconds; 11 | uint64_t length; 12 | }; 13 | 14 | struct{ 15 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 16 | __type(key, __u32); 17 | __type(value, uint8_t); 18 | __uint(max_entries, HASH_NUM * LENGTH); 19 | } counters SEC(".maps"); 20 | 21 | struct{ 22 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 23 | __type(key, __u32); 24 | __type(value, uint64_t); 25 | __uint(max_entries, HASH_NUM * LENGTH); 26 | } keys SEC(".maps"); 27 | 28 | struct { 29 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 30 | __type(key, __u32); 31 | __type(value, uint64_t); 32 | __uint(max_entries, 1); 33 | } packets SEC(".maps"); 34 | 35 | struct { 36 | __uint(type, BPF_MAP_TYPE_ARRAY); 37 | __type(key, __u32); 38 | __type(value, int32_t); 39 | __uint(max_entries, 1); 40 | } threshold SEC(".maps"); 41 | 42 | struct { 43 | __uint(type, BPF_MAP_TYPE_RINGBUF); 44 | __uint(max_entries, 64 * 1024); 45 | } buffer SEC(".maps"); 46 | 47 | struct { 48 | __uint(type, BPF_MAP_TYPE_ARRAY); 49 | __type(key, __u32); 50 | __type(value, struct Length); 51 | __uint(max_entries, 1); 52 | } buffer_length SEC(".maps"); 53 | 54 | int32_t fill_entry(uint64_t _key, uint16_t _pos0, uint16_t _pos1, 55 | uint8_t _value, bool _pos_valid){ 56 | struct Entry *entry = bpf_ringbuf_reserve(&buffer, sizeof(struct Entry), 0); 57 | if(entry){ 58 | entry->key = _key; 59 | entry->pos[0] = _pos0; 60 | entry->pos[1] = _pos1; 61 | entry->value = _value; 62 | bpf_ringbuf_submit(entry, 0); 63 | } 64 | return 0; 65 | } 66 | 67 | SEC("xdp") 68 | int xdp_prog(struct xdp_md *skb) 69 | { 70 | uint64_t packet; 71 | 72 | if(parse_key(skb, (struct Packet*)&packet) < 0) 73 | return XDP_DROP; 74 | 75 | uint64_t *number = bpf_map_lookup_elem(&packets, &zero); 76 | if(number){ 77 | *number += 1; 78 | if(((*number) & 0xff) == 0xff){ 79 | struct Length *len = bpf_map_lookup_elem(&buffer_length, &zero); 80 | if(len){ 81 | uint64_t ns = bpf_ktime_get_ns(); 82 | if(ns > len->nanoseconds + 100000){ 83 | len->nanoseconds = ns; 84 | len->length = bpf_ringbuf_query(&buffer, BPF_RB_AVAIL_DATA); 85 | } 86 | } 87 | } 88 | } 89 | 90 | int32_t *thd = bpf_map_lookup_elem(&threshold, &zero); 91 | 92 | if(!thd){ 93 | return XDP_DROP; 94 | } 95 | 96 | uint32_t choice; 97 | uint32_t hashNum = hash(packet, seed[0]); 98 | uint32_t pos[2] = {hashNum >> 16, (hashNum & 0xffff) + LENGTH}; 99 | 100 | uint64_t* key[2]; 101 | uint8_t* counter[2]; 102 | 103 | key[0] = bpf_map_lookup_elem(&keys, &pos[0]); 104 | counter[0] = bpf_map_lookup_elem(&counters, &pos[0]); 105 | 106 | if(!key[0] || !counter[0]) 107 | return XDP_DROP; 108 | 109 | if(*key[0] == packet){ 110 | *counter[0] += 1; 111 | if(*counter[0] > *thd){ 112 | fill_entry(packet, pos[0], pos[1] - LENGTH, *counter[0], true); 113 | *counter[0] = 0; 114 | } 115 | return XDP_DROP; 116 | } 117 | 118 | 119 | key[1] = bpf_map_lookup_elem(&keys, &pos[1]); 120 | counter[1] = bpf_map_lookup_elem(&counters, &pos[1]); 121 | 122 | if(!key[1] || !counter[1]) 123 | return XDP_DROP; 124 | 125 | if(*key[1] == packet){ 126 | *counter[1] += 1; 127 | if(*counter[1] > *thd){ 128 | fill_entry(packet, pos[0], pos[1] - LENGTH, *counter[1], true); 129 | *counter[1] = 0; 130 | } 131 | return XDP_DROP; 132 | } 133 | 134 | choice = (*counter[0] > *counter[1]); 135 | uint64_t* tmp_key = key[choice]; 136 | uint8_t* tmp_counter = counter[choice]; 137 | 138 | *tmp_counter += 1; 139 | 140 | if(bpf_get_prandom_u32() % (*tmp_counter) == 0){ 141 | *tmp_key = packet; 142 | if(*tmp_counter > *thd){ 143 | fill_entry(packet, pos[0], pos[1] - LENGTH, *tmp_counter, true); 144 | *tmp_counter = 0; 145 | } 146 | } 147 | else{ 148 | if(*tmp_counter > *thd){ 149 | fill_entry(*tmp_key, pos[0], pos[1] - LENGTH, *tmp_counter, false); 150 | *tmp_counter = 0; 151 | } 152 | } 153 | 154 | return XDP_DROP; 155 | } 156 | 157 | char _license[] SEC("license") = "GPL"; 158 | -------------------------------------------------------------------------------- /XDP/Count/XDP_Count.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_COUNT_H 2 | #define XDP_COUNT_H 3 | 4 | #include 5 | 6 | struct Entry{ 7 | uint64_t key; 8 | uint16_t hashPos; 9 | uint16_t pos; 10 | int8_t value; 11 | }; 12 | 13 | #endif -------------------------------------------------------------------------------- /XDP/Count/compile.sh: -------------------------------------------------------------------------------- 1 | clang -O2 -g -Wall -target bpf -c xdp.c -o xdp.o 2 | g++ main.c -o main -lbpf -lxdp 3 | 4 | sudo /local/xdp-tools/xdp-loader/xdp-loader unload -a enp65s0f0np0 -------------------------------------------------------------------------------- /XDP/Count/main.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/Abstract.h" 2 | 3 | #include "XDP_Count.h" 4 | 5 | #include "../../sketch/Count.h" 6 | 7 | #define HASH_NUM 3 8 | #define LENGTH 65536 9 | #define HEAP_SIZE 0x3ff 10 | 11 | template 12 | using MyCount = Count; 13 | 14 | static uint32_t processed = 0; 15 | static int32_t old_length = 0; 16 | static int32_t threshold = 64; 17 | 18 | static int32_t out_thd_fd, out_len_fd; 19 | 20 | static MyCount* p; 21 | 22 | int handle_entry(void *ctx, void *data, size_t data_sz){ 23 | Count_Entry *e = (Count_Entry*)data; 24 | p->Merge(*e); 25 | 26 | processed += 1; 27 | 28 | if((processed & 0x3f) == 0x3f){ 29 | struct Length len; 30 | bpf_map_lookup_elem(out_len_fd, &zero, &len); 31 | 32 | int32_t new_length = len.length; 33 | int32_t gap = new_length - old_length; 34 | int32_t tmp_length = new_length + gap; 35 | int32_t new_thres = threshold; 36 | 37 | //printf("Length: %d\n", new_length); 38 | 39 | if(tmp_length < LOWER_LENGTH && gap < 32){ 40 | new_thres = threshold - 1; 41 | new_thres = MIN(0x7e, new_thres); 42 | new_thres = MAX(0x3f, new_thres); 43 | } 44 | else if(tmp_length > UPPER_LENGTH && gap > -32){ 45 | new_thres = threshold + 1; 46 | new_thres = MIN(0x7e, new_thres); 47 | new_thres = MAX(0x3f, new_thres); 48 | } 49 | 50 | if(new_thres != threshold){ 51 | bpf_map_update_elem(out_thd_fd, &zero, &new_thres, BPF_EXIST); 52 | } 53 | 54 | //printf("Threshold: %d\n", new_thres); 55 | 56 | threshold = new_thres; 57 | old_length = new_length; 58 | } 59 | 60 | return 0; 61 | } 62 | 63 | class Ours : public Abstract{ 64 | public: 65 | 66 | int32_t merge(){ 67 | p = new MyCount(); 68 | 69 | out_len_fd = len_fd; 70 | out_thd_fd = thd_fd; 71 | 72 | struct ring_buffer *rb = ring_buffer__new(buf_fd, handle_entry, NULL, NULL); 73 | 74 | if(!rb){ 75 | printf("Error on ring buffer\n"); 76 | return -1; 77 | } 78 | 79 | int32_t err = 0; 80 | while(true){ 81 | err = ring_buffer__consume(rb); 82 | if (err == -EINTR) { 83 | printf("EINTR\n"); 84 | break; 85 | } 86 | if (err < 0) { 87 | printf("Error polling ring buffer: %d\n", err); 88 | break; 89 | } 90 | } 91 | 92 | delete p; 93 | return 0; 94 | } 95 | 96 | }; 97 | 98 | int main(int argc, char *argv[]){ 99 | Abstract* abs = new Ours(); 100 | abs->update(); 101 | delete abs; 102 | return 0; 103 | } 104 | -------------------------------------------------------------------------------- /XDP/Count/xdp.c: -------------------------------------------------------------------------------- 1 | #include "../xdp_common/hash.h" 2 | #include "../xdp_common/parse.h" 3 | 4 | #include "XDP_Count.h" 5 | 6 | #define HASH_NUM 3 7 | #define LENGTH 65536 8 | 9 | struct Length{ 10 | uint64_t nanoseconds; 11 | uint64_t length; 12 | }; 13 | 14 | struct{ 15 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 16 | __type(key, __u32); 17 | __type(value, uint8_t); 18 | __uint(max_entries, HASH_NUM * LENGTH); 19 | } sketch SEC(".maps"); 20 | 21 | struct { 22 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 23 | __type(key, __u32); 24 | __type(value, uint64_t); 25 | __uint(max_entries, 1); 26 | } packets SEC(".maps"); 27 | 28 | struct { 29 | __uint(type, BPF_MAP_TYPE_ARRAY); 30 | __type(key, __u32); 31 | __type(value, int32_t); 32 | __uint(max_entries, 1); 33 | } threshold SEC(".maps"); 34 | 35 | struct { 36 | __uint(type, BPF_MAP_TYPE_RINGBUF); 37 | __uint(max_entries, 64 * 1024); 38 | } buffer SEC(".maps"); 39 | 40 | struct { 41 | __uint(type, BPF_MAP_TYPE_ARRAY); 42 | __type(key, __u32); 43 | __type(value, struct Length); 44 | __uint(max_entries, 1); 45 | } buffer_length SEC(".maps"); 46 | 47 | 48 | SEC("xdp") 49 | int xdp_prog(struct xdp_md *skb) 50 | { 51 | uint64_t packet; 52 | 53 | if(parse_key(skb, (struct Packet*)&packet) < 0) 54 | return XDP_DROP; 55 | 56 | uint64_t *number = bpf_map_lookup_elem(&packets, &zero); 57 | if(number){ 58 | *number += 1; 59 | if(((*number) & 0xff) == 0xff){ 60 | struct Length *len = bpf_map_lookup_elem(&buffer_length, &zero); 61 | if(len){ 62 | uint64_t ns = bpf_ktime_get_ns(); 63 | if(ns > len->nanoseconds + 100000){ 64 | len->nanoseconds = ns; 65 | len->length = bpf_ringbuf_query(&buffer, BPF_RB_AVAIL_DATA); 66 | } 67 | } 68 | } 69 | } 70 | 71 | int32_t *thd = bpf_map_lookup_elem(&threshold, &zero); 72 | 73 | if(!thd){ 74 | return XDP_DROP; 75 | } 76 | 77 | int32_t increment[2]; 78 | increment[0] = 1; 79 | increment[1] = -1; 80 | 81 | for(uint32_t i = 0;i < HASH_NUM;++i){ 82 | uint32_t hashNum = hash(packet, seed[i]); 83 | uint32_t index = (hashNum >> 1) % (uint32_t)LENGTH + i * LENGTH; 84 | int32_t incre = increment[hashNum & 1]; 85 | 86 | int8_t* counter = bpf_map_lookup_elem(&sketch, &index); 87 | if(counter){ 88 | *counter += incre; 89 | if((*counter) * incre > *thd){ 90 | struct Entry *entry = bpf_ringbuf_reserve(&buffer, sizeof(struct Entry), 0); 91 | if(entry){ 92 | entry->key = packet; 93 | entry->hashPos = i; 94 | entry->pos = index; 95 | entry->value = *counter; 96 | bpf_ringbuf_submit(entry, 0); 97 | *counter = 0; 98 | } 99 | } 100 | } 101 | } 102 | 103 | return XDP_DROP; 104 | } 105 | 106 | char _license[] SEC("license") = "GPL"; 107 | -------------------------------------------------------------------------------- /XDP/README.md: -------------------------------------------------------------------------------- 1 | XDP Integration 2 | ============ 3 | 4 | Environment 5 | -------------------- 6 | We have tested on 7 | - xdp-tools-1.2.9 8 | - Linux kernel version 5.15.0 9 | 10 | Repository structure 11 | -------------------- 12 | - `CM`: OctoSketch for the Count-Min Sketch 13 | - `Coco`: OctoSketch for the CocoSketch 14 | - `Count`: OctoSketch for the Count Sketch 15 | - `xdp_common`: Common functions used in OctoSketch of XDP 16 | 17 | How to run (example for the Count-Min Sketch) 18 | ------- 19 | ```bash 20 | $ cd ./CM 21 | $ sudo ./compile.sh 22 | $ sudo ./main 23 | ``` -------------------------------------------------------------------------------- /XDP/xdp_common/Abstract.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_ABSTRACT_H 2 | #define XDP_ABSTRACT_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "../../common/util.h" 17 | 18 | #define UPPER_LENGTH 0xa0 19 | #define TARGET_LENGTH 0x80 20 | #define LOWER_LENGTH 0x60 21 | 22 | struct Length{ 23 | uint64_t nanoseconds; 24 | uint64_t length; 25 | }; 26 | 27 | static const uint32_t zero = 0; 28 | static int ncpus = 0; 29 | /* This function will count the per-CPU number of packets and print out 30 | * the total number of dropped packets number and PPS (packets per second). 31 | */ 32 | void poll_stats(int fd){ 33 | ncpus = libbpf_num_possible_cpus(); 34 | uint64_t* values = new uint64_t [ncpus]; 35 | uint64_t* prev = new uint64_t [ncpus]; 36 | uint64_t sum = 0; 37 | 38 | memset(values, 0, sizeof(uint64_t) * ncpus); 39 | memset(prev, 0, sizeof(uint64_t) * ncpus); 40 | 41 | printf("CPU: %d\n", ncpus); 42 | 43 | TP last_time = now(), t; 44 | 45 | while (1) { 46 | t = now(); 47 | 48 | double seconds = durationms(t, last_time); 49 | 50 | if(seconds >= 1000000){ 51 | sum = 0; 52 | bpf_map_lookup_elem(fd, &zero, values); 53 | for (uint32_t i = 0; i < ncpus; i++){ 54 | sum += (values[i] - prev[i]); 55 | if(values[i] - prev[i] > 0){ 56 | printf("%d : %10lu pkt/s\n", i, values[i] - prev[i]); 57 | } 58 | } 59 | 60 | printf("%10lu pkt/s\n", sum); 61 | memcpy(prev, values, sizeof(uint64_t) * ncpus); 62 | last_time = t; 63 | } 64 | } 65 | 66 | delete [] values; 67 | delete [] prev; 68 | } 69 | 70 | class Abstract{ 71 | public: 72 | 73 | virtual int32_t merge() = 0; 74 | 75 | int update(){ 76 | if(xdp_load() < 0) 77 | return -1; 78 | 79 | std::thread stats; 80 | stats = std::thread(poll_stats, stats_fd); 81 | 82 | merge(); 83 | 84 | stats.join(); 85 | return 0; 86 | } 87 | 88 | int32_t xdp_load(){ 89 | int32_t ifindex = if_nametoindex(nic); 90 | if (!ifindex) { 91 | printf("get ifindex from interface name failed\n"); 92 | return -1; 93 | } 94 | 95 | /* load XDP object by libxdp */ 96 | struct bpf_object * object = bpf_object__open_file("xdp.o", NULL); 97 | if (!object) { 98 | printf("Error, open object failed\n"); 99 | return -1; 100 | } 101 | bpf_object__load(object); 102 | 103 | struct bpf_program * prog = bpf_object__find_program_by_name(object, "xdp_prog"); 104 | if (!prog) { 105 | printf("Error, find prog failed\n"); 106 | return -1; 107 | } 108 | 109 | int prog_fd = bpf_program__fd(prog); 110 | int ret = bpf_set_link_xdp_fd(ifindex, prog_fd, XDP_FLAGS_SKB_MODE); 111 | if (ret) { 112 | printf("Error, load prog failed\n"); 113 | printf("Return %d\n", ret); 114 | return -1; 115 | } 116 | 117 | /* Find the map fd from the bpf object */ 118 | stats_fd = bpf_object__find_map_fd_by_name(object, "packets"); 119 | thd_fd = bpf_object__find_map_fd_by_name(object, "threshold"); 120 | buf_fd = bpf_object__find_map_fd_by_name(object, "buffer"); 121 | len_fd = bpf_object__find_map_fd_by_name(object, "buffer_length"); 122 | 123 | if (stats_fd < 0 || thd_fd < 0 || buf_fd < 0 || len_fd < 0) { 124 | printf("Error, get stats/thd fd from bpf obj failed\n"); 125 | return -1; 126 | } 127 | 128 | int32_t threshold = 64; 129 | bpf_map_update_elem(thd_fd, &zero, &threshold, BPF_EXIST); 130 | 131 | return 0; 132 | } 133 | 134 | // TODO: You should change it based on your NIC 135 | const char nic[13] = "enp65s0f0np0"; 136 | 137 | struct bpf_object *bpf_obj; 138 | int32_t stats_fd, thd_fd, buf_fd, len_fd; 139 | }; 140 | 141 | #endif -------------------------------------------------------------------------------- /XDP/xdp_common/hash.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_HASH_H 2 | #define XDP_HASH_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #define MAX(a, b) (a > b? a:b) 9 | #define MIN(a, b) (a < b? a:b) 10 | 11 | static const uint32_t zero = 0; 12 | static const uint32_t seed[] = {8017, 3593, 1427}; 13 | static const uint32_t Prime[] = { 14 | 2654435761U,246822519U,3266489917U,668265263U,374761393U}; 15 | 16 | uint32_t rotateLeft(uint32_t x, uint8_t bits){ 17 | return (x << bits) | (x >> (32 - bits)); 18 | } 19 | 20 | uint32_t hash(uint64_t data, uint32_t seed){ 21 | uint32_t state[4] = {seed + Prime[0] + Prime[1], 22 | seed + Prime[1], seed, seed - Prime[0]}; 23 | uint32_t result = sizeof(uint64_t) + state[2] + Prime[4]; 24 | 25 | result = rotateLeft(result + (data & 0xffffffff) * Prime[2], 17) * Prime[3]; 26 | result = rotateLeft(result + (data >> 32) * Prime[2], 17) * Prime[3]; 27 | 28 | result ^= result >> 15; 29 | result *= Prime[1]; 30 | result ^= result >> 13; 31 | result *= Prime[2]; 32 | result ^= result >> 16; 33 | return result; 34 | } 35 | 36 | #endif -------------------------------------------------------------------------------- /XDP/xdp_common/parse.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_PARSE_H 2 | #define XDP_PARSE_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | struct Packet{ 14 | uint32_t src; 15 | uint32_t dst; 16 | }; 17 | 18 | inline void init_key(struct Packet* key, uint64_t src, uint64_t dst){ 19 | key->src = src; 20 | key->dst = dst; 21 | } 22 | 23 | inline int32_t parse_key(struct xdp_md *skb, struct Packet* key){ 24 | void *data_end = (void *)(long)skb->data_end; 25 | void *data = (void *)(unsigned long long)skb->data; 26 | 27 | if(data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct tcphdr) > data_end) 28 | return -1; 29 | 30 | struct iphdr *iph = data + sizeof(struct ethhdr); 31 | if(iph->protocol == IPPROTO_TCP){ 32 | struct tcphdr *h = ((void*)iph) + sizeof(struct iphdr); 33 | init_key(key, h->source, h->dest); 34 | return 0; 35 | } 36 | 37 | return -1; 38 | } 39 | 40 | #endif -------------------------------------------------------------------------------- /XDP/xdp_common/shll.h: -------------------------------------------------------------------------------- 1 | #ifndef XDP_SHLL_H 2 | #define XDP_SHLL_H 3 | 4 | #include 5 | 6 | struct Buckets{ 7 | uint8_t counter0 : 4; 8 | uint8_t counter1 : 4; 9 | }; 10 | 11 | struct SHLL{ 12 | struct Buckets buckets[8]; 13 | }; 14 | 15 | #endif -------------------------------------------------------------------------------- /common/bitmap.h: -------------------------------------------------------------------------------- 1 | #ifndef BITMAP_H 2 | #define BITMAP_H 3 | 4 | #include 5 | #include 6 | 7 | struct BitMap { 8 | 9 | uint8_t* bitset; 10 | 11 | BitMap(uint32_t length){ 12 | uint32_t size = ((length + 7) >> 3); 13 | bitset = new uint8_t[size]; 14 | memset(bitset, 0, size * sizeof(uint8_t)); 15 | } 16 | 17 | ~BitMap(){ 18 | delete [] bitset; 19 | } 20 | 21 | inline void Set(uint32_t index){ 22 | uint32_t position = (index >> 3); 23 | uint32_t offset = (index & 0x7); 24 | bitset[position] |= (1 << offset); 25 | } 26 | 27 | inline bool Get(uint32_t index){ 28 | uint32_t position = (index >> 3); 29 | uint32_t offset = (index & 0x7); 30 | return (bitset[position] & (1 << offset)); 31 | } 32 | 33 | inline void Clear(uint32_t index){ 34 | uint32_t position = (index >> 3); 35 | uint32_t offset = (index & 0x7); 36 | bitset[position] &= (~(1 << offset)); 37 | } 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /common/cuckooMap.h: -------------------------------------------------------------------------------- 1 | #ifndef CUCKOOMAP_H 2 | #define CUCKOOMAP_H 3 | 4 | #include 5 | 6 | #include "hash.h" 7 | #include "bitmap.h" 8 | 9 | #define LOAD 0.5 10 | 11 | #define CUCKOOSEED 201 12 | 13 | #define MAX_KICK 50 14 | #define ARRAY_NUM 2 15 | #define SLOT_PER_BUCKET 4 16 | 17 | 18 | template 19 | class CuckooMap{ 20 | public: 21 | struct Bucket{ 22 | KEY_TYPE keys[SLOT_PER_BUCKET]; 23 | VALUE_TYPE values[SLOT_PER_BUCKET]; 24 | }; 25 | 26 | struct Entry{ 27 | Bucket* bucket; 28 | uint32_t position; 29 | 30 | Entry(Bucket* _bucket = nullptr, uint32_t _position = -1): 31 | bucket(_bucket), position(_position){}; 32 | }; 33 | 34 | CuckooMap(uint32_t SIZE){ 35 | inserted = 0; 36 | length = SIZE / LOAD / ARRAY_NUM / SLOT_PER_BUCKET + 1; 37 | 38 | std::random_device rd; 39 | rng.seed(rd()); 40 | 41 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 42 | bitmaps[i] = new BitMap(length * SLOT_PER_BUCKET); 43 | buckets[i] = new Bucket[length]; 44 | memset(buckets[i], 0, length * sizeof(Bucket)); 45 | } 46 | } 47 | 48 | ~CuckooMap(){ 49 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 50 | delete bitmaps[i]; 51 | delete [] buckets[i]; 52 | } 53 | } 54 | 55 | static uint32_t Size2Memory(uint32_t size){ 56 | return size / LOAD * (sizeof(KEY_TYPE) + sizeof(VALUE_TYPE)); 57 | } 58 | 59 | static uint32_t Memory2Size(uint32_t memory){ 60 | return memory * LOAD / (sizeof(KEY_TYPE) + sizeof(VALUE_TYPE)); 61 | } 62 | 63 | inline uint32_t size(){ 64 | return inserted; 65 | } 66 | 67 | bool Insert(KEY_TYPE key, VALUE_TYPE value){ 68 | Entry ret; 69 | inserted += 1; 70 | uint32_t pos[ARRAY_NUM]; 71 | 72 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 73 | pos[i] = hash(key, i + CUCKOOSEED) % length; 74 | uint32_t start = pos[i] * SLOT_PER_BUCKET; 75 | for(uint32_t slot = 0;slot < SLOT_PER_BUCKET;++slot){ 76 | if(!bitmaps[i]->Get(start + slot)){ 77 | bitmaps[i]->Set(start + slot); 78 | buckets[i][pos[i]].keys[slot] = key; 79 | buckets[i][pos[i]].values[slot] = value; 80 | return true; 81 | } 82 | } 83 | } 84 | 85 | uint32_t choice = 0; 86 | for(uint32_t kick_num = 0;kick_num < MAX_KICK;++kick_num){ 87 | uint32_t slot = rng() % SLOT_PER_BUCKET; 88 | 89 | KEY_TYPE tempKey = buckets[choice][pos[choice]].keys[slot]; 90 | VALUE_TYPE tempValue = buckets[choice][pos[choice]].values[slot]; 91 | 92 | buckets[choice][pos[choice]].keys[slot] = key; 93 | buckets[choice][pos[choice]].values[slot] = value; 94 | 95 | key = tempKey; 96 | value = tempValue; 97 | 98 | choice = 1 - choice; 99 | pos[choice] = hash(key, choice + CUCKOOSEED) % length; 100 | 101 | uint32_t start = pos[choice] * SLOT_PER_BUCKET; 102 | for(slot = 0;slot < SLOT_PER_BUCKET;++slot){ 103 | if(!bitmaps[choice]->Get(start + slot)){ 104 | bitmaps[choice]->Set(start + slot); 105 | buckets[choice][pos[choice]].keys[slot] = key; 106 | buckets[choice][pos[choice]].values[slot] = value; 107 | return true; 108 | } 109 | } 110 | } 111 | 112 | std::cerr << "Hash Insert Error" << std::endl; 113 | throw; 114 | } 115 | 116 | void Replace(KEY_TYPE key, VALUE_TYPE value){ 117 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 118 | uint32_t pos = hash(key, i + CUCKOOSEED) % length; 119 | uint32_t start = pos * SLOT_PER_BUCKET; 120 | for(uint32_t slot = 0;slot < SLOT_PER_BUCKET;++slot){ 121 | if(bitmaps[i]->Get(start + slot) && 122 | buckets[i][pos].keys[slot] == key){ 123 | buckets[i][pos].values[slot] = value; 124 | return; 125 | } 126 | } 127 | } 128 | std::cerr << "Hash Replace Error" << std::endl; 129 | throw; 130 | } 131 | 132 | Entry Lookup(KEY_TYPE key){ 133 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 134 | uint32_t pos = hash(key, i + CUCKOOSEED) % length; 135 | uint32_t start = pos * SLOT_PER_BUCKET; 136 | for(uint32_t slot = 0;slot < SLOT_PER_BUCKET;++slot){ 137 | if(bitmaps[i]->Get(start + slot) && 138 | buckets[i][pos].keys[slot] == key){ 139 | return Entry((&buckets[i][pos]), slot); 140 | } 141 | } 142 | } 143 | return Entry(nullptr, 0); 144 | } 145 | 146 | VALUE_TYPE operator [] (KEY_TYPE key){ 147 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 148 | uint32_t pos = hash(key, i + CUCKOOSEED) % length; 149 | uint32_t start = pos * SLOT_PER_BUCKET; 150 | for(uint32_t slot = 0;slot < SLOT_PER_BUCKET;++slot){ 151 | if(bitmaps[i]->Get(start + slot) && 152 | buckets[i][pos].keys[slot] == key){ 153 | return buckets[i][pos].values[slot]; 154 | } 155 | } 156 | } 157 | std::cerr << "Hash Find Error" << std::endl; 158 | throw; 159 | } 160 | 161 | void Delete(KEY_TYPE key){ 162 | inserted -= 1; 163 | 164 | for(uint32_t i = 0;i < ARRAY_NUM;++i){ 165 | uint32_t pos = hash(key, i + CUCKOOSEED) % length; 166 | uint32_t start = pos * SLOT_PER_BUCKET; 167 | for(uint32_t slot = 0;slot < SLOT_PER_BUCKET;++slot){ 168 | if(bitmaps[i]->Get(start + slot) && 169 | buckets[i][pos].keys[slot] == key){ 170 | bitmaps[i]->Clear(start + slot); 171 | return; 172 | } 173 | } 174 | } 175 | std::cerr << "Hash Delete Error" << std::endl; 176 | throw; 177 | } 178 | 179 | protected: 180 | uint32_t length; 181 | uint32_t inserted; 182 | 183 | BitMap* bitmaps[ARRAY_NUM]; 184 | Bucket* buckets[ARRAY_NUM]; 185 | 186 | std::mt19937 rng; 187 | }; 188 | 189 | #endif 190 | -------------------------------------------------------------------------------- /common/heap.h: -------------------------------------------------------------------------------- 1 | #ifndef HEAP_H 2 | #define HEAP_H 3 | 4 | #include 5 | #include 6 | 7 | #include "cuckooMap.h" 8 | 9 | template 10 | class Heap{ 11 | public: 12 | 13 | struct KV{ 14 | DATA_TYPE key; 15 | COUNT_TYPE value; 16 | }; 17 | 18 | typedef CuckooMap Cuckoo; 19 | typedef std::unordered_map HashMap; 20 | typedef typename Cuckoo::Entry Entry; 21 | 22 | Heap(uint32_t _SIZE){ 23 | SIZE = _SIZE; 24 | mp = new Cuckoo(SIZE); 25 | heap = new KV[SIZE]; 26 | memset(heap, 0, sizeof(KV) * SIZE); 27 | } 28 | 29 | ~Heap(){ 30 | delete mp; 31 | delete [] heap; 32 | } 33 | 34 | static uint32_t Size2Memory(uint32_t size){ 35 | return size * ((sizeof(DATA_TYPE) + sizeof(uint32_t)) / LOAD 36 | + sizeof(DATA_TYPE) + sizeof(COUNT_TYPE)); 37 | } 38 | 39 | static uint32_t Memory2Size(uint32_t memory){ 40 | return memory / ((sizeof(DATA_TYPE) + sizeof(uint32_t)) / LOAD 41 | + sizeof(DATA_TYPE) + sizeof(COUNT_TYPE)); 42 | } 43 | 44 | inline COUNT_TYPE min(){ 45 | return heap[0].value; 46 | } 47 | 48 | void Insert(const DATA_TYPE item, const COUNT_TYPE frequency){ 49 | if(this->isFull()){ 50 | if(frequency > heap[0].value){ 51 | Entry temp = mp->Lookup(item); 52 | if(temp.bucket != nullptr){ 53 | uint32_t pos = temp.bucket->values[temp.position]; 54 | heap[pos].value = frequency; 55 | Heap_Down(temp, pos); 56 | } 57 | else{ 58 | mp->Delete(heap[0].key); 59 | heap[0].value = frequency; 60 | heap[0].key = item; 61 | mp->Insert(item, 0); 62 | Entry temp = mp->Lookup(item); 63 | this->Heap_Down(temp, 0); 64 | } 65 | } 66 | return; 67 | } 68 | 69 | Entry temp = mp->Lookup(item); 70 | if(temp.bucket != nullptr){ 71 | uint32_t pos = temp.bucket->values[temp.position]; 72 | heap[pos].value = frequency; 73 | Heap_Down(temp, pos); 74 | } 75 | else{ 76 | uint32_t pos = mp->size(); 77 | heap[pos].value = frequency; 78 | heap[pos].key = item; 79 | mp->Insert(item, pos); 80 | Heap_Up(pos); 81 | } 82 | } 83 | 84 | //COUNT_TYPE Query(const DATA_TYPE& item){ 85 | // return mp->Lookup(item)? heap[(*mp)[item]].value : 0; 86 | //} 87 | 88 | HashMap AllQuery(){ 89 | HashMap ret; 90 | uint32_t size = mp->size(); 91 | for(uint32_t i = 0;i < size;++i){ 92 | ret[heap[i].key] = heap[i].value; 93 | } 94 | return ret; 95 | } 96 | 97 | //protected: 98 | uint32_t SIZE; 99 | Cuckoo* mp; 100 | KV* heap; 101 | 102 | inline bool isFull(){ 103 | return mp->size() >= SIZE; 104 | } 105 | 106 | void Heap_Down(Entry temp, uint32_t pos){ 107 | uint32_t upper = mp->size(); 108 | 109 | while (pos < upper / 2) { 110 | uint32_t left = 2 * pos + 1, right = 2 * pos + 2; 111 | uint32_t replace = pos; 112 | 113 | if (left < upper && heap[left].value < heap[replace].value) 114 | replace = left; 115 | if (right < upper && heap[right].value< heap[replace].value) 116 | replace = right; 117 | 118 | if (replace != pos) { 119 | temp.bucket->values[temp.position] = replace; 120 | KV temp = heap[pos]; 121 | heap[pos] = heap[replace]; 122 | heap[replace] = temp; 123 | mp->Replace(heap[pos].key, pos); 124 | pos = replace; 125 | } 126 | else return; 127 | } 128 | } 129 | 130 | void Heap_Up(uint32_t pos) { 131 | while (pos > 1) { 132 | uint32_t parent = (pos - 1) / 2; 133 | if (heap[parent].value <= heap[pos].value) 134 | break; 135 | 136 | KV temp = heap[pos]; 137 | heap[pos] = heap[parent]; 138 | heap[parent] = temp; 139 | mp->Replace(heap[pos].key, pos); 140 | mp->Replace(heap[parent].key, parent); 141 | pos = parent; 142 | } 143 | } 144 | }; 145 | 146 | #endif 147 | -------------------------------------------------------------------------------- /common/loader.h: -------------------------------------------------------------------------------- 1 | #ifndef MMAP_H 2 | #define MMAP_H 3 | 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | /** 11 | * Use MMap to load binary dataset 12 | */ 13 | 14 | struct LoadResult{ 15 | void* start; 16 | uint64_t length; 17 | }; 18 | 19 | LoadResult Load(const char* PATH); 20 | void UnLoad(LoadResult result); 21 | 22 | #include 23 | #include 24 | 25 | LoadResult Load(const char* PATH){ 26 | LoadResult ret; 27 | 28 | int32_t fd = open(PATH, O_RDONLY); 29 | if(fd == -1) { 30 | std::cerr << "Cannot open " << PATH << std::endl; 31 | throw; 32 | } 33 | 34 | struct stat sb; 35 | if(fstat(fd, &sb) == -1){ 36 | std::cerr << "Fstat Error" << std::endl; 37 | throw; 38 | } 39 | 40 | ret.length = sb.st_size; 41 | ret.start = mmap(nullptr, ret.length, PROT_READ, MAP_PRIVATE, fd, 0u); 42 | 43 | if (ret.start == MAP_FAILED) { 44 | std::cerr << "Cannot mmap " << PATH << " of length " << ret.length << std::endl; 45 | throw; 46 | } 47 | 48 | return ret; 49 | } 50 | 51 | void UnLoad(LoadResult result){ 52 | munmap(result.start, result.length); 53 | } 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /common/shll.h: -------------------------------------------------------------------------------- 1 | #ifndef SHLL_H 2 | #define SHLL_H 3 | 4 | #include "util.h" 5 | #include "hash.h" 6 | #include 7 | 8 | /** 9 | * Bucket used in HyperLogLog-related data structures 10 | */ 11 | 12 | struct Buckets{ 13 | uint8_t counter0 : 4; 14 | uint8_t counter1 : 4; 15 | 16 | void insert(uint32_t index, uint8_t rank){ 17 | switch(index){ 18 | case 0: 19 | if(counter0 < rank){ 20 | counter0 = rank; 21 | } 22 | return; 23 | case 1: 24 | if(counter1 < rank){ 25 | counter1 = rank; 26 | } 27 | return; 28 | } 29 | } 30 | 31 | double query(){ 32 | return 1.0 / (1 << counter0) + 1.0 / (1 << counter1); 33 | } 34 | 35 | void merge(Buckets& other){ 36 | counter0 = MAX(counter0, other.counter0); 37 | counter1 = MAX(counter1, other.counter1); 38 | } 39 | }; 40 | 41 | struct SHLL{ 42 | 43 | template 44 | void Insert(Key key, uint32_t seed = 0){ 45 | uint32_t temp = hash(key, seed); 46 | uint32_t inbucket_index = (temp & 0x1); 47 | uint32_t bucket_index = ((temp >> 1) & 0x7); 48 | uint8_t rank = MIN(15, __builtin_clz(temp) + 1); 49 | 50 | buckets[bucket_index].insert(inbucket_index, rank); 51 | } 52 | 53 | double Query(){ 54 | double sum = 0; 55 | for(uint32_t i = 0;i < 8;++i){ 56 | sum += buckets[i].query(); 57 | } 58 | return 0.673 * 256 / sum; 59 | } 60 | 61 | void Merge(SHLL& other){ 62 | for(uint32_t i = 0;i < 8;++i){ 63 | buckets[i].merge(other.buckets[i]); 64 | } 65 | } 66 | 67 | Buckets buckets[8]; 68 | }; 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /common/util.h: -------------------------------------------------------------------------------- 1 | #ifndef UTIL_H 2 | #define UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define MAX(a, b) (a > b? a:b) 11 | #define MIN(a, b) (a < b? a:b) 12 | 13 | typedef std::chrono::high_resolution_clock::time_point TP; 14 | 15 | inline TP now(){ 16 | return std::chrono::high_resolution_clock::now(); 17 | } 18 | 19 | inline double durationms(TP finish, TP start){ 20 | return std::chrono::duration_cast>>(finish - start).count(); 21 | } 22 | 23 | inline double durationns(TP finish, TP start){ 24 | return std::chrono::duration_cast(finish - start).count(); 25 | } 26 | 27 | template 28 | T MEDIAN3(T array[3]){ 29 | if(array[0] < array[1]){ 30 | if(array[2] < array[0]){ 31 | return array[0]; 32 | } 33 | else if(array[2] < array[1]){ 34 | return array[2]; 35 | } 36 | else{ 37 | return array[1]; 38 | } 39 | } 40 | else{ 41 | if(array[2] < array[1]){ 42 | return array[1]; 43 | } 44 | else if(array[2] < array[0]){ 45 | return array[2]; 46 | } 47 | else{ 48 | return array[0]; 49 | } 50 | } 51 | } 52 | 53 | #ifdef __linux__ 54 | static bool setaffinity(std::thread* thd, uint32_t coreId){ 55 | cpu_set_t cpuset; 56 | CPU_ZERO(&cpuset); 57 | CPU_SET(coreId, &cpuset); 58 | int rc = pthread_setaffinity_np(thd->native_handle(), 59 | sizeof(cpu_set_t), &cpuset); 60 | if (rc != 0) { 61 | std::cerr << "Error calling pthread_setaffinity_np: " << rc << "\n"; 62 | return false; 63 | } 64 | return true; 65 | } 66 | #endif 67 | 68 | static std::mutex mtx; 69 | 70 | static void CDF(uint32_t* array, uint32_t length){ 71 | double sum = 0, preSum = 0; 72 | 73 | for(uint32_t i = 0;i < length;++i){ 74 | sum += array[i]; 75 | } 76 | 77 | mtx.lock(); 78 | for(uint32_t i = 0;i < length;++i){ 79 | preSum += array[i]; 80 | std::cout << preSum / sum << ","; 81 | } 82 | 83 | std::cout << std::endl << std::endl; 84 | mtx.unlock(); 85 | } 86 | 87 | inline static void Rank(double* array, uint32_t length){ 88 | std::sort(array, array + length); 89 | 90 | mtx.lock(); 91 | for(uint32_t i = 1;i < 1000;++i){ 92 | std::cout << array[uint32_t(0.001 * i * length)] << ","; 93 | } 94 | 95 | std::cout << std::endl << std::endl; 96 | mtx.unlock(); 97 | } 98 | 99 | #endif 100 | -------------------------------------------------------------------------------- /queue/atomicops.h: -------------------------------------------------------------------------------- 1 | #ifndef ATOMICOPS_H 2 | #define ATOMICOPS_H 3 | 4 | // ©2013-2016 Cameron Desrochers. 5 | // Distributed under the simplified BSD license (see the license file that 6 | // should have come with this header). 7 | // Uses Jeff Preshing's semaphore implementation (under the terms of its 8 | // separate zlib license, embedded below). 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | // AE_UNUSED 21 | #define AE_UNUSED(x) ((void)x) 22 | 23 | // AE_NO_TSAN/AE_TSAN_ANNOTATE_* 24 | #if defined(__has_feature) 25 | #if __has_feature(thread_sanitizer) 26 | #if __cplusplus >= 201703L // inline variables require C++17 27 | namespace moodycamel { inline int ae_tsan_global; } 28 | #define AE_TSAN_ANNOTATE_RELEASE() AnnotateHappensBefore(__FILE__, __LINE__, (void *)(&::moodycamel::ae_tsan_global)) 29 | #define AE_TSAN_ANNOTATE_ACQUIRE() AnnotateHappensAfter(__FILE__, __LINE__, (void *)(&::moodycamel::ae_tsan_global)) 30 | extern "C" void AnnotateHappensBefore(const char*, int, void*); 31 | extern "C" void AnnotateHappensAfter(const char*, int, void*); 32 | #else // when we can't work with tsan, attempt to disable its warnings 33 | #define AE_NO_TSAN __attribute__((no_sanitize("thread"))) 34 | #endif 35 | #endif 36 | #endif 37 | #ifndef AE_NO_TSAN 38 | #define AE_NO_TSAN 39 | #endif 40 | #ifndef AE_TSAN_ANNOTATE_RELEASE 41 | #define AE_TSAN_ANNOTATE_RELEASE() 42 | #define AE_TSAN_ANNOTATE_ACQUIRE() 43 | #endif 44 | 45 | 46 | 47 | #define AE_FORCEINLINE inline 48 | #define AE_ALIGN(x) __attribute__((aligned(x))) 49 | 50 | 51 | #include 52 | #include 53 | 54 | // WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY: 55 | // Provides basic support for atomic variables -- no memory ordering guarantees are provided. 56 | // The guarantee of atomicity is only made for types that already have atomic load and store guarantees 57 | // at the hardware level -- on most platforms this generally means aligned pointers and integers (only). 58 | 59 | // Portable atomic fences implemented below: 60 | 61 | enum memory_order { 62 | memory_order_relaxed, 63 | memory_order_acquire, 64 | memory_order_release, 65 | memory_order_acq_rel, 66 | memory_order_seq_cst, 67 | 68 | // memory_order_sync: Forces a full sync: 69 | // #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad 70 | memory_order_sync = memory_order_seq_cst 71 | }; 72 | 73 | AE_FORCEINLINE void compiler_fence(memory_order order) AE_NO_TSAN 74 | { 75 | switch (order) { 76 | case memory_order_relaxed: break; 77 | case memory_order_acquire: std::atomic_signal_fence(std::memory_order_acquire); break; 78 | case memory_order_release: std::atomic_signal_fence(std::memory_order_release); break; 79 | case memory_order_acq_rel: std::atomic_signal_fence(std::memory_order_acq_rel); break; 80 | case memory_order_seq_cst: std::atomic_signal_fence(std::memory_order_seq_cst); break; 81 | default: assert(false); 82 | } 83 | } 84 | 85 | AE_FORCEINLINE void fence(memory_order order) AE_NO_TSAN 86 | { 87 | switch (order) { 88 | case memory_order_relaxed: break; 89 | case memory_order_acquire: AE_TSAN_ANNOTATE_ACQUIRE(); std::atomic_thread_fence(std::memory_order_acquire); break; 90 | case memory_order_release: AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_release); break; 91 | case memory_order_acq_rel: AE_TSAN_ANNOTATE_ACQUIRE(); AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_acq_rel); break; 92 | case memory_order_seq_cst: AE_TSAN_ANNOTATE_ACQUIRE(); AE_TSAN_ANNOTATE_RELEASE(); std::atomic_thread_fence(std::memory_order_seq_cst); break; 93 | default: assert(false); 94 | } 95 | } 96 | 97 | template 98 | class weak_atomic 99 | { 100 | public: 101 | AE_NO_TSAN weak_atomic() : value() { } 102 | 103 | template AE_NO_TSAN weak_atomic(U&& x) : value(std::forward(x)) { } 104 | 105 | AE_NO_TSAN weak_atomic(weak_atomic const& other) : value(other.load()) { } 106 | AE_NO_TSAN weak_atomic(weak_atomic&& other) : value(std::move(other.load())) { } 107 | 108 | AE_FORCEINLINE operator T() const AE_NO_TSAN { return load(); } 109 | 110 | template 111 | AE_FORCEINLINE weak_atomic const& operator=(U&& x) AE_NO_TSAN 112 | { 113 | value.store(std::forward(x), std::memory_order_relaxed); 114 | return *this; 115 | } 116 | 117 | AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) AE_NO_TSAN 118 | { 119 | value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed); 120 | return *this; 121 | } 122 | 123 | AE_FORCEINLINE T load() const AE_NO_TSAN { return value.load(std::memory_order_relaxed); } 124 | 125 | private: 126 | std::atomic value; 127 | }; 128 | 129 | #endif 130 | 131 | -------------------------------------------------------------------------------- /sketch/CM.h: -------------------------------------------------------------------------------- 1 | #ifndef CM_H 2 | #define CM_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef uint8_t Value; 7 | 8 | template 9 | struct CM_Entry{ 10 | Key key; 11 | uint16_t hashPos; 12 | uint16_t pos; 13 | Value value; 14 | 15 | CM_Entry(Key _key = 0, uint16_t _hashPos = 0, uint16_t _pos = 0, Value _value = 0): 16 | key(_key), hashPos(_hashPos), pos(_pos), value(_value){}; 17 | }; 18 | 19 | template 20 | class Child_CM : public Sketch{ 21 | public: 22 | 23 | Value sketch[_HASH_NUM][_LENGTH]; 24 | 25 | Child_CM(){ 26 | memset(sketch, 0, sizeof(Value) * _HASH_NUM * _LENGTH); 27 | } 28 | 29 | ~Child_CM(){} 30 | 31 | void insert_one(const Key& packet){} 32 | 33 | HashMap query_all(){ 34 | HashMap ret; 35 | return ret; 36 | } 37 | }; 38 | 39 | template 40 | class CM : public Sketch{ 41 | public: 42 | typedef Heap myHeap; 43 | 44 | int32_t sketch[_HASH_NUM][_LENGTH]; 45 | myHeap* heap; 46 | 47 | CM(){ 48 | memset(sketch, 0, sizeof(int32_t) * _HASH_NUM * _LENGTH); 49 | heap = new myHeap(_HEAP_SIZE); 50 | } 51 | 52 | ~CM(){ 53 | delete heap; 54 | } 55 | 56 | void insert_one(const Key& packet){ 57 | int32_t minimum = 0x7fffffff; 58 | 59 | for(uint32_t hashPos = 0;hashPos < _HASH_NUM;++hashPos){ 60 | uint32_t pos = hash(packet, hashPos) % _LENGTH; 61 | sketch[hashPos][pos] += 1; 62 | minimum = MIN(minimum, sketch[hashPos][pos]); 63 | } 64 | 65 | heap->Insert(packet, minimum); 66 | } 67 | 68 | HashMap query_all(){ 69 | return heap->AllQuery(); 70 | } 71 | 72 | void Merge(CM* temp){ 73 | for(uint32_t i = 0;i < _HASH_NUM;++i){ 74 | for(uint32_t j = 0;j < _LENGTH;++j){ 75 | sketch[i][j] += temp->sketch[i][j]; 76 | } 77 | } 78 | myHeap* check[2] = {heap, temp->heap}; 79 | 80 | for(auto p : check){ 81 | for(uint32_t i = 0;i < p->mp->size();++i){ 82 | int32_t minimum = 0x7fffffff; 83 | for(uint32_t hashPos = 0;hashPos < _HASH_NUM;++hashPos){ 84 | uint32_t pos = hash(p->heap[i].key, hashPos) % _LENGTH; 85 | minimum = MIN(minimum, sketch[hashPos][pos]); 86 | } 87 | heap->Insert(p->heap[i].key, minimum); 88 | } 89 | } 90 | 91 | delete temp; 92 | } 93 | 94 | void Merge(const CM_Entry& temp){ 95 | const uint16_t& hashPos = temp.hashPos; 96 | const uint16_t& pos = temp.pos; 97 | 98 | sketch[hashPos][pos] += temp.value; 99 | 100 | if(sketch[hashPos][pos] > heap->min()){ 101 | int32_t minimum = sketch[hashPos][pos]; 102 | for(uint32_t tempHash = 0;tempHash < _HASH_NUM;++tempHash){ 103 | uint32_t tempPos = hash(temp.key, tempHash) % _LENGTH; 104 | minimum = MIN(minimum, sketch[tempHash][tempPos]); 105 | } 106 | heap->Insert(temp.key, minimum); 107 | } 108 | } 109 | }; 110 | 111 | #endif -------------------------------------------------------------------------------- /sketch/Coco.h: -------------------------------------------------------------------------------- 1 | #ifndef COCO_H 2 | #define COCO_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef uint8_t Value; 7 | 8 | template 9 | struct Coco_Entry{ 10 | Key key; 11 | uint16_t pos[2]; 12 | Value value; 13 | bool pos_valid; 14 | 15 | Coco_Entry(Key _key = 0, uint16_t _pos0 = 0, uint16_t _pos1 = 0, 16 | Value _value = 0, uint16_t _pos_valid = true): 17 | key(_key), value(_value), pos_valid(_pos_valid){ 18 | pos[0] = _pos0; 19 | pos[1] = _pos1; 20 | }; 21 | }; 22 | 23 | template 24 | class Child_Coco : public Sketch{ 25 | public: 26 | 27 | Key keys[HASH_NUM][LENGTH]; 28 | Value counters[HASH_NUM][LENGTH]; 29 | 30 | std::mt19937 rng; 31 | 32 | Child_Coco(){ 33 | memset(keys, 0, sizeof(Key) * HASH_NUM * LENGTH); 34 | memset(counters, 0, sizeof(Value) * HASH_NUM * LENGTH); 35 | 36 | std::random_device rd; 37 | rng.seed(rd()); 38 | } 39 | 40 | ~Child_Coco(){} 41 | 42 | void insert_one(const Key& packet){} 43 | 44 | HashMap query_all(){ 45 | HashMap ret; 46 | return ret; 47 | } 48 | }; 49 | 50 | template 51 | class Coco : public Sketch{ 52 | public: 53 | 54 | Key keys[HASH_NUM][LENGTH]; 55 | int32_t counters[HASH_NUM][LENGTH]; 56 | 57 | std::mt19937 rng; 58 | 59 | Coco(){ 60 | memset(keys, 0, sizeof(Key) * HASH_NUM * LENGTH); 61 | memset(counters, 0, sizeof(int32_t) * HASH_NUM * LENGTH); 62 | 63 | std::random_device rd; 64 | rng.seed(rd()); 65 | } 66 | 67 | ~Coco(){} 68 | 69 | void insert_one(const Key& packet){ 70 | uint32_t choice; 71 | uint16_t pos[2]; 72 | 73 | *((uint32_t*)pos) = hash(packet, 0); 74 | 75 | if(keys[0][pos[0]] == packet){ 76 | counters[0][pos[0]] += 1; 77 | return; 78 | } 79 | 80 | if(keys[1][pos[1]] == packet){ 81 | counters[1][pos[1]] += 1; 82 | return; 83 | } 84 | 85 | choice = (counters[0][pos[0]] > counters[1][pos[1]]); 86 | counters[choice][pos[choice]] += 1; 87 | if(rng() % counters[choice][pos[choice]] == 0){ 88 | keys[choice][pos[choice]] = packet; 89 | } 90 | } 91 | 92 | HashMap query_all(){ 93 | HashMap ret; 94 | 95 | for(uint32_t i = 0;i < HASH_NUM;++i){ 96 | for(uint32_t j = 0;j < LENGTH;++j){ 97 | ret[keys[i][j]] = counters[i][j]; 98 | } 99 | } 100 | 101 | return ret; 102 | } 103 | 104 | void Merge(Coco* temp){ 105 | for(uint32_t i = 0;i < HASH_NUM;++i){ 106 | for(uint32_t j = 0;j < LENGTH;++j){ 107 | counters[i][j] += temp->counters[i][j]; 108 | if(counters[i][j] == 0 || rng() % counters[i][j] < temp->counters[i][j]){ 109 | keys[i][j] = temp->keys[i][j]; 110 | } 111 | } 112 | } 113 | 114 | delete temp; 115 | } 116 | 117 | void Merge(Coco_Entry temp){ 118 | if(!temp.pos_valid){ 119 | *((uint32_t*)temp.pos) = hash(temp.key, 0); 120 | } 121 | 122 | if(keys[0][temp.pos[0]] == temp.key){ 123 | counters[0][temp.pos[0]] += temp.value; 124 | return; 125 | } 126 | 127 | if(keys[1][temp.pos[1]] == temp.key){ 128 | counters[1][temp.pos[1]] += temp.value; 129 | return; 130 | } 131 | 132 | uint32_t choice = (counters[0][temp.pos[0]] > counters[1][temp.pos[1]]); 133 | counters[choice][temp.pos[choice]] += temp.value; 134 | if(rng() % counters[choice][temp.pos[choice]] < temp.value){ 135 | keys[choice][temp.pos[choice]] = temp.key; 136 | } 137 | } 138 | 139 | }; 140 | 141 | #endif -------------------------------------------------------------------------------- /sketch/Count.h: -------------------------------------------------------------------------------- 1 | #ifndef COUNT_H 2 | #define COUNT_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef int8_t Value; 7 | 8 | constexpr static int32_t increment[2] = {1, -1}; 9 | 10 | template 11 | struct Count_Entry{ 12 | Key key; 13 | uint16_t hashPos; 14 | uint16_t pos; 15 | Value value; 16 | 17 | Count_Entry(Key _key = 0, uint16_t _hashPos = 0, uint16_t _pos = 0, Value _value = 0): 18 | key(_key), hashPos(_hashPos), pos(_pos), value(_value){}; 19 | }; 20 | 21 | template 22 | class Child_Count : public Sketch{ 23 | public: 24 | 25 | Value sketch[HASH_NUM][LENGTH]; 26 | 27 | Child_Count(){ 28 | memset(sketch, 0, sizeof(Value) * HASH_NUM * LENGTH); 29 | } 30 | 31 | ~Child_Count(){} 32 | 33 | void insert_one(const Key& packet){} 34 | 35 | HashMap query_all(){ 36 | HashMap ret; 37 | return ret; 38 | } 39 | }; 40 | 41 | template 42 | class Count : public Sketch{ 43 | public: 44 | typedef Heap myHeap; 45 | 46 | int32_t sketch[HASH_NUM][LENGTH]; 47 | myHeap* heap; 48 | 49 | Count(){ 50 | memset(sketch, 0, sizeof(int32_t) * HASH_NUM * LENGTH); 51 | heap = new myHeap(HEAP_SIZE); 52 | } 53 | 54 | ~Count(){ 55 | delete heap; 56 | } 57 | 58 | void insert_one(const Key& packet){ 59 | int32_t number[HASH_NUM] = {0}; 60 | 61 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 62 | uint32_t hashNum = hash(packet, hashPos); 63 | uint32_t pos = (hashNum >> 1) % LENGTH; 64 | int32_t incre = increment[hashNum & 1]; 65 | 66 | sketch[hashPos][pos] += incre; 67 | number[hashPos] = sketch[hashPos][pos] * incre; 68 | } 69 | 70 | heap->Insert(packet, MEDIAN3(number)); 71 | } 72 | 73 | HashMap query_all(){ 74 | return heap->AllQuery(); 75 | } 76 | 77 | void Merge(Count* temp){ 78 | for(uint32_t i = 0;i < HASH_NUM;++i){ 79 | for(uint32_t j = 0;j < LENGTH;++j){ 80 | sketch[i][j] += temp->sketch[i][j]; 81 | } 82 | } 83 | 84 | myHeap* check[2] = {heap, temp->heap}; 85 | 86 | for(auto p : check){ 87 | for(uint32_t i = 0;i < p->mp->size();++i){ 88 | int32_t count[HASH_NUM] = {0}; 89 | for (uint32_t hashPos = 0; hashPos < HASH_NUM; ++hashPos) { 90 | uint32_t hashNum = hash(p->heap[i].key, hashPos); 91 | uint32_t pos = (hashNum >> 1) % LENGTH; 92 | int32_t incre = increment[hashNum & 1]; 93 | count[hashPos] = sketch[hashPos][pos] * incre; 94 | } 95 | heap->Insert(p->heap[i].key, MEDIAN3(count)); 96 | } 97 | } 98 | 99 | delete temp; 100 | } 101 | 102 | void Merge(const Count_Entry& temp){ 103 | const uint16_t& hashPos = temp.hashPos; 104 | const uint16_t& pos = temp.pos; 105 | 106 | sketch[hashPos][pos] += temp.value; 107 | 108 | if(abs(sketch[hashPos][pos]) > heap->min()){ 109 | int32_t count[HASH_NUM] = {0}; 110 | 111 | for(uint32_t tempHash = 0;tempHash < HASH_NUM;++tempHash){ 112 | uint32_t hashNum = hash(temp.key, tempHash); 113 | uint32_t tempPos = (hashNum >> 1) % LENGTH; 114 | int32_t incre = increment[hashNum & 1]; 115 | 116 | count[tempHash] = sketch[tempHash][tempPos] * incre; 117 | } 118 | heap->Insert(temp.key, MEDIAN3(count)); 119 | } 120 | } 121 | }; 122 | 123 | #endif -------------------------------------------------------------------------------- /sketch/DD.h: -------------------------------------------------------------------------------- 1 | #ifndef DD_H 2 | #define DD_H 3 | 4 | #include "sketch.h" 5 | #include 6 | 7 | typedef uint8_t Value; 8 | 9 | template 10 | struct DD_Entry{ 11 | uint16_t pos; 12 | Value value; 13 | 14 | DD_Entry(uint16_t _pos = 0, Value _value = 0): 15 | pos(_pos), value(_value){}; 16 | }; 17 | 18 | template 19 | class Child_DD : public Sketch { 20 | public: 21 | 22 | Value sketch[LENGTH]; 23 | 24 | Child_DD(){ 25 | memset(sketch, 0, sizeof(Value) * LENGTH); 26 | } 27 | 28 | ~Child_DD(){} 29 | 30 | void insert_one(const Key& packet){} 31 | 32 | HashMap query_all(){ 33 | HashMap ret; 34 | return ret; 35 | } 36 | 37 | }; 38 | 39 | #define GAMMA 1.001 40 | 41 | template 42 | class DD : public Sketch{ 43 | public: 44 | 45 | int32_t sketch[LENGTH]; 46 | 47 | DD(){ 48 | memset(sketch, 0, sizeof(int32_t) * LENGTH); 49 | } 50 | 51 | ~DD(){} 52 | 53 | void insert_one(const Key& packet){ 54 | uint32_t pos = log(packet) / log(GAMMA); 55 | if (pos >= LENGTH) { 56 | pos = LENGTH - 1; 57 | } 58 | sketch[pos] += 1; // if cannot fit, put in the last bucket 59 | } 60 | 61 | HashMap query_all(){ 62 | HashMap ret; 63 | 64 | double sum = 0; 65 | for(uint32_t i = 0;i < LENGTH;++i){ 66 | sum += sketch[i]; 67 | } 68 | 69 | uint32_t index = 0; 70 | double preSum = 0; 71 | for(uint32_t i = 0; i < 1000; ++i){ 72 | while(preSum <= i * 0.001 * sum){ 73 | preSum += sketch[index]; 74 | index += 1; 75 | } 76 | ret[i] = 2 * pow(GAMMA, index) / (1 + GAMMA); 77 | } 78 | 79 | return ret; 80 | } 81 | 82 | void Merge(DD* temp){ 83 | for(uint32_t i = 0;i < LENGTH;++i){ 84 | sketch[i] += temp->sketch[i]; 85 | } 86 | delete temp; 87 | } 88 | 89 | void Merge(const DD_Entry& temp){ 90 | const uint16_t& pos = temp.pos; 91 | sketch[pos] += temp.value; 92 | } 93 | }; 94 | 95 | #endif -------------------------------------------------------------------------------- /sketch/Elastic.h: -------------------------------------------------------------------------------- 1 | #ifndef ELASTIC_H 2 | #define ELASTIC_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef uint8_t Value; 7 | 8 | template 9 | struct Elastic_Entry{ 10 | Key key; 11 | uint32_t pos; 12 | Value value; 13 | bool bucket; 14 | 15 | Elastic_Entry(Key _key = 0, uint32_t _pos = 0, Value _value = 0, bool _bucket = false): 16 | key(_key), pos(_pos), value(_value), bucket(_bucket){}; 17 | }; 18 | 19 | template 20 | class Child_Elastic : public Sketch{ 21 | public: 22 | 23 | struct Bucket{ 24 | int32_t vote; 25 | Key ID[COUNTER_PER_BUCKET]; 26 | Value count[COUNTER_PER_BUCKET]; 27 | }; 28 | 29 | Value sketch[SKETCH_LENGTH]; 30 | Bucket buckets[BUCKET_LENGTH]; 31 | 32 | Child_Elastic(){ 33 | memset(sketch, 0, sizeof(Value) * SKETCH_LENGTH); 34 | memset(buckets, 0, sizeof(Bucket) * BUCKET_LENGTH); 35 | } 36 | 37 | ~Child_Elastic(){} 38 | 39 | void insert_one(const Key& packet){} 40 | 41 | HashMap query_all(){ 42 | HashMap ret; 43 | return ret; 44 | } 45 | }; 46 | 47 | template 48 | class Elastic : public Sketch{ 49 | public: 50 | 51 | struct Bucket{ 52 | int32_t vote; 53 | Key ID[COUNTER_PER_BUCKET]; 54 | int32_t count[COUNTER_PER_BUCKET]; 55 | }; 56 | 57 | int32_t sketch[SKETCH_LENGTH]; 58 | Bucket buckets[BUCKET_LENGTH]; 59 | 60 | Elastic(){ 61 | memset(sketch, 0, sizeof(int32_t) * SKETCH_LENGTH); 62 | memset(buckets, 0, sizeof(Bucket) * BUCKET_LENGTH); 63 | } 64 | 65 | ~Elastic(){} 66 | 67 | void insert_one(const Key& packet){ 68 | uint32_t pos = hash(packet) % BUCKET_LENGTH, minPos = 0; 69 | int32_t minVal = 0x7fffffff; 70 | 71 | for (uint32_t j = 0; j < COUNTER_PER_BUCKET; j++){ 72 | if(buckets[pos].ID[j] == packet) { 73 | buckets[pos].count[j] += 1; 74 | return; 75 | } 76 | 77 | if(buckets[pos].count[j] == 0){ 78 | buckets[pos].ID[j] = packet; 79 | buckets[pos].count[j] = 1; 80 | return; 81 | } 82 | 83 | if(buckets[pos].count[j] < minVal){ 84 | minPos = j; 85 | minVal = buckets[pos].count[j]; 86 | } 87 | } 88 | 89 | if((buckets[pos].vote + 1) >= minVal * 8){ 90 | buckets[pos].vote = 0; 91 | 92 | uint32_t position = hash(buckets[pos].ID[minPos], 101) % SKETCH_LENGTH; 93 | sketch[position] = sketch[position] + buckets[pos].count[minPos]; 94 | 95 | buckets[pos].ID[minPos] = packet; 96 | buckets[pos].count[minPos] = 1; 97 | } 98 | else { 99 | buckets[pos].vote += 1; 100 | uint32_t position = hash(packet, 101) % SKETCH_LENGTH; 101 | sketch[position] = sketch[position] + 1; 102 | } 103 | } 104 | 105 | HashMap query_all(){ 106 | HashMap ret; 107 | 108 | for(uint32_t i = 0;i < BUCKET_LENGTH;++i){ 109 | for(uint32_t j = 0;j < COUNTER_PER_BUCKET;++j){ 110 | ret[buckets[i].ID[j]] = buckets[i].count[j] + 111 | sketch[hash(buckets[i].ID[j], 101) % SKETCH_LENGTH]; 112 | } 113 | } 114 | 115 | return ret; 116 | } 117 | 118 | void Merge(Elastic* temp){ 119 | for(uint32_t i = 0;i < SKETCH_LENGTH;++i){ 120 | sketch[i] += temp->sketch[i]; 121 | } 122 | 123 | for(uint32_t i = 0;i < BUCKET_LENGTH;++i){ 124 | for(uint32_t j = 0;j < COUNTER_PER_BUCKET;++j){ 125 | int32_t minVal = 0x7fffffff; 126 | uint32_t minPos = 0; 127 | 128 | for (uint32_t k = 0; k < COUNTER_PER_BUCKET; k++){ 129 | if(buckets[i].ID[k] == temp->buckets[i].ID[j]){ 130 | buckets[i].count[k] += temp->buckets[i].count[j]; 131 | goto MergeEnd; 132 | } 133 | 134 | if(buckets[i].count[k] < minVal){ 135 | minPos = k; 136 | minVal = buckets[i].count[k]; 137 | } 138 | } 139 | 140 | if((buckets[i].vote + temp->buckets[i].count[j]) >= minVal * 8){ 141 | buckets[i].vote = 0; 142 | 143 | if(minVal != 0){ 144 | uint32_t position = hash(buckets[i].ID[minPos], 101) % SKETCH_LENGTH; 145 | sketch[position] = sketch[position] + buckets[i].count[minPos]; 146 | } 147 | 148 | buckets[i].ID[minPos] = temp->buckets[i].ID[j]; 149 | buckets[i].count[minPos] = temp->buckets[i].count[j]; 150 | } 151 | else { 152 | buckets[i].vote += temp->buckets[i].count[j]; 153 | uint32_t position = hash(temp->buckets[i].ID[j], 101) % SKETCH_LENGTH; 154 | sketch[position] = sketch[position] + temp->buckets[i].count[j]; 155 | } 156 | 157 | MergeEnd: 158 | {}; 159 | } 160 | } 161 | 162 | delete temp; 163 | } 164 | 165 | void Merge(const Elastic_Entry& temp){ 166 | if(temp.bucket){ 167 | int32_t minVal = 0x7fffffff; 168 | uint32_t minPos = 0; 169 | 170 | for(uint32_t j = 0; j < COUNTER_PER_BUCKET; j++){ 171 | if(buckets[temp.pos].ID[j] == temp.key) { 172 | buckets[temp.pos].count[j] += temp.value; 173 | return; 174 | } 175 | 176 | if(buckets[temp.pos].count[j] < minVal){ 177 | minPos = j; 178 | minVal = buckets[temp.pos].count[j]; 179 | } 180 | } 181 | 182 | if((buckets[temp.pos].vote + temp.value) >= minVal * 8){ 183 | buckets[temp.pos].vote = 0; 184 | 185 | if(minVal != 0){ 186 | uint32_t position = hash(buckets[temp.pos].ID[minPos], 101) % SKETCH_LENGTH; 187 | sketch[position] = sketch[position] + buckets[temp.pos].count[minPos]; 188 | } 189 | 190 | buckets[temp.pos].ID[minPos] = temp.key; 191 | buckets[temp.pos].count[minPos] = temp.value; 192 | } 193 | else { 194 | buckets[temp.pos].vote += temp.value; 195 | uint32_t position = hash(temp.key, 101) % SKETCH_LENGTH; 196 | sketch[position] = sketch[position] + temp.value; 197 | } 198 | 199 | } 200 | else{ 201 | sketch[temp.pos] += temp.value; 202 | } 203 | } 204 | }; 205 | 206 | #endif -------------------------------------------------------------------------------- /sketch/HLL.h: -------------------------------------------------------------------------------- 1 | #ifndef HLL_H 2 | #define HLL_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef uint8_t Value; 7 | 8 | template 9 | struct HLL_Entry{ 10 | uint16_t pos; 11 | Value value; 12 | 13 | HLL_Entry(uint16_t _pos = 0, Value _value = 0): 14 | pos(_pos), value(_value){}; 15 | }; 16 | 17 | template 18 | class Child_HLL : public Sketch{ 19 | public: 20 | 21 | Value sketch[LENGTH]; 22 | 23 | Child_HLL(){ 24 | memset(sketch, 0, sizeof(Value) * LENGTH); 25 | } 26 | 27 | ~Child_HLL(){} 28 | 29 | void insert_one(const Key& packet){} 30 | 31 | HashMap query_all(){ 32 | HashMap ret; 33 | return ret; 34 | } 35 | }; 36 | 37 | template 38 | class HLL : public Sketch{ 39 | public: 40 | 41 | Value sketch[LENGTH]; 42 | 43 | HLL(){ 44 | memset(sketch, 0, sizeof(Value) * LENGTH); 45 | } 46 | 47 | ~HLL(){} 48 | 49 | void insert_one(const Key& packet){ 50 | uint32_t pos = hash(packet) % LENGTH; 51 | uint32_t temp = hash(packet, 101); 52 | uint8_t rank = MIN(31, __builtin_clz(temp) + 1); 53 | 54 | if(sketch[pos] < rank) 55 | sketch[pos] = rank; 56 | } 57 | 58 | HashMap query_all(){ 59 | HashMap ret; 60 | double sum = 0; 61 | for(uint32_t i = 0;i < LENGTH;++i){ 62 | sum += (1.0 / (1 << sketch[i])); 63 | } 64 | ret[0] = 0.7213 * LENGTH / (LENGTH + 1.079) * LENGTH / sum * LENGTH; 65 | return ret; 66 | } 67 | 68 | void Merge(HLL* temp){ 69 | for(uint32_t i = 0;i < LENGTH;++i){ 70 | sketch[i] = MAX(sketch[i], temp->sketch[i]); 71 | } 72 | delete temp; 73 | } 74 | 75 | void Merge(const HLL_Entry& temp){ 76 | const uint16_t& pos = temp.pos; 77 | sketch[pos] = MAX(sketch[pos], temp.value); 78 | } 79 | }; 80 | 81 | #endif -------------------------------------------------------------------------------- /sketch/LL.h: -------------------------------------------------------------------------------- 1 | #ifndef LL_H 2 | #define LL_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef uint8_t Value; 7 | 8 | template 9 | struct LL_Entry{ 10 | uint16_t pos; 11 | Value value; 12 | 13 | LL_Entry(uint16_t _pos = 0, Value _value = 0): 14 | pos(_pos), value(_value){}; 15 | }; 16 | 17 | template 18 | class Child_LL : public Sketch{ 19 | public: 20 | 21 | Value sketch[LENGTH]; 22 | 23 | Child_LL(){ 24 | memset(sketch, 0, sizeof(Value) * LENGTH); 25 | } 26 | 27 | ~Child_LL(){} 28 | 29 | void insert_one(const Key& packet){} 30 | 31 | HashMap query_all(){ 32 | HashMap ret; 33 | return ret; 34 | } 35 | }; 36 | 37 | template 38 | class LL : public Sketch{ 39 | public: 40 | 41 | Value sketch[LENGTH]; 42 | 43 | LL(){ 44 | memset(sketch, 0, sizeof(Value) * LENGTH); 45 | } 46 | 47 | ~LL(){} 48 | 49 | void insert_one(const Key& packet){ 50 | uint32_t pos = hash(packet) % LENGTH; 51 | uint32_t temp = hash(packet, 101); 52 | uint8_t rank = MIN(31, __builtin_clz(temp) + 1); 53 | 54 | if(sketch[pos] < rank) 55 | sketch[pos] = rank; 56 | } 57 | 58 | HashMap query_all(){ 59 | HashMap ret; 60 | double sum = 0; 61 | for(uint32_t i = 0;i < LENGTH;++i){ 62 | sum += sketch[i]; 63 | } 64 | ret[0] = 0.39701 * LENGTH * pow(2, sum / LENGTH); 65 | return ret; 66 | } 67 | 68 | void Merge(LL* temp){ 69 | for(uint32_t i = 0;i < LENGTH;++i){ 70 | sketch[i] = MAX(sketch[i], temp->sketch[i]); 71 | } 72 | delete temp; 73 | } 74 | 75 | void Merge(const LL_Entry& temp){ 76 | const uint16_t& pos = temp.pos; 77 | sketch[pos] = MAX(sketch[pos], temp.value); 78 | } 79 | }; 80 | 81 | #endif -------------------------------------------------------------------------------- /sketch/Locher.h: -------------------------------------------------------------------------------- 1 | #ifndef LOCHER_H 2 | #define LOCHER_H 3 | 4 | #include "../common/shll.h" 5 | 6 | #include "sketch.h" 7 | 8 | struct Locher_Entry{ 9 | uint32_t src; 10 | uint16_t hashPos; 11 | uint16_t pos; 12 | uint32_t value; 13 | 14 | Locher_Entry(uint32_t _src = 0, uint32_t _hashPos = 0, uint32_t _pos = 0, uint32_t _value = 0): 15 | src(_src), hashPos(_hashPos), pos(_pos), value(_value){}; 16 | }; 17 | 18 | template 19 | class Child_Locher : public Sketch{ 20 | public: 21 | 22 | SHLL sketch[HASH_NUM][LENGTH]; 23 | 24 | Child_Locher(){ 25 | memset(sketch, 0, sizeof(SHLL) * HASH_NUM * LENGTH); 26 | } 27 | 28 | ~Child_Locher(){} 29 | 30 | void insert_one(const Packet& packet){} 31 | 32 | HashMap query_all(){ 33 | HashMap ret; 34 | return ret; 35 | } 36 | }; 37 | 38 | template 39 | class Locher : public Sketch{ 40 | public: 41 | typedef Heap myHeap; 42 | 43 | SHLL sketch[HASH_NUM][LENGTH]; 44 | double distinct[HASH_NUM] = {0}; 45 | myHeap* heap; 46 | 47 | Locher(){ 48 | memset(sketch, 0, sizeof(SHLL) * HASH_NUM * LENGTH); 49 | heap = new myHeap(HEAP_SIZE); 50 | } 51 | 52 | ~Locher(){ 53 | delete heap; 54 | } 55 | 56 | void insert_one(const Packet& packet){ 57 | double estimation[HASH_NUM] = {0}; 58 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 59 | uint32_t pos = hash(packet.src, hashPos) % LENGTH; 60 | distinct[hashPos] -= sketch[hashPos][pos].Query(); 61 | sketch[hashPos][pos].Insert(packet.dst, hashPos); 62 | double est = sketch[hashPos][pos].Query(); 63 | estimation[hashPos] = est - distinct[hashPos] / (LENGTH - 1); 64 | distinct[hashPos] += est; 65 | } 66 | heap->Insert(packet.src, MEDIAN3(estimation)); 67 | } 68 | 69 | HashMap query_all(){ 70 | return heap->AllQuery(); 71 | } 72 | 73 | void Merge(Locher* temp){ 74 | for(uint32_t i = 0;i < HASH_NUM;++i){ 75 | distinct[i] = 0; 76 | for(uint32_t j = 0;j < LENGTH;++j){ 77 | sketch[i][j].Merge(temp->sketch[i][j]); 78 | distinct[i] += sketch[i][j].Query(); 79 | } 80 | } 81 | 82 | myHeap* check[2] = {heap, temp->heap}; 83 | 84 | for(auto p : check) { 85 | for (uint32_t i = 0; i < p->mp->size(); ++i) { 86 | double estimation[HASH_NUM] = {0}; 87 | for (uint32_t hashPos = 0; hashPos < HASH_NUM; ++hashPos) { 88 | uint32_t pos = hash((uint32_t)p->heap[i].key, hashPos) % LENGTH; 89 | double est = sketch[hashPos][pos].Query(); 90 | estimation[hashPos] = est - (distinct[hashPos] - est) / (LENGTH - 1); 91 | } 92 | heap->Insert(p->heap[i].key, MEDIAN3(estimation)); 93 | } 94 | } 95 | 96 | delete temp; 97 | } 98 | 99 | void Merge(const Locher_Entry& temp){ 100 | const uint16_t& hashPos = temp.hashPos; 101 | const uint16_t& pos = temp.pos; 102 | 103 | double before = sketch[hashPos][pos].Query(); 104 | 105 | uint32_t inbucket_index = (temp.value & 0x1); 106 | uint32_t bucket_index = ((temp.value >> 1) & 0x7); 107 | uint8_t rank = MIN(15, __builtin_clz(temp.value) + 1); 108 | bool modify = false; 109 | 110 | Buckets& tempBucket = sketch[hashPos][pos].buckets[bucket_index]; 111 | switch(inbucket_index){ 112 | case 0: 113 | if(tempBucket.counter0 < rank){ 114 | tempBucket.counter0 = rank; 115 | modify = true; 116 | break; 117 | } 118 | case 1: 119 | if(tempBucket.counter1 < rank){ 120 | tempBucket.counter1 = rank; 121 | modify = true; 122 | break; 123 | } 124 | } 125 | 126 | if(modify){ 127 | double after = sketch[hashPos][pos].Query(); 128 | distinct[hashPos] += (after - before); 129 | after = after - (distinct[hashPos] - after) / (LENGTH - 1); 130 | 131 | if(after > heap->min()){ 132 | double estimation[HASH_NUM] = {0}; 133 | 134 | for(uint32_t tempHash = 0;tempHash < HASH_NUM;++tempHash){ 135 | uint32_t tempPos = hash(temp.src, tempHash) % LENGTH; 136 | double est = sketch[tempHash][tempPos].Query(); 137 | estimation[tempHash] = est - (distinct[tempHash] - est) / (LENGTH - 1); 138 | } 139 | heap->Insert(temp.src, MEDIAN3(estimation)); 140 | } 141 | } 142 | } 143 | }; 144 | 145 | #endif -------------------------------------------------------------------------------- /sketch/UnivMon.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIVMON_H 2 | #define UNIVMON_H 3 | 4 | #include "sketch.h" 5 | 6 | typedef int8_t Value; 7 | 8 | constexpr static int32_t increment[2] = {1, -1}; 9 | 10 | template 11 | struct Univ_Entry{ 12 | Key key; 13 | uint16_t level; 14 | uint16_t hashPos; 15 | uint16_t pos; 16 | Value value; 17 | 18 | Univ_Entry(Key _key = 0, uint16_t _level = 0, uint16_t _hashPos = 0, 19 | uint16_t _pos = 0, Value _value = 0): 20 | key(_key), level(_level), hashPos(_hashPos), pos(_pos), value(_value){}; 21 | }; 22 | 23 | template 24 | class Child_UnivMon : public Sketch{ 25 | public: 26 | 27 | Value sketch[MAX_LEVEL][HASH_NUM][LENGTH]; 28 | 29 | Child_UnivMon(){ 30 | memset(sketch, 0, sizeof(Value) * MAX_LEVEL * HASH_NUM * LENGTH); 31 | } 32 | 33 | ~Child_UnivMon(){} 34 | 35 | void insert_one(const Key& packet){} 36 | 37 | HashMap query_all(){ 38 | HashMap ret; 39 | return ret; 40 | } 41 | }; 42 | 43 | template 44 | class UnivMon : public Sketch{ 45 | public: 46 | typedef Heap myHeap; 47 | 48 | int32_t sketch[MAX_LEVEL][HASH_NUM][LENGTH]; 49 | myHeap* heap[MAX_LEVEL]; 50 | 51 | UnivMon(){ 52 | memset(sketch, 0, sizeof(int32_t) * MAX_LEVEL * HASH_NUM * LENGTH); 53 | for(uint32_t i = 0;i < MAX_LEVEL;++i){ 54 | heap[i] = new myHeap(HEAP_SIZE); 55 | } 56 | } 57 | 58 | ~UnivMon(){ 59 | for(uint32_t i = 0;i < MAX_LEVEL;++i){ 60 | delete heap[i]; 61 | } 62 | } 63 | 64 | void insert_one(const Key& packet){ 65 | uint32_t polar = hash(packet, 199); 66 | uint32_t max_level = MIN(MAX_LEVEL - 1, __builtin_clz(polar)); 67 | 68 | for(uint32_t level = 0; level <= max_level;++level){ 69 | int32_t number[HASH_NUM] = {0}; 70 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 71 | uint32_t hashNum = hash(packet, level * HASH_NUM + hashPos); 72 | uint32_t pos = (hashNum >> 1) % LENGTH; 73 | int32_t incre = increment[hashNum & 1]; 74 | 75 | sketch[level][hashPos][pos] += incre; 76 | number[hashPos] = sketch[level][hashPos][pos] * incre; 77 | } 78 | heap[level]->Insert(packet, MEDIAN3(number)); 79 | } 80 | } 81 | 82 | HashMap query_all(){ 83 | HashMap ret; 84 | 85 | for(int32_t level = 0;level < MAX_LEVEL;++level){ 86 | HashMap temp = heap[level]->AllQuery(); 87 | for(auto it = temp.begin();it != temp.end();++it){ 88 | if(ret.find(it->first) == ret.end()){ 89 | ret[it->first] = it->second; 90 | } 91 | } 92 | } 93 | 94 | return ret; 95 | } 96 | 97 | void Merge(UnivMon* temp){ 98 | for(uint32_t level = 0;level < MAX_LEVEL;++level){ 99 | for(uint32_t i = 0;i < HASH_NUM;++i){ 100 | for(uint32_t j = 0;j < LENGTH;++j){ 101 | sketch[level][i][j] += temp->sketch[level][i][j]; 102 | } 103 | } 104 | 105 | myHeap* check[2] = {heap[level], temp->heap[level]}; 106 | 107 | for(auto p : check) { 108 | for(uint32_t i = 0;i < p->mp->size();++i){ 109 | int32_t number[HASH_NUM] = {0}; 110 | for(uint32_t hashPos = 0;hashPos < HASH_NUM;++hashPos){ 111 | uint32_t hashNum = hash(p->heap[i].key, level * HASH_NUM + hashPos); 112 | uint32_t pos = (hashNum >> 1) % LENGTH; 113 | int32_t incre = increment[hashNum & 1]; 114 | number[hashPos] = sketch[level][hashPos][pos] * incre; 115 | } 116 | heap[level]->Insert(p->heap[i].key, MEDIAN3(number)); 117 | } 118 | } 119 | } 120 | 121 | delete temp; 122 | } 123 | 124 | void Merge(const Univ_Entry& temp){ 125 | const uint16_t& level = temp.level; 126 | const uint16_t& hashPos = temp.hashPos; 127 | const uint16_t& pos = temp.pos; 128 | 129 | sketch[level][hashPos][pos] += temp.value; 130 | 131 | if(abs(sketch[level][hashPos][pos]) > heap[level]->min()){ 132 | int32_t count[HASH_NUM] = {0}; 133 | 134 | for(uint32_t tempHash = 0;tempHash < HASH_NUM;++tempHash){ 135 | uint32_t hashNum = hash(temp.key, tempHash); 136 | uint32_t tempPos = (hashNum >> 1) % LENGTH; 137 | int32_t incre = increment[hashNum & 1]; 138 | 139 | count[tempHash] = sketch[level][tempHash][tempPos] * incre; 140 | } 141 | heap[level]->Insert(temp.key, MEDIAN3(count)); 142 | } 143 | } 144 | }; 145 | 146 | 147 | #endif -------------------------------------------------------------------------------- /sketch/sketch.h: -------------------------------------------------------------------------------- 1 | #ifndef SKETCH_H 2 | #define SKETCH_H 3 | 4 | #include "../common/hash.h" 5 | #include "../common/heap.h" 6 | 7 | typedef std::unordered_map HashMap; 8 | 9 | struct Packet{ 10 | uint32_t src; 11 | uint32_t dst; 12 | 13 | operator uint64_t() const{ 14 | return *((uint64_t*)(this)); 15 | } 16 | }; 17 | 18 | /** 19 | * Template for sketches 20 | */ 21 | 22 | template 23 | class Sketch{ 24 | public: 25 | virtual ~Sketch(){}; 26 | 27 | // Insert one element in the data stream 28 | virtual void insert_one(const Key& p) = 0; 29 | // Get statistics (e.g., heavy hitters) from sketches 30 | virtual HashMap query_all() = 0; 31 | }; 32 | 33 | #endif --------------------------------------------------------------------------------