├── _config.yml ├── src ├── Router │ ├── protoc_files │ │ ├── lookup.pb.o │ │ ├── router.pb.o │ │ ├── lookup.grpc.pb.o │ │ ├── router.grpc.pb.o │ │ ├── lookup.proto │ │ ├── router.proto │ │ ├── lookup.grpc.pb.cc │ │ ├── Makefile │ │ └── router.grpc.pb.cc │ ├── mid_tier_service │ │ ├── service │ │ │ ├── helper_files │ │ │ │ ├── timing.o │ │ │ │ ├── utils.o │ │ │ │ ├── timing.h │ │ │ │ ├── timing.cc │ │ │ │ ├── utils.cc │ │ │ │ └── utils.h │ │ │ └── Makefile │ │ └── src │ │ │ ├── thread_safe_flag.cpp │ │ │ ├── thread_safe_queue.cpp │ │ │ └── thread_safe_circ_buffer.cpp │ ├── lookup_service │ │ ├── service │ │ │ ├── helper_files │ │ │ │ ├── utils.cc │ │ │ │ ├── utils.h │ │ │ │ ├── timing.h │ │ │ │ ├── timing.cc │ │ │ │ ├── server_helper.h │ │ │ │ └── server_helper.cc │ │ │ └── Makefile │ │ └── src │ │ │ └── thread_safe_map.cpp │ └── load_generator │ │ └── Makefile ├── HDSearch │ ├── bucket_service │ │ ├── src │ │ │ ├── utils.cpp │ │ │ ├── utils.h │ │ │ ├── custom_priority_queue.cpp │ │ │ ├── point.cpp │ │ │ ├── custom_priority_queue.h │ │ │ ├── point.h │ │ │ ├── multiple_points.h │ │ │ └── dist_calc.h │ │ ├── service │ │ │ ├── helper_files │ │ │ │ ├── timing.h │ │ │ │ ├── timing.cc │ │ │ │ └── client_helper.h │ │ │ └── Makefile │ │ └── README.txt │ ├── mid_tier_service │ │ ├── src │ │ │ ├── CMakeLists.txt │ │ │ ├── cpp │ │ │ │ └── flann │ │ │ │ │ ├── mpi │ │ │ │ │ ├── flann_mpi_server.cpp │ │ │ │ │ ├── flann_mpi_client.cpp │ │ │ │ │ ├── matrix.h │ │ │ │ │ ├── client.h │ │ │ │ │ └── queries.h │ │ │ │ │ ├── flann_cpp.cpp │ │ │ │ │ ├── config.h │ │ │ │ │ ├── config.h.in │ │ │ │ │ ├── util │ │ │ │ │ ├── timer.h │ │ │ │ │ ├── sampling.h │ │ │ │ │ ├── object_factory.h │ │ │ │ │ ├── matrix.h │ │ │ │ │ ├── saving.h │ │ │ │ │ ├── logger.h │ │ │ │ │ └── random.h │ │ │ │ │ └── nn │ │ │ │ │ └── ground_truth.h │ │ │ ├── thread_safe_flag.cpp │ │ │ ├── thread_safe_map.cpp │ │ │ ├── thread_safe_queue.cpp │ │ │ └── thread_safe_circ_buffer.cpp │ │ ├── cmake │ │ │ ├── CMakeLists.txt │ │ │ ├── flann.pc.in │ │ │ ├── uninstall_target.cmake.in │ │ │ └── FindFlann.cmake │ │ ├── service │ │ │ ├── helper_files │ │ │ │ ├── timing.h │ │ │ │ ├── utils.cc │ │ │ │ ├── utils.h │ │ │ │ └── timing.cc │ │ │ └── Makefile │ │ └── ChangeLog │ ├── load_generator │ │ ├── helper_files │ │ │ ├── timing.h │ │ │ ├── utils.cc │ │ │ ├── utils.h │ │ │ └── timing.cc │ │ └── Makefile │ └── protoc_files │ │ ├── bucket.proto │ │ ├── mid_tier.proto │ │ └── Makefile ├── Recommend │ ├── cf_service │ │ └── service │ │ │ ├── helper_files │ │ │ ├── utils.cc │ │ │ ├── utils.h │ │ │ ├── timing.h │ │ │ ├── server_helper.cc │ │ │ ├── server_helper.h │ │ │ └── timing.cc │ │ │ └── Makefile │ ├── recommender_service │ │ ├── service │ │ │ ├── helper_files │ │ │ │ ├── timing.h │ │ │ │ ├── timing.cc │ │ │ │ ├── utils.cc │ │ │ │ └── utils.h │ │ │ └── Makefile │ │ └── src │ │ │ ├── thread_safe_flag.cpp │ │ │ ├── thread_safe_queue.cpp │ │ │ └── thread_safe_circ_buffer.cpp │ ├── protoc_files │ │ ├── cf.proto │ │ ├── recommender.proto │ │ └── Makefile │ └── load_generator │ │ └── Makefile └── SetAlgebra │ ├── intersection_service │ ├── service │ │ ├── helper_files │ │ │ ├── utils.cc │ │ │ ├── utils.h │ │ │ ├── timing.h │ │ │ ├── server_helper.h │ │ │ ├── timing.cc │ │ │ └── server_helper.cc │ │ └── Makefile │ └── src │ │ ├── intersection.h │ │ └── intersection.cc │ ├── union_service │ ├── service │ │ ├── helper_files │ │ │ ├── timing.h │ │ │ ├── timing.cc │ │ │ ├── utils.cc │ │ │ └── utils.h │ │ └── Makefile │ └── src │ │ ├── thread_safe_flag.cpp │ │ ├── thread_safe_queue.cpp │ │ └── thread_safe_circ_buffer.cpp │ ├── protoc_files │ ├── intersection.proto │ ├── union.proto │ └── Makefile │ └── load_generator │ └── Makefile ├── LICENSE └── install.py /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-midnight -------------------------------------------------------------------------------- /src/Router/protoc_files/lookup.pb.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/protoc_files/lookup.pb.o -------------------------------------------------------------------------------- /src/Router/protoc_files/router.pb.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/protoc_files/router.pb.o -------------------------------------------------------------------------------- /src/Router/protoc_files/lookup.grpc.pb.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/protoc_files/lookup.grpc.pb.o -------------------------------------------------------------------------------- /src/Router/protoc_files/router.grpc.pb.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/protoc_files/router.grpc.pb.o -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | unsigned int GetNumProcs() 4 | { 5 | return std::thread::hardware_concurrency(); 6 | } 7 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/timing.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/mid_tier_service/service/helper_files/timing.o -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/utils.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wenischlab/MicroSuite/HEAD/src/Router/mid_tier_service/service/helper_files/utils.o -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | unsigned int GetNumProcs() 4 | { 5 | return std::thread::hardware_concurrency(); 6 | } 7 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | unsigned int GetNumProcs() 4 | { 5 | return std::thread::hardware_concurrency(); 6 | } 7 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | unsigned int GetNumProcs(); 5 | 6 | #endif //__UTILS_H_INCLUDED__ 7 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | unsigned int GetNumProcs() 4 | { 5 | return std::thread::hardware_concurrency(); 6 | } 7 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | unsigned int GetNumProcs(); 5 | 6 | #endif //__UTILS_H_INCLUDED__ 7 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | unsigned int GetNumProcs(); 5 | 6 | #endif //__UTILS_H_INCLUDED__ 7 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | unsigned int GetNumProcs(); 5 | 6 | #endif //__UTILS_H_INCLUDED__ 7 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory( cpp ) 3 | 4 | #if (BUILD_MATLAB_BINDINGS) 5 | # add_subdirectory( matlab ) 6 | #endif() 7 | 8 | #if (BUILD_PYTHON_BINDINGS) 9 | # add_subdirectory( python ) 10 | #endif() 11 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | uint64_t GetTimeInMicro(); 6 | void GetCpuTimes(uint64_t* user_time, 7 | uint64_t* system_time, 8 | uint64_t* io_time, 9 | uint64_t* idle_time); 10 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | uint64_t GetTimeInMicro(); 6 | void GetCpuTimes(uint64_t* user_time, 7 | uint64_t* system_time, 8 | uint64_t* io_time, 9 | uint64_t* idle_time); 10 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | uint64_t GetTimeInMicro(); 6 | void GetCpuTimes(uint64_t* user_time, 7 | uint64_t* system_time, 8 | uint64_t* io_time, 9 | uint64_t* idle_time); 10 | 11 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | uint64_t GetTimeInMicro(); 6 | void GetCpuTimes(uint64_t* user_time, 7 | uint64_t* system_time, 8 | uint64_t* io_time, 9 | uint64_t* idle_time); 10 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/cmake/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PKG_DESC "Fast Library for Approximate Nearest Neighbors") 2 | set(pkg_conf_file ${CMAKE_CURRENT_BINARY_DIR}/flann.pc) 3 | configure_file(flann.pc.in ${pkg_conf_file} @ONLY) 4 | install(FILES ${pkg_conf_file} 5 | DESTINATION ${FLANN_LIB_INSTALL_DIR}/pkgconfig/ COMPONENT pkgconfig) 6 | 7 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/cmake/flann.pc.in: -------------------------------------------------------------------------------- 1 | # This file was generated by CMake for @PROJECT_NAME@ 2 | prefix=@CMAKE_INSTALL_PREFIX@ 3 | exec_prefix=${prefix} 4 | libdir=${prefix}/@FLANN_LIB_INSTALL_DIR@ 5 | includedir=${prefix}/include 6 | 7 | Name: @PROJECT_NAME@ 8 | Description: @PKG_DESC@ 9 | Version: @FLANN_VERSION@ 10 | Requires: @PKG_EXTERNAL_DEPS@ 11 | Libs: -L${libdir} -lflann -lflann_cpp 12 | Cflags: -I${includedir} 13 | 14 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #ifndef __TIMING_H_INCLUDED__ 2 | #define __TIMING_H_INCLUDED__ 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | uint64_t GetTimeInMicro(); 9 | void GetCpuTimes(uint64_t* user_time, 10 | uint64_t* system_time, 11 | uint64_t* io_time, 12 | uint64_t* idle_time); 13 | 14 | uint64_t GetTimeInSec(); 15 | 16 | #endif //__TIMING_H_INCLUDED__ 17 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #ifndef __TIMING_H_INCLUDED__ 2 | #define __TIMING_H_INCLUDED__ 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | uint64_t GetTimeInMicro(); 9 | void GetCpuTimes(uint64_t* user_time, 10 | uint64_t* system_time, 11 | uint64_t* io_time, 12 | uint64_t* idle_time); 13 | 14 | uint64_t GetTimeInSec(); 15 | 16 | #endif //__TIMING_H_INCLUDED__ 17 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #ifndef __TIMING_H_INCLUDED__ 2 | #define __TIMING_H_INCLUDED__ 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | uint64_t GetTimeInMicro(); 9 | void GetCpuTimes(uint64_t* user_time, 10 | uint64_t* system_time, 11 | uint64_t* io_time, 12 | uint64_t* idle_time); 13 | 14 | uint64_t GetTimeInSec(); 15 | 16 | #endif //__TIMING_H_INCLUDED__ 17 | -------------------------------------------------------------------------------- /src/HDSearch/load_generator/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro(); 12 | uint64_t GetTimeInSec(); 13 | double Rdtsc(); 14 | uint64_t RdtscCycles(); 15 | void GetCpuTimes(uint64_t* user_time, 16 | uint64_t* system_time, 17 | uint64_t* io_time, 18 | uint64_t* idle_time); 19 | 20 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/service/helper_files/timing.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro(); 12 | uint64_t GetTimeInSec(); 13 | double Rdtsc(); 14 | uint64_t RdtscCycles(); 15 | void GetCpuTimes(uint64_t* user_time, 16 | uint64_t* system_time, 17 | uint64_t* io_time, 18 | uint64_t* idle_time); 19 | 20 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/src/intersection.h: -------------------------------------------------------------------------------- 1 | #ifndef __INTERSECTION_H_INCLUDED__ 2 | #define __INTERSECTION_H_INCLUDED__ 3 | 4 | #define CHECK(condition, error_message) if (!condition) {std::cerr << __FILE__ << ": " << __LINE__ << ": " << error_message << "\n"; exit(-1);} 5 | 6 | #include 7 | 8 | typedef uint32_t Docids; 9 | 10 | void ComputeIntersection(const std::vector &word_one, 11 | const std::vector &word_two, 12 | std::vector* result); 13 | 14 | #endif // __INTERSECTION_H_INCLUDED__ 15 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/README.txt: -------------------------------------------------------------------------------- 1 | Dataset directory: 2 | --Contains dataset and query files 3 | 4 | Service directory: 5 | --Install grpc and protoc 6 | --Paths to "grpc/libs/opt/pkgconfig" and "grpc/libs/opt" must be exported as environment variables: PKG_CONFIG_PATH and LD_LIBRARY_PATH in the ~/.bashrc file 7 | --Makefile compiles bucket.proto to create ".grpc.pb.cc" and ".pb.cc" files 8 | --bucket_server and bucket_client use classes defined in src/ for distance calculations 9 | 10 | Src directory: 11 | --Contains all the necessary files to perform distance calculations between dataset and query points 12 | 13 | Test directory: 14 | --Unit tests for distance calculations 15 | 16 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/mpi/flann_mpi_server.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char* argv[]) 7 | { 8 | boost::mpi::environment env(argc, argv); 9 | 10 | try { 11 | if (argc != 4) { 12 | std::cout << "Usage: " << argv[0] << " \n"; 13 | return 1; 14 | } 15 | flann::mpi::Server > server(argv[1], argv[2], std::atoi(argv[3]), 16 | flann::KDTreeIndexParams(4)); 17 | 18 | server.run(); 19 | } 20 | catch (std::exception& e) { 21 | std::cerr << "Exception: " << e.what() << "\n"; 22 | } 23 | 24 | return 0; 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/src/intersection.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "intersection.h" 4 | 5 | void ComputeIntersection(const std::vector &word_one, 6 | const std::vector &word_two, 7 | std::vector* result) 8 | { 9 | Docids word_one_size = word_one.size(); 10 | Docids word_two_size = word_two.size(); 11 | 12 | long i = 0, j = 0; 13 | 14 | while( (i < word_one_size) && (j < word_two_size) ) 15 | { 16 | if (word_one[i] < word_two[j]) { 17 | i++; 18 | } else if (word_two[j] < word_one[i]) { 19 | j++; 20 | } else { 21 | result->emplace_back(word_one[i]); 22 | i++; 23 | j++; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/server_helper.h: -------------------------------------------------------------------------------- 1 | #ifndef __SERVER_HELPER_H_INCLUDED__ 2 | #define __SERVER_HELPER_H_INCLUDED__ 3 | 4 | #include 5 | #include 6 | 7 | #include "intersection_service/src/intersection.h" 8 | #include "protoc_files/intersection.grpc.pb.h" 9 | 10 | void CreateIndexFromFile(const std::string dataset_file_name, 11 | std::map >* index); 12 | 13 | void UnpackIntersectionServiceRequest(const intersection::IntersectionRequest &request, 14 | std::vector* word_ids); 15 | 16 | bool ExtractDocids(const std::vector &word_ids, 17 | const std::map > &word_to_docids, 18 | std::vector >* doc_ids_for_all_words); 19 | 20 | void PackIntersectionServiceResponse(const std::vector &intersection_res, 21 | intersection::IntersectionResponse* reply); 22 | 23 | #endif //__SERVER_HELPER_H_INCLUDED__ 24 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/thread_safe_flag.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeFlag 8 | { 9 | public: 10 | 11 | void Wait() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (!flag_) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | } 19 | 20 | void Set() 21 | { 22 | std::unique_lock mlock(mutex_); 23 | flag_ = true; 24 | mlock.unlock(); 25 | cond_.notify_all(); 26 | } 27 | 28 | void Reset() 29 | { 30 | std::unique_lock mlock(mutex_); 31 | flag_ = false; 32 | mlock.unlock(); 33 | } 34 | 35 | 36 | private: 37 | bool flag_; 38 | std::mutex mutex_; 39 | std::condition_variable cond_; 40 | }; 41 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/src/thread_safe_flag.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeFlag 8 | { 9 | public: 10 | 11 | void Wait() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (!flag_) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | } 19 | 20 | void Set() 21 | { 22 | std::unique_lock mlock(mutex_); 23 | flag_ = true; 24 | mlock.unlock(); 25 | cond_.notify_all(); 26 | } 27 | 28 | void Reset() 29 | { 30 | std::unique_lock mlock(mutex_); 31 | flag_ = false; 32 | mlock.unlock(); 33 | } 34 | 35 | 36 | private: 37 | bool flag_; 38 | std::mutex mutex_; 39 | std::condition_variable cond_; 40 | }; 41 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/src/thread_safe_flag.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeFlag 8 | { 9 | public: 10 | 11 | void Wait() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (!flag_) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | } 19 | 20 | void Set() 21 | { 22 | std::unique_lock mlock(mutex_); 23 | flag_ = true; 24 | mlock.unlock(); 25 | cond_.notify_all(); 26 | } 27 | 28 | void Reset() 29 | { 30 | std::unique_lock mlock(mutex_); 31 | flag_ = false; 32 | mlock.unlock(); 33 | } 34 | 35 | 36 | private: 37 | bool flag_; 38 | std::mutex mutex_; 39 | std::condition_variable cond_; 40 | }; 41 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/src/thread_safe_flag.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeFlag 8 | { 9 | public: 10 | 11 | void Wait() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (!flag_) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | } 19 | 20 | void Set() 21 | { 22 | std::unique_lock mlock(mutex_); 23 | flag_ = true; 24 | mlock.unlock(); 25 | cond_.notify_all(); 26 | } 27 | 28 | void Reset() 29 | { 30 | std::unique_lock mlock(mutex_); 31 | flag_ = false; 32 | mlock.unlock(); 33 | } 34 | 35 | 36 | private: 37 | bool flag_; 38 | std::mutex mutex_; 39 | std::condition_variable cond_; 40 | }; 41 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/server_helper.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "server_helper.h" 5 | #include 6 | 7 | using namespace mlpack; 8 | using namespace mlpack::cf; 9 | 10 | std::mutex test; 11 | 12 | 13 | void CreateDatasetFromFile(std::string dataset_file_name, 14 | Matrix* dataset) 15 | { 16 | data::Load(dataset_file_name, *dataset); 17 | } 18 | 19 | void UnpackCFServiceRequest(const collaborative_filtering::CFRequest &request, 20 | Request* user_item) 21 | { 22 | user_item->user = request.user(); 23 | user_item->item = request.item(); 24 | } 25 | 26 | void CalculateRating(const Request &user_item, 27 | CF* cf, 28 | float* rating) 29 | { 30 | *rating = cf->Predict(user_item.user, user_item.item); 31 | } 32 | 33 | void PackCFServiceResponse(const float rating, 34 | collaborative_filtering::CFResponse* reply) 35 | { 36 | reply->set_rating(rating); 37 | } 38 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/cmake/uninstall_target.cmake.in: -------------------------------------------------------------------------------- 1 | if(NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt") 2 | message(FATAL_ERROR "Cannot find install manifest: \"@PROJECT_BINARY_DIR@/install_manifest.txt\"") 3 | endif(NOT EXISTS "@PROJECT_BINARY_DIR@/install_manifest.txt") 4 | 5 | file(READ "@PROJECT_BINARY_DIR@/install_manifest.txt" files) 6 | string(REGEX REPLACE "\n" ";" files "${files}") 7 | foreach(file ${files}) 8 | message(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") 9 | if(EXISTS "$ENV{DESTDIR}${file}") 10 | exec_program("@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" 11 | OUTPUT_VARIABLE rm_out RETURN_VALUE rm_retval) 12 | if(NOT "${rm_retval}" STREQUAL 0) 13 | message(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") 14 | endif(NOT "${rm_retval}" STREQUAL 0) 15 | else(EXISTS "$ENV{DESTDIR}${file}") 16 | message(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") 17 | endif(EXISTS "$ENV{DESTDIR}${file}") 18 | endforeach(file) 19 | 20 | -------------------------------------------------------------------------------- /src/Router/lookup_service/src/thread_safe_map.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include // For std::unique_lock 3 | #include 4 | #include 5 | #include 6 | 7 | class ThreadSafeMap { 8 | public: 9 | ThreadSafeMap() = default; 10 | 11 | // Multiple threads/readers can read the counter's value at the same time. 12 | std::string Get(std::string key) { 13 | std::shared_lock lock(mutex_); 14 | try { 15 | map_.at(key); 16 | } catch( ... ) { 17 | return "nack"; 18 | } 19 | return map_[key]; 20 | } 21 | 22 | // Only one thread/writer can increment/write the counter's value. 23 | void Set(std::string key, std::string value) { 24 | std::unique_lock lock(mutex_); 25 | map_[key] = value; 26 | } 27 | 28 | private: 29 | mutable std::shared_mutex mutex_; 30 | std::map map_; 31 | }; 32 | 33 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/thread_safe_map.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include // For std::unique_lock 3 | #include 4 | #include 5 | #include 6 | 7 | class ThreadSafeMap { 8 | public: 9 | ThreadSafeMap() = default; 10 | 11 | // Multiple threads/readers can read the counter's value at the same time. 12 | std::string Get(std::string key) { 13 | std::shared_lock lock(mutex_); 14 | try { 15 | map_.at(key); 16 | } catch( ... ) { 17 | return "nack"; 18 | } 19 | return map_[key]; 20 | } 21 | 22 | // Only one thread/writer can increment/write the counter's value. 23 | void Set(std::string key, std::string value) { 24 | std::unique_lock lock(mutex_); 25 | map_[key] = value; 26 | } 27 | 28 | private: 29 | mutable std::shared_mutex mutex_; 30 | std::map map_; 31 | }; 32 | 33 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/cmake/FindFlann.cmake: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Find Flann 3 | # 4 | # This sets the following variables: 5 | # FLANN_FOUND - True if FLANN was found. 6 | # FLANN_INCLUDE_DIRS - Directories containing the FLANN include files. 7 | # FLANN_LIBRARIES - Libraries needed to use FLANN. 8 | # FLANN_DEFINITIONS - Compiler flags for FLANN. 9 | 10 | find_package(PkgConfig) 11 | pkg_check_modules(PC_FLANN flann) 12 | set(FLANN_DEFINITIONS ${PC_FLANN_CFLAGS_OTHER}) 13 | 14 | find_path(FLANN_INCLUDE_DIR flann/flann.hpp 15 | HINTS ${PC_FLANN_INCLUDEDIR} ${PC_FLANN_INCLUDE_DIRS}) 16 | 17 | find_library(FLANN_LIBRARY flann 18 | HINTS ${PC_FLANN_LIBDIR} ${PC_FLANN_LIBRARY_DIRS}) 19 | 20 | set(FLANN_INCLUDE_DIRS ${FLANN_INCLUDE_DIR}) 21 | set(FLANN_LIBRARIES ${FLANN_LIBRARY}) 22 | 23 | include(FindPackageHandleStandardArgs) 24 | find_package_handle_standard_args(Flann DEFAULT_MSG 25 | FLANN_LIBRARY FLANN_INCLUDE_DIR) 26 | 27 | mark_as_advanced(FLANN_LIBRARY FLANN_INCLUDE_DIR) 28 | 29 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/custom_priority_queue.cpp: -------------------------------------------------------------------------------- 1 | #include "custom_priority_queue.h" 2 | 3 | bool CustomPriorityQueue::IsEmpty() const 4 | { 5 | return knn_priority_queue_.empty(); 6 | } 7 | 8 | unsigned CustomPriorityQueue::GetSize() const 9 | { 10 | return knn_priority_queue_.size(); 11 | } 12 | 13 | void CustomPriorityQueue::AddToPriorityQueue(PointIDDistPair &knn_point) 14 | { 15 | knn_priority_queue_.push(knn_point); 16 | } 17 | 18 | void CustomPriorityQueue::RemoveTopElement() 19 | { 20 | CHECK(!knn_priority_queue_.empty(), "ERROR: Tring to remove an element from a NULL data structure\n"); 21 | knn_priority_queue_.pop(); 22 | } 23 | 24 | uint32_t CustomPriorityQueue::GetTopPointID() const 25 | { 26 | CHECK(!knn_priority_queue_.empty(), "ERROR: Tring to get an element from a NULL data structure\n"); 27 | return knn_priority_queue_.top().first; 28 | } 29 | 30 | float CustomPriorityQueue::GetTopDistance() const 31 | { 32 | CHECK(!knn_priority_queue_.empty(), "ERROR: Tring to get an element from a NULL data structure\n"); 33 | return knn_priority_queue_.top().second; 34 | } 35 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/server_helper.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "protoc_files/cf.grpc.pb.h" 8 | 9 | #ifndef __CLIENT_HELPER_H_INCLUDED__ 10 | #define __CLIENT_HELPER_H_INCLUDED__ 11 | #define CHECK(condition, error_message) if (!condition) {std::cerr << __FILE__ << ": " << __LINE__ << ": " << error_message << "\n"; exit(-1);} 12 | 13 | #define EMPTY -2.0 14 | 15 | typedef arma::mat Matrix; 16 | 17 | struct Request { 18 | int user = 0; 19 | int item = 0; 20 | }; 21 | 22 | void CreateDatasetFromFile(const std::string dataset_file_name, 23 | Matrix* dataset); 24 | 25 | void UnpackCFServiceRequest(const collaborative_filtering::CFRequest &request, 26 | Request* user_item); 27 | 28 | void CalculateRating(const Request &user_item, 29 | mlpack::cf::CF* cf_matrix, 30 | float* rating); 31 | 32 | void PackCFServiceResponse(const float rating, 33 | collaborative_filtering::CFResponse* reply); 34 | 35 | #endif // __CLIENT_HELPER_H_INCLUDED__ 36 | 37 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/point.cpp: -------------------------------------------------------------------------------- 1 | #include "point.h" 2 | 3 | void Point::Resize(const int size, const float value) 4 | { 5 | point_.assign(size, value); 6 | } 7 | void Point::AddValueToIndex(const int index, const float value) 8 | { 9 | point_[index] = value; 10 | } 11 | 12 | float Point::GetValueAtIndex(const int index) const 13 | { 14 | return point_[index]; 15 | } 16 | 17 | void Point::CreatePointFromFloatArray(const float float_arr[], 18 | const int float_arr_size) 19 | { 20 | CHECK((point_.size() == float_arr_size), "ERROR: Size of point must be initialized to the size of float array\n"); 21 | for(int i = 0; i < float_arr_size; i++) 22 | { 23 | point_[i] = float_arr[i]; 24 | } 25 | } 26 | 27 | unsigned Point::GetSize() const 28 | { 29 | return point_.size(); 30 | } 31 | 32 | bool Point::Equal(const Point &p) const 33 | { 34 | if (point_.size() != p.GetSize()) 35 | { 36 | return false; 37 | } 38 | return std::equal(point_.begin(), point_.end(), p.point_.begin()); 39 | } 40 | 41 | void Point::PrintPoint() const 42 | { 43 | for(auto& p : point_) 44 | { 45 | std::cout << p << " "; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/HDSearch/load_generator/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | 4 | std::string ExecuteShellCommand(const char* cmd) 5 | { 6 | char buffer[128]; 7 | std::string result = ""; 8 | std::shared_ptr pipe(popen(cmd, "r"), pclose); 9 | if (!pipe) throw std::runtime_error("popen() failed!"); 10 | while (!feof(pipe.get())) { 11 | if (fgets(buffer, 128, pipe.get()) != NULL) 12 | result += buffer; 13 | } 14 | return result; 15 | } 16 | 17 | void StringSplit(std::string str, 18 | char delimiter, 19 | std::vector* queries_str_vec) 20 | { 21 | std::stringstream ss(str); // Turn the string into a stream. 22 | std::string tok; 23 | 24 | while(std::getline(ss, tok, delimiter)) { 25 | if (queries_str_vec->size() == 2048) break; 26 | queries_str_vec->push_back(tok); 27 | } 28 | } 29 | 30 | void GetPerf(pid_t pid) { 31 | std::ostringstream str_pid; 32 | str_pid << pid; 33 | const std::string my_pid(str_pid.str()); 34 | std::string s = "perf stat -e context-switches -I 60000 -p " + my_pid + " &"; 35 | char* cmd = new char[s.length() + 1]; 36 | std::strcpy(cmd, s.c_str()); 37 | ExecuteShellCommand(cmd); 38 | } 39 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | 4 | std::string ExecuteShellCommand(const char* cmd) 5 | { 6 | char buffer[128]; 7 | std::string result = ""; 8 | std::shared_ptr pipe(popen(cmd, "r"), pclose); 9 | if (!pipe) throw std::runtime_error("popen() failed!"); 10 | while (!feof(pipe.get())) { 11 | if (fgets(buffer, 128, pipe.get()) != NULL) 12 | result += buffer; 13 | } 14 | return result; 15 | } 16 | 17 | void StringSplit(std::string str, 18 | char delimiter, 19 | std::vector* queries_str_vec) 20 | { 21 | std::stringstream ss(str); // Turn the string into a stream. 22 | std::string tok; 23 | 24 | while(std::getline(ss, tok, delimiter)) { 25 | if (queries_str_vec->size() == 2048) break; 26 | queries_str_vec->push_back(tok); 27 | } 28 | } 29 | 30 | void GetPerf(pid_t pid) { 31 | std::ostringstream str_pid; 32 | str_pid << pid; 33 | const std::string my_pid(str_pid.str()); 34 | std::string s = "perf stat -e context-switches -I 60000 -p " + my_pid + " &"; 35 | char* cmd = new char[s.length() + 1]; 36 | std::strcpy(cmd, s.c_str()); 37 | ExecuteShellCommand(cmd); 38 | } 39 | -------------------------------------------------------------------------------- /src/Router/protoc_files/lookup.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package lookup; 4 | 5 | service LookupService{ 6 | rpc KeyLookup(Key) returns (Value) {} 7 | } 8 | 9 | // The request message containing the query point. 10 | 11 | message UtilRequest { 12 | bool util_request = 1; 13 | } 14 | 15 | message Key { 16 | uint32 operation = 1; 17 | string key = 2; 18 | string value = 3; 19 | UtilRequest util_request = 4; 20 | uint64 request_id = 5; 21 | uint64 index_view = 6; 22 | uint32 lookup_server_id = 7; 23 | } 24 | 25 | message TimingDataInMicro{ 26 | uint64 unpack_lookup_srv_req_time_in_micro = 1; 27 | uint64 lookup_srv_time_in_micro = 2; 28 | uint64 pack_lookup_srv_resp_time_in_micro = 3; 29 | float cpu_util = 4; 30 | } 31 | 32 | message UtilResponse { 33 | bool util_present = 1; 34 | uint64 user_time = 2; 35 | uint64 system_time = 3; 36 | uint64 io_time = 4; 37 | uint64 idle_time = 5; 38 | } 39 | 40 | message Value { 41 | string value = 1; 42 | TimingDataInMicro timing_data_in_micro = 2; 43 | UtilResponse util_response = 3; 44 | uint64 request_id = 4; 45 | uint64 recv_stamp = 5; 46 | uint64 send_stamp = 6; 47 | uint64 index_view = 7; 48 | uint32 bucket_server_id = 8; 49 | } 50 | 51 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | 38 | uint64_t GetTimeInSec() 39 | { 40 | struct timeval tv; 41 | gettimeofday(&tv, NULL); 42 | return tv.tv_sec; 43 | } 44 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | 38 | uint64_t GetTimeInSec() 39 | { 40 | struct timeval tv; 41 | gettimeofday(&tv, NULL); 42 | return tv.tv_sec; 43 | } 44 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/ChangeLog: -------------------------------------------------------------------------------- 1 | Version 1.6.11 2 | * bug fixes 3 | 4 | Version 1.6.10 5 | * fixed a radiusSearch bug introduced in 1.6.9 6 | 7 | Version 1.6.9 8 | * bug fixes 9 | * fixed radius search bug on MSVC compiler 10 | * fixed windows linking problems 11 | 12 | Version 1.6.8 13 | * bug fixes, low dimensional search speedup 14 | 15 | Version 1.6.7 16 | * bug fixes 17 | 18 | Version 1.6.6 19 | * misc bug fixes 20 | 21 | Version 1.6.5 22 | * fix compilation problem on some C++ compilers 23 | * fixes in the python bindings 24 | 25 | Version 1.6.4 26 | * small bug fix 27 | 28 | Version 1.6.3 29 | * radius search speedup 30 | 31 | Version 1.6.2 32 | * slight API changes to the C++ bindings, now the main index type is templated 33 | * on the distance functor which makes it easier to use custom distances 34 | * new kd-tree implementation optimized for low dimensionality data 35 | * experimental MPI support for cluster computing 36 | 37 | Version 1.5 38 | * new C++ templated API 39 | * saving/loading of indices to disk 40 | * threadsafe search 41 | * new distance types (thanks to Radu Bogdan Rusu and Romain Thibaux for the patch) 42 | * (api change) autotuned is no longer selected by passing a precision >0, it's used when the algorithm type is set to autotuned 43 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "timing.h" 8 | #include 9 | #include 10 | 11 | uint64_t GetTimeInMicro() 12 | { 13 | struct timeval tv; 14 | gettimeofday(&tv, NULL); 15 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 16 | } 17 | 18 | void GetCpuTimes(uint64_t* user_time, 19 | uint64_t* system_time, 20 | uint64_t* io_time, 21 | uint64_t* idle_time) 22 | { 23 | std::ifstream proc_stat("/proc/stat"); 24 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 25 | std::vector cpu_times; 26 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 27 | if (cpu_times.size() < 4) 28 | { 29 | std::cout << "Size of CPU times cannot be less than 4\n"; 30 | exit(0); 31 | } 32 | (*user_time) = cpu_times[0] + cpu_times[1]; 33 | (*system_time) = cpu_times[2]; 34 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 35 | (*idle_time) = cpu_times[3]; 36 | } 37 | 38 | uint64_t GetTimeInSec() 39 | { 40 | struct timeval tv; 41 | gettimeofday(&tv, NULL); 42 | return tv.tv_sec; 43 | } 44 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | #include 4 | 5 | std::string ExecuteShellCommand(const char* cmd) 6 | { 7 | char buffer[128]; 8 | std::string result = ""; 9 | std::shared_ptr pipe(popen(cmd, "r"), pclose); 10 | if (!pipe) throw std::runtime_error("popen() failed!"); 11 | while (!feof(pipe.get())) { 12 | if (fgets(buffer, 128, pipe.get()) != NULL) 13 | result += buffer; 14 | } 15 | return result; 16 | } 17 | 18 | void StringSplit(std::string str, 19 | char delimiter, 20 | std::vector* queries_str_vec) 21 | { 22 | std::stringstream ss(str); // Turn the string into a stream. 23 | std::string tok; 24 | 25 | while(std::getline(ss, tok, delimiter)) { 26 | if (queries_str_vec->size() == 2048) break; 27 | queries_str_vec->push_back(tok); 28 | } 29 | } 30 | 31 | void GetPerf(pid_t pid) { 32 | std::ostringstream str_pid; 33 | str_pid << pid; 34 | const std::string my_pid(str_pid.str()); 35 | std::string s = "perf stat -e context-switches -I 60000 -p " + my_pid + " &"; 36 | char* cmd = new char[s.length() + 1]; 37 | std::strcpy(cmd, s.c_str()); 38 | ExecuteShellCommand(cmd); 39 | } 40 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | #include 4 | 5 | std::string ExecuteShellCommand(const char* cmd) 6 | { 7 | char buffer[128]; 8 | std::string result = ""; 9 | std::shared_ptr pipe(popen(cmd, "r"), pclose); 10 | if (!pipe) throw std::runtime_error("popen() failed!"); 11 | while (!feof(pipe.get())) { 12 | if (fgets(buffer, 128, pipe.get()) != NULL) 13 | result += buffer; 14 | } 15 | return result; 16 | } 17 | 18 | void StringSplit(std::string str, 19 | char delimiter, 20 | std::vector* queries_str_vec) 21 | { 22 | std::stringstream ss(str); // Turn the string into a stream. 23 | std::string tok; 24 | 25 | while(std::getline(ss, tok, delimiter)) { 26 | if (queries_str_vec->size() == 2048) break; 27 | queries_str_vec->push_back(tok); 28 | } 29 | } 30 | 31 | void GetPerf(pid_t pid) { 32 | std::ostringstream str_pid; 33 | str_pid << pid; 34 | const std::string my_pid(str_pid.str()); 35 | std::string s = "perf stat -e context-switches -I 60000 -p " + my_pid + " &"; 36 | char* cmd = new char[s.length() + 1]; 37 | std::strcpy(cmd, s.c_str()); 38 | ExecuteShellCommand(cmd); 39 | } 40 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/service/helper_files/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | #include 4 | 5 | std::string ExecuteShellCommand(const char* cmd) 6 | { 7 | char buffer[128]; 8 | std::string result = ""; 9 | std::shared_ptr pipe(popen(cmd, "r"), pclose); 10 | if (!pipe) throw std::runtime_error("popen() failed!"); 11 | while (!feof(pipe.get())) { 12 | if (fgets(buffer, 128, pipe.get()) != NULL) 13 | result += buffer; 14 | } 15 | return result; 16 | } 17 | 18 | void StringSplit(std::string str, 19 | char delimiter, 20 | std::vector* queries_str_vec) 21 | { 22 | std::stringstream ss(str); // Turn the string into a stream. 23 | std::string tok; 24 | 25 | while(std::getline(ss, tok, delimiter)) { 26 | if (queries_str_vec->size() == 2048) break; 27 | queries_str_vec->push_back(tok); 28 | } 29 | } 30 | 31 | void GetPerf(pid_t pid) { 32 | std::ostringstream str_pid; 33 | str_pid << pid; 34 | const std::string my_pid(str_pid.str()); 35 | std::string s = "perf stat -e context-switches -I 60000 -p " + my_pid + " &"; 36 | char* cmd = new char[s.length() + 1]; 37 | std::strcpy(cmd, s.c_str()); 38 | ExecuteShellCommand(cmd); 39 | } 40 | -------------------------------------------------------------------------------- /src/Recommend/protoc_files/cf.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package collaborative_filtering; 4 | 5 | service CFService{ 6 | rpc CF(CFRequest) returns (CFResponse) {} 7 | } 8 | 9 | // The request message containing the query point. 10 | 11 | message UtilRequest { 12 | bool util_request = 1; 13 | } 14 | 15 | message CFRequest { 16 | uint32 user = 1; 17 | uint32 item = 2; 18 | UtilRequest util_request = 4; 19 | uint64 request_id = 5; 20 | uint64 recommender_view = 6; 21 | uint32 cf_server_id = 7; 22 | } 23 | 24 | message TimingDataInMicro{ 25 | uint64 unpack_cf_srv_req_time_in_micro = 1; 26 | uint64 calculate_cf_srv_time_in_micro = 2; 27 | uint64 pack_cf_srv_resp_time_in_micro = 3; 28 | uint64 bucket_time = 5; 29 | float cpu_util = 4; 30 | } 31 | 32 | message UtilResponse { 33 | bool util_present = 1; 34 | uint64 user_time = 2; 35 | uint64 system_time = 3; 36 | uint64 io_time = 4; 37 | uint64 idle_time = 5; 38 | } 39 | 40 | message CFResponse { 41 | float rating = 1; 42 | TimingDataInMicro timing_data_in_micro = 2; 43 | UtilResponse util_response = 3; 44 | uint64 request_id = 4; 45 | uint64 recv_stamp = 5; 46 | uint64 send_stamp = 6; 47 | uint64 recommender_view = 7; 48 | uint32 cf_server_id = 8; 49 | } 50 | 51 | -------------------------------------------------------------------------------- /src/SetAlgebra/protoc_files/intersection.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package intersection; 4 | 5 | service IntersectionService{ 6 | rpc Intersection(IntersectionRequest) returns (IntersectionResponse) {} 7 | } 8 | 9 | 10 | message UtilRequest { 11 | bool util_request = 1; 12 | } 13 | 14 | message IntersectionRequest { 15 | repeated uint64 word_ids = 1; 16 | uint32 intersection_server_id = 2; 17 | uint32 shard_size = 3; 18 | UtilRequest util_request = 4; 19 | uint64 request_id = 5; 20 | uint64 index_view = 6; 21 | } 22 | 23 | message TimingDataInMicro { 24 | uint64 unpack_intersection_srv_req_time_in_micro = 1; 25 | uint64 calculate_intersection_time_in_micro = 2; 26 | uint64 pack_intersection_srv_resp_time_in_micro = 3; 27 | float cpu_util = 4; 28 | } 29 | 30 | message UtilResponse { 31 | bool util_present = 1; 32 | uint64 user_time = 2; 33 | uint64 system_time = 3; 34 | uint64 io_time = 4; 35 | uint64 idle_time = 5; 36 | } 37 | 38 | message IntersectionResponse { 39 | repeated uint64 doc_ids = 1; 40 | TimingDataInMicro timing_data_in_micro = 2; 41 | UtilResponse util_response = 3; 42 | uint64 request_id = 4; 43 | uint64 recv_stamp = 5; 44 | uint64 send_stamp = 6; 45 | uint64 index_view = 7; 46 | uint32 request_server_id = 8; 47 | } 48 | 49 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/src/thread_safe_queue.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeQueue 8 | { 9 | public: 10 | 11 | T pop() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (queue_.empty()) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | auto item = queue_.front(); 19 | queue_.pop(); 20 | return item; 21 | } 22 | 23 | void pop(T& item) 24 | { 25 | std::unique_lock mlock(mutex_); 26 | while (queue_.empty()) 27 | { 28 | cond_.wait(mlock); 29 | } 30 | item = queue_.front(); 31 | queue_.pop(); 32 | } 33 | 34 | void push(const T& item) 35 | { 36 | std::unique_lock mlock(mutex_); 37 | queue_.push(item); 38 | mlock.unlock(); 39 | cond_.notify_one(); 40 | } 41 | 42 | void push(T&& item) 43 | { 44 | std::unique_lock mlock(mutex_); 45 | queue_.push(std::move(item)); 46 | mlock.unlock(); 47 | cond_.notify_one(); 48 | } 49 | 50 | unsigned int size() 51 | { 52 | return queue_.size(); 53 | } 54 | 55 | private: 56 | std::queue queue_; 57 | std::mutex mutex_; 58 | std::condition_variable cond_; 59 | }; 60 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/src/thread_safe_queue.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeQueue 8 | { 9 | public: 10 | 11 | T pop() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (queue_.empty()) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | auto item = queue_.front(); 19 | queue_.pop(); 20 | return item; 21 | } 22 | 23 | void pop(T& item) 24 | { 25 | std::unique_lock mlock(mutex_); 26 | while (queue_.empty()) 27 | { 28 | cond_.wait(mlock); 29 | } 30 | item = queue_.front(); 31 | queue_.pop(); 32 | } 33 | 34 | void push(const T& item) 35 | { 36 | std::unique_lock mlock(mutex_); 37 | queue_.push(item); 38 | mlock.unlock(); 39 | cond_.notify_one(); 40 | } 41 | 42 | void push(T&& item) 43 | { 44 | std::unique_lock mlock(mutex_); 45 | queue_.push(std::move(item)); 46 | mlock.unlock(); 47 | cond_.notify_one(); 48 | } 49 | 50 | unsigned int size() 51 | { 52 | return queue_.size(); 53 | } 54 | 55 | private: 56 | std::queue queue_; 57 | std::mutex mutex_; 58 | std::condition_variable cond_; 59 | }; 60 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/src/thread_safe_queue.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeQueue 8 | { 9 | public: 10 | 11 | T pop() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (queue_.empty()) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | auto item = queue_.front(); 19 | queue_.pop(); 20 | return item; 21 | } 22 | 23 | void pop(T& item) 24 | { 25 | std::unique_lock mlock(mutex_); 26 | while (queue_.empty()) 27 | { 28 | cond_.wait(mlock); 29 | } 30 | item = queue_.front(); 31 | queue_.pop(); 32 | } 33 | 34 | void push(const T& item) 35 | { 36 | std::unique_lock mlock(mutex_); 37 | queue_.push(item); 38 | mlock.unlock(); 39 | cond_.notify_one(); 40 | } 41 | 42 | void push(T&& item) 43 | { 44 | std::unique_lock mlock(mutex_); 45 | queue_.push(std::move(item)); 46 | mlock.unlock(); 47 | cond_.notify_one(); 48 | } 49 | 50 | unsigned int size() 51 | { 52 | return queue_.size(); 53 | } 54 | 55 | private: 56 | std::queue queue_; 57 | std::mutex mutex_; 58 | std::condition_variable cond_; 59 | }; 60 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/server_helper.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "protoc_files/lookup.grpc.pb.h" 7 | 8 | #include "lookup_service/src/thread_safe_map.cpp" 9 | 10 | #ifndef __CLIENT_HELPER_H_INCLUDED__ 11 | #define __CLIENT_HELPER_H_INCLUDED__ 12 | #define CHECK(condition, error_message) if (!condition) {std::cerr << __FILE__ << ": " << __LINE__ << ": " << error_message << "\n"; exit(-1);} 13 | 14 | void CreateMemcachedConn(const int memcached_port, 15 | memcached_st *memc, 16 | memcached_return* rc); 17 | 18 | void UnpackBucketServiceRequest(const lookup::Key &request, 19 | uint32_t* operation, 20 | std::string* key, 21 | std::string* value); 22 | 23 | void Get(memcached_st* memc, 24 | memcached_return* rc, 25 | const std::string key, 26 | std::string* value); 27 | 28 | void Set(memcached_st* memc, 29 | memcached_return* rc, 30 | const std::string key, 31 | std::string* value); 32 | 33 | #if 0 34 | void Get(memcached_st* memc, 35 | memcached_return* rc, 36 | ThreadSafeMap &key_value_store, 37 | std::string key, 38 | std::string* value); 39 | 40 | void Set(memcached_st* memc, 41 | memcached_return* rc, 42 | ThreadSafeMap* key_value_store, 43 | std::string key, 44 | std::string* value); 45 | #endif 46 | 47 | void Update(const std::string key, 48 | const std::string value); 49 | 50 | #endif // __CLIENT_HELPER_H_INCLUDED__ 51 | 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, University of Michigan 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /src/HDSearch/load_generator/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "bucket_service/src/dist_calc.h" 5 | 6 | //#include 7 | #include 8 | #if 0 9 | #include 10 | #include 11 | 12 | namespace std { 13 | template struct _Unique_if { 14 | typedef unique_ptr _Single_object; 15 | }; 16 | 17 | template struct _Unique_if { 18 | typedef unique_ptr _Unknown_bound; 19 | }; 20 | 21 | template struct _Unique_if { 22 | typedef void _Known_bound; 23 | }; 24 | 25 | template 26 | typename _Unique_if::_Single_object 27 | make_unique(Args&&... args) { 28 | return unique_ptr(new T(std::forward(args)...)); 29 | } 30 | 31 | template 32 | typename _Unique_if::_Unknown_bound 33 | make_unique(size_t n) { 34 | typedef typename remove_extent::type U; 35 | return unique_ptr(new U[n]()); 36 | } 37 | 38 | template 39 | typename _Unique_if::_Known_bound 40 | make_unique(Args&&...) = delete; 41 | } 42 | #endif 43 | 44 | #if 0 45 | template 46 | std::unique_ptr make_unique(Args&&... args) { 47 | return std::unique_ptr(new T(std::forward(args)...)); 48 | } 49 | #endif 50 | 51 | std::string ExecuteShellCommand(const char* cmd); 52 | void StringSplit(std::string str, 53 | char delimiter, 54 | std::vector* queries_str_vec); 55 | 56 | void GetPerf(pid_t pid); 57 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/thread_safe_queue.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | template 7 | class ThreadSafeQueue 8 | { 9 | public: 10 | 11 | T pop() 12 | { 13 | std::unique_lock mlock(mutex_); 14 | while (queue_.empty()) 15 | { 16 | cond_.wait(mlock); 17 | } 18 | auto item = queue_.front(); 19 | queue_.pop(); 20 | return item; 21 | } 22 | 23 | void pop(T& item) 24 | { 25 | std::unique_lock mlock(mutex_); 26 | while (queue_.empty()) 27 | { 28 | cond_.wait(mlock); 29 | } 30 | item = queue_.front(); 31 | queue_.pop(); 32 | } 33 | 34 | void push(const T& item) 35 | { 36 | std::unique_lock mlock(mutex_); 37 | queue_.push(item); 38 | mlock.unlock(); 39 | cond_.notify_one(); 40 | } 41 | 42 | void push(T&& item) 43 | { 44 | std::unique_lock mlock(mutex_); 45 | queue_.push(std::move(item)); 46 | mlock.unlock(); 47 | cond_.notify_one(); 48 | } 49 | 50 | int peek() 51 | { 52 | return queue_.size(); 53 | } 54 | 55 | unsigned int size() 56 | { 57 | return queue_.size(); 58 | } 59 | 60 | private: 61 | std::queue queue_; 62 | std::mutex mutex_; 63 | std::condition_variable cond_; 64 | }; 65 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/custom_priority_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __CUSTOM_PRIORITY_QUEUE_INCLUDED__ 2 | #define __CUSTOM_PRIORITY_QUEUE_INCLUDED__ 3 | 4 | #include 5 | #include "multiple_points.h" 6 | 7 | typedef std::pair PointIDDistPair; 8 | // Create top element of priority queue based on the second element of a pair. 9 | class CompareDist 10 | { 11 | public: 12 | bool operator()(PointIDDistPair n1, PointIDDistPair n2) 13 | { 14 | return n1.second < n2.second; 15 | } 16 | }; 17 | 18 | // Class implementation right now, because we might change the data structure. 19 | class CustomPriorityQueue 20 | { 21 | public: 22 | CustomPriorityQueue() = default; 23 | // Checks & returns true if the priority queue is empty. 24 | bool IsEmpty() const; 25 | // Returns the size of the priority queue. 26 | unsigned GetSize() const; 27 | // Adds a pair of to the priority queue. 28 | void AddToPriorityQueue(PointIDDistPair &knn_point); 29 | // Removes the pair with the largest distance from query. 30 | void RemoveTopElement(); 31 | // Returns the Point with the largest distance from query. 32 | uint32_t GetTopPointID() const; 33 | // Returns the largest distance from query. 34 | float GetTopDistance() const; 35 | private: 36 | // Priority queue of a pair of the Point and its distance from query. 37 | std::priority_queue, 39 | CompareDist> knn_priority_queue_; 40 | }; 41 | #endif //__CUSTOM_PRIORITY_QUEUE_INCLUDED__ 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "../../../bucket_service/src/dist_calc.h" 5 | 6 | //#include 7 | #include 8 | #if 0 9 | #include 10 | #include 11 | 12 | namespace std { 13 | template struct _Unique_if { 14 | typedef unique_ptr _Single_object; 15 | }; 16 | 17 | template struct _Unique_if { 18 | typedef unique_ptr _Unknown_bound; 19 | }; 20 | 21 | template struct _Unique_if { 22 | typedef void _Known_bound; 23 | }; 24 | 25 | template 26 | typename _Unique_if::_Single_object 27 | make_unique(Args&&... args) { 28 | return unique_ptr(new T(std::forward(args)...)); 29 | } 30 | 31 | template 32 | typename _Unique_if::_Unknown_bound 33 | make_unique(size_t n) { 34 | typedef typename remove_extent::type U; 35 | return unique_ptr(new U[n]()); 36 | } 37 | 38 | template 39 | typename _Unique_if::_Known_bound 40 | make_unique(Args&&...) = delete; 41 | } 42 | #endif 43 | 44 | #if 0 45 | template 46 | std::unique_ptr make_unique(Args&&... args) { 47 | return std::unique_ptr(new T(std::forward(args)...)); 48 | } 49 | #endif 50 | 51 | std::string ExecuteShellCommand(const char* cmd); 52 | void StringSplit(std::string str, 53 | char delimiter, 54 | std::vector* queries_str_vec); 55 | 56 | void GetPerf(pid_t pid); 57 | -------------------------------------------------------------------------------- /src/HDSearch/load_generator/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "timing.h" 6 | #include 7 | 8 | uint64_t GetTimeInMicro() 9 | { 10 | struct timeval tv; 11 | gettimeofday(&tv, NULL); 12 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 13 | } 14 | 15 | uint64_t GetTimeInSec() 16 | { 17 | struct timeval tv; 18 | gettimeofday(&tv, NULL); 19 | return tv.tv_sec; 20 | } 21 | 22 | double Rdtsc() 23 | { 24 | unsigned int lo, hi; 25 | __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); 26 | // A.S TODO: Get the frequency using lscpu 27 | uint64_t frequency = 1200632000; 28 | double seconds = (double)(((uint64_t)hi << 32) | lo)/(double)frequency; 29 | return seconds; 30 | } 31 | 32 | uint64_t RdtscCycles() 33 | { 34 | unsigned int lo, hi; 35 | __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); 36 | return (((uint64_t)hi << 32) | lo); 37 | } 38 | 39 | void GetCpuTimes(uint64_t* user_time, 40 | uint64_t* system_time, 41 | uint64_t* io_time, 42 | uint64_t* idle_time) 43 | { 44 | std::ifstream proc_stat("/proc/stat"); 45 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 46 | std::vector cpu_times; 47 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 48 | if (cpu_times.size() < 4) 49 | { 50 | std::cout << "Size of CPU times cannot be less than 4\n"; 51 | exit(0); 52 | } 53 | (*user_time) = cpu_times[0] + cpu_times[1]; 54 | (*system_time) = cpu_times[2]; 55 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 56 | (*idle_time) = cpu_times[3]; 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/service/helper_files/timing.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "timing.h" 6 | #include 7 | 8 | uint64_t GetTimeInMicro() 9 | { 10 | struct timeval tv; 11 | gettimeofday(&tv, NULL); 12 | return (tv.tv_sec*(uint64_t)1000000+tv.tv_usec); 13 | } 14 | 15 | uint64_t GetTimeInSec() 16 | { 17 | struct timeval tv; 18 | gettimeofday(&tv, NULL); 19 | return tv.tv_sec; 20 | } 21 | 22 | double Rdtsc() 23 | { 24 | unsigned int lo, hi; 25 | __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); 26 | // A.S TODO: Get the frequency using lscpu 27 | uint64_t frequency = 1200632000; 28 | double seconds = (double)(((uint64_t)hi << 32) | lo)/(double)frequency; 29 | return seconds; 30 | } 31 | 32 | uint64_t RdtscCycles() 33 | { 34 | unsigned int lo, hi; 35 | __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); 36 | return (((uint64_t)hi << 32) | lo); 37 | } 38 | 39 | void GetCpuTimes(uint64_t* user_time, 40 | uint64_t* system_time, 41 | uint64_t* io_time, 42 | uint64_t* idle_time) 43 | { 44 | std::ifstream proc_stat("/proc/stat"); 45 | proc_stat.ignore(5, ' '); // Skip the 'cpu' prefix. 46 | std::vector cpu_times; 47 | for (size_t time; proc_stat >> time; cpu_times.push_back(time)); 48 | if (cpu_times.size() < 4) 49 | { 50 | std::cout << "Size of CPU times cannot be less than 4\n"; 51 | exit(0); 52 | } 53 | (*user_time) = cpu_times[0] + cpu_times[1]; 54 | (*system_time) = cpu_times[2]; 55 | (*io_time) = cpu_times[4] + cpu_times[5] + cpu_times[6]; 56 | (*idle_time) = cpu_times[3]; 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/flann_cpp.cpp: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #include "flann/flann.hpp" 31 | -------------------------------------------------------------------------------- /src/HDSearch/protoc_files/bucket.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package bucket; 4 | 5 | service DistanceService{ 6 | rpc GetNearestNeighbors(NearestNeighborRequest) returns (NearestNeighborResponse) {} 7 | } 8 | 9 | // The request message containing the query point. 10 | message DataPoint { 11 | repeated float data_point = 1; 12 | } 13 | 14 | message MultipleDataPoints{ 15 | repeated DataPoint multiple_data_points = 1; 16 | } 17 | 18 | message PointIdList { 19 | repeated uint32 point_id = 1; 20 | uint32 bucket_server_id = 2; 21 | } 22 | 23 | message UtilRequest { 24 | bool util_request = 1; 25 | } 26 | 27 | message NearestNeighborRequest { 28 | repeated uint64 queries = 1; 29 | repeated PointIdList maybe_neighbor_list = 2; 30 | uint32 requested_neighbor_count = 3; 31 | uint32 bucket_server_id = 4; 32 | uint32 shard_size = 5; 33 | UtilRequest util_request = 6; 34 | uint64 request_id = 7; 35 | uint64 index_view = 8; 36 | } 37 | 38 | message TimingDataInMicro{ 39 | uint64 unpack_bucket_req_time_in_micro = 1; 40 | uint64 calculate_knn_time_in_micro = 2; 41 | uint64 pack_bucket_resp_time_in_micro = 3; 42 | float cpu_util = 4; 43 | } 44 | 45 | message UtilResponse { 46 | bool util_present = 1; 47 | uint64 user_time = 2; 48 | uint64 system_time = 3; 49 | uint64 io_time = 4; 50 | uint64 idle_time = 5; 51 | } 52 | 53 | message NearestNeighborResponse { 54 | repeated PointIdList neighbor_ids = 1; 55 | repeated uint64 queries = 2; 56 | TimingDataInMicro timing_data_in_micro = 3; 57 | UtilResponse util_response = 4; 58 | uint64 request_id = 5; 59 | uint64 recv_stamp = 6; 60 | uint64 send_stamp = 7; 61 | uint64 index_view = 8; 62 | uint32 bucket_server_id = 9; 63 | } 64 | 65 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | //#include 10 | #include 11 | #if 0 12 | #include 13 | #include 14 | 15 | namespace std { 16 | template struct _Unique_if { 17 | typedef unique_ptr _Single_object; 18 | }; 19 | 20 | template struct _Unique_if { 21 | typedef unique_ptr _Unknown_bound; 22 | }; 23 | 24 | template struct _Unique_if { 25 | typedef void _Known_bound; 26 | }; 27 | 28 | template 29 | typename _Unique_if::_Single_object 30 | make_unique(Args&&... args) { 31 | return unique_ptr(new T(std::forward(args)...)); 32 | } 33 | 34 | template 35 | typename _Unique_if::_Unknown_bound 36 | make_unique(size_t n) { 37 | typedef typename remove_extent::type U; 38 | return unique_ptr(new U[n]()); 39 | } 40 | 41 | template 42 | typename _Unique_if::_Known_bound 43 | make_unique(Args&&...) = delete; 44 | } 45 | #endif 46 | 47 | #if 0 48 | template 49 | std::unique_ptr make_unique(Args&&... args) { 50 | return std::unique_ptr(new T(std::forward(args)...)); 51 | } 52 | #endif 53 | 54 | std::string ExecuteShellCommand(const char* cmd); 55 | void StringSplit(std::string str, 56 | char delimiter, 57 | std::vector* queries_str_vec); 58 | 59 | void GetPerf(pid_t pid); 60 | 61 | #endif //__UTILS_H_INCLUDED__ 62 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | //#include 10 | #include 11 | #if 0 12 | #include 13 | #include 14 | 15 | namespace std { 16 | template struct _Unique_if { 17 | typedef unique_ptr _Single_object; 18 | }; 19 | 20 | template struct _Unique_if { 21 | typedef unique_ptr _Unknown_bound; 22 | }; 23 | 24 | template struct _Unique_if { 25 | typedef void _Known_bound; 26 | }; 27 | 28 | template 29 | typename _Unique_if::_Single_object 30 | make_unique(Args&&... args) { 31 | return unique_ptr(new T(std::forward(args)...)); 32 | } 33 | 34 | template 35 | typename _Unique_if::_Unknown_bound 36 | make_unique(size_t n) { 37 | typedef typename remove_extent::type U; 38 | return unique_ptr(new U[n]()); 39 | } 40 | 41 | template 42 | typename _Unique_if::_Known_bound 43 | make_unique(Args&&...) = delete; 44 | } 45 | #endif 46 | 47 | #if 0 48 | template 49 | std::unique_ptr make_unique(Args&&... args) { 50 | return std::unique_ptr(new T(std::forward(args)...)); 51 | } 52 | #endif 53 | 54 | std::string ExecuteShellCommand(const char* cmd); 55 | void StringSplit(std::string str, 56 | char delimiter, 57 | std::vector* queries_str_vec); 58 | 59 | void GetPerf(pid_t pid); 60 | 61 | #endif //__UTILS_H_INCLUDED__ 62 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/service/helper_files/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __UTILS_H_INCLUDED__ 2 | #define __UTILS_H_INCLUDED__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | //#include 10 | #include 11 | #if 0 12 | #include 13 | #include 14 | 15 | namespace std { 16 | template struct _Unique_if { 17 | typedef unique_ptr _Single_object; 18 | }; 19 | 20 | template struct _Unique_if { 21 | typedef unique_ptr _Unknown_bound; 22 | }; 23 | 24 | template struct _Unique_if { 25 | typedef void _Known_bound; 26 | }; 27 | 28 | template 29 | typename _Unique_if::_Single_object 30 | make_unique(Args&&... args) { 31 | return unique_ptr(new T(std::forward(args)...)); 32 | } 33 | 34 | template 35 | typename _Unique_if::_Unknown_bound 36 | make_unique(size_t n) { 37 | typedef typename remove_extent::type U; 38 | return unique_ptr(new U[n]()); 39 | } 40 | 41 | template 42 | typename _Unique_if::_Known_bound 43 | make_unique(Args&&...) = delete; 44 | } 45 | #endif 46 | 47 | #if 0 48 | template 49 | std::unique_ptr make_unique(Args&&... args) { 50 | return std::unique_ptr(new T(std::forward(args)...)); 51 | } 52 | #endif 53 | 54 | std::string ExecuteShellCommand(const char* cmd); 55 | void StringSplit(std::string str, 56 | char delimiter, 57 | std::vector* queries_str_vec); 58 | 59 | void GetPerf(pid_t pid); 60 | 61 | #endif //__UTILS_H_INCLUDED__ 62 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/point.h: -------------------------------------------------------------------------------- 1 | #ifndef __POINT_H_INCLUDED__ 2 | #define __POINT_H_INCLUDED__ 3 | #include 4 | #include 5 | 6 | #define CHECK(condition, error_message) if (!condition) {std::cerr << __FILE__ << ": " << __LINE__ << ": " << error_message << "\n"; exit(-1);} 7 | 8 | // Class to define/access a Point: floats for each dimension. 9 | class Point 10 | { 11 | public: 12 | // Point constructor: initializes a point with one float value: 0.0 13 | Point() = default; 14 | // To initialize point with "size" dimensions, each dim = value. 15 | Point(const int size, const float &value) 16 | { 17 | point_.assign(size, value); 18 | } 19 | // Resize a point. 20 | void Resize(const int size, const float value); 21 | // Add a float to a particular dimension (index). 22 | void AddValueToIndex(const int index, const float value); 23 | // Returns the float at a particular dimension (index). 24 | float GetValueAtIndex(const int index) const; 25 | /* Given a float array, create a point from it. 26 | Helps when reading a binary dataset file and creating points. 27 | In: float array 28 | Out: the point object itself.*/ 29 | void CreatePointFromFloatArray(const float float_arr[], 30 | const int float_arr_size); 31 | // Returns the number of floats in the point (dimension). 32 | unsigned GetSize() const; 33 | // Check if two points are equal. 34 | bool Equal(const Point &p) const; 35 | // Prints a point to terminal. 36 | void PrintPoint() const; 37 | // private: 38 | // Defines the data structure that holds collection of floats i.e point. 39 | std::vector point_; 40 | }; 41 | #endif // __POINT_H_INCLUDED__. 42 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/config.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef FLANN_CONFIG_H_ 31 | #define FLANN_CONFIG_H_ 32 | 33 | #ifdef FLANN_VERSION_ 34 | #undef FLANN_VERSION_ 35 | #endif 36 | #define FLANN_VERSION_ "1.9.1" 37 | 38 | #endif /* FLANN_CONFIG_H_ */ 39 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/config.h.in: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef FLANN_CONFIG_H_ 31 | #define FLANN_CONFIG_H_ 32 | 33 | #ifdef FLANN_VERSION_ 34 | #undef FLANN_VERSION_ 35 | #endif 36 | #define FLANN_VERSION_ "${FLANN_VERSION}" 37 | 38 | #endif /* FLANN_CONFIG_H_ */ 39 | -------------------------------------------------------------------------------- /src/Router/protoc_files/router.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/descriptor.proto"; 4 | 5 | package router; 6 | 7 | service RouterService { 8 | // Defining service between load generator and index. 9 | // Input: Queries, number of nearest neighbors to compute. 10 | // Output: K-nn 11 | rpc Router (RouterRequest) returns (LookupResponse) {} 12 | } 13 | 14 | 15 | message UtilRequest { 16 | bool util_request = 1; 17 | } 18 | 19 | message RouterRequest { 20 | string key = 1; 21 | string value = 2; 22 | uint32 operation = 3; 23 | UtilRequest util_request = 4; 24 | bool last_request = 5; 25 | uint64 resp_time = 6; 26 | bool kill = 7; 27 | uint64 request_id = 8; 28 | uint32 load = 9; 29 | } 30 | 31 | message Util { 32 | uint64 user_time = 1; 33 | uint64 system_time = 2; 34 | uint64 io_time = 3; 35 | uint64 idle_time = 4; 36 | } 37 | 38 | message UtilResponse { 39 | bool util_present = 1; 40 | repeated Util lookup_srv_util = 2; 41 | Util router_util = 3; 42 | } 43 | 44 | message LookupResponse { 45 | uint64 create_router_req_time = 1; 46 | uint64 unpack_router_req_time = 2; 47 | uint64 update_router_util_time = 3; 48 | uint64 get_lookup_srv_responses_time = 5; 49 | uint64 create_lookup_srv_req_time = 6; 50 | uint64 unpack_lookup_srv_resp_time = 7; 51 | uint64 unpack_lookup_srv_req_time = 8; 52 | uint64 lookup_srv_time = 9; 53 | uint64 pack_lookup_srv_resp_time = 10; 54 | uint64 pack_router_resp_time = 11; 55 | uint32 number_of_lookup_servers = 12; 56 | UtilResponse util_response = 13; 57 | uint64 resp_time = 14; 58 | uint64 router_time = 15; 59 | uint64 start_stamp = 16; 60 | uint64 end_stamp = 17; 61 | uint64 request_id = 18; 62 | uint64 num_inline = 19; 63 | uint64 num_workers = 20; 64 | uint64 num_resp = 21; 65 | bool kill_ack = 22; 66 | string value = 23; 67 | uint64 merge_time = 24; 68 | } 69 | -------------------------------------------------------------------------------- /src/SetAlgebra/protoc_files/union.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/descriptor.proto"; 4 | 5 | package union_service; 6 | 7 | service UnionService { 8 | // Defining service between load generator and union. 9 | // Input: Queries, number of nearest neighbors to compute. 10 | // Output: K-nn 11 | rpc Union (UnionRequest) returns (UnionResponse) {} 12 | } 13 | 14 | message UtilRequest { 15 | bool util_request = 1; 16 | } 17 | 18 | message UnionRequest { 19 | repeated uint64 word_ids = 1; 20 | UtilRequest util_request = 2; 21 | bool last_request = 3; 22 | uint64 resp_time = 4; 23 | bool kill = 5; 24 | uint64 request_id = 6; 25 | uint32 load = 7; 26 | } 27 | 28 | message Util { 29 | uint64 user_time = 1; 30 | uint64 system_time = 2; 31 | uint64 io_time = 3; 32 | uint64 idle_time = 4; 33 | } 34 | 35 | message UtilResponse { 36 | bool util_present = 1; 37 | repeated Util intersection_srv_util = 2; 38 | Util union_util = 3; 39 | } 40 | 41 | message UnionResponse { 42 | repeated uint64 doc_ids = 1; 43 | uint64 create_union_req_time = 2; 44 | uint64 unpack_union_req_time = 3; 45 | uint64 update_union_util_time = 4; 46 | uint64 get_intersection_srv_responses_time = 5; 47 | uint64 create_intersection_srv_req_time = 6; 48 | uint64 unpack_intersection_srv_resp_time = 7; 49 | uint64 unpack_intersection_srv_req_time = 8; 50 | uint64 calculate_intersection_time = 9; 51 | uint64 pack_intersection_srv_resp_time = 10; 52 | uint64 calculate_union_time = 11; 53 | uint64 pack_union_resp_time = 12; 54 | uint32 number_of_intersection_servers = 13; 55 | UtilResponse util_response = 14; 56 | uint64 resp_time = 15; 57 | uint64 union_time = 16; 58 | uint64 start_stamp = 17; 59 | uint64 end_stamp = 18; 60 | uint64 request_id = 19; 61 | uint64 num_inline = 20; 62 | uint64 num_workers = 21; 63 | uint64 num_resp = 22; 64 | bool kill_ack = 23; 65 | } 66 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/thread_safe_circ_buffer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | // Thread safe circular buffer 7 | template 8 | class CircBuffer : private boost::noncopyable 9 | { 10 | public: 11 | typedef boost::mutex::scoped_lock lock; 12 | CircBuffer(){}; 13 | CircBuffer(int n) {cb.set_capacity(n);} 14 | void send (T imdata) { 15 | lock lk(monitor); 16 | cb.push_back(imdata); 17 | buffer_not_empty.notify_one(); 18 | } 19 | T receive() { 20 | lock lk(monitor); 21 | while (cb.empty()) 22 | buffer_not_empty.wait(lk); 23 | T imdata = cb.front(); 24 | cb.pop_front(); 25 | return imdata; 26 | } 27 | void clear() { 28 | lock lk(monitor); 29 | cb.clear(); 30 | } 31 | int size() { 32 | lock lk(monitor); 33 | return cb.size(); 34 | } 35 | void set_capacity(int capacity) { 36 | lock lk(monitor); 37 | cb.set_capacity(capacity); 38 | } 39 | private: 40 | boost::condition buffer_not_empty; 41 | boost::mutex monitor; 42 | boost::circular_buffer cb; 43 | }; 44 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/src/thread_safe_circ_buffer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | // Thread safe circular buffer 7 | template 8 | class CircBuffer : private boost::noncopyable 9 | { 10 | public: 11 | typedef boost::mutex::scoped_lock lock; 12 | CircBuffer(){}; 13 | CircBuffer(int n) {cb.set_capacity(n);} 14 | void send (T imdata) { 15 | lock lk(monitor); 16 | cb.push_back(imdata); 17 | buffer_not_empty.notify_one(); 18 | } 19 | T receive() { 20 | lock lk(monitor); 21 | while (cb.empty()) 22 | buffer_not_empty.wait(lk); 23 | T imdata = cb.front(); 24 | cb.pop_front(); 25 | return imdata; 26 | } 27 | void clear() { 28 | lock lk(monitor); 29 | cb.clear(); 30 | } 31 | int size() { 32 | lock lk(monitor); 33 | return cb.size(); 34 | } 35 | void set_capacity(int capacity) { 36 | lock lk(monitor); 37 | cb.set_capacity(capacity); 38 | } 39 | private: 40 | boost::condition buffer_not_empty; 41 | boost::mutex monitor; 42 | boost::circular_buffer cb; 43 | }; 44 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/src/thread_safe_circ_buffer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | // Thread safe circular buffer 7 | template 8 | class CircBuffer : private boost::noncopyable 9 | { 10 | public: 11 | typedef boost::mutex::scoped_lock lock; 12 | CircBuffer(){}; 13 | CircBuffer(int n) {cb.set_capacity(n);} 14 | void send (T imdata) { 15 | lock lk(monitor); 16 | cb.push_back(imdata); 17 | buffer_not_empty.notify_one(); 18 | } 19 | T receive() { 20 | lock lk(monitor); 21 | while (cb.empty()) 22 | buffer_not_empty.wait(lk); 23 | T imdata = cb.front(); 24 | cb.pop_front(); 25 | return imdata; 26 | } 27 | void clear() { 28 | lock lk(monitor); 29 | cb.clear(); 30 | } 31 | int size() { 32 | lock lk(monitor); 33 | return cb.size(); 34 | } 35 | void set_capacity(int capacity) { 36 | lock lk(monitor); 37 | cb.set_capacity(capacity); 38 | } 39 | private: 40 | boost::condition buffer_not_empty; 41 | boost::mutex monitor; 42 | boost::circular_buffer cb; 43 | }; 44 | -------------------------------------------------------------------------------- /src/Recommend/recommender_service/src/thread_safe_circ_buffer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | // Thread safe circular buffer 7 | template 8 | class CircBuffer : private boost::noncopyable 9 | { 10 | public: 11 | typedef boost::mutex::scoped_lock lock; 12 | CircBuffer(){}; 13 | CircBuffer(int n) {cb.set_capacity(n);} 14 | void send (T imdata) { 15 | lock lk(monitor); 16 | cb.push_back(imdata); 17 | buffer_not_empty.notify_one(); 18 | } 19 | T receive() { 20 | lock lk(monitor); 21 | while (cb.empty()) 22 | buffer_not_empty.wait(lk); 23 | T imdata = cb.front(); 24 | cb.pop_front(); 25 | return imdata; 26 | } 27 | void clear() { 28 | lock lk(monitor); 29 | cb.clear(); 30 | } 31 | int size() { 32 | lock lk(monitor); 33 | return cb.size(); 34 | } 35 | void set_capacity(int capacity) { 36 | lock lk(monitor); 37 | cb.set_capacity(capacity); 38 | } 39 | private: 40 | boost::condition buffer_not_empty; 41 | boost::mutex monitor; 42 | boost::circular_buffer cb; 43 | }; 44 | -------------------------------------------------------------------------------- /src/Recommend/protoc_files/recommender.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/descriptor.proto"; 4 | 5 | package recommender; 6 | 7 | service RecommenderService { 8 | // Defining service between load generator and index. 9 | // Input: Queries, number of nearest neighbors to compute. 10 | // Output: K-nn 11 | rpc Recommender (RecommenderRequest) returns (RecommenderResponse) {} 12 | } 13 | 14 | 15 | message UtilRequest { 16 | bool util_request = 1; 17 | } 18 | 19 | message RecommenderRequest { 20 | uint32 user = 1; 21 | uint32 item = 2; 22 | UtilRequest util_request = 4; 23 | bool last_request = 5; 24 | uint64 resp_time = 6; 25 | bool kill = 7; 26 | uint64 request_id = 8; 27 | uint32 load = 9; 28 | } 29 | 30 | message Util { 31 | uint64 user_time = 1; 32 | uint64 system_time = 2; 33 | uint64 io_time = 3; 34 | uint64 idle_time = 4; 35 | } 36 | 37 | message UtilResponse { 38 | bool util_present = 1; 39 | repeated Util cf_srv_util = 2; 40 | Util recommender_util = 3; 41 | } 42 | 43 | message RecommenderResponse { 44 | float rating = 25; 45 | uint64 create_recommender_req_time = 1; 46 | uint64 unpack_recommender_req_time = 2; 47 | uint64 update_recommender_util_time = 3; 48 | uint64 get_cf_srv_responses_time = 5; 49 | uint64 create_cf_srv_req_time = 6; 50 | uint64 unpack_cf_srv_resp_time = 7; 51 | uint64 unpack_cf_srv_req_time = 8; 52 | uint64 calculate_cf_srv_time = 9; 53 | uint64 pack_cf_srv_resp_time = 10; 54 | uint64 pack_recommender_resp_time = 11; 55 | uint32 number_of_cf_servers = 12; 56 | UtilResponse util_response = 13; 57 | uint64 resp_time = 14; 58 | uint64 recommender_time = 15; 59 | uint64 start_stamp = 16; 60 | uint64 end_stamp = 17; 61 | uint64 request_id = 18; 62 | uint64 num_inline = 19; 63 | uint64 num_workers = 20; 64 | uint64 num_resp = 21; 65 | bool kill_ack = 22; 66 | string value = 23; 67 | uint64 merge_time = 24; 68 | uint64 total_calc_time = 26; 69 | } 70 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/mpi/flann_mpi_client.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | 12 | #define IF_RANK0 if (world.rank()==0) 13 | 14 | timeval start_time_; 15 | void start_timer(const std::string& message = "") 16 | { 17 | if (!message.empty()) { 18 | printf("%s", message.c_str()); 19 | fflush(stdout); 20 | } 21 | gettimeofday(&start_time_,NULL); 22 | } 23 | 24 | double stop_timer() 25 | { 26 | timeval end_time; 27 | gettimeofday(&end_time,NULL); 28 | 29 | return double(end_time.tv_sec-start_time_.tv_sec)+ double(end_time.tv_usec-start_time_.tv_usec)/1000000; 30 | } 31 | 32 | float compute_precision(const flann::Matrix& match, const flann::Matrix& indices) 33 | { 34 | int count = 0; 35 | 36 | assert(match.rows == indices.rows); 37 | size_t nn = std::min(match.cols, indices.cols); 38 | 39 | for(size_t i=0; i query; 58 | flann::Matrix match; 59 | 60 | flann::load_from_file(query, "sift100K.h5","query"); 61 | flann::load_from_file(match, "sift100K.h5","match"); 62 | // flann::load_from_file(gt_dists, "sift100K.h5","dists"); 63 | 64 | flann::mpi::Client index("localhost","9999"); 65 | 66 | int nn = 1; 67 | flann::Matrix indices(new int[query.rows*nn], query.rows, nn); 68 | flann::Matrix dists(new float[query.rows*nn], query.rows, nn); 69 | 70 | start_timer("Performing search...\n"); 71 | index.knnSearch(query, indices, dists, nn, flann::SearchParams(64)); 72 | printf("Search done (%g seconds)\n", stop_timer()); 73 | 74 | printf("Checking results\n"); 75 | float precision = compute_precision(match, indices); 76 | printf("Precision is: %g\n", precision); 77 | 78 | } 79 | catch (std::exception& e) { 80 | std::cerr << "Exception: " << e.what() << "\n"; 81 | } 82 | 83 | return 0; 84 | } 85 | 86 | -------------------------------------------------------------------------------- /src/HDSearch/protoc_files/mid_tier.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/descriptor.proto"; 4 | 5 | package loadgen_index; 6 | 7 | service LoadGenIndex { 8 | // Defining service between load generator and index. 9 | // Input: Queries, number of nearest neighbors to compute. 10 | // Output: K-nn 11 | rpc LoadGen_Index (LoadGenRequest) returns (ResponseIndexKnnQueries) {} 12 | } 13 | 14 | 15 | message UtilRequest { 16 | bool util_request = 1; 17 | } 18 | 19 | message LoadGenRequest { 20 | repeated uint64 query_id = 1; 21 | uint32 number_nearest_neighbors = 2; 22 | UtilRequest util_request = 3; 23 | bool last_request = 4; 24 | uint64 resp_time = 5; 25 | bool kill = 6; 26 | uint64 request_id = 7; 27 | uint32 load = 8; 28 | } 29 | 30 | message MyDefault { 31 | repeated uint32 default = 1; 32 | } 33 | 34 | extend google.protobuf.FieldOptions { 35 | MyDefault my_default = 123456; 36 | } 37 | 38 | message PointIds { 39 | repeated uint32 point_id = 1 [(my_default) = {default: [1, 2, 3]}]; 40 | } 41 | 42 | message Util { 43 | uint64 user_time = 1; 44 | uint64 system_time = 2; 45 | uint64 io_time = 3; 46 | uint64 idle_time = 4; 47 | } 48 | 49 | message UtilResponse { 50 | bool util_present = 1; 51 | repeated Util bucket_util = 2; 52 | Util index_util = 3; 53 | } 54 | 55 | message ResponseIndexKnnQueries { 56 | repeated PointIds neighbor_ids = 1; 57 | uint64 create_index_req_time = 3; 58 | uint64 unpack_loadgen_req_time = 4; 59 | uint64 update_index_util_time = 5; 60 | uint64 get_point_ids_time = 6; 61 | uint64 get_bucket_responses_time = 7; 62 | uint64 create_bucket_req_time = 8; 63 | uint64 unpack_bucket_resp_time = 9; 64 | uint64 unpack_bucket_req_time = 10; 65 | uint64 calculate_knn_time = 11; 66 | uint64 pack_bucket_resp_time = 12; 67 | uint64 merge_time = 13; 68 | uint64 pack_index_resp_time = 14; 69 | // TODO A.S Temporary - REMOVE 70 | float percent_data_sent = 15; 71 | uint32 number_of_bucket_servers = 16; 72 | UtilResponse util_response = 17; 73 | uint64 resp_time = 18; 74 | uint64 index_time = 19; 75 | uint64 start_stamp = 20; 76 | uint64 end_stamp = 21; 77 | uint64 request_id = 22; 78 | uint64 num_inline = 23; 79 | uint64 num_workers = 24; 80 | uint64 num_resp = 25; 81 | bool kill_ack = 26; 82 | } 83 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/mpi/matrix.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef MPI_MATRIX_H_ 31 | #define MPI_MATRIX_H_ 32 | 33 | #include 34 | #include 35 | 36 | 37 | namespace boost { 38 | namespace serialization { 39 | 40 | template 41 | void serialize(Archive & ar, flann::Matrix & matrix, const unsigned int version) 42 | { 43 | ar & matrix.rows & matrix.cols & matrix.stride; 44 | if (Archive::is_loading::value) { 45 | matrix = flann::Matrix(new T[matrix.rows*matrix.cols], matrix.rows, matrix.cols, matrix.stride); 46 | } 47 | ar & boost::serialization::make_array(matrix.ptr(), matrix.rows*matrix.cols); 48 | } 49 | 50 | } 51 | } 52 | 53 | 54 | #endif /* MPI_MATRIX_H_ */ 55 | -------------------------------------------------------------------------------- /install.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | def InstallGrpc(): 7 | subprocess.call(["sudo", "apt-get", "install", "build-essential", "autoconf", "libtool", "curl", "cmake", "git", "pkg-config"]) 8 | os.system("git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc") 9 | subprocess.Popen(["git", "submodule", "update", "--init"], cwd="grpc/") 10 | subprocess.Popen(["make"], cwd="grpc/") 11 | subprocess.Popen(["sudo", "make", "install"], cwd="grpc/") 12 | 13 | def InstallProtobuf(): 14 | subprocess.call(["wget", "https://github.com/google/protobuf/releases/download/v3.2.0/protobuf-cpp-3.2.0.tar.gz"]) 15 | subprocess.call(["tar", "-xzvf", "protobuf-cpp-3.2.0.tar.gz"]) 16 | subprocess.Popen(["./configure"], cwd="protobuf-3.2.0") 17 | subprocess.Popen(["make"], cwd="protobuf-3.2.0") 18 | subprocess.Popen(["make", "check"], cwd="protobuf-3.2.0") 19 | subprocess.Popen(["sudo", "make", "install"], cwd="protobuf-3.2.0") 20 | subprocess.Popen(["sudo", "ldconfig"], cwd="protobuf-3.2.0") 21 | 22 | def InstallHDS(): 23 | subprocess.Popen(["cd", "HDSearch"], cwd=".") 24 | subprocess.Popen(["mkdir", "build"], cwd="HDSearch/mid_tier_service") 25 | subprocess.Popen(["cmake", ".."], cwd="HDSearch/mid_tier_service/build") 26 | subprocess.Popen(["sudo", "make", "install"], cwd="HDSearch/mid_tier_service/build") 27 | subprocess.Popen(["make"], cwd="HDSearch/mid_tier_service/build") 28 | 29 | subprocess.Popen(["cd", "protoc_files"], cwd="HDSearch/protoc_files") 30 | subprocess.Popen(["make"], cwd="HDSearch/protoc_files") 31 | 32 | subprocess.Popen(["cd", "bucket_service/service"], cwd="HDSearch/bucket_service/service") 33 | subprocess.Popen(["make"], cwd="HDSearch/bucket_service/service") 34 | 35 | 36 | 37 | def InstallOpenSSL(): 38 | subprocess.call(["sudo", "apt-get", "install", "openssl"]) 39 | subprocess.call(["sudo", "apt-get", "install", "libssl-dev"]) 40 | 41 | def InstallMKL(): 42 | subprocess.call(["wget", "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/11306/l_mkl_2017.2.174.tgz"]) 43 | subprocess.call(["tar", "xzvf", "l_mkl_2017.2.174.tgz"]) 44 | subprocess.call(["./install.sh"], cwd="l_mkl_2017.2.174") 45 | 46 | def main(): 47 | print "Installing gRPC....\n" 48 | InstallGrpc() 49 | print "Installed gRPC....\n" 50 | print "Installing Protobuf....\n" 51 | InstallProtobuf() 52 | print "Installed Protobuf....\n" 53 | InstallOpenSSL() 54 | InstallMKL() 55 | print "Installing HDSearch....\n" 56 | InstallHDS() 57 | print "Installed HDSearch....\n" 58 | InstallOpenSSL() 59 | InstallMKL() 60 | 61 | main() 62 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/timer.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_TIMER_H 32 | #define FLANN_TIMER_H 33 | 34 | #include 35 | 36 | 37 | namespace flann 38 | { 39 | 40 | /** 41 | * A start-stop timer class. 42 | * 43 | * Can be used to time portions of code. 44 | */ 45 | class StartStopTimer 46 | { 47 | clock_t startTime; 48 | 49 | public: 50 | /** 51 | * Value of the timer. 52 | */ 53 | double value; 54 | 55 | 56 | /** 57 | * Constructor. 58 | */ 59 | StartStopTimer() 60 | { 61 | reset(); 62 | } 63 | 64 | /** 65 | * Starts the timer. 66 | */ 67 | void start() 68 | { 69 | startTime = clock(); 70 | } 71 | 72 | /** 73 | * Stops the timer and updates timer value. 74 | */ 75 | double stop() 76 | { 77 | clock_t stopTime = clock(); 78 | value += ( (double)stopTime - startTime) / CLOCKS_PER_SEC; 79 | 80 | return value; 81 | } 82 | 83 | /** 84 | * Resets the timer value to 0. 85 | */ 86 | void reset() 87 | { 88 | value = 0; 89 | } 90 | 91 | }; 92 | 93 | } 94 | 95 | #endif // FLANN_TIMER_H 96 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/sampling.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef FLANN_SAMPLING_H_ 31 | #define FLANN_SAMPLING_H_ 32 | 33 | #include "flann/util/matrix.h" 34 | #include "flann/util/random.h" 35 | 36 | namespace flann 37 | { 38 | 39 | template 40 | Matrix random_sample(Matrix& srcMatrix, size_t size, bool remove = false) 41 | { 42 | UniqueRandom rand_unique(srcMatrix.rows); 43 | Matrix newSet(new T[size * srcMatrix.cols], size,srcMatrix.cols); 44 | 45 | T* src,* dest; 46 | for (size_t i=0; i(rand_int(srcMatrix.rows-i)); 50 | } 51 | else { 52 | r = static_cast(rand_unique.next()); 53 | } 54 | dest = newSet[i]; 55 | src = srcMatrix[r]; 56 | std::copy(src, src+srcMatrix.cols, dest); 57 | if (remove) { 58 | src = srcMatrix[srcMatrix.rows-i-1]; 59 | dest = srcMatrix[r]; 60 | std::copy(src, src+srcMatrix.cols, dest); 61 | } 62 | } 63 | if (remove) { 64 | srcMatrix.rows -= size; 65 | } 66 | return newSet; 67 | } 68 | 69 | } // namespace 70 | 71 | 72 | #endif /* FLANN_SAMPLING_H_ */ 73 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/multiple_points.h: -------------------------------------------------------------------------------- 1 | #ifndef __MULTIPLE_POINTS_H_INCLUDED__ 2 | #define __MULTIPLE_POINTS_H_INCLUDED__ 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "point.h" 10 | 11 | class MultiplePoints 12 | { 13 | public: 14 | MultiplePoints() = default; 15 | // Second constructor to initialize "size"# points to a value. 16 | MultiplePoints(int size, const Point &p) 17 | { 18 | multiple_points_.assign(size, p); 19 | } 20 | /* Read text file and create a structure: vector of points. 21 | file format: float11 float12 float13 ... --> all dimensions of pt 1. 22 | float21 float22 float23 ... --> all dimensions of pt 2. 23 | In: path to text file. 24 | Out: multiple_points_ private variable takes up point values 25 | based on the text file.*/ 26 | void CreateMultiplePoints(const std::string &file_name); 27 | void CreateMultiplePointsFromBinFile(const std::string &file_name); 28 | // Dimension of all points in dataset/queries must be the same. 29 | void ValidateDimensions(const int dataset_dimension, 30 | const int point_dimension) const; 31 | /* Read float elements of a line and store them as point dimensions. 32 | In: a set of string elements (represents floats of point dimensions). 33 | Out: a variable of type "Point" - has floats for all dimensions.*/ 34 | void CreatePoint(const std::vector &tokens, Point* p) ; 35 | // Return total number of points in this collection. 36 | unsigned GetSize() const; 37 | /* Resize a set of multiplepoints. 38 | In: new size, new points 39 | Out: the multiplepoints private member gets modified.*/ 40 | void Resize(const int size, const Point &p); 41 | // Return the dimension of each point (points must have equal dimension). 42 | unsigned GetPointDimension() const; 43 | // Remove all points, makes data structure empty. 44 | void Clear(); 45 | // Get the Point at a given index. 46 | const Point& GetPointAtIndex(const int index) const; 47 | // Add a point to the end of the vector. 48 | void PushBack(const Point &point); 49 | // Get point at back. 50 | Point& GetPointAtBack(); 51 | 52 | // Add point to a given index. 53 | void SetPoint(const unsigned int index, const Point &point); 54 | std::vector::iterator Begin(); 55 | void Erase(std::vector::iterator element); 56 | void PopBack(); 57 | // Prints a collection of points. 58 | void Print() const; 59 | private: 60 | // Define a data structure that stores a collection of points. 61 | std::vector multiple_points_; 62 | }; 63 | #endif // __MULTIPLE_POINTS_H_INCLUDED__ 64 | -------------------------------------------------------------------------------- /src/Router/protoc_files/lookup.grpc.pb.cc: -------------------------------------------------------------------------------- 1 | // Generated by the gRPC C++ plugin. 2 | // If you make any local change, they will be lost. 3 | // source: lookup.proto 4 | 5 | #include "lookup.pb.h" 6 | #include "lookup.grpc.pb.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | namespace lookup { 17 | 18 | static const char* LookupService_method_names[] = { 19 | "/lookup.LookupService/KeyLookup", 20 | }; 21 | 22 | std::unique_ptr< LookupService::Stub> LookupService::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) { 23 | std::unique_ptr< LookupService::Stub> stub(new LookupService::Stub(channel)); 24 | return stub; 25 | } 26 | 27 | LookupService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel) 28 | : channel_(channel), rpcmethod_KeyLookup_(LookupService_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) 29 | {} 30 | 31 | ::grpc::Status LookupService::Stub::KeyLookup(::grpc::ClientContext* context, const ::lookup::Key& request, ::lookup::Value* response) { 32 | return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_KeyLookup_, context, request, response); 33 | } 34 | 35 | ::grpc::ClientAsyncResponseReader< ::lookup::Value>* LookupService::Stub::AsyncKeyLookupRaw(::grpc::ClientContext* context, const ::lookup::Key& request, ::grpc::CompletionQueue* cq) { 36 | return ::grpc::internal::ClientAsyncResponseReaderFactory< ::lookup::Value>::Create(channel_.get(), cq, rpcmethod_KeyLookup_, context, request, true); 37 | } 38 | 39 | ::grpc::ClientAsyncResponseReader< ::lookup::Value>* LookupService::Stub::PrepareAsyncKeyLookupRaw(::grpc::ClientContext* context, const ::lookup::Key& request, ::grpc::CompletionQueue* cq) { 40 | return ::grpc::internal::ClientAsyncResponseReaderFactory< ::lookup::Value>::Create(channel_.get(), cq, rpcmethod_KeyLookup_, context, request, false); 41 | } 42 | 43 | LookupService::Service::Service() { 44 | AddMethod(new ::grpc::internal::RpcServiceMethod( 45 | LookupService_method_names[0], 46 | ::grpc::internal::RpcMethod::NORMAL_RPC, 47 | new ::grpc::internal::RpcMethodHandler< LookupService::Service, ::lookup::Key, ::lookup::Value>( 48 | std::mem_fn(&LookupService::Service::KeyLookup), this))); 49 | } 50 | 51 | LookupService::Service::~Service() { 52 | } 53 | 54 | ::grpc::Status LookupService::Service::KeyLookup(::grpc::ServerContext* context, const ::lookup::Key* request, ::lookup::Value* response) { 55 | (void) context; 56 | (void) request; 57 | (void) response; 58 | return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); 59 | } 60 | 61 | 62 | } // namespace lookup 63 | 64 | -------------------------------------------------------------------------------- /src/Router/protoc_files/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = . 11 | 12 | vpath %.proto $(PROTOS_PATH) 13 | 14 | all: system-check protoc_files 15 | 16 | protoc_files: lookup.pb.o lookup.grpc.pb.o router.pb.o router.grpc.pb.o 17 | $(CXX) $^ -o $@ $(LDFLAGS) 18 | 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/HDSearch/protoc_files/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = . 11 | 12 | vpath %.proto $(PROTOS_PATH) 13 | 14 | all: system-check protoc_files 15 | 16 | protoc_files: mid_tier.pb.o mid_tier.grpc.pb.o bucket.pb.o bucket.grpc.pb.o 17 | $(CXX) $^ -o $@ $(LDFLAGS) 18 | 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/Recommend/protoc_files/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = . 11 | 12 | vpath %.proto $(PROTOS_PATH) 13 | 14 | all: system-check protoc_files 15 | 16 | protoc_files: cf.pb.o cf.grpc.pb.o recommender.pb.o recommender.grpc.pb.o 17 | $(CXX) $^ -o $@ $(LDFLAGS) 18 | 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/SetAlgebra/protoc_files/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = . 11 | 12 | vpath %.proto $(PROTOS_PATH) 13 | 14 | all: system-check protoc_files 15 | 16 | protoc_files: intersection.pb.o intersection.grpc.pb.o union.pb.o union.grpc.pb.o 17 | $(CXX) $^ -o $@ $(LDFLAGS) 18 | 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/Recommend/cf_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -I$(HDS_PATH) 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I$(HDS_PATH) 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I$(HDS_PATH) -lmlpack -larmadillo 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | 12 | vpath %.proto $(PROTOS_PATH) 13 | 14 | all: system-check cf_server 15 | 16 | cf_server: $(PROTOS_PATH)/cf.pb.o $(PROTOS_PATH)/cf.grpc.pb.o helper_files/timing.o helper_files/utils.o helper_files/server_helper.o cf_server.o 17 | $(CXX) $^ -O3 $(LDFLAGS) -o $@ 18 | 19 | .PRECIOUS: %.grpc.pb.cc 20 | %.grpc.pb.cc: %.proto 21 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 22 | 23 | .PRECIOUS: %.pb.cc 24 | %.pb.cc: %.proto 25 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 26 | 27 | clean: 28 | rm -f *.o *.pb.cc *.pb.h cf_server 29 | 30 | 31 | # The following is to test your system and ensure a smoother experience. 32 | # They are by no means necessary to actually compile a grpc-enabled software. 33 | 34 | PROTOC_CMD = which $(PROTOC) 35 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 36 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 37 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 38 | ifeq ($(HAS_PROTOC),true) 39 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 40 | endif 41 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 42 | 43 | SYSTEM_OK = false 44 | ifeq ($(HAS_VALID_PROTOC),true) 45 | ifeq ($(HAS_PLUGIN),true) 46 | SYSTEM_OK = true 47 | endif 48 | endif 49 | 50 | system-check: 51 | ifneq ($(HAS_VALID_PROTOC),true) 52 | @echo " DEPENDENCY ERROR" 53 | @echo 54 | @echo "You don't have protoc 3.0.0 installed in your path." 55 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 56 | @echo "You can find it here:" 57 | @echo 58 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 59 | @echo 60 | @echo "Here is what I get when trying to evaluate your version of protoc:" 61 | @echo 62 | -$(PROTOC) --version 63 | @echo 64 | @echo 65 | endif 66 | ifneq ($(HAS_PLUGIN),true) 67 | @echo " DEPENDENCY ERROR" 68 | @echo 69 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 70 | @echo "Please install grpc. You can find it here:" 71 | @echo 72 | @echo " https://github.com/grpc/grpc" 73 | @echo 74 | @echo "Here is what I get when trying to detect if you have the plugin:" 75 | @echo 76 | -which $(GRPC_CPP_PLUGIN) 77 | @echo 78 | @echo 79 | endif 80 | ifneq ($(SYSTEM_OK),true) 81 | @false 82 | endif 83 | -------------------------------------------------------------------------------- /src/Router/protoc_files/router.grpc.pb.cc: -------------------------------------------------------------------------------- 1 | // Generated by the gRPC C++ plugin. 2 | // If you make any local change, they will be lost. 3 | // source: router.proto 4 | 5 | #include "router.pb.h" 6 | #include "router.grpc.pb.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | namespace router { 17 | 18 | static const char* RouterService_method_names[] = { 19 | "/router.RouterService/Router", 20 | }; 21 | 22 | std::unique_ptr< RouterService::Stub> RouterService::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) { 23 | std::unique_ptr< RouterService::Stub> stub(new RouterService::Stub(channel)); 24 | return stub; 25 | } 26 | 27 | RouterService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel) 28 | : channel_(channel), rpcmethod_Router_(RouterService_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) 29 | {} 30 | 31 | ::grpc::Status RouterService::Stub::Router(::grpc::ClientContext* context, const ::router::RouterRequest& request, ::router::LookupResponse* response) { 32 | return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Router_, context, request, response); 33 | } 34 | 35 | ::grpc::ClientAsyncResponseReader< ::router::LookupResponse>* RouterService::Stub::AsyncRouterRaw(::grpc::ClientContext* context, const ::router::RouterRequest& request, ::grpc::CompletionQueue* cq) { 36 | return ::grpc::internal::ClientAsyncResponseReaderFactory< ::router::LookupResponse>::Create(channel_.get(), cq, rpcmethod_Router_, context, request, true); 37 | } 38 | 39 | ::grpc::ClientAsyncResponseReader< ::router::LookupResponse>* RouterService::Stub::PrepareAsyncRouterRaw(::grpc::ClientContext* context, const ::router::RouterRequest& request, ::grpc::CompletionQueue* cq) { 40 | return ::grpc::internal::ClientAsyncResponseReaderFactory< ::router::LookupResponse>::Create(channel_.get(), cq, rpcmethod_Router_, context, request, false); 41 | } 42 | 43 | RouterService::Service::Service() { 44 | AddMethod(new ::grpc::internal::RpcServiceMethod( 45 | RouterService_method_names[0], 46 | ::grpc::internal::RpcMethod::NORMAL_RPC, 47 | new ::grpc::internal::RpcMethodHandler< RouterService::Service, ::router::RouterRequest, ::router::LookupResponse>( 48 | std::mem_fn(&RouterService::Service::Router), this))); 49 | } 50 | 51 | RouterService::Service::~Service() { 52 | } 53 | 54 | ::grpc::Status RouterService::Service::Router(::grpc::ServerContext* context, const ::router::RouterRequest* request, ::router::LookupResponse* response) { 55 | (void) context; 56 | (void) request; 57 | (void) response; 58 | return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); 59 | } 60 | 61 | 62 | } // namespace router 63 | 64 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/helper_files/server_helper.cc: -------------------------------------------------------------------------------- 1 | #include "server_helper.h" 2 | 3 | std::mutex test; 4 | 5 | void CreateMemcachedConn(const int memcached_port, 6 | memcached_st* memc, 7 | memcached_return* rc) 8 | { 9 | 10 | memcached_server_st *servers = NULL; 11 | 12 | memcached_server_st *memcached_servers_parse (char *server_strings); 13 | servers = memcached_server_list_append(servers, "localhost", memcached_port, rc); 14 | *rc = memcached_server_push(memc, servers); 15 | 16 | if (*rc == MEMCACHED_SUCCESS) { 17 | fprintf(stderr,"Added server successfully\n"); 18 | } else { 19 | CHECK(false, "Could not add server\n"); 20 | } 21 | } 22 | 23 | void UnpackBucketServiceRequest(const lookup::Key &request, 24 | uint32_t* operation, 25 | std::string* key, 26 | std::string* value) 27 | { 28 | *operation = request.operation(); 29 | *key = request.key(); 30 | *value = request.value(); 31 | } 32 | 33 | void Get(memcached_st* memc, 34 | memcached_return* rc, 35 | const std::string key, 36 | std::string* value) 37 | { 38 | size_t value_length; 39 | uint32_t flags; 40 | char* retrieved_value; 41 | try { 42 | test.lock(); 43 | retrieved_value = memcached_get(memc, key.c_str(), strlen(key.c_str()), &value_length, &flags, rc); 44 | test.unlock(); 45 | } catch(...) { 46 | CHECK(false, "Exception\n"); 47 | } 48 | if (*rc == MEMCACHED_SUCCESS) { 49 | if (retrieved_value == NULL) { 50 | *(value) = "nack"; 51 | return; 52 | } 53 | *(value) = std::string(retrieved_value); 54 | } 55 | else { 56 | *(value) = "nack"; 57 | } 58 | free(retrieved_value); 59 | } 60 | 61 | #if 0 62 | void Get(memcached_st* memc, 63 | memcached_return* rc, 64 | ThreadSafeMap &key_value_store, 65 | std::string key, 66 | std::string* value) 67 | { 68 | *value = key_value_store.Get(key); 69 | } 70 | 71 | void Set(memcached_st* memc, 72 | memcached_return* rc, 73 | ThreadSafeMap* key_value_store, 74 | std::string key, 75 | std::string* value) 76 | { 77 | key_value_store->Set(key, *value); 78 | *value = "ack"; 79 | } 80 | #endif 81 | void Set(memcached_st* memc, 82 | memcached_return* rc, 83 | const std::string key, 84 | std::string* value) 85 | { 86 | std::string val = *value; 87 | 88 | try { 89 | test.lock(); 90 | *rc = memcached_set(memc, key.c_str(), strlen(key.c_str()), val.c_str(), strlen(val.c_str()), (time_t)0, (uint32_t)0); 91 | test.unlock(); 92 | } catch(...) { 93 | CHECK(false, "Exception\n"); 94 | } 95 | 96 | if (*rc == MEMCACHED_SUCCESS) 97 | { 98 | *value = "ack"; 99 | } 100 | else { 101 | *value = "nack"; 102 | } 103 | } 104 | 105 | void Update(const std::string key, 106 | const std::string Value) 107 | { 108 | 109 | } 110 | 111 | -------------------------------------------------------------------------------- /src/Router/lookup_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -I$(HDS_PATH) -g 4 | CXXFLAGS += -std=c++17 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I$(HDS_PATH) -g 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I$(HDS_PATH) -lmemcached 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | BUCKET_PATH = ../ 12 | 13 | vpath %.proto $(PROTOS_PATH) 14 | 15 | all: system-check lookup_server 16 | 17 | lookup_server: $(PROTOS_PATH)/lookup.pb.o $(PROTOS_PATH)/lookup.grpc.pb.o $(BUCKET_PATH)/service/helper_files/server_helper.o $(BUCKET_PATH)/service/helper_files/timing.o $(BUCKET_PATH)/service/helper_files/utils.o lookup_server.o 18 | $(CXX) $^ -O3 $(LDFLAGS) -o $@ 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h lookup_server 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -I$(HDS_PATH) -g 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I$(HDS_PATH) -g 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I$(HDS_PATH) 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | BUCKET_PATH = ../ 12 | 13 | vpath %.proto $(PROTOS_PATH) 14 | 15 | all: system-check intersection_server 16 | 17 | intersection_server: $(PROTOS_PATH)/intersection.pb.o $(PROTOS_PATH)/intersection.grpc.pb.o $(BUCKET_PATH)/src/intersection.o $(BUCKET_PATH)/service/helper_files/server_helper.o $(BUCKET_PATH)/service/helper_files/timing.o $(BUCKET_PATH)/service/helper_files/utils.o intersection_server.o 18 | $(CXX) $^ -O3 $(LDFLAGS) -o $@ 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h intersection_server 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -I$(HDS_PATH) 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I$(HDS_PATH) 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I$(HDS_PATH) 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | BUCKET_PATH = ../ 12 | 13 | vpath %.proto $(PROTOS_PATH) 14 | 15 | all: system-check bucket_server 16 | 17 | bucket_server: $(PROTOS_PATH)/bucket.pb.o $(PROTOS_PATH)/bucket.grpc.pb.o $(BUCKET_PATH)/src/custom_priority_queue.o $(BUCKET_PATH)/src/utils.o $(BUCKET_PATH)/src/point.o $(BUCKET_PATH)/src/multiple_points.o $(BUCKET_PATH)/src/dist_calc.o $(BUCKET_PATH)/service/helper_files/timing.o $(BUCKET_PATH)/service/helper_files/server_helper.o bucket_server.o 18 | $(CXX) $^ -O3 $(LDFLAGS) -o $@ 19 | 20 | .PRECIOUS: %.grpc.pb.cc 21 | %.grpc.pb.cc: %.proto 22 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 23 | 24 | .PRECIOUS: %.pb.cc 25 | %.pb.cc: %.proto 26 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 27 | 28 | clean: 29 | rm -f *.o *.pb.cc *.pb.h bucket_server 30 | 31 | 32 | # The following is to test your system and ensure a smoother experience. 33 | # They are by no means necessary to actually compile a grpc-enabled software. 34 | 35 | PROTOC_CMD = which $(PROTOC) 36 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 37 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 38 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 39 | ifeq ($(HAS_PROTOC),true) 40 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 41 | endif 42 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 43 | 44 | SYSTEM_OK = false 45 | ifeq ($(HAS_VALID_PROTOC),true) 46 | ifeq ($(HAS_PLUGIN),true) 47 | SYSTEM_OK = true 48 | endif 49 | endif 50 | 51 | system-check: 52 | ifneq ($(HAS_VALID_PROTOC),true) 53 | @echo " DEPENDENCY ERROR" 54 | @echo 55 | @echo "You don't have protoc 3.0.0 installed in your path." 56 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 57 | @echo "You can find it here:" 58 | @echo 59 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 60 | @echo 61 | @echo "Here is what I get when trying to evaluate your version of protoc:" 62 | @echo 63 | -$(PROTOC) --version 64 | @echo 65 | @echo 66 | endif 67 | ifneq ($(HAS_PLUGIN),true) 68 | @echo " DEPENDENCY ERROR" 69 | @echo 70 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 71 | @echo "Please install grpc. You can find it here:" 72 | @echo 73 | @echo " https://github.com/grpc/grpc" 74 | @echo 75 | @echo "Here is what I get when trying to detect if you have the plugin:" 76 | @echo 77 | -which $(GRPC_CPP_PLUGIN) 78 | @echo 79 | @echo 80 | endif 81 | ifneq ($(SYSTEM_OK),true) 82 | @false 83 | endif 84 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/object_factory.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_OBJECT_FACTORY_H_ 32 | #define FLANN_OBJECT_FACTORY_H_ 33 | 34 | #include 35 | 36 | namespace flann 37 | { 38 | 39 | class CreatorNotFound 40 | { 41 | }; 42 | 43 | template 46 | class ObjectFactory 47 | { 48 | typedef ObjectFactory ThisClass; 49 | typedef std::map ObjectRegistry; 50 | 51 | // singleton class, private constructor 52 | ObjectFactory() {} 53 | 54 | public: 55 | 56 | bool subscribe(UniqueIdType id, ObjectCreator creator) 57 | { 58 | if (object_registry.find(id) != object_registry.end()) return false; 59 | 60 | object_registry[id] = creator; 61 | return true; 62 | } 63 | 64 | bool unregister(UniqueIdType id) 65 | { 66 | return object_registry.erase(id) == 1; 67 | } 68 | 69 | ObjectCreator create(UniqueIdType id) 70 | { 71 | typename ObjectRegistry::const_iterator iter = object_registry.find(id); 72 | 73 | if (iter == object_registry.end()) { 74 | throw CreatorNotFound(); 75 | } 76 | 77 | return iter->second; 78 | } 79 | 80 | static ThisClass& instance() 81 | { 82 | static ThisClass the_factory; 83 | return the_factory; 84 | } 85 | private: 86 | ObjectRegistry object_registry; 87 | }; 88 | 89 | } 90 | 91 | #endif /* FLANN_OBJECT_FACTORY_H_ */ 92 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/mpi/client.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef MPI_CLIENT_H_ 31 | #define MPI_CLIENT_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "queries.h" 38 | 39 | namespace flann { 40 | namespace mpi { 41 | 42 | 43 | class Client 44 | { 45 | public: 46 | Client(const std::string& host, const std::string& service) 47 | { 48 | tcp::resolver resolver(io_service_); 49 | tcp::resolver::query query(tcp::v4(), host, service); 50 | iterator_ = resolver.resolve(query); 51 | } 52 | 53 | 54 | template 55 | void knnSearch(const flann::Matrix& queries, flann::Matrix& indices, flann::Matrix& dists, int knn, const SearchParams& params) 56 | { 57 | std::cout << "knn client\n"; 58 | tcp::socket sock(io_service_); 59 | sock.connect(*iterator_); 60 | 61 | Request req; 62 | req.nn = knn; 63 | req.queries = queries; 64 | req.checks = params.checks; 65 | // send request 66 | write_object(sock,req); 67 | 68 | Response resp; 69 | // read response 70 | read_object(sock, resp); 71 | 72 | for (size_t i=0;i /dev/null && echo true || echo false) 40 | ifeq ($(HAS_PROTOC),true) 41 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 42 | endif 43 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 44 | 45 | SYSTEM_OK = false 46 | ifeq ($(HAS_VALID_PROTOC),true) 47 | ifeq ($(HAS_PLUGIN),true) 48 | SYSTEM_OK = true 49 | endif 50 | endif 51 | 52 | system-check: 53 | ifneq ($(HAS_VALID_PROTOC),true) 54 | @echo " DEPENDENCY ERROR" 55 | @echo 56 | @echo "You don't have protoc 3.0.0 installed in your path." 57 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 58 | @echo "You can find it here:" 59 | @echo 60 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 61 | @echo 62 | @echo "Here is what I get when trying to evaluate your version of protoc:" 63 | @echo 64 | -$(PROTOC) --version 65 | @echo 66 | @echo 67 | endif 68 | ifneq ($(HAS_PLUGIN),true) 69 | @echo " DEPENDENCY ERROR" 70 | @echo 71 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 72 | @echo "Please install grpc. You can find it here:" 73 | @echo 74 | @echo " https://github.com/grpc/grpc" 75 | @echo 76 | @echo "Here is what I get when trying to detect if you have the plugin:" 77 | @echo 78 | -which $(GRPC_CPP_PLUGIN) 79 | @echo 80 | @echo 81 | endif 82 | ifneq ($(SYSTEM_OK),true) 83 | @false 84 | endif 85 | -------------------------------------------------------------------------------- /src/Router/mid_tier_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -Wall -fopenmp -I../../ 4 | CXXFLAGS += -std=c++14 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../../ -lboost_system -lboost_thread 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files/ 11 | BUCKET_PATH = ../../lookup_service 12 | INDEX_PATH = ../../mid_tier_service 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check mid_tier_server 17 | 18 | mid_tier_server: $(PROTOS_PATH)/lookup.pb.o $(PROTOS_PATH)/lookup.grpc.pb.o $(PROTOS_PATH)/router.pb.o $(PROTOS_PATH)/router.grpc.pb.o $(INDEX_PATH)/src/spookyhash.o $(BUCKET_PATH)/service/helper_files/client_helper.o helper_files/router_server_helper.o helper_files/timing.o helper_files/utils.o mid_tier_server.o 19 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 20 | 21 | .PRECIOUS: %.grpc.pb.cc 22 | %.grpc.pb.cc: %.proto 23 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 24 | 25 | .PRECIOUS: %.pb.cc 26 | %.pb.cc: %.proto 27 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 28 | 29 | clean: 30 | rm -f *.o *.pb.cc *.pb.h mid_tier_server 31 | 32 | 33 | # The following is to test your system and ensure a smoother experience. 34 | # They are by no means necessary to actually compile a grpc-enabled software. 35 | 36 | PROTOC_CMD = which $(PROTOC) 37 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 38 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 39 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 40 | ifeq ($(HAS_PROTOC),true) 41 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 42 | endif 43 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 44 | 45 | SYSTEM_OK = false 46 | ifeq ($(HAS_VALID_PROTOC),true) 47 | ifeq ($(HAS_PLUGIN),true) 48 | SYSTEM_OK = true 49 | endif 50 | endif 51 | 52 | system-check: 53 | ifneq ($(HAS_VALID_PROTOC),true) 54 | @echo " DEPENDENCY ERROR" 55 | @echo 56 | @echo "You don't have protoc 3.0.0 installed in your path." 57 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 58 | @echo "You can find it here:" 59 | @echo 60 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 61 | @echo 62 | @echo "Here is what I get when trying to evaluate your version of protoc:" 63 | @echo 64 | -$(PROTOC) --version 65 | @echo 66 | @echo 67 | endif 68 | ifneq ($(HAS_PLUGIN),true) 69 | @echo " DEPENDENCY ERROR" 70 | @echo 71 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 72 | @echo "Please install grpc. You can find it here:" 73 | @echo 74 | @echo " https://github.com/grpc/grpc" 75 | @echo 76 | @echo "Here is what I get when trying to detect if you have the plugin:" 77 | @echo 78 | -which $(GRPC_CPP_PLUGIN) 79 | @echo 80 | @echo 81 | endif 82 | ifneq ($(SYSTEM_OK),true) 83 | @false 84 | endif 85 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/src/dist_calc.h: -------------------------------------------------------------------------------- 1 | #ifndef __DIST_CALC_H_INCLUDED__ 2 | #define __DIST_CALC_H_INCLUDED__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "custom_priority_queue.h" 8 | 9 | /* This is the function called by individual worker threads and 10 | is therefore not a member of the DistCalc class.*/ 11 | void* QueriesShardedDistanceCalculation(void* parameter); 12 | 13 | typedef std::vector PointIDs; 14 | class DistCalc 15 | { 16 | public: 17 | DistCalc() = default; 18 | // Calculates the k-nn points for all queries. 19 | void DistanceCalculation(const MultiplePoints &dataset, 20 | const MultiplePoints &queries, 21 | const std::vector> &point_ids_vec, 22 | const unsigned number_of_nearest_neighbors, 23 | const int num_cores); 24 | 25 | /* Faster function when only one NN needs to be computed for 26 | one query.*/ 27 | void GetNN(const MultiplePoints &dataset, 28 | const Point &query_point, 29 | const std::vector &point_id_vec, 30 | const int num_cores); 31 | 32 | void CreateThreadsShardingQueries(const MultiplePoints* dataset, 33 | const MultiplePoints* queries, 34 | const unsigned int number_of_nearest_neighbors, 35 | const unsigned int num_procs, 36 | const std::vector> &point_ids_vec); 37 | // Calculates k-points between one query and all points in the dataset. 38 | void CalculateKnn(const MultiplePoints &dataset, 39 | const Point &query_point, 40 | const std::vector &point_id_vec, 41 | const unsigned number_of_nearest_neighbors, 42 | CustomPriorityQueue *knn_priority_queue); 43 | void CalculateShardedKnn(const MultiplePoints &dataset, 44 | const Point &query_point, 45 | const std::vector &point_id_vec, 46 | const unsigned number_of_nearest_neighbors, 47 | CustomPriorityQueue* knn_priority_queue); 48 | // Add the K-NN result to each query in the batch. 49 | void AddKnnAnswer(PointIDs& answer_curr_query, 50 | const unsigned index); 51 | // Calculates euclidean distance between a query & dataset point. 52 | float EuclideanDistance(const Point &query, 53 | const Point &dataset_point) const; 54 | // Initializes knn_all_queries_ with a given size & value. 55 | void Initialize(const int size, 56 | const PointIDs &point_ids); 57 | // Returns size of knn_all_queries_. 58 | unsigned GetSize() const; 59 | // Returns the MultiplePoints at the given index of knn_all_queries_. 60 | PointIDs GetValueAtIndex(const int index) const; 61 | // Adds given value to given index of knn_all_queries_. 62 | void AddValueToIndex(const int index, const PointIDs &value); 63 | // Adds value to the back of the vector knn_all_queries_. 64 | void AddValueToBack(const PointIDs &value); 65 | private: 66 | // Data structure that holds k-nn (MultiplePoints) for all queries. 67 | std::vector knn_all_queries_; 68 | }; 69 | #endif //__DIST_CALC_H_INCLUDED__ 70 | -------------------------------------------------------------------------------- /src/SetAlgebra/union_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -Wall -fopenmp -I../../ 4 | CXXFLAGS += -std=c++14 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../../ -lboost_system -lboost_thread 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | INTERSECTION_SRV_PATH = ../../intersection_service 12 | UNION_PATH = ../../union_service/ 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check mid_tier_server 17 | 18 | mid_tier_server: $(PROTOS_PATH)/intersection.pb.o $(PROTOS_PATH)/intersection.grpc.pb.o $(PROTOS_PATH)/union.pb.o $(PROTOS_PATH)/union.grpc.pb.o $(INTERSECTION_SRV_PATH)/service/helper_files/client_helper.o $(UNION_PATH)/service/helper_files/union_server_helper.o $(UNION_PATH)/service/helper_files/timing.o $(UNION_PATH)/service/helper_files/utils.o mid_tier_server.o 19 | $(CXX) $^ -o $@ $(LDFLAGS) 20 | 21 | .PRECIOUS: %.grpc.pb.cc 22 | %.grpc.pb.cc: %.proto 23 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 24 | 25 | .PRECIOUS: %.pb.cc 26 | %.pb.cc: %.proto 27 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 28 | 29 | clean: 30 | rm -f *.o *.pb.cc *.pb.h mid_tier_server 31 | 32 | 33 | # The following is to test your system and ensure a smoother experience. 34 | # They are by no means necessary to actually compile a grpc-enabled software. 35 | 36 | PROTOC_CMD = which $(PROTOC) 37 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 38 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 39 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 40 | ifeq ($(HAS_PROTOC),true) 41 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 42 | endif 43 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 44 | 45 | SYSTEM_OK = false 46 | ifeq ($(HAS_VALID_PROTOC),true) 47 | ifeq ($(HAS_PLUGIN),true) 48 | SYSTEM_OK = true 49 | endif 50 | endif 51 | 52 | system-check: 53 | ifneq ($(HAS_VALID_PROTOC),true) 54 | @echo " DEPENDENCY ERROR" 55 | @echo 56 | @echo "You don't have protoc 3.0.0 installed in your path." 57 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 58 | @echo "You can find it here:" 59 | @echo 60 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 61 | @echo 62 | @echo "Here is what I get when trying to evaluate your version of protoc:" 63 | @echo 64 | -$(PROTOC) --version 65 | @echo 66 | @echo 67 | endif 68 | ifneq ($(HAS_PLUGIN),true) 69 | @echo " DEPENDENCY ERROR" 70 | @echo 71 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 72 | @echo "Please install grpc. You can find it here:" 73 | @echo 74 | @echo " https://github.com/grpc/grpc" 75 | @echo 76 | @echo "Here is what I get when trying to detect if you have the plugin:" 77 | @echo 78 | -which $(GRPC_CPP_PLUGIN) 79 | @echo 80 | @echo 81 | endif 82 | ifneq ($(SYSTEM_OK),true) 83 | @false 84 | endif 85 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/mpi/queries.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2011 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2011 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | 30 | #ifndef MPI_QUERIES_H_ 31 | #define MPI_QUERIES_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace flann 39 | { 40 | 41 | template 42 | struct Request 43 | { 44 | flann::Matrix queries; 45 | int nn; 46 | int checks; 47 | 48 | template 49 | void serialize(Archive& ar, const unsigned int version) 50 | { 51 | ar & queries & nn & checks; 52 | } 53 | }; 54 | 55 | template 56 | struct Response 57 | { 58 | flann::Matrix indices; 59 | flann::Matrix dists; 60 | 61 | template 62 | void serialize(Archive& ar, const unsigned int version) 63 | { 64 | ar & indices & dists; 65 | } 66 | }; 67 | 68 | 69 | using boost::asio::ip::tcp; 70 | 71 | template 72 | void read_object(tcp::socket& sock, T& val) 73 | { 74 | uint32_t size; 75 | boost::asio::read(sock, boost::asio::buffer(&size, sizeof(size))); 76 | size = ntohl(size); 77 | 78 | boost::asio::streambuf archive_stream; 79 | boost::asio::read(sock, archive_stream, boost::asio::transfer_at_least(size)); 80 | 81 | boost::archive::binary_iarchive archive(archive_stream); 82 | archive >> val; 83 | } 84 | 85 | template 86 | void write_object(tcp::socket& sock, const T& val) 87 | { 88 | boost::asio::streambuf archive_stream; 89 | boost::archive::binary_oarchive archive(archive_stream); 90 | archive << val; 91 | 92 | uint32_t size = archive_stream.size(); 93 | size = htonl(size); 94 | boost::asio::write(sock, boost::asio::buffer(&size, sizeof(size))); 95 | boost::asio::write(sock, archive_stream); 96 | 97 | } 98 | 99 | } 100 | 101 | 102 | 103 | #endif /* MPI_QUERIES_H_ */ 104 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/service/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -Wall -fopenmp -I../../ 4 | CXXFLAGS += -std=c++17 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../../ 5 | LDFLAGS += -L/usr/local/lib -lgrpc++ -lgrpc -lgpr -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../../ -lboost_system -lboost_thread 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../../protoc_files 11 | BUCKET_PATH = ../../bucket_service 12 | INDEX_PATH = ../../mid_tier_service/ 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check mid_tier_server 17 | 18 | mid_tier_server: $(PROTOS_PATH)/bucket.pb.o $(PROTOS_PATH)/bucket.grpc.pb.o $(PROTOS_PATH)/mid_tier.pb.o $(PROTOS_PATH)/mid_tier.grpc.pb.o $(BUCKET_PATH)/src/multiple_points.o $(BUCKET_PATH)/src/point.o $(BUCKET_PATH)/src/utils.o $(BUCKET_PATH)/src/dist_calc.o $(BUCKET_PATH)/src/custom_priority_queue.o $(BUCKET_PATH)/service/helper_files/client_helper.o $(INDEX_PATH)/service/helper_files/mid_tier_server_helper.o $(INDEX_PATH)/service/helper_files/timing.o $(INDEX_PATH)/service/helper_files/utils.o mid_tier_server.o 19 | $(CXX) $^ -o $@ $(LDFLAGS) 20 | 21 | .PRECIOUS: %.grpc.pb.cc 22 | %.grpc.pb.cc: %.proto 23 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 24 | 25 | .PRECIOUS: %.pb.cc 26 | %.pb.cc: %.proto 27 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $< 28 | 29 | clean: 30 | rm -f *.o *.pb.cc *.pb.h mid_tier_server 31 | 32 | 33 | # The following is to test your system and ensure a smoother experience. 34 | # They are by no means necessary to actually compile a grpc-enabled software. 35 | 36 | PROTOC_CMD = which $(PROTOC) 37 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 38 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 39 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 40 | ifeq ($(HAS_PROTOC),true) 41 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 42 | endif 43 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 44 | 45 | SYSTEM_OK = false 46 | ifeq ($(HAS_VALID_PROTOC),true) 47 | ifeq ($(HAS_PLUGIN),true) 48 | SYSTEM_OK = true 49 | endif 50 | endif 51 | 52 | system-check: 53 | ifneq ($(HAS_VALID_PROTOC),true) 54 | @echo " DEPENDENCY ERROR" 55 | @echo 56 | @echo "You don't have protoc 3.0.0 installed in your path." 57 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 58 | @echo "You can find it here:" 59 | @echo 60 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 61 | @echo 62 | @echo "Here is what I get when trying to evaluate your version of protoc:" 63 | @echo 64 | -$(PROTOC) --version 65 | @echo 66 | @echo 67 | endif 68 | ifneq ($(HAS_PLUGIN),true) 69 | @echo " DEPENDENCY ERROR" 70 | @echo 71 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 72 | @echo "Please install grpc. You can find it here:" 73 | @echo 74 | @echo " https://github.com/grpc/grpc" 75 | @echo 76 | @echo "Here is what I get when trying to detect if you have the plugin:" 77 | @echo 78 | -which $(GRPC_CPP_PLUGIN) 79 | @echo 80 | @echo 81 | endif 82 | ifneq ($(SYSTEM_OK),true) 83 | @false 84 | endif 85 | -------------------------------------------------------------------------------- /src/SetAlgebra/intersection_service/service/helper_files/server_helper.cc: -------------------------------------------------------------------------------- 1 | #include "server_helper.h" 2 | 3 | #define BASE (10) 4 | 5 | void CreateIndexFromFile(const std::string dataset_file_name, 6 | std::map >* index) 7 | { 8 | std::ifstream file(dataset_file_name); 9 | // If file does not exist, error out. 10 | CHECK((file.good()), "Cannot create words_ids->docids index from file because file does not exist\n"); 11 | std::string line; 12 | Docids token_no = 0, word_id = 0, doc_id = 0; 13 | char* err_check; 14 | 15 | // Process each file line. 16 | for(int i = 0; std::getline(file, line); i++) 17 | { 18 | std::istringstream buf(line); 19 | std::istream_iterator begin(buf), end; 20 | std::vector tokens(begin, end); 21 | 22 | /* Read values and return a set of doc 23 | ids corresponding to the word id.*/ 24 | token_no = 0; 25 | 26 | // Set a size to the empty point. 27 | 28 | for(auto& s: tokens) 29 | { 30 | // All values read from be floats, else catch exception. 31 | if(token_no == 0) { 32 | word_id = strtol(s.c_str(), &err_check, BASE); 33 | } else { 34 | if (token_no == 1) { 35 | std::vector docids; 36 | docids.emplace_back(strtol(s.c_str(), &err_check, BASE)); 37 | (*index)[word_id] = docids; 38 | } else { 39 | index->at(word_id).emplace_back(strtol(s.c_str(), &err_check, BASE)); 40 | } 41 | } 42 | token_no++; 43 | } 44 | } 45 | } 46 | 47 | void UnpackIntersectionServiceRequest(const intersection::IntersectionRequest &request, 48 | std::vector* word_ids) 49 | { 50 | int num_ids = request.word_ids_size(); 51 | Docids id_value = 0; 52 | 53 | for(int i = 0; i < num_ids; i++) { 54 | id_value = request.word_ids(i); 55 | word_ids->emplace_back(id_value); 56 | } 57 | } 58 | 59 | bool ExtractDocids(const std::vector &word_ids, 60 | const std::map > &word_to_docids, 61 | std::vector >* doc_ids_for_all_words) 62 | { 63 | Docids num_words_in_query = word_ids.size(); 64 | 65 | for(Docids i = 0; i < num_words_in_query; i++) { 66 | try { 67 | word_to_docids.at(word_ids[i]); 68 | } catch(...) { 69 | return false; 70 | //CHECK(false, "Word not present in wikipedia\n"); 71 | /* Return result for other words if this 72 | word if this word does not exist in wikipedia.*/ 73 | continue; 74 | } 75 | // AS HACK 76 | if (word_to_docids.at(word_ids[i]).size() > 1000) 77 | { 78 | continue; 79 | } 80 | doc_ids_for_all_words->emplace_back(word_to_docids.at(word_ids[i])); 81 | } 82 | /* If all words must have doc ids, then you should 83 | activate the following invariant.*/ 84 | //CHECK( (doc_ids_for_all_words->size() == num_words_in_query), "Could not populate doc ids for all words in the query\n"); 85 | return true; 86 | } 87 | 88 | void PackIntersectionServiceResponse(const std::vector &intersection_res, 89 | intersection::IntersectionResponse* reply) 90 | { 91 | Docids intersection_res_size = intersection_res.size(); 92 | 93 | for(Docids i = 0; i < intersection_res_size; i++) 94 | { 95 | reply->add_doc_ids(intersection_res[i]); 96 | 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/nn/ground_truth.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_GROUND_TRUTH_H_ 32 | #define FLANN_GROUND_TRUTH_H_ 33 | 34 | #include "flann/algorithms/dist.h" 35 | #include "flann/util/matrix.h" 36 | 37 | 38 | namespace flann 39 | { 40 | 41 | template 42 | void find_nearest(const Matrix& dataset, typename Distance::ElementType* query, size_t* matches, size_t nn, 43 | size_t skip = 0, Distance distance = Distance()) 44 | { 45 | //typedef typename Distance::ElementType ElementType; 46 | typedef typename Distance::ResultType DistanceType; 47 | int n = nn + skip; 48 | 49 | int* match = new int[n]; 50 | DistanceType* dists = new DistanceType[n]; 51 | 52 | dists[0] = distance(dataset[0], query, dataset.cols); 53 | match[0] = 0; 54 | int dcnt = 1; 55 | 56 | for (size_t i=1; i=1 && dists[j] 87 | void compute_ground_truth(const Matrix& dataset, const Matrix& testset, Matrix& matches, 88 | int skip=0, Distance d = Distance()) 89 | { 90 | for (size_t i=0; i(dataset, testset[i], matches[i], matches.cols, skip, d); 92 | } 93 | } 94 | 95 | 96 | } 97 | 98 | #endif //FLANN_GROUND_TRUTH_H_ 99 | -------------------------------------------------------------------------------- /src/Router/load_generator/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../protoc_files 11 | BUCKET_PATH = ../lookup_service 12 | INDEX_PATH = ../mid_tier_service 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check load_generator_closed_loop load_generator_open_loop kill_router_server_empty 17 | 18 | load_generator_closed_loop: $(PROTOS_PATH)/router.pb.o $(PROTOS_PATH)/router.grpc.pb.o helper_files/loadgen_router_client_helper.o $(INDEX_PATH)/service/helper_files/timing.o $(INDEX_PATH)/service/helper_files/utils.o load_generator_closed_loop.o 19 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 20 | 21 | load_generator_open_loop: $(PROTOS_PATH)/router.pb.o $(PROTOS_PATH)/router.grpc.pb.o helper_files/loadgen_router_client_helper.o $(INDEX_PATH)/service/helper_files/timing.o $(INDEX_PATH)/service/helper_files/utils.o load_generator_open_loop.o 22 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 23 | 24 | kill_router_server_empty: $(PROTOS_PATH)/router.pb.o $(PROTOS_PATH)/router.grpc.pb.o helper_files/loadgen_router_client_helper.o $(INDEX_PATH)/service/helper_files/timing.o $(INDEX_PATH)/service/helper_files/utils.o kill_router_server_empty.o 25 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 26 | 27 | .PRECIOUS: %.grpc.pb.cc 28 | %.grpc.pb.cc: %.proto 29 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 30 | 31 | .PRECIOUS: %.pb.cc 32 | %.pb.cc: %.proto 33 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 34 | 35 | clean: 36 | rm -f *.o *.pb.cc *.pb.h load_generator_closed_loop load_generator_open_loop kill_router_server_empty 37 | 38 | 39 | # The following is to test your system and ensure a smoother experience. 40 | # They are by no means necessary to actually compile a grpc-enabled software. 41 | 42 | PROTOC_CMD = which $(PROTOC) 43 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 44 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 45 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 46 | ifeq ($(HAS_PROTOC),true) 47 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 48 | endif 49 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 50 | 51 | SYSTEM_OK = false 52 | ifeq ($(HAS_VALID_PROTOC),true) 53 | ifeq ($(HAS_PLUGIN),true) 54 | SYSTEM_OK = true 55 | endif 56 | endif 57 | 58 | system-check: 59 | ifneq ($(HAS_VALID_PROTOC),true) 60 | @echo " DEPENDENCY ERROR" 61 | @echo 62 | @echo "You don't have protoc 3.0.0 installed in your path." 63 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 64 | @echo "You can find it here:" 65 | @echo 66 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 67 | @echo 68 | @echo "Here is what I get when trying to evaluate your version of protoc:" 69 | @echo 70 | -$(PROTOC) --version 71 | @echo 72 | @echo 73 | endif 74 | ifneq ($(HAS_PLUGIN),true) 75 | @echo " DEPENDENCY ERROR" 76 | @echo 77 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 78 | @echo "Please install grpc. You can find it here:" 79 | @echo 80 | @echo " https://github.com/grpc/grpc" 81 | @echo 82 | @echo "Here is what I get when trying to detect if you have the plugin:" 83 | @echo 84 | -which $(GRPC_CPP_PLUGIN) 85 | @echo 86 | @echo 87 | endif 88 | ifneq ($(SYSTEM_OK),true) 89 | @false 90 | endif 91 | -------------------------------------------------------------------------------- /src/SetAlgebra/load_generator/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../protoc_files 11 | INTERSECTION_PATH = ../intersection_service 12 | UNION_PATH = ../union_service 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check load_generator_closed_loop load_generator_open_loop kill_union_server_empty 17 | 18 | load_generator_closed_loop: $(PROTOS_PATH)/union.pb.o $(PROTOS_PATH)/union.grpc.pb.o helper_files/loadgen_union_client_helper.o $(UNION_PATH)/service/helper_files/timing.o $(UNION_PATH)/service/helper_files/utils.o load_generator_closed_loop.o 19 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 20 | 21 | load_generator_open_loop: $(PROTOS_PATH)/union.pb.o $(PROTOS_PATH)/union.grpc.pb.o helper_files/loadgen_union_client_helper.o $(UNION_PATH)/service/helper_files/timing.o $(UNION_PATH)/service/helper_files/utils.o load_generator_open_loop.o 22 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 23 | 24 | kill_union_server_empty: $(PROTOS_PATH)/union.pb.o $(PROTOS_PATH)/union.grpc.pb.o helper_files/loadgen_union_client_helper.o $(UNION_PATH)/service/helper_files/timing.o $(UNION_PATH)/service/helper_files/utils.o kill_union_server_empty.o 25 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 26 | 27 | .PRECIOUS: %.grpc.pb.cc 28 | %.grpc.pb.cc: %.proto 29 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 30 | 31 | .PRECIOUS: %.pb.cc 32 | %.pb.cc: %.proto 33 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 34 | 35 | clean: 36 | rm -f *.o *.pb.cc *.pb.h load_generator_closed_loop load_generator_open_loop kill_union_server_empty 37 | 38 | 39 | # The following is to test your system and ensure a smoother experience. 40 | # They are by no means necessary to actually compile a grpc-enabled software. 41 | 42 | PROTOC_CMD = which $(PROTOC) 43 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 44 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 45 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 46 | ifeq ($(HAS_PROTOC),true) 47 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 48 | endif 49 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 50 | 51 | SYSTEM_OK = false 52 | ifeq ($(HAS_VALID_PROTOC),true) 53 | ifeq ($(HAS_PLUGIN),true) 54 | SYSTEM_OK = true 55 | endif 56 | endif 57 | 58 | system-check: 59 | ifneq ($(HAS_VALID_PROTOC),true) 60 | @echo " DEPENDENCY ERROR" 61 | @echo 62 | @echo "You don't have protoc 3.0.0 installed in your path." 63 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 64 | @echo "You can find it here:" 65 | @echo 66 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 67 | @echo 68 | @echo "Here is what I get when trying to evaluate your version of protoc:" 69 | @echo 70 | -$(PROTOC) --version 71 | @echo 72 | @echo 73 | endif 74 | ifneq ($(HAS_PLUGIN),true) 75 | @echo " DEPENDENCY ERROR" 76 | @echo 77 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 78 | @echo "Please install grpc. You can find it here:" 79 | @echo 80 | @echo " https://github.com/grpc/grpc" 81 | @echo 82 | @echo "Here is what I get when trying to detect if you have the plugin:" 83 | @echo 84 | -which $(GRPC_CPP_PLUGIN) 85 | @echo 86 | @echo 87 | endif 88 | ifneq ($(SYSTEM_OK),true) 89 | @false 90 | endif 91 | -------------------------------------------------------------------------------- /src/Recommend/load_generator/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../protoc_files 11 | CF_PATH = ../cf_service 12 | RECOMMENDER_PATH = ../recommender_service 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check load_generator_closed_loop load_generator_open_loop kill_recommender_server_empty 17 | 18 | load_generator_closed_loop: $(PROTOS_PATH)/recommender.pb.o $(PROTOS_PATH)/recommender.grpc.pb.o helper_files/loadgen_recommender_client_helper.o $(RECOMMENDER_PATH)/service/helper_files/timing.o $(RECOMMENDER_PATH)/service/helper_files/utils.o load_generator_closed_loop.o 19 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 20 | 21 | load_generator_open_loop: $(PROTOS_PATH)/recommender.pb.o $(PROTOS_PATH)/recommender.grpc.pb.o helper_files/loadgen_recommender_client_helper.o $(RECOMMENDER_PATH)/service/helper_files/timing.o $(RECOMMENDER_PATH)/service/helper_files/utils.o load_generator_open_loop.o 22 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 23 | 24 | kill_recommender_server_empty: $(PROTOS_PATH)/recommender.pb.o $(PROTOS_PATH)/recommender.grpc.pb.o helper_files/loadgen_recommender_client_helper.o $(RECOMMENDER_PATH)/service/helper_files/timing.o $(RECOMMENDER_PATH)/service/helper_files/utils.o kill_recommender_server_empty.o 25 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 26 | 27 | .PRECIOUS: %.grpc.pb.cc 28 | %.grpc.pb.cc: %.proto 29 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 30 | 31 | .PRECIOUS: %.pb.cc 32 | %.pb.cc: %.proto 33 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 34 | 35 | clean: 36 | rm -f *.o *.pb.cc *.pb.h load_generator_closed_loop load_generator_open_loop kill_recommender_server_empty 37 | 38 | 39 | # The following is to test your system and ensure a smoother experience. 40 | # They are by no means necessary to actually compile a grpc-enabled software. 41 | 42 | PROTOC_CMD = which $(PROTOC) 43 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 44 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 45 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 46 | ifeq ($(HAS_PROTOC),true) 47 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 48 | endif 49 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 50 | 51 | SYSTEM_OK = false 52 | ifeq ($(HAS_VALID_PROTOC),true) 53 | ifeq ($(HAS_PLUGIN),true) 54 | SYSTEM_OK = true 55 | endif 56 | endif 57 | 58 | system-check: 59 | ifneq ($(HAS_VALID_PROTOC),true) 60 | @echo " DEPENDENCY ERROR" 61 | @echo 62 | @echo "You don't have protoc 3.0.0 installed in your path." 63 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 64 | @echo "You can find it here:" 65 | @echo 66 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 67 | @echo 68 | @echo "Here is what I get when trying to evaluate your version of protoc:" 69 | @echo 70 | -$(PROTOC) --version 71 | @echo 72 | @echo 73 | endif 74 | ifneq ($(HAS_PLUGIN),true) 75 | @echo " DEPENDENCY ERROR" 76 | @echo 77 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 78 | @echo "Please install grpc. You can find it here:" 79 | @echo 80 | @echo " https://github.com/grpc/grpc" 81 | @echo 82 | @echo "Here is what I get when trying to detect if you have the plugin:" 83 | @echo 84 | -which $(GRPC_CPP_PLUGIN) 85 | @echo 86 | @echo 87 | endif 88 | ifneq ($(SYSTEM_OK),true) 89 | @false 90 | endif 91 | -------------------------------------------------------------------------------- /src/HDSearch/bucket_service/service/helper_files/client_helper.h: -------------------------------------------------------------------------------- 1 | #ifndef __CLIENT_HELPER_H_INCLUDED__ 2 | #define __CLIENT_HELPER_H_INCLUDED__ 3 | 4 | #include "bucket_service/src/dist_calc.h" 5 | #include "protoc_files/bucket.grpc.pb.h" 6 | 7 | struct BucketTimingInfo { 8 | uint64_t create_bucket_request_time = 0; 9 | uint64_t unpack_bucket_req_time = 0; 10 | uint64_t calculate_knn_time = 0; 11 | uint64_t pack_bucket_resp_time = 0; 12 | uint64_t unpack_bucket_resp_time = 0; 13 | float cpu_util_bucket = 0.0; 14 | }; 15 | 16 | struct BucketUtil { 17 | bool util_present = false; 18 | uint64_t user_time = 0; 19 | uint64_t system_time = 0; 20 | uint64_t io_time = 0; 21 | uint64_t idle_time = 0; 22 | }; 23 | 24 | struct BucketClientCommandLineArgs{ 25 | unsigned number_of_nearest_neighbors = 0; 26 | }; 27 | 28 | struct ResponseData { 29 | DistCalc* knn_answer = new DistCalc(); 30 | BucketTimingInfo* bucket_timing_info = new BucketTimingInfo(); 31 | BucketUtil* bucket_util = new BucketUtil(); 32 | }; 33 | 34 | /* Parses command line args and puts them into a BucketClientCommandLineArgs 35 | struct. 36 | In: user command line input 37 | Out: pointer to struct BucketClientCommandLineArgs. 38 | A.S As of now, the command line accepts number of nearest neighbors to be computed.*/ 39 | BucketClientCommandLineArgs* ParseBucketClientCommandLine(const int &argc, 40 | char** argv); 41 | // A.S Spare function - not using it any more. 42 | void ReadPointIDsFromFile(const std::string &point_ids_file_name, 43 | std::vector >* point_ids); 44 | /* Packs data that needs to be sent to the bucket server, into a 45 | protobuf message. 46 | In: queries, point IDs, queries_size, number of neighbors that must 47 | be computed, number of dimensions of each point, ID of the bucket server, 48 | shard size. 49 | Out: Protobuf message - request.*/ 50 | void CreateBucketServiceRequest(const MultiplePoints &queries, 51 | const std::vector> &point_ids, 52 | const unsigned queries_size, 53 | const unsigned number_of_nearest_neighbors, 54 | const int dimension, 55 | const uint32_t bucket_server_id, 56 | const int shard_size, 57 | const bool util_present, 58 | bucket::NearestNeighborRequest* request); 59 | /* Convert the bucket server's protobuf response message 60 | into a bunch of different response values. 61 | In: Reply from the bucket server, number of nearest neighbors. 62 | Out: k-nn answers for all queries, timing info: unpacking bucket server req, 63 | knn distance calc time, packing bucket server respose time (in micro).*/ 64 | void UnpackBucketServiceResponse(const bucket::NearestNeighborResponse &reply, 65 | const unsigned number_of_nearest_neighbors, 66 | DistCalc* knn_answer, 67 | BucketTimingInfo* bucket_timing_info, 68 | BucketUtil* bucket_util); 69 | 70 | /* Following two functions are just helper unpack 71 | functions to the function above.*/ 72 | void UnpackTimingInfo(const bucket::NearestNeighborResponse &reply, 73 | BucketTimingInfo* bucket_timing_info); 74 | 75 | void UnpackUtilInfo(const bucket::NearestNeighborResponse &reply, 76 | BucketUtil* bucket_util); 77 | 78 | void AddBucketResponseToMap(); 79 | 80 | /* Create an output text file containing k-nn for all input queries. 81 | Program exits if file cannot be opened. 82 | In: file name, final k-nn answer for all queries.*/ 83 | void WriteKNNToFile(const std::string &knn_file_name, 84 | const DistCalc &knn_answer); 85 | /* Prints contents of an object of type: DistCalc. 86 | In: object of type DistCalc.*/ 87 | void PrintKNNForAllQueries(const DistCalc &knn_answer); 88 | /* Prints point IDs returned by the index for all queries. 89 | In: point IDs for all queries.*/ 90 | void PrintPointIDs(const std::vector > &point_ids); 91 | /* Reads a text file, and converts it into a vector of points. 92 | Program exits if file canot be opened/ does not exist. 93 | In: name of the text file. 94 | Out: vector of points.*/ 95 | void CreatePointsFromFile(const std::string &file_name, 96 | MultiplePoints* multiple_points); 97 | 98 | #endif // __CLIENT_HELPER_H_INCLUDED__ 99 | -------------------------------------------------------------------------------- /src/HDSearch/load_generator/Makefile: -------------------------------------------------------------------------------- 1 | HDS_PATH = ../ 2 | CXX = g++ 3 | CPPFLAGS += -I/usr/local/include -pthread -O3 -I$(HDS_PATH) -g -Wall -fopenmp -I../ 4 | CXXFLAGS += -std=c++11 -O3 -mavx2 -mavx -fopenmp -DMKL_ILP64 -m64 -I/opt/intel/mkl/include -I../ 5 | LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -I /usr/local/include -lflann -fopenmp -L/usr/lib64 -lstdc++ -lssl -lcrypto -fopenmp -Wl,--start-group /opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.a /opt/intel/mkl/lib/intel64/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl -I../ 6 | PROTOC = protoc 7 | GRPC_CPP_PLUGIN = grpc_cpp_plugin 8 | GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` 9 | 10 | PROTOS_PATH = ../protoc_files 11 | BUCKET_PATH = ../bucket_service 12 | INDEX_PATH = ../mid_tier_service 13 | 14 | vpath %.proto $(PROTOS_PATH) 15 | 16 | all: system-check load_generator_open_loop load_generator_closed_loop kill_index_server_empty 17 | 18 | load_generator_open_loop: $(PROTOS_PATH)/mid_tier.pb.o $(PROTOS_PATH)/mid_tier.grpc.pb.o $(BUCKET_PATH)/src/multiple_points.o $(BUCKET_PATH)/src/point.o $(BUCKET_PATH)/src/utils.o $(BUCKET_PATH)/src/dist_calc.o $(BUCKET_PATH)/src/custom_priority_queue.o helper_files/mid_tier_client_helper.o helper_files/timing.o helper_files/utils.o load_generator_open_loop.o 19 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 20 | 21 | load_generator_closed_loop: $(PROTOS_PATH)/mid_tier.pb.o $(PROTOS_PATH)/mid_tier.grpc.pb.o $(BUCKET_PATH)/src/multiple_points.o $(BUCKET_PATH)/src/point.o $(BUCKET_PATH)/src/utils.o $(BUCKET_PATH)/src/dist_calc.o $(BUCKET_PATH)/src/custom_priority_queue.o helper_files/mid_tier_client_helper.o helper_files/timing.o helper_files/utils.o load_generator_closed_loop.o 22 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 23 | 24 | kill_index_server_empty: $(PROTOS_PATH)/mid_tier.pb.o $(PROTOS_PATH)/mid_tier.grpc.pb.o $(BUCKET_PATH)/src/multiple_points.o $(BUCKET_PATH)/src/point.o $(BUCKET_PATH)/src/utils.o $(BUCKET_PATH)/src/dist_calc.o $(BUCKET_PATH)/src/custom_priority_queue.o helper_files/mid_tier_client_helper.o helper_files/timing.o helper_files/utils.o kill_index_server_empty.o 25 | $(CXX) $^ -O3 -o $@ $(LDFLAGS) 26 | 27 | .PRECIOUS: %.grpc.pb.cc 28 | %.grpc.pb.cc: %.proto 29 | $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< 30 | 31 | .PRECIOUS: %.pb.cc 32 | %.pb.cc: %.proto 33 | $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< 34 | 35 | clean: 36 | rm -f *.o *.pb.cc *.pb.h load_generator_open_loop load_generator_closed_loop kill_index_server_empty 37 | 38 | 39 | # The following is to test your system and ensure a smoother experience. 40 | # They are by no means necessary to actually compile a grpc-enabled software. 41 | 42 | PROTOC_CMD = which $(PROTOC) 43 | PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3 44 | PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN) 45 | HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false) 46 | ifeq ($(HAS_PROTOC),true) 47 | HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false) 48 | endif 49 | HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false) 50 | 51 | SYSTEM_OK = false 52 | ifeq ($(HAS_VALID_PROTOC),true) 53 | ifeq ($(HAS_PLUGIN),true) 54 | SYSTEM_OK = true 55 | endif 56 | endif 57 | 58 | system-check: 59 | ifneq ($(HAS_VALID_PROTOC),true) 60 | @echo " DEPENDENCY ERROR" 61 | @echo 62 | @echo "You don't have protoc 3.0.0 installed in your path." 63 | @echo "Please install Google protocol buffers 3.0.0 and its compiler." 64 | @echo "You can find it here:" 65 | @echo 66 | @echo " https://github.com/google/protobuf/releases/tag/v3.0.0-beta-2" 67 | @echo 68 | @echo "Here is what I get when trying to evaluate your version of protoc:" 69 | @echo 70 | -$(PROTOC) --version 71 | @echo 72 | @echo 73 | endif 74 | ifneq ($(HAS_PLUGIN),true) 75 | @echo " DEPENDENCY ERROR" 76 | @echo 77 | @echo "You don't have the grpc c++ protobuf plugin installed in your path." 78 | @echo "Please install grpc. You can find it here:" 79 | @echo 80 | @echo " https://github.com/grpc/grpc" 81 | @echo 82 | @echo "Here is what I get when trying to detect if you have the plugin:" 83 | @echo 84 | -which $(GRPC_CPP_PLUGIN) 85 | @echo 86 | @echo 87 | endif 88 | ifneq ($(SYSTEM_OK),true) 89 | @false 90 | endif 91 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/matrix.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_DATASET_H_ 32 | #define FLANN_DATASET_H_ 33 | 34 | #include "flann/general.h" 35 | #include "flann/util/serialization.h" 36 | #include 37 | 38 | namespace flann 39 | { 40 | 41 | typedef unsigned char uchar; 42 | 43 | class Matrix_ 44 | { 45 | public: 46 | 47 | Matrix_() : rows(0), cols(0), stride(0), type(FLANN_NONE), data(NULL) 48 | { 49 | }; 50 | 51 | Matrix_(void* data_, size_t rows_, size_t cols_, flann_datatype_t type_, size_t stride_ = 0) : 52 | rows(rows_), cols(cols_), stride(stride_), type(type_) 53 | { 54 | data = static_cast(data_); 55 | 56 | if (stride==0) stride = flann_datatype_size(type)*cols; 57 | } 58 | 59 | /** 60 | * Operator that returns a (pointer to a) row of the data. 61 | */ 62 | inline void* operator[](size_t index) const 63 | { 64 | return data+index*stride; 65 | } 66 | 67 | void* ptr() const 68 | { 69 | return data; 70 | } 71 | 72 | size_t rows; 73 | size_t cols; 74 | size_t stride; 75 | flann_datatype_t type; 76 | protected: 77 | uchar* data; 78 | 79 | template 80 | void serialize(Archive& ar) 81 | { 82 | ar & rows; 83 | ar & cols; 84 | ar & stride; 85 | ar & type; 86 | if (Archive::is_loading::value) { 87 | data = new uchar[rows*stride]; 88 | } 89 | ar & serialization::make_binary_object(data, rows*stride); 90 | } 91 | friend struct serialization::access; 92 | }; 93 | 94 | 95 | /** 96 | * Class that implements a simple rectangular matrix stored in a memory buffer and 97 | * provides convenient matrix-like access using the [] operators. 98 | * 99 | * This class has the same memory structure as the un-templated class flann::Matrix_ and 100 | * it's directly convertible from it. 101 | */ 102 | template 103 | class Matrix : public Matrix_ 104 | { 105 | public: 106 | typedef T type; 107 | 108 | Matrix() : Matrix_() 109 | { 110 | } 111 | 112 | Matrix(T* data_, size_t rows_, size_t cols_, size_t stride_ = 0) : 113 | Matrix_(data_, rows_, cols_, flann_datatype_value::value, stride_) 114 | { 115 | if (stride==0) stride = sizeof(T)*cols; 116 | } 117 | 118 | /** 119 | * Operator that returns a (pointer to a) row of the data. 120 | */ 121 | inline T* operator[](size_t index) const 122 | { 123 | return reinterpret_cast(data+index*stride); 124 | } 125 | 126 | 127 | T* ptr() const 128 | { 129 | return reinterpret_cast(data); 130 | } 131 | }; 132 | 133 | } 134 | 135 | #endif //FLANN_DATASET_H_ 136 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/saving.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * 1. Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 2. Redistributions in binary form must reproduce the above copyright 14 | * notice, this list of conditions and the following disclaimer in the 15 | * documentation and/or other materials provided with the distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE NNIndexGOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | *************************************************************************/ 28 | 29 | #ifndef FLANN_SAVING_H_ 30 | #define FLANN_SAVING_H_ 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include "flann/general.h" 37 | #include "flann/util/serialization.h" 38 | 39 | 40 | #ifdef FLANN_SIGNATURE_ 41 | #undef FLANN_SIGNATURE_ 42 | #endif 43 | #define FLANN_SIGNATURE_ "FLANN_INDEX_v1.1" 44 | 45 | namespace flann 46 | { 47 | 48 | /** 49 | * Structure representing the index header. 50 | */ 51 | struct IndexHeader 52 | { 53 | IndexHeaderStruct h; 54 | 55 | IndexHeader() 56 | { 57 | memset(h.signature, 0, sizeof(h.signature)); 58 | strcpy(h.signature, FLANN_SIGNATURE_); 59 | memset(h.version, 0, sizeof(h.version)); 60 | strcpy(h.version, FLANN_VERSION_); 61 | 62 | h.compression = 0; 63 | h.first_block_size = 0; 64 | } 65 | 66 | private: 67 | template 68 | void serialize(Archive& ar) 69 | { 70 | ar & h.signature; 71 | ar & h.version; 72 | ar & h.data_type; 73 | ar & h.index_type; 74 | ar & h.rows; 75 | ar & h.cols; 76 | ar & h.compression; 77 | ar & h.first_block_size; 78 | } 79 | friend struct serialization::access; 80 | }; 81 | 82 | /** 83 | * Saves index header to stream 84 | * 85 | * @param stream - Stream to save to 86 | * @param index - The index to save 87 | */ 88 | template 89 | void save_header(FILE* stream, const Index& index) 90 | { 91 | IndexHeader header; 92 | header.h.data_type = flann_datatype_value::value; 93 | header.h.index_type = index.getType(); 94 | header.h.rows = index.size(); 95 | header.h.cols = index.veclen(); 96 | 97 | fwrite(&header, sizeof(header),1,stream); 98 | } 99 | 100 | 101 | /** 102 | * 103 | * @param stream - Stream to load from 104 | * @return Index header 105 | */ 106 | inline IndexHeader load_header(FILE* stream) 107 | { 108 | IndexHeader header; 109 | int read_size = fread(&header,sizeof(header),1,stream); 110 | 111 | if (read_size != 1) { 112 | throw FLANNException("Invalid index file, cannot read"); 113 | } 114 | 115 | if (strncmp(header.h.signature, 116 | FLANN_SIGNATURE_, 117 | strlen(FLANN_SIGNATURE_) - strlen("v0.0")) != 0) { 118 | throw FLANNException("Invalid index file, wrong signature"); 119 | } 120 | 121 | return header; 122 | } 123 | 124 | 125 | namespace serialization 126 | { 127 | ENUM_SERIALIZER(flann_algorithm_t); 128 | ENUM_SERIALIZER(flann_centers_init_t); 129 | ENUM_SERIALIZER(flann_log_level_t); 130 | ENUM_SERIALIZER(flann_datatype_t); 131 | } 132 | 133 | } 134 | 135 | #endif /* FLANN_SAVING_H_ */ 136 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/logger.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_LOGGER_H 32 | #define FLANN_LOGGER_H 33 | 34 | #include 35 | #include 36 | 37 | #include "flann/defines.h" 38 | 39 | 40 | namespace flann 41 | { 42 | 43 | class Logger 44 | { 45 | Logger() : stream(stdout), logLevel(FLANN_LOG_WARN) {} 46 | 47 | ~Logger() 48 | { 49 | if ((stream!=NULL)&&(stream!=stdout)) { 50 | fclose(stream); 51 | } 52 | } 53 | 54 | static Logger& instance() 55 | { 56 | static Logger logger; 57 | return logger; 58 | } 59 | 60 | void _setDestination(const char* name) 61 | { 62 | if (name==NULL) { 63 | stream = stdout; 64 | } 65 | else { 66 | stream = fopen(name,"w"); 67 | if (stream == NULL) { 68 | stream = stdout; 69 | } 70 | } 71 | } 72 | 73 | int _log(int level, const char* fmt, va_list arglist) 74 | { 75 | if (level > logLevel ) return -1; 76 | int ret = vfprintf(stream, fmt, arglist); 77 | return ret; 78 | } 79 | 80 | public: 81 | /** 82 | * Sets the logging level. All messages with lower priority will be ignored. 83 | * @param level Logging level 84 | */ 85 | static void setLevel(int level) { instance().logLevel = level; } 86 | 87 | /** 88 | * Returns the currently set logging level. 89 | * @return current logging level 90 | */ 91 | static int getLevel() { return instance().logLevel; } 92 | 93 | /** 94 | * Sets the logging destination 95 | * @param name Filename or NULL for console 96 | */ 97 | static void setDestination(const char* name) { instance()._setDestination(name); } 98 | 99 | /** 100 | * Print log message 101 | * @param level Log level 102 | * @param fmt Message format 103 | * @return 104 | */ 105 | static int log(int level, const char* fmt, ...) 106 | { 107 | va_list arglist; 108 | va_start(arglist, fmt); 109 | int ret = instance()._log(level,fmt,arglist); 110 | va_end(arglist); 111 | return ret; 112 | } 113 | 114 | #define LOG_METHOD(NAME,LEVEL) \ 115 | static int NAME(const char* fmt, ...) \ 116 | { \ 117 | va_list ap; \ 118 | va_start(ap, fmt); \ 119 | int ret = instance()._log(LEVEL, fmt, ap); \ 120 | va_end(ap); \ 121 | return ret; \ 122 | } 123 | 124 | LOG_METHOD(fatal, FLANN_LOG_FATAL) 125 | LOG_METHOD(error, FLANN_LOG_ERROR) 126 | LOG_METHOD(warn, FLANN_LOG_WARN) 127 | LOG_METHOD(info, FLANN_LOG_INFO) 128 | LOG_METHOD(debug, FLANN_LOG_DEBUG) 129 | 130 | private: 131 | FILE* stream; 132 | int logLevel; 133 | }; 134 | 135 | } 136 | 137 | #endif //FLANN_LOGGER_H 138 | -------------------------------------------------------------------------------- /src/HDSearch/mid_tier_service/src/cpp/flann/util/random.h: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * Software License Agreement (BSD License) 3 | * 4 | * Copyright 2008-2009 Marius Muja (mariusm@cs.ubc.ca). All rights reserved. 5 | * Copyright 2008-2009 David G. Lowe (lowe@cs.ubc.ca). All rights reserved. 6 | * 7 | * THE BSD LICENSE 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted provided that the following conditions 11 | * are met: 12 | * 13 | * 1. Redistributions of source code must retain the above copyright 14 | * notice, this list of conditions and the following disclaimer. 15 | * 2. Redistributions in binary form must reproduce the above copyright 16 | * notice, this list of conditions and the following disclaimer in the 17 | * documentation and/or other materials provided with the distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | *************************************************************************/ 30 | 31 | #ifndef FLANN_RANDOM_H 32 | #define FLANN_RANDOM_H 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #include "flann/general.h" 40 | 41 | namespace flann 42 | { 43 | 44 | /** 45 | * Seeds the random number generator 46 | * @param seed Random seed 47 | */ 48 | inline void seed_random(unsigned int seed) 49 | { 50 | srand(seed); 51 | } 52 | 53 | /* 54 | * Generates a random double value. 55 | */ 56 | /** 57 | * Generates a random double value. 58 | * @param high Upper limit 59 | * @param low Lower limit 60 | * @return Random double value 61 | */ 62 | inline double rand_double(double high = 1.0, double low = 0) 63 | { 64 | return low + ((high-low) * (std::rand() / (RAND_MAX + 1.0))); 65 | } 66 | 67 | /** 68 | * Generates a random integer value. 69 | * @param high Upper limit 70 | * @param low Lower limit 71 | * @return Random integer value 72 | */ 73 | inline int rand_int(int high = RAND_MAX, int low = 0) 74 | { 75 | return low + (int) ( double(high-low) * (std::rand() / (RAND_MAX + 1.0))); 76 | } 77 | 78 | 79 | class RandomGenerator 80 | { 81 | public: 82 | ptrdiff_t operator() (ptrdiff_t i) { return rand_int(i); } 83 | }; 84 | 85 | 86 | /** 87 | * Random number generator that returns a distinct number from 88 | * the [0,n) interval each time. 89 | */ 90 | class UniqueRandom 91 | { 92 | std::vector vals_; 93 | int size_; 94 | int counter_; 95 | 96 | public: 97 | /** 98 | * Constructor. 99 | * @param n Size of the interval from which to generate 100 | * @return 101 | */ 102 | UniqueRandom(int n) 103 | { 104 | init(n); 105 | } 106 | 107 | /** 108 | * Initializes the number generator. 109 | * @param n the size of the interval from which to generate random numbers. 110 | */ 111 | void init(int n) 112 | { 113 | static RandomGenerator generator; 114 | // create and initialize an array of size n 115 | vals_.resize(n); 116 | size_ = n; 117 | for (int i = 0; i < size_; ++i) vals_[i] = i; 118 | 119 | // shuffle the elements in the array 120 | std::random_shuffle(vals_.begin(), vals_.end(), generator); 121 | 122 | counter_ = 0; 123 | } 124 | 125 | /** 126 | * Return a distinct random integer in greater or equal to 0 and less 127 | * than 'n' on each call. It should be called maximum 'n' times. 128 | * Returns: a random integer 129 | */ 130 | int next() 131 | { 132 | if (counter_ == size_) { 133 | return -1; 134 | } 135 | else { 136 | return vals_[counter_++]; 137 | } 138 | } 139 | }; 140 | 141 | } 142 | 143 | #endif //FLANN_RANDOM_H 144 | 145 | 146 | --------------------------------------------------------------------------------