├── .gitattributes ├── .gitignore ├── CMakeLists.txt ├── CPPLINT.cfg ├── LICENSE ├── README.md ├── bin ├── cli_unit_test.ustore ├── init_test.ustore ├── manip_chunkable.ustore ├── run_preload.sh ├── test_blob_store.ustore ├── test_lucene_cli.sh ├── test_ustore_cli.sh ├── ustore_clean.sh ├── ustore_env.sh ├── ustore_start.sh ├── ustore_stop.sh └── ustore_stop_worker.sh ├── cmake ├── Dependencies.cmake ├── Modules │ ├── FindCryptoPP.cmake │ ├── FindCzmq.cmake │ ├── FindGFlags.cmake │ ├── FindGlog.cmake │ ├── FindLevelDB.cmake │ ├── FindRocksdb.cmake │ └── FindSnappy.cmake └── Utils.cmake ├── conf ├── config.cfg └── workers.lst ├── docker ├── README.md ├── build.sh └── ubuntu │ └── 16.04 │ └── Dockerfile ├── docs ├── api_cmd.md ├── api_http.md ├── arch.jpg ├── arch.md ├── depend.md ├── guide.md ├── intro.md └── pr_guide.md ├── example ├── CMakeLists.txt ├── ca │ ├── README.md │ ├── analytics.cc │ ├── analytics.h │ ├── ca_arguments.h │ ├── ca_main.cc │ ├── utils.cc │ └── utils.h ├── lucene_cli │ ├── lucene_blob_store.cc │ ├── lucene_blob_store.h │ ├── lucene_cli_arguments.h │ ├── lucene_cli_main.cc │ ├── lucene_client.cc │ └── lucene_client.h └── table_op │ ├── README.md │ ├── table_gen.cc │ ├── table_gen.h │ ├── table_op.cc │ ├── table_op.h │ ├── table_op_arguments.h │ └── table_op_main.cc ├── include ├── CPPLINT.cfg ├── benchmark │ ├── bench_config.h │ ├── bench_utils.h │ └── benchmark.h ├── chunk │ ├── chunk.h │ ├── chunk_loader.h │ ├── chunk_writer.h │ ├── chunker.h │ └── segment.h ├── cli │ ├── command.h │ ├── config.h │ └── console.h ├── cluster │ ├── access_logging.h │ ├── chunk_client.h │ ├── chunk_client_service.h │ ├── chunk_service.h │ ├── client.h │ ├── client_service.h │ ├── host_service.h │ ├── master_service.h │ ├── partitioner.h │ ├── port_helper.h │ ├── response_blob.h │ ├── service.h │ ├── worker_client.h │ ├── worker_client_service.h │ └── worker_service.h ├── hash │ ├── buzhash.h │ ├── hash.h │ ├── murmurhash.h │ ├── murmurhash3.h │ └── sha2.h ├── http │ ├── event.h │ ├── http_client.h │ ├── http_msg.h │ ├── lock.h │ ├── net.h │ ├── request.h │ ├── server.h │ └── settings.h ├── net │ ├── ae.h │ ├── anet.h │ ├── hashtable.h │ ├── net.h │ ├── rdma_config.h │ ├── rdma_net.h │ └── zmq_net.h ├── node │ ├── blob_node.h │ ├── cell_node.h │ ├── cursor.h │ ├── list_node.h │ ├── map_node.h │ ├── meta_node.h │ ├── node.h │ ├── node_builder.h │ ├── node_comparator.h │ ├── node_merger.h │ ├── orderedkey.h │ ├── rolling_hash.h │ └── set_node.h ├── recovery │ ├── log_cursor.h │ ├── log_data_writer.h │ ├── log_entry.h │ ├── log_generator.h │ ├── log_reader.h │ ├── log_record.h │ ├── log_thread.h │ ├── log_worker.h │ ├── log_writer.h │ ├── record_header.h │ └── single_log_reader.h ├── spec │ ├── blob_store.h │ ├── db.h │ ├── object_db.h │ ├── object_meta.h │ ├── relational.h │ ├── slice.h │ └── value.h ├── store │ ├── chunk_store.h │ ├── iterator.h │ ├── ldb_store.h │ ├── lst_store.h │ ├── rocks_store.h │ └── store_initializer.h ├── types │ ├── base.h │ ├── client │ │ ├── vblob.h │ │ ├── vhandler.h │ │ ├── vlist.h │ │ ├── vmap.h │ │ ├── vmeta.h │ │ ├── vobject.h │ │ ├── vref.h │ │ ├── vset.h │ │ └── vstring.h │ ├── server │ │ ├── factory.h │ │ ├── sblob.h │ │ ├── slist.h │ │ ├── smap.h │ │ ├── sset.h │ │ └── sstring.h │ ├── type.h │ ├── ublob.h │ ├── ucell.h │ ├── uiterator.h │ ├── ulist.h │ ├── umap.h │ ├── uset.h │ └── ustring.h ├── utils │ ├── arguments.h │ ├── blocking_queue.h │ ├── chars.h │ ├── debug.h │ ├── enum.h │ ├── env.h │ ├── logging.h │ ├── map_check_policy.h │ ├── message_parser.h │ ├── noncopyable.h │ ├── rocksdb.h │ ├── shared_lock.h │ ├── singleton.h │ ├── sync_task_line.h │ ├── thread_model.h │ ├── timer.h │ ├── type_traits.h │ ├── uniform_random.h │ └── utils.h └── worker │ ├── head_version.h │ ├── rocks_head_version.h │ ├── simple_head_version.h │ ├── worker.h │ └── worker_ext.h ├── lucene ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ ├── application │ │ │ └── Application.java │ │ ├── configuration │ │ │ └── LuceneConfiguration.java │ │ ├── controller │ │ │ └── IndexController.java │ │ ├── pojo │ │ │ ├── Const.java │ │ │ ├── IndexRequest.java │ │ │ ├── IndexResponse.java │ │ │ ├── SearchRequest.java │ │ │ ├── SearchResponse.java │ │ │ └── SearchResult.java │ │ └── service │ │ │ ├── AsyncService.java │ │ │ ├── AsyncServiceImpl.java │ │ │ ├── IndexService.java │ │ │ └── IndexServiceImpl.java │ └── resources │ │ ├── application.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── service │ └── IndexServiceImplTest.java ├── src ├── CMakeLists.txt ├── benchmark │ ├── bench_config.cc │ ├── bench_utils.cc │ └── benchmark.cc ├── chunk │ ├── chunk.cc │ ├── chunk_loader.cc │ ├── chunk_writer.cc │ └── segment.cc ├── cli │ ├── command.cc │ ├── config.cc │ └── console.cc ├── cli_main.cc ├── cluster │ ├── chunk_client.cc │ ├── chunk_client_service.cc │ ├── chunk_service.cc │ ├── client.cc │ ├── client_service.cc │ ├── partitioner.cc │ ├── service.cc │ ├── worker_client.cc │ ├── worker_client_service.cc │ └── worker_service.cc ├── hash │ ├── hash.cc │ └── murmurhash3.cc ├── http │ ├── event.cc │ ├── http_client.cc │ ├── http_msg.cc │ ├── net.cc │ ├── request.cc │ └── server.cc ├── http_main.cc ├── main.cc ├── net │ ├── ae.cc │ ├── ae_epoll.cc │ ├── anet.cc │ ├── net.cc │ ├── rdma_net.cc │ └── zmq_net.cc ├── node │ ├── blob_node.cc │ ├── cell_node.cc │ ├── cursor.cc │ ├── list_node.cc │ ├── map_node.cc │ ├── meta_node.cc │ ├── node.cc │ ├── node_builder.cc │ ├── node_comparator.cc │ ├── orderedkey.cc │ ├── rolling_hash.cc │ └── set_node.cc ├── proto │ ├── config.proto │ ├── head_version.proto │ └── messages.proto ├── recovery │ ├── log_record.cc │ ├── log_thread.cc │ └── log_worker.cc ├── spec │ ├── blob_store.cc │ ├── object_db.cc │ ├── object_meta.cc │ └── relational.cc ├── store │ ├── chunk_store.cc │ ├── ldb_store.cc │ ├── lst_store.cc │ └── rocks_store.cc ├── types │ ├── client │ │ ├── vblob.cc │ │ ├── vhandler.cc │ │ ├── vlist.cc │ │ ├── vmap.cc │ │ ├── vset.cc │ │ └── vstring.cc │ ├── server │ │ ├── sblob.cc │ │ ├── slist.cc │ │ ├── smap.cc │ │ └── sset.cc │ ├── ublob.cc │ ├── ucell.cc │ ├── uiterator.cc │ ├── ulist.cc │ ├── umap.cc │ └── uset.cc ├── utils │ ├── arguments.cc │ ├── debug.cc │ ├── env.cc │ ├── logging.cc │ ├── rocksdb.cc │ ├── uniform_random.cc │ └── utils.cc └── worker │ ├── README.md │ ├── rocks_head_version.cc │ ├── simple_head_version.cc │ ├── worker.cc │ └── worker_ext.cc ├── test ├── CMakeLists.txt ├── benchmark │ ├── dist_bench_main.cc │ ├── micro_bench_main.cc │ └── toy_test.cc ├── gtest │ ├── gtest-all.cc │ ├── gtest.h │ └── gtest_main.cc └── ustore │ ├── test_blob_node.cc │ ├── test_buzhash.cc │ ├── test_ca.cc │ ├── test_cell_node.cc │ ├── test_chunk.cc │ ├── test_chunk_loader.cc │ ├── test_chunk_service.cc │ ├── test_chunker.cc │ ├── test_hash.cc │ ├── test_head_version.cc │ ├── test_http_client.cc │ ├── test_ldb_store.cc │ ├── test_list_node.cc │ ├── test_log_record.cc │ ├── test_log_worker.cc │ ├── test_logging.cc │ ├── test_lst_chunk_store.cc │ ├── test_map_node.cc │ ├── test_messages.cc │ ├── test_meta_node.cc │ ├── test_net.cc │ ├── test_node.cc │ ├── test_node_builder.cc │ ├── test_node_comparator.cc │ ├── test_node_cursor.cc │ ├── test_orderedkey.cc │ ├── test_random_generator.cc │ ├── test_rocks_head_version.cc │ ├── test_rocks_store.cc │ ├── test_rolling_hash.cc │ ├── test_sblob.cc │ ├── test_set_node.cc │ ├── test_slist.cc │ ├── test_smap.cc │ ├── test_sset.cc │ ├── test_ucell.cc │ ├── test_uiterator.cc │ ├── test_utils_debug.cc │ ├── test_vblob.cc │ ├── test_vlist.cc │ ├── test_vmap.cc │ ├── test_vset.cc │ ├── test_vstring.cc │ ├── test_worker.cc │ └── test_worker_client.cc └── tool └── cpplint.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Handle line endings automatically for files detected as text 2 | # and leave all files detected as binary untouched. 3 | * text=auto 4 | 5 | # These files are text and should be normalized (Convert crlf => lf) 6 | *.php text 7 | *.css text 8 | *.js text 9 | *.json text 10 | *.htm text 11 | *.html text 12 | *.xml text 13 | *.txt text 14 | *.ini text 15 | *.inc text 16 | *.pl text 17 | *.rb text 18 | *.py text 19 | *.scm text 20 | *.sql text 21 | .htaccess text 22 | *.sh text 23 | 24 | # These files are binary and should be left untouched 25 | # (binary is a macro for -text -diff) 26 | *.png binary 27 | *.jpg binary 28 | *.jpeg binary 29 | *.gif binary 30 | *.ico binary 31 | *.mov binary 32 | *.mp4 binary 33 | *.mp3 binary 34 | *.flv binary 35 | *.fla binary 36 | *.swf binary 37 | *.gz binary 38 | *.zip binary 39 | *.7z binary 40 | *.ttf binary 41 | *.pyc binary 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | *.pyc 31 | 32 | # Build folder 33 | build/ 34 | 35 | # Perfmon logs 36 | perfmon/logs/ 37 | perfmon/tmp/ 38 | 39 | # Lucene files 40 | lucene/index/ 41 | lucene/docs/ 42 | lucene/logs/ 43 | lucene/target/ 44 | 45 | # Project files 46 | *sublime* 47 | 48 | # Temporary files from vim 49 | *.swp 50 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | # Stop searching for additional CPPLINT.cfg in parent directories. 2 | set noparent 3 | 4 | # Limit line length. In case that a longer line is more readable, add 5 | # '// NOLINT' at the end of the line. 6 | linelength=80 7 | 8 | # Ignore the following categories of errors, as specified by the filter: 9 | filter=-legal/copyright 10 | filter=-build/c++11 11 | filter=-build/include_subdir 12 | -------------------------------------------------------------------------------- /bin/init_test.ustore: -------------------------------------------------------------------------------- 1 | # For test_blob_store.ustore 2 | create-ds -t ds1 -b master 3 | create-ds -t ds1 -b dev 4 | create-ds -t ds2 -b master 5 | -------------------------------------------------------------------------------- /bin/run_preload.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | if [ $# -ge 2 ]; then 4 | ustore_lib_path=$1 5 | bin=$2 6 | else 7 | echo "Usage: ./run_preload.sh ustore_lib_path executable [args]" 8 | exit 9 | fi 10 | 11 | # ustore_lib_path=ustorelib 12 | pre_libs="" 13 | 14 | for lib in `ls $ustore_lib_path` 15 | do 16 | pre_libs="$pre_libs $ustore_lib_path/$lib" 17 | done 18 | 19 | shift 20 | shift 21 | echo "LD_PRELOAD=$pre_libs $bin $@" 22 | LD_PRELOAD="$pre_libs" $bin $@ 23 | -------------------------------------------------------------------------------- /bin/test_blob_store.ustore: -------------------------------------------------------------------------------- 1 | # Init: comment out the following lines for the first run 2 | drop-ds -t ds1 -b master 3 | drop-ds -t ds1 -b dev 4 | drop-ds -t ds2 -b master 5 | 6 | # Basic operations 7 | create-ds -t ds1 -b master 8 | ls-ds 9 | exists-ds -t ds1 -b master 10 | put-de -t ds1 -b master /etc/hosts 11 | exists-de -t ds1 -b master /etc/hosts 12 | put-de -t ds1 -b master -x "Hello, world!" -m f1 13 | exists-de -t ds1 -m f1 14 | get-ds -t ds1 -b master 15 | get-de -t ds1 -b master -m f1 16 | get-de -t ds1 -b master -m f1 /tmp/f1.ustore_test 17 | 18 | # Branch dataset 19 | branch-ds -t ds1 -b dev -c master --time 20 | ls-ds-branch -t ds1 21 | ls-de-branch -t ds1 -m f1 22 | put-de -t ds1 -b dev /etc/networks 23 | del-de -t ds1 -b dev -m f1 24 | get-ds -t ds1 -b dev -1 25 | diff-ds -t ds1 -b master -c dev 26 | 27 | # Diff datasets 28 | exist-ds -t ds2 -b master 29 | create-ds -t ds2 -b master 30 | lsds -1 31 | put-de -t ds2 -b master /etc/hosts 32 | put-de -t ds2 -b master -m f1 -x "foo" 33 | diff-ds -t ds1 -b master -s ds2 -c master 34 | 35 | # Batch operation 36 | put-de-bat -t ds2 -b master /etc/init --time 37 | get-de -t ds2 -b master -m hostname.conf 38 | ### For Debug ### 39 | #put-de-bat -t ds2 -b master /etc/network --time 40 | #get-de -t ds2 -b master -m system/syslog.service 41 | # 42 | #put-de-bat -t ds2 -b master /etc/systemd --time 43 | #get-de -t ds2 -b master -m interfaces 44 | ################# 45 | get-ds -t ds2 -b master 46 | get-de-bat -t ds2 -b master /tmp/ds2.ustore_test --time 47 | export-ds-bin -t ds2 -b master /tmp/ds2.ustore_test.bin 48 | -------------------------------------------------------------------------------- /bin/test_ustore_cli.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # get environment variables 4 | . $(dirname "${BASH_SOURCE-$0}")/ustore_env.sh 5 | 6 | # go to ustore home to execute binary 7 | cd ${USTORE_HOME} 8 | 9 | set -o nounset 10 | set -o pipefail 11 | 12 | #### Begin of Configurations #### 13 | 14 | if [ $# -ge 1 ]; then 15 | libs=$1 16 | echo "Using preload $libs" 17 | shift 18 | CLI="./bin/run_preload.sh $libs bin/ustore_cli $@" 19 | else 20 | libs="" 21 | CLI="./bin/ustore_cli $@" 22 | fi 23 | 24 | USTORE_CLEAN="./bin/ustore_clean.sh $@" 25 | USTORE_START="./bin/ustore_start.sh $libs $@" 26 | 27 | UNIT_TEST_SCRIPT="./bin/cli_unit_test.ustore" 28 | 29 | #### End of Configurations #### 30 | 31 | RED="\033[1m\033[31m" 32 | GREEN="\033[1m\033[32m" 33 | YELLOW="\033[1m\033[33m" 34 | NE="\033[0m" 35 | 36 | ${USTORE_CLEAN} && ${USTORE_START} 37 | 38 | if [[ $? -ne 0 ]]; then 39 | echo "Failed to start/restart the UStore service" 40 | exit -1 41 | fi 42 | 43 | ${CLI} --script ${UNIT_TEST_SCRIPT} 44 | RST=$? 45 | 46 | if [[ ${RST} -eq 0 ]]; then 47 | echo -e "${GREEN}[SUCCESS]${NE} All tests have passed." 48 | else 49 | echo -e "${RED}[FAILED]${NE} Failed test is found!" 50 | fi 51 | 52 | exit ${RST} 53 | -------------------------------------------------------------------------------- /bin/ustore_clean.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # stop worker services and clean data 4 | # 5 | 6 | # get environment variables 7 | . `dirname "${BASH_SOURCE-$0}"`/ustore_env.sh 8 | cd $USTORE_HOME 9 | 10 | # stop service 11 | ./bin/ustore_stop.sh 12 | 13 | # remove data 14 | ssh_options="-oStrictHostKeyChecking=no \ 15 | -oUserKnownHostsFile=/dev/null \ 16 | -oLogLevel=quiet" 17 | 18 | # clean ustore data 19 | hosts=`cat $USTORE_CONF_HOST_FILE | cut -d ':' -f 1` 20 | for i in ${hosts[@]}; do 21 | echo Clean ustore @ $i ... 22 | if [ $i == localhost ]; then 23 | rm -rf $USTORE_CONF_DATA_DIR/* 24 | else 25 | ssh $ssh_options $i rm $USTORE_CONF_DATA_DIR/* 26 | fi 27 | done 28 | echo "----------- All data removed ------------" 29 | -------------------------------------------------------------------------------- /bin/ustore_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # set ustore environment variables, includes: 4 | # * USTORE_HOME 5 | # * USTORE_BIN 6 | # * USTORE_CONF 7 | # * USTORE_LOG 8 | # 9 | 10 | # exit if varaiables already set 11 | [ -z $USTORE_ENV_DONE ] || exit 0 12 | USTORE_ENV_DONE=1 13 | 14 | # set USTORE_HOME 15 | if [ -z $USTORE_HOME ]; then 16 | USTORE_HOME=`dirname "${BASH_SOURCE-$0}"` 17 | USTORE_HOME=`cd "$USTORE_HOME/..">/dev/null; pwd` 18 | fi 19 | 20 | # set USTORE_BIN 21 | if [ -z $USTORE_BIN ]; then 22 | USTORE_BIN="$USTORE_HOME/bin" 23 | fi 24 | 25 | # set USTORE_CONF 26 | if [ -z $USTORE_CONF ]; then 27 | USTORE_CONF="$USTORE_HOME/conf" 28 | fi 29 | 30 | # set USTORE_CONF_FILE 31 | if [ -z $USTORE_CONF_FILE ]; then 32 | USTORE_CONF_FILE="$USTORE_CONF/config.cfg" 33 | fi 34 | 35 | # set USTORE_CONF_HOST_FILE 36 | if [ -z $USTORE_CONF_HOST_FILE ]; then 37 | USTORE_CONF_HOST_FILE=`grep "worker_file:" $USTORE_CONF_FILE | cut -d '"' -f 2` 38 | fi 39 | 40 | # set USTORE_CONF_DATA_DIR 41 | if [ -z $USTORE_CONF_DATA_DIR ]; then 42 | USTORE_CONF_DATA_DIR=`grep "data_dir:" $USTORE_CONF_FILE | cut -d '"' -f 2` 43 | fi 44 | 45 | # set USTORE_LOG 46 | if [ -z $USTORE_LOG ]; then 47 | USTORE_LOG="$USTORE_HOME/log" 48 | fi 49 | 50 | # check command existence 51 | [ "$(command -v killall)" ] || echo "WARNING: killall command not found" 52 | 53 | # function for killing process 54 | wait_and_fkill() { 55 | fkill=$1 56 | pid=$2 57 | stime=$3 58 | # echo "wait for $pid for $stime and then $fkill" 59 | 60 | counter=0 61 | while [ true ]; 62 | do 63 | kill -0 $pid > /dev/null 2>&1 64 | status=$? 65 | if [ $status = 0 ]; then 66 | sleep 1 67 | counter=$((counter+1)) 68 | if [ $counter = $stime ]; then 69 | # echo "force kill the process" 70 | $fkill > /dev/null 2>&1 71 | fi 72 | else 73 | return 74 | fi 75 | done 76 | } 77 | -------------------------------------------------------------------------------- /bin/ustore_start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # start worker and client services 4 | # 5 | 6 | usage="Usage: ustore-start.sh [ arguments ]\n 7 | --config : if the configuration file is not in the default location (conf/config)\n" 8 | 9 | # get environment variables 10 | . `dirname "${BASH_SOURCE-$0}"`/ustore_env.sh 11 | cd $USTORE_HOME 12 | 13 | # create log directory 14 | if [ ! -d $USTORE_LOG ]; then 15 | mkdir $USTORE_LOG 16 | fi 17 | 18 | if [ $# -ge 1 ]; then 19 | libs=$1 20 | echo "Using preload $libs" 21 | shift 22 | ustore_run="bin/run_preload.sh $libs bin/ustored $@" 23 | else 24 | ustore_run="bin/ustored $@" 25 | fi 26 | 27 | # ssh and start ustore processes 28 | ssh_options="-oStrictHostKeyChecking=no \ 29 | -oUserKnownHostsFile=/dev/null \ 30 | -oLogLevel=quiet" 31 | ustore_sshrun="cd $USTORE_HOME; $ustore_run" 32 | 33 | # start all the workers 34 | echo "------------- Starting workers -------------" 35 | for i in `cat $USTORE_CONF_HOST_FILE` ; do 36 | host=`echo $i | cut -d ':' -f 1` 37 | port=`echo $i | cut -d ':' -f 2` 38 | if [ $host = localhost ] ; then 39 | echo Starting worker @ $host : $ustore_run --node_id $i 40 | $ustore_run --node_id $i >> $USTORE_LOG/worker-$host-$port.log 2>&1 & 41 | sleep 1 42 | else 43 | echo Starting worker @ $host : $ustore_sshrun --worker --node_id $i 44 | ssh $ssh_options $host $ustore_sshrun --node_id $i >> $USTORE_LOG/worker-$host-$port.log 2>&1 & 45 | sleep 1 46 | fi 47 | done 48 | 49 | echo "----------- All processes started ------------" 50 | -------------------------------------------------------------------------------- /bin/ustore_stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # stop worker services 4 | # 5 | 6 | # get environment variables 7 | . `dirname "${BASH_SOURCE-$0}"`/ustore_env.sh 8 | cd $USTORE_HOME 9 | 10 | ssh_options="-oStrictHostKeyChecking=no \ 11 | -oUserKnownHostsFile=/dev/null \ 12 | -oLogLevel=quiet" 13 | 14 | # kill ustore worker processes 15 | hosts=`cat $USTORE_CONF_HOST_FILE | cut -d ':' -f 1` 16 | for i in ${hosts[@]}; do 17 | echo Stop ustore @ $i ... 18 | if [ $i == localhost ]; then 19 | ./bin/ustore_stop_worker.sh 20 | else 21 | ssh $ssh_options $i $USTORE_HOME/bin/ustore_stop_worker.sh 22 | fi 23 | done 24 | 25 | echo "----------- All processes stopped ------------" 26 | -------------------------------------------------------------------------------- /bin/ustore_stop_worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # stop worker services 4 | # 5 | 6 | # get environment variables 7 | . `dirname "${BASH_SOURCE-$0}"`/ustore_env.sh 8 | cd $USTORE_HOME 9 | 10 | ustore_kill="killall -q -w -s SIGTERM -r ustored" 11 | ustore_fkill="killall -q -s SIGKILL -r ustored" 12 | 13 | $ustore_kill > /dev/null 2>&1 & 14 | kpid=$! 15 | wait_and_fkill "$ustore_fkill" $kpid 20 16 | -------------------------------------------------------------------------------- /cmake/Dependencies.cmake: -------------------------------------------------------------------------------- 1 | SET(USTORE_INCLUDE_DIRS "") 2 | SET(USTORE_LINKER_LIBS "") 3 | SET(USTORE_DEFINITIONS "") 4 | 5 | FIND_PACKAGE(Protobuf REQUIRED) 6 | if (NOT DYNAMIC_LIB) 7 | SET(Boost_USE_STATIC_LIBS ON) 8 | endif() 9 | FIND_PACKAGE(Boost REQUIRED COMPONENTS ${BOOST_COMPONENT}) 10 | IF(Boost_FOUND) 11 | LIST(APPEND USTORE_INCLUDE_DIRS ${Boost_INCLUDE_DIRS}) 12 | LIST(APPEND USTORE_LINKER_LIBS ${Boost_LIBRARIES}) 13 | ENDIF() 14 | 15 | # Rocksdb (optional) 16 | IF (USE_ROCKSDB) 17 | MESSAGE(STATUS "Use RocksDB") 18 | FIND_PACKAGE(Rocksdb REQUIRED) 19 | IF(ROCKSDB_FOUND) 20 | LIST(APPEND USTORE_INCLUDE_DIRS ${ROCKSDB_INCLUDES}) 21 | LIST(APPEND USTORE_LINKER_LIBS ${ROCKSDB_LIBRARIES}) 22 | ADD_DEFINITIONS(-DUSE_ROCKSDB) 23 | ENDIF() 24 | ENDIF() 25 | 26 | # --- [ LevelDB (optional) 27 | IF (USE_LEVELDB) 28 | MESSAGE(STATUS "Use LevelDB as chunk storage") 29 | FIND_PACKAGE(LevelDB REQUIRED) 30 | IF(LEVELDB_FOUND) 31 | LIST(APPEND USTORE_INCLUDE_DIRS ${LevelDB_INCLUDES}) 32 | LIST(APPEND USTORE_LINKER_LIBS ${LevelDB_LIBRARIES}) 33 | ADD_DEFINITIONS(-DUSE_LEVELDB) 34 | ENDIF() 35 | 36 | FIND_PACKAGE(Snappy REQUIRED) 37 | IF(Snappy_FOUND) 38 | LIST(APPEND USTORE_INCLUDE_DIRS ${Snappy_INCLUDE_DIR}) 39 | LIST(APPEND USTORE_LINKER_LIBS ${Snappy_LIBRARIES}) 40 | ENDIF() 41 | ENDIF() 42 | 43 | # --- [ Crypto++ (optional) 44 | IF (USE_CRYPTOPP) 45 | FIND_PACKAGE(CryptoPP REQUIRED) 46 | IF(CRYPTOPP_FOUND) 47 | LIST(APPEND USTORE_INCLUDE_DIRS ${CRYPTOPP_INCLUDE_DIR}) 48 | LIST(APPEND USTORE_LINKER_LIBS ${CRYPTOPP_LIBRARIES}) 49 | ENDIF() 50 | ENDIF() 51 | 52 | FIND_PACKAGE(GFlags REQUIRED) 53 | IF(GFLAGS_FOUND) 54 | LIST(APPEND USTORE_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 55 | LIST(APPEND USTORE_LINKER_LIBS ${GFLAGS_LIBRARIES}) 56 | ENDIF() 57 | 58 | FIND_PACKAGE(Czmq REQUIRED) 59 | IF(CZMQ_FOUND) 60 | LIST(APPEND USTORE_INCLUDE_DIRS ${Czmq_INCLUDE_DIR}) 61 | LIST(APPEND USTORE_LINKER_LIBS ${Czmq_LIBRARIES}) 62 | ENDIF() 63 | 64 | INCLUDE_DIRECTORIES(SYSTEM ${USTORE_INCLUDE_DIRS}) 65 | # MESSAGE(STATUS "INCLUDE ${USTORE_INCLUDE_DIRS}") 66 | -------------------------------------------------------------------------------- /cmake/Modules/FindCzmq.cmake: -------------------------------------------------------------------------------- 1 | 2 | FIND_PATH(Czmq_INCLUDE_DIR NAMES czmq.h PATHS "$ENV{Czmq_ROOT_DIR}/include") 3 | FIND_LIBRARY(Czmq_LIBRARIES NAMES czmq PATHS "$ENV{Czmq_ROOT_DIR}/lib") 4 | 5 | INCLUDE(FindPackageHandleStandardArgs) 6 | find_package_handle_standard_args(Czmq DEFAULT_MSG Czmq_INCLUDE_DIR Czmq_LIBRARIES) 7 | IF(CZMQ_FOUND) 8 | MESSAGE(STATUS "Found czmq at ${Czmq_INCLUDE_DIR}") 9 | MARK_AS_ADVANCED(Czmq_INCLUDE_DIR Czmq_LIBRARIES) 10 | 11 | if(EXISTS "${Czmq_INCLUDE_DIR}/czmq_library.h") 12 | file(STRINGS "${Czmq_INCLUDE_DIR}/czmq_library.h" __version_lines 13 | REGEX "#define[ \t]+CZMQ_VERSION_[^V]+[ \t]+[0-9]+") 14 | 15 | foreach(__line ${__version_lines}) 16 | if(__line MATCHES "#define[ \t]+CZMQ_VERSION_MAJOR[ \t]+([0-9]+)") 17 | set(Czmq_VERSION_MAJOR ${CMAKE_MATCH_1}) 18 | elseif(__line MATCHES "#define[ \t]+CZMQ_VERSION_MINOR[ \t]+([0-9]+)") 19 | set(Czmq_VERSION_MINOR ${CMAKE_MATCH_1}) 20 | elseif(__line MATCHES "#define[ \t]+CZMQ_VERSION_PATCH[ \t]+([0-9]+)") 21 | set(Czmq_VERSION_PATCH ${CMAKE_MATCH_1}) 22 | endif() 23 | endforeach() 24 | 25 | set(Czmq_VERSION 26 | "${Czmq_VERSION_MAJOR}.${Czmq_VERSION_MINOR}.${Czmq_VERSION_PATCH}") 27 | if(NOT Czmq_VERSION STREQUAL "") 28 | message(STATUS "Czmq Version: ${Czmq_VERSION}") 29 | endif() 30 | 31 | ustore_clear_vars(__line __version_lines) 32 | endif() 33 | ELSE() 34 | MESSAGE(FATAL_ERROR "Czmq NOT FOUND") 35 | ENDIF() 36 | -------------------------------------------------------------------------------- /cmake/Modules/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | 2 | FIND_PATH(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h PATHS "$ENV{GFLAGS_DIR}/include") 3 | if (DYNAMIC_LIB) 4 | FIND_LIBRARY(GFLAGS_LIBRARIES NAMES gflags PATHS "$ENV{GFLAGS_DIR}/lib") 5 | else() 6 | FIND_LIBRARY(GFLAGS_LIBRARIES NAMES libgflags.a gflags PATHS "$ENV{GFLAGS_DIR}/lib") 7 | endif() 8 | 9 | INCLUDE(FindPackageHandleStandardArgs) 10 | find_package_handle_standard_args(GFLAGS DEFAULT_MSG GFLAGS_INCLUDE_DIR GFLAGS_LIBRARIES) 11 | 12 | IF(GFLAGS_FOUND) 13 | MESSAGE(STATUS "Found gflags at ${GFLAGS_INCLUDE_DIR}") 14 | MARK_AS_ADVANCED(GFLAGS_INCLUDE_DIR GFLAGS_LIBRARIES) 15 | ELSE() 16 | MESSAGE(FATAL_ERROR "GFlags NOT FOUND") 17 | ENDIF() 18 | -------------------------------------------------------------------------------- /cmake/Modules/FindGlog.cmake: -------------------------------------------------------------------------------- 1 | 2 | FIND_PATH(GLOG_INCLUDE_DIR NAMES glog/logging.h PATHS "$ENV{GLOG_DIR}/include") 3 | if (DYNAMIC_LIB) 4 | FIND_LIBRARY(GLOG_LIBRARIES NAMES glog) 5 | else() 6 | FIND_LIBRARY(GLOG_LIBRARIES NAMES libglog.a glog) 7 | endif() 8 | 9 | INCLUDE(FindPackageHandleStandardArgs) 10 | find_package_handle_standard_args(GLOG DEFAULT_MSG GLOG_INCLUDE_DIR GLOG_LIBRARIES) 11 | 12 | IF(GLOG_FOUND) 13 | # MESSAGE(STATUS "Found glog at ${GLOG_INCLUDE_DIR}") 14 | MARK_AS_ADVANCED(GLOG_INCLUDE_DIR GLOG_LIBRARIES) 15 | ELSE() 16 | MESSAGE(FATAL_ERROR "Glog NOT FOUND") 17 | ENDIF(GLOG_FOUND) 18 | -------------------------------------------------------------------------------- /cmake/Modules/FindLevelDB.cmake: -------------------------------------------------------------------------------- 1 | # - Find LevelDB, adapted from BVLC/caffe 2 | # 3 | # LevelDB_INCLUDES - List of LevelDB includes 4 | # LevelDB_LIBRARIES - List of libraries when using LevelDB. 5 | # LevelDB_FOUND - True if LevelDB found. 6 | 7 | # Look for the header file. 8 | find_path(LevelDB_INCLUDE NAMES leveldb/db.h 9 | PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include 10 | DOC "Path in which the file leveldb/db.h is located." ) 11 | 12 | # Look for the library. 13 | find_library(LevelDB_LIBRARY NAMES leveldb 14 | PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib 15 | DOC "Path to leveldb library." ) 16 | 17 | include(FindPackageHandleStandardArgs) 18 | find_package_handle_standard_args(LevelDB DEFAULT_MSG LevelDB_INCLUDE LevelDB_LIBRARY) 19 | 20 | if(LEVELDB_FOUND) 21 | message(STATUS "Found LevelDB (include: ${LevelDB_INCLUDE}, library: ${LevelDB_LIBRARY})") 22 | set(LevelDB_INCLUDES ${LevelDB_INCLUDE}) 23 | set(LevelDB_LIBRARIES ${LevelDB_LIBRARY}) 24 | mark_as_advanced(LevelDB_INCLUDE LevelDB_LIBRARY) 25 | 26 | if(EXISTS "${LevelDB_INCLUDE}/leveldb/db.h") 27 | file(STRINGS "${LevelDB_INCLUDE}/leveldb/db.h" __version_lines 28 | REGEX "static const int k[^V]+Version[ \t]+=[ \t]+[0-9]+;") 29 | 30 | foreach(__line ${__version_lines}) 31 | if(__line MATCHES "[^k]+kMajorVersion[ \t]+=[ \t]+([0-9]+);") 32 | set(LEVELDB_VERSION_MAJOR ${CMAKE_MATCH_1}) 33 | elseif(__line MATCHES "[^k]+kMinorVersion[ \t]+=[ \t]+([0-9]+);") 34 | set(LEVELDB_VERSION_MINOR ${CMAKE_MATCH_1}) 35 | endif() 36 | endforeach() 37 | 38 | set(LEVELDB_VERSION "${LEVELDB_VERSION_MAJOR}.${LEVELDB_VERSION_MINOR}") 39 | if(NOT LEVELDB_VERSION STREQUAL "") 40 | message(STATUS "LevelDB Version: ${LEVELDB_VERSION}") 41 | endif() 42 | 43 | ustore_clear_vars(__line __version_lines) 44 | endif() 45 | endif() 46 | -------------------------------------------------------------------------------- /cmake/Modules/FindRocksdb.cmake: -------------------------------------------------------------------------------- 1 | 2 | FIND_PATH(ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h PATHS "$ENV{ROCKSDB_DIR}/include") 3 | if (DYNAMIC_LIB) 4 | FIND_LIBRARY(ROCKSDB_LIBRARIES NAMES rocksdb PATHS "$ENV{ROCKSDB_DIR}/lib") 5 | else() 6 | FIND_LIBRARY(ROCKSDB_LIBRARIES NAMES librocksdb.a rocksdb PATHS "$ENV{ROCKSDB_DIR}/lib") 7 | endif() 8 | 9 | INCLUDE(FindPackageHandleStandardArgs) 10 | find_package_handle_standard_args(ROCKSDB DEFAULT_MSG ROCKSDB_INCLUDE_DIR ROCKSDB_LIBRARIES) 11 | 12 | IF(ROCKSDB_FOUND) 13 | MESSAGE(STATUS "Found Rocksdb at ${ROCKSDB_INCLUDE_DIR}") 14 | MARK_AS_ADVANCED(ROCKSDB_INCLUDE_DIR ROCKSDB_LIBRARIES) 15 | ELSE() 16 | MESSAGE(FATAL_ERROR "ROCKSDB NOT FOUND") 17 | ENDIF() 18 | -------------------------------------------------------------------------------- /cmake/Modules/FindSnappy.cmake: -------------------------------------------------------------------------------- 1 | # Find the Snappy libraries, adapted from BVLC/caffe 2 | # 3 | # The following variables are optionally searched for defaults 4 | # Snappy_ROOT_DIR: Base directory where all Snappy components are found 5 | # 6 | # The following are set after configuration is done: 7 | # SNAPPY_FOUND 8 | # Snappy_INCLUDE_DIR 9 | # Snappy_LIBRARIES 10 | 11 | find_path(Snappy_INCLUDE_DIR NAMES snappy.h 12 | PATHS $ENV{SNAPPY_ROOT_DIR}/include) 13 | 14 | find_library(Snappy_LIBRARIES NAMES snappy 15 | PATHS $ENV{SNAPPY_ROOT_DIR}/lib) 16 | 17 | include(FindPackageHandleStandardArgs) 18 | find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_INCLUDE_DIR Snappy_LIBRARIES) 19 | 20 | if(SNAPPY_FOUND) 21 | message(STATUS "Found Snappy (include: ${Snappy_INCLUDE_DIR}, library: ${Snappy_LIBRARIES})") 22 | mark_as_advanced(Snappy_INCLUDE_DIR Snappy_LIBRARIES) 23 | 24 | ustore_parse_header(${Snappy_INCLUDE_DIR}/snappy-stubs-public.h 25 | SNAPPY_VERION_LINES SNAPPY_MAJOR SNAPPY_MINOR SNAPPY_PATCHLEVEL) 26 | set(Snappy_VERSION "${SNAPPY_MAJOR}.${SNAPPY_MINOR}.${SNAPPY_PATCHLEVEL}") 27 | if(Snappy_VERSION) 28 | message(STATUS "Snappy version: ${Snappy_VERSION}") 29 | endif() 30 | endif() 31 | -------------------------------------------------------------------------------- /cmake/Utils.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################################ 2 | # Reads set of version defines from the header file 3 | # Usage: 4 | # ustore_parse_header( ..) 5 | macro(ustore_parse_header FILENAME FILE_VAR) 6 | set(vars_regex "") 7 | set(__parnet_scope OFF) 8 | set(__add_cache OFF) 9 | foreach(name ${ARGN}) 10 | if("${name}" STREQUAL "PARENT_SCOPE") 11 | set(__parnet_scope ON) 12 | elseif("${name}" STREQUAL "CACHE") 13 | set(__add_cache ON) 14 | elseif(vars_regex) 15 | set(vars_regex "${vars_regex}|${name}") 16 | else() 17 | set(vars_regex "${name}") 18 | endif() 19 | endforeach() 20 | if(EXISTS "${FILENAME}") 21 | file(STRINGS "${FILENAME}" ${FILE_VAR} REGEX "#define[ \t]+(${vars_regex})[ \t]+[0-9]+" ) 22 | else() 23 | unset(${FILE_VAR}) 24 | endif() 25 | foreach(name ${ARGN}) 26 | if(NOT "${name}" STREQUAL "PARENT_SCOPE" AND NOT "${name}" STREQUAL "CACHE") 27 | if(${FILE_VAR}) 28 | if(${FILE_VAR} MATCHES ".+[ \t]${name}[ \t]+([0-9]+).*") 29 | string(REGEX REPLACE ".+[ \t]${name}[ \t]+([0-9]+).*" "\\1" ${name} "${${FILE_VAR}}") 30 | else() 31 | set(${name} "") 32 | endif() 33 | if(__add_cache) 34 | set(${name} ${${name}} CACHE INTERNAL "${name} parsed from ${FILENAME}" FORCE) 35 | elseif(__parnet_scope) 36 | set(${name} "${${name}}" PARENT_SCOPE) 37 | endif() 38 | else() 39 | unset(${name} CACHE) 40 | endif() 41 | endif() 42 | endforeach() 43 | endmacro() 44 | 45 | ################################################################################################ 46 | # Clears variables from list 47 | # Usage: 48 | # ustore_clear_vars() 49 | macro(ustore_clear_vars) 50 | foreach(_var ${ARGN}) 51 | unset(${_var}) 52 | endforeach() 53 | endmacro() 54 | 55 | -------------------------------------------------------------------------------- /conf/config.cfg: -------------------------------------------------------------------------------- 1 | data_dir: "ustore_data" 2 | data_file_pattern: "ustore" 3 | access_log_dir: "" 4 | max_segments: 10000 5 | enable_dist_store: false 6 | get_chunk_bypass_worker: true 7 | 8 | worker_file: "conf/workers.lst" 9 | 10 | recv_threads: 2 11 | 12 | http_port: 60600 13 | -------------------------------------------------------------------------------- /conf/workers.lst: -------------------------------------------------------------------------------- 1 | localhost:50500 2 | localhost:50502 3 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # UStore Docker Images 2 | 3 | ## Build Images 4 | 5 | ``` 6 | ./build.sh 7 | ``` 8 | 9 | ## Run Containers 10 | 11 | ``` 12 | docker run -it ustore: /bin/bash 13 | docker exec -it /bin/bash 14 | ``` 15 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # build ustore docker images 4 | # 5 | 6 | # change to root dir 7 | cd `dirname "${BASH_SOURCE-$0}"`/.. 8 | 9 | echo "#########################" 10 | echo "Build ustore:ubuntu-16.04" 11 | echo "#########################" 12 | docker build -f docker/ubuntu/16.04/Dockerfile --force-rm -t ustore:ubuntu-16.04 . 13 | -------------------------------------------------------------------------------- /docker/ubuntu/16.04/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | # install dependencies 4 | RUN apt-get update && apt-get install -y --no-install-recommends psmisc 5 | 6 | # import ustore executable 7 | COPY . /ustore 8 | 9 | # set environment variables 10 | ENV USTORE_HOME=/ustore/build 11 | ENV PATH=$USTORE_HOME/bin:$PATH 12 | ENV LD_LIBRARY_PATH=$USTORE_HOME/lib:$LD_LIBRARY_PATH 13 | -------------------------------------------------------------------------------- /docs/arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nusdbsystem/forkbase/bbef77055ea7afa2abe0c48503221bb701cd11fb/docs/arch.jpg -------------------------------------------------------------------------------- /docs/arch.md: -------------------------------------------------------------------------------- 1 | # System Architecture 2 | 3 | ![ForkBase Architecture](arch.jpg) 4 | 5 | ## Remote Client Service 6 | ``Remote Client Service`` receives all requests from application, 7 | and forwards the requests to the corresponding workers to process. 8 | The requests are dispatched based on the hash value of the request key. 9 | 10 | ## Worker 11 | ``Workers`` process requests from disjoint subsets of keys. 12 | 13 | ## Worker Service 14 | ``Worker Service`` receives requests from the client service and translates the request messages 15 | to the corresponding data structures. 16 | 17 | ## Access Control 18 | ``Access Control`` checks the permission of each request before it is further processed. 19 | Unauthorized requests will be rejected directly. 20 | 21 | ## Head Version Table 22 | ``Head Version Table`` maintains the up-to-date head information for all named and unnamed branches. 23 | Each update of a branch head is captured in the table and persisted on disk. 24 | 25 | ## Data Type Manager 26 | ``Data Type Manager`` hides chunk storage and internal data representation from worker logics. 27 | When loading an object, it loads the corresponding data chunks and de-serializes them. 28 | After an update operation is done, the object will be chunked and only new chunks are written back to chunk storage. 29 | 30 | ## Chunk Storage 31 | ``Chunk Storage`` is distributed across the cluster. It handles read/write chunk requests from the data type manager. 32 | -------------------------------------------------------------------------------- /docs/depend.md: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | 3 | ## Compulsory 4 | 5 | ### Core 6 | 7 | * cmake >= 2.8 8 | * g++ >= 4.9 9 | * protobuf >= 2.6.1 10 | * boost >= 1.66.0 11 | * czmq >= 4.0.2 12 | * libzmq >= 4.2.1 13 | * gflags >= 2.1.0 14 | 15 | ### Performance Monitor 16 | 17 | * libpcap (1.8.1) 18 | * dbus (1.0) 19 | * cpp-netlib == 0.11.0 20 | * python >= 2.7 21 | 22 | ## Optional 23 | 24 | ### For RDMA 25 | 26 | * libibverbs >= 1.0 27 | * libboost_thread >= 1.5.4 (Thread) 28 | 29 | ### For Cryptographic Hashing 30 | 31 | * libcrypto++ 32 | ``` 33 | Download from https://github.com/weidai11/cryptopp/releases 34 | ``` 35 | 36 | ### For Storage 37 | 38 | * LevelDB 39 | ``` 40 | Install from https://github.com/google/leveldb 41 | Export `$LEVELDB_ROOT` variable to point to the installed directory. 42 | Include `$LEVELDB_ROOT` to `$CPLUS_INCLUDE_PATH` so that cmake can find the header files. 43 | ``` 44 | 45 | * Snappy 46 | ``` 47 | Install from https://github.com/google/snappy 48 | Export `$SNAPPY_ROOT_DIR` variable to point to the installed directory. 49 | ``` 50 | 51 | * RocksDB >= 5.8 ([download](https://github.com/facebook/rocksdb/releases)) 52 | 53 | Install RocksDB 54 | ``` 55 | $ make shared_lib -j "${NCORES}" USE_RTTI=1 DISABLE_WARNING_AS_ERROR=ON && make install-shared INSTALL_PATH="${LIB_HOME}/rocksdb" 56 | ``` 57 | Set RocksDB environment variables 58 | ``` 59 | # RocksDB 60 | export ROCKSDB_ROOT="${LIB_HOME}/rocksdb" 61 | export CPATH="${ROCKSDB_ROOT}/include:${CPATH}" 62 | export LD_LIBRARY_PATH="${ROCKSDB_ROOT}/lib:${LD_LIBRARY_PATH}" 63 | export LIBRARY_PATH="${ROCKSDB_ROOT}/lib:${LIBRARY_PATH}" 64 | export CMAKE_INCLUDE_PATH="${ROCKSDB_ROOT}/include:${CMAKE_INCLUDE_PATH}" 65 | export CMAKE_LIBRARY_PATH="${ROCKSDB_ROOT}/lib:${CMAKE_LIBRARY_PATH}" 66 | ``` 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /docs/guide.md: -------------------------------------------------------------------------------- 1 | # ForkBase Usage Guide 2 | 3 | ## Preparation 4 | 5 | Install all required [dependencies](depend.md). 6 | 7 | ## Compilation 8 | 9 | ```console 10 | # Go to ForkBase root dir 11 | $ mkdir build && cd build 12 | $ cmake .. 13 | $ make 14 | ``` 15 | ForkBase has many compilation options, which can be found in ``CMakeList.txt``. 16 | ```console 17 | # compile with RDMA and examples 18 | $ cmake -DUSE_RDMA=ON -DENABLE_EXAMPLE=ON .. 19 | $ make 20 | ``` 21 | 22 | ## Unit Test 23 | 24 | ```console 25 | # go to build dir 26 | $ cd build 27 | $ ./bin/test_ustore 28 | ``` 29 | Unit test data are stored in ``ustore_data/ustore_2017``. 30 | 31 | ## Micro-Benchmark 32 | 33 | ```console 34 | $ ./bin/micro_bench 35 | ``` 36 | Unit benchmark data are stored in ``ustore_data/ustore_2018``. 37 | 38 | ## Setup ForkBase Service 39 | 40 | All configurations can be set in ``conf/config``. 41 | Import settings are: 42 | * ``num_segments``: maximum data segments a worker can store 43 | * ``worker_file``: a list of worker nodes 44 | * ``http_port``: port for default RESTful service 45 | 46 | Start ForkBase service, which will launch all worker processes and a default 47 | HTTP server. 48 | ```console 49 | $ ./bin/ustore_start.sh 50 | ``` 51 | 52 | Stop ForkBase service. 53 | ```console 54 | $ ./bin/ustore_stop.sh 55 | ``` 56 | 57 | Or stop ForkBase service and remove related data. 58 | ```console 59 | $ ./bin/ustore_clean.sh 60 | ``` 61 | 62 | ## Distributed Benchmark 63 | 64 | Ensure ForkBase service is on before running the benchmark. 65 | Better remove benchmarked data in the end. 66 | ```console 67 | $ ./bin/ustore_start.sh 68 | $ ./bin/dist_bench 69 | $ ./bin/ustore_clean.sh 70 | ``` 71 | 72 | ## Commandline Client 73 | 74 | Ensure ForkBase service is on. 75 | ```console 76 | $ ./bin/ustore_cli 77 | # e.g., put a value 78 | $ ./bin/ustore_cli put -k key -b branch -v value 79 | ``` 80 | All APIs can be found in [API doc](api_cmd.md). 81 | 82 | ## HTTP Request 83 | 84 | Ensure ForkBase service is on. 85 | ```console 86 | # e.g., put a value 87 | $ curl http://localhost:60600/put -X POST -d "key=key&branch=branch&value=value" 88 | ``` 89 | All APIs can be found in [API doc](api_http.md). 90 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) 2 | INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/include) 3 | INCLUDE_DIRECTORIES(${USTORE_INCLUDE_DIR}) 4 | 5 | AUX_SOURCE_DIRECTORY(ca example_ca_source) 6 | # MESSAGE(STATUS "include path ${USTORE_INCLUDE_DIR}") 7 | 8 | ADD_EXECUTABLE(ustore_example_ca "ca/ca_main.cc" ${example_ca_source}) 9 | ADD_DEPENDENCIES(ustore_example_ca ustore) 10 | TARGET_LINK_LIBRARIES(ustore_example_ca ustore) 11 | 12 | AUX_SOURCE_DIRECTORY(table_op example_table_op_source) 13 | ADD_EXECUTABLE(ustore_example_table_op "table_op/table_op_main.cc" ${example_table_op_source}) 14 | ADD_DEPENDENCIES(ustore_example_table_op ustore) 15 | TARGET_LINK_LIBRARIES(ustore_example_table_op ustore) 16 | 17 | AUX_SOURCE_DIRECTORY(lucene_cli lucene_cli_source) 18 | ADD_EXECUTABLE(ustore_lucene_cli "lucene_cli/lucene_cli_main.cc" ${lucene_cli_source}) 19 | ADD_DEPENDENCIES(ustore_lucene_cli ustore) 20 | TARGET_LINK_LIBRARIES(ustore_lucene_cli ustore) 21 | -------------------------------------------------------------------------------- /example/ca/ca_arguments.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_EXAMPLE_CA_CA_ARGUMENTS_H_ 4 | #define USTORE_EXAMPLE_CA_CA_ARGUMENTS_H_ 5 | 6 | #include 7 | #include "utils/arguments.h" 8 | 9 | namespace ustore { 10 | namespace example { 11 | namespace ca { 12 | 13 | class CAArguments : public ::ustore::Arguments { 14 | public: 15 | int task_id; 16 | int64_t n_columns; 17 | int64_t n_records; 18 | double p; 19 | int64_t iters; 20 | 21 | CAArguments() { 22 | Add(&task_id, "task", "t", "ID of analytics task", 0); 23 | Add(&n_columns, "columns", "c", "number of columns in a simple table", 3); 24 | Add(&n_records, "records", "n", "number of records in a simple table", 10); 25 | Add(&p, "probability", "p", 26 | "probability used in the analytical simulation", 0.01); 27 | Add(&iters, "iterations", "i", 28 | "number of iterations in the analytical simulation", 1000); 29 | } 30 | 31 | bool CheckArgs() override { 32 | GUARD(CheckGE(task_id, 0, "Task ID")); 33 | GUARD(CheckGE(n_columns, 3, "Number of columns")); 34 | GUARD(CheckGT(n_records, 0, "Number of records")); 35 | GUARD(CheckInRange(p, 0, 1, "Probability")); 36 | GUARD(CheckGT(iters, 0, "Number of iterations")); 37 | return true; 38 | } 39 | 40 | ~CAArguments() = default; 41 | }; 42 | 43 | } // namespace ca 44 | } // namespace example 45 | } // namespace ustore 46 | 47 | #endif // USTORE_EXAMPLE_CA_CA_ARGUMENTS_H_ 48 | -------------------------------------------------------------------------------- /example/ca/utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "ca/utils.h" 4 | 5 | namespace ustore { 6 | namespace example { 7 | namespace ca { 8 | 9 | const int Utils::kKeyPrintWidth = 16; 10 | const int Utils::kBrnachPrintWidth = 8; 11 | 12 | void Utils::PrintListDiff(DuallyDiffIndexIterator& it) { 13 | auto f_print_entry = [&it]() { 14 | std::cout << it.index() << ":("; 15 | auto lhs = it.lhs_value(); 16 | std::cout << (lhs.empty() ? "_" : lhs.ToString()) << ','; 17 | auto rhs = it.rhs_value(); 18 | std::cout << (rhs.empty() ? "_" : rhs.ToString()) << ')'; 19 | }; 20 | std::cout << "["; 21 | if (!it.end()) { 22 | f_print_entry(); 23 | for (it.next(); !it.end(); it.next()) { 24 | std::cout << ", "; 25 | f_print_entry(); 26 | } 27 | } 28 | std::cout << "]"; 29 | } 30 | 31 | void Utils::Print(const std::string& table_name, 32 | const std::string& branch_name, 33 | const std::string& list_name, const UList& list) { 34 | const std::string list_key(table_name + "::" + list_name); 35 | std::cout << std::left << std::setw(kKeyPrintWidth) << list_key 36 | << " @" << std::setw(kBrnachPrintWidth) << branch_name << ": "; 37 | ::ustore::Utils::Print(list); 38 | std::cout << std::endl; 39 | } 40 | 41 | } // namespace ca 42 | } // namespace example 43 | } // namespace ustore 44 | -------------------------------------------------------------------------------- /example/ca/utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_EXAMPLE_CA_UTILS_H_ 4 | #define USTORE_EXAMPLE_CA_UTILS_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "types/uiterator.h" 10 | #include "types/ulist.h" 11 | #include "utils/utils.h" 12 | 13 | namespace ustore { 14 | namespace example { 15 | namespace ca { 16 | 17 | class Utils : public ::ustore::Utils { 18 | public: 19 | static void PrintListDiff(DuallyDiffIndexIterator& it); 20 | 21 | static inline void PrintListDiff(DuallyDiffIndexIterator&& it) { 22 | PrintListDiff(it); 23 | } 24 | 25 | static void Print(const std::string& table_name, 26 | const std::string& branch_name, 27 | const std::string& list_name, const UList& list); 28 | 29 | private: 30 | static const int kKeyPrintWidth; 31 | static const int kBrnachPrintWidth; 32 | }; 33 | 34 | } // namespace ca 35 | } // namespace example 36 | } // namespace ustore 37 | 38 | #endif // USTORE_EXAMPLE_CA_UTILS_H_ 39 | -------------------------------------------------------------------------------- /example/lucene_cli/lucene_cli_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "cluster/worker_client_service.h" 4 | #include "worker/worker.h" 5 | 6 | #include "lucene_cli_arguments.h" 7 | #include "lucene_client.h" 8 | 9 | namespace ustore { 10 | namespace example { 11 | namespace lucene_cli { 12 | 13 | int main(int argc, char* argv[]) { 14 | SetStderrLogging(ERROR); 15 | // parse command-line arguments 16 | LuceneCLIArguments args; 17 | if (!args.ParseCmdArgs(argc, argv)) { 18 | std::cerr << BOLD_RED("[ERROR] ") 19 | << "Found invalid command-line option" << std::endl; 20 | return static_cast(ErrorCode::kInvalidCommandArgument); 21 | } 22 | if (args.is_help) return static_cast(ErrorCode::kOK); 23 | // execution 24 | WorkerClientService svc; 25 | svc.Run(); 26 | auto db = svc.CreateWorkerClient(); 27 | return static_cast(LuceneClient(args, &db).Run()); 28 | } 29 | 30 | } // namespace lucene_cli 31 | } // namespace example 32 | } // namespace ustore 33 | 34 | int main(int argc, char* argv[]) { 35 | return ustore::example::lucene_cli::main(argc, argv); 36 | } 37 | -------------------------------------------------------------------------------- /example/lucene_cli/lucene_client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_EXAMPLE_LUCENE_CLI_LUCENE_CLIENT_H_ 4 | #define USTORE_EXAMPLE_LUCENE_CLI_LUCENE_CLIENT_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "lucene_blob_store.h" 10 | #include "lucene_cli_arguments.h" 11 | 12 | namespace ustore { 13 | namespace example { 14 | namespace lucene_cli { 15 | 16 | class LuceneClient { 17 | public: 18 | explicit LuceneClient(LuceneCLIArguments& args, DB* db) noexcept; 19 | ~LuceneClient() = default; 20 | 21 | ErrorCode Run(); 22 | 23 | private: 24 | ErrorCode ExecCommand(const std::string& command); 25 | 26 | ErrorCode ExecPutDataEntryByCSV(); 27 | ErrorCode ExecGetDataEntryByIndexQuery(); 28 | ErrorCode ExecGetDataEntryNameByIndexQuery(); 29 | ErrorCode ExecGetDataEntryByIndexQueryAndJoin(); 30 | ErrorCode ExecGetDatasetSchema(); 31 | 32 | LuceneBlobStore bs_; 33 | LuceneCLIArguments& args_; 34 | std::unordered_map> cmd_exec_; 35 | std::unordered_map*> alias_exec_; 36 | }; 37 | 38 | } // namespace lucene_cli 39 | } // namespace example 40 | } // namespace ustore 41 | 42 | #endif // USTORE_EXAMPLE_LUCENE_CLI_LUCENE_CLIENT_H_ 43 | -------------------------------------------------------------------------------- /example/table_op/README.md: -------------------------------------------------------------------------------- 1 | # Example of Table Operations # 2 | 3 | This example code is in purpose of demonstrating how to operate on table in UStore. To this end, the following operations are executed in order: 4 | 5 | 1. Loading CSV data into table. 6 | 2. Updating data with predicate whose values refer to the external parameter file. This generates multiple versions (branches) of the table. 7 | 3. Performing `diff` operation upon different versions (branches) of the table. 8 | 4. Performing simple aggregation on one column of the table. 9 | 10 | Meanwhile, we also benchmark the above operations and output the results to screen. 11 | 12 | ## Compilation ## 13 | 14 | Run the following commands to compile source code: 15 | 16 | $ cd /path/to/ustore 17 | $ mkdir -p build && cd build && rm -rf * 18 | $ cmake -DENABLE_TEST=OFF .. 19 | $ make 20 | 21 | The generated executable binary is `ustore_example_table_op`, which is placed under the `build/bin` directory. 22 | 23 | ## Execution ## 24 | 25 | Run `ustore_example_table_op` with `--help` (or `-?`) to check for available command-line parameters. 26 | 27 | Suppose all the input data are placed in the `data` folder which resides at the same directory level as the `build` folder. Run the sample command below at the `build` folder: 28 | 29 | $ ./bin/ustore_example_table_op ../data/rpc_r100_a5_e3.csv -r Age_Region -v ../data/rpc_update_ref_val.txt -e Profile -a Num_Departure 30 | 31 | The `rpc_r100_a5_e3.csv` is the input CSV file, and the `rpc_update_ref_val` file contains the update-referring values organized in a one-line-per-value manner. Each update refers to the `Age_Region` column and effects on the `Profile` column. Regarding aggregation, the `Num_Departure` column is to be aggregated. 32 | 33 | Along with the execution, table `Test` is created and to be operated on. Each update on the table generates a new version (branch). The branch name corresponds to the input data is `master`, and the branches generated by the subsequent updates are named by `dev-XXX` where `XXX` is the update-referring value for each update. 34 | -------------------------------------------------------------------------------- /example/table_op/table_gen.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_EXAMPLE_TABLE_OP_TABLE_GEN_H_ 4 | #define USTORE_EXAMPLE_TABLE_OP_TABLE_GEN_H_ 5 | 6 | #include 7 | #include 8 | #include "table_op_arguments.h" 9 | #include "utils/uniform_random.h" 10 | 11 | namespace ustore { 12 | namespace example { 13 | namespace table_op { 14 | 15 | class TableGen { 16 | public: 17 | explicit TableGen(TableOpArguments& args) noexcept : args_(args) {} 18 | 19 | ~TableGen() = default; 20 | 21 | ErrorCode Run(); 22 | 23 | private: 24 | ErrorCode GenData(const std::vector& vals); 25 | ErrorCode GenQueries(const std::vector& vals); 26 | 27 | TableOpArguments& args_; 28 | uio::UniformRandom rand_; 29 | }; 30 | 31 | } // namespace table_op 32 | } // namespace example 33 | } // namespace ustore 34 | 35 | #endif // USTORE_EXAMPLE_TABLE_OP_TABLE_GEN_H_ 36 | -------------------------------------------------------------------------------- /example/table_op/table_op.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_EXAMPLE_TABLE_OP_TABLE_OP_H_ 4 | #define USTORE_EXAMPLE_TABLE_OP_TABLE_OP_H_ 5 | 6 | #include 7 | #include "spec/relational.h" 8 | #include "utils/timer.h" 9 | 10 | #include "table_op_arguments.h" 11 | 12 | namespace ustore { 13 | namespace example { 14 | namespace table_op { 15 | 16 | class TableOp { 17 | public: 18 | explicit TableOp(TableOpArguments& args, DB* db) noexcept 19 | : cs_(db), args_(args) {} 20 | 21 | ~TableOp() = default; 22 | 23 | ErrorCode Run(); 24 | 25 | private: 26 | ErrorCode Init(); 27 | ErrorCode Load(); 28 | ErrorCode Update(const std::string& ref_val); 29 | ErrorCode Aggregate(); 30 | ErrorCode Diff(const std::string& lhs_branch, const std::string& rhs_branch); 31 | 32 | ErrorCode DropIfExists(const std::string& tab); 33 | 34 | ErrorCode VerifyColumn(const std::string& col); 35 | 36 | ErrorCode MeasureByteIncrement(size_t* bytes_inc, 37 | const std::function& f); 38 | 39 | ColumnStore cs_; 40 | TableOpArguments& args_; 41 | }; 42 | 43 | } // namespace table_op 44 | } // namespace example 45 | } // namespace ustore 46 | 47 | #endif // USTORE_EXAMPLE_TABLE_OP_TABLE_OP_H_ 48 | -------------------------------------------------------------------------------- /example/table_op/table_op_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "cluster/worker_client_service.h" 4 | #include "worker/worker.h" 5 | 6 | #include "table_gen.h" 7 | #include "table_op.h" 8 | #include "table_op_arguments.h" 9 | 10 | namespace ustore { 11 | namespace example { 12 | namespace table_op { 13 | 14 | int main(int argc, char* argv[]) { 15 | SetStderrLogging(WARNING); 16 | // parse command-line arguments 17 | TableOpArguments args; 18 | if (!args.ParseCmdArgs(argc, argv)) { 19 | std::cerr << BOLD_RED("[ERROR] ") 20 | << "Found invalid command-line option" << std::endl; 21 | return static_cast(ErrorCode::kInvalidCommandArgument); 22 | } 23 | if (args.is_help) return static_cast(ErrorCode::kOK); 24 | // execution 25 | if (args.to_gen_data) { // for data generation 26 | auto ec = TableGen(args).Run(); 27 | if (ec == ErrorCode::kOK) { 28 | std::cout << BOLD_GREEN("[SUCCESS: Table Gen] ") 29 | << "Data: " << YELLOW(args.file) 30 | << ", Query: " << YELLOW(args.update_ref_val) << std::endl; 31 | } else { 32 | std::cout << BOLD_RED("[FAILED: Table Gen] ") 33 | << "Error(" << ec << "): " << Utils::ToString(ec) << std::endl; 34 | } 35 | return static_cast(ec); 36 | } else { // for data operation 37 | auto ec = ErrorCode::kUnknownOp; 38 | if (args.is_at_svr) { 39 | Worker db(0, nullptr, true); 40 | ec = TableOp(args, &db).Run(); 41 | } else { 42 | WorkerClientService svc; 43 | svc.Run(); 44 | auto db = svc.CreateWorkerClient(); 45 | ec = TableOp(args, &db).Run(); 46 | } 47 | if (ec != ErrorCode::kOK) { 48 | std::cout << BOLD_RED("[FAILED: Table Op] ") 49 | << "Error(" << ec << "): " << Utils::ToString(ec) << std::endl; 50 | } 51 | return static_cast(ec); 52 | } 53 | } 54 | 55 | } // namespace table_op 56 | } // namespace example 57 | } // namespace ustore 58 | 59 | int main(int argc, char* argv[]) { 60 | return ustore::example::table_op::main(argc, argv); 61 | } 62 | -------------------------------------------------------------------------------- /include/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | # Construct header guard from the this directory level. 2 | root=. 3 | -------------------------------------------------------------------------------- /include/chunk/chunk_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CHUNK_CHUNK_WRITER_H_ 4 | #define USTORE_CHUNK_CHUNK_WRITER_H_ 5 | 6 | #include "chunk/chunk.h" 7 | #include "hash/hash.h" 8 | #include "store/chunk_store.h" 9 | #include "utils/noncopyable.h" 10 | 11 | namespace ustore { 12 | 13 | class ChunkClient; 14 | class Partitioner; 15 | 16 | // ChunkWriter is responsible to persist chunks into storage 17 | class ChunkWriter : private Noncopyable { 18 | public: 19 | virtual ~ChunkWriter() = default; 20 | 21 | virtual bool Write(const Hash& key, const Chunk& chunk) = 0; 22 | 23 | protected: 24 | ChunkWriter() = default; 25 | }; 26 | 27 | // Local chunk writer write chunks to local storage 28 | class LocalChunkWriter : public ChunkWriter { 29 | public: 30 | LocalChunkWriter() : cs_(store::GetChunkStore()) {} 31 | ~LocalChunkWriter() = default; 32 | 33 | bool Write(const Hash& key, const Chunk& chunk) override; 34 | 35 | private: 36 | ChunkStore* const cs_; 37 | }; 38 | 39 | // Partitioned chunk loader write chunks based on hash-based partitions 40 | class PartitionedChunkWriter : public ChunkWriter { 41 | public: 42 | explicit PartitionedChunkWriter(const Partitioner* ptt, ChunkClient* client) 43 | : cs_(store::GetChunkStore()), ptt_(ptt), client_(client) {} 44 | ~PartitionedChunkWriter() = default; 45 | 46 | bool Write(const Hash& key, const Chunk& chunk) override; 47 | 48 | private: 49 | ChunkStore* const cs_; 50 | const Partitioner* ptt_; 51 | ChunkClient* client_; 52 | }; 53 | 54 | } // namespace ustore 55 | 56 | #endif // USTORE_CHUNK_CHUNK_WRITER_H_ 57 | -------------------------------------------------------------------------------- /include/chunk/chunker.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CHUNK_CHUNKER_H_ 4 | #define USTORE_CHUNK_CHUNKER_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "chunk/chunk.h" 10 | #include "chunk/segment.h" 11 | #include "node/rolling_hash.h" 12 | 13 | namespace ustore { 14 | 15 | struct ChunkInfo { 16 | Chunk chunk; 17 | // a Segment that holding a single MetaEntry bytes 18 | std::unique_ptr meta_seg; 19 | }; 20 | 21 | class Chunker { 22 | // An interface to make chunk from multiple segments. 23 | // Each type, e.g, Blob, MetaNode shall have one. 24 | public: 25 | virtual ~Chunker() = default; 26 | virtual ChunkInfo Make(const std::vector& segments) const = 0; 27 | virtual bool isFixedEntryLen() const = 0; 28 | // Derived class can customize rolling hashes 29 | inline void SetRHasherParams(uint32_t chunk_pattern, size_t chunk_window, 30 | size_t max_chunk_size) { 31 | chunk_pattern_ = chunk_pattern; 32 | chunk_window_ = chunk_window; 33 | max_chunk_size_ = max_chunk_size; 34 | } 35 | virtual inline std::unique_ptr GetRHasher() const { 36 | #ifdef TEST_NODEBUILDER 37 | return std::unique_ptr(RollingHasher::TestHasher()); 38 | #else 39 | return std::unique_ptr( 40 | new RollingHasher(chunk_pattern_, chunk_window_, max_chunk_size_)); 41 | #endif 42 | } 43 | 44 | private: 45 | uint32_t chunk_pattern_ = (1 << 12) - 1; // 4KB 46 | size_t chunk_window_ = 256; 47 | size_t max_chunk_size_ = 1 << 15; // 32KB 48 | }; 49 | 50 | } // namespace ustore 51 | #endif // USTORE_CHUNK_CHUNKER_H_ 52 | -------------------------------------------------------------------------------- /include/cli/console.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLI_CONSOLE_H_ 4 | #define USTORE_CLI_CONSOLE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "cli/command.h" 11 | 12 | namespace ustore { 13 | namespace cli { 14 | 15 | #define CONSOLE_CMD_HANDLER(cmd, handler) do { \ 16 | CMD_HANDLER(cmd, handler); \ 17 | console_commands_.insert(cmd); \ 18 | } while(0) 19 | 20 | #define CONSOLE_CMD_ALIAS(cmd, alias) do { \ 21 | CMD_ALIAS(cmd, alias); \ 22 | console_commands_.insert(alias); \ 23 | } while(0) 24 | 25 | class Console : public Command { 26 | public: 27 | explicit Console(DB* db) noexcept; 28 | ~Console() = default; 29 | 30 | ErrorCode Run(int argc, char* argv[]) = delete; 31 | ErrorCode Run(); 32 | 33 | protected: 34 | void PrintHelp() override; 35 | void PrintConsoleCommandHelp(std::ostream& os = std::cout); 36 | 37 | private: 38 | void Run(const std::string& cmd_line); 39 | bool ReplaceWithHistory(std::string& cmd_line); 40 | void MarkCurrentCommandLineToComment(); 41 | 42 | ErrorCode ExecHistory(); 43 | ErrorCode ExecDumpHistory(); 44 | 45 | std::unordered_set console_commands_; 46 | std::vector history_; 47 | std::list comment_history_lines; 48 | }; 49 | 50 | } // namespace cli 51 | } // namespace ustore 52 | 53 | #endif // USTORE_CLI_CONSOLE_H_ 54 | -------------------------------------------------------------------------------- /include/cluster/chunk_client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_CHUNK_CLIENT_H_ 4 | #define USTORE_CLUSTER_CHUNK_CLIENT_H_ 5 | 6 | #include "cluster/client.h" 7 | 8 | namespace ustore { 9 | 10 | 11 | /** 12 | * The client part of chunk service, providing APIs for fetching, writing and checking 13 | * certain chunk hash. 14 | * An extension of ClientDB 15 | */ 16 | 17 | class ChunkClient : public Client { 18 | public: 19 | ChunkClient(ResponseBlob* blob, const Partitioner* ptt) 20 | : Client(blob), ptt_(ptt) {} 21 | ~ChunkClient() = default; 22 | 23 | ErrorCode Get(const Hash& hash, Chunk* chunk) const; 24 | ErrorCode Put(const Hash& hash, const Chunk& chunk); 25 | ErrorCode Exists(const Hash& hash, bool* exist) const; 26 | 27 | // only used by worker client while enable_dist_store = false 28 | ErrorCode Get(const Slice& key, const Hash& hash, Chunk* chunk) const; 29 | 30 | private: 31 | void CreateChunkMessage(const Hash& hash, UMessage *msg) const; 32 | 33 | const Partitioner* const ptt_; 34 | }; 35 | 36 | } // namespace ustore 37 | 38 | #endif // USTORE_CLUSTER_CHUNK_CLIENT_H_ 39 | -------------------------------------------------------------------------------- /include/cluster/chunk_client_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_CHUNK_CLIENT_SERVICE_H_ 4 | #define USTORE_CLUSTER_CHUNK_CLIENT_SERVICE_H_ 5 | 6 | #include "cluster/chunk_client.h" 7 | #include "cluster/client_service.h" 8 | #include "cluster/partitioner.h" 9 | #include "utils/env.h" 10 | 11 | namespace ustore { 12 | 13 | class ChunkClientService : public ClientService { 14 | public: 15 | ChunkClientService() 16 | : ClientService(&ptt_), 17 | ptt_(Env::Instance()->config().worker_file(), "") {} 18 | ~ChunkClientService() = default; 19 | 20 | void Init() override; 21 | ChunkClient CreateChunkClient(); 22 | 23 | private: 24 | const ChunkPartitioner ptt_; 25 | }; 26 | 27 | } // namespace ustore 28 | 29 | #endif // USTORE_CLUSTER_CHUNK_CLIENT_SERVICE_H_ 30 | -------------------------------------------------------------------------------- /include/cluster/chunk_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_CHUNK_SERVICE_H_ 4 | #define USTORE_CLUSTER_CHUNK_SERVICE_H_ 5 | 6 | #include "cluster/host_service.h" 7 | #include "cluster/port_helper.h" 8 | #include "proto/messages.pb.h" 9 | #include "store/chunk_store.h" 10 | #include "utils/env.h" 11 | 12 | namespace ustore { 13 | 14 | /** 15 | * The server side of chunk service, serving requests for ChunkDb requests. 16 | */ 17 | class ChunkService : public HostService { 18 | public: 19 | explicit ChunkService(const node_id_t& addr) 20 | : HostService(PortHelper::ChunkPort(addr)), 21 | store_(store::GetChunkStore()) {} 22 | ~ChunkService() = default; 23 | 24 | void Init() override; 25 | void HandleRequest(const void *msg, int size, const node_id_t& source) 26 | override; 27 | 28 | private: 29 | void HandleGetChunkRequest(const UMessage& umsg, ResponsePayload* reponse); 30 | void HandlePutChunkRequest(const UMessage& umsg, ResponsePayload* response); 31 | void HandleExistChunkRequest(const UMessage& umsg, 32 | ResponsePayload* response); 33 | 34 | ChunkStore* const store_; 35 | }; 36 | } // namespace ustore 37 | 38 | #endif // USTORE_CLUSTER_CHUNK_SERVICE_H_ 39 | -------------------------------------------------------------------------------- /include/cluster/client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_CLIENT_H_ 4 | #define USTORE_CLUSTER_CLIENT_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "chunk/chunk.h" 10 | #include "cluster/partitioner.h" 11 | #include "cluster/response_blob.h" 12 | #include "hash/hash.h" 13 | #include "net/net.h" 14 | #include "proto/messages.pb.h" 15 | #include "types/ucell.h" 16 | #include "store/chunk_store.h" 17 | 18 | namespace ustore { 19 | 20 | /** 21 | * Client is an abstracted class to parse response messages from ClientService 22 | * into app-level data structures. 23 | * It contains impls for parsing all return types of response messages. 24 | */ 25 | 26 | class Client { 27 | public: 28 | ~Client() = default; 29 | 30 | protected: 31 | explicit Client(ResponseBlob* blob) 32 | : id_(blob->id), net_(blob->net), res_blob_(blob) {} 33 | 34 | bool Send(UMessage* msg, const node_id_t& node_id) const; 35 | // helper methods for getting response 36 | ErrorCode GetEmptyResponse() const; 37 | ErrorCode GetVersionResponse(Hash* version) const; 38 | ErrorCode GetUCellResponse(UCell* value) const; 39 | ErrorCode GetStringListResponse(std::vector* vals) const; 40 | ErrorCode GetVersionListResponse(std::vector* versions) const; 41 | ErrorCode GetBoolResponse(bool* value) const; 42 | ErrorCode GetChunkResponse(Chunk* chunk) const; 43 | ErrorCode GetInfoResponse(std::vector* info) const; 44 | 45 | private: 46 | std::unique_ptr WaitForResponse() const; 47 | // send request to a node. Return false if there are 48 | // errors with network communication. 49 | 50 | const int id_ = 0; 51 | Net* const net_ = nullptr; // for network communication 52 | ResponseBlob* const res_blob_ = nullptr; // response blob 53 | }; 54 | 55 | } // namespace ustore 56 | 57 | #endif // USTORE_CLUSTER_CLIENT_H_ 58 | -------------------------------------------------------------------------------- /include/cluster/client_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_CLIENT_SERVICE_H_ 4 | #define USTORE_CLUSTER_CLIENT_SERVICE_H_ 5 | 6 | #include 7 | #include 8 | #include "boost/thread.hpp" 9 | #include "cluster/partitioner.h" 10 | #include "cluster/response_blob.h" 11 | #include "cluster/service.h" 12 | #include "net/net.h" 13 | 14 | namespace ustore { 15 | 16 | /** 17 | * ClientService is an abstracted class to handle response from server. 18 | * A ClientService receives responses from Server and invokes corresponding 19 | * classes to process the message. 20 | */ 21 | class ClientService : public Service { 22 | public: 23 | explicit ClientService(const Partitioner* ptt) 24 | : nclients_(0), ptt_(ptt) {} 25 | virtual ~ClientService() = default; 26 | 27 | void Init(std::unique_ptr callback); 28 | void HandleResponse(const void *msg, int size, const node_id_t& source); 29 | 30 | protected: 31 | ResponseBlob* CreateResponseBlob(); 32 | 33 | private: 34 | int nclients_; // how many RequestHandler thread it uses 35 | const Partitioner* const ptt_; 36 | std::vector> responses_; // the response queue 37 | boost::shared_mutex lock_; 38 | }; 39 | } // namespace ustore 40 | 41 | #endif // USTORE_CLUSTER_CLIENT_SERVICE_H_ 42 | -------------------------------------------------------------------------------- /include/cluster/host_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_HOST_SERVICE_H_ 4 | #define USTORE_CLUSTER_HOST_SERVICE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "cluster/service.h" 10 | #include "net/net.h" 11 | #include "utils/env.h" 12 | 13 | namespace ustore { 14 | 15 | /** 16 | * HostService is an abstracted class to handle request from server side. 17 | * A HostService receives requests from Client and invokes corresponding 18 | * classes to process the message. 19 | * Derived class from HostService should provide impl for handling requests. 20 | */ 21 | class HostService : public Service { 22 | public: 23 | // use another port if xor_port = true 24 | explicit HostService(const node_id_t& addr) : node_addr_(addr) {} 25 | ~HostService() = default; 26 | 27 | /** 28 | * Handle requests: 29 | * 1. It parse msg into a UStoreMessage 30 | * 2. Invoke the processing logic correspondingly. 31 | * 3. Construct a response and send back to source. 32 | */ 33 | virtual void HandleRequest(const void *msg, int size, 34 | const node_id_t& source) = 0; 35 | 36 | protected: 37 | inline void Init(std::unique_ptr callback) { 38 | Net* net = net::CreateServerNetwork(node_addr_, 39 | Env::Instance()->config().recv_threads()); 40 | Service::Init(std::unique_ptr(net), std::move(callback)); 41 | } 42 | inline void Send(const node_id_t& source, byte_t* ptr, int len) { 43 | net_->GetNetContext(source)->Send(ptr, static_cast(len)); 44 | } 45 | 46 | node_id_t node_addr_; 47 | }; 48 | } // namespace ustore 49 | 50 | #endif // USTORE_CLUSTER_HOST_SERVICE_H_ 51 | -------------------------------------------------------------------------------- /include/cluster/master_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_MASTER_SERVICE_H_ 4 | #define USTORE_CLUSTER_MASTER_SERVICE_H_ 5 | 6 | #include 7 | #include "cluster/service.h" 8 | #include "net/net.h" 9 | 10 | namespace ustore { 11 | 12 | /** 13 | * The MasterService receives requests from ClientService about key 14 | * range changes (RangeInfo). It then invokes Master to return 15 | * the ranges. 16 | * Basically a simplified version of WorkerService. 17 | */ 18 | class MasterService : public Service { 19 | public: 20 | MasterService(const node_id_t& id, const string& config_path) 21 | : Service(id), config_path_(config_path) {} 22 | ~MasterService(); 23 | 24 | /** 25 | * Handle requests: 26 | * 1. It parse msg into a message (RangeRequest, e.g) 27 | * 2. Invoke the processing logic from Master. 28 | * 3. Construct a response (RangeResponse, e.g.) and send back. 29 | */ 30 | void HandleRequest(const void *msg, int size, const node_id_t& source) 31 | override; 32 | 33 | private: 34 | string config_path_; // the master may read from a global config file 35 | }; 36 | } // namespace ustore 37 | 38 | #endif // USTORE_CLUSTER_MASTER_SERVICE_H_ 39 | -------------------------------------------------------------------------------- /include/cluster/port_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_PORT_HELPER_H_ 4 | #define USTORE_CLUSTER_PORT_HELPER_H_ 5 | 6 | #include 7 | 8 | namespace ustore { 9 | 10 | /* 11 | * Router is responsible for calculate corresponding port for a service 12 | */ 13 | class PortHelper { 14 | public: 15 | // worker service port is even 16 | static std::string WorkerPort(std::string addr) { 17 | if (addr.back() & 1) addr.back() ^= 1; 18 | return addr; 19 | } 20 | // chunk service port is odd 21 | static std::string ChunkPort(std::string addr) { 22 | if (!(addr.back() & 1)) addr.back() ^= 1; 23 | return addr; 24 | } 25 | }; 26 | 27 | } // namespace ustore 28 | 29 | #endif // USTORE_CLUSTER_PORT_HELPER_H_ 30 | -------------------------------------------------------------------------------- /include/cluster/response_blob.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_RESPONSE_BLOB_H_ 4 | #define USTORE_CLUSTER_RESPONSE_BLOB_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "net/net.h" 10 | #include "proto/messages.pb.h" 11 | 12 | namespace ustore { 13 | 14 | using google::protobuf::Message; 15 | 16 | /** 17 | * A unit on the response queue. Each client request thread 18 | * waits on one of this object. The thread goes to sleep waiting for has_msg 19 | * condition to hold true. The has_msg variable will be set by the registered 20 | * callback method of the network thread 21 | * 22 | * This object is used under the assumption that the client issues requests in 23 | * a synchronous manner. As a result, the msg must be cleared before another 24 | * response is set. 25 | */ 26 | struct ResponseBlob { 27 | int id; 28 | Net* net = nullptr; 29 | std::mutex lock; 30 | std::condition_variable condition; 31 | 32 | bool has_msg; 33 | // message will be takeover by corresponding client, no memory leak here 34 | Message* message = nullptr; 35 | }; 36 | 37 | } // namespace ustore 38 | 39 | #endif // USTORE_CLUSTER_RESPONSE_BLOB_H_ 40 | -------------------------------------------------------------------------------- /include/cluster/service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_SERVICE_H_ 4 | #define USTORE_CLUSTER_SERVICE_H_ 5 | 6 | #include 7 | #include 8 | #include "net/net.h" 9 | #include "utils/noncopyable.h" 10 | 11 | namespace ustore { 12 | 13 | /** 14 | * Service Run() calls Start() in a background thread 15 | */ 16 | class Service : private Noncopyable { 17 | public: 18 | Service() = default; 19 | virtual ~Service() { Stop(); } 20 | 21 | // Init network before start 22 | virtual void Init() = 0; 23 | // Start in current thread (blocking) 24 | void Start(); 25 | // Run in another thread (non-blocking) 26 | void Run(); 27 | // Stop service 28 | void Stop(); 29 | // check status 30 | inline bool IsRunning() { return is_running_; } 31 | 32 | protected: 33 | // Need to be called in Init() 34 | void Init(std::unique_ptr net, std::unique_ptr callback); 35 | 36 | std::unique_ptr cb_; 37 | std::unique_ptr net_; 38 | 39 | private: 40 | // thread run in background 41 | std::unique_ptr thread_; 42 | volatile bool is_running_ = false; 43 | volatile bool is_init_ = false; 44 | }; 45 | } // namespace ustore 46 | 47 | #endif // USTORE_CLUSTER_SERVICE_H_ 48 | -------------------------------------------------------------------------------- /include/cluster/worker_client_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_CLUSTER_WORKER_CLIENT_SERVICE_H_ 4 | #define USTORE_CLUSTER_WORKER_CLIENT_SERVICE_H_ 5 | 6 | #include 7 | #include 8 | #include "cluster/chunk_client_service.h" 9 | #include "cluster/client_service.h" 10 | #include "cluster/worker_client.h" 11 | #include "cluster/partitioner.h" 12 | #include "utils/env.h" 13 | 14 | namespace ustore { 15 | 16 | 17 | /** 18 | * Service that handle remote client requests to the database. 19 | * Multiple clients/threads share the same service, thus avoiding 20 | * creating one connection for each client. 21 | * 22 | * To interact with the database, the user first creates a ClientDb 23 | * object via CreateClientDb(). 24 | * 25 | * The database servers (Worker) knows about every RemoteClientService 26 | * in the system, and can send responses asynchronously. 27 | */ 28 | class WorkerClientService : public ClientService { 29 | public: 30 | WorkerClientService() 31 | : ClientService(&ptt_), ptt_(Env::Instance()->config().worker_file(), "") { 32 | // only need chunk client when want to get chunk bypass worker 33 | if (Env::Instance()->config().get_chunk_bypass_worker()) 34 | ck_svc_.reset(new ChunkClientService()); 35 | } 36 | ~WorkerClientService() = default; 37 | 38 | void Init() override; 39 | /** 40 | * Create a new ClientDb connecting to the database. 41 | * Interaction with the database is through this object. 42 | */ 43 | WorkerClient CreateWorkerClient(); 44 | 45 | private: 46 | const WorkerPartitioner ptt_; 47 | std::unique_ptr ck_svc_; 48 | }; 49 | 50 | } // namespace ustore 51 | 52 | #endif // USTORE_CLUSTER_WORKER_CLIENT_SERVICE_H_ 53 | -------------------------------------------------------------------------------- /include/hash/murmurhash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include 4 | #include "hash/murmurhash3.h" 5 | 6 | #ifndef USTORE_HASH_MURMURHASH_H_ 7 | #define USTORE_HASH_MURMURHASH_H_ 8 | 9 | namespace ustore { 10 | 11 | static const uint32_t kMurmurHashSeed = 0xbc9f1d34; // from LevelDB 12 | 13 | inline const uint32_t MurmurHash32(const void* key, const int& len) { 14 | uint32_t hash; 15 | MurmurHash3_x86_32(key, len, kMurmurHashSeed, &hash); 16 | return hash; 17 | } 18 | 19 | inline const size_t MurmurHash(const void* key, const int& len) { 20 | return static_cast(MurmurHash32(key, len)); 21 | } 22 | 23 | } // namespace ustore 24 | 25 | #endif // USTORE_HASH_MURMURHASH_H_ 26 | -------------------------------------------------------------------------------- /include/hash/murmurhash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef USTORE_HASH_MURMURHASH3_H_ 6 | #define USTORE_HASH_MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32(const void* key, int len, uint32_t seed, void* out); 30 | 31 | void MurmurHash3_x86_128(const void* key, int len, uint32_t seed, void* out); 32 | 33 | void MurmurHash3_x64_128(const void* key, int len, uint32_t seed, void* out); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // USTORE_HASH_MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /include/http/event.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_HTTP_EVENT_H_ 4 | #define USTORE_HTTP_EVENT_H_ 5 | 6 | #include 7 | #include 8 | 9 | namespace ustore { 10 | 11 | constexpr int kNone = 0; 12 | constexpr int kReadable = 1; // EPOLLIN 13 | constexpr int kWritable = 2; // EPOLLOUT | EPOLLERR | EPOLLHUP 14 | constexpr int kEdege = 4; 15 | 16 | constexpr int kDefaultTimeout = 10; // or 0: NON_BLOCKING, -1: BLOCKING 17 | constexpr int kDefaultMaxFd = 10000; 18 | 19 | struct EventLoop; 20 | 21 | // Types and data structures 22 | typedef void FileProc(struct EventLoop *event_loop, int fd, void *client_data, 23 | int mask); 24 | 25 | // File event structure 26 | struct FileEvent { 27 | int mask; // one of AE_(kReadable|kWritable) 28 | FileProc *rfile_proc; 29 | FileProc *wfile_proc; 30 | void *client_data; 31 | }; 32 | 33 | // A fired event 34 | struct FiredEvent { 35 | int fd; 36 | int mask; 37 | }; 38 | 39 | struct EpollState { 40 | int epfd; 41 | struct epoll_event *events; 42 | }; 43 | 44 | /* 45 | * event loop wrapper of epoll for easy use 46 | */ 47 | class EventLoop { 48 | public: 49 | explicit EventLoop(int setsize = kDefaultMaxFd); 50 | ~EventLoop(); 51 | 52 | void Start(); 53 | void Stop(); 54 | 55 | int CreateFileEvent(int fd, int mask, FileProc *proc, void *client_data); 56 | void DeleteFileEvent(int fd, int mask); 57 | int GetFileEvents(int fd); 58 | 59 | private: 60 | int ResizeSetSize(int setsize); 61 | 62 | // return number of processed events 63 | int ProcessEvents(int timeout = kDefaultTimeout); 64 | int EpollAddEvent(int fd, int mask); 65 | void EpollDelEvent(int fd, int delmask); 66 | int EpollPoll(int timeout = kDefaultTimeout); 67 | 68 | int maxfd_; // highest file descriptor currently registered 69 | int setsize_; // max number of file descriptors tracked 70 | std::vector events_; // Registered events 71 | std::vector fired_; // Fired events 72 | volatile int stop_; 73 | EpollState* estate_; 74 | }; 75 | } // namespace ustore 76 | 77 | #endif // USTORE_HTTP_EVENT_H_ 78 | -------------------------------------------------------------------------------- /include/http/http_client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_HTTP_HTTP_CLIENT_H_ 4 | #define USTORE_HTTP_HTTP_CLIENT_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "http/http_msg.h" 10 | 11 | namespace ustore { 12 | namespace http { 13 | 14 | class HttpClient { 15 | public: 16 | HttpClient() = default; 17 | ~HttpClient() = default; 18 | 19 | /* 20 | * Connect to a http server in host:port format 21 | */ 22 | bool Connect(const std::string& host, const std::string& port); 23 | /* 24 | * Send request of type verb to http://host:port/url 25 | */ 26 | bool Send(Request* request); 27 | /* 28 | * Get response message from server 29 | */ 30 | bool Receive(Response* response); 31 | /* 32 | * Shutdown the connection 33 | */ 34 | bool Shutdown(); 35 | 36 | private: 37 | boost::asio::io_context io_context_; 38 | boost::asio::ip::tcp::socket socket_{io_context_}; 39 | std::string host_; 40 | }; 41 | 42 | } // namespace http 43 | } // namespace ustore 44 | 45 | #endif // USTORE_HTTP_HTTP_CLIENT_H_ 46 | -------------------------------------------------------------------------------- /include/http/http_msg.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_HTTP_HTTP_MSG_H_ 4 | #define USTORE_HTTP_HTTP_MSG_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace ustore { 12 | namespace http { 13 | 14 | namespace beast = boost::beast::http; 15 | 16 | enum class Verb { 17 | kGet = 0, 18 | kPut = 1, 19 | kPost = 2 20 | }; 21 | 22 | class Request { 23 | friend class HttpClient; 24 | 25 | public: 26 | Request() { 27 | SetDefaultFields(); 28 | } 29 | Request(const std::string& target, Verb method) : Request() { 30 | SetTarget(target); 31 | SetMethod(method); 32 | } 33 | ~Request() = default; 34 | 35 | // Set target 36 | inline void SetTarget(const std::string& target) { target_ = target; } 37 | // Set method verb 38 | void SetMethod(Verb method); 39 | // Add customized header fields 40 | inline void SetHeaderField(const std::string& fld, const std::string& val) { 41 | req_.set(fld, val); 42 | } 43 | inline void AddParameter(const std::string& key, const std::string& val) { 44 | param_[key] = val; 45 | } 46 | // Set data body 47 | inline void SetBody(const std::string& data) { req_.body() = data; } 48 | 49 | private: 50 | static constexpr int kDefaultHttpVersion = 11; 51 | 52 | // add default header fields 53 | // called by constructor 54 | void SetDefaultFields(); 55 | // Prepare payload 56 | void PreparePayload(); 57 | 58 | beast::request req_; 59 | std::string target_; 60 | std::map param_; 61 | }; 62 | 63 | class Response { 64 | friend class HttpClient; 65 | 66 | public: 67 | Response() = default; 68 | ~Response() = default; 69 | 70 | const std::map& headers(); 71 | inline const std::string& body() { return res_.body(); } 72 | inline int code() { return res_.result_int(); } 73 | 74 | private: 75 | beast::response res_; 76 | boost::beast::flat_buffer buffer_; 77 | std::map headers_; // to hold header strings 78 | }; 79 | 80 | } // namespace http 81 | } // namespace ustore 82 | 83 | #endif // USTORE_HTTP_HTTP_MSG_H_ 84 | -------------------------------------------------------------------------------- /include/http/lock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_HTTP_LOCK_H_ 4 | #define USTORE_HTTP_LOCK_H_ 5 | 6 | #include "http/settings.h" 7 | 8 | namespace ustore { 9 | #if !defined(USE_ATOMIC) 10 | #include 11 | #endif 12 | 13 | /* 14 | * a lock wrapper which can be changed to mutex-based or atomic-based easily 15 | */ 16 | class Locker { 17 | #ifdef USE_ATOMIC 18 | bool lock_ = 0; 19 | #else 20 | std::mutex lock_; 21 | #endif 22 | 23 | public: 24 | inline void lock() { 25 | #ifdef USE_ATOMIC 26 | while (__atomic_test_and_set(&lock_, __ATOMIC_RELAXED)) {} 27 | #else 28 | lock_.lock(); 29 | #endif 30 | } 31 | 32 | inline void unlock() { 33 | #ifdef USE_ATOMIC 34 | __atomic_clear(&lock_, __ATOMIC_RELAXED); 35 | #else 36 | lock_.unlock(); 37 | #endif 38 | } 39 | 40 | inline bool try_lock() { 41 | #ifdef USE_ATOMIC 42 | return !__atomic_test_and_set(&lock_, __ATOMIC_RELAXED); 43 | #else 44 | return lock_.try_lock(); 45 | #endif 46 | } 47 | }; 48 | } // namespace ustore 49 | 50 | #endif // USTORE_HTTP_LOCK_H_ 51 | -------------------------------------------------------------------------------- /include/http/settings.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_HTTP_SETTINGS_H_ 4 | #define USTORE_HTTP_SETTINGS_H_ 5 | 6 | namespace ustore { 7 | 8 | /* 9 | * whether use atomic or mutex 10 | * if defined, use atomic 11 | */ 12 | #define USE_ATOMIC 13 | 14 | /* 15 | * whether use cache or not 16 | * if defined, use cache 17 | */ 18 | #define USE_CACHE 19 | constexpr size_t kDefaultCacheSize = 100; // default cache size 20 | 21 | constexpr size_t kMaxHeaderSize = 10240; // max http header size 22 | 23 | /* 24 | * max file size 25 | * if actual file size > kMaxFileSize, 26 | * it is ok since we have a dynamic strategy to decide the file size 27 | * and allocate the response buffer 28 | */ 29 | constexpr size_t kMaxOutputSize = 1UL << 35; // max response message size 30 | constexpr size_t kMaxResponseSize = kMaxHeaderSize + kMaxOutputSize; 31 | 32 | /* 33 | * default receive buffer size 34 | * TODO: if data is larger than the kDefaultRecvSize, how to handle? 35 | */ 36 | constexpr size_t kMaxInputSize = 1UL << 20; // max input message size 37 | constexpr size_t kDefaultRecvSize = kMaxHeaderSize + kMaxInputSize; 38 | 39 | } // namespace ustore 40 | 41 | #endif // USTORE_HTTP_SETTINGS_H_ 42 | -------------------------------------------------------------------------------- /include/node/blob_node.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_NODE_BLOB_NODE_H_ 4 | #define USTORE_NODE_BLOB_NODE_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "node/node.h" 10 | #include "utils/singleton.h" 11 | 12 | namespace ustore { 13 | 14 | class BlobChunker : public Singleton, public Chunker { 15 | friend class Singleton; 16 | 17 | public: 18 | ChunkInfo Make(const std::vector& segments) const override; 19 | inline bool isFixedEntryLen() const override { return true; } 20 | 21 | private: 22 | BlobChunker() = default; 23 | ~BlobChunker() = default; 24 | }; 25 | 26 | class BlobNode : public LeafNode { 27 | /* 28 | BlobNode is a leaf node in pos tree that contains 29 | actual blob data 30 | 31 | Encoding Scheme: 32 | | ------blob bytes ------| 33 | | ------variable size 34 | */ 35 | public: 36 | explicit BlobNode(const Chunk* chunk) : LeafNode(chunk) {} 37 | ~BlobNode() = default; 38 | 39 | const byte_t* data(size_t idx) const override { return chunk_->data() + idx; } 40 | // return the byte len of the idx-th entry 41 | inline size_t len(size_t idx) const override { return 1; } 42 | inline size_t numEntries() const override { return chunk_->capacity(); } 43 | inline size_t GetLength(size_t start, size_t end) const override { 44 | return end - start; 45 | } 46 | OrderedKey key(size_t idx) const override; 47 | size_t Copy(size_t start, size_t num_bytes, byte_t* buffer) const override; 48 | uint64_t FindIndexForKey(const OrderedKey& key, 49 | ChunkLoader* loader) const override; 50 | std::unique_ptr GetSegment( 51 | size_t start, size_t num_elements) const override; 52 | }; 53 | 54 | } // namespace ustore 55 | 56 | #endif // USTORE_NODE_BLOB_NODE_H_ 57 | -------------------------------------------------------------------------------- /include/node/orderedkey.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_NODE_ORDEREDKEY_H_ 4 | #define USTORE_NODE_ORDEREDKEY_H_ 5 | 6 | #include 7 | 8 | #include "spec/slice.h" 9 | #include "types/type.h" 10 | #include "utils/logging.h" 11 | 12 | namespace ustore { 13 | class OrderedKey { 14 | /* OrderedKey can either be a hash value (byte array) or an uint64_t integer 15 | 16 | Encoding Scheme by OrderedKey (variable size) 17 | |-----hash value/uint64 | 18 | 0 -- variable size 19 | */ 20 | public: 21 | inline static OrderedKey FromSlice(const Slice& key) { 22 | return OrderedKey(false, key.data(), key.len()); 23 | } 24 | // Set an integer value for key 25 | // own set to false 26 | OrderedKey() = default; 27 | explicit OrderedKey(uint64_t value) noexcept; 28 | // Set the hash data for key 29 | OrderedKey(bool by_value, const byte_t* data, size_t num_bytes) noexcept; 30 | OrderedKey(const OrderedKey&) = default; 31 | ~OrderedKey() = default; 32 | 33 | OrderedKey& operator=(const OrderedKey&) = default; 34 | 35 | inline const byte_t* data() const { return slice_.data(); } 36 | inline size_t numBytes() const { 37 | return by_value_ ? sizeof(uint64_t) : slice_.len(); 38 | } 39 | // encode OrderedKey into buffer 40 | // given buffer capacity > numBytes 41 | size_t Encode(byte_t* buffer) const; 42 | inline bool byValue() const { return by_value_; } 43 | 44 | bool operator>(const OrderedKey& otherKey) const; 45 | bool operator<(const OrderedKey& otherKey) const; 46 | bool operator==(const OrderedKey& otherKey) const; 47 | inline bool operator<=(const OrderedKey& otherKey) const { 48 | return *this < otherKey || *this == otherKey; 49 | } 50 | inline bool operator>=(const OrderedKey& otherKey) const { 51 | return *this > otherKey || *this == otherKey; 52 | } 53 | 54 | inline Slice ToSlice() const { 55 | CHECK(!by_value_); 56 | return slice_; 57 | } 58 | 59 | private: 60 | // Parse the data as a number to compare 61 | // Otherwise as hash value 62 | bool by_value_; 63 | uint64_t value_; 64 | Slice slice_; 65 | }; 66 | 67 | } // namespace ustore 68 | 69 | #endif // USTORE_NODE_ORDEREDKEY_H_ 70 | -------------------------------------------------------------------------------- /include/recovery/log_cursor.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_RECOVERY_LOG_CURSOR_H_ 4 | #define USTORE_RECOVERY_LOG_CURSOR_H_ 5 | 6 | #include 7 | #include "recovery/log_entry.h" 8 | 9 | namespace ustore { 10 | namespace recovery { 11 | 12 | class LogCursor { 13 | public: 14 | LogCursor(); 15 | ~LogCursor(); 16 | 17 | /* 18 | * @brief make sure all the fields are >= 0 19 | * */ 20 | bool IsValid() const; 21 | /* 22 | * @brief reset all the fields to be zero 23 | * */ 24 | void Reset(); 25 | /* 26 | * @brief set the field content, all the parameters are input parameters 27 | * */ 28 | void Set(uint64_t file_id, uint64_t log_id, uint64_t offset); 29 | /* 30 | * @biref put the cursor content to buffer 31 | * @param [in,out] buf 32 | * @param [in] buf_length 33 | * @param [in] pos: where the put the data in the buffer 34 | * */ 35 | int Serialize(char* buf, uint64_t buf_length, uint64_t pos) const; 36 | /* 37 | * @brief deserialize the cursor from the buffer 38 | * */ 39 | int Deserialize(const char* buf, uint64_t buf_length, uint64_t pos); 40 | std::string ToString() const; 41 | uint64_t ToString(char* buf, uint64_t length) const; 42 | /* 43 | * @brief Read Log entry from log buffer according to the cursor 44 | * */ 45 | int LoadEntry(LogCommand cmd, const char* log_data, uint64_t data_length, 46 | LogEntry* entry) const; 47 | /* 48 | * @brief forward the curpos forward 49 | * */ 50 | int Advance(LogCommand cmd, uint64_t seq_id, uint64_t data_length); 51 | int Advance(const LogEntry* entry); 52 | /* 53 | * compare the age of the cursor 54 | * */ 55 | bool operator<(const LogCursor& other) const; 56 | bool operator>(const LogCursor& other) const; 57 | bool operator==(const LogCursor& other) const; 58 | 59 | private: 60 | uint64_t file_id_; // log file id 61 | uint64_t log_id_; // log sequence id 62 | uint64_t offset_; 63 | }; // LogCursor 64 | 65 | // TODO(yaochang): add atomic log cursor 66 | 67 | } // namespace recovery 68 | } // namespace ustore 69 | 70 | #endif // USTORE_RECOVERY_LOG_CURSOR_H_ 71 | -------------------------------------------------------------------------------- /include/recovery/log_entry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_RECOVERY_LOG_ENTRY_H_ 4 | #define USTORE_RECOVERY_LOG_ENTRY_H_ 5 | 6 | #include 7 | #include "recovery/record_header.h" 8 | 9 | namespace ustore { 10 | namespace recovery { 11 | 12 | enum LogCommand : uint8_t { 13 | kCheckpoint = 101; // write checkpoints 14 | // TODO(yaochang): add log command when all logics are done 15 | }; 16 | 17 | /* 18 | * @brief Introduction to LogEntry 19 | * Log Entry for UStore 20 | * one Log record consists of four parts: 21 | * RecordHeader + Sequence ID + Log Command ID + Log content 22 | * */ 23 | class LogEntry { 24 | public: 25 | static constexpr uint16_t kLogVersion = 1; 26 | 27 | LogEntry() { 28 | memset(this, 0x00, sizeof(LogEntry)); 29 | } 30 | ~LogEntry() {} 31 | 32 | inline void setSeqId(uint64_t seq_id) { seq_id_ = seq_id; } 33 | inline void setCmdId(uint32_t cmd_id) { cmd_id_ = cmd_id; } 34 | std::string ToString() const; 35 | int64_t ToString(char* buf, uint64_t len) const; 36 | /* 37 | * @brief Set record header in the Log Entry 38 | * @param log buffer address 39 | * @param content length 40 | * */ 41 | int FillRecordHeader(const char* log_data, const data_len); 42 | /* 43 | * @brief Calculating the checksum of the log entry: 44 | * sequence_id + cmd_id + log content 45 | * @param log buffer address 46 | * @param content length 47 | * */ 48 | int64_t ComputeChecksum(const char* log_data, uint64_t data_len) const; 49 | /* 50 | * @brief Return the length of the log data 51 | * */ 52 | uint32_t GetLogdataLength() const; 53 | /* 54 | * @brief Check the structure correctnesss of the record header 55 | * */ 56 | bool CheckRecordHeader() const; 57 | /* 58 | * @brief Check the correctness of the log data 59 | * */ 60 | bool CheckLogdata() const; 61 | 62 | private: 63 | RecordHeader header_; 64 | uint64_t seq_id_; 65 | uint32_t cmd_id_; 66 | }; // LogEntry 67 | 68 | } // namespace recovery 69 | } // namespace ustore 70 | 71 | #endif // USTORE_RECOVERY_LOG_ENTRY_H_ 72 | -------------------------------------------------------------------------------- /include/recovery/log_generator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_RECOVERY_LOG_GENERATOR_H_ 4 | #define USTORE_RECOVERY_LOG_GENERATOR_H_ 5 | 6 | #include "recovery/log_cursor.h" 7 | #include "recovery/log_entry.h" 8 | 9 | namespace ustore { 10 | namespace recovery { 11 | 12 | class LogGenerator { 13 | public: 14 | LogGenerator(); 15 | ~LogGenerator(); 16 | int Init(uint64_t log_buf_size, uint64_t log_file_max_size, 17 | UStoreServer* server = NULL); 18 | int Reset(); 19 | bool IsLogStart(); 20 | int StartLog(LogCursor* start_cursor); 21 | int UpdateCursor(LogCursor* log_cursor); 22 | int WriteLog(LogCommand cmd, const char* log_data, uint64_t data_length); 23 | template 24 | int WriteLog(LogCommand cmd, T* data); 25 | int GetLog(LogCursor* start_cursor, LogCursor* end_cursor, 26 | char** buf, uint64_t* length); 27 | int Commit(LogCursor* end_cursor); 28 | int SwitchLog(uint64_t* new_file_id); 29 | int Checkpoint(uint64_t* cur_log_file_id); 30 | uint64_t ToString(char* buf, uint64_t length) const; 31 | }; 32 | 33 | } // namespace recovery 34 | } // namespace ustore 35 | 36 | #endif // USTORE_RECOVERY_LOG_GENERATOR_H_ 37 | -------------------------------------------------------------------------------- /include/recovery/log_record.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #ifndef USTORE_RECOVERY_LOG_RECORD_H_ 4 | #define USTORE_RECOVERY_LOG_RECORD_H_ 5 | 6 | #include 7 | #include 8 | #include "types/type.h" 9 | 10 | namespace ustore { 11 | namespace recovery { 12 | 13 | enum class LogCommand : int16_t { 14 | kNull = 0, // Invalid 15 | kUpdate = 111, // Update(branch_name, version) 16 | kRename = 112, // Rename(branch_name, new_branch_name) 17 | kRemove = 113 // Remove(branch_name) 18 | }; 19 | /* 20 | * the structure of one log record is: 21 | * total_length||checksum||version||logcmd||lsn||key_len||value_len||key||value 22 | **/ 23 | struct LogRecord { 24 | public: 25 | LogRecord() {} 26 | ~LogRecord() = default; 27 | 28 | // generate all the content to a string 29 | std::string ToString(); 30 | // generate all the content to a string 31 | void FromString(const char* log_data); 32 | // Compute the checksum according to the content 33 | int64_t ComputeChecksum(); 34 | 35 | // compute the checksum after the other fields are filled 36 | int64_t checksum = 0; 37 | int16_t version = 1; // by default 38 | LogCommand logcmd = LogCommand::kNull; // what kind of operation 39 | int64_t log_sequence_number = 0; 40 | const byte_t* key = nullptr; 41 | const byte_t* value = nullptr; 42 | int64_t key_length = 0; 43 | int64_t value_length = 0; 44 | int64_t data_length = 0; 45 | std::unique_ptr key_data; 46 | std::unique_ptr value_data; 47 | }; 48 | 49 | } // namespace recovery 50 | } // namespace ustore 51 | 52 | #endif // USTORE_RECOVERY_LOG_RECORD_H_ 53 | -------------------------------------------------------------------------------- /include/recovery/log_thread.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The UStore Authors. 2 | 3 | #ifndef USTORE_RECOVERY_LOG_THREAD_H_ 4 | #define USTORE_RECOVERY_LOG_THREAD_H_ 5 | 6 | #include 7 | 8 | namespace ustore { 9 | namespace recovery { 10 | 11 | /* 12 | * The base class of LogWriter and LogReader 13 | * */ 14 | class LogThread { 15 | public: 16 | LogThread(); 17 | ~LogThread(); 18 | 19 | int Start(); 20 | int Join(); 21 | int Detach(); 22 | pthread_t Self(); 23 | virtual void* Run() = 0; // this function should be implemented!! 24 | private: 25 | pthread_t m_tid_; // the containing thread ID 26 | int m_running_; // flag to indicate the thread is running or not: 1 yes, 0 no 27 | int m_detached_; // flag the thread is detached or not: 1 yes, 0 no 28 | }; 29 | 30 | static void* runThread(void* args); 31 | 32 | } // namespace recovery 33 | } // namespace ustore 34 | 35 | #endif // USTORE_RECOVERY_LOG_THREAD_H_ 36 | -------------------------------------------------------------------------------- /include/spec/object_meta.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The UStore Authors. 2 | 3 | #ifndef USTORE_SPEC_OBJECT_META_H_ 4 | #define USTORE_SPEC_OBJECT_META_H_ 5 | 6 | #include 7 | 8 | #include "spec/object_db.h" 9 | 10 | namespace ustore { 11 | 12 | class ObjectMeta { 13 | public: 14 | static const std::string kKeyPrefix; 15 | 16 | static inline std::string MetaTableKey(const std::string& obj_name) { 17 | return kKeyPrefix + obj_name; 18 | } 19 | 20 | explicit ObjectMeta(DB* db) noexcept : odb_(db) {} 21 | ~ObjectMeta() = default; 22 | 23 | protected: 24 | ErrorCode CreateMetaTable(const std::string& obj_name, 25 | const std::string& branch); 26 | 27 | ErrorCode BranchMetaTable(const std::string& obj_name, 28 | const std::string& old_branch, 29 | const std::string& new_branch); 30 | 31 | ErrorCode DeleteMetaTable(const std::string& obj_name, 32 | const std::string& branch); 33 | 34 | ErrorCode GetMetaTable(const std::string& obj_name, 35 | const std::string& branch, 36 | VMap* meta_tab) const; 37 | 38 | ErrorCode SetMeta(const std::string& obj_name, 39 | const std::string& branch, 40 | const std::string& meta_name, 41 | const std::string& meta_val); 42 | 43 | ErrorCode GetMeta(const std::string& obj_name, 44 | const std::string& branch, 45 | const std::string& meta_name, 46 | std::string* meta_val) const; 47 | 48 | ErrorCode DeleteMeta(const std::string& obj_name, 49 | const std::string& branch, 50 | const std::string& meta_name); 51 | 52 | private: 53 | ErrorCode ReadMetaTable(const Slice& meta_tab_key, const Slice& branch, 54 | VMap* meta_tab) const; 55 | 56 | ObjectDB odb_; 57 | }; 58 | 59 | } // namespace ustore 60 | 61 | #endif // USTORE_SPEC_OBJECT_META_H_ 62 | -------------------------------------------------------------------------------- /include/spec/value.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_SPEC_VALUE_H_ 4 | #define USTORE_SPEC_VALUE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "hash/hash.h" 10 | #include "types/type.h" 11 | #include "spec/slice.h" 12 | #include "utils/logging.h" 13 | 14 | namespace ustore { 15 | 16 | // This struct is generalized to inserts/updates any UTypes 17 | struct Value { 18 | // Type of the value 19 | UType type; 20 | // Hash::kNull if it is a new insertion 21 | // Otherwise, it is an update from the content that has the Hash 22 | Hash base; 23 | // Only used for an update 24 | // Indicate where to start the insertion/deletion 25 | size_t pos; 26 | // Number of deletions 27 | size_t dels; 28 | // Content of Insertions 29 | // size = 1 for Blob/String 30 | // size > 1 for Map/List 31 | std::vector vals; 32 | // Only used my Map, and has keys.size() = vals.size() 33 | std::vector keys; 34 | 35 | // Application specific context 36 | Slice ctx; 37 | }; 38 | 39 | } // namespace ustore 40 | 41 | #endif // USTORE_SPEC_VALUE_H_ 42 | -------------------------------------------------------------------------------- /include/store/chunk_store.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_STORE_CHUNK_STORE_H_ 4 | #define USTORE_STORE_CHUNK_STORE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "chunk/chunk.h" 10 | #include "store/iterator.h" 11 | #include "types/type.h" 12 | 13 | namespace ustore { 14 | 15 | struct StoreInfo { 16 | using MapType = std::unordered_map; 17 | 18 | size_t chunks; 19 | size_t chunkBytes; 20 | size_t validChunks; 21 | size_t validChunkBytes; 22 | 23 | size_t maxSegments; 24 | size_t allocSegments; 25 | size_t freeSegments; 26 | size_t usedSegments; 27 | 28 | MapType chunksPerType; 29 | MapType bytesPerType; 30 | 31 | std::string nodeId; 32 | 33 | friend std::ostream& operator<<(std::ostream& os, const StoreInfo& obj); 34 | }; 35 | 36 | class ChunkStore { 37 | public: 38 | /* 39 | * store allocates the returned chunck, 40 | * caller need to release memory after use. 41 | */ 42 | ChunkStore() = default; 43 | virtual ~ChunkStore() noexcept(false) {} 44 | 45 | virtual Chunk Get(const Hash& key) = 0; 46 | virtual bool Put(const Hash& key, const Chunk& chunk) = 0; 47 | virtual bool Exists(const Hash& key) = 0; 48 | virtual StoreInfo GetInfo() = 0; 49 | 50 | virtual StoreIterator begin() const = 0; 51 | virtual StoreIterator cbegin() const = 0; 52 | virtual StoreIterator end() const = 0; 53 | virtual StoreIterator cend() const = 0; 54 | }; 55 | 56 | // wrap global functions inside a namespace 57 | namespace store { 58 | 59 | // have to be called before calling GetChunkStore(), otherwise has no effect 60 | ChunkStore* InitChunkStore(const std::string& dir, const std::string& file, 61 | bool persist); 62 | ChunkStore* GetChunkStore(); 63 | 64 | } // namespace store 65 | } // namespace ustore 66 | 67 | #endif // USTORE_STORE_CHUNK_STORE_H_ 68 | -------------------------------------------------------------------------------- /include/store/iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_STORE_ITERATOR_H_ 4 | #define USTORE_STORE_ITERATOR_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "chunk/chunk.h" 11 | 12 | namespace ustore { 13 | 14 | class StoreIteratorBase { 15 | public: 16 | virtual ~StoreIteratorBase() {} 17 | 18 | bool operator==(const StoreIteratorBase& other) const { 19 | return typeid(*this) == typeid(other) && equal(other); 20 | } 21 | 22 | virtual void operator++() = 0; 23 | virtual Chunk operator*() const = 0; 24 | // return a newly allocated object 25 | virtual StoreIteratorBase* clone() const = 0; 26 | 27 | protected: 28 | virtual bool equal(const StoreIteratorBase& other) const = 0; 29 | }; 30 | 31 | class StoreIterator : public std::iterator { 33 | public: 34 | // take ownership of the pointer 35 | explicit StoreIterator(StoreIteratorBase* ptr) : itr_(ptr) {} 36 | StoreIterator(const StoreIterator& other) : itr_(other.itr_->clone()) {} 37 | StoreIterator(StoreIterator&& other) = default; 38 | 39 | // copy & swap idiom; strong exception guarantee 40 | // note there is no extra cost 41 | StoreIterator& operator=(StoreIterator other) { 42 | std::swap(itr_, other.itr_); 43 | return *this; 44 | } 45 | 46 | inline bool operator==(const StoreIterator& other) const { 47 | return itr_ == other.itr_ || *itr_ == *other.itr_; 48 | } 49 | 50 | inline bool operator!=(const StoreIterator& other) const { 51 | return !(*this == other); 52 | } 53 | 54 | inline StoreIterator& operator++() { 55 | ++(*itr_); 56 | return *this; 57 | } 58 | 59 | inline StoreIterator operator++(int) { 60 | StoreIterator it = *this; 61 | ++(*this); 62 | return it; 63 | } 64 | 65 | reference operator*() const { 66 | return itr_->operator*(); 67 | } 68 | 69 | private: 70 | std::unique_ptr itr_; 71 | }; 72 | 73 | } // namespace ustore 74 | 75 | #endif // USTORE_STORE_ITERATOR_H_ 76 | -------------------------------------------------------------------------------- /include/store/ldb_store.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_STORE_LDB_STORE_H_ 4 | #define USTORE_STORE_LDB_STORE_H_ 5 | 6 | #include 7 | #include 8 | #include "leveldb/db.h" 9 | #include "chunk/chunk.h" 10 | #include "hash/hash.h" 11 | #include "store/chunk_store.h" 12 | #include "types/type.h" 13 | #include "utils/noncopyable.h" 14 | #include "utils/singleton.h" 15 | 16 | namespace ustore { 17 | 18 | class LDBStore 19 | : private Noncopyable, public Singleton, public ChunkStore { 20 | friend class Singleton; 21 | 22 | public: 23 | ~LDBStore(); 24 | /* 25 | * store allocates the returned chunck, 26 | * caller need to release memory after use. 27 | */ 28 | Chunk Get(const Hash& key) override; 29 | bool Put(const Hash& key, const Chunk& chunk) override; 30 | bool Exists(const Hash& key) override; 31 | const StoreInfo& GetInfo() override; 32 | 33 | private: 34 | LDBStore(); 35 | explicit LDBStore(const std::string& dbpath); 36 | 37 | leveldb::DB* db_; 38 | leveldb::Options opt_; 39 | leveldb::WriteOptions wr_opt_; 40 | leveldb::ReadOptions rd_opt_; 41 | }; 42 | } // namespace ustore 43 | 44 | #endif // USTORE_STORE_LDB_STORE_H_ 45 | -------------------------------------------------------------------------------- /include/store/rocks_store.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_STORE_ROCKS_STORE_H_ 4 | #define USTORE_STORE_ROCKS_STORE_H_ 5 | 6 | #include 7 | #include 8 | #include "chunk/chunk.h" 9 | #include "hash/hash.h" 10 | #include "store/chunk_store.h" 11 | #include "types/type.h" 12 | #include "utils/logging.h" 13 | #include "utils/rocksdb.h" 14 | #include "utils/singleton.h" 15 | 16 | namespace ustore { 17 | 18 | class RocksStore 19 | : public RocksDB, public Singleton, public ChunkStore { 20 | friend class Singleton; 21 | 22 | public: 23 | ~RocksStore(); 24 | 25 | Chunk Get(const Hash& key) override; 26 | bool Exists(const Hash& key) override; 27 | bool Put(const Hash& key, const Chunk& chunk) override; 28 | StoreInfo GetInfo() override; 29 | 30 | StoreIterator begin() const override; 31 | StoreIterator cbegin() const override; 32 | StoreIterator end() const override; 33 | StoreIterator cend() const override; 34 | 35 | private: 36 | RocksStore(); 37 | explicit RocksStore(const std::string& db_path, const bool persist = true); 38 | 39 | void InitStoreInfo(); 40 | void UpdateStoreInfoForNewChunk(const Chunk& chunk); 41 | 42 | bool persist_; 43 | StoreInfo store_info_; 44 | std::mutex mtx_store_info_; 45 | }; 46 | 47 | } // namespace ustore 48 | 49 | #endif // USTORE_STORE_ROCKS_STORE_H_ 50 | -------------------------------------------------------------------------------- /include/store/store_initializer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_STORE_STORE_INITIALIZER_H_ 4 | #define USTORE_STORE_STORE_INITIALIZER_H_ 5 | 6 | #include 7 | #include 8 | #include "store/chunk_store.h" 9 | #include "utils/env.h" 10 | 11 | namespace ustore { 12 | 13 | namespace fs = boost::filesystem; 14 | 15 | /* 16 | * StoreInitialier is responsible for init the chunk store singleton 17 | * in the process. It should be inherited by the first class that interact 18 | * with chunk store. 19 | */ 20 | class StoreInitializer { 21 | public: 22 | virtual ~StoreInitializer() = default; 23 | 24 | inline bool persist() { return persist_; } 25 | inline const std::string& dataDir() { return data_dir_; } 26 | inline const std::string& dataFile() { return data_file_; } 27 | inline const std::string& dataPath() { return data_path_; } 28 | 29 | protected: 30 | StoreInitializer(size_t seed, bool persist) : persist_(persist) { 31 | // create data dir 32 | data_dir_ = Env::Instance()->config().data_dir(); 33 | fs::create_directory(fs::path(data_dir_)); 34 | // set data path 35 | auto pattern = Env::Instance()->config().data_file_pattern(); 36 | data_file_ = pattern + "_" + std::to_string(seed); 37 | // init chunk store before using it 38 | store::InitChunkStore(data_dir_, data_file_, persist_); 39 | // data path for later use 40 | data_path_ = data_dir_ + "/" + data_file_; 41 | } 42 | 43 | private: 44 | bool persist_; 45 | std::string data_dir_; 46 | std::string data_file_; 47 | std::string data_path_; 48 | }; 49 | 50 | } // namespace ustore 51 | 52 | #endif // USTORE_STORE_STORE_INITIALIZER_H_ 53 | 54 | -------------------------------------------------------------------------------- /include/types/base.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_BASE_H_ 4 | #define USTORE_TYPES_BASE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "chunk/chunk_loader.h" 11 | #include "node/node.h" 12 | #include "utils/noncopyable.h" 13 | 14 | namespace ustore { 15 | 16 | // A genric type for all utypes 17 | class BaseType : private Moveable { 18 | public: 19 | virtual bool empty() const = 0; 20 | 21 | protected: 22 | BaseType() = default; 23 | BaseType(BaseType&&) = default; 24 | BaseType& operator=(BaseType&&) = default; 25 | virtual ~BaseType() = default; 26 | }; 27 | 28 | class ChunkableType : public BaseType { 29 | // A genric type for parent class 30 | // all other types shall inherit from this 31 | public: 32 | inline bool empty() const override { return root_node_.get() == nullptr; } 33 | inline const Hash& hash() const { 34 | return empty() ? Hash::kNull : root_node_->hash(); 35 | } 36 | inline uint64_t numElements() const { 37 | CHECK(!empty()); 38 | return root_node_->numElements(); 39 | } 40 | 41 | // TODO(wangsh): expose it in a safer way 42 | std::shared_ptr& GetChunkLoader() { return chunk_loader_; } 43 | 44 | protected: 45 | ChunkableType() = default; 46 | ChunkableType(ChunkableType&&) = default; 47 | ChunkableType& operator=(ChunkableType&&) = default; 48 | explicit ChunkableType(std::shared_ptr loader) noexcept : 49 | chunk_loader_(std::move(loader)) {} 50 | ~ChunkableType() = default; 51 | 52 | // Must be called at the last step of construction 53 | virtual bool SetNodeForHash(const Hash& hash) = 0; 54 | 55 | std::shared_ptr chunk_loader_; 56 | std::unique_ptr root_node_; 57 | }; 58 | 59 | } // namespace ustore 60 | 61 | #endif // USTORE_TYPES_BASE_H_ 62 | -------------------------------------------------------------------------------- /include/types/client/vblob.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VBLOB_H_ 4 | #define USTORE_TYPES_CLIENT_VBLOB_H_ 5 | 6 | #include 7 | #include "types/client/vobject.h" 8 | #include "types/ublob.h" 9 | 10 | namespace ustore { 11 | 12 | class VBlob : public UBlob, public VObject { 13 | friend class VHandler; 14 | 15 | public: 16 | VBlob() noexcept : VBlob(Slice()) {} 17 | VBlob(VBlob&&) = default; 18 | VBlob& operator=(VBlob&&) = default; 19 | // Create new VBlob 20 | explicit VBlob(const Slice& slice) noexcept; 21 | explicit VBlob(std::istream* is) noexcept; 22 | ~VBlob() = default; 23 | 24 | Hash Splice(size_t pos, size_t num_delete, const byte_t* data, 25 | size_t num_insert) const override; 26 | 27 | Hash Splice(size_t pos, size_t num_delete, std::istream* is, 28 | size_t num_insert) const; 29 | 30 | protected: 31 | // Load exsiting VBlob 32 | VBlob(std::shared_ptr loader, const Hash& root_hash) noexcept; 33 | }; 34 | 35 | } // namespace ustore 36 | 37 | #endif // USTORE_TYPES_CLIENT_VBLOB_H_ 38 | -------------------------------------------------------------------------------- /include/types/client/vhandler.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VHANDLER_H_ 4 | #define USTORE_TYPES_CLIENT_VHANDLER_H_ 5 | 6 | #include 7 | #include 8 | #include "spec/db.h" 9 | #include "types/client/vblob.h" 10 | #include "types/client/vlist.h" 11 | #include "types/client/vmap.h" 12 | #include "types/client/vset.h" 13 | #include "types/client/vstring.h" 14 | #include "utils/utils.h" 15 | 16 | namespace ustore { 17 | 18 | /* 19 | * Main entry for initializing objects in all types. 20 | */ 21 | class VHandler { 22 | public: 23 | VHandler() = default; 24 | explicit VHandler(DB* db) : db_(db) {} 25 | VHandler(DB* db, std::shared_ptr loader) 26 | : db_(db), loader_(loader) {} 27 | virtual ~VHandler() = default; 28 | 29 | inline bool empty() const { return !db_; } 30 | virtual UType type() const = 0; 31 | virtual Hash dataHash() const = 0; 32 | virtual Slice partitionKey() const = 0; 33 | 34 | // primitive types are not supported by default 35 | virtual VString String() const; 36 | 37 | // chunkable types 38 | VBlob Blob() const; 39 | VList List() const; 40 | VMap Map() const; 41 | VSet Set() const; 42 | 43 | friend std::ostream& operator<<(std::ostream& os, const VHandler& obj); 44 | 45 | private: 46 | DB* db_ = nullptr; 47 | std::shared_ptr loader_; 48 | }; 49 | 50 | } // namespace ustore 51 | 52 | #endif // USTORE_TYPES_CLIENT_VHANDLER_H_ 53 | -------------------------------------------------------------------------------- /include/types/client/vlist.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VLIST_H_ 4 | #define USTORE_TYPES_CLIENT_VLIST_H_ 5 | 6 | #include 7 | #include 8 | #include "types/client/vobject.h" 9 | #include "types/ulist.h" 10 | 11 | namespace ustore { 12 | 13 | class VList : public UList, public VObject { 14 | friend class VHandler; 15 | 16 | public: 17 | VList() noexcept : VList(std::vector()) {} 18 | VList(VList&&) = default; 19 | VList& operator=(VList&&) = default; 20 | // Create new VList 21 | explicit VList(const std::vector& elements) noexcept; 22 | ~VList() = default; 23 | 24 | // entry vector can be empty 25 | Hash Splice(uint64_t start_idx, uint64_t num_to_delete, 26 | const std::vector& entries) const override; 27 | 28 | protected: 29 | // Load an existing VList 30 | VList(std::shared_ptr, const Hash& root_hash) noexcept; 31 | }; 32 | 33 | } // namespace ustore 34 | 35 | #endif // USTORE_TYPES_CLIENT_VLIST_H_ 36 | -------------------------------------------------------------------------------- /include/types/client/vmap.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VMAP_H_ 4 | #define USTORE_TYPES_CLIENT_VMAP_H_ 5 | 6 | #include 7 | #include 8 | #include "types/client/vobject.h" 9 | #include "types/umap.h" 10 | 11 | namespace ustore { 12 | 13 | class VMap : public UMap, public VObject { 14 | friend class VHandler; 15 | 16 | public: 17 | VMap() noexcept : VMap(std::vector(), std::vector()) {} 18 | VMap(VMap&&) = default; 19 | VMap& operator=(VMap&&) = default; 20 | // Create new VMap 21 | VMap(const std::vector& keys, const std::vector& vals) noexcept; 22 | ~VMap() = default; 23 | 24 | Hash Set(const Slice& key, const Slice& val) const override; 25 | Hash Set(const std::vector& keys, 26 | const std::vector& vals) const override; 27 | Hash Remove(const Slice& key) const override; 28 | 29 | protected: 30 | // Load an existing VMap 31 | VMap(std::shared_ptr, const Hash& root_hash) noexcept; 32 | }; 33 | 34 | } // namespace ustore 35 | 36 | #endif // USTORE_TYPES_CLIENT_VMAP_H_ 37 | -------------------------------------------------------------------------------- /include/types/client/vmeta.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VMETA_H_ 4 | #define USTORE_TYPES_CLIENT_VMETA_H_ 5 | 6 | #include 7 | #include 8 | #include "types/ucell.h" 9 | #include "types/client/vhandler.h" 10 | #include "utils/noncopyable.h" 11 | 12 | namespace ustore { 13 | 14 | // TODO(wangsh): modify DB api and remove version in VMeta 15 | class VMeta : public VHandler, private Moveable { 16 | public: 17 | VMeta() = default; 18 | VMeta(DB* db, UCell&& cell) : VHandler(db), cell_(std::move(cell)) {} 19 | VMeta(DB* db, UCell&& cell, std::shared_ptr loader) 20 | : VHandler(db, loader), cell_(std::move(cell)) {} 21 | VMeta(VMeta&&) = default; 22 | VMeta& operator=(VMeta&&) = default; 23 | ~VMeta() = default; 24 | 25 | UType type() const override { return cell_.type(); } 26 | Hash dataHash() const override { return cell_.dataHash(); } 27 | Slice partitionKey() const override { return cell_.key(); } 28 | 29 | inline const UCell& cell() const { return cell_; } 30 | 31 | VString String() const override { 32 | if (!empty() && type() == UType::kString) return VString(cell_); 33 | LOG(WARNING) << "Get empty VString, actual type: " << type(); 34 | return VString(); 35 | } 36 | 37 | friend std::ostream& operator<<(std::ostream& os, const VMeta& obj) { 38 | os << static_cast(obj); 39 | return os; 40 | } 41 | 42 | private: 43 | UCell cell_; 44 | }; 45 | 46 | } // namespace ustore 47 | 48 | #endif // USTORE_TYPES_CLIENT_VMETA_H_ 49 | -------------------------------------------------------------------------------- /include/types/client/vobject.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VOBJECT_H_ 4 | #define USTORE_TYPES_CLIENT_VOBJECT_H_ 5 | 6 | #include 7 | #include "spec/value.h" 8 | 9 | namespace ustore { 10 | 11 | class VObject { 12 | public: 13 | VObject(): useStream_(false) {} 14 | virtual ~VObject() = default; 15 | 16 | bool empty() const { return buffer_.type == UType::kUnknown; } 17 | const Value& value() const { return buffer_; } 18 | std::istream* dataStream() const { return dataStream_; } 19 | bool useStream() const { return useStream_; } 20 | 21 | void SetContext(Slice ctx) { buffer_.ctx = ctx; } 22 | void Clear() { buffer_ = {}; } 23 | 24 | protected: 25 | mutable Value buffer_; 26 | mutable bool useStream_; 27 | mutable std::istream* dataStream_; 28 | }; 29 | 30 | } // namespace ustore 31 | 32 | #endif // USTORE_TYPES_CLIENT_VOBJECT_H_ 33 | -------------------------------------------------------------------------------- /include/types/client/vref.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VREF_H_ 4 | #define USTORE_TYPES_CLIENT_VREF_H_ 5 | 6 | #include 7 | #include 8 | #include "hash/hash.h" 9 | #include "spec/slice.h" 10 | #include "types/type.h" 11 | #include "types/client/vhandler.h" 12 | 13 | namespace ustore { 14 | 15 | class VRef : public VHandler { 16 | public: 17 | VRef() = default; 18 | VRef(DB* db, Slice route_key, UType type, Hash hash) 19 | : VHandler(db), route_key_(route_key), type_(type), hash_(hash) {} 20 | VRef(DB* db, Slice route_key, UType type, Hash hash, 21 | std::shared_ptr loader) 22 | : VHandler(db, loader), route_key_(route_key), type_(type), hash_(hash) {} 23 | ~VRef() = default; 24 | 25 | UType type() const override { return type_; } 26 | Hash dataHash() const override { return hash_; } 27 | Slice partitionKey() const override { return route_key_; } 28 | 29 | friend std::ostream& operator<<(std::ostream& os, const VRef& obj) { 30 | os << static_cast(obj); 31 | return os; 32 | } 33 | 34 | private: 35 | Slice route_key_; 36 | UType type_; 37 | Hash hash_; 38 | }; 39 | 40 | } // namespace ustore 41 | 42 | #endif // USTORE_TYPES_CLIENT_VREF_H_ 43 | -------------------------------------------------------------------------------- /include/types/client/vset.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VSET_H_ 4 | #define USTORE_TYPES_CLIENT_VSET_H_ 5 | 6 | #include 7 | #include 8 | #include "types/client/vobject.h" 9 | #include "types/uset.h" 10 | 11 | namespace ustore { 12 | 13 | class VSet : public USet, public VObject { 14 | friend class VHandler; 15 | 16 | public: 17 | VSet() noexcept : VSet(std::vector()) {} 18 | VSet(VSet&&) = default; 19 | VSet& operator=(VSet&&) = default; 20 | // Create new VSet 21 | explicit VSet(const std::vector& keys) noexcept; 22 | ~VSet() = default; 23 | 24 | Hash Set(const Slice& key) const override; 25 | Hash Remove(const Slice& key) const override; 26 | 27 | protected: 28 | // Load an existing VSet 29 | VSet(std::shared_ptr, const Hash& root_hash) noexcept; 30 | }; 31 | 32 | } // namespace ustore 33 | 34 | #endif // USTORE_TYPES_CLIENT_VSET_H_ 35 | -------------------------------------------------------------------------------- /include/types/client/vstring.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_CLIENT_VSTRING_H_ 4 | #define USTORE_TYPES_CLIENT_VSTRING_H_ 5 | 6 | #include 7 | #include "types/client/vobject.h" 8 | #include "types/ustring.h" 9 | 10 | namespace ustore { 11 | 12 | class VString : public UString, public VObject { 13 | friend class VHandler; 14 | friend class VMeta; 15 | 16 | public: 17 | VString() noexcept : VString(Slice()) {} 18 | VString(VString&&) = default; 19 | VString& operator=(VString&&) = default; 20 | // Create new VString 21 | explicit VString(const Slice& slice) noexcept; 22 | ~VString() = default; 23 | 24 | protected: 25 | // Load existing VString 26 | explicit VString(const UCell& cell) noexcept; 27 | }; 28 | 29 | } // namespace ustore 30 | 31 | #endif // USTORE_TYPES_CLIENT_VSTRING_H_ 32 | -------------------------------------------------------------------------------- /include/types/server/factory.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_FACTORY_H_ 4 | #define USTORE_TYPES_SERVER_FACTORY_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "cluster/chunk_client_service.h" 11 | #include "cluster/partitioner.h" 12 | #include "types/server/sblob.h" 13 | #include "types/server/slist.h" 14 | #include "types/server/smap.h" 15 | #include "utils/noncopyable.h" 16 | 17 | namespace ustore { 18 | 19 | class ChunkableTypeFactory : private Noncopyable { 20 | public: 21 | // local chunkable types 22 | ChunkableTypeFactory() : ChunkableTypeFactory(nullptr) {} 23 | // distributed chunkable types 24 | explicit ChunkableTypeFactory(const Partitioner* ptt) : ptt_(ptt) { 25 | if (ptt_) { 26 | // start chunk client service 27 | client_svc_.Run(); 28 | cli_.push_back(client_svc_.CreateChunkClient()); 29 | writer_.reset(new PartitionedChunkWriter(ptt, &cli_[0])); 30 | } else { 31 | writer_.reset(new LocalChunkWriter()); 32 | } 33 | } 34 | ~ChunkableTypeFactory() = default; 35 | 36 | // Load exsiting SObject 37 | template 38 | T Load(const Hash& root_hash) { 39 | return T(loader(), writer(), root_hash); 40 | } 41 | 42 | // Create new SObject 43 | template 44 | T Create(Args&&... args) { 45 | return T(loader(), writer(), std::forward(args)...); 46 | } 47 | 48 | inline std::shared_ptr loader() { 49 | if (ptt_) 50 | return std::make_shared(ptt_, &cli_[0]); 51 | else 52 | return std::make_shared(); 53 | } 54 | 55 | inline ChunkWriter* writer() { return writer_.get(); } 56 | 57 | private: 58 | const Partitioner* const ptt_; 59 | std::unique_ptr writer_; // chunk writer is shared 60 | ChunkClientService client_svc_; 61 | std::vector cli_; 62 | }; 63 | 64 | } // namespace ustore 65 | #endif // USTORE_TYPES_SERVER_FACTORY_H_ 66 | -------------------------------------------------------------------------------- /include/types/server/sblob.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_SBLOB_H_ 4 | #define USTORE_TYPES_SERVER_SBLOB_H_ 5 | 6 | #include 7 | 8 | #include "chunk/chunk_loader.h" 9 | #include "chunk/chunk_writer.h" 10 | #include "types/ublob.h" 11 | 12 | namespace ustore { 13 | 14 | class SBlob : public UBlob { 15 | friend class ChunkableTypeFactory; 16 | public: 17 | SBlob(SBlob&&) = default; 18 | SBlob& operator=(SBlob&&) = default; 19 | ~SBlob() = default; 20 | 21 | Hash Splice(size_t pos, size_t num_delete, 22 | const byte_t* data, size_t num_insert) const override; 23 | 24 | protected: 25 | // Load exsiting SBlob 26 | SBlob(std::shared_ptr loader, ChunkWriter* writer, 27 | const Hash& root_hash) noexcept; 28 | // Create new SBlob 29 | SBlob(std::shared_ptr loader, ChunkWriter* writer, 30 | const Slice& slice) noexcept; 31 | 32 | private: 33 | ChunkWriter* chunk_writer_; 34 | }; 35 | 36 | } // namespace ustore 37 | #endif // USTORE_TYPES_SERVER_SBLOB_H_ 38 | -------------------------------------------------------------------------------- /include/types/server/slist.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_SLIST_H_ 4 | #define USTORE_TYPES_SERVER_SLIST_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "chunk/chunk_loader.h" 10 | #include "chunk/chunk_writer.h" 11 | #include "types/ulist.h" 12 | 13 | namespace ustore { 14 | 15 | class SList : public UList { 16 | friend class ChunkableTypeFactory; 17 | public: 18 | SList(SList&&) = default; 19 | SList& operator=(SList&&) = default; 20 | ~SList() = default; 21 | 22 | // entry vector can be empty 23 | Hash Splice(uint64_t start_idx, uint64_t num_to_delete, 24 | const std::vector& entries) const override; 25 | 26 | // Use this List as base to perform three-way merging 27 | // return empty hash when merging fails 28 | Hash Merge(const SList& node1, const SList& node2) const; 29 | 30 | protected: 31 | // Load existing SList 32 | SList(std::shared_ptr loader, ChunkWriter* writer, 33 | const Hash& root_hash) noexcept; 34 | // create new SList 35 | SList(std::shared_ptr loader, ChunkWriter* writer, 36 | const std::vector& elements) noexcept; 37 | 38 | private: 39 | ChunkWriter* chunk_writer_; 40 | }; 41 | } // namespace ustore 42 | 43 | #endif // USTORE_TYPES_SERVER_SLIST_H_ 44 | -------------------------------------------------------------------------------- /include/types/server/smap.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_SMAP_H_ 4 | #define USTORE_TYPES_SERVER_SMAP_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "chunk/chunk_writer.h" 10 | #include "types/umap.h" 11 | 12 | namespace ustore { 13 | 14 | class SMap : public UMap { 15 | friend class ChunkableTypeFactory; 16 | public: 17 | SMap(SMap&&) = default; 18 | SMap& operator=(SMap&&) = default; 19 | ~SMap() = default; 20 | 21 | // Both Use chunk builder to do splice 22 | // this kv_items must be sorted in descending order before 23 | Hash Set(const Slice& key, const Slice& val) const override; 24 | Hash Set(const std::vector& keys, 25 | const std::vector& vals) const override; 26 | Hash Remove(const Slice& key) const override; 27 | 28 | // Use this map as base to perform three-way merging 29 | // return empty hash when merging fails 30 | Hash Merge(const SMap& node1, const SMap& node2) const; 31 | 32 | protected: 33 | // Load existing SMap 34 | SMap(std::shared_ptr loader, ChunkWriter* writer, 35 | const Hash& root_hash) noexcept; 36 | // Create new SMap 37 | // kv_items must be sorted in strict ascending order based on key 38 | SMap(std::shared_ptr loader, ChunkWriter* writer, 39 | const std::vector& keys, const std::vector& vals) noexcept; 40 | 41 | private: 42 | ChunkWriter* chunk_writer_; 43 | }; 44 | 45 | } // namespace ustore 46 | 47 | #endif // USTORE_TYPES_SERVER_SMAP_H_ 48 | -------------------------------------------------------------------------------- /include/types/server/sset.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_SSET_H_ 4 | #define USTORE_TYPES_SERVER_SSET_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "chunk/chunk_writer.h" 10 | #include "types/uset.h" 11 | 12 | namespace ustore { 13 | 14 | class SSet : public USet { 15 | friend class ChunkableTypeFactory; 16 | public: 17 | SSet(SSet&&) = default; 18 | SSet& operator=(SSet&&) = default; 19 | ~SSet() = default; 20 | 21 | // Both Use chunk builder to do splice 22 | // this kv_items must be sorted in descending order before 23 | Hash Set(const Slice& key) const override; 24 | Hash Remove(const Slice& key) const override; 25 | 26 | // Use this Set as base to perform three-way merging 27 | // return empty hash when merging fails 28 | Hash Merge(const SSet& node1, const SSet& node2) const; 29 | 30 | protected: 31 | // Load existing SSet 32 | SSet(std::shared_ptr loader, ChunkWriter* writer, 33 | const Hash& root_hash) noexcept; 34 | // Create new SSet 35 | // kv_items must be sorted in strict ascending order based on key 36 | SSet(std::shared_ptr loader, ChunkWriter* writer, 37 | const std::vector& keys) noexcept; 38 | 39 | private: 40 | ChunkWriter* chunk_writer_; 41 | }; 42 | 43 | } // namespace ustore 44 | 45 | #endif // USTORE_TYPES_SERVER_SSET_H_ 46 | -------------------------------------------------------------------------------- /include/types/server/sstring.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_SERVER_SSTRING_H_ 4 | #define USTORE_TYPES_SERVER_SSTRING_H_ 5 | 6 | #include "types/ustring.h" 7 | 8 | namespace ustore { 9 | 10 | class SString : public UString { 11 | public: 12 | // Load existing VString 13 | explicit SString(const UCell& cell) noexcept : UString(cell) {} 14 | }; // namespace ustore 15 | 16 | } // namespace ustore 17 | 18 | #endif // USTORE_TYPES_SERVER_SSTRING_H_ 19 | -------------------------------------------------------------------------------- /include/types/ustring.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_TYPES_USTRING_H_ 4 | #define USTORE_TYPES_USTRING_H_ 5 | 6 | #include 7 | 8 | #include "types/base.h" 9 | #include "types/ucell.h" 10 | 11 | namespace ustore { 12 | 13 | class UString : public BaseType { 14 | public: 15 | inline bool empty() const override { return node_.get() == nullptr; } 16 | inline size_t len() const { return node_->dataLength(); } 17 | inline const byte_t* data() const { return node_->data(); } 18 | inline Slice slice() const { return Slice(data(), len()); } 19 | 20 | friend inline std::ostream& operator<<(std::ostream& os, const UString& obj) { 21 | os << obj.slice(); 22 | return os; 23 | } 24 | 25 | protected: 26 | UString() = default; 27 | UString(UString&&) = default; 28 | UString& operator=(UString&&) = default; 29 | explicit UString(const UCell& cell) noexcept : node_(cell.node()) {} 30 | ~UString() = default; 31 | 32 | protected: 33 | // Responsible to remove during destructing 34 | std::shared_ptr node_; 35 | }; 36 | 37 | } // namespace ustore 38 | #endif // USTORE_TYPES_USTRING_H_ 39 | -------------------------------------------------------------------------------- /include/utils/blocking_queue.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The UStore Authors. 2 | 3 | #ifndef USTORE_UTILS_BLOCKING_QUEUE_H_ 4 | #define USTORE_UTILS_BLOCKING_QUEUE_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "utils/noncopyable.h" 11 | 12 | namespace ustore { 13 | 14 | template 15 | class BlockingQueue : private Noncopyable { 16 | public: 17 | explicit BlockingQueue(size_t capacity) : capacity_(capacity) {} 18 | ~BlockingQueue() = default; 19 | 20 | void Put(const T& x) { 21 | { 22 | std::lock_guard lock(mutex_); 23 | while (queue_.size() == capacity_) cv_not_full_.wait(mutex_); 24 | queue_.emplace_back(x); 25 | } 26 | cv_not_empty_.notify_one(); 27 | } 28 | 29 | void Put(const T&& x) { 30 | { 31 | std::lock_guard lock(mutex_); 32 | cv_not_full_.wait(mutex_, [this] { return queue_.size() != capacity_; }); 33 | queue_.push_back(std::move(x)); 34 | } 35 | cv_not_empty_.notify_one(); 36 | } 37 | 38 | T Take() { 39 | std::lock_guard lock(mutex_); 40 | cv_not_empty_.wait(mutex_, [this] { return !queue_.empty(); }); 41 | auto x = std::move(queue_.front()); 42 | queue_.pop_front(); 43 | cv_not_full_.notify_one(); 44 | return x; 45 | } 46 | 47 | size_t size() const { 48 | std::lock_guard lock(mutex_); 49 | return queue_.size(); 50 | } 51 | 52 | size_t capacity() const { return capacity_; } 53 | 54 | private: 55 | const size_t capacity_; 56 | std::list queue_; 57 | std::mutex mutex_; 58 | std::condition_variable_any cv_not_empty_, cv_not_full_; 59 | }; 60 | 61 | } // namespace ustore 62 | 63 | #endif // USTORE_UTILS_BLOCKING_QUEUE_H_ 64 | -------------------------------------------------------------------------------- /include/utils/chars.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_CHARS_H_ 4 | #define USTORE_UTILS_CHARS_H_ 5 | 6 | #include 7 | 8 | #include "utils/type_traits.h" 9 | #include "utils/logging.h" 10 | 11 | static inline size_t AppendInteger(char* buf) { return 0; } 12 | static inline size_t ReadInteger(const char* buf) { return 0; } 13 | 14 | template > 16 | static size_t AppendInteger(char* buf, Type1 value, Types ... values) { 17 | // make sure alignment 18 | CHECK_EQ((uintptr_t)buf % sizeof(Type1), Type1(0)); 19 | *(reinterpret_cast(buf)) = value; 20 | return sizeof(Type1) + AppendInteger(buf + sizeof(Type1), values...); 21 | } 22 | 23 | template > 25 | static size_t ReadInteger(const char* buf, Type1& value, Types&... values) { 26 | // make sure alignment 27 | CHECK_EQ((uintptr_t)buf % sizeof(Type1), Type1(0)); 28 | value = *reinterpret_cast(const_cast(buf)); 29 | return sizeof(Type1) 30 | + ReadInteger(const_cast(buf) + sizeof(Type1), values...); 31 | } 32 | 33 | template 34 | static size_t ReadInteger(const char* buf, T* array); 35 | 36 | template 37 | static size_t ReadInteger(const char* buf, T (&array)[N] ) { 38 | return ReadInteger(buf, reinterpret_cast(array)); 39 | } 40 | 41 | template 42 | static size_t ReadInteger(const char* buf, T* array, std::true_type) { 43 | return ReadInteger(buf, array) 44 | + ReadInteger(buf + sizeof(T), array + 1); 45 | } 46 | 47 | template 48 | static size_t ReadInteger(const char* buf, T* array, std::false_type) { 49 | return ReadInteger(buf, *array); 50 | } 51 | 52 | template 53 | static size_t ReadInteger(const char* buf, T* array) { 54 | return ReadInteger(buf, array, std::integral_constant1)>()); 55 | } 56 | 57 | #endif // USTORE_UTILS_CHARS_H_ 58 | -------------------------------------------------------------------------------- /include/utils/debug.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_DEBUG_H_ 4 | #define USTORE_UTILS_DEBUG_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "types/type.h" 10 | 11 | namespace ustore { 12 | 13 | std::string byte2str(const byte_t* data, size_t num_bytes); 14 | 15 | // A utility method to splice bytes array, delete @arg num_delete bytes, 16 | // from @arg start, then insert @arg append_size number of bytes from 17 | // input buffer @arg append. This method will not rewrite the original 18 | // @arg src but copy the content of it. Caller of this method is responsible 19 | // for delete of the created buffer. 20 | const ustore::byte_t* SpliceBytes(const ustore::byte_t* src, size_t src_size, 21 | size_t start, size_t num_delete, 22 | const ustore::byte_t* append, size_t append_size); 23 | 24 | // Perform multiple splice operation on the original content 25 | // idxs is a vector of starting index of original content 26 | // to perform splice on. Must be in strict ascending order. 27 | const ustore::byte_t* MultiSplice( 28 | const ustore::byte_t* original_content, 29 | size_t original_num_bytes, std::vector idxs, 30 | std::vector num_bytes_remove, 31 | std::vector inserted_data, 32 | std::vector inserted_num_bytes, 33 | size_t* result_num_bytes); 34 | 35 | } // namespace ustore 36 | #endif // USTORE_UTILS_DEBUG_H_ 37 | -------------------------------------------------------------------------------- /include/utils/enum.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #ifndef USTORE_UTILS_ENUM_H_ 3 | #define USTORE_UTILS_ENUM_H_ 4 | 5 | #include 6 | 7 | #include "utils/type_traits.h" 8 | 9 | namespace ustore { 10 | 11 | template ::value>> 12 | struct Enum { 13 | class Iterator : public std::iterator { 14 | public: 15 | explicit Iterator(int pos) : pos_(pos) {} 16 | 17 | Iterator& operator++() { 18 | ++pos_; 19 | return *this; 20 | } 21 | 22 | Iterator& operator++(int) { 23 | Iterator it = *this; 24 | ++(*this); 25 | return it; 26 | } 27 | 28 | T operator*() const { 29 | return (T)pos_; 30 | } 31 | 32 | Iterator(const Iterator&) noexcept = default; 33 | Iterator& operator=(const Iterator&) noexcept = default; 34 | bool operator==(const Iterator& other) const { return pos_ == other.pos_; } 35 | bool operator!=(const Iterator& other) const { return !(*this == other); } 36 | 37 | private: 38 | int pos_; 39 | }; 40 | }; 41 | 42 | template ::value>> 43 | typename Enum::Iterator begin(Enum) { 44 | return typename Enum::Iterator(static_cast(T::First)); 45 | } 46 | 47 | template ::value>> 48 | typename Enum::Iterator end(Enum) { 49 | return typename Enum::Iterator(static_cast(T::Last) + 1); 50 | } 51 | 52 | } // namespace ustore 53 | 54 | #endif // USTORE_UTILS_ENUM_H_ 55 | -------------------------------------------------------------------------------- /include/utils/env.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_ENV_H_ 4 | #define USTORE_UTILS_ENV_H_ 5 | 6 | #include "proto/config.pb.h" 7 | #include "utils/singleton.h" 8 | 9 | namespace ustore { 10 | 11 | /* 12 | * USTORE environment 13 | * This is the place place for storing global configuration 14 | * parameters. 15 | */ 16 | 17 | class Env : public Singleton, private Noncopyable { 18 | friend class Singleton; 19 | public: 20 | static const char* kDefaultConfigFile; 21 | 22 | const Config& config() const { return config_; } 23 | Config& m_config() { return config_; } 24 | 25 | private: 26 | Env(); 27 | ~Env() = default; 28 | 29 | Config config_; 30 | }; 31 | 32 | } // namespace ustore 33 | #endif // USTORE_UTILS_ENV_H_ 34 | -------------------------------------------------------------------------------- /include/utils/map_check_policy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_MAP_CHECK_POLICY_H_ 4 | #define USTORE_UTILS_MAP_CHECK_POLICY_H_ 5 | 6 | #include 7 | #include "utils/type_traits.h" 8 | 9 | namespace ustore { 10 | 11 | template 12 | class NoCheckPolicy { 13 | protected: 14 | using key_type = typename MapType::key_type; 15 | // to avoid the construction of a temporary key_type object 16 | template ::value>> 18 | constexpr bool check(const MapType&, T&&...) noexcept { 19 | return true; 20 | } 21 | 22 | constexpr bool check(const MapType& map, const key_type&) noexcept { 23 | return true; 24 | } 25 | }; 26 | 27 | template 28 | class CheckExistPolicy { 29 | protected: 30 | using key_type = typename MapType::key_type; 31 | // to avoid the construction of a temporary key_type object 32 | template ::value>> 34 | constexpr bool check(const MapType& map, T&&... keyArgs) { 35 | return (map.end() != map.find(key_type{std::forward(keyArgs)...})); 36 | } 37 | 38 | template >, 41 | remove_cv_t>>::value>> 42 | constexpr bool check(const MapType& map, T&& key) { 43 | return (map.end() != map.find(std::forward(key))); 44 | } 45 | }; 46 | 47 | } // namespace ustore 48 | 49 | #endif // USTORE_UTILS_MAP_CHECK_POLICY_H_ 50 | -------------------------------------------------------------------------------- /include/utils/message_parser.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The UStore Authors. 2 | 3 | #ifndef USTORE_UTILS_MESSAGE_PARSER_H_ 4 | #define USTORE_UTILS_MESSAGE_PARSER_H_ 5 | 6 | #include 7 | #include 8 | #include "proto/messages.pb.h" 9 | #include "utils/logging.h" 10 | 11 | namespace ustore { 12 | 13 | class MessageParser { 14 | public: 15 | // TODO(wangsh): handle large messages in a robust way 16 | static bool Parse(const void* msg, int size, UMessage* umsg) { 17 | // parse small message (< 60M) 18 | if (size < (60 << 20)) { 19 | return umsg->ParseFromArray(msg, size); 20 | } 21 | // parse large message (< 2GB) 22 | ::google::protobuf::io::ArrayInputStream buf(msg, size); 23 | ::google::protobuf::io::CodedInputStream decoder(&buf); 24 | decoder.SetTotalBytesLimit((1L << 31) - 1, 1L << 30); 25 | bool success = umsg->ParseFromCodedStream(&decoder) 26 | && decoder.ConsumedEntireMessage(); 27 | if (!success) LOG(ERROR) << "Fail to consume entire message"; 28 | return success; 29 | } 30 | }; 31 | 32 | } // namespace ustore 33 | 34 | #endif // USTORE_UTILS_MESSAGE_PARSER_H_ 35 | -------------------------------------------------------------------------------- /include/utils/noncopyable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_NONCOPYABLE_H_ 4 | #define USTORE_UTILS_NONCOPYABLE_H_ 5 | 6 | namespace ustore { 7 | 8 | struct Noncopyable { 9 | Noncopyable() = default; 10 | ~Noncopyable() = default; 11 | Noncopyable(const Noncopyable&) = delete; 12 | Noncopyable& operator=(const Noncopyable&) = delete; 13 | }; 14 | 15 | struct Moveable : private Noncopyable { 16 | Moveable() = default; 17 | ~Moveable() = default; 18 | Moveable(Moveable&&) {} 19 | Moveable& operator=(Moveable&&) { return *this; } 20 | }; 21 | 22 | } // namespace ustore 23 | #endif // USTORE_UTILS_NONCOPYABLE_H_ 24 | -------------------------------------------------------------------------------- /include/utils/shared_lock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_SHARED_LOCK_H_ 4 | #define USTORE_UTILS_SHARED_LOCK_H_ 5 | 6 | #if __cplusplus >= 201402L 7 | #include 8 | #else 9 | #include 10 | #endif 11 | 12 | namespace ustore { 13 | #if __cplusplus == 201402L // c++ 14 14 | using shared_mutex = std::shared_timed_mutex; 15 | template 16 | using shared_lock = std::shared_lock; 17 | #elif __cplusplus > 201402L // c++ 17 18 | using shared_mutex = std::shared_mutex; 19 | template 20 | using shared_lock = std::shared_lock; 21 | #else // using boost::shared_mutex 22 | using shared_mutex = boost::shared_mutex; 23 | template 24 | using shared_lock = boost::shared_lock; 25 | #endif 26 | } // namespace ustore 27 | 28 | #endif // USTORE_UTILS_SHARED_LOCK_H_ 29 | -------------------------------------------------------------------------------- /include/utils/singleton.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_SINGLETON_H_ 4 | #define USTORE_UTILS_SINGLETON_H_ 5 | 6 | #include 7 | 8 | #include "utils/thread_model.h" 9 | 10 | namespace ustore { 11 | 12 | // For thread safety, pass ClassLevelLockable 13 | // as the template parameter 14 | template class ThreadedPolicy = SingleThreaded> 16 | class Singleton : private ThreadedPolicy { 17 | public: 18 | // Get instance 19 | // or init it without constructor parameter 20 | static T* Instance() { 21 | return data_ == nullptr ? MakeSingleton() : data_; 22 | } 23 | 24 | // Init instance explicitly with constructor parameters 25 | template 26 | static T* MakeSingleton(Args&&... args) { 27 | typename ThreadedPolicy::Lock lock(); 28 | // double check 29 | if (data_ == nullptr) { 30 | static T data{std::forward(args)...}; 31 | data_ = &data; 32 | } 33 | return data_; 34 | } 35 | 36 | private: 37 | static T* data_; 38 | }; 39 | 40 | template class ThreadedPolicy> 42 | T* Singleton::data_ = nullptr; 43 | } // namespace ustore 44 | 45 | #endif // USTORE_UTILS_SINGLETON_H_ 46 | -------------------------------------------------------------------------------- /include/utils/sync_task_line.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_SYNC_TASK_LINE_H_ 4 | #define USTORE_UTILS_SYNC_TASK_LINE_H_ 5 | 6 | #include 7 | #include 8 | #include "utils/blocking_queue.h" 9 | 10 | namespace ustore { 11 | 12 | template 13 | class SyncTaskLine { 14 | public: 15 | virtual T2 Consume(const T1& data) = 0; 16 | virtual bool Terminate(const T1& data) = 0; 17 | 18 | SyncTaskLine() 19 | : queue_(new BlockingQueue(1)), 20 | mtx_(new std::mutex()), 21 | cv_(new std::condition_variable()), 22 | processed_(false) {} 23 | 24 | ~SyncTaskLine() { 25 | delete queue_; 26 | delete mtx_; 27 | delete cv_; 28 | } 29 | 30 | inline std::thread Launch() { 31 | return std::thread([this] { Run(); }); 32 | } 33 | 34 | void Produce(const T1& data) { 35 | UnsetProcessed(); 36 | queue_->Put(data); 37 | } 38 | 39 | void Produce(const T1&& data) { 40 | UnsetProcessed(); 41 | queue_->Put(std::move(data)); 42 | } 43 | 44 | T2 Sync() { 45 | std::unique_lock lock(*mtx_); 46 | cv_->wait(lock, [this] { return processed_; }); 47 | return stat_; 48 | } 49 | 50 | private: 51 | inline void UnsetProcessed() { 52 | std::lock_guard lock(*mtx_); 53 | processed_ = false; 54 | } 55 | 56 | void Run(); 57 | 58 | ustore::BlockingQueue* queue_; 59 | std::mutex* mtx_; 60 | std::condition_variable* cv_; 61 | bool processed_; 62 | T2 stat_; 63 | }; 64 | 65 | template 66 | void SyncTaskLine::Run() { 67 | T1 data; 68 | do { 69 | data = queue_->Take(); 70 | { 71 | std::lock_guard lock(*mtx_); 72 | stat_ = Consume(data); 73 | processed_ = true; 74 | } 75 | cv_->notify_one(); 76 | } while (!Terminate(data)); 77 | } 78 | 79 | } // namespace ustore 80 | 81 | #endif // USTORE_UTILS_SYNC_TASK_LINE_H_ 82 | -------------------------------------------------------------------------------- /include/utils/thread_model.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_THREAD_MODEL_H_ 4 | #define USTORE_UTILS_THREAD_MODEL_H_ 5 | 6 | #include 7 | #include "utils/noncopyable.h" 8 | 9 | namespace ustore { 10 | 11 | template 12 | struct SingleThreaded { 13 | protected: 14 | SingleThreaded() = default; 15 | 16 | struct Lock : private Noncopyable { 17 | Lock() = default; 18 | explicit Lock(const SingleThreaded&) : Lock() {} 19 | explicit Lock(const SingleThreaded*) : Lock() {} 20 | ~Lock() = default; 21 | }; 22 | }; 23 | 24 | template 25 | struct ObjectLevelLockable { 26 | protected: 27 | ObjectLevelLockable() = default; 28 | 29 | struct Lock : private Noncopyable { 30 | // should be noncopyable; otherwise deadlock occurs in copy ctor 31 | // and assignment operator 32 | Lock() = delete; 33 | explicit Lock(const ObjectLevelLockable& host) : host_(host) { 34 | host_.mtx_.lock(); 35 | } 36 | explicit Lock(const ObjectLevelLockable* host) : Lock(*host) {} 37 | ~Lock() { host_.mtx_.unlock(); } 38 | 39 | private: 40 | const ObjectLevelLockable& host_; 41 | }; 42 | 43 | private: 44 | mutable std::mutex mtx_; 45 | friend class Lock; 46 | }; 47 | 48 | template 49 | struct ClassLevelLockable { 50 | protected: 51 | ClassLevelLockable() = default; 52 | 53 | struct Lock : private Noncopyable { 54 | // should be noncopyable; otherwise deadlock occurs in copy ctor 55 | // and assignment operator 56 | Lock() { ClassLevelLockable::mtx_.lock(); } 57 | explicit Lock(const ClassLevelLockable& host) : Lock() {} 58 | explicit Lock(const ClassLevelLockable* host) : Lock(*host) {} 59 | ~Lock() { ClassLevelLockable::mtx_.unlock(); } 60 | }; 61 | 62 | private: 63 | static std::mutex mtx_; 64 | friend class Lock; 65 | }; 66 | 67 | template 68 | std::mutex ClassLevelLockable::mtx_; 69 | 70 | } // namespace ustore 71 | #endif // USTORE_UTILS_THREAD_MODEL_H_ 72 | -------------------------------------------------------------------------------- /include/utils/type_traits.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_UTILS_TYPE_TRAITS_H_ 4 | #define USTORE_UTILS_TYPE_TRAITS_H_ 5 | 6 | #include 7 | 8 | namespace ustore { 9 | 10 | template 11 | struct Int2Type{ 12 | enum {value = N}; 13 | }; 14 | 15 | template 16 | struct is_char{ 17 | constexpr static bool value = false; 18 | }; 19 | 20 | template <> 21 | struct is_char{ 22 | constexpr static bool value = true; 23 | }; 24 | 25 | template <> 26 | struct is_char{ 27 | constexpr static bool value = true; 28 | }; 29 | 30 | template <> 31 | struct is_char{ 32 | constexpr static bool value = true; 33 | }; 34 | 35 | template 36 | using enable_if_t = typename std::enable_if::type; 37 | 38 | template 39 | using remove_reference_t = typename ::std::remove_reference::type; 40 | 41 | template 42 | using remove_cv_t = typename ::std::remove_cv::type; 43 | 44 | template 45 | using is_integral_t 46 | = typename ::ustore::enable_if_t::value, T>; 47 | 48 | template 49 | using not_integral_t 50 | = typename ::ustore::enable_if_t::value, T>; 51 | 52 | template 53 | using is_char_t 54 | = typename ::ustore::enable_if_t<::ustore::is_char::value, T>; 55 | 56 | template 57 | using not_char_t 58 | = typename ::ustore::enable_if_t::value, T>; 59 | 60 | } // namespace ustore 61 | 62 | #endif // USTORE_UTILS_TYPE_TRAITS_H_ 63 | -------------------------------------------------------------------------------- /include/utils/uniform_random.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_UNIFORM_RANDOM_H_ 2 | #define UTILS_UNIFORM_RANDOM_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace uio { 10 | namespace utils { 11 | 12 | class UniformRandom { 13 | public: 14 | UniformRandom(); 15 | ~UniformRandom() = default; 16 | 17 | std::string FixedString(int length); 18 | 19 | std::vector NFixedString(int size, int length); 20 | 21 | std::string RandomString(int max_length); 22 | 23 | std::vector NRandomString(int size, int max_length); 24 | 25 | std::vector SequentialNumString( 26 | const std::string& prefix, int size); 27 | 28 | std::vector 29 | PrefixSeqString(const std::string& prefix, 30 | int size, 31 | int mod); 32 | 33 | std::vector 34 | PrefixRandString(const std::string& prefix, 35 | int size, 36 | int mod); 37 | 38 | inline int RandomInt(const int min, 39 | const int max) { 40 | std::uniform_int_distribution<> dist(min, max); 41 | return dist(engine_); 42 | } 43 | 44 | inline bool RandomBool() { 45 | static std::uniform_int_distribution<> dist(0, 1); 46 | return (dist(engine_) == 0); 47 | } 48 | 49 | template 50 | inline void Shuffle(std::vector* elems) { 51 | std::shuffle(elems->begin(), elems->end(), engine_); 52 | } 53 | 54 | const char* RandomWeekDay(); 55 | const char* RandomWeekDayAbbr(); 56 | 57 | inline const char* RandomGender() { 58 | return (RandomBool() ? "Male" : "Female"); 59 | } 60 | 61 | inline const char* RandomGenderAbbr() { 62 | return (RandomBool() ? "M" : "F"); 63 | } 64 | 65 | private: 66 | std::default_random_engine engine_; 67 | std::uniform_int_distribution<> alph_dist_; 68 | }; 69 | 70 | } // namespace utils 71 | 72 | // pull names to the uio namespace 73 | using utils::UniformRandom; 74 | 75 | } // namespace uio 76 | 77 | #endif // UTILS_UNIFORM_RANDOM_H_ 78 | -------------------------------------------------------------------------------- /include/worker/head_version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_WORKER_HEAD_VERSION_H_ 4 | #define USTORE_WORKER_HEAD_VERSION_H_ 5 | 6 | #include 7 | #include 8 | #include "hash/hash.h" 9 | #include "spec/slice.h" 10 | #include "utils/noncopyable.h" 11 | 12 | namespace ustore { 13 | 14 | class HeadVersion : private Noncopyable { 15 | public: 16 | // TODO(yaochang): persist the log of branch update. 17 | virtual void LogBranchUpdate(const Slice& key, const Slice& branch, 18 | const Hash& ver) const {} 19 | 20 | virtual bool Load(const std::string& persist_path) = 0; 21 | 22 | virtual bool Dump(const std::string& persist_path) = 0; 23 | 24 | virtual bool GetBranch(const Slice& key, const Slice& branch, 25 | Hash* ver) const = 0; 26 | 27 | virtual std::vector GetLatest(const Slice& key) const = 0; 28 | 29 | virtual void PutBranch(const Slice& key, const Slice& branch, 30 | const Hash& ver) = 0; 31 | 32 | virtual void PutLatest(const Slice& key, const Hash& prev_ver1, 33 | const Hash& prev_ver2, const Hash& ver) = 0; 34 | 35 | virtual void RemoveBranch(const Slice& key, const Slice& branch) = 0; 36 | 37 | virtual void RenameBranch(const Slice& key, const Slice& old_branch, 38 | const Slice& new_branch) = 0; 39 | 40 | virtual bool Exists(const Slice& key) const = 0; 41 | 42 | virtual bool Exists(const Slice& key, const Slice& branch) const = 0; 43 | 44 | virtual bool IsLatest(const Slice& key, const Hash& ver) const = 0; 45 | 46 | virtual bool IsBranchHead(const Slice& key, const Slice& branch, 47 | const Hash& ver) const = 0; 48 | 49 | virtual std::vector ListKey() const = 0; 50 | 51 | virtual std::vector ListBranch(const Slice& key) const = 0; 52 | }; 53 | 54 | } // namespace ustore 55 | 56 | #endif // USTORE_WORKER_HEAD_VERSION_H_ 57 | -------------------------------------------------------------------------------- /include/worker/worker_ext.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #ifndef USTORE_WORKER_WORKER_EXT_H_ 4 | #define USTORE_WORKER_WORKER_EXT_H_ 5 | 6 | #include 7 | #include "worker/worker.h" 8 | 9 | namespace ustore { 10 | 11 | class WorkerExt : public Worker { 12 | public: 13 | explicit WorkerExt(const WorkerID& id, const Partitioner* ptt, bool persist) 14 | : Worker(id, ptt, persist) {} 15 | ~WorkerExt() = default; 16 | 17 | ErrorCode GetForType(const UType& type, const Slice& key, 18 | const Hash& ver, UCell* ucell); 19 | 20 | inline ErrorCode GetForType(const UType& type, const Slice& key, 21 | const Slice& branch, UCell* ucell) { 22 | Hash head; 23 | head_ver_.GetBranch(key, branch, &head); 24 | return GetForType(type, key, head, ucell); 25 | } 26 | }; 27 | 28 | } // namespace ustore 29 | 30 | #endif // USTORE_WORKER_WORKER_EXT_H_ 31 | -------------------------------------------------------------------------------- /lucene/README.md: -------------------------------------------------------------------------------- 1 | # Provide RESTful service for data indexing and searching using Apache Lucene 2 | 3 | ## Installation 4 | 5 | 1. Run "mvn clean package" under lucene directory. 6 | 2. Run "java -jar target/LuceneJAR-0.0.1-SNAPSHOT.jar" to start the service. 7 | 8 | ## RESTful API 9 | 10 | 1. Index 11 | ``` 12 | URI : host:61000/index 13 | HTTP Method : Post 14 | Request Body: JSON string 15 | { 16 | "dir" : "", 17 | "dataset": "", 18 | "branch" : "" 19 | } 20 | Response : JSON string with fields 21 | status - 0: successful; 1: failed 22 | msg - error message 23 | ``` 24 | 25 | 2. Search 26 | ``` 27 | URI : host:61000/search 28 | HTTP Method : Get 29 | Request Body: JSON string 30 | { 31 | "query" : "", 32 | "dataset": "", 33 | "branch" : "" 34 | } 35 | Response : JSON string with fields 36 | status - 0: successful; 1: failed 37 | msg - error message 38 | docs - list of matched result 39 | ``` 40 | 41 | ## Usage 42 | 43 | 1. Put files to be indexed under lucene/file_path 44 | 2. Call index API with parameter "file_path", dataset name and branch name 45 | -------------------------------------------------------------------------------- /lucene/src/main/java/application/Application.java: -------------------------------------------------------------------------------- 1 | package application; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.ComponentScan; 6 | import org.springframework.scheduling.annotation.EnableAsync; 7 | 8 | @SpringBootApplication 9 | @ComponentScan(basePackages = {"service", "controller"}) 10 | @EnableAsync 11 | public class Application { 12 | public static void main(String[] args) { 13 | SpringApplication.run(Application.class, args); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lucene/src/main/java/configuration/LuceneConfiguration.java: -------------------------------------------------------------------------------- 1 | package configuration; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | import service.IndexService; 7 | import service.IndexServiceImpl; 8 | 9 | @Configuration 10 | public class LuceneConfiguration { 11 | @Bean 12 | public IndexService indexService() { 13 | return new IndexServiceImpl(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lucene/src/main/java/controller/IndexController.java: -------------------------------------------------------------------------------- 1 | package controller; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.stereotype.Controller; 5 | import org.springframework.web.bind.annotation.RequestBody; 6 | import org.springframework.web.bind.annotation.RequestMapping; 7 | import org.springframework.web.bind.annotation.RequestMethod; 8 | import org.springframework.web.bind.annotation.ResponseBody; 9 | 10 | import com.google.gson.Gson; 11 | 12 | import pojo.IndexRequest; 13 | import pojo.SearchRequest; 14 | import service.IndexService; 15 | 16 | @Controller 17 | public class IndexController { 18 | 19 | @Autowired IndexService service; 20 | 21 | /** 22 | * Index data in files by line 23 | * 24 | * @param param The directory of files to be indicated 25 | * @return JSON string with execution status field 26 | */ 27 | @RequestMapping(value = "/index", method = RequestMethod.POST) 28 | @ResponseBody 29 | public String indexFiles(@RequestBody IndexRequest param) { 30 | Gson gson = new Gson(); 31 | return gson.toJson(service.indexFile(param)); 32 | } 33 | 34 | /** 35 | * Search indexed data 36 | * 37 | * @param param The Lucene query string 38 | * @return JSON string with fields status - execution status docs - list of matched results 39 | */ 40 | @RequestMapping(value = "/search", method = RequestMethod.POST) 41 | @ResponseBody 42 | public String searchFiles(@RequestBody SearchRequest param) { 43 | Gson gson = new Gson(); 44 | return gson.toJson(service.searchFile(param)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/Const.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | public class Const { 4 | public static final String INDEX_ROOT = "index/"; 5 | public static final String FIELD_PRIMARY_KEY = "PRIMARY_KEY"; 6 | public static final String FIELD_ALL_FIELDS = "ALL_FIELDS"; 7 | public static final int SUCCESS = 0; 8 | public static final int INVALID_PATH = 1; 9 | public static final int IO_EXCEPTION = 2; 10 | public static final int PARSE_ERROR = 3; 11 | public static final String[] EMPTY_ARRAY = new String[0]; 12 | } 13 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/IndexRequest.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | public class IndexRequest { 4 | String dataset; 5 | String branch; 6 | String dir; 7 | 8 | public String getDataset() { 9 | return dataset; 10 | } 11 | 12 | public void setDataset(String dataset) { 13 | this.dataset = dataset; 14 | } 15 | 16 | public String getBranch() { 17 | return branch; 18 | } 19 | 20 | public void setBranch(String branch) { 21 | this.branch = branch; 22 | } 23 | 24 | public String getDir() { 25 | return dir; 26 | } 27 | 28 | public void setDir(String dir) { 29 | this.dir = dir; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/IndexResponse.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | public class IndexResponse { 4 | int status; 5 | String msg; 6 | 7 | public IndexResponse(int status, String msg) { 8 | this.status = status; 9 | this.msg = msg; 10 | } 11 | 12 | public int getStatus() { 13 | return status; 14 | } 15 | 16 | public void setStatus(int status) { 17 | this.status = status; 18 | } 19 | 20 | public String getMsg() { 21 | return msg; 22 | } 23 | 24 | public void setMsg(String msg) { 25 | this.msg = msg; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/SearchRequest.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | public class SearchRequest { 4 | String dataset; 5 | String branch; 6 | String query; 7 | 8 | public String getDataset() { 9 | return dataset; 10 | } 11 | 12 | public void setDataset(String dataset) { 13 | this.dataset = dataset; 14 | } 15 | 16 | public String getBranch() { 17 | return branch; 18 | } 19 | 20 | public void setBranch(String branch) { 21 | this.branch = branch; 22 | } 23 | 24 | public String getQuery() { 25 | return query; 26 | } 27 | 28 | public void setQuery(String query) { 29 | this.query = query; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/SearchResponse.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | import java.util.List; 4 | 5 | public class SearchResponse { 6 | int status; 7 | String msg; 8 | List docs; 9 | 10 | public SearchResponse(int status, String msg) { 11 | this.status = status; 12 | this.msg = msg; 13 | } 14 | 15 | public SearchResponse(int status, String msg, List docs) { 16 | this(status, msg); 17 | this.docs = docs; 18 | } 19 | 20 | public int getStatus() { 21 | return status; 22 | } 23 | 24 | public void setStatus(int status) { 25 | this.status = status; 26 | } 27 | 28 | public String getMsg() { 29 | return msg; 30 | } 31 | 32 | public void setMsg(String msg) { 33 | this.msg = msg; 34 | } 35 | 36 | public List getDocs() { 37 | return docs; 38 | } 39 | 40 | public void setDocs(List docs) { 41 | this.docs = docs; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /lucene/src/main/java/pojo/SearchResult.java: -------------------------------------------------------------------------------- 1 | package pojo; 2 | 3 | public class SearchResult { 4 | String key; 5 | 6 | public SearchResult(String key) { 7 | this.key = key; 8 | } 9 | 10 | public String getKey() { 11 | return key; 12 | } 13 | 14 | public void setKey(String key) { 15 | this.key = key; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /lucene/src/main/java/service/AsyncService.java: -------------------------------------------------------------------------------- 1 | package service; 2 | 3 | import java.nio.file.Path; 4 | 5 | public interface AsyncService { 6 | public void asyncIndexFile(Path path, String dataset, String branch); 7 | } 8 | -------------------------------------------------------------------------------- /lucene/src/main/java/service/IndexService.java: -------------------------------------------------------------------------------- 1 | package service; 2 | 3 | import pojo.IndexRequest; 4 | import pojo.IndexResponse; 5 | import pojo.SearchRequest; 6 | import pojo.SearchResponse; 7 | 8 | public interface IndexService { 9 | public IndexResponse indexFile(IndexRequest param); 10 | 11 | public SearchResponse searchFile(SearchRequest param); 12 | } 13 | -------------------------------------------------------------------------------- /lucene/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=61000 -------------------------------------------------------------------------------- /lucene/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/chunk/chunk.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "chunk/chunk.h" 4 | 5 | #include 6 | 7 | namespace ustore { 8 | 9 | Chunk::Chunk(ChunkType type, uint32_t capacity) 10 | : own_(new byte_t[kMetaLength + capacity]) { 11 | head_ = own_.get(); 12 | *reinterpret_cast(&own_[kNumBytesOffset]) = kMetaLength + capacity; 13 | *reinterpret_cast(&own_[kChunkTypeOffset]) = type; 14 | } 15 | 16 | Chunk::Chunk(std::unique_ptr head) noexcept : own_(std::move(head)) { 17 | head_ = own_.get(); 18 | } 19 | 20 | } // namespace ustore 21 | -------------------------------------------------------------------------------- /src/chunk/chunk_loader.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "chunk/chunk_loader.h" 4 | #include "cluster/chunk_client.h" 5 | #include "cluster/partitioner.h" 6 | #include "spec/db.h" 7 | 8 | namespace ustore { 9 | 10 | const Chunk* ChunkLoader::Load(const Hash& key) { 11 | CHECK(!key.empty()); 12 | auto it = cache_.find(key); 13 | if (it != cache_.end()) return &(it->second); 14 | cache_.emplace(key.Clone(), GetChunk(key)); 15 | return &cache_[key]; 16 | } 17 | 18 | Chunk LocalChunkLoader::GetChunk(const Hash& key) { 19 | return cs_->Get(key); 20 | } 21 | 22 | Chunk PartitionedChunkLoader::GetChunk(const Hash& key) { 23 | int id = ptt_->GetDestId(key); 24 | if (id == ptt_->id()) { 25 | return cs_->Get(key); 26 | } else { 27 | Chunk c; 28 | auto stat = client_->Get(key, &c); 29 | CHECK(stat == ErrorCode::kOK) << "Failed to load remote chunk " << key; 30 | return c; 31 | } 32 | LOG(FATAL) << "Failed to load chunk"; 33 | return Chunk(); 34 | } 35 | 36 | Chunk ClientChunkLoader::GetChunk(const Hash& key) { 37 | Chunk chunk; 38 | ErrorCode code = db_->GetChunk(Slice(key_), key, &chunk); 39 | if (code != ErrorCode::kOK) 40 | LOG(ERROR) << "Failed to fetch chunk " << key 41 | << ", error code: " << static_cast(code); 42 | return chunk; 43 | } 44 | 45 | } // namespace ustore 46 | -------------------------------------------------------------------------------- /src/chunk/chunk_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "chunk/chunk_writer.h" 4 | #include "cluster/chunk_client.h" 5 | #include "cluster/partitioner.h" 6 | 7 | namespace ustore { 8 | 9 | bool LocalChunkWriter::Write(const Hash& key, const Chunk& chunk) { 10 | return cs_->Put(key, chunk); 11 | } 12 | 13 | bool PartitionedChunkWriter::Write(const Hash& key, const Chunk& chunk) { 14 | int id = ptt_->GetDestId(key); 15 | if (id == ptt_->id()) { 16 | return cs_->Put(key, chunk); 17 | } else { 18 | // check if already exist 19 | // TODO(wangsh): net overhead is high now, not worth checking existence 20 | // bool exists; 21 | // auto stat = client_->Exists(key, &exists); 22 | // CHECK(stat == ErrorCode::kOK) << "Failed to check remote chunk"; 23 | // if (exists) return true; 24 | 25 | // send chunk 26 | auto stat = client_->Put(key, chunk); 27 | CHECK(stat == ErrorCode::kOK) << "Failed to put remote chunk"; 28 | return true; 29 | } 30 | LOG(FATAL) << "Failed to write chunk"; 31 | return false; 32 | } 33 | 34 | } // namespace ustore 35 | -------------------------------------------------------------------------------- /src/cli_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "cli/command.h" 4 | #include "cli/console.h" 5 | #include "cluster/worker_client_service.h" 6 | 7 | namespace ustore { 8 | namespace cli { 9 | 10 | int main(int argc, char* argv[]) { 11 | SetStderrLogging(ERROR); 12 | // connect to UStore servcie 13 | WorkerClientService svc; 14 | svc.Run(); 15 | auto db = svc.CreateWorkerClient(); 16 | // conditional execution 17 | return static_cast( 18 | argc == 1 ? Console(&db).Run() : Command(&db).Run(argc, argv)); 19 | } 20 | 21 | } // namespace cli 22 | } // namespace ustore 23 | 24 | int main(int argc, char* argv[]) { 25 | return ustore::cli::main(argc, argv); 26 | } 27 | -------------------------------------------------------------------------------- /src/cluster/chunk_client.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "cluster/chunk_client.h" 4 | #include "proto/messages.pb.h" 5 | #include "utils/logging.h" 6 | 7 | namespace ustore { 8 | 9 | void ChunkClient::CreateChunkMessage(const Hash& hash, UMessage *msg) const { 10 | // request 11 | auto request = msg->mutable_request_payload(); 12 | request->set_version(hash.value(), Hash::kByteLength); 13 | } 14 | 15 | ErrorCode ChunkClient::Get(const Hash& hash, Chunk* chunk) const { 16 | UMessage msg; 17 | // header 18 | msg.set_type(UMessage::GET_CHUNK_REQUEST); 19 | // request 20 | CreateChunkMessage(hash, &msg); 21 | // send 22 | Send(&msg, ptt_->GetDestAddr(hash)); 23 | return GetChunkResponse(chunk); 24 | } 25 | 26 | ErrorCode ChunkClient::Get(const Slice& key, const Hash& hash, Chunk* chunk) 27 | const { 28 | UMessage msg; 29 | // header 30 | msg.set_type(UMessage::GET_CHUNK_REQUEST); 31 | // request 32 | CreateChunkMessage(hash, &msg); 33 | // send 34 | Send(&msg, ptt_->GetDestAddr(key)); 35 | return GetChunkResponse(chunk); 36 | } 37 | 38 | ErrorCode ChunkClient::Put(const Hash& hash, const Chunk& chunk) { 39 | UMessage msg; 40 | // header 41 | msg.set_type(UMessage::PUT_CHUNK_REQUEST); 42 | // request 43 | CreateChunkMessage(hash, &msg); 44 | auto payload = msg.mutable_value_payload(); 45 | payload->set_base(chunk.head(), chunk.numBytes()); 46 | // send 47 | Send(&msg, ptt_->GetDestAddr(hash)); 48 | return GetEmptyResponse(); 49 | } 50 | 51 | ErrorCode ChunkClient::Exists(const Hash& hash, bool* exist) const { 52 | UMessage msg; 53 | // header 54 | msg.set_type(UMessage::EXISTS_CHUNK_REQUEST); 55 | // request 56 | CreateChunkMessage(hash, &msg); 57 | // send 58 | Send(&msg, ptt_->GetDestAddr(hash)); 59 | return GetBoolResponse(exist); 60 | } 61 | 62 | } // namespace ustore 63 | -------------------------------------------------------------------------------- /src/cluster/chunk_client_service.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include "cluster/chunk_client_service.h" 4 | 5 | namespace ustore { 6 | 7 | class ChunkClientServiceCallBack : public CallBack { 8 | public: 9 | explicit ChunkClientServiceCallBack(void* handler) : CallBack(handler) {} 10 | void operator()(const void *msg, int size, const node_id_t& source) override { 11 | (reinterpret_cast(handler_))->HandleResponse( 12 | msg, size, source); 13 | } 14 | }; 15 | 16 | void ChunkClientService::Init() { 17 | CallBack* callback = new ChunkClientServiceCallBack(this); 18 | ClientService::Init(std::unique_ptr(callback)); 19 | } 20 | 21 | ChunkClient ChunkClientService::CreateChunkClient() { 22 | // adding a new response blob 23 | ResponseBlob* resblob = CreateResponseBlob(); 24 | return ChunkClient(resblob, &ptt_); 25 | } 26 | 27 | } // namespace ustore 28 | -------------------------------------------------------------------------------- /src/cluster/client_service.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include "cluster/client_service.h" 4 | 5 | #include "utils/env.h" 6 | #include "utils/logging.h" 7 | #include "utils/message_parser.h" 8 | 9 | namespace ustore { 10 | 11 | void ClientService::Init(std::unique_ptr callback) { 12 | Net* net = net::CreateClientNetwork(Env::Instance()->config().recv_threads()); 13 | Service::Init(std::unique_ptr(net), std::move(callback)); 14 | // NOTE(wangsh): 15 | // client service need to init context before connect to host service 16 | net_->CreateNetContexts(ptt_->destAddrs()); 17 | } 18 | 19 | void ClientService::HandleResponse(const void *msg, int size, 20 | const node_id_t& source) { 21 | // parse the request 22 | UMessage *ustore_msg = new UMessage(); 23 | MessageParser::Parse(msg, size, ustore_msg); 24 | ResponseBlob* res_blob; 25 | { 26 | boost::shared_lock lock(lock_); 27 | res_blob = responses_[ustore_msg->source()].get(); 28 | } 29 | 30 | std::unique_lock lck(res_blob->lock); 31 | res_blob->message = ustore_msg; 32 | res_blob->has_msg = true; 33 | res_blob->condition.notify_all(); 34 | } 35 | 36 | ResponseBlob* ClientService::CreateResponseBlob() { 37 | boost::unique_lock lock(lock_); 38 | ResponseBlob* ret = new ResponseBlob(); 39 | responses_.emplace_back(ret); 40 | for (size_t i = 0; i < responses_.size(); ++i) 41 | if (responses_[i].get() == ret) { 42 | ret->id = i; 43 | ret->net = net_.get(); 44 | return ret; 45 | } 46 | LOG(FATAL) << "Failed to find response blob id"; 47 | return nullptr; 48 | } 49 | 50 | } // namespace ustore 51 | -------------------------------------------------------------------------------- /src/cluster/partitioner.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include 4 | #include "cluster/partitioner.h" 5 | #include "utils/env.h" 6 | 7 | namespace ustore { 8 | 9 | Partitioner::Partitioner(const std::string& hostfile, 10 | const std::string& self_addr, 11 | std::function f_port) { 12 | // load worker file 13 | std::ifstream fin(hostfile); 14 | std::string dest_addr; 15 | for (int id = 0; fin >> dest_addr; ++id) { 16 | if (dest_addr == self_addr) id_ = id; 17 | dest_list_.push_back(f_port(dest_addr)); 18 | } 19 | fin.close(); 20 | CHECK(dest_list_.size()) << "IP:PORT list cannot be empty"; 21 | } 22 | 23 | } // namespace ustore 24 | -------------------------------------------------------------------------------- /src/cluster/service.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include "cluster/service.h" 4 | 5 | #include 6 | #include "utils/env.h" 7 | 8 | namespace ustore { 9 | 10 | constexpr int kInitForMs = 75; 11 | 12 | void Service::Init(std::unique_ptr net, 13 | std::unique_ptr callback) { 14 | net_ = std::move(net); 15 | cb_ = std::move(callback); 16 | net_->RegisterRecv(cb_.get()); 17 | is_init_ = true; 18 | } 19 | 20 | void Service::Start() { 21 | if (is_running_) { 22 | LOG(WARNING) << "service is already started"; 23 | return; 24 | } 25 | if (!is_init_) Init(); 26 | is_running_ = true; 27 | net_->Start(); 28 | } 29 | 30 | void Service::Run() { 31 | if (thread_) { 32 | LOG(WARNING) << "service thread is already running"; 33 | return; 34 | } 35 | if (!is_init_) Init(); 36 | thread_.reset(new std::thread(&Service::Start, this)); 37 | std::this_thread::sleep_for(std::chrono::milliseconds(kInitForMs)); 38 | } 39 | 40 | void Service::Stop() { 41 | if (!is_running_) return; 42 | net_->Stop(); 43 | if (thread_) { 44 | thread_->join(); 45 | thread_.reset(nullptr); 46 | } 47 | is_running_ = false; 48 | } 49 | 50 | } // namespace ustore 51 | -------------------------------------------------------------------------------- /src/cluster/worker_client_service.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include "cluster/worker_client_service.h" 4 | 5 | namespace ustore { 6 | 7 | class WorkerClientServiceCallBack : public CallBack { 8 | public: 9 | explicit WorkerClientServiceCallBack(void* handler) : CallBack(handler) {} 10 | void operator()(const void *msg, int size, const node_id_t& source) override { 11 | (reinterpret_cast(handler_))->HandleResponse( 12 | msg, size, source); 13 | } 14 | }; 15 | 16 | void WorkerClientService::Init() { 17 | CallBack* callback = new WorkerClientServiceCallBack(this); 18 | ClientService::Init(std::unique_ptr(callback)); 19 | // start chunk client service as well 20 | if (ck_svc_) ck_svc_->Run(); 21 | } 22 | 23 | WorkerClient WorkerClientService::CreateWorkerClient() { 24 | // adding a new response blob 25 | ResponseBlob* resblob = CreateResponseBlob(); 26 | return WorkerClient(resblob, &ptt_, ck_svc_.get()); 27 | } 28 | 29 | } // namespace ustore 30 | -------------------------------------------------------------------------------- /src/http/http_client.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "http/http_client.h" 4 | 5 | #include 6 | #include "utils/logging.h" 7 | 8 | namespace ustore { 9 | namespace http { 10 | 11 | using tcp = boost::asio::ip::tcp; 12 | 13 | bool HttpClient::Connect(const std::string& host, const std::string& port) { 14 | try { 15 | host_ = host; 16 | tcp::resolver resolver{io_context_}; 17 | auto results = resolver.resolve(host, port); 18 | boost::asio::connect(socket_, results.begin(), results.end()); 19 | return true; 20 | } catch(std::exception const& e) { 21 | LOG(FATAL) << e.what(); 22 | return false; 23 | } 24 | } 25 | 26 | bool HttpClient::Send(Request* request) { 27 | try { 28 | // initialize payload 29 | request->PreparePayload(); 30 | beast::write(socket_, request->req_); 31 | return true; 32 | } catch(std::exception const& e) { 33 | LOG(FATAL) << e.what(); 34 | return false; 35 | } 36 | } 37 | 38 | bool HttpClient::Receive(Response* response) { 39 | try { 40 | // Receive the HTTP response 41 | beast::read(socket_, response->buffer_, response->res_); 42 | return true; 43 | } catch(std::exception const& e) { 44 | LOG(FATAL) << e.what(); 45 | return false; 46 | } 47 | } 48 | 49 | bool HttpClient::Shutdown() { 50 | try { 51 | // Gracefully close the socket 52 | boost::system::error_code ec; 53 | socket_.shutdown(tcp::socket::shutdown_both, ec); 54 | // report not_connected if happens 55 | if (ec && ec != boost::system::errc::not_connected) 56 | throw boost::system::system_error{ec}; 57 | return true; 58 | } catch(std::exception const& e) { 59 | LOG(FATAL) << e.what(); 60 | return false; 61 | } 62 | } 63 | 64 | } // namespace http 65 | } // namespace ustore 66 | -------------------------------------------------------------------------------- /src/http/http_msg.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "http/http_msg.h" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "utils/logging.h" 10 | 11 | namespace ustore { 12 | namespace http { 13 | 14 | void Request::SetMethod(Verb method) { 15 | switch (method) { 16 | case Verb::kGet: 17 | req_.method(beast::verb::get); 18 | break; 19 | case Verb::kPut: 20 | req_.method(beast::verb::put); 21 | break; 22 | case Verb::kPost: 23 | req_.method(beast::verb::post); 24 | break; 25 | default: 26 | LOG(FATAL) << "Unsupported http verb"; 27 | } 28 | } 29 | 30 | void Request::PreparePayload() { 31 | // append parameters to target in the form: /target?k1=v1&k2=v2&... 32 | int cnt = 0; 33 | std::string param_str; 34 | for (auto const& entry : param_) { 35 | param_str += (cnt++ ? "&" : "?") + entry.first + "=" + entry.second; 36 | } 37 | req_.target(target_ + param_str); 38 | req_.prepare_payload(); 39 | } 40 | 41 | void Request::SetDefaultFields() { 42 | req_.version(kDefaultHttpVersion); 43 | req_.set(beast::field::user_agent, BOOST_BEAST_VERSION_STRING); 44 | } 45 | 46 | const std::map& Response::headers() { 47 | if (headers_.empty()) { 48 | for (auto const& field : res_) { 49 | headers_[std::string(field.name_string())] = std::string(field.value()); 50 | } 51 | } 52 | return headers_; 53 | } 54 | 55 | } // namespace http 56 | } // namespace ustore 57 | -------------------------------------------------------------------------------- /src/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include 4 | #include 5 | #include "utils/env.h" 6 | #include "cluster/chunk_service.h" 7 | #include "cluster/worker_service.h" 8 | 9 | #ifndef GFLAGS_GFLAGS_H_ 10 | namespace gflags = google; 11 | #endif // GFLAGS_GFLAGS_H_ 12 | 13 | DEFINE_string(node_id, "", "ip address of this node (ib0 interface)"); 14 | DEFINE_int32(loglevel, ustore::INFO, "logging severity level"); 15 | 16 | int main(int argc, char **argv) { 17 | if (argc < 2) { 18 | std::cout << "Args: --node_id ip:port [--loglevel 0/1/2/3]\n"; 19 | return 0; 20 | } 21 | 22 | gflags::ParseCommandLineFlags(&argc, &argv, true); 23 | 24 | // set the logging severity level 25 | ustore::SetStderrLogging(FLAGS_loglevel); 26 | 27 | // create worker/chunk service 28 | ustore::WorkerService ws(FLAGS_node_id, true); 29 | ustore::ChunkService cs(FLAGS_node_id); 30 | 31 | // Start chunk service if dist store is enabled 32 | // if (ustore::Env::Instance()->config().enable_dist_store()) cs.Run(); 33 | // start worker and blocking here 34 | ws.Start(); 35 | 36 | return 0; 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/net/net.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "net/net.h" 4 | 5 | #include 6 | #include 7 | #ifdef USE_RDMA 8 | #include "net/rdma_net.h" 9 | #else 10 | #include "net/zmq_net.h" 11 | #endif // USE_RDMA 12 | 13 | namespace ustore { 14 | 15 | Net::~Net() { 16 | for (auto val : netmap_) 17 | delete val.second; 18 | DLOG(INFO) << "Destroy Net "; 19 | } 20 | 21 | void Net::CreateNetContexts(const std::vector& nodes) { 22 | std::set tmp; 23 | for (size_t i = 0; i < nodes.size(); i++) 24 | tmp.insert(nodes[i]); 25 | for (auto val = netmap_.begin(); val != netmap_.end(); val++) { 26 | if (tmp.find(val->first) == tmp.end()) 27 | delete val->second; 28 | else 29 | tmp.erase(val->first); 30 | } 31 | 32 | for (auto val = tmp.begin(); val != tmp.end(); val++) { 33 | if (*val != cur_node_) 34 | CreateNetContext(*val); 35 | } 36 | } 37 | 38 | void Net::DeleteNetContext(NetContext* ctx) { delete ctx; } 39 | 40 | namespace net { 41 | 42 | Net* CreateServerNetwork(const node_id_t& id, int n_threads) { 43 | #ifdef USE_RDMA 44 | return new RdmaNet(id, n_threads); 45 | #else 46 | return new ServerZmqNet(id, n_threads); 47 | #endif 48 | LOG(FATAL) << "Failed to create network instance"; 49 | return nullptr; 50 | } 51 | 52 | Net* CreateClientNetwork(int n_threads) { 53 | #ifdef USE_RDMA 54 | return new RdmaNet("", n_threads); 55 | #else 56 | return new ClientZmqNet(n_threads); 57 | #endif 58 | LOG(FATAL) << "Failed to create network instance"; 59 | return nullptr; 60 | } 61 | 62 | } // namespace net 63 | } // namespace ustore 64 | -------------------------------------------------------------------------------- /src/node/blob_node.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "node/blob_node.h" 4 | 5 | #include // for memcpy 6 | #include "utils/logging.h" 7 | 8 | namespace ustore { 9 | 10 | ChunkInfo BlobChunker::Make(const std::vector& segments) 11 | const { 12 | size_t chunk_num_bytes = 0; 13 | size_t num_entries = 0; 14 | for (size_t i = 0; i < segments.size(); i++) { 15 | chunk_num_bytes += segments[i]->numBytes(); 16 | num_entries += segments[i]->numEntries(); 17 | } 18 | Chunk chunk(ChunkType::kBlob, chunk_num_bytes); 19 | size_t seg_offset = 0; 20 | for (const Segment* seg : segments) { 21 | seg->AppendForChunk(chunk.m_data() + seg_offset); 22 | seg_offset += seg->numBytes(); 23 | } 24 | size_t me_num_bytes; 25 | std::unique_ptr meta_data(MetaEntry::Encode( 26 | 1, num_entries, chunk.hash(), OrderedKey(0), &me_num_bytes)); 27 | std::unique_ptr meta_seg( 28 | new VarSegment(std::move(meta_data), me_num_bytes, {0})); 29 | return {std::move(chunk), std::move(meta_seg)}; 30 | } 31 | 32 | OrderedKey BlobNode::key(size_t idx) const { 33 | LOG(FATAL) << "Not Supported"; 34 | return OrderedKey(); 35 | } 36 | 37 | uint64_t BlobNode::FindIndexForKey( 38 | const OrderedKey& key, ChunkLoader* loader) const { 39 | LOG(FATAL) << "Not Supported to Get Blob Elemeny by Key. "; 40 | return 0; 41 | } 42 | 43 | size_t BlobNode::Copy(size_t start, size_t num_bytes, byte_t* buffer) const { 44 | size_t len = num_bytes; 45 | if (start + len > numElements()) { 46 | LOG(WARNING) 47 | << "start + len > BlobNode capacity. Will copy until chunk end."; 48 | len = numElements() - start; 49 | } 50 | std::memcpy(buffer, chunk_->data() + start, len); 51 | return len; 52 | } 53 | 54 | std::unique_ptr BlobNode::GetSegment( 55 | size_t start, size_t num_elements) const { 56 | CHECK_LE(start + num_elements, numEntries()); 57 | std::unique_ptr seg( 58 | new FixedSegment(data(start), num_elements, 1)); 59 | return seg; 60 | } 61 | 62 | } // namespace ustore 63 | -------------------------------------------------------------------------------- /src/node/cell_node.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "node/cell_node.h" 4 | 5 | #include 6 | #include // for memcpy 7 | 8 | namespace ustore { 9 | 10 | Chunk CellNode::NewChunk(const UType type, const Slice& key, const Slice& data, 11 | const Slice& ctx, const Hash& preHash1, const Hash& preHash2) { 12 | // First hash can not be empty 13 | CHECK(!preHash1.empty()); 14 | // key/data/ctx size can not be larger than 1<<16 15 | CHECK(key.len() < kFieldMaxLength) << "Key length ("<< key.len() <<") exceeds limit"; // NOLINT 16 | CHECK(data.len() < kFieldMaxLength) << "Data length ("<< data.len() <<") exceeds limit"; // NOLINT 17 | CHECK(ctx.len() < kFieldMaxLength) << "Context length ("<< ctx.len() <<") exceeds limit"; // NOLINT 18 | size_t num_pre_hash = preHash2.empty() ? 1 : 2; 19 | size_t key_offset = ComputeKeyOffset(num_pre_hash); 20 | size_t data_offset = key_offset + key.len(); 21 | size_t ctx_offset = data_offset + data.len(); 22 | size_t chunk_len = ctx_offset + ctx.len(); 23 | Chunk chunk(ChunkType::kCell, chunk_len); 24 | // mete fields 25 | *reinterpret_cast(chunk.m_data() + kUTypePos) = type; 26 | *reinterpret_cast(chunk.m_data() + kNumPreHashPos) = num_pre_hash; 27 | *reinterpret_cast(chunk.m_data() + kKeyLengthPos) = key.len(); 28 | *reinterpret_cast(chunk.m_data() + kDataLengthPos) = data.len(); 29 | *reinterpret_cast(chunk.m_data() + kCtxLengthPos) = ctx.len(); 30 | // pre hash 31 | std::memcpy(chunk.m_data() + ComputePreHashOffset(0), preHash1.value(), 32 | Hash::kByteLength); 33 | if (num_pre_hash == 2) 34 | std::memcpy(chunk.m_data() + ComputePreHashOffset(1), preHash2.value(), 35 | Hash::kByteLength); 36 | // key 37 | std::memcpy(chunk.m_data() + key_offset, key.data(), key.len()); 38 | // data 39 | std::memcpy(chunk.m_data() + data_offset, data.data(), data.len()); 40 | // context 41 | std::memcpy(chunk.m_data() + ctx_offset, ctx.data(), ctx.len()); 42 | return chunk; 43 | } 44 | 45 | } // namespace ustore 46 | -------------------------------------------------------------------------------- /src/node/orderedkey.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "node/orderedkey.h" 4 | 5 | #include // for memcpy 6 | 7 | namespace ustore { 8 | OrderedKey::OrderedKey(uint64_t value) noexcept 9 | : by_value_(true), value_(value), slice_() {} 10 | 11 | // NOTE: num_bytes_ count the first byte of by_value field 12 | // data_ points to the first byte of key value, skipping by_value field. 13 | OrderedKey::OrderedKey(bool by_value, const byte_t* data, 14 | size_t num_bytes) noexcept 15 | : by_value_(by_value), slice_(data, num_bytes) { 16 | // parse by_value from paremeter data 17 | if (by_value_) { 18 | // parse value from data_ (class members) 19 | this->value_ = *(reinterpret_cast(slice_.data())); 20 | } 21 | } 22 | 23 | size_t OrderedKey::Encode(byte_t* buffer) const { 24 | if (by_value_) { 25 | std::memcpy(buffer, &value_, sizeof(value_)); 26 | } else { 27 | std::memcpy(buffer, slice_.data(), slice_.len()); 28 | } 29 | return numBytes(); 30 | } 31 | 32 | bool OrderedKey::operator>(const OrderedKey& otherKey) const { 33 | CHECK_EQ(by_value_, otherKey.by_value_); 34 | return by_value_ ? value_ > otherKey.value_ : slice_ > otherKey.slice_; 35 | } 36 | 37 | bool OrderedKey::operator<(const OrderedKey& otherKey) const { 38 | CHECK_EQ(by_value_, otherKey.by_value_); 39 | return by_value_ ? value_ < otherKey.value_ : slice_ < otherKey.slice_; 40 | } 41 | 42 | bool OrderedKey::operator==(const OrderedKey& otherKey) const { 43 | CHECK_EQ(by_value_, otherKey.by_value_); 44 | return by_value_ ? value_ == otherKey.value_ : slice_ == otherKey.slice_; 45 | } 46 | } // namespace ustore 47 | -------------------------------------------------------------------------------- /src/node/rolling_hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "node/rolling_hash.h" 4 | 5 | namespace ustore { 6 | 7 | RollingHasher::RollingHasher() 8 | : RollingHasher(kDefaultChunkPattern, kDefaultChunkWindow, 9 | kDefaultMaxChunkSize) {} 10 | 11 | RollingHasher::RollingHasher(uint32_t chunk_pattern, size_t window_size, 12 | size_t max_size) 13 | : chunk_pattern_{chunk_pattern}, 14 | window_size_{window_size}, 15 | max_size_{max_size}, 16 | buz_{unsigned(window_size)} {} 17 | 18 | void RollingHasher::HashBytes(const byte_t* data, size_t numBytes) { 19 | for (size_t i = 0; i < numBytes; i++) { 20 | HashByte(*(data + i)); 21 | } 22 | } 23 | 24 | size_t RollingHasher::TryHashBytes(const byte_t* data, size_t numBytes) { 25 | for (size_t i = 0; i < numBytes; i++) { 26 | HashByte(*(data + i)); 27 | if (crossed_boundary_) return i; 28 | } 29 | return numBytes; 30 | } 31 | 32 | } // namespace ustore 33 | -------------------------------------------------------------------------------- /src/proto/config.proto: -------------------------------------------------------------------------------- 1 | /// Copyright (c) 2017 The Ustore Authors. 2 | 3 | package ustore; 4 | 5 | /** 6 | * Configuration parameters 7 | */ 8 | message Config { 9 | /* data storage related */ 10 | // folder to store data files 11 | optional string data_dir = 1 [default = "ustore_data"]; 12 | // pattern of data file, the final file name is {$pattern}_{$worker_id}.dat 13 | optional string data_file_pattern = 2 [default = "ustore"]; 14 | // destination of access log from worker service (disabled by default) 15 | optional string access_log_dir = 3 [default = ""]; 16 | // maximum number of data segments allocated by chunk store 17 | optional int32 max_segments = 4 [default = 64]; 18 | // distributed chunk store maintain data globally without duplication 19 | optional bool enable_dist_store = 5 [default = false]; 20 | // client reads chunks via chunk service to bypass worker service 21 | optional bool get_chunk_bypass_worker = 6 [default = true]; 22 | 23 | /* cluster related */ 24 | // file containing worker list in format of hostname:port 25 | optional string worker_file = 10 [default = "conf/workers.lst"]; 26 | 27 | /* service related */ 28 | optional int32 recv_threads = 21 [default = 2]; // number of receiving threads 29 | // optional int32 service_threads = 22 [default = 1]; // number of server threads 30 | 31 | /* http server related */ 32 | optional int32 http_port = 50 [default = 60000]; // the port for the http client 33 | } 34 | -------------------------------------------------------------------------------- /src/proto/head_version.proto: -------------------------------------------------------------------------------- 1 | /// Copyright (c) 2017 The Ustore Authors. 2 | 3 | package ustore; 4 | 5 | message BranchVersion { 6 | required bytes branch = 1; 7 | required bytes version = 2; 8 | } 9 | 10 | message KeyVersion { 11 | required bytes key = 1; 12 | repeated BranchVersion branches = 2; 13 | } 14 | 15 | message KeyVersions { 16 | repeated KeyVersion key_versions = 1; 17 | } 18 | 19 | -------------------------------------------------------------------------------- /src/recovery/log_thread.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The UStore Authors. 2 | 3 | #include"recovery/log_thread.h" 4 | 5 | namespace ustore { 6 | namespace recovery { 7 | 8 | LogThread::LogThread() { 9 | m_tid_ = 0; 10 | m_running_ = 0; 11 | m_detached_ = 0; 12 | } 13 | 14 | LogThread::~LogThread() { 15 | if (m_running_ == 1 && m_detached_ == 0) { 16 | pthread_detach(m_tid_); 17 | } 18 | if (m_running_ == 1) { 19 | pthread_cancel(m_tid_); 20 | } 21 | } 22 | 23 | int LogThread::Start() { 24 | int tmpret = pthread_create(&m_tid_, NULL, &runThread, this); 25 | if (tmpret == 0) { 26 | m_running_ = 1; 27 | } 28 | return tmpret; 29 | } 30 | 31 | int LogThread::Join() { 32 | int tmpret = -1; 33 | if (m_running_ == 1) { 34 | tmpret = pthread_join(m_tid_, NULL); 35 | } 36 | if (tmpret == 0) { 37 | m_detached_ = 1; 38 | } 39 | return tmpret; 40 | } 41 | 42 | int LogThread::Detach() { 43 | int tmpret = -1; 44 | if (m_running_ == 1 && m_detached_ == 0) { 45 | tmpret = pthread_detach(m_tid_); 46 | if (tmpret == 0) { 47 | m_detached_ = 1; 48 | } 49 | } 50 | return tmpret; 51 | } 52 | 53 | pthread_t LogThread::Self() { 54 | return m_tid_; 55 | } 56 | 57 | static void* runThread(void* args) { 58 | return (reinterpret_cast(args))->Run(); 59 | // return ((LogThread*)args)->Run(); 60 | } 61 | 62 | } // end of namespace recovery 63 | } // end of namespace ustore 64 | -------------------------------------------------------------------------------- /src/types/client/vblob.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/client/vblob.h" 4 | 5 | namespace ustore { 6 | 7 | VBlob::VBlob(const Slice& data) noexcept : UBlob(nullptr) { 8 | buffer_ = {UType::kBlob, {}, 0, 0, {data}, {}}; 9 | } 10 | 11 | VBlob::VBlob(std::istream* is) noexcept : UBlob(nullptr) { 12 | buffer_ = {UType::kBlob, {}, 0, 0, {}, {}}; 13 | useStream_ = true; 14 | dataStream_ = is; 15 | } 16 | 17 | VBlob::VBlob(std::shared_ptr loader, const Hash& root_hash) 18 | noexcept : UBlob(loader) { 19 | SetNodeForHash(root_hash); 20 | buffer_ = {UType::kBlob, root_node_->hash(), 0, 0, {}, {}}; 21 | } 22 | 23 | Hash VBlob::Splice(size_t pos, size_t num_delete, const byte_t* data, 24 | size_t num_append) const { 25 | buffer_ = {UType::kBlob, root_node_->hash(), pos, num_delete, 26 | {Slice(reinterpret_cast(data), num_append)}, {}}; 27 | return Hash::kNull; 28 | } 29 | 30 | Hash VBlob::Splice(size_t pos, size_t num_delete, std::istream* is, 31 | size_t num_append) const { 32 | buffer_ = {UType::kBlob, root_node_->hash(), pos, num_delete, {}, {}}; 33 | useStream_ = true; 34 | dataStream_ = is; 35 | return Hash::kNull; 36 | } 37 | 38 | } // namespace ustore 39 | -------------------------------------------------------------------------------- /src/types/client/vlist.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/client/vlist.h" 4 | 5 | namespace ustore { 6 | 7 | VList::VList(const std::vector& data) noexcept : UList(nullptr) { 8 | buffer_ = {UType::kList, {}, 0, 0, data, {}}; 9 | } 10 | 11 | VList::VList(std::shared_ptr loader, const Hash& root_hash) 12 | noexcept : UList(loader) { 13 | SetNodeForHash(root_hash); 14 | buffer_ = {UType::kList, root_node_->hash(), 0, 0, {}, {}}; 15 | } 16 | 17 | Hash VList::Splice(uint64_t start_idx, uint64_t num_to_delete, 18 | const std::vector& entries) const { 19 | buffer_ = {UType::kList, root_node_->hash(), start_idx, num_to_delete, 20 | entries, {}}; 21 | return Hash::kNull; 22 | } 23 | 24 | } // namespace ustore 25 | -------------------------------------------------------------------------------- /src/types/client/vmap.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/client/vmap.h" 4 | 5 | namespace ustore { 6 | 7 | VMap::VMap(const std::vector& keys, const std::vector& vals) 8 | noexcept { 9 | buffer_ = {UType::kMap, {}, 0, 0, vals, keys}; 10 | } 11 | 12 | VMap::VMap(std::shared_ptr loader, const Hash& root_hash) 13 | noexcept : UMap(loader) { 14 | SetNodeForHash(root_hash); 15 | buffer_ = {UType::kMap, root_node_->hash(), 0, 0, {}, {}}; 16 | } 17 | 18 | Hash VMap::Set(const Slice& key, const Slice& val) const { 19 | buffer_ = {UType::kMap, root_node_->hash(), 0, 0, {val}, {key}}; 20 | return Hash::kNull; 21 | } 22 | 23 | Hash VMap::Set(const std::vector& keys, 24 | const std::vector& vals) const { 25 | buffer_ = {UType::kMap, root_node_->hash(), 0, 0, vals, keys}; 26 | return Hash::kNull; 27 | } 28 | 29 | Hash VMap::Remove(const Slice& key) const { 30 | buffer_ = {UType::kMap, root_node_->hash(), 0, 1, {}, {key}}; 31 | return Hash::kNull; 32 | } 33 | 34 | } // namespace ustore 35 | -------------------------------------------------------------------------------- /src/types/client/vset.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/client/vset.h" 4 | 5 | namespace ustore { 6 | 7 | VSet::VSet(const std::vector& keys) 8 | noexcept { 9 | buffer_ = {UType::kSet, {}, 0, 0, {}, keys}; 10 | } 11 | 12 | VSet::VSet(std::shared_ptr loader, const Hash& root_hash) 13 | noexcept : USet(loader) { 14 | SetNodeForHash(root_hash); 15 | buffer_ = {UType::kSet, root_node_->hash(), 0, 0, {}, {}}; 16 | } 17 | 18 | Hash VSet::Set(const Slice& key) const { 19 | buffer_ = {UType::kSet, root_node_->hash(), 0, 0, {}, {key}}; 20 | return Hash::kNull; 21 | } 22 | 23 | Hash VSet::Remove(const Slice& key) const { 24 | buffer_ = {UType::kSet, root_node_->hash(), 0, 1, {}, {key}}; 25 | return Hash::kNull; 26 | } 27 | 28 | } // namespace ustore 29 | -------------------------------------------------------------------------------- /src/types/client/vstring.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/client/vstring.h" 4 | 5 | namespace ustore { 6 | 7 | VString::VString(const Slice& data) noexcept { 8 | buffer_ = {UType::kString, {}, 0, 0, {data}, {}}; 9 | } 10 | 11 | VString::VString(const UCell& cell) noexcept : UString(cell) { 12 | buffer_ = {UType::kString, {}, 0, 0, {slice()}, {}}; 13 | } 14 | 15 | } // namespace ustore 16 | -------------------------------------------------------------------------------- /src/types/server/sblob.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/server/sblob.h" 4 | 5 | #include "node/blob_node.h" 6 | #include "node/meta_node.h" 7 | #include "node/node_builder.h" 8 | 9 | namespace ustore { 10 | 11 | SBlob::SBlob(std::shared_ptr loader, ChunkWriter* writer, 12 | const Hash& root_hash) noexcept : UBlob(loader), chunk_writer_(writer) { 13 | SetNodeForHash(root_hash); 14 | } 15 | 16 | SBlob::SBlob(std::shared_ptr loader, ChunkWriter* writer, 17 | const Slice& data) noexcept : UBlob(loader), chunk_writer_(writer) { 18 | if (data.empty()) { 19 | ChunkInfo chunk_info = BlobChunker::Instance()->Make({}); 20 | chunk_writer_->Write(chunk_info.chunk.hash(), chunk_info.chunk); 21 | SetNodeForHash(chunk_info.chunk.hash()); 22 | } else { 23 | NodeBuilder nb(chunk_writer_, BlobChunker::Instance(), 24 | MetaChunker::Instance()); 25 | FixedSegment seg(data.data(), data.len(), 1); 26 | nb.SpliceElements(0, &seg); 27 | Hash root_hash(nb.Commit()); 28 | SetNodeForHash(root_hash); 29 | } 30 | } 31 | 32 | Hash SBlob::Splice(size_t pos, size_t num_delete, const byte_t* data, 33 | size_t num_append) const { 34 | NodeBuilder nb(root_node_->hash(), pos, chunk_loader_.get(), chunk_writer_, 35 | BlobChunker::Instance(), MetaChunker::Instance()); 36 | FixedSegment seg(data, num_append, 1); 37 | nb.SpliceElements(num_delete, &seg); 38 | return nb.Commit(); 39 | } 40 | 41 | } // namespace ustore 42 | -------------------------------------------------------------------------------- /src/types/ucell.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "types/ucell.h" 4 | 5 | #include 6 | #include "node/node_builder.h" 7 | #include "store/chunk_store.h" 8 | #include "utils/logging.h" 9 | 10 | namespace ustore { 11 | 12 | UCell UCell::Create(UType type, const Slice& key, const Slice& data, 13 | const Slice& ctx, const Hash& preHash1, 14 | const Hash& preHash2) { 15 | Chunk chunk = CellNode::NewChunk(type, key, data, ctx, preHash1, preHash2); 16 | store::GetChunkStore()->Put(chunk.hash(), chunk); 17 | return UCell(std::move(chunk)); 18 | } 19 | 20 | UCell UCell::Create(UType type, const Slice& key, const Hash& data, 21 | const Slice& ctx, const Hash& preHash1, 22 | const Hash& preHash2) { 23 | return Create(type, key, Slice(data.value(), Hash::kByteLength), ctx, 24 | preHash1, preHash2); 25 | } 26 | 27 | UCell UCell::Load(const Hash& hash) { 28 | // ucell do not need chunk loader, as it has only one chunk 29 | return UCell(store::GetChunkStore()->Get(hash)); 30 | } 31 | 32 | UCell::UCell(Chunk&& chunk) { 33 | if (chunk.empty()) { 34 | LOG(WARNING) << "Empty Chunk. Loading Failed. "; 35 | } else if (chunk.type() == ChunkType::kCell) { 36 | node_.reset(new CellNode(std::move(chunk))); 37 | } else { 38 | LOG(FATAL) << "Cannot be other chunk type for UCell" 39 | << " (actual type: " << static_cast(chunk.type()) << ")"; 40 | } 41 | } 42 | 43 | } // namespace ustore 44 | -------------------------------------------------------------------------------- /src/utils/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include "utils/env.h" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "utils/logging.h" 11 | 12 | namespace ustore { 13 | 14 | const char* Env::kDefaultConfigFile = "conf/config.cfg"; 15 | 16 | Env::Env() { 17 | const char* home_path = getenv("USTORE_HOME"); 18 | std::string config_path = kDefaultConfigFile; 19 | if (home_path == nullptr) 20 | LOG(WARNING) << "Use working dir (env USTORE_HOME not set)"; 21 | else 22 | config_path = home_path + std::string("/") + config_path; 23 | int fd = open(config_path.c_str(), O_RDONLY); 24 | if (fd == -1) { 25 | LOG(FATAL) << "Fail to load configuration (file \"" 26 | << config_path << "\" not found)"; 27 | return; 28 | } 29 | LOG(INFO) << "Load config \"" << config_path << "\""; 30 | google::protobuf::io::FileInputStream input{fd}; 31 | google::protobuf::TextFormat::Parse(&input, &config_); 32 | LOG(INFO) << "Loaded config:" << std::endl << config_.DebugString(); 33 | // handle relative paths for data_dir and worker_file 34 | if (home_path && config_.data_dir()[0] != '/') 35 | config_.set_data_dir(home_path + std::string("/") + config_.data_dir()); 36 | if (home_path && config_.worker_file()[0] != '/') 37 | config_.set_worker_file(home_path + std::string("/") 38 | + config_.worker_file()); 39 | close(fd); 40 | } 41 | } // namespace ustore 42 | -------------------------------------------------------------------------------- /src/worker/README.md: -------------------------------------------------------------------------------- 1 | # UStore Worker # 2 | 3 | ## Dependency Resolution ## 4 | 5 | ### RocksDB ### 6 | 7 | Since [RocksDB v5.8](http://rocksdb.org/blog/2017/09/28/rocksdb-5-8-released.html), the library compiled with release mode is built with `-fno-rtti`, whereas that with debug mode is without it. However, abandoning the RTTI information with release build may cause the "Undefined Reference to Typeinfo" error. Therefore, when building the RocksDB library, the `USE_RTTI` environmental variable should be set to 1 prior to launch the `make` command. For example, 8 | 9 | $ USE_RTTI=1 DISABLE_WARNING_AS_ERROR=ON bash -c 'make shared_lib' && sudo make install-shared 10 | 11 | this is to build the RocksDB shared library and install it to the default path. For building/installing the static library, use the command below: 12 | 13 | $ USE_RTTI=1 DISABLE_WARNING_AS_ERROR=ON bash -c 'make static_lib' && sudo make install-static 14 | 15 | At times, you may prefer customized install path: 16 | 17 | $ USE_RTTI=1 DISABLE_WARNING_AS_ERROR=ON bash -c 'make -j shared_lib' && INSTALL_PATH=/path/to/install bash -c 'make install-shared' 18 | 19 | ## Compilation ## 20 | 21 | To enable RocksDB-based head version table, `USE_SIMPLE_HEAD_VERSION=OFF` and `USE_ROCKSDB=ON` should be passed to the `cmake` command. For example, 22 | 23 | $ cmake -DUSE_SIMPLE_HEAD_VERSION=OFF -DUSE_ROCKSDB=ON .. 24 | -------------------------------------------------------------------------------- /src/worker/worker_ext.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "utils/utils.h" 4 | #include "worker/worker_ext.h" 5 | 6 | namespace ustore { 7 | 8 | ErrorCode WorkerExt::GetForType(const UType& type, const Slice& key, 9 | const Hash& ver, UCell* ucell) { 10 | USTORE_GUARD(Get(key, ver, ucell)); 11 | if (ucell->type() != type) { 12 | LOG(ERROR) << "Type mismatch: [Actual] " << ucell->type() 13 | << ", [Expected] " << type; 14 | return ErrorCode::kTypeMismatch; 15 | } 16 | return ErrorCode::kOK; 17 | } 18 | 19 | } // namespace ustore 20 | -------------------------------------------------------------------------------- /test/benchmark/dist_bench_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include 4 | #include 5 | #include 6 | #include "benchmark/benchmark.h" 7 | #include "benchmark/bench_config.h" 8 | #include "cluster/worker_client_service.h" 9 | #include "spec/object_db.h" 10 | #include "utils/env.h" 11 | 12 | using namespace ustore; 13 | 14 | constexpr int kSleepTime = 100000; 15 | 16 | void BenchmarkClient() { 17 | // create client service 18 | WorkerClientService service; 19 | service.Run(); 20 | 21 | // create client 22 | size_t n_client = BenchmarkConfig::num_clients; 23 | std::vector clientdbs; 24 | std::vector dbs; 25 | for (size_t i = 0; i < n_client; ++i) 26 | clientdbs.push_back(service.CreateWorkerClient()); 27 | for (auto& db : clientdbs) 28 | dbs.push_back(new ObjectDB(&db)); 29 | Benchmark bm(dbs); 30 | std::cout << "============================\n"; 31 | std::cout << "Benchmarking " << n_client 32 | << " clients connected to ustore service.......\n"; 33 | bm.Run(); 34 | 35 | service.Stop(); 36 | for (auto& p : dbs) delete p; 37 | } 38 | 39 | int main(int argc, char* argv[]) { 40 | if (BenchmarkConfig::ParseCmdArgs(argc, argv)) { 41 | if (BenchmarkConfig::is_help) return 0; 42 | } else { 43 | std::cerr << BOLD_RED("[FAILURE] ") 44 | << "Found invalid command-line option" << std::endl; 45 | return -1; 46 | } 47 | BenchmarkClient(); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /test/gtest/gtest_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2006, Google Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // * Neither the name of Google Inc. nor the names of its 15 | // contributors may be used to endorse or promote products derived from 16 | // this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | #include 31 | #include 32 | 33 | #include "gtest/gtest.h" 34 | 35 | #ifndef GFLAGS_GFLAGS_H_ 36 | namespace gflags = google; 37 | #endif // GFLAGS_GFLAGS_H_ 38 | 39 | GTEST_API_ int main(int argc, char **argv) { 40 | printf("Running main() from gtest_main.cc\n"); 41 | testing::InitGoogleTest(&argc, argv); 42 | gflags::ParseCommandLineFlags(&argc, &argv, true); 43 | return RUN_ALL_TESTS(); 44 | } 45 | -------------------------------------------------------------------------------- /test/ustore/test_ca.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "gtest/gtest.h" 4 | #include "spec/relational.h" 5 | #include "worker/worker.h" 6 | 7 | using ustore::ColumnStore; 8 | using ustore::Column; 9 | 10 | inline void ASSERT_SUCCEEDED(ustore::ErrorCode code) { 11 | ASSERT_EQ(code, ustore::ErrorCode::kOK); 12 | } 13 | 14 | void ASSERT_FAILED(ustore::ErrorCode code) { 15 | ASSERT_NE(code, ustore::ErrorCode::kOK); 16 | } 17 | 18 | TEST(Relational, Basic) { 19 | ustore::Worker worker(2017, nullptr, false); 20 | ColumnStore cs(&worker); 21 | 22 | const std::string table_name("Test"); 23 | ASSERT_SUCCEEDED(cs.CreateTable(table_name, "master")); 24 | 25 | const std::string col1_name("col1"); 26 | const std::vector expected_col1{"col1_v1", "col1_v2"}; 27 | 28 | ASSERT_SUCCEEDED( 29 | cs.PutColumn(table_name, "master", col1_name, expected_col1)); 30 | 31 | Column actual_col1; 32 | ASSERT_SUCCEEDED( 33 | cs.GetColumn(table_name, "master", col1_name, &actual_col1)); 34 | 35 | ASSERT_EQ(size_t(2), actual_col1.numElements()); 36 | 37 | ASSERT_SUCCEEDED( 38 | cs.DeleteColumn(table_name, "master", col1_name)); 39 | 40 | ASSERT_FAILED( 41 | cs.GetColumn(table_name, "master", col1_name, &actual_col1)); 42 | 43 | std::string new_branch("branch"); 44 | ASSERT_SUCCEEDED( 45 | cs.BranchTable(table_name, "master", new_branch)); 46 | 47 | const std::string col2_name("col2"); 48 | const std::vector expected_col2{"col2_v1", "col2_v2", "col2_v3"}; 49 | 50 | ASSERT_SUCCEEDED( 51 | cs.PutColumn(table_name, new_branch, col2_name, expected_col2)); 52 | 53 | Column actual_col2; 54 | ASSERT_SUCCEEDED( 55 | cs.GetColumn(table_name, new_branch, col2_name, &actual_col2)); 56 | 57 | ASSERT_EQ(size_t(3), actual_col2.numElements()); 58 | 59 | const std::string col3_name("col3"); 60 | const std::vector expected_col3{"col3_v1"}; 61 | 62 | ASSERT_SUCCEEDED( 63 | cs.MergeTable(table_name, new_branch, "master", col3_name, expected_col3)); 64 | 65 | Column actual_col3; 66 | ASSERT_SUCCEEDED( 67 | cs.GetColumn(table_name, new_branch, col3_name, &actual_col3)); 68 | 69 | ASSERT_EQ(size_t(1), actual_col3.numElements()); 70 | } 71 | -------------------------------------------------------------------------------- /test/ustore/test_chunk.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include 3 | #include 4 | #include "gtest/gtest.h" 5 | #include "chunk/chunk.h" 6 | #include "hash/hash.h" 7 | 8 | const ustore::byte_t raw_data[] = "The quick brown fox jumps over the lazy dog"; 9 | 10 | TEST(Chunk, CreateNewChunk) { 11 | const ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 12 | std::memcpy(chunk.m_data(), raw_data, sizeof(raw_data)); 13 | // check chunk type 14 | EXPECT_EQ(ustore::ChunkType::kBlob, chunk.type()); 15 | // check chunk size 16 | size_t expected_size = sizeof(raw_data) + ustore::Chunk::kMetaLength; 17 | EXPECT_EQ(expected_size, chunk.numBytes()); 18 | // check capacity 19 | EXPECT_EQ(sizeof(raw_data), chunk.capacity()); 20 | // check content 21 | EXPECT_EQ(0, std::memcmp(raw_data, chunk.data(), sizeof(raw_data))); 22 | // check chunk hash 23 | ustore::Hash h = ustore::Hash::ComputeFrom(chunk.head(), chunk.numBytes()); 24 | EXPECT_EQ(h.ToBase32(), chunk.hash().ToBase32()); 25 | } 26 | 27 | TEST(Chunk, LoadChunk) { 28 | const ustore::Chunk origin(ustore::ChunkType::kBlob, sizeof(raw_data)); 29 | std::memcpy(origin.m_data(), raw_data, sizeof(raw_data)); 30 | ustore::byte_t* buffer = new ustore::byte_t[origin.numBytes()]; 31 | std::memcpy(buffer, origin.head(), origin.numBytes()); 32 | const ustore::Chunk chunk(buffer); 33 | // check chunk type 34 | EXPECT_EQ(ustore::ChunkType::kBlob, chunk.type()); 35 | // check chunk size 36 | size_t expected_size = sizeof(raw_data) + ustore::Chunk::kMetaLength; 37 | EXPECT_EQ(expected_size, chunk.numBytes()); 38 | // check capacity 39 | EXPECT_EQ(sizeof(raw_data), chunk.capacity()); 40 | // check content 41 | EXPECT_EQ(0, std::memcmp(raw_data, chunk.data(), sizeof(raw_data))); 42 | // check chunk hash 43 | ustore::Hash h = ustore::Hash::ComputeFrom(chunk.head(), chunk.numBytes()); 44 | EXPECT_EQ(h.ToBase32(), chunk.hash().ToBase32()); 45 | delete[] buffer; 46 | } 47 | -------------------------------------------------------------------------------- /test/ustore/test_chunk_loader.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include 4 | #include 5 | #include 6 | #include "gtest/gtest.h" 7 | #include "chunk/chunk_loader.h" 8 | #include "store/chunk_store.h" 9 | 10 | const ustore::byte_t raw_data[] = "The quick brown fox jumps over the lazy dog"; 11 | 12 | TEST(ChunkLoader, GetChunk) { 13 | ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 14 | std::copy(raw_data, raw_data + sizeof(raw_data), chunk.m_data()); 15 | ustore::ChunkStore* cs = ustore::store::GetChunkStore(); 16 | EXPECT_TRUE(cs->Put(chunk.hash(), chunk)); 17 | ustore::LocalChunkLoader cl; 18 | // load from stroage 19 | const ustore::Chunk* c = cl.Load(chunk.hash()); 20 | EXPECT_EQ(c->hash(), chunk.hash()); 21 | EXPECT_EQ(c->type(), ustore::ChunkType::kBlob); 22 | EXPECT_EQ(c->numBytes(), chunk.numBytes()); 23 | EXPECT_EQ(c->capacity(), chunk.capacity()); 24 | // load from cache 25 | c = cl.Load(chunk.hash()); 26 | EXPECT_EQ(c->hash(), chunk.hash()); 27 | EXPECT_EQ(c->type(), ustore::ChunkType::kBlob); 28 | EXPECT_EQ(c->numBytes(), chunk.numBytes()); 29 | EXPECT_EQ(c->capacity(), chunk.capacity()); 30 | } 31 | -------------------------------------------------------------------------------- /test/ustore/test_hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "gtest/gtest.h" 7 | #include "hash/hash.h" 8 | 9 | const ustore::byte_t raw_str[] = "The quick brown fox jumps over the lazy dog"; 10 | #ifdef USE_BLAKE2b 11 | const char base32_encoded[] = "VCW5JPO57WJ6JB35E5DOMKAXWELDMSQ7"; 12 | const char hash_hex_str[] = "a8add4bdddfd93e4877d2746e62817b116364a1f"; 13 | #else 14 | const char base32_encoded[] = "26UPXMYH26AJI2OKTK6LACBOJ6GVMUPE"; 15 | const char hash_hex_str[] = "d7a8fbb307d7809469ca9abcb0082e4f8d5651e4"; 16 | #endif 17 | TEST(Hash, FromBase32) { 18 | ustore::Hash h = ustore::Hash::FromBase32(base32_encoded); 19 | EXPECT_EQ(base32_encoded, h.ToBase32()); 20 | } 21 | 22 | TEST(Hash, ComputeHash) { 23 | ustore::Hash h = ustore::Hash::ComputeFrom(raw_str, 43); 24 | EXPECT_EQ(base32_encoded, h.ToBase32()); 25 | std::ostringstream stm; 26 | for (size_t i = 0; i < ustore::Hash::kByteLength; ++i) { 27 | stm << std::hex << std::setfill('0') << std::setw(2) 28 | << uint32_t(*(h.value() + i)); 29 | } 30 | EXPECT_EQ(hash_hex_str, stm.str()); 31 | } 32 | 33 | TEST(Hash, IsEmpty) { 34 | ustore::Hash h; 35 | EXPECT_TRUE(h.empty()); 36 | h = ustore::Hash::kNull; 37 | EXPECT_FALSE(h.empty()); 38 | EXPECT_EQ(h, ustore::Hash::kNull); 39 | } 40 | 41 | TEST(Hash, Clone) { 42 | ustore::Hash h = ustore::Hash::kNull.Clone(); 43 | EXPECT_FALSE(h.empty()); 44 | EXPECT_EQ(h, ustore::Hash::kNull); 45 | } 46 | 47 | TEST(Hash, Movable) { 48 | ustore::Hash old = ustore::Hash::FromBase32(base32_encoded); 49 | EXPECT_TRUE(old.own()); 50 | ustore::Hash h = std::move(old); 51 | EXPECT_FALSE(old.own()); 52 | EXPECT_TRUE(h.own()); 53 | EXPECT_EQ(base32_encoded, h.ToBase32()); 54 | } 55 | -------------------------------------------------------------------------------- /test/ustore/test_ldb_store.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #ifdef USE_LEVELDB 3 | #include 4 | #include 5 | #include 6 | #include "gtest/gtest.h" 7 | #include "store/ldb_store.h" 8 | 9 | static ustore::LDBStore* ldb = ustore::LDBStore::Instance(); 10 | const ustore::byte_t raw_data[] = "The quick brown fox jumps over the lazy dog"; 11 | 12 | TEST(LDBStore, PutChunk) { 13 | ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 14 | std::copy(raw_data, raw_data + sizeof(raw_data), chunk.m_data()); 15 | EXPECT_TRUE(ldb->Put(chunk.hash(), chunk)); 16 | } 17 | 18 | TEST(LDBStore, GetChunk) { 19 | ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 20 | std::copy(raw_data, raw_data + sizeof(raw_data), chunk.m_data()); 21 | ustore::Chunk c = ldb->Get(chunk.hash()); 22 | EXPECT_EQ(c.forceHash(), chunk.hash()); 23 | EXPECT_EQ(c.type(), ustore::ChunkType::kBlob); 24 | EXPECT_EQ(c.numBytes(), chunk.numBytes()); 25 | EXPECT_EQ(c.capacity(), chunk.capacity()); 26 | } 27 | #endif // USE_LEVELDB 28 | -------------------------------------------------------------------------------- /test/ustore/test_log_record.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include"gtest/gtest.h" 9 | #include"recovery/log_record.h" 10 | #include"utils/logging.h" 11 | 12 | char testkey[] = "12djasfkj4iadusf4efjsdf"; 13 | char testvalue[] = "odadesades1sda1645f"; 14 | int64_t checksum_result = 1237; 15 | 16 | TEST(Recovery, LogRecordChecksum) { 17 | ustore::recovery::LogRecord record; 18 | EXPECT_EQ(record.version, 1); 19 | EXPECT_EQ(record.key, nullptr); 20 | EXPECT_EQ(record.value, nullptr); 21 | record.logcmd = ustore::recovery::LogCommand::kUpdate; 22 | record.log_sequence_number = 11; 23 | record.key = reinterpret_cast(testkey); 24 | record.value = reinterpret_cast(testvalue); 25 | record.key_length = strlen(testkey); 26 | record.value_length = strlen(testvalue); 27 | int64_t checksum_ret = record.ComputeChecksum(); 28 | DLOG(INFO) << "returned checksum: " << checksum_ret; 29 | } 30 | 31 | TEST(Recovery, LogRecordToString) { 32 | ustore::recovery::LogRecord record; 33 | record.logcmd = ustore::recovery::LogCommand::kUpdate; 34 | record.log_sequence_number = 11; 35 | record.key = reinterpret_cast(testkey); 36 | record.value = reinterpret_cast(testvalue); 37 | record.key_length = strlen(testkey); 38 | record.value_length = strlen(testvalue); 39 | record.ComputeChecksum(); 40 | std::string str = record.ToString(); 41 | EXPECT_EQ(record.data_length, 86); 42 | } 43 | -------------------------------------------------------------------------------- /test/ustore/test_logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include "utils/logging.h" 3 | #include "gtest/gtest.h" 4 | 5 | TEST(Logging, InfoLogging) { 6 | ustore::InitLogging(""); 7 | int a = 3; 8 | CHECK_EQ(a, 3); 9 | LOG(INFO) << "test info logging"; 10 | } 11 | 12 | TEST(Logging, WarningLogging) { 13 | int a = 4; 14 | CHECK_EQ(a, 4); 15 | LOG(WARNING) << "test warning logging"; 16 | } 17 | 18 | TEST(Logging, ErrorLogging) { 19 | int a = 5; 20 | CHECK_EQ(a, 5); 21 | LOG(ERROR) << "test error logging"; 22 | } 23 | 24 | /* 25 | TEST(Logging, FatalLogging) { 26 | int a = 6; 27 | CHECK_EQ(a, 6); 28 | LOG(FATAL) << "test fatal logging"; 29 | } 30 | */ 31 | 32 | TEST(Logging, SetLogDestination) { 33 | int a = 6; 34 | ustore::SetLogDestination(ustore::WARNING, "test_logging.log"); 35 | CHECK_EQ(a, 6); 36 | LOG(WARNING) << "test warning logging to file"; 37 | } 38 | 39 | TEST(Logging, StderrLoggingLevel) { 40 | int a = 6; 41 | ustore::SetStderrLogging(ustore::WARNING); 42 | CHECK_EQ(a, 6); 43 | LOG(INFO) << "test info logging to stderr"; 44 | LOG(WARNING) << "test warning logging to stderr and file"; 45 | LOG(ERROR) << "test error logging to stderr and file"; 46 | ustore::SetStderrLogging(ustore::INFO); 47 | } 48 | 49 | #ifdef DEBUG 50 | TEST(Logging, DebugInfoLogging) { 51 | ustore::InitLogging(""); 52 | int a = 3; 53 | DCHECK_EQ(a, 3); 54 | DLOG(INFO) << "test debug info logging"; 55 | } 56 | 57 | TEST(Logging, DebugWarningLogging) { 58 | int a = 4; 59 | DCHECK_EQ(a, 4); 60 | DLOG(WARNING) << "test debug warning logging"; 61 | } 62 | 63 | TEST(Logging, DebugErrorLogging) { 64 | int a = 5; 65 | DCHECK_EQ(a, 5); 66 | DLOG(ERROR) << "test debug error logging"; 67 | } 68 | #endif // ifdef DEBUG 69 | -------------------------------------------------------------------------------- /test/ustore/test_node.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "gtest/gtest.h" 8 | 9 | #include "node/node.h" 10 | 11 | TEST(MetaEntry, EncodeDecode) { 12 | uint32_t num_leaves = 5; 13 | uint64_t num_elements = 10; 14 | 15 | const char encoded_hash_value[] = "26UPXMYH26AJI2OKTK6LACBOJ6GVMUPE"; 16 | ustore::Hash data_hash = ustore::Hash::FromBase32(encoded_hash_value); 17 | 18 | // construct order key from above hash 19 | // padding a false bool in front of hash value to indicate not by value 20 | size_t key_num_bytes = ustore::Hash::kByteLength; 21 | ustore::byte_t* key_value = new ustore::byte_t[key_num_bytes]; 22 | std::memcpy(key_value, data_hash.value(), 23 | ustore::Hash::kByteLength); 24 | ustore::OrderedKey key(false, key_value, key_num_bytes); 25 | 26 | size_t encode_len = 0; 27 | const ustore::byte_t* me_bytes = ustore::MetaEntry::Encode( 28 | num_leaves, num_elements, data_hash, key, &encode_len); 29 | 30 | EXPECT_EQ(encode_len, 2 * sizeof(uint32_t) 31 | + sizeof(uint64_t) 32 | + ustore::Hash::kByteLength 33 | + key_num_bytes 34 | + sizeof(bool)); 35 | 36 | const ustore::MetaEntry me(me_bytes); 37 | 38 | EXPECT_EQ(encode_len, me.numBytes()); 39 | EXPECT_EQ(num_leaves, me.numLeaves()); 40 | EXPECT_EQ(num_elements, me.numElements()); 41 | EXPECT_EQ(data_hash, me.targetHash()); 42 | EXPECT_EQ(key, me.orderedKey()); 43 | 44 | delete[] key_value; 45 | delete[] me_bytes; 46 | } 47 | -------------------------------------------------------------------------------- /test/ustore/test_orderedkey.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | #include 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "node/orderedkey.h" 8 | 9 | TEST(OrderedKey, CompareValueKey) { 10 | ustore::OrderedKey k1(10); 11 | ustore::OrderedKey k2(5); 12 | ustore::OrderedKey k3(15); 13 | ustore::OrderedKey kk2(5); 14 | 15 | EXPECT_TRUE(k1 > k2); 16 | EXPECT_TRUE(k1 < k3); 17 | EXPECT_TRUE(k1 <= k3); 18 | EXPECT_TRUE(k2 == kk2); 19 | EXPECT_TRUE(k2 >= kk2); 20 | } 21 | 22 | TEST(OrderedKey, CompareByte) { 23 | ustore::byte_t d1[] = "abc"; 24 | ustore::byte_t d2[] = "efg"; 25 | ustore::byte_t dd2[] = "efg"; 26 | ustore::byte_t d3[] = "aaaa"; 27 | 28 | bool by_value = false; 29 | ustore::OrderedKey k1(by_value, d1, 3); 30 | ustore::OrderedKey k2(by_value, d2, 3); 31 | ustore::OrderedKey kk2(by_value, dd2, 3); 32 | ustore::OrderedKey k3(by_value, d3, 4); 33 | 34 | EXPECT_TRUE(k1 > k3); 35 | EXPECT_TRUE(k1 < k2); 36 | EXPECT_TRUE(k3 <= k2); 37 | EXPECT_TRUE(k2 == kk2); 38 | EXPECT_TRUE(k2 >= kk2); 39 | } 40 | 41 | TEST(OrderedKey, Encode) { 42 | ustore::byte_t* buffer = new ustore::byte_t[10]; 43 | ustore::OrderedKey k1(10); 44 | size_t len = k1.Encode(buffer); 45 | 46 | EXPECT_EQ(len, sizeof(uint64_t)); 47 | uint64_t key = *(reinterpret_cast(buffer)); 48 | EXPECT_EQ(size_t(10), key); 49 | 50 | delete[] buffer; 51 | 52 | buffer = new ustore::byte_t[10]; 53 | ustore::byte_t d[] = "efg"; 54 | ustore::OrderedKey k2(false, d, 3); 55 | len = k2.Encode(buffer); 56 | 57 | EXPECT_EQ(size_t(3), len); 58 | EXPECT_EQ(0, std::memcmp(buffer, d, 3)); 59 | 60 | ustore::OrderedKey kk2(false, buffer, 3); 61 | EXPECT_EQ(k2.numBytes(), kk2.numBytes()); 62 | delete[] buffer; 63 | } 64 | 65 | -------------------------------------------------------------------------------- /test/ustore/test_random_generator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "gtest/gtest.h" 4 | #include "benchmark/bench_utils.h" 5 | 6 | using namespace ustore; 7 | using Vec = std::vector; 8 | 9 | size_t test_size = 10; 10 | size_t str_length = 32; 11 | size_t blob_size = 4096; 12 | 13 | TEST(RandomGenerator, RG_SequentialNumString) { 14 | RandomGenerator rg; 15 | Vec keys = rg.SequentialNumString("", test_size); 16 | EXPECT_EQ(test_size, keys.capacity()); 17 | EXPECT_EQ(size_t(1), keys[1].length()); 18 | } 19 | 20 | TEST(RandomGenerator, RG_NFixedString) { 21 | RandomGenerator rg; 22 | Vec keys = rg.NFixedString(test_size, str_length); 23 | EXPECT_EQ(test_size, keys.capacity()); 24 | EXPECT_EQ(str_length, keys[1].length()); 25 | } 26 | 27 | TEST(RandomGenerator, RG_NRandomString) { 28 | RandomGenerator rg; 29 | Vec keys = rg.NRandomString(test_size, str_length); 30 | EXPECT_EQ(test_size, keys.capacity()); 31 | EXPECT_GE(str_length, keys[1].length()); 32 | } 33 | 34 | TEST(RandomGenerator, RG_FixedString) { 35 | RandomGenerator rg; 36 | std::string str = rg.FixedString(str_length); 37 | EXPECT_EQ(str_length, str.length()); 38 | } 39 | 40 | TEST(RandomGenerator, RG_RandomString) { 41 | RandomGenerator rg; 42 | std::string str = rg.RandomString(str_length); 43 | EXPECT_GE(str_length, str.length()); 44 | } 45 | -------------------------------------------------------------------------------- /test/ustore/test_rocks_store.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #if defined(USE_ROCKSDB) 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "store/rocks_store.h" 8 | 9 | static ustore::RocksStore* rs = ustore::RocksStore::Instance(); 10 | const ustore::byte_t raw_data[] = "The quick brown fox jumps over the lazy dog"; 11 | 12 | TEST(RocksStore, PutChunk) { 13 | ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 14 | std::copy(raw_data, raw_data + sizeof(raw_data), chunk.m_data()); 15 | EXPECT_TRUE(rs->Put(chunk.hash(), chunk)); 16 | } 17 | 18 | TEST(RocksStore, GetChunk) { 19 | ustore::Chunk chunk(ustore::ChunkType::kBlob, sizeof(raw_data)); 20 | std::copy(raw_data, raw_data + sizeof(raw_data), chunk.m_data()); 21 | ustore::Chunk c = rs->Get(chunk.hash()); 22 | EXPECT_EQ(c.forceHash(), chunk.hash()); 23 | EXPECT_EQ(c.type(), ustore::ChunkType::kBlob); 24 | EXPECT_EQ(c.numBytes(), chunk.numBytes()); 25 | EXPECT_EQ(c.capacity(), chunk.capacity()); 26 | } 27 | 28 | #endif // USE_ROCKSDB 29 | -------------------------------------------------------------------------------- /test/ustore/test_vstring.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The Ustore Authors. 2 | 3 | #include "gtest/gtest.h" 4 | 5 | #include "spec/object_db.h" 6 | #include "types/client/vstring.h" 7 | #include "worker/worker.h" 8 | 9 | const char raw_data[] = "The quick brown fox jumps over the lazy dog"; 10 | 11 | using ustore::byte_t; 12 | using ustore::Slice; 13 | using ustore::VMeta; 14 | using ustore::ErrorCode; 15 | using ustore::Hash; 16 | 17 | const char key_vstring[] = "key_vstring"; 18 | const char branch_vstring[] = "branch_vstring"; 19 | const char ctx_vstring[] = "ctx_vstring"; 20 | 21 | ustore::Worker& worker_vstring() { 22 | static ustore::Worker* worker = new ustore::Worker(1995, nullptr, false); 23 | return *worker; 24 | } 25 | 26 | TEST(VString, CreateFromEmpty) { 27 | ustore::ObjectDB db(&worker_vstring()); 28 | Slice empty; 29 | ustore::VString string(empty); 30 | 31 | auto put = db.Put(Slice(key_vstring), string, Slice(branch_vstring)); 32 | EXPECT_TRUE(ErrorCode::kOK == put.stat); 33 | // get string 34 | auto get = db.Get(Slice(key_vstring), Slice(branch_vstring)); 35 | EXPECT_TRUE(ErrorCode::kOK == get.stat); 36 | auto v = get.value.String(); 37 | 38 | // check data 39 | EXPECT_EQ(size_t(0), v.len()); 40 | EXPECT_EQ(nullptr, v.data()); 41 | } 42 | 43 | 44 | TEST(VString, CreateNewVString) { 45 | ustore::ObjectDB db(&worker_vstring()); 46 | // create buffered new string 47 | ustore::VString string{Slice(raw_data)}; 48 | string.SetContext(Slice(ctx_vstring)); 49 | // put new string 50 | auto put = db.Put(Slice(key_vstring), string, Slice(branch_vstring)); 51 | EXPECT_TRUE(ErrorCode::kOK == put.stat); 52 | // get string 53 | auto get = db.Get(Slice(key_vstring), Slice(branch_vstring)); 54 | EXPECT_TRUE(ErrorCode::kOK == get.stat); 55 | // check context 56 | EXPECT_EQ(Slice(ctx_vstring), get.value.cell().context()); 57 | auto v = get.value.String(); 58 | // check data 59 | EXPECT_EQ(0, memcmp(raw_data, v.data(), v.len())); 60 | } 61 | 62 | TEST(VString, DestructWorker) { 63 | delete &worker_vstring(); 64 | } 65 | --------------------------------------------------------------------------------