├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── connection_handler.cc ├── db.cc ├── example_client.c ├── facebook_combined.txt ├── include ├── connection_handler.hh ├── db.hh ├── net_server.hh ├── redis_protocol_parser.c ├── redis_protocol_parser.hh ├── redis_protocol_parser.rl ├── reply_builder.hh ├── req_server.hh └── seastarkv.hh ├── main.cc ├── net_server.cc ├── reply_builder.cc ├── req_server.cc ├── scripts.js ├── setup_huge_pages.sh ├── setup_seastar.sh ├── setup_v8.sh └── ycsb ├── Makefile ├── ycsb.cc ├── ycsb_graph.cc ├── ycsb_nn.cc └── ycsb_throughput.cc /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "seastar"] 2 | path = seastar 3 | url = https://github.com/scylladb/seastar.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Utah Scalable Computer Systems 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all : shredder example_client 2 | shredder : main.cc connection_handler.cc net_server.cc req_server.cc db.cc 3 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -o shredder.o -c main.cc 4 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -o connection_handler.o -c connection_handler.cc 5 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -o net_server.o -c net_server.cc 6 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -o req_server.o -c req_server.cc 7 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -o db.o -c db.cc 8 | gcc -std=c++14 -g -O2 -Wall -Werror -Wno-unused-variable -Wno-error=deprecated-declarations -fexceptions -fvisibility=hidden -pthread -U_FORTIFY_SOURCE -DHAVE_DPDK -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -Iv8/ -Iv8/include -Iv8/include/libplatform/ -Iv8/third_party/icu/source/i18n -Iv8/third_party/icu/source/common -fdata-sections -ffunction-sections -g -MMD -DMODE=NATIVE -I./seastar -I./seastar/build/dpdk/include -L./seastar/build/release -L./seastar/build/dpdk/lib -Wl,--whole-archive,-lseastar,--no-whole-archive -g -Wall -Werror -Wno-error=deprecated-declarations -fvisibility=hidden -I./seastar -U_FORTIFY_SOURCE -DHAVE_DPDK -I./seastar/build/dpdk/include -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 -DRTE_MACHINE_CPUFLAG_AES -DRTE_MACHINE_CPUFLAG_PCLMULQDQ -DRTE_MACHINE_CPUFLAG_AVX -DRTE_MACHINE_CPUFLAG_RDRAND -DRTE_MACHINE_CPUFLAG_FSGSBASE -DRTE_MACHINE_CPUFLAG_F16C -DRTE_MACHINE_CPUFLAG_AVX2 -Wno-error=literal-suffix -Wno-literal-suffix -Wno-invalid-offsetof -I./seastar/fmt -I. -DBOOST_TEST_DYN_LINK -Wno-overloaded-virtual -Wno-maybe-uninitialized -DFMT_HEADER_ONLY -DHAVE_HWLOC -DHAVE_NUMA -DHAVE_LZ4_COMPRESS_DEFAULT -laio -lboost_program_options -lboost_system -lboost_filesystem -lstdc++ -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -lprotobuf -ldl -lgcc_s -lunwind -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_bnxt -lrte_pmd_cxgbe -lrte_pmd_ena -lrte_pmd_enic -lrte_pmd_fm10k -lrte_pmd_nfp -lrte_pmd_qede -lrte_pmd_sfc_efx -lrte_hash -lrte_kvargs -lrte_mbuf -lrte_ethdev -lrte_eal -lrte_mempool -lrte_ring -lrte_cmdline -lrte_cfgfile -Wl,--no-whole-archive -lrt -lm -ldl -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar -L./seastar/build/release -Lv8/x64.release/obj/ -o shredder -Wl,--start-group shredder.o db.o req_server.o net_server.o connection_handler.o -lv8_monolith -Wl,--end-group -ldl -lrt -pthread 9 | 10 | example_client : example_client.c 11 | gcc -o example_client example_client.c -lhiredis 12 | 13 | .PHONY: clean 14 | 15 | clean: 16 | rm -f *.o shredder example_client shredder.d 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shredder 2 | Shredder has been tested on Ubuntu 18.04. 3 | To compile Shredder: 4 | 1. Run ./setup_v8.sh to fetch V8 code and compile V8. 5 | 2. Run ./setup_seastar.sh to fetch Seastar code and compile Seastar. 6 | 3. Run make 7 | 8 | To run Shredder: 9 | 1. Run ./shredder & in background. Use -c to run Shredder on multiple cores, for example "./shredder -c2" runs Shredder on 2 cores. 10 | 2. example_client.c is an example Shredder client implementation. 11 | Shredder uses Redis protocol, so Redis client library such as hredis can be used to communicate with Shredder server. 12 | Shredder adds a new command "JS" to run JavaScript functions. Example JS functions can be found in "script.js", including a dataset setup function, a simple key value query function, a graph traversal function and a neural model predict function. 13 | 14 | For DPDK, please follow instructions in seastar/dpdk to setup DPDK. To run Shredder with DPDK, run "sudo ./shredder --network-stack native --dpdk-pmd". 15 | 16 | # Caveats 17 | 18 | This is a research prototype, so many features are incomplete or hacky. 19 | For example, tenant support is not completely implemented, currently Shredder assigns a new tenant ID for each new incoming network connection, which cannot be used in real production. 20 | -------------------------------------------------------------------------------- /connection_handler.cc: -------------------------------------------------------------------------------- 1 | #include "include/connection_handler.hh" 2 | #include "include/req_server.hh" 3 | 4 | using namespace seastar; 5 | void connection_handler::prepare_request() 6 | { 7 | _request_args._command_args_count = _parser._args_count - 1; 8 | _request_args._command_args = std::move(_parser._args_list); 9 | _request_args._tmp_keys.clear(); 10 | _request_args._tmp_key_values.clear(); 11 | _request_args._tmp_key_scores.clear(); 12 | _request_args._tmp_key_value_pairs.clear(); 13 | } 14 | 15 | inline unsigned getcpu(const sstring& key) { 16 | return std::hash()(key) % smp::count; 17 | } 18 | 19 | future<> connection_handler::handle(input_stream& in, output_stream& out, int& tid) { 20 | _parser.init(); 21 | 22 | // NOTE: The command is handled sequentially. The parser will control the lifetime 23 | // of every parameters for command. 24 | return in.consume(_parser).then([this, &in, &out, &tid] () -> future<> { 25 | switch (_parser._state) { 26 | case redis_protocol_parser::state::eof: 27 | printf("Parser eof\n"); 28 | case redis_protocol_parser::state::error: 29 | printf("Parser error\n"); 30 | return make_ready_future<>(); 31 | 32 | case redis_protocol_parser::state::ok: 33 | { 34 | prepare_request(); 35 | switch (_parser._command) { 36 | case redis_protocol_parser::command::set: 37 | { 38 | return local_req_server().set(_request_args, std::ref(out), tid); 39 | } 40 | case redis_protocol_parser::command::get: 41 | { 42 | return local_req_server().get(std::move(std::ref(_request_args)), std::ref(out)); 43 | } 44 | // Shredder adds JS command to run JavaScript functions 45 | case redis_protocol_parser::command::js: 46 | return local_req_server().js_req(std::move(std::ref(_request_args)), std::ref(out), tid); 47 | case redis_protocol_parser::command::info: 48 | return out.write("$0\r\n\r\n").then([&out] () { 49 | return out.flush(); 50 | }); 51 | case redis_protocol_parser::command::quit: 52 | return out.write(msg_ok); 53 | 54 | default: 55 | //tracer.incr_number_exceptions(); 56 | return out.write("+Not Implemented"); 57 | }; 58 | } 59 | default: 60 | //tracer.incr_number_exceptions(); 61 | return out.write("+Error\r\n"); 62 | }; 63 | std::abort(); 64 | }); 65 | } 66 | -------------------------------------------------------------------------------- /db.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "include/seastarkv.hh" 3 | #include "include/db.hh" 4 | #include "include/reply_builder.hh" 5 | 6 | using namespace std; 7 | 8 | distributed db; 9 | 10 | future<> database::start() { 11 | for (int i = 0; i < NUM_CONTEXTS; i++) 12 | ht[i].table = NULL; 13 | return make_ready_future<>(); 14 | } 15 | 16 | future<> database::stop() { 17 | return make_ready_future<>(); 18 | } 19 | 20 | future>> 21 | database::get_direct(uint32_t key, int tid) 22 | { 23 | using return_type = foreign_ptr>; 24 | db_val* val = ht_get(&ht[tid], key); 25 | if (!val) { 26 | val = (db_val*)malloc(sizeof(db_val)); 27 | val->length = 0; 28 | } 29 | return make_ready_future( 30 | foreign_ptr>(make_lw_shared(*val))); 31 | } 32 | 33 | future>> 34 | database::set_direct(uint32_t key, db_val* val, int tid) 35 | { 36 | using return_type = foreign_ptr>; 37 | if (ht[tid].table == NULL) 38 | hashtable_init(&ht[tid], 1000*1000); 39 | ht_set(&ht[tid], val); 40 | 41 | const sstring v = msg_ok; 42 | return make_ready_future( 43 | foreign_ptr>(make_lw_shared(v))); 44 | } 45 | 46 | // TO DO: not implemented for hash table 47 | future>> 48 | database::del_direct(const redis_key& rk, int tid) 49 | { 50 | using return_type = foreign_ptr>; 51 | const sstring v = "ok\n"; 52 | return make_ready_future( 53 | foreign_ptr>(make_lw_shared(v))); 54 | } 55 | 56 | // Get pointer to hash table 57 | future>> 58 | database::get_table(int tid) 59 | { 60 | using return_type = foreign_ptr>; 61 | return make_ready_future( 62 | foreign_ptr>(make_lw_shared(ht[tid].table))); 63 | } 64 | 65 | db_val** database::get_table_direct(int tid) 66 | { 67 | return ht[tid].table; 68 | } 69 | -------------------------------------------------------------------------------- /example_client.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | int main(int argc, char **argv) { 8 | unsigned int j, isunix = 0; 9 | redisContext *c; 10 | redisReply *reply; 11 | const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; 12 | 13 | if (argc > 2) { 14 | if (*argv[2] == 'u' || *argv[2] == 'U') { 15 | isunix = 1; 16 | /* in this case, host is the path to the unix socket */ 17 | printf("Will connect to unix socket @%s\n", hostname); 18 | } 19 | } 20 | 21 | int port = (argc > 2) ? atoi(argv[2]) : 11211; 22 | 23 | struct timeval timeout = { 1, 500000 }; // 1.5 seconds 24 | if (isunix) { 25 | c = redisConnectUnixWithTimeout(hostname, timeout); 26 | } else { 27 | c = redisConnectWithTimeout(hostname, port, timeout); 28 | } 29 | if (c == NULL || c->err) { 30 | if (c) { 31 | printf("Connection error: %s\n", c->errstr); 32 | redisFree(c); 33 | } else { 34 | printf("Connection error: can't allocate redis context\n"); 35 | } 36 | exit(1); 37 | } 38 | 39 | // Run JavaScript function 'setup' to setup test data, 40 | // including Facebook social graphs and neural network model. 41 | reply = redisCommand(c,"JS %s", "setup"); 42 | printf("JS setup: %s\n", reply->str); 43 | freeReplyObject(reply); 44 | 45 | // Run JavaScript function 'get' to read value of key 0, 46 | // which is the friend list of user id 0 in Facebook socail graph 47 | reply = redisCommand(c,"JS %s %s", "get", "0"); 48 | printf("JS get 0: %s\n", reply->str); 49 | freeReplyObject(reply); 50 | 51 | // count_friend_list is a graph traversal function which counts a user's 52 | // friends and friends's friends and so on. The first parameter is the 53 | // user ID and the second is the depth of the traversal. 54 | reply = redisCommand(c,"JS %s %s %s", "count_friend_list", "0", "1"); 55 | printf("JS count_friend_list 0 1: %s\n", reply->str); 56 | freeReplyObject(reply); 57 | 58 | // Run JavaScript neural predict function, the result should be 1 59 | reply = redisCommand(c,"JS %s", "predict"); 60 | printf("JS predict: %s\n", reply->str); 61 | freeReplyObject(reply); 62 | 63 | /* Disconnects and frees the context */ 64 | redisFree(c); 65 | 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /include/connection_handler.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "redis_protocol_parser.hh" 3 | #include "seastarkv.hh" 4 | 5 | using namespace seastar; 6 | using namespace redis; 7 | 8 | class connection_handler { 9 | private: 10 | redis_protocol_parser _parser; 11 | args_collection _request_args; 12 | 13 | public: 14 | connection_handler() {}; 15 | void prepare_request(); 16 | future<> handle(input_stream& in, output_stream& out, int& tid); 17 | }; 18 | 19 | -------------------------------------------------------------------------------- /include/db.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "include/seastarkv.hh" 3 | #include "include/reply_builder.hh" 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace redis; 9 | using namespace std; 10 | 11 | class database; 12 | extern distributed db; 13 | inline distributed& get_database() { 14 | return db; 15 | } 16 | inline database* get_local_database() { 17 | return &db.local(); 18 | } 19 | 20 | struct db_val { 21 | uint32_t key; 22 | struct db_val *next; 23 | void* data; 24 | uint32_t length; 25 | }__attribute__((__packed__)); 26 | 27 | struct node 28 | { 29 | void* data; 30 | node *next; 31 | }; 32 | 33 | class database { 34 | public: 35 | typedef std::chrono::time_point ts; 36 | typedef std::chrono::duration ns; 37 | 38 | struct hashtable { 39 | uint32_t size; 40 | struct db_val **table; 41 | }; 42 | 43 | typedef struct hashtable hashtable_t; 44 | typedef struct db_val db_val_t; 45 | 46 | void hashtable_init(hashtable_t* ht, uint32_t size) { 47 | ht->table = (db_val**)calloc(size, sizeof(void*)); 48 | assert(ht->table != NULL); 49 | ht->size = size; 50 | return; 51 | } 52 | 53 | // Thomas Wang, Integer Hash Functions. 54 | // http://www.concentric.net/~Ttwang/tech/inthash.htm 55 | inline uint32_t get_hash(uint32_t key, uint32_t seed) { 56 | uint32_t hash = key; 57 | hash = hash ^ seed; 58 | hash = ~hash + (hash << 15); // hash = (hash << 15) - hash - 1; 59 | hash = hash ^ (hash >> 12); 60 | hash = hash + (hash << 2); 61 | hash = hash ^ (hash >> 4); 62 | hash = hash * 2057; // hash = (hash + (hash << 3)) + (hash << 11); 63 | hash = hash ^ (hash >> 16); 64 | return hash & 0x3fffffff; 65 | } 66 | 67 | void ht_set(hashtable_t* ht, db_val_t* val) { 68 | val->next = NULL; 69 | uint32_t hash = get_hash(val->key, 0) % ht->size; 70 | db_val_t* next = ht->table[hash]; 71 | db_val_t* before = NULL; 72 | while (next != NULL && next->key != val->key) { 73 | before = next; 74 | next = before->next; 75 | } 76 | 77 | if (next != NULL) { 78 | free(next->data); 79 | next->data = val->data; 80 | next->length = val->length; 81 | free(val); 82 | } else if (before != NULL){ 83 | before->next = val; 84 | } else 85 | ht->table[hash] = val; 86 | } 87 | 88 | db_val_t* ht_get(hashtable_t* ht, uint32_t key) { 89 | uint32_t hash = get_hash(key, 0) % ht->size; 90 | db_val_t* p = ht->table[hash]; 91 | while (p && p->key != key) 92 | p = p->next; 93 | return p; 94 | } 95 | 96 | database() 97 | { 98 | } 99 | 100 | ~database() { 101 | }; 102 | 103 | future<> set_all(redis_key rk, db_val val); 104 | future set(const redis_key& rk, sstring& val); 105 | future get(const redis_key& key, int tid); 106 | bool del(const redis_key& key); 107 | future>> get_direct(uint32_t key, int tid); 108 | future>> set_direct(uint32_t key, db_val* val, int tid); 109 | future>> del_direct(const redis_key& rk, int tid); 110 | future>> get_iterator(const redis_key& rk, int tid); 111 | future>> get_next(int tid); 112 | future>> get_table(int tid); 113 | db_val** get_table_direct(int tid); 114 | 115 | future<> start(); 116 | future<> stop(); 117 | // One hashtable for each V8 context 118 | hashtable_t ht[NUM_CONTEXTS]; 119 | }; 120 | -------------------------------------------------------------------------------- /include/net_server.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "seastarkv.hh" 3 | #include "connection_handler.hh" 4 | 5 | using namespace v8; 6 | using namespace seastar; 7 | 8 | class network_server; 9 | extern distributed net_server; 10 | inline distributed& get_net_server() { 11 | return net_server; 12 | } 13 | 14 | class network_server { 15 | private: 16 | lw_shared_ptr _listener; 17 | uint16_t _port = 11211; 18 | // Tenant id to be assigned to an incoming connection. 19 | int tid = 0; 20 | 21 | struct connection { 22 | connected_socket _socket; 23 | socket_address _addr; 24 | input_stream _in; 25 | output_stream _out; 26 | connection_handler handler; 27 | int tenant_id; 28 | connection(connected_socket&& socket, socket_address addr, int tid) 29 | : _socket(std::move(socket)) 30 | , _addr(addr) 31 | , _in(_socket.input()) 32 | , _out(_socket.output()) 33 | , tenant_id(tid) 34 | { 35 | } 36 | ~connection() { 37 | } 38 | }; 39 | 40 | seastar::gate _request_gate; 41 | 42 | public: 43 | network_server() 44 | { 45 | } 46 | 47 | void start(); 48 | 49 | future<> stop() { 50 | return make_ready_future<>(); 51 | } 52 | }; 53 | -------------------------------------------------------------------------------- /include/redis_protocol_parser.rl: -------------------------------------------------------------------------------- 1 | /* 2 | * Pedis is free software: you can redistribute it and/or modify 3 | * it under the terms of the GNU Affero General Public License as published by 4 | * the Free Software Foundation, either version 3 of the License, or 5 | * (at your option) any later version. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.gnu.org/licenses 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | * 18 | * Copyright (c) 2016-2026, Peng Jian, pstack@163.com. All rights reserved. 19 | * 20 | */ 21 | 22 | /** This protocol parser was inspired by the memcached app, 23 | which is the example in Seastar project. 24 | **/ 25 | 26 | #include "core/ragel.hh" 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | using namespace seastar; 33 | 34 | %%{ 35 | 36 | machine redis_resp_protocol; 37 | 38 | access _fsm_; 39 | 40 | action mark { 41 | g.mark_start(p); 42 | } 43 | 44 | action start_blob { 45 | g.mark_start(p); 46 | _size_left = _arg_size; 47 | } 48 | action start_command { 49 | g.mark_start(p); 50 | _size_left = _arg_size; 51 | } 52 | 53 | action advance_blob { 54 | auto len = std::min((uint32_t)(pe - p), _size_left); 55 | _size_left -= len; 56 | p += len; 57 | if (_size_left == 0) { 58 | _args_list.push_back(str()); 59 | p--; 60 | fret; 61 | } 62 | p--; 63 | } 64 | 65 | action advance_command { 66 | auto len = std::min((uint32_t)(pe - p), _size_left); 67 | _size_left -= len; 68 | p += len; 69 | if (_size_left == 0) { 70 | _command = str(); 71 | p--; 72 | fret; 73 | } 74 | p--; 75 | } 76 | 77 | 78 | crlf = '\r\n'; 79 | u32 = digit+ >{ _u32 = 0;} ${ _u32 *= 10; _u32 += fc - '0';}; 80 | args_count = '*' u32 crlf ${_args_count = _u32;}; 81 | blob := any+ >start_blob $advance_blob; 82 | quit = "quit"i ${_command = command::quit;}; 83 | info = "info"i ${_command = command::info;}; 84 | js = "js"i ${_command = command::js;}; 85 | set = "set"i ${_command = command::set;}; 86 | mset = "mset"i ${_command = command::mset;}; 87 | get = "get"i ${_command = command::get;}; 88 | mget = "mget"i ${_command = command::mget;}; 89 | del = "del"i ${_command = command::del;}; 90 | echo = "echo"i ${_command = command::echo;}; 91 | ping = "ping"i ${_command = command::ping;}; 92 | incr = "incr"i ${_command = command::incr;}; 93 | decr = "decr"i ${_command = command::decr;}; 94 | incrby = "incrby"i ${_command = command::incrby;}; 95 | decrby = "decrby"i ${_command = command::decrby;}; 96 | command_ = "command"i ${_command = command::command;}; 97 | exists = "exists"i ${_command = command::exists;}; 98 | append = "append"i ${_command = command::append;}; 99 | strlen = "strlen"i ${_command = command::strlen;}; 100 | lpush = "lpush"i ${_command = command::lpush;}; 101 | lpushx = "lpushx"i ${_command = command::lpushx;}; 102 | lpop = "lpop"i ${_command = command::lpop;}; 103 | llen = "llen"i ${_command = command::llen;}; 104 | lindex = "lindex"i ${_command = command::lindex;}; 105 | linsert = "linsert"i ${_command = command::linsert;}; 106 | lrange = "lrange"i ${_command = command::lrange;}; 107 | lset = "lset"i ${_command = command::lset;}; 108 | rpush = "rpush"i ${_command = command::rpush;}; 109 | rpushx = "rpushx"i ${_command = command::rpushx;}; 110 | rpop = "rpop"i ${_command = command::rpop;}; 111 | lrem = "lrem"i ${_command = command::lrem;}; 112 | ltrim = "ltrim"i ${_command = command::ltrim;}; 113 | hset = "hset"i ${_command = command::hset;}; 114 | hmset = "hmset"i ${_command = command::hmset;}; 115 | hdel = "hdel"i ${_command = command::hdel;}; 116 | hget = "hget"i ${_command = command::hget;}; 117 | hlen = "hlen"i ${_command = command::hlen;}; 118 | hexists = "hexists"i ${_command = command::hexists;}; 119 | hstrlen = "hstrlen"i ${_command = command::hstrlen;}; 120 | hincrby = "hincrby"i ${_command = command::hincrby;}; 121 | hincrbyfloat = "hincrbyfloat"i ${_command = command::hincrbyfloat;}; 122 | hkeys = "hkeys"i ${_command = command::hkeys;}; 123 | hvals = "hvals"i ${_command = command::hvals;}; 124 | hmget = "hmget"i ${_command = command::hmget;}; 125 | hgetall = "hgetall"i ${_command = command::hgetall;}; 126 | sadd = "sadd"i ${_command = command::sadd;}; 127 | scard = "scard"i ${_command = command::scard;}; 128 | sismember = "sismember"i ${_command = command::sismember;}; 129 | smembers = "smembers"i ${_command = command::smembers;}; 130 | srandmember = "srandmember"i ${_command = command::srandmember;}; 131 | srem = "srem"i ${_command = command::srem;}; 132 | sdiff = "sdiff"i ${_command = command::sdiff;}; 133 | sdiffstore = "sdiffstore"i ${_command = command::sdiffstore;}; 134 | sinter = "sinter"i ${_command = command::sinter;}; 135 | sinterstore = "sinterstore"i ${_command = command::sinterstore;}; 136 | sunion = "sunion"i ${_command = command::sunion;}; 137 | sunionstore = "sunionstore"i ${_command = command::sunionstore;}; 138 | smove = "smove"i ${_command = command::smove;}; 139 | spop = "spop"i ${_command = command::spop;}; 140 | type = "type"i ${_command = command::type; }; 141 | expire = "expire"i ${_command = command::expire; }; 142 | pexpire = "pexpire"i ${_command = command::pexpire; }; 143 | ttl = "ttl"i ${_command = command::ttl; }; 144 | pttl = "pttl"i ${_command = command::pttl; }; 145 | persist = "persist"i ${_command = command::persist; }; 146 | zadd = "zadd"i ${_command = command::zadd; }; 147 | zcard = "zcard"i ${_command = command::zcard; }; 148 | zcount = "zcount"i ${_command = command::zcount; }; 149 | zincrby = "zincrby"i ${_command = command::zincrby; }; 150 | zrange = "zrange"i ${_command = command::zrange;}; 151 | zrank = "zrank"i ${_command = command::zrank; }; 152 | zrem = "zrem"i ${_command = command::zrem; }; 153 | zremrangebyrank = "zremrangebyrank"i ${_command = command::zremrangebyrank; }; 154 | zremrangebyscore = "zremrangebyscore"i ${_command = command::zremrangebyscore; }; 155 | zrevrange = "zrevrange"i ${_command = command::zrevrange; }; 156 | zrevrangebyscore = "zrevrangebyscore"i ${_command = command::zrevrangebyscore; }; 157 | zrevrank = "zrevrank"i ${_command = command::zrevrank; }; 158 | zscore = "zscore"i ${_command = command::zscore; }; 159 | zunionstore = "zunionstore"i ${_command = command::zunionstore; }; 160 | zinterstore = "zinterstore"i ${_command = command::zinterstore; }; 161 | zdiffstore = "zdiffstore"i ${_command = command::zdiffstore; }; 162 | zunion = "zunion"i ${_command = command::zunion; }; 163 | zinter = "zinter"i ${_command = command::zinter; }; 164 | zdiff = "zunion"i ${_command = command::zdiff; }; 165 | zscan = "zscan"i ${_command = command::zscan; }; 166 | zrangebylex = "zrangebylex"i ${_command = command::zrangebylex; }; 167 | zrangebyscore = "zrangebyscore"i ${_command = command::zrangebyscore; }; 168 | zlexcount = "zlexcount"i ${_command = command::zlexcount;}; 169 | zremrangebylex = "zremrangebylex"i ${_command = command::zremrangebylex; }; 170 | select = "select"i ${_command = command::select; }; 171 | geoadd = "geoadd"i ${_command = command::geoadd; }; 172 | geodist = "geodist"i ${_command = command::geodist; }; 173 | geohash = "geohash"i ${_command = command::geohash; }; 174 | geopos = "geopos"i ${_command = command::geopos; }; 175 | georadius = "georadius"i ${_command = command::georadius; }; 176 | georadiusbymember = "georadiusbymember"i ${_command = command::georadiusbymember; }; 177 | setbit = "setbit"i ${_command = command::setbit; }; 178 | getbit = "getbit"i ${_command = command::getbit; }; 179 | bitcount = "bitcount"i ${_command = command::bitcount; }; 180 | bitop = "bitop"i ${_command = command::bitop; }; 181 | bitfield = "bitfield"i ${_command = command::bitfield; }; 182 | bitpos = "bitpos"i ${_command = command::bitpos; }; 183 | pfadd = "pfadd"i ${_command = command::pfadd; }; 184 | pfcount = "pfcount"i ${_command = command::pfcount; }; 185 | pfmerge = "pfmerge"i ${_command = command::pfmerge; }; 186 | 187 | command = (quit| info| js | setbit | set | getbit | get | del | mget | mset | echo | ping | incr | decr | incrby | decrby | command_ | exists | append | 188 | strlen | lpushx | lpush | lpop | llen | lindex | linsert | lrange | lset | rpushx | rpush | rpop | lrem | 189 | ltrim | hset | hgetall |hget | hdel | hlen | hexists | hstrlen | hincrby | hincrbyfloat | hkeys | hvals | hmget | hmset | 190 | sadd | scard | sismember | smembers | srem | sdiffstore | sdiff | sinterstore | sinter| sunionstore | sunion | smove | srandmember | spop | 191 | type | expire | pexpire | persist | ttl | pttl | zadd | zcard | zcount | zincrby | 192 | zrangebyscore | zrank | zremrangebyrank | zremrangebyscore | zremrangebylex | zrem | zrevrangebyscore | zrevrange| zrevrank | 193 | zscore | zunionstore | zinterstore | zdiffstore | zunion | zinter | zdiff | zscan | zrangebylex | zlexcount | 194 | zrange | select | geoadd | geodist | geohash | geopos | georadiusbymember | georadius | bitcount | 195 | bitpos | bitop | bitfield | 196 | pfadd | pfcount | pfmerge ); 197 | arg = '$' u32 crlf ${ _arg_size = _u32;}; 198 | 199 | main := (args_count (arg command crlf) (arg @{fcall blob; } crlf)+) ${_state = state::ok;}; 200 | 201 | prepush { 202 | prepush(); 203 | } 204 | 205 | postpop { 206 | postpop(); 207 | } 208 | 209 | }%% 210 | 211 | class redis_protocol_parser : public ragel_parser_base { 212 | %% write data nofinal noprefix; 213 | public: 214 | enum class state { 215 | error, 216 | eof, 217 | ok, 218 | }; 219 | enum class command { 220 | quit, 221 | info, 222 | js, 223 | set, 224 | mset, 225 | get, 226 | mget, 227 | del, 228 | echo, 229 | ping, 230 | incr, 231 | decr, 232 | incrby, 233 | decrby, 234 | command, 235 | exists, 236 | append, 237 | strlen, 238 | lpush, 239 | lpushx, 240 | lpop, 241 | llen, 242 | lindex, 243 | linsert, 244 | lrange, 245 | lset, 246 | rpush, 247 | rpushx, 248 | rpop, 249 | lrem, 250 | ltrim, 251 | hset, 252 | hdel, 253 | hget, 254 | hlen, 255 | hexists, 256 | hstrlen, 257 | hincrby, 258 | hincrbyfloat, 259 | hkeys, 260 | hvals, 261 | hmget, 262 | hmset, 263 | hgetall, 264 | sadd, 265 | scard, 266 | sismember, 267 | smembers, 268 | srem, 269 | sdiff, 270 | sdiffstore, 271 | sinter, 272 | sinterstore, 273 | sunion, 274 | sunionstore, 275 | smove, 276 | srandmember, 277 | spop, 278 | type, 279 | expire, 280 | pexpire, 281 | ttl, 282 | pttl, 283 | persist, 284 | zadd, 285 | zcard, 286 | zcount, 287 | zincrby, 288 | zrange, 289 | zrangebyscore, 290 | zrank, 291 | zrem, 292 | zremrangebyrank, 293 | zremrangebyscore, 294 | zrevrange, 295 | zrevrangebyscore, 296 | zrevrank, 297 | zscore, 298 | zunionstore, 299 | zinterstore, 300 | zdiffstore, 301 | zunion, 302 | zinter, 303 | zdiff, 304 | zscan, 305 | zrangebylex, 306 | zlexcount, 307 | zremrangebylex, 308 | select, 309 | geoadd, 310 | geohash, 311 | geodist, 312 | geopos, 313 | georadius, 314 | georadiusbymember, 315 | setbit, 316 | getbit, 317 | bitcount, 318 | bitop, 319 | bitpos, 320 | bitfield, 321 | pfadd, 322 | pfcount, 323 | pfmerge, 324 | }; 325 | 326 | state _state; 327 | command _command; 328 | uint32_t _u32; 329 | uint32_t _arg_size; 330 | uint32_t _args_count; 331 | uint32_t _size_left; 332 | std::vector _args_list; 333 | public: 334 | void init() { 335 | init_base(); 336 | _state = state::error; 337 | _args_list.clear(); 338 | _args_count = 0; 339 | _size_left = 0; 340 | _arg_size = 0; 341 | %% write init; 342 | } 343 | 344 | char* parse(char* p, char* pe, char* eof) { 345 | sstring_builder::guard g(_builder, p, pe); 346 | auto str = [this, &g, &p] { g.mark_end(p); return get_str(); }; 347 | %% write exec; 348 | if (_state != state::error) { 349 | return p; 350 | } 351 | // error ? 352 | if (p != pe) { 353 | p = pe; 354 | return p; 355 | } 356 | return nullptr; 357 | } 358 | bool eof() const { 359 | return _state == state::eof; 360 | } 361 | }; 362 | -------------------------------------------------------------------------------- /include/reply_builder.hh: -------------------------------------------------------------------------------- 1 | /* 2 | * Pedis is free software: you can redistribute it and/or modify 3 | * it under the terms of the GNU Affero General Public License as published by 4 | * the Free Software Foundation, either version 3 of the License, or 5 | * (at your option) any later version. 6 | * 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.gnu.org/licenses 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | * 18 | * Copyright (c) 2016-2026, Peng Jian, pstack@163.com. All rights reserved. 19 | * 20 | */ 21 | #pragma once 22 | #include "core/app-template.hh" 23 | #include "core/future-util.hh" 24 | #include "core/timer-set.hh" 25 | #include "core/shared_ptr.hh" 26 | #include "core/stream.hh" 27 | #include "core/memory.hh" 28 | #include "core/units.hh" 29 | #include "core/distributed.hh" 30 | #include "core/vector-data-sink.hh" 31 | #include "core/bitops.hh" 32 | #include "core/slab.hh" 33 | #include "core/align.hh" 34 | #include "net/api.hh" 35 | #include "net/packet-data-source.hh" 36 | 37 | namespace redis { 38 | using scattered_message_ptr = foreign_ptr>>; 39 | 40 | class reply_builder final { 41 | public: 42 | static future build(size_t size) 43 | { 44 | auto m = make_lw_shared>(); 45 | m->append_static(msg_num_tag); 46 | m->append(to_sstring(size)); 47 | m->append_static(msg_crlf); 48 | return make_ready_future(foreign_ptr>>(m)); 49 | } 50 | 51 | static future build(double number) 52 | { 53 | auto m = make_lw_shared>(); 54 | auto&& n = to_sstring(number); 55 | m->append_static(msg_batch_tag); 56 | m->append(to_sstring(n.size())); 57 | m->append_static(msg_crlf); 58 | m->append(std::move(n)); 59 | m->append_static(msg_crlf); 60 | return make_ready_future(foreign_ptr>>(m)); 61 | } 62 | 63 | static future build(const sstring& message) 64 | { 65 | auto m = make_lw_shared>(); 66 | m->append(message); 67 | return make_ready_future(foreign_ptr>>(m)); 68 | } 69 | 70 | static future build(const sstring& message, size_t size) 71 | { 72 | auto m = make_lw_shared>(); 73 | m->append_static(msg_batch_tag); 74 | m->append(to_sstring(size)); 75 | m->append_static(msg_crlf); 76 | m->append(message); 77 | m->append_static(msg_crlf); 78 | return make_ready_future(foreign_ptr>>(m)); 79 | } 80 | 81 | static future<> build_local(output_stream& out, size_t size) 82 | { 83 | auto m = make_lw_shared>(); 84 | m->append_static(msg_num_tag); 85 | m->append(to_sstring(size)); 86 | m->append_static(msg_crlf); 87 | return out.write(std::move(*m)); 88 | } 89 | 90 | static scattered_message build_direct(const sstring& message, size_t size) 91 | { 92 | auto m = scattered_message(); 93 | m.append_static(msg_batch_tag); 94 | m.append(to_sstring(size)); 95 | m.append_static(msg_crlf); 96 | m.append(message); 97 | m.append_static(msg_crlf); 98 | return m; 99 | } 100 | 101 | inline static future<> build_local(output_stream& out, const sstring& message) 102 | { 103 | return out.write(message); 104 | } 105 | 106 | }; // end of class 107 | } 108 | -------------------------------------------------------------------------------- /include/req_server.hh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "include/seastarkv.hh" 3 | #include "include/reply_builder.hh" 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | using namespace redis; 9 | using namespace seastar; 10 | using namespace v8; 11 | 12 | class req_service; 13 | extern distributed req_server; 14 | inline distributed& get_req_server() { 15 | return req_server; 16 | } 17 | inline req_service& local_req_server() { 18 | return req_server.local(); 19 | } 20 | 21 | inline unsigned get_cpu(const sstring& key) { 22 | return std::hash()(key) % smp::count; 23 | } 24 | inline unsigned get_cpu(const redis_key& key) { 25 | return key.hash() % smp::count; 26 | } 27 | 28 | void db_get(const v8::FunctionCallbackInfo& args); 29 | void db_set(const v8::FunctionCallbackInfo& args); 30 | void db_del(const v8::FunctionCallbackInfo& args); 31 | void js_print(const v8::FunctionCallbackInfo& args); 32 | void init_iterator(const v8::FunctionCallbackInfo& args); 33 | void iterator_next(const v8::FunctionCallbackInfo& args); 34 | void get_hash_table(const v8::FunctionCallbackInfo& args); 35 | void load_fb_graph(const v8::FunctionCallbackInfo& args); 36 | 37 | using message = scattered_message; 38 | class req_service { 39 | private: 40 | // The tenant id tied to the currently running script. 41 | // The JavaScript context of the currently running script. 42 | Local* current_context; 43 | char** argv; 44 | Isolate::CreateParams create_params; 45 | // JavaScript contexts for each tenant. 46 | Global contexts[NUM_CONTEXTS]; 47 | // Stores the function object of the last run JS function. 48 | Global prev_fun[NUM_CONTEXTS]; 49 | // Stores the name of the last run JS function. 50 | sstring prev_fun_name[NUM_CONTEXTS]; 51 | Global args_templ; 52 | 53 | // Semaphore to signal incoming requests to JS thread. 54 | semaphore sem{0}; 55 | 56 | // The stucture to store an incoming request. 57 | struct rqst { 58 | args_collection& args; 59 | sstring key; 60 | sstring val; 61 | int tenant_id; 62 | bool put; 63 | lw_shared_ptr>>> p; 64 | rqst(args_collection& a) 65 | :args(a) 66 | {} 67 | ~rqst(){} 68 | }; 69 | 70 | MaybeLocal read_file(Isolate* isolate, const string& name) { 71 | FILE* file = fopen(name.c_str(), "rb"); 72 | if (file == NULL) return MaybeLocal(); 73 | 74 | fseek(file, 0, SEEK_END); 75 | size_t size = ftell(file); 76 | rewind(file); 77 | 78 | char* chars = new char[size + 1]; 79 | chars[size] = '\0'; 80 | for (size_t i = 0; i < size;) { 81 | i += fread(&chars[i], 1, size - i, file); 82 | if (ferror(file)) { 83 | fclose(file); 84 | return MaybeLocal(); 85 | } 86 | } 87 | fclose(file); 88 | MaybeLocal result = String::NewFromUtf8( 89 | isolate, chars, NewStringType::kNormal, static_cast(size)); 90 | delete[] chars; 91 | return result; 92 | } 93 | 94 | public: 95 | int current_tid; 96 | Isolate* isolate; 97 | 98 | req_service(char** a) 99 | : argv(a) 100 | { 101 | create_params.array_buffer_allocator = 102 | v8::ArrayBuffer::Allocator::NewDefaultAllocator(); 103 | isolate = Isolate::New(create_params); 104 | Isolate::Scope isolate_scope(isolate); 105 | { 106 | HandleScope handle_scope(isolate); 107 | 108 | // Setup the JS contexts for every tenant.. 109 | for (int i = 0; i < NUM_CONTEXTS; i++) { 110 | v8::Local global = v8::ObjectTemplate::New(isolate); 111 | // Set C++ bindings 112 | global->Set( 113 | v8::String::NewFromUtf8(isolate, "DBGet", v8::NewStringType::kNormal) 114 | .ToLocalChecked(), 115 | v8::FunctionTemplate::New(isolate, db_get) 116 | ); 117 | global->Set( 118 | v8::String::NewFromUtf8(isolate, "DBSet", v8::NewStringType::kNormal) 119 | .ToLocalChecked(), 120 | v8::FunctionTemplate::New(isolate, db_set) 121 | ); 122 | global->Set( 123 | v8::String::NewFromUtf8(isolate, "DBDel", v8::NewStringType::kNormal) 124 | .ToLocalChecked(), 125 | v8::FunctionTemplate::New(isolate, db_del) 126 | ); 127 | global->Set( 128 | v8::String::NewFromUtf8(isolate, "print", v8::NewStringType::kNormal) 129 | .ToLocalChecked(), 130 | v8::FunctionTemplate::New(isolate, js_print) 131 | ); 132 | global->Set( 133 | v8::String::NewFromUtf8(isolate, "GetHashTable", v8::NewStringType::kNormal) 134 | .ToLocalChecked(), 135 | v8::FunctionTemplate::New(isolate, get_hash_table) 136 | ); 137 | global->Set( 138 | v8::String::NewFromUtf8(isolate, "LoadFBGraph", v8::NewStringType::kNormal) 139 | .ToLocalChecked(), 140 | v8::FunctionTemplate::New(isolate, load_fb_graph) 141 | ); 142 | 143 | // JS script can get the current tenant id by this global variable. 144 | global->Set( 145 | v8::String::NewFromUtf8(isolate, "TID", v8::NewStringType::kNormal) 146 | .ToLocalChecked(), 147 | v8::Number::New(isolate, i) 148 | ); 149 | 150 | Local c = Context::New(isolate, NULL, global); 151 | Context::Scope contextScope(c); 152 | contexts[i].Reset(isolate, c); 153 | } 154 | 155 | for (int i = 0; i < NUM_CONTEXTS; i++) { 156 | Local c = Local::New(isolate, contexts[i]); 157 | Context::Scope contextScope(c); 158 | 159 | Local source; 160 | // Read the JS script file. 161 | if (!read_file(isolate, "scripts.js").ToLocal(&source)) { 162 | fprintf(stderr, "Error reading file\n"); 163 | } 164 | 165 | Local