├── .clang-format ├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake ├── CodeCoverage.cmake ├── DownloadProject.cmake ├── DownloadProjectHelper.cmake └── clang-format.cmake ├── conf ├── kvs-base.yml └── kvs-local.yml ├── dockerfiles ├── fluent-base.dockerfile ├── functions │ ├── cache.dockerfile │ ├── executor.dockerfile │ ├── start-cache.sh │ └── start-funcs.sh └── kvs │ ├── anna.dockerfile │ ├── kops.dockerfile │ ├── start.sh │ └── start_kops.sh ├── docs ├── function-execution.md ├── getting-started-aws.md ├── getting-started.md └── index.md ├── functions ├── CMakeLists.txt ├── benchmarks │ ├── __init__.py │ ├── composition.py │ ├── dist_avg.py │ ├── lambda_locality.py │ ├── locality.py │ ├── predserving.py │ ├── scaling.py │ ├── server.py │ ├── summa.py │ └── utils.py ├── cache │ ├── CMakeLists.txt │ ├── include │ │ ├── causal_cache_handlers.hpp │ │ └── causal_cache_utils.hpp │ ├── src │ │ ├── CMakeLists.txt │ │ ├── async_cache.cpp │ │ ├── cache.cpp │ │ ├── causal_cache.cpp │ │ ├── causal_cache_get_request_handler.cpp │ │ ├── causal_cache_kvs_response_handler.cpp │ │ ├── causal_cache_periodic_migration_handler.cpp │ │ ├── causal_cache_put_request_handler.cpp │ │ ├── causal_cache_utils.cpp │ │ └── causal_cache_versioned_key_handlers.cpp │ └── tests │ │ ├── CMakeLists.txt │ │ ├── causal_cache_base.hpp │ │ ├── run_causal_cache_tests.cpp │ │ ├── test_causal_lattice_comparison.hpp │ │ ├── test_cross_obj_causal.hpp │ │ └── test_single_obj_causal.hpp ├── client.py ├── executor │ ├── __init__.py │ ├── call.py │ ├── pin.py │ ├── server.py │ ├── user_library.py │ └── utils.py ├── include │ ├── __init__.py │ ├── serializer.py │ ├── server_utils.py │ └── shared.py ├── run_benchmark.py ├── scheduler │ ├── __init__.py │ ├── call.py │ ├── create.py │ ├── server.py │ └── utils.py ├── server.py └── trigger_bench.py ├── include ├── CMakeLists.txt ├── common.hpp ├── kvs_async_client.hpp ├── kvs_client.hpp ├── kvs_mock_client.hpp ├── lattices │ ├── core_lattices.hpp │ ├── cross_causal_lattice.hpp │ ├── lattice.hpp │ ├── lww_pair_lattice.hpp │ └── vector_clock_pair_lattice.hpp ├── proto │ ├── functions.proto │ └── kvs.proto ├── requests.hpp ├── threads.hpp ├── types.hpp └── zmq │ ├── socket_cache.cpp │ ├── socket_cache.hpp │ ├── zmq_util.cpp │ └── zmq_util.hpp ├── k8s ├── add_nodes.py ├── create_cluster.py ├── k8s_server.py ├── kops │ ├── create_cluster_object.sh │ ├── modify_ig.sh │ ├── validate_cluster.sh │ └── yaml │ │ └── igs │ │ ├── benchmark-ig.yml │ │ ├── ebs-ig.yml │ │ ├── function-ig.yml │ │ ├── general-ig.yml │ │ ├── memory-ig.yml │ │ ├── routing-ig.yml │ │ └── scheduler-ig.yml ├── management_server.py ├── remove_node.py ├── util.py └── yaml │ ├── ds │ ├── benchmark-ds.yml │ ├── ebs-ds.yml │ ├── function-ds.yml │ ├── memory-ds.yml │ ├── routing-ds.yml │ └── scheduler-ds.yml │ ├── pods │ ├── kops-pod.yml │ └── monitoring-pod.yml │ └── services │ ├── function.yml │ └── routing.yml ├── kvs ├── CMakeLists.txt ├── README.md ├── client │ ├── cpp │ │ ├── CMakeLists.txt │ │ ├── async_cli.cpp │ │ └── cli.cpp │ └── python │ │ ├── LICENSE.txt │ │ ├── anna │ │ ├── __init__.py │ │ ├── client.py │ │ ├── common.py │ │ ├── ipc_client.py │ │ ├── lattices.py │ │ └── zmq_util.py │ │ └── setup.py ├── include │ ├── CMakeLists.txt │ ├── consistent_hash_map.hpp │ ├── hash_ring.hpp │ ├── hashers.hpp │ ├── kvs │ │ ├── base_kv_store.hpp │ │ ├── kvs_handlers.hpp │ │ └── server_utils.hpp │ ├── kvs_common.hpp │ ├── kvs_threads.hpp │ ├── kvs_types.hpp │ ├── metadata.hpp │ ├── monitor │ │ ├── monitoring_handlers.hpp │ │ ├── monitoring_utils.hpp │ │ └── policies.hpp │ ├── proto │ │ ├── metadata.proto │ │ └── replication.proto │ └── route │ │ └── routing_handlers.hpp ├── src │ ├── CMakeLists.txt │ ├── benchmark │ │ ├── CMakeLists.txt │ │ ├── benchmark.cpp │ │ └── trigger.cpp │ ├── hash_ring │ │ ├── CMakeLists.txt │ │ └── hash_ring.cpp │ ├── kvs │ │ ├── CMakeLists.txt │ │ ├── cache_ip_response_handler.cpp │ │ ├── gossip_handler.cpp │ │ ├── node_depart_handler.cpp │ │ ├── node_join_handler.cpp │ │ ├── replication_change_handler.cpp │ │ ├── replication_response_handler.cpp │ │ ├── self_depart_handler.cpp │ │ ├── server.cpp │ │ ├── user_request_handler.cpp │ │ └── utils.cpp │ ├── monitor │ │ ├── CMakeLists.txt │ │ ├── depart_done_handler.cpp │ │ ├── elasticity.cpp │ │ ├── feedback_handler.cpp │ │ ├── membership_handler.cpp │ │ ├── monitoring.cpp │ │ ├── movement_policy.cpp │ │ ├── replication_helpers.cpp │ │ ├── slo_policy.cpp │ │ ├── stats_helpers.cpp │ │ └── storage_policy.cpp │ └── route │ │ ├── CMakeLists.txt │ │ ├── address_handler.cpp │ │ ├── membership_handler.cpp │ │ ├── replication_change_handler.cpp │ │ ├── replication_response_handler.cpp │ │ ├── routing.cpp │ │ └── seed_handler.cpp └── tests │ ├── CMakeLists.txt │ ├── include │ ├── CMakeLists.txt │ └── lattices │ │ ├── CMakeLists.txt │ │ ├── run_lattice_tests.cpp │ │ ├── test_bool_lattice.hpp │ │ ├── test_map_lattice.hpp │ │ ├── test_max_lattice.hpp │ │ ├── test_ordered_set_lattice.hpp │ │ └── test_set_lattice.hpp │ ├── kvs │ ├── CMakeLists.txt │ ├── run_server_handler_tests.cpp │ ├── server_handler_base.hpp │ ├── test_gossip_handler.hpp │ ├── test_node_depart_handler.hpp │ ├── test_node_join_handler.hpp │ ├── test_rep_factor_change_handler.hpp │ ├── test_rep_factor_response_handler.hpp │ ├── test_self_depart_handler.hpp │ └── test_user_request_handler.hpp │ ├── mock │ ├── mock_utils.cpp │ └── mock_utils.hpp │ ├── route │ ├── CMakeLists.txt │ ├── routing_handler_base.hpp │ ├── run_routing_handler_tests.cpp │ ├── test_address_handler.hpp │ ├── test_membership_handler.hpp │ ├── test_replication_change_handler.hpp │ ├── test_replication_response_handler.hpp │ └── test_seed_handler.hpp │ ├── simple │ ├── expected │ ├── expected_async │ ├── input │ ├── input_async │ ├── test-simple-async.sh │ └── test-simple.sh │ └── test_all.cpp ├── scripts ├── build-all.sh ├── check-format.sh ├── install-dependencies-osx.sh ├── install-dependencies.sh ├── make.sh ├── start-kvs-local.sh ├── stop-kvs-local.sh └── travis │ ├── docker-build.sh │ ├── llvm-gcov.sh │ ├── run-tests.sh │ ├── travis-build.sh │ ├── travis-install.sh │ └── upload-codecov.sh └── vendor ├── spdlog └── CMakeLists.txt ├── yamlcpp └── CMakeLists.txt ├── zeromq └── CMakeLists.txt └── zeromqcpp └── CMakeLists.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # project specific 16 | build 17 | vendor/gtest/build 18 | *log*.txt 19 | *tmp* 20 | 21 | # ignore compiled byte code 22 | target 23 | 24 | # ignore output files from testing 25 | output* 26 | 27 | # ignore standard Mac OS X files/dirs 28 | .DS_Store 29 | default.profaw 30 | 31 | ################################################################################ 32 | # vim 33 | ################################################################################ 34 | # swap 35 | [._]*.s[a-w][a-z] 36 | [._]s[a-w][a-z] 37 | # session 38 | Session.vim 39 | # temporary 40 | .netrwhist 41 | *~ 42 | # auto-generated tag files 43 | tags 44 | # syntastic 45 | .syntastic_clang_tidy_config 46 | .syntastic_cpp_config 47 | 48 | ################################################################################ 49 | # C++ 50 | ################################################################################ 51 | # Prerequisites 52 | *.d 53 | 54 | # Compiled Object files 55 | *.slo 56 | *.lo 57 | *.o 58 | *.obj 59 | 60 | # Precompiled Headers 61 | *.gch 62 | *.pch 63 | 64 | # Compiled Dynamic libraries 65 | *.so 66 | *.dylib 67 | *.dll 68 | 69 | # Fortran module files 70 | *.mod 71 | *.smod 72 | 73 | # Compiled Static libraries 74 | *.lai 75 | *.la 76 | *.a 77 | *.lib 78 | 79 | # Executables 80 | *.exe 81 | *.out 82 | *.app 83 | 84 | ################################################################################ 85 | # Python 86 | ################################################################################ 87 | # Compiled Object files 88 | __pycache__/ 89 | 90 | # Compiled protobuf files 91 | *_pb2.py 92 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | language: cpp 16 | sudo: required 17 | 18 | os: 19 | - linux 20 | 21 | dist: trusty 22 | 23 | compiler: 24 | - clang 25 | 26 | services: 27 | - docker 28 | 29 | env: 30 | global: 31 | - PROTOBUF_DIR="$HOME/protobuf" 32 | - LCOV_VERSION=1.13 33 | 34 | cache: 35 | directories: 36 | - $PROTOBUF_DIR 37 | 38 | install: 39 | - ./scripts/travis/travis-install.sh 40 | 41 | script: 42 | - ./scripts/travis/travis-build.sh 43 | 44 | after_success: 45 | - ./scripts/travis/upload-codecov.sh 46 | - ./scripts/travis/docker-build.sh 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ***NOTE***: The Fluent project has been renamed to Hydro, and the project has been broken up into multiple repositories. You can find all of them in the [Hydro Project](https://github.com/hydro-project) organization on Github. This repository has been archived and will be deleted eventually. 2 | 3 | # Fluent 4 | 5 | [![Build Status](https://travis-ci.com/hydro-project/fluent.svg?branch=master)](https://travis-ci.com/hydro-project/fluent) 6 | [![codecov](https://codecov.io/gh/hydro-project/fluent/branch/master/graph/badge.svg)](https://codecov.io/gh/hydro-project/fluent) 7 | [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 8 | 9 | 10 | Fluent is a fully managed, data-first computation framework under development that [U.C. Berkeley](https://www.berkeley.edu) [RISE Lab](https://rise.cs.berkeley.edu). There are two main components in Fluent. 11 | 12 | The first is a key-value store based on prior open-source work from the RISE lab (fka Anna). The Fluent KVS is an elastic, cloud-native storage engine that uses coordination-avoiding techniques and asynchronous message passing to provide very low latency. You can find more information about running and using the Fluent KVS in the `kvs` directory. 13 | 14 | The second component is a data-centric programming framework, built on top of the Anna KVS. The goal of the programming framework is to provide users a general-purpose API and runtime for executing programs on data stored in the Anna KVS. Users are able to submit arbitrary code or containers for execution, and we plan to support performance SLOs for function execution. 15 | 16 | To get started with the full project, please see the [Getting Started](docs/getting-started.md) page. You can find the index of all the docs [here](docs/index.md). 17 | 18 | ## Contributing 19 | 20 | If you run into any issues, please open an issue and make sure to include relevant information (e.g., stack traces) as well as operating system, dependency versions, etc. 21 | 22 | If you are looking to contribute to the project, please look at our [issues list](https://github.com/fluent-project/fluent/issues), particularly those marked as [good for beginners](https://github.com/fluent-project/fluent/issues?q=is%3Aopen+is%3Aissue+label%3Abeginners) and [help wanted](https://github.com/fluent-project/fluent/labels/help%20wanted). 23 | -------------------------------------------------------------------------------- /cmake/DownloadProjectHelper.cmake: -------------------------------------------------------------------------------- 1 | # Distributed under the OSI-approved MIT License. See accompanying 2 | # file LICENSE or https://github.com/Crascit/DownloadProject for details. 3 | 4 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 5 | 6 | project(${DL_ARGS_PROJ}-download NONE) 7 | 8 | include(ExternalProject) 9 | ExternalProject_Add(${DL_ARGS_PROJ}-download 10 | ${DL_ARGS_UNPARSED_ARGUMENTS} 11 | SOURCE_DIR "${DL_ARGS_SOURCE_DIR}" 12 | BINARY_DIR "${DL_ARGS_BINARY_DIR}" 13 | CONFIGURE_COMMAND "" 14 | BUILD_COMMAND "" 15 | INSTALL_COMMAND "" 16 | TEST_COMMAND "" 17 | ) 18 | -------------------------------------------------------------------------------- /cmake/clang-format.cmake: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Script copied from https://arcanis.me/en/2015/10/17/cppcheck-and-clang-format 16 | 17 | # get list of all project files 18 | FILE(GLOB_RECURSE ALL_SOURCE_FILES *.cpp *.hpp) 19 | foreach (SOURCE_FILE ${ALL_SOURCE_FILES}) 20 | STRING(FIND ${SOURCE_FILE} ${VENDOR_DIR} VENDOR_DIR_FOUND) 21 | if (NOT ${VENDOR_DIR_FOUND} EQUAL -1) 22 | LIST(REMOVE_ITEM ALL_SOURCE_FILES ${SOURCE_FILE}) 23 | endif () 24 | endforeach () 25 | 26 | ADD_CUSTOM_TARGET( 27 | clang-format 28 | COMMAND clang-format 29 | -style=file 30 | -i 31 | ${ALL_SOURCE_FILES} 32 | ) 33 | 34 | -------------------------------------------------------------------------------- /conf/kvs-base.yml: -------------------------------------------------------------------------------- 1 | ebs: /ebs 2 | capacities: # in GB 3 | memory-cap: 45 4 | ebs-cap: 256 5 | threads: 6 | memory: 4 7 | ebs: 4 8 | routing: 4 9 | benchmark: 4 10 | replication: 11 | memory: 1 12 | ebs: 0 13 | minimum: 1 14 | local: 1 15 | policy: 16 | elasticity: true 17 | selective-rep: true 18 | tiering: false 19 | -------------------------------------------------------------------------------- /conf/kvs-local.yml: -------------------------------------------------------------------------------- 1 | monitoring: 2 | mgmt_ip: 127.0.0.1 3 | ip: 127.0.0.1 4 | routing: 5 | monitoring: 6 | - 127.0.0.1 7 | ip: 127.0.0.1 8 | user: 9 | monitoring: 10 | - 127.0.0.1 11 | routing: 12 | - 127.0.0.1 13 | ip: 127.0.0.1 14 | server: 15 | monitoring: 16 | - 127.0.0.1 17 | routing: 18 | - 127.0.0.1 19 | seed_ip: 127.0.0.1 20 | public_ip: 127.0.0.1 21 | private_ip: 127.0.0.1 22 | mgmt_ip: "NULL" 23 | policy: 24 | elasticity: true 25 | selective-rep: true 26 | tiering: false 27 | ebs: ./ 28 | capacities: # in GB 29 | memory-cap: 2 30 | ebs-cap: 4 31 | threads: 32 | memory: 1 33 | ebs: 1 34 | routing: 1 35 | benchmark: 1 36 | replication: 37 | memory: 1 38 | ebs: 0 39 | minimum: 1 40 | local: 1 41 | -------------------------------------------------------------------------------- /dockerfiles/functions/cache.dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM fluentproject/base:latest 16 | 17 | MAINTAINER Vikram Sreekanti version: 0.1 18 | 19 | ARG repo_org=fluent-project 20 | ARG source_branch=master 21 | ARG build_branch=docker-build 22 | 23 | USER root 24 | 25 | # check out to the appropriate branch and build the C++ project 26 | WORKDIR /fluent 27 | RUN git remote remove origin && git remote add origin https://github.com/$repo_org/fluent 28 | RUN git fetch origin && git checkout -b $build_branch origin/$source_branch 29 | RUN bash scripts/build-all.sh -j4 -bRelease 30 | WORKDIR / 31 | 32 | COPY start-cache.sh / 33 | 34 | CMD bash start-cache.sh $SERVER_TYPE 35 | -------------------------------------------------------------------------------- /dockerfiles/functions/executor.dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM fluentproject/base:latest 16 | 17 | MAINTAINER Vikram Sreekanti version: 0.1 18 | 19 | ARG repo_org=fluent-project 20 | ARG source_branch=master 21 | ARG build_branch=docker-build 22 | 23 | USER root 24 | 25 | # pipeline specific installs 26 | RUN pip3 install numpy sklearn pyarrow pandas 27 | 28 | # check out to the appropriate branch and install the Python KVS client 29 | WORKDIR /fluent 30 | RUN git remote remove origin && git remote add origin https://github.com/$repo_org/fluent 31 | RUN git fetch -p origin && git checkout -b $build_branch origin/$source_branch 32 | RUN cd kvs/client/python && python3.6 setup.py install --prefix=$HOME/.local 33 | WORKDIR / 34 | 35 | COPY start-funcs.sh /start-funcs.sh 36 | 37 | # start the executor server 38 | CMD bash start-funcs.sh 39 | -------------------------------------------------------------------------------- /dockerfiles/functions/start-cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | cd fluent 18 | mkdir -p conf 19 | 20 | IP=`ifconfig eth0 | grep 'inet addr:' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1 }'` 21 | 22 | # download latest version of the code from relevant repository & branch 23 | git remote remove origin 24 | if [[ -z "$REPO_ORG" ]]; then 25 | REPO_ORG="fluent-project" 26 | fi 27 | 28 | if [[ -z "$REPO_BRANCH" ]]; then 29 | REPO_BRANCH="master" 30 | fi 31 | 32 | # switch to the desired branch; by default we run with master on 33 | # fluent-project/fluent 34 | git remote add origin https://github.com/$REPO_ORG/fluent 35 | git fetch -p origin 36 | git checkout -b brnch origin/$REPO_BRANCH 37 | 38 | cd build && make -j4 && cd .. 39 | 40 | while [[ ! -f "conf/kvs-config.yml" ]]; do 41 | continue 42 | done 43 | 44 | 45 | echo -e "user:" >> conf/kvs-config.yml 46 | echo -e " ip: $IP" >> conf/kvs-config.yml 47 | echo -e " routing-elb: $ROUTE_ADDR" >> conf/kvs-config.yml 48 | 49 | ./build/functions/cache/src/flfunc-async-cache 50 | -------------------------------------------------------------------------------- /dockerfiles/functions/start-funcs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | IS_EC2=`curl -s http://instance-data.ec2.internal` 18 | IP=`ifconfig eth0 | grep 'inet addr:' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1 }'` 19 | 20 | # move into the fluent directory for the rest of the script 21 | cd fluent 22 | 23 | # download latest version of the code from relevant repository & branch 24 | git remote remove origin 25 | if [[ -z "$REPO_ORG" ]]; then 26 | REPO_ORG="fluent-project" 27 | fi 28 | 29 | if [[ -z "$REPO_BRANCH" ]]; then 30 | REPO_BRANCH="master" 31 | fi 32 | 33 | # switch to the desired branch; by default we run with master on 34 | # fluent-project/fluent 35 | git remote add origin https://github.com/$REPO_ORG/fluent 36 | git fetch -p origin 37 | git checkout -b brnch origin/$REPO_BRANCH 38 | 39 | # generate Python protobufs 40 | cd include/proto 41 | protoc -I=./ --python_out=../../functions/include functions.proto 42 | protoc -I=./ --python_out=../../functions/include kvs.proto 43 | cd ../.. 44 | 45 | # TODO: this might not be necessary permanently -- depends on whether you're 46 | # changing the client or not 47 | cd kvs/client/python 48 | python3.6 setup.py install --prefix=$HOME/.local 49 | cd ../../.. 50 | 51 | # start python server 52 | cd functions 53 | while true; do 54 | export MY_IP=$IP && python3.6 server.py 55 | done 56 | -------------------------------------------------------------------------------- /dockerfiles/kvs/anna.dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM fluentproject/base:latest 16 | 17 | MAINTAINER Vikram Sreekanti version: 0.1 18 | 19 | ARG repo_org=fluent-project 20 | ARG source_branch=master 21 | ARG build_branch=docker-build 22 | 23 | USER root 24 | 25 | # check out to the appropriate branch and build the C++ project 26 | WORKDIR /fluent 27 | RUN git remote remove origin && git remote add origin https://github.com/$repo_org/fluent 28 | RUN git fetch origin && git checkout -b $build_branch origin/$source_branch 29 | RUN bash scripts/build-all.sh -j4 -bRelease 30 | WORKDIR / 31 | 32 | COPY start.sh / 33 | CMD bash start.sh $SERVER_TYPE 34 | -------------------------------------------------------------------------------- /dockerfiles/kvs/kops.dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM fluentproject/base:latest 16 | MAINTAINER Vikram Sreekanti version: 0.1 17 | 18 | ARG repo_org=fluent-project 19 | ARG source_branch=master 20 | ARG build_branch=docker-build 21 | 22 | USER root 23 | 24 | # install kops 25 | RUN wget -O kops https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep -Po '"tag_name": "\K.*?(?=")')/kops-linux-amd64 26 | RUN chmod +x ./kops 27 | RUN mv ./kops /usr/local/bin/ 28 | 29 | # install kubectl 30 | RUN wget -O kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl 31 | RUN chmod +x ./kubectl 32 | RUN mv ./kubectl /usr/local/bin/kubectl 33 | 34 | # NOTE: Doesn't make sense to set up the kops user at build time because we 35 | # need the user's AWS creds... should have a script to do this at runtime 36 | # eventually; for now, going to assume that the user is already set up or we 37 | # can just provide a script to this generally, independent of running it here 38 | WORKDIR /fluent 39 | RUN git remote remove origin && git remote add origin https://github.com/$repo_org/fluent 40 | RUN git fetch origin && git checkout -b $build_branch origin/$source_branch 41 | WORKDIR / 42 | 43 | # make kube root dir 44 | RUN mkdir /root/.kube 45 | 46 | COPY start_kops.sh / 47 | CMD bash start_kops.sh 48 | -------------------------------------------------------------------------------- /dockerfiles/kvs/start_kops.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # set AWS environment variables 18 | mkdir -p ~/.aws 19 | echo "[default]\nregion = us-east-1" > ~/.aws/config 20 | echo "[default]\naws_access_key_id = $AWS_ACCESS_KEY_ID\naws_secret_access_key = $AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials 21 | mkdir -p ~/.ssh 22 | 23 | # move into the fluent directory for the rest of the script 24 | cd fluent 25 | 26 | # download latest version of the code from relevant repository & branch 27 | git remote remove origin 28 | if [[ -z "$REPO_ORG" ]]; then 29 | REPO_ORG="fluent-project" 30 | fi 31 | 32 | if [[ -z "$REPO_BRANCH" ]]; then 33 | REPO_BRANCH="master" 34 | fi 35 | 36 | # switch to the desired branch; by default we run with master on 37 | # fluent-project/fluent 38 | git remote add origin https://github.com/$REPO_ORG/fluent 39 | git fetch -p origin 40 | git checkout -b brnch origin/$REPO_BRANCH 41 | 42 | # generate Python protobuf libraries 43 | cd include/proto 44 | protoc -I=./ --python_out=../../k8s kvs.proto 45 | protoc -I=./ --python_out=../../k8s functions.proto 46 | cd ../../kvs/include/proto 47 | protoc -I=./ --python_out=../../../k8s metadata.proto 48 | cd ../../.. 49 | 50 | # start k8s actuation and management servers 51 | cd k8s 52 | python3.6 k8s_server.py & 53 | python3.6 management_server.py 54 | -------------------------------------------------------------------------------- /docs/function-execution.md: -------------------------------------------------------------------------------- 1 | # Executing Functions in Fluent 2 | 3 | Assuming you have a Fluent cluster running, as described [here](getting-started-aws.md), you can run functions and DAGs in Fluent. 4 | 5 | First, we'll create two new functions: 6 | 7 | ```python3 8 | >>> from client import FluentConnection 9 | >>> flconn = FluentConnection(AWS_FUNCTION_ELB, MY_IP) 10 | >>> incr = lambda fluent, a: a + 1 11 | >>> cloud_incr = flconn.register(incr, 'incr') 12 | >>> cloud_incr(1).get() 13 | 2 14 | >>> square = lambda fluent, a: a * a 15 | >>> cloud_square = flconn.register(square, 'square') 16 | >>> cloud_square(2).get() 17 | 4 18 | ``` 19 | 20 | Now we'll chain those functions together and execute them at once: 21 | 22 | ```python3 23 | # create a DAG with two functions, incr and square, where incr comes before square 24 | >>> flconn.register_dag('test_dag', ['incr', 'square'], [('incr', 'square')]) 25 | True # returns False if registration fails, e.g., if one of the referenced functions does not exist 26 | >>> flconn.call_dag('test_dag', { 'incr': 1 }).get() 27 | 4 28 | ``` 29 | 30 | * All functions take a `fluent` argument as their first parameter. See [below](#Fluent-API) for the full API for this library, which includes message passing and KVS access. 31 | * All calls to functions and DAGs are by default asynchronous. Results are stored in the key-value store, and object IDs are returned. DAG calls can optionally specify synchronous calls by setting the `direct_response` argument to `True`. 32 | * DAGs can have arbitrary branches and connections and have multiple sources, but there must be only one sink function in the DAG. The result of this sink function is what is returned to the caller. 33 | 34 | 35 | ## Fluent API 36 | 37 | | API Name | Functionality | 38 | |-----------|---------------| 39 | | `get(key)`| Retrieves `key` from the KVS | 40 | | `put(key, value)`| Puts `value` into the KVS at key `key` | 41 | | `get_id()`| Returns the unique messaging identifier for this function | 42 | | `send(id, msg)`| Sends message contents `msg` to the function at ID `id` | 43 | | `recv()`| Receives any messages sent to this function | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | You can install the KVS and function serving environment locally either to play around with the system or for development. If you would like to start a cluster, please see the [Getting Started on AWS](getting-started-aws.md) docs. 4 | 5 | ```bash 6 | # install required depenencies 7 | $ ./scripts/install-dependencies.sh # please use install-dependencies-osx.sh if you are on a Mac 8 | 9 | # compile and build all C++ components 10 | $ ./scripts/build-kvs.sh -j4 -bRelease # to build with tests, change -bRelease to -bDebug -t 11 | 12 | # start the KVS locally 13 | $ ./scripts/start-kvs-local.sh n n 14 | 15 | # start the FaaS server 16 | $ cd functions && python3 function_server.py 17 | 18 | ``` 19 | 20 | Once you have the FaaS server running, you can start the client in a separate Python shell: 21 | 22 | ```python3 23 | >>> import client 24 | >>> cloud = FluentConnnection('127.0.0.1', '127.0.0.1') 25 | >>> cloud.list() 26 | sum 27 | square 28 | >>> sum = cloud.get('sum') 29 | >>> sum(2, 2).get() # make a remote function call 30 | 4 31 | ``` 32 | 33 | You can see the [function API](function-api.md) docs for more on how to use the function server client. -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Documentation Index 2 | 3 | ## Getting Started 4 | 5 | * [Getting Started Locally](getting-started.md) 6 | * [Getting Started on AWS](getting-started-aws.md) 7 | 8 | ## Function Execution Docs 9 | 10 | * [API](function-execution .md) 11 | 12 | ## KVS Docs 13 | 14 | **Coming Soon!** 15 | -------------------------------------------------------------------------------- /functions/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | PROJECT(Fluent-Function) 17 | 18 | SET(FLUENT_FUNC_MAJOR_VERSION ${FLUENT_MAJOR_VERSION}) 19 | SET(FLUENT_FUNC_MINOR_VERSION ${FLUENT_MINOR_VERSION}) 20 | SET(FLUENT_FUNC_PATCH_VERSION ${FLUENT_PATCH_VERSION}) 21 | 22 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 23 | INCLUDE_DIRECTORIES(include) 24 | INCLUDE_DIRECTORIES(tests) 25 | 26 | # INCLUDE(FindProtobuf) 27 | # FIND_PACKAGE(Protobuf REQUIRED) 28 | # INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) 29 | # PROTOBUF_GENERATE_CPP(PROTO_SRC PROTO_HEADER ./include/proto/functions.proto) 30 | 31 | # need to build a target at this level or subdirs won't have the 32 | # protobuf files generated. 33 | # ADD_LIBRARY(flfunc-proto ${PROTO_HEADER} ${PROTO_SRC}) 34 | 35 | LINK_DIRECTORIES(${ZEROMQ_LINK_DIRS} ${YAMLCPP_LINK_DIRS}) 36 | 37 | ADD_SUBDIRECTORY(cache) 38 | -------------------------------------------------------------------------------- /functions/benchmarks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydro-project/fluent/51dc6de82334af4b8d5991d1b13ff2fc6a6fd650/functions/benchmarks/__init__.py -------------------------------------------------------------------------------- /functions/benchmarks/composition.py: -------------------------------------------------------------------------------- 1 | import cloudpickle as cp 2 | import logging 3 | import sys 4 | import time 5 | 6 | from include.functions_pb2 import * 7 | from include.serializer import * 8 | 9 | 10 | def run(flconn, kvs, num_requests, sckt): 11 | ''' DEFINE AND REGISTER FUNCTIONS ''' 12 | def incr(fluent, x): 13 | return x + 1 14 | 15 | def square(fluent, x): 16 | return x * x 17 | 18 | cloud_incr = flconn.register(incr, 'incr') 19 | cloud_square = flconn.register(square, 'square') 20 | 21 | if cloud_incr and cloud_square: 22 | print('Successfully registered incr and square functions.') 23 | else: 24 | sys.exit(1) 25 | 26 | ''' TEST REGISTERED FUNCTIONS ''' 27 | incr_test = cloud_incr(2).get() 28 | if incr_test != 3: 29 | print('Unexpected result from incr(2): %s' % (str(incr_test))) 30 | sys.exit(1) 31 | 32 | square_test = cloud_square(2).get() 33 | if square_test != 4: 34 | print('Unexpected result from square(2): %s' % (str(square_test))) 35 | sys.exit(1) 36 | 37 | print('Successfully tested functions!') 38 | 39 | ''' CREATE DAG ''' 40 | dag_name = 'composition' 41 | 42 | functions = ['incr', 'square'] 43 | connections = [('incr', 'square')] 44 | success, error = flconn.register_dag(dag_name, functions, connections) 45 | 46 | if not success: 47 | print('Failed to register DAG: %s' % (ErrorType.Name(error))) 48 | sys.exit(1) 49 | 50 | ''' RUN DAG ''' 51 | arg_map = {'incr': [1]} 52 | 53 | total_time = [] 54 | scheduler_time = [] 55 | kvs_time = [] 56 | 57 | retries = 0 58 | 59 | for _ in range(num_requests): 60 | start = time.time() 61 | rid = flconn.call_dag(dag_name, arg_map) 62 | end = time.time() 63 | 64 | stime = end - start 65 | 66 | start = time.time() 67 | res = kvs.get(rid) 68 | while not res: 69 | retries += 1 70 | res = kvs.get(rid) 71 | 72 | res = deserialize_val(res.reveal()[1]) 73 | end = time.time() 74 | 75 | ktime = end - start 76 | 77 | total_time += [stime + ktime] 78 | scheduler_time += [stime] 79 | kvs_time += [ktime] 80 | 81 | if sckt: 82 | sckt.send(cp.dumps(total_time)) 83 | return total_time, scheduler_time, kvs_time, retries 84 | -------------------------------------------------------------------------------- /functions/benchmarks/lambda_locality.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import cloudpickle as cp 3 | import json 4 | import logging 5 | import random 6 | import time 7 | import uuid 8 | 9 | from . import utils 10 | 11 | sys_random = random.SystemRandom() 12 | 13 | 14 | def run(name, kvs, num_requests, sckt): 15 | name = 'locality-' + name 16 | oids = cp.loads(kvs.get(name).reveal()[1]) 17 | 18 | lambd = boto3.client('lambda', 'us-east-1') 19 | 20 | latencies = [] 21 | epoch_latencies = [] 22 | epoch_kvs = [] 23 | epoch_comp = [] 24 | epoch_start = time.time() 25 | 26 | epoch = 0 27 | for _ in range(num_requests): 28 | args = [] 29 | for _ in range(10): 30 | args.append(sys_random.choice(oids)) 31 | 32 | start = time.time() 33 | loc = str(uuid.uuid4()) 34 | body = {'args': args, 'loc': loc} 35 | 36 | res = lambd.invoke(FunctionName=name, Payload=json.dumps(body)) 37 | res = json.loads(res['Payload'].read()) 38 | kvs, comp = res 39 | end = time.time() 40 | invoke = end - start 41 | 42 | epoch_kvs.append(kvs) 43 | epoch_comp.append(comp) 44 | 45 | total = invoke + kvs 46 | latencies.append(total) 47 | epoch_latencies.append(total) 48 | epoch_end = time.time() 49 | 50 | if (epoch_end - epoch_start) > 10: 51 | sckt.send(cp.dumps(epoch_latencies)) 52 | utils.print_latency_stats(epoch_latencies, 'EPOCH %d E2E' % 53 | (epoch), True) 54 | utils.print_latency_stats(epoch_comp, 'EPOCH %d COMP' % 55 | (epoch), True) 56 | utils.print_latency_stats(epoch_kvs, 'EPOCH %d KVS' % 57 | (epoch), True) 58 | epoch += 1 59 | 60 | epoch_latencies.clear() 61 | epoch_kvs.clear() 62 | epoch_comp.clear() 63 | epoch_start = time.time() 64 | 65 | return latencies, [], [], 0 66 | -------------------------------------------------------------------------------- /functions/benchmarks/scaling.py: -------------------------------------------------------------------------------- 1 | import cloudpickle as cp 2 | import logging 3 | import sys 4 | import time 5 | 6 | from include.functions_pb2 import * 7 | from include.serializer import * 8 | from . import utils 9 | 10 | 11 | def run(flconn, kvs, num_requests, sckt, create): 12 | ''' DEFINE AND REGISTER FUNCTIONS ''' 13 | dag_name = 'scaling' 14 | 15 | if create: 16 | def slp(fluent, x): 17 | import time 18 | time.sleep(.050) 19 | return x 20 | 21 | cloud_sleep = flconn.register(slp, 'sleep') 22 | 23 | if cloud_sleep: 24 | print('Successfully registered sleep function.') 25 | else: 26 | sys.exit(1) 27 | 28 | ''' TEST REGISTERED FUNCTIONS ''' 29 | sleep_test = cloud_sleep(2).get() 30 | if sleep_test != 2: 31 | print('Unexpected result from sleep(2): %s' % (str(incr_test))) 32 | sys.exit(1) 33 | print('Successfully tested functions!') 34 | 35 | ''' CREATE DAG ''' 36 | functions = ['sleep'] 37 | success, error = flconn.register_dag(dag_name, functions, []) 38 | 39 | if not success: 40 | print('Failed to register DAG: %s' % (ErrorType.Name(error))) 41 | sys.exit(1) 42 | 43 | return [], [], [], 0 44 | else: 45 | ''' RUN DAG ''' 46 | arg_map = {'sleep': [1]} 47 | 48 | total_time = [] 49 | epoch_req_count = 0 50 | epoch_latencies = [] 51 | 52 | epoch_start = time.time() 53 | epoch = 0 54 | for _ in range(num_requests): 55 | start = time.time() 56 | res = flconn.call_dag(dag_name, arg_map, True) 57 | end = time.time() 58 | 59 | if res is not None: 60 | epoch_req_count += 1 61 | 62 | total_time += [end - start] 63 | epoch_latencies += [end - start] 64 | 65 | epoch_end = time.time() 66 | if epoch_end - epoch_start > 10: 67 | if sckt: 68 | sckt.send(cp.dumps((epoch_req_count, epoch_latencies))) 69 | 70 | logging.info('EPOCH %d THROUGHPUT: %.2f' % 71 | (epoch, (epoch_req_count / 10))) 72 | utils.print_latency_stats(epoch_latencies, 73 | 'EPOCH %d E2E' % epoch, True) 74 | epoch += 1 75 | 76 | epoch_req_count = 0 77 | epoch_latencies.clear() 78 | epoch_start = time.time() 79 | 80 | return total_time, [], [], 0 81 | -------------------------------------------------------------------------------- /functions/benchmarks/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import numpy as np 3 | import scipy.stats 4 | 5 | 6 | def mean_confidence_interval(data, confidence=0.95): 7 | n = len(data) 8 | m, se = np.mean(data), scipy.stats.sem(data) 9 | h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1) 10 | return m, m-h, m+h 11 | 12 | 13 | def print_latency_stats(data, ident, log=False): 14 | npdata = np.array(data) 15 | interval = mean_confidence_interval(npdata) 16 | 17 | median = np.percentile(npdata, 50) 18 | p75 = np.percentile(npdata, 75) 19 | p95 = np.percentile(npdata, 95) 20 | p99 = np.percentile(npdata, 99) 21 | mx = np.max(npdata) 22 | 23 | p25 = np.percentile(npdata, 25) 24 | p05 = np.percentile(npdata, 5) 25 | p01 = np.percentile(npdata, 1) 26 | mn = np.min(npdata) 27 | 28 | output = ('%s LATENCY:\n\tsample size: %d\n' + 29 | '\tmean: %.6f, median: %.6f\n' + 30 | '\t95%% confidence: (%.6f, %.6f)\n' + 31 | '\tmin/max: (%.6f, %.6f)\n' + 32 | '\tp25/p75: (%.6f, %.6f) ' + 33 | '\n\tp5/p95: (%.6f, %.6f)\n' + 34 | '\tp1/p99: (%.6f, %.6f)') % (ident, len(data), interval[0], 35 | median, interval[1], interval[2], 36 | mn, mx, p25, p75, p05, p95, p01, 37 | p99) 38 | 39 | if log: 40 | logging.info(output) 41 | else: 42 | print(output) 43 | -------------------------------------------------------------------------------- /functions/cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | 18 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 19 | INCLUDE_DIRECTORIES(include) 20 | 21 | SET(FUNC_DEPENDENCIES 22 | ${PROTO_SRC} 23 | ${PROTO_HEADER} 24 | ) 25 | 26 | SET(FUNC_DEPENDENCIES 27 | protobuf 28 | flproto 29 | pthread 30 | zmq 31 | flzmq 32 | yaml-cpp 33 | ) 34 | 35 | ADD_SUBDIRECTORY(src) 36 | ADD_SUBDIRECTORY(tests) 37 | -------------------------------------------------------------------------------- /functions/cache/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(FUNC_CACHE_SOURCE 18 | cache.cpp 19 | ) 20 | 21 | ADD_EXECUTABLE(flfunc-cache ${FUNC_CACHE_SOURCE}) 22 | TARGET_LINK_LIBRARIES(flfunc-cache ${FUNC_DEPENDENCIES}) 23 | ADD_DEPENDENCIES(flfunc-cache flzmq zeromq zeromqcpp) 24 | 25 | SET(FUNC_ASYNC_CACHE_SOURCE 26 | async_cache.cpp 27 | ) 28 | 29 | ADD_EXECUTABLE(flfunc-async-cache ${FUNC_ASYNC_CACHE_SOURCE}) 30 | TARGET_LINK_LIBRARIES(flfunc-async-cache ${FUNC_DEPENDENCIES}) 31 | ADD_DEPENDENCIES(flfunc-async-cache flzmq zeromq zeromqcpp) 32 | 33 | SET(FUNC_CAUSAL_CACHE_SOURCE 34 | causal_cache.cpp 35 | causal_cache_utils.cpp 36 | causal_cache_get_request_handler.cpp 37 | causal_cache_put_request_handler.cpp 38 | causal_cache_versioned_key_handlers.cpp 39 | causal_cache_kvs_response_handler.cpp 40 | causal_cache_periodic_migration_handler.cpp 41 | ) 42 | 43 | ADD_EXECUTABLE(flfunc-causal-cache ${FUNC_CAUSAL_CACHE_SOURCE}) 44 | TARGET_LINK_LIBRARIES(flfunc-causal-cache ${FUNC_DEPENDENCIES}) 45 | ADD_DEPENDENCIES(flfunc-causal-cache flzmq zeromq zeromqcpp) 46 | -------------------------------------------------------------------------------- /functions/cache/src/causal_cache_periodic_migration_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "causal_cache_utils.hpp" 16 | 17 | void periodic_migration_handler( 18 | const StoreType& unmerged_store, InPreparationType& in_preparation, 19 | StoreType& causal_cut_store, VersionStoreType& version_store, 20 | map& pending_cross_metadata, 21 | map>& to_fetch_map, 22 | map, VectorClockHash>>& 23 | cover_map, 24 | SocketCache& pushers, KvsAsyncClientInterface* client, 25 | const CausalCacheThread& cct, 26 | map>& client_id_to_address_map, logger log) { 27 | for (const auto& pair : unmerged_store) { 28 | if ((causal_cut_store.find(pair.first) == causal_cut_store.end() || 29 | causal_comparison(causal_cut_store[pair.first], pair.second) != 30 | kCausalGreaterOrEqual) && 31 | find_lattice_from_in_preparation(in_preparation, pair.first) == 32 | nullptr) { 33 | to_fetch_map[pair.first] = set(); 34 | in_preparation[pair.first].second[pair.first] = pair.second; 35 | recursive_dependency_check(pair.first, pair.second, in_preparation, 36 | causal_cut_store, unmerged_store, to_fetch_map, 37 | cover_map, client, log); 38 | if (to_fetch_map[pair.first].size() == 0) { 39 | // all dependency met 40 | merge_into_causal_cut(pair.first, causal_cut_store, in_preparation, 41 | version_store, pending_cross_metadata, pushers, 42 | cct, client_id_to_address_map, log, 43 | unmerged_store); 44 | to_fetch_map.erase(pair.first); 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /functions/cache/src/causal_cache_put_request_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "causal_cache_utils.hpp" 16 | 17 | void put_request_handler(const string& serialized, StoreType& unmerged_store, 18 | StoreType& causal_cut_store, 19 | VersionStoreType& version_store, 20 | map& request_id_to_address_map, 21 | KvsAsyncClientInterface* client, logger log) { 22 | CausalRequest request; 23 | request.ParseFromString(serialized); 24 | for (CausalTuple tuple : request.tuples()) { 25 | Key key = tuple.key(); 26 | auto lattice = std::make_shared>>( 27 | to_cross_causal_payload(deserialize_cross_causal(tuple.payload()))); 28 | // first, update unmerged store 29 | if (unmerged_store.find(key) == unmerged_store.end()) { 30 | unmerged_store[key] = lattice; 31 | } else { 32 | unsigned comp_result = causal_comparison(unmerged_store[key], lattice); 33 | if (comp_result == kCausalLess) { 34 | unmerged_store[key] = lattice; 35 | } else if (comp_result == kCausalConcurrent) { 36 | unmerged_store[key] = causal_merge(unmerged_store[key], lattice); 37 | } 38 | } 39 | // write to KVS 40 | string req_id = client->put_async(key, serialize(*unmerged_store[key]), 41 | LatticeType::CROSSCAUSAL); 42 | request_id_to_address_map[req_id] = request.response_address(); 43 | } 44 | } -------------------------------------------------------------------------------- /functions/cache/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(CAUSAL_SRC_DIR ${CMAKE_SOURCE_DIR}/functions/cache/src) 18 | 19 | FILE(GLOB CAUSAL_SOURCES "${CAUSAL_SRC_DIR}/*handler.cpp" "${CAUSAL_SRC_DIR}/*handlers.cpp" "${CAUSAL_SRC_DIR}/*utils.cpp") 20 | FILE(GLOB MOCK_SOURCES "${CMAKE_SOURCE_DIR}/kvs/tests/mock/*.cpp") 21 | 22 | ADD_EXECUTABLE(run_causal_cache_tests 23 | run_causal_cache_tests.cpp 24 | ${MOCK_SOURCES} 25 | ${CAUSAL_SOURCES}) 26 | 27 | TARGET_LINK_LIBRARIES(run_causal_cache_tests ${FUNC_DEPENDENCIES} gtest gmock pthread zmq) 28 | ADD_DEPENDENCIES(run_causal_cache_tests gtest) 29 | 30 | ADD_TEST(NAME CausalTests COMMAND run_causal_cache_tests) 31 | -------------------------------------------------------------------------------- /functions/cache/tests/run_causal_cache_tests.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "gmock/gmock.h" 19 | #include "gtest/gtest.h" 20 | 21 | #include "causal_cache_base.hpp" 22 | #include "test_causal_lattice_comparison.hpp" 23 | #include "test_cross_obj_causal.hpp" 24 | #include "test_single_obj_causal.hpp" 25 | 26 | int main(int argc, char* argv[]) { 27 | log_->set_level(spdlog::level::info); 28 | testing::InitGoogleTest(&argc, argv); 29 | return RUN_ALL_TESTS(); 30 | } 31 | -------------------------------------------------------------------------------- /functions/executor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydro-project/fluent/51dc6de82334af4b8d5991d1b13ff2fc6a6fd650/functions/executor/__init__.py -------------------------------------------------------------------------------- /functions/executor/pin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | 18 | from . import utils 19 | from include.functions_pb2 import * 20 | from include import server_utils as sutils 21 | 22 | 23 | def pin(pin_socket, pusher_cache, client, status, pinned_functions, runtimes, 24 | exec_counts): 25 | msg = pin_socket.recv_string() 26 | splits = msg.split(':') 27 | 28 | resp_ip, name = splits[0], splits[1] 29 | sckt = pusher_cache.get(sutils._get_pin_accept_port(resp_ip)) 30 | 31 | if len(pinned_functions) > 0 or not status.running: 32 | resp = sutils.error.SerializeToString() 33 | sckt.send(sutils.error.SerializeToString()) 34 | return 35 | 36 | logging.info('Adding function %s to my local pinned functions.' % (name)) 37 | sckt.send(sutils.ok_resp) 38 | 39 | func = utils._retrieve_function(name, client) 40 | 41 | # the function must exist -- because otherwise the DAG couldn't be 42 | # registered -- so we keep trying to retrieve it 43 | while not func: 44 | func = utils._retrieve_function(name, client) 45 | 46 | if name not in pinned_functions: 47 | pinned_functions[name] = func 48 | status.functions.append(name) 49 | runtimes[name] = 0.0 50 | exec_counts[name] = 0 51 | 52 | 53 | def unpin(unpin_socket, status, pinned_functions, runtimes, exec_counts): 54 | name = unpin_socket.recv_string() # the name of the func to unpin 55 | logging.info('Removing function %s from my local pinned functions.' % 56 | (name)) 57 | 58 | os._exit(0) 59 | -------------------------------------------------------------------------------- /functions/executor/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | from include.functions_pb2 import * 18 | from include.kvs_pb2 import * 19 | from include import server_utils, serializer 20 | 21 | import zmq 22 | 23 | UTILIZATION_REPORT_PORT = 7003 24 | EXECUTOR_DEPART_PORT = 7005 25 | 26 | 27 | def _retrieve_function(name, kvs, consistency=NORMAL): 28 | kvs_name = server_utils._get_func_kvs_name(name) 29 | 30 | if consistency == NORMAL: 31 | result = kvs.get(kvs_name) 32 | if result: 33 | latt = result[kvs_name] 34 | return serializer.function_ser.load(latt.reveal()[1]) 35 | else: 36 | return None 37 | else: 38 | result = kvs.causal_get([kvs_name], set(), {}, SINGLE, 0) 39 | if result: 40 | return serializer.function_ser.load(result[1][kvs_name][1]) 41 | else: 42 | return None 43 | 44 | 45 | def _push_status(schedulers, pusher_cache, status): 46 | msg = status.SerializeToString() 47 | 48 | # tell all the schedulers your new status 49 | for sched in schedulers: 50 | sckt = pusher_cache.get(_get_status_address(sched)) 51 | sckt.send(msg) 52 | 53 | 54 | def _get_status_address(ip): 55 | return 'tcp://' + ip + ':' + str(server_utils.STATUS_PORT) 56 | 57 | 58 | def _get_util_report_address(mgmt_ip): 59 | return 'tcp://' + mgmt_ip + ':' + str(UTILIZATION_REPORT_PORT) 60 | 61 | 62 | def _get_depart_done_addr(mgmt_ip): 63 | return 'tcp://' + mgmt_ip + ':' + str(EXECUTOR_DEPART_PORT) 64 | -------------------------------------------------------------------------------- /functions/include/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydro-project/fluent/51dc6de82334af4b8d5991d1b13ff2fc6a6fd650/functions/include/__init__.py -------------------------------------------------------------------------------- /functions/include/server_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .functions_pb2 import * 16 | 17 | # shared constants 18 | FUNC_PREFIX = 'funcs/' 19 | BIND_ADDR_TEMPLATE = 'tcp://*:%d' 20 | 21 | PIN_PORT = 4000 22 | UNPIN_PORT = 4010 23 | FUNC_EXEC_PORT = 4020 24 | DAG_QUEUE_PORT = 4030 25 | DAG_EXEC_PORT = 4040 26 | SELF_DEPART_PORT = 4050 27 | 28 | STATUS_PORT = 5007 29 | SCHED_UPDATE_PORT = 5008 30 | BACKOFF_PORT = 5009 31 | PIN_ACCEPT_PORT = 5010 32 | 33 | # For message sending via the user library. 34 | RECV_INBOX_PORT = 5500 35 | 36 | STATISTICS_REPORT_PORT = 7006 37 | 38 | # create generic error response 39 | error = GenericResponse() 40 | error.success = False 41 | 42 | # create generic OK response 43 | ok = GenericResponse() 44 | ok.success = True 45 | ok_resp = ok.SerializeToString() 46 | 47 | 48 | def _get_func_kvs_name(fname): 49 | return FUNC_PREFIX + fname 50 | 51 | 52 | def _get_dag_trigger_address(ip_tid): 53 | ip, tid = ip_tid.split(':') 54 | return 'tcp://' + ip + ':' + str(int(tid) + DAG_EXEC_PORT) 55 | 56 | 57 | def _get_statistics_report_address(mgmt_ip): 58 | return 'tcp://' + mgmt_ip + ':' + str(STATISTICS_REPORT_PORT) 59 | 60 | 61 | def _get_backoff_addresss(ip): 62 | return 'tcp://' + ip + ':' + str(BACKOFF_PORT) 63 | 64 | 65 | def _get_pin_accept_port(ip): 66 | return 'tcp://' + ip + ':' + str(PIN_ACCEPT_PORT) 67 | 68 | 69 | def _get_dag_predecessors(dag, fname): 70 | result = [] 71 | 72 | for connection in dag.connections: 73 | if connection.sink == fname: 74 | result.append(connection.source) 75 | 76 | return result 77 | 78 | 79 | def _get_user_msg_inbox_addr(ip, tid): 80 | return 'tcp://' + ip + ':' + str(int(tid) + RECV_INBOX_PORT) 81 | -------------------------------------------------------------------------------- /functions/include/shared.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import time 16 | 17 | from anna.lattices import * 18 | from .functions_pb2 import * 19 | from .kvs_pb2 import * 20 | from . import serializer 21 | 22 | CONNECT_PORT = 5000 23 | FUNC_CREATE_PORT = 5001 24 | FUNC_CALL_PORT = 5002 25 | LIST_PORT = 5003 26 | DAG_CREATE_PORT = 5004 27 | DAG_CALL_PORT = 5005 28 | DAG_DELETE_PORT = 5006 29 | 30 | 31 | def generate_timestamp(tid=1): 32 | t = time.time() 33 | 34 | p = 10 35 | while tid >= p: 36 | p *= 10 37 | 38 | return int(t * p + tid) 39 | 40 | 41 | class FluentFuture(): 42 | def __init__(self, obj_id, kvs_client): 43 | self.obj_id = obj_id 44 | self.kvs_client = kvs_client 45 | 46 | def get(self): 47 | obj = self.kvs_client.get(self.obj_id) 48 | 49 | while not obj: 50 | obj = self.kvs_client.get(self.obj_id) 51 | 52 | if type(obj).__name__ == 'LWWPairLattice': 53 | return serializer.deserialize_val(obj.reveal()[1]) 54 | elif type(obj).__name__ == 'CrossCausalValue': 55 | return serializer.deserialize_val(obj.values[0]) 56 | 57 | 58 | class FluentFunction(): 59 | def __init__(self, name, conn, kvs_client): 60 | self.name = name 61 | self._conn = conn 62 | self._kvs_client = kvs_client 63 | 64 | def __call__(self, *args): 65 | obj_id = self._conn.exec_func(self.name, args) 66 | return FluentFuture(obj_id, self._kvs_client) 67 | 68 | 69 | class FluentReference(): 70 | def __init__(self, key, deserialize, obj_type): 71 | self.key = key 72 | self.deserialize = deserialize 73 | self.obj_type = obj_type 74 | -------------------------------------------------------------------------------- /functions/run_benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import logging 4 | import sys 5 | from benchmarks import composition 6 | from benchmarks import dist_avg 7 | from benchmarks import locality 8 | from benchmarks import predserving 9 | from benchmarks import scaling 10 | from benchmarks import summa 11 | from benchmarks import utils 12 | import client as flclient 13 | 14 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 15 | 16 | if len(sys.argv) < 4: 17 | print('Usage: ./run_benchmark.py benchmark_name function_elb num_requests ' 18 | + '{ip}') 19 | sys.exit(1) 20 | 21 | f_elb = sys.argv[2] 22 | num_requests = int(sys.argv[3]) 23 | 24 | if len(sys.argv) == 5: 25 | ip = sys.argv[4] 26 | flconn = flclient.FluentConnection(f_elb, ip) 27 | else: 28 | flconn = flclient.FluentConnection(f_elb) 29 | 30 | kvs = flconn.kvs_client 31 | 32 | bname = sys.argv[1] 33 | 34 | if bname == 'composition': 35 | total, scheduler, kvs, retries = composition.run(flconn, kvs, num_requests, 36 | None) 37 | elif bname == 'locality': 38 | locality.run(flconn, kvs, num_requests, True, None) 39 | total, scheduler, kvs, retries = locality.run(flconn, kvs, num_requests, 40 | False, None) 41 | elif bname == 'pred_serving': 42 | total, scheduler, kvs, retries = predserving.run(flconn, kvs, 43 | num_requests, None) 44 | elif bname == 'avg': 45 | total, scheduler, kvs, retries = dist_avg.run(flconn, kvs, num_requests, 46 | None) 47 | elif bname == 'summa': 48 | total, scheduler, kvs, retries = summa.run(flconn, kvs, num_requests, None) 49 | elif bname == 'scaling': 50 | total, scheduler, kvs, retries = scaling.run(flconn, kvs, num_requests, 51 | None) 52 | else: 53 | print('Unknown benchmark type: %s!' % (bname)) 54 | 55 | print('Total computation time: %.4f' % (sum(total))) 56 | 57 | if total: 58 | utils.print_latency_stats(total, 'E2E') 59 | if scheduler: 60 | utils.print_latency_stats(scheduler, 'SCHEDULER') 61 | if kvs: 62 | utils.print_latency_stats(kvs, 'KVS') 63 | 64 | if retries > 0: 65 | print('Number of KVS get retries: %d' % (retries)) 66 | -------------------------------------------------------------------------------- /functions/scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydro-project/fluent/51dc6de82334af4b8d5991d1b13ff2fc6a6fd650/functions/scheduler/__init__.py -------------------------------------------------------------------------------- /functions/server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | import os 17 | import time 18 | import zmq 19 | 20 | from benchmarks.server import * 21 | from executor.server import * 22 | import client as flclient 23 | from scheduler.server import * 24 | 25 | REPORT_THRESH = 30 26 | global_util = 0.0 27 | 28 | 29 | def run(): 30 | mgmt_ip = os.environ['MGMT_IP'] 31 | ip = os.environ['MY_IP'] 32 | 33 | sys_func = os.environ['SYSTEM_FUNC'] 34 | if sys_func == 'scheduler': 35 | route_addr = os.environ['ROUTE_ADDR'] 36 | scheduler(ip, mgmt_ip, route_addr) 37 | if sys_func == 'benchmark': 38 | function_addr = os.environ['FUNCTION_ADDR'] 39 | thread_id = int(os.environ['THREAD_ID']) 40 | 41 | flconn = flclient.FluentConnection(function_addr, ip, thread_id) 42 | benchmark(flconn, thread_id) 43 | else: 44 | schedulers = os.environ['SCHED_IPS'].split(' ') 45 | thread_id = int(os.environ['THREAD_ID']) 46 | executor(ip, mgmt_ip, schedulers, thread_id) 47 | 48 | 49 | if __name__ == '__main__': 50 | run() 51 | -------------------------------------------------------------------------------- /functions/trigger_bench.py: -------------------------------------------------------------------------------- 1 | import cloudpickle as cp 2 | import logging 3 | import sys 4 | import time 5 | import zmq 6 | 7 | from benchmarks import utils 8 | 9 | logging.basicConfig(filename='log_trigger.txt', level=logging.INFO, 10 | format='%(asctime)s %(message)s') 11 | 12 | NUM_THREADS = 4 13 | 14 | ips = [] 15 | with open('bench_ips.txt', 'r') as f: 16 | line = f.readline() 17 | while line: 18 | ips.append(line.strip()) 19 | line = f.readline() 20 | 21 | msg = sys.argv[1] 22 | ctx = zmq.Context(1) 23 | 24 | recv_socket = ctx.socket(zmq.PULL) 25 | recv_socket.bind('tcp://*:3000') 26 | 27 | sent_msgs = 0 28 | 29 | if 'create' in msg: 30 | sckt = ctx.socket(zmq.PUSH) 31 | sckt.connect('tcp://' + ips[0] + ':3000') 32 | 33 | sckt.send_string(msg) 34 | sent_msgs += 1 35 | else: 36 | for ip in ips: 37 | for tid in range(NUM_THREADS): 38 | sckt = ctx.socket(zmq.PUSH) 39 | sckt.connect('tcp://' + ip + ':' + str(3000 + tid)) 40 | 41 | sckt.send_string(msg) 42 | sent_msgs += 1 43 | 44 | epoch_total = [] 45 | total = [] 46 | end_recv = 0 47 | 48 | epoch_recv = 0 49 | epoch = 1 50 | epoch_thruput = 0 51 | epoch_start = time.time() 52 | 53 | while end_recv < sent_msgs: 54 | msg = recv_socket.recv() 55 | 56 | if b'END' in msg: 57 | end_recv += 1 58 | else: 59 | msg = cp.loads(msg) 60 | 61 | if type(msg) == tuple: 62 | epoch_thruput += msg[0] 63 | new_tot = msg[1] 64 | else: 65 | new_tot = msg 66 | 67 | epoch_total += new_tot 68 | total += new_tot 69 | epoch_recv += 1 70 | 71 | if epoch_recv == sent_msgs: 72 | epoch_end = time.time() 73 | elapsed = epoch_end - epoch_start 74 | thruput = epoch_thruput / elapsed 75 | 76 | logging.info('\n\n*** EPOCH %d ***' % (epoch)) 77 | logging.info('\tTHROUGHPUT: %.2f' % (thruput)) 78 | utils.print_latency_stats(epoch_total, 'E2E', True) 79 | 80 | epoch_recv = 0 81 | epoch_thruput = 0 82 | epoch_total.clear() 83 | epoch_start = time.time() 84 | epoch += 1 85 | 86 | logging.info('*** END ***') 87 | 88 | if len(total) > 0: 89 | utils.print_latency_stats(total, 'E2E', True) 90 | -------------------------------------------------------------------------------- /include/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 16 | 17 | ADD_SUBDIRECTORY(lattices) 18 | -------------------------------------------------------------------------------- /include/kvs_mock_client.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef SRC_INCLUDE_MOCK_CLIENT_HPP_ 16 | #define SRC_INCLUDE_MOCK_CLIENT_HPP_ 17 | 18 | #include "common.hpp" 19 | #include "kvs.pb.h" 20 | #include "requests.hpp" 21 | #include "threads.hpp" 22 | #include "types.hpp" 23 | 24 | #include "kvs_async_client.hpp" 25 | 26 | class KvsMockClient : public KvsAsyncClientInterface { 27 | public: 28 | /** 29 | * @addrs A vector of routing addresses. 30 | * @routing_thread_count The number of thread sone ach routing node 31 | * @ip My node's IP address 32 | * @tid My client's thread ID 33 | * @timeout Length of request timeouts in ms 34 | */ 35 | KvsMockClient() { rid_ = 0; } 36 | 37 | ~KvsMockClient() {} 38 | 39 | /** 40 | * Issue an async PUT request to the KVS for a certain lattice typed value. 41 | */ 42 | string put_async(const Key& key, const string& payload, 43 | LatticeType lattice_type) { 44 | keys_put_.push_back(key); 45 | return get_request_id(); 46 | } 47 | 48 | /** 49 | * Issue an async GET request to the KVS. 50 | */ 51 | void get_async(const Key& key) { keys_get_.push_back(key); } 52 | 53 | vector receive_async(ZmqUtilInterface* kZmqUtil) { 54 | return responses_; 55 | } 56 | 57 | zmq::context_t* get_context() { return nullptr; } 58 | 59 | void clear() { 60 | keys_put_.clear(); 61 | keys_get_.clear(); 62 | responses_.clear(); 63 | } 64 | 65 | // keep track of the keys being put 66 | vector keys_put_; 67 | // keep track of the keys being get 68 | vector keys_get_; 69 | // responses to send back 70 | vector responses_; 71 | 72 | private: 73 | /** 74 | * Generates a unique request ID. 75 | */ 76 | string get_request_id() { 77 | if (++rid_ % 10000 == 0) rid_ = 0; 78 | return std::to_string(rid_++); 79 | } 80 | 81 | // the current request id 82 | unsigned rid_; 83 | }; 84 | 85 | #endif // SRC_INCLUDE_MOCK_CLIENT_HPP_ 86 | -------------------------------------------------------------------------------- /include/lattices/lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef INCLUDE_LATTICES_LATTICE_HPP_ 16 | #define INCLUDE_LATTICES_LATTICE_HPP_ 17 | 18 | template 19 | class Lattice { 20 | protected: 21 | T element; 22 | virtual void do_merge(const T &e) = 0; 23 | 24 | public: 25 | // Lattice() { assign(bot()); } 26 | 27 | Lattice(const T &e) { assign(e); } 28 | 29 | Lattice(const Lattice &other) { assign(other.reveal()); } 30 | 31 | virtual ~Lattice() = default; 32 | Lattice &operator=(const Lattice &rhs) { 33 | assign(rhs.reveal()); 34 | return *this; 35 | } 36 | 37 | bool operator==(const Lattice &rhs) const { 38 | return this->reveal() == rhs.reveal(); 39 | } 40 | 41 | const T &reveal() const { return element; } 42 | 43 | void merge(const T &e) { return do_merge(e); } 44 | 45 | void merge(const Lattice &e) { return do_merge(e.reveal()); } 46 | 47 | void assign(const T e) { element = e; } 48 | 49 | void assign(const Lattice &e) { element = e.reveal(); } 50 | }; 51 | 52 | #endif // INCLUDE_LATTICES_LATTICE_HPP_ 53 | -------------------------------------------------------------------------------- /include/lattices/lww_pair_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef INCLUDE_LATTICES_LWW_PAIR_LATTICE_HPP_ 16 | #define INCLUDE_LATTICES_LWW_PAIR_LATTICE_HPP_ 17 | 18 | #include "core_lattices.hpp" 19 | 20 | template 21 | struct TimestampValuePair { 22 | // MapLattice> v_map; 23 | unsigned long long timestamp{0}; 24 | T value; 25 | 26 | TimestampValuePair() { 27 | timestamp = 0; 28 | value = T(); 29 | } 30 | 31 | // need this because of static cast 32 | TimestampValuePair(const unsigned long long& a) { 33 | timestamp = 0; 34 | value = T(); 35 | } 36 | 37 | TimestampValuePair(const unsigned long long& ts, const T& v) { 38 | timestamp = ts; 39 | value = v; 40 | } 41 | unsigned size() { return value.size() + sizeof(unsigned long long); } 42 | }; 43 | 44 | template 45 | class LWWPairLattice : public Lattice> { 46 | protected: 47 | void do_merge(const TimestampValuePair& p) { 48 | if (p.timestamp >= this->element.timestamp) { 49 | this->element.timestamp = p.timestamp; 50 | this->element.value = p.value; 51 | } 52 | } 53 | 54 | public: 55 | LWWPairLattice() : Lattice>(TimestampValuePair()) {} 56 | LWWPairLattice(const TimestampValuePair& p) : 57 | Lattice>(p) {} 58 | MaxLattice size() { return {this->element.size()}; } 59 | }; 60 | 61 | #endif // INCLUDE_LATTICES_LWW_PAIR_LATTICE_HPP_ 62 | -------------------------------------------------------------------------------- /include/lattices/vector_clock_pair_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef SRC_INCLUDE_KVS_VECTOR_CLOCK_PAIR_LATTICE_HPP_ 16 | #define SRC_INCLUDE_KVS_VECTOR_CLOCK_PAIR_LATTICE_HPP_ 17 | 18 | #include "core_lattices.hpp" 19 | 20 | using VectorClock = MapLattice>; 21 | 22 | template 23 | struct VectorClockValuePair { 24 | VectorClock vector_clock; 25 | T value; 26 | 27 | VectorClockValuePair() { 28 | vector_clock = VectorClock(); 29 | value = T(); 30 | } 31 | 32 | // need this because of static cast 33 | VectorClockValuePair(unsigned) { 34 | vector_clock = VectorClock(); 35 | value = T(); 36 | } 37 | 38 | VectorClockValuePair(VectorClock vc, T v) { 39 | vector_clock = vc; 40 | value = v; 41 | } 42 | 43 | unsigned size() { 44 | return vector_clock.size().reveal() * 2 * sizeof(unsigned) + 45 | value.size().reveal(); 46 | } 47 | }; 48 | 49 | template 50 | class CausalPairLattice : public Lattice> { 51 | protected: 52 | void do_merge(const VectorClockValuePair &p) { 53 | VectorClock prev = this->element.vector_clock; 54 | this->element.vector_clock.merge(p.vector_clock); 55 | 56 | if (this->element.vector_clock == p.vector_clock) { 57 | this->element.value.assign(p.value); 58 | } else if (!(this->element.vector_clock == prev)) { 59 | this->element.value.merge(p.value); 60 | } 61 | } 62 | 63 | public: 64 | CausalPairLattice() : 65 | Lattice>(VectorClockValuePair()) {} 66 | CausalPairLattice(const VectorClockValuePair &p) : 67 | Lattice>(p) {} 68 | MaxLattice size() { return {this->element.size()}; } 69 | }; 70 | 71 | #endif // SRC_INCLUDE_KVS_VECTOR_CLOCK_PAIR_LATTICE_HPP_ 72 | -------------------------------------------------------------------------------- /include/types.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef INCLUDE_TYPES_HPP_ 16 | #define INCLUDE_TYPES_HPP_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "spdlog/spdlog.h" 24 | 25 | using string = std::string; 26 | 27 | template 28 | using map = std::unordered_map; 29 | 30 | template 31 | using ordered_set = std::set; 32 | 33 | template 34 | using set = std::unordered_set; 35 | 36 | template 37 | using vector = std::vector; 38 | 39 | template 40 | using pair = std::pair; 41 | 42 | using Address = std::string; 43 | 44 | using Key = std::string; 45 | 46 | using logger = std::shared_ptr; 47 | 48 | #endif // INCLUDE_TYPES_HPP_ 49 | -------------------------------------------------------------------------------- /include/zmq/socket_cache.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "socket_cache.hpp" 16 | 17 | #include 18 | 19 | zmq::socket_t& SocketCache::At(const Address& addr) { 20 | auto iter = cache_.find(addr); 21 | if (iter != cache_.end()) { 22 | return iter->second; 23 | } 24 | 25 | zmq::socket_t socket(*context_, type_); 26 | socket.connect(addr); 27 | auto p = cache_.insert(std::make_pair(addr, std::move(socket))); 28 | 29 | return p.first->second; 30 | } 31 | 32 | zmq::socket_t& SocketCache::operator[](const Address& addr) { return At(addr); } 33 | 34 | void SocketCache::clear_cache() { cache_.clear(); } 35 | -------------------------------------------------------------------------------- /include/zmq/socket_cache.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef SRC_INCLUDE_ZMQ_SOCKET_CACHE_HPP_ 16 | #define SRC_INCLUDE_ZMQ_SOCKET_CACHE_HPP_ 17 | 18 | #include 19 | #include 20 | 21 | #include "types.hpp" 22 | #include "zmq.hpp" 23 | 24 | // A SocketCache is a map from ZeroMQ addresses to PUSH ZeroMQ sockets. The 25 | // socket corresponding to address `address` can be retrieved from a 26 | // SocketCache `cache` with `cache[address]` or `cache.At(address)`. If a 27 | // socket with a given address is not in the cache when it is requested, one is 28 | // created and connected to the address. An example: 29 | // 30 | // zmq::context_t context(1); 31 | // SocketCache cache(&context); 32 | // // This will create a socket and connect it to "inproc://a". 33 | // zmq::socket_t& a = cache["inproc://a"]; 34 | // // This will not createa new socket. It will return the socket created in 35 | // // the previous line. In other words, a and the_same_a_as_before are 36 | // // references to the same socket. 37 | // zmq::socket_t& the_same_a_as_before = cache["inproc://a"]; 38 | // // cache.At("inproc://a") is 100% equivalent to cache["inproc://a"]. 39 | // zmq::socket_t& another_a = cache.At("inproc://a"); 40 | class SocketCache { 41 | public: 42 | explicit SocketCache(zmq::context_t* context, int type) : 43 | context_(context), 44 | type_(type) {} 45 | zmq::socket_t& At(const Address& addr); 46 | zmq::socket_t& operator[](const Address& addr); 47 | void clear_cache(); 48 | 49 | private: 50 | zmq::context_t* context_; 51 | std::map cache_; 52 | int type_; 53 | }; 54 | 55 | #endif // SRC_INCLUDE_ZMQ_SOCKET_CACHE_HPP_ 56 | -------------------------------------------------------------------------------- /include/zmq/zmq_util.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "zmq_util.hpp" 16 | 17 | #include 18 | #include 19 | 20 | string ZmqUtilInterface::message_to_string(const zmq::message_t& message) { 21 | return string(static_cast(message.data()), message.size()); 22 | } 23 | 24 | zmq::message_t ZmqUtilInterface::string_to_message(const string& s) { 25 | zmq::message_t msg(s.size()); 26 | memcpy(msg.data(), s.c_str(), s.size()); 27 | return msg; 28 | } 29 | 30 | void ZmqUtil::send_string(const string& s, zmq::socket_t* socket) { 31 | socket->send(string_to_message(s)); 32 | } 33 | 34 | string ZmqUtil::recv_string(zmq::socket_t* socket) { 35 | zmq::message_t message; 36 | socket->recv(&message); 37 | return message_to_string(message); 38 | } 39 | 40 | int ZmqUtil::poll(long timeout, vector* items) { 41 | return zmq::poll(items->data(), items->size(), timeout); 42 | } 43 | -------------------------------------------------------------------------------- /include/zmq/zmq_util.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef SRC_INCLUDE_ZMQ_ZMQ_UTIL_HPP_ 16 | #define SRC_INCLUDE_ZMQ_ZMQ_UTIL_HPP_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "types.hpp" 25 | #include "zmq.hpp" 26 | 27 | class ZmqUtilInterface { 28 | public: 29 | // Converts the data within a `zmq::message_t` into a string. 30 | string message_to_string(const zmq::message_t& message); 31 | // Converts a string into a `zmq::message_t`. 32 | zmq::message_t string_to_message(const string& s); 33 | // `send` a string over the socket. 34 | virtual void send_string(const string& s, zmq::socket_t* socket) = 0; 35 | // `recv` a string over the socket. 36 | virtual string recv_string(zmq::socket_t* socket) = 0; 37 | // `poll` is a wrapper around `zmq::poll` that takes a vector instead of a 38 | // pointer and a size. 39 | virtual int poll(long timeout, vector* items) = 0; 40 | }; 41 | 42 | class ZmqUtil : public ZmqUtilInterface { 43 | public: 44 | virtual void send_string(const string& s, zmq::socket_t* socket); 45 | virtual string recv_string(zmq::socket_t* socket); 46 | virtual int poll(long timeout, vector* items); 47 | }; 48 | 49 | extern ZmqUtilInterface* kZmqUtil; 50 | 51 | #endif // SRC_INCLUDE_ZMQ_ZMQ_UTIL_HPP_ 52 | -------------------------------------------------------------------------------- /k8s/k8s_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import os 19 | import zmq 20 | 21 | from add_nodes import add_nodes 22 | from remove_node import remove_node 23 | import util 24 | 25 | logging.basicConfig(filename='log_k8s.txt', level=logging.INFO) 26 | 27 | 28 | def run(): 29 | context = zmq.Context(1) 30 | client, apps_client = util.init_k8s() 31 | 32 | node_add_socket = context.socket(zmq.PULL) 33 | node_add_socket.bind('ipc:///tmp/node_add') 34 | 35 | node_remove_socket = context.socket(zmq.PULL) 36 | node_remove_socket.bind('ipc:///tmp/node_remove') 37 | 38 | poller = zmq.Poller() 39 | poller.register(node_add_socket, zmq.POLLIN) 40 | poller.register(node_remove_socket, zmq.POLLIN) 41 | 42 | cfile = '/fluent/conf/kvs-base.yml' 43 | 44 | while True: 45 | socks = dict(poller.poll(timeout=1000)) 46 | 47 | if node_add_socket in socks and socks[node_add_socket] == zmq.POLLIN: 48 | msg = node_add_socket.recv_string() 49 | args = msg.split(':') 50 | 51 | ntype = args[0] 52 | num = int(args[1]) 53 | logging.info('Adding %d new %s node(s)...' % (num, ntype)) 54 | 55 | add_nodes(client, apps_client, cfile, [ntype], [num]) 56 | logging.info('Successfully added %d %s node(s).' % (num, ntype)) 57 | 58 | if node_remove_socket in socks and socks[node_remove_socket] == \ 59 | zmq.POLLIN: 60 | msg = node_remove_socket.recv_string() 61 | args = msg.split(':') 62 | 63 | ntype = args[0] 64 | ip = args[1] 65 | 66 | remove_node(ip, ntype) 67 | logging.info('Successfully removed node %s.' % (ip)) 68 | 69 | 70 | if __name__ == '__main__': 71 | # wait for this file to appear before starting 72 | while not os.path.isfile('/fluent/setup_complete'): 73 | pass 74 | 75 | run() 76 | -------------------------------------------------------------------------------- /k8s/kops/create_cluster_object.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [[ -z "$1" ]] && [[ -z "$2" ]]; then 18 | echo "Usage: ./create_cluster_object.sh cluster-name cluster-state-store " 19 | echo "" 20 | echo "Cluster name and S3 Bucket used as kops state store must be specified." 21 | echo "If no SSH key is specified, the default SSH key (~/.ssh/id_rsa) will be used." 22 | 23 | exit 1 24 | fi 25 | 26 | if [[ -z "$3" ]]; then 27 | if [[ ! -f "~/.ssh/id_rsa" ]]; then 28 | echo "No SSH key specified and default SSH key (~/.ssh/id_rsa) does not exist." 29 | 30 | exit 1 31 | fi 32 | 33 | SSH_KEY=~/.ssh/id_rsa 34 | else 35 | SSH_KEY=$3 36 | fi 37 | 38 | if [[ -z "$AWS_ACCESS_KEY_ID" ]] || [[ -z "$AWS_SECRET_ACCESS_KEY" ]]; then 39 | echo "AWS access credentials are required to be stored in local environment variables for cluster creation." 40 | echo "Please use the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY variables." 41 | 42 | exit 1 43 | fi 44 | 45 | FLUSTER_CLUSTER_NAME=$1 46 | KOPS_STATE_STORE=$2 47 | 48 | echo "Creating cluster object..." 49 | kops create cluster --master-size c4.large --zones us-east-1a --ssh-public-key ${SSH_KEY}.pub ${FLUENT_CLUSTER_NAME} --networking kube-router > /dev/null 2>&1 50 | # delete default instance group that we won't use 51 | kops delete ig nodes --name ${FLUENT_CLUSTER_NAME} --yes > /dev/null 2>&1 52 | 53 | echo "Adding general instance group" 54 | sed "s|CLUSTER_NAME|$FLUENT_CLUSTER_NAME|g" yaml/igs/general-ig.yml > tmp.yml 55 | kops create -f tmp.yml > /dev/null 2>&1 56 | rm tmp.yml 57 | 58 | # create the cluster with just the routing instance group 59 | echo "Creating cluster on AWS..." 60 | kops update cluster --name ${FLUENT_CLUSTER_NAME} --yes > /dev/null 2>&1 61 | 62 | ./validate_cluster.sh 63 | -------------------------------------------------------------------------------- /k8s/kops/modify_ig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -z "$1" ] && [ -z "$2" ]; then 18 | echo "Usage: ./add_servers.sh node-type instance-count" 19 | echo "Valid node types are memory, ebs, benchmark, and routing." 20 | echo "If number of previous instances is not specified, it is assumed to be 0." 21 | exit 1 22 | fi 23 | 24 | YML_FILE=yaml/igs/$1-ig.yml 25 | 26 | sed "s|CLUSTER_NAME|$FLUENT_CLUSTER_NAME|g" $YML_FILE > tmp.yml 27 | sed -i "s|NUM_DUMMY|$2|g" tmp.yml 28 | 29 | kops replace -f tmp.yml --force > /dev/null 2>&1 30 | rm tmp.yml 31 | 32 | kops update cluster --name ${FLUENT_CLUSTER_NAME} --yes > /dev/null 2>&1 33 | -------------------------------------------------------------------------------- /k8s/kops/validate_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | echo "Validating cluster..." 18 | kops validate cluster > /dev/null 2>&1 19 | while [ $? -ne 0 ] 20 | do 21 | kops validate cluster > /dev/null 2>&1 22 | done 23 | 24 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/benchmark-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: benchmark-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: m4.2xlarge 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | role: Node 27 | rootVolumeSize: 32 28 | nodeLabels: 29 | role: benchmark 30 | subnets: 31 | - us-east-1a 32 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/ebs-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: ebs-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: r4.large 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | rootVolumeSize: 32 27 | role: Node 28 | nodeLabels: 29 | role: ebs 30 | subnets: 31 | - us-east-1a 32 | volumes: 33 | - device: /dev/xvdb 34 | encrypted: false 35 | size: 64 36 | type: gp2 37 | - device: /dev/xvdc 38 | encrypted: false 39 | size: 64 40 | type: gp2 41 | - device: /dev/xvdd 42 | encrypted: false 43 | size: 64 44 | type: gp2 45 | - device: /dev/xvde 46 | encrypted: false 47 | size: 64 48 | type: gp2 49 | volumeMounts: 50 | - device: /dev/xvdb 51 | filesystem: ext4 52 | path: /ebs/ebs_0 53 | - device: /dev/xvdc 54 | filesystem: ext4 55 | path: /ebs/ebs_1 56 | - device: /dev/xvdd 57 | filesystem: ext4 58 | path: /ebs/ebs_2 59 | - device: /dev/xvde 60 | filesystem: ext4 61 | path: /ebs/ebs_3 62 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/function-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: function-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: c5.2xlarge 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | rootVolumeSize: 32 27 | role: Node 28 | nodeLabels: 29 | role: function 30 | subnets: 31 | - us-east-1a 32 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/general-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: misc-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: m4.xlarge 24 | maxSize: 1 25 | minSize: 1 26 | role: Node 27 | rootVolumeSize: 32 28 | nodeLabels: 29 | role: general 30 | subnets: 31 | - us-east-1a 32 | 33 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/memory-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: memory-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: r4.2xlarge 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | rootVolumeSize: 32 27 | role: Node 28 | nodeLabels: 29 | role: memory 30 | subnets: 31 | - us-east-1a 32 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/routing-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: routing-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: r4.2xlarge 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | role: Node 27 | rootVolumeSize: 32 28 | nodeLabels: 29 | role: routing 30 | subnets: 31 | - us-east-1a 32 | -------------------------------------------------------------------------------- /k8s/kops/yaml/igs/scheduler-ig.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kops/v1alpha2 16 | kind: InstanceGroup 17 | metadata: 18 | labels: 19 | kops.k8s.io/cluster: CLUSTER_NAME 20 | name: scheduler-instances 21 | spec: 22 | image: kope.io/k8s-1.11-debian-stretch-amd64-hvm-ebs-2018-08-17 23 | machineType: c5.large 24 | maxSize: NUM_DUMMY 25 | minSize: NUM_DUMMY 26 | rootVolumeSize: 32 27 | role: Node 28 | nodeLabels: 29 | role: scheduler 30 | subnets: 31 | - us-east-1a 32 | -------------------------------------------------------------------------------- /k8s/remove_node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | import os 17 | 18 | import boto3 19 | import kubernetes as k8s 20 | import util 21 | 22 | ec2_client = boto3.client('ec2', os.getenv('AWS_REGION', 'us-east-1')) 23 | 24 | 25 | def remove_node(ip, ntype): 26 | client, _ = util.init_k8s() 27 | 28 | pod = util.get_pod_from_ip(client, ip) 29 | hostname = 'ip-%s.ec2.internal' % (ip.replace('.', '-')) 30 | 31 | podname = pod.metadata.name 32 | client.delete_namespaced_pod(name=podname, namespace=util.NAMESPACE, 33 | body=k8s.client.V1DeleteOptions()) 34 | client.delete_node(name=hostname, body=k8s.client.V1DeleteOptions()) 35 | 36 | prev_count = util.get_previous_count(client, ntype) 37 | util.run_process(['./modify_ig.sh', ntype, str(prev_count - 1)]) 38 | -------------------------------------------------------------------------------- /k8s/yaml/ds/benchmark-ds.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: benchmark-nodes 19 | labels: 20 | role: benchmark 21 | spec: 22 | selector: 23 | matchLabels: 24 | role: benchmark 25 | template: 26 | metadata: 27 | labels: 28 | role: benchmark 29 | spec: 30 | nodeSelector: 31 | role: benchmark 32 | containers: 33 | - name: benchmark-1 34 | image: fluentproject/executor 35 | env: 36 | - name: SYSTEM_FUNC 37 | value: benchmark 38 | - name: FUNCTION_ADDR 39 | value: FUNCTION_ADDR_DUMMY 40 | - name: THREAD_ID 41 | value: "0" 42 | - name: MGMT_IP 43 | value: "" 44 | - name: benchmark-2 45 | image: fluentproject/executor 46 | env: 47 | - name: SYSTEM_FUNC 48 | value: benchmark 49 | - name: FUNCTION_ADDR 50 | value: FUNCTION_ADDR_DUMMY 51 | - name: THREAD_ID 52 | value: "1" 53 | - name: MGMT_IP 54 | value: "" 55 | - name: benchmark-3 56 | image: fluentproject/executor 57 | env: 58 | - name: SYSTEM_FUNC 59 | value: benchmark 60 | - name: FUNCTION_ADDR 61 | value: FUNCTION_ADDR_DUMMY 62 | - name: THREAD_ID 63 | value: "2" 64 | - name: MGMT_IP 65 | value: "" 66 | - name: benchmark-4 67 | image: fluentproject/executor 68 | env: 69 | - name: SYSTEM_FUNC 70 | value: benchmark 71 | - name: FUNCTION_ADDR 72 | value: FUNCTION_ADDR_DUMMY 73 | - name: THREAD_ID 74 | value: "3" 75 | - name: MGMT_IP 76 | value: "" 77 | -------------------------------------------------------------------------------- /k8s/yaml/ds/ebs-ds.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: ebs-nodes 19 | labels: 20 | role: ebs 21 | spec: 22 | selector: 23 | matchLabels: 24 | role: ebs 25 | template: 26 | metadata: 27 | labels: 28 | role: ebs 29 | spec: 30 | nodeSelector: 31 | role: ebs 32 | hostNetwork: true 33 | containers: 34 | - name: ebs-container 35 | image: fluentproject/annakvs 36 | env: 37 | - name: SERVER_TYPE 38 | value: "ebs" 39 | - name: ROUTING_IPS 40 | value: ROUTING_IPS_DUMMY 41 | - name: SEED_IP 42 | value: SEED_IP_DUMMY 43 | - name: MGMT_IP 44 | value: MGMT_IP_DUMMY 45 | - name: MON_IPS 46 | value: MON_IPS_DUMMY 47 | volumeMounts: 48 | - mountPath: /ebs/ebs_0 49 | name: ebs0 50 | - mountPath: /ebs/ebs_1 51 | name: ebs1 52 | - mountPath: /ebs/ebs_2 53 | name: ebs2 54 | - mountPath: /ebs/ebs_3 55 | name: ebs3 56 | volumes: 57 | - name: ebs0 58 | hostPath: 59 | path: /ebs/ebs_0 60 | type: Directory 61 | - name: ebs1 62 | hostPath: 63 | path: /ebs/ebs_1 64 | type: Directory 65 | - name: ebs2 66 | hostPath: 67 | path: /ebs/ebs_2 68 | type: Directory 69 | - name: ebs3 70 | hostPath: 71 | path: /ebs/ebs_3 72 | type: Directory 73 | -------------------------------------------------------------------------------- /k8s/yaml/ds/memory-ds.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: memory-nodes 19 | labels: 20 | role: memory 21 | spec: 22 | selector: 23 | matchLabels: 24 | role: memory 25 | template: 26 | metadata: 27 | labels: 28 | role: memory 29 | spec: 30 | nodeSelector: 31 | role: memory 32 | hostNetwork: true 33 | containers: 34 | - name: memory-container 35 | image: fluentproject/annakvs 36 | imagePullPolicy: Always 37 | env: 38 | - name: SERVER_TYPE 39 | value: "memory" 40 | - name: ROUTING_IPS 41 | value: ROUTING_IPS_DUMMY 42 | - name: MGMT_IP 43 | value: MGMT_IP_DUMMY 44 | - name: SEED_IP 45 | value: SEED_IP_DUMMY 46 | - name: MON_IPS 47 | value: MON_IPS_DUMMY 48 | -------------------------------------------------------------------------------- /k8s/yaml/ds/routing-ds.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: routing-nodes 19 | labels: 20 | role: routing 21 | spec: 22 | selector: 23 | matchLabels: 24 | role: routing 25 | template: 26 | metadata: 27 | labels: 28 | role: routing 29 | spec: 30 | nodeSelector: 31 | role: routing 32 | hostNetwork: true 33 | containers: 34 | - name: routing-container 35 | image: fluentproject/annakvs 36 | env: 37 | - name: SERVER_TYPE 38 | value: r 39 | - name: MON_IPS 40 | value: MON_IPS_DUMMY 41 | -------------------------------------------------------------------------------- /k8s/yaml/ds/scheduler-ds.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: scheduler-nodes 19 | labels: 20 | role: scheduler 21 | spec: 22 | selector: 23 | matchLabels: 24 | role: scheduler 25 | template: 26 | metadata: 27 | labels: 28 | role: scheduler 29 | spec: 30 | nodeSelector: 31 | role: scheduler 32 | hostNetwork: true 33 | hostIPC: true 34 | containers: 35 | - name: scheduler-container 36 | image: fluentproject/executor 37 | imagePullPolicy: Always 38 | env: 39 | - name: ROUTE_ADDR 40 | value: ROUTE_ADDR_DUMMY 41 | - name: MGMT_IP 42 | value: MGMT_IP_DUMMY 43 | - name: SYSTEM_FUNC 44 | value: scheduler 45 | -------------------------------------------------------------------------------- /k8s/yaml/pods/kops-pod.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Pod 17 | metadata: 18 | name: kops-pod 19 | labels: 20 | role: kops 21 | spec: 22 | restartPolicy: Never 23 | containers: 24 | - name: kops-container 25 | image: fluentproject/kops 26 | env: 27 | - name: AWS_ACCESS_KEY_ID 28 | value: ACCESS_KEY_ID_DUMMY 29 | - name: AWS_SECRET_ACCESS_KEY 30 | value: SECRET_KEY_DUMMY 31 | - name: KOPS_STATE_STORE 32 | value: KOPS_BUCKET_DUMMY 33 | - name: FLUENT_CLUSTER_NAME 34 | value: CLUSTER_NAME 35 | nodeSelector: 36 | role: general 37 | -------------------------------------------------------------------------------- /k8s/yaml/pods/monitoring-pod.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Pod 17 | metadata: 18 | name: monitoring-pod 19 | labels: 20 | role: monitoring 21 | spec: 22 | containers: 23 | - name: monitoring-container 24 | image: fluentproject/annakvs 25 | env: 26 | - name: SERVER_TYPE 27 | value: mn 28 | - name: MGMT_IP 29 | value: MGMT_IP_DUMMY 30 | nodeSelector: 31 | role: general 32 | -------------------------------------------------------------------------------- /k8s/yaml/services/function.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: function-service 19 | spec: 20 | type: LoadBalancer 21 | ports: 22 | - port: 5000 23 | targetPort: 5000 24 | name: connect 25 | - port: 5001 26 | targetPort: 5001 27 | name: func-create 28 | - port: 5002 29 | targetPort: 5002 30 | name: func-call 31 | - port: 5003 32 | targetPort: 5003 33 | name: list 34 | - port: 5004 35 | targetPort: 5004 36 | name: dag-create 37 | - port: 5005 38 | targetPort: 5005 39 | name: dag-call 40 | - port: 5006 41 | targetPort: 5006 42 | name: dag-delete 43 | selector: 44 | role: scheduler 45 | -------------------------------------------------------------------------------- /k8s/yaml/services/routing.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: routing-service 19 | spec: 20 | type: LoadBalancer 21 | ports: 22 | - port: 6450 23 | targetPort: 6450 24 | name: routing-0 25 | - port: 6451 26 | targetPort: 6451 27 | name: routing-1 28 | - port: 6452 29 | targetPort: 6452 30 | name: routing-2 31 | - port: 6453 32 | targetPort: 6453 33 | name: routing-3 34 | selector: 35 | role: routing 36 | -------------------------------------------------------------------------------- /kvs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | PROJECT(Fluent-KVS) 17 | 18 | SET(FLUENT_KVS_MAJOR_VERSION ${FLUENT_MAJOR_VERSION}) 19 | SET(FLUENT_KVS_MINOR_VERSION ${FLUENT_MINOR_VERSION}) 20 | SET(FLUENT_KVS_PATCH_VERSION ${FLUENT_PATCH_VERSION}) 21 | 22 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 23 | INCLUDE_DIRECTORIES(include) 24 | INCLUDE_DIRECTORIES(tests) 25 | 26 | INCLUDE(FindProtobuf) 27 | FIND_PACKAGE(Protobuf REQUIRED) 28 | INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) 29 | PROTOBUF_GENERATE_CPP(PROTO_SRC PROTO_HEADER ./include/proto/metadata.proto 30 | ./include/proto/replication.proto) 31 | 32 | # need to build a target at this level or subdirs won't have the 33 | # protobuf files generated. 34 | ADD_LIBRARY(flkvs-proto ${PROTO_HEADER} ${PROTO_SRC}) 35 | 36 | LINK_DIRECTORIES(${ZEROMQ_LINK_DIRS} ${YAMLCPP_LINK_DIRS}) 37 | 38 | ADD_SUBDIRECTORY(src) 39 | ADD_SUBDIRECTORY(client/cpp) 40 | ADD_SUBDIRECTORY(tests) 41 | -------------------------------------------------------------------------------- /kvs/README.md: -------------------------------------------------------------------------------- 1 | # Anna 2 | 3 | Anna is a cloud-native, autoscaling, tiered storage engine that provides a range of options for consistency. To build and run Anna on your local machine, please run `./scripts/build-kvs.sh` followed by `./scripts/start-kvs-local.sh` from the project root directory. 4 | 5 | **TODO**: Add a longer description here. Also describe how to run just the KVS in Kubernetes. 6 | -------------------------------------------------------------------------------- /kvs/client/cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(LIBRARY_DEPENDENCIES 18 | protobuf 19 | flproto 20 | zmq 21 | flzmq 22 | yaml-cpp 23 | ) 24 | 25 | ADD_EXECUTABLE(flkvs-cli cli.cpp) 26 | TARGET_LINK_LIBRARIES(flkvs-cli ${LIBRARY_DEPENDENCIES}) 27 | ADD_DEPENDENCIES(flkvs-cli zeromq zeromqcpp) 28 | 29 | ADD_EXECUTABLE(flkvs-async-cli async_cli.cpp) 30 | TARGET_LINK_LIBRARIES(flkvs-async-cli ${LIBRARY_DEPENDENCIES}) 31 | ADD_DEPENDENCIES(flkvs-async-cli zeromq zeromqcpp) -------------------------------------------------------------------------------- /kvs/client/python/anna/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydro-project/fluent/51dc6de82334af4b8d5991d1b13ff2fc6a6fd650/kvs/client/python/anna/__init__.py -------------------------------------------------------------------------------- /kvs/client/python/anna/common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Define the number of memory threads 16 | MEMORY_THREAD_NUM = 4 17 | 18 | # Define the number of ebs threads 19 | EBS_THREAD_NUM = 4 20 | 21 | # Define the number of proxy worker threads 22 | PROXY_THREAD_NUM = 4 23 | 24 | # Number of tiers 25 | MIN_TIER = 1 26 | MAX_TIER = 2 27 | 28 | # Define port offset 29 | SERVER_PORT = 6560 30 | NODE_JOIN_BASE_PORT = 6660 31 | NODE_DEPART_BASE_PORT = 6760 32 | SELF_DEPART_BASE_PORT = 6860 33 | REPLICATION_FACTOR_BASE_PORT = 6960 34 | REQUEST_PULLING_BASE_PORT = 6460 35 | GOSSIP_BASE_PORT = 7060 36 | REPLICATION_FACTOR_CHANGE_BASE_PORT = 7160 37 | 38 | # used by proxies 39 | SEED_BASE_PORT = 6560 40 | NOTIFY_BASE_PORT = 6660 41 | KEY_ADDRESS_BASE_PORT = 6760 42 | 43 | # used by monitoring nodes 44 | DEPART_DONE_BASE_PORT = 6760 45 | LATENCY_REPORT_BASE_PORT = 6860 46 | 47 | # used by benchmark threads 48 | COMMAND_BASE_PORT = 6560 49 | 50 | class Thread(): 51 | def __init__(self, ip, tid): 52 | self.ip = ip 53 | self.tid = tid 54 | 55 | self._base = 'tcp://*:' 56 | self._ip_base = 'tcp://' + self.ip + ':' 57 | 58 | def get_ip(self): 59 | return self.ip 60 | 61 | def get_tid(self): 62 | return self.tid 63 | 64 | 65 | class UserThread(Thread): 66 | def get_request_pull_connect_addr(self): 67 | return self._ip_base + str(self.tid + REQUEST_PULLING_BASE_PORT) 68 | 69 | def get_request_pull_bind_addr(self): 70 | return self._base + str(self.tid + REQUEST_PULLING_BASE_PORT) 71 | 72 | def get_key_address_connect_addr(self): 73 | return self._ip_base + str(self.tid + KEY_ADDRESS_BASE_PORT) 74 | 75 | def get_key_address_bind_addr(self): 76 | return self._base + str(self.tid + KEY_ADDRESS_BASE_PORT) 77 | -------------------------------------------------------------------------------- /kvs/client/python/anna/zmq_util.py: -------------------------------------------------------------------------------- 1 | import zmq 2 | 3 | def send_request(req_obj, send_sock): 4 | req_string = req_obj.SerializeToString() 5 | 6 | send_sock.send(req_string) 7 | 8 | def recv_response(req_ids, rcv_sock, resp_class): 9 | responses = [] 10 | 11 | while len(responses) < len(req_ids): 12 | resp_obj = resp_class() 13 | resp = rcv_sock.recv() 14 | resp_obj.ParseFromString(resp) 15 | 16 | while resp_obj.response_id not in req_ids: 17 | resp_obj.Clear() 18 | resp_obj.ParseFromString(rcv_sock.recv()) 19 | 20 | responses.append(resp_obj) 21 | 22 | return responses 23 | 24 | class SocketCache(): 25 | def __init__(self, context, zmq_type): 26 | self.context = context 27 | self._cache = {} 28 | self.zmq_type = zmq_type 29 | 30 | def get(self, addr): 31 | if addr not in self._cache: 32 | sock = self.context.socket(self.zmq_type) 33 | sock.connect(addr) 34 | 35 | self._cache[addr] = sock 36 | 37 | return sock 38 | else: 39 | return self._cache[addr] 40 | -------------------------------------------------------------------------------- /kvs/client/python/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | import os 3 | from setuptools.command.install import install 4 | 5 | class InstallWrapper(install): 6 | def run(self): 7 | # compile the relevant protobufs 8 | self.compile_proto() 9 | 10 | # Run the standard PyPi copy 11 | install.run(self) 12 | 13 | # remove the compiled protobufs 14 | self.cleanup() 15 | 16 | def compile_proto(self): 17 | # compile the protobufs 18 | os.system('cd anna && protoc -I=../../../../include/proto --python_out=. ' + 19 | 'kvs.proto') 20 | 21 | os.system('cd anna && protoc -I=../../../../include/proto --python_out=. ' + 22 | 'functions.proto') 23 | 24 | def cleanup(self): 25 | os.system('rm anna/kvs_pb2.py') 26 | 27 | setup( 28 | name='Anna', 29 | version='0.1', 30 | packages=['anna', ], 31 | license='Apache v2', 32 | long_description='Client for the Anna KVS', 33 | install_requires=['zmq', 'protobuf'], 34 | cmdclass={'install': InstallWrapper} 35 | ) 36 | -------------------------------------------------------------------------------- /kvs/include/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 16 | 17 | ADD_SUBDIRECTORY(kvs) 18 | ADD_SUBDIRECTORY(route) 19 | ADD_SUBDIRECTORY(monitor) 20 | -------------------------------------------------------------------------------- /kvs/include/consistent_hash_map.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_CONSISTENT_HASH_MAP_HPP_ 16 | #define KVS_INCLUDE_CONSISTENT_HASH_MAP_HPP_ 17 | 18 | #include 19 | #include 20 | 21 | template >> 24 | class ConsistentHashMap { 25 | public: 26 | typedef typename Hash::ResultType size_type; 27 | typedef std::map, Alloc> map_type; 28 | typedef typename map_type::value_type value_type; 29 | typedef value_type& reference; 30 | typedef const value_type& const_reference; 31 | typedef typename map_type::iterator iterator; 32 | typedef Alloc allocator_type; 33 | 34 | public: 35 | ConsistentHashMap() {} 36 | 37 | ~ConsistentHashMap() {} 38 | 39 | public: 40 | std::size_t size() const { return nodes_.size(); } 41 | 42 | bool empty() const { return nodes_.empty(); } 43 | 44 | std::pair insert(const T& node) { 45 | size_type hash = hasher_(node); 46 | return nodes_.insert(value_type(hash, node)); 47 | } 48 | 49 | void erase(iterator it) { nodes_.erase(it); } 50 | 51 | std::size_t erase(const T& node) { 52 | size_type hash = hasher_(node); 53 | return nodes_.erase(hash); 54 | } 55 | 56 | iterator find(size_type hash) { 57 | if (nodes_.empty()) { 58 | return nodes_.end(); 59 | } 60 | 61 | iterator it = nodes_.lower_bound(hash); 62 | 63 | if (it == nodes_.end()) { 64 | it = nodes_.begin(); 65 | } 66 | 67 | return it; 68 | } 69 | 70 | iterator find(Key key) { return find(hasher_(key)); } 71 | 72 | iterator begin() { return nodes_.begin(); } 73 | 74 | iterator end() { return nodes_.end(); } 75 | 76 | private: 77 | Hash hasher_; 78 | map_type nodes_; 79 | }; 80 | 81 | #endif // KVS_INCLUDE_CONSISTENT_HASH_MAP_HPP_ 82 | -------------------------------------------------------------------------------- /kvs/include/hashers.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_HASHERS_HPP_ 16 | #define KVS_INCLUDE_HASHERS_HPP_ 17 | 18 | #include 19 | #include "kvs_threads.hpp" 20 | 21 | struct GlobalHasher { 22 | uint32_t operator()(const ServerThread& th) { 23 | // prepend a string to make the hash value different than 24 | // what it would be on the naked input 25 | return std::hash{}("GLOBAL" + th.virtual_id()); 26 | } 27 | 28 | uint32_t operator()(const Key& key) { 29 | // prepend a string to make the hash value different than 30 | // what it would be on the naked input 31 | return std::hash{}("GLOBAL" + key); 32 | } 33 | 34 | typedef uint32_t ResultType; 35 | }; 36 | 37 | struct LocalHasher { 38 | typedef std::hash::result_type ResultType; 39 | 40 | ResultType operator()(const ServerThread& th) { 41 | return std::hash{}(std::to_string(th.tid()) + "_" + 42 | std::to_string(th.virtual_num())); 43 | } 44 | 45 | ResultType operator()(const Key& key) { return std::hash{}(key); } 46 | }; 47 | 48 | #endif // KVS_INCLUDE_HASHERS_HPP_ 49 | -------------------------------------------------------------------------------- /kvs/include/kvs/base_kv_store.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_KVS_BASE_KV_STORE_HPP_ 16 | #define KVS_INCLUDE_KVS_BASE_KV_STORE_HPP_ 17 | 18 | #include "lattices/core_lattices.hpp" 19 | 20 | template 21 | class KVStore { 22 | protected: 23 | MapLattice db; 24 | 25 | public: 26 | KVStore() {} 27 | 28 | KVStore(MapLattice& other) { db = other; } 29 | 30 | V get(const K& k, unsigned& err_number) { 31 | if (!db.contains(k).reveal()) { 32 | err_number = 1; 33 | } 34 | return db.at(k); 35 | } 36 | 37 | void put(const K& k, const V& v) { return db.at(k).merge(v); } 38 | 39 | unsigned size(const K& k) { return db.at(k).size().reveal(); } 40 | 41 | void remove(const K& k) { db.remove(k); } 42 | }; 43 | 44 | #endif // KVS_INCLUDE_KVS_BASE_KV_STORE_HPP_ 45 | -------------------------------------------------------------------------------- /kvs/include/kvs_common.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_KVS_COMMON_HPP_ 16 | #define KVS_INCLUDE_KVS_COMMON_HPP_ 17 | 18 | #include "kvs_types.hpp" 19 | 20 | const unsigned kMetadataReplicationFactor = 1; 21 | const unsigned kMetadataLocalReplicationFactor = 1; 22 | 23 | const unsigned kVirtualThreadNum = 3000; 24 | 25 | const unsigned kMemoryTierId = 0; 26 | const unsigned kEbsTierId = 1; 27 | const unsigned kRoutingTierId = 100; 28 | 29 | const unsigned kMaxTier = 1; 30 | const vector kAllTierIds = {0, 1}; 31 | 32 | const unsigned kSloWorst = 3000; 33 | 34 | // run-time constants 35 | extern unsigned kSelfTierId; 36 | extern vector kSelfTierIdVector; 37 | 38 | extern unsigned kMemoryNodeCapacity; 39 | extern unsigned kEbsNodeCapacity; 40 | 41 | // the number of threads running in this executable 42 | extern unsigned kThreadNum; 43 | extern unsigned kMemoryThreadCount; 44 | extern unsigned kEbsThreadCount; 45 | extern unsigned kRoutingThreadCount; 46 | 47 | extern unsigned kDefaultGlobalMemoryReplication; 48 | extern unsigned kDefaultGlobalEbsReplication; 49 | extern unsigned kDefaultLocalReplication; 50 | extern unsigned kMinimumReplicaNumber; 51 | 52 | #endif // KVS_INCLUDE_KVS_COMMON_HPP_ 53 | -------------------------------------------------------------------------------- /kvs/include/kvs_types.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_TYPES_HPP_ 16 | #define KVS_INCLUDE_TYPES_HPP_ 17 | 18 | #include 19 | #include "kvs_threads.hpp" 20 | #include "types.hpp" 21 | 22 | using StorageStats = map>; 23 | 24 | using OccupancyStats = map>>; 25 | 26 | using AccessStats = map>; 27 | 28 | using TimePoint = std::chrono::time_point; 29 | 30 | using TierId = unsigned; 31 | 32 | using ServerThreadList = vector; 33 | 34 | using ServerThreadSet = std::unordered_set; 35 | 36 | #endif // KVS_INCLUDE_TYPES_HPP_ 37 | -------------------------------------------------------------------------------- /kvs/include/monitor/monitoring_handlers.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_MONITOR_MONITORING_HANDLERS_HPP_ 16 | #define KVS_INCLUDE_MONITOR_MONITORING_HANDLERS_HPP_ 17 | 18 | #include "hash_ring.hpp" 19 | #include "metadata.pb.h" 20 | 21 | void membership_handler(logger log, string& serialized, 22 | map& global_hash_rings, 23 | unsigned& new_memory_count, unsigned& new_ebs_count, 24 | TimePoint& grace_start, vector
& routing_ips, 25 | StorageStats& memory_storage, StorageStats& ebs_storage, 26 | OccupancyStats& memory_occupancy, 27 | OccupancyStats& ebs_occupancy, 28 | map>& key_access_frequency); 29 | 30 | void depart_done_handler(logger log, string& serialized, 31 | map& departing_node_map, 32 | Address management_ip, bool& removing_memory_node, 33 | bool& removing_ebs_node, SocketCache& pushers, 34 | TimePoint& grace_start); 35 | 36 | void feedback_handler( 37 | string& serialized, map& user_latency, 38 | map& user_throughput, 39 | map>& latency_miss_ratio_map); 40 | 41 | #endif // KVS_INCLUDE_MONITOR_MONITORING_HANDLERS_HPP_ 42 | -------------------------------------------------------------------------------- /kvs/include/proto/metadata.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto2"; 16 | 17 | message ServerThreadStatistics { 18 | required uint64 storage_consumption = 1; 19 | required double occupancy = 2; 20 | required uint32 epoch = 3; 21 | required uint32 access_count = 4; 22 | } 23 | 24 | message KeyAccessData { 25 | message KeyCount { 26 | required string key = 1; 27 | required uint32 access_count = 2; 28 | } 29 | 30 | repeated KeyCount keys = 1; 31 | } 32 | 33 | message TierMembership { 34 | message Tier { 35 | message Server { 36 | required string public_ip = 1; 37 | required string private_ip = 2; 38 | } 39 | 40 | required uint32 tier_id = 1; 41 | repeated Server servers = 2; 42 | } 43 | 44 | repeated Tier tiers = 1; 45 | } 46 | 47 | message KeySizeData { 48 | message KeySize { 49 | required string key = 1; 50 | required uint32 size = 2; 51 | } 52 | repeated KeySize key_sizes = 1; 53 | } 54 | -------------------------------------------------------------------------------- /kvs/include/proto/replication.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto2"; 16 | 17 | message Replication { 18 | required uint32 tier_id = 1; 19 | required uint32 replication_factor = 2; 20 | } 21 | 22 | message ReplicationFactor { 23 | required string key = 1; 24 | repeated Replication global = 2; 25 | repeated Replication local = 3; 26 | } 27 | 28 | message ReplicationFactorUpdate { 29 | repeated ReplicationFactor key_reps = 1; 30 | } 31 | -------------------------------------------------------------------------------- /kvs/include/route/routing_handlers.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_INCLUDE_ROUTE_ROUTING_HANDLERS_HPP_ 16 | #define KVS_INCLUDE_ROUTE_ROUTING_HANDLERS_HPP_ 17 | 18 | #include "hash_ring.hpp" 19 | #include "metadata.pb.h" 20 | #include "replication.pb.h" 21 | 22 | string seed_handler(logger log, map& global_hash_rings); 23 | 24 | void membership_handler(logger log, string& serialized, SocketCache& pushers, 25 | map& global_hash_rings, 26 | unsigned thread_id, Address ip); 27 | 28 | void replication_response_handler( 29 | logger log, string& serialized, SocketCache& pushers, RoutingThread& rt, 30 | map& global_hash_rings, 31 | map& local_hash_rings, 32 | map& key_replication_map, 33 | map>>& pending_requests, unsigned& seed); 34 | 35 | void replication_change_handler(logger log, string& serialized, 36 | SocketCache& pushers, 37 | map& key_replication_map, 38 | unsigned thread_id, Address ip); 39 | 40 | void address_handler(logger log, string& serialized, SocketCache& pushers, 41 | RoutingThread& rt, 42 | map& global_hash_rings, 43 | map& local_hash_rings, 44 | map& key_replication_map, 45 | map>>& pending_requests, 46 | unsigned& seed); 47 | 48 | #endif // KVS_INCLUDE_ROUTE_ROUTING_HANDLERS_HPP_ 49 | -------------------------------------------------------------------------------- /kvs/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(KV_SRC_DEPENDENCIES 18 | ${PROTO_SRC} 19 | ${PROTO_HEADER} 20 | ) 21 | 22 | SET(KV_LIBRARY_DEPENDENCIES 23 | protobuf 24 | flproto 25 | flkvs-proto 26 | pthread 27 | zmq 28 | flzmq 29 | yaml-cpp 30 | ) 31 | 32 | ADD_SUBDIRECTORY(hash_ring) 33 | ADD_SUBDIRECTORY(kvs) 34 | ADD_SUBDIRECTORY(monitor) 35 | ADD_SUBDIRECTORY(route) 36 | ADD_SUBDIRECTORY(benchmark) 37 | -------------------------------------------------------------------------------- /kvs/src/benchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | ADD_EXECUTABLE(flkvs-bench benchmark.cpp) 18 | TARGET_LINK_LIBRARIES(flkvs-bench flkvs-ring ${KV_LIBRARY_DEPENDENCIES}) 19 | ADD_DEPENDENCIES(flkvs-bench zeromq zeromqcpp) 20 | 21 | ADD_EXECUTABLE(flkvs-bench-trigger trigger.cpp) 22 | TARGET_LINK_LIBRARIES(flkvs-bench-trigger flkvs-ring ${KV_LIBRARY_DEPENDENCIES}) 23 | ADD_DEPENDENCIES(flkvs-bench-trigger flkvs-ring zeromq zeromqcpp) 24 | -------------------------------------------------------------------------------- /kvs/src/benchmark/trigger.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "common.hpp" 18 | #include "hash_ring.hpp" 19 | #include "kvs_common.hpp" 20 | #include "threads.hpp" 21 | #include "yaml-cpp/yaml.h" 22 | 23 | // TODO(vikram): We probably don't want to have to define all of these here? 24 | ZmqUtil zmq_util; 25 | ZmqUtilInterface* kZmqUtil = &zmq_util; 26 | 27 | HashRingUtil hash_ring_util; 28 | HashRingUtilInterface* kHashRingUtil = &hash_ring_util; 29 | 30 | unsigned kBenchmarkThreadNum = 1; 31 | unsigned kRoutingThreadCount = 1; 32 | unsigned kDefaultLocalReplication = 1; 33 | 34 | int main(int argc, char* argv[]) { 35 | if (argc != 2) { 36 | std::cerr << "Usage: " << argv[0] << " " << std::endl; 37 | return 1; 38 | } 39 | 40 | unsigned thread_num = atoi(argv[1]); 41 | 42 | // read the YAML conf 43 | vector
benchmark_address; 44 | YAML::Node conf = YAML::LoadFile("conf/kvs-config.yml"); 45 | YAML::Node benchmark = conf["benchmark"]; 46 | 47 | for (const YAML::Node& node : benchmark) { 48 | benchmark_address.push_back(node.as
()); 49 | } 50 | 51 | zmq::context_t context(1); 52 | SocketCache pushers(&context, ZMQ_PUSH); 53 | 54 | string command; 55 | while (true) { 56 | std::cout << "command> "; 57 | getline(std::cin, command); 58 | 59 | for (const Address address : benchmark_address) { 60 | for (unsigned tid = 0; tid < thread_num; tid++) { 61 | BenchmarkThread bt = BenchmarkThread(address, tid); 62 | 63 | kZmqUtil->send_string(command, 64 | &pushers[bt.benchmark_command_address()]); 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /kvs/src/hash_ring/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | ADD_LIBRARY(flkvs-ring STATIC hash_ring.cpp) 18 | TARGET_LINK_LIBRARIES(flkvs-ring flkvs-proto ${KV_LIBRARY_DEPENDENCIES}) 19 | ADD_DEPENDENCIES(flkvs-ring zeromq zeromqcpp) 20 | -------------------------------------------------------------------------------- /kvs/src/kvs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(KVS_SOURCE 18 | server.cpp 19 | node_join_handler.cpp 20 | node_depart_handler.cpp 21 | self_depart_handler.cpp 22 | user_request_handler.cpp 23 | gossip_handler.cpp 24 | replication_response_handler.cpp 25 | replication_change_handler.cpp 26 | cache_ip_response_handler.cpp 27 | utils.cpp) 28 | 29 | ADD_EXECUTABLE(flkvs ${KVS_SOURCE}) 30 | TARGET_LINK_LIBRARIES(flkvs flkvs-ring ${KV_LIBRARY_DEPENDENCIES}) 31 | ADD_DEPENDENCIES(flkvs flzmq zeromq zeromqcpp) 32 | -------------------------------------------------------------------------------- /kvs/src/kvs/node_depart_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | void node_depart_handler(unsigned thread_id, Address public_ip, 18 | Address private_ip, 19 | map& global_hash_rings, 20 | logger log, string& serialized, SocketCache& pushers) { 21 | vector v; 22 | split(serialized, ':', v); 23 | 24 | unsigned tier = stoi(v[0]); 25 | Address departing_public_ip = v[1]; 26 | Address departing_private_ip = v[2]; 27 | log->info("Received departure for node {}/{} on tier {}.", 28 | departing_public_ip, departing_private_ip, tier); 29 | 30 | // update hash ring 31 | global_hash_rings[tier].remove(departing_public_ip, departing_private_ip, 0); 32 | 33 | if (thread_id == 0) { 34 | // tell all worker threads about the node departure 35 | for (unsigned tid = 1; tid < kThreadNum; tid++) { 36 | kZmqUtil->send_string(serialized, 37 | &pushers[ServerThread(public_ip, private_ip, tid) 38 | .node_depart_connect_address()]); 39 | } 40 | 41 | for (const auto& pair : global_hash_rings) { 42 | log->info("Hash ring for tier {} size is {}.", pair.first, 43 | pair.second.size()); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /kvs/src/monitor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(MONITORING_SOURCE 18 | monitoring.cpp 19 | membership_handler.cpp 20 | depart_done_handler.cpp 21 | feedback_handler.cpp 22 | stats_helpers.cpp 23 | replication_helpers.cpp 24 | elasticity.cpp 25 | storage_policy.cpp 26 | movement_policy.cpp 27 | slo_policy.cpp) 28 | 29 | ADD_EXECUTABLE(flmonitor ${MONITORING_SOURCE}) 30 | TARGET_LINK_LIBRARIES(flmonitor flkvs-ring ${KV_LIBRARY_DEPENDENCIES}) 31 | ADD_DEPENDENCIES(flmonitor zeromq zeromqcpp) 32 | -------------------------------------------------------------------------------- /kvs/src/monitor/depart_done_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "monitor/monitoring_handlers.hpp" 16 | 17 | void depart_done_handler(logger log, string& serialized, 18 | map& departing_node_map, 19 | Address management_ip, bool& removing_memory_node, 20 | bool& removing_ebs_node, SocketCache& pushers, 21 | TimePoint& grace_start) { 22 | vector tokens; 23 | split(serialized, '_', tokens); 24 | 25 | Address departed_public_ip = tokens[0]; 26 | Address departed_private_ip = tokens[1]; 27 | unsigned tier_id = stoi(tokens[2]); 28 | 29 | if (departing_node_map.find(departed_private_ip) != 30 | departing_node_map.end()) { 31 | departing_node_map[departed_private_ip] -= 1; 32 | 33 | if (departing_node_map[departed_private_ip] == 0) { 34 | string ntype; 35 | if (tier_id == 1) { 36 | ntype = "memory"; 37 | removing_memory_node = false; 38 | } else { 39 | ntype = "ebs"; 40 | removing_ebs_node = false; 41 | } 42 | 43 | log->info("Removing {} node {}/{}.", ntype, departed_public_ip, 44 | departed_private_ip); 45 | 46 | string mgmt_addr = "tcp://" + management_ip + ":7001"; 47 | string message = "remove:" + departed_private_ip + ":" + ntype; 48 | 49 | kZmqUtil->send_string(message, &pushers[mgmt_addr]); 50 | 51 | // reset grace period timer 52 | grace_start = std::chrono::system_clock::now(); 53 | departing_node_map.erase(departed_private_ip); 54 | } 55 | } else { 56 | log->error("Missing entry in the depart done map."); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /kvs/src/monitor/elasticity.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "monitor/monitoring_utils.hpp" 16 | 17 | void add_node(logger log, string tier, unsigned number, unsigned& adding, 18 | SocketCache& pushers, const Address& management_ip) { 19 | log->info("Adding {} node(s) in tier {}.", std::to_string(number), tier); 20 | 21 | string mgmt_addr = "tcp://" + management_ip + ":7001"; 22 | string message = "add:" + std::to_string(number) + ":" + tier; 23 | 24 | kZmqUtil->send_string(message, &pushers[mgmt_addr]); 25 | adding = number; 26 | } 27 | 28 | void remove_node(logger log, ServerThread& node, string tier, bool& removing, 29 | SocketCache& pushers, 30 | map& departing_node_map, 31 | MonitoringThread& mt) { 32 | auto connection_addr = node.self_depart_connect_address(); 33 | departing_node_map[node.private_ip()] = 34 | kTierMetadata[kMemoryTierId].thread_number_; 35 | auto ack_addr = mt.depart_done_connect_address(); 36 | 37 | kZmqUtil->send_string(ack_addr, &pushers[connection_addr]); 38 | removing = true; 39 | } 40 | -------------------------------------------------------------------------------- /kvs/src/monitor/feedback_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "monitor/monitoring_handlers.hpp" 16 | 17 | void feedback_handler( 18 | string& serialized, map& user_latency, 19 | map& user_throughput, 20 | map>& latency_miss_ratio_map) { 21 | UserFeedback fb; 22 | fb.ParseFromString(serialized); 23 | 24 | if (fb.has_finish() && fb.finish()) { 25 | user_latency.erase(fb.uid()); 26 | } else { 27 | // collect latency and throughput feedback 28 | user_latency[fb.uid()] = fb.latency(); 29 | user_throughput[fb.uid()] = fb.throughput(); 30 | 31 | // collect replication factor adjustment factors 32 | for (const auto& key_latency_pair : fb.key_latency()) { 33 | Key key = key_latency_pair.key(); 34 | double observed_key_latency = key_latency_pair.latency(); 35 | 36 | if (latency_miss_ratio_map.find(key) == latency_miss_ratio_map.end()) { 37 | latency_miss_ratio_map[key].first = observed_key_latency / kSloWorst; 38 | latency_miss_ratio_map[key].second = 1; 39 | } else { 40 | latency_miss_ratio_map[key].first = 41 | (latency_miss_ratio_map[key].first * 42 | latency_miss_ratio_map[key].second + 43 | observed_key_latency / kSloWorst) / 44 | (latency_miss_ratio_map[key].second + 1); 45 | latency_miss_ratio_map[key].second += 1; 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /kvs/src/route/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(ROUTING_SOURCE 18 | routing.cpp 19 | seed_handler.cpp 20 | membership_handler.cpp 21 | replication_response_handler.cpp 22 | replication_change_handler.cpp 23 | address_handler.cpp) 24 | 25 | ADD_EXECUTABLE(flroute ${ROUTING_SOURCE}) 26 | TARGET_LINK_LIBRARIES(flroute flkvs-ring ${KV_LIBRARY_DEPENDENCIES}) 27 | ADD_DEPENDENCIES(flroute flkvs-ring zeromq zeromqcpp) 28 | -------------------------------------------------------------------------------- /kvs/src/route/replication_change_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | void replication_change_handler(logger log, string& serialized, 18 | SocketCache& pushers, 19 | map& key_replication_map, 20 | unsigned thread_id, Address ip) { 21 | if (thread_id == 0) { 22 | // tell all worker threads about the replication factor change 23 | for (unsigned tid = 1; tid < kRoutingThreadCount; tid++) { 24 | kZmqUtil->send_string( 25 | serialized, &pushers[RoutingThread(ip, tid) 26 | .replication_change_connect_address()]); 27 | } 28 | } 29 | 30 | ReplicationFactorUpdate update; 31 | update.ParseFromString(serialized); 32 | 33 | for (const auto& key_rep : update.key_reps()) { 34 | Key key = key_rep.key(); 35 | log->info("Received a replication factor change for key {}.", key); 36 | 37 | for (const Replication& global : key_rep.global()) { 38 | key_replication_map[key].global_replication_[global.tier_id()] = 39 | global.replication_factor(); 40 | } 41 | 42 | for (const Replication& local : key_rep.local()) { 43 | key_replication_map[key].local_replication_[local.tier_id()] = 44 | local.replication_factor(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /kvs/src/route/seed_handler.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | string seed_handler(logger log, 18 | map& global_hash_rings) { 19 | log->info("Received an address request."); 20 | 21 | TierMembership membership; 22 | 23 | for (const auto& pair : global_hash_rings) { 24 | TierId tid = pair.first; 25 | GlobalHashRing hash_ring = pair.second; 26 | 27 | TierMembership_Tier* tier = membership.add_tiers(); 28 | tier->set_tier_id(tid); 29 | 30 | for (const ServerThread& st : hash_ring.get_unique_servers()) { 31 | auto server = tier->add_servers(); 32 | server->set_private_ip(st.private_ip()); 33 | server->set_public_ip(st.public_ip()); 34 | } 35 | } 36 | 37 | string serialized; 38 | membership.SerializeToString(&serialized); 39 | return serialized; 40 | } 41 | -------------------------------------------------------------------------------- /kvs/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | ADD_SUBDIRECTORY(include) 18 | ADD_SUBDIRECTORY(kvs) 19 | ADD_SUBDIRECTORY(route) 20 | 21 | ADD_CUSTOM_TARGET(ctest ${CMAKE_CTEST_COMMAND}) 22 | 23 | IF(${CMAKE_BUILD_TYPE} STREQUAL "Debug") 24 | SETUP_TARGET_FOR_COVERAGE(test-coverage ctest coverage) 25 | ENDIF() 26 | -------------------------------------------------------------------------------- /kvs/tests/include/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | ADD_SUBDIRECTORY(lattices) 18 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | ADD_EXECUTABLE(run_lattice_tests run_lattice_tests.cpp) 18 | TARGET_LINK_LIBRARIES (run_lattice_tests gtest gmock) 19 | ADD_DEPENDENCIES(run_lattice_tests gtest) 20 | 21 | ADD_TEST(NAME LatticeTest COMMAND run_lattice_tests) 22 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/run_lattice_tests.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "test_bool_lattice.hpp" 21 | #include "test_map_lattice.hpp" 22 | #include "test_max_lattice.hpp" 23 | #include "test_ordered_set_lattice.hpp" 24 | #include "test_set_lattice.hpp" 25 | 26 | int main(int argc, char *argv[]) { 27 | testing::InitGoogleTest(&argc, argv); 28 | return RUN_ALL_TESTS(); 29 | } 30 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/test_bool_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "lattices/core_lattices.hpp" 22 | 23 | class BoolLatticeTest : public ::testing::Test { 24 | protected: 25 | BoolLattice* bl; 26 | BoolLatticeTest() { bl = new BoolLattice; } 27 | virtual ~BoolLatticeTest() { delete bl; } 28 | }; 29 | 30 | const int foo() { return 5; } 31 | 32 | TEST_F(BoolLatticeTest, Assign) { 33 | EXPECT_EQ(false, bl->reveal()); 34 | bl->assign(true); 35 | EXPECT_EQ(true, bl->reveal()); 36 | bl->assign(false); 37 | EXPECT_EQ(false, bl->reveal()); 38 | } 39 | 40 | TEST_F(BoolLatticeTest, MergeByValue) { 41 | EXPECT_EQ(false, bl->reveal()); 42 | bl->merge(true); 43 | EXPECT_EQ(true, bl->reveal()); 44 | bl->merge(false); 45 | EXPECT_EQ(true, bl->reveal()); 46 | } 47 | 48 | TEST_F(BoolLatticeTest, MergeByLattice) { 49 | EXPECT_EQ(false, bl->reveal()); 50 | bl->merge(BoolLattice(true)); 51 | EXPECT_EQ(true, bl->reveal()); 52 | bl->merge(BoolLattice(false)); 53 | EXPECT_EQ(true, bl->reveal()); 54 | } 55 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/test_max_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "lattices/core_lattices.hpp" 22 | 23 | template 24 | class MaxLatticeTest : public ::testing::Test { 25 | protected: 26 | MaxLattice* ml; 27 | MaxLatticeTest() { ml = new MaxLattice; } 28 | virtual ~MaxLatticeTest() { delete ml; } 29 | }; 30 | 31 | typedef ::testing::Types MaxTypes; 32 | TYPED_TEST_CASE(MaxLatticeTest, MaxTypes); 33 | 34 | TYPED_TEST(MaxLatticeTest, Assign) { 35 | EXPECT_EQ(0, this->ml->reveal()); 36 | this->ml->assign(10); 37 | EXPECT_EQ(10, this->ml->reveal()); 38 | this->ml->assign(5); 39 | EXPECT_EQ(5, this->ml->reveal()); 40 | } 41 | 42 | TYPED_TEST(MaxLatticeTest, MergeByValue) { 43 | EXPECT_EQ(0, this->ml->reveal()); 44 | this->ml->merge(10); 45 | EXPECT_EQ(10, this->ml->reveal()); 46 | this->ml->merge(5); 47 | EXPECT_EQ(10, this->ml->reveal()); 48 | } 49 | 50 | TYPED_TEST(MaxLatticeTest, MergeByLattice) { 51 | EXPECT_EQ(0, this->ml->reveal()); 52 | this->ml->merge(MaxLattice(10)); 53 | EXPECT_EQ(10, this->ml->reveal()); 54 | this->ml->merge(MaxLattice(5)); 55 | EXPECT_EQ(10, this->ml->reveal()); 56 | } 57 | 58 | TYPED_TEST(MaxLatticeTest, Add) { 59 | MaxLattice res = this->ml->add(5); 60 | EXPECT_EQ(5, res.reveal()); 61 | } 62 | 63 | TYPED_TEST(MaxLatticeTest, Subtract) { 64 | MaxLattice res = this->ml->subtract(5); 65 | EXPECT_EQ(-5, res.reveal()); 66 | } 67 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/test_ordered_set_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "lattices/core_lattices.hpp" 22 | 23 | class OrderedSetLatticeTest : public ::testing::Test { 24 | protected: 25 | OrderedSetLattice* sl; 26 | // Note that the order in these initializer lists is not preserved; 27 | // ordered_set will sort the items itself. 28 | ordered_set set1{'a', 'b', 'c'}; 29 | ordered_set set2{'c', 'd', 'e'}; 30 | ordered_set set3{'a', 'b', 'c', 'd', 'e'}; 31 | OrderedSetLatticeTest() { sl = new OrderedSetLattice; } 32 | virtual ~OrderedSetLatticeTest() { delete sl; } 33 | }; 34 | 35 | const int flow_test_ordered_set() { return 5; } 36 | 37 | TEST_F(OrderedSetLatticeTest, Assign) { 38 | EXPECT_EQ(0, sl->size().reveal()); 39 | sl->assign(set1); 40 | EXPECT_EQ(3, sl->size().reveal()); 41 | EXPECT_EQ(set1, sl->reveal()); 42 | } 43 | 44 | TEST_F(OrderedSetLatticeTest, MergeByValue) { 45 | EXPECT_EQ(0, sl->size().reveal()); 46 | sl->merge(set1); 47 | EXPECT_EQ(3, sl->size().reveal()); 48 | EXPECT_EQ(set1, sl->reveal()); 49 | sl->merge(set2); 50 | EXPECT_EQ(5, sl->size().reveal()); 51 | EXPECT_EQ(set3, sl->reveal()); 52 | } 53 | 54 | TEST_F(OrderedSetLatticeTest, MergeByLattice) { 55 | EXPECT_EQ(0, sl->size().reveal()); 56 | sl->merge(OrderedSetLattice(set1)); 57 | EXPECT_EQ(3, sl->size().reveal()); 58 | EXPECT_EQ(set1, sl->reveal()); 59 | sl->merge(OrderedSetLattice(set2)); 60 | EXPECT_EQ(5, sl->size().reveal()); 61 | EXPECT_EQ(set3, sl->reveal()); 62 | } 63 | 64 | TEST_F(OrderedSetLatticeTest, Intersection) { 65 | sl->merge(set1); 66 | OrderedSetLattice res = sl->intersect(set2); 67 | EXPECT_EQ(ordered_set({'c'}), res.reveal()); 68 | } 69 | 70 | TEST_F(OrderedSetLatticeTest, Ordering) { 71 | sl->merge(set2); 72 | EXPECT_EQ('c', *(sl->reveal().cbegin())); 73 | sl->merge(set1); 74 | EXPECT_EQ('a', *(sl->reveal().cbegin())); 75 | } 76 | -------------------------------------------------------------------------------- /kvs/tests/include/lattices/test_set_lattice.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "lattices/core_lattices.hpp" 22 | 23 | class SetLatticeTest : public ::testing::Test { 24 | protected: 25 | SetLattice* sl; 26 | set set1 = {'a', 'b', 'c'}; 27 | set set2 = {'c', 'd', 'e'}; 28 | set set3 = {'a', 'd', 'e', 'b', 'c'}; 29 | SetLatticeTest() { sl = new SetLattice; } 30 | virtual ~SetLatticeTest() { delete sl; } 31 | }; 32 | 33 | const int flow_test_set() { return 5; } 34 | 35 | TEST_F(SetLatticeTest, Assign) { 36 | EXPECT_EQ(0, sl->size().reveal()); 37 | sl->assign(set1); 38 | EXPECT_EQ(3, sl->size().reveal()); 39 | EXPECT_EQ(set1, sl->reveal()); 40 | } 41 | 42 | TEST_F(SetLatticeTest, MergeByValue) { 43 | EXPECT_EQ(0, sl->size().reveal()); 44 | sl->merge(set1); 45 | EXPECT_EQ(3, sl->size().reveal()); 46 | EXPECT_EQ(set1, sl->reveal()); 47 | sl->merge(set2); 48 | EXPECT_EQ(5, sl->size().reveal()); 49 | EXPECT_EQ(set3, sl->reveal()); 50 | } 51 | 52 | TEST_F(SetLatticeTest, MergeByLattice) { 53 | EXPECT_EQ(0, sl->size().reveal()); 54 | sl->merge(SetLattice(set1)); 55 | EXPECT_EQ(3, sl->size().reveal()); 56 | EXPECT_EQ(set1, sl->reveal()); 57 | sl->merge(SetLattice(set2)); 58 | EXPECT_EQ(5, sl->size().reveal()); 59 | EXPECT_EQ(set3, sl->reveal()); 60 | } 61 | 62 | TEST_F(SetLatticeTest, Intersection) { 63 | sl->merge(set1); 64 | SetLattice res = sl->intersect(set2); 65 | EXPECT_EQ(set({'c'}), res.reveal()); 66 | } 67 | -------------------------------------------------------------------------------- /kvs/tests/kvs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(KVS_SRC_DIR ${CMAKE_SOURCE_DIR}/kvs/src/kvs) 18 | 19 | FILE(GLOB HANDLER_SOURCES "${KVS_SRC_DIR}/*handler.cpp") 20 | FILE(GLOB MOCK_SOURCES "${CMAKE_SOURCE_DIR}/kvs/tests/mock/*.cpp") 21 | 22 | ADD_EXECUTABLE(run_server_handler_tests 23 | run_server_handler_tests.cpp 24 | ${MOCK_SOURCES} 25 | ${HANDLER_SOURCES} 26 | ${KVS_SRC_DIR}/utils.cpp) 27 | 28 | TARGET_LINK_LIBRARIES(run_server_handler_tests gtest gmock pthread flkvs-ring zmq) 29 | ADD_DEPENDENCIES(run_server_handler_tests gtest) 30 | 31 | ADD_TEST(NAME ServerTests COMMAND run_server_handler_tests) 32 | -------------------------------------------------------------------------------- /kvs/tests/kvs/run_server_handler_tests.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "gmock/gmock.h" 19 | #include "gtest/gtest.h" 20 | 21 | #include "kvs.pb.h" 22 | #include "kvs/server_utils.hpp" 23 | #include "metadata.pb.h" 24 | #include "replication.pb.h" 25 | #include "types.hpp" 26 | 27 | #include "server_handler_base.hpp" 28 | #include "test_node_depart_handler.hpp" 29 | #include "test_node_join_handler.hpp" 30 | #include "test_self_depart_handler.hpp" 31 | #include "test_user_request_handler.hpp" 32 | 33 | unsigned kDefaultLocalReplication = 1; 34 | unsigned kSelfTierId = kMemoryTierId; 35 | unsigned kThreadNum = 1; 36 | 37 | vector kSelfTierIdVector = {kSelfTierId}; 38 | map kTierMetadata = {}; 39 | 40 | unsigned kEbsThreadNum = 1; 41 | unsigned kMemoryThreadNum = 1; 42 | unsigned kRoutingThreadNum = 1; 43 | 44 | int main(int argc, char* argv[]) { 45 | log_->set_level(spdlog::level::info); 46 | testing::InitGoogleTest(&argc, argv); 47 | return RUN_ALL_TESTS(); 48 | } 49 | -------------------------------------------------------------------------------- /kvs/tests/kvs/test_gossip_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | TEST_F(ServerHandlerTest, SimpleGossipReceive) { 18 | Key key = "key"; 19 | string value = "value"; 20 | string put_request = put_key_request(key, value, ip); 21 | 22 | unsigned access_count = 0; 23 | unsigned seed = 0; 24 | unsigned error; 25 | auto now = std::chrono::system_clock::now(); 26 | 27 | EXPECT_EQ(local_changeset.size(), 0); 28 | 29 | gossip_handler(seed, put_request, global_hash_rings, local_hash_rings, 30 | key_size_map, pending_gossip, metadata_map, wt, serializer, 31 | pushers); 32 | 33 | EXPECT_EQ(pending_gossip.size(), 0); 34 | EXPECT_EQ(serializer->get(key, error).reveal().value, value); 35 | } 36 | 37 | TEST_F(ServerHandlerTest, GossipUpdate) { 38 | Key key = "key"; 39 | string value = "value1"; 40 | serializer->put(key, value, (unsigned)0); 41 | 42 | value = "value2"; 43 | 44 | string put_request = put_key_request(key, value, ip); 45 | 46 | unsigned access_count = 0; 47 | unsigned seed = 0; 48 | unsigned error; 49 | auto now = std::chrono::system_clock::now(); 50 | 51 | EXPECT_EQ(local_changeset.size(), 0); 52 | 53 | gossip_handler(seed, put_request, global_hash_rings, local_hash_rings, 54 | key_size_map, pending_gossip, metadata_map, wt, serializer, 55 | pushers); 56 | 57 | EXPECT_EQ(pending_gossip.size(), 0); 58 | EXPECT_EQ(serializer->get(key, error).reveal().value, value); 59 | } 60 | 61 | // TODO: test pending gossip 62 | // TODO: test gossip forwarding 63 | -------------------------------------------------------------------------------- /kvs/tests/kvs/test_node_depart_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | TEST_F(ServerHandlerTest, SimpleNodeDepart) { 18 | kThreadNum = 2; 19 | global_hash_rings[kMemoryTierId].insert("127.0.0.2", "127.0.0.2", 0, 0); 20 | 21 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 6000); 22 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 2); 23 | 24 | string serialized = std::to_string(kMemoryTierId) + ":127.0.0.2:127.0.0.2"; 25 | node_depart_handler(thread_id, ip, ip, global_hash_rings, log_, serialized, 26 | pushers); 27 | 28 | vector messages = get_zmq_messages(); 29 | 30 | EXPECT_EQ(messages.size(), 1); 31 | EXPECT_EQ(messages[0], serialized); 32 | 33 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 34 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 1); 35 | } 36 | 37 | TEST_F(ServerHandlerTest, FakeNodeDepart) { 38 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 39 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 1); 40 | 41 | string serialized = std::to_string(kMemoryTierId) + ":127.0.0.2:127.0.0.2"; 42 | node_depart_handler(thread_id, ip, ip, global_hash_rings, log_, serialized, 43 | pushers); 44 | 45 | vector messages = get_zmq_messages(); 46 | 47 | EXPECT_EQ(messages.size(), 0); 48 | 49 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 50 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 1); 51 | } 52 | -------------------------------------------------------------------------------- /kvs/tests/kvs/test_rep_factor_change_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | // TODO: write actual tests 18 | -------------------------------------------------------------------------------- /kvs/tests/kvs/test_rep_factor_response_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | // TODO: write actual tests 18 | -------------------------------------------------------------------------------- /kvs/tests/kvs/test_self_depart_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "kvs/kvs_handlers.hpp" 16 | 17 | TEST_F(ServerHandlerTest, SelfDepart) { 18 | unsigned seed = 0; 19 | vector
routing_ips; 20 | vector
monitoring_ips; 21 | 22 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 23 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 1); 24 | 25 | string serialized = "tcp://127.0.0.2:6560"; 26 | 27 | self_depart_handler(thread_id, seed, ip, ip, log_, serialized, 28 | global_hash_rings, local_hash_rings, stored_key_map, 29 | key_replication_map, routing_ips, monitoring_ips, wt, 30 | pushers, serializers); 31 | 32 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 0); 33 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 0); 34 | 35 | vector zmq_messages = get_zmq_messages(); 36 | EXPECT_EQ(zmq_messages.size(), 1); 37 | EXPECT_EQ(zmq_messages[0], ip + "_" + ip + "_" + std::to_string(kSelfTierId)); 38 | } 39 | 40 | // TODO: test should add keys and make sure that they are gossiped elsewhere 41 | // TODO: test should make sure that depart messages are sent to the worker 42 | // threads 43 | -------------------------------------------------------------------------------- /kvs/tests/mock/mock_utils.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "mock_utils.hpp" 16 | 17 | void MockZmqUtil::send_string(const string& s, zmq::socket_t* socket) { 18 | sent_messages.push_back(s); 19 | } 20 | 21 | string MockZmqUtil::recv_string(zmq::socket_t* socket) { return ""; } 22 | 23 | int MockZmqUtil::poll(long timeout, vector* items) { 24 | return 0; 25 | } 26 | 27 | // get all threads responsible for a key from the "node_type" tier 28 | // metadata flag = 0 means the key is metadata; otherwise, it is regular data 29 | ServerThreadList MockHashRingUtil::get_responsible_threads( 30 | Address respond_address, const Key& key, bool metadata, 31 | map& global_hash_rings, 32 | map& local_hash_rings, 33 | map& key_replication_map, SocketCache& pushers, 34 | const vector& tier_ids, bool& succeed, unsigned& seed) { 35 | ServerThreadList threads; 36 | succeed = true; 37 | 38 | threads.push_back(ServerThread("127.0.0.1", "127.0.0.1", 0)); 39 | return threads; 40 | } 41 | -------------------------------------------------------------------------------- /kvs/tests/mock/mock_utils.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef KVS_TESTS_MOCKED_HPP_ 16 | #define KVS_TESTS_MOCKED_HPP_ 17 | 18 | #include "kvs/include/hash_ring.hpp" 19 | #include "zmq/zmq_util.hpp" 20 | 21 | class MockZmqUtil : public ZmqUtilInterface { 22 | public: 23 | vector sent_messages; 24 | 25 | virtual void send_string(const string& s, zmq::socket_t* socket); 26 | virtual string recv_string(zmq::socket_t* socket); 27 | virtual int poll(long timeout, vector* items); 28 | }; 29 | 30 | class MockHashRingUtil : public HashRingUtilInterface { 31 | public: 32 | virtual ServerThreadList get_responsible_threads( 33 | Address respond_address, const Key& key, bool metadata, 34 | map& global_hash_rings, 35 | map& local_hash_rings, 36 | map& key_replication_map, SocketCache& pushers, 37 | const vector& tier_ids, bool& succeed, unsigned& seed); 38 | }; 39 | 40 | #endif // KVS_TESTS_MOCKED_HPP_ 41 | -------------------------------------------------------------------------------- /kvs/tests/route/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6 FATAL_ERROR) 16 | 17 | SET(ROUTE_SRC_DIR ${CMAKE_SOURCE_DIR}/kvs/src/route) 18 | 19 | FILE(GLOB HANDLER_SOURCES "${ROUTE_SRC_DIR}/*handler.cpp") 20 | FILE(GLOB MOCK_SOURCES "${CMAKE_SOURCE_DIR}/kvs/tests/mock/*.cpp") 21 | 22 | ADD_EXECUTABLE(run_routing_handler_tests 23 | run_routing_handler_tests.cpp 24 | ${MOCK_SOURCES} 25 | ${HANDLER_SOURCES}) 26 | 27 | TARGET_LINK_LIBRARIES(run_routing_handler_tests gtest gmock pthread flkvs-ring zmq) 28 | ADD_DEPENDENCIES(run_routing_handler_tests gtest) 29 | 30 | ADD_TEST(NAME RouteTests COMMAND run_routing_handler_tests) 31 | -------------------------------------------------------------------------------- /kvs/tests/route/run_routing_handler_tests.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | 22 | #include "kvs.pb.h" 23 | #include "metadata.pb.h" 24 | #include "replication.pb.h" 25 | #include "types.hpp" 26 | 27 | #include "routing_handler_base.hpp" 28 | #include "test_address_handler.hpp" 29 | #include "test_membership_handler.hpp" 30 | #include "test_replication_change_handler.hpp" 31 | #include "test_replication_response_handler.hpp" 32 | #include "test_seed_handler.hpp" 33 | 34 | unsigned kDefaultLocalReplication = 1; 35 | unsigned kDefaultGlobalMemoryReplication = 1; 36 | unsigned kDefaultGlobalEbsReplication = 1; 37 | unsigned kThreadNum = 1; 38 | 39 | unsigned kSelfTierId = kRoutingTierId; 40 | 41 | vector kSelfTierIdVector = {kSelfTierId}; 42 | map kTierMetadata = {}; 43 | 44 | unsigned kEbsThreadNum = 1; 45 | unsigned kMemoryThreadNum = 1; 46 | unsigned kRoutingThreadCount = 1; 47 | 48 | int main(int argc, char* argv[]) { 49 | log_->set_level(spdlog::level::off); 50 | testing::InitGoogleTest(&argc, argv); 51 | return RUN_ALL_TESTS(); 52 | } 53 | -------------------------------------------------------------------------------- /kvs/tests/route/test_address_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | TEST_F(RoutingHandlerTest, Address) { 18 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 19 | 20 | unsigned seed = 0; 21 | 22 | KeyAddressRequest req; 23 | req.set_request_id("1"); 24 | req.set_response_address("tcp://127.0.0.1:5000"); 25 | req.add_keys("key"); 26 | 27 | string serialized; 28 | req.SerializeToString(&serialized); 29 | 30 | address_handler(log_, serialized, pushers, rt, global_hash_rings, 31 | local_hash_rings, key_replication_map, pending_requests, 32 | seed); 33 | 34 | vector messages = get_zmq_messages(); 35 | 36 | EXPECT_EQ(messages.size(), 1); 37 | string serialized_resp = messages[0]; 38 | 39 | KeyAddressResponse resp; 40 | resp.ParseFromString(serialized_resp); 41 | 42 | EXPECT_EQ(resp.response_id(), "1"); 43 | EXPECT_EQ(resp.error(), 0); 44 | 45 | for (const KeyAddressResponse_KeyAddress& addr : resp.addresses()) { 46 | string key = addr.key(); 47 | EXPECT_EQ(key, "key"); 48 | for (const string& ip : addr.ips()) { 49 | EXPECT_EQ(ip, "tcp://127.0.0.1:6200"); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /kvs/tests/route/test_membership_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | TEST_F(RoutingHandlerTest, Membership) { 18 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 19 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 1); 20 | 21 | string message_base = 22 | std::to_string(kMemoryTierId) + ":127.0.0.2:127.0.0.2:0"; 23 | 24 | string serialized = "join:" + message_base; 25 | membership_handler(log_, serialized, pushers, global_hash_rings, thread_id, 26 | ip); 27 | 28 | vector messages = get_zmq_messages(); 29 | 30 | EXPECT_EQ(messages.size(), 1); 31 | EXPECT_EQ(messages[0], message_base); 32 | 33 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 6000); 34 | EXPECT_EQ(global_hash_rings[kMemoryTierId].get_unique_servers().size(), 2); 35 | } 36 | -------------------------------------------------------------------------------- /kvs/tests/route/test_replication_change_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | TEST_F(RoutingHandlerTest, ReplicationChange) { 18 | kRoutingThreadCount = 3; 19 | vector keys = {"key0, key1, key2"}; 20 | warmup_key_replication_map_to_defaults(keys); 21 | 22 | ReplicationFactorUpdate update; 23 | for (string key : keys) { 24 | ReplicationFactor* rf = update.add_key_reps(); 25 | rf->set_key(key); 26 | 27 | for (unsigned tier_id : kAllTierIds) { 28 | Replication* rep_global = rf->add_global(); 29 | rep_global->set_tier_id(tier_id); 30 | rep_global->set_replication_factor(2); 31 | } 32 | 33 | for (unsigned tier_id : kAllTierIds) { 34 | Replication* rep_local = rf->add_local(); 35 | rep_local->set_tier_id(tier_id); 36 | rep_local->set_replication_factor(3); 37 | } 38 | } 39 | 40 | string serialized; 41 | update.SerializeToString(&serialized); 42 | 43 | replication_change_handler(log_, serialized, pushers, key_replication_map, 44 | thread_id, ip); 45 | 46 | vector messages = get_zmq_messages(); 47 | 48 | EXPECT_EQ(messages.size(), 2); 49 | for (unsigned i = 0; i < messages.size(); i++) { 50 | EXPECT_EQ(messages[i], serialized); 51 | } 52 | 53 | for (string key : keys) { 54 | EXPECT_EQ(key_replication_map[key].global_replication_[kMemoryTierId], 2); 55 | EXPECT_EQ(key_replication_map[key].global_replication_[kEbsTierId], 2); 56 | EXPECT_EQ(key_replication_map[key].local_replication_[kMemoryTierId], 3); 57 | EXPECT_EQ(key_replication_map[key].local_replication_[kEbsTierId], 3); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /kvs/tests/route/test_seed_handler.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "route/routing_handlers.hpp" 16 | 17 | TEST_F(RoutingHandlerTest, Seed) { 18 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 19 | 20 | string serialized = seed_handler(log_, global_hash_rings); 21 | 22 | EXPECT_EQ(global_hash_rings[kMemoryTierId].size(), 3000); 23 | 24 | TierMembership membership; 25 | membership.ParseFromString(serialized); 26 | 27 | // check serialized tier size, tier_id, ip 28 | EXPECT_EQ(membership.tiers_size(), 1); 29 | for (const auto& tier : membership.tiers()) { 30 | for (const auto& other : tier.servers()) { 31 | EXPECT_EQ(tier.tier_id(), kMemoryTierId); 32 | EXPECT_EQ(other.private_ip(), ip); 33 | EXPECT_EQ(other.public_ip(), ip); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /kvs/tests/simple/expected: -------------------------------------------------------------------------------- 1 | Success! 2 | 1 3 | Success! 4 | 2 5 | Success! 6 | 10 7 | Success! 8 | { 3 2 1 } 9 | Success! 10 | { 4 3 2 1 } 11 | -------------------------------------------------------------------------------- /kvs/tests/simple/expected_async: -------------------------------------------------------------------------------- 1 | Success! 2 | 1 3 | Success! 4 | 2 5 | Success! 6 | 10 7 | Success! 8 | {test : 1} 9 | dep1 : {test1 : 1} 10 | hello 11 | -------------------------------------------------------------------------------- /kvs/tests/simple/input: -------------------------------------------------------------------------------- 1 | PUT a 1 2 | GET a 3 | PUT b 2 4 | GET b 5 | PUT a 10 6 | GET a 7 | PUT_SET_ALL set 1 2 3 8 | GET_SET set 9 | PUT_SET_ALL set 1 2 4 10 | GET_SET set 11 | -------------------------------------------------------------------------------- /kvs/tests/simple/input_async: -------------------------------------------------------------------------------- 1 | PUT a 1 2 | GET a 3 | PUT b 2 4 | GET b 5 | PUT a 10 6 | GET a 7 | PUT_CAUSAL c hello 8 | GET_CAUSAL c 9 | -------------------------------------------------------------------------------- /kvs/tests/simple/test-simple-async.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | 17 | if [ $# -gt 2 ]; then 18 | echo "Usage: $0 " 19 | echo "If no build option is specified, the test will default to not building." 20 | 21 | exit 1 22 | fi 23 | 24 | if [ -z "$1" ]; then 25 | BUILD="n" 26 | else 27 | BUILD=$1 28 | fi 29 | 30 | echo "Starting local server..." 31 | ./scripts/start-kvs-local.sh $BUILD n 32 | 33 | echo "Running tests..." 34 | ./build/kvs/client/cpp/flkvs-async-cli conf/kvs-local.yml kvs/tests/simple/input_async > tmp.out 35 | 36 | DIFF=`diff tmp.out kvs/tests/simple/expected_async` 37 | 38 | if [ "$DIFF" != "" ]; then 39 | echo "Output did not match expected output (tests/simple/expected.out). Observed output was: " 40 | echo $DIFF 41 | CODE=1 42 | else 43 | echo "Test succeeded!" 44 | CODE=0 45 | fi 46 | 47 | rm tmp.out 48 | echo "Stopping local server..." 49 | ./scripts/stop-kvs-local.sh y 50 | exit $CODE 51 | -------------------------------------------------------------------------------- /kvs/tests/simple/test-simple.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2018 U.C. Berkeley RISE Lab 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | 17 | if [ $# -gt 2 ]; then 18 | echo "Usage: $0 " 19 | echo "If no build option is specified, the test will default to not building." 20 | 21 | exit 1 22 | fi 23 | 24 | if [ -z "$1" ]; then 25 | BUILD="n" 26 | else 27 | BUILD=$1 28 | fi 29 | 30 | echo "Starting local server..." 31 | ./scripts/start-kvs-local.sh $BUILD n 32 | 33 | echo "Running tests..." 34 | ./build/kvs/client/cpp/flkvs-cli conf/kvs-local.yml kvs/tests/simple/input > tmp.out 35 | 36 | DIFF=`diff tmp.out kvs/tests/simple/expected` 37 | 38 | if [ "$DIFF" != "" ]; then 39 | echo "Output did not match expected output (tests/simple/expected.out). Observed output was: " 40 | echo $DIFF 41 | CODE=1 42 | else 43 | echo "Test succeeded!" 44 | CODE=0 45 | fi 46 | 47 | rm tmp.out 48 | echo "Stopping local server..." 49 | ./scripts/stop-kvs-local.sh y 50 | exit $CODE 51 | -------------------------------------------------------------------------------- /kvs/tests/test_all.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2018 U.C. Berkeley RISE Lab 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "gtest/gtest.h" 19 | 20 | #include "kvs.pb.h" 21 | #include "misc.pb.h" 22 | #include "replication.pb.h" 23 | #include "types.hpp" 24 | #include "utils/server_utils.hpp" 25 | 26 | #include "kvs/server_handler_base.hpp" 27 | #include "kvs/test_gossip_handler.hpp" 28 | #include "kvs/test_node_depart_handler.hpp" 29 | #include "kvs/test_node_join_handler.hpp" 30 | #include "kvs/test_rep_factor_change_handler.hpp" 31 | #include "kvs/test_rep_factor_response_handler.hpp" 32 | #include "kvs/test_self_depart_handler.hpp" 33 | #include "kvs/test_user_request_handler.hpp" 34 | 35 | #include "include/lattices/test_bool_lattice.hpp" 36 | #include "include/lattices/test_map_lattice.hpp" 37 | #include "include/lattices/test_max_lattice.hpp" 38 | #include "include/lattices/test_set_lattice.hpp" 39 | 40 | unsigned kDefaultLocalReplication = 1; 41 | unsigned kSelfTierId = kMemoryTierId; 42 | unsigned kThreadNum = 1; 43 | vector kSelfTierIdVector = {kSelfTierId}; 44 | map kTierMetadata = {}; 45 | 46 | unsigned kEbsThreadCount = 1; 47 | unsigned kMemoryThreadCount = 1; 48 | unsigned kRoutingThreadCount = 1; 49 | 50 | int main(int argc, char* argv[]) { 51 | log->set_level(spdlog::level::info); 52 | testing::InitGoogleTest(&argc, argv); 53 | return RUN_ALL_TESTS(); 54 | } 55 | -------------------------------------------------------------------------------- /scripts/build-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | args=( -j -b -t ) 18 | containsElement() { 19 | local e match="$1" 20 | shift 21 | for e; do [[ "$e" == "$match" ]] && return 0; done 22 | return 1 23 | } 24 | 25 | while getopts ":j:b:tg" opt; do 26 | case $opt in 27 | j ) 28 | MAKE_THREADS=$OPTARG 29 | if containsElement $OPTARG "${args[@]}" 30 | then 31 | echo "Missing argument to flag $opt" 32 | exit 1 33 | else 34 | echo "make set to run on $OPTARG threads" >&2 35 | fi 36 | ;; 37 | b ) 38 | TYPE=$OPTARG 39 | if containsElement $OPTARG "${args[@]}" 40 | then 41 | echo "Missing argument to flag $opt" 42 | exit 1 43 | else 44 | echo "build type set to $OPTARG" >&2 45 | fi 46 | ;; 47 | t ) 48 | TEST="-DBUILD_TEST=ON" 49 | echo "Testing enabled..." 50 | ;; 51 | g ) 52 | COMPILER="/usr/bin/g++" 53 | RUN_FORMAT="" 54 | echo "Compiler set to GNU g++..." 55 | ;; 56 | \? ) 57 | echo "Invalid option: -$OPTARG" >&2 58 | exit 1 59 | ;; 60 | esac 61 | done 62 | 63 | if [[ -z "$MAKE_THREADS" ]]; then MAKE_THREADS=2; fi 64 | if [[ -z "$TYPE" ]]; then TYPE=Release; fi 65 | if [[ -z "$TEST" ]]; then TEST=""; fi 66 | if [[ -z "$COMPILER" ]]; then 67 | COMPILER="/usr/bin/clang++" 68 | RUN_FORMAT="yes" 69 | fi 70 | 71 | rm -rf build 72 | mkdir build 73 | cd build 74 | 75 | cmake -std=c++11 "-GUnix Makefiles" -DCMAKE_BUILD_TYPE=$TYPE -DCMAKE_CXX_COMPILER=$COMPILER $TEST .. 76 | make -j${MAKE_THREADS} 77 | 78 | if [[ "$TYPE" = "Debug" ]] && [[ ! -z "$RUN_FORMAT" ]]; then 79 | make clang-format 80 | fi 81 | -------------------------------------------------------------------------------- /scripts/check-format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | cd build && make clang-format && cd .. 18 | 19 | FILES=`git status --porcelain` 20 | MODIFIED=`git status --porcelain | wc -l` 21 | 22 | if [[ $MODIFIED -gt 0 ]]; then 23 | echo -e "clang-format check failed. The following files had style issues:\n" 24 | echo -e $FILES 25 | 26 | exit 1 27 | else 28 | echo "clang-format check succeeded!" 29 | fi 30 | 31 | ERRORS=`find */*py | grep -v pb2 | xargs pycodestyle` 32 | MODIFIED=`find */*py | grep -v pb2 | xargs pycodestyle | wc -l` 33 | 34 | if [[ $MODIFIED -gt 0 ]]; then 35 | echo -e "pycodestyle check failed. These were the issues reported:\n" 36 | echo -e $ERRORs 37 | 38 | exit 1 39 | else 40 | echo "pycodestyle check succeeded!" 41 | exit 0 42 | fi 43 | -------------------------------------------------------------------------------- /scripts/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | find . -name "*.gcda" -print0 | xargs -0 rm 18 | make -C build -j4 -------------------------------------------------------------------------------- /scripts/start-kvs-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -z "$1" ] && [ -z "$2" ]; then 18 | echo "Usage: ./scripts/start_local.sh build start-user" 19 | echo "" 20 | echo "You must run this from the project root directory." 21 | exit 1 22 | fi 23 | 24 | if [ "$1" = "y" ] || [ "$1" = "yes" ]; then 25 | ./scripts/build.sh 26 | fi 27 | 28 | cp conf/kvs-local.yml conf/kvs-config.yml 29 | 30 | ./build/kvs/src/monitor/flmonitor & 31 | MPID=$! 32 | ./build/kvs/src/route/flroute & 33 | RPID=$! 34 | export SERVER_TYPE="memory" 35 | ./build/kvs/src/kvs/flkvs & 36 | SPID=$! 37 | 38 | echo $MPID > pids 39 | echo $RPID >> pids 40 | echo $SPID >> pids 41 | 42 | if [ "$2" = "y" ] || [ "$2" = "yes" ]; then 43 | ./build/kvs/src/cli/flkvs-cli 44 | fi 45 | -------------------------------------------------------------------------------- /scripts/stop-kvs-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [ -z "$1" ]; then 18 | echo "Usage: ./scripts/stop_local.sh remove-logs" 19 | exit 1 20 | fi 21 | 22 | while IFS='' read -r line || [[ -n "$line" ]] ; do 23 | kill $line 24 | done < "pids" 25 | 26 | if [ "$1" = "y" ]; then 27 | rm *log* 28 | fi 29 | 30 | rm conf/kvs-config.yml 31 | rm pids 32 | -------------------------------------------------------------------------------- /scripts/travis/docker-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # only build a new Docker image if this is a master branch build -- ignore this 18 | # for PR builds 19 | if [[ "$TRAVIS_BRANCH" = "master" ]] && [[ "$TRAVIS_PULL_REQUEST" = "false" ]]; then 20 | docker pull fluentproject/base 21 | docker pull fluentproject/annakvs 22 | docker pull fluentproject/kops 23 | docker pull fluentproject/executor 24 | docker pull fluentproject/cache 25 | 26 | cd dockerfiles/kvs 27 | docker build . -f anna.dockerfile -t fluentproject/annakvs --cache-from fluentproject/annakvs:latest 28 | docker build . -f kops.dockerfile -t fluentproject/kops --cache-from fluentproject/kops:latest 29 | 30 | cd ../functions 31 | docker build . -f executor.dockerfile -t fluentproject/executor --cache-from fluentproject/executor:latest 32 | docker build . -f cache.dockerfile -t fluentproject/cache --cache-from fluentproject/cache:latest 33 | 34 | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 35 | 36 | docker push fluentproject/annakvs 37 | docker push fluentproject/kops 38 | docker push fluentproject/executor 39 | docker push fluentproject/cache 40 | fi 41 | -------------------------------------------------------------------------------- /scripts/travis/llvm-gcov.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | exec llvm-cov gcov "$@" 18 | -------------------------------------------------------------------------------- /scripts/travis/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | cd build && make test 18 | -------------------------------------------------------------------------------- /scripts/travis/travis-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | SCRIPTS=("scripts/check-format.sh" "kvs/tests/simple/test-simple.sh" "kvs/tests/simple/test-simple-async.sh" "scripts/travis/run-tests.sh") 18 | 19 | ./scripts/build-all.sh -bDebug -t 20 | EXIT=$? 21 | if [[ $EXIT -ne 0 ]]; then 22 | echo "$SCRIPT failed with exit code $EXIT." 23 | exit $EXIT 24 | fi 25 | 26 | for SCRIPT in "${SCRIPTS[@]}"; do 27 | ./"$SCRIPT" 28 | EXIT=$? 29 | if [[ $EXIT -ne 0 ]]; then 30 | echo "$SCRIPT failed with exit code $EXIT." 31 | exit $EXIT 32 | fi 33 | done 34 | 35 | exit 0 36 | -------------------------------------------------------------------------------- /scripts/travis/travis-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | download_protobuf() { 18 | rm -rf $PROTOBUF_DIR 19 | 20 | wget https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-all-3.5.1.zip 21 | unzip protobuf-all-3.5.1.zip > /dev/null 2>&1 22 | mv protobuf-3.5.1 $PROTOBUF_DIR 23 | rm protobuf-all-3.5.1.zip 24 | 25 | cd $PROTOBUF_DIR && ./autogen.sh 26 | cd $PROTOBUF_DIR && ./configure CXX=clang++ CXXFLAGS='-std=c++11 -stdlib=libc++ -O3 -g' 27 | } 28 | 29 | install_protobuf() { 30 | cd $PROTOBUF_DIR && make -j4 31 | 32 | cd $PROTOBUF_DIR && sudo make install 33 | cd $PROTOBUF_DIR && sudo ldconfig 34 | } 35 | 36 | download_lcov() { 37 | wget http://sourceforge.net/projects/ltp/files/Coverage%20Analysis/LCOV-1.13/lcov-1.13.tar.gz 38 | tar xvzf lcov-${LCOV_VERSION}.tar.gz > /dev/null 2>&1 39 | rm -rf lcov-${LCOV_VERSION}.tar.gz 40 | } 41 | 42 | install_lcov() { 43 | cd $LCOV_DIR && sudo make install 44 | which lcov 45 | lcov -v 46 | } 47 | 48 | sudo apt-get update 49 | sudo apt-get install -y build-essential autoconf automake libtool curl make unzip pkg-config wget 50 | sudo apt-get install -y libc++-dev libc++abi-dev awscli jq python3-pip 51 | 52 | sudo ln -s $(which clang) /usr/bin/clang 53 | sudo ln -s $(which clang++) /usr/bin/clang++ 54 | 55 | # if the protobuf directory doesn't exist or is empty 56 | if [ ! -d "$PROTOBUF_DIR" ] || [ -z "$(ls -A $PROTOBUF_DIR)" ] ; then 57 | download_protobuf 58 | fi 59 | 60 | LCOV_DIR="lcov-${LCOV_VERSION}" 61 | 62 | install_protobuf 63 | 64 | cd $HOME 65 | 66 | download_lcov 67 | install_lcov 68 | 69 | sudo pip3 install pycodestyle 70 | -------------------------------------------------------------------------------- /scripts/travis/upload-codecov.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 U.C. Berkeley RISE Lab 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # NOTE: This script assumes it is run from the fluent project root directory. 18 | cd build 19 | make test-coverage 20 | lcov --list coverage.info 21 | bash <(curl -s https://codecov.io/bash) 22 | -------------------------------------------------------------------------------- /vendor/spdlog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 2 | 3 | INCLUDE(ExternalProject) 4 | 5 | EXTERNALPROJECT_ADD(spdlog 6 | GIT_REPOSITORY "https://github.com/gabime/spdlog" 7 | GIT_TAG "master" 8 | BUILD_IN_SOURCE 1 9 | UPDATE_COMMAND "" 10 | CONFIGURE_COMMAND "" 11 | BUILD_COMMAND "" 12 | INSTALL_COMMAND "" 13 | ) 14 | 15 | SET(SPDLOG_INCLUDE_DIRS 16 | ${CMAKE_CURRENT_BINARY_DIR}/spdlog-prefix/src/spdlog/include PARENT_SCOPE) 17 | -------------------------------------------------------------------------------- /vendor/yamlcpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 2 | 3 | INCLUDE(ExternalProject) 4 | 5 | IF(${CMAKE_CXX_COMPILER} STREQUAL "/usr/bin/clang++") 6 | SET(CMAKE_CXX_FLAGS_YCPP "-stdlib=libc++") 7 | ENDIF() 8 | 9 | IF(${CMAKE_CXX_COMPILER} STREQUAL "/usr/bin/g++") 10 | SET(CMAKE_CXX_FLAGS_YCPP "") 11 | ENDIF() 12 | 13 | EXTERNALPROJECT_ADD(yamlcpp 14 | URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.2.tar.gz" 15 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 16 | BUILD_IN_SOURCE 1 17 | INSTALL_DIR "${CMAKE_BINARY_DIR}/" 18 | CMAKE_ARGS -DBUILD_SHARED_LIBS=ON -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS_YCPP} 19 | BUILD_COMMAND make 20 | INSTALL_COMMAND "" 21 | ) 22 | 23 | SET(YAMLCPP_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/yamlcpp/include PARENT_SCOPE) 24 | SET(YAMLCPP_LINK_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/yamlcpp PARENT_SCOPE) 25 | -------------------------------------------------------------------------------- /vendor/zeromq/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://zeromq.org/intro:get-the-software 3 | # - http://bit.ly/2dK0UBT 4 | # 5 | # Remember to have libtool, pkg-config, build-essential, autoconf, and automake 6 | # installed. 7 | 8 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 9 | 10 | INCLUDE(ExternalProject) 11 | 12 | SET(CXX "clang++ -stdlib=c++") 13 | 14 | if(APPLE) 15 | set(PREFIX /usr/local) 16 | elseif(UNIX) 17 | set(PREFIX /usr) 18 | endif() 19 | 20 | EXTERNALPROJECT_ADD(zeromq 21 | URL "https://github.com/zeromq/libzmq/releases/download/v4.2.5/zeromq-4.2.5.tar.gz" 22 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 23 | BUILD_IN_SOURCE 1 24 | UPDATE_COMMAND "" 25 | CONFIGURE_COMMAND ./configure 26 | BUILD_COMMAND make 27 | INSTALL_COMMAND "" 28 | ) 29 | 30 | SET(ZEROMQ_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/zeromq/include PARENT_SCOPE) 31 | SET(ZEROMQ_LINK_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/zeromq/src/.libs PARENT_SCOPE) 32 | -------------------------------------------------------------------------------- /vendor/zeromqcpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://zeromq.org/intro:get-the-software 3 | # - http://bit.ly/2dK0UBT 4 | # 5 | # Remember to have libtool, pkg-config, build-essential, autoconf, and automake 6 | # installed. 7 | 8 | CMAKE_MINIMUM_REQUIRED(VERSION 3.6) 9 | 10 | INCLUDE(ExternalProject) 11 | 12 | EXTERNALPROJECT_ADD(zeromqcpp 13 | GIT_REPOSITORY "https://github.com/zeromq/cppzmq.git" 14 | GIT_TAG "v4.2.3" 15 | BUILD_IN_SOURCE 1 16 | UPDATE_COMMAND "" 17 | CONFIGURE_COMMAND "" 18 | BUILD_COMMAND "" 19 | INSTALL_COMMAND "" 20 | ) 21 | 22 | SET(ZEROMQCPP_INCLUDE_DIRS 23 | ${CMAKE_CURRENT_BINARY_DIR}/zeromqcpp-prefix/src/zeromqcpp PARENT_SCOPE) 24 | --------------------------------------------------------------------------------