├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── cmake └── Modules │ └── FindZMQ.cmake ├── docs ├── Doxyfile ├── Makefile ├── api.md ├── conf.py ├── env.md ├── get_started.md ├── history.md ├── how_to.md ├── index.md ├── overview.md ├── requirements.txt └── sphinx_util.py ├── include ├── dmlc │ ├── base.h │ └── logging.h └── ps │ ├── base.h │ ├── internal │ ├── assign_op.h │ ├── customer.h │ ├── env.h │ ├── message.h │ ├── parallel_kv_match.h │ ├── parallel_sort.h │ ├── postoffice.h │ ├── threadsafe_pqueue.h │ ├── threadsafe_queue.h │ ├── utils.h │ └── van.h │ ├── kv_app.h │ ├── ps.h │ ├── range.h │ ├── sarray.h │ └── simple_app.h ├── make ├── deps.mk └── ps.mk ├── src ├── customer.cc ├── ibverbs_van.h ├── meta.proto ├── network_utils.h ├── p3_van.h ├── postoffice.cc ├── resender.h ├── van.cc ├── windows │ └── unistd.h └── zmq_van.h ├── tests ├── CMakeLists.txt ├── README.md ├── lint.py ├── local.sh ├── local_multi_workers.sh ├── repeat.sh ├── test.mk ├── test_connection.cc ├── test_kv_app.cc ├── test_kv_app_benchmark.cc ├── test_kv_app_multi_workers.cc ├── test_simple_app.cc └── travis │ ├── travis_before_cache.sh │ ├── travis_script.sh │ └── travis_setup_env.sh └── tracker ├── README.md ├── dmlc_local.py ├── dmlc_mpi.py ├── dmlc_ssh.py └── tracker.py /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | deps 3 | *.pb.* 4 | .* 5 | *core* 6 | docs/html 7 | docs/xml 8 | docs/latex 9 | docs/_build 10 | tests/test_* 11 | *.pyc 12 | *.d 13 | 14 | cmake-build-debug/ 15 | CMakeFiles/ 16 | recommonmark/ 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # disable sudo to use container based build 2 | sudo: false 3 | 4 | # Use Build Matrix to do lint and build seperately 5 | env: 6 | matrix: 7 | - TASK=lint 8 | - TASK=test CXX=g++-4.8 9 | # - TASK=build CXX=g++-4.8 10 | # - TASK=build CXX=g++-5 11 | 12 | # dependent apt packages 13 | addons: 14 | apt: 15 | sources: 16 | - ubuntu-toolchain-r-test 17 | packages: 18 | - g++-4.8 19 | # - g++-5 20 | - wget 21 | - git 22 | # - unzip 23 | 24 | before_install: 25 | - export TRAVIS=tests/travis 26 | - source ${TRAVIS}/travis_setup_env.sh 27 | 28 | 29 | install: 30 | - pip install cpplint pylint --user `whoami` 31 | 32 | 33 | script: ${TRAVIS}/travis_script.sh 34 | 35 | 36 | before_cache: 37 | - ${TRAVIS}/travis_before_cache.sh 38 | 39 | cache: 40 | directories: 41 | - ${HOME}/.cache/usr 42 | 43 | 44 | notifications: 45 | # Emails are sent to the committer's git-configured email address by default, 46 | email: 47 | on_success: change 48 | on_failure: always 49 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4) 2 | 3 | project(pslite C CXX) 4 | set(CMAKE_CXX_STANDARD 11) 5 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 6 | set(CMAKE_CXX_EXTENSIONS OFF) 7 | list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) 8 | 9 | add_library(pslite) 10 | 11 | if(POLICY CMP0074) # Support Protobuf_ROOT etc environment variables on CMake 3.12+ 12 | cmake_policy(SET CMP0074 NEW) 13 | endif() 14 | 15 | # ---[ zmq 16 | find_package(ZMQ) 17 | if(NOT ZMQ_FOUND) 18 | cmake_minimum_required(VERSION 3.11) 19 | include(FetchContent) 20 | FetchContent_Declare(zmq 21 | URL https://github.com/zeromq/libzmq/releases/download/v4.3.2/zeromq-4.3.2.zip 22 | URL_HASH SHA256=d01517983ded9ff1b6c40f8206fc2b44ac96157a5aea7b974e8b0079547edfda) 23 | FetchContent_GetProperties(zmq) 24 | if(NOT zmq_POPULATED) 25 | FetchContent_Populate(zmq) 26 | if(POLICY CMP0077) # Avoid building shared library and tests on CMake 3.13+ 27 | cmake_policy(SET CMP0077 NEW) 28 | set(BUILD_SHARED OFF CACHE BOOL "") 29 | set(BUILD_TESTS OFF CACHE BOOL "") 30 | endif() 31 | add_subdirectory(${zmq_SOURCE_DIR} ${zmq_BINARY_DIR}) 32 | endif() 33 | target_link_libraries(pslite PUBLIC libzmq-static) 34 | else() 35 | target_include_directories(pslite PRIVATE ${ZMQ_INCLUDE_DIRS}) 36 | target_link_libraries(pslite PUBLIC ${ZMQ_LIBRARIES}) 37 | endif() 38 | 39 | # ---[ Google-protobuf 40 | # Workaround broken DLAMI. DLAMI ships a broken protoc at /home/ubuntu/anaconda3/bin 41 | # https://docs.aws.amazon.com/dlami/latest/devguide/overview-base.html 42 | set(CMAKE_IGNORE_PATH "/home/ubuntu/anaconda3/bin") 43 | set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) 44 | find_package(Protobuf REQUIRED) 45 | target_link_libraries(pslite PUBLIC ${PROTOBUF_LIBRARY}) 46 | target_include_directories(pslite PUBLIC ${PROTOBUF_INCLUDE_DIR}) 47 | 48 | # Generate protobuf headers and sources 49 | file(GLOB_RECURSE PROTO_FILES "src/*.proto") 50 | protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES}) 51 | target_sources(pslite PRIVATE ${PROTO_SRCS}) 52 | target_include_directories(pslite PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) # PROTO_HDRS files are here 53 | 54 | # Generate protobuf python interface 55 | protobuf_generate_python(PROTO_PYS ${PROTO_FILES}) 56 | add_custom_target(proto_python ALL DEPENDS ${PROTO_PYS}) 57 | add_dependencies(pslite proto_python) 58 | 59 | if(MSVC) 60 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") 61 | FILE(GLOB getopt_SOURCE "src/windows/getopt.c") 62 | target_sources(pslite PRIVATE ${getopt_SOURCE}) 63 | add_definitions(-DSTATIC_GETOPT) 64 | target_include_directories(pslite PRIVATE "${PROJECT_SOURCE_DIR}/src/windows") 65 | target_link_libraries(pslite PUBLIC "ipHlpApi.lib" "ws2_32.lib") 66 | foreach(flag_var 67 | CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE 68 | CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) 69 | if(${flag_var} MATCHES "/MD") 70 | string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") 71 | endif(${flag_var} MATCHES "/MD") 72 | endforeach(flag_var) 73 | endif() 74 | 75 | file(GLOB_RECURSE SOURCE "src/*.cc") 76 | target_include_directories(pslite PUBLIC "${PROJECT_SOURCE_DIR}/include/") 77 | target_sources(pslite PRIVATE ${SOURCE}) 78 | 79 | add_subdirectory(tests) 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2015 Carnegie Mellon University 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifdef config 2 | include $(config) 3 | endif 4 | 5 | include make/ps.mk 6 | 7 | ifndef CXX 8 | CXX = g++ 9 | endif 10 | 11 | ifndef DEPS_PATH 12 | DEPS_PATH = $(shell pwd)/deps 13 | endif 14 | 15 | 16 | ifndef PROTOC 17 | PROTOC = ${DEPS_PATH}/bin/protoc 18 | endif 19 | 20 | INCPATH = -I./src -I./include -I$(DEPS_PATH)/include 21 | CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) 22 | LIBS = -pthread 23 | 24 | ifdef USE_IBVERBS 25 | LIBS += -lrdmacm -libverbs 26 | CFLAGS += -DDMLC_USE_IBVERBS 27 | endif 28 | 29 | ifdef ASAN 30 | CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls 31 | endif 32 | 33 | 34 | all: ps test 35 | 36 | include make/deps.mk 37 | 38 | clean: 39 | rm -rf build $(TEST) tests/*.d 40 | find src -name "*.pb.[ch]*" -delete 41 | 42 | lint: 43 | python tests/lint.py ps all include/ps src 44 | 45 | ps: build/libps.a 46 | 47 | OBJS = $(addprefix build/, customer.o postoffice.o van.o meta.pb.o) 48 | build/libps.a: $(OBJS) 49 | ar crv $@ $(filter %.o, $?) 50 | 51 | build/%.o: src/%.cc ${ZMQ} src/meta.pb.h 52 | @mkdir -p $(@D) 53 | $(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d 54 | $(CXX) $(CFLAGS) $(LIBS) -c $< -o $@ 55 | 56 | src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF} 57 | $(PROTOC) --cpp_out=./src --proto_path=./src $< 58 | 59 | -include build/*.d 60 | -include build/*/*.d 61 | 62 | include tests/test.mk 63 | test: $(TEST) 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/dmlc/ps-lite.svg?branch=master)](https://travis-ci.org/dmlc/ps-lite) 2 | [![GitHub license](http://dmlc.github.io/img/apache2.svg)](./LICENSE) 3 | 4 | A light and efficient implementation of the parameter server 5 | framework. It provides clean yet powerful APIs. For example, a worker node can 6 | communicate with the server nodes by 7 | - `Push(keys, values)`: push a list of (key, value) pairs to the server nodes 8 | - `Pull(keys)`: pull the values from servers for a list of keys 9 | - `Wait`: wait untill a push or pull finished. 10 | 11 | A simple example: 12 | 13 | ```c++ 14 | std::vector key = {1, 3, 5}; 15 | std::vector val = {1, 1, 1}; 16 | std::vector recv_val; 17 | ps::KVWorker w; 18 | w.Wait(w.Push(key, val)); 19 | w.Wait(w.Pull(key, &recv_val)); 20 | ``` 21 | 22 | More features: 23 | 24 | - Flexible and high-performance communication: zero-copy push/pull, supporting 25 | dynamic length values, user-defined filters for communication compression 26 | - Server-side programming: supporting user-defined handles on server nodes 27 | 28 | ### Build 29 | 30 | `ps-lite` requires a C++11 compiler such as `g++ >= 4.8`. On Ubuntu >= 13.10, we 31 | can install it by 32 | ``` 33 | sudo apt-get update && sudo apt-get install -y build-essential git 34 | ``` 35 | Instructions for gcc 4.8 installation on other platforms: 36 | - [Ubuntu 12.04](http://ubuntuhandbook.org/index.php/2013/08/install-gcc-4-8-via-ppa-in-ubuntu-12-04-13-04/) 37 | - [Centos](http://linux.web.cern.ch/linux/devtoolset/) 38 | - [Mac Os X](http://hpc.sourceforge.net/). 39 | 40 | Then clone and build 41 | 42 | ```bash 43 | git clone https://github.com/dmlc/ps-lite 44 | cd ps-lite && make -j4 45 | ``` 46 | 47 | ### How to use 48 | 49 | `ps-lite` provides asynchronous communication for other projects: 50 | - Distributed deep neural networks: 51 | [MXNet](https://github.com/dmlc/mxnet), 52 | [CXXNET](https://github.com/dmlc/cxxnet), 53 | [Minverva](https://github.com/minerva-developers/minerva), and 54 | [BytePS](https://github.com/bytedance/byteps/) 55 | - Distributed high dimensional inference, such as sparse logistic regression, 56 | factorization machines: 57 | [DiFacto](https://github.com/dmlc/difacto) 58 | [Wormhole](https://github.com/dmlc/wormhole) 59 | 60 | 61 | ### Research papers 62 | 1. Mu Li, Dave Andersen, Alex Smola, Junwoo Park, Amr Ahmed, Vanja Josifovski, 63 | James Long, Eugene Shekita, Bor-Yiing 64 | Su. [Scaling Distributed Machine Learning with the Parameter Server](http://www.cs.cmu.edu/~muli/file/parameter_server_osdi14.pdf). In 65 | Operating Systems Design and Implementation (OSDI), 2014 66 | 2. Mu Li, Dave Andersen, Alex Smola, and Kai 67 | Yu. [Communication Efficient Distributed Machine Learning with the Parameter Server](http://www.cs.cmu.edu/~muli/file/parameter_server_nips14.pdf). In 68 | Neural Information Processing Systems (NIPS), 2014 69 | -------------------------------------------------------------------------------- /cmake/Modules/FindZMQ.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find ZMQ 2 | # Once done this will define 3 | # ZMQ_FOUND - System has ZMQ 4 | # ZMQ_INCLUDE_DIRS - The ZMQ include directories 5 | # ZMQ_LIBRARIES - The libraries needed to use ZMQ 6 | # ZMQ_DEFINITIONS - Compiler switches required for using ZMQ 7 | 8 | find_path ( ZMQ_INCLUDE_DIR zmq.h ) 9 | find_library ( ZMQ_LIBRARY NAMES zmq ) 10 | 11 | set ( ZMQ_LIBRARIES ${ZMQ_LIBRARY} ) 12 | set ( ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR} ) 13 | 14 | include ( FindPackageHandleStandardArgs ) 15 | # handle the QUIETLY and REQUIRED arguments and set ZMQ_FOUND to TRUE 16 | # if all listed variables are TRUE 17 | find_package_handle_standard_args ( ZMQ DEFAULT_MSG ZMQ_LIBRARY ZMQ_INCLUDE_DIR ) -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | @echo " coverage to run coverage check of the documentation (if enabled)" 49 | 50 | .PHONY: clean 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | .PHONY: html 55 | html: 56 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 59 | 60 | .PHONY: dirhtml 61 | dirhtml: 62 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 63 | @echo 64 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 65 | 66 | .PHONY: singlehtml 67 | singlehtml: 68 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 69 | @echo 70 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 71 | 72 | .PHONY: pickle 73 | pickle: 74 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 75 | @echo 76 | @echo "Build finished; now you can process the pickle files." 77 | 78 | .PHONY: json 79 | json: 80 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 81 | @echo 82 | @echo "Build finished; now you can process the JSON files." 83 | 84 | .PHONY: htmlhelp 85 | htmlhelp: 86 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 87 | @echo 88 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 89 | ".hhp project file in $(BUILDDIR)/htmlhelp." 90 | 91 | .PHONY: qthelp 92 | qthelp: 93 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 94 | @echo 95 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 96 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 97 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ps-lite.qhcp" 98 | @echo "To view the help file:" 99 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ps-lite.qhc" 100 | 101 | .PHONY: applehelp 102 | applehelp: 103 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 104 | @echo 105 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 106 | @echo "N.B. You won't be able to view it unless you put it in" \ 107 | "~/Library/Documentation/Help or install it in your application" \ 108 | "bundle." 109 | 110 | .PHONY: devhelp 111 | devhelp: 112 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 113 | @echo 114 | @echo "Build finished." 115 | @echo "To view the help file:" 116 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ps-lite" 117 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ps-lite" 118 | @echo "# devhelp" 119 | 120 | .PHONY: epub 121 | epub: 122 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 123 | @echo 124 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 125 | 126 | .PHONY: latex 127 | latex: 128 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 129 | @echo 130 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 131 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 132 | "(use \`make latexpdf' here to do that automatically)." 133 | 134 | .PHONY: latexpdf 135 | latexpdf: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo "Running LaTeX files through pdflatex..." 138 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 139 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 140 | 141 | .PHONY: latexpdfja 142 | latexpdfja: 143 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 144 | @echo "Running LaTeX files through platex and dvipdfmx..." 145 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 146 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 147 | 148 | .PHONY: text 149 | text: 150 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 151 | @echo 152 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 153 | 154 | .PHONY: man 155 | man: 156 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 157 | @echo 158 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 159 | 160 | .PHONY: texinfo 161 | texinfo: 162 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 163 | @echo 164 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 165 | @echo "Run \`make' in that directory to run these through makeinfo" \ 166 | "(use \`make info' here to do that automatically)." 167 | 168 | .PHONY: info 169 | info: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo "Running Texinfo files through makeinfo..." 172 | make -C $(BUILDDIR)/texinfo info 173 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 174 | 175 | .PHONY: gettext 176 | gettext: 177 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 178 | @echo 179 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 180 | 181 | .PHONY: changes 182 | changes: 183 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 184 | @echo 185 | @echo "The overview file is in $(BUILDDIR)/changes." 186 | 187 | .PHONY: linkcheck 188 | linkcheck: 189 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 190 | @echo 191 | @echo "Link check complete; look for any errors in the above output " \ 192 | "or in $(BUILDDIR)/linkcheck/output.txt." 193 | 194 | .PHONY: doctest 195 | doctest: 196 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 197 | @echo "Testing of doctests in the sources finished, look at the " \ 198 | "results in $(BUILDDIR)/doctest/output.txt." 199 | 200 | .PHONY: coverage 201 | coverage: 202 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 203 | @echo "Testing of coverage in the sources finished, look at the " \ 204 | "results in $(BUILDDIR)/coverage/python.txt." 205 | 206 | .PHONY: xml 207 | xml: 208 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 209 | @echo 210 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 211 | 212 | .PHONY: pseudoxml 213 | pseudoxml: 214 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 215 | @echo 216 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 217 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | # APIs 2 | 3 | 4 | The data communicated are presented as key-value 5 | pairs, where the key might be the `uint64_t` (defined by `ps::Key`) feature 6 | index and the value might be the according `float` gradient. 7 | 1. Basic synchronization functions: \ref ps::KVWorker::Push, \ref 8 | ps::KVWorker::Pull, and \ref ps::KVWorker::Wait 9 | 2. Dynamic length value push and pull: \ref ps::KVWorker::VPush and \ref 10 | ps::KVWorker::VPull 11 | 3. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref 12 | ps::KVWorker::ZPull, \ref ps::KVWorker::ZVPush and \ref 13 | ps::KVWorker::ZVPull 14 | 15 | 16 | often server *i* handles the keys (feature indices) within the i-th 17 | segment of [0, uint64_max]. The server node allows user-defined handles to 18 | process the `push` and `pull` requests from the workers. 19 | 1. Online key-value store \ref ps::OnlineServer 20 | 2. Example user-defined value: \ref ps::IVal 21 | 3. Example user-defined handle: \ref ps::IOnlineHandle 22 | 23 | 24 | 25 | also We can 26 | also implement 27 | 28 | , which is often used to monitor and control the 29 | progress of the machine learning application. It also can be used to deal with node 30 | failures. See an example in [asynchronous SGD](https://github.com/dmlc/wormhole/blob/master/learn/solver/async_sgd.h#L27). 31 | 32 | ```eval_rst 33 | .. automodule:: ps::KVWorker 34 | :members: 35 | ``` 36 | 37 | ```eval_rst 38 | .. doxygenstruct:: ps::KVPairs 39 | :members: 40 | ``` 41 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # ps-lite documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Mar 20 20:12:23 2016. 5 | # 6 | # Mu: additional changes 7 | # - add breathe into extensions 8 | # - change html theme into sphinx_rtd_theme 9 | # - add sphnix_util.py 10 | # - add .md into source_suffix 11 | # - add setup() at the end 12 | # 13 | # This file is execfile()d with the current directory set to its 14 | # containing dir. 15 | # 16 | # Note that not all possible configuration values are present in this 17 | # autogenerated file. 18 | # 19 | # All configuration values have a default; values that are commented out 20 | # serve to show the default. 21 | 22 | 23 | import sys 24 | import os 25 | 26 | # If extensions (or modules to document with autodoc) are in another directory, 27 | # add these directories to sys.path here. If the directory is relative to the 28 | # documentation root, use os.path.abspath to make it absolute, like shown here. 29 | #sys.path.insert(0, os.path.abspath('.')) 30 | 31 | # -- General configuration ------------------------------------------------ 32 | 33 | # If your documentation needs a minimal Sphinx version, state it here. 34 | #needs_sphinx = '1.0' 35 | 36 | # Add any Sphinx extension module names here, as strings. They can be 37 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 38 | # ones. 39 | extensions = [ 40 | 'sphinx.ext.mathjax', 41 | 'breathe', 42 | ] 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ['_templates'] 46 | 47 | # The suffix(es) of source filenames. 48 | # You can specify multiple suffix as a list of string: 49 | source_suffix = ['.md'] 50 | # source_suffix = '.rst' 51 | 52 | # The encoding of source files. 53 | #source_encoding = 'utf-8-sig' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # General information about the project. 59 | project = u'ps-lite' 60 | copyright = u'2016, ps-lite developers' 61 | author = u'ps-lite developers' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = u'1.0' 69 | # The full version, including alpha/beta/rc tags. 70 | release = u'1.0' 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = None 78 | 79 | # There are two options for replacing |today|: either, you set today to some 80 | # non-false value, then it is used: 81 | #today = '' 82 | # Else, today_fmt is used as the format for a strftime call. 83 | #today_fmt = '%B %d, %Y' 84 | 85 | # List of patterns, relative to source directory, that match files and 86 | # directories to ignore when looking for source files. 87 | exclude_patterns = ['_build'] 88 | 89 | # The reST default role (used for this markup: `text`) to use for all 90 | # documents. 91 | #default_role = None 92 | 93 | # If true, '()' will be appended to :func: etc. cross-reference text. 94 | #add_function_parentheses = True 95 | 96 | # If true, the current module name will be prepended to all description 97 | # unit titles (such as .. function::). 98 | #add_module_names = True 99 | 100 | # If true, sectionauthor and moduleauthor directives will be shown in the 101 | # output. They are ignored by default. 102 | #show_authors = False 103 | 104 | # The name of the Pygments (syntax highlighting) style to use. 105 | pygments_style = 'sphinx' 106 | 107 | # A list of ignored prefixes for module index sorting. 108 | #modindex_common_prefix = [] 109 | 110 | # If true, keep warnings as "system message" paragraphs in the built documents. 111 | #keep_warnings = False 112 | 113 | # If true, `todo` and `todoList` produce output, else they produce nothing. 114 | todo_include_todos = False 115 | 116 | 117 | # -- Options for HTML output ---------------------------------------------- 118 | 119 | # The theme to use for HTML and HTML Help pages. See the documentation for 120 | # a list of builtin themes. 121 | html_theme = 'sphinx_rtd_theme' 122 | 123 | # Theme options are theme-specific and customize the look and feel of a theme 124 | # further. For a list of options available for each theme, see the 125 | # documentation. 126 | #html_theme_options = {} 127 | 128 | # Add any paths that contain custom themes here, relative to this directory. 129 | #html_theme_path = [] 130 | 131 | # The name for this set of Sphinx documents. If None, it defaults to 132 | # " v documentation". 133 | #html_title = None 134 | 135 | # A shorter title for the navigation bar. Default is the same as html_title. 136 | #html_short_title = None 137 | 138 | # The name of an image file (relative to this directory) to place at the top 139 | # of the sidebar. 140 | #html_logo = None 141 | 142 | # The name of an image file (relative to this directory) to use as a favicon of 143 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 144 | # pixels large. 145 | #html_favicon = None 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = ['_static'] 151 | 152 | # Add any extra paths that contain custom files (such as robots.txt or 153 | # .htaccess) here, relative to this directory. These files are copied 154 | # directly to the root of the documentation. 155 | #html_extra_path = [] 156 | 157 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 158 | # using the given strftime format. 159 | #html_last_updated_fmt = '%b %d, %Y' 160 | 161 | # If true, SmartyPants will be used to convert quotes and dashes to 162 | # typographically correct entities. 163 | #html_use_smartypants = True 164 | 165 | # Custom sidebar templates, maps document names to template names. 166 | #html_sidebars = {} 167 | 168 | # Additional templates that should be rendered to pages, maps page names to 169 | # template names. 170 | #html_additional_pages = {} 171 | 172 | # If false, no module index is generated. 173 | #html_domain_indices = True 174 | 175 | # If false, no index is generated. 176 | #html_use_index = True 177 | 178 | # If true, the index is split into individual pages for each letter. 179 | #html_split_index = False 180 | 181 | # If true, links to the reST sources are added to the pages. 182 | #html_show_sourcelink = True 183 | 184 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 185 | #html_show_sphinx = True 186 | 187 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 188 | #html_show_copyright = True 189 | 190 | # If true, an OpenSearch description file will be output, and all pages will 191 | # contain a tag referring to it. The value of this option must be the 192 | # base URL from which the finished HTML is served. 193 | #html_use_opensearch = '' 194 | 195 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 196 | #html_file_suffix = None 197 | 198 | # Language to be used for generating the HTML full-text search index. 199 | # Sphinx supports the following languages: 200 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 201 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 202 | #html_search_language = 'en' 203 | 204 | # A dictionary with options for the search language support, empty by default. 205 | # Now only 'ja' uses this config value 206 | #html_search_options = {'type': 'default'} 207 | 208 | # The name of a javascript file (relative to the configuration directory) that 209 | # implements a search results scorer. If empty, the default will be used. 210 | #html_search_scorer = 'scorer.js' 211 | 212 | # Output file base name for HTML help builder. 213 | htmlhelp_basename = 'ps-litedoc' 214 | 215 | # -- Options for LaTeX output --------------------------------------------- 216 | 217 | latex_elements = { 218 | # The paper size ('letterpaper' or 'a4paper'). 219 | #'papersize': 'letterpaper', 220 | 221 | # The font size ('10pt', '11pt' or '12pt'). 222 | #'pointsize': '10pt', 223 | 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | 227 | # Latex figure (float) alignment 228 | #'figure_align': 'htbp', 229 | } 230 | 231 | # Grouping the document tree into LaTeX files. List of tuples 232 | # (source start file, target name, title, 233 | # author, documentclass [howto, manual, or own class]). 234 | latex_documents = [ 235 | (master_doc, 'ps-lite.tex', u'ps-lite Documentation', 236 | u'ps-lite developers', 'manual'), 237 | ] 238 | 239 | # The name of an image file (relative to this directory) to place at the top of 240 | # the title page. 241 | #latex_logo = None 242 | 243 | # For "manual" documents, if this is true, then toplevel headings are parts, 244 | # not chapters. 245 | #latex_use_parts = False 246 | 247 | # If true, show page references after internal links. 248 | #latex_show_pagerefs = False 249 | 250 | # If true, show URL addresses after external links. 251 | #latex_show_urls = False 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #latex_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #latex_domain_indices = True 258 | 259 | 260 | # -- Options for manual page output --------------------------------------- 261 | 262 | # One entry per manual page. List of tuples 263 | # (source start file, name, description, authors, manual section). 264 | man_pages = [ 265 | (master_doc, 'ps-lite', u'ps-lite Documentation', 266 | [author], 1) 267 | ] 268 | 269 | # If true, show URL addresses after external links. 270 | #man_show_urls = False 271 | 272 | 273 | # -- Options for Texinfo output ------------------------------------------- 274 | 275 | # Grouping the document tree into Texinfo files. List of tuples 276 | # (source start file, target name, title, author, 277 | # dir menu entry, description, category) 278 | texinfo_documents = [ 279 | (master_doc, 'ps-lite', u'ps-lite Documentation', 280 | author, 'ps-lite', 'One line description of project.', 281 | 'Miscellaneous'), 282 | ] 283 | 284 | # Documents to append as an appendix to all manuals. 285 | #texinfo_appendices = [] 286 | 287 | # If false, no module index is generated. 288 | #texinfo_domain_indices = True 289 | 290 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 291 | #texinfo_show_urls = 'footnote' 292 | 293 | # If true, do not generate a @detailmenu in the "Top" node's menu. 294 | #texinfo_no_detailmenu = False 295 | 296 | curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) 297 | sys.path.insert(0, curr_path) 298 | from sphinx_util import MarkdownParser, AutoStructify, generate_doxygen_xml 299 | 300 | MarkdownParser.github_doc_root = 'https://github.com/dmlc/ps-lite/tree/master/' 301 | source_parsers = { 302 | '.md': MarkdownParser, 303 | '.Rmd': MarkdownParser, 304 | } 305 | 306 | breathe_projects = {'ps-lite' : 'xml/'} 307 | breathe_default_project = 'ps-lite' 308 | 309 | doc_root = 'http://dmlc.ml' 310 | def setup(app): 311 | app.connect("builder-inited", generate_doxygen_xml) 312 | app.add_config_value('recommonmark_config', { 313 | 'url_resolver': lambda url: doc_root + url, 314 | }, True) 315 | app.add_transform(AutoStructify) 316 | -------------------------------------------------------------------------------- /docs/env.md: -------------------------------------------------------------------------------- 1 | # environment variables 2 | 3 | The variables must be set for starting 4 | 5 | - `DMLC_NUM_WORKER` : the number of workers 6 | - `DMLC_NUM_SERVER` : the number of servers 7 | - `DMLC_ROLE` : the role of the current node, can be `worker`, `server`, or `scheduler` 8 | - `DMLC_PS_ROOT_URI` : the ip or hostname of the scheduler node 9 | - `DMLC_PS_ROOT_PORT` : the port that the scheduler node is listening 10 | 11 | additional variables: 12 | 13 | - `DMLC_INTERFACE` : the network interface a node should use. in default choose 14 | automatically 15 | - `DMLC_LOCAL` : runs in local machines, no network is needed 16 | - `DMLC_PS_WATER_MARK` : limit on the maximum number of outstanding messages 17 | - `DMLC_PS_VAN_TYPE` : the type of the Van for transport, can be `ibverbs` for RDMA, `zmq` for TCP, `p3` for TCP with [priority based parameter propagation](https://anandj.in/wp-content/uploads/sysml.pdf). -------------------------------------------------------------------------------- /docs/get_started.md: -------------------------------------------------------------------------------- 1 | # Get Started 2 | -------------------------------------------------------------------------------- /docs/history.md: -------------------------------------------------------------------------------- 1 | # History 2 | 3 | We started to work on the parameter server framework since 2010. 4 | 5 | 1. The first generation was 6 | designed and optimized for specific algorithms, such as logistic regression and 7 | LDA, to serve the sheer size industrial machine learning tasks (hundreds billions of 8 | examples and features with 10-100TB data size) . 9 | 10 | 2. Later we tried to build a open-source general purpose framework for machine learning 11 | algorithms. The project is available at [dmlc/parameter_server](https://github.com/dmlc/parameter_server). 12 | 13 | 3. Given the growing demands from other projects, we created `ps-lite`, which provides a clean data communication API and a 14 | lightweight implementation. The implementation is based on `dmlc/parameter_server`, but we refactored the job launchers, file I/O and machine 15 | learning algorithms codes into different projects such as `dmlc-core` and 16 | `wormhole`. 17 | 18 | 4. From the experience we learned during developing 19 | [dmlc/mxnet](https://github.com/dmlc/mxnet), we further refactored the API and implementation from [v1](https://github.com/dmlc/ps-lite/releases/tag/v1). The main 20 | changes include 21 | - less library dependencies 22 | - more flexible user-defined callbacks, which facilitate other language 23 | bindings 24 | - let the users, such as the dependency 25 | engine of mxnet, manage the data consistency 26 | -------------------------------------------------------------------------------- /docs/how_to.md: -------------------------------------------------------------------------------- 1 | # How To 2 | 3 | ## Debug PS-Lite 4 | 5 | One way to debug is loggining all communications. We can do it by specifying 6 | the environment variable `PS_VERBOSE`: 7 | - `PS_VERBOSE=1`: logging connection information 8 | - `PS_VERBOSE=2`: logging all data communication information 9 | 10 | For example, first run `make test; cd tests` in the root directory. Then 11 | ```bash 12 | export PS_VERBOSE=1; ./local.sh 1 1 ./test_connection 13 | ``` 14 | Possible outputs are 15 | ```bash 16 | [19:57:18] src/van.cc:72: Node Info: role=schedulerid=1, ip=127.0.0.1, port=8000 17 | [19:57:18] src/van.cc:72: Node Info: role=worker, ip=128.2.211.110, port=58442 18 | [19:57:18] src/van.cc:72: Node Info: role=server, ip=128.2.211.110, port=40112 19 | [19:57:18] src/van.cc:336: assign rank=8 to node role=server, ip=128.2.211.110, port=40112 20 | [19:57:18] src/van.cc:336: assign rank=9 to node role=worker, ip=128.2.211.110, port=58442 21 | [19:57:18] src/van.cc:347: the scheduler is connected to 1 workers and 1 servers 22 | [19:57:18] src/van.cc:354: S[8] is connected to others 23 | [19:57:18] src/van.cc:354: W[9] is connected to others 24 | [19:57:18] src/van.cc:296: H[1] is stopped 25 | [19:57:18] src/van.cc:296: S[8] is stopped 26 | [19:57:18] src/van.cc:296: W[9] is stopped 27 | ``` 28 | where `H`, `S` and `W` stand for scheduler, server, and worker respectively. 29 | 30 | ## Use a Particular Network Interface 31 | 32 | In default PS-Lite automatically chooses an available network interface. But for 33 | machines have multiple interfaces, we can specify the network interface to use 34 | by the environment variable `DMLC_INTERFACE`. For example, to use the 35 | infinite-band interface `ib0`, we can 36 | ```bash 37 | export DMLC_INTERFACE=ib0; commands_to_run 38 | ``` 39 | 40 | If all PS-Lite nodes run in the same machine, we can set `DMLC_LOCAL` to use 41 | memory copy rather than the local network interface, which may improve the 42 | performance: 43 | ```bash 44 | export DMLC_LOCAL=1; commands_to_run 45 | ``` 46 | 47 | ## Environment Variables to Start PS-Lite 48 | 49 | This section is useful if we want to port PS-Lite to other cluster resource 50 | managers besides the provided ones such as `ssh`, `mpirun`, `yarn` and `sge`. 51 | 52 | To start a PS-Lite node, we need to give proper values to the following 53 | environment variables. 54 | - `DMLC_NUM_WORKER` : the number of workers 55 | - `DMLC_NUM_SERVER` : the number of servers 56 | - `DMLC_ROLE` : the role of the current node, can be `worker`, `server`, or `scheduler` 57 | - `DMLC_PS_ROOT_URI` : the ip or hostname of the scheduler node 58 | - `DMLC_PS_ROOT_PORT` : the port that the scheduler node is listening 59 | 60 | ## Retransmission for Unreliable Network 61 | 62 | It's not uncommon that a message disappear when sending from one node to another 63 | node. The program hangs when a critical message is not delivered 64 | successfully. In that case, we can let PS-Lite send an additional ACK for each 65 | message, and resend that message if the ACK is not received within a given 66 | time. To enable this feature, we can set the environment variables 67 | 68 | - `PS_RESEND` : if or not enable retransmission. Default is 0. 69 | - `PS_RESEND_TIMEOUT` : timeout in millisecond if an ACK message if not 70 | received. PS-Lite then will resend that message. Default is 1000. 71 | 72 | We can set `PS_DROP_MSG`, the percent of probability to drop a received 73 | message, for testing. For example, `PS_DROP_MSG=10` will let a node drop a 74 | received message with 10% probability. 75 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # PS-Lite Documents 2 | 3 | PS-Lite is a lightweight implementation of the parameter server. It provides 4 | asynchronous and zero-copy key-value pair communication between machines. 5 | 6 | 7 | ```eval_rst 8 | .. toctree:: 9 | :numbered: 10 | 11 | overview 12 | get_started 13 | how_to 14 | api 15 | history 16 | ``` 17 | -------------------------------------------------------------------------------- /docs/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The parameter server model supports high-performance distributed machine learning 4 | applications. In this framework, multiple nodes run over multiple machines to 5 | solve machine learning problems. 6 | There is often a single scheduler node, and several worker and server nodes. 7 | 8 | ![ps arch](https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/ps-arch.png) 9 | 10 | - **Worker**. A worker node performs the main computations of reading data and 11 | computing the gradient. It communicates with the server nodes via `push` and 12 | `pull`. For example, it pushes the computed gradient to the servers, or pulls 13 | the recent model from them. 14 | 15 | - **Server.** A server node maintains and updates the model weights. Each node 16 | maintains only a segment of the model weights. 17 | 18 | - **Scheduler.** The scheduler node monitors the aliveness of other nodes. It can 19 | be also used to send control signals to other nodes and collect their 20 | progress. 21 | 22 | 23 | ## Distributed Optimization 24 | 25 | Assume we are going to solve the following 26 | ```eval_rst 27 | .. math:: 28 | \min_w \sum_{i=1}^n f(x_i, y_i, w) 29 | ``` 30 | where *(yi, xi)* are example pairs and *w* denotes model weights. 31 | 32 | We consider solving the above problem by minibatch stochastic gradient descent 33 | (SGD) with batch size *b*. At step *t*, this algorithm first randomly picks up 34 | *b* examples, and then updates the weight *w* by 35 | ```eval_rst 36 | .. math:: 37 | w = w - \eta_t \sum_{i=1}^b \nabla f(x_{k_i}, y_{k_i}, w) 38 | ``` 39 | 40 | We give two examples to illusrate the basic idea of how to implement a 41 | distributed optimization algorithm in ps-lite. 42 | 43 | ### Asynchronous SGD 44 | 45 | The servers maintain model weights *w*, where server *k* gets the *k*-th segment of *w*, denoted by 46 | *wk*. Once received gradient from a worker, the server *k* will 47 | update the weight it maintained: 48 | 49 | ```c++ 50 | t = 0; 51 | while (Received(&grad)) { 52 | w_k -= eta(t) * grad; 53 | t++; 54 | } 55 | ``` 56 | where the function `received` returns if received gradient from any worker 57 | node, and `eta` returns the learning rate at step *t*. 58 | 59 | A worker at each step does four things 60 | 61 | ```c++ 62 | Read(&X, &Y); // read a minibatch X and Y 63 | Pull(&w); // pull the latest weight from the servers 64 | ComputeGrad(X, Y, w, &grad); // compute the gradient 65 | Push(grad); // push the gradients to the servers 66 | ``` 67 | where ps-lite will provide functions `push` and `pull` which will communicate 68 | with servers with the right part of data. 69 | 70 | Note that asynchronous SGD is semantically different from the single machine 71 | version. Since there is no communication between workers, it is possible that 72 | the weight is updated while one worker is calculating the gradients. In other 73 | words, each worker may used the **delayed** weights. The following figure 74 | shows communication between 2 server nodes and 3 worker nodes. 75 | 76 | 77 | 78 | 79 | ### Synchronized SGD 80 | 81 | Unlike asynchronous SGD, synchronized SGD is semantically identical to the single 82 | machine algorithm. We use the scheduler to manage the data synchronization 83 | 84 | ```c++ 85 | for (t = 0, t < num_iteration; ++t) { 86 | for (i = 0; i < num_worker; ++i) { 87 | IssueComputeGrad(i, t); 88 | } 89 | for (i = 0; i < num_server; ++i) { 90 | IssueUpdateWeight(i, t); 91 | } 92 | WaitAllFinished(); 93 | } 94 | ``` 95 | 96 | where `IssueComputeGrad` and `IssueUpdateWeight` issue commands to worker and 97 | servers, while `WaitAllFinished` wait until all issued commands are finished. 98 | 99 | When a worker receives a command, it executes the following function 100 | ```c++ 101 | ExecComputeGrad(i, t) { 102 | Read(&X, &Y); // read minibatch with b / num_workers examples 103 | Pull(&w); // pull the latest weight from the servers 104 | ComputeGrad(X, Y, w, &grad); // compute the gradient 105 | Push(grad); // push the gradients to the servers 106 | } 107 | ``` 108 | which is almost identical to asynchronous SGD but only *b/num_workers* examples 109 | are processed at each step. 110 | 111 | While for a server node, it has an additional aggregation step comparing to 112 | asynchronous SGD 113 | 114 | ```c++ 115 | ExecUpdateWeight(i, t) { 116 | for (j = 0; j < num_workers; ++j) { 117 | Receive(&grad); 118 | aggregated_grad += grad; 119 | } 120 | w_i -= eta(t) * aggregated_grad; 121 | } 122 | ``` 123 | 124 | ### Which one to use? 125 | 126 | Compared to a single machine algorithm, the distributed algorithms have two 127 | additional costs: one is the data communication cost, namely sending data over 128 | the network, and the other one is synchronization cost due to the imperfect load 129 | balance and performance variance across machines. These two costs may dominate 130 | performance for large scale applications with hundreds of machines and 131 | terabytes of data. 132 | 133 | Notation: 134 | ```eval_rst 135 | ======================== === 136 | :math:`{f}` convex function 137 | :math:`{n}` number of examples 138 | :math:`{m}` number of workers 139 | :math:`{b}` minibatch size 140 | :math:`{\tau}` maximal delay 141 | :math:`T_{\text{comm}}` data communication overhead of one minibatch 142 | :math:`T_{\text{sync}}` synchronization overhead 143 | ======================== === 144 | ``` 145 | 146 | The trade-offs are summarized by 147 | ```eval_rst 148 | ============ ========================= =================== 149 | SGD slowdown of convergence additional overhead 150 | ============ ========================= =================== 151 | synchronized :math:`\sqrt{b}` :math:`\frac {n}b (T_{\text{comm}} + T_{\textrm{sync}})` 152 | asynchronous :math:`\sqrt{b\tau}` :math:`\frac n{mb} T_{\textrm{comm}}` 153 | ============ ========================= =================== 154 | ``` 155 | 156 | What we can see are 157 | - the minibatch size trade-offs the convergence and communication cost 158 | - the maximal allowed delay trade-offs the convergence and synchronization 159 | cost. In synchronized SGD, we have *τ=0* and therefore it suffers a large 160 | synchronization cost. While asynchronous SGD uses an infinite *τ* to eliminate 161 | this cost. In practice, an infinite *τ* is unlikely happens. But we also place 162 | a upper bound of *τ* to guarantee the convergence with some synchronization 163 | costs. 164 | 165 | ## Further Reads 166 | 167 | Distributed optimization algorithms are an active research topic in recent years. To 168 | name some of them 169 | 170 | - [Dean, NIPS'13](), [Li, OSDI'14]() The parameter server architecture 171 | - [Langford, NIPS'09](https://papers.nips.cc/paper/3888-slow-learners-are-fast.pdf), 172 | [Agarwal, NIPS'11](http://arxiv.org/pdf/1104.5525.pdf) theoretical convergence 173 | of asynchronous SGD 174 | - [Li, NIPS'14](http://www.cs.cmu.edu/~muli/file/parameter_server_nips14.pdf) 175 | trade-offs with bounded maximal delays *τ* 176 | - [Li, KDD'14](http://www.cs.cmu.edu/~muli/file/minibatch_sgd.pdf) improves the 177 | convergence rate with large minibatch size *b* 178 | - [Sra, AISTATS'16](http://arxiv.org/abs/1508.05003) asynchronous SGD adaptive 179 | to the actually delay rather than the worst maximal delay 180 | - [Li, WSDM'16](http://www.cs.cmu.edu/~yuxiangw/docs/fm.pdf) practical 181 | considerations for asynchronous SGD with the parameter server 182 | - [Chen, LearningSys'16]() synchronized SGD for deep learning. 183 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | breathe 2 | -------------------------------------------------------------------------------- /docs/sphinx_util.py: -------------------------------------------------------------------------------- 1 | import sys, os, subprocess 2 | 3 | 4 | if not os.path.exists('../recommonmark'): 5 | subprocess.call('cd ..; git clone https://github.com/tqchen/recommonmark', shell = True) 6 | else: 7 | subprocess.call('cd ../recommonmark; git pull', shell=True) 8 | 9 | sys.path.insert(0, os.path.abspath('../recommonmark/')) 10 | 11 | from recommonmark import parser, transform 12 | MarkdownParser = parser.CommonMarkParser 13 | AutoStructify = transform.AutoStructify 14 | 15 | # MarkdownParser.github_doc_root = github_doc_root 16 | 17 | def generate_doxygen_xml(app): 18 | """Run the doxygen make commands""" 19 | subprocess.call('doxygen') 20 | -------------------------------------------------------------------------------- /include/dmlc/base.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2015 by Contributors 3 | * \file base.h 4 | * \brief defines configuration macros 5 | */ 6 | #ifndef DMLC_BASE_H_ 7 | #define DMLC_BASE_H_ 8 | 9 | /*! \brief whether use glog for logging */ 10 | #ifndef DMLC_USE_GLOG 11 | #define DMLC_USE_GLOG 0 12 | #endif 13 | 14 | /*! 15 | * \brief whether throw dmlc::Error instead of 16 | * directly calling abort when FATAL error occured 17 | * NOTE: this may still not be perfect. 18 | * do not use FATAL and CHECK in destructors 19 | */ 20 | #ifndef DMLC_LOG_FATAL_THROW 21 | #define DMLC_LOG_FATAL_THROW 1 22 | #endif 23 | 24 | /*! 25 | * \brief Whether to print stack trace for fatal error, 26 | * enabled on linux when using gcc. 27 | */ 28 | #if (defined(__GNUC__) && !defined(__MINGW32__) && !defined(__sun) && !defined(__SVR4) && \ 29 | !(defined __MINGW64__) && !(defined __ANDROID__)) 30 | #if (!defined(DMLC_LOG_STACK_TRACE)) 31 | #define DMLC_LOG_STACK_TRACE 1 32 | #endif 33 | #if (!defined(DMLC_LOG_STACK_TRACE_SIZE)) 34 | #define DMLC_LOG_STACK_TRACE_SIZE 10 35 | #endif 36 | #endif 37 | 38 | /*! \brief whether compile with hdfs support */ 39 | #ifndef DMLC_USE_HDFS 40 | #define DMLC_USE_HDFS 0 41 | #endif 42 | 43 | /*! \brief whether compile with s3 support */ 44 | #ifndef DMLC_USE_S3 45 | #define DMLC_USE_S3 0 46 | #endif 47 | 48 | /*! \brief whether or not use parameter server */ 49 | #ifndef DMLC_USE_PS 50 | #define DMLC_USE_PS 0 51 | #endif 52 | 53 | /*! \brief whether or not use c++11 support */ 54 | #ifndef DMLC_USE_CXX11 55 | #define DMLC_USE_CXX11 (defined(__GXX_EXPERIMENTAL_CXX0X__) ||\ 56 | __cplusplus >= 201103L || defined(_MSC_VER)) 57 | #endif 58 | 59 | /// check if g++ is before 4.6 60 | #if DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__) 61 | #if __GNUC__ == 4 && __GNUC_MINOR__ < 6 62 | #pragma message("Will need g++-4.6 or higher to compile all" \ 63 | "the features in dmlc-core, " \ 64 | "compile without c++11, some features may be disabled") 65 | #undef DMLC_USE_CXX11 66 | #define DMLC_USE_CXX11 0 67 | #endif 68 | #endif 69 | 70 | /*! 71 | * \brief Disable copy constructor and assignment operator. 72 | * 73 | * If C++11 is supported, both copy and move constructors and 74 | * assignment operators are deleted explicitly. Otherwise, they are 75 | * only declared but not implemented. Place this macro in private 76 | * section if C++11 is not available. 77 | */ 78 | #ifndef DISALLOW_COPY_AND_ASSIGN 79 | # if DMLC_USE_CXX11 80 | # define DISALLOW_COPY_AND_ASSIGN(T) \ 81 | T(T const&) = delete; \ 82 | T(T&&) = delete; \ 83 | T& operator=(T const&) = delete; \ 84 | T& operator=(T&&) = delete 85 | # else 86 | # define DISALLOW_COPY_AND_ASSIGN(T) \ 87 | T(T const&); \ 88 | T& operator=(T const&) 89 | # endif 90 | #endif 91 | 92 | /// 93 | /// code block to handle optionally loading 94 | /// 95 | #if !defined(__GNUC__) 96 | #define fopen64 std::fopen 97 | #endif 98 | #ifdef _MSC_VER 99 | #if _MSC_VER < 1900 100 | // NOTE: sprintf_s is not equivalent to snprintf, 101 | // they are equivalent when success, which is sufficient for our case 102 | #define snprintf sprintf_s 103 | #define vsnprintf vsprintf_s 104 | #endif 105 | #else 106 | #ifdef _FILE_OFFSET_BITS 107 | #if _FILE_OFFSET_BITS == 32 108 | #pragma message("Warning: FILE OFFSET BITS defined to be 32 bit") 109 | #endif 110 | #endif 111 | 112 | #ifdef __APPLE__ 113 | #define off64_t off_t 114 | #define fopen64 std::fopen 115 | #endif 116 | 117 | extern "C" { 118 | #include 119 | } 120 | #endif 121 | 122 | #ifdef _MSC_VER 123 | //! \cond Doxygen_Suppress 124 | typedef signed char int8_t; 125 | typedef __int16 int16_t; 126 | typedef __int32 int32_t; 127 | typedef __int64 int64_t; 128 | typedef unsigned char uint8_t; 129 | typedef unsigned __int16 uint16_t; 130 | typedef unsigned __int32 uint32_t; 131 | typedef unsigned __int64 uint64_t; 132 | //! \endcond 133 | #else 134 | #include 135 | #endif 136 | #include 137 | #include 138 | 139 | /*! \brief namespace for dmlc */ 140 | namespace dmlc { 141 | /*! 142 | * \brief safely get the beginning address of a vector 143 | * \param vec input vector 144 | * \return beginning address of a vector 145 | */ 146 | template 147 | inline T *BeginPtr(std::vector &vec) { // NOLINT(*) 148 | if (vec.size() == 0) { 149 | return NULL; 150 | } else { 151 | return &vec[0]; 152 | } 153 | } 154 | /*! 155 | * \brief get the beginning address of a vector 156 | * \param vec input vector 157 | * \return beginning address of a vector 158 | */ 159 | template 160 | inline const T *BeginPtr(const std::vector &vec) { 161 | if (vec.size() == 0) { 162 | return NULL; 163 | } else { 164 | return &vec[0]; 165 | } 166 | } 167 | /*! 168 | * \brief get the beginning address of a vector 169 | * \param str input string 170 | * \return beginning address of a string 171 | */ 172 | inline char* BeginPtr(std::string &str) { // NOLINT(*) 173 | if (str.length() == 0) return NULL; 174 | return &str[0]; 175 | } 176 | /*! 177 | * \brief get the beginning address of a vector 178 | * \param str input string 179 | * \return beginning address of a string 180 | */ 181 | inline const char* BeginPtr(const std::string &str) { 182 | if (str.length() == 0) return NULL; 183 | return &str[0]; 184 | } 185 | } // namespace dmlc 186 | 187 | #if defined(_MSC_VER) && _MSC_VER < 1900 188 | #define constexpr const 189 | #define alignof __alignof 190 | #endif 191 | 192 | #endif // DMLC_BASE_H_ 193 | -------------------------------------------------------------------------------- /include/dmlc/logging.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2015 by Contributors 3 | * \file logging.h 4 | * \brief defines logging macros of dmlc 5 | * allows use of GLOG, fall back to internal 6 | * implementation when disabled 7 | */ 8 | #ifndef DMLC_LOGGING_H_ 9 | #define DMLC_LOGGING_H_ 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "./base.h" 17 | 18 | #if DMLC_LOG_STACK_TRACE 19 | #include 20 | #endif 21 | 22 | #if DMLC_LOG_STACK_TRACE 23 | #include 24 | #endif 25 | 26 | namespace dmlc { 27 | /*! 28 | * \brief exception class that will be thrown by 29 | * default logger if DMLC_LOG_FATAL_THROW == 1 30 | */ 31 | struct Error : public std::runtime_error { 32 | /*! 33 | * \brief constructor 34 | * \param s the error message 35 | */ 36 | explicit Error(const std::string &s) : std::runtime_error(s) {} 37 | }; 38 | } // namespace dmlc 39 | 40 | #if defined(_MSC_VER) && _MSC_VER < 1900 41 | #define noexcept(a) 42 | #endif 43 | 44 | #if DMLC_USE_CXX11 45 | #define DMLC_THROW_EXCEPTION noexcept(false) 46 | #else 47 | #define DMLC_THROW_EXCEPTION 48 | #endif 49 | 50 | #if DMLC_USE_GLOG 51 | #include 52 | 53 | namespace dmlc { 54 | inline void InitLogging(const char* argv0) { 55 | google::InitGoogleLogging(argv0); 56 | } 57 | } // namespace dmlc 58 | 59 | #else 60 | // use a light version of glog 61 | #include 62 | #include 63 | #include 64 | #include 65 | 66 | #if defined(_MSC_VER) 67 | #pragma warning(disable : 4722) 68 | #endif 69 | 70 | namespace dmlc { 71 | inline void InitLogging(const char* argv0) { 72 | // DO NOTHING 73 | } 74 | 75 | // Always-on checking 76 | #define CHECK(x) \ 77 | if (!(x)) \ 78 | dmlc::LogMessageFatal(__FILE__, __LINE__).stream() << "Check " \ 79 | "failed: " #x << ' ' 80 | #define CHECK_LT(x, y) CHECK((x) < (y)) 81 | #define CHECK_GT(x, y) CHECK((x) > (y)) 82 | #define CHECK_LE(x, y) CHECK((x) <= (y)) 83 | #define CHECK_GE(x, y) CHECK((x) >= (y)) 84 | #define CHECK_EQ(x, y) CHECK((x) == (y)) 85 | #define CHECK_NE(x, y) CHECK((x) != (y)) 86 | #define CHECK_NOTNULL(x) \ 87 | ((x) == NULL ? dmlc::LogMessageFatal(__FILE__, __LINE__).stream() << "Check notnull: " #x << ' ', (x) : (x)) // NOLINT(*) 88 | // Debug-only checking. 89 | #ifdef NDEBUG 90 | #define DCHECK(x) \ 91 | while (false) CHECK(x) 92 | #define DCHECK_LT(x, y) \ 93 | while (false) CHECK((x) < (y)) 94 | #define DCHECK_GT(x, y) \ 95 | while (false) CHECK((x) > (y)) 96 | #define DCHECK_LE(x, y) \ 97 | while (false) CHECK((x) <= (y)) 98 | #define DCHECK_GE(x, y) \ 99 | while (false) CHECK((x) >= (y)) 100 | #define DCHECK_EQ(x, y) \ 101 | while (false) CHECK((x) == (y)) 102 | #define DCHECK_NE(x, y) \ 103 | while (false) CHECK((x) != (y)) 104 | #else 105 | #define DCHECK(x) CHECK(x) 106 | #define DCHECK_LT(x, y) CHECK((x) < (y)) 107 | #define DCHECK_GT(x, y) CHECK((x) > (y)) 108 | #define DCHECK_LE(x, y) CHECK((x) <= (y)) 109 | #define DCHECK_GE(x, y) CHECK((x) >= (y)) 110 | #define DCHECK_EQ(x, y) CHECK((x) == (y)) 111 | #define DCHECK_NE(x, y) CHECK((x) != (y)) 112 | #endif // NDEBUG 113 | 114 | #define LOG_INFO dmlc::LogMessage(__FILE__, __LINE__) 115 | #define LOG_ERROR LOG_INFO 116 | #define LOG_WARNING LOG_INFO 117 | #define LOG_FATAL dmlc::LogMessageFatal(__FILE__, __LINE__) 118 | #define LOG_QFATAL LOG_FATAL 119 | 120 | // Poor man version of VLOG 121 | #define VLOG(x) LOG_INFO.stream() 122 | 123 | #define LOG(severity) LOG_##severity.stream() 124 | #define LG LOG_INFO.stream() 125 | #define LOG_IF(severity, condition) \ 126 | !(condition) ? (void)0 : dmlc::LogMessageVoidify() & LOG(severity) 127 | 128 | #ifdef NDEBUG 129 | #define LOG_DFATAL LOG_ERROR 130 | #define DFATAL ERROR 131 | #define DLOG(severity) true ? (void)0 : dmlc::LogMessageVoidify() & LOG(severity) 132 | #define DLOG_IF(severity, condition) \ 133 | (true || !(condition)) ? (void)0 : dmlc::LogMessageVoidify() & LOG(severity) 134 | #else 135 | #define LOG_DFATAL LOG_FATAL 136 | #define DFATAL FATAL 137 | #define DLOG(severity) LOG(severity) 138 | #define DLOG_IF(severity, condition) LOG_IF(severity, condition) 139 | #endif 140 | 141 | // Poor man version of LOG_EVERY_N 142 | #define LOG_EVERY_N(severity, n) LOG(severity) 143 | 144 | class DateLogger { 145 | public: 146 | DateLogger() { 147 | #if defined(_MSC_VER) 148 | _tzset(); 149 | #endif 150 | } 151 | const char* HumanDate() { 152 | #if defined(_MSC_VER) 153 | _strtime_s(buffer_, sizeof(buffer_)); 154 | #else 155 | time_t time_value = time(NULL); 156 | struct tm now; 157 | localtime_r(&time_value, &now); 158 | snprintf(buffer_, sizeof(buffer_), "%02d:%02d:%02d", now.tm_hour, 159 | now.tm_min, now.tm_sec); 160 | #endif 161 | return buffer_; 162 | } 163 | private: 164 | char buffer_[9]; 165 | }; 166 | 167 | class LogMessage { 168 | public: 169 | LogMessage(const char* file, int line) 170 | : 171 | #ifdef __ANDROID__ 172 | log_stream_(std::cout) 173 | #else 174 | log_stream_(std::cerr) 175 | #endif 176 | { 177 | log_stream_ << "[" << pretty_date_.HumanDate() << "] " << file << ":" 178 | << line << ": "; 179 | } 180 | ~LogMessage() { log_stream_ << "\n"; } 181 | std::ostream& stream() { return log_stream_; } 182 | 183 | protected: 184 | std::ostream& log_stream_; 185 | 186 | private: 187 | DateLogger pretty_date_; 188 | LogMessage(const LogMessage&); 189 | void operator=(const LogMessage&); 190 | }; 191 | 192 | #if DMLC_LOG_STACK_TRACE 193 | inline std::string Demangle(char const *msg_str) { 194 | using std::string; 195 | string msg(msg_str); 196 | size_t symbol_start = string::npos; 197 | size_t symbol_end = string::npos; 198 | if (((symbol_start = msg.find("_Z")) != string::npos) && 199 | (symbol_end = msg.find_first_of(" +", symbol_start))) { 200 | string left_of_symbol(msg, 0, symbol_start); 201 | string symbol(msg, symbol_start, symbol_end - symbol_start); 202 | string right_of_symbol(msg, symbol_end); 203 | 204 | int status = 0; 205 | size_t length = string::npos; 206 | std::unique_ptr demangled_symbol = { 207 | abi::__cxa_demangle(symbol.c_str(), 0, &length, &status), &std::free}; 208 | if (demangled_symbol && status == 0 && length > 0) { 209 | string symbol_str(demangled_symbol.get()); 210 | std::ostringstream os; 211 | os << left_of_symbol << symbol_str << right_of_symbol; 212 | return os.str(); 213 | } 214 | } 215 | return string(msg_str); 216 | } 217 | 218 | inline std::string StackTrace() { 219 | using std::string; 220 | std::ostringstream stacktrace_os; 221 | const int MAX_STACK_SIZE = DMLC_LOG_STACK_TRACE_SIZE; 222 | void *stack[MAX_STACK_SIZE]; 223 | int nframes = backtrace(stack, MAX_STACK_SIZE); 224 | stacktrace_os << "Stack trace returned " << nframes << " entries:" << std::endl; 225 | char **msgs = backtrace_symbols(stack, nframes); 226 | if (msgs != nullptr) { 227 | for (int frameno = 0; frameno < nframes; ++frameno) { 228 | string msg = dmlc::Demangle(msgs[frameno]); 229 | stacktrace_os << "[bt] (" << frameno << ") " << msg << "\n"; 230 | } 231 | } 232 | free(msgs); 233 | string stack_trace = stacktrace_os.str(); 234 | return stack_trace; 235 | } 236 | 237 | #else // DMLC_LOG_STACK_TRACE is off 238 | 239 | inline std::string demangle(char const *msg_str) { return std::string(); } 240 | 241 | inline std::string StackTrace() { 242 | return std::string( 243 | "stack traces not available when " 244 | "DMLC_LOG_STACK_TRACE is disabled at compile time."); 245 | } 246 | 247 | #endif // DMLC_LOG_STACK_TRACE 248 | 249 | #if DMLC_LOG_FATAL_THROW == 0 250 | class LogMessageFatal : public LogMessage { 251 | public: 252 | LogMessageFatal(const char* file, int line) : LogMessage(file, line) {} 253 | ~LogMessageFatal() { 254 | log_stream_ << "\n"; 255 | abort(); 256 | } 257 | 258 | private: 259 | LogMessageFatal(const LogMessageFatal&); 260 | void operator=(const LogMessageFatal&); 261 | }; 262 | #else 263 | class LogMessageFatal { 264 | public: 265 | LogMessageFatal(const char* file, int line) { 266 | log_stream_ << "[" << pretty_date_.HumanDate() << "] " << file << ":" 267 | << line << ": "; 268 | } 269 | std::ostringstream &stream() { return log_stream_; } 270 | ~LogMessageFatal() DMLC_THROW_EXCEPTION { 271 | #if DMLC_LOG_STACK_TRACE 272 | log_stream_ << "\n\n" << StackTrace() << "\n"; 273 | #endif 274 | // throwing out of destructor is evil 275 | // hopefully we can do it here 276 | // also log the message before throw 277 | LOG(ERROR) << log_stream_.str(); 278 | throw Error(log_stream_.str()); 279 | } 280 | 281 | private: 282 | std::ostringstream log_stream_; 283 | DateLogger pretty_date_; 284 | LogMessageFatal(const LogMessageFatal &); 285 | void operator=(const LogMessageFatal &); 286 | }; 287 | #endif 288 | 289 | // This class is used to explicitly ignore values in the conditional 290 | // logging macros. This avoids compiler warnings like "value computed 291 | // is not used" and "statement has no effect". 292 | class LogMessageVoidify { 293 | public: 294 | LogMessageVoidify() {} 295 | // This has to be an operator with a precedence lower than << but 296 | // higher than "?:". See its usage. 297 | void operator&(std::ostream&) {} 298 | }; 299 | 300 | } // namespace dmlc 301 | 302 | #endif 303 | #endif // DMLC_LOGGING_H_ 304 | -------------------------------------------------------------------------------- /include/ps/base.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_BASE_H_ 5 | #define PS_BASE_H_ 6 | #include 7 | #include "ps/internal/utils.h" 8 | namespace ps { 9 | 10 | #if USE_KEY32 11 | /*! \brief Use unsigned 32-bit int as the key type */ 12 | using Key = uint32_t; 13 | #else 14 | /*! \brief Use unsigned 64-bit int as the key type */ 15 | using Key = uint64_t; 16 | #endif 17 | /*! \brief The maximal allowed key value */ 18 | static const Key kMaxKey = std::numeric_limits::max(); 19 | /** \brief node ID for the scheduler */ 20 | static const int kScheduler = 1; 21 | /** 22 | * \brief the server node group ID 23 | * 24 | * group id can be combined: 25 | * - kServerGroup + kScheduler means all server nodes and the scheuduler 26 | * - kServerGroup + kWorkerGroup means all server and worker nodes 27 | */ 28 | static const int kServerGroup = 2; 29 | /** \brief the worker node group ID */ 30 | static const int kWorkerGroup = 4; 31 | 32 | } // namespace ps 33 | #endif // PS_BASE_H_ 34 | -------------------------------------------------------------------------------- /include/ps/internal/assign_op.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | * \file assign_op.h 4 | * \brief assignment operator 5 | * http://en.cppreference.com/w/cpp/language/operator_assignment 6 | */ 7 | #ifndef PS_INTERNAL_ASSIGN_OP_H_ 8 | #define PS_INTERNAL_ASSIGN_OP_H_ 9 | #include "ps/internal/utils.h" 10 | namespace ps { 11 | 12 | enum AssignOp { 13 | ASSIGN, // a = b 14 | PLUS, // a += b 15 | MINUS, // a -= b 16 | TIMES, // a *= b 17 | DIVIDE, // a -= b 18 | AND, // a &= b 19 | OR, // a |= b 20 | XOR // a ^= b 21 | }; 22 | 23 | /** 24 | * \brief return an assignment function: right op= left 25 | */ 26 | template 27 | inline void AssignFunc(const T& lhs, AssignOp op, T* rhs) { 28 | switch (op) { 29 | case ASSIGN: 30 | *right = left; break; 31 | case PLUS: 32 | *right += left; break; 33 | case MINUS: 34 | *right -= left; break; 35 | case TIMES: 36 | *right *= left; break; 37 | case DIVIDE: 38 | *right /= left; break; 39 | default: 40 | LOG(FATAL) << "use AssignOpInt.."; 41 | } 42 | } 43 | 44 | /** 45 | * \brief return an assignment function including bit operations, only 46 | * works for integers 47 | */ 48 | template 49 | inline void AssignFuncInt(const T& lhs, AssignOp op, T* rhs) { 50 | switch (op) { 51 | case ASSIGN: 52 | *right = left; break; 53 | case PLUS: 54 | *right += left; break; 55 | case MINUS: 56 | *right -= left; break; 57 | case TIMES: 58 | *right *= left; break; 59 | case DIVIDE: 60 | *right /= left; break; 61 | case AND: 62 | *right &= left; break; 63 | case OR: 64 | *right |= left; break; 65 | case XOR: 66 | *right ^= left; break; 67 | } 68 | } 69 | 70 | 71 | } // namespace ps 72 | #endif // PS_INTERNAL_ASSIGN_OP_H_ 73 | -------------------------------------------------------------------------------- /include/ps/internal/customer.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_CUSTOMER_H_ 5 | #define PS_INTERNAL_CUSTOMER_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "ps/internal/message.h" 15 | #include "ps/internal/threadsafe_pqueue.h" 16 | namespace ps { 17 | 18 | /** 19 | * \brief The object for communication. 20 | * 21 | * As a sender, a customer tracks the responses for each request sent. 22 | * 23 | * It has its own receiving thread which is able to process any message received 24 | * from a remote node with `msg.meta.customer_id` equal to this customer's id 25 | */ 26 | class Customer { 27 | public: 28 | /** 29 | * \brief the handle for a received message 30 | * \param recved the received message 31 | */ 32 | using RecvHandle = std::function; 33 | 34 | /** 35 | * \brief constructor 36 | * \param app_id the globally unique id indicating the application the postoffice 37 | * serving for 38 | * \param customer_id the locally unique id indicating the customer of a postoffice 39 | * \param recv_handle the functino for processing a received message 40 | */ 41 | Customer(int app_id, int customer_id, const RecvHandle& recv_handle); 42 | 43 | /** 44 | * \brief desconstructor 45 | */ 46 | ~Customer(); 47 | 48 | /** 49 | * \brief return the globally unique application id 50 | */ 51 | inline int app_id() { return app_id_; } 52 | 53 | 54 | /** 55 | * \brief return the locally unique customer id 56 | */ 57 | inline int customer_id() { return customer_id_; } 58 | 59 | /** 60 | * \brief get a timestamp for a new request. threadsafe 61 | * \param recver the receive node id of this request 62 | * \return the timestamp of this request 63 | */ 64 | int NewRequest(int recver); 65 | 66 | 67 | /** 68 | * \brief wait until the request is finished. threadsafe 69 | * \param timestamp the timestamp of the request 70 | */ 71 | void WaitRequest(int timestamp); 72 | 73 | /** 74 | * \brief return the number of responses received for the request. threadsafe 75 | * \param timestamp the timestamp of the request 76 | */ 77 | int NumResponse(int timestamp); 78 | 79 | /** 80 | * \brief add a number of responses to timestamp 81 | */ 82 | void AddResponse(int timestamp, int num = 1); 83 | 84 | /** 85 | * \brief accept a received message from \ref Van. threadsafe 86 | * \param recved the received the message 87 | */ 88 | inline void Accept(const Message& recved) { 89 | recv_queue_.Push(recved); 90 | } 91 | 92 | private: 93 | /** 94 | * \brief the thread function 95 | */ 96 | void Receiving(); 97 | 98 | int app_id_; 99 | 100 | int customer_id_; 101 | 102 | RecvHandle recv_handle_; 103 | ThreadsafePQueue recv_queue_; 104 | std::unique_ptr recv_thread_; 105 | 106 | std::mutex tracker_mu_; 107 | std::condition_variable tracker_cond_; 108 | std::vector> tracker_; 109 | 110 | DISALLOW_COPY_AND_ASSIGN(Customer); 111 | }; 112 | 113 | } // namespace ps 114 | #endif // PS_INTERNAL_CUSTOMER_H_ 115 | -------------------------------------------------------------------------------- /include/ps/internal/env.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_ENV_H_ 5 | #define PS_INTERNAL_ENV_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | namespace ps { 11 | 12 | /** 13 | * \brief Environment configurations 14 | */ 15 | class Environment { 16 | public: 17 | /** 18 | * \brief return the singleton instance 19 | */ 20 | static inline Environment* Get() { 21 | return _GetSharedRef(nullptr).get(); 22 | } 23 | /** 24 | * \brief return a shared ptr of the singleton instance 25 | */ 26 | static inline std::shared_ptr _GetSharedRef() { 27 | return _GetSharedRef(nullptr); 28 | } 29 | /** 30 | * \brief initialize the environment 31 | * \param envs key-value environment variables 32 | * \return the initialized singleton instance 33 | */ 34 | static inline Environment* Init(const std::unordered_map& envs) { 35 | Environment* env = _GetSharedRef(&envs).get(); 36 | env->kvs = envs; 37 | return env; 38 | } 39 | 40 | /** 41 | * \brief find the env value. 42 | * User-defined env vars first. If not found, check system's environment 43 | * \param k the environment key 44 | * \return the related environment value, nullptr when not found 45 | */ 46 | const char* find(const char* k) { 47 | std::string key(k); 48 | return kvs.find(key) == kvs.end() ? getenv(k) : kvs[key].c_str(); 49 | } 50 | 51 | private: 52 | explicit Environment(const std::unordered_map* envs) { 53 | if (envs) kvs = *envs; 54 | } 55 | 56 | static std::shared_ptr _GetSharedRef( 57 | const std::unordered_map* envs) { 58 | static std::shared_ptr inst_ptr(new Environment(envs)); 59 | return inst_ptr; 60 | } 61 | 62 | std::unordered_map kvs; 63 | }; 64 | 65 | } // namespace ps 66 | #endif // PS_INTERNAL_ENV_H_ 67 | -------------------------------------------------------------------------------- /include/ps/internal/message.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_MESSAGE_H_ 5 | #define PS_INTERNAL_MESSAGE_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "ps/sarray.h" 11 | namespace ps { 12 | /** \brief data type */ 13 | enum DataType { 14 | CHAR, INT8, INT16, INT32, INT64, 15 | UINT8, UINT16, UINT32, UINT64, 16 | FLOAT, DOUBLE, OTHER 17 | }; 18 | /** \brief data type name */ 19 | static const char* DataTypeName[] = { 20 | "CHAR", "INT8", "INT16", "INT32", "INT64", 21 | "UINT8", "UINT16", "UINT32", "UINT64", 22 | "FLOAT", "DOUBLE", "OTHER" 23 | }; 24 | /** 25 | * \brief compare if V and W are the same type 26 | */ 27 | template 28 | inline bool SameType() { 29 | return std::is_same::type, W>::value; 30 | } 31 | /** 32 | * \brief return the DataType of V 33 | */ 34 | template 35 | DataType GetDataType() { 36 | if (SameType()) { 37 | return INT8; 38 | } else if (SameType()) { 39 | return INT16; 40 | } else if (SameType()) { 41 | return INT32; 42 | } else if (SameType()) { 43 | return INT64; 44 | } else if (SameType()) { 45 | return UINT8; 46 | } else if (SameType()) { 47 | return UINT16; 48 | } else if (SameType()) { 49 | return UINT32; 50 | } else if (SameType()) { 51 | return UINT64; 52 | } else if (SameType()) { 53 | return FLOAT; 54 | } else if (SameType()) { 55 | return DOUBLE; 56 | } else { 57 | return OTHER; 58 | } 59 | } 60 | /** 61 | * \brief information about a node 62 | */ 63 | struct Node { 64 | /** \brief the empty value */ 65 | static const int kEmpty; 66 | /** \brief default constructor */ 67 | Node() : id(kEmpty), port(kEmpty), is_recovery(false) {} 68 | /** \brief node roles */ 69 | enum Role { SERVER, WORKER, SCHEDULER }; 70 | /** \brief get debug string */ 71 | std::string DebugString() const { 72 | std::stringstream ss; 73 | ss << "role=" << (role == SERVER ? "server" : (role == WORKER ? "worker" : "scheduler")) 74 | << (id != kEmpty ? ", id=" + std::to_string(id) : "") 75 | << ", ip=" << hostname << ", port=" << port << ", is_recovery=" << is_recovery; 76 | 77 | return ss.str(); 78 | } 79 | /** \brief get short debug string */ 80 | std::string ShortDebugString() const { 81 | std::string str = role == SERVER ? "S" : (role == WORKER ? "W" : "H"); 82 | if (id != kEmpty) str += "[" + std::to_string(id) + "]"; 83 | return str; 84 | } 85 | /** \brief the role of this node */ 86 | Role role; 87 | /** \brief node id */ 88 | int id; 89 | /** \brief customer id */ 90 | int customer_id; 91 | /** \brief hostname or ip */ 92 | std::string hostname; 93 | /** \brief the port this node is binding */ 94 | int port; 95 | /** \brief whether this node is created by failover */ 96 | bool is_recovery; 97 | }; 98 | /** 99 | * \brief meta info of a system control message 100 | */ 101 | struct Control { 102 | /** \brief empty constructor */ 103 | Control() : cmd(EMPTY) { } 104 | /** \brief return true is empty */ 105 | inline bool empty() const { return cmd == EMPTY; } 106 | /** \brief get debug string */ 107 | std::string DebugString() const { 108 | if (empty()) return ""; 109 | std::vector cmds = { 110 | "EMPTY", "TERMINATE", "ADD_NODE", "BARRIER", "ACK", "HEARTBEAT"}; 111 | std::stringstream ss; 112 | ss << "cmd=" << cmds[cmd]; 113 | if (node.size()) { 114 | ss << ", node={"; 115 | for (const Node& n : node) ss << " " << n.DebugString(); 116 | ss << " }"; 117 | } 118 | if (cmd == BARRIER) ss << ", barrier_group=" << barrier_group; 119 | if (cmd == ACK) ss << ", msg_sig=" << msg_sig; 120 | return ss.str(); 121 | } 122 | /** \brief all commands */ 123 | enum Command { EMPTY, TERMINATE, ADD_NODE, BARRIER, ACK, HEARTBEAT }; 124 | /** \brief the command */ 125 | Command cmd; 126 | /** \brief node infos */ 127 | std::vector node; 128 | /** \brief the node group for a barrier, such as kWorkerGroup */ 129 | int barrier_group; 130 | /** message signature */ 131 | uint64_t msg_sig; 132 | }; 133 | /** 134 | * \brief meta info of a message 135 | */ 136 | struct Meta { 137 | /** \brief the empty value */ 138 | static const int kEmpty; 139 | /** \brief default constructor */ 140 | Meta() : head(kEmpty), app_id(kEmpty), customer_id(kEmpty), 141 | timestamp(kEmpty), sender(kEmpty), recver(kEmpty), 142 | request(false), push(false), pull(false), simple_app(false) {} 143 | std::string DebugString() const { 144 | std::stringstream ss; 145 | if (sender == Node::kEmpty) { 146 | ss << "?"; 147 | } else { 148 | ss << sender; 149 | } 150 | ss << " => " << recver; 151 | ss << ". Meta: request=" << request; 152 | if (timestamp != kEmpty) ss << ", timestamp=" << timestamp; 153 | if (!control.empty()) { 154 | ss << ", control={ " << control.DebugString() << " }"; 155 | } else { 156 | ss << ", app_id=" << app_id 157 | << ", customer_id=" << customer_id 158 | << ", simple_app=" << simple_app 159 | << ", push=" << push; 160 | } 161 | if (head != kEmpty) ss << ", head=" << head; 162 | if (body.size()) ss << ", body=" << body; 163 | if (data_type.size()) { 164 | ss << ", data_type={"; 165 | for (auto d : data_type) ss << " " << DataTypeName[static_cast(d)]; 166 | ss << " }"; 167 | } 168 | return ss.str(); 169 | } 170 | /** \brief an int head */ 171 | int head; 172 | /** \brief the unique id of the application of messsage is for*/ 173 | int app_id; 174 | /** \brief customer id*/ 175 | int customer_id; 176 | /** \brief the timestamp of this message */ 177 | int timestamp; 178 | /** \brief the node id of the sender of this message */ 179 | int sender; 180 | /** \brief the node id of the receiver of this message */ 181 | int recver; 182 | /** \brief whether or not this is a request message*/ 183 | bool request; 184 | /** \brief whether or not a push message */ 185 | bool push; 186 | /** \brief whether or not a pull message */ 187 | bool pull; 188 | /** \brief whether or not it's for SimpleApp */ 189 | bool simple_app; 190 | /** \brief an string body */ 191 | std::string body; 192 | /** \brief data type of message.data[i] */ 193 | std::vector data_type; 194 | /** \brief system control message */ 195 | Control control; 196 | /** \brief the byte size */ 197 | int data_size = 0; 198 | /** \brief message priority */ 199 | int priority = 0; 200 | }; 201 | /** 202 | * \brief messages that communicated amaong nodes. 203 | */ 204 | struct Message { 205 | /** \brief the meta info of this message */ 206 | Meta meta; 207 | /** \brief the large chunk of data of this message */ 208 | std::vector > data; 209 | /** 210 | * \brief push array into data, and add the data type 211 | */ 212 | template 213 | void AddData(const SArray& val) { 214 | CHECK_EQ(data.size(), meta.data_type.size()); 215 | meta.data_type.push_back(GetDataType()); 216 | SArray bytes(val); 217 | meta.data_size += bytes.size(); 218 | data.push_back(bytes); 219 | } 220 | std::string DebugString() const { 221 | std::stringstream ss; 222 | ss << meta.DebugString(); 223 | if (data.size()) { 224 | ss << " Body:"; 225 | for (const auto& d : data) ss << " data_size=" << d.size(); 226 | } 227 | return ss.str(); 228 | } 229 | }; 230 | } // namespace ps 231 | #endif // PS_INTERNAL_MESSAGE_H_ 232 | -------------------------------------------------------------------------------- /include/ps/internal/parallel_kv_match.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | * \file parallel_kv_match.h 4 | * \brief paralle key-value pairs matching 5 | */ 6 | #ifndef PS_INTERNAL_PARALLEL_KV_MATCH_H_ 7 | #define PS_INTERNAL_PARALLEL_KV_MATCH_H_ 8 | #include 9 | #include 10 | #include "ps/sarray.h" 11 | #include "ps/internal/assign_op.h" 12 | 13 | namespace ps { 14 | namespace { 15 | /** 16 | * \brief thread function, internal use 17 | * 18 | * \param src_key start of source key 19 | * \param src_key_end end of source key 20 | * \param src_val start of source val 21 | * \param dst_key start of destination key 22 | * \param dst_key_end end of denstination key 23 | * \param dst_val start of destination val 24 | * \param k length of a single value 25 | * \param op assignment operator 26 | * \param grainsize thread grainsize size 27 | * \param n number of matched kv pairs 28 | */ 29 | template 30 | void ParallelOrderedMatch( 31 | const K* src_key, const K* src_key_end, const V* src_val, 32 | const K* dst_key, const K* dst_key_end, V* dst_val, 33 | int k, AsOp op, size_t grainsize, size_t* n) { 34 | size_t src_len = std::distance(src_key, src_key_end); 35 | size_t dst_len = std::distance(dst_key, dst_key_end); 36 | if (dst_len == 0 || src_len == 0) return; 37 | 38 | // drop the unmatched tail of src 39 | src_key = std::lower_bound(src_key, src_key_end, *dst_key); 40 | src_val += (src_key - (src_key_end - src_len)) * k; 41 | 42 | if (dst_len <= grainsize) { 43 | while (dst_key != dst_key_end && src_key != src_key_end) { 44 | if (*src_key < *dst_key) { 45 | ++src_key; src_val += k; 46 | } else { 47 | if (!(*dst_key < *src_key)) { 48 | for (int i = 0; i < k; ++i) { 49 | AssignOp(dst_val[i], src_val[i], op); 50 | } 51 | ++src_key; src_val += k; 52 | *n += k; 53 | } 54 | ++dst_key; dst_val += k; 55 | } 56 | } 57 | } else { 58 | std::thread thr( 59 | ParallelOrderedMatch, src_key, src_key_end, src_val, 60 | dst_key, dst_key + dst_len / 2, dst_val, 61 | k, op, grainsize, n); 62 | size_t m = 0; 63 | ParallelOrderedMatch( 64 | src_key, src_key_end, src_val, 65 | dst_key + dst_len / 2, dst_key_end, dst_val + (dst_len / 2) * k, 66 | k, op, grainsize, &m); 67 | thr.join(); 68 | *n += m; 69 | } 70 | } 71 | } // namespace 72 | 73 | 74 | /** 75 | * \brief Merge \a src_val into \a dst_val by matching keys. Keys must be unique 76 | * and sorted. 77 | * 78 | * \code 79 | * if (dst_key[i] == src_key[j]) { 80 | * dst_val[i] op= src_val[j] 81 | * } 82 | * \endcode 83 | * 84 | * When finished, \a dst_val will have length `k * dst_key.size()` and filled 85 | * with matched value. Umatched value will be untouched if exists or filled with 0. 86 | * 87 | * \tparam K type of key 88 | * \tparam V type of value 89 | * \tparam C type of the container such as \ref SArray or \ref std::vector 90 | * \param src_key the source keys 91 | * \param src_val the source values 92 | * \param dst_key the destination keys 93 | * \param dst_val the destination values. 94 | * \param k the length of a single value (default is 1) 95 | * \param op the assignment operator (default is ASSIGN) 96 | * \param num_threads number of thread (default is 1) 97 | * \return the number of matched kv pairs 98 | */ 99 | template 100 | size_t ParallelOrderedMatch( 101 | const SArray& src_key, const SArray& src_val, 102 | const SArray& dst_key, C* dst_val, 103 | int k = 1, AssignOp op = ASSIGN, int num_threads = 1) { 104 | // do check 105 | CHECK_GT(num_threads, 0); 106 | CHECK_EQ(src_key.size() * k, src_val.size()); 107 | CHECK_NOTNULL(dst_val->resize(dst_key.size() * k)); 108 | if (dst_key.empty()) return 0; 109 | 110 | // shorten the matching range 111 | Range range = FindRange(dst_key, src_key.begin(), src_key.end()); 112 | size_t grainsize = std::max(range.size() * k / num_threads + 5, 113 | static_cast(1024*1024)); 114 | size_t n = 0; 115 | ParallelOrderedMatch( 116 | src_key.begin(), src_key.end(), src_val.begin(), 117 | dst_key.begin() + range.begin(), dst_key.begin() + range.end(), 118 | dst_val->begin() + range.begin()*k, k, op, grainsize, &n); 119 | return n; 120 | } 121 | 122 | } // namespace ps 123 | #endif // PS_INTERNAL_PARALLEL_KV_MATCH_H_ 124 | -------------------------------------------------------------------------------- /include/ps/internal/parallel_sort.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | * @file parallel_sort.h 4 | * @brief Parallel sort 5 | */ 6 | #ifndef PS_INTERNAL_PARALLEL_SORT_H_ 7 | #define PS_INTERNAL_PARALLEL_SORT_H_ 8 | #include 9 | #include 10 | #include 11 | #include "ps/sarray.h" 12 | namespace ps { 13 | 14 | namespace { 15 | /** 16 | * \brief the thread function 17 | * 18 | * \param data start pointer of data 19 | * \param len length of data 20 | * \param grainsize max data length of one thread 21 | * \param cmp comparison function 22 | */ 23 | template 24 | void ParallelSort(T* data, size_t len, size_t grainsize, const Fn& cmp) { 25 | if (len <= grainsize) { 26 | std::sort(data, data + len, cmp); 27 | } else { 28 | std::thread thr(ParallelSort, data, len/2, grainsize, cmp); 29 | ParallelSort(data + len/2, len - len/2, grainsize, cmp); 30 | thr.join(); 31 | 32 | std::inplace_merge(data, data + len/2, data + len, cmp); 33 | } 34 | } 35 | } // namespace 36 | 37 | /** 38 | * \brief Parallel Sort 39 | * 40 | * \param arr the array for sorting 41 | * \param num_threads number of thread 42 | * \param cmp the comparision function such as 43 | * [](const T& a, const T& b) {* return a < b; } 44 | * or an even simplier version: 45 | * std::less() 46 | */ 47 | template 48 | void ParallelSort(SArray* arr, 49 | int num_threads = 2, 50 | const Fn& cmp = std::less()) { 51 | CHECK_GT(num_threads, 0); 52 | CHECK(cmp); 53 | size_t grainsize = std::max(arr->size() / num_threads + 5, (size_t)1024*16); 54 | ParallelSort(arr->data(), arr->size(), grainsize, cmp); 55 | } 56 | 57 | } // namespace ps 58 | #endif // PS_INTERNAL_PARALLEL_SORT_H_ 59 | -------------------------------------------------------------------------------- /include/ps/internal/postoffice.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_POSTOFFICE_H_ 5 | #define PS_INTERNAL_POSTOFFICE_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "ps/range.h" 12 | #include "ps/internal/env.h" 13 | #include "ps/internal/customer.h" 14 | #include "ps/internal/van.h" 15 | namespace ps { 16 | /** 17 | * \brief the center of the system 18 | */ 19 | class Postoffice { 20 | public: 21 | /** 22 | * \brief return the singleton object 23 | */ 24 | static Postoffice* Get() { 25 | static Postoffice e; return &e; 26 | } 27 | /** \brief get the van */ 28 | Van* van() { return van_; } 29 | /** 30 | * \brief start the system 31 | * 32 | * This function will block until every nodes are started. 33 | * \param argv0 the program name, used for logging. 34 | * \param do_barrier whether to block until every nodes are started. 35 | */ 36 | void Start(int customer_id, const char* argv0, const bool do_barrier); 37 | /** 38 | * \brief terminate the system 39 | * 40 | * All nodes should call this function before existing. 41 | * \param do_barrier whether to do block until every node is finalized, default true. 42 | */ 43 | void Finalize(const int customer_id, const bool do_barrier = true); 44 | /** 45 | * \brief add an customer to the system. threadsafe 46 | */ 47 | void AddCustomer(Customer* customer); 48 | /** 49 | * \brief remove a customer by given it's id. threasafe 50 | */ 51 | void RemoveCustomer(Customer* customer); 52 | /** 53 | * \brief get the customer by id, threadsafe 54 | * \param app_id the application id 55 | * \param customer_id the customer id 56 | * \param timeout timeout in sec 57 | * \return return nullptr if doesn't exist and timeout 58 | */ 59 | Customer* GetCustomer(int app_id, int customer_id, int timeout = 0) const; 60 | /** 61 | * \brief get the id of a node (group), threadsafe 62 | * 63 | * if it is a node group, return the list of node ids in this 64 | * group. otherwise, return {node_id} 65 | */ 66 | const std::vector& GetNodeIDs(int node_id) const { 67 | const auto it = node_ids_.find(node_id); 68 | CHECK(it != node_ids_.cend()) << "node " << node_id << " doesn't exist"; 69 | return it->second; 70 | } 71 | /** 72 | * \brief return the key ranges of all server nodes 73 | */ 74 | const std::vector& GetServerKeyRanges(); 75 | /** 76 | * \brief the template of a callback 77 | */ 78 | using Callback = std::function; 79 | /** 80 | * \brief Register a callback to the system which is called after Finalize() 81 | * 82 | * The following codes are equal 83 | * \code {cpp} 84 | * RegisterExitCallback(cb); 85 | * Finalize(); 86 | * \endcode 87 | * 88 | * \code {cpp} 89 | * Finalize(); 90 | * cb(); 91 | * \endcode 92 | * \param cb the callback function 93 | */ 94 | void RegisterExitCallback(const Callback& cb) { 95 | exit_callback_ = cb; 96 | } 97 | /** 98 | * \brief convert from a worker rank into a node id 99 | * \param rank the worker rank 100 | */ 101 | static inline int WorkerRankToID(int rank) { 102 | return rank * 2 + 9; 103 | } 104 | /** 105 | * \brief convert from a server rank into a node id 106 | * \param rank the server rank 107 | */ 108 | static inline int ServerRankToID(int rank) { 109 | return rank * 2 + 8; 110 | } 111 | /** 112 | * \brief convert from a node id into a server or worker rank 113 | * \param id the node id 114 | */ 115 | static inline int IDtoRank(int id) { 116 | #ifdef _MSC_VER 117 | #undef max 118 | #endif 119 | return std::max((id - 8) / 2, 0); 120 | } 121 | /** \brief Returns the number of worker nodes */ 122 | int num_workers() const { return num_workers_; } 123 | /** \brief Returns the number of server nodes */ 124 | int num_servers() const { return num_servers_; } 125 | /** \brief Returns the rank of this node in its group 126 | * 127 | * Each worker will have a unique rank within [0, NumWorkers()). So are 128 | * servers. This function is available only after \ref Start has been called. 129 | */ 130 | int my_rank() const { return IDtoRank(van_->my_node().id); } 131 | /** \brief Returns true if this node is a worker node */ 132 | int is_worker() const { return is_worker_; } 133 | /** \brief Returns true if this node is a server node. */ 134 | int is_server() const { return is_server_; } 135 | /** \brief Returns true if this node is a scheduler node. */ 136 | int is_scheduler() const { return is_scheduler_; } 137 | /** \brief Returns the verbose level. */ 138 | int verbose() const { return verbose_; } 139 | /** \brief Return whether this node is a recovery node */ 140 | bool is_recovery() const { return van_->my_node().is_recovery; } 141 | /** 142 | * \brief barrier 143 | * \param node_id the barrier group id 144 | */ 145 | void Barrier(int customer_id, int node_group); 146 | /** 147 | * \brief process a control message, called by van 148 | * \param the received message 149 | */ 150 | void Manage(const Message& recv); 151 | /** 152 | * \brief update the heartbeat record map 153 | * \param node_id the \ref Node id 154 | * \param t the last received heartbeat time 155 | */ 156 | void UpdateHeartbeat(int node_id, time_t t) { 157 | std::lock_guard lk(heartbeat_mu_); 158 | heartbeats_[node_id] = t; 159 | } 160 | /** 161 | * \brief get node ids that haven't reported heartbeats for over t seconds 162 | * \param t timeout in sec 163 | */ 164 | std::vector GetDeadNodes(int t = 60); 165 | 166 | private: 167 | Postoffice(); 168 | ~Postoffice() { delete van_; } 169 | 170 | void InitEnvironment(); 171 | Van* van_; 172 | mutable std::mutex mu_; 173 | // app_id -> (customer_id -> customer pointer) 174 | std::unordered_map> customers_; 175 | std::unordered_map> node_ids_; 176 | std::mutex server_key_ranges_mu_; 177 | std::vector server_key_ranges_; 178 | bool is_worker_, is_server_, is_scheduler_; 179 | int num_servers_, num_workers_; 180 | std::unordered_map > barrier_done_; 181 | int verbose_; 182 | std::mutex barrier_mu_; 183 | std::condition_variable barrier_cond_; 184 | std::mutex heartbeat_mu_; 185 | std::mutex start_mu_; 186 | int init_stage_ = 0; 187 | std::unordered_map heartbeats_; 188 | Callback exit_callback_; 189 | /** \brief Holding a shared_ptr to prevent it from being destructed too early */ 190 | std::shared_ptr env_ref_; 191 | time_t start_time_; 192 | DISALLOW_COPY_AND_ASSIGN(Postoffice); 193 | }; 194 | 195 | /** \brief verbose log */ 196 | #define PS_VLOG(x) LOG_IF(INFO, x <= Postoffice::Get()->verbose()) 197 | } // namespace ps 198 | #endif // PS_INTERNAL_POSTOFFICE_H_ 199 | -------------------------------------------------------------------------------- /include/ps/internal/threadsafe_pqueue.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_THREADSAFE_PQUEUE_H_ 5 | #define PS_INTERNAL_THREADSAFE_PQUEUE_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "ps/base.h" 13 | namespace ps { 14 | 15 | /** 16 | * \brief thread-safe queue allowing push and waited pop 17 | */ 18 | class ThreadsafePQueue { 19 | public: 20 | ThreadsafePQueue() { } 21 | ~ThreadsafePQueue() { } 22 | 23 | /** 24 | * \brief push an value into the end. threadsafe. 25 | * \param new_value the value 26 | */ 27 | void Push(Message new_value) { 28 | mu_.lock(); 29 | queue_.push(std::move(new_value)); 30 | mu_.unlock(); 31 | cond_.notify_all(); 32 | } 33 | 34 | /** 35 | * \brief wait until pop an element from the beginning, threadsafe 36 | * \param value the poped value 37 | */ 38 | void WaitAndPop(Message* value) { 39 | std::unique_lock lk(mu_); 40 | cond_.wait(lk, [this]{return !queue_.empty();}); 41 | *value = std::move(queue_.top()); 42 | queue_.pop(); 43 | } 44 | 45 | private: 46 | class Compare { 47 | public: 48 | bool operator()(const Message &l, const Message &r) { 49 | return l.meta.priority <= r.meta.priority; 50 | } 51 | }; 52 | mutable std::mutex mu_; 53 | std::priority_queue, Compare> queue_; 54 | std::condition_variable cond_; 55 | }; 56 | 57 | } // namespace ps 58 | 59 | #endif // PS_INTERNAL_THREADSAFE_PQUEUE_H_ 60 | -------------------------------------------------------------------------------- /include/ps/internal/threadsafe_queue.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_THREADSAFE_QUEUE_H_ 5 | #define PS_INTERNAL_THREADSAFE_QUEUE_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "ps/base.h" 12 | namespace ps { 13 | 14 | /** 15 | * \brief thread-safe queue allowing push and waited pop 16 | */ 17 | template class ThreadsafeQueue { 18 | public: 19 | ThreadsafeQueue() { } 20 | ~ThreadsafeQueue() { } 21 | 22 | /** 23 | * \brief push an value into the end. threadsafe. 24 | * \param new_value the value 25 | */ 26 | void Push(T new_value) { 27 | mu_.lock(); 28 | queue_.push(std::move(new_value)); 29 | mu_.unlock(); 30 | cond_.notify_all(); 31 | } 32 | 33 | /** 34 | * \brief wait until pop an element from the beginning, threadsafe 35 | * \param value the poped value 36 | */ 37 | void WaitAndPop(T* value) { 38 | std::unique_lock lk(mu_); 39 | cond_.wait(lk, [this]{return !queue_.empty();}); 40 | *value = std::move(queue_.front()); 41 | queue_.pop(); 42 | } 43 | 44 | private: 45 | mutable std::mutex mu_; 46 | std::queue queue_; 47 | std::condition_variable cond_; 48 | }; 49 | 50 | } // namespace ps 51 | 52 | // bool TryPop(T& value) { 53 | // std::lock_guard lk(mut); 54 | // if(data_queue.empty()) 55 | // return false; 56 | // value=std::move(data_queue.front()); 57 | // data_queue.pop(); 58 | // return true; 59 | // } 60 | #endif // PS_INTERNAL_THREADSAFE_QUEUE_H_ 61 | -------------------------------------------------------------------------------- /include/ps/internal/utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_UTILS_H_ 5 | #define PS_INTERNAL_UTILS_H_ 6 | #include "dmlc/logging.h" 7 | #include "ps/internal/env.h" 8 | namespace ps { 9 | 10 | #ifdef _MSC_VER 11 | typedef signed char int8_t; 12 | typedef __int16 int16_t; 13 | typedef __int32 int32_t; 14 | typedef __int64 int64_t; 15 | typedef unsigned char uint8_t; 16 | typedef unsigned __int16 uint16_t; 17 | typedef unsigned __int32 uint32_t; 18 | typedef unsigned __int64 uint64_t; 19 | #else 20 | #include 21 | #endif 22 | 23 | /*! 24 | * \brief Get environment variable as int with default. 25 | * \param key the name of environment variable. 26 | * \param default_val the default value of environment variable. 27 | * \return The value received 28 | */ 29 | template 30 | inline V GetEnv(const char *key, V default_val) { 31 | const char *val = Environment::Get()->find(key); 32 | if (val == nullptr) { 33 | return default_val; 34 | } else { 35 | return V(val); 36 | } 37 | } 38 | 39 | inline int GetEnv(const char *key, int default_val) { 40 | const char *val = Environment::Get()->find(key); 41 | if (val == nullptr) { 42 | return default_val; 43 | } else { 44 | return atoi(val); 45 | } 46 | } 47 | 48 | #ifndef DISALLOW_COPY_AND_ASSIGN 49 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 50 | TypeName(const TypeName&); \ 51 | void operator=(const TypeName&) 52 | #endif 53 | 54 | #define LL LOG(ERROR) 55 | 56 | } // namespace ps 57 | #endif // PS_INTERNAL_UTILS_H_ 58 | -------------------------------------------------------------------------------- /include/ps/internal/van.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_INTERNAL_VAN_H_ 5 | #define PS_INTERNAL_VAN_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "ps/base.h" 16 | #include "ps/internal/message.h" 17 | namespace ps { 18 | class Resender; 19 | class PBMeta; 20 | /** 21 | * \brief Van sends messages to remote nodes 22 | * 23 | * If environment variable PS_RESEND is set to be 1, then van will resend a 24 | * message if it no ACK messsage is received within PS_RESEND_TIMEOUT 25 | * millisecond 26 | */ 27 | class Van { 28 | public: 29 | /** 30 | * \brief create Van 31 | * \param type zmq, socket, ... 32 | */ 33 | static Van *Create(const std::string &type); 34 | 35 | /** \brief constructer, do nothing. use \ref Start for real start */ 36 | Van() {} 37 | 38 | /**\brief deconstructer, do nothing. use \ref Stop for real stop */ 39 | virtual ~Van() {} 40 | 41 | /** 42 | * \brief start van 43 | * 44 | * must call it before calling Send 45 | * 46 | * it initializes all connections to other nodes. start the receiving 47 | * thread, which keeps receiving messages. if it is a system 48 | * control message, give it to postoffice::manager, otherwise, give it to the 49 | * corresponding app. 50 | */ 51 | virtual void Start(int customer_id); 52 | 53 | /** 54 | * \brief send a message, It is thread-safe 55 | * \return the number of bytes sent. -1 if failed 56 | */ 57 | int Send(const Message &msg); 58 | 59 | /** 60 | * \brief return my node 61 | */ 62 | inline const Node &my_node() const { 63 | CHECK(ready_) << "call Start() first"; 64 | return my_node_; 65 | } 66 | 67 | /** 68 | * \brief stop van 69 | * stop receiving threads 70 | */ 71 | virtual void Stop(); 72 | 73 | /** 74 | * \brief get next available timestamp. thread safe 75 | */ 76 | inline int GetTimestamp() { return timestamp_++; } 77 | 78 | /** 79 | * \brief whether it is ready for sending. thread safe 80 | */ 81 | inline bool IsReady() { return ready_; } 82 | 83 | protected: 84 | /** 85 | * \brief connect to a node 86 | */ 87 | virtual void Connect(const Node &node) = 0; 88 | 89 | /** 90 | * \brief bind to my node 91 | * do multiple retries on binding the port. since it's possible that 92 | * different nodes on the same machine picked the same port 93 | * \return return the port binded, -1 if failed. 94 | */ 95 | virtual int Bind(const Node &node, int max_retry) = 0; 96 | 97 | /** 98 | * \brief block until received a message 99 | * \return the number of bytes received. -1 if failed or timeout 100 | */ 101 | virtual int RecvMsg(Message *msg) = 0; 102 | 103 | /** 104 | * \brief send a mesage 105 | * \return the number of bytes sent 106 | */ 107 | virtual int SendMsg(const Message &msg) = 0; 108 | 109 | /** 110 | * \brief pack meta into a string 111 | */ 112 | void PackMeta(const Meta &meta, char **meta_buf, int *buf_size); 113 | 114 | /** 115 | * \brief pack meta into protobuf 116 | */ 117 | void PackMetaPB(const Meta &meta, PBMeta *pb); 118 | 119 | /** 120 | * \brief unpack meta from a string 121 | */ 122 | void UnpackMeta(const char *meta_buf, int buf_size, Meta *meta); 123 | 124 | Node scheduler_; 125 | Node my_node_; 126 | bool is_scheduler_; 127 | std::mutex start_mu_; 128 | 129 | private: 130 | /** thread function for receving */ 131 | void Receiving(); 132 | 133 | /** thread function for heartbeat */ 134 | void Heartbeat(); 135 | 136 | // node's address string (i.e. ip:port) -> node id 137 | // this map is updated when ip:port is received for the first time 138 | std::unordered_map connected_nodes_; 139 | // maps the id of node which is added later to the id of node 140 | // which is with the same ip:port and added first 141 | std::unordered_map shared_node_mapping_; 142 | 143 | /** whether it is ready for sending */ 144 | std::atomic ready_{false}; 145 | std::atomic send_bytes_{0}; 146 | size_t recv_bytes_ = 0; 147 | int num_servers_ = 0; 148 | int num_workers_ = 0; 149 | /** the thread for receiving messages */ 150 | std::unique_ptr receiver_thread_; 151 | /** the thread for sending heartbeat */ 152 | std::unique_ptr heartbeat_thread_; 153 | std::vector barrier_count_; 154 | /** msg resender */ 155 | Resender *resender_ = nullptr; 156 | int drop_rate_ = 0; 157 | std::atomic timestamp_{0}; 158 | int init_stage = 0; 159 | 160 | /** 161 | * \brief processing logic of AddNode message for scheduler 162 | */ 163 | void ProcessAddNodeCommandAtScheduler(Message *msg, Meta *nodes, 164 | Meta *recovery_nodes); 165 | 166 | /** 167 | * \brief processing logic of Terminate message 168 | */ 169 | void ProcessTerminateCommand(); 170 | 171 | /** 172 | * \brief processing logic of AddNode message (run on each node) 173 | */ 174 | void ProcessAddNodeCommand(Message *msg, Meta *nodes, Meta *recovery_nodes); 175 | 176 | /** 177 | * \brief processing logic of Barrier message (run on each node) 178 | */ 179 | void ProcessBarrierCommand(Message *msg); 180 | 181 | /** 182 | * \brief processing logic of AddNode message (run on each node) 183 | */ 184 | void ProcessHearbeat(Message *msg); 185 | 186 | /** 187 | * \brief processing logic of Data message 188 | */ 189 | void ProcessDataMsg(Message *msg); 190 | 191 | /** 192 | * \brief called by ProcessAddNodeCommand, in scheduler it assigns an id to 193 | * the newly added node; in other nodes, it updates the node id with what is 194 | * received from scheduler 195 | */ 196 | void UpdateLocalID(Message *msg, std::unordered_set *deadnodes_set, 197 | Meta *nodes, Meta *recovery_nodes); 198 | 199 | const char *heartbeat_timeout_val = 200 | Environment::Get()->find("PS_HEARTBEAT_TIMEOUT"); 201 | int heartbeat_timeout_ = 202 | heartbeat_timeout_val ? atoi(heartbeat_timeout_val) : 0; 203 | 204 | DISALLOW_COPY_AND_ASSIGN(Van); 205 | }; 206 | } // namespace ps 207 | #endif // PS_INTERNAL_VAN_H_ 208 | -------------------------------------------------------------------------------- /include/ps/ps.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2015 by Contributors 3 | * @file ps.h 4 | * \brief The parameter server interface 5 | */ 6 | #ifndef PS_PS_H_ 7 | #define PS_PS_H_ 8 | /** \brief basic setups in ps */ 9 | #include "ps/base.h" 10 | /** \brief communicating with a pair of (int, string). */ 11 | #include "ps/simple_app.h" 12 | /** \brief communcating with a list of key-value paris. */ 13 | #include "ps/kv_app.h" 14 | namespace ps { 15 | /** \brief Returns the number of worker nodes */ 16 | inline int NumWorkers() { return Postoffice::Get()->num_workers(); } 17 | /** \brief Returns the number of server nodes */ 18 | inline int NumServers() { return Postoffice::Get()->num_servers(); } 19 | /** \brief Returns true if this node is a worker node */ 20 | inline bool IsWorker() { return Postoffice::Get()->is_worker(); } 21 | /** \brief Returns true if this node is a server node. */ 22 | inline bool IsServer() { return Postoffice::Get()->is_server(); } 23 | /** \brief Returns true if this node is a scheduler node. */ 24 | inline bool IsScheduler() { return Postoffice::Get()->is_scheduler(); } 25 | /** \brief Returns the rank of this node in its group 26 | * 27 | * Each worker will have a unique rank within [0, NumWorkers()). So are 28 | * servers. This function is available only after \ref Start has been called. 29 | */ 30 | inline int MyRank() { return Postoffice::Get()->my_rank(); } 31 | /** 32 | * \brief start the system 33 | * 34 | * This function will block until every nodes are started. 35 | * \param argv0 the program name, used for logging 36 | */ 37 | inline void Start(int customer_id, const char* argv0 = nullptr) { 38 | Postoffice::Get()->Start(customer_id, argv0, true); 39 | } 40 | /** 41 | * \brief start the system 42 | * 43 | * This function will NOT block. 44 | * \param argv0 the program name, used for logging 45 | */ 46 | inline void StartAsync(int customer_id, const char* argv0 = nullptr) { 47 | Postoffice::Get()->Start(customer_id, argv0, false); 48 | } 49 | /** 50 | * \brief terminate the system 51 | * 52 | * All nodes should call this function before existing. 53 | * \param do_barrier whether to block until every node is finalized, default true. 54 | */ 55 | inline void Finalize(int customer_id, const bool do_barrier = true) { 56 | Postoffice::Get()->Finalize(customer_id, do_barrier); 57 | } 58 | /** 59 | * \brief Register a callback to the system which is called after Finalize() 60 | * 61 | * The following codes are equal 62 | * \code {cpp} 63 | * RegisterExitCallback(cb); 64 | * Finalize(); 65 | * \endcode 66 | * 67 | * \code {cpp} 68 | * Finalize(); 69 | * cb(); 70 | * \endcode 71 | * \param cb the callback function 72 | */ 73 | inline void RegisterExitCallback(const std::function& cb) { 74 | Postoffice::Get()->RegisterExitCallback(cb); 75 | } 76 | 77 | } // namespace ps 78 | #endif // PS_PS_H_ 79 | -------------------------------------------------------------------------------- /include/ps/range.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_RANGE_H_ 5 | #define PS_RANGE_H_ 6 | #include "ps/internal/utils.h" 7 | namespace ps { 8 | 9 | /** 10 | * \brief a range [begin, end) 11 | */ 12 | class Range { 13 | public: 14 | Range() : Range(0, 0) {} 15 | Range(uint64_t begin, uint64_t end) : begin_(begin), end_(end) { } 16 | 17 | uint64_t begin() const { return begin_; } 18 | uint64_t end() const { return end_; } 19 | uint64_t size() const { return end_ - begin_; } 20 | private: 21 | uint64_t begin_; 22 | uint64_t end_; 23 | }; 24 | 25 | } // namespace ps 26 | #endif // PS_RANGE_H_ 27 | -------------------------------------------------------------------------------- /include/ps/sarray.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_SARRAY_H_ 5 | #define PS_SARRAY_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "ps/internal/utils.h" 12 | #include "ps/range.h" 13 | namespace ps { 14 | 15 | /** 16 | * \brief Shared array 17 | * 18 | * A smart array that retains shared ownership. It provides similar 19 | * functionalities comparing to std::vector, including data(), size(), 20 | * operator[], resize(), clear(). SArray can be easily constructed from 21 | * std::vector, such as 22 | * 23 | * \code 24 | * std::vector a(10); SArray b(a); // copying 25 | * std::shared_ptr> c(new std::vector(10)); 26 | * SArray d(c); // only pointer copying 27 | * \endcode 28 | * 29 | * SArray is also like a C pointer when copying and assigning, namely 30 | * both copy are assign are passing by pointers. The memory will be release only 31 | * if there is no copy exists. It is also can be cast without memory copy, such as 32 | * 33 | * \code 34 | * SArray a(10); 35 | * SArray b(a); // now b.size() = 10 * sizeof(int); 36 | * \endcode 37 | * 38 | * \tparam V the value type 39 | */ 40 | template 41 | class SArray { 42 | public: 43 | /** \brief empty constructor */ 44 | SArray() { } 45 | 46 | /** \brief empty deconstrcutor */ 47 | ~SArray() { } 48 | 49 | /** 50 | * \brief Create an array with length n with initialized value 51 | * \param size the length 52 | * \param val the initial length (0 in default) 53 | */ 54 | explicit SArray(size_t size, V val = 0) { resize(size, val); } 55 | 56 | 57 | /** 58 | * \brief construct from another SArray. 59 | * 60 | * Zero-copy constructor, namely just copy the pointer 61 | * 62 | * \tparam W the value type of the source array 63 | * \param arr the source array 64 | */ 65 | template 66 | explicit SArray(const SArray& arr) { *this = arr; } 67 | 68 | /** 69 | * \brief construct from another SArray. 70 | * 71 | * Zero-copy constructor, namely just copy the pointer 72 | * 73 | * \tparam W the value type of the source array 74 | * \param arr the source array 75 | */ 76 | template void operator=(const SArray& arr) { 77 | size_ = arr.size() * sizeof(W) / sizeof(V); 78 | CHECK_EQ(size_ * sizeof(V), arr.size() * sizeof(W)) << "cannot be divided"; 79 | capacity_ = arr.capacity() * sizeof(W) / sizeof(V); 80 | ptr_ = std::shared_ptr(arr.ptr(), reinterpret_cast(arr.data())); 81 | } 82 | 83 | /** 84 | * \brief construct from a c-array 85 | * 86 | * Zero-copy constructor, namely just copy the pointer 87 | * 88 | * \param data the source data 89 | * \param size the length 90 | * \param deletable whether or not can call `delete [] data` when the reference 91 | * count goes 0 92 | */ 93 | 94 | SArray(V* data, size_t size, bool deletable = false) { 95 | if (deletable) { 96 | reset(data, size, [](V* data){ delete [] data; }); 97 | } else { 98 | reset(data, size, [](V* data) { }); 99 | } 100 | } 101 | 102 | /** 103 | * \brief copy from a c-array 104 | * 105 | * \param data the source data 106 | * \param size the length 107 | */ 108 | void CopyFrom(const V* data, size_t size) { 109 | resize(size); 110 | memcpy(this->data(), data, size*sizeof(V)); 111 | } 112 | 113 | /** 114 | * \brief copy from another SArray 115 | * 116 | * \param other the source data 117 | */ 118 | void CopyFrom(const SArray& other) { 119 | if (this == &other) return; 120 | CopyFrom(other.data(), other.size()); 121 | } 122 | 123 | /** 124 | * \brief copy from an iterator 125 | */ 126 | template 127 | void CopyFrom(const ForwardIt& first, const ForwardIt& last) { 128 | int size = static_cast(std::distance(first, last)); 129 | V* data = new V[size]; 130 | reset(data, size, [](V* data){ delete [] data; }); 131 | auto it = first; 132 | while (size-- > 0) { *data = *it; ++data; ++it; } 133 | } 134 | 135 | /** 136 | * \brief construct from a std::vector, copy the data 137 | */ 138 | explicit SArray(const std::vector& vec) { CopyFrom(vec.data(), vec.size()); } 139 | 140 | /** 141 | * \brief construct from a shared std::vector pinter, no data copy 142 | */ 143 | explicit SArray(const std::shared_ptr>& vec) { 144 | ptr_ = std::shared_ptr(vec, vec->data()); 145 | size_ = vec->size(); 146 | capacity_ = size_; 147 | } 148 | 149 | /** @brief Copy from a initializer_list */ 150 | template SArray(const std::initializer_list& list) { 151 | CopyFrom(list.begin(), list.end()); 152 | } 153 | 154 | /** @brief Copy from a initializer_list */ 155 | template void operator=(const std::initializer_list& list) { 156 | CopyFrom(list.begin(), list.end()); 157 | } 158 | 159 | /** 160 | * @brief Reset the current data pointer with a deleter 161 | */ 162 | template 163 | void reset(V* data, size_t size, Deleter del) { 164 | size_ = size; capacity_ = size; ptr_.reset(data, del); 165 | } 166 | 167 | /** 168 | * @brief Resizes the array to size elements 169 | * 170 | * If size <= capacity_, then only change the size. otherwise, append size - 171 | * current_size entries, and then set new value to val 172 | */ 173 | void resize(size_t size, V val = 0) { 174 | size_t cur_n = size_; 175 | if (capacity_ >= size) { 176 | size_ = size; 177 | } else { 178 | V* new_data = new V[size+5]; 179 | memcpy(new_data, data(), size_*sizeof(V)); 180 | reset(new_data, size, [](V* data){ delete [] data; }); 181 | } 182 | if (size <= cur_n) return; 183 | V* p = data() + cur_n; 184 | if (val == 0) { 185 | memset(p, 0, (size - cur_n)*sizeof(V)); 186 | } else { 187 | for (size_t i = 0; i < size - cur_n; ++i) { *p = val; ++p; } 188 | } 189 | } 190 | 191 | /** 192 | * @brief Requests that the capacity be at least enough to contain n elements. 193 | */ 194 | void reserve(size_t size) { 195 | if (capacity_ >= size) { return; } 196 | size_t old_size = size_; 197 | resize(size); 198 | size_ = old_size; 199 | } 200 | 201 | /** @brief release the memory */ 202 | void clear() { reset(nullptr, 0, [](V* data) {}); } 203 | 204 | 205 | inline bool empty() const { return size() == 0; } 206 | inline size_t size() const { return size_; } 207 | inline size_t capacity() const { return capacity_; } 208 | 209 | inline V* begin() { return data(); } 210 | inline const V* begin() const { return data(); } 211 | inline V* end() { return data() + size(); } 212 | inline const V* end() const { return data() + size(); } 213 | 214 | inline V* data() const { return ptr_.get(); } 215 | 216 | /** \brief get the shared pointer */ 217 | inline std::shared_ptr& ptr() { return ptr_; } 218 | /** \brief get the const shared pointer */ 219 | inline const std::shared_ptr& ptr() const { return ptr_; } 220 | 221 | inline V back() const { CHECK(!empty()); return data()[size_-1]; } 222 | inline V front() const { CHECK(!empty()); return data()[0]; } 223 | inline V& operator[] (int i) { return data()[i]; } 224 | inline const V& operator[] (int i) const { return data()[i]; } 225 | 226 | inline void push_back(const V& val) { 227 | if (size_ == capacity_) reserve(size_*2+5); 228 | data()[size_++] = val; 229 | } 230 | 231 | void pop_back() { if (size_) --size_; } 232 | 233 | void append(const SArray& arr) { 234 | if (arr.empty()) return; 235 | auto orig_size = size_; 236 | resize(size_ + arr.size()); 237 | memcpy(data()+orig_size, arr.data(), arr.size()*sizeof(V)); 238 | } 239 | 240 | 241 | /** 242 | * @brief Slice a segment, zero-copy 243 | * 244 | * @param begin the start index segment 245 | * @param end the end index segment 246 | * @return the segment [begin, end) 247 | */ 248 | SArray segment(size_t begin, size_t end) const { 249 | CHECK_GE(end, begin); CHECK_LE(end, size()); 250 | SArray ret; 251 | ret.ptr_ = std::shared_ptr(ptr_, data() + begin); 252 | ret.size_ = end - begin; 253 | ret.capacity_ = end - begin; 254 | return ret; 255 | } 256 | 257 | private: 258 | size_t size_ = 0; 259 | size_t capacity_ = 0; 260 | std::shared_ptr ptr_; 261 | }; 262 | 263 | 264 | /** 265 | * \brief Find the index range of a segment of a sorted array such that the 266 | * entries in this segment is within [lower, upper). Assume 267 | * array values are ordered. 268 | * 269 | * An example 270 | * \code{cpp} 271 | * SArray a{1 3 5 7 9}; 272 | * CHECK_EQ(Range(1,3), FindRange(a, 2, 7); 273 | * \endcode 274 | * 275 | * \param arr the source array 276 | * \param lower the lower bound 277 | * \param upper the upper bound 278 | * 279 | * \return the index range 280 | */ 281 | template 282 | Range FindRange(const SArray& arr, V lower, V upper) { 283 | if (upper <= lower) return Range(0, 0); 284 | auto lb = std::lower_bound(arr.begin(), arr.end(), lower); 285 | auto ub = std::lower_bound(arr.begin(), arr.end(), upper); 286 | return Range(lb - arr.begin(), ub - arr.begin()); 287 | } 288 | 289 | 290 | /*! \brief returns a short debug string */ 291 | template 292 | inline std::string DebugStr(const V* data, int n, int m = 5) { 293 | std::stringstream ss; 294 | ss << "[" << n << "]: "; 295 | if (n < 2 * m) { 296 | for (int i = 0; i < n; ++i) ss << data[i] << " "; 297 | } else { 298 | for (int i = 0; i < m; ++i) ss << data[i] << " "; 299 | ss << "... "; 300 | for (int i = n-m; i < n; ++i) ss << data[i] << " "; 301 | } 302 | return ss.str(); 303 | } 304 | 305 | /** 306 | * \brief print a debug string 307 | */ 308 | template 309 | std::ostream& operator<<(std::ostream& os, const SArray& obj) { 310 | os << DebugStr(obj.data(), obj.size()); 311 | return os; 312 | } 313 | 314 | } // namespace ps 315 | #endif // PS_SARRAY_H_ 316 | -------------------------------------------------------------------------------- /include/ps/simple_app.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_SIMPLE_APP_H_ 5 | #define PS_SIMPLE_APP_H_ 6 | #include 7 | #include "ps/internal/message.h" 8 | #include "ps/internal/postoffice.h" 9 | namespace ps { 10 | 11 | /** 12 | * \brief the format for a received request or reponse for \ref SimpleApp 13 | */ 14 | struct SimpleData { 15 | /** \brief the int head */ 16 | int head; 17 | /** \brief the string body */ 18 | std::string body; 19 | /** \brief sender's node id */ 20 | int sender; 21 | /** \brief the associated timestamp */ 22 | int timestamp; 23 | /** \brief sender's customer id */ 24 | int customer_id; 25 | }; 26 | 27 | /** 28 | * \brief a simple app 29 | * 30 | * It provides basic communcation with a pair of int (head) and string (body) 31 | */ 32 | class SimpleApp { 33 | public: 34 | /** 35 | * \brief constructor 36 | * @param app_id the app id, should match with the remote node app with which this app 37 | * @param customer_id the customer_id, should be node-locally unique 38 | * is communicated 39 | */ 40 | explicit SimpleApp(int app_id, int customer_id); 41 | 42 | /** \brief deconstructor */ 43 | virtual ~SimpleApp() { delete obj_; obj_ = nullptr; } 44 | 45 | /** 46 | * \brief send a request to a remote node 47 | * 48 | * \param req_head request head 49 | * \param req_body request body 50 | * \param recv_id remote node id 51 | * 52 | * @return the timestamp of this request 53 | */ 54 | virtual inline int Request(int req_head, const std::string& req_body, int recv_id); 55 | 56 | /** 57 | * \brief wait until a request is finished 58 | * 59 | * \param timestamp 60 | */ 61 | virtual inline void Wait(int timestamp) { obj_->WaitRequest(timestamp); } 62 | 63 | 64 | /** 65 | * \brief send back a response for a request 66 | * \param recv_req the received request 67 | * \param the response body 68 | */ 69 | virtual inline void Response(const SimpleData& recv_req, const std::string& res_body = ""); 70 | 71 | /** 72 | * \brief the handle to proces a received request/respoonse 73 | * 74 | * \param recved the received request or response 75 | * \param app this pointer 76 | */ 77 | using Handle = std::function; 78 | 79 | /** 80 | * \brief set the request handle 81 | * \param request_handle the request handle 82 | */ 83 | virtual inline void set_request_handle(const Handle& request_handle) { 84 | CHECK(request_handle) << "invalid request handle"; 85 | request_handle_ = request_handle; 86 | } 87 | 88 | /** 89 | * \brief set the response handle 90 | * \param response_handle the response handle 91 | */ 92 | virtual inline void set_response_handle(const Handle& response_handle) { 93 | CHECK(response_handle) << "invalid response handle"; 94 | response_handle_ = response_handle; 95 | } 96 | 97 | /** 98 | * \brief returns the customer 99 | */ 100 | virtual inline Customer* get_customer() { return obj_; } 101 | 102 | protected: 103 | /** \brief empty construct */ 104 | inline SimpleApp() : obj_(nullptr) { 105 | request_handle_ = [](const SimpleData& recved, SimpleApp* app) { 106 | app->Response(recved); 107 | }; 108 | response_handle_ = [](const SimpleData& recved, SimpleApp* app) { }; 109 | } 110 | 111 | /** \brief process a received message */ 112 | virtual inline void Process(const Message& msg); 113 | 114 | /** \brief ps internal object */ 115 | Customer* obj_; 116 | 117 | private: 118 | /** \brief request handle */ 119 | Handle request_handle_; 120 | /** \brief request handle */ 121 | Handle response_handle_; 122 | }; 123 | 124 | //////////////////////////////////////////////////////////////////////////////// 125 | 126 | inline SimpleApp::SimpleApp(int app_id, int customer_id) : SimpleApp() { 127 | using namespace std::placeholders; 128 | obj_ = new Customer(app_id, customer_id, std::bind(&SimpleApp::Process, this, _1)); 129 | } 130 | 131 | inline int SimpleApp::Request(int req_head, const std::string& req_body, int recv_id) { 132 | // setup message 133 | Message msg; 134 | msg.meta.head = req_head; 135 | if (req_body.size()) msg.meta.body = req_body; 136 | int ts = obj_->NewRequest(recv_id); 137 | msg.meta.timestamp = ts; 138 | msg.meta.request = true; 139 | msg.meta.simple_app = true; 140 | msg.meta.app_id = obj_->app_id(); 141 | msg.meta.customer_id = obj_->customer_id(); 142 | 143 | // send 144 | for (int r : Postoffice::Get()->GetNodeIDs(recv_id)) { 145 | msg.meta.recver = r; 146 | Postoffice::Get()->van()->Send(msg); 147 | } 148 | return ts; 149 | } 150 | 151 | inline void SimpleApp::Response(const SimpleData& req, const std::string& res_body) { 152 | // setup message 153 | Message msg; 154 | msg.meta.head = req.head; 155 | if (res_body.size()) msg.meta.body = res_body; 156 | msg.meta.timestamp = req.timestamp; 157 | msg.meta.request = false; 158 | msg.meta.simple_app = true; 159 | msg.meta.app_id = obj_->app_id(); 160 | msg.meta.customer_id = req.customer_id; 161 | msg.meta.recver = req.sender; 162 | 163 | // send 164 | Postoffice::Get()->van()->Send(msg); 165 | } 166 | 167 | 168 | inline void SimpleApp::Process(const Message& msg) { 169 | SimpleData recv; 170 | recv.sender = msg.meta.sender; 171 | recv.head = msg.meta.head; 172 | recv.body = msg.meta.body; 173 | recv.timestamp = msg.meta.timestamp; 174 | recv.customer_id = msg.meta.customer_id; 175 | if (msg.meta.request) { 176 | CHECK(request_handle_); 177 | request_handle_(recv, this); 178 | } else { 179 | CHECK(response_handle_); 180 | response_handle_(recv, this); 181 | } 182 | } 183 | 184 | } // namespace ps 185 | #endif // PS_SIMPLE_APP_H_ 186 | -------------------------------------------------------------------------------- /make/deps.mk: -------------------------------------------------------------------------------- 1 | # Install dependencies 2 | 3 | URL1=https://raw.githubusercontent.com/mli/deps/master/build 4 | URL2=https://github.com/google/protobuf/releases/download/v3.5.1 5 | ifndef WGET 6 | WGET = wget 7 | endif 8 | 9 | # protobuf 10 | PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h 11 | ${PROTOBUF}: 12 | $(eval FILE=protobuf-cpp-3.5.1.tar.gz) 13 | $(eval DIR=protobuf-3.5.1) 14 | rm -rf $(FILE) $(DIR) 15 | $(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE) 16 | cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install 17 | rm -rf $(FILE) $(DIR) 18 | 19 | # zmq 20 | ZMQ = ${DEPS_PATH}/include/zmq.h 21 | 22 | ${ZMQ}: 23 | $(eval FILE=zeromq-4.1.4.tar.gz) 24 | $(eval DIR=zeromq-4.1.4) 25 | rm -rf $(FILE) $(DIR) 26 | $(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) 27 | cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install 28 | rm -rf $(FILE) $(DIR) 29 | 30 | # lz4 31 | LZ4 = ${DEPS_PATH}/include/lz4.h 32 | ${LZ4}: 33 | $(eval FILE=lz4-r129.tar.gz) 34 | $(eval DIR=lz4-r129) 35 | rm -rf $(FILE) $(DIR) 36 | wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) 37 | cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install 38 | rm -rf $(FILE) $(DIR) 39 | 40 | # cityhash 41 | CITYHASH = ${DEPS_PATH}/include/city.h 42 | ${CITYHASH}: 43 | $(eval FILE=cityhash-1.1.1.tar.gz) 44 | $(eval DIR=cityhash-1.1.1) 45 | rm -rf $(FILE) $(DIR) 46 | wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) 47 | cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install 48 | rm -rf $(FILE) $(DIR) 49 | 50 | 51 | # # gflags 52 | # ${DEPS_PATH}/include/google/gflags.h: 53 | # $(eval FILE=gflags-2.0-no-svn-files.tar.gz) 54 | # $(eval DIR=gflags-2.0) 55 | # rm -rf $(FILE) $(DIR) 56 | # wget $(URL)/$(FILE) && tar -zxf $(FILE) 57 | # cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install 58 | # rm -rf $(FILE) $(DIR) 59 | # gflags: | ${DEPS_PATH}/include/google/gflags.h 60 | 61 | # # glog 62 | # ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h 63 | # $(eval FILE=v0.3.4.tar.gz) 64 | # $(eval DIR=glog-0.3.4) 65 | # rm -rf $(FILE) $(DIR) 66 | # wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE) 67 | # cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install 68 | # rm -rf $(FILE) $(DIR) 69 | # glog: | ${DEPS_PATH}/include/glog/logging.h 70 | -------------------------------------------------------------------------------- /make/ps.mk: -------------------------------------------------------------------------------- 1 | #--------------------------------------------------------------------------------------- 2 | # parameter server configuration script 3 | # 4 | # include ps.mk after the variables are set 5 | # 6 | #---------------------------------------------------------------------------------------- 7 | 8 | ifeq ($(USE_KEY32), 1) 9 | ADD_CFLAGS += -DUSE_KEY32=1 10 | endif 11 | 12 | PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq 13 | PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a) 14 | -------------------------------------------------------------------------------- /src/customer.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #include "ps/internal/customer.h" 5 | #include "ps/internal/postoffice.h" 6 | namespace ps { 7 | 8 | const int Node::kEmpty = std::numeric_limits::max(); 9 | const int Meta::kEmpty = std::numeric_limits::max(); 10 | 11 | Customer::Customer(int app_id, int customer_id, const Customer::RecvHandle& recv_handle) 12 | : app_id_(app_id), customer_id_(customer_id), recv_handle_(recv_handle) { 13 | Postoffice::Get()->AddCustomer(this); 14 | recv_thread_ = std::unique_ptr(new std::thread(&Customer::Receiving, this)); 15 | } 16 | 17 | Customer::~Customer() { 18 | Postoffice::Get()->RemoveCustomer(this); 19 | Message msg; 20 | msg.meta.control.cmd = Control::TERMINATE; 21 | recv_queue_.Push(msg); 22 | recv_thread_->join(); 23 | } 24 | 25 | int Customer::NewRequest(int recver) { 26 | std::lock_guard lk(tracker_mu_); 27 | int num = Postoffice::Get()->GetNodeIDs(recver).size(); 28 | tracker_.push_back(std::make_pair(num, 0)); 29 | return tracker_.size() - 1; 30 | } 31 | 32 | void Customer::WaitRequest(int timestamp) { 33 | std::unique_lock lk(tracker_mu_); 34 | tracker_cond_.wait(lk, [this, timestamp]{ 35 | return tracker_[timestamp].first == tracker_[timestamp].second; 36 | }); 37 | } 38 | 39 | int Customer::NumResponse(int timestamp) { 40 | std::lock_guard lk(tracker_mu_); 41 | return tracker_[timestamp].second; 42 | } 43 | 44 | void Customer::AddResponse(int timestamp, int num) { 45 | std::lock_guard lk(tracker_mu_); 46 | tracker_[timestamp].second += num; 47 | } 48 | 49 | void Customer::Receiving() { 50 | while (true) { 51 | Message recv; 52 | recv_queue_.WaitAndPop(&recv); 53 | if (!recv.meta.control.empty() && 54 | recv.meta.control.cmd == Control::TERMINATE) { 55 | break; 56 | } 57 | recv_handle_(recv); 58 | if (!recv.meta.request) { 59 | std::lock_guard lk(tracker_mu_); 60 | tracker_[recv.meta.timestamp].second++; 61 | tracker_cond_.notify_all(); 62 | } 63 | } 64 | } 65 | 66 | } // namespace ps 67 | -------------------------------------------------------------------------------- /src/meta.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | syntax = "proto2"; 5 | package ps; 6 | option optimize_for = LITE_RUNTIME; 7 | 8 | message PBNode { 9 | // the node role 10 | required int32 role = 1; 11 | // node id 12 | optional int32 id = 2; 13 | // hostname or ip 14 | optional string hostname = 3; 15 | // the port this node is binding 16 | optional int32 port = 4; 17 | // whether this node is created by failover 18 | optional bool is_recovery = 5; 19 | // the locally unique id of an customer 20 | optional int32 customer_id = 10; 21 | } 22 | 23 | // system control info 24 | message PBControl { 25 | required int32 cmd = 1; 26 | repeated PBNode node = 2; 27 | optional int32 barrier_group = 3; 28 | optional uint64 msg_sig = 4; 29 | } 30 | 31 | // mete information about a message 32 | message PBMeta { 33 | // message.head 34 | optional int32 head = 1; 35 | // message.body 36 | optional bytes body = 2; 37 | // if set, then it is system control task. otherwise, it is for app 38 | optional PBControl control = 3; 39 | // true: a request task 40 | // false: the response task to the request task with the same *time* 41 | optional bool request = 4 [default = false]; 42 | // the unique id of an application 43 | optional int32 app_id = 7; 44 | // the timestamp of this message 45 | optional int32 timestamp = 8; 46 | // data type of message.data[i] 47 | repeated int32 data_type = 9 [packed=true]; 48 | // the locally unique id of an customer 49 | optional int32 customer_id = 10; 50 | // whether or not a push message 51 | optional bool push = 5; 52 | // whether or not a pull message 53 | optional bool pull = 12; 54 | // whether or not it's for SimpleApp 55 | optional bool simple_app = 6 [default = false]; 56 | // message.data_size 57 | optional int32 data_size = 11; 58 | // priority 59 | optional int32 priority = 13 [default = 0]; 60 | } 61 | -------------------------------------------------------------------------------- /src/network_utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | * @file network_utils.h 4 | * @brief network utilities 5 | */ 6 | #ifndef PS_NETWORK_UTILS_H_ 7 | #define PS_NETWORK_UTILS_H_ 8 | #include 9 | #ifdef _MSC_VER 10 | #include 11 | #include 12 | #include 13 | #include 14 | #undef interface 15 | #else 16 | #include 17 | #include 18 | #include 19 | #include 20 | #endif 21 | #include 22 | 23 | namespace ps { 24 | 25 | /** 26 | * \brief return the IP address for given interface eth0, eth1, ... 27 | */ 28 | void GetIP(const std::string& interface, std::string* ip) { 29 | #ifdef _MSC_VER 30 | typedef std::basic_string tstring; 31 | // Try to get the Adapters-info table, so we can given useful names to the IP 32 | // addresses we are returning. Gotta call GetAdaptersInfo() up to 5 times to handle 33 | // the potential race condition between the size-query call and the get-data call. 34 | // I love a well-designed API :^P 35 | IP_ADAPTER_INFO * pAdapterInfo = NULL; 36 | { 37 | ULONG bufLen = 0; 38 | for (int i = 0; i < 5; i++) { 39 | DWORD apRet = GetAdaptersInfo(pAdapterInfo, &bufLen); 40 | if (apRet == ERROR_BUFFER_OVERFLOW) { 41 | free(pAdapterInfo); // in case we had previously allocated it 42 | pAdapterInfo = static_cast(malloc(bufLen)); 43 | } else if (apRet == ERROR_SUCCESS) { 44 | break; 45 | } else { 46 | free(pAdapterInfo); 47 | pAdapterInfo = NULL; 48 | break; 49 | } 50 | } 51 | } 52 | if (pAdapterInfo) { 53 | tstring keybase = _T( 54 | "SYSTEM\\CurrentControlSet\\Control\\Network\\{4D36E972-E325-11CE-BFC1-08002BE10318}\\"); 55 | tstring connection = _T("\\Connection"); 56 | 57 | IP_ADAPTER_INFO *curpAdapterInfo = pAdapterInfo; 58 | while (curpAdapterInfo) { 59 | HKEY hKEY; 60 | std::string AdapterName = curpAdapterInfo->AdapterName; 61 | // GUID only ascii 62 | tstring key_set = keybase + tstring(AdapterName.begin(), AdapterName.end()) + connection; 63 | LPCTSTR data_Set = key_set.c_str(); 64 | LPCTSTR dwValue = NULL; 65 | if (ERROR_SUCCESS == 66 | ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, data_Set, 0, KEY_READ, &hKEY)) { 67 | DWORD dwSize = 0; 68 | DWORD dwType = REG_SZ; 69 | if (ERROR_SUCCESS == 70 | ::RegQueryValueEx(hKEY, _T("Name"), 0, &dwType, (LPBYTE)dwValue, &dwSize)) { 71 | dwValue = new TCHAR[dwSize]; 72 | if (ERROR_SUCCESS == 73 | ::RegQueryValueEx(hKEY, _T("Name"), 0, &dwType, (LPBYTE)dwValue, &dwSize)) { 74 | // interface name must only ascii 75 | tstring tstr = dwValue; 76 | std::string s(tstr.begin(), tstr.end()); 77 | if (s == interface) { 78 | *ip = curpAdapterInfo->IpAddressList.IpAddress.String; 79 | break; 80 | } 81 | } 82 | } 83 | ::RegCloseKey(hKEY); 84 | } 85 | curpAdapterInfo = curpAdapterInfo->Next; 86 | } 87 | free(pAdapterInfo); 88 | } 89 | #else 90 | struct ifaddrs * ifAddrStruct = NULL; 91 | struct ifaddrs * ifa = NULL; 92 | void * tmpAddrPtr = NULL; 93 | 94 | getifaddrs(&ifAddrStruct); 95 | for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) { 96 | if (ifa->ifa_addr == NULL) continue; 97 | if (ifa->ifa_addr->sa_family == AF_INET) { 98 | // is a valid IP4 Address 99 | tmpAddrPtr = &(reinterpret_cast(ifa->ifa_addr))->sin_addr; 100 | char addressBuffer[INET_ADDRSTRLEN]; 101 | inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN); 102 | if (strncmp(ifa->ifa_name, 103 | interface.c_str(), 104 | interface.size()) == 0) { 105 | *ip = addressBuffer; 106 | break; 107 | } 108 | } 109 | } 110 | if (ifAddrStruct != NULL) freeifaddrs(ifAddrStruct); 111 | #endif 112 | } 113 | 114 | 115 | /** 116 | * \brief return the IP address and Interface the first interface which is not 117 | * loopback 118 | * 119 | * only support IPv4 120 | */ 121 | void GetAvailableInterfaceAndIP( 122 | std::string* interface, std::string* ip) { 123 | #ifdef _MSC_VER 124 | typedef std::basic_string tstring; 125 | IP_ADAPTER_INFO * pAdapterInfo = NULL; 126 | { 127 | ULONG bufLen = 0; 128 | for (int i = 0; i < 5; i++) { 129 | DWORD apRet = GetAdaptersInfo(pAdapterInfo, &bufLen); 130 | if (apRet == ERROR_BUFFER_OVERFLOW) { 131 | free(pAdapterInfo); // in case we had previously allocated it 132 | pAdapterInfo = static_cast(malloc(bufLen)); 133 | } else if (apRet == ERROR_SUCCESS) { 134 | break; 135 | } else { 136 | free(pAdapterInfo); 137 | pAdapterInfo = NULL; 138 | break; 139 | } 140 | } 141 | } 142 | if (pAdapterInfo) { 143 | tstring keybase = _T( 144 | "SYSTEM\\CurrentControlSet\\Control\\Network\\{4D36E972-E325-11CE-BFC1-08002BE10318}\\"); 145 | tstring connection = _T("\\Connection"); 146 | 147 | IP_ADAPTER_INFO *curpAdapterInfo = pAdapterInfo; 148 | HKEY hKEY = NULL; 149 | while (curpAdapterInfo) { 150 | std::string curip(curpAdapterInfo->IpAddressList.IpAddress.String); 151 | if (curip == "127.0.0.1") { 152 | curpAdapterInfo = curpAdapterInfo->Next; 153 | continue; 154 | } 155 | if (curip == "0.0.0.0") { 156 | curpAdapterInfo = curpAdapterInfo->Next; 157 | continue; 158 | } 159 | 160 | std::string AdapterName = curpAdapterInfo->AdapterName; 161 | // GUID only ascii 162 | tstring key_set = keybase + tstring(AdapterName.begin(), AdapterName.end()) + connection; 163 | LPCTSTR data_Set = key_set.c_str(); 164 | LPCTSTR dwValue = NULL; 165 | if (ERROR_SUCCESS == 166 | ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, data_Set, 0, KEY_READ, &hKEY)) { 167 | DWORD dwSize = 0; 168 | DWORD dwType = REG_SZ; 169 | if (ERROR_SUCCESS == 170 | ::RegQueryValueEx(hKEY, _T("Name"), 0, &dwType, (LPBYTE)dwValue, &dwSize)) { 171 | dwValue = new TCHAR[dwSize]; 172 | if (ERROR_SUCCESS == 173 | ::RegQueryValueEx(hKEY, _T("Name"), 0, &dwType, (LPBYTE)dwValue, &dwSize)) { 174 | // interface name must only ascii 175 | tstring tstr = dwValue; 176 | std::string s(tstr.begin(), tstr.end()); 177 | 178 | *interface = s; 179 | *ip = curip; 180 | break; 181 | } 182 | } 183 | ::RegCloseKey(hKEY); 184 | hKEY = NULL; 185 | } 186 | curpAdapterInfo = curpAdapterInfo->Next; 187 | } 188 | if (hKEY != NULL) { 189 | ::RegCloseKey(hKEY); 190 | } 191 | free(pAdapterInfo); 192 | } 193 | #else 194 | struct ifaddrs * ifAddrStruct = nullptr; 195 | struct ifaddrs * ifa = nullptr; 196 | 197 | interface->clear(); 198 | ip->clear(); 199 | getifaddrs(&ifAddrStruct); 200 | for (ifa = ifAddrStruct; ifa != nullptr; ifa = ifa->ifa_next) { 201 | if (nullptr == ifa->ifa_addr) continue; 202 | 203 | if (AF_INET == ifa->ifa_addr->sa_family && 204 | 0 == (ifa->ifa_flags & IFF_LOOPBACK)) { 205 | char address_buffer[INET_ADDRSTRLEN]; 206 | void* sin_addr_ptr = &(reinterpret_cast(ifa->ifa_addr))->sin_addr; 207 | inet_ntop(AF_INET, sin_addr_ptr, address_buffer, INET_ADDRSTRLEN); 208 | 209 | *ip = address_buffer; 210 | *interface = ifa->ifa_name; 211 | 212 | break; 213 | } 214 | } 215 | if (nullptr != ifAddrStruct) freeifaddrs(ifAddrStruct); 216 | return; 217 | #endif 218 | } 219 | 220 | /** 221 | * \brief return an available port on local machine 222 | * 223 | * only support IPv4 224 | * \return 0 on failure 225 | */ 226 | int GetAvailablePort() { 227 | struct sockaddr_in addr; 228 | addr.sin_port = htons(0); // have system pick up a random port available for me 229 | addr.sin_family = AF_INET; // IPV4 230 | addr.sin_addr.s_addr = htonl(INADDR_ANY); // set our addr to any interface 231 | 232 | int sock = socket(AF_INET, SOCK_STREAM, 0); 233 | if (0 != bind(sock, (struct sockaddr*)&addr, sizeof(struct sockaddr_in))) { 234 | perror("bind():"); 235 | return 0; 236 | } 237 | #ifdef _MSC_VER 238 | int addr_len = sizeof(struct sockaddr_in); 239 | #else 240 | socklen_t addr_len = sizeof(struct sockaddr_in); 241 | #endif 242 | 243 | if (0 != getsockname(sock, (struct sockaddr*)&addr, &addr_len)) { 244 | perror("getsockname():"); 245 | return 0; 246 | } 247 | 248 | int ret_port = ntohs(addr.sin_port); 249 | #ifdef _MSC_VER 250 | closesocket(sock); 251 | #else 252 | close(sock); 253 | #endif 254 | return ret_port; 255 | } 256 | 257 | } // namespace ps 258 | #endif // PS_NETWORK_UTILS_H_ 259 | -------------------------------------------------------------------------------- /src/p3_van.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_P3_VAN_H_ 5 | #define PS_P3_VAN_H_ 6 | #include 7 | namespace ps { 8 | 9 | /** 10 | * \brief P3 based Van implementation 11 | */ 12 | class P3Van : public ZMQVan { 13 | public: 14 | P3Van() {} 15 | virtual ~P3Van() {} 16 | 17 | protected: 18 | void Start(int customer_id) override { 19 | start_mu_.lock(); 20 | if (init_stage == 0) { 21 | // start sender 22 | sender_thread_ = std::unique_ptr( 23 | new std::thread(&P3Van::Sending, this)); 24 | init_stage++; 25 | } 26 | start_mu_.unlock(); 27 | ZMQVan::Start(customer_id); 28 | } 29 | 30 | void Stop() override { 31 | ZMQVan::Stop(); 32 | sender_thread_->join(); 33 | } 34 | 35 | int SendMsg(const Message& msg) override { 36 | send_queue_.Push(msg); 37 | return 0; 38 | } 39 | 40 | void Sending() { 41 | while (true) { 42 | Message msg; 43 | send_queue_.WaitAndPop(&msg); 44 | ZMQVan::SendMsg(msg); 45 | if (!msg.meta.control.empty() && 46 | msg.meta.control.cmd == Control::TERMINATE) { 47 | break; 48 | } 49 | } 50 | } 51 | 52 | private: 53 | /** the thread for sending messages */ 54 | std::unique_ptr sender_thread_; 55 | ThreadsafePQueue send_queue_; 56 | int init_stage = 0; 57 | }; 58 | } // namespace ps 59 | 60 | #endif // PS_P3_VAN_H_ 61 | -------------------------------------------------------------------------------- /src/postoffice.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include "ps/internal/postoffice.h" 8 | #include "ps/internal/message.h" 9 | #include "ps/base.h" 10 | 11 | namespace ps { 12 | Postoffice::Postoffice() { 13 | env_ref_ = Environment::_GetSharedRef(); 14 | } 15 | 16 | void Postoffice::InitEnvironment() { 17 | const char* val = NULL; 18 | std::string van_type = GetEnv("DMLC_PS_VAN_TYPE", "zmq"); 19 | van_ = Van::Create(van_type); 20 | val = CHECK_NOTNULL(Environment::Get()->find("DMLC_NUM_WORKER")); 21 | num_workers_ = atoi(val); 22 | val = CHECK_NOTNULL(Environment::Get()->find("DMLC_NUM_SERVER")); 23 | num_servers_ = atoi(val); 24 | val = CHECK_NOTNULL(Environment::Get()->find("DMLC_ROLE")); 25 | std::string role(val); 26 | is_worker_ = role == "worker"; 27 | is_server_ = role == "server"; 28 | is_scheduler_ = role == "scheduler"; 29 | verbose_ = GetEnv("PS_VERBOSE", 0); 30 | } 31 | 32 | void Postoffice::Start(int customer_id, const char* argv0, const bool do_barrier) { 33 | start_mu_.lock(); 34 | if (init_stage_ == 0) { 35 | InitEnvironment(); 36 | // init glog 37 | if (argv0) { 38 | dmlc::InitLogging(argv0); 39 | } else { 40 | dmlc::InitLogging("ps-lite\0"); 41 | } 42 | 43 | // init node info. 44 | for (int i = 0; i < num_workers_; ++i) { 45 | int id = WorkerRankToID(i); 46 | for (int g : {id, kWorkerGroup, kWorkerGroup + kServerGroup, 47 | kWorkerGroup + kScheduler, 48 | kWorkerGroup + kServerGroup + kScheduler}) { 49 | node_ids_[g].push_back(id); 50 | } 51 | } 52 | 53 | for (int i = 0; i < num_servers_; ++i) { 54 | int id = ServerRankToID(i); 55 | for (int g : {id, kServerGroup, kWorkerGroup + kServerGroup, 56 | kServerGroup + kScheduler, 57 | kWorkerGroup + kServerGroup + kScheduler}) { 58 | node_ids_[g].push_back(id); 59 | } 60 | } 61 | 62 | for (int g : {kScheduler, kScheduler + kServerGroup + kWorkerGroup, 63 | kScheduler + kWorkerGroup, kScheduler + kServerGroup}) { 64 | node_ids_[g].push_back(kScheduler); 65 | } 66 | init_stage_++; 67 | } 68 | start_mu_.unlock(); 69 | 70 | // start van 71 | van_->Start(customer_id); 72 | 73 | start_mu_.lock(); 74 | if (init_stage_ == 1) { 75 | // record start time 76 | start_time_ = time(NULL); 77 | init_stage_++; 78 | } 79 | start_mu_.unlock(); 80 | // do a barrier here 81 | if (do_barrier) Barrier(customer_id, kWorkerGroup + kServerGroup + kScheduler); 82 | } 83 | 84 | void Postoffice::Finalize(const int customer_id, const bool do_barrier) { 85 | if (do_barrier) Barrier(customer_id, kWorkerGroup + kServerGroup + kScheduler); 86 | if (customer_id == 0) { 87 | num_workers_ = 0; 88 | num_servers_ = 0; 89 | van_->Stop(); 90 | init_stage_ = 0; 91 | customers_.clear(); 92 | node_ids_.clear(); 93 | barrier_done_.clear(); 94 | server_key_ranges_.clear(); 95 | heartbeats_.clear(); 96 | if (exit_callback_) exit_callback_(); 97 | } 98 | } 99 | 100 | 101 | void Postoffice::AddCustomer(Customer* customer) { 102 | std::lock_guard lk(mu_); 103 | int app_id = CHECK_NOTNULL(customer)->app_id(); 104 | // check if the customer id has existed 105 | int customer_id = CHECK_NOTNULL(customer)->customer_id(); 106 | CHECK_EQ(customers_[app_id].count(customer_id), (size_t) 0) << "customer_id " \ 107 | << customer_id << " already exists\n"; 108 | customers_[app_id].insert(std::make_pair(customer_id, customer)); 109 | std::unique_lock ulk(barrier_mu_); 110 | barrier_done_[app_id].insert(std::make_pair(customer_id, false)); 111 | } 112 | 113 | 114 | void Postoffice::RemoveCustomer(Customer* customer) { 115 | std::lock_guard lk(mu_); 116 | int app_id = CHECK_NOTNULL(customer)->app_id(); 117 | int customer_id = CHECK_NOTNULL(customer)->customer_id(); 118 | customers_[app_id].erase(customer_id); 119 | if (customers_[app_id].empty()) { 120 | customers_.erase(app_id); 121 | } 122 | } 123 | 124 | 125 | Customer* Postoffice::GetCustomer(int app_id, int customer_id, int timeout) const { 126 | Customer* obj = nullptr; 127 | for (int i = 0; i < timeout * 1000 + 1; ++i) { 128 | { 129 | std::lock_guard lk(mu_); 130 | const auto it = customers_.find(app_id); 131 | if (it != customers_.end()) { 132 | std::unordered_map customers_in_app = it->second; 133 | obj = customers_in_app[customer_id]; 134 | break; 135 | } 136 | } 137 | std::this_thread::sleep_for(std::chrono::milliseconds(1)); 138 | } 139 | return obj; 140 | } 141 | 142 | void Postoffice::Barrier(int customer_id, int node_group) { 143 | if (GetNodeIDs(node_group).size() <= 1) return; 144 | auto role = van_->my_node().role; 145 | if (role == Node::SCHEDULER) { 146 | CHECK(node_group & kScheduler); 147 | } else if (role == Node::WORKER) { 148 | CHECK(node_group & kWorkerGroup); 149 | } else if (role == Node::SERVER) { 150 | CHECK(node_group & kServerGroup); 151 | } 152 | 153 | std::unique_lock ulk(barrier_mu_); 154 | barrier_done_[0][customer_id] = false; 155 | Message req; 156 | req.meta.recver = kScheduler; 157 | req.meta.request = true; 158 | req.meta.control.cmd = Control::BARRIER; 159 | req.meta.app_id = 0; 160 | req.meta.customer_id = customer_id; 161 | req.meta.control.barrier_group = node_group; 162 | req.meta.timestamp = van_->GetTimestamp(); 163 | van_->Send(req); 164 | barrier_cond_.wait(ulk, [this, customer_id] { 165 | return barrier_done_[0][customer_id]; 166 | }); 167 | } 168 | 169 | const std::vector& Postoffice::GetServerKeyRanges() { 170 | server_key_ranges_mu_.lock(); 171 | if (server_key_ranges_.empty()) { 172 | for (int i = 0; i < num_servers_; ++i) { 173 | server_key_ranges_.push_back(Range( 174 | kMaxKey / num_servers_ * i, 175 | kMaxKey / num_servers_ * (i+1))); 176 | } 177 | } 178 | server_key_ranges_mu_.unlock(); 179 | return server_key_ranges_; 180 | } 181 | 182 | void Postoffice::Manage(const Message& recv) { 183 | CHECK(!recv.meta.control.empty()); 184 | const auto& ctrl = recv.meta.control; 185 | if (ctrl.cmd == Control::BARRIER && !recv.meta.request) { 186 | barrier_mu_.lock(); 187 | auto size = barrier_done_[recv.meta.app_id].size(); 188 | for (size_t customer_id = 0; customer_id < size; customer_id++) { 189 | barrier_done_[recv.meta.app_id][customer_id] = true; 190 | } 191 | barrier_mu_.unlock(); 192 | barrier_cond_.notify_all(); 193 | } 194 | } 195 | 196 | std::vector Postoffice::GetDeadNodes(int t) { 197 | std::vector dead_nodes; 198 | if (!van_->IsReady() || t == 0) return dead_nodes; 199 | 200 | time_t curr_time = time(NULL); 201 | const auto& nodes = is_scheduler_ 202 | ? GetNodeIDs(kWorkerGroup + kServerGroup) 203 | : GetNodeIDs(kScheduler); 204 | { 205 | std::lock_guard lk(heartbeat_mu_); 206 | for (int r : nodes) { 207 | auto it = heartbeats_.find(r); 208 | if ((it == heartbeats_.end() || it->second + t < curr_time) 209 | && start_time_ + t < curr_time) { 210 | dead_nodes.push_back(r); 211 | } 212 | } 213 | } 214 | return dead_nodes; 215 | } 216 | } // namespace ps 217 | -------------------------------------------------------------------------------- /src/resender.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015 by Contributors 3 | */ 4 | #ifndef PS_RESENDER_H_ 5 | #define PS_RESENDER_H_ 6 | #include 7 | #include 8 | #include 9 | #include 10 | namespace ps { 11 | 12 | /** 13 | * \brief resend a messsage if no ack is received within a given time 14 | */ 15 | class Resender { 16 | public: 17 | /** 18 | * \param timeout timeout in millisecond 19 | */ 20 | Resender(int timeout, int max_num_retry, Van* van) { 21 | timeout_ = timeout; 22 | max_num_retry_ = max_num_retry; 23 | van_ = van; 24 | monitor_ = new std::thread(&Resender::Monitoring, this); 25 | } 26 | ~Resender() { 27 | exit_ = true; 28 | monitor_->join(); 29 | delete monitor_; 30 | } 31 | 32 | /** 33 | * \brief add an outgoining message 34 | * 35 | */ 36 | void AddOutgoing(const Message& msg) { 37 | if (msg.meta.control.cmd == Control::ACK) return; 38 | CHECK_NE(msg.meta.timestamp, Meta::kEmpty) << msg.DebugString(); 39 | auto key = GetKey(msg); 40 | std::lock_guard lk(mu_); 41 | // already buffered, which often due to call Send by the monitor thread 42 | if (send_buff_.find(key) != send_buff_.end()) return; 43 | 44 | auto& ent = send_buff_[key]; 45 | ent.msg = msg; 46 | ent.send = Now(); 47 | ent.num_retry = 0; 48 | } 49 | 50 | /** 51 | * \brief add an incomming message 52 | * \brief return true if msg has been added before or a ACK message 53 | */ 54 | bool AddIncomming(const Message& msg) { 55 | // a message can be received by multiple times 56 | if (msg.meta.control.cmd == Control::TERMINATE) { 57 | return false; 58 | } else if (msg.meta.control.cmd == Control::ACK) { 59 | mu_.lock(); 60 | auto key = msg.meta.control.msg_sig; 61 | auto it = send_buff_.find(key); 62 | if (it != send_buff_.end()) send_buff_.erase(it); 63 | mu_.unlock(); 64 | return true; 65 | } else { 66 | mu_.lock(); 67 | auto key = GetKey(msg); 68 | auto it = acked_.find(key); 69 | bool duplicated = it != acked_.end(); 70 | if (!duplicated) acked_.insert(key); 71 | mu_.unlock(); 72 | // send back ack message (even if it is duplicated) 73 | Message ack; 74 | ack.meta.recver = msg.meta.sender; 75 | ack.meta.sender = msg.meta.recver; 76 | ack.meta.control.cmd = Control::ACK; 77 | ack.meta.control.msg_sig = key; 78 | van_->Send(ack); 79 | // warning 80 | if (duplicated) LOG(WARNING) << "Duplicated message: " << msg.DebugString(); 81 | return duplicated; 82 | } 83 | } 84 | 85 | private: 86 | using Time = std::chrono::milliseconds; 87 | // the buffer entry 88 | struct Entry { 89 | Message msg; 90 | Time send; 91 | int num_retry = 0; 92 | }; 93 | std::unordered_map send_buff_; 94 | 95 | uint64_t GetKey(const Message& msg) { 96 | CHECK_NE(msg.meta.timestamp, Meta::kEmpty) << msg.DebugString(); 97 | uint16_t id = msg.meta.app_id; 98 | uint8_t sender = msg.meta.sender == Node::kEmpty ? 99 | van_->my_node().id : msg.meta.sender; 100 | uint8_t recver = msg.meta.recver; 101 | return (static_cast(id) << 48) | 102 | (static_cast(sender) << 40) | 103 | (static_cast(recver) << 32) | 104 | (msg.meta.timestamp << 1) | msg.meta.request; 105 | } 106 | Time Now() { 107 | return std::chrono::duration_cast