├── cbindgen.toml ├── .gitignore ├── src ├── implement │ ├── mod.rs │ ├── nthread_per_socket_backend.rs │ └── tokio_backend.rs ├── interface.rs ├── lib.rs └── utils.rs ├── .github ├── workflows │ ├── ci.yml │ └── release.yml └── dependabot.yml ├── cc ├── Makefile ├── nccl_types.h ├── bagua_net.h ├── bagua_net.cc ├── v3 │ ├── nccl_net_v3.h │ └── nccl_net_v3.cc └── v4 │ ├── nccl_net_v4.h │ └── nccl_net_v4.cc ├── Cargo.toml ├── LICENSE ├── README.md └── Cargo.lock /cbindgen.toml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /src/implement/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod tokio_backend; 2 | pub mod nthread_per_socket_backend; 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Unit Test 20 | run: cd cc && make test 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "cargo" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /cc/Makefile: -------------------------------------------------------------------------------- 1 | INC:= -I. -I./v4 -I./v3 2 | PLUGIN_SO:=libnccl-net.so 3 | BAGUA_NET_LIB:=libbagua_net.a 4 | NCCL_NET_V4:=./v4/nccl_net_v4.cc 5 | NCCL_NET_V3:=./v3/nccl_net_v3.cc 6 | 7 | default: $(PLUGIN_SO) $(BAGUA_NET_LIB) 8 | 9 | $(PLUGIN_SO): bagua_net.cc $(NCCL_NET_V4) $(NCCL_NET_V3) $(BAGUA_NET_LIB) 10 | $(CXX) -v $(INC) \ 11 | -std=c++17 -fPIC -shared \ 12 | -o $@ $^ \ 13 | -L. -l:$(BAGUA_NET_LIB) -lz 14 | 15 | $(BAGUA_NET_LIB): 16 | cargo build --release && cp ../target/release/$(BAGUA_NET_LIB) ./$(BAGUA_NET_LIB) 17 | 18 | test: 19 | cargo test --verbose 20 | 21 | clean: 22 | rm -f $(PLUGIN_SO) $(BAGUA_NET_LIB) 23 | 24 | tar: $(PLUGIN_SO) 25 | mkdir build && mv $(PLUGIN_SO) build \ 26 | && tar czf build.tar.gz build 27 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bagua-net" 3 | version = "0.1.0" 4 | authors = ["Jianbin Chang "] 5 | edition = "2018" 6 | license = "MIT License Copyright (c) 2021 Kuaishou AI Platform & DS3 Lab" 7 | 8 | [lib] 9 | crate-type = ["staticlib"] 10 | 11 | [dependencies] 12 | nix = "0.22.1" 13 | tracing = "0.1" 14 | tracing-subscriber = "0.2" 15 | thiserror = "1" 16 | bytes = "1.1" 17 | libc = "0.2" 18 | ffi-convert = "0.5" 19 | flume = "0.10" 20 | socket2 = "0.4" 21 | opentelemetry = { version = "0.16", features = [ 22 | "trace", 23 | "metrics", 24 | "rt-async-std", 25 | ] } 26 | opentelemetry-jaeger = { version = "*", features = [ 27 | "rt-async-std", 28 | "collector_client", 29 | "isahc_collector_client", 30 | ] } 31 | opentelemetry-prometheus = { version = "*" } 32 | openssl = { version = "0.10", features = ["vendored"] } 33 | prometheus = { version = "0.12", features = ["push"] } 34 | lazy_static = "1.4" 35 | regex = "1.5" 36 | tokio = { version = "1", features = ["full"] } 37 | futures = "0.3" 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kuaishou AI Platform & DS3 Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Create Release 2 | 3 | on: 4 | push: 5 | # Sequence of patterns matched against refs/tags 6 | tags: 7 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 8 | 9 | jobs: 10 | build: 11 | name: Create Release 12 | runs-on: ubuntu-18.04 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v2 16 | - name: Build Project # This would actually build your project, using zip for an example artifact 17 | run: | 18 | cd cc && make && make tar 19 | - name: Create Release 20 | id: create_release 21 | uses: actions/create-release@v1 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | with: 25 | tag_name: ${{ github.ref }} 26 | release_name: Release ${{ github.ref }} 27 | draft: false 28 | prerelease: false 29 | - name: Upload Release Asset 30 | id: upload-release-asset 31 | uses: actions/upload-release-asset@v1 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | with: 35 | upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 36 | asset_path: ./cc/build.tar.gz 37 | asset_name: bagua-net_${{ github.ref }}_x86_64.tar.gz 38 | asset_content_type: application/tar+gzip 39 | -------------------------------------------------------------------------------- /cc/nccl_types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /** COPY FROM nccl.h.in - START **/ 4 | 5 | /* Error type */ 6 | typedef enum { ncclSuccess = 0, 7 | ncclUnhandledCudaError = 1, 8 | ncclSystemError = 2, 9 | ncclInternalError = 3, 10 | ncclInvalidArgument = 4, 11 | ncclInvalidUsage = 5, 12 | ncclNumResults = 6 } ncclResult_t; 13 | 14 | /* Reduction operation selector */ 15 | typedef enum { ncclSum = 0, 16 | ncclProd = 1, 17 | ncclMax = 2, 18 | ncclMin = 3, 19 | ncclAvg = 4, 20 | ncclNumOps = 5 } ncclRedOp_t; 21 | 22 | /* Data types */ 23 | typedef enum { ncclInt8 = 0, ncclChar = 0, 24 | ncclUint8 = 1, 25 | ncclInt32 = 2, ncclInt = 2, 26 | ncclUint32 = 3, 27 | ncclInt64 = 4, 28 | ncclUint64 = 5, 29 | ncclFloat16 = 6, ncclHalf = 6, 30 | ncclFloat32 = 7, ncclFloat = 7, 31 | ncclFloat64 = 8, ncclDouble = 8, 32 | #if defined(__CUDA_BF16_TYPES_EXIST__) 33 | ncclBfloat16 = 9, 34 | ncclNumTypes = 10 35 | #else 36 | ncclNumTypes = 9 37 | #endif 38 | } ncclDataType_t; 39 | 40 | /** COPY FROM nccl.h.in - END **/ 41 | 42 | /** COPY FROM nccl_net.h - START **/ 43 | 44 | #define NCCL_NET_HANDLE_MAXSIZE 64 45 | 46 | #define NCCL_PTR_HOST 0x1 47 | #define NCCL_PTR_CUDA 0x2 48 | 49 | // Maximum number of requests per comm object 50 | #define NCCL_NET_MAX_REQUESTS 8 51 | 52 | typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; 53 | typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_ALL=~0} ncclDebugLogSubSys; 54 | 55 | typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); 56 | 57 | /** COPY FROM nccl_net.h - END **/ -------------------------------------------------------------------------------- /src/interface.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Error, Debug, Clone)] 4 | pub enum BaguaNetError { 5 | #[error("io error")] 6 | IOError(String), 7 | #[error("tcp error")] 8 | TCPError(String), 9 | #[error("inner error")] 10 | InnerError(String), 11 | } 12 | 13 | #[derive(Debug)] 14 | pub struct NCCLNetProperties { 15 | pub name: String, 16 | pub pci_path: String, 17 | pub guid: u64, 18 | pub ptr_support: i32, // NCCL_PTR_HOST or NCCL_PTR_HOST|NCCL_PTR_CUDA 19 | pub speed: i32, // Port speed in Mbps. 20 | pub port: i32, 21 | pub max_comms: i32, 22 | } 23 | 24 | #[derive(Debug)] 25 | pub struct SocketHandle { 26 | pub addr: nix::sys::socket::SockAddr, 27 | } 28 | 29 | pub type SocketListenCommID = usize; 30 | pub type SocketSendCommID = usize; 31 | pub type SocketRecvCommID = usize; 32 | pub type SocketRequestID = usize; 33 | 34 | pub trait Net { 35 | fn devices(&self) -> Result; 36 | 37 | fn get_properties(&self, dev_id: usize) -> Result; 38 | 39 | fn listen( 40 | &mut self, 41 | dev_id: usize, 42 | ) -> Result<(SocketHandle, SocketListenCommID), BaguaNetError>; 43 | 44 | fn connect( 45 | &mut self, 46 | _dev_id: usize, 47 | socket_handle: SocketHandle, 48 | ) -> Result; 49 | 50 | fn accept( 51 | &mut self, 52 | listen_comm_id: SocketListenCommID, 53 | ) -> Result; 54 | 55 | fn isend( 56 | &mut self, 57 | send_comm_id: SocketSendCommID, 58 | data: &'static [u8], 59 | ) -> Result; 60 | 61 | fn irecv( 62 | &mut self, 63 | recv_comm_id: SocketRecvCommID, 64 | data: &'static mut [u8], 65 | ) -> Result; 66 | 67 | fn test(&mut self, request_id: SocketRequestID) -> Result<(bool, usize), BaguaNetError>; 68 | 69 | fn close_send(&mut self, send_comm_id: SocketSendCommID) -> Result<(), BaguaNetError>; 70 | 71 | fn close_recv(&mut self, recv_comm_id: SocketRecvCommID) -> Result<(), BaguaNetError>; 72 | 73 | fn close_listen(&mut self, listen_comm_id: SocketListenCommID) -> Result<(), BaguaNetError>; 74 | } 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Bagua-Net has been merged into [Bagua](https://github.com/BaguaSys/bagua)! 2 | ===== 3 | 4 | [![GitHub license](https://img.shields.io/github/license/BaguaSys/bagua-core)](https://github.com/BaguaSys/bagua-core/blob/master/LICENSE) 5 | 6 | Bagua-Net is a high performance NCCL plugin for [Bagua](https://github.com/BaguaSys/bagua). By optimizing the fairness among multiple TCP streams, Bagua-Net can improve the overall throughput under TCP network by more than 30% for typical distributed learning tasks. 7 | 8 | ## Requirement 9 | 10 | - NCCL >= v2.6.4 11 | 12 | ## Quick Start 13 | 14 | ```bash 15 | # Install Bagua-Net 16 | cd cc && make 17 | export BAGUA_NET_LIBRARY_PATH=$(readlink -f .) 18 | 19 | # Install nccl and nccl-test 20 | git clone https://github.com/NVIDIA/nccl.git && cd nccl && git checkout v2.10.3-1 21 | make -j src.build && make install 22 | git clone https://github.com/NVIDIA/nccl-tests.git 23 | cd nccl-tests 24 | make MPI=1 25 | 26 | # Run nccl baseline 27 | mpirun \ 28 | --allow-run-as-root \ 29 | -H ${HOST1}:1,${HOST2}:1 --np 2 \ 30 | -mca pml ob1 -mca btl ^openib \ 31 | -mca btl_tcp_if_include eth01 \ 32 | -x NCCL_DEBUG=INFO \ 33 | -x LD_LIBRARY_PATH \ 34 | ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 1 35 | 36 | # Run nccl with bagua-net 37 | mpirun \ 38 | --allow-run-as-root \ 39 | -H ${HOST1}:1,${HOST2}:1 --np 2 \ 40 | -mca pml ob1 -mca btl ^openib \ 41 | -mca btl_tcp_if_include eth01 \ 42 | -x NCCL_DEBUG=INFO \ 43 | -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BAGUA_NET_LIBRARY_PATH \ 44 | ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 1 45 | # If the installation is successful, there will be a log like this `NCCL INFO Using network BaguaNet`. 46 | ``` 47 | 48 | ## Benchmark 49 | 50 | On 4 nodes, each one equipped with 8 V100 GPUs and 100Gb ethernet connection, [the throughput of AllReduce can be improved by 50%](https://github.com/BaguaSys/bagua-net/wiki/NCCL-benchmark-bagua-net-vs-google-fastsocket-vs-baseline). 51 | 52 | In an [end-to-end test](https://github.com/BaguaSys/examples/blob/main/benchmark/synthetic_benchmark.py) with VGG16 as the model to be trained, with Bagua-Net enabled, Bagua can be more than 35% faster than other distributed solutions (such as PyTorch DDP): 53 | 54 | ``` 55 | # VGG16 on 4x8xV100 bagua-net 56 | Running benchmark... 57 | Iter #0: 4081.0 img/sec GPU 58 | Iter #1: 4072.0 img/sec GPU 59 | Iter #2: 4106.4 img/sec GPU 60 | Iter #3: 4081.7 img/sec GPU 61 | Iter #4: 4064.8 img/sec GPU 62 | Iter #5: 4122.1 img/sec GPU 63 | Iter #6: 3857.7 img/sec GPU 64 | Iter #7: 4128.3 img/sec GPU 65 | Iter #8: 4125.5 img/sec GPU 66 | Iter #9: 3826.6 img/sec GPU 67 | Img/sec per GPU: 126.5 +-6.4 68 | Total img/sec on 32 GPU(s): 4046.6 +-205.2 69 | 70 | # VGG16 on 4x8xV100 baseline 71 | Running benchmark... 72 | Iter #0: 2620.2 img/sec GPU 73 | Iter #1: 2771.9 img/sec GPU 74 | Iter #2: 2772.6 img/sec GPU 75 | Iter #3: 2794.5 img/sec GPU 76 | Iter #4: 2627.9 img/sec GPU 77 | Iter #5: 2787.8 img/sec GPU 78 | Iter #6: 2775.9 img/sec GPU 79 | Iter #7: 2741.6 img/sec GPU 80 | Iter #8: 2760.0 img/sec GPU 81 | Iter #9: 2796.6 img/sec GPU 82 | Img/sec per GPU: 85.8 +-3.8 83 | Total img/sec on 32 GPU(s): 2744.9 +-122.3 84 | ``` 85 | 86 | ## Limitations 87 | 88 | Bagua-Net currently only accelerates AllReduce on TCP. Support for other operations and RDMA is still under development. 89 | -------------------------------------------------------------------------------- /cc/bagua_net.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | struct BaguaNetC; 14 | 15 | struct NCCLNetPropertiesC 16 | { 17 | const char *name; 18 | const char *pci_path; 19 | uint64_t guid; 20 | int32_t ptr_support; 21 | int32_t speed; 22 | int32_t port; 23 | int32_t max_comms; 24 | }; 25 | 26 | struct SocketHandleC 27 | { 28 | struct sockaddr sockaddr; 29 | }; 30 | 31 | struct Buffer 32 | { 33 | uint8_t *data; 34 | uintptr_t len; 35 | }; 36 | 37 | extern "C" 38 | { 39 | 40 | BaguaNetC *bagua_net_c_create(); 41 | 42 | void bagua_net_c_destroy(BaguaNetC **ptr); 43 | 44 | /// Error code 45 | /// 0: success 46 | /// -1: null pointer 47 | int32_t bagua_net_c_devices(BaguaNetC *ptr, int32_t *ndev); 48 | 49 | /// Error code 50 | /// 0: success 51 | /// -1: null pointer 52 | int32_t bagua_net_c_get_properties(BaguaNetC *ptr, int32_t dev_id, NCCLNetPropertiesC *props); 53 | 54 | /// Error code 55 | /// 0: success 56 | /// -1: null pointer 57 | int32_t bagua_net_c_listen(BaguaNetC *ptr, 58 | int32_t dev_id, 59 | SocketHandleC *socket_handle, 60 | uintptr_t *socket_listen_comm_id); 61 | 62 | /// Error code 63 | /// 0: success 64 | /// -1: null pointer 65 | int32_t bagua_net_c_connect(BaguaNetC *ptr, 66 | int32_t dev_id, 67 | SocketHandleC *socket_handle, 68 | uintptr_t *socket_send_comm_id); 69 | 70 | /// Error code 71 | /// 0: success 72 | /// -1: null pointer 73 | int32_t bagua_net_c_accept(BaguaNetC *ptr, uintptr_t listen_comm_id, uintptr_t *recv_comm_id); 74 | 75 | /// Error code 76 | /// 0: success 77 | /// -1: null pointer 78 | int32_t bagua_net_c_isend(BaguaNetC *ptr, 79 | uintptr_t send_comm_id, 80 | Buffer buf, 81 | uintptr_t *request_id); 82 | 83 | /// Error code 84 | /// 0: success 85 | /// -1: null pointer 86 | int32_t bagua_net_c_irecv(BaguaNetC *ptr, 87 | uintptr_t recv_comm_id, 88 | Buffer buf, 89 | uintptr_t *request_id); 90 | 91 | /// Error code 92 | /// 0: success 93 | /// -1: null pointer 94 | int32_t bagua_net_c_test(BaguaNetC *ptr, uintptr_t request_id, bool *done, uintptr_t *bytes); 95 | 96 | /// Error code 97 | /// 0: success 98 | /// -1: null pointer 99 | int32_t bagua_net_c_close_send(BaguaNetC *ptr, uintptr_t send_comm_id); 100 | 101 | /// Error code 102 | /// 0: success 103 | /// -1: null pointer 104 | int32_t bagua_net_c_close_recv(BaguaNetC *ptr, uintptr_t recv_comm_id); 105 | 106 | /// Error code 107 | /// 0: success 108 | /// -1: null pointer 109 | int32_t bagua_net_c_close_listen(BaguaNetC *ptr, uintptr_t listen_comm_id); 110 | 111 | } // extern "C" 112 | 113 | class BaguaNet 114 | { 115 | public: 116 | static BaguaNet &instance() 117 | { 118 | static BaguaNet instance; 119 | return instance; 120 | } 121 | 122 | BaguaNet(BaguaNet const &) = delete; 123 | void operator=(BaguaNet const &) = delete; 124 | 125 | int32_t devices(int32_t *ndev); 126 | 127 | int32_t get_properties(int32_t dev_id, NCCLNetPropertiesC *props); 128 | 129 | int32_t listen(int32_t dev_id, void *handle, void **listen_comm); 130 | 131 | int32_t connect(int32_t dev_id, void *handle, void **send_comm); 132 | 133 | int32_t accept(void *listen_comm, void **recv_comm); 134 | 135 | int32_t isend(void *send_comm, void *data, int size, void *mhandle, void **request); 136 | 137 | int32_t irecv(void *recv_comm, void *data, int size, void *mhandle, void **request); 138 | 139 | int32_t test(void *request, bool *done, uintptr_t *bytes); 140 | 141 | int32_t close_send(void *send_comm); 142 | 143 | int32_t close_recv(void *recv_comm); 144 | 145 | int32_t close_listen(void *listen_comm); 146 | 147 | private: 148 | BaguaNet(); 149 | 150 | private: 151 | std::unique_ptr > inner; 152 | std::vector > device_props; 153 | }; 154 | -------------------------------------------------------------------------------- /cc/bagua_net.cc: -------------------------------------------------------------------------------- 1 | #include "bagua_net.h" 2 | 3 | int32_t BaguaNet::devices(int32_t *ndev) 4 | { 5 | return bagua_net_c_devices(inner.get(), ndev); 6 | } 7 | 8 | int32_t BaguaNet::get_properties(int32_t dev_id, NCCLNetPropertiesC *props) 9 | { 10 | // HINT: The name and pci_path of ncclNetProperties_v4_t are both 11 | // references and cannot be passed in directly 12 | auto &inner_props = device_props.at(dev_id); 13 | if (!inner_props) 14 | { 15 | inner_props = std::shared_ptr( 16 | new NCCLNetPropertiesC{.name = NULL, .pci_path = NULL}, 17 | [](NCCLNetPropertiesC *ptr) 18 | { 19 | delete ptr->name; 20 | delete ptr->pci_path; 21 | }); 22 | int ret = bagua_net_c_get_properties(inner.get(), dev_id, inner_props.get()); 23 | if (ret != 0) 24 | { 25 | return ret; 26 | } 27 | } 28 | 29 | *props = *inner_props; 30 | return 0; 31 | } 32 | 33 | int32_t BaguaNet::listen(int32_t dev_id, void *handle, void **listen_comm) 34 | { 35 | auto socket_listen_comm_id = std::make_unique(-1); 36 | int32_t ret = bagua_net_c_listen(inner.get(), dev_id, static_cast(handle), socket_listen_comm_id.get()); 37 | if (ret != 0) 38 | { 39 | return ret; 40 | } 41 | 42 | *listen_comm = socket_listen_comm_id.release(); 43 | return 0; 44 | } 45 | 46 | int32_t BaguaNet::connect(int32_t dev_id, void *handle, void **send_comm) 47 | { 48 | auto socket_send_comm_id = std::make_unique(-1); 49 | int32_t ret = bagua_net_c_connect(inner.get(), dev_id, static_cast(handle), socket_send_comm_id.get()); 50 | if (ret != 0) 51 | { 52 | return ret; 53 | } 54 | 55 | *send_comm = socket_send_comm_id.release(); 56 | return 0; 57 | } 58 | 59 | int32_t BaguaNet::accept(void *listen_comm, void **recv_comm) 60 | { 61 | uintptr_t listen_comm_id = *static_cast(listen_comm); 62 | auto recv_comm_id = std::make_unique(-1); 63 | int32_t ret = bagua_net_c_accept(inner.get(), listen_comm_id, recv_comm_id.get()); 64 | if (ret != 0) 65 | { 66 | return ret; 67 | } 68 | 69 | *recv_comm = recv_comm_id.release(); 70 | return 0; 71 | } 72 | 73 | int32_t BaguaNet::isend(void *send_comm, void *data, int size, void *mhandle, void **request) 74 | { 75 | uintptr_t send_comm_id = *static_cast(send_comm); 76 | Buffer buf{ 77 | .data = static_cast(data), 78 | .len = (uintptr_t)(size), 79 | }; 80 | auto request_id = std::make_unique(-1); 81 | 82 | int32_t ret = bagua_net_c_isend(inner.get(), send_comm_id, buf, request_id.get()); 83 | if (ret != 0) 84 | { 85 | return ret; 86 | } 87 | 88 | *request = request_id.release(); 89 | return 0; 90 | } 91 | 92 | int32_t BaguaNet::irecv(void *recv_comm, void *data, int size, void *mhandle, void **request) 93 | { 94 | uintptr_t recv_comm_id = *static_cast(recv_comm); 95 | Buffer buf{ 96 | .data = static_cast(data), 97 | .len = (uintptr_t)(size), 98 | }; 99 | auto request_id = std::make_unique(-1); 100 | 101 | int32_t ret = bagua_net_c_irecv(inner.get(), recv_comm_id, buf, request_id.get()); 102 | if (ret != 0) 103 | { 104 | return ret; 105 | } 106 | 107 | *request = request_id.release(); 108 | return 0; 109 | } 110 | 111 | int32_t BaguaNet::test(void *request, bool *done, uintptr_t *bytes) 112 | { 113 | uintptr_t request_id = *static_cast(request); 114 | int32_t ret = bagua_net_c_test(inner.get(), request_id, done, bytes); 115 | if (ret != 0) 116 | { 117 | return ret; 118 | } 119 | 120 | return 0; 121 | } 122 | 123 | int32_t BaguaNet::close_send(void *send_comm) 124 | { 125 | auto send_comm_id = std::unique_ptr(static_cast(send_comm)); 126 | return bagua_net_c_close_send(inner.get(), *send_comm_id); 127 | } 128 | 129 | int32_t BaguaNet::close_recv(void *recv_comm) 130 | { 131 | auto recv_comm_id = std::unique_ptr(static_cast(recv_comm)); 132 | return bagua_net_c_close_recv(inner.get(), *recv_comm_id); 133 | } 134 | 135 | int32_t BaguaNet::close_listen(void *listen_comm) 136 | { 137 | auto listen_comm_id = std::unique_ptr(static_cast(listen_comm)); 138 | return bagua_net_c_close_listen(inner.get(), *listen_comm_id); 139 | } 140 | 141 | BaguaNet::BaguaNet() 142 | { 143 | inner = std::unique_ptr >( 144 | bagua_net_c_create(), 145 | [](BaguaNetC *ptr) 146 | { 147 | bagua_net_c_destroy(&ptr); 148 | }); 149 | int32_t ndev = -1; 150 | if (bagua_net_c_devices(inner.get(), &ndev) == 0 && ndev != -1) 151 | { 152 | device_props.resize(ndev); 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /cc/v3/nccl_net_v3.h: -------------------------------------------------------------------------------- 1 | /************************************************************************* 2 | * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * See LICENSE.txt for license information 5 | ************************************************************************/ 6 | 7 | #pragma once 8 | 9 | #include "nccl_types.h" 10 | 11 | #include 12 | 13 | typedef struct { 14 | char* name; // Used mostly for logging. 15 | char* pciPath; // Path to the PCI device in /sys. 16 | uint64_t guid; // Unique identifier for the NIC chip. Important for 17 | // cards with multiple PCI functions (Physical or virtual). 18 | int ptrSupport; // NCCL_PTR_HOST or NCCL_PTR_HOST|NCCL_PTR_CUDA 19 | int speed; // Port speed in Mbps. 20 | int port; // Port number. 21 | int maxComms; // Maximum number of comms we can create 22 | } ncclNetProperties_v3_t; 23 | 24 | typedef struct { 25 | // Name of the network (mainly for logs) 26 | const char* name; 27 | // Initialize the network. 28 | ncclResult_t (*init)(ncclDebugLogger_t logFunction); 29 | // Return the number of adapters. 30 | ncclResult_t (*devices)(int* ndev); 31 | // Get various device properties. 32 | ncclResult_t (*getProperties)(int dev, ncclNetProperties_v3_t* props); 33 | // Create a receiving object and provide a handle to connect to it. The 34 | // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged 35 | // between ranks to create a connection. 36 | ncclResult_t (*listen)(int dev, void* handle, void** listenComm); 37 | // Connect to a handle and return a sending comm object for that peer. 38 | ncclResult_t (*connect)(int dev, void* handle, void** sendComm); 39 | // Finalize connection establishment after remote peer has called connectHandle 40 | ncclResult_t (*accept)(void* listenComm, void** recvComm); 41 | // Register/Deregister memory. Comm can be either a sendComm or a recvComm. 42 | // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. 43 | ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); 44 | ncclResult_t (*deregMr)(void* comm, void* mhandle); 45 | // Asynchronous send to a peer. 46 | // May return request == NULL if the call cannot be performed (or would block) 47 | ncclResult_t (*isend)(void* sendComm, void* data, int size, void* mhandle, void** request); 48 | // Asynchronous recv from a peer. 49 | // May return request == NULL if the call cannot be performed (or would block) 50 | ncclResult_t (*irecv)(void* recvComm, void* data, int size, void* mhandle, void** request); 51 | // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is 52 | // visible to the GPU 53 | ncclResult_t (*flush)(void* recvComm, void* data, int size, void* mhandle); 54 | // Test whether a request is complete. If size is not NULL, it returns the 55 | // number of bytes sent/received. 56 | ncclResult_t (*test)(void* request, int* done, int* size); 57 | // Close and free send/recv comm objects 58 | ncclResult_t (*closeSend)(void* sendComm); 59 | ncclResult_t (*closeRecv)(void* recvComm); 60 | ncclResult_t (*closeListen)(void* listenComm); 61 | } ncclNet_v3_t; 62 | 63 | typedef struct { 64 | // Name of the collective network (mainly for logs) 65 | const char* name; 66 | // Initialize the collective network. 67 | ncclResult_t (*init)(ncclDebugLogger_t logFunction); 68 | // Return the number of adapters capable of doing collective operations. 69 | // If ndev returns 0, all other functions might be set to NULL. 70 | ncclResult_t (*devices)(int* ndev); 71 | // Get various device properties. 72 | ncclResult_t (*getProperties)(int dev, ncclNetProperties_v3_t* props); 73 | // Create a receiving object and provide a handle to connect to it. The 74 | // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged 75 | // between ranks to create connections. 76 | ncclResult_t (*listen)(int dev, void* handle, void** listenComm); 77 | // Create a group for collective operations. handles have been created 78 | // using listen() above. rank indicates caller's rank in the collective network. 79 | ncclResult_t (*connect)(void* handles[], int nranks, int rank, void* listenComm, void** collComm); 80 | // Returns whether a reduction operation on a data type is supported. 81 | // 1 for supported, 0 otherwise. 82 | ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int* supported); 83 | // Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. 84 | ncclResult_t (*regMr)(void* collComm, void* data, int size, int type, void** mhandle); 85 | ncclResult_t (*deregMr)(void* collComm, void* mhandle); 86 | // Performs an asynchronous allreduce operation on the collective group. 87 | // May return request == NULL if the call cannot be performed (or would block). 88 | ncclResult_t (*iallreduce)(void* collComm, void* sendData, void* recvData, int count, 89 | ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request); 90 | // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is 91 | // visible to the GPU 92 | ncclResult_t (*flush)(void* collComm, void* data, int size, void* mhandle); 93 | // Test whether a request is complete. If size is not NULL, it returns the 94 | // number of bytes sent/received. 95 | ncclResult_t (*test)(void* request, int* done, int* size); 96 | // Close and free collective comm objects 97 | ncclResult_t (*closeColl)(void* collComm); 98 | ncclResult_t (*closeListen)(void* listenComm); 99 | } ncclCollNet_v3_t; 100 | -------------------------------------------------------------------------------- /cc/v4/nccl_net_v4.h: -------------------------------------------------------------------------------- 1 | /************************************************************************* 2 | * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * See LICENSE.txt for license information 5 | ************************************************************************/ 6 | 7 | #pragma once 8 | 9 | #include "nccl_types.h" 10 | #include 11 | 12 | typedef struct 13 | { 14 | char *name; // Used mostly for logging. 15 | char *pciPath; // Path to the PCI device in /sys. 16 | uint64_t guid; // Unique identifier for the NIC chip. Important for 17 | // cards with multiple PCI functions (Physical or virtual). 18 | int ptrSupport; // NCCL_PTR_HOST or NCCL_PTR_HOST|NCCL_PTR_CUDA 19 | int speed; // Port speed in Mbps. 20 | int port; // Port number. 21 | int maxComms; // Maximum number of comms we can create 22 | } ncclNetProperties_v4_t; 23 | 24 | typedef struct 25 | { 26 | // Name of the network (mainly for logs) 27 | const char *name; 28 | // Initialize the network. 29 | ncclResult_t (*init)(ncclDebugLogger_t logFunction); 30 | // Return the number of adapters. 31 | ncclResult_t (*devices)(int *ndev); 32 | // Get various device properties. 33 | ncclResult_t (*getProperties)(int dev, ncclNetProperties_v4_t *props); 34 | // Create a receiving object and provide a handle to connect to it. The 35 | // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged 36 | // between ranks to create a connection. 37 | ncclResult_t (*listen)(int dev, void *handle, void **listenComm); 38 | // Connect to a handle and return a sending comm object for that peer. 39 | ncclResult_t (*connect)(int dev, void *handle, void **sendComm); 40 | // Finalize connection establishment after remote peer has called connectHandle 41 | ncclResult_t (*accept)(void *listenComm, void **recvComm); 42 | // Register/Deregister memory. Comm can be either a sendComm or a recvComm. 43 | // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. 44 | ncclResult_t (*regMr)(void *comm, void *data, int size, int type, void **mhandle); 45 | ncclResult_t (*deregMr)(void *comm, void *mhandle); 46 | // Asynchronous send to a peer. 47 | // May return request == NULL if the call cannot be performed (or would block) 48 | ncclResult_t (*isend)(void *sendComm, void *data, int size, void *mhandle, void **request); 49 | // Asynchronous recv from a peer. 50 | // May return request == NULL if the call cannot be performed (or would block) 51 | ncclResult_t (*irecv)(void *recvComm, void *data, int size, void *mhandle, void **request); 52 | // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is 53 | // visible to the GPU 54 | ncclResult_t (*iflush)(void *recvComm, void *data, int size, void *mhandle, void **request); 55 | // Test whether a request is complete. If size is not NULL, it returns the 56 | // number of bytes sent/received. 57 | ncclResult_t (*test)(void *request, int *done, int *size); 58 | // Close and free send/recv comm objects 59 | ncclResult_t (*closeSend)(void *sendComm); 60 | ncclResult_t (*closeRecv)(void *recvComm); 61 | ncclResult_t (*closeListen)(void *listenComm); 62 | } ncclNet_v4_t; 63 | 64 | typedef struct 65 | { 66 | // Name of the collective network (mainly for logs) 67 | const char *name; 68 | // Initialize the collective network. 69 | ncclResult_t (*init)(ncclDebugLogger_t logFunction); 70 | // Return the number of adapters capable of doing collective operations. 71 | // If ndev returns 0, all other functions might be set to NULL. 72 | ncclResult_t (*devices)(int *ndev); 73 | // Get various device properties. 74 | ncclResult_t (*getProperties)(int dev, ncclNetProperties_v4_t *props); 75 | // Create a receiving object and provide a handle to connect to it. The 76 | // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged 77 | // between ranks to create connections. 78 | ncclResult_t (*listen)(int dev, void *handle, void **listenComm); 79 | // Create a group for collective operations. handles have been created 80 | // using listen() above. rank indicates caller's rank in the collective network. 81 | ncclResult_t (*connect)(void *handles[], int nranks, int rank, void *listenComm, void **collComm); 82 | // Returns whether a reduction operation on a data type is supported. 83 | // 1 for supported, 0 otherwise. 84 | ncclResult_t (*reduceSupport)(ncclDataType_t dataType, ncclRedOp_t redOp, int *supported); 85 | // Register/Deregister memory. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. 86 | ncclResult_t (*regMr)(void *collComm, void *data, int size, int type, void **mhandle); 87 | ncclResult_t (*deregMr)(void *collComm, void *mhandle); 88 | // Performs an asynchronous allreduce operation on the collective group. 89 | // May return request == NULL if the call cannot be performed (or would block). 90 | ncclResult_t (*iallreduce)(void *collComm, void *sendData, void *recvData, int count, 91 | ncclDataType_t dataType, ncclRedOp_t redOp, void *sendMhandle, void *recvMhandle, void **request); 92 | // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is 93 | // visible to the GPU 94 | ncclResult_t (*iflush)(void *collComm, void *data, int size, void *mhandle, void **request); 95 | // Test whether a request is complete. If size is not NULL, it returns the 96 | // number of bytes sent/received. 97 | ncclResult_t (*test)(void *request, int *done, int *size); 98 | // Close and free collective comm objects 99 | ncclResult_t (*closeColl)(void *collComm); 100 | ncclResult_t (*closeListen)(void *listenComm); 101 | } ncclCollNet_v4_t; 102 | -------------------------------------------------------------------------------- /cc/v4/nccl_net_v4.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nccl_net_v4.h" 9 | #include "bagua_net.h" 10 | 11 | #define __hidden __attribute__((visibility("hidden"))) 12 | 13 | ncclDebugLogger_t NCCL_DEBUG_LOG; 14 | #define NCCL_TRACE(FLAGS, ...) NCCL_DEBUG_LOG(NCCL_LOG_TRACE, (FLAGS), __func__, __LINE__, __VA_ARGS__) 15 | #define NCCL_INFO(FLAGS, ...) NCCL_DEBUG_LOG(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__) 16 | #define NCCL_WARN(...) NCCL_DEBUG_LOG(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) 17 | 18 | void set_properties(ncclNetProperties_v4_t &lhs, NCCLNetPropertiesC &rhs) 19 | { 20 | lhs.name = const_cast(rhs.name); 21 | lhs.pciPath = const_cast(rhs.pci_path); 22 | lhs.guid = rhs.guid; 23 | lhs.ptrSupport = rhs.ptr_support; 24 | lhs.speed = rhs.speed; 25 | lhs.port = rhs.port; 26 | lhs.maxComms = rhs.max_comms; 27 | } 28 | 29 | __hidden ncclResult_t baguaNetInit_v4(ncclDebugLogger_t logFunction) 30 | { 31 | NCCL_DEBUG_LOG = logFunction; 32 | BaguaNet::instance(); 33 | 34 | NCCL_TRACE(NCCL_ALL, "baguaNetInit_v4!"); 35 | 36 | return ncclSuccess; 37 | } 38 | 39 | __hidden ncclResult_t baguaNetDevices_v4(int *ndev) 40 | { 41 | if (BaguaNet::instance().devices((int32_t *)ndev) != 0) 42 | { 43 | NCCL_WARN("baguaNetDevices_v4 failed, ndev=%d", *ndev); 44 | return ncclInternalError; 45 | } 46 | 47 | NCCL_TRACE(NCCL_ALL, "baguaNetDevices_v4, ndev=%d", *ndev); 48 | return ncclSuccess; 49 | } 50 | 51 | __hidden ncclResult_t baguaNetGetProperties_v4(int dev, ncclNetProperties_v4_t *props) 52 | { 53 | NCCLNetPropertiesC inner_props; 54 | int ret = BaguaNet::instance().get_properties(dev, &inner_props); 55 | if (ret != 0) 56 | { 57 | NCCL_WARN("baguaNetGetProperties_v4 failed, ret=%d, dev=%d", ret, dev); 58 | return ncclInternalError; 59 | } 60 | set_properties(*props, inner_props); 61 | NCCL_TRACE(NCCL_ALL, "baguaNetGetProperties_v4, dev=%d", dev); 62 | 63 | return ncclSuccess; 64 | } 65 | 66 | __hidden ncclResult_t baguaNetListen_v4(int dev, void *handle, void **listenComm) 67 | { 68 | int ret = BaguaNet::instance().listen(dev, handle, listenComm); 69 | if (ret != 0) 70 | { 71 | NCCL_WARN("baguaNetListen_v4 failed, ret=%d, dev=%d", ret, dev); 72 | return ncclInternalError; 73 | } 74 | NCCL_TRACE(NCCL_ALL, "baguaNetListen_v4, dev=%d", dev); 75 | 76 | return ncclSuccess; 77 | } 78 | 79 | __hidden ncclResult_t baguaNetConnect_v4(int dev, void *handle, void **sendComm) 80 | { 81 | int ret = BaguaNet::instance().connect(dev, handle, sendComm); 82 | if (ret != 0) 83 | { 84 | NCCL_WARN("baguaNetConnect_v4 failed, ret=%d", ret); 85 | return ncclInternalError; 86 | } 87 | NCCL_TRACE(NCCL_ALL, "baguaNetConnect_v4 ok, dev=%d", dev); 88 | 89 | return ncclSuccess; 90 | } 91 | 92 | __hidden ncclResult_t baguaNetAccept_v4(void *listenComm, void **recvComm) 93 | { 94 | int ret = BaguaNet::instance().accept(listenComm, recvComm); 95 | if (ret != 0) 96 | { 97 | NCCL_WARN("baguaNetAccept_v4 failed, ret=%d", ret); 98 | return ncclInternalError; 99 | } 100 | NCCL_TRACE(NCCL_ALL, "baguaNetAccept_v4, listenComm=%p", listenComm); 101 | 102 | return ncclSuccess; 103 | } 104 | 105 | __hidden ncclResult_t baguaNetRegMr_v4(void *comm, void *data, int size, int type, void **mhandle) 106 | { 107 | NCCL_TRACE(NCCL_ALL, "baguaNetRegMr_v4, comm=%p, data=%p, type=%d", comm, data, type); 108 | return (type != NCCL_PTR_HOST) ? ncclInternalError : ncclSuccess; 109 | } 110 | 111 | __hidden ncclResult_t baguaNetDeregMr_v4(void *comm, void *mhandle) 112 | { 113 | NCCL_TRACE(NCCL_ALL, "baguaNetDeregMr_v4, comm=%p", comm); 114 | return ncclSuccess; 115 | } 116 | 117 | __hidden ncclResult_t baguaNetIsend_v4(void *sendComm, void *data, int size, void *mhandle, void **request) 118 | { 119 | int ret = BaguaNet::instance().isend(sendComm, data, size, mhandle, request); 120 | if (ret != 0) 121 | { 122 | NCCL_WARN("baguaNetIsend_v4 failed, ret=%d, sendComm=%p, data=%p, size=%d", ret, sendComm, data, size); 123 | return ncclInternalError; 124 | } 125 | NCCL_TRACE(NCCL_ALL, "baguaNetIsend_v4, sendComm=%p, data=%p, size=%d, request_id=%d", 126 | sendComm, data, size, *(uintptr_t *)(*request)); 127 | 128 | return ncclSuccess; 129 | } 130 | 131 | __hidden ncclResult_t baguaNetIrecv_v4(void *recvComm, void *data, int size, void *mhandle, void **request) 132 | { 133 | int ret = BaguaNet::instance().irecv(recvComm, data, size, mhandle, request); 134 | if (ret != 0) 135 | { 136 | NCCL_WARN("baguaNetIrecv_v4 failed, ret=%d, sendComm=%p, data=%p, size=%d", ret, recvComm, data, size); 137 | return ncclInternalError; 138 | } 139 | NCCL_TRACE(NCCL_ALL, "baguaNetIrecv_v4, recvComm=%p, data=%p, size=%d, request_id=%d", 140 | recvComm, data, size, *(uintptr_t *)(*request)); 141 | 142 | return ncclSuccess; 143 | } 144 | 145 | __hidden ncclResult_t baguaNetFlush_v4(void *recvComm, void *data, int size, void *mhandle, void **request) 146 | { 147 | // We don't support CUDA pointers, so we don't need a flush operation 148 | return ncclInternalError; 149 | } 150 | 151 | __hidden ncclResult_t baguaNetTest_v4(void *request, int *done, int *size) 152 | { 153 | bool b_done = false; 154 | uintptr_t nbytes = 0; 155 | int ret = BaguaNet::instance().test(request, &b_done, &nbytes); 156 | if (ret != 0) 157 | { 158 | NCCL_WARN("baguaNetTest_v4 failed, ret=%d, request_id=%d", 159 | ret, *static_cast(request)); 160 | return ncclInternalError; 161 | } 162 | *done = b_done ? 1 : 0; 163 | if (b_done && size != NULL) 164 | { 165 | *size = nbytes; 166 | NCCL_TRACE(NCCL_ALL, "size=%d", *size); 167 | } 168 | NCCL_TRACE(NCCL_ALL, "baguaNetTest_v4, request_id=%d, done=%d, nbytes=%d", 169 | *static_cast(request), *done, nbytes); 170 | 171 | return ncclSuccess; 172 | } 173 | 174 | __hidden ncclResult_t baguaNetCloseSend_v4(void *sendComm) 175 | { 176 | int ret = BaguaNet::instance().close_send(sendComm); 177 | if (ret != 0) 178 | { 179 | NCCL_WARN("baguaNetCloseSend_v4 failed, ret=%d, sendComm=%p", ret, sendComm); 180 | return ncclInternalError; 181 | } 182 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseSend_v4, sendComm=%p", sendComm); 183 | return ncclSuccess; 184 | } 185 | 186 | __hidden ncclResult_t baguaNetCloseRecv_v4(void *recvComm) 187 | { 188 | int ret = BaguaNet::instance().close_recv(recvComm); 189 | if (ret != 0) 190 | { 191 | NCCL_WARN("baguaNetCloseRecv_v4 failed, ret=%d, recvComm=%p", ret, recvComm); 192 | return ncclInternalError; 193 | } 194 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseRecv_v4, recvComm=%p", recvComm); 195 | return ncclSuccess; 196 | } 197 | 198 | __hidden ncclResult_t baguaNetCloseListen_v4(void *listenComm) 199 | { 200 | int ret = BaguaNet::instance().close_listen(listenComm); 201 | if (ret != 0) 202 | { 203 | NCCL_WARN("baguaNetCloseListen_v4 failed, ret=%d, listenComm=%p", ret, listenComm); 204 | return ncclInternalError; 205 | } 206 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseListen_v4, listenComm=%p", listenComm); 207 | return ncclSuccess; 208 | } 209 | 210 | ncclNet_v4_t ncclNetPlugin_v4 = { 211 | "BaguaNet", 212 | baguaNetInit_v4, 213 | baguaNetDevices_v4, 214 | baguaNetGetProperties_v4, 215 | baguaNetListen_v4, 216 | baguaNetConnect_v4, 217 | baguaNetAccept_v4, 218 | baguaNetRegMr_v4, 219 | baguaNetDeregMr_v4, 220 | baguaNetIsend_v4, 221 | baguaNetIrecv_v4, 222 | baguaNetFlush_v4, 223 | baguaNetTest_v4, 224 | baguaNetCloseSend_v4, 225 | baguaNetCloseRecv_v4, 226 | baguaNetCloseListen_v4}; 227 | -------------------------------------------------------------------------------- /cc/v3/nccl_net_v3.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "nccl_net_v3.h" 9 | #include "bagua_net.h" 10 | 11 | #define __hidden __attribute__((visibility("hidden"))) 12 | 13 | ncclDebugLogger_t NCCL_DEBUG_LOG_V3; 14 | #define NCCL_TRACE(FLAGS, ...) NCCL_DEBUG_LOG_V3(NCCL_LOG_TRACE, (FLAGS), __func__, __LINE__, __VA_ARGS__) 15 | #define NCCL_INFO(FLAGS, ...) NCCL_DEBUG_LOG_V3(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__) 16 | #define NCCL_WARN(...) NCCL_DEBUG_LOG_V3(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) 17 | 18 | void set_properties_v3(ncclNetProperties_v3_t &lhs, NCCLNetPropertiesC &rhs) 19 | { 20 | lhs.name = const_cast(rhs.name); 21 | lhs.pciPath = const_cast(rhs.pci_path); 22 | lhs.guid = rhs.guid; 23 | lhs.ptrSupport = rhs.ptr_support; 24 | lhs.speed = rhs.speed; 25 | lhs.port = rhs.port; 26 | lhs.maxComms = rhs.max_comms; 27 | } 28 | 29 | __hidden ncclResult_t baguaNetInit_v3(ncclDebugLogger_t logFunction) 30 | { 31 | NCCL_DEBUG_LOG_V3 = logFunction; 32 | BaguaNet::instance(); 33 | 34 | NCCL_TRACE(NCCL_ALL, "baguaNetInit_v3!"); 35 | 36 | return ncclSuccess; 37 | } 38 | 39 | __hidden ncclResult_t baguaNetDevices_v3(int *ndev) 40 | { 41 | if (BaguaNet::instance().devices((int32_t *)ndev) != 0) 42 | { 43 | NCCL_WARN("baguaNetDevices_v3 failed, ndev=%d", *ndev); 44 | return ncclInternalError; 45 | } 46 | 47 | NCCL_TRACE(NCCL_ALL, "baguaNetDevices_v3, ndev=%d", *ndev); 48 | return ncclSuccess; 49 | } 50 | 51 | __hidden ncclResult_t baguaNetGetProperties_v3(int dev, ncclNetProperties_v3_t *props) 52 | { 53 | NCCLNetPropertiesC inner_props; 54 | int ret = BaguaNet::instance().get_properties(dev, &inner_props); 55 | if (ret != 0) 56 | { 57 | NCCL_WARN("baguaNetGetProperties_v3 failed, ret=%d, dev=%d", ret, dev); 58 | return ncclInternalError; 59 | } 60 | set_properties_v3(*props, inner_props); 61 | NCCL_TRACE(NCCL_ALL, "baguaNetGetProperties_v3, dev=%d", dev); 62 | 63 | return ncclSuccess; 64 | } 65 | 66 | __hidden ncclResult_t baguaNetListen_v3(int dev, void *handle, void **listenComm) 67 | { 68 | int ret = BaguaNet::instance().listen(dev, handle, listenComm); 69 | if (ret != 0) 70 | { 71 | NCCL_WARN("baguaNetListen_v3 failed, ret=%d, dev=%d", ret, dev); 72 | return ncclInternalError; 73 | } 74 | NCCL_TRACE(NCCL_ALL, "baguaNetListen_v3, dev=%d", dev); 75 | 76 | return ncclSuccess; 77 | } 78 | 79 | __hidden ncclResult_t baguaNetConnect_v3(int dev, void *handle, void **sendComm) 80 | { 81 | int ret = BaguaNet::instance().connect(dev, handle, sendComm); 82 | if (ret != 0) 83 | { 84 | NCCL_WARN("baguaNetConnect_v3 failed, ret=%d", ret); 85 | return ncclInternalError; 86 | } 87 | NCCL_TRACE(NCCL_ALL, "baguaNetConnect_v3 ok, dev=%d", dev); 88 | 89 | return ncclSuccess; 90 | } 91 | 92 | __hidden ncclResult_t baguaNetAccept_v3(void *listenComm, void **recvComm) 93 | { 94 | int ret = BaguaNet::instance().accept(listenComm, recvComm); 95 | if (ret != 0) 96 | { 97 | NCCL_WARN("baguaNetAccept_v3 failed, ret=%d", ret); 98 | return ncclInternalError; 99 | } 100 | NCCL_TRACE(NCCL_ALL, "baguaNetAccept_v3, listenComm=%p", listenComm); 101 | 102 | return ncclSuccess; 103 | } 104 | 105 | __hidden ncclResult_t baguaNetRegMr_v3(void *comm, void *data, int size, int type, void **mhandle) 106 | { 107 | NCCL_TRACE(NCCL_ALL, "baguaNetRegMr_v3, comm=%p, data=%p, type=%d", comm, data, type); 108 | return (type != NCCL_PTR_HOST) ? ncclInternalError : ncclSuccess; 109 | } 110 | 111 | __hidden ncclResult_t baguaNetDeregMr_v3(void *comm, void *mhandle) 112 | { 113 | NCCL_TRACE(NCCL_ALL, "baguaNetDeregMr_v3, comm=%p", comm); 114 | return ncclSuccess; 115 | } 116 | 117 | __hidden ncclResult_t baguaNetIsend_v3(void *sendComm, void *data, int size, void *mhandle, void **request) 118 | { 119 | int ret = BaguaNet::instance().isend(sendComm, data, size, mhandle, request); 120 | if (ret != 0) 121 | { 122 | NCCL_WARN("baguaNetIsend_v3 failed, ret=%d, sendComm=%p, data=%p, size=%d", ret, sendComm, data, size); 123 | return ncclInternalError; 124 | } 125 | NCCL_TRACE(NCCL_ALL, "baguaNetIsend_v3, sendComm=%p, data=%p, size=%d, request_id=%d", 126 | sendComm, data, size, *(uintptr_t *)(*request)); 127 | 128 | return ncclSuccess; 129 | } 130 | 131 | __hidden ncclResult_t baguaNetIrecv_v3(void *recvComm, void *data, int size, void *mhandle, void **request) 132 | { 133 | int ret = BaguaNet::instance().irecv(recvComm, data, size, mhandle, request); 134 | if (ret != 0) 135 | { 136 | NCCL_WARN("baguaNetIrecv_v3 failed, ret=%d, sendComm=%p, data=%p, size=%d", ret, recvComm, data, size); 137 | return ncclInternalError; 138 | } 139 | NCCL_TRACE(NCCL_ALL, "baguaNetIrecv_v3, recvComm=%p, data=%p, size=%d, request_id=%d", 140 | recvComm, data, size, *(uintptr_t *)(*request)); 141 | 142 | return ncclSuccess; 143 | } 144 | 145 | __hidden ncclResult_t baguaNetFlush_v3(void *recvComm, void *data, int size, void *mhandle) 146 | { 147 | // We don't support CUDA pointers, so we don't need a flush operation 148 | return ncclInternalError; 149 | } 150 | 151 | __hidden ncclResult_t baguaNetTest_v3(void *request, int *done, int *size) 152 | { 153 | bool b_done = false; 154 | uintptr_t nbytes = 0; 155 | int ret = BaguaNet::instance().test(request, &b_done, &nbytes); 156 | if (ret != 0) 157 | { 158 | NCCL_WARN("baguaNetTest_v3 failed, ret=%d, request_id=%d", 159 | ret, *static_cast(request)); 160 | return ncclInternalError; 161 | } 162 | *done = b_done ? 1 : 0; 163 | if (b_done && size != NULL) 164 | { 165 | *size = nbytes; 166 | NCCL_TRACE(NCCL_ALL, "size=%d", *size); 167 | } 168 | NCCL_TRACE(NCCL_ALL, "baguaNetTest_v3, request_id=%d, done=%d, nbytes=%d", 169 | *static_cast(request), *done, nbytes); 170 | 171 | return ncclSuccess; 172 | } 173 | 174 | __hidden ncclResult_t baguaNetCloseSend_v3(void *sendComm) 175 | { 176 | int ret = BaguaNet::instance().close_send(sendComm); 177 | if (ret != 0) 178 | { 179 | NCCL_WARN("baguaNetCloseSend_v3 failed, ret=%d, sendComm=%p", ret, sendComm); 180 | return ncclInternalError; 181 | } 182 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseSend_v3, sendComm=%p", sendComm); 183 | return ncclSuccess; 184 | } 185 | 186 | __hidden ncclResult_t baguaNetCloseRecv_v3(void *recvComm) 187 | { 188 | int ret = BaguaNet::instance().close_recv(recvComm); 189 | if (ret != 0) 190 | { 191 | NCCL_WARN("baguaNetCloseRecv_v3 failed, ret=%d, recvComm=%p", ret, recvComm); 192 | return ncclInternalError; 193 | } 194 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseRecv_v3, recvComm=%p", recvComm); 195 | return ncclSuccess; 196 | } 197 | 198 | __hidden ncclResult_t baguaNetCloseListen_v3(void *listenComm) 199 | { 200 | int ret = BaguaNet::instance().close_listen(listenComm); 201 | if (ret != 0) 202 | { 203 | NCCL_WARN("baguaNetCloseListen_v3 failed, ret=%d, listenComm=%p", ret, listenComm); 204 | return ncclInternalError; 205 | } 206 | NCCL_TRACE(NCCL_ALL, "baguaNetCloseListen_v3, listenComm=%p", listenComm); 207 | return ncclSuccess; 208 | } 209 | 210 | ncclNet_v3_t ncclNetPlugin_v3 = { 211 | "BaguaNet", 212 | baguaNetInit_v3, 213 | baguaNetDevices_v3, 214 | baguaNetGetProperties_v3, 215 | baguaNetListen_v3, 216 | baguaNetConnect_v3, 217 | baguaNetAccept_v3, 218 | baguaNetRegMr_v3, 219 | baguaNetDeregMr_v3, 220 | baguaNetIsend_v3, 221 | baguaNetIrecv_v3, 222 | baguaNetFlush_v3, 223 | baguaNetTest_v3, 224 | baguaNetCloseSend_v3, 225 | baguaNetCloseRecv_v3, 226 | baguaNetCloseListen_v3}; 227 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate lazy_static; 3 | 4 | mod implement; 5 | mod interface; 6 | mod utils; 7 | 8 | use ffi_convert::{AsRust, CDrop, CReprOf}; 9 | use implement::{nthread_per_socket_backend, tokio_backend}; 10 | use interface::{NCCLNetProperties, Net, SocketHandle}; 11 | use libc; 12 | use std::sync::{Arc, Mutex}; 13 | 14 | pub struct BaguaNetC { 15 | inner: Arc>>, 16 | } 17 | 18 | #[no_mangle] 19 | pub extern "C" fn bagua_net_c_create() -> *mut BaguaNetC { 20 | let config = std::env::var("BAGUA_NET_IMPLEMENT") 21 | .unwrap_or("BASIC".to_owned()) 22 | .to_uppercase(); 23 | let bagua_net: Box = match &config[..] { 24 | "TOKIO" => Box::new(tokio_backend::BaguaNet::new().unwrap()), 25 | "BASIC" => Box::new(nthread_per_socket_backend::BaguaNet::new().unwrap()), 26 | _ => { 27 | return std::ptr::null_mut(); 28 | } 29 | }; 30 | let obj = BaguaNetC { 31 | inner: Arc::new(Mutex::new(bagua_net)), 32 | }; 33 | 34 | // into_raw turns the Box into a *mut, which the borrow checker 35 | // ignores, without calling its destructor. 36 | Box::into_raw(Box::new(obj)) 37 | } 38 | 39 | #[no_mangle] 40 | pub extern "C" fn bagua_net_c_destroy(ptr: &mut *mut BaguaNetC) { 41 | // First, we **must** check to see if the pointer is null. 42 | if ptr.is_null() { 43 | // Do nothing. 44 | return; 45 | } 46 | 47 | // Now we know the pointer is non-null, we can continue. from_raw is the 48 | // inverse of into_raw: it turns the *mut Dramatic back into a 49 | // Box. You must only call from_raw once per pointer. 50 | let obj: Box = unsafe { Box::from_raw(*ptr) }; 51 | 52 | // We don't *have* to do anything else; once obj goes out of scope, it will 53 | // be dropped. I'm going to drop it explicitly, however, for clarity. 54 | drop(obj); 55 | 56 | // I am, however, going to null out the `ptr` we were passed just so the 57 | // calling code is less likely to accidentally re-use the pointer. 58 | *ptr = ::std::ptr::null_mut(); 59 | } 60 | 61 | /// Error code 62 | /// 0: success 63 | /// -1: null pointer 64 | #[no_mangle] 65 | pub extern "C" fn bagua_net_c_devices(ptr: *mut BaguaNetC, ndev: *mut i32) -> i32 { 66 | // First, we **must** check to see if the pointer is null. 67 | if ptr.is_null() { 68 | // Do nothing. 69 | return -1; 70 | } 71 | 72 | unsafe { 73 | *ndev = (*ptr).inner.lock().unwrap().devices().unwrap() as i32; 74 | } 75 | return 0; 76 | } 77 | 78 | #[repr(C)] 79 | #[derive(Debug, CReprOf, AsRust, CDrop)] 80 | #[target_type(NCCLNetProperties)] 81 | pub struct NCCLNetPropertiesC { 82 | pub name: *const libc::c_char, 83 | pub pci_path: *const libc::c_char, 84 | guid: u64, 85 | ptr_support: i32, 86 | speed: i32, 87 | port: i32, 88 | max_comms: i32, 89 | } 90 | 91 | /// Error code 92 | /// 0: success 93 | /// -1: null pointer 94 | #[no_mangle] 95 | pub extern "C" fn bagua_net_c_get_properties( 96 | ptr: *mut BaguaNetC, 97 | dev_id: i32, 98 | props: *mut NCCLNetPropertiesC, 99 | ) -> i32 { 100 | // First, we **must** check to see if the pointer is null. 101 | if ptr.is_null() { 102 | // Do nothing. 103 | return -1; 104 | } 105 | if dev_id < 0 { 106 | return -2; 107 | } 108 | 109 | unsafe { 110 | let props_raw = (*ptr) 111 | .inner 112 | .lock() 113 | .unwrap() 114 | .get_properties(dev_id as usize) 115 | .unwrap(); 116 | *props = NCCLNetPropertiesC::c_repr_of(props_raw).unwrap(); 117 | } 118 | return 0; 119 | } 120 | 121 | #[repr(C)] 122 | pub struct SocketHandleC { 123 | pub sockaddr: libc::sockaddr, 124 | } 125 | 126 | #[repr(C)] 127 | pub struct SocketListenCommC { 128 | pub id: usize, 129 | } 130 | 131 | /// Error code 132 | /// 0: success 133 | /// -1: null pointer 134 | /// -2: invalid parameter 135 | /// -3: listen failed 136 | #[no_mangle] 137 | pub extern "C" fn bagua_net_c_listen( 138 | ptr: *mut BaguaNetC, 139 | dev_id: i32, 140 | socket_handle: *mut SocketHandleC, 141 | socket_listen_comm_id: *mut usize, 142 | ) -> i32 { 143 | // First, we **must** check to see if the pointer is null. 144 | if ptr.is_null() { 145 | // Do nothing. 146 | return -1; 147 | } 148 | if dev_id < 0 { 149 | return -2; 150 | } 151 | 152 | unsafe { 153 | let (handle, id) = match (*ptr).inner.lock().unwrap().listen(dev_id as usize) { 154 | Ok(result) => result, 155 | Err(_err) => return -3, 156 | }; 157 | let (sockaddr, _) = handle.addr.as_ffi_pair(); 158 | (*socket_handle).sockaddr = *sockaddr; 159 | *socket_listen_comm_id = id as usize; 160 | } 161 | return 0; 162 | } 163 | 164 | /// Error code 165 | /// 0: success 166 | /// -1: null pointer 167 | /// -2: invalid parameter 168 | /// -3: connect failed 169 | #[no_mangle] 170 | #[cfg(target_os = "linux")] 171 | pub extern "C" fn bagua_net_c_connect( 172 | ptr: *mut BaguaNetC, 173 | dev_id: i32, 174 | socket_handle: *mut SocketHandleC, 175 | socket_send_comm_id: *mut usize, 176 | ) -> i32 { 177 | // First, we **must** check to see if the pointer is null. 178 | if ptr.is_null() { 179 | // Do nothing. 180 | return -1; 181 | } 182 | if dev_id < 0 { 183 | return -2; 184 | } 185 | 186 | unsafe { 187 | let sockaddr = (*socket_handle).sockaddr; 188 | 189 | let sockaddr = libc::sockaddr { 190 | sa_family: sockaddr.sa_family, 191 | sa_data: sockaddr.sa_data, 192 | }; 193 | 194 | *socket_send_comm_id = match (*ptr).inner.lock().unwrap().connect( 195 | dev_id as usize, 196 | SocketHandle { 197 | addr: utils::from_libc_sockaddr(&sockaddr).unwrap(), 198 | }, 199 | ) { 200 | Ok(id) => id, 201 | Err(_err) => return -3, 202 | } 203 | } 204 | return 0; 205 | } 206 | 207 | /// Error code 208 | /// 0: success 209 | /// -1: null pointer 210 | #[no_mangle] 211 | pub extern "C" fn bagua_net_c_accept( 212 | ptr: *mut BaguaNetC, 213 | listen_comm_id: usize, 214 | recv_comm_id: *mut usize, 215 | ) -> i32 { 216 | // First, we **must** check to see if the pointer is null. 217 | if ptr.is_null() { 218 | // Do nothing. 219 | return -1; 220 | } 221 | 222 | unsafe { 223 | *recv_comm_id = (*ptr).inner.lock().unwrap().accept(listen_comm_id).unwrap(); 224 | } 225 | return 0; 226 | } 227 | 228 | #[repr(C)] 229 | pub struct Buffer { 230 | data: *mut u8, 231 | len: usize, 232 | } 233 | 234 | /// Error code 235 | /// 0: success 236 | /// -1: null pointer 237 | #[no_mangle] 238 | pub extern "C" fn bagua_net_c_isend( 239 | ptr: *mut BaguaNetC, 240 | send_comm_id: usize, 241 | buf: Buffer, 242 | request_id: *mut usize, 243 | ) -> i32 { 244 | // First, we **must** check to see if the pointer is null. 245 | if ptr.is_null() { 246 | // Do nothing. 247 | return -1; 248 | } 249 | 250 | unsafe { 251 | let data: &'static mut [u8] = std::slice::from_raw_parts_mut(buf.data, buf.len); 252 | *request_id = (*ptr) 253 | .inner 254 | .lock() 255 | .unwrap() 256 | .isend(send_comm_id, data) 257 | .unwrap(); 258 | } 259 | return 0; 260 | } 261 | 262 | /// Error code 263 | /// 0: success 264 | /// -1: null pointer 265 | #[no_mangle] 266 | pub extern "C" fn bagua_net_c_irecv( 267 | ptr: *mut BaguaNetC, 268 | recv_comm_id: usize, 269 | buf: Buffer, 270 | request_id: *mut usize, 271 | ) -> i32 { 272 | // First, we **must** check to see if the pointer is null. 273 | if ptr.is_null() { 274 | // Do nothing. 275 | return -1; 276 | } 277 | 278 | unsafe { 279 | let data: &'static mut [u8] = std::slice::from_raw_parts_mut(buf.data, buf.len); 280 | *request_id = (*ptr) 281 | .inner 282 | .lock() 283 | .unwrap() 284 | .irecv(recv_comm_id, data) 285 | .unwrap(); 286 | } 287 | return 0; 288 | } 289 | 290 | /// Error code 291 | /// 0: success 292 | /// -1: null pointer 293 | /// -2: invalid parameter 294 | /// -3: bagua-net inner error 295 | #[no_mangle] 296 | pub extern "C" fn bagua_net_c_test( 297 | ptr: *mut BaguaNetC, 298 | request_id: usize, 299 | done: *mut bool, 300 | bytes: *mut usize, 301 | ) -> i32 { 302 | // First, we **must** check to see if the pointer is null. 303 | if ptr.is_null() { 304 | // Do nothing. 305 | return -1; 306 | } 307 | if done.is_null() { 308 | return -2; 309 | } 310 | 311 | unsafe { 312 | match (*ptr).inner.lock().unwrap().test(request_id) { 313 | Ok((let_done, let_bytes)) => { 314 | *done = let_done; 315 | if let_done && !bytes.is_null() { 316 | *bytes = let_bytes; 317 | } 318 | } 319 | Err(err) => { 320 | tracing::warn!("{:?}", err); 321 | return -3; 322 | } 323 | } 324 | } 325 | return 0; 326 | } 327 | 328 | /// Error code 329 | /// 0: success 330 | /// -1: null pointer 331 | #[no_mangle] 332 | pub extern "C" fn bagua_net_c_close_send(ptr: *mut BaguaNetC, send_comm_id: usize) -> i32 { 333 | // First, we **must** check to see if the pointer is null. 334 | if ptr.is_null() { 335 | // Do nothing. 336 | return -1; 337 | } 338 | 339 | unsafe { 340 | (*ptr) 341 | .inner 342 | .lock() 343 | .unwrap() 344 | .close_send(send_comm_id) 345 | .unwrap(); 346 | } 347 | return 0; 348 | } 349 | 350 | /// Error code 351 | /// 0: success 352 | /// -1: null pointer 353 | #[no_mangle] 354 | pub extern "C" fn bagua_net_c_close_recv(ptr: *mut BaguaNetC, recv_comm_id: usize) -> i32 { 355 | // First, we **must** check to see if the pointer is null. 356 | if ptr.is_null() { 357 | // Do nothing. 358 | return -1; 359 | } 360 | 361 | unsafe { 362 | (*ptr) 363 | .inner 364 | .lock() 365 | .unwrap() 366 | .close_recv(recv_comm_id) 367 | .unwrap(); 368 | } 369 | return 0; 370 | } 371 | 372 | /// Error code 373 | /// 0: success 374 | /// -1: null pointer 375 | #[no_mangle] 376 | pub extern "C" fn bagua_net_c_close_listen(ptr: *mut BaguaNetC, listen_comm_id: usize) -> i32 { 377 | // First, we **must** check to see if the pointer is null. 378 | if ptr.is_null() { 379 | // Do nothing. 380 | return -1; 381 | } 382 | 383 | unsafe { 384 | (*ptr) 385 | .inner 386 | .lock() 387 | .unwrap() 388 | .close_listen(listen_comm_id) 389 | .unwrap(); 390 | } 391 | return 0; 392 | } 393 | -------------------------------------------------------------------------------- /src/utils.rs: -------------------------------------------------------------------------------- 1 | use nix::net::if_::InterfaceFlags; 2 | use nix::sys::socket::{AddressFamily, InetAddr, SockAddr}; 3 | use std::fs; 4 | use std::io; 5 | use std::io::{Read, Write}; 6 | 7 | pub fn get_net_if_speed(device: &str) -> i32 { 8 | const DEFAULT_SPEED: i32 = 10000; 9 | 10 | let speed_path = format!("/sys/class/net/{}/speed", device); 11 | match fs::read_to_string(speed_path.clone()) { 12 | Ok(speed_str) => { 13 | return speed_str.trim().parse().unwrap_or(DEFAULT_SPEED); 14 | } 15 | Err(_) => { 16 | tracing::debug!( 17 | "Could not get speed from {}. Defaulting to 10 Gbps.", 18 | speed_path 19 | ); 20 | DEFAULT_SPEED 21 | } 22 | } 23 | } 24 | 25 | #[derive(Debug, Clone)] 26 | pub struct NCCLSocketDev { 27 | pub interface_name: String, 28 | pub addr: SockAddr, 29 | pub pci_path: String, 30 | } 31 | 32 | pub fn find_interfaces() -> Vec { 33 | let nccl_socket_family = std::env::var("NCCL_SOCKET_FAMILY") 34 | .unwrap_or("-1".to_string()) 35 | .parse::() 36 | .unwrap_or(-1); 37 | let nccl_socket_ifname = 38 | std::env::var("NCCL_SOCKET_IFNAME").unwrap_or("^docker,lo".to_string()); 39 | // TODO @shjwudp: support parse sockaddr from NCCL_COMM_ID 40 | 41 | let mut search_not = Vec::<&str>::new(); 42 | let mut search_exact = Vec::<&str>::new(); 43 | if nccl_socket_ifname.starts_with("^") { 44 | search_not = nccl_socket_ifname[1..].split(",").collect(); 45 | } else if nccl_socket_ifname.starts_with("=") { 46 | search_exact = nccl_socket_ifname[1..].split(",").collect(); 47 | } else { 48 | search_exact = nccl_socket_ifname.split(",").collect(); 49 | } 50 | 51 | let mut socket_devs = Vec::::new(); 52 | const MAX_IF_NAME_SIZE: usize = 16; 53 | let addrs = nix::ifaddrs::getifaddrs().unwrap(); 54 | for ifaddr in addrs { 55 | match ifaddr.address { 56 | Some(addr) => { 57 | if addr.family() != AddressFamily::Inet && addr.family() != AddressFamily::Inet6 { 58 | continue; 59 | } 60 | if ifaddr.flags.contains(InterfaceFlags::IFF_LOOPBACK) { 61 | continue; 62 | } 63 | 64 | assert_eq!(ifaddr.interface_name.len() < MAX_IF_NAME_SIZE, true); 65 | let found_ifs: Vec<&NCCLSocketDev> = socket_devs 66 | .iter() 67 | .filter(|scoket_dev| scoket_dev.interface_name == ifaddr.interface_name) 68 | .collect(); 69 | if found_ifs.len() > 0 { 70 | continue; 71 | } 72 | 73 | let pci_path = format!("/sys/class/net/{}/device", ifaddr.interface_name); 74 | let pci_path: String = match std::fs::canonicalize(pci_path) { 75 | Ok(pci_path) => pci_path.to_str().unwrap_or("").to_string(), 76 | Err(_) => "".to_string(), 77 | }; 78 | 79 | socket_devs.push(NCCLSocketDev { 80 | addr: addr, 81 | interface_name: ifaddr.interface_name.clone(), 82 | pci_path: pci_path, 83 | }) 84 | } 85 | None => { 86 | tracing::warn!( 87 | "interface {} with unsupported address family", 88 | ifaddr.interface_name 89 | ); 90 | } 91 | } 92 | } 93 | 94 | let search_not = &mut search_not; 95 | let search_exact = &mut search_exact; 96 | let socket_devs = socket_devs 97 | .iter() 98 | .filter({ 99 | |socket_dev| -> bool { 100 | let (sockaddr, _) = socket_dev.addr.as_ffi_pair(); 101 | if nccl_socket_family != -1 && sockaddr.sa_family as i32 != nccl_socket_family { 102 | return false; 103 | } 104 | 105 | for not_interface in &*search_not { 106 | if socket_dev.interface_name.starts_with(not_interface) { 107 | return false; 108 | } 109 | } 110 | if !(&*search_exact).is_empty() { 111 | let mut ok = false; 112 | for exact_interface in &*search_exact { 113 | if socket_dev.interface_name.starts_with(exact_interface) { 114 | ok = true; 115 | break; 116 | } 117 | } 118 | if !ok { 119 | return false; 120 | } 121 | } 122 | 123 | return true; 124 | } 125 | }) 126 | .cloned() 127 | .collect(); 128 | 129 | socket_devs 130 | } 131 | 132 | pub fn nonblocking_write_all(stream: &mut std::net::TcpStream, mut buf: &[u8]) -> io::Result<()> { 133 | while !buf.is_empty() { 134 | match stream.write(buf) { 135 | Ok(0) => { 136 | return Err(io::Error::new( 137 | io::ErrorKind::WriteZero, 138 | "failed to write whole buffer", 139 | )); 140 | } 141 | Ok(n) => buf = &buf[n..], 142 | Err(ref e) 143 | if e.kind() == io::ErrorKind::Interrupted 144 | || e.kind() == io::ErrorKind::WouldBlock => {} 145 | Err(e) => return Err(e), 146 | } 147 | std::thread::yield_now(); 148 | } 149 | Ok(()) 150 | } 151 | 152 | pub fn nonblocking_read_exact( 153 | stream: &mut std::net::TcpStream, 154 | mut buf: &mut [u8], 155 | ) -> io::Result<()> { 156 | while !buf.is_empty() { 157 | match stream.read(buf) { 158 | Ok(0) => break, 159 | Ok(n) => { 160 | let tmp = buf; 161 | buf = &mut tmp[n..]; 162 | } 163 | Err(ref e) 164 | if e.kind() == io::ErrorKind::Interrupted 165 | || e.kind() == io::ErrorKind::WouldBlock => {} 166 | Err(e) => return Err(e), 167 | } 168 | std::thread::yield_now(); 169 | } 170 | if !buf.is_empty() { 171 | Err(io::Error::new( 172 | io::ErrorKind::UnexpectedEof, 173 | "failed to fill whole buffer", 174 | )) 175 | } else { 176 | Ok(()) 177 | } 178 | } 179 | 180 | pub fn parse_user_pass_and_addr(raw_url: &str) -> Option<(String, String, String)> { 181 | let re = regex::Regex::new(r"^(?:([^:]+):([^@]+)@)?(\S+)$").unwrap(); 182 | match re.captures(raw_url) { 183 | Some(caps) => { 184 | let username = match caps.get(1) { 185 | Some(username) => username.as_str().to_owned(), 186 | None => Default::default(), 187 | }; 188 | let password = match caps.get(2) { 189 | Some(password) => password.as_str().to_owned(), 190 | None => Default::default(), 191 | }; 192 | let address = caps.get(3).unwrap().as_str().to_owned(); 193 | 194 | Some((username, password, address)) 195 | } 196 | None => None, 197 | } 198 | } 199 | 200 | pub fn chunk_size(total: usize, min_chunksize: usize, expected_nchunks: usize) -> usize { 201 | let chunk_size = (total + expected_nchunks - 1) / expected_nchunks; 202 | let chunk_size = std::cmp::max(chunk_size, min_chunksize); 203 | 204 | chunk_size 205 | } 206 | 207 | /// Creates a `SockAddr` struct from libc's sockaddr. 208 | /// 209 | /// Supports only the following address families: Unix, Inet (v4 & v6), Netlink and System. 210 | /// Returns None for unsupported families. 211 | /// 212 | /// # Safety 213 | /// 214 | /// unsafe because it takes a raw pointer as argument. The caller must 215 | /// ensure that the pointer is valid. 216 | #[cfg(not(target_os = "fuchsia"))] 217 | pub(crate) unsafe fn from_libc_sockaddr(addr: *const libc::sockaddr) -> Option { 218 | if addr.is_null() { 219 | None 220 | } else { 221 | match AddressFamily::from_i32(i32::from((*addr).sa_family)) { 222 | Some(AddressFamily::Unix) => None, 223 | Some(AddressFamily::Inet) => Some(SockAddr::Inet(InetAddr::V4( 224 | *(addr as *const libc::sockaddr_in), 225 | ))), 226 | Some(AddressFamily::Inet6) => Some(SockAddr::Inet(InetAddr::V6( 227 | *(addr as *const libc::sockaddr_in6), 228 | ))), 229 | // #[cfg(any(target_os = "android", target_os = "linux"))] 230 | // Some(AddressFamily::Netlink) => Some(SockAddr::Netlink( 231 | // NetlinkAddr(*(addr as *const libc::sockaddr_nl)))), 232 | // #[cfg(any(target_os = "ios", target_os = "macos"))] 233 | // Some(AddressFamily::System) => Some(SockAddr::SysControl( 234 | // SysControlAddr(*(addr as *const libc::sockaddr_ctl)))), 235 | // #[cfg(any(target_os = "android", target_os = "linux"))] 236 | // Some(AddressFamily::Packet) => Some(SockAddr::Link( 237 | // LinkAddr(*(addr as *const libc::sockaddr_ll)))), 238 | // #[cfg(any(target_os = "dragonfly", 239 | // target_os = "freebsd", 240 | // target_os = "ios", 241 | // target_os = "macos", 242 | // target_os = "netbsd", 243 | // target_os = "illumos", 244 | // target_os = "openbsd"))] 245 | // Some(AddressFamily::Link) => { 246 | // let ether_addr = LinkAddr(*(addr as *const libc::sockaddr_dl)); 247 | // if ether_addr.is_empty() { 248 | // None 249 | // } else { 250 | // Some(SockAddr::Link(ether_addr)) 251 | // } 252 | // }, 253 | // #[cfg(any(target_os = "android", target_os = "linux"))] 254 | // Some(AddressFamily::Vsock) => Some(SockAddr::Vsock( 255 | // VsockAddr(*(addr as *const libc::sockaddr_vm)))), 256 | // Other address families are currently not supported and simply yield a None 257 | // entry instead of a proper conversion to a `SockAddr`. 258 | Some(_) | None => None, 259 | } 260 | } 261 | } 262 | 263 | #[cfg(test)] 264 | mod tests { 265 | use super::*; 266 | use nix::sys::socket::{InetAddr, IpAddr, SockAddr}; 267 | 268 | #[test] 269 | fn test_parse() { 270 | let username = "nagle"; 271 | let password = "1984"; 272 | let address = "127.0.0.1:9090"; 273 | 274 | let (user, pass, addr) = 275 | parse_user_pass_and_addr(&format!("{}:{}@{}", username, password, address)).unwrap(); 276 | assert_eq!(user, username); 277 | assert_eq!(pass, password); 278 | assert_eq!(addr, address); 279 | 280 | let (user, pass, addr) = parse_user_pass_and_addr(address).unwrap(); 281 | assert_eq!(user, ""); 282 | assert_eq!(pass, ""); 283 | assert_eq!(addr, address); 284 | } 285 | 286 | #[test] 287 | fn test_socket_handle() { 288 | let addr = InetAddr::new(IpAddr::new_v4(127, 0, 0, 1), 8123); 289 | let addr = SockAddr::new_inet(addr); 290 | let addr = unsafe { 291 | let (c_sockaddr, _) = addr.as_ffi_pair(); 292 | from_libc_sockaddr(c_sockaddr).unwrap() 293 | }; 294 | 295 | assert_eq!(addr.to_str(), "127.0.0.1:8123"); 296 | } 297 | 298 | #[test] 299 | fn test_chunks() { 300 | let chunks = |total: usize, min_chunksize: usize, expected_nchunks: usize| -> usize { 301 | let size = chunk_size(total, min_chunksize, expected_nchunks); 302 | 303 | let mut chunk_count = total / size; 304 | if total % size != 0 { 305 | chunk_count += 1; 306 | } 307 | 308 | chunk_count 309 | }; 310 | 311 | assert_eq!(chunks(1024, 1, 20), 20); 312 | assert_eq!(chunks(1024, 1000, 20), 2); 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /src/implement/nthread_per_socket_backend.rs: -------------------------------------------------------------------------------- 1 | use crate::interface; 2 | use crate::interface::{ 3 | BaguaNetError, NCCLNetProperties, Net, SocketHandle, SocketListenCommID, SocketRecvCommID, 4 | SocketRequestID, SocketSendCommID, 5 | }; 6 | use crate::utils; 7 | use crate::utils::NCCLSocketDev; 8 | use nix::sys::socket::{InetAddr, SockAddr}; 9 | use opentelemetry::{ 10 | metrics::{BoundValueRecorder, ObserverResult}, 11 | trace::{Span, TraceContextExt, Tracer}, 12 | KeyValue, 13 | }; 14 | use socket2::{Domain, Socket, Type}; 15 | use std::collections::HashMap; 16 | use std::io::{Read, Write}; 17 | use std::net; 18 | use std::sync::{Arc, Mutex}; 19 | 20 | const NCCL_PTR_HOST: i32 = 1; 21 | const NCCL_PTR_CUDA: i32 = 2; 22 | 23 | lazy_static! { 24 | static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; 25 | } 26 | 27 | pub struct SocketListenComm { 28 | pub tcp_listener: Arc>, 29 | } 30 | 31 | // TODO: make Rotating communicator 32 | #[derive(Clone)] 33 | pub struct SocketSendComm { 34 | pub tcp_sender: Arc>, 35 | pub msg_sender: flume::Sender<(&'static [u8], Arc>)>, 36 | } 37 | 38 | #[derive(Clone)] 39 | pub struct SocketRecvComm { 40 | pub tcp_sender: Arc>, 41 | pub msg_sender: flume::Sender<(&'static mut [u8], Arc>)>, 42 | } 43 | 44 | pub struct SocketSendRequest { 45 | pub state: Arc>, 46 | pub trace_span: opentelemetry::global::BoxedSpan, 47 | } 48 | 49 | pub struct SocketRecvRequest { 50 | pub state: Arc>, 51 | pub trace_span: opentelemetry::global::BoxedSpan, 52 | } 53 | 54 | #[derive(Debug)] 55 | pub struct RequestState { 56 | pub nsubtasks: usize, 57 | pub completed_subtasks: usize, 58 | pub nbytes_transferred: usize, 59 | pub err: Option, 60 | } 61 | 62 | pub enum SocketRequest { 63 | SendRequest(SocketSendRequest), 64 | RecvRequest(SocketRecvRequest), 65 | } 66 | 67 | static TELEMETRY_INIT_ONCE: std::sync::Once = std::sync::Once::new(); 68 | // static TELEMETRY_GUARD: Option = None; 69 | 70 | struct AppState { 71 | exporter: opentelemetry_prometheus::PrometheusExporter, 72 | isend_nbytes_gauge: BoundValueRecorder<'static, u64>, 73 | irecv_nbytes_gauge: BoundValueRecorder<'static, u64>, 74 | isend_nbytes_per_second: Arc>, 75 | isend_percentage_of_effective_time: Arc>, 76 | // isend_nbytes_gauge: BoundValueRecorder<'static, u64>, 77 | // irecv_nbytes_gauge: BoundValueRecorder<'static, u64>, 78 | uploader: std::thread::JoinHandle<()>, 79 | } 80 | 81 | pub struct BaguaNet { 82 | pub socket_devs: Vec, 83 | pub listen_comm_next_id: usize, 84 | pub listen_comm_map: HashMap, 85 | pub send_comm_next_id: usize, 86 | pub send_comm_map: HashMap, 87 | pub recv_comm_next_id: usize, 88 | pub recv_comm_map: HashMap, 89 | pub socket_request_next_id: usize, 90 | pub socket_request_map: HashMap, 91 | pub trace_span_context: opentelemetry::Context, 92 | pub trace_on_flag: bool, 93 | pub rank: i32, 94 | state: Arc, 95 | nstreams: usize, 96 | min_chunksize: usize, 97 | } 98 | 99 | impl BaguaNet { 100 | const DEFAULT_SOCKET_MAX_COMMS: i32 = 65536; 101 | const DEFAULT_LISTEN_BACKLOG: i32 = 16384; 102 | 103 | pub fn new() -> Result { 104 | let rank: i32 = std::env::var("RANK") 105 | .unwrap_or("-1".to_string()) 106 | .parse() 107 | .unwrap(); 108 | TELEMETRY_INIT_ONCE.call_once(|| { 109 | if rank == -1 || rank > 7 { 110 | return; 111 | } 112 | 113 | let jaeger_addr = match std::env::var("BAGUA_NET_JAEGER_ADDRESS") { 114 | Ok(jaeger_addr) => { 115 | tracing::info!("detected auto tuning server, connecting"); 116 | jaeger_addr 117 | } 118 | Err(_) => { 119 | tracing::warn!("Jaeger server not detected."); 120 | return; 121 | } 122 | }; 123 | 124 | opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); 125 | opentelemetry_jaeger::new_pipeline() 126 | .with_collector_endpoint(format!("http://{}/api/traces", jaeger_addr)) 127 | .with_service_name("bagua-net") 128 | .install_batch(opentelemetry::runtime::AsyncStd) 129 | .unwrap(); 130 | }); 131 | 132 | let tracer = opentelemetry::global::tracer("bagua-net"); 133 | let mut span = tracer.start(format!("BaguaNet-{}", rank)); 134 | span.set_attribute(KeyValue::new( 135 | "socket_devs", 136 | format!("{:?}", utils::find_interfaces()), 137 | )); 138 | 139 | let prom_exporter = opentelemetry_prometheus::exporter() 140 | .with_default_histogram_boundaries(vec![16., 1024., 4096., 1048576.]) 141 | .init(); 142 | 143 | let isend_nbytes_per_second = Arc::new(Mutex::new(0.)); 144 | let isend_percentage_of_effective_time = Arc::new(Mutex::new(0.)); 145 | 146 | let meter = opentelemetry::global::meter("bagua-net"); 147 | let isend_nbytes_per_second_clone = isend_nbytes_per_second.clone(); 148 | meter 149 | .f64_value_observer( 150 | "isend_nbytes_per_second", 151 | move |res: ObserverResult| { 152 | res.observe( 153 | *isend_nbytes_per_second_clone.lock().unwrap(), 154 | HANDLER_ALL.as_ref(), 155 | ); 156 | }, 157 | ) 158 | .init(); 159 | let isend_percentage_of_effective_time_clone = isend_percentage_of_effective_time.clone(); 160 | meter 161 | .f64_value_observer( 162 | "isend_percentage_of_effective_time", 163 | move |res: ObserverResult| { 164 | res.observe( 165 | *isend_percentage_of_effective_time_clone.lock().unwrap(), 166 | HANDLER_ALL.as_ref(), 167 | ); 168 | }, 169 | ) 170 | .init(); 171 | let state = Arc::new(AppState { 172 | exporter: prom_exporter.clone(), 173 | isend_nbytes_gauge: meter 174 | .u64_value_recorder("isend_nbytes") 175 | .init() 176 | .bind(HANDLER_ALL.as_ref()), 177 | irecv_nbytes_gauge: meter 178 | .u64_value_recorder("irecv_nbytes") 179 | .init() 180 | .bind(HANDLER_ALL.as_ref()), 181 | isend_nbytes_per_second: isend_nbytes_per_second, 182 | isend_percentage_of_effective_time: isend_percentage_of_effective_time, 183 | uploader: std::thread::spawn(move || { 184 | let prometheus_addr = 185 | std::env::var("BAGUA_NET_PROMETHEUS_ADDRESS").unwrap_or_default(); 186 | let (user, pass, address) = match utils::parse_user_pass_and_addr(&prometheus_addr) 187 | { 188 | Some(ret) => ret, 189 | None => return, 190 | }; 191 | 192 | loop { 193 | std::thread::sleep(std::time::Duration::from_micros(200)); 194 | let metric_families = prom_exporter.registry().gather(); 195 | match prometheus::push_metrics( 196 | "BaguaNet", 197 | prometheus::labels! { "rank".to_owned() => rank.to_string(), }, 198 | &address, 199 | metric_families, 200 | Some(prometheus::BasicAuthentication { 201 | username: user.clone(), 202 | password: pass.clone(), 203 | }), 204 | ) { 205 | Ok(_) => {} 206 | Err(err) => { 207 | tracing::warn!("{:?}", err); 208 | } 209 | } 210 | } 211 | }), 212 | }); 213 | 214 | Ok(Self { 215 | socket_devs: utils::find_interfaces(), 216 | listen_comm_next_id: 0, 217 | listen_comm_map: Default::default(), 218 | send_comm_next_id: 0, 219 | send_comm_map: Default::default(), 220 | recv_comm_next_id: 0, 221 | recv_comm_map: Default::default(), 222 | socket_request_next_id: 0, 223 | socket_request_map: Default::default(), 224 | trace_span_context: opentelemetry::Context::current_with_span(span), 225 | rank: rank, 226 | trace_on_flag: rank < 8, 227 | state: state, 228 | nstreams: std::env::var("BAGUA_NET_NSTREAMS") 229 | .unwrap_or("2".to_owned()) 230 | .parse() 231 | .unwrap(), 232 | min_chunksize: std::env::var("BAGUA_NET_MIN_CHUNKSIZE") 233 | .unwrap_or("1048576".to_owned()) 234 | .parse() 235 | .unwrap(), 236 | }) 237 | } 238 | } 239 | 240 | impl Net for BaguaNet { 241 | fn devices(&self) -> Result { 242 | Ok(self.socket_devs.len()) 243 | } 244 | 245 | fn get_properties(&self, dev_id: usize) -> Result { 246 | let socket_dev = &self.socket_devs[dev_id]; 247 | 248 | Ok(NCCLNetProperties { 249 | name: socket_dev.interface_name.clone(), 250 | pci_path: socket_dev.pci_path.clone(), 251 | guid: dev_id as u64, 252 | ptr_support: NCCL_PTR_HOST, 253 | speed: utils::get_net_if_speed(&socket_dev.interface_name), 254 | port: 0, 255 | max_comms: BaguaNet::DEFAULT_SOCKET_MAX_COMMS, 256 | }) 257 | } 258 | 259 | fn listen( 260 | &mut self, 261 | dev_id: usize, 262 | ) -> Result<(SocketHandle, SocketListenCommID), BaguaNetError> { 263 | let socket_dev = &self.socket_devs[dev_id]; 264 | let addr = match socket_dev.addr.clone() { 265 | SockAddr::Inet(inet_addr) => inet_addr, 266 | others => { 267 | return Err(BaguaNetError::InnerError(format!( 268 | "Got invalid socket address, which is {:?}", 269 | others 270 | ))) 271 | } 272 | }; 273 | 274 | let socket = match Socket::new( 275 | match addr { 276 | InetAddr::V4(_) => Domain::IPV4, 277 | InetAddr::V6(_) => Domain::IPV6, 278 | }, 279 | Type::STREAM, 280 | None, 281 | ) { 282 | Ok(sock) => sock, 283 | Err(err) => return Err(BaguaNetError::IOError(format!("{:?}", err))), 284 | }; 285 | socket.bind(&addr.to_std().into()).unwrap(); 286 | socket.listen(BaguaNet::DEFAULT_LISTEN_BACKLOG).unwrap(); 287 | 288 | let listener: net::TcpListener = socket.into(); 289 | let socket_addr = listener.local_addr().unwrap(); 290 | let socket_handle = SocketHandle { 291 | addr: SockAddr::new_inet(InetAddr::from_std(&socket_addr)), 292 | }; 293 | let id = self.listen_comm_next_id; 294 | self.listen_comm_next_id += 1; 295 | self.listen_comm_map.insert( 296 | id, 297 | SocketListenComm { 298 | tcp_listener: Arc::new(Mutex::new(listener)), 299 | }, 300 | ); 301 | 302 | Ok((socket_handle, id)) 303 | } 304 | 305 | fn connect( 306 | &mut self, 307 | _dev_id: usize, 308 | socket_handle: SocketHandle, 309 | ) -> Result { 310 | let mut parallel_streams = Vec::new(); 311 | let mut streams_input = Vec::new(); 312 | for stream_id in 0..self.nstreams { 313 | let mut stream = match net::TcpStream::connect(socket_handle.addr.clone().to_str()) { 314 | Ok(stream) => stream, 315 | Err(err) => { 316 | tracing::warn!( 317 | "net::TcpStream::connect failed, err={:?}, socket_handle={:?}", 318 | err, 319 | socket_handle 320 | ); 321 | return Err(BaguaNetError::TCPError(format!( 322 | "socket_handle={:?}, err={:?}", 323 | socket_handle, err 324 | ))); 325 | } 326 | }; 327 | stream.write_all(&stream_id.to_be_bytes()[..]).unwrap(); 328 | 329 | stream.set_nodelay(true).unwrap(); 330 | stream.set_nonblocking(true).unwrap(); 331 | 332 | let (msg_sender, msg_receiver) = 333 | flume::unbounded::<(&'static [u8], Arc>)>(); 334 | let metrics = self.state.clone(); 335 | // TODO: Consider dynamically assigning tasks to make the least stream full 336 | parallel_streams.push(std::thread::spawn(move || { 337 | let out_timer = std::time::Instant::now(); 338 | let mut sum_in_time = 0.; 339 | for (data, state) in msg_receiver.iter() { 340 | let in_timer = std::time::Instant::now(); 341 | utils::nonblocking_write_all(&mut stream, &data[..]).unwrap(); 342 | 343 | let dur = in_timer.elapsed().as_secs_f64(); 344 | sum_in_time += dur; 345 | 346 | *metrics.isend_nbytes_per_second.lock().unwrap() = data.len() as f64 / dur; 347 | *metrics.isend_percentage_of_effective_time.lock().unwrap() = 348 | sum_in_time / out_timer.elapsed().as_secs_f64(); 349 | 350 | metrics.isend_nbytes_gauge.record(data.len() as u64); 351 | match state.lock() { 352 | Ok(mut state) => { 353 | state.completed_subtasks += 1; 354 | state.nbytes_transferred += data.len(); 355 | } 356 | Err(poisoned) => { 357 | tracing::warn!("{:?}", poisoned); 358 | } 359 | }; 360 | } 361 | })); 362 | streams_input.push(msg_sender); 363 | } 364 | 365 | let nstreams = self.nstreams; 366 | let mut ctrl_stream = match net::TcpStream::connect(socket_handle.addr.clone().to_str()) { 367 | Ok(ctrl_stream) => ctrl_stream, 368 | Err(err) => { 369 | tracing::warn!( 370 | "net::TcpStream::connect failed, err={:?}, socket_handle={:?}", 371 | err, 372 | socket_handle 373 | ); 374 | return Err(BaguaNetError::TCPError(format!( 375 | "socket_handle={:?}, err={:?}", 376 | socket_handle, err 377 | ))); 378 | } 379 | }; 380 | ctrl_stream.write_all(&nstreams.to_be_bytes()[..]).unwrap(); 381 | ctrl_stream.set_nodelay(true).unwrap(); 382 | ctrl_stream.set_nonblocking(true).unwrap(); 383 | 384 | let (msg_sender, msg_receiver) = flume::unbounded(); 385 | let min_chunksize = self.min_chunksize; 386 | let id = self.send_comm_next_id; 387 | self.send_comm_next_id += 1; 388 | self.send_comm_map.insert( 389 | id, 390 | SocketSendComm { 391 | msg_sender: msg_sender, 392 | tcp_sender: Arc::new(std::thread::spawn(move || { 393 | let mut downstream_id = 0; 394 | for (data, state) in msg_receiver.iter() { 395 | let send_nbytes = data.len().to_be_bytes(); 396 | if let Err(err) = 397 | utils::nonblocking_write_all(&mut ctrl_stream, &send_nbytes[..]) 398 | { 399 | state.lock().unwrap().err = 400 | Some(BaguaNetError::IOError(format!("{:?}", err))); 401 | break; 402 | } 403 | 404 | if data.len() != 0 { 405 | let chunk_size = utils::chunk_size(data.len(), min_chunksize, nstreams); 406 | 407 | for bucket in data.chunks(chunk_size) { 408 | state.lock().unwrap().nsubtasks += 1; 409 | streams_input[downstream_id] 410 | .send((bucket, state.clone())) 411 | .unwrap(); 412 | downstream_id = (downstream_id + 1) % parallel_streams.len(); 413 | } 414 | } 415 | 416 | state.lock().unwrap().completed_subtasks += 1; 417 | } 418 | })), 419 | }, 420 | ); 421 | 422 | Ok(id) 423 | } 424 | 425 | fn accept( 426 | &mut self, 427 | listen_comm_id: SocketListenCommID, 428 | ) -> Result { 429 | let listen_comm = self.listen_comm_map.get(&listen_comm_id).unwrap(); 430 | let mut parallel_streams = Vec::new(); 431 | let mut ctrl_stream = None; 432 | let mut streams_input = std::collections::BTreeMap::new(); 433 | for _ in 0..=self.nstreams { 434 | let (mut stream, _addr) = match listen_comm.tcp_listener.lock().unwrap().accept() { 435 | Ok(listen) => listen, 436 | Err(err) => { 437 | return Err(BaguaNetError::TCPError(format!("{:?}", err))); 438 | } 439 | }; 440 | let mut stream_id = (0 as usize).to_be_bytes(); 441 | stream.read_exact(&mut stream_id[..]).unwrap(); 442 | let stream_id = usize::from_be_bytes(stream_id); 443 | 444 | if stream_id == self.nstreams { 445 | ctrl_stream = Some(stream); 446 | continue; 447 | } 448 | 449 | stream.set_nodelay(true).unwrap(); 450 | stream.set_nonblocking(true).unwrap(); 451 | 452 | let (msg_sender, msg_receiver) = 453 | flume::unbounded::<(&'static mut [u8], Arc>)>(); 454 | let metrics = self.state.clone(); 455 | parallel_streams.push(std::thread::spawn(move || { 456 | for (data, state) in msg_receiver.iter() { 457 | utils::nonblocking_read_exact(&mut stream, &mut data[..]).unwrap(); 458 | 459 | metrics.irecv_nbytes_gauge.record(data.len() as u64); 460 | match state.lock() { 461 | Ok(mut state) => { 462 | state.completed_subtasks += 1; 463 | state.nbytes_transferred += data.len(); 464 | } 465 | Err(poisoned) => { 466 | tracing::warn!("{:?}", poisoned); 467 | } 468 | }; 469 | } 470 | })); 471 | streams_input.insert(stream_id, msg_sender); 472 | } 473 | let mut ctrl_stream = ctrl_stream.unwrap(); 474 | let streams_input: Vec<_> = streams_input 475 | .into_iter() 476 | .map(|(_, stream)| stream) 477 | .collect(); 478 | 479 | ctrl_stream.set_nodelay(true).unwrap(); 480 | ctrl_stream.set_nonblocking(true).unwrap(); 481 | 482 | let nstreams = self.nstreams; 483 | let (msg_sender, msg_receiver) = flume::unbounded(); 484 | let min_chunksize = self.min_chunksize; 485 | let id = self.recv_comm_next_id; 486 | self.recv_comm_next_id += 1; 487 | self.recv_comm_map.insert( 488 | id, 489 | SocketRecvComm { 490 | msg_sender: msg_sender, 491 | tcp_sender: Arc::new(std::thread::spawn(move || { 492 | let mut downstream_id = 0; 493 | for (data, state) in msg_receiver.iter() { 494 | let mut target_nbytes = data.len().to_be_bytes(); 495 | if let Err(err) = 496 | utils::nonblocking_read_exact(&mut ctrl_stream, &mut target_nbytes[..]) 497 | { 498 | state.lock().unwrap().err = 499 | Some(BaguaNetError::IOError(format!("{:?}", err))); 500 | break; 501 | } 502 | let target_nbytes = usize::from_be_bytes(target_nbytes); 503 | 504 | if target_nbytes != 0 { 505 | let chunk_size = 506 | utils::chunk_size(target_nbytes, min_chunksize, nstreams); 507 | for bucket in data[..target_nbytes].chunks_mut(chunk_size) { 508 | state.lock().unwrap().nsubtasks += 1; 509 | streams_input[downstream_id] 510 | .send((&mut bucket[..], state.clone())) 511 | .unwrap(); 512 | downstream_id = (downstream_id + 1) % parallel_streams.len(); 513 | } 514 | } 515 | state.lock().unwrap().completed_subtasks += 1; 516 | } 517 | })), 518 | }, 519 | ); 520 | 521 | Ok(id) 522 | } 523 | 524 | fn isend( 525 | &mut self, 526 | send_comm_id: SocketSendCommID, 527 | data: &'static [u8], 528 | ) -> Result { 529 | let tracer = opentelemetry::global::tracer("bagua-net"); 530 | let mut span = tracer 531 | .span_builder(format!("isend-{}", send_comm_id)) 532 | .with_parent_context(self.trace_span_context.clone()) 533 | .start(&tracer); 534 | let send_comm = self.send_comm_map.get(&send_comm_id).unwrap(); 535 | let id = self.socket_request_next_id; 536 | 537 | span.set_attribute(KeyValue::new("id", id as i64)); 538 | span.set_attribute(KeyValue::new("nbytes", data.len() as i64)); 539 | 540 | self.socket_request_next_id += 1; 541 | let task_state = Arc::new(Mutex::new(RequestState { 542 | nsubtasks: 1, 543 | completed_subtasks: 0, 544 | nbytes_transferred: 0, 545 | err: None, 546 | })); 547 | self.socket_request_map.insert( 548 | id, 549 | SocketRequest::SendRequest(SocketSendRequest { 550 | state: task_state.clone(), 551 | trace_span: span, 552 | }), 553 | ); 554 | 555 | send_comm.msg_sender.send((data, task_state)).unwrap(); 556 | 557 | Ok(id) 558 | } 559 | 560 | fn irecv( 561 | &mut self, 562 | recv_comm_id: SocketRecvCommID, 563 | data: &'static mut [u8], 564 | ) -> Result { 565 | let tracer = opentelemetry::global::tracer("bagua-net"); 566 | let mut span = tracer 567 | .span_builder(format!("irecv-{}", recv_comm_id)) 568 | .with_parent_context(self.trace_span_context.clone()) 569 | .start(&tracer); 570 | let recv_comm = self.recv_comm_map.get(&recv_comm_id).unwrap(); 571 | let id = self.socket_request_next_id; 572 | 573 | span.set_attribute(KeyValue::new("id", id as i64)); 574 | 575 | self.socket_request_next_id += 1; 576 | let task_state = Arc::new(Mutex::new(RequestState { 577 | nsubtasks: 1, 578 | completed_subtasks: 0, 579 | nbytes_transferred: 0, 580 | err: None, 581 | })); 582 | self.socket_request_map.insert( 583 | id, 584 | SocketRequest::RecvRequest(SocketRecvRequest { 585 | state: task_state.clone(), 586 | trace_span: span, 587 | }), 588 | ); 589 | 590 | recv_comm.msg_sender.send((data, task_state)).unwrap(); 591 | 592 | Ok(id) 593 | } 594 | 595 | fn test(&mut self, request_id: SocketRequestID) -> Result<(bool, usize), BaguaNetError> { 596 | let request = self.socket_request_map.get_mut(&request_id).unwrap(); 597 | let ret = match request { 598 | SocketRequest::SendRequest(send_req) => { 599 | let state = send_req.state.lock().unwrap(); 600 | if let Some(err) = state.err.clone() { 601 | return Err(err); 602 | } 603 | 604 | let task_completed = state.nsubtasks == state.completed_subtasks; 605 | if task_completed { 606 | send_req.trace_span.end(); 607 | } 608 | Ok((task_completed, state.nbytes_transferred)) 609 | } 610 | SocketRequest::RecvRequest(recv_req) => { 611 | let state = recv_req.state.lock().unwrap(); 612 | if let Some(err) = state.err.clone() { 613 | return Err(err); 614 | } 615 | 616 | let task_completed = state.nsubtasks == state.completed_subtasks; 617 | if task_completed { 618 | recv_req.trace_span.end(); 619 | } 620 | Ok((task_completed, state.nbytes_transferred)) 621 | } 622 | }; 623 | 624 | if let Ok(ret) = ret { 625 | if ret.0 { 626 | self.socket_request_map.remove(&request_id).unwrap(); 627 | } 628 | } 629 | 630 | ret 631 | } 632 | 633 | fn close_send(&mut self, send_comm_id: SocketSendCommID) -> Result<(), BaguaNetError> { 634 | self.send_comm_map.remove(&send_comm_id); 635 | 636 | Ok(()) 637 | } 638 | 639 | fn close_recv(&mut self, recv_comm_id: SocketRecvCommID) -> Result<(), BaguaNetError> { 640 | self.recv_comm_map.remove(&recv_comm_id); 641 | 642 | Ok(()) 643 | } 644 | 645 | fn close_listen(&mut self, listen_comm_id: SocketListenCommID) -> Result<(), BaguaNetError> { 646 | self.listen_comm_map.remove(&listen_comm_id); 647 | 648 | Ok(()) 649 | } 650 | } 651 | 652 | impl Drop for BaguaNet { 653 | fn drop(&mut self) { 654 | // TODO: make shutdown global 655 | self.trace_span_context.span().end(); 656 | opentelemetry::global::shutdown_tracer_provider(); 657 | } 658 | } 659 | -------------------------------------------------------------------------------- /src/implement/tokio_backend.rs: -------------------------------------------------------------------------------- 1 | use crate::interface; 2 | use crate::interface::{ 3 | BaguaNetError, NCCLNetProperties, Net, SocketHandle, SocketListenCommID, SocketRecvCommID, 4 | SocketRequestID, SocketSendCommID, 5 | }; 6 | use crate::utils; 7 | use crate::utils::NCCLSocketDev; 8 | use nix::sys::socket::{InetAddr, SockAddr}; 9 | use opentelemetry::{ 10 | metrics::{BoundValueRecorder, ObserverResult}, 11 | trace::{Span, TraceContextExt, Tracer}, 12 | KeyValue, 13 | }; 14 | use socket2::{Domain, Socket, Type}; 15 | use std::collections::HashMap; 16 | use std::io::{Read, Write}; 17 | use std::net; 18 | use std::sync::{Arc, Mutex}; 19 | use tokio::io::{AsyncReadExt, AsyncWriteExt}; 20 | use tokio::sync::mpsc; 21 | 22 | const NCCL_PTR_HOST: i32 = 1; 23 | const NCCL_PTR_CUDA: i32 = 2; 24 | 25 | lazy_static! { 26 | static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; 27 | } 28 | 29 | pub struct SocketListenComm { 30 | pub tcp_listener: Arc>, 31 | } 32 | 33 | // TODO: make Rotating communicator 34 | #[derive(Clone)] 35 | pub struct SocketSendComm { 36 | pub msg_sender: mpsc::UnboundedSender<(&'static [u8], Arc>)>, 37 | } 38 | 39 | #[derive(Clone)] 40 | pub struct SocketRecvComm { 41 | pub msg_sender: mpsc::UnboundedSender<(&'static mut [u8], Arc>)>, 42 | } 43 | 44 | pub struct SocketSendRequest { 45 | pub state: Arc>, 46 | pub trace_span: opentelemetry::global::BoxedSpan, 47 | } 48 | 49 | pub struct SocketRecvRequest { 50 | pub state: Arc>, 51 | pub trace_span: opentelemetry::global::BoxedSpan, 52 | } 53 | 54 | #[derive(Debug)] 55 | pub struct RequestState { 56 | pub nsubtasks: usize, 57 | pub completed_subtasks: usize, 58 | pub nbytes_transferred: usize, 59 | pub err: Option, 60 | } 61 | 62 | pub enum SocketRequest { 63 | SendRequest(SocketSendRequest), 64 | RecvRequest(SocketRecvRequest), 65 | } 66 | 67 | static TELEMETRY_INIT_ONCE: std::sync::Once = std::sync::Once::new(); 68 | // static TELEMETRY_GUARD: Option = None; 69 | 70 | struct AppState { 71 | exporter: opentelemetry_prometheus::PrometheusExporter, 72 | isend_nbytes_gauge: BoundValueRecorder<'static, u64>, 73 | irecv_nbytes_gauge: BoundValueRecorder<'static, u64>, 74 | isend_per_second: Arc>, 75 | request_count: Arc>, 76 | isend_nbytes_per_second: Arc>, 77 | isend_percentage_of_effective_time: Arc>, 78 | // isend_nbytes_gauge: BoundValueRecorder<'static, u64>, 79 | // irecv_nbytes_gauge: BoundValueRecorder<'static, u64>, 80 | uploader: std::thread::JoinHandle<()>, 81 | } 82 | 83 | pub struct BaguaNet { 84 | pub socket_devs: Vec, 85 | pub listen_comm_next_id: usize, 86 | pub listen_comm_map: HashMap, 87 | pub send_comm_next_id: usize, 88 | pub send_comm_map: HashMap, 89 | pub recv_comm_next_id: usize, 90 | pub recv_comm_map: HashMap, 91 | pub socket_request_next_id: usize, 92 | pub socket_request_map: HashMap, 93 | pub trace_span_context: opentelemetry::Context, 94 | pub rank: i32, 95 | state: Arc, 96 | nstreams: usize, 97 | min_chunksize: usize, 98 | tokio_rt: tokio::runtime::Runtime, 99 | } 100 | 101 | impl BaguaNet { 102 | const DEFAULT_SOCKET_MAX_COMMS: i32 = 65536; 103 | const DEFAULT_LISTEN_BACKLOG: i32 = 16384; 104 | 105 | pub fn new() -> Result { 106 | let rank: i32 = std::env::var("RANK") 107 | .unwrap_or("-1".to_string()) 108 | .parse() 109 | .unwrap(); 110 | TELEMETRY_INIT_ONCE.call_once(|| { 111 | if rank == -1 || rank > 7 { 112 | return; 113 | } 114 | 115 | let jaeger_addr = match std::env::var("BAGUA_NET_JAEGER_ADDRESS") { 116 | Ok(jaeger_addr) => { 117 | tracing::info!("detected auto tuning server, connecting"); 118 | jaeger_addr 119 | } 120 | Err(_) => { 121 | tracing::warn!("Jaeger server not detected."); 122 | return; 123 | } 124 | }; 125 | 126 | opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); 127 | opentelemetry_jaeger::new_pipeline() 128 | .with_collector_endpoint(format!("http://{}/api/traces", jaeger_addr)) 129 | .with_service_name("bagua-net") 130 | .install_batch(opentelemetry::runtime::AsyncStd) 131 | .unwrap(); 132 | }); 133 | 134 | let tracer = opentelemetry::global::tracer("bagua-net"); 135 | let mut span = tracer.start(format!("BaguaNet-{}", rank)); 136 | span.set_attribute(KeyValue::new( 137 | "socket_devs", 138 | format!("{:?}", utils::find_interfaces()), 139 | )); 140 | 141 | let prom_exporter = opentelemetry_prometheus::exporter() 142 | .with_default_histogram_boundaries(vec![16., 1024., 4096., 1048576.]) 143 | .init(); 144 | 145 | let isend_nbytes_per_second = Arc::new(Mutex::new(0.)); 146 | let isend_percentage_of_effective_time = Arc::new(Mutex::new(0.)); 147 | let isend_per_second = Arc::new(Mutex::new(0.)); 148 | let request_count = Arc::new(Mutex::new(0)); 149 | 150 | let meter = opentelemetry::global::meter("bagua-net"); 151 | let isend_nbytes_per_second_clone = isend_nbytes_per_second.clone(); 152 | meter 153 | .f64_value_observer( 154 | "isend_nbytes_per_second", 155 | move |res: ObserverResult| { 156 | res.observe( 157 | *isend_nbytes_per_second_clone.lock().unwrap(), 158 | HANDLER_ALL.as_ref(), 159 | ); 160 | }, 161 | ) 162 | .init(); 163 | let isend_per_second_clone = isend_per_second.clone(); 164 | meter 165 | .f64_value_observer("isend_per_second", move |res: ObserverResult| { 166 | res.observe( 167 | *isend_per_second_clone.lock().unwrap(), 168 | HANDLER_ALL.as_ref(), 169 | ); 170 | }) 171 | .init(); 172 | let isend_percentage_of_effective_time_clone = isend_percentage_of_effective_time.clone(); 173 | meter 174 | .f64_value_observer( 175 | "isend_percentage_of_effective_time", 176 | move |res: ObserverResult| { 177 | res.observe( 178 | *isend_percentage_of_effective_time_clone.lock().unwrap(), 179 | HANDLER_ALL.as_ref(), 180 | ); 181 | }, 182 | ) 183 | .init(); 184 | let request_count_clone = request_count.clone(); 185 | meter.i64_value_observer("hold_on_request", move |res: ObserverResult| { 186 | res.observe( 187 | *request_count_clone.lock().unwrap() as i64, 188 | HANDLER_ALL.as_ref(), 189 | ); 190 | }); 191 | let state = Arc::new(AppState { 192 | exporter: prom_exporter.clone(), 193 | isend_nbytes_gauge: meter 194 | .u64_value_recorder("isend_nbytes") 195 | .init() 196 | .bind(HANDLER_ALL.as_ref()), 197 | irecv_nbytes_gauge: meter 198 | .u64_value_recorder("irecv_nbytes") 199 | .init() 200 | .bind(HANDLER_ALL.as_ref()), 201 | request_count: request_count, 202 | isend_per_second: isend_per_second, 203 | isend_nbytes_per_second: isend_nbytes_per_second, 204 | isend_percentage_of_effective_time: isend_percentage_of_effective_time, 205 | uploader: std::thread::spawn(move || { 206 | let prometheus_addr = 207 | std::env::var("BAGUA_NET_PROMETHEUS_ADDRESS").unwrap_or_default(); 208 | let (user, pass, address) = match utils::parse_user_pass_and_addr(&prometheus_addr) 209 | { 210 | Some(ret) => ret, 211 | None => return, 212 | }; 213 | 214 | loop { 215 | std::thread::sleep(std::time::Duration::from_micros(200)); 216 | let metric_families = prom_exporter.registry().gather(); 217 | match prometheus::push_metrics( 218 | "BaguaNet", 219 | prometheus::labels! { "rank".to_owned() => rank.to_string(), }, 220 | &address, 221 | metric_families, 222 | Some(prometheus::BasicAuthentication { 223 | username: user.clone(), 224 | password: pass.clone(), 225 | }), 226 | ) { 227 | Ok(_) => {} 228 | Err(err) => { 229 | tracing::warn!("{:?}", err); 230 | } 231 | } 232 | } 233 | }), 234 | }); 235 | 236 | let tokio_rt = match std::env::var("BAGUA_NET_TOKIO_WORKER_THREADS") { 237 | Ok(nworker_thread) => tokio::runtime::Builder::new_multi_thread() 238 | .worker_threads(nworker_thread.parse().unwrap()) 239 | .enable_all() 240 | .build() 241 | .unwrap(), 242 | Err(_) => tokio::runtime::Runtime::new().unwrap(), 243 | }; 244 | 245 | Ok(Self { 246 | socket_devs: utils::find_interfaces(), 247 | listen_comm_next_id: 0, 248 | listen_comm_map: Default::default(), 249 | send_comm_next_id: 0, 250 | send_comm_map: Default::default(), 251 | recv_comm_next_id: 0, 252 | recv_comm_map: Default::default(), 253 | socket_request_next_id: 0, 254 | socket_request_map: Default::default(), 255 | trace_span_context: opentelemetry::Context::current_with_span(span), 256 | rank: rank, 257 | state: state, 258 | nstreams: std::env::var("BAGUA_NET_NSTREAMS") 259 | .unwrap_or("2".to_owned()) 260 | .parse() 261 | .unwrap(), 262 | min_chunksize: std::env::var("BAGUA_NET_MIN_CHUNKSIZE") 263 | .unwrap_or("65535".to_owned()) 264 | .parse() 265 | .unwrap(), 266 | tokio_rt: tokio_rt, 267 | }) 268 | } 269 | } 270 | 271 | impl interface::Net for BaguaNet { 272 | fn devices(&self) -> Result { 273 | Ok(self.socket_devs.len()) 274 | } 275 | 276 | fn get_properties(&self, dev_id: usize) -> Result { 277 | let socket_dev = &self.socket_devs[dev_id]; 278 | 279 | Ok(NCCLNetProperties { 280 | name: socket_dev.interface_name.clone(), 281 | pci_path: socket_dev.pci_path.clone(), 282 | guid: dev_id as u64, 283 | ptr_support: NCCL_PTR_HOST, 284 | speed: utils::get_net_if_speed(&socket_dev.interface_name), 285 | port: 0, 286 | max_comms: BaguaNet::DEFAULT_SOCKET_MAX_COMMS, 287 | }) 288 | } 289 | 290 | fn listen( 291 | &mut self, 292 | dev_id: usize, 293 | ) -> Result<(SocketHandle, SocketListenCommID), BaguaNetError> { 294 | let socket_dev = &self.socket_devs[dev_id]; 295 | let addr = match socket_dev.addr.clone() { 296 | SockAddr::Inet(inet_addr) => inet_addr, 297 | others => { 298 | return Err(BaguaNetError::InnerError(format!( 299 | "Got invalid socket address, which is {:?}", 300 | others 301 | ))) 302 | } 303 | }; 304 | 305 | let socket = match Socket::new( 306 | match addr { 307 | InetAddr::V4(_) => Domain::IPV4, 308 | InetAddr::V6(_) => Domain::IPV6, 309 | }, 310 | Type::STREAM, 311 | None, 312 | ) { 313 | Ok(sock) => sock, 314 | Err(err) => return Err(BaguaNetError::IOError(format!("{:?}", err))), 315 | }; 316 | socket.bind(&addr.to_std().into()).unwrap(); 317 | socket.listen(BaguaNet::DEFAULT_LISTEN_BACKLOG).unwrap(); 318 | 319 | let listener: net::TcpListener = socket.into(); 320 | let socket_addr = listener.local_addr().unwrap(); 321 | let socket_handle = SocketHandle { 322 | addr: SockAddr::new_inet(InetAddr::from_std(&socket_addr)), 323 | }; 324 | let id = self.listen_comm_next_id; 325 | self.listen_comm_next_id += 1; 326 | self.listen_comm_map.insert( 327 | id, 328 | SocketListenComm { 329 | tcp_listener: Arc::new(Mutex::new(listener)), 330 | }, 331 | ); 332 | 333 | Ok((socket_handle, id)) 334 | } 335 | 336 | fn connect( 337 | &mut self, 338 | _dev_id: usize, 339 | socket_handle: SocketHandle, 340 | ) -> Result { 341 | // Init datapass tcp stream 342 | let mut stream_vec = Vec::new(); 343 | for stream_id in 0..self.nstreams { 344 | let mut stream = match net::TcpStream::connect(socket_handle.addr.clone().to_str()) { 345 | Ok(stream) => stream, 346 | Err(err) => { 347 | tracing::warn!( 348 | "net::TcpStream::connect failed, err={:?}, socket_handle={:?}", 349 | err, 350 | socket_handle 351 | ); 352 | return Err(BaguaNetError::TCPError(format!( 353 | "socket_handle={:?}, err={:?}", 354 | socket_handle, err 355 | ))); 356 | } 357 | }; 358 | tracing::debug!( 359 | "{:?} connect to {:?}", 360 | stream.local_addr(), 361 | socket_handle.addr.clone().to_str() 362 | ); 363 | stream.write_all(&stream_id.to_be_bytes()[..]).unwrap(); 364 | 365 | stream_vec.push(stream); 366 | } 367 | 368 | // Launch async datapass pipeline 369 | let min_chunksize = self.min_chunksize; 370 | let (datapass_sender, mut datapass_receiver) = 371 | mpsc::unbounded_channel::<(&'static [u8], Arc>)>(); 372 | self.tokio_rt.spawn(async move { 373 | let mut stream_vec: Vec = stream_vec 374 | .into_iter() 375 | .map(|s| tokio::net::TcpStream::from_std(s).unwrap()) 376 | .collect(); 377 | for stream in stream_vec.iter_mut() { 378 | stream.set_nodelay(true).unwrap(); 379 | } 380 | let nstreams = stream_vec.len(); 381 | 382 | loop { 383 | let (data, state) = match datapass_receiver.recv().await { 384 | Some(it) => it, 385 | None => break, 386 | }; 387 | if data.len() == 0 { 388 | state.lock().unwrap().completed_subtasks += 1; 389 | continue; 390 | } 391 | 392 | let mut chunks = 393 | data.chunks(utils::chunk_size(data.len(), min_chunksize, nstreams)); 394 | 395 | let mut datapass_fut = Vec::with_capacity(stream_vec.len()); 396 | for stream in stream_vec.iter_mut() { 397 | let chunk = match chunks.next() { 398 | Some(b) => b, 399 | None => break, 400 | }; 401 | 402 | datapass_fut.push(stream.write_all(&chunk[..])); 403 | } 404 | futures::future::join_all(datapass_fut).await; 405 | 406 | match state.lock() { 407 | Ok(mut state) => { 408 | state.completed_subtasks += 1; 409 | state.nbytes_transferred += data.len(); 410 | } 411 | Err(poisoned) => { 412 | tracing::warn!("{:?}", poisoned); 413 | } 414 | }; 415 | } 416 | }); 417 | 418 | let mut ctrl_stream = match net::TcpStream::connect(socket_handle.addr.clone().to_str()) { 419 | Ok(ctrl_stream) => ctrl_stream, 420 | Err(err) => { 421 | tracing::warn!( 422 | "net::TcpStream::connect failed, err={:?}, socket_handle={:?}", 423 | err, 424 | socket_handle 425 | ); 426 | return Err(BaguaNetError::TCPError(format!( 427 | "socket_handle={:?}, err={:?}", 428 | socket_handle, err 429 | ))); 430 | } 431 | }; 432 | ctrl_stream 433 | .write_all(&self.nstreams.to_be_bytes()[..]) 434 | .unwrap(); 435 | tracing::debug!( 436 | "ctrl_stream {:?} connect to {:?}", 437 | ctrl_stream.local_addr(), 438 | ctrl_stream.peer_addr() 439 | ); 440 | 441 | let (msg_sender, mut msg_receiver) = tokio::sync::mpsc::unbounded_channel(); 442 | let id = self.send_comm_next_id; 443 | self.send_comm_next_id += 1; 444 | let send_comm = SocketSendComm { 445 | msg_sender: msg_sender, 446 | }; 447 | self.tokio_rt.spawn(async move { 448 | let mut ctrl_stream = tokio::net::TcpStream::from_std(ctrl_stream).unwrap(); 449 | ctrl_stream.set_nodelay(true).unwrap(); 450 | loop { 451 | let (data, state) = match msg_receiver.recv().await { 452 | Some(it) => it, 453 | None => break, 454 | }; 455 | 456 | match ctrl_stream.write_u32(data.len() as u32).await { 457 | Ok(_) => {} 458 | Err(err) => { 459 | state.lock().unwrap().err = 460 | Some(BaguaNetError::IOError(format!("{:?}", err))); 461 | break; 462 | } 463 | }; 464 | tracing::debug!( 465 | "send to {:?} target_nbytes={}", 466 | ctrl_stream.peer_addr(), 467 | data.len() 468 | ); 469 | 470 | datapass_sender.send((data, state)).unwrap(); 471 | } 472 | }); 473 | self.send_comm_map.insert(id, send_comm); 474 | 475 | Ok(id) 476 | } 477 | 478 | fn accept( 479 | &mut self, 480 | listen_comm_id: SocketListenCommID, 481 | ) -> Result { 482 | let listen_comm = self.listen_comm_map.get(&listen_comm_id).unwrap(); 483 | 484 | let mut ctrl_stream = None; 485 | let mut stream_vec = std::collections::BTreeMap::new(); 486 | for _ in 0..=self.nstreams { 487 | let (mut stream, _addr) = match listen_comm.tcp_listener.lock().unwrap().accept() { 488 | Ok(listen) => listen, 489 | Err(err) => { 490 | return Err(BaguaNetError::TCPError(format!("{:?}", err))); 491 | } 492 | }; 493 | 494 | let mut stream_id = (0 as usize).to_be_bytes(); 495 | stream.read_exact(&mut stream_id[..]).unwrap(); 496 | let stream_id = usize::from_be_bytes(stream_id); 497 | 498 | if stream_id == self.nstreams { 499 | ctrl_stream = Some(stream); 500 | } else { 501 | stream_vec.insert(stream_id, stream); 502 | } 503 | } 504 | let ctrl_stream = ctrl_stream.unwrap(); 505 | 506 | let min_chunksize = self.min_chunksize; 507 | let (datapass_sender, mut datapass_receiver) = 508 | mpsc::unbounded_channel::<(&'static mut [u8], Arc>)>(); 509 | self.tokio_rt.spawn(async move { 510 | let mut stream_vec: Vec = stream_vec 511 | .into_iter() 512 | .map(|(_, stream)| tokio::net::TcpStream::from_std(stream).unwrap()) 513 | .collect(); 514 | for stream in stream_vec.iter_mut() { 515 | stream.set_nodelay(true).unwrap(); 516 | } 517 | let nstreams = stream_vec.len(); 518 | 519 | loop { 520 | let (data, state) = match datapass_receiver.recv().await { 521 | Some(it) => it, 522 | None => break, 523 | }; 524 | if data.len() == 0 { 525 | state.lock().unwrap().completed_subtasks += 1; 526 | continue; 527 | } 528 | 529 | let mut chunks = 530 | data.chunks_mut(utils::chunk_size(data.len(), min_chunksize, nstreams)); 531 | let mut datapass_fut = Vec::with_capacity(stream_vec.len()); 532 | for stream in stream_vec.iter_mut() { 533 | let chunk = match chunks.next() { 534 | Some(b) => b, 535 | None => break, 536 | }; 537 | 538 | datapass_fut.push(stream.read_exact(&mut chunk[..])); 539 | } 540 | futures::future::join_all(datapass_fut).await; 541 | 542 | match state.lock() { 543 | Ok(mut state) => { 544 | state.completed_subtasks += 1; 545 | state.nbytes_transferred += data.len(); 546 | } 547 | Err(poisoned) => { 548 | tracing::warn!("{:?}", poisoned); 549 | } 550 | }; 551 | } 552 | }); 553 | 554 | let (msg_sender, mut msg_receiver) = mpsc::unbounded_channel(); 555 | let id = self.recv_comm_next_id; 556 | self.recv_comm_next_id += 1; 557 | let recv_comm = SocketRecvComm { 558 | msg_sender: msg_sender, 559 | }; 560 | self.tokio_rt.spawn(async move { 561 | let mut ctrl_stream = tokio::net::TcpStream::from_std(ctrl_stream).unwrap(); 562 | ctrl_stream.set_nodelay(true).unwrap(); 563 | loop { 564 | let (data, state) = match msg_receiver.recv().await { 565 | Some(it) => it, 566 | None => break, 567 | }; 568 | 569 | let target_nbytes = match ctrl_stream.read_u32().await { 570 | Ok(n) => n as usize, 571 | Err(err) => { 572 | state.lock().unwrap().err = 573 | Some(BaguaNetError::IOError(format!("{:?}", err))); 574 | break; 575 | } 576 | }; 577 | 578 | tracing::debug!( 579 | "{:?} recv target_nbytes={}", 580 | ctrl_stream.local_addr(), 581 | target_nbytes 582 | ); 583 | 584 | datapass_sender 585 | .send((&mut data[..target_nbytes], state)) 586 | .unwrap(); 587 | } 588 | }); 589 | self.recv_comm_map.insert(id, recv_comm); 590 | 591 | Ok(id) 592 | } 593 | 594 | fn isend( 595 | &mut self, 596 | send_comm_id: SocketSendCommID, 597 | data: &'static [u8], 598 | ) -> Result { 599 | let tracer = opentelemetry::global::tracer("bagua-net"); 600 | let mut span = tracer 601 | .span_builder(format!("isend-{}", send_comm_id)) 602 | .with_parent_context(self.trace_span_context.clone()) 603 | .start(&tracer); 604 | let send_comm = self.send_comm_map.get(&send_comm_id).unwrap(); 605 | let id = self.socket_request_next_id; 606 | 607 | span.set_attribute(KeyValue::new("id", id as i64)); 608 | span.set_attribute(KeyValue::new("nbytes", data.len() as i64)); 609 | 610 | self.socket_request_next_id += 1; 611 | let task_state = Arc::new(Mutex::new(RequestState { 612 | nsubtasks: 1, 613 | completed_subtasks: 0, 614 | nbytes_transferred: 0, 615 | err: None, 616 | })); 617 | self.socket_request_map.insert( 618 | id, 619 | SocketRequest::SendRequest(SocketSendRequest { 620 | state: task_state.clone(), 621 | trace_span: span, 622 | }), 623 | ); 624 | 625 | send_comm.msg_sender.send((data, task_state)).unwrap(); 626 | 627 | Ok(id) 628 | } 629 | 630 | fn irecv( 631 | &mut self, 632 | recv_comm_id: SocketRecvCommID, 633 | data: &'static mut [u8], 634 | ) -> Result { 635 | let tracer = opentelemetry::global::tracer("bagua-net"); 636 | let mut span = tracer 637 | .span_builder(format!("irecv-{}", recv_comm_id)) 638 | .with_parent_context(self.trace_span_context.clone()) 639 | .start(&tracer); 640 | let recv_comm = self.recv_comm_map.get(&recv_comm_id).unwrap(); 641 | let id = self.socket_request_next_id; 642 | 643 | span.set_attribute(KeyValue::new("id", id as i64)); 644 | 645 | self.socket_request_next_id += 1; 646 | let task_state = Arc::new(Mutex::new(RequestState { 647 | nsubtasks: 1, 648 | completed_subtasks: 0, 649 | nbytes_transferred: 0, 650 | err: None, 651 | })); 652 | self.socket_request_map.insert( 653 | id, 654 | SocketRequest::RecvRequest(SocketRecvRequest { 655 | state: task_state.clone(), 656 | trace_span: span, 657 | }), 658 | ); 659 | 660 | recv_comm.msg_sender.send((data, task_state)).unwrap(); 661 | 662 | Ok(id) 663 | } 664 | 665 | fn test(&mut self, request_id: SocketRequestID) -> Result<(bool, usize), BaguaNetError> { 666 | *self.state.request_count.lock().unwrap() = self.socket_request_map.len(); 667 | let request = self.socket_request_map.get_mut(&request_id).unwrap(); 668 | let ret = match request { 669 | SocketRequest::SendRequest(send_req) => { 670 | let state = send_req.state.lock().unwrap(); 671 | if let Some(err) = state.err.clone() { 672 | return Err(err); 673 | } 674 | 675 | let task_completed = state.nsubtasks == state.completed_subtasks; 676 | if task_completed { 677 | send_req.trace_span.end(); 678 | } 679 | Ok((task_completed, state.nbytes_transferred)) 680 | } 681 | SocketRequest::RecvRequest(recv_req) => { 682 | let state = recv_req.state.lock().unwrap(); 683 | if let Some(err) = state.err.clone() { 684 | return Err(err); 685 | } 686 | 687 | let task_completed = state.nsubtasks == state.completed_subtasks; 688 | if task_completed { 689 | recv_req.trace_span.end(); 690 | } 691 | Ok((task_completed, state.nbytes_transferred)) 692 | } 693 | }; 694 | 695 | if let Ok(ret) = ret { 696 | if ret.0 { 697 | self.socket_request_map.remove(&request_id).unwrap(); 698 | } 699 | } 700 | 701 | ret 702 | } 703 | 704 | fn close_send(&mut self, send_comm_id: SocketSendCommID) -> Result<(), BaguaNetError> { 705 | self.send_comm_map.remove(&send_comm_id); 706 | tracing::debug!("close_send send_comm_id={}", send_comm_id); 707 | 708 | Ok(()) 709 | } 710 | 711 | fn close_recv(&mut self, recv_comm_id: SocketRecvCommID) -> Result<(), BaguaNetError> { 712 | self.recv_comm_map.remove(&recv_comm_id); 713 | tracing::debug!("close_recv recv_comm_id={}", recv_comm_id); 714 | 715 | Ok(()) 716 | } 717 | 718 | fn close_listen(&mut self, listen_comm_id: SocketListenCommID) -> Result<(), BaguaNetError> { 719 | self.listen_comm_map.remove(&listen_comm_id); 720 | 721 | Ok(()) 722 | } 723 | } 724 | 725 | impl Drop for BaguaNet { 726 | fn drop(&mut self) { 727 | // TODO: make shutdown global 728 | self.trace_span_context.span().end(); 729 | opentelemetry::global::shutdown_tracer_provider(); 730 | } 731 | } 732 | 733 | #[cfg(test)] 734 | mod tests { 735 | use super::*; 736 | 737 | #[test] 738 | fn it_works() { 739 | BaguaNet::new().unwrap(); 740 | } 741 | } 742 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.7.18" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "ansi_term" 16 | version = "0.12.1" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" 19 | dependencies = [ 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "async-channel" 25 | version = "1.6.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" 28 | dependencies = [ 29 | "concurrent-queue", 30 | "event-listener", 31 | "futures-core", 32 | ] 33 | 34 | [[package]] 35 | name = "async-executor" 36 | version = "1.4.1" 37 | source = "registry+https://github.com/rust-lang/crates.io-index" 38 | checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" 39 | dependencies = [ 40 | "async-task", 41 | "concurrent-queue", 42 | "fastrand", 43 | "futures-lite", 44 | "once_cell", 45 | "slab", 46 | ] 47 | 48 | [[package]] 49 | name = "async-global-executor" 50 | version = "2.0.2" 51 | source = "registry+https://github.com/rust-lang/crates.io-index" 52 | checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" 53 | dependencies = [ 54 | "async-channel", 55 | "async-executor", 56 | "async-io", 57 | "async-mutex", 58 | "blocking", 59 | "futures-lite", 60 | "num_cpus", 61 | "once_cell", 62 | ] 63 | 64 | [[package]] 65 | name = "async-io" 66 | version = "1.6.0" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "a811e6a479f2439f0c04038796b5cfb3d2ad56c230e0f2d3f7b04d68cfee607b" 69 | dependencies = [ 70 | "concurrent-queue", 71 | "futures-lite", 72 | "libc", 73 | "log", 74 | "once_cell", 75 | "parking", 76 | "polling", 77 | "slab", 78 | "socket2", 79 | "waker-fn", 80 | "winapi", 81 | ] 82 | 83 | [[package]] 84 | name = "async-lock" 85 | version = "2.4.0" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" 88 | dependencies = [ 89 | "event-listener", 90 | ] 91 | 92 | [[package]] 93 | name = "async-mutex" 94 | version = "1.4.0" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" 97 | dependencies = [ 98 | "event-listener", 99 | ] 100 | 101 | [[package]] 102 | name = "async-process" 103 | version = "1.2.0" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "b21b63ab5a0db0369deb913540af2892750e42d949faacc7a61495ac418a1692" 106 | dependencies = [ 107 | "async-io", 108 | "blocking", 109 | "cfg-if", 110 | "event-listener", 111 | "futures-lite", 112 | "libc", 113 | "once_cell", 114 | "signal-hook", 115 | "winapi", 116 | ] 117 | 118 | [[package]] 119 | name = "async-std" 120 | version = "1.9.0" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" 123 | dependencies = [ 124 | "async-channel", 125 | "async-global-executor", 126 | "async-io", 127 | "async-lock", 128 | "async-process", 129 | "crossbeam-utils", 130 | "futures-channel", 131 | "futures-core", 132 | "futures-io", 133 | "futures-lite", 134 | "gloo-timers", 135 | "kv-log-macro", 136 | "log", 137 | "memchr", 138 | "num_cpus", 139 | "once_cell", 140 | "pin-project-lite", 141 | "pin-utils", 142 | "slab", 143 | "wasm-bindgen-futures", 144 | ] 145 | 146 | [[package]] 147 | name = "async-task" 148 | version = "4.0.3" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" 151 | 152 | [[package]] 153 | name = "async-trait" 154 | version = "0.1.51" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" 157 | dependencies = [ 158 | "proc-macro2", 159 | "quote", 160 | "syn", 161 | ] 162 | 163 | [[package]] 164 | name = "atomic-waker" 165 | version = "1.0.0" 166 | source = "registry+https://github.com/rust-lang/crates.io-index" 167 | checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" 168 | 169 | [[package]] 170 | name = "autocfg" 171 | version = "1.0.1" 172 | source = "registry+https://github.com/rust-lang/crates.io-index" 173 | checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" 174 | 175 | [[package]] 176 | name = "bagua-net" 177 | version = "0.1.0" 178 | dependencies = [ 179 | "bytes", 180 | "ffi-convert", 181 | "flume", 182 | "futures", 183 | "lazy_static", 184 | "libc", 185 | "nix", 186 | "openssl", 187 | "opentelemetry", 188 | "opentelemetry-jaeger", 189 | "opentelemetry-prometheus", 190 | "prometheus", 191 | "regex", 192 | "socket2", 193 | "thiserror", 194 | "tokio", 195 | "tracing", 196 | "tracing-subscriber", 197 | ] 198 | 199 | [[package]] 200 | name = "base64" 201 | version = "0.13.0" 202 | source = "registry+https://github.com/rust-lang/crates.io-index" 203 | checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" 204 | 205 | [[package]] 206 | name = "bitflags" 207 | version = "1.2.1" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 210 | 211 | [[package]] 212 | name = "blocking" 213 | version = "1.0.2" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" 216 | dependencies = [ 217 | "async-channel", 218 | "async-task", 219 | "atomic-waker", 220 | "fastrand", 221 | "futures-lite", 222 | "once_cell", 223 | ] 224 | 225 | [[package]] 226 | name = "bumpalo" 227 | version = "3.7.0" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" 230 | 231 | [[package]] 232 | name = "byteorder" 233 | version = "1.4.3" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" 236 | 237 | [[package]] 238 | name = "bytes" 239 | version = "1.1.0" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" 242 | 243 | [[package]] 244 | name = "cache-padded" 245 | version = "1.1.1" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" 248 | 249 | [[package]] 250 | name = "cc" 251 | version = "1.0.69" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" 254 | 255 | [[package]] 256 | name = "cfg-if" 257 | version = "1.0.0" 258 | source = "registry+https://github.com/rust-lang/crates.io-index" 259 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 260 | 261 | [[package]] 262 | name = "chrono" 263 | version = "0.4.19" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" 266 | dependencies = [ 267 | "libc", 268 | "num-integer", 269 | "num-traits", 270 | "winapi", 271 | ] 272 | 273 | [[package]] 274 | name = "concurrent-queue" 275 | version = "1.2.2" 276 | source = "registry+https://github.com/rust-lang/crates.io-index" 277 | checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" 278 | dependencies = [ 279 | "cache-padded", 280 | ] 281 | 282 | [[package]] 283 | name = "core-foundation" 284 | version = "0.9.1" 285 | source = "registry+https://github.com/rust-lang/crates.io-index" 286 | checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" 287 | dependencies = [ 288 | "core-foundation-sys", 289 | "libc", 290 | ] 291 | 292 | [[package]] 293 | name = "core-foundation-sys" 294 | version = "0.8.2" 295 | source = "registry+https://github.com/rust-lang/crates.io-index" 296 | checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" 297 | 298 | [[package]] 299 | name = "crossbeam-channel" 300 | version = "0.5.1" 301 | source = "registry+https://github.com/rust-lang/crates.io-index" 302 | checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" 303 | dependencies = [ 304 | "cfg-if", 305 | "crossbeam-utils", 306 | ] 307 | 308 | [[package]] 309 | name = "crossbeam-utils" 310 | version = "0.8.5" 311 | source = "registry+https://github.com/rust-lang/crates.io-index" 312 | checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" 313 | dependencies = [ 314 | "cfg-if", 315 | "lazy_static", 316 | ] 317 | 318 | [[package]] 319 | name = "ctor" 320 | version = "0.1.20" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" 323 | dependencies = [ 324 | "quote", 325 | "syn", 326 | ] 327 | 328 | [[package]] 329 | name = "curl" 330 | version = "0.4.38" 331 | source = "registry+https://github.com/rust-lang/crates.io-index" 332 | checksum = "003cb79c1c6d1c93344c7e1201bb51c2148f24ec2bd9c253709d6b2efb796515" 333 | dependencies = [ 334 | "curl-sys", 335 | "libc", 336 | "openssl-probe", 337 | "openssl-sys", 338 | "schannel", 339 | "socket2", 340 | "winapi", 341 | ] 342 | 343 | [[package]] 344 | name = "curl-sys" 345 | version = "0.4.45+curl-7.78.0" 346 | source = "registry+https://github.com/rust-lang/crates.io-index" 347 | checksum = "de9e5a72b1c744eb5dd20b2be4d7eb84625070bb5c4ab9b347b70464ab1e62eb" 348 | dependencies = [ 349 | "cc", 350 | "libc", 351 | "libz-sys", 352 | "openssl-sys", 353 | "pkg-config", 354 | "vcpkg", 355 | "winapi", 356 | ] 357 | 358 | [[package]] 359 | name = "dashmap" 360 | version = "4.0.2" 361 | source = "registry+https://github.com/rust-lang/crates.io-index" 362 | checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" 363 | dependencies = [ 364 | "cfg-if", 365 | "num_cpus", 366 | ] 367 | 368 | [[package]] 369 | name = "encoding_rs" 370 | version = "0.8.28" 371 | source = "registry+https://github.com/rust-lang/crates.io-index" 372 | checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" 373 | dependencies = [ 374 | "cfg-if", 375 | ] 376 | 377 | [[package]] 378 | name = "event-listener" 379 | version = "2.5.1" 380 | source = "registry+https://github.com/rust-lang/crates.io-index" 381 | checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" 382 | 383 | [[package]] 384 | name = "fastrand" 385 | version = "1.5.0" 386 | source = "registry+https://github.com/rust-lang/crates.io-index" 387 | checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e" 388 | dependencies = [ 389 | "instant", 390 | ] 391 | 392 | [[package]] 393 | name = "ffi-convert" 394 | version = "0.5.0" 395 | source = "registry+https://github.com/rust-lang/crates.io-index" 396 | checksum = "36e8b2c8fd8f9948c8db1848bef2e9aade535b1461e59204d67e18a9a35b4af6" 397 | dependencies = [ 398 | "ffi-convert-derive", 399 | "libc", 400 | "thiserror", 401 | ] 402 | 403 | [[package]] 404 | name = "ffi-convert-derive" 405 | version = "0.5.0" 406 | source = "registry+https://github.com/rust-lang/crates.io-index" 407 | checksum = "ced52c8e0b09123ed593c2e7f581f42d1cf035d3289732d2c627b5cd5e165bf3" 408 | dependencies = [ 409 | "proc-macro2", 410 | "quote", 411 | "syn", 412 | ] 413 | 414 | [[package]] 415 | name = "flume" 416 | version = "0.10.9" 417 | source = "registry+https://github.com/rust-lang/crates.io-index" 418 | checksum = "24c3fd473b3a903a62609e413ed7538f99e10b665ecb502b5e481a95283f8ab4" 419 | dependencies = [ 420 | "futures-core", 421 | "futures-sink", 422 | "nanorand", 423 | "pin-project", 424 | "spin", 425 | ] 426 | 427 | [[package]] 428 | name = "fnv" 429 | version = "1.0.7" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 432 | 433 | [[package]] 434 | name = "foreign-types" 435 | version = "0.3.2" 436 | source = "registry+https://github.com/rust-lang/crates.io-index" 437 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 438 | dependencies = [ 439 | "foreign-types-shared", 440 | ] 441 | 442 | [[package]] 443 | name = "foreign-types-shared" 444 | version = "0.1.1" 445 | source = "registry+https://github.com/rust-lang/crates.io-index" 446 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 447 | 448 | [[package]] 449 | name = "form_urlencoded" 450 | version = "1.0.1" 451 | source = "registry+https://github.com/rust-lang/crates.io-index" 452 | checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" 453 | dependencies = [ 454 | "matches", 455 | "percent-encoding", 456 | ] 457 | 458 | [[package]] 459 | name = "futures" 460 | version = "0.3.16" 461 | source = "registry+https://github.com/rust-lang/crates.io-index" 462 | checksum = "1adc00f486adfc9ce99f77d717836f0c5aa84965eb0b4f051f4e83f7cab53f8b" 463 | dependencies = [ 464 | "futures-channel", 465 | "futures-core", 466 | "futures-executor", 467 | "futures-io", 468 | "futures-sink", 469 | "futures-task", 470 | "futures-util", 471 | ] 472 | 473 | [[package]] 474 | name = "futures-channel" 475 | version = "0.3.16" 476 | source = "registry+https://github.com/rust-lang/crates.io-index" 477 | checksum = "74ed2411805f6e4e3d9bc904c95d5d423b89b3b25dc0250aa74729de20629ff9" 478 | dependencies = [ 479 | "futures-core", 480 | "futures-sink", 481 | ] 482 | 483 | [[package]] 484 | name = "futures-core" 485 | version = "0.3.16" 486 | source = "registry+https://github.com/rust-lang/crates.io-index" 487 | checksum = "af51b1b4a7fdff033703db39de8802c673eb91855f2e0d47dcf3bf2c0ef01f99" 488 | 489 | [[package]] 490 | name = "futures-executor" 491 | version = "0.3.16" 492 | source = "registry+https://github.com/rust-lang/crates.io-index" 493 | checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c" 494 | dependencies = [ 495 | "futures-core", 496 | "futures-task", 497 | "futures-util", 498 | ] 499 | 500 | [[package]] 501 | name = "futures-io" 502 | version = "0.3.16" 503 | source = "registry+https://github.com/rust-lang/crates.io-index" 504 | checksum = "0b0e06c393068f3a6ef246c75cdca793d6a46347e75286933e5e75fd2fd11582" 505 | 506 | [[package]] 507 | name = "futures-lite" 508 | version = "1.12.0" 509 | source = "registry+https://github.com/rust-lang/crates.io-index" 510 | checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" 511 | dependencies = [ 512 | "fastrand", 513 | "futures-core", 514 | "futures-io", 515 | "memchr", 516 | "parking", 517 | "pin-project-lite", 518 | "waker-fn", 519 | ] 520 | 521 | [[package]] 522 | name = "futures-macro" 523 | version = "0.3.16" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "c54913bae956fb8df7f4dc6fc90362aa72e69148e3f39041fbe8742d21e0ac57" 526 | dependencies = [ 527 | "autocfg", 528 | "proc-macro-hack", 529 | "proc-macro2", 530 | "quote", 531 | "syn", 532 | ] 533 | 534 | [[package]] 535 | name = "futures-sink" 536 | version = "0.3.16" 537 | source = "registry+https://github.com/rust-lang/crates.io-index" 538 | checksum = "c0f30aaa67363d119812743aa5f33c201a7a66329f97d1a887022971feea4b53" 539 | 540 | [[package]] 541 | name = "futures-task" 542 | version = "0.3.16" 543 | source = "registry+https://github.com/rust-lang/crates.io-index" 544 | checksum = "bbe54a98670017f3be909561f6ad13e810d9a51f3f061b902062ca3da80799f2" 545 | 546 | [[package]] 547 | name = "futures-util" 548 | version = "0.3.16" 549 | source = "registry+https://github.com/rust-lang/crates.io-index" 550 | checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78" 551 | dependencies = [ 552 | "autocfg", 553 | "futures-channel", 554 | "futures-core", 555 | "futures-io", 556 | "futures-macro", 557 | "futures-sink", 558 | "futures-task", 559 | "memchr", 560 | "pin-project-lite", 561 | "pin-utils", 562 | "proc-macro-hack", 563 | "proc-macro-nested", 564 | "slab", 565 | ] 566 | 567 | [[package]] 568 | name = "getrandom" 569 | version = "0.2.3" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" 572 | dependencies = [ 573 | "cfg-if", 574 | "js-sys", 575 | "libc", 576 | "wasi", 577 | "wasm-bindgen", 578 | ] 579 | 580 | [[package]] 581 | name = "gloo-timers" 582 | version = "0.2.1" 583 | source = "registry+https://github.com/rust-lang/crates.io-index" 584 | checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" 585 | dependencies = [ 586 | "futures-channel", 587 | "futures-core", 588 | "js-sys", 589 | "wasm-bindgen", 590 | "web-sys", 591 | ] 592 | 593 | [[package]] 594 | name = "h2" 595 | version = "0.3.4" 596 | source = "registry+https://github.com/rust-lang/crates.io-index" 597 | checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" 598 | dependencies = [ 599 | "bytes", 600 | "fnv", 601 | "futures-core", 602 | "futures-sink", 603 | "futures-util", 604 | "http", 605 | "indexmap", 606 | "slab", 607 | "tokio", 608 | "tokio-util", 609 | "tracing", 610 | ] 611 | 612 | [[package]] 613 | name = "hashbrown" 614 | version = "0.11.2" 615 | source = "registry+https://github.com/rust-lang/crates.io-index" 616 | checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" 617 | 618 | [[package]] 619 | name = "hermit-abi" 620 | version = "0.1.19" 621 | source = "registry+https://github.com/rust-lang/crates.io-index" 622 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 623 | dependencies = [ 624 | "libc", 625 | ] 626 | 627 | [[package]] 628 | name = "http" 629 | version = "0.2.4" 630 | source = "registry+https://github.com/rust-lang/crates.io-index" 631 | checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" 632 | dependencies = [ 633 | "bytes", 634 | "fnv", 635 | "itoa", 636 | ] 637 | 638 | [[package]] 639 | name = "http-body" 640 | version = "0.4.3" 641 | source = "registry+https://github.com/rust-lang/crates.io-index" 642 | checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" 643 | dependencies = [ 644 | "bytes", 645 | "http", 646 | "pin-project-lite", 647 | ] 648 | 649 | [[package]] 650 | name = "httparse" 651 | version = "1.5.1" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" 654 | 655 | [[package]] 656 | name = "httpdate" 657 | version = "1.0.1" 658 | source = "registry+https://github.com/rust-lang/crates.io-index" 659 | checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" 660 | 661 | [[package]] 662 | name = "hyper" 663 | version = "0.14.11" 664 | source = "registry+https://github.com/rust-lang/crates.io-index" 665 | checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11" 666 | dependencies = [ 667 | "bytes", 668 | "futures-channel", 669 | "futures-core", 670 | "futures-util", 671 | "h2", 672 | "http", 673 | "http-body", 674 | "httparse", 675 | "httpdate", 676 | "itoa", 677 | "pin-project-lite", 678 | "socket2", 679 | "tokio", 680 | "tower-service", 681 | "tracing", 682 | "want", 683 | ] 684 | 685 | [[package]] 686 | name = "hyper-tls" 687 | version = "0.5.0" 688 | source = "registry+https://github.com/rust-lang/crates.io-index" 689 | checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" 690 | dependencies = [ 691 | "bytes", 692 | "hyper", 693 | "native-tls", 694 | "tokio", 695 | "tokio-native-tls", 696 | ] 697 | 698 | [[package]] 699 | name = "idna" 700 | version = "0.2.3" 701 | source = "registry+https://github.com/rust-lang/crates.io-index" 702 | checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" 703 | dependencies = [ 704 | "matches", 705 | "unicode-bidi", 706 | "unicode-normalization", 707 | ] 708 | 709 | [[package]] 710 | name = "indexmap" 711 | version = "1.7.0" 712 | source = "registry+https://github.com/rust-lang/crates.io-index" 713 | checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" 714 | dependencies = [ 715 | "autocfg", 716 | "hashbrown", 717 | ] 718 | 719 | [[package]] 720 | name = "instant" 721 | version = "0.1.10" 722 | source = "registry+https://github.com/rust-lang/crates.io-index" 723 | checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" 724 | dependencies = [ 725 | "cfg-if", 726 | ] 727 | 728 | [[package]] 729 | name = "integer-encoding" 730 | version = "1.1.7" 731 | source = "registry+https://github.com/rust-lang/crates.io-index" 732 | checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" 733 | 734 | [[package]] 735 | name = "ipnet" 736 | version = "2.3.1" 737 | source = "registry+https://github.com/rust-lang/crates.io-index" 738 | checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" 739 | 740 | [[package]] 741 | name = "isahc" 742 | version = "1.4.0" 743 | source = "registry+https://github.com/rust-lang/crates.io-index" 744 | checksum = "81c01404730bb4574bbacb59ca0855f969f8eabd688ca22866f2cc333f1a4f69" 745 | dependencies = [ 746 | "async-channel", 747 | "crossbeam-utils", 748 | "curl", 749 | "curl-sys", 750 | "event-listener", 751 | "futures-lite", 752 | "http", 753 | "log", 754 | "once_cell", 755 | "polling", 756 | "slab", 757 | "sluice", 758 | "tracing", 759 | "tracing-futures", 760 | "url", 761 | "waker-fn", 762 | ] 763 | 764 | [[package]] 765 | name = "itoa" 766 | version = "0.4.7" 767 | source = "registry+https://github.com/rust-lang/crates.io-index" 768 | checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" 769 | 770 | [[package]] 771 | name = "js-sys" 772 | version = "0.3.52" 773 | source = "registry+https://github.com/rust-lang/crates.io-index" 774 | checksum = "ce791b7ca6638aae45be056e068fc756d871eb3b3b10b8efa62d1c9cec616752" 775 | dependencies = [ 776 | "wasm-bindgen", 777 | ] 778 | 779 | [[package]] 780 | name = "kv-log-macro" 781 | version = "1.0.7" 782 | source = "registry+https://github.com/rust-lang/crates.io-index" 783 | checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" 784 | dependencies = [ 785 | "log", 786 | ] 787 | 788 | [[package]] 789 | name = "lazy_static" 790 | version = "1.4.0" 791 | source = "registry+https://github.com/rust-lang/crates.io-index" 792 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 793 | 794 | [[package]] 795 | name = "libc" 796 | version = "0.2.101" 797 | source = "registry+https://github.com/rust-lang/crates.io-index" 798 | checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" 799 | 800 | [[package]] 801 | name = "libz-sys" 802 | version = "1.1.3" 803 | source = "registry+https://github.com/rust-lang/crates.io-index" 804 | checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" 805 | dependencies = [ 806 | "cc", 807 | "libc", 808 | "pkg-config", 809 | "vcpkg", 810 | ] 811 | 812 | [[package]] 813 | name = "lock_api" 814 | version = "0.4.4" 815 | source = "registry+https://github.com/rust-lang/crates.io-index" 816 | checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" 817 | dependencies = [ 818 | "scopeguard", 819 | ] 820 | 821 | [[package]] 822 | name = "log" 823 | version = "0.4.14" 824 | source = "registry+https://github.com/rust-lang/crates.io-index" 825 | checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" 826 | dependencies = [ 827 | "cfg-if", 828 | "value-bag", 829 | ] 830 | 831 | [[package]] 832 | name = "matchers" 833 | version = "0.0.1" 834 | source = "registry+https://github.com/rust-lang/crates.io-index" 835 | checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" 836 | dependencies = [ 837 | "regex-automata", 838 | ] 839 | 840 | [[package]] 841 | name = "matches" 842 | version = "0.1.9" 843 | source = "registry+https://github.com/rust-lang/crates.io-index" 844 | checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" 845 | 846 | [[package]] 847 | name = "memchr" 848 | version = "2.4.0" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" 851 | 852 | [[package]] 853 | name = "memoffset" 854 | version = "0.6.4" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" 857 | dependencies = [ 858 | "autocfg", 859 | ] 860 | 861 | [[package]] 862 | name = "mime" 863 | version = "0.3.16" 864 | source = "registry+https://github.com/rust-lang/crates.io-index" 865 | checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" 866 | 867 | [[package]] 868 | name = "mio" 869 | version = "0.7.13" 870 | source = "registry+https://github.com/rust-lang/crates.io-index" 871 | checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" 872 | dependencies = [ 873 | "libc", 874 | "log", 875 | "miow", 876 | "ntapi", 877 | "winapi", 878 | ] 879 | 880 | [[package]] 881 | name = "miow" 882 | version = "0.3.7" 883 | source = "registry+https://github.com/rust-lang/crates.io-index" 884 | checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" 885 | dependencies = [ 886 | "winapi", 887 | ] 888 | 889 | [[package]] 890 | name = "nanorand" 891 | version = "0.6.1" 892 | source = "registry+https://github.com/rust-lang/crates.io-index" 893 | checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" 894 | dependencies = [ 895 | "getrandom", 896 | ] 897 | 898 | [[package]] 899 | name = "native-tls" 900 | version = "0.2.8" 901 | source = "registry+https://github.com/rust-lang/crates.io-index" 902 | checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" 903 | dependencies = [ 904 | "lazy_static", 905 | "libc", 906 | "log", 907 | "openssl", 908 | "openssl-probe", 909 | "openssl-sys", 910 | "schannel", 911 | "security-framework", 912 | "security-framework-sys", 913 | "tempfile", 914 | ] 915 | 916 | [[package]] 917 | name = "nix" 918 | version = "0.22.1" 919 | source = "registry+https://github.com/rust-lang/crates.io-index" 920 | checksum = "e7555d6c7164cc913be1ce7f95cbecdabda61eb2ccd89008524af306fb7f5031" 921 | dependencies = [ 922 | "bitflags", 923 | "cc", 924 | "cfg-if", 925 | "libc", 926 | "memoffset", 927 | ] 928 | 929 | [[package]] 930 | name = "ntapi" 931 | version = "0.3.6" 932 | source = "registry+https://github.com/rust-lang/crates.io-index" 933 | checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" 934 | dependencies = [ 935 | "winapi", 936 | ] 937 | 938 | [[package]] 939 | name = "num-integer" 940 | version = "0.1.44" 941 | source = "registry+https://github.com/rust-lang/crates.io-index" 942 | checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" 943 | dependencies = [ 944 | "autocfg", 945 | "num-traits", 946 | ] 947 | 948 | [[package]] 949 | name = "num-traits" 950 | version = "0.2.14" 951 | source = "registry+https://github.com/rust-lang/crates.io-index" 952 | checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" 953 | dependencies = [ 954 | "autocfg", 955 | ] 956 | 957 | [[package]] 958 | name = "num_cpus" 959 | version = "1.13.0" 960 | source = "registry+https://github.com/rust-lang/crates.io-index" 961 | checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" 962 | dependencies = [ 963 | "hermit-abi", 964 | "libc", 965 | ] 966 | 967 | [[package]] 968 | name = "once_cell" 969 | version = "1.8.0" 970 | source = "registry+https://github.com/rust-lang/crates.io-index" 971 | checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" 972 | 973 | [[package]] 974 | name = "openssl" 975 | version = "0.10.36" 976 | source = "registry+https://github.com/rust-lang/crates.io-index" 977 | checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" 978 | dependencies = [ 979 | "bitflags", 980 | "cfg-if", 981 | "foreign-types", 982 | "libc", 983 | "once_cell", 984 | "openssl-sys", 985 | ] 986 | 987 | [[package]] 988 | name = "openssl-probe" 989 | version = "0.1.4" 990 | source = "registry+https://github.com/rust-lang/crates.io-index" 991 | checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" 992 | 993 | [[package]] 994 | name = "openssl-src" 995 | version = "111.15.0+1.1.1k" 996 | source = "registry+https://github.com/rust-lang/crates.io-index" 997 | checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a" 998 | dependencies = [ 999 | "cc", 1000 | ] 1001 | 1002 | [[package]] 1003 | name = "openssl-sys" 1004 | version = "0.9.66" 1005 | source = "registry+https://github.com/rust-lang/crates.io-index" 1006 | checksum = "1996d2d305e561b70d1ee0c53f1542833f4e1ac6ce9a6708b6ff2738ca67dc82" 1007 | dependencies = [ 1008 | "autocfg", 1009 | "cc", 1010 | "libc", 1011 | "openssl-src", 1012 | "pkg-config", 1013 | "vcpkg", 1014 | ] 1015 | 1016 | [[package]] 1017 | name = "opentelemetry" 1018 | version = "0.16.0" 1019 | source = "registry+https://github.com/rust-lang/crates.io-index" 1020 | checksum = "e1cf9b1c4e9a6c4de793c632496fa490bdc0e1eea73f0c91394f7b6990935d22" 1021 | dependencies = [ 1022 | "async-std", 1023 | "async-trait", 1024 | "crossbeam-channel", 1025 | "dashmap", 1026 | "fnv", 1027 | "futures", 1028 | "js-sys", 1029 | "lazy_static", 1030 | "percent-encoding", 1031 | "pin-project", 1032 | "rand", 1033 | "thiserror", 1034 | ] 1035 | 1036 | [[package]] 1037 | name = "opentelemetry-http" 1038 | version = "0.5.0" 1039 | source = "registry+https://github.com/rust-lang/crates.io-index" 1040 | checksum = "d50ceb0b0e8b75cb3e388a2571a807c8228dabc5d6670f317b6eb21301095373" 1041 | dependencies = [ 1042 | "async-trait", 1043 | "bytes", 1044 | "futures-util", 1045 | "http", 1046 | "isahc", 1047 | "opentelemetry", 1048 | ] 1049 | 1050 | [[package]] 1051 | name = "opentelemetry-jaeger" 1052 | version = "0.15.0" 1053 | source = "registry+https://github.com/rust-lang/crates.io-index" 1054 | checksum = "db22f492873ea037bc267b35a0e8e4fb846340058cb7c864efe3d0bf23684593" 1055 | dependencies = [ 1056 | "async-std", 1057 | "async-trait", 1058 | "http", 1059 | "isahc", 1060 | "lazy_static", 1061 | "opentelemetry", 1062 | "opentelemetry-http", 1063 | "opentelemetry-semantic-conventions", 1064 | "thiserror", 1065 | "thrift", 1066 | ] 1067 | 1068 | [[package]] 1069 | name = "opentelemetry-prometheus" 1070 | version = "0.9.0" 1071 | source = "registry+https://github.com/rust-lang/crates.io-index" 1072 | checksum = "4ee9c06c1366665e7d4dba6540a42ea48900a9c92dc5b963f3ae05fbba76dc63" 1073 | dependencies = [ 1074 | "opentelemetry", 1075 | "prometheus", 1076 | "protobuf", 1077 | ] 1078 | 1079 | [[package]] 1080 | name = "opentelemetry-semantic-conventions" 1081 | version = "0.8.0" 1082 | source = "registry+https://github.com/rust-lang/crates.io-index" 1083 | checksum = "ffeac823339e8b0f27b961f4385057bf9f97f2863bc745bd015fd6091f2270e9" 1084 | dependencies = [ 1085 | "opentelemetry", 1086 | ] 1087 | 1088 | [[package]] 1089 | name = "ordered-float" 1090 | version = "1.1.1" 1091 | source = "registry+https://github.com/rust-lang/crates.io-index" 1092 | checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" 1093 | dependencies = [ 1094 | "num-traits", 1095 | ] 1096 | 1097 | [[package]] 1098 | name = "parking" 1099 | version = "2.0.0" 1100 | source = "registry+https://github.com/rust-lang/crates.io-index" 1101 | checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" 1102 | 1103 | [[package]] 1104 | name = "parking_lot" 1105 | version = "0.11.1" 1106 | source = "registry+https://github.com/rust-lang/crates.io-index" 1107 | checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" 1108 | dependencies = [ 1109 | "instant", 1110 | "lock_api", 1111 | "parking_lot_core", 1112 | ] 1113 | 1114 | [[package]] 1115 | name = "parking_lot_core" 1116 | version = "0.8.3" 1117 | source = "registry+https://github.com/rust-lang/crates.io-index" 1118 | checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" 1119 | dependencies = [ 1120 | "cfg-if", 1121 | "instant", 1122 | "libc", 1123 | "redox_syscall", 1124 | "smallvec", 1125 | "winapi", 1126 | ] 1127 | 1128 | [[package]] 1129 | name = "percent-encoding" 1130 | version = "2.1.0" 1131 | source = "registry+https://github.com/rust-lang/crates.io-index" 1132 | checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" 1133 | 1134 | [[package]] 1135 | name = "pin-project" 1136 | version = "1.0.8" 1137 | source = "registry+https://github.com/rust-lang/crates.io-index" 1138 | checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" 1139 | dependencies = [ 1140 | "pin-project-internal", 1141 | ] 1142 | 1143 | [[package]] 1144 | name = "pin-project-internal" 1145 | version = "1.0.8" 1146 | source = "registry+https://github.com/rust-lang/crates.io-index" 1147 | checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" 1148 | dependencies = [ 1149 | "proc-macro2", 1150 | "quote", 1151 | "syn", 1152 | ] 1153 | 1154 | [[package]] 1155 | name = "pin-project-lite" 1156 | version = "0.2.7" 1157 | source = "registry+https://github.com/rust-lang/crates.io-index" 1158 | checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" 1159 | 1160 | [[package]] 1161 | name = "pin-utils" 1162 | version = "0.1.0" 1163 | source = "registry+https://github.com/rust-lang/crates.io-index" 1164 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 1165 | 1166 | [[package]] 1167 | name = "pkg-config" 1168 | version = "0.3.19" 1169 | source = "registry+https://github.com/rust-lang/crates.io-index" 1170 | checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" 1171 | 1172 | [[package]] 1173 | name = "polling" 1174 | version = "2.1.0" 1175 | source = "registry+https://github.com/rust-lang/crates.io-index" 1176 | checksum = "92341d779fa34ea8437ef4d82d440d5e1ce3f3ff7f824aa64424cd481f9a1f25" 1177 | dependencies = [ 1178 | "cfg-if", 1179 | "libc", 1180 | "log", 1181 | "wepoll-ffi", 1182 | "winapi", 1183 | ] 1184 | 1185 | [[package]] 1186 | name = "ppv-lite86" 1187 | version = "0.2.10" 1188 | source = "registry+https://github.com/rust-lang/crates.io-index" 1189 | checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" 1190 | 1191 | [[package]] 1192 | name = "proc-macro-hack" 1193 | version = "0.5.19" 1194 | source = "registry+https://github.com/rust-lang/crates.io-index" 1195 | checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" 1196 | 1197 | [[package]] 1198 | name = "proc-macro-nested" 1199 | version = "0.1.7" 1200 | source = "registry+https://github.com/rust-lang/crates.io-index" 1201 | checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" 1202 | 1203 | [[package]] 1204 | name = "proc-macro2" 1205 | version = "1.0.28" 1206 | source = "registry+https://github.com/rust-lang/crates.io-index" 1207 | checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" 1208 | dependencies = [ 1209 | "unicode-xid", 1210 | ] 1211 | 1212 | [[package]] 1213 | name = "prometheus" 1214 | version = "0.12.0" 1215 | source = "registry+https://github.com/rust-lang/crates.io-index" 1216 | checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c" 1217 | dependencies = [ 1218 | "cfg-if", 1219 | "fnv", 1220 | "lazy_static", 1221 | "libc", 1222 | "memchr", 1223 | "parking_lot", 1224 | "protobuf", 1225 | "reqwest", 1226 | "thiserror", 1227 | ] 1228 | 1229 | [[package]] 1230 | name = "protobuf" 1231 | version = "2.25.1" 1232 | source = "registry+https://github.com/rust-lang/crates.io-index" 1233 | checksum = "23129d50f2c9355ced935fce8a08bd706ee2e7ce2b3b33bf61dace0e379ac63a" 1234 | 1235 | [[package]] 1236 | name = "quote" 1237 | version = "1.0.9" 1238 | source = "registry+https://github.com/rust-lang/crates.io-index" 1239 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 1240 | dependencies = [ 1241 | "proc-macro2", 1242 | ] 1243 | 1244 | [[package]] 1245 | name = "rand" 1246 | version = "0.8.4" 1247 | source = "registry+https://github.com/rust-lang/crates.io-index" 1248 | checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" 1249 | dependencies = [ 1250 | "libc", 1251 | "rand_chacha", 1252 | "rand_core", 1253 | "rand_hc", 1254 | ] 1255 | 1256 | [[package]] 1257 | name = "rand_chacha" 1258 | version = "0.3.1" 1259 | source = "registry+https://github.com/rust-lang/crates.io-index" 1260 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 1261 | dependencies = [ 1262 | "ppv-lite86", 1263 | "rand_core", 1264 | ] 1265 | 1266 | [[package]] 1267 | name = "rand_core" 1268 | version = "0.6.3" 1269 | source = "registry+https://github.com/rust-lang/crates.io-index" 1270 | checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" 1271 | dependencies = [ 1272 | "getrandom", 1273 | ] 1274 | 1275 | [[package]] 1276 | name = "rand_hc" 1277 | version = "0.3.1" 1278 | source = "registry+https://github.com/rust-lang/crates.io-index" 1279 | checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" 1280 | dependencies = [ 1281 | "rand_core", 1282 | ] 1283 | 1284 | [[package]] 1285 | name = "redox_syscall" 1286 | version = "0.2.10" 1287 | source = "registry+https://github.com/rust-lang/crates.io-index" 1288 | checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" 1289 | dependencies = [ 1290 | "bitflags", 1291 | ] 1292 | 1293 | [[package]] 1294 | name = "regex" 1295 | version = "1.5.4" 1296 | source = "registry+https://github.com/rust-lang/crates.io-index" 1297 | checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" 1298 | dependencies = [ 1299 | "aho-corasick", 1300 | "memchr", 1301 | "regex-syntax", 1302 | ] 1303 | 1304 | [[package]] 1305 | name = "regex-automata" 1306 | version = "0.1.10" 1307 | source = "registry+https://github.com/rust-lang/crates.io-index" 1308 | checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" 1309 | dependencies = [ 1310 | "regex-syntax", 1311 | ] 1312 | 1313 | [[package]] 1314 | name = "regex-syntax" 1315 | version = "0.6.25" 1316 | source = "registry+https://github.com/rust-lang/crates.io-index" 1317 | checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" 1318 | 1319 | [[package]] 1320 | name = "remove_dir_all" 1321 | version = "0.5.3" 1322 | source = "registry+https://github.com/rust-lang/crates.io-index" 1323 | checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" 1324 | dependencies = [ 1325 | "winapi", 1326 | ] 1327 | 1328 | [[package]] 1329 | name = "reqwest" 1330 | version = "0.11.4" 1331 | source = "registry+https://github.com/rust-lang/crates.io-index" 1332 | checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" 1333 | dependencies = [ 1334 | "base64", 1335 | "bytes", 1336 | "encoding_rs", 1337 | "futures-core", 1338 | "futures-util", 1339 | "http", 1340 | "http-body", 1341 | "hyper", 1342 | "hyper-tls", 1343 | "ipnet", 1344 | "js-sys", 1345 | "lazy_static", 1346 | "log", 1347 | "mime", 1348 | "native-tls", 1349 | "percent-encoding", 1350 | "pin-project-lite", 1351 | "serde", 1352 | "serde_urlencoded", 1353 | "tokio", 1354 | "tokio-native-tls", 1355 | "url", 1356 | "wasm-bindgen", 1357 | "wasm-bindgen-futures", 1358 | "web-sys", 1359 | "winreg", 1360 | ] 1361 | 1362 | [[package]] 1363 | name = "ryu" 1364 | version = "1.0.5" 1365 | source = "registry+https://github.com/rust-lang/crates.io-index" 1366 | checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" 1367 | 1368 | [[package]] 1369 | name = "schannel" 1370 | version = "0.1.19" 1371 | source = "registry+https://github.com/rust-lang/crates.io-index" 1372 | checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" 1373 | dependencies = [ 1374 | "lazy_static", 1375 | "winapi", 1376 | ] 1377 | 1378 | [[package]] 1379 | name = "scopeguard" 1380 | version = "1.1.0" 1381 | source = "registry+https://github.com/rust-lang/crates.io-index" 1382 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 1383 | 1384 | [[package]] 1385 | name = "security-framework" 1386 | version = "2.3.1" 1387 | source = "registry+https://github.com/rust-lang/crates.io-index" 1388 | checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" 1389 | dependencies = [ 1390 | "bitflags", 1391 | "core-foundation", 1392 | "core-foundation-sys", 1393 | "libc", 1394 | "security-framework-sys", 1395 | ] 1396 | 1397 | [[package]] 1398 | name = "security-framework-sys" 1399 | version = "2.3.0" 1400 | source = "registry+https://github.com/rust-lang/crates.io-index" 1401 | checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" 1402 | dependencies = [ 1403 | "core-foundation-sys", 1404 | "libc", 1405 | ] 1406 | 1407 | [[package]] 1408 | name = "serde" 1409 | version = "1.0.127" 1410 | source = "registry+https://github.com/rust-lang/crates.io-index" 1411 | checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" 1412 | 1413 | [[package]] 1414 | name = "serde_json" 1415 | version = "1.0.66" 1416 | source = "registry+https://github.com/rust-lang/crates.io-index" 1417 | checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" 1418 | dependencies = [ 1419 | "itoa", 1420 | "ryu", 1421 | "serde", 1422 | ] 1423 | 1424 | [[package]] 1425 | name = "serde_urlencoded" 1426 | version = "0.7.0" 1427 | source = "registry+https://github.com/rust-lang/crates.io-index" 1428 | checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" 1429 | dependencies = [ 1430 | "form_urlencoded", 1431 | "itoa", 1432 | "ryu", 1433 | "serde", 1434 | ] 1435 | 1436 | [[package]] 1437 | name = "sharded-slab" 1438 | version = "0.1.3" 1439 | source = "registry+https://github.com/rust-lang/crates.io-index" 1440 | checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" 1441 | dependencies = [ 1442 | "lazy_static", 1443 | ] 1444 | 1445 | [[package]] 1446 | name = "signal-hook" 1447 | version = "0.3.9" 1448 | source = "registry+https://github.com/rust-lang/crates.io-index" 1449 | checksum = "470c5a6397076fae0094aaf06a08e6ba6f37acb77d3b1b91ea92b4d6c8650c39" 1450 | dependencies = [ 1451 | "libc", 1452 | "signal-hook-registry", 1453 | ] 1454 | 1455 | [[package]] 1456 | name = "signal-hook-registry" 1457 | version = "1.4.0" 1458 | source = "registry+https://github.com/rust-lang/crates.io-index" 1459 | checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" 1460 | dependencies = [ 1461 | "libc", 1462 | ] 1463 | 1464 | [[package]] 1465 | name = "slab" 1466 | version = "0.4.4" 1467 | source = "registry+https://github.com/rust-lang/crates.io-index" 1468 | checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" 1469 | 1470 | [[package]] 1471 | name = "sluice" 1472 | version = "0.5.5" 1473 | source = "registry+https://github.com/rust-lang/crates.io-index" 1474 | checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" 1475 | dependencies = [ 1476 | "async-channel", 1477 | "futures-core", 1478 | "futures-io", 1479 | ] 1480 | 1481 | [[package]] 1482 | name = "smallvec" 1483 | version = "1.6.1" 1484 | source = "registry+https://github.com/rust-lang/crates.io-index" 1485 | checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" 1486 | 1487 | [[package]] 1488 | name = "socket2" 1489 | version = "0.4.1" 1490 | source = "registry+https://github.com/rust-lang/crates.io-index" 1491 | checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" 1492 | dependencies = [ 1493 | "libc", 1494 | "winapi", 1495 | ] 1496 | 1497 | [[package]] 1498 | name = "spin" 1499 | version = "0.9.2" 1500 | source = "registry+https://github.com/rust-lang/crates.io-index" 1501 | checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" 1502 | dependencies = [ 1503 | "lock_api", 1504 | ] 1505 | 1506 | [[package]] 1507 | name = "syn" 1508 | version = "1.0.74" 1509 | source = "registry+https://github.com/rust-lang/crates.io-index" 1510 | checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" 1511 | dependencies = [ 1512 | "proc-macro2", 1513 | "quote", 1514 | "unicode-xid", 1515 | ] 1516 | 1517 | [[package]] 1518 | name = "tempfile" 1519 | version = "3.2.0" 1520 | source = "registry+https://github.com/rust-lang/crates.io-index" 1521 | checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" 1522 | dependencies = [ 1523 | "cfg-if", 1524 | "libc", 1525 | "rand", 1526 | "redox_syscall", 1527 | "remove_dir_all", 1528 | "winapi", 1529 | ] 1530 | 1531 | [[package]] 1532 | name = "thiserror" 1533 | version = "1.0.29" 1534 | source = "registry+https://github.com/rust-lang/crates.io-index" 1535 | checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88" 1536 | dependencies = [ 1537 | "thiserror-impl", 1538 | ] 1539 | 1540 | [[package]] 1541 | name = "thiserror-impl" 1542 | version = "1.0.29" 1543 | source = "registry+https://github.com/rust-lang/crates.io-index" 1544 | checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c" 1545 | dependencies = [ 1546 | "proc-macro2", 1547 | "quote", 1548 | "syn", 1549 | ] 1550 | 1551 | [[package]] 1552 | name = "thread_local" 1553 | version = "1.1.3" 1554 | source = "registry+https://github.com/rust-lang/crates.io-index" 1555 | checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" 1556 | dependencies = [ 1557 | "once_cell", 1558 | ] 1559 | 1560 | [[package]] 1561 | name = "threadpool" 1562 | version = "1.8.1" 1563 | source = "registry+https://github.com/rust-lang/crates.io-index" 1564 | checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" 1565 | dependencies = [ 1566 | "num_cpus", 1567 | ] 1568 | 1569 | [[package]] 1570 | name = "thrift" 1571 | version = "0.13.0" 1572 | source = "registry+https://github.com/rust-lang/crates.io-index" 1573 | checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b" 1574 | dependencies = [ 1575 | "byteorder", 1576 | "integer-encoding", 1577 | "log", 1578 | "ordered-float", 1579 | "threadpool", 1580 | ] 1581 | 1582 | [[package]] 1583 | name = "tinyvec" 1584 | version = "1.3.1" 1585 | source = "registry+https://github.com/rust-lang/crates.io-index" 1586 | checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" 1587 | dependencies = [ 1588 | "tinyvec_macros", 1589 | ] 1590 | 1591 | [[package]] 1592 | name = "tinyvec_macros" 1593 | version = "0.1.0" 1594 | source = "registry+https://github.com/rust-lang/crates.io-index" 1595 | checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" 1596 | 1597 | [[package]] 1598 | name = "tokio" 1599 | version = "1.10.0" 1600 | source = "registry+https://github.com/rust-lang/crates.io-index" 1601 | checksum = "01cf844b23c6131f624accf65ce0e4e9956a8bb329400ea5bcc26ae3a5c20b0b" 1602 | dependencies = [ 1603 | "autocfg", 1604 | "bytes", 1605 | "libc", 1606 | "memchr", 1607 | "mio", 1608 | "num_cpus", 1609 | "once_cell", 1610 | "parking_lot", 1611 | "pin-project-lite", 1612 | "signal-hook-registry", 1613 | "tokio-macros", 1614 | "winapi", 1615 | ] 1616 | 1617 | [[package]] 1618 | name = "tokio-macros" 1619 | version = "1.3.0" 1620 | source = "registry+https://github.com/rust-lang/crates.io-index" 1621 | checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" 1622 | dependencies = [ 1623 | "proc-macro2", 1624 | "quote", 1625 | "syn", 1626 | ] 1627 | 1628 | [[package]] 1629 | name = "tokio-native-tls" 1630 | version = "0.3.0" 1631 | source = "registry+https://github.com/rust-lang/crates.io-index" 1632 | checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" 1633 | dependencies = [ 1634 | "native-tls", 1635 | "tokio", 1636 | ] 1637 | 1638 | [[package]] 1639 | name = "tokio-util" 1640 | version = "0.6.7" 1641 | source = "registry+https://github.com/rust-lang/crates.io-index" 1642 | checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" 1643 | dependencies = [ 1644 | "bytes", 1645 | "futures-core", 1646 | "futures-sink", 1647 | "log", 1648 | "pin-project-lite", 1649 | "tokio", 1650 | ] 1651 | 1652 | [[package]] 1653 | name = "tower-service" 1654 | version = "0.3.1" 1655 | source = "registry+https://github.com/rust-lang/crates.io-index" 1656 | checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" 1657 | 1658 | [[package]] 1659 | name = "tracing" 1660 | version = "0.1.26" 1661 | source = "registry+https://github.com/rust-lang/crates.io-index" 1662 | checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" 1663 | dependencies = [ 1664 | "cfg-if", 1665 | "log", 1666 | "pin-project-lite", 1667 | "tracing-attributes", 1668 | "tracing-core", 1669 | ] 1670 | 1671 | [[package]] 1672 | name = "tracing-attributes" 1673 | version = "0.1.15" 1674 | source = "registry+https://github.com/rust-lang/crates.io-index" 1675 | checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" 1676 | dependencies = [ 1677 | "proc-macro2", 1678 | "quote", 1679 | "syn", 1680 | ] 1681 | 1682 | [[package]] 1683 | name = "tracing-core" 1684 | version = "0.1.18" 1685 | source = "registry+https://github.com/rust-lang/crates.io-index" 1686 | checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" 1687 | dependencies = [ 1688 | "lazy_static", 1689 | ] 1690 | 1691 | [[package]] 1692 | name = "tracing-futures" 1693 | version = "0.2.5" 1694 | source = "registry+https://github.com/rust-lang/crates.io-index" 1695 | checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" 1696 | dependencies = [ 1697 | "pin-project", 1698 | "tracing", 1699 | ] 1700 | 1701 | [[package]] 1702 | name = "tracing-log" 1703 | version = "0.1.2" 1704 | source = "registry+https://github.com/rust-lang/crates.io-index" 1705 | checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" 1706 | dependencies = [ 1707 | "lazy_static", 1708 | "log", 1709 | "tracing-core", 1710 | ] 1711 | 1712 | [[package]] 1713 | name = "tracing-serde" 1714 | version = "0.1.2" 1715 | source = "registry+https://github.com/rust-lang/crates.io-index" 1716 | checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" 1717 | dependencies = [ 1718 | "serde", 1719 | "tracing-core", 1720 | ] 1721 | 1722 | [[package]] 1723 | name = "tracing-subscriber" 1724 | version = "0.2.20" 1725 | source = "registry+https://github.com/rust-lang/crates.io-index" 1726 | checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe" 1727 | dependencies = [ 1728 | "ansi_term", 1729 | "chrono", 1730 | "lazy_static", 1731 | "matchers", 1732 | "regex", 1733 | "serde", 1734 | "serde_json", 1735 | "sharded-slab", 1736 | "smallvec", 1737 | "thread_local", 1738 | "tracing", 1739 | "tracing-core", 1740 | "tracing-log", 1741 | "tracing-serde", 1742 | ] 1743 | 1744 | [[package]] 1745 | name = "try-lock" 1746 | version = "0.2.3" 1747 | source = "registry+https://github.com/rust-lang/crates.io-index" 1748 | checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" 1749 | 1750 | [[package]] 1751 | name = "unicode-bidi" 1752 | version = "0.3.6" 1753 | source = "registry+https://github.com/rust-lang/crates.io-index" 1754 | checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" 1755 | 1756 | [[package]] 1757 | name = "unicode-normalization" 1758 | version = "0.1.19" 1759 | source = "registry+https://github.com/rust-lang/crates.io-index" 1760 | checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" 1761 | dependencies = [ 1762 | "tinyvec", 1763 | ] 1764 | 1765 | [[package]] 1766 | name = "unicode-xid" 1767 | version = "0.2.2" 1768 | source = "registry+https://github.com/rust-lang/crates.io-index" 1769 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 1770 | 1771 | [[package]] 1772 | name = "url" 1773 | version = "2.2.2" 1774 | source = "registry+https://github.com/rust-lang/crates.io-index" 1775 | checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" 1776 | dependencies = [ 1777 | "form_urlencoded", 1778 | "idna", 1779 | "matches", 1780 | "percent-encoding", 1781 | ] 1782 | 1783 | [[package]] 1784 | name = "value-bag" 1785 | version = "1.0.0-alpha.7" 1786 | source = "registry+https://github.com/rust-lang/crates.io-index" 1787 | checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" 1788 | dependencies = [ 1789 | "ctor", 1790 | "version_check", 1791 | ] 1792 | 1793 | [[package]] 1794 | name = "vcpkg" 1795 | version = "0.2.15" 1796 | source = "registry+https://github.com/rust-lang/crates.io-index" 1797 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1798 | 1799 | [[package]] 1800 | name = "version_check" 1801 | version = "0.9.3" 1802 | source = "registry+https://github.com/rust-lang/crates.io-index" 1803 | checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" 1804 | 1805 | [[package]] 1806 | name = "waker-fn" 1807 | version = "1.1.0" 1808 | source = "registry+https://github.com/rust-lang/crates.io-index" 1809 | checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" 1810 | 1811 | [[package]] 1812 | name = "want" 1813 | version = "0.3.0" 1814 | source = "registry+https://github.com/rust-lang/crates.io-index" 1815 | checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" 1816 | dependencies = [ 1817 | "log", 1818 | "try-lock", 1819 | ] 1820 | 1821 | [[package]] 1822 | name = "wasi" 1823 | version = "0.10.2+wasi-snapshot-preview1" 1824 | source = "registry+https://github.com/rust-lang/crates.io-index" 1825 | checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" 1826 | 1827 | [[package]] 1828 | name = "wasm-bindgen" 1829 | version = "0.2.75" 1830 | source = "registry+https://github.com/rust-lang/crates.io-index" 1831 | checksum = "b608ecc8f4198fe8680e2ed18eccab5f0cd4caaf3d83516fa5fb2e927fda2586" 1832 | dependencies = [ 1833 | "cfg-if", 1834 | "serde", 1835 | "serde_json", 1836 | "wasm-bindgen-macro", 1837 | ] 1838 | 1839 | [[package]] 1840 | name = "wasm-bindgen-backend" 1841 | version = "0.2.75" 1842 | source = "registry+https://github.com/rust-lang/crates.io-index" 1843 | checksum = "580aa3a91a63d23aac5b6b267e2d13cb4f363e31dce6c352fca4752ae12e479f" 1844 | dependencies = [ 1845 | "bumpalo", 1846 | "lazy_static", 1847 | "log", 1848 | "proc-macro2", 1849 | "quote", 1850 | "syn", 1851 | "wasm-bindgen-shared", 1852 | ] 1853 | 1854 | [[package]] 1855 | name = "wasm-bindgen-futures" 1856 | version = "0.4.25" 1857 | source = "registry+https://github.com/rust-lang/crates.io-index" 1858 | checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" 1859 | dependencies = [ 1860 | "cfg-if", 1861 | "js-sys", 1862 | "wasm-bindgen", 1863 | "web-sys", 1864 | ] 1865 | 1866 | [[package]] 1867 | name = "wasm-bindgen-macro" 1868 | version = "0.2.75" 1869 | source = "registry+https://github.com/rust-lang/crates.io-index" 1870 | checksum = "171ebf0ed9e1458810dfcb31f2e766ad6b3a89dbda42d8901f2b268277e5f09c" 1871 | dependencies = [ 1872 | "quote", 1873 | "wasm-bindgen-macro-support", 1874 | ] 1875 | 1876 | [[package]] 1877 | name = "wasm-bindgen-macro-support" 1878 | version = "0.2.75" 1879 | source = "registry+https://github.com/rust-lang/crates.io-index" 1880 | checksum = "6c2657dd393f03aa2a659c25c6ae18a13a4048cebd220e147933ea837efc589f" 1881 | dependencies = [ 1882 | "proc-macro2", 1883 | "quote", 1884 | "syn", 1885 | "wasm-bindgen-backend", 1886 | "wasm-bindgen-shared", 1887 | ] 1888 | 1889 | [[package]] 1890 | name = "wasm-bindgen-shared" 1891 | version = "0.2.75" 1892 | source = "registry+https://github.com/rust-lang/crates.io-index" 1893 | checksum = "2e0c4a743a309662d45f4ede961d7afa4ba4131a59a639f29b0069c3798bbcc2" 1894 | 1895 | [[package]] 1896 | name = "web-sys" 1897 | version = "0.3.52" 1898 | source = "registry+https://github.com/rust-lang/crates.io-index" 1899 | checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" 1900 | dependencies = [ 1901 | "js-sys", 1902 | "wasm-bindgen", 1903 | ] 1904 | 1905 | [[package]] 1906 | name = "wepoll-ffi" 1907 | version = "0.1.2" 1908 | source = "registry+https://github.com/rust-lang/crates.io-index" 1909 | checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" 1910 | dependencies = [ 1911 | "cc", 1912 | ] 1913 | 1914 | [[package]] 1915 | name = "winapi" 1916 | version = "0.3.9" 1917 | source = "registry+https://github.com/rust-lang/crates.io-index" 1918 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1919 | dependencies = [ 1920 | "winapi-i686-pc-windows-gnu", 1921 | "winapi-x86_64-pc-windows-gnu", 1922 | ] 1923 | 1924 | [[package]] 1925 | name = "winapi-i686-pc-windows-gnu" 1926 | version = "0.4.0" 1927 | source = "registry+https://github.com/rust-lang/crates.io-index" 1928 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1929 | 1930 | [[package]] 1931 | name = "winapi-x86_64-pc-windows-gnu" 1932 | version = "0.4.0" 1933 | source = "registry+https://github.com/rust-lang/crates.io-index" 1934 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1935 | 1936 | [[package]] 1937 | name = "winreg" 1938 | version = "0.7.0" 1939 | source = "registry+https://github.com/rust-lang/crates.io-index" 1940 | checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" 1941 | dependencies = [ 1942 | "winapi", 1943 | ] 1944 | --------------------------------------------------------------------------------