├── .clang-format ├── .clang-tidy ├── .dir-locals.el ├── .dockerignore ├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .gitmodules ├── .style.yapf ├── .travis.yml ├── CMakeLists.txt ├── CMakeLists.txt.in ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── CPPLINT.cfg ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── ci ├── build-in-docker.sh └── test.sh ├── cmake ├── FindFlatbuffers.cmake ├── FindHdrhistogram.cmake ├── FindZstd.cmake ├── FindxxHash.cmake ├── ccache.cmake ├── doxygen_gen.cmake ├── dpdk_configure.cmake ├── set_option.cmake ├── smf-config.cmake.in ├── smfc_generator.cmake └── tests.cmake ├── demo_apps ├── CMakeLists.txt ├── cpp │ ├── CMakeLists.txt │ ├── client │ │ └── main.cc │ └── server │ │ └── main.cc └── demo_service.fbs ├── docs ├── .gitignore ├── 404.html ├── Gemfile ├── Gemfile.lock ├── _config.yml ├── _includes │ ├── head.html │ └── sidebar.html ├── _layouts │ ├── default.html │ ├── page.html │ └── post.html ├── _posts │ ├── 2017-10-12-introducing-smf.md │ ├── 2018-10-29-demo-projects.md │ ├── 2019-02-10-xfsprogs.md │ └── 2019-03-25-getting-started.md ├── _wip │ └── LDFI.md ├── atom.xml ├── contact.md ├── doxy.in ├── getting_started.md ├── index.html ├── presentations.md ├── public │ ├── annotated_rpc.png │ ├── apple-touch-icon-144-precomposed.png │ ├── clock_pro_cache.png │ ├── css │ │ ├── hyde.css │ │ ├── poole.css │ │ └── syntax.css │ ├── kafka_vs_smf_latency.png │ ├── kafka_vs_smf_qps.png │ ├── logo.png │ ├── services.png │ ├── smf_wal_filesystem.png │ ├── smf_write_ahead_log.png │ ├── wal_kafka_latency_vs_smf_1producer.png │ └── wal_kafka_latency_vs_smf_3producers.png ├── rpc.md └── server.sh ├── dpdk_config ├── install-deps.sh ├── src ├── .gitignore ├── CMakeLists.txt ├── afl_tests │ └── rpc │ │ └── rpc.dict ├── benchmarks │ ├── CMakeLists.txt │ ├── checksum_bench │ │ └── main.cc │ └── fbs_alloc │ │ ├── CMakeLists.txt │ │ ├── kv.fbs │ │ └── main.cc ├── core │ ├── README.md │ ├── compression.cc │ ├── histogram.cc │ ├── histogram_seastar_utils.cc │ ├── lz4_filter.cc │ ├── random.cc │ ├── rpc.fbs │ ├── rpc_client.cc │ ├── rpc_envelope.cc │ ├── rpc_handle_router.cc │ ├── rpc_letter.cc │ ├── rpc_recv_context.cc │ ├── rpc_server.cc │ └── zstd_filter.cc ├── include │ └── smf │ │ ├── compression.h │ │ ├── fbs_typed_buf.h │ │ ├── futurize_utils.h │ │ ├── histogram.h │ │ ├── histogram_seastar_utils.h │ │ ├── human_bytes.h │ │ ├── load_channel.h │ │ ├── load_generator.h │ │ ├── load_generator_args.h │ │ ├── load_generator_duration.h │ │ ├── log.h │ │ ├── lz4_filter.h │ │ ├── macros.h │ │ ├── native_type_utils.h │ │ ├── random.h │ │ ├── reconnect_client.h │ │ ├── rpc_client.h │ │ ├── rpc_connection.h │ │ ├── rpc_connection_limits.h │ │ ├── rpc_envelope.h │ │ ├── rpc_filter.h │ │ ├── rpc_handle_router.h │ │ ├── rpc_header_ostream.h │ │ ├── rpc_header_utils.h │ │ ├── rpc_letter.h │ │ ├── rpc_recv_context.h │ │ ├── rpc_recv_typed_context.h │ │ ├── rpc_server.h │ │ ├── rpc_server_args.h │ │ ├── rpc_server_connection.h │ │ ├── rpc_server_stats.h │ │ ├── rpc_service.h │ │ ├── rpc_typed_envelope.h │ │ ├── std-compat.h │ │ ├── time_utils.h │ │ ├── unique_histogram_adder.h │ │ └── zstd_filter.h ├── integration_tests │ ├── CMakeLists.txt │ ├── demo_service.fbs │ ├── histograms │ │ ├── main.cc │ │ └── test.json │ ├── hystrix │ │ └── main.cc │ ├── non_root_port.h │ ├── rpc │ │ ├── main.cc │ │ └── test.json │ ├── rpc_backpressure │ │ ├── main.cc │ │ └── test.json │ ├── rpc_bad_msg_t │ │ ├── CMakeLists.txt │ │ ├── bad_msg.fbs │ │ ├── bad_svc.fbs │ │ └── main.cc │ ├── rpc_multiple_remote_addrs │ │ ├── main.cc │ │ └── test.json │ ├── rpc_reconnect_with_timeout │ │ ├── main.cc │ │ └── test.json │ ├── rpc_recv_timeout │ │ ├── main.cc │ │ └── test.json │ └── rpc_send_timeout │ │ ├── main.cc │ │ └── test.json ├── smfc │ ├── .gitignore │ ├── README.md │ ├── codegen.cc │ ├── codegen.h │ ├── cpp_generator.cc │ ├── cpp_generator.h │ ├── crc.h │ ├── generator.h │ ├── go_generator.cc │ ├── go_generator.h │ ├── language.h │ ├── main.cc │ ├── python_generator.cc │ ├── python_generator.h │ ├── smf_method.h │ ├── smf_printer.h │ └── smf_service.h ├── test_runner.py └── tests │ ├── CMakeLists.txt │ ├── histgen.cc │ ├── histogram_tests.cc │ └── randomstr.cc └── tools ├── base └── Dockerfile ├── build.sh ├── docker-deps.sh ├── docker └── Dockerfile ├── fmt.py ├── git_email_merge_confirmation.sh ├── git_merge_patch.sh ├── local.sh ├── one.sh ├── shellcheck.sh └── travis_stdout.sh /.clang-format: -------------------------------------------------------------------------------- 1 | # After playing around: http://cf.monofraps.net/ 2 | # 3 | BasedOnStyle: LLVM 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: true 6 | AlignEscapedNewlinesLeft: false 7 | AlignOperands: true 8 | AlignTrailingComments: true 9 | AllowAllParametersOfDeclarationOnNextLine: true 10 | AllowShortBlocksOnASingleLine: true 11 | AllowShortCaseLabelsOnASingleLine: false 12 | AllowShortFunctionsOnASingleLine: All 13 | AllowShortIfStatementsOnASingleLine: true 14 | AllowShortLoopsOnASingleLine: false 15 | AlwaysBreakAfterDefinitionReturnType: false 16 | AlwaysBreakAfterReturnType: AllDefinitions 17 | AlwaysBreakBeforeMultilineStrings: false 18 | AlwaysBreakTemplateDeclarations: true 19 | BinPackArguments: true 20 | BinPackParameters: true 21 | BreakBeforeBinaryOperators: None 22 | BreakBeforeBraces: Attach 23 | BreakBeforeTernaryOperators: true 24 | BreakConstructorInitializersBeforeComma: false 25 | ColumnLimit: 80 26 | CommentPragmas: '^ IWYU pragma:' 27 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 28 | ConstructorInitializerIndentWidth: 2 29 | ContinuationIndentWidth: 2 30 | Cpp11BracedListStyle: true 31 | DerivePointerAlignment: false 32 | DisableFormat: false 33 | ExperimentalAutoDetectBinPacking: false 34 | IndentCaseLabels: false 35 | IndentWidth: 2 36 | IndentWrappedFunctionNames: false 37 | KeepEmptyLinesAtTheStartOfBlocks: true 38 | Language: Cpp 39 | MaxEmptyLinesToKeep: 1 40 | NamespaceIndentation: None 41 | ObjCBlockIndentWidth: 2 42 | ObjCSpaceAfterProperty: false 43 | ObjCSpaceBeforeProtocolList: true 44 | PenaltyBreakBeforeFirstCallParameter: 19 45 | PenaltyBreakComment: 300 46 | PenaltyBreakFirstLessLess: 120 47 | PenaltyBreakString: 1000 48 | PenaltyExcessCharacter: 1000000 49 | PenaltyReturnTypeOnItsOwnLine: 60 50 | PointerAlignment: Right 51 | SpaceAfterCStyleCast: false 52 | SpaceBeforeAssignmentOperators: true 53 | SpaceBeforeParens: ControlStatements 54 | SpaceInEmptyParentheses: false 55 | SpacesBeforeTrailingComments: 2 56 | SpacesInAngles: false 57 | SpacesInCStyleCastParentheses: false 58 | SpacesInContainerLiterals: true 59 | SpacesInParentheses: false 60 | SpacesInSquareBrackets: false 61 | Standard: Cpp11 62 | UseTab: Never 63 | TabWidth: 2 64 | # IncludeCategories: 65 | # - Regex: '^' 66 | # Priority: 2 67 | -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,readability-identifier-naming' 2 | CheckOptions: 3 | - key: readability-identifier-naming.ClassCase 4 | value: snake_case 5 | - key: readability-identifier-naming.EnumCase 6 | value: snake_case 7 | - key: readability-identifier-naming.FunctionCase 8 | value: lowerCase 9 | - key: readability-identifier-naming.UnionCase 10 | value: snake_case 11 | - key: readability-identifier-naming.VariableCase 12 | value: snake_case 13 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;; Project-wide Emacs settings 2 | ;; We use clang-format for the whole project 3 | ;; see misc/fmt.py - but this sets up the basic 4 | ;; environment for c++ 5 | ((c++-mode (helm-make-build-dir . "build/debug"))) 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Asciinema** 24 | 25 | see https://asciinema.org/ for recording video in terminal 26 | 27 | 28 | ** Environment Info ** 29 | 30 | - operating system 31 | - git hash 32 | - compiler 33 | 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | build/ 30 | 31 | # bazel 32 | bazel-* 33 | 34 | # Vagrant 35 | .vagrant/ 36 | 37 | .gdb_history 38 | *.smf.fb.h* 39 | *_generated.h 40 | build_debug/ 41 | build_release/ 42 | 43 | .DS_Store 44 | 45 | # clang-tidy 46 | compile_commands.json 47 | 48 | # vim 49 | .*.swp 50 | 51 | CMakeCache.txt 52 | CMakeFiles/ 53 | Makefile 54 | cmake_install.cmake 55 | CPackConfig.cmake 56 | CPackSourceConfig.cmake 57 | CTestTestfile.cmake 58 | CMakeDoxyfile.in 59 | CMakeDoxygenDefaults.cmake 60 | deps-build/ 61 | smf-config-version.cmake 62 | smf-config.cmake 63 | smf-deps-build/ 64 | smf-deps-install/ 65 | smfTargets.cmake 66 | 67 | src/demo_apps/demo_client 68 | src/demo_apps/demo_server 69 | bin/ 70 | 71 | # tests 72 | src/integration_tests/*_test 73 | src/test/*_unit_test 74 | *.test 75 | 76 | # docs 77 | Doxyfile 78 | vendor/ 79 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/.gitmodules -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = pep8 3 | split_before_logical_operator = true -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: required 3 | language: minimal 4 | services: 5 | - docker 6 | 7 | matrix: 8 | include: 9 | - os: linux 10 | env: 11 | - DOCKER_IMAGE=fedora:30 12 | - BUILD_GENERATOR=Ninja 13 | - CI=1 14 | - os: linux 15 | env: 16 | - DOCKER_IMAGE=fedora:29 17 | - BUILD_GENERATOR=Ninja 18 | - CI=1 19 | - os: linux 20 | env: 21 | - DOCKER_IMAGE=ubuntu:disco 22 | - CI=1 23 | - CC=gcc-9 24 | - CXX=g++-9 25 | - os: linux 26 | env: 27 | - DOCKER_IMAGE=ubuntu:bionic 28 | - BUILD_GENERATOR=Ninja 29 | - CI=1 30 | - CC=gcc-8 31 | - CXX=g++-8 32 | script: 33 | - ci/build-in-docker.sh ${DOCKER_IMAGE} 34 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at gallego.alexx@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to smf (/smɝf/) 2 | ====================================== 3 | 4 | # Sending Patches 5 | 6 | #### Note on corporate contributions. 7 | We do not have the bandwith to deal w/ the legalities from corporate contributions. Please 8 | make all your contributions as an individual with DCO. 9 | Copy right should be `Copyright (c) SMF Authors. All rights reserved.` going forward. 10 | 11 | ## tl;dr 12 | 13 | `smf` follows a patch submission similar to Linux. 14 | Send patches to smf-dev, with a 15 | [DCO](http://elinux.org/Developer_Certificate_Of_Origin) 16 | signed-off-message. 17 | 18 | Use `git send-email` to send your patch. 19 | 20 | ## Example: 21 | 22 | 1. When you commit, use `"-s"` in your git commit command, which adds a DCO 23 | signed off message. DCO is a 24 | [Developer's Certificate of Origin](http://elinux.org/Developer_Certificate_Of_Origin) 25 | . For the commit message, you can prefix a tag for an area of the codebase the patch is addressing 26 | 27 | ``` 28 | git commit -s -m "core: some descriptive commit message" 29 | ``` 30 | 31 | 2. then send an email to the google group 32 | 33 | ``` 34 | git send-email .. --to smf-dev@googlegroups.com 35 | ``` 36 | 37 | NOTE: for sending replies to patches, use --in-reply-to with the message ID of 38 | the original message. Also, if you are sending out a new version of the change, 39 | use git rebase and then a `git send-email` with a `-v2`, for instance, to 40 | denote that it is a second version. 41 | 42 | ## Testing and Approval 43 | 44 | Run `$ROOT/tools/build.sh -rt` and ensure tests are passing (at least) as well as before the 45 | patch. 46 | 47 | 48 | 49 | ## References: 50 | 51 | [Linux Kernel docs on mail clients](https://www.kernel.org/doc/Documentation/email-clients.txt) 52 | 53 | 54 | ## Reminders: 55 | 56 | * Ensure there are docs & tests 57 | * Make sure clang-format is ran on all the cpp buffers 58 | * Check the log levels. Use `LOG_INFO` with tender loving care. 59 | * Make sure the patch was submitted with --sign-off 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | # Do not continue looking up the directory hierarchy 2 | # for more config files. 3 | set noparent 4 | linelength=80 5 | root=src 6 | # already protected by the build - won't compile 7 | filter=-build/header_guard 8 | filter=-build/c++11 9 | filter=-build/include_order 10 | filter=-runtime/printf 11 | filter=-build/include 12 | filter=-readability/inheritance 13 | # enable c++14 stuff 14 | filter=+build/c++14 15 | filter=+build/include_alpha 16 | filter=+readability/casting 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Alexander Gallego 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # References 2 | 3 | Site | Link 4 | ------------ | -------- 5 | Mailing List | https://groups.google.com/forum/#!forum/smf-dev 6 | Documentation | https://smfrpc.github.io/smf/ 7 | 8 | # Big changes 9 | 10 | Please discuss them on the mailing list first 11 | 12 | # Minor changes 13 | 14 | We prefer mailing list patches, as described in CONTRIBUTING.md. 15 | 16 | We will also accept them via github pull requests. 17 | 18 | Take a look at the open issues with the tag good first issue - https://github.com/smfrpc/smf/labels/good%20first%20issue 19 | 20 | # For additional guidelines see our CONTRIBUTING.md document 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [smf - the fastest RPC in the West](http://smfrpc.github.io/smf/) [![Travis-ci Build Status](https://travis-ci.org/smfrpc/smf.svg?branch=master)](https://travis-ci.org/smfrpc/smf) 2 | 3 | ![](docs/public/logo.png) 4 | 5 | 6 | **smf** is pronounced **/smɝf/** 7 | 8 | Site | Link 9 | ------------ | -------- 10 | Mailing List | https://groups.google.com/forum/#!forum/smf-dev 11 | Documentation | https://smfrpc.github.io/smf/ 12 | 13 | # [Official Documentation](https://smfrpc.github.io/smf) 14 | 15 | Please visit our official documentation, 16 | it'll get you up and running in no time! 17 | 18 | If you are using **smf**, drop us a line on the mailing list introducing your project. 19 | 20 | 21 | # What is smf? 22 | 23 | **smf** is a new RPC system and code generation like gRPC, Cap n Proto, 24 | Apache Thrift, etc, but designed for **microsecond tail latency***. 25 | 26 | Current benchmarks in microseconds 27 | 28 | | 60 byte payload | latency | 29 | | ---------------- | --------- | 30 | | p50 | 7us | 31 | | p90 | 8us | 32 | | p99 | 8us | 33 | | p9999 | 15us | 34 | | p100 | 26us | 35 | 36 | 37 | 38 | # Getting started 39 | 40 | Please see our quick 41 | [getting started on our official docs!](https://smfrpc.github.io/smf//getting_started/) 42 | -------------------------------------------------------------------------------- /ci/build-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 SMF Authors 4 | # 5 | 6 | set -e 7 | set -x 8 | 9 | this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 10 | proj_dir=$(realpath "${this_dir}"/../) 11 | 12 | base_img=${1:-fedora:latest} 13 | 14 | # don't attach stdin/tty in CI environment 15 | if [ "${CI}" == "true" ]; then 16 | extra="" 17 | else 18 | extra="-it" 19 | fi 20 | 21 | docker run --rm --privileged \ 22 | -v "${proj_dir}":/src/smf:z,ro ${extra} \ 23 | -e BUILD_GENERATOR -e CI="${CI}" \ 24 | -e CC="${CC}" -e CXX="${CXX}" \ 25 | -w /src/smf "${base_img}" \ 26 | /bin/bash -c "./install-deps.sh && ci/test.sh" 27 | -------------------------------------------------------------------------------- /ci/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 SMF Authors 4 | # 5 | 6 | set -e 7 | set -x 8 | 9 | this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 10 | proj_dir=$(realpath "${this_dir}"/../) 11 | 12 | # shellcheck disable=SC1091 13 | source /etc/os-release 14 | 15 | CMAKE="cmake" 16 | CTEST="ctest" 17 | case ${ID} in 18 | centos|rhel) 19 | MAJOR_VERSION="$(echo "$VERSION_ID" | cut -d. -f1)" 20 | if test "$MAJOR_VERSION" = 7 ; then 21 | CMAKE="cmake3" 22 | CTEST="ctest3" 23 | # shellcheck disable=SC1091 24 | source /opt/rh/devtoolset-8/enable 25 | fi 26 | esac 27 | 28 | : "${BUILD_GENERATOR:=Unix Makefiles}" 29 | build_cmd="make VERBOSE=1" 30 | if [ "${BUILD_GENERATOR}" = "Ninja" ]; then 31 | build_cmd="ninja -v" 32 | fi 33 | 34 | # setup temp dirs 35 | build_dir=$(mktemp -d) 36 | trap 'rm -rf ${build_dir}' EXIT 37 | 38 | ( 39 | cd "${build_dir}" 40 | ${CMAKE} -G"${BUILD_GENERATOR}" -DSMF_MANAGE_DEPS=ON "${proj_dir}" 41 | ${build_cmd} -j"$(nproc)" 42 | ls -l bin 43 | ldd bin/smf_demo_server | grep smf-deps-install 44 | ldd bin/smf_demo_server | grep -v smf-deps-install 45 | ${CTEST} -V 46 | ) 47 | -------------------------------------------------------------------------------- /cmake/FindFlatbuffers.cmake: -------------------------------------------------------------------------------- 1 | find_package (PkgConfig REQUIRED) 2 | pkg_search_module (Flatbuffers_PC QUIET flatbuffers) 3 | find_library (Flatbuffers_LIBRARY 4 | NAMES flatbuffers 5 | HINTS 6 | ${Flatbuffers_PC_LIBDIR} 7 | ${Flatbuffers_PC_LIBRARY_DIRS}) 8 | find_path (Flatbuffers_INCLUDE_DIR 9 | NAMES "flatbuffers/flatbuffers.h" 10 | HINTS 11 | ${Flatbuffers_PC_INCLUDEDIR} 12 | ${Flatbuffers_PC_INCLUDEDIRS}) 13 | mark_as_advanced ( 14 | Flatbuffers_LIBRARY 15 | Flatbuffers_INCLUDE_DIR) 16 | 17 | include (FindPackageHandleStandardArgs) 18 | find_package_handle_standard_args (Flatbuffers 19 | REQUIRED_VARS 20 | Flatbuffers_LIBRARY 21 | Flatbuffers_INCLUDE_DIR 22 | VERSION_VAR Flatbuffers_PC_VERSION) 23 | 24 | set (Flatbuffers_LIBRARIES ${Flatbuffers_LIBRARY}) 25 | set (Flatbuffers_INCLUDE_DIRS ${Flatbuffers_INCLUDE_DIR}) 26 | if (Flatbuffers_FOUND AND NOT (TARGET Flatbuffers::flatbuffers)) 27 | add_library (Flatbuffers::flatbuffers UNKNOWN IMPORTED) 28 | set_target_properties (Flatbuffers::flatbuffers 29 | PROPERTIES 30 | IMPORTED_LOCATION ${Flatbuffers_LIBRARY} 31 | INTERFACE_INCLUDE_DIRECTORIES ${Flatbuffers_INCLUDE_DIRS}) 32 | endif () 33 | -------------------------------------------------------------------------------- /cmake/FindHdrhistogram.cmake: -------------------------------------------------------------------------------- 1 | find_path (Hdrhistogram_INCLUDE_DIR 2 | NAMES hdr_histogram.h 3 | PATH_SUFFIXES hdr) 4 | 5 | find_library (Hdrhistogram_LIBRARY 6 | NAMES hdr_histogram_static) 7 | 8 | mark_as_advanced ( 9 | Hdrhistogram_INCLUDE_DIR 10 | Hdrhistogram_LIBRARY) 11 | 12 | include (FindPackageHandleStandardArgs) 13 | 14 | find_package_handle_standard_args (Hdrhistogram 15 | REQUIRED_VARS 16 | Hdrhistogram_LIBRARY 17 | Hdrhistogram_INCLUDE_DIR) 18 | 19 | set (Hdrhistogram_INCLUDE_DIRS ${Hdrhistogram_INCLUDE_DIR}) 20 | set (Hdrhistogram_LIBRARIES ${Hdrhistogram_LIBRARY}) 21 | 22 | if (Hdrhistogram_FOUND AND NOT (TARGET Hdrhistogram::hdr_histogram)) 23 | add_library (Hdrhistogram::hdr_histogram UNKNOWN IMPORTED) 24 | 25 | set_target_properties (Hdrhistogram::hdr_histogram 26 | PROPERTIES 27 | IMPORTED_LOCATION ${Hdrhistogram_LIBRARIES} 28 | INTERFACE_INCLUDE_DIRECTORIES ${Hdrhistogram_INCLUDE_DIRS}) 29 | endif () 30 | -------------------------------------------------------------------------------- /cmake/FindZstd.cmake: -------------------------------------------------------------------------------- 1 | find_package (PkgConfig REQUIRED) 2 | 3 | pkg_search_module (Zstd_PC 4 | QUIET 5 | libzstd) 6 | 7 | find_library (Zstd_LIBRARY 8 | NAMES zstd 9 | HINTS 10 | ${Zstd_PC_LIBRARY_DIRS}) 11 | 12 | find_path (Zstd_INCLUDE_DIR 13 | NAMES zstd.h 14 | HINTS 15 | ${Zstd_PC_INCLUDE_DIRS}) 16 | 17 | mark_as_advanced ( 18 | Zstd_LIBRARY 19 | Zstd_INCLUDE_DIR) 20 | 21 | include (FindPackageHandleStandardArgs) 22 | 23 | find_package_handle_standard_args (Zstd 24 | REQUIRED_VARS 25 | Zstd_LIBRARY 26 | Zstd_INCLUDE_DIR 27 | VERSION_VAR Zstd_PC_VERSION) 28 | 29 | set (Zstd_LIBRARIES ${Zstd_LIBRARY}) 30 | set (Zstd_INCLUDE_DIRS ${Zstd_INCLUDE_DIR}) 31 | 32 | if (Zstd_FOUND AND NOT (TARGET Zstd::zstd)) 33 | add_library (Zstd::zstd UNKNOWN IMPORTED) 34 | 35 | set_target_properties (Zstd::zstd 36 | PROPERTIES 37 | IMPORTED_LOCATION ${Zstd_LIBRARY} 38 | INTERFACE_INCLUDE_DIRECTORIES ${Zstd_INCLUDE_DIRS}) 39 | endif () 40 | -------------------------------------------------------------------------------- /cmake/FindxxHash.cmake: -------------------------------------------------------------------------------- 1 | find_package (PkgConfig REQUIRED) 2 | pkg_search_module(xxHash_PC 3 | QUIET 4 | xxhash) 5 | find_path(xxHash_INCLUDE_DIR 6 | NAMES xxhash.h 7 | HINTS 8 | ${xxHash_PC_INCLUDEDIR} 9 | ${xxHash_PC_INCLUDEDIRS}) 10 | find_library (xxHash_LIBRARY 11 | NAMES xxhash 12 | HINTS 13 | ${xxHash_PC_LIBDIR} 14 | ${xxHash_PC_LIBRARY_DIRS}) 15 | mark_as_advanced( 16 | xxHash_INCLUDE_DIR 17 | xxHash_LIBRARY 18 | ) 19 | 20 | 21 | include(FindPackageHandleStandardArgs) 22 | 23 | find_package_handle_standard_args(xxHash 24 | REQUIRED_VARS 25 | xxHash_INCLUDE_DIR 26 | xxHash_LIBRARY 27 | VERSION_VAR xxHash_PC_VERSION) 28 | 29 | set(xxHash_LIBRARIES ${xxHash_LIBRARY}) 30 | set(xxHash_INCLUDE_DIRS ${xxHash_INCLUDE_DIR}) 31 | 32 | if (xxHash_FOUND AND NOT (TARGET xxHash::xxhash)) 33 | add_library(xxHash::xxhash UNKNOWN IMPORTED) 34 | set_target_properties(xxHash::xxhash 35 | PROPERTIES 36 | IMPORTED_LOCATION ${xxHash_LIBRARY} 37 | INTERFACE_INCLUDE_DIRECTORIES ${xxHash_INCLUDE_DIRS}) 38 | endif () 39 | 40 | # always allow for inline 41 | # To debug that the installation worked, uncomment the following lines 42 | # it verifies that there exist an override to the ExternalProjectAdd 43 | # INSTALL_COMMAND with a copy of the xxhash.c into the installation directory 44 | # set_target_properties( 45 | # xxHash::xxhash 46 | # PROPERTIES 47 | # COMPILE_DEFINITIONS -DXXH_PRIVATE_API) 48 | -------------------------------------------------------------------------------- /cmake/ccache.cmake: -------------------------------------------------------------------------------- 1 | # https://github.com/ceph/ceph/blob/master/CMakeLists.txt 2 | # Use CCACHE_DIR environment variable 3 | option(WITH_CCACHE "Build with ccache.") 4 | if(WITH_CCACHE) 5 | find_program (CCACHE_FOUND ccache) 6 | if(CCACHE_FOUND) 7 | message(STATUS "Building with ccache: ${CCACHE_FOUND}, CCACHE_DIR=$ENV{CCACHE_DIR}") 8 | set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) 9 | set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) 10 | else(CCACHE_FOUND) 11 | message(FATAL_ERROR "Can't find ccache. Is it installed?") 12 | endif(CCACHE_FOUND) 13 | endif(WITH_CCACHE) 14 | -------------------------------------------------------------------------------- /cmake/doxygen_gen.cmake: -------------------------------------------------------------------------------- 1 | find_package(Doxygen) 2 | if(DOXYGEN_FOUND AND NOT TARGET doc) 3 | configure_file(${CMAKE_CURRENT_SOURCE_DIR}/docs/doxy.in Doxyfile @ONLY) 4 | add_custom_target(doc 5 | ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile 6 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 7 | COMMENT "Generating API documentation with Doxygen" VERBATIM) 8 | endif() 9 | -------------------------------------------------------------------------------- /cmake/dpdk_configure.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # This file is open source software, licensed to you under the terms 3 | # of the Apache License, Version 2.0 (the "License"). See the NOTICE file 4 | # distributed with this work for additional information regarding copyright 5 | # ownership. You may not use this file except in compliance with the License. 6 | # 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | 19 | # 20 | # Copyright (C) 2018 Scylladb, Ltd. 21 | # 22 | 23 | file (READ ${Seastar_DPDK_CONFIG_FILE_IN} dpdk_config) 24 | file (STRINGS ${Seastar_DPDK_CONFIG_FILE_CHANGES} dpdk_config_changes) 25 | set (word_pattern "[^\n\r \t]+") 26 | 27 | foreach (var ${dpdk_config_changes}) 28 | if (var MATCHES "(${word_pattern})=(${word_pattern})") 29 | set (key ${CMAKE_MATCH_1}) 30 | set (value ${CMAKE_MATCH_2}) 31 | 32 | string (REGEX REPLACE 33 | "${key}=${word_pattern}" 34 | "${key}=${value}" 35 | dpdk_config 36 | ${dpdk_config}) 37 | endif () 38 | endforeach () 39 | 40 | file (WRITE ${Seastar_DPDK_CONFIG_FILE_OUT} ${dpdk_config}) 41 | -------------------------------------------------------------------------------- /cmake/set_option.cmake: -------------------------------------------------------------------------------- 1 | # source 2 | # https://github.com/edsiper/cmake-options 3 | # 4 | macro(SMF_SET_OPTION option value) 5 | set(${option} ${value} CACHE "" INTERNAL FORCE) 6 | endmacro() 7 | -------------------------------------------------------------------------------- /cmake/smf-config.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | include(CMakeFindDependencyMacro) 3 | list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) 4 | find_dependency(Seastar) 5 | find_dependency(xxHash) 6 | find_dependency(Flatbuffers) 7 | find_dependency(Zstd) 8 | find_dependency(Hdrhistogram) 9 | if (NOT TARGET smf::smf) 10 | include("${CMAKE_CURRENT_LIST_DIR}/smfTargets.cmake") 11 | endif () 12 | check_required_components("@PROJECT_NAME@") 13 | -------------------------------------------------------------------------------- /cmake/smfc_generator.cmake: -------------------------------------------------------------------------------- 1 | function(smfc_gen) 2 | set(options CPP GOLANG VERBOSE) 3 | set(oneValueArgs TARGET_NAME OUTPUT_DIRECTORY) 4 | set(multiValueArgs SOURCES INCLUDE_DIRS) 5 | cmake_parse_arguments(SMFC_GEN "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) 6 | 7 | # Do not put outside of function. We need them discovered later 8 | # per invocation 9 | find_program (FLATBUFFERS_COMPILER flatc) 10 | # for smfc do it in 2 stages. 11 | # first stage is for downstream consumers. 12 | # Second stage is for internal targets 13 | find_program (SMF_COMPILER smfc) 14 | if(NOT SMF_COMPILER) 15 | set(SMF_COMPILER $) 16 | endif() 17 | 18 | # We need one per generator so we can pass it to the flatc compiler 19 | # 20 | set(flatc_generated_includes) 21 | set(smfc_generated_includes) 22 | 23 | file(MAKE_DIRECTORY ${SMFC_GEN_OUTPUT_DIRECTORY}) 24 | 25 | # need to know the language we are generating 26 | set(smfc_language) 27 | set(flatc_language) 28 | if(SMFC_GEN_CPP) 29 | set(smfc_language "--language=cpp") 30 | set(flatc_language "--cpp") 31 | endif() 32 | if(SMFC_GEN_GOLANG) 33 | set(smfc_language "--language=go") 34 | set(flatc_language "--go") 35 | endif() 36 | # needed for the 'return' value 37 | set(SMF_GEN_OUTPUTS) 38 | 39 | # gen include dirs for each 40 | if(SMFC_GEN_INCLUDE_DIRS) 41 | foreach(d ${SMFC_GEN_INCLUDE_DIRS}) 42 | list(APPEND flatc_generated_includes -I ${d}) 43 | endforeach() 44 | string(REGEX REPLACE ";" "," 45 | smfc_generated_includes ${SMFC_GEN_INCLUDE_DIRS}) 46 | set(smfc_generated_includes 47 | "--include_dirs=${smfc_generated_includes}") 48 | endif() 49 | 50 | foreach(FILE ${SMFC_GEN_SOURCES}) 51 | get_filename_component(BASE_NAME ${FILE} NAME_WE) 52 | set(FLATC_OUTPUT 53 | "${SMFC_GEN_OUTPUT_DIRECTORY}/${BASE_NAME}_generated.h") 54 | set(SMF_GEN_OUTPUT 55 | "${SMFC_GEN_OUTPUT_DIRECTORY}/${BASE_NAME}.smf.fb.h") 56 | list(APPEND SMF_GEN_OUTPUTS ${FLATC_OUTPUT} ${SMF_GEN_OUTPUT}) 57 | # flatc 58 | add_custom_command(OUTPUT ${FLATC_OUTPUT} 59 | COMMAND ${FLATBUFFERS_COMPILER} 60 | ARGS --gen-name-strings --gen-object-api ${flatc_language} 61 | ARGS ${flatc_generated_includes} --force-empty --gen-compare 62 | ARGS --keep-prefix --json --reflect-names --defaults-json 63 | ARGS --gen-mutable --cpp-str-type 'seastar::sstring' 64 | ARGS --cpp-include 'seastar/core/sstring.hh' 65 | ARGS --cpp-str-flex-ctor 66 | ARGS -o "${SMFC_GEN_OUTPUT_DIRECTORY}/" "${FILE}" 67 | COMMENT "Building C++ header for ${FILE}" 68 | DEPENDS ${FILE} 69 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 70 | # smfc 71 | add_custom_command(OUTPUT ${SMF_GEN_OUTPUT} 72 | COMMAND ${SMF_COMPILER} 73 | ARGS --filename ${FILE} 74 | ARGS "${smfc_language}" 75 | ARGS "${smfc_generated_includes}" 76 | ARGS --output_path="${SMFC_GEN_OUTPUT_DIRECTORY}" 77 | COMMENT "Generating smf rpc stubs for ${FILE}" 78 | DEPENDS ${SMF_COMPILER} 79 | DEPENDS ${FILE} 80 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 81 | endforeach() 82 | set(${SMFC_GEN_TARGET_NAME} ${SMF_GEN_OUTPUTS} PARENT_SCOPE) 83 | endfunction() 84 | -------------------------------------------------------------------------------- /cmake/tests.cmake: -------------------------------------------------------------------------------- 1 | include(CMakeParseArguments) 2 | enable_testing() 3 | set(INTEGRATION_TESTS "") 4 | set(UNIT_TESTS "") 5 | set(BENCHMARK_TESTS "") 6 | set(TEST_RUNNER ${PROJECT_SOURCE_DIR}/src/test_runner.py) 7 | 8 | message(STATUS "SMF_ENABLE_INTEGRATION_TESTS=${SMF_ENABLE_INTEGRATION_TESTS}") 9 | message(STATUS "SMF_ENABLE_UNIT_TESTS=${SMF_ENABLE_UNIT_TESTS}") 10 | message(STATUS "SMF_ENABLE_BENCHMARK_TESTS=${SMF_ENABLE_BENCHMARK_TESTS}") 11 | 12 | function (smf_test) 13 | set(options INTEGRATION_TEST UNIT_TEST BENCHMARK_TEST) 14 | set(oneValueArgs BINARY_NAME SOURCE_DIRECTORY) 15 | set(multiValueArgs SOURCES LIBRARIES INCLUDES) 16 | cmake_parse_arguments(SMF_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) 17 | 18 | if(SMF_TEST_INTEGRATION_TEST AND SMF_ENABLE_INTEGRATION_TESTS) 19 | set(SMF_TEST_BINARY_NAME "smf_${SMF_TEST_BINARY_NAME}_integration_test") 20 | set(INTEGRATION_TESTS "${INTEGRATION_TESTS} ${SMF_TEST_BINARY_NAME}") 21 | add_executable( 22 | ${SMF_TEST_BINARY_NAME} "${SMF_TEST_SOURCES}") 23 | install(TARGETS ${SMF_TEST_BINARY_NAME} DESTINATION bin) 24 | target_link_libraries( 25 | ${SMF_TEST_BINARY_NAME} 26 | PUBLIC "${SMF_TEST_LIBRARIES}") 27 | add_test ( 28 | NAME ${SMF_TEST_BINARY_NAME} 29 | COMMAND 30 | ${TEST_RUNNER} 31 | --type integration 32 | --binary $ 33 | --directory ${SMF_TEST_SOURCE_DIRECTORY} 34 | ) 35 | endif() 36 | if(SMF_TEST_UNIT_TEST AND SMF_ENABLE_UNIT_TESTS) 37 | set(SMF_TEST_BINARY_NAME "smf_${SMF_TEST_BINARY_NAME}_unit_test") 38 | set(UNIT_TESTS "${UNIT_TESTS} ${SMF_TEST_BINARY_NAME}") 39 | add_executable( 40 | ${SMF_TEST_BINARY_NAME} "${SMF_TEST_SOURCES}") 41 | install(TARGETS ${SMF_TEST_BINARY_NAME} DESTINATION bin) 42 | target_link_libraries( 43 | ${SMF_TEST_BINARY_NAME} "${SMF_TEST_LIBRARIES}") 44 | add_test ( 45 | NAME ${SMF_TEST_BINARY_NAME} 46 | COMMAND 47 | ${TEST_RUNNER} 48 | --type unit 49 | --binary $ 50 | --directory ${SMF_TEST_SOURCE_DIRECTORY} 51 | ) 52 | endif() 53 | if(SMF_TEST_BENCHMARK_TEST AND SMF_ENABLE_BENCHMARK_TESTS) 54 | set(SMF_TEST_BINARY_NAME "smf_${SMF_TEST_BINARY_NAME}_benchmark_test") 55 | set(BENCHMARK_TESTS "${BENCHMARK_TESTS} ${SMF_TEST_BINARY_NAME}") 56 | add_executable( 57 | ${SMF_TEST_BINARY_NAME} "${SMF_TEST_SOURCES}") 58 | target_link_libraries( 59 | ${SMF_TEST_BINARY_NAME} "${SMF_TEST_LIBRARIES}") 60 | add_test ( 61 | NAME ${SMF_TEST_BINARY_NAME} 62 | COMMAND 63 | ${TEST_RUNNER} 64 | --type benchmark 65 | --binary $ 66 | --directory ${SMF_TEST_SOURCE_DIRECTORY} 67 | ) 68 | endif() 69 | foreach(i ${SMF_TEST_INCLUDES}) 70 | target_include_directories(${SMF_TEST_BINARY_NAME} PUBLIC ${i}) 71 | endforeach() 72 | endfunction () 73 | if(SMF_ENABLE_TESTS) 74 | add_custom_target(check 75 | COMMAND ctest --output-on-failure -N -R "^smf" 76 | DEPENDS "${UNIT_TESTS} ${INTEGRATION_TESTS} ${BENCHMARK_TESTS}") 77 | endif() 78 | -------------------------------------------------------------------------------- /demo_apps/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(cpp) 2 | -------------------------------------------------------------------------------- /demo_apps/cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | smfc_gen( 2 | CPP 3 | TARGET_NAME demo_fbs 4 | OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 5 | SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/../demo_service.fbs) 6 | 7 | add_executable(demo_server server/main.cc "${demo_fbs}") 8 | set_target_properties(demo_server PROPERTIES 9 | OUTPUT_NAME smf_demo_server) 10 | target_link_libraries(demo_server smf) 11 | target_include_directories(demo_server 12 | PUBLIC ${CMAKE_CURRENT_BINARY_DIR} 13 | PUBLIC ${PROJECT_SOURCE_DIR}/src/include 14 | ) 15 | 16 | add_executable(demo_client client/main.cc "${demo_fbs}") 17 | set_target_properties(demo_client PROPERTIES 18 | OUTPUT_NAME smf_demo_client) 19 | target_link_libraries(demo_client smf) 20 | target_include_directories(demo_client 21 | PUBLIC ${CMAKE_CURRENT_BINARY_DIR} 22 | PUBLIC ${PROJECT_SOURCE_DIR}/src/include 23 | ) 24 | 25 | 26 | 27 | install(TARGETS 28 | demo_server 29 | demo_client 30 | DESTINATION bin) 31 | -------------------------------------------------------------------------------- /demo_apps/cpp/server/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "smf/histogram_seastar_utils.h" 12 | #include "smf/log.h" 13 | #include "smf/rpc_filter.h" 14 | #include "smf/rpc_server.h" 15 | #include "smf/unique_histogram_adder.h" 16 | #include "smf/zstd_filter.h" 17 | 18 | #include "demo_service.smf.fb.h" 19 | 20 | class storage_service final : public smf_gen::demo::SmfStorage { 21 | virtual seastar::future> 22 | Get(smf::rpc_recv_typed_context &&rec) final { 23 | smf::rpc_typed_envelope data; 24 | // return the same payload 25 | if (rec) { data.data->name = rec->name()->c_str(); } 26 | data.envelope.set_status(200); 27 | return seastar::make_ready_future< 28 | smf::rpc_typed_envelope>(std::move(data)); 29 | } 30 | }; 31 | 32 | void 33 | cli_opts(boost::program_options::options_description_easy_init o) { 34 | namespace po = boost::program_options; 35 | o("ip", po::value()->default_value("127.0.0.1"), 36 | "ip to connect to"); 37 | o("port", po::value()->default_value(20776), "port for service"); 38 | o("httpport", po::value()->default_value(20777), 39 | "port for http stats service"); 40 | } 41 | 42 | int 43 | main(int args, char **argv, char **env) { 44 | std::setvbuf(stdout, nullptr, _IOLBF, 1024); 45 | seastar::distributed rpc; 46 | seastar::app_template app; 47 | cli_opts(app.add_options()); 48 | 49 | return app.run_deprecated(args, argv, [&] { 50 | seastar::engine().at_exit([&] { 51 | return rpc 52 | .map_reduce(smf::unique_histogram_adder(), 53 | &smf::rpc_server::copy_histogram) 54 | .then([](auto h) { 55 | LOG_INFO("Writing server histograms"); 56 | return smf::histogram_seastar_utils::write("server_latency.hgrm", 57 | std::move(h)); 58 | }) 59 | .then([&rpc] { return rpc.stop(); }); 60 | }); 61 | 62 | auto &cfg = app.configuration(); 63 | 64 | smf::rpc_server_args args; 65 | args.ip = cfg["ip"].as().c_str(); 66 | args.rpc_port = cfg["port"].as(); 67 | args.http_port = cfg["httpport"].as(); 68 | args.memory_avail_per_core = 69 | static_cast(0.9 * seastar::memory::stats().total_memory()); 70 | 71 | return rpc.start(args) 72 | .then([&rpc] { 73 | LOG_INFO("Registering smf_gen::demo::storage_service"); 74 | return rpc.invoke_on_all( 75 | &smf::rpc_server::register_service); 76 | }) 77 | .then([&rpc] { 78 | return rpc.invoke_on_all(&smf::rpc_server::register_incoming_filter< 79 | smf::zstd_decompression_filter>); 80 | }) 81 | .then([&rpc] { 82 | LOG_INFO("Invoking rpc start on all cores"); 83 | return rpc.invoke_on_all(&smf::rpc_server::start); 84 | }); 85 | }); 86 | } 87 | -------------------------------------------------------------------------------- /demo_apps/demo_service.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | namespace smf_gen.demo; 5 | 6 | table Request { 7 | name: string; 8 | } 9 | 10 | table Response { 11 | name: string; 12 | } 13 | 14 | 15 | rpc_service SmfStorage { 16 | Get(Request):Response; 17 | 18 | // -- test for the compiler generator 19 | 20 | Put(Request):Response; 21 | Fire(Request):Response; 22 | Forget(Request):Response; 23 | SomethingElse(Request):Response; 24 | } 25 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore docs files 2 | _gh_pages 3 | _site 4 | .ruby-version 5 | 6 | # Numerous always-ignore extensions 7 | *.diff 8 | *.err 9 | *.orig 10 | *.log 11 | *.rej 12 | *.swo 13 | *.swp 14 | *.zip 15 | *.vi 16 | *~ 17 | 18 | # OS or Editor folders 19 | .DS_Store 20 | ._* 21 | Thumbs.db 22 | .cache 23 | .project 24 | .settings 25 | .tmproj 26 | *.esproj 27 | nbproject 28 | *.sublime-project 29 | *.sublime-workspace 30 | .idea 31 | 32 | # Komodo 33 | *.komodoproject 34 | .komodotools 35 | 36 | # grunt-html-validation 37 | validation-status.json 38 | validation-report.json 39 | 40 | # Folders to ignore 41 | node_modules 42 | .jekyll-metadata 43 | -------------------------------------------------------------------------------- /docs/404.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "404: Page not found" 4 | permalink: 404.html 5 | --- 6 | 7 |
8 |

404: Page not found

9 |

Sorry, we've misplaced that URL or it's pointing to something that doesn't exist. Head back home to try finding it again.

10 |
11 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gem 'jemoji' 3 | gem 'jekyll-gist' 4 | gem 'jekyll-paginate' 5 | gem 'kramdown' 6 | gem 'rouge' 7 | -------------------------------------------------------------------------------- /docs/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activesupport (5.2.2) 5 | concurrent-ruby (~> 1.0, >= 1.0.2) 6 | i18n (>= 0.7, < 2) 7 | minitest (~> 5.1) 8 | tzinfo (~> 1.1) 9 | addressable (2.5.2) 10 | public_suffix (>= 2.0.2, < 4.0) 11 | colorator (1.1.0) 12 | concurrent-ruby (1.1.4) 13 | em-websocket (0.5.1) 14 | eventmachine (>= 0.12.9) 15 | http_parser.rb (~> 0.6.0) 16 | eventmachine (1.2.7) 17 | faraday (0.15.4) 18 | multipart-post (>= 1.2, < 3) 19 | ffi (1.10.0) 20 | forwardable-extended (2.6.0) 21 | gemoji (3.0.0) 22 | html-pipeline (2.10.0) 23 | activesupport (>= 2) 24 | nokogiri (>= 1.4) 25 | http_parser.rb (0.6.0) 26 | i18n (0.9.5) 27 | concurrent-ruby (~> 1.0) 28 | jekyll (3.8.5) 29 | addressable (~> 2.4) 30 | colorator (~> 1.0) 31 | em-websocket (~> 0.5) 32 | i18n (~> 0.7) 33 | jekyll-sass-converter (~> 1.0) 34 | jekyll-watch (~> 2.0) 35 | kramdown (~> 1.14) 36 | liquid (~> 4.0) 37 | mercenary (~> 0.3.3) 38 | pathutil (~> 0.9) 39 | rouge (>= 1.7, < 4) 40 | safe_yaml (~> 1.0) 41 | jekyll-gist (1.5.0) 42 | octokit (~> 4.2) 43 | jekyll-paginate (1.1.0) 44 | jekyll-sass-converter (1.5.2) 45 | sass (~> 3.4) 46 | jekyll-watch (2.1.2) 47 | listen (~> 3.0) 48 | jemoji (0.10.1) 49 | gemoji (~> 3.0) 50 | html-pipeline (~> 2.2) 51 | jekyll (~> 3.0) 52 | kramdown (1.17.0) 53 | liquid (4.0.1) 54 | listen (3.1.5) 55 | rb-fsevent (~> 0.9, >= 0.9.4) 56 | rb-inotify (~> 0.9, >= 0.9.7) 57 | ruby_dep (~> 1.2) 58 | mercenary (0.3.6) 59 | mini_portile2 (2.4.0) 60 | minitest (5.11.3) 61 | multipart-post (2.0.0) 62 | nokogiri (1.10.5) 63 | mini_portile2 (~> 2.4.0) 64 | octokit (4.13.0) 65 | sawyer (~> 0.8.0, >= 0.5.3) 66 | pathutil (0.16.2) 67 | forwardable-extended (~> 2.6) 68 | public_suffix (3.0.3) 69 | rb-fsevent (0.10.3) 70 | rb-inotify (0.10.0) 71 | ffi (~> 1.0) 72 | rouge (3.3.0) 73 | ruby_dep (1.5.0) 74 | safe_yaml (1.0.4) 75 | sass (3.7.3) 76 | sass-listen (~> 4.0.0) 77 | sass-listen (4.0.0) 78 | rb-fsevent (~> 0.9, >= 0.9.4) 79 | rb-inotify (~> 0.9, >= 0.9.7) 80 | sawyer (0.8.1) 81 | addressable (>= 2.3.5, < 2.6) 82 | faraday (~> 0.8, < 1.0) 83 | thread_safe (0.3.6) 84 | tzinfo (1.2.5) 85 | thread_safe (~> 0.1) 86 | 87 | PLATFORMS 88 | ruby 89 | 90 | DEPENDENCIES 91 | jekyll-gist 92 | jekyll-paginate 93 | jemoji 94 | kramdown 95 | rouge 96 | 97 | BUNDLED WITH 98 | 1.16.1 99 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | markdown: kramdown 3 | highlighter: rouge 4 | 5 | plugins: ['jekyll-paginate', 'jekyll-gist', 'jemoji'] 6 | 7 | # Permalinks 8 | permalink: pretty 9 | 10 | # Setup 11 | title: 'smf' 12 | tagline: 'smf - **/smɝf/** - the fastest RPC in the West' 13 | description: '' 14 | url: 'https://smfrpc.github.io' 15 | baseurl: /smf/ 16 | 17 | author: 18 | name: 'emaxerrno' 19 | url: 'https://twitter.com/emaxerrno' 20 | 21 | paginate: 5 22 | 23 | # Custom vars 24 | version: 2.1.0 25 | 26 | github: 27 | repo: 'https://github.com/smfrpc/smf' 28 | -------------------------------------------------------------------------------- /docs/_includes/head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | {% if page.title == "Home" %} 10 | {{ site.title }} · {{ site.tagline }} 11 | {% else %} 12 | {{ page.title }} · {{ site.title }} 13 | {% endif %} 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/_includes/sidebar.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

5 | smf 6 |

7 |

fastest RPC in the west

8 |
9 | 10 | 34 | 35 |

36 | © {{ site.time | date: '%Y' }}. All rights reserved.

37 |
38 |
39 | -------------------------------------------------------------------------------- /docs/_layouts/default.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head.html %} 5 | 6 | 7 | 8 | {% include sidebar.html %} 9 | 10 |
11 | {{ content }} 12 |
13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/_layouts/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 |

{{ page.title }}

7 | {{ content }} 8 |
9 | -------------------------------------------------------------------------------- /docs/_layouts/post.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 |

{{ page.title }}

7 | {{ page.date | date_to_string }} 8 | {{ content }} 9 |
10 | 11 |
12 |

Related Posts

13 | 25 |
26 | -------------------------------------------------------------------------------- /docs/_posts/2017-10-12-introducing-smf.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Introducing smf 4 | --- 5 | 6 | 7 | # **smf** is pronounced **/smɝf/** 8 | 9 | --- 10 | 11 | | Important Links 12 | | ---------------- 13 | | [Mailing List](https://groups.google.com/forum/#!forum/smf-dev) 14 | | [Documentation](https://smfrpc.github.io/smf/) 15 | 16 | 17 | ## [**smf** RPC]({{site.baseurl}}rpc) 18 | 19 | 20 | **smf** is a new RPC system and code generation like gRPC, Cap n Proto, 21 | Apache Thrift, etc, but designed for **microsecond tail latency**. 22 | 23 | Current benchmarks in microseconds 24 | 25 | | 60 byte payload | latency | 26 | | ---------------- | --------- | 27 | | p50 | 7us | 28 | | p90 | 8us | 29 | | p99 | 8us | 30 | | p9999 | 15us | 31 | | p100 | 26us | 32 | 33 | 34 | Highlights: 35 | --- 36 | 37 | * Code generation compiler for RPC 38 | * Load testing framework for RPC subsystem 39 | * Kernel-bypass RPC via DPDK 40 | * 0-copy Serialization framework based on Google's Flatbuffers project 41 | * Arbitrary filter chaining on incoming and outgoing channels (like twitter's Finagle) 42 | * Small 16byte overhead 43 | * Binary with pluggable compressors 44 | 45 | ## We need your help! 46 | 47 | We have a lot of issues marked as [good first issue](https://github.com/smfrpc/smf/labels/good%20first%20issue) 48 | 49 | Look at the [contributing](https://github.com/smfrpc/smf/blob/master/CONTRIBUTING.md) 50 | guideline for more details. 51 | 52 | Take a look at the issue list or send an email to the 53 | [smf-dev mailing List](https://groups.google.com/forum/#!forum/smf-dev) 54 | to get started. 55 | 56 | ## Getting started 57 | 58 | Please see our quick 59 | [getting started!](https://smfrpc.github.io/smf/getting_started/) 60 | -------------------------------------------------------------------------------- /docs/_posts/2018-10-29-demo-projects.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Getting started projects 4 | --- 5 | 6 | It's now very easy to get started with **smf** in C++. 7 | Simply clone this project and `./build.sh` to have a 8 | fully working project 9 | 10 | # C++ 11 | 12 | A fully working project with build system. 13 | 14 | https://github.com/smfrpc/smf-getting-started-cpp 15 | 16 | 17 | 18 | ```sh 19 | 20 | # 1 clone 21 | git clone --recursive https://github.com/smfrpc/smf-getting-started-cpp 22 | 23 | # 2 go to project 24 | cd smf-getting-started-cpp 25 | 26 | # 3 build 27 | ./build.sh 28 | 29 | ``` 30 | -------------------------------------------------------------------------------- /docs/_posts/2019-02-10-xfsprogs.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: reproducible deps 4 | --- 5 | 6 | 7 | # Dependencies: 8 | 9 | `smf` comes with the ability to fetch and build transitive dependencies on it's own at known 10 | revisions for reproducible builds. 11 | 12 | To enable reproducible builds, simply invoke cmake like so: 13 | 14 | `mkdir build && cmake -GNinja -DSMF_MANAGE_DEPS=ON -DCMAKE_BUILD_TYPE=Release .. ` 15 | 16 | By default automatic dep management is off (i.e.: `-DSMF_MANAGE_DEPS=OFF`) so you can 17 | use smf as part of a larger cmake project 18 | 19 | # Example app 20 | 21 | ``` 22 | 23 | ➜ release git:(master) ✗ ldd bin/smf_demo_client 24 | 25 | # system deps required 26 | 27 | linux-vdso.so.1 (0x00007ffdd8364000) 28 | libpthread.so.0 => /lib64/libpthread.so.0 (0x00007fe397aec000) 29 | libdl.so.2 => /lib64/libdl.so.2 (0x00007fe397749000) 30 | libatomic.so.1 => /lib64/libatomic.so.1 (0x00007fe39754b000) 31 | librt.so.1 => /lib64/librt.so.1 (0x00007fe3972e3000) 32 | libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007fe3970fe000) 33 | libm.so.6 => /lib64/libm.so.6 (0x00007fe396f7a000) 34 | libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007fe396f5f000) 35 | libc.so.6 => /lib64/libc.so.6 (0x00007fe396d99000) 36 | /lib64/ld-linux-x86-64.so.2 (0x00007fe397b35000) 37 | 38 | # optionally manage deps 39 | 40 | libgnutls.so.30 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libgnutls.so.30 (0x00007fe397555000) 41 | libsctp.so.1 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libsctp.so.1 (0x00007fe397546000) 42 | libprotobuf.so.13 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libprotobuf.so.13 (0x00007fe3972ed000) 43 | libhwloc.so.5 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libhwloc.so.5 (0x00007fe3972a5000) 44 | libnuma.so.1 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libnuma.so.1 (0x00007fe397298000) 45 | libcares.so.2 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libcares.so.2 (0x00007fe397ad8000) 46 | libcryptopp.so.5.6 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib64/libcryptopp.so.5.6 (0x00007fe397766000) 47 | liblz4.so.1 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib64/liblz4.so.1 (0x00007fe39774f000) 48 | libz.so.1 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libz.so.1 (0x00007fe396d7c000) 49 | libnettle.so.6 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libnettle.so.6 (0x00007fe396d36000) 50 | libhogweed.so.4 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libhogweed.so.4 (0x00007fe396cf6000) 51 | libgmp.so.10 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libgmp.so.10 (0x00007fe396c80000) 52 | libpciaccess.so.0 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libpciaccess.so.0 (0x00007fe396c76000) 53 | libxml2.so.2 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/libxml2.so.2 (0x00007fe396b11000) 54 | liblzma.so.5 => /home/agallego/workspace/smf/build/release/smf-deps-install/lib/liblzma.so.5 (0x00007fe396ae9000) 55 | 56 | ``` 57 | 58 | # Next steps / help wanted 59 | 60 | I'd love to make most of the dependencies statically compiled into the final binary. 61 | 62 | The only issue I ran into was w/ the TLS libraries. 63 | 64 | `protobuf, cares, lz4, z, lzma, xml2` are easy to change. They already produce the `.a` files for linking. 65 | 66 | The only remaining problematic libs are `hwloc, numa, gnutls, nettle, hogweed, gmp` 67 | 68 | -------------------------------------------------------------------------------- /docs/atom.xml: -------------------------------------------------------------------------------- 1 | --- 2 | layout: null 3 | --- 4 | 5 | 6 | 7 | 8 | {{ site.title }} 9 | 10 | 11 | {{ site.time | date_to_xmlschema }} 12 | {{ site.url }} 13 | 14 | {{ site.author.name }} 15 | {{ site.author.email }} 16 | 17 | 18 | {% for post in site.posts %} 19 | 20 | {{ post.title }} 21 | 22 | {{ post.date | date_to_xmlschema }} 23 | {{ site.url }}{{ post.id }} 24 | {{ post.content | xml_escape }} 25 | 26 | {% endfor %} 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/contact.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: contact us 4 | --- 5 | 6 | # preferred 7 | 8 | [mailing list](https://groups.google.com/forum/#!forum/smf-dev) 9 | 10 | 11 | ---- 12 | 13 | for general discussions, questions, comments, concerns, etc, send us a 14 | [mailing list](https://groups.google.com/forum/#!forum/smf-dev) 15 | email 16 | 17 | come say hi on twitter [@emaxerrno](https://twitter.com/emaxerrno) 18 | 19 | if you are using the software and find a bug, 20 | please file a [github issue](https://github.com/smfrpc/smf/issues) 21 | in addition to contacting the mailing list 22 | -------------------------------------------------------------------------------- /docs/doxy.in: -------------------------------------------------------------------------------- 1 | PROJECT_NAME = "SMF" 2 | # filled via cmake 3 | INPUT = @PROJECT_SOURCE_DIR@/src 4 | RECURSIVE = YES 5 | EXCLUDE_PATTERNS = @PROJECT_SOURCE_DIR@/src/test/* 6 | EXCLUDE_PATTERNS += @PROJECT_SOURCE_DIR@/src/alf_tests/* 7 | EXCLUDE_PATTERNS += @PROJECT_SOURCE_DIR@/src/integration_tests/* 8 | # out to current build directory in cmake 9 | OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@ 10 | QUIET = YES 11 | # We want to have warnings for functions/methods which are documented, 12 | # but have no documentation for their parameters or the return value. 13 | WARN_NO_PARAMDOC = YES 14 | GENERATE_LATEX = NO 15 | # generate the xml commands for markdown 16 | GENERATE_XML = YES 17 | -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: getting started 4 | --- 5 | 6 |

7 | Nice! thanks for trying and don't forget to 8 | send the mailing list an email if you get stuck! 9 |

10 | 11 | 12 | Currently we have tested the system to work on Fedora, Debian, Ubuntu. 13 | 14 | We require gcc7 or greater. 15 | 16 | It is a fully compliant CMake project. If you know CMake you can skip this part. 17 | 18 | 19 | ```bash 20 | git clone https://github.com/smfrpc/smf.git 21 | 22 | # Install smf deps 23 | # 24 | 25 | # this is a work in progress, and very soon will be 26 | # removed. Most dependencies are now built by cmake 27 | # and built from source to provide consistent link & build flags 28 | ./install-deps.sh 29 | 30 | 31 | # We have a wrapper script to build the project 32 | # try build.sh -h for more advanced usage! 33 | # -r == release 34 | # 35 | ./tools/build.sh -r 36 | ``` 37 | 38 | Please drop us a line and tell us what you are using 39 | **smf** for. 40 | 41 | ## Sample programs: 42 | 43 | Server: 44 | 45 | ``` 46 | # Run on one core (-c 1)! change --ip and --port as needed 47 | # 48 | ./build/release/demo_apps/cpp/demo_server -c 1 49 | ``` 50 | 51 | Client: 52 | ``` 53 | # Run on one core (-c 1)! change --ip and --port as needed 54 | # 55 | ./build/release/demo_apps/cpp/demo_client -c 1 56 | ``` 57 | 58 | 59 | ## Docker 60 | 61 | Alternatively, you can use Dockerfile, 62 | enter `cd tools/local_development`, and then: 63 | 64 | ```bash 65 | docker build -t smf . 66 | ``` 67 | 68 | To run a shell inside the docker: 69 | 70 | ```bash 71 | docker run -it smf /usr/bin/bash 72 | ``` 73 | 74 | Note: This re-clones the repo inside the Docker image 75 | from the `master` tag. It does not use the current local copy. 76 | The base image is `fedora27` 77 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: home 4 | --- 5 | 6 |
7 | {% for post in paginator.posts %} 8 |
9 |

10 | 11 | {{ post.title }} 12 | 13 |

14 | 15 | {{ post.date | date_to_string }} 16 | 17 | {{ post.content }} 18 |
19 | {% endfor %} 20 |
21 | 22 | 38 | 39 | 49 | -------------------------------------------------------------------------------- /docs/presentations.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: presentations 4 | --- 5 | 6 | - [ScyllaDB Summit 2017](https://scyllasummit2017.sched.com/event/CEwd) 7 | 8 | -------------------------------------------------------------------------------- /docs/public/annotated_rpc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/annotated_rpc.png -------------------------------------------------------------------------------- /docs/public/apple-touch-icon-144-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/apple-touch-icon-144-precomposed.png -------------------------------------------------------------------------------- /docs/public/clock_pro_cache.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/clock_pro_cache.png -------------------------------------------------------------------------------- /docs/public/css/syntax.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffc; } 2 | .highlight .c { color: #999; } /* Comment */ 3 | .highlight .err { color: #a00; background-color: #faa } /* Error */ 4 | .highlight .k { color: #069; } /* Keyword */ 5 | .highlight .o { color: #555 } /* Operator */ 6 | .highlight .cm { color: #09f; font-style: italic } /* Comment.Multiline */ 7 | .highlight .cp { color: #099 } /* Comment.Preproc */ 8 | .highlight .c1 { color: #999; } /* Comment.Single */ 9 | .highlight .cs { color: #999; } /* Comment.Special */ 10 | .highlight .gd { background-color: #fcc; border: 1px solid #c00 } /* Generic.Deleted */ 11 | .highlight .ge { font-style: italic } /* Generic.Emph */ 12 | .highlight .gr { color: #f00 } /* Generic.Error */ 13 | .highlight .gh { color: #030; } /* Generic.Heading */ 14 | .highlight .gi { background-color: #cfc; border: 1px solid #0c0 } /* Generic.Inserted */ 15 | .highlight .go { color: #aaa } /* Generic.Output */ 16 | .highlight .gp { color: #009; } /* Generic.Prompt */ 17 | .highlight .gs { } /* Generic.Strong */ 18 | .highlight .gu { color: #030; } /* Generic.Subheading */ 19 | .highlight .gt { color: #9c6 } /* Generic.Traceback */ 20 | .highlight .kc { color: #069; } /* Keyword.Constant */ 21 | .highlight .kd { color: #069; } /* Keyword.Declaration */ 22 | .highlight .kn { color: #069; } /* Keyword.Namespace */ 23 | .highlight .kp { color: #069 } /* Keyword.Pseudo */ 24 | .highlight .kr { color: #069; } /* Keyword.Reserved */ 25 | .highlight .kt { color: #078; } /* Keyword.Type */ 26 | .highlight .m { color: #f60 } /* Literal.Number */ 27 | .highlight .s { color: #d44950 } /* Literal.String */ 28 | .highlight .na { color: #4f9fcf } /* Name.Attribute */ 29 | .highlight .nb { color: #366 } /* Name.Builtin */ 30 | .highlight .nc { color: #0a8; } /* Name.Class */ 31 | .highlight .no { color: #360 } /* Name.Constant */ 32 | .highlight .nd { color: #99f } /* Name.Decorator */ 33 | .highlight .ni { color: #999; } /* Name.Entity */ 34 | .highlight .ne { color: #c00; } /* Name.Exception */ 35 | .highlight .nf { color: #c0f } /* Name.Function */ 36 | .highlight .nl { color: #99f } /* Name.Label */ 37 | .highlight .nn { color: #0cf; } /* Name.Namespace */ 38 | .highlight .nt { color: #2f6f9f; } /* Name.Tag */ 39 | .highlight .nv { color: #033 } /* Name.Variable */ 40 | .highlight .ow { color: #000; } /* Operator.Word */ 41 | .highlight .w { color: #bbb } /* Text.Whitespace */ 42 | .highlight .mf { color: #f60 } /* Literal.Number.Float */ 43 | .highlight .mh { color: #f60 } /* Literal.Number.Hex */ 44 | .highlight .mi { color: #f60 } /* Literal.Number.Integer */ 45 | .highlight .mo { color: #f60 } /* Literal.Number.Oct */ 46 | .highlight .sb { color: #c30 } /* Literal.String.Backtick */ 47 | .highlight .sc { color: #c30 } /* Literal.String.Char */ 48 | .highlight .sd { color: #c30; font-style: italic } /* Literal.String.Doc */ 49 | .highlight .s2 { color: #c30 } /* Literal.String.Double */ 50 | .highlight .se { color: #c30; } /* Literal.String.Escape */ 51 | .highlight .sh { color: #c30 } /* Literal.String.Heredoc */ 52 | .highlight .si { color: #a00 } /* Literal.String.Interpol */ 53 | .highlight .sx { color: #c30 } /* Literal.String.Other */ 54 | .highlight .sr { color: #3aa } /* Literal.String.Regex */ 55 | .highlight .s1 { color: #c30 } /* Literal.String.Single */ 56 | .highlight .ss { color: #fc3 } /* Literal.String.Symbol */ 57 | .highlight .bp { color: #366 } /* Name.Builtin.Pseudo */ 58 | .highlight .vc { color: #033 } /* Name.Variable.Class */ 59 | .highlight .vg { color: #033 } /* Name.Variable.Global */ 60 | .highlight .vi { color: #033 } /* Name.Variable.Instance */ 61 | .highlight .il { color: #f60 } /* Literal.Number.Integer.Long */ 62 | 63 | .css .o, 64 | .css .o + .nt, 65 | .css .nt + .nt { color: #999; } 66 | -------------------------------------------------------------------------------- /docs/public/kafka_vs_smf_latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/kafka_vs_smf_latency.png -------------------------------------------------------------------------------- /docs/public/kafka_vs_smf_qps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/kafka_vs_smf_qps.png -------------------------------------------------------------------------------- /docs/public/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/logo.png -------------------------------------------------------------------------------- /docs/public/services.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/services.png -------------------------------------------------------------------------------- /docs/public/smf_wal_filesystem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/smf_wal_filesystem.png -------------------------------------------------------------------------------- /docs/public/smf_write_ahead_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/smf_write_ahead_log.png -------------------------------------------------------------------------------- /docs/public/wal_kafka_latency_vs_smf_1producer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/wal_kafka_latency_vs_smf_1producer.png -------------------------------------------------------------------------------- /docs/public/wal_kafka_latency_vs_smf_3producers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emaxerrno/smf/4b862275a91af5b0f6ab6bf673ceb0e082c001a7/docs/public/wal_kafka_latency_vs_smf_3producers.png -------------------------------------------------------------------------------- /docs/server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 SMF Authors 4 | # 5 | 6 | set -ex 7 | bundle update 8 | exec bundle exec jekyll serve 9 | -------------------------------------------------------------------------------- /dpdk_config: -------------------------------------------------------------------------------- 1 | CONFIG_RTE_LIBRTE_PMD_BOND=n 2 | CONFIG_RTE_MBUF_SCATTER_GATHER=n 3 | CONFIG_RTE_LIBRTE_IP_FRAG=n 4 | CONFIG_RTE_APP_TEST=n 5 | CONFIG_RTE_TEST_PMD=n 6 | CONFIG_RTE_MBUF_REFCNT_ATOMIC=n 7 | CONFIG_RTE_MAX_MEMSEG=8192 8 | CONFIG_RTE_EAL_IGB_UIO=n 9 | CONFIG_RTE_LIBRTE_KNI=n 10 | CONFIG_RTE_KNI_KMOD=n 11 | CONFIG_RTE_LIBRTE_JOBSTATS=n 12 | CONFIG_RTE_LIBRTE_LPM=n 13 | CONFIG_RTE_LIBRTE_ACL=n 14 | CONFIG_RTE_LIBRTE_POWER=n 15 | CONFIG_RTE_LIBRTE_IP_FRAG=n 16 | CONFIG_RTE_LIBRTE_METER=n 17 | CONFIG_RTE_LIBRTE_SCHED=n 18 | CONFIG_RTE_LIBRTE_DISTRIBUTOR=n 19 | CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER=n 20 | CONFIG_RTE_LIBRTE_REORDER=n 21 | CONFIG_RTE_LIBRTE_PORT=n 22 | CONFIG_RTE_LIBRTE_TABLE=n 23 | CONFIG_RTE_LIBRTE_PIPELINE=n 24 | -------------------------------------------------------------------------------- /install-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 SMF Authors 4 | # 5 | 6 | set -e 7 | if [ -n "${CI}" ]; then 8 | set -x 9 | fi 10 | # shellcheck disable=SC1091 11 | source /etc/os-release 12 | 13 | function apt_gcc { 14 | local gccver=$1 15 | local cc="gcc-${gccver}" 16 | local cxx="g++-${gccver}" 17 | apt-get install -y "${cc}" "${cxx}" 18 | if [ -n "${CI}" ]; then 19 | update-alternatives \ 20 | --install /usr/bin/gcc gcc "/usr/bin/${cc}" 800 \ 21 | --slave /usr/bin/g++ g++ "/usr/bin/${cxx}" 22 | fi 23 | 24 | } 25 | 26 | function debs() { 27 | apt-get update -y 28 | 29 | local extra="" 30 | if [ -n "${USE_CLANG}" ]; then 31 | extra=clang 32 | else 33 | # ensure gcc is installed so we can test its version 34 | apt-get install -y build-essential 35 | if ! command -v add-apt-repository; then 36 | apt-get -y install software-properties-common 37 | fi 38 | apt-get update -y 39 | if [ "$(apt-cache search '^gcc-9$' | awk '{print $1}')" == "gcc-9" ]; then 40 | apt_gcc 9 41 | else 42 | gcc_ver=$(gcc -dumpfullversion -dumpversion) 43 | if dpkg --compare-versions "${gcc_ver}" lt 8.0; then 44 | # as of may 29, 2019, ubuntu:disco did not have this ppa enabled 45 | add-apt-repository -y ppa:ubuntu-toolchain-r/test 46 | apt_gcc 8 47 | fi 48 | fi 49 | fi 50 | if [ "${UBUNTU_CODENAME}" == "xenial" ] && [ -n "${CI}" ]; then 51 | cmake_version="3.14.0-rc2" 52 | cmake_full_name="cmake-${cmake_version}-Linux-x86_64.sh" 53 | apt-get install -y wget 54 | wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_full_name} -O /tmp/${cmake_full_name} 55 | chmod +x /tmp/${cmake_full_name} 56 | /tmp/${cmake_full_name} --skip-license --prefix=/usr 57 | else 58 | apt-get install -y cmake 59 | fi 60 | apt-get install -y \ 61 | build-essential \ 62 | libtool \ 63 | m4 \ 64 | ninja-build \ 65 | automake \ 66 | pkg-config \ 67 | xfslibs-dev \ 68 | systemtap-sdt-dev \ 69 | ragel ${extra} 70 | } 71 | 72 | function rpms() { 73 | yumdnf="yum" 74 | if command -v dnf > /dev/null; then 75 | yumdnf="dnf" 76 | fi 77 | 78 | case ${ID} in 79 | centos|rhel) 80 | MAJOR_VERSION="$(echo "$VERSION_ID" | cut -d. -f1)" 81 | $SUDO yum-config-manager --add-repo https://dl.fedoraproject.org/pub/epel/"$MAJOR_VERSION"/x86_64/ 82 | $SUDO yum install --nogpgcheck -y epel-release 83 | $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-"$MAJOR_VERSION" 84 | $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org* 85 | if test "$ID" = centos -a "$MAJOR_VERSION" = 7 ; then 86 | yum install -y centos-release-scl 87 | yum install -y devtoolset-8 88 | dts_ver=8 89 | fi 90 | ;; 91 | esac 92 | 93 | if [ -n "${USE_CLANG}" ]; then 94 | extra=clang 95 | fi 96 | 97 | cmake="cmake" 98 | case ${ID} in 99 | centos|rhel) 100 | MAJOR_VERSION="$(echo "$VERSION_ID" | cut -d. -f1)" 101 | if test "$MAJOR_VERSION" = 7 ; then 102 | cmake="cmake3" 103 | fi 104 | esac 105 | 106 | ${yumdnf} install -y \ 107 | ${cmake} \ 108 | gcc-c++ \ 109 | ninja-build \ 110 | m4 \ 111 | libtool \ 112 | make \ 113 | ragel \ 114 | xfsprogs-devel \ 115 | systemtap-sdt-devel \ 116 | libasan \ 117 | libubsan \ 118 | libatomic \ 119 | doxygen ${extra} 120 | 121 | if [ -n "$dts_ver" ]; then 122 | if test -t 1; then 123 | # interactive shell 124 | cat < 46 | $ 47 | $) 48 | 49 | target_compile_definitions(smf PUBLIC -DXXH_PRIVATE_API) 50 | 51 | target_link_libraries(smf 52 | PUBLIC 53 | Seastar::seastar 54 | Flatbuffers::flatbuffers 55 | Zstd::zstd 56 | Hdrhistogram::hdr_histogram 57 | xxHash::xxhash) 58 | 59 | if(CMAKE_BUILD_TYPE MATCHES Debug) 60 | target_compile_options(smf PRIVATE 61 | -fdiagnostics-color=auto 62 | -Wall 63 | -Werror 64 | -Wextra 65 | -Wformat 66 | -Wmissing-braces 67 | -Wparentheses 68 | -Wpointer-arith 69 | -Wformat-security 70 | -Wunused 71 | -Wno-unused-parameter 72 | -Wcast-align 73 | -Wno-missing-field-initializers 74 | -Wdelete-non-virtual-dtor 75 | -Wno-ignored-qualifiers) 76 | endif() 77 | 78 | install(TARGETS smf 79 | EXPORT smf-export 80 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 81 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) 82 | 83 | if(SMF_ENABLE_BENCHMARK_TESTS) 84 | add_subdirectory(benchmarks) 85 | endif() 86 | 87 | if(SMF_ENABLE_INTEGRATION_TESTS) 88 | add_subdirectory(integration_tests) 89 | endif() 90 | 91 | if(SMF_ENABLE_UNIT_TESTS) 92 | add_subdirectory(tests) 93 | endif() 94 | -------------------------------------------------------------------------------- /src/afl_tests/rpc/rpc.dict: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | # 3 | # 4 | # AFL dictionary for smf RPC 5 | # -------------------------- 6 | # 7 | # Several basic syntax elements and attributes, modeled on rpc.fbs. 8 | # 9 | # 10 | 11 | header_size="1" 12 | header_flags="1" 13 | header_checksum="1" 14 | payload="\x89\x0d\x0a\x1a\x0a" 15 | -------------------------------------------------------------------------------- /src/benchmarks/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(benchmark REQUIRED) 2 | add_subdirectory(fbs_alloc) 3 | set(BENCH_ROOT ${PROJECT_SOURCE_DIR}/src/benchmarks) 4 | smf_test( 5 | BENCHMARK_TEST 6 | BINARY_NAME checksum 7 | SOURCES ${BENCH_ROOT}/checksum_bench/main.cc 8 | SOURCE_DIRECTORY ${BENCH_ROOT}/checksum_bench 9 | LIBRARIES benchmark::benchmark smf 10 | ) 11 | -------------------------------------------------------------------------------- /src/benchmarks/checksum_bench/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Alexander Gallego 2 | // 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | static constexpr uint32_t kPayloadSize = 1 << 29; 12 | static char kPayload[kPayloadSize]{}; 13 | 14 | static void 15 | BM_hash32(benchmark::State &state) { 16 | for (auto _ : state) { 17 | state.PauseTiming(); 18 | std::memset(kPayload, 'x', state.range(0)); 19 | state.ResumeTiming(); 20 | benchmark::DoNotOptimize(XXH32(kPayload, state.range(0), 0)); 21 | } 22 | } 23 | BENCHMARK(BM_hash32) 24 | ->Args({1 << 16, 1 << 16}) 25 | ->Args({1 << 20, 1 << 20}) 26 | ->Args({1 << 29, 1 << 29}); 27 | 28 | static void 29 | BM_hash64(benchmark::State &state) { 30 | for (auto _ : state) { 31 | state.PauseTiming(); 32 | std::memset(kPayload, 'x', state.range(0)); 33 | state.ResumeTiming(); 34 | benchmark::DoNotOptimize(std::numeric_limits::max() & 35 | XXH64(kPayload, state.range(0), 0)); 36 | } 37 | } 38 | BENCHMARK(BM_hash64) 39 | ->Args({1 << 16, 1 << 16}) 40 | ->Args({1 << 20, 1 << 20}) 41 | ->Args({1 << 29, 1 << 29}); 42 | 43 | BENCHMARK_MAIN(); 44 | -------------------------------------------------------------------------------- /src/benchmarks/fbs_alloc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(benchmark REQUIRED) 2 | include(smfc_generator) 3 | smfc_gen( 4 | CPP 5 | TARGET_NAME kv_fbs 6 | OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 7 | SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/kv.fbs) 8 | smf_test( 9 | BENCHMARK_TEST 10 | BINARY_NAME fbsalloc 11 | SOURCES 12 | ${CMAKE_CURRENT_SOURCE_DIR}/main.cc 13 | ${kv_fbs} 14 | SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 15 | TIMEOUT 420 # on my laptop test takes 320 secs 16 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR} 17 | LIBRARIES benchmark::benchmark smf 18 | ) 19 | -------------------------------------------------------------------------------- /src/benchmarks/fbs_alloc/kv.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | table kvpair { 5 | key: string; 6 | value: string; 7 | } -------------------------------------------------------------------------------- /src/core/histogram_seastar_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | #include "smf/histogram_seastar_utils.h" 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | // smf 10 | #include "smf/histogram.h" 11 | #include "smf/log.h" 12 | 13 | namespace smf { 14 | 15 | seastar::future> 16 | histogram_seastar_utils::print_histogram(histogram *h) { 17 | // HdrHistogram/GoogleChartsExample/example1.txt is 5K 18 | char *buf; 19 | std::size_t len; 20 | FILE *fp = open_memstream(&buf, &len); 21 | LOG_THROW_IF(fp == nullptr, "Failed to allocate filestream"); 22 | h->print(fp); 23 | // MUST fflush in order to have len update 24 | fflush(fp); 25 | fclose(fp); 26 | 27 | auto ret = 28 | seastar::temporary_buffer(buf, len, seastar::make_free_deleter(buf)); 29 | return seastar::make_ready_future(std::move(ret)); 30 | } 31 | seastar::future<> 32 | histogram_seastar_utils::write_histogram(seastar::sstring filename, 33 | histogram *h) { 34 | return open_file_dma(filename, seastar::open_flags::rw | 35 | seastar::open_flags::create | 36 | seastar::open_flags::truncate) 37 | .then([h = std::move(h)](seastar::file file) mutable { 38 | auto f = seastar::make_lw_shared>( 39 | seastar::make_file_output_stream(std::move(file))); 40 | return histogram_seastar_utils::print_histogram(h).then( 41 | [f](seastar::temporary_buffer buf) { 42 | return f->write(buf.get(), buf.size()).then([f]() mutable { 43 | return f->flush().then( 44 | [f]() mutable { return f->close().finally([f] {}); }); 45 | }); 46 | }); 47 | }); 48 | } 49 | 50 | } // namespace smf 51 | -------------------------------------------------------------------------------- /src/core/lz4_filter.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include "smf/lz4_filter.h" 4 | 5 | #include 6 | 7 | #include "smf/compression.h" 8 | #include "smf/rpc_header_utils.h" 9 | #include "smf/rpc_recv_context.h" 10 | 11 | namespace smf { 12 | 13 | static thread_local auto compressor = 14 | codec::make_unique(codec_type::lz4, compression_level::fastest); 15 | 16 | seastar::future 17 | lz4_compression_filter::operator()(rpc_envelope &&e) { 18 | if (e.letter.header.compression() != 19 | rpc::compression_flags::compression_flags_none) { 20 | return seastar::make_ready_future(std::move(e)); 21 | } 22 | if (e.letter.body.size() <= min_compression_size) { 23 | return seastar::make_ready_future(std::move(e)); 24 | } 25 | 26 | auto buf = compressor->compress(e.letter.body); 27 | e.letter.body = std::move(buf); 28 | e.letter.header.mutate_compression( 29 | rpc::compression_flags::compression_flags_lz4); 30 | checksum_rpc(e.letter.header, e.letter.body.get(), e.letter.body.size()); 31 | 32 | return seastar::make_ready_future(std::move(e)); 33 | } 34 | 35 | seastar::future 36 | lz4_decompression_filter::operator()(rpc_recv_context &&ctx) { 37 | if (ctx.header.compression() == 38 | rpc::compression_flags::compression_flags_lz4) { 39 | auto buf = compressor->uncompress(ctx.payload); 40 | ctx.payload = std::move(buf); 41 | ctx.header.mutate_compression( 42 | rpc::compression_flags::compression_flags_none); 43 | checksum_rpc(ctx.header, ctx.payload.get(), ctx.payload.size()); 44 | } 45 | return seastar::make_ready_future(std::move(ctx)); 46 | } 47 | 48 | } // namespace smf 49 | -------------------------------------------------------------------------------- /src/core/random.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #include "smf/random.h" 5 | 6 | namespace smf { 7 | 8 | random::random() { 9 | std::random_device rd; 10 | rand_.seed(rd()); 11 | } 12 | random::~random() {} 13 | 14 | uint64_t 15 | random::next() { 16 | return dist_(rand_); 17 | } 18 | 19 | seastar::sstring 20 | randstr(const seastar::sstring &dict, 21 | std::uniform_int_distribution &dist, std::mt19937 &rand, 22 | uint32_t size) { 23 | seastar::sstring retval; 24 | retval.resize(size); 25 | // this could use a simd 26 | for (uint32_t i = 0; i < size; ++i) { 27 | uint32_t idx = dist(rand) % dict.size(); 28 | retval[i] = dict[idx]; 29 | } 30 | return retval; 31 | } 32 | 33 | seastar::sstring 34 | random::next_str(uint32_t size) { 35 | static const seastar::sstring kDict = "abcdefghijklmnopqrstuvwxyz" 36 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 37 | "1234567890" 38 | "!@#$%^&*()" 39 | "`~-_=+[{]}|;:'\",<.>/?"; 40 | static std::uniform_int_distribution kDictDist(0, kDict.size()); 41 | return randstr(kDict, kDictDist, rand_, size); 42 | } 43 | seastar::sstring 44 | random::next_alphanum(uint32_t size) { 45 | static const seastar::sstring kDict = "abcdefghijklmnopqrstuvwxyz" 46 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 47 | "1234567890"; 48 | static std::uniform_int_distribution kDictDist(0, kDict.size()); 49 | return randstr(kDict, kDictDist, rand_, size); 50 | } 51 | } // namespace smf 52 | -------------------------------------------------------------------------------- /src/core/rpc.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | namespace smf.rpc; 5 | 6 | /// \brief: headers that are stored in an int 7 | /// so they need to be inclusive. That is, you can turn on 8 | /// many flags at the same time, i.e.: enable checksum and 9 | /// have the payload be zlib compressed. 10 | /// 11 | enum compression_flags:byte { 12 | /// \brief compression not set 13 | none, 14 | /// \brief useful for preventing outgoing filters from enabling some compression 15 | disabled, 16 | /// \brief zstd compression 17 | zstd, 18 | /// \brief lz4 compression 19 | lz4 20 | } 21 | enum header_bit_flags:ubyte (bit_flags) { 22 | has_payload_headers 23 | } 24 | 25 | 26 | 27 | /// \brief: header parsed by rpc engine 28 | /// must be sizeof()'able 29 | /// that is, must be a struct in fbs language 30 | /// 31 | /// layout 32 | /// [ 8bits(compression) + 8bits(bitflags) + 16bits(session) + 32bits(size) + 32bits(checksum) + 32bits(meta) ] 33 | /// total = 128bits == 16bytes 34 | /// 35 | struct header { 36 | compression: compression_flags; 37 | bitflags: header_bit_flags; 38 | /// 16 bits for storing the actual session id. 39 | /// used for streaming client and slot allocation 40 | session: ushort; 41 | /// size of the next payload 42 | size: uint; 43 | /// currently we use (xxhash64 & UINT32_MAX) 44 | checksum: uint; 45 | /// \brief used for sending and receiving, read carefully. 46 | /// 47 | /// Receiving: 48 | /// 49 | /// Uses the same as HTTP status - on the receiving end 50 | /// We don't want to pay the cost of parsing a header 51 | /// On every response as does HTTP. std::to_string and std::stol() 52 | /// are needlesly expensive 53 | /// 54 | /// Sending: 55 | /// 56 | /// Used with the xor hash of Service::ID() ^ Service::Method::ID() 57 | /// This is how the server multiplexer figures out what function pointer 58 | /// to call 59 | /// 60 | meta: uint; 61 | } 62 | 63 | /// \brief used for extra headers, ala HTTP 64 | /// The use case for the core is to support 65 | /// zipkin/google-Dapper style tracing 66 | table dynamic_header { 67 | /// alows for binary search lookup 68 | /// use with CreateVectorOfSortedTables<> instead of the CreateVector 69 | key: string (key); 70 | value: string; 71 | } 72 | 73 | table payload_headers { 74 | /// Headers for forward compat. 75 | dynamic_headers: [dynamic_header]; 76 | /// We need to chain the actual payload 77 | size: uint = 0; 78 | checksum: uint = 0; 79 | compression: compression_flags = none; 80 | } 81 | 82 | /// \brief, useful when the type is empty 83 | /// i.e.: void foo(); 84 | /// rpc my_rpc { null_type MutateOnlyOnServerMethod(int); } 85 | /// 86 | table null_type {} 87 | -------------------------------------------------------------------------------- /src/core/rpc_envelope.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | #include "smf/rpc_header_ostream.h" 13 | #include "smf/rpc_header_utils.h" 14 | 15 | namespace smf { 16 | 17 | seastar::future<> 18 | rpc_envelope::send(seastar::output_stream *out, rpc_envelope e) { 19 | seastar::temporary_buffer header_buf(kHeaderSize); 20 | DLOG_THROW_IF(e.letter.header.size() == 0, "Invalid header size"); 21 | DLOG_THROW_IF(e.letter.header.checksum() == 0, "Invalid header checksum"); 22 | DLOG_ERROR_IF(e.letter.body.size() == 0, "Invalid payload. 0-size"); 23 | // use 0 copy iface in seastar 24 | // prepare the header locally 25 | std::memcpy(header_buf.get_write(), 26 | reinterpret_cast(&e.letter.header), kHeaderSize); 27 | // needs to be moved so we can do zero copy output buffer 28 | return out->write(std::move(header_buf)) 29 | .then([out, e = std::move(e)]() mutable { 30 | return out->write(std::move(e.letter.body)); 31 | }) 32 | .then([out] { return out->flush(); }); 33 | } 34 | 35 | rpc_envelope::rpc_envelope(rpc_letter &&l) : letter(std::move(l)) {} 36 | rpc_envelope::~rpc_envelope() {} 37 | rpc_envelope::rpc_envelope() {} 38 | 39 | rpc_envelope & 40 | rpc_envelope::operator=(rpc_envelope &&o) noexcept { 41 | if (this != &o) { letter = std::move(o.letter); } 42 | return *this; 43 | } 44 | 45 | rpc_envelope::rpc_envelope(rpc_envelope &&o) noexcept 46 | : letter(std::move(o.letter)) {} 47 | 48 | void 49 | rpc_envelope::add_dynamic_header(const char *header, const char *value) { 50 | DLOG_THROW_IF(header != nullptr, "Cannot add header with empty key"); 51 | DLOG_THROW_IF(value != nullptr, "Cannot add header with empty value"); 52 | letter.dynamic_headers.emplace(header, value); 53 | } 54 | } // namespace smf 55 | -------------------------------------------------------------------------------- /src/core/rpc_handle_router.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include "smf/rpc_handle_router.h" 4 | 5 | namespace smf { 6 | 7 | smf::rpc_service_method_handle * 8 | rpc_handle_router::get_handle_for_request(const uint32_t &request_id) { 9 | for (auto &p : services_) { 10 | auto x = p->method_for_request_id(request_id); 11 | if (x != nullptr) return x; 12 | } 13 | return nullptr; 14 | } 15 | 16 | void 17 | rpc_handle_router::register_service(std::unique_ptr s) { 18 | assert(s != nullptr); 19 | services_.push_back(std::move(s)); 20 | } 21 | } // namespace smf 22 | -------------------------------------------------------------------------------- /src/core/rpc_letter.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | #include "smf/rpc_letter.h" 4 | 5 | namespace smf { 6 | 7 | rpc_letter::rpc_letter() {} 8 | rpc_letter::rpc_letter( 9 | rpc::header _h, std::unordered_map _hdrs, 10 | seastar::temporary_buffer _buf) 11 | : header(_h), dynamic_headers(std::move(_hdrs)), body(std::move(_buf)) {} 12 | 13 | rpc_letter & 14 | rpc_letter::operator=(rpc_letter &&l) noexcept { 15 | header = l.header; 16 | dynamic_headers = std::move(l.dynamic_headers); 17 | body = std::move(l.body); 18 | return *this; 19 | } 20 | rpc_letter 21 | rpc_letter::share() { 22 | return rpc_letter(header, dynamic_headers, body.share()); 23 | } 24 | 25 | rpc_letter::rpc_letter(rpc_letter &&o) noexcept 26 | : header(o.header), dynamic_headers(std::move(o.dynamic_headers)), 27 | body(std::move(o.body)) {} 28 | 29 | rpc_letter::~rpc_letter() {} 30 | 31 | size_t 32 | rpc_letter::size() const { 33 | return sizeof(header) + body.size(); 34 | } 35 | bool 36 | rpc_letter::empty() const { 37 | return body.size() == 0; 38 | } 39 | 40 | } // namespace smf 41 | -------------------------------------------------------------------------------- /src/core/zstd_filter.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include "smf/zstd_filter.h" 4 | 5 | #include 6 | 7 | #include "smf/compression.h" 8 | #include "smf/log.h" 9 | #include "smf/rpc_header_utils.h" 10 | #include "smf/rpc_recv_context.h" 11 | 12 | namespace smf { 13 | 14 | static thread_local auto compressor = 15 | codec::make_unique(codec_type::zstd, compression_level::fastest); 16 | 17 | seastar::future 18 | zstd_compression_filter::operator()(rpc_envelope &&e) { 19 | if (e.letter.header.compression() != 20 | rpc::compression_flags::compression_flags_none) { 21 | return seastar::make_ready_future(std::move(e)); 22 | } 23 | if (e.letter.body.size() <= min_compression_size) { 24 | return seastar::make_ready_future(std::move(e)); 25 | } 26 | 27 | e.letter.body = compressor->compress(e.letter.body); 28 | e.letter.header.mutate_compression( 29 | rpc::compression_flags::compression_flags_zstd); 30 | checksum_rpc(e.letter.header, e.letter.body.get(), e.letter.body.size()); 31 | 32 | return seastar::make_ready_future(std::move(e)); 33 | } 34 | 35 | seastar::future 36 | zstd_decompression_filter::operator()(rpc_recv_context &&ctx) { 37 | if (ctx.header.compression() == 38 | rpc::compression_flags::compression_flags_zstd) { 39 | ctx.payload = compressor->uncompress(ctx.payload); 40 | ctx.header.mutate_compression( 41 | rpc::compression_flags::compression_flags_none); 42 | checksum_rpc(ctx.header, ctx.payload.get(), ctx.payload.size()); 43 | } 44 | return seastar::make_ready_future(std::move(ctx)); 45 | } 46 | 47 | } // namespace smf 48 | -------------------------------------------------------------------------------- /src/include/smf/compression.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | // modeled after the folly::io::compression.h 12 | 13 | namespace smf { 14 | 15 | enum class codec_type { lz4, zstd }; 16 | enum class compression_level { fastest, best }; 17 | 18 | /** 19 | * Uncompress data. Throws std::runtime_error on decompression error. 20 | */ 21 | class codec { 22 | public: 23 | codec(codec_type type, compression_level level) 24 | : type_(type), level_(level) {} 25 | virtual ~codec() {} 26 | virtual codec_type 27 | type() const final { 28 | return type_; 29 | } 30 | virtual compression_level 31 | level() const final { 32 | return level_; 33 | } 34 | 35 | virtual seastar::temporary_buffer 36 | compress(const seastar::temporary_buffer &data) = 0; 37 | virtual seastar::temporary_buffer compress(const char *data, 38 | std::size_t size) = 0; 39 | 40 | virtual seastar::temporary_buffer 41 | uncompress(const seastar::temporary_buffer &data) = 0; 42 | virtual seastar::temporary_buffer uncompress(const char *data, 43 | std::size_t sz) = 0; 44 | 45 | static std::unique_ptr make_unique(codec_type type, 46 | compression_level level); 47 | 48 | private: 49 | codec_type type_; 50 | compression_level level_; 51 | }; 52 | 53 | } // namespace smf 54 | -------------------------------------------------------------------------------- /src/include/smf/fbs_typed_buf.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include "smf/log.h" 10 | #include "smf/macros.h" 11 | 12 | namespace smf { 13 | 14 | /// No validation done here. Just a buffer containing a type 15 | /// 16 | template 17 | class fbs_typed_buf { 18 | public: 19 | static_assert(std::is_base_of::value, 20 | "Should ONLY be Table derived classes"); 21 | using type = T; 22 | explicit fbs_typed_buf(seastar::temporary_buffer body) 23 | : buf_(std::move(body)) { 24 | DLOG_THROW_IF(buf_.size() == 0, "Empty flatbuffers buffer"); 25 | auto ptr = static_cast(buf_.get_write()); 26 | cache_ = flatbuffers::GetMutableRoot(ptr); 27 | } 28 | fbs_typed_buf(fbs_typed_buf &&o) noexcept 29 | : buf_(std::move(o.buf_)), cache_(std::move(o.cache_)) {} 30 | fbs_typed_buf & 31 | operator=(fbs_typed_buf &&o) { 32 | buf_ = std::move(o.buf_); 33 | cache_ = std::move(o.cache_); 34 | return *this; 35 | } 36 | T *operator->() const { return cache_; } 37 | T &operator*() const { return *cache_; } 38 | T * 39 | get() const { 40 | return cache_; 41 | } 42 | 43 | // needed to share the payload at specific ranges - i.e.: internal 44 | // structures for saving to files, creating subranges for 45 | // nested flatbuffers types,etc. 46 | // 47 | seastar::temporary_buffer & 48 | buf() { 49 | return buf_; 50 | } 51 | seastar::temporary_buffer && 52 | move_buf() { 53 | cache_ = nullptr; 54 | return std::move(buf_); 55 | } 56 | 57 | SMF_DISALLOW_COPY_AND_ASSIGN(fbs_typed_buf); 58 | 59 | private: 60 | seastar::temporary_buffer buf_; 61 | T *cache_{nullptr}; 62 | }; 63 | 64 | } // namespace smf 65 | -------------------------------------------------------------------------------- /src/include/smf/futurize_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | namespace smf { 6 | 7 | /// \brief used in almost all RPC applications like 8 | /// \code 9 | /// if (!record) { 10 | /// return futurize_status_for_type(400); 11 | /// } 12 | /// \endcode 13 | template 14 | auto 15 | futurize_status_for_type(uint32_t status) { 16 | using t = smf::rpc_typed_envelope; 17 | t data; 18 | data.envelope.set_status(status); 19 | return seastar::make_ready_future(std::move(data)); 20 | } 21 | 22 | } // namespace smf 23 | -------------------------------------------------------------------------------- /src/include/smf/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "smf/macros.h" 14 | 15 | namespace smf { 16 | class histogram; 17 | struct histogram_measure; 18 | } // namespace smf 19 | 20 | namespace smf { 21 | // 1 hour in microsecs - max value 22 | static constexpr const int64_t kDefaultHistogramMaxValue = 3600000000; 23 | 24 | // VERY Expensive object. At this granularity is about 185KB 25 | // per instance 26 | struct hist_t { 27 | explicit hist_t(int64_t max_value) { 28 | ::hdr_init(1, // 1 microsec - minimum value 29 | max_value, // 1 hour in microsecs - max value 30 | 3, // Number of significant figures 31 | &hist); // Pointer to initialize 32 | } 33 | 34 | hist_t(hist_t &&o) noexcept : hist(std::move(o.hist)) {} 35 | 36 | SMF_DISALLOW_COPY_AND_ASSIGN(hist_t); 37 | 38 | ~hist_t() { 39 | if (hist) { 40 | hdr_close(hist); 41 | hist = nullptr; 42 | } 43 | } 44 | hdr_histogram *hist = nullptr; 45 | uint64_t sample_count = 0; 46 | uint64_t sample_sum = 0; 47 | }; 48 | 49 | /// brief - simple wrapper for hdr_histogram_c project 50 | /// 51 | class histogram final : public seastar::enable_lw_shared_from_this { 52 | public: 53 | static seastar::lw_shared_ptr 54 | make_lw_shared(int64_t max_value = kDefaultHistogramMaxValue); 55 | 56 | static std::unique_ptr 57 | make_unique(int64_t max_value = kDefaultHistogramMaxValue); 58 | 59 | SMF_DISALLOW_COPY_AND_ASSIGN(histogram); 60 | 61 | histogram(histogram &&o) noexcept; 62 | 63 | histogram &operator=(histogram &&o) noexcept; 64 | histogram &operator+=(const histogram &o); 65 | histogram &operator+=(const hist_t *o); 66 | 67 | void record(const uint64_t &v); 68 | 69 | void record_multiple_times(const uint64_t &v, const uint32_t ×); 70 | void record_corrected(const uint64_t &v, const uint64_t &interval); 71 | int64_t value_at(double percentile) const; 72 | double stddev() const; 73 | double mean() const; 74 | size_t memory_size() const; 75 | 76 | hdr_histogram *get(); 77 | 78 | std::unique_ptr auto_measure(); 79 | 80 | int print(FILE *fp) const; 81 | 82 | seastar::metrics::histogram seastar_histogram_logform() const; 83 | 84 | ~histogram(); 85 | 86 | private: 87 | explicit histogram(int64_t max_value); 88 | friend seastar::lw_shared_ptr; 89 | 90 | private: 91 | std::unique_ptr hist_; 92 | }; 93 | /// simple struct that records the measurement at the dtor 94 | /// similar to boost_scope_exit; 95 | struct histogram_measure { 96 | explicit histogram_measure(seastar::lw_shared_ptr ptr) 97 | : h(ptr), begin_t(std::chrono::high_resolution_clock::now()) {} 98 | 99 | SMF_DISALLOW_COPY_AND_ASSIGN(histogram_measure); 100 | 101 | histogram_measure(histogram_measure &&o) noexcept 102 | : trace_(o.trace_), h(std::move(o.h)), begin_t(o.begin_t) {} 103 | 104 | void 105 | set_trace(bool b) { 106 | trace_ = b; 107 | } 108 | 109 | ~histogram_measure() { 110 | if (h && trace_) { 111 | auto duration = std::chrono::duration_cast( 112 | std::chrono::high_resolution_clock::now() - begin_t) 113 | .count(); 114 | h->record(duration); 115 | } 116 | } 117 | 118 | bool trace_ = true; 119 | seastar::lw_shared_ptr h = nullptr; 120 | std::chrono::high_resolution_clock::time_point begin_t; 121 | }; 122 | } // namespace smf 123 | 124 | inline std::ostream & 125 | operator<<(std::ostream &o, const smf::histogram &h) { 126 | o << "smf::histogram={p50=" << h.value_at(.5) << "μs,p99=" << h.value_at(.99) 127 | << "μs,p999=" << h.value_at(.999) << "μs}"; 128 | return o; 129 | }; 130 | -------------------------------------------------------------------------------- /src/include/smf/histogram_seastar_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | 7 | #include 8 | 9 | #include "smf/histogram.h" 10 | 11 | namespace smf { 12 | struct histogram_seastar_utils { 13 | static seastar::future> 14 | print_histogram(histogram *h); 15 | inline static seastar::future<> 16 | write(seastar::sstring filename, std::unique_ptr h) { 17 | auto p = h.get(); 18 | return write_histogram(std::move(filename), p).finally([hh = std::move(h)] { 19 | }); 20 | } 21 | inline static seastar::future<> 22 | write(seastar::sstring filename, seastar::lw_shared_ptr h) { 23 | return write_histogram(std::move(filename), h.get()).finally([h] {}); 24 | } 25 | static seastar::future<> write_histogram(seastar::sstring filename, 26 | histogram *h); 27 | }; 28 | 29 | } // namespace smf 30 | -------------------------------------------------------------------------------- /src/include/smf/human_bytes.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | namespace smf { 7 | 8 | /// \brief prints a double as bytes, KB, MB, GB, TB, etc. 9 | /// \code `std::cout << human_bytes(x) << std::endl;` \endcode 10 | /// 11 | struct human_bytes { 12 | explicit human_bytes(double d) : data(d) {} 13 | double data; 14 | }; 15 | 16 | // take by value 17 | inline std::ostream & 18 | operator<<(std::ostream &o, smf::human_bytes h) { 19 | auto const orig_precision = o.precision(); 20 | o << std::fixed << std::setprecision(3); 21 | if (h.data < 1024) { 22 | if (h.data < 1) { h.data = 0; } 23 | o << h.data << " bytes"; 24 | } else if ((h.data /= static_cast(1024)) < 25 | static_cast(1000)) { 26 | o << h.data << " KB"; 27 | } else if ((h.data /= static_cast(1024)) < 28 | static_cast(1000)) { 29 | o << h.data << " MB"; 30 | } else if ((h.data /= static_cast(1024)) < 31 | static_cast(1000)) { 32 | o << h.data << " GB"; 33 | } else { 34 | o << h.data << " TB"; 35 | } 36 | // reset to original 37 | o << std::setprecision(orig_precision); 38 | return o; 39 | } 40 | 41 | } // namespace smf 42 | -------------------------------------------------------------------------------- /src/include/smf/load_channel.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | 7 | #include "smf/load_generator_duration.h" 8 | #include "smf/log.h" 9 | #include "smf/lz4_filter.h" 10 | #include "smf/macros.h" 11 | #include "smf/rpc_client.h" 12 | #include "smf/rpc_envelope.h" 13 | #include "smf/zstd_filter.h" 14 | 15 | namespace smf { 16 | 17 | template 18 | struct load_channel { 19 | using func_t = 20 | std::function(ClientService *, smf::rpc_envelope &&)>; 21 | using generator_t = std::function; 23 | 24 | load_channel(uint64_t id, const char *ip, uint16_t port, uint64_t mem, 25 | smf::rpc::compression_flags compression) 26 | : channel_id_(id) { 27 | smf::rpc_client_opts opts{}; 28 | opts.server_addr = seastar::ipv4_addr{ip, port}; 29 | opts.memory_avail_for_client = mem; 30 | client = seastar::make_shared(std::move(opts)); 31 | client->enable_histogram_metrics(); 32 | if (compression == smf::rpc::compression_flags::compression_flags_zstd) { 33 | client->incoming_filters().push_back(smf::zstd_decompression_filter()); 34 | client->outgoing_filters().push_back(smf::zstd_compression_filter(1024)); 35 | } else if (compression == 36 | smf::rpc::compression_flags::compression_flags_lz4) { 37 | client->incoming_filters().push_back(smf::lz4_decompression_filter()); 38 | client->outgoing_filters().push_back(smf::lz4_compression_filter(1024)); 39 | } 40 | } 41 | 42 | SMF_DISALLOW_COPY_AND_ASSIGN(load_channel); 43 | 44 | load_channel(load_channel &&o) : client(std::move(o.client)) {} 45 | 46 | ~load_channel() {} 47 | 48 | inline auto 49 | get_histogram() const { 50 | return client->get_histogram(); 51 | } 52 | 53 | seastar::future<> 54 | connect() { 55 | return client->connect(); 56 | } 57 | seastar::future<> 58 | stop() { 59 | return client->stop(); 60 | } 61 | 62 | seastar::future<> 63 | invoke(uint32_t reqs, const boost::program_options::variables_map &opts, 64 | seastar::lw_shared_ptr stats, generator_t gen, 65 | func_t func) { 66 | LOG_THROW_IF(reqs == 0, "bad number of requests"); 67 | LOG_INFO("Channel: {}. Launching {} serial reqs", channel_id_, reqs); 68 | // explicitly make copies of opts, gen, and func 69 | // happens once per reqs call 70 | // 71 | return seastar::do_for_each( 72 | boost::counting_iterator(0), 73 | boost::counting_iterator(reqs), 74 | [this, opts, stats, gen, func](uint32_t i) mutable { 75 | auto e = gen(opts); 76 | stats->total_bytes += e.size(); 77 | return func(client.get(), std::move(e)); 78 | }); 79 | } 80 | uint64_t channel_id_ = 0; 81 | seastar::shared_ptr client; 82 | }; 83 | 84 | } // namespace smf 85 | -------------------------------------------------------------------------------- /src/include/smf/load_generator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include "smf/load_generator_args.h" 11 | #include "smf/load_generator_duration.h" 12 | #include "smf/macros.h" 13 | #include "smf/random.h" 14 | #include "smf/rpc_envelope.h" 15 | 16 | namespace smf { 17 | 18 | /// \brief enable load generator for any smf::rpc_client* 19 | /// You can only have one client per active stream 20 | /// the problem comes when you try to read, 2 function calls to read, even 21 | /// read_exactly might interpolate 22 | /// which yields incorrect results for test. That use case has to be manually 23 | /// optimized and don't expect it to be the main use case 24 | /// In effect, you need a socket per concurrency item in the command line 25 | /// flags 26 | /// 27 | template 28 | class __attribute__((visibility("default"))) load_generator { 29 | public: 30 | using channel_t = load_channel; 31 | using channel_t_ptr = std::unique_ptr; 32 | 33 | using method_cb_t = 34 | std::function(ClientService *, smf::rpc_envelope &&)>; 35 | using generator_cb_t = std::function; 37 | 38 | explicit load_generator(load_generator_args _args) : args(_args) { 39 | random rand; 40 | channels_.reserve(args.concurrency); 41 | for (uint32_t i = 0u; i < args.concurrency; ++i) { 42 | channels_.push_back(std::make_unique( 43 | rand.next(), args.ip, args.port, 44 | args.memory_per_core / args.concurrency, args.compression)); 45 | } 46 | } 47 | ~load_generator() {} 48 | 49 | SMF_DISALLOW_COPY_AND_ASSIGN(load_generator); 50 | 51 | const load_generator_args args; 52 | 53 | std::unique_ptr 54 | copy_histogram() const { 55 | auto h = smf::histogram::make_unique(); 56 | for (auto &c : channels_) { 57 | auto p = c->get_histogram(); 58 | *h += *p; 59 | } 60 | return std::move(h); 61 | } 62 | 63 | seastar::future<> 64 | stop() { 65 | return seastar::do_for_each(channels_.begin(), channels_.end(), 66 | [](auto &c) { return c->stop(); }); 67 | } 68 | seastar::future<> 69 | connect() { 70 | LOG_INFO("Making {} connections on this core.", channels_.size()); 71 | return seastar::do_for_each(channels_.begin(), channels_.end(), 72 | [](auto &c) { return c->connect(); }); 73 | } 74 | seastar::future 75 | benchmark(generator_cb_t gen, method_cb_t method_cb) { 76 | namespace co = std::chrono; 77 | const uint32_t reqs_per_channel = 78 | std::max(1, std::ceil(args.num_of_req / args.concurrency)); 79 | auto duration = 80 | seastar::make_lw_shared(reqs_per_channel); 81 | 82 | return seastar::do_with( 83 | seastar::semaphore(args.concurrency), 84 | [this, duration, method_cb, gen, 85 | reqs_per_channel](auto &limit) mutable { 86 | duration->begin(); 87 | return seastar::do_for_each( 88 | channels_.begin(), channels_.end(), 89 | [this, &limit, gen, method_cb, reqs_per_channel, 90 | duration](auto &c) mutable { 91 | return limit.wait(1).then([this, gen, &c, &limit, 92 | reqs_per_channel, duration, 93 | method_cb]() mutable { 94 | // notice that this does not return, hence 95 | // executing concurrently 96 | (void)c 97 | ->invoke(reqs_per_channel, args.cfg, duration, 98 | gen, method_cb) 99 | .finally([&limit] { limit.signal(1); }); 100 | }); 101 | }) 102 | .then([this, &limit, duration] { 103 | // now let's wait for ALL to finish 104 | return limit.wait(args.concurrency).finally([duration] { 105 | duration->end(); 106 | }); 107 | }); 108 | }) 109 | .then([duration] { 110 | return seastar::make_ready_future( 111 | std::move(*duration)); 112 | }); 113 | } 114 | 115 | private: 116 | std::vector channels_{}; 117 | }; 118 | 119 | } // namespace smf 120 | -------------------------------------------------------------------------------- /src/include/smf/load_generator_args.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | namespace smf { 12 | // missing timeout 13 | // tracer probability 14 | // exponential distribution, poisson distribution of load 15 | // 16 | struct load_generator_args { 17 | load_generator_args(const char *_ip, uint16_t _port, size_t _num_of_req, 18 | size_t _concurrency, size_t _memory_per_core, 19 | smf::rpc::compression_flags _compression, 20 | boost::program_options::variables_map _cfg) 21 | : ip(_ip), port(_port), num_of_req(_num_of_req), concurrency(_concurrency), 22 | memory_per_core(_memory_per_core), compression(_compression), 23 | cfg(std::move(_cfg)) { 24 | static const uint64_t kMinMemory = 1 << 24; // 16MB 25 | LOG_THROW_IF(memory_per_core < kMinMemory && concurrency > 1, 26 | "Cfg error. memory < 16MB per connection."); 27 | } 28 | 29 | const char *ip; 30 | uint16_t port; 31 | size_t num_of_req; 32 | size_t concurrency; 33 | size_t memory_per_core; 34 | smf::rpc::compression_flags compression; 35 | const boost::program_options::variables_map cfg; 36 | }; 37 | 38 | } // namespace smf 39 | 40 | namespace std { 41 | inline std::ostream & 42 | operator<<(std::ostream &o, const smf::load_generator_args &args) { 43 | o << "generator_args{ip=" << args.ip << ", port=" << args.port 44 | << ", num_of_req=" << args.num_of_req 45 | << ", concurrency=" << args.concurrency 46 | << ", memory_per_client=" << args.memory_per_core / args.concurrency 47 | << ", compression=" << smf::rpc::EnumNamecompression_flags(args.compression) 48 | << ", cfg_size=" << args.cfg.size() << "}"; 49 | return o; 50 | } 51 | } // namespace std 52 | -------------------------------------------------------------------------------- /src/include/smf/load_generator_duration.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "smf/histogram.h" 10 | #include "smf/human_bytes.h" 11 | 12 | namespace smf { 13 | 14 | struct load_generator_duration { 15 | explicit load_generator_duration(uint64_t reqs) : num_of_req(reqs) {} 16 | load_generator_duration(load_generator_duration &&d) noexcept 17 | : num_of_req(std::move(d.num_of_req)), test_begin(std::move(d.test_begin)), 18 | test_end(std::move(d.test_end)), total_bytes(std::move(d.total_bytes)) {} 19 | 20 | uint64_t num_of_req; 21 | std::chrono::high_resolution_clock::time_point test_begin; 22 | std::chrono::high_resolution_clock::time_point test_end; 23 | 24 | uint64_t total_bytes{0}; 25 | 26 | void 27 | begin() { 28 | test_begin = std::chrono::high_resolution_clock::now(); 29 | } 30 | void 31 | end() { 32 | test_end = std::chrono::high_resolution_clock::now(); 33 | } 34 | 35 | inline uint64_t 36 | duration_in_millis() const { 37 | namespace co = std::chrono; 38 | auto d = test_end - test_begin; 39 | return co::duration_cast(d).count(); 40 | } 41 | 42 | inline uint64_t 43 | qps() const { 44 | auto milli = duration_in_millis(); 45 | const auto reqs = static_cast(num_of_req); 46 | 47 | // some times the test run under 1 millisecond 48 | const auto safe_denom = std::max(milli, 1); 49 | const auto denom = static_cast(safe_denom); 50 | 51 | auto queries_per_milli = reqs / denom; 52 | 53 | return queries_per_milli * 1000.0; 54 | } 55 | }; 56 | 57 | inline std::ostream & 58 | operator<<(std::ostream &o, const smf::load_generator_duration &d) { 59 | o << "generator_duration={ test_duration= " << d.duration_in_millis() 60 | << "ms, qps=" << d.qps() 61 | << ", total_bytes=" << smf::human_bytes(d.total_bytes) << "(" 62 | << d.total_bytes << ") }"; 63 | return o; 64 | } 65 | 66 | } // namespace smf 67 | -------------------------------------------------------------------------------- /src/include/smf/lz4_filter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include "smf/rpc_envelope.h" 6 | #include "smf/rpc_filter.h" 7 | #include "smf/rpc_recv_context.h" 8 | 9 | namespace smf { 10 | 11 | struct lz4_compression_filter : rpc_filter { 12 | explicit lz4_compression_filter(uint32_t _min_compression_size) 13 | : min_compression_size(_min_compression_size) {} 14 | 15 | seastar::future operator()(rpc_envelope &&e); 16 | 17 | const uint32_t min_compression_size; 18 | }; 19 | 20 | struct lz4_decompression_filter : rpc_filter { 21 | seastar::future operator()(rpc_recv_context &&ctx); 22 | }; 23 | 24 | } // namespace smf 25 | -------------------------------------------------------------------------------- /src/include/smf/macros.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | 6 | template 7 | char (&smf_array_size_helper(T (&array)[N]))[N]; 8 | #define SMF_ARRAYSIZE(array) (sizeof(::smf_array_size_helper(array))) 9 | 10 | #define SMF_DISALLOW_COPY(TypeName) TypeName(const TypeName &) = delete 11 | 12 | #define SMF_DISALLOW_ASSIGN(TypeName) void operator=(const TypeName &) = delete 13 | 14 | // A macro to disallow the copy constructor and operator= functions 15 | // This is usually placed in the private: declarations for a class. 16 | #define SMF_DISALLOW_COPY_AND_ASSIGN(TypeName) \ 17 | TypeName(const TypeName &) = delete; \ 18 | void operator=(const TypeName &) = delete 19 | 20 | #define SMF_DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ 21 | TypeName() = delete; \ 22 | SMF_DISALLOW_COPY_AND_ASSIGN(TypeName) 23 | 24 | // GCC can be told that a certain branch is not likely to be taken (for 25 | // instance, a CHECK failure), and use that information in static analysis. 26 | // Giving it this information can help it optimize for the common case in 27 | // the absence of better information (ie. -fprofile-arcs). 28 | #if defined(__GNUC__) 29 | #define SMF_LIKELY(x) (__builtin_expect((x), 1)) 30 | #define SMF_UNLIKELY(x) (__builtin_expect((x), 0)) 31 | #else 32 | #define SMF_LIKELY(x) (x) 33 | #define SMF_UNLIKELY(x) (x) 34 | #endif 35 | 36 | // we only compile on linux. both clang & gcc support this 37 | #if defined(__clang__) || defined(__GNUC__) 38 | #define SMF_ALWAYS_INLINE inline __attribute__((__always_inline__)) 39 | #else 40 | #define SMF_ALWAYS_INLINE inline 41 | #endif 42 | 43 | #if defined(__clang__) || defined(__GNUC__) 44 | #define SMF_NOINLINE __attribute__((__noinline__)) 45 | #else 46 | #define SMF_NOINLINE 47 | #endif 48 | -------------------------------------------------------------------------------- /src/include/smf/native_type_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "smf/log.h" 11 | 12 | namespace smf { 13 | /// \brief converts a flatbuffers::NativeTableType into a buffer. 14 | /// See full benchmarks/comparisons here: 15 | /// https://github.com/smfrpc/smf/pull/259 16 | /// 17 | /// Makes 2 copies in the worst case. See PR #259 18 | /// If the buffer that we encode is large, we generate a new fbs::builder 19 | /// to keep memory usage predicatable. 20 | /// 21 | /// 22 | template 23 | seastar::temporary_buffer 24 | native_table_as_buffer(const typename RootType::NativeTableType &t) { 25 | flatbuffers::FlatBufferBuilder bdr; 26 | bdr.Finish(RootType::Pack(bdr, &t, nullptr)); 27 | auto mem = bdr.Release(); 28 | auto ptr = reinterpret_cast(mem.data()); 29 | auto sz = mem.size(); 30 | return seastar::temporary_buffer( 31 | ptr, sz, seastar::make_object_deleter(std::move(mem))); 32 | } 33 | 34 | } // namespace smf 35 | -------------------------------------------------------------------------------- /src/include/smf/random.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace smf { 11 | class random { 12 | public: 13 | random(); 14 | ~random(); 15 | 16 | uint64_t next(); 17 | 18 | seastar::sstring next_str(uint32_t size); 19 | seastar::sstring next_alphanum(uint32_t size); 20 | 21 | private: 22 | std::mt19937 rand_; 23 | // by default range [0, MAX] 24 | std::uniform_int_distribution dist_; 25 | }; 26 | 27 | } // namespace smf 28 | -------------------------------------------------------------------------------- /src/include/smf/reconnect_client.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace smf { 15 | enum class reconnect_backoff : uint16_t { 16 | none = 0, 17 | wait_1_sec = 1, 18 | wait_3_sec = 3, 19 | wait_5_sec = 5, 20 | wait_10_sec = 10, 21 | wait_20_sec = 20, 22 | wait_30_sec = 30, 23 | wait_60_sec = 60, 24 | wait_300_sec = 300, // 5min 25 | wait_600_sec = 600, // 10min 26 | wait_1800_sec = 1800, // 30min 27 | // always update this one 28 | max = wait_1800_sec 29 | }; 30 | 31 | template 32 | /// allows any smf::rpc_client to have an exponential backoff 33 | /// background reconnect. 34 | class reconnect_client { 35 | public: 36 | using type = std::enable_if_t::value, T>; 37 | SMF_DISALLOW_COPY_AND_ASSIGN(reconnect_client); 38 | explicit reconnect_client(seastar::ipv4_addr server_addr) 39 | : client_(seastar::make_shared(std::move(server_addr))) {} 40 | explicit reconnect_client(smf::rpc_client_opts o) 41 | : client_(seastar::make_shared(std::move(o))) {} 42 | reconnect_client(reconnect_client &&o) noexcept 43 | : client_(std::move(o.client_)), bo_(o.bo_), rand_(std::move(o.rand_)), 44 | reconnect_gate_(std::move(o.reconnect_gate_)) {} 45 | 46 | /// \brief main method 47 | seastar::future<> connect(); 48 | seastar::future<> 49 | stop() { 50 | return reconnect_gate_.close().then([this] { return client_->stop(); }); 51 | } 52 | /// \brief returns pointer to client (this) if 53 | /// everything is good. 54 | seastar::shared_ptr 55 | get() { 56 | return client_; 57 | } 58 | 59 | reconnect_backoff 60 | backoff() const { 61 | return bo_; 62 | } 63 | 64 | private: 65 | seastar::shared_ptr client_; 66 | reconnect_backoff bo_{reconnect_backoff::none}; 67 | random rand_; 68 | seastar::gate reconnect_gate_; 69 | }; 70 | 71 | /// \brief prefix operator i.e.: `++backoff;` 72 | inline reconnect_backoff & 73 | operator++(reconnect_backoff &b) { 74 | const auto as_num = static_cast>(b); 75 | // increment always, so clamp table below works on 'next' 76 | b = static_cast(as_num + 1); 77 | // clamp table 78 | constexpr std::array arr{ 79 | reconnect_backoff::none, reconnect_backoff::wait_1_sec, 80 | reconnect_backoff::wait_3_sec, reconnect_backoff::wait_5_sec, 81 | reconnect_backoff::wait_10_sec, reconnect_backoff::wait_20_sec, 82 | reconnect_backoff::wait_30_sec, reconnect_backoff::wait_60_sec, 83 | reconnect_backoff::wait_300_sec, reconnect_backoff::wait_600_sec, 84 | reconnect_backoff::wait_1800_sec}; 85 | for (const reconnect_backoff &x : arr) { 86 | if (b <= x) { 87 | b = x; 88 | return b; 89 | } 90 | } 91 | b = reconnect_backoff::wait_1800_sec; 92 | return b; 93 | } 94 | template 95 | seastar::future<> 96 | reconnect_client::connect() { 97 | if (client_->is_conn_valid()) { return seastar::make_ready_future<>(); } 98 | if (reconnect_gate_.is_closed()) { return seastar::make_ready_future<>(); } 99 | return seastar::with_gate(reconnect_gate_, [this] { 100 | return client_->reconnect() 101 | .then([this] { bo_ = reconnect_backoff::none; }) 102 | .handle_exception([this](auto eptr) { 103 | LOG_INFO("Recovering from '{}' for {}", eptr, client_->server_addr); 104 | // perform in background; increase the backoff to next version 105 | (void)seastar::with_gate(reconnect_gate_, [this] { 106 | // ensure 100ms random jitter 107 | auto secs = 108 | std::chrono::milliseconds(rand_.next() % 100) + 109 | std::chrono::seconds( 110 | static_cast>(++bo_)); 111 | DLOG_TRACE("Sleeping for {}", secs.count()); 112 | return seastar::sleep(secs) 113 | .then([this] { return connect(); }) 114 | .discard_result(); 115 | }); 116 | }); 117 | }); 118 | } 119 | 120 | } // namespace smf 121 | 122 | namespace std { 123 | inline ostream & 124 | operator<<(ostream &o, smf::reconnect_backoff b) { 125 | return o << "smf::reconnect_backoff{ " 126 | << static_cast>(b) 127 | << "secs }"; 128 | } 129 | } // namespace std 130 | -------------------------------------------------------------------------------- /src/include/smf/rpc_connection.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | // seastar 6 | #include 7 | #include 8 | #include 9 | 10 | #include "smf/std-compat.h" 11 | #include "smf/macros.h" 12 | #include "smf/rpc_connection_limits.h" 13 | 14 | namespace smf { 15 | 16 | /// \brief this class is called a *lot* 17 | /// to test if we should keep consuming or not. 18 | /// Do not make virtual. Instead use type embedding. 19 | /// Currently only the rpc_server_connection is used - instead of doing 20 | /// seastar::shared_ptr we can use seastar::lw_shared_ptr and have 21 | /// effectively no cost of pointer ownership. 22 | /// 23 | class rpc_connection final { 24 | public: 25 | explicit rpc_connection( 26 | seastar::connected_socket fd, seastar::socket_address address, 27 | seastar::lw_shared_ptr conn_limits = nullptr) 28 | : socket(std::move(fd)), remote_address(address), istream(socket.input()), 29 | ostream(socket.output()), limits(conn_limits) { 30 | socket.set_nodelay(true); 31 | socket.set_keepalive(true); 32 | } 33 | 34 | seastar::connected_socket socket; 35 | const seastar::socket_address remote_address; 36 | seastar::input_stream istream; 37 | seastar::output_stream ostream; 38 | seastar::lw_shared_ptr limits; 39 | uint32_t istream_active_parser{0}; 40 | 41 | inline void 42 | disable() { 43 | enabled_ = false; 44 | } 45 | SMF_ALWAYS_INLINE bool 46 | is_enabled() const { 47 | return enabled_; 48 | } 49 | SMF_ALWAYS_INLINE bool 50 | is_valid() { 51 | return enabled_ && !has_error() && !istream.eof(); 52 | } 53 | SMF_ALWAYS_INLINE bool 54 | has_error() const { 55 | return error_.operator bool(); 56 | } 57 | SMF_ALWAYS_INLINE void 58 | set_error(seastar::sstring e) { 59 | if (!error_) { 60 | error_ = e; 61 | return; 62 | } 63 | // keep history of errors 64 | error_ = error_.value() + " :: " + e; 65 | } 66 | inline seastar::sstring 67 | get_error() const { 68 | if (!error_) return ""; 69 | return error_.value(); 70 | } 71 | 72 | ~rpc_connection() {} 73 | 74 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_connection); 75 | 76 | private: 77 | smf::compat::optional error_; 78 | bool enabled_{true}; 79 | }; 80 | } // namespace smf 81 | -------------------------------------------------------------------------------- /src/include/smf/rpc_connection_limits.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | namespace smf { 13 | /// Currently, it contains the limit to prase the body of the connection to be 14 | /// 1minute after successfully parsing the header. If the RPC doesn't finish 15 | /// parsing the body, it will throw an exception causing a connection close on 16 | /// the server side 17 | /// 18 | struct rpc_connection_limits { 19 | using timer_duration_t = seastar::timer<>::duration; 20 | explicit rpc_connection_limits(uint64_t max_mem_per_core, 21 | timer_duration_t body_timeout_duration) 22 | : max_memory(max_mem_per_core), 23 | max_body_parsing_duration(body_timeout_duration), 24 | resources_available(max_mem_per_core) {} 25 | 26 | ~rpc_connection_limits() = default; 27 | 28 | const uint64_t max_memory; 29 | const timer_duration_t max_body_parsing_duration; 30 | 31 | seastar::semaphore resources_available; 32 | }; 33 | inline std::ostream & 34 | operator<<(std::ostream &o, const ::smf::rpc_connection_limits &l) { 35 | o << "rpc_connection_limits{max_mem:" << ::smf::human_bytes(l.max_memory) 36 | << ", max_body_parsing_duration: " 37 | << std::chrono::duration_cast( 38 | l.max_body_parsing_duration) 39 | .count() 40 | << "ms, res_avail:" << ::smf::human_bytes(l.resources_available.current()) 41 | << " (" << l.resources_available.current() << ")}"; 42 | return o; 43 | } 44 | 45 | } // namespace smf 46 | -------------------------------------------------------------------------------- /src/include/smf/rpc_envelope.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | // seastar 5 | #include 6 | #include 7 | // smf 8 | #include "smf/macros.h" 9 | #include "smf/rpc_letter.h" 10 | 11 | namespace smf { 12 | /// \brief send rpc request to stablished remote host. 13 | /// This class will async send the request to the remote host in rpc_client.h 14 | /// and will ensure that the request is valid, enabling tracing if available, 15 | /// compression, etc. Send arbitrary byte arrays to remote host 16 | /// 17 | struct rpc_envelope { 18 | constexpr static size_t kHeaderSize = sizeof(rpc::header); 19 | static seastar::future<> send(seastar::output_stream *out, 20 | rpc_envelope req); 21 | 22 | rpc_envelope(); 23 | ~rpc_envelope(); 24 | explicit rpc_envelope(rpc_letter &&_letter); 25 | 26 | /// \brief copy ctor deleted 27 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_envelope); 28 | 29 | /// \brief move ctor 30 | rpc_envelope(rpc_envelope &&o) noexcept; 31 | rpc_envelope &operator=(rpc_envelope &&o) noexcept; 32 | 33 | /// \brief add a key=value pair ala HTTP/1.1 34 | /// Useful for the framework to send trace information etc. 35 | /// 36 | void add_dynamic_header(const char *header, const char *value); 37 | 38 | /// \brief used on the client-sender side 39 | SMF_ALWAYS_INLINE void 40 | set_request_id(uint32_t request_id) { 41 | letter.header.mutate_meta(request_id); 42 | } 43 | 44 | /// \brief typically used on the server-returning-content side. 45 | /// usually it acts like the HTTP status codes 46 | SMF_ALWAYS_INLINE void 47 | set_status(const uint32_t &status) { 48 | letter.header.mutate_meta(status); 49 | } 50 | 51 | SMF_ALWAYS_INLINE size_t 52 | size() const { 53 | return letter.size(); 54 | } 55 | 56 | /// \brief, sometimes you know/understand the lifecycle and want a read 57 | /// only copy of this rpc_envelope - note that headers are 'copied', the 58 | /// payload, however is 'shared()' 59 | rpc_envelope 60 | share() { 61 | return rpc_envelope(letter.share()); 62 | } 63 | 64 | rpc_letter letter; 65 | }; 66 | } // namespace smf 67 | -------------------------------------------------------------------------------- /src/include/smf/rpc_filter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | namespace smf { 6 | 7 | /// brief - generic filter interface (c++ conept'ish) that gets 8 | /// applied from the server and client(todo) before and after a request 9 | /// 10 | template 11 | struct rpc_filter { 12 | seastar::future operator()(T t); 13 | }; 14 | 15 | /// brief - applies a functor `future operator()(T t)` to all the filters 16 | /// useful for incoming and outgoing filters. Taking a pair of iterators 17 | /// 18 | template 19 | seastar::future 20 | rpc_filter_apply(const Iterator &b, const Iterator &end, Arg &&arg) { 21 | auto begin = b; 22 | if (begin == end) { 23 | return seastar::make_ready_future(std::forward(arg)); 24 | } 25 | return (*begin)(std::forward(arg)) 26 | .then([begin = std::next(begin), end](Arg &&a) { 27 | return rpc_filter_apply(begin, end, 28 | std::forward(a)); 29 | }); 30 | } 31 | 32 | template 33 | seastar::future 34 | rpc_filter_apply(Container *c, Arg &&arg) { 35 | return rpc_filter_apply( 36 | c->begin(), c->end(), std::forward(arg)); 37 | } 38 | 39 | } // namespace smf 40 | -------------------------------------------------------------------------------- /src/include/smf/rpc_handle_router.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | 7 | #include "smf/macros.h" 8 | #include "smf/rpc_envelope.h" 9 | #include "smf/rpc_recv_context.h" 10 | #include "smf/rpc_service.h" 11 | 12 | namespace smf { 13 | /// \brief used to host many services 14 | /// multiple services can use this class to handle the routing for them 15 | /// 16 | class rpc_handle_router { 17 | public: 18 | rpc_handle_router() {} 19 | ~rpc_handle_router() {} 20 | void register_service(std::unique_ptr s); 21 | 22 | seastar::future<> stop(); 23 | 24 | smf::rpc_service_method_handle * 25 | get_handle_for_request(const uint32_t &request_id); 26 | 27 | /// \brief multiple rpc_services can register w/ this handle router 28 | void register_rpc_service(rpc_service *s); 29 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_handle_router); 30 | 31 | const std::vector> & 32 | services() const { 33 | return services_; 34 | } 35 | 36 | private: 37 | std::vector> services_{}; 38 | }; 39 | } // namespace smf 40 | 41 | namespace std { 42 | static inline ostream & 43 | operator<<(std::ostream &o, const smf::rpc_handle_router &r) { 44 | o << "rpc_handle_router{"; 45 | for (const auto &service : r.services()) { 46 | service->print(o); 47 | } 48 | o << "}"; 49 | return o; 50 | } 51 | } // namespace std 52 | -------------------------------------------------------------------------------- /src/include/smf/rpc_header_ostream.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | namespace std { 10 | inline ostream & 11 | operator<<(ostream &o, const ::smf::rpc::header &h) { 12 | o << "rpc::header={compression:" 13 | << ::smf::rpc::EnumNamecompression_flags(h.compression()) 14 | << ", header_bit_flags:" 15 | << std::bitset<8>(static_cast(h.bitflags())) 16 | << ", session:" << h.session() << ", size:" << h.size() 17 | << ", checksum:" << h.checksum() << ", meta:" << h.meta() << "}"; 18 | return o; 19 | } 20 | } // namespace std 21 | -------------------------------------------------------------------------------- /src/include/smf/rpc_header_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #pragma once 5 | #include 6 | 7 | #include "smf/rpc_generated.h" 8 | 9 | namespace smf { 10 | 11 | SMF_ALWAYS_INLINE static uint32_t 12 | rpc_checksum_payload(const char *payload, uint32_t size) { 13 | return std::numeric_limits::max() & XXH64(payload, size, 0); 14 | } 15 | 16 | template 17 | SMF_ALWAYS_INLINE void 18 | checksum_rpc(T &hdr, const char *payload, uint32_t size) { 19 | hdr.mutate_checksum(rpc_checksum_payload(payload, size)); 20 | hdr.mutate_size(size); 21 | } 22 | 23 | } // namespace smf 24 | -------------------------------------------------------------------------------- /src/include/smf/rpc_letter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | #include "smf/macros.h" 9 | #include "smf/rpc_generated.h" 10 | 11 | namespace smf { 12 | 13 | struct rpc_letter { 14 | rpc_letter(); 15 | rpc_letter(rpc::header, 16 | std::unordered_map, 17 | seastar::temporary_buffer); 18 | rpc_letter &operator=(rpc_letter &&l) noexcept; 19 | rpc_letter(rpc_letter &&) noexcept; 20 | ~rpc_letter(); 21 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_letter); 22 | 23 | /// \brief returns a share'd copy of the body payload 24 | /// Do not modify the body itself - replace it 25 | /// and make an explicit copy instead 26 | /// 27 | rpc_letter share(); 28 | /// \brief size including headers 29 | size_t size() const; 30 | /// \brief does it have a valid body 31 | bool empty() const; 32 | 33 | rpc::header header; 34 | std::unordered_map dynamic_headers; 35 | seastar::temporary_buffer body; 36 | }; 37 | 38 | } // namespace smf 39 | -------------------------------------------------------------------------------- /src/include/smf/rpc_recv_context.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | // std 5 | // seastar 6 | #include 7 | #include 8 | // smf 9 | #include "smf/macros.h" 10 | #include "smf/rpc_connection.h" 11 | #include "smf/rpc_connection_limits.h" 12 | #include "smf/rpc_generated.h" 13 | #include "smf/std-compat.h" 14 | 15 | namespace smf { 16 | struct rpc_recv_context { 17 | /// \brief determines if we've correctly parsed the request 18 | /// \return optional fully parsed request, iff the request is supported 19 | /// i.e: we support the compression algorithm, etc 20 | /// 21 | /// Parses the incoming rpc in 2 steps. First we statically know the 22 | /// size of the header, so we parse sizeof(Header). We with this information 23 | /// we parse the body of the request 24 | /// 25 | static seastar::future> 26 | parse_header(rpc_connection *conn); 27 | static seastar::future> 28 | parse_payload(rpc_connection *conn, rpc::header hdr); 29 | 30 | explicit rpc_recv_context( 31 | seastar::lw_shared_ptr server_instance_limits, 32 | seastar::socket_address remote_address, rpc::header hdr, 33 | seastar::temporary_buffer body); 34 | rpc_recv_context(rpc_recv_context &&o) noexcept; 35 | ~rpc_recv_context(); 36 | 37 | /// \brief used by the server side to determine the actual RPC 38 | SMF_ALWAYS_INLINE uint32_t 39 | request_id() const { 40 | return header.meta(); 41 | } 42 | 43 | /// \brief used by the client side to determine the status from the server 44 | /// follows the HTTP status codes 45 | SMF_ALWAYS_INLINE uint32_t 46 | status() const { 47 | return header.meta(); 48 | } 49 | 50 | SMF_ALWAYS_INLINE uint16_t 51 | session() const { 52 | return header.session(); 53 | } 54 | 55 | seastar::lw_shared_ptr rpc_server_limits; 56 | const seastar::socket_address remote_address; 57 | rpc::header header; 58 | seastar::temporary_buffer payload; 59 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_recv_context); 60 | }; 61 | } // namespace smf 62 | -------------------------------------------------------------------------------- /src/include/smf/rpc_recv_typed_context.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include "smf/macros.h" 6 | #include "smf/rpc_recv_context.h" 7 | 8 | namespace smf { 9 | 10 | template 11 | class rpc_recv_typed_context { 12 | static_assert(std::is_base_of::value, 13 | "Should ONLY be Table derived classes"); 14 | 15 | public: 16 | using type = T; 17 | using opt_recv_ctx_t = smf::compat::optional; 18 | 19 | rpc_recv_typed_context() : ctx(smf::compat::nullopt) {} 20 | 21 | explicit rpc_recv_typed_context(opt_recv_ctx_t t) : ctx(std::move(t)) { 22 | if (SMF_LIKELY(!!ctx)) { 23 | auto ptr = ctx.value().payload.get_write(); 24 | cache_ = flatbuffers::GetMutableRoot(ptr); 25 | } 26 | } 27 | 28 | rpc_recv_typed_context(rpc_recv_typed_context &&o) noexcept 29 | : ctx(std::move(o.ctx)), cache_(std::move(o.cache_)) {} 30 | 31 | rpc_recv_typed_context & 32 | operator=(rpc_recv_typed_context &&o) noexcept { 33 | if (this != &o) { 34 | this->~rpc_recv_typed_context(); 35 | new (this) rpc_recv_typed_context(std::move(o)); 36 | } 37 | return *this; 38 | } 39 | 40 | SMF_ALWAYS_INLINE T *operator->() { return cache_; } 41 | SMF_ALWAYS_INLINE T * 42 | get() const { 43 | return cache_; 44 | } 45 | 46 | seastar::lw_shared_ptr> 47 | move_to_lw_shared() { 48 | return seastar::make_lw_shared>( 49 | std::move(ctx.value())); 50 | } 51 | 52 | /// \brief uses the flatbuffers verifier for payload 53 | /// can be expensive depending on type - traverses the full type tree 54 | bool 55 | verify_fbs() const { 56 | if (!this->operator bool()) { return false; } 57 | auto &buf = ctx.value().payload; 58 | flatbuffers::Verifier verifier((const uint8_t *)buf.get(), buf.size()); 59 | return get()->Verify(verifier); 60 | } 61 | /// \brief returns true of this obj 62 | /// so that it works with 63 | /// \code 64 | /// if(obj){} 65 | /// \endcode 66 | /// simply forward the bool operator to the option 67 | inline operator bool() const { return ctx.operator bool(); } 68 | opt_recv_ctx_t ctx; 69 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_recv_typed_context); 70 | 71 | private: 72 | T *cache_ = nullptr; 73 | }; 74 | } // namespace smf 75 | -------------------------------------------------------------------------------- /src/include/smf/rpc_server.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "smf/histogram.h" 16 | #include "smf/macros.h" 17 | #include "smf/rpc_connection_limits.h" 18 | #include "smf/rpc_filter.h" 19 | #include "smf/rpc_handle_router.h" 20 | #include "smf/rpc_server_args.h" 21 | #include "smf/rpc_server_connection.h" 22 | #include "smf/rpc_server_stats.h" 23 | #include "smf/zstd_filter.h" 24 | 25 | namespace smf { 26 | 27 | class rpc_server { 28 | 29 | /// \brief filter type to process data *before* it hits main handle 30 | using in_filter_t = 31 | std::function(rpc_recv_context)>; 32 | 33 | /// \brief filter type for sending data back out to clients 34 | using out_filter_t = 35 | std::function(rpc_envelope)>; 36 | 37 | public: 38 | explicit rpc_server(rpc_server_args args); 39 | ~rpc_server(); 40 | 41 | void start(); 42 | 43 | seastar::future<> stop(); 44 | 45 | /// \brief copy histogram. Cannot be made const due to seastar::map_reduce 46 | /// const-ness bugs 47 | seastar::future> copy_histogram(); 48 | 49 | template 50 | void 51 | register_service(Args &&... args) { 52 | static_assert(std::is_base_of::value, 53 | "register_service can only be called with a derived class of " 54 | "smf::rpc_service"); 55 | routes_.register_service(std::make_unique(std::forward(args)...)); 56 | } 57 | template 58 | void 59 | register_incoming_filter(Args &&... args) { 60 | in_filters_.push_back(Function(std::forward(args)...)); 61 | } 62 | 63 | template 64 | void 65 | register_outgoing_filter(Args &&... args) { 66 | out_filters_.push_back(Function(std::forward(args)...)); 67 | } 68 | 69 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_server); 70 | 71 | seastar::future apply_incoming_filters(rpc_recv_context); 72 | seastar::future apply_outgoing_filters(rpc_envelope); 73 | 74 | private: 75 | seastar::future<> 76 | handle_client_connection(seastar::lw_shared_ptr conn); 77 | 78 | seastar::future<> 79 | handle_one_client_session(seastar::lw_shared_ptr conn); 80 | 81 | seastar::future<> 82 | dispatch_rpc(int32_t payload_size, 83 | seastar::lw_shared_ptr conn, 84 | smf::compat::optional ctx); 85 | 86 | /// \brief main difference between dispatch_rpc and do_dispatch_rpc 87 | /// is that the former just wraps the calls in a safe seastar::gate 88 | seastar::future<> 89 | do_dispatch_rpc(seastar::lw_shared_ptr conn, 90 | rpc_recv_context &&ctx); 91 | 92 | seastar::future<> 93 | cleanup_dispatch_rpc(seastar::lw_shared_ptr conn); 94 | 95 | // SEDA piplines 96 | seastar::future 97 | stage_apply_incoming_filters(rpc_recv_context); 98 | 99 | seastar::future stage_apply_outgoing_filters(rpc_envelope); 100 | 101 | private: 102 | const rpc_server_args args_; 103 | seastar::lw_shared_ptr limits_; 104 | // -- the actual requests go through these 105 | rpc_handle_router routes_; 106 | std::vector in_filters_; 107 | std::vector out_filters_; 108 | // -- http & rpc sockets 109 | seastar::lw_shared_ptr listener_; 110 | seastar::lw_shared_ptr admin_ = nullptr; 111 | // connection counting happens in different future 112 | // must survive this instance 113 | seastar::lw_shared_ptr stats_ = 114 | seastar::make_lw_shared(); 115 | 116 | /// \brief keeps latency measurements per request flow 117 | seastar::lw_shared_ptr hist_ = histogram::make_lw_shared(); 118 | 119 | // this is needed for shutdown procedures 120 | uint64_t connection_idx_{0}; 121 | std::unordered_map> 122 | open_connections_; 123 | 124 | /// \brief set once keep_listening has exited 125 | seastar::promise<> stopped_; 126 | /// \brief keeps server alive until all continuations have finished 127 | seastar::gate reply_gate_; 128 | /// \brief prometheus metrics exposed 129 | seastar::metrics::metric_groups metrics_{}; 130 | 131 | private: 132 | friend std::ostream &operator<<(std::ostream &, const smf::rpc_server &); 133 | }; 134 | 135 | } // namespace smf 136 | -------------------------------------------------------------------------------- /src/include/smf/rpc_server_args.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | namespace smf { 12 | enum rpc_server_flags : uint32_t { rpc_server_flags_disable_http_server = 1 }; 13 | 14 | struct rpc_server_args { 15 | seastar::sstring ip = ""; 16 | uint16_t rpc_port = 11225; 17 | uint16_t http_port = 33140; 18 | 19 | /// \brief rpc_server_flags are bitwise flags. 20 | /// 21 | uint32_t flags = 0; 22 | 23 | /// \ brief The default timeout PER connection body. After we parse the 24 | /// header of the connection we need to make sure that we at some point 25 | /// receive some bytes or expire the connection. 26 | /// 27 | typename seastar::timer<>::duration recv_timeout = std::chrono::minutes(1); 28 | /// \brief 4GB usually. After this limit, each connection to this 29 | /// server-core will block until there are enough bytes free in memory to 30 | /// continue 31 | /// 32 | uint64_t memory_avail_per_core = uint64_t(1) << 31 /*2GB per core*/; 33 | }; 34 | 35 | } // namespace smf 36 | -------------------------------------------------------------------------------- /src/include/smf/rpc_server_connection.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | // std 5 | #include 6 | // seastar 7 | #include 8 | // smf 9 | #include "smf/log.h" 10 | #include "smf/rpc_connection.h" 11 | #include "smf/rpc_server_stats.h" 12 | #include "smf/std-compat.h" 13 | namespace smf { 14 | struct rpc_server_connection_options { 15 | explicit rpc_server_connection_options(bool _nodelay = false, 16 | bool _keepalive = false) 17 | : nodelay(_nodelay), enable_keepalive(_keepalive) {} 18 | rpc_server_connection_options(rpc_server_connection_options &&o) noexcept 19 | : nodelay(o.nodelay), enable_keepalive(o.enable_keepalive), 20 | keepalive(std::move(o.keepalive)) {} 21 | 22 | const bool nodelay; 23 | const bool enable_keepalive; 24 | 25 | // struct tcp_keepalive_params { 26 | // TCP_KEEPIDLE (since Linux 2.4) 27 | // The time (in seconds) the connection needs to remain idle before 28 | // TCP starts sending keepalive probes, if the socket option 29 | // SO_KEEPALIVE has been set on this socket. This option should 30 | // not be used in code intended to be portable. 31 | // 32 | // std::chrono::seconds idle; 33 | // 34 | // TCP_KEEPINTVL (since Linux 2.4) 35 | // The time (in seconds) between individual keepalive probes. This 36 | // option should not be used in code intended to be portable 37 | // 38 | // std::chrono::seconds interval; 39 | // 40 | // 41 | // TCP_KEEPCNT (since Linux 2.4) 42 | // The maximum number of keepalive probes TCP should send before 43 | // dropping the connection. This option should not be used in code 44 | // intended to be portable. 45 | // 46 | // unsigned count; 47 | // }; 48 | //// see linux sctp(7) for parameter explanation - not yet supported 49 | // struct sctp_keepalive_params { 50 | // std::chrono::seconds interval; // spp_hbinterval 51 | // unsigned count; // spp_pathmaxrt 52 | // }; 53 | // 54 | // 55 | // keepalive_params=boost::variant 56 | // 57 | seastar::net::keepalive_params keepalive{seastar::net::tcp_keepalive_params{ 58 | std::chrono::seconds(5), /*Idle secs, Looks like a good default?*/ 59 | std::chrono::seconds(75), /*Interval secs, Linux default*/ 60 | 9 /*Probs count, Linux default*/}}; 61 | }; 62 | 63 | class rpc_server_connection final { 64 | public: 65 | rpc_server_connection( 66 | seastar::connected_socket sock, 67 | seastar::lw_shared_ptr conn_limits, 68 | seastar::socket_address address, 69 | seastar::lw_shared_ptr _stats, uint64_t connection_id, 70 | rpc_server_connection_options opts = rpc_server_connection_options(true, 71 | true)) 72 | : conn(std::move(sock), address, conn_limits), id(connection_id), 73 | stats(_stats), opts_(std::move(opts)) { 74 | // TODO(agallego) - maybe set to true? 75 | conn.socket.set_nodelay(opts_.nodelay); 76 | if (opts_.enable_keepalive) { 77 | conn.socket.set_keepalive(true); 78 | conn.socket.set_keepalive_parameters(opts_.keepalive); 79 | } 80 | stats->active_connections++; 81 | stats->total_connections++; 82 | } 83 | 84 | ~rpc_server_connection() { stats->active_connections--; } 85 | 86 | SMF_ALWAYS_INLINE void 87 | set_error(seastar::sstring e) { 88 | conn.set_error(std::move(e)); 89 | } 90 | SMF_ALWAYS_INLINE bool 91 | has_error() const { 92 | return conn.has_error(); 93 | } 94 | SMF_ALWAYS_INLINE seastar::sstring 95 | get_error() const { 96 | return conn.get_error(); 97 | } 98 | 99 | SMF_ALWAYS_INLINE bool 100 | is_valid() { 101 | return conn.is_valid(); 102 | } 103 | SMF_ALWAYS_INLINE seastar::lw_shared_ptr 104 | limits() { 105 | return conn.limits; 106 | } 107 | 108 | rpc_connection conn; 109 | const uint64_t id; 110 | seastar::lw_shared_ptr stats; 111 | seastar::semaphore serialize_writes{1}; 112 | 113 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_server_connection); 114 | 115 | private: 116 | rpc_server_connection_options opts_; 117 | }; 118 | } // namespace smf 119 | -------------------------------------------------------------------------------- /src/include/smf/rpc_server_stats.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #pragma once 5 | 6 | namespace smf { 7 | 8 | // NEEDED on their *own* header 9 | // to decouple rpc_server.h from rpc_server_connection.h 10 | struct rpc_server_stats { 11 | uint64_t active_connections{}; 12 | uint64_t total_connections{}; 13 | uint64_t in_bytes{}; 14 | uint64_t out_bytes{}; 15 | uint64_t bad_requests{}; 16 | uint64_t no_route_requests{}; 17 | uint64_t completed_requests{}; 18 | uint64_t too_large_requests{}; 19 | }; 20 | 21 | } // namespace smf 22 | -------------------------------------------------------------------------------- /src/include/smf/rpc_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | 7 | #include "smf/rpc_envelope.h" 8 | #include "smf/rpc_recv_context.h" 9 | 10 | namespace smf { 11 | // https://github.com/grpc/grpc/blob/d0fbba52d6e379b76a69016bc264b96a2318315f/include/grpc%2B%2B/impl/codegen/rpc_method.h 12 | struct rpc_service_method_handle { 13 | // TODO(agallego) - expose through generator 14 | // now it the default is server_streaming 15 | enum rpc_type { 16 | NORMAL_RPC = 0, 17 | CLIENT_STREAMING, // request streaming 18 | SERVER_STREAMING, // response streaming 19 | BIDI_STREAMING 20 | }; 21 | 22 | using fn_t = seastar::noncopyable_function( 23 | rpc_recv_context &&recv)>; 24 | 25 | rpc_service_method_handle(fn_t &&f) : apply(std::move(f)) {} 26 | ~rpc_service_method_handle() = default; 27 | 28 | fn_t apply; 29 | }; 30 | 31 | struct rpc_service { 32 | virtual const char *service_name() const = 0; 33 | virtual uint32_t service_id() const = 0; 34 | virtual rpc_service_method_handle *method_for_request_id(uint32_t idx) = 0; 35 | virtual std::ostream &print(std::ostream &) const = 0; 36 | virtual ~rpc_service() {} 37 | rpc_service() {} 38 | }; 39 | } // namespace smf 40 | 41 | // namespace std { 42 | // ostream &operator<<(ostream &o, const smf::rpc_service *s); 43 | // } // namespace std 44 | -------------------------------------------------------------------------------- /src/include/smf/rpc_typed_envelope.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include "smf/macros.h" 6 | #include "smf/native_type_utils.h" 7 | #include "smf/rpc_envelope.h" 8 | #include "smf/rpc_header_utils.h" 9 | 10 | namespace smf { 11 | template 12 | struct rpc_typed_envelope { 13 | using type = RootType; 14 | using native_type = typename RootType::NativeTableType; 15 | 16 | rpc_envelope envelope; 17 | std::unique_ptr data; 18 | 19 | rpc_typed_envelope() : data(std::make_unique()) {} 20 | ~rpc_typed_envelope() {} 21 | explicit rpc_typed_envelope(std::unique_ptr &&ptr) noexcept 22 | : data(std::move(ptr)) {} 23 | rpc_typed_envelope & 24 | operator=(rpc_typed_envelope &&te) noexcept { 25 | envelope = std::move(te.envelope); 26 | data = std::move(te.data); 27 | return *this; 28 | } 29 | rpc_typed_envelope(rpc_typed_envelope &&te) noexcept { 30 | *this = std::move(te); 31 | } 32 | /// \brief this copies this->data into this->envelope 33 | /// and retuns a *moved* copy of the envelope. That is 34 | /// the envelope will be invalid after this method call 35 | rpc_envelope && 36 | serialize_data() { 37 | envelope.letter.body = 38 | std::move(smf::native_table_as_buffer(*(data.get()))); 39 | smf::checksum_rpc(envelope.letter.header, envelope.letter.body.get(), 40 | envelope.letter.body.size()); 41 | data = nullptr; 42 | return std::move(envelope); 43 | } 44 | /// \brief copy ctor deleted 45 | SMF_DISALLOW_COPY_AND_ASSIGN(rpc_typed_envelope); 46 | }; 47 | } // namespace smf 48 | -------------------------------------------------------------------------------- /src/include/smf/std-compat.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Alexander Gallego 2 | // 3 | #pragma once 4 | 5 | #include 6 | 7 | namespace smf { 8 | namespace compat = seastar::compat; 9 | } // namespace smf 10 | -------------------------------------------------------------------------------- /src/include/smf/time_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | namespace smf { 10 | inline uint64_t 11 | time_now_millis() { 12 | return std::chrono::duration_cast( 13 | std::chrono::system_clock::now().time_since_epoch()) 14 | .count(); 15 | } 16 | 17 | inline uint64_t 18 | time_now_micros() { 19 | return std::chrono::duration_cast( 20 | std::chrono::high_resolution_clock::now().time_since_epoch()) 21 | .count(); 22 | } 23 | 24 | inline uint64_t 25 | lowres_time_now_millis() { 26 | return std::chrono::duration_cast( 27 | seastar::lowres_clock::now().time_since_epoch()) 28 | .count(); 29 | } 30 | // Extracted out of gmock 31 | // Converts the given epoch time in milliseconds to a date string in the ISO 32 | // 8601 format, without the timezone information. 33 | inline seastar::sstring 34 | time_as_iso_8601(uint64_t millisecs) { 35 | // Using non-reentrant version as localtime_r is not portable. 36 | time_t seconds = static_cast(millisecs / 1000); 37 | const struct tm *const time_struct = localtime(&seconds); // NOLINT 38 | if (time_struct == NULL) { 39 | return ""; // Invalid ms value 40 | } 41 | // YYYY-MM-DDThh:mm:ss 42 | return seastar::to_sstring(time_struct->tm_year + 1900) + "-" + 43 | seastar::to_sstring(time_struct->tm_mon + 1) + "-" + 44 | seastar::to_sstring(time_struct->tm_mday) + "T" + 45 | seastar::to_sstring(time_struct->tm_hour) + ":" + 46 | seastar::to_sstring(time_struct->tm_min) + ":" + 47 | seastar::to_sstring(time_struct->tm_sec); 48 | } 49 | } // namespace smf 50 | -------------------------------------------------------------------------------- /src/include/smf/unique_histogram_adder.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | #pragma once 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | 12 | namespace smf { 13 | class unique_histogram_adder { 14 | private: 15 | std::unique_ptr result_ = smf::histogram::make_unique(); 16 | 17 | public: 18 | seastar::future<> 19 | operator()(std::unique_ptr value) { 20 | if (value) { *result_ += *value; } 21 | return seastar::make_ready_future<>(); 22 | } 23 | seastar::future<> 24 | operator()(const smf::histogram *value) { 25 | if (value) { *result_ += *value; } 26 | return seastar::make_ready_future<>(); 27 | } 28 | std::unique_ptr 29 | get() && { 30 | return std::move(result_); 31 | } 32 | }; 33 | 34 | } // namespace smf 35 | -------------------------------------------------------------------------------- /src/include/smf/zstd_filter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | // smf 5 | #include "smf/rpc_envelope.h" 6 | #include "smf/rpc_filter.h" 7 | #include "smf/rpc_recv_context.h" 8 | 9 | namespace smf { 10 | 11 | struct zstd_compression_filter : rpc_filter { 12 | explicit zstd_compression_filter(uint32_t _min_compression_size) 13 | : min_compression_size(_min_compression_size) {} 14 | 15 | seastar::future operator()(rpc_envelope &&e); 16 | 17 | const uint32_t min_compression_size; 18 | }; 19 | 20 | struct zstd_decompression_filter : rpc_filter { 21 | seastar::future operator()(rpc_recv_context &&ctx); 22 | }; 23 | 24 | } // namespace smf 25 | -------------------------------------------------------------------------------- /src/integration_tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | smfc_gen( 2 | CPP 3 | TARGET_NAME demo_test_fbs 4 | OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 5 | SOURCES ${PROJECT_SOURCE_DIR}/demo_apps/demo_service.fbs) 6 | 7 | set(IT_ROOT ${PROJECT_SOURCE_DIR}/src/integration_tests) 8 | 9 | smf_test( 10 | INTEGRATION_TEST 11 | BINARY_NAME histograms 12 | SOURCES ${IT_ROOT}/histograms/main.cc 13 | SOURCE_DIRECTORY ${IT_ROOT}/histograms 14 | INCLUDES ${PROJECT_SOURCE_DIR}/src 15 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 16 | LIBRARIES smf 17 | ) 18 | smf_test( 19 | INTEGRATION_TEST 20 | BINARY_NAME rpc 21 | SOURCES ${IT_ROOT}/rpc/main.cc ${demo_test_fbs} 22 | SOURCE_DIRECTORY ${IT_ROOT}/rpc 23 | INCLUDES ${PROJECT_SOURCE_DIR}/src 24 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 25 | LIBRARIES smf 26 | ) 27 | smf_test( 28 | INTEGRATION_TEST 29 | BINARY_NAME rpc_recv_timeout 30 | SOURCES ${IT_ROOT}/rpc_recv_timeout/main.cc ${demo_test_fbs} 31 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_recv_timeout 32 | INCLUDES ${PROJECT_SOURCE_DIR}/src 33 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 34 | LIBRARIES smf 35 | ) 36 | smf_test( 37 | INTEGRATION_TEST 38 | BINARY_NAME rpc_multiple_remote_addrs 39 | SOURCES ${IT_ROOT}/rpc_multiple_remote_addrs/main.cc ${demo_test_fbs} 40 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_multiple_remote_addrs 41 | INCLUDES ${PROJECT_SOURCE_DIR}/src 42 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 43 | LIBRARIES smf 44 | ) 45 | smf_test( 46 | INTEGRATION_TEST 47 | BINARY_NAME rpc_backpressure 48 | SOURCES ${IT_ROOT}/rpc_backpressure/main.cc ${demo_test_fbs} 49 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_backpressure 50 | INCLUDES ${PROJECT_SOURCE_DIR}/src 51 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 52 | LIBRARIES smf 53 | ) 54 | smf_test( 55 | INTEGRATION_TEST 56 | BINARY_NAME rpc_send_client_timeout 57 | SOURCES ${IT_ROOT}/rpc_send_timeout/main.cc ${demo_test_fbs} 58 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_send_timeout 59 | INCLUDES ${PROJECT_SOURCE_DIR}/src 60 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 61 | LIBRARIES smf 62 | ) 63 | smf_test( 64 | INTEGRATION_TEST 65 | BINARY_NAME rpc_reconnect_with_timeout 66 | SOURCES ${IT_ROOT}/rpc_reconnect_with_timeout/main.cc ${demo_test_fbs} 67 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_reconnect_with_timeout 68 | INCLUDES ${PROJECT_SOURCE_DIR}/src 69 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 70 | LIBRARIES smf 71 | ) 72 | smf_test( 73 | INTEGRATION_TEST 74 | BINARY_NAME rpc_hystrix_basic 75 | SOURCES ${IT_ROOT}/hystrix/main.cc ${demo_test_fbs} 76 | SOURCE_DIRECTORY ${IT_ROOT}/hystrix 77 | INCLUDES ${PROJECT_SOURCE_DIR}/src 78 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR}/.. 79 | LIBRARIES smf 80 | ) 81 | 82 | 83 | add_subdirectory(rpc_bad_msg_t) 84 | -------------------------------------------------------------------------------- /src/integration_tests/demo_service.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Alexander Gallego 2 | // 3 | 4 | namespace smf_gen.demo; 5 | 6 | table Request { 7 | name: string; 8 | } 9 | 10 | table Response { 11 | name: string; 12 | } 13 | 14 | 15 | rpc_service SmfStorage { 16 | Get(Request):Response; 17 | } 18 | -------------------------------------------------------------------------------- /src/integration_tests/histograms/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include 4 | // seastar 5 | #include 6 | #include 7 | // smf 8 | #include "smf/histogram_seastar_utils.h" 9 | #include "smf/log.h" 10 | 11 | int 12 | main(int args, char **argv, char **env) { 13 | LOG_DEBUG("Starting test for histogram write"); 14 | seastar::app_template app; 15 | try { 16 | return app.run(args, argv, [&app]() -> seastar::future { 17 | DLOG_TRACE("Test debug trace"); 18 | LOG_TRACE("Test non-debug trace"); 19 | auto h = smf::histogram::make_lw_shared(); 20 | for (auto i = 0u; i < 1000; i++) { 21 | h->record(i * i); 22 | } 23 | LOG_DEBUG("Writing histogram"); 24 | return smf::histogram_seastar_utils::write("hist.testing.hgrm", h) 25 | .then([] { return seastar::make_ready_future(0); }); 26 | }); // app.run 27 | } catch (const std::exception &e) { 28 | std::cerr << "Fatal exception: " << e.what() << std::endl; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/integration_tests/histograms/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 1", "-m 1G"], 3 | "tmp_home": true 4 | } 5 | -------------------------------------------------------------------------------- /src/integration_tests/hystrix/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include // timer::arm/re-arm here 9 | #include 10 | #include 11 | #include 12 | 13 | #include "smf/reconnect_client.h" 14 | // generated-templates 15 | #include "integration_tests/demo_service.smf.fb.h" 16 | 17 | static seastar::future<> 18 | test() { 19 | smf::rpc_client_opts opts{}; 20 | opts.server_addr = seastar::ipv4_addr{"127.0.0.1", 7897}; 21 | auto client = seastar::make_shared< 22 | smf::reconnect_client>(std::move(opts)); 23 | return client->connect() 24 | .then([=] { 25 | LOG_THROW_IF(static_cast>( 26 | client->backoff()) != 1, 27 | "Must be 1 second backoff"); 28 | LOG_INFO("{}", client->backoff()); 29 | }) 30 | .then([client] { return client->stop().finally([client] {}); }); 31 | } 32 | 33 | int 34 | main(int args, char **argv, char **env) { 35 | std::cout.setf(std::ios::unitbuf); 36 | smf::app_run_log_level(seastar::log_level::trace); 37 | seastar::app_template app; 38 | return app.run(args, argv, [&] { 39 | return test().then([] { return seastar::make_ready_future(0); }); 40 | }); 41 | } 42 | -------------------------------------------------------------------------------- /src/integration_tests/non_root_port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | namespace smf { 5 | inline uint16_t 6 | non_root_port(uint16_t port) { 7 | if (port < 1024) return port + 1024; 8 | return port; 9 | } 10 | } // namespace smf 11 | -------------------------------------------------------------------------------- /src/integration_tests/rpc/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 2", 3 | "-m 2G", 4 | "--req-num 1", 5 | "--concurrency 2", 6 | "--ip 127.0.0.1"], 7 | "tmp_home": true 8 | } 9 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_backpressure/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 2", "-m 2G"], 3 | "tmp_home": true 4 | } 5 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_bad_msg_t/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(IT_ROOT ${PROJECT_SOURCE_DIR}/src/integration_tests) 2 | smfc_gen( 3 | CPP 4 | TARGET_NAME bad_msg_fbs 5 | OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 6 | SOURCES 7 | ${CMAKE_CURRENT_SOURCE_DIR}/bad_msg.fbs 8 | ${CMAKE_CURRENT_SOURCE_DIR}/bad_svc.fbs) 9 | smf_test( 10 | INTEGRATION_TEST 11 | BINARY_NAME rpc_bad_msg_t 12 | SOURCES main.cc ${bad_msg_fbs} 13 | SOURCE_DIRECTORY ${IT_ROOT}/rpc_bad_msg_t 14 | INCLUDES ${CMAKE_CURRENT_BINARY_DIR} 15 | INCLUDES ${PROJECT_SOURCE_DIR}/src 16 | LIBRARIES smf 17 | ) 18 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_bad_msg_t/bad_msg.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | namespace bad.msg; 5 | 6 | table ack {} 7 | table node { 8 | name: string; 9 | } 10 | table test { 11 | test_node: node; 12 | } 13 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_bad_msg_t/bad_svc.fbs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | include "bad_msg.fbs"; 5 | 6 | namespace bad.svc; 7 | 8 | rpc_service bad_svc { 9 | go(bad.msg.test):bad.msg.ack; 10 | } 11 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_bad_msg_t/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | // stdlib++ 5 | #include 6 | 7 | // smf headers 8 | #include 9 | #include 10 | 11 | // smfc generated headers 12 | #include "bad_svc.smf.fb.h" 13 | 14 | using test_msg_t = smf::rpc_typed_envelope; 15 | 16 | int 17 | main(int argc, char **argv) { 18 | test_msg_t test; 19 | // TEST that nested types work 20 | test.data->test_node = std::make_unique(); 21 | test.data->test_node->name = "TEST"; 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_multiple_remote_addrs/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 1"] 3 | } 4 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_reconnect_with_timeout/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | // std 4 | #include 5 | #include 6 | // seastar 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | // smf 13 | #include "integration_tests/non_root_port.h" 14 | #include "smf/log.h" 15 | #include "smf/random.h" 16 | #include "smf/rpc_generated.h" 17 | #include "smf/rpc_handle_router.h" 18 | #include "smf/rpc_recv_context.h" 19 | #include "smf/rpc_server.h" 20 | #include "smf/time_utils.h" 21 | // generated-templates 22 | #include "integration_tests/demo_service.smf.fb.h" 23 | 24 | using namespace std::chrono_literals; // NOLINT 25 | constexpr const uint32_t kCoreMemory = 1 << 20; 26 | constexpr const auto kMinRequestDurationOnServer = 100ms; 27 | 28 | class storage_service final : public smf_gen::demo::SmfStorage { 29 | int internal_counter = 0; 30 | 31 | seastar::future<> 32 | simple_state() { 33 | if (internal_counter++ == 0) { 34 | return seastar::sleep(kMinRequestDurationOnServer); 35 | } 36 | return seastar::make_ready_future<>(); 37 | } 38 | 39 | virtual seastar::future> 40 | Get(smf::rpc_recv_typed_context &&rec) final { 41 | return simple_state().then([this] { 42 | smf::rpc_typed_envelope data; 43 | // lazy to create a new .fbs file 44 | data.data->name = seastar::to_sstring(internal_counter); 45 | data.envelope.set_status(200); 46 | return seastar::make_ready_future< 47 | smf::rpc_typed_envelope>(std::move(data)); 48 | }); 49 | } 50 | }; 51 | 52 | static seastar::future<> 53 | backpressure_request(uint16_t port) { 54 | smf::rpc_client_opts opts{}; 55 | opts.server_addr = seastar::ipv4_addr{"127.0.0.1", port}; 56 | opts.recv_timeout = 20ms; 57 | auto client = 58 | seastar::make_shared(std::move(opts)); 59 | return client->connect() 60 | .then([=] { 61 | return seastar::do_for_each( 62 | boost::counting_iterator(0), 63 | boost::counting_iterator(3), [=](uint32_t counter) { 64 | LOG_INFO("client iteration: {}", counter); 65 | smf::random r; 66 | smf::rpc_typed_envelope req; 67 | req.data->name = r.next_alphanum(100); 68 | return client->Get(std::move(req)) 69 | .then([](auto r) { 70 | const char *n = r->name()->c_str(); 71 | int server_counter = std::stoi(n); 72 | LOG_THROW_IF(server_counter == 1, 73 | "The first try we are supposed to timeout"); 74 | LOG_INFO("Success. Reconnected and method returned. Server " 75 | "Iteration: {}", 76 | server_counter); 77 | }) 78 | .handle_exception( 79 | [counter, client](auto err) { return client->reconnect(); }); 80 | }); 81 | }) 82 | .then([client] { return client->stop().finally([client] {}); }); 83 | } 84 | 85 | int 86 | main(int args, char **argv, char **env) { 87 | seastar::distributed rpc; 88 | seastar::app_template app; 89 | smf::random rand; 90 | uint16_t random_port = 91 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 92 | return app.run(args, argv, [&]() -> seastar::future { 93 | seastar::engine().at_exit([&] { return rpc.stop(); }); 94 | smf::rpc_server_args sargs; 95 | sargs.ip = "127.0.0.1"; 96 | sargs.rpc_port = random_port; 97 | sargs.http_port = 98 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 99 | sargs.flags |= smf::rpc_server_flags::rpc_server_flags_disable_http_server; 100 | return rpc.start(sargs) 101 | .then([&rpc] { 102 | return rpc.invoke_on_all( 103 | &smf::rpc_server::register_service); 104 | }) 105 | .then([&rpc] { return rpc.invoke_on_all(&smf::rpc_server::start); }) 106 | .then([&] { return backpressure_request(random_port); }) 107 | .then([] { return seastar::make_ready_future(0); }); 108 | }); 109 | } 110 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_reconnect_with_timeout/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 2", "-m 2G"], 3 | "tmp_home": true 4 | } 5 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_recv_timeout/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | // std 4 | #include 5 | #include 6 | // seastar 7 | #include 8 | #include 9 | #include 10 | #include 11 | // smf 12 | #include "integration_tests/non_root_port.h" 13 | #include "smf/log.h" 14 | #include "smf/random.h" 15 | #include "smf/rpc_generated.h" 16 | #include "smf/rpc_handle_router.h" 17 | #include "smf/rpc_recv_context.h" 18 | #include "smf/rpc_server.h" 19 | // templates 20 | #include "integration_tests/demo_service.smf.fb.h" 21 | 22 | class storage_service final : public smf_gen::demo::SmfStorage { 23 | virtual seastar::future> 24 | Get(smf::rpc_recv_typed_context &&rec) final { 25 | LOG_THROW("SHOULD NOT REACH THIS W/ A TIMEOUT"); 26 | } 27 | }; 28 | 29 | int 30 | main(int args, char **argv, char **env) { 31 | DLOG_DEBUG("About to start the RPC test"); 32 | seastar::distributed rpc; 33 | 34 | seastar::app_template app; 35 | 36 | smf::random rand; 37 | uint16_t random_port = 38 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 39 | return app.run(args, argv, [&]() -> seastar::future { 40 | DLOG_DEBUG("Setting up at_exit hooks"); 41 | seastar::engine().at_exit([&] { return rpc.stop(); }); 42 | 43 | smf::random r; 44 | smf::rpc_server_args sargs; 45 | sargs.rpc_port = random_port; 46 | sargs.http_port = 47 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 48 | sargs.flags |= smf::rpc_server_flags::rpc_server_flags_disable_http_server; 49 | sargs.recv_timeout = std::chrono::milliseconds(1); // immediately 50 | return rpc.start(sargs) 51 | .then([&rpc] { 52 | return rpc.invoke_on_all( 53 | &smf::rpc_server::register_service); 54 | }) 55 | .then([&rpc] { 56 | DLOG_DEBUG("Invoking rpc start on all cores"); 57 | return rpc.invoke_on_all(&smf::rpc_server::start); 58 | }) 59 | .then([&random_port] { 60 | DLOG_DEBUG("Sening only header ane expecting timeout"); 61 | auto local = 62 | seastar::socket_address(sockaddr_in{AF_INET, INADDR_ANY, {0}}); 63 | 64 | return seastar::engine() 65 | .net() 66 | .connect(seastar::make_ipv4_address(random_port), local, 67 | seastar::transport::TCP) 68 | .then([local](auto skt) { 69 | auto conn = seastar::make_lw_shared( 70 | std::move(skt), local); 71 | 72 | uint32_t kHeaderSize = sizeof(smf::rpc::header); 73 | 74 | smf::rpc::header header; 75 | header.mutate_size(300); 76 | header.mutate_checksum(1234234); 77 | seastar::temporary_buffer header_buf(kHeaderSize); 78 | std::copy(reinterpret_cast(&header), 79 | reinterpret_cast(&header) + kHeaderSize, 80 | header_buf.get_write()); 81 | return conn->ostream.write(std::move(header_buf)) 82 | .then([conn] { return conn->ostream.flush(); }) 83 | .then([conn] { 84 | return seastar::sleep(std::chrono::milliseconds(10)); 85 | }) 86 | .finally([conn] {}); 87 | }); 88 | }) 89 | .then([] { 90 | DLOG_DEBUG("Exiting"); 91 | return seastar::make_ready_future(0); 92 | }); 93 | }); 94 | } 95 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_recv_timeout/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 1"] 3 | } 4 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_send_timeout/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | // std 4 | #include 5 | #include 6 | // seastar 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | // smf 13 | #include "integration_tests/non_root_port.h" 14 | #include "smf/log.h" 15 | #include "smf/random.h" 16 | #include "smf/rpc_generated.h" 17 | #include "smf/rpc_handle_router.h" 18 | #include "smf/rpc_recv_context.h" 19 | #include "smf/rpc_server.h" 20 | #include "smf/time_utils.h" 21 | // generated-templates 22 | #include "integration_tests/demo_service.smf.fb.h" 23 | 24 | using namespace std::chrono_literals; // NOLINT 25 | constexpr const uint32_t kCoreMemory = 1 << 20; 26 | constexpr const auto kMinRequestDurationOnServer = 100ms; 27 | 28 | class storage_service final : public smf_gen::demo::SmfStorage { 29 | virtual seastar::future> 30 | Get(smf::rpc_recv_typed_context &&rec) final { 31 | return seastar::sleep(1s).then([] { 32 | smf::rpc_typed_envelope data; 33 | data.data->name = seastar::to_sstring(smf::lowres_time_now_millis()); 34 | data.envelope.set_status(200); 35 | return seastar::make_ready_future< 36 | smf::rpc_typed_envelope>(std::move(data)); 37 | }); 38 | } 39 | }; 40 | 41 | static seastar::future<> 42 | backpressure_request(uint16_t port) { 43 | smf::rpc_client_opts opts{}; 44 | opts.server_addr = seastar::ipv4_addr{"127.0.0.1", port}; 45 | opts.recv_timeout = 20ms; 46 | auto client = 47 | seastar::make_shared(std::move(opts)); 48 | return client->connect() 49 | .then([=] { 50 | smf::random r; 51 | smf::rpc_typed_envelope req; 52 | req.data->name = r.next_alphanum(kCoreMemory); 53 | return client->Get(std::move(req)) 54 | .then([](auto _) { LOG_THROW("SHOULD HAVE TIMED OUT"); }) 55 | .handle_exception( 56 | [](auto err) { LOG_INFO("EXPECTED!!! Exception: {}", err); }); 57 | }) 58 | .then([client] { return client->stop().finally([client] {}); }); 59 | } 60 | 61 | int 62 | main(int args, char **argv, char **env) { 63 | seastar::distributed rpc; 64 | seastar::app_template app; 65 | smf::random rand; 66 | uint16_t random_port = 67 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 68 | return app.run(args, argv, [&]() -> seastar::future { 69 | seastar::engine().at_exit([&] { return rpc.stop(); }); 70 | smf::rpc_server_args sargs; 71 | sargs.ip = "127.0.0.1"; 72 | sargs.rpc_port = random_port; 73 | sargs.http_port = 74 | smf::non_root_port(rand.next() % std::numeric_limits::max()); 75 | sargs.flags |= smf::rpc_server_flags::rpc_server_flags_disable_http_server; 76 | 77 | // -- MAIN TEST -- 78 | sargs.memory_avail_per_core = 79 | static_cast(kCoreMemory + 200 /*some slack bytes*/); 80 | 81 | return rpc.start(sargs) 82 | .then([&rpc] { 83 | return rpc.invoke_on_all( 84 | &smf::rpc_server::register_service); 85 | }) 86 | .then([&rpc] { return rpc.invoke_on_all(&smf::rpc_server::start); }) 87 | .then([&] { return backpressure_request(random_port); }) 88 | .then([] { return seastar::make_ready_future(0); }); 89 | }); 90 | } 91 | -------------------------------------------------------------------------------- /src/integration_tests/rpc_send_timeout/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "args": ["-c 2", "-m 2G"], 3 | "tmp_home": true 4 | } 5 | -------------------------------------------------------------------------------- /src/smfc/.gitignore: -------------------------------------------------------------------------------- 1 | *.smf.* 2 | demo_service.bfbs 3 | demo_service_generated.h 4 | -------------------------------------------------------------------------------- /src/smfc/README.md: -------------------------------------------------------------------------------- 1 | # **smfc** is the code generator & compiler 2 | 3 | 4 | To build *ONLY* the compiler: 5 | 6 | ``` 7 | cd $ROOT 8 | mkdir build 9 | cd build 10 | cmake -DSMF_ENABLE_TESTS=OFF \ 11 | -DSMF_BUILD_PROGRAMS=OFF $ROOT 12 | make 13 | ``` 14 | -------------------------------------------------------------------------------- /src/smfc/codegen.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | 4 | #include "codegen.h" 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include "cpp_generator.h" 12 | #include "go_generator.h" 13 | #include "python_generator.h" 14 | 15 | namespace smf_gen { 16 | 17 | codegen::codegen(std::string ifname, std::string out_dir, 18 | std::vector includes, std::vector langs) 19 | : input_filename(std::move(ifname)), output_dir(std::move(out_dir)), 20 | languages(std::move(langs)), include_dirs_(std::move(includes)) { 21 | LOG_IF(FATAL, languages.empty()) << "No programming langues"; 22 | LOG_IF(FATAL, output_dir.empty()) << "No place to put generated code"; 23 | 24 | parser_ = std::make_unique(opts_); 25 | 26 | // make sure that the current file's dir is added to the include dirs 27 | // 28 | if (include_dirs_.end() == 29 | std::find_if(include_dirs_.begin(), include_dirs_.end(), 30 | [this](auto &s) { return s == input_filename; })) { 31 | include_dirs_.push_back(flatbuffers::StripFileName(input_filename)); 32 | } 33 | } 34 | 35 | std::size_t 36 | codegen::service_count() const { 37 | if (!parsed_) { 38 | LOG(ERROR) << "Generator not parsed, please call ::parse() first"; 39 | return 0; 40 | } 41 | return parser_->services_.vec.size(); 42 | } 43 | 44 | smf::compat::optional 45 | codegen::parse() { 46 | if (parsed_) { 47 | if (!parser_->error_.empty()) { 48 | return "Parser in error state: " + parser_->error_; 49 | } 50 | return smf::compat::nullopt; 51 | } 52 | parsed_ = true; 53 | 54 | // UGH!!! Flatbuffers C-looking API .... is... 55 | // 56 | std::vector cincludes; 57 | cincludes.reserve(include_dirs_.size() + 1); 58 | for (auto &s : include_dirs_) { 59 | cincludes.push_back(s.c_str()); 60 | } 61 | // always end in null :'( 62 | cincludes.push_back(nullptr); 63 | 64 | std::string contents; 65 | if (!flatbuffers::LoadFile(input_filename.c_str(), true, &contents)) { 66 | return "Could not load file: " + input_filename; 67 | } 68 | 69 | if (!parser_->Parse(contents.c_str(), cincludes.data(), 70 | input_filename.c_str())) { 71 | return "Could not PARSE file: " + parser_->error_; 72 | } 73 | 74 | return smf::compat::nullopt; 75 | } 76 | 77 | smf::compat::optional 78 | codegen::gen() { 79 | auto x = parse(); 80 | if (x) { return x; } 81 | for (const auto &l : languages) { 82 | switch (l) { 83 | case language::cpp: { 84 | VLOG(1) << "Adding cpp generator"; 85 | auto g = std::make_unique(*parser_.get(), input_filename, 86 | output_dir); 87 | x = g->gen(); 88 | if (x) { return x; } 89 | VLOG(1) << "Generated: " << g->output_filename(); 90 | break; 91 | } 92 | case language::go: { 93 | VLOG(1) << "Adding golang generator"; 94 | auto g = std::make_unique(*parser_.get(), input_filename, 95 | output_dir); 96 | x = g->gen(); 97 | if (x) { return x; } 98 | VLOG(1) << "Generated: " << g->output_filename(); 99 | break; 100 | } 101 | case language::python: { 102 | VLOG(1) << "Adding python generator"; 103 | auto g = std::make_unique(*parser_.get(), 104 | input_filename, output_dir); 105 | x = g->gen(); 106 | if (x) { return x; } 107 | VLOG(1) << "Generated: " << g->output_filename(); 108 | break; 109 | } 110 | default: 111 | LOG(ERROR) << "Uknown code generator"; 112 | } 113 | } 114 | return smf::compat::nullopt; 115 | } 116 | 117 | } // namespace smf_gen 118 | -------------------------------------------------------------------------------- /src/smfc/codegen.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | 12 | #include "smf/std-compat.h" 13 | #include "language.h" 14 | 15 | namespace smf_gen { 16 | 17 | class codegen { 18 | public: 19 | using status = smf::compat::optional; 20 | 21 | codegen(std::string ifname, std::string output_dir, 22 | std::vector include_dirs, std::vector langs); 23 | ~codegen() = default; 24 | 25 | status gen(); 26 | status parse(); 27 | std::size_t service_count() const; 28 | 29 | const std::string input_filename; 30 | const std::string output_dir; 31 | const std::vector languages; 32 | 33 | private: 34 | std::vector include_dirs_; 35 | flatbuffers::IDLOptions opts_; 36 | std::unique_ptr parser_; 37 | bool parsed_ = false; 38 | }; 39 | } // namespace smf_gen 40 | -------------------------------------------------------------------------------- /src/smfc/cpp_generator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "generator.h" 14 | #include "smf/std-compat.h" 15 | 16 | namespace smf_gen { 17 | 18 | class cpp_generator : public generator { 19 | public: 20 | cpp_generator(const flatbuffers::Parser &p, const std::string &ifname, 21 | const std::string &output_dir) 22 | : generator(p, ifname, output_dir) {} 23 | virtual ~cpp_generator() = default; 24 | 25 | virtual std::string 26 | output_filename() final { 27 | return output_dir + "/" + input_filename_without_ext() + ".smf.fb.h"; 28 | } 29 | 30 | virtual smf::compat::optional 31 | gen() final { 32 | generate_header_prologue(); 33 | generate_header_prologue_includes(); 34 | 35 | generate_header_prologue_namespace(); 36 | generate_header_prologue_forward_decl(); 37 | generate_header_epilogue_namespace(); 38 | 39 | generate_header_prologue_forward_decl_external(); 40 | 41 | generate_header_prologue_namespace(); 42 | generate_header_services(); 43 | 44 | generate_header_epilogue_namespace(); 45 | generate_header_epilogue(); 46 | 47 | return save_conents_to_file(); 48 | } 49 | 50 | std::string 51 | message_header_ext() const { 52 | return "_generated.h"; 53 | } 54 | 55 | private: 56 | void generate_header_prologue(); 57 | void generate_header_prologue_includes(); 58 | void generate_header_prologue_forward_decl(); 59 | void generate_header_prologue_forward_decl_external(); 60 | void generate_header_prologue_namespace(); 61 | void generate_header_services(); 62 | void generate_header_epilogue(); 63 | void generate_header_epilogue_namespace(); 64 | }; 65 | 66 | } // namespace smf_gen 67 | -------------------------------------------------------------------------------- /src/smfc/crc.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #pragma once 5 | #include 6 | namespace smf_gen { 7 | inline uint32_t 8 | crc32(const char *data, const size_t &length) { 9 | boost::crc_32_type result; 10 | result.process_bytes(data, length); 11 | return result.checksum(); 12 | } 13 | } // namespace smf_gen 14 | -------------------------------------------------------------------------------- /src/smfc/generator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #pragma once 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "smf_printer.h" 14 | #include "smf_service.h" 15 | #include "smf/std-compat.h" 16 | 17 | namespace smf_gen { 18 | class generator { 19 | public: 20 | generator(const flatbuffers::Parser &p, const std::string &ifname, 21 | const std::string &out_dir) 22 | : parser(p), input_filename(ifname), output_dir(out_dir) { 23 | for (const auto *s : parser.services_.vec) { 24 | if (s->generated) { 25 | // True for transitive flatbuffer files. 26 | // If you have a.fbs include b.fbs, then the services on b.fbs 27 | // will be marked as generated 28 | continue; 29 | } 30 | services_.emplace_back(std::make_unique(s)); 31 | } 32 | } 33 | virtual ~generator() = default; 34 | 35 | virtual std::string output_filename() = 0; 36 | virtual smf::compat::optional gen() = 0; 37 | 38 | virtual const std::string & 39 | contents() final { 40 | return printer_.contents(); 41 | } 42 | 43 | const flatbuffers::Parser &parser; 44 | const std::string &input_filename; 45 | const std::string &output_dir; 46 | 47 | virtual const std::map> & 48 | fbs_files_included_per_file() const final { 49 | return parser.files_included_per_file_; 50 | } 51 | virtual const std::vector & 52 | native_included_files() const final { 53 | return parser.native_included_files_; 54 | } 55 | virtual const std::map & 56 | included_files() const final { 57 | return parser.included_files_; 58 | }; 59 | 60 | virtual const std::vector> & 61 | services() final { 62 | return services_; 63 | } 64 | 65 | virtual std::string 66 | package() const final { 67 | return parser.current_namespace_->GetFullyQualifiedName(""); 68 | } 69 | 70 | virtual std::vector 71 | package_parts() const final { 72 | return parser.current_namespace_->components; 73 | } 74 | 75 | virtual std::string 76 | input_filename_without_path() const final { 77 | return flatbuffers::StripPath(input_filename); 78 | } 79 | 80 | virtual std::string 81 | input_filename_without_ext() const final { 82 | return flatbuffers::StripExtension(input_filename_without_path()); 83 | } 84 | 85 | virtual smf::compat::optional 86 | save_conents_to_file() { 87 | auto outname = output_filename(); 88 | if (boost::filesystem::exists(outname)) { 89 | boost::filesystem::remove(outname); 90 | } 91 | if (!flatbuffers::SaveFile(outname.c_str(), printer_.contents(), false)) { 92 | return "Could not create filename: " + outname; 93 | } 94 | return smf::compat::nullopt; 95 | } 96 | 97 | protected: 98 | smf_printer printer_; 99 | 100 | private: 101 | std::vector> services_; 102 | }; 103 | } // namespace smf_gen 104 | -------------------------------------------------------------------------------- /src/smfc/go_generator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #pragma once 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include "generator.h" 12 | #include "smf/std-compat.h" 13 | 14 | namespace smf_gen { 15 | 16 | class go_generator : public generator { 17 | public: 18 | go_generator(const flatbuffers::Parser &p, const std::string &ifname, 19 | const std::string &output_dir) 20 | : generator(p, ifname, output_dir) { 21 | // go uses tabs 22 | printer_.set_indent_char('\t'); 23 | printer_.set_indent_step(1); 24 | } 25 | virtual ~go_generator() = default; 26 | 27 | virtual std::string 28 | output_filename() final { 29 | std::stringstream str; 30 | str << output_dir << "/"; 31 | for (auto i = 0u; i < package_parts().size(); ++i) { 32 | str << package_parts()[i] << "/"; 33 | } 34 | str << input_filename_without_ext() + ".smf.fb.go"; 35 | return str.str(); 36 | } 37 | 38 | virtual smf::compat::optional 39 | gen() final { 40 | generate_header_prologue(); 41 | generate_header_includes(); 42 | generate_header_services(); 43 | // for Go make sure that the directories exist 44 | boost::filesystem::create_directories( 45 | boost::algorithm::join(package_parts(), "/")); 46 | return save_conents_to_file(); 47 | } 48 | 49 | private: 50 | void generate_header_prologue(); 51 | void generate_header_includes(); 52 | void generate_header_services(); 53 | }; 54 | 55 | } // namespace smf_gen 56 | -------------------------------------------------------------------------------- /src/smfc/language.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | #pragma once 4 | 5 | namespace smf_gen { 6 | enum class language { none, cpp, go, python, all }; 7 | } // namespace smf_gen 8 | -------------------------------------------------------------------------------- /src/smfc/main.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "codegen.h" 15 | #include "smf/std-compat.h" 16 | 17 | DEFINE_string(filename, "", "filename to parse"); 18 | DEFINE_string(include_dirs, "", "extra include directories"); 19 | DEFINE_string(language, "cpp", 20 | "coma separated list of language to generate: go, cpp"); 21 | DEFINE_string(output_path, ".", "output path of the generated files"); 22 | 23 | std::vector 24 | split_coma(const std::string &dirs) { 25 | std::vector retval; 26 | boost::algorithm::split(retval, dirs, boost::is_any_of(",")); 27 | if (retval.empty() && !dirs.empty()) { retval.push_back(dirs); } 28 | for (auto &s : retval) { 29 | boost::algorithm::trim(s); 30 | } 31 | return retval; 32 | } 33 | 34 | std::vector 35 | split_langs(const std::string &lang) { 36 | // Note: be sure to add the generator in codegen.cc 37 | std::vector retval; 38 | auto str_langs = split_coma(lang); 39 | for (auto &l : str_langs) { 40 | if (l == "cpp") { 41 | retval.push_back(smf_gen::language::cpp); 42 | } else if (l == "go") { 43 | retval.push_back(smf_gen::language::go); 44 | } else if (l == "python") { 45 | retval.push_back(smf_gen::language::python); 46 | } else { 47 | LOG(ERROR) << "Skipping unknown language: " << l; 48 | } 49 | } 50 | return retval; 51 | } 52 | 53 | int 54 | main(int argc, char **argv, char **env) { 55 | google::SetUsageMessage("Generate smf services"); 56 | google::ParseCommandLineFlags(&argc, &argv, true); 57 | google::InstallFailureSignalHandler(); 58 | google::InitGoogleLogging(argv[0]); 59 | 60 | // validate flags 61 | if (FLAGS_filename.empty()) { 62 | LOG(ERROR) << "No filename to parse"; 63 | std::exit(1); 64 | } 65 | if (!boost::filesystem::exists(FLAGS_filename)) { 66 | LOG(ERROR) << " ` " << FLAGS_filename 67 | << " ' - does not exists or could not be found"; 68 | std::exit(1); 69 | } 70 | if (FLAGS_output_path.empty()) { 71 | LOG(ERROR) << " ` " << FLAGS_output_path << " ' - empty output path"; 72 | std::exit(1); 73 | } 74 | FLAGS_output_path = 75 | boost::filesystem::canonical(FLAGS_output_path.c_str()).string(); 76 | if (!flatbuffers::DirExists(FLAGS_output_path.c_str())) { 77 | LOG(ERROR) << "--output_path specified, but directory: " 78 | << FLAGS_output_path << " does not exist;"; 79 | std::exit(1); 80 | } 81 | 82 | auto codegenerator = std::make_unique( 83 | FLAGS_filename, FLAGS_output_path, split_coma(FLAGS_include_dirs), 84 | split_langs(FLAGS_language)); 85 | // generate code! 86 | auto status = codegenerator->parse(); 87 | if (status) { 88 | LOG(ERROR) << "Failed to parse file: " << status.value(); 89 | std::exit(1); 90 | } 91 | if (codegenerator->service_count() == 0) { 92 | LOG(INFO) << "No services need to be generated"; 93 | // if we return 0, the cmake module cannot detect if we generate a file or 94 | // not and always calls smfc return 0; 95 | } 96 | status = codegenerator->gen(); 97 | if (status) { 98 | LOG(ERROR) << "Errors generating code: " << *status; 99 | std::exit(1); 100 | } 101 | VLOG(1) << "Success"; 102 | return 0; 103 | } 104 | -------------------------------------------------------------------------------- /src/smfc/python_generator.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | #include "python_generator.h" 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace smf_gen { 11 | 12 | static inline std::string 13 | python_public_name(std::string s) { 14 | s[0] = std::toupper(s[0]); 15 | return s; 16 | } 17 | 18 | static void 19 | print_client_method(smf_printer &printer, const smf_method *method) { 20 | std::map vars; 21 | vars["MethodName"] = python_public_name(method->name()); 22 | vars["MethodID"] = std::to_string(method->method_id()); 23 | vars["ServiceID"] = std::to_string(method->service_id()); 24 | vars["OutType"] = method->output_type_name(language::python); 25 | vars["RequestID"] = 26 | std::to_string(method->service_id() ^ method->method_id()); 27 | 28 | printer.print(vars, "async def $MethodName$(self, x):\n"); 29 | printer.indent(); 30 | printer.print(vars, "# request id = $ServiceID$ ^ $MethodID$\n"); 31 | printer.print(vars, "buf, status = await self._conn.call(x, $RequestID$)\n"); 32 | printer.print(vars, 33 | "return $OutType$.GetRootAsPutResponse(buf, 0), status\n"); 34 | printer.outdent(); 35 | } 36 | 37 | static void 38 | print_client(smf_printer &printer, const smf_service *service) { 39 | VLOG(1) << "print_client"; 40 | std::map vars; 41 | 42 | vars["InterfaceName"] = python_public_name(service->name()); 43 | vars["ClientName"] = vars["InterfaceName"] + "Client"; 44 | vars["ServiceID"] = std::to_string(service->service_id()); 45 | 46 | printer.print(vars, "class $ClientName$:\n"); 47 | printer.indent(); 48 | printer.print("def __init__(self, conn):\n"); 49 | printer.indent(); 50 | printer.print("self._conn = conn\n\n"); 51 | printer.outdent(); 52 | printer.outdent(); 53 | 54 | for (auto i = 0u; i < service->methods().size(); i++) { 55 | auto &method = service->methods()[i]; 56 | printer.indent(); 57 | print_client_method(printer, method.get()); 58 | printer.outdent(); 59 | if ((i + 1) < service->methods().size()) { printer.print("\n"); } 60 | } 61 | } 62 | 63 | void 64 | python_generator::generate_header_prologue() { 65 | VLOG(1) << "generate_header_prologue"; 66 | std::map vars; 67 | vars["filename"] = input_filename; 68 | printer_.print("# Generated by smfc.\n"); 69 | printer_.print("# Any local changes WILL BE LOST.\n"); 70 | printer_.print(vars, "# source: $filename$\n\n"); 71 | } 72 | 73 | void 74 | python_generator::generate_header_includes() { 75 | VLOG(1) << "generate_header_includes"; 76 | 77 | std::set imports; 78 | for (auto &service : services()) { 79 | for (auto &method : service->methods()) { 80 | auto type = method->output_type_name(language::python); 81 | auto point = type.rfind("."); 82 | LOG_IF(FATAL, point == std::string::npos) << "Invalid type"; 83 | imports.insert(type.substr(0, point)); 84 | } 85 | } 86 | 87 | for (auto &import : imports) { 88 | printer_.print("import "); 89 | printer_.print(import.c_str()); 90 | printer_.print("\n"); 91 | } 92 | printer_.print("\n"); 93 | } 94 | 95 | void 96 | python_generator::generate_header_services() { 97 | VLOG(1) << "Generating (" << services().size() << ") services"; 98 | for (auto &srv : services()) { 99 | print_client(printer_, srv.get()); 100 | } 101 | } 102 | 103 | } // namespace smf_gen 104 | -------------------------------------------------------------------------------- /src/smfc/python_generator.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include "generator.h" 11 | #include "smf/std-compat.h" 12 | 13 | namespace smf_gen { 14 | 15 | class python_generator final : public generator { 16 | public: 17 | python_generator(const flatbuffers::Parser &p, const std::string &ifname, 18 | const std::string &output_dir) 19 | : generator(p, ifname, output_dir) { 20 | printer_.set_indent_char(' '); 21 | printer_.set_indent_step(4); 22 | } 23 | 24 | virtual std::string 25 | output_filename() final { 26 | std::stringstream str; 27 | str << output_dir << "/"; 28 | for (auto i = 0u; i < package_parts().size(); ++i) { 29 | str << package_parts()[i] << "/"; 30 | } 31 | str << input_filename_without_ext() + "_smf_client.py"; 32 | return str.str(); 33 | } 34 | 35 | virtual smf::compat::optional 36 | gen() final { 37 | generate_header_prologue(); 38 | generate_header_includes(); 39 | generate_header_services(); 40 | // for Python make sure that the directories exist 41 | boost::filesystem::create_directories( 42 | boost::algorithm::join(package_parts(), "/")); 43 | return save_conents_to_file(); 44 | } 45 | 46 | private: 47 | void generate_header_prologue(); 48 | void generate_header_includes(); 49 | void generate_header_services(); 50 | }; 51 | 52 | } // namespace smf_gen 53 | -------------------------------------------------------------------------------- /src/smfc/smf_method.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include "crc.h" 11 | #include "language.h" 12 | 13 | namespace smf_gen { 14 | class smf_method { 15 | public: 16 | enum Streaming { kNone, kClient, kServer, kBiDi }; 17 | 18 | smf_method(const flatbuffers::RPCCall *method, std::string service_name, 19 | uint32_t service_id) 20 | : method_(method), service_name_(service_name), service_id_(service_id) { 21 | streaming_ = kNone; 22 | // you can have the same method name w/ different arguments, so in that 23 | // case you should change the hash id 24 | std::string method_id_str = 25 | method_->name + ":" + input_type_name() + ":" + output_type_name(); 26 | id_ = crc32(method_id_str.c_str(), method_id_str.length()); 27 | } 28 | std::string 29 | name() const { 30 | return method_->name; 31 | } 32 | uint32_t 33 | method_id() const { 34 | return id_; 35 | } 36 | 37 | std::string 38 | service_name() const { 39 | return service_name_; 40 | } 41 | uint32_t 42 | service_id() const { 43 | return service_id_; 44 | } 45 | 46 | std::string 47 | type(const flatbuffers::StructDef &sd, language lang) const { 48 | switch (lang) { 49 | case language::cpp: { 50 | std::deque tmp(sd.defined_namespace->components.begin(), 51 | sd.defined_namespace->components.end()); 52 | tmp.push_back(sd.name); 53 | return boost::algorithm::join(tmp, "::"); 54 | } 55 | case language::go: 56 | return sd.name; 57 | case language::python: { 58 | std::deque tmp(sd.defined_namespace->components.begin(), 59 | sd.defined_namespace->components.end()); 60 | tmp.push_back(sd.name); 61 | tmp.push_back(sd.name); // name is inside module 62 | return boost::algorithm::join(tmp, "."); 63 | } 64 | default: 65 | throw std::runtime_error("Unknown language for the method.h"); 66 | } 67 | } 68 | 69 | std::string 70 | input_type_name(language l = language::cpp) const { 71 | return type(*method_->request, l); 72 | } 73 | std::string 74 | output_type_name(language l = language::cpp) const { 75 | return type(*method_->response, l); 76 | } 77 | 78 | private: 79 | const flatbuffers::RPCCall *method_; 80 | const std::string service_name_; 81 | const uint32_t service_id_; 82 | Streaming streaming_; 83 | 84 | uint32_t id_; 85 | }; 86 | } // namespace smf_gen 87 | -------------------------------------------------------------------------------- /src/smfc/smf_printer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | namespace smf_gen { 8 | class smf_printer { 9 | public: 10 | explicit smf_printer(char escape = '$') : escape_char(escape) {} 11 | 12 | void 13 | print(const std::map &vars, 14 | const char *string_template) { 15 | std::string s = string_template; 16 | // Replace any occurrences of strings in "vars" that are surrounded 17 | // by the escape character by what they're mapped to. 18 | size_t pos; 19 | while ((pos = s.find(escape_char)) != std::string::npos) { 20 | // Found an escape char, must also find the closing one. 21 | size_t pos2 = s.find(escape_char, pos + 1); 22 | // If placeholder not closed, ignore. 23 | if (pos2 == std::string::npos) break; 24 | auto it = vars.find(s.substr(pos + 1, pos2 - pos - 1)); 25 | // If unknown placeholder, ignore. 26 | if (it == vars.end()) break; 27 | // Subtitute placeholder. 28 | s.replace(pos, pos2 - pos + 1, it->second); 29 | } 30 | print(s.c_str()); 31 | } 32 | 33 | void 34 | print(const char *s) { 35 | // Add this string, but for each part separated by \n, add indentation. 36 | for (;;) { 37 | // Current indentation. 38 | out_.insert(out_.end(), indent_ * indent_step_, indent_char_); 39 | // See if this contains more than one line. 40 | const char *lf = strchr(s, '\n'); 41 | if (lf) { 42 | out_ += std::string(s, lf + 1); 43 | s = lf + 1; 44 | if (!*s) break; // Only continue if there's more lines. 45 | } else { 46 | out_ += s; 47 | break; 48 | } 49 | } 50 | } 51 | 52 | void 53 | indent() { 54 | indent_++; 55 | } 56 | void 57 | outdent() { 58 | indent_--; 59 | assert(indent_ >= 0); 60 | } 61 | 62 | const char escape_char; 63 | 64 | const std::string & 65 | contents() const { 66 | return out_; 67 | } 68 | 69 | void 70 | set_indent_char(char i) { 71 | indent_char_ = i; 72 | } 73 | void 74 | set_indent_step(uint16_t step) { 75 | indent_step_ = step; 76 | } 77 | 78 | private: 79 | std::string out_; 80 | int32_t indent_{0}; 81 | char indent_char_ = ' '; 82 | uint16_t indent_step_ = 2; 83 | }; 84 | } // namespace smf_gen 85 | -------------------------------------------------------------------------------- /src/smfc/smf_service.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Alexander Gallego. All rights reserved. 2 | // 3 | #pragma once 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "crc.h" 10 | #include "smf_method.h" 11 | 12 | namespace smf_gen { 13 | class smf_service { 14 | public: 15 | explicit smf_service(const flatbuffers::ServiceDef *service) 16 | : service_(service) { 17 | id_ = crc32(service_->name.c_str(), service_->name.length()); 18 | 19 | // populate methods 20 | for (auto i = 0u; i < service_->calls.vec.size(); ++i) { 21 | methods_.emplace_back(std::make_unique(service_->calls.vec[i], 22 | name(), service_id())); 23 | } 24 | } 25 | 26 | std::string 27 | name() const { 28 | return service_->name; 29 | } 30 | 31 | // hash 32 | uint32_t 33 | service_id() const { 34 | return id_; 35 | } 36 | 37 | const std::vector> & 38 | methods() const { 39 | return methods_; 40 | } 41 | 42 | const flatbuffers::ServiceDef * 43 | raw_service() const { 44 | return service_; 45 | }; 46 | 47 | private: 48 | const flatbuffers::ServiceDef *service_; 49 | uint32_t id_{0}; 50 | std::vector> methods_; 51 | }; 52 | } // namespace smf_gen 53 | -------------------------------------------------------------------------------- /src/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package (GTest REQUIRED CONFIG) 2 | 3 | set(TOOR ${PROJECT_SOURCE_DIR}/src/tests) 4 | smf_test( 5 | UNIT_TEST 6 | BINARY_NAME randomstr 7 | SOURCES ${TOOR}/randomstr.cc 8 | SOURCE_DIRECTORY ${TOOR} 9 | LIBRARIES smf GTest::gtest 10 | ) 11 | smf_test( 12 | UNIT_TEST 13 | BINARY_NAME simple_hist 14 | SOURCES ${TOOR}/histogram_tests.cc 15 | SOURCE_DIRECTORY ${TOOR} 16 | LIBRARIES smf GTest::gtest 17 | ) 18 | 19 | add_executable(smf_histgen histgen.cc) 20 | target_link_libraries(smf_histgen smf) 21 | target_include_directories(smf_histgen 22 | PUBLIC ${CMAKE_CURRENT_BINARY_DIR} 23 | PUBLIC ${PROJECT_SOURCE_DIR}/src/include 24 | ) 25 | -------------------------------------------------------------------------------- /src/tests/histgen.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SMF Authors 2 | // 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "smf/histogram.h" 15 | 16 | struct histgen_server_args { 17 | seastar::sstring ip = ""; 18 | uint16_t port = 33140; 19 | uint64_t min_val; 20 | uint64_t max_val; 21 | uint32_t num_samples; 22 | }; 23 | 24 | class histgen_server { 25 | public: 26 | histgen_server(histgen_server_args args) 27 | : args(args), hist(smf::histogram::make_lw_shared()), 28 | http_server("histgen server") { 29 | namespace sm = seastar::metrics; 30 | metrics.add_group( 31 | "histgen_server", 32 | { 33 | sm::make_histogram( 34 | "synthetic_histogram", sm::description("Synthetic histogram data"), 35 | [this] { return hist->seastar_histogram_logform(); }), 36 | }); 37 | } 38 | 39 | ~histgen_server() {} 40 | 41 | seastar::future<> 42 | serve() { 43 | std::random_device rd; 44 | std::mt19937 gen(rd()); 45 | std::uniform_int_distribution dist(args.min_val, args.max_val); 46 | boost::counting_iterator from(0); 47 | boost::counting_iterator to(args.num_samples); 48 | return seastar::do_for_each(from, to, 49 | [&](int i) mutable { 50 | hist->record(dist(gen)); 51 | return seastar::make_ready_future<>(); 52 | }) 53 | .then([&] { 54 | seastar::prometheus::config conf; 55 | conf.metric_help = "synthetic histogram"; 56 | conf.prefix = "histgen"; 57 | return seastar::prometheus::add_prometheus_routes(http_server, conf) 58 | .then([&] { 59 | return http_server.listen(seastar::make_ipv4_address( 60 | args.ip.empty() ? seastar::ipv4_addr{args.port} 61 | : seastar::ipv4_addr{args.ip, args.port})); 62 | }); 63 | }); 64 | } 65 | 66 | seastar::future<> 67 | stop() { 68 | return http_server.stop(); 69 | } 70 | 71 | histgen_server_args args; 72 | seastar::lw_shared_ptr hist; 73 | seastar::http_server http_server; 74 | seastar::metrics::metric_groups metrics; 75 | }; 76 | 77 | void 78 | cli_opts(boost::program_options::options_description_easy_init o) { 79 | namespace po = boost::program_options; 80 | o("ip", po::value()->default_value("127.0.0.1"), 81 | "ip to connect to"); 82 | o("port", po::value()->default_value(20777), 83 | "port for http stats service"); 84 | o("min_val", po::value()->default_value(0), "minimum sample value"); 85 | o("max_val", 86 | po::value()->default_value(smf::kDefaultHistogramMaxValue), 87 | "maximum sample value"); 88 | o("num_samples", po::value()->default_value(10000), 89 | "number of sample values"); 90 | } 91 | 92 | int 93 | main(int args, char **argv, char **env) { 94 | seastar::sharded server; 95 | 96 | seastar::app_template app; 97 | cli_opts(app.add_options()); 98 | 99 | return app.run_deprecated(args, argv, [&] { 100 | seastar::engine().at_exit([&] { return server.stop(); }); 101 | 102 | auto &cfg = app.configuration(); 103 | 104 | histgen_server_args args; 105 | args.ip = cfg["ip"].as(); 106 | args.port = cfg["port"].as(); 107 | args.min_val = cfg["min_val"].as(); 108 | args.max_val = cfg["max_val"].as(); 109 | args.num_samples = cfg["num_samples"].as(); 110 | 111 | return server.start(args).then( 112 | [&server] { return server.invoke_on_all(&histgen_server::serve); }); 113 | }); 114 | } 115 | -------------------------------------------------------------------------------- /src/tests/histogram_tests.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "smf/histogram.h" 10 | #include "smf/random.h" 11 | 12 | static constexpr const int64_t kMaxValue = 10240; 13 | 14 | TEST(histogram, simple) { 15 | smf::random r; 16 | auto h = smf::histogram::make_unique(kMaxValue); 17 | for (auto i = 0u; i < 1000; ++i) { 18 | auto x = r.next() % kMaxValue; 19 | h->record_corrected(x, kMaxValue); 20 | } 21 | } 22 | 23 | int 24 | main(int argc, char **argv) { 25 | ::testing::InitGoogleTest(&argc, argv); 26 | return RUN_ALL_TESTS(); 27 | } 28 | -------------------------------------------------------------------------------- /src/tests/randomstr.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018 SMF Authors 2 | // 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "smf/random.h" 10 | 11 | TEST(bloom, random_alphanum) { 12 | smf::random r; 13 | for (auto i = 0u; i < 1000; ++i) { 14 | auto x = r.next_alphanum(i); 15 | for (char c : x) { 16 | ASSERT_TRUE(std::isalnum(c)); 17 | } 18 | } 19 | } 20 | 21 | int 22 | main(int argc, char **argv) { 23 | ::testing::InitGoogleTest(&argc, argv); 24 | return RUN_ALL_TESTS(); 25 | } 26 | -------------------------------------------------------------------------------- /tools/base/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE 2 | FROM ${BASE} 3 | ENV CI=1 4 | ENV TRAVIS=1 5 | COPY . /smf 6 | RUN /smf/tools/docker-deps.sh 7 | RUN /smf/install-deps.sh 8 | -------------------------------------------------------------------------------- /tools/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 SMF Authors 3 | # 4 | 5 | set -e 6 | set -x 7 | if [[ -n ${CI} ]]; then 8 | echo "In continous integration system..." 9 | # disable colors so we can parse travis better 10 | export GCC_COLORS="" 11 | set -x 12 | fi 13 | 14 | this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 15 | rootdir="$(cd "${this_dir}"/.. && pwd)" 16 | 17 | build_rootdir=${rootdir}/build 18 | builddir="" 19 | 20 | # shellcheck disable=SC1091 21 | source /etc/os-release 22 | 23 | function buildcmd() { 24 | ninja -v -C "${1}" 25 | } 26 | 27 | case $ID in 28 | debian|ubuntu|linuxmint) 29 | echo "$ID supported" 30 | ;; 31 | centos|fedora) 32 | echo "$ID supported" 33 | ;; 34 | *) 35 | echo "$ID not supported" 36 | exit 1 37 | ;; 38 | esac 39 | 40 | function debug { 41 | echo "Debug" 42 | builddir="${build_rootdir}/debug" 43 | 44 | mkdir -p "$builddir" 45 | cd "${rootdir}" 46 | cmake -B"${builddir}" -GNinja \ 47 | -DSMF_MANAGE_DEPS=ON \ 48 | -DCMAKE_BUILD_TYPE=Debug -H"${rootdir}" 49 | 50 | # for fmt.py 51 | ln -sfn "${builddir}/compile_commands.json" "${rootdir}/compile_commands.json" 52 | buildcmd "${builddir}" 53 | } 54 | 55 | function tests { 56 | echo "Testing" 57 | cd "${builddir}" 58 | ctest --output-on-failure -V -R smf 59 | } 60 | 61 | function release { 62 | echo "Release" 63 | builddir="${build_rootdir}/release" 64 | 65 | mkdir -p "$builddir" 66 | cd "${rootdir}" 67 | cmake -B"${builddir}" -GNinja \ 68 | -DSMF_MANAGE_DEPS=ON \ 69 | -DSMF_ENABLE_BENCHMARK_TESTS=ON \ 70 | -DCMAKE_BUILD_TYPE=Release -H"${rootdir}" 71 | 72 | # for fmt.py 73 | ln -sfn "${builddir}/compile_commands.json" "${rootdir}/compile_commands.json" 74 | buildcmd "${builddir}" 75 | } 76 | 77 | function format { 78 | echo "Format" 79 | cd "${rootdir}" 80 | "${rootdir}"/tools/fmt.py 81 | } 82 | 83 | function usage { 84 | cat < /dev/null; then 11 | echo "shellcheck not installed. see: https://github.com/koalaman/shellcheck" 12 | exit 1 13 | fi 14 | 15 | (cd "${rootdir}" && for f in $(git ls-files --full-name '*.sh'); do shellcheck "$f"; done) 16 | -------------------------------------------------------------------------------- /tools/travis_stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 SMF Authors 4 | # 5 | 6 | 7 | cmd="$*" 8 | echo "Launching command in background: $cmd" 9 | eval "${cmd}" & 10 | while kill -0 $! >/dev/null 2>&1; do 11 | sleep 1m && echo "travis background: $(date)"; 12 | done 13 | --------------------------------------------------------------------------------