├── .clang-format ├── .github ├── actions │ ├── do-build │ │ └── action.yml │ └── install-deps │ │ └── action.yml └── workflows │ ├── codeql.yml │ ├── integration-tests.yml │ └── unit-tests.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── anyblob.cmake ├── data ├── aws.csv ├── azure.csv ├── gcp.csv ├── generate.py ├── scraper.py └── sources.txt ├── example ├── benchmark │ ├── CMakeLists.txt │ ├── README.md │ ├── include │ │ ├── benchmark │ │ │ └── bandwidth.hpp │ │ ├── local.cmake │ │ └── network │ │ │ └── s3_send_receiver.hpp │ ├── scripts │ │ ├── create.sh │ │ ├── https.py │ │ ├── https.sh │ │ ├── latency.py │ │ ├── latency.sh │ │ ├── latency_sparse.sh │ │ ├── model.py │ │ ├── model.sh │ │ ├── sizes.py │ │ ├── sizes.sh │ │ ├── sizes_latency.py │ │ ├── sizes_latency.sh │ │ ├── throughput.py │ │ ├── throughput.sh │ │ ├── throughput_sparse.py │ │ └── throughput_sparse.sh │ ├── src │ │ ├── benchmark │ │ │ └── bandwidth.cpp │ │ ├── local.cmake │ │ └── main.cpp │ └── thirdparty │ │ ├── anyblob.cmake │ │ └── awssdk.cmake └── simple │ ├── CMakeLists.txt │ ├── README.md │ ├── anyblob.cmake │ └── main.cpp ├── include ├── cloud │ ├── aws.hpp │ ├── aws_cache.hpp │ ├── aws_instances.hpp │ ├── aws_signer.hpp │ ├── azure.hpp │ ├── azure_instances.hpp │ ├── azure_signer.hpp │ ├── gcp.hpp │ ├── gcp_instances.hpp │ ├── gcp_signer.hpp │ ├── http.hpp │ ├── ibm.hpp │ ├── ibm_instances.hpp │ ├── minio.hpp │ ├── oracle.hpp │ ├── oracle_instances.hpp │ └── provider.hpp ├── local.cmake ├── network │ ├── cache.hpp │ ├── config.hpp │ ├── connection_manager.hpp │ ├── http_helper.hpp │ ├── http_message.hpp │ ├── http_request.hpp │ ├── http_response.hpp │ ├── https_message.hpp │ ├── io_uring_socket.hpp │ ├── message_result.hpp │ ├── message_task.hpp │ ├── original_message.hpp │ ├── tasked_send_receiver.hpp │ ├── throughput_cache.hpp │ ├── tls_connection.hpp │ ├── tls_context.hpp │ └── transaction.hpp └── utils │ ├── data_vector.hpp │ ├── load_tracker.hpp │ ├── ring_buffer.hpp │ ├── timer.hpp │ ├── unordered_map.hpp │ └── utils.hpp ├── src ├── cloud │ ├── aws.cpp │ ├── aws_cache.cpp │ ├── aws_instances.cpp │ ├── aws_signer.cpp │ ├── azure.cpp │ ├── azure_instances.cpp │ ├── azure_signer.cpp │ ├── gcp.cpp │ ├── gcp_instances.cpp │ ├── gcp_signer.cpp │ ├── http.cpp │ ├── ibm.cpp │ ├── ibm_instances.cpp │ ├── minio.cpp │ ├── oracle.cpp │ ├── oracle_instances.cpp │ └── provider.cpp ├── local.cmake ├── network │ ├── cache.cpp │ ├── connection_manager.cpp │ ├── http_helper.cpp │ ├── http_message.cpp │ ├── http_request.cpp │ ├── http_response.cpp │ ├── https_message.cpp │ ├── io_uring_socket.cpp │ ├── message_result.cpp │ ├── message_task.cpp │ ├── tasked_send_receiver.cpp │ ├── throughput_cache.cpp │ ├── tls_connection.cpp │ ├── tls_context.cpp │ └── transaction.cpp └── utils │ ├── timer.cpp │ └── utils.cpp └── test ├── integration ├── local.cmake ├── minio_async.cpp ├── minio_sync.cpp └── test.cpp └── unit ├── cloud ├── aws_test.cpp ├── azure_test.cpp ├── gcp_test.cpp └── provider_test.cpp ├── local.cmake ├── network ├── http_helper_test.cpp ├── http_request_test.cpp ├── io_uring_socket_test.cpp ├── messages_test.cpp ├── resolver_test.cpp ├── send_receiver_test.cpp └── tasked_send_receiver_test.cpp ├── test.cpp └── utils ├── data_vector_test.cpp ├── ring_buffer_test.cpp ├── unordered_map_test.cpp └── utils_test.cpp /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | AccessModifierOffset: 0 4 | AlignAfterOpenBracket: Align 5 | AlignArrayOfStructures: None 6 | AlignConsecutiveAssignments: 7 | Enabled: false 8 | AcrossEmptyLines: false 9 | AcrossComments: false 10 | AlignCompound: false 11 | PadOperators: true 12 | AlignConsecutiveBitFields: 13 | Enabled: false 14 | AcrossEmptyLines: false 15 | AcrossComments: false 16 | AlignCompound: false 17 | PadOperators: true 18 | AlignConsecutiveDeclarations: 19 | Enabled: false 20 | AcrossEmptyLines: false 21 | AcrossComments: false 22 | AlignCompound: false 23 | PadOperators: true 24 | AlignConsecutiveMacros: 25 | Enabled: false 26 | AcrossEmptyLines: false 27 | AcrossComments: false 28 | AlignCompound: false 29 | PadOperators: true 30 | AlignEscapedNewlines: Left 31 | AlignOperands: Align 32 | AlignTrailingComments: 33 | Kind: Never 34 | OverEmptyLines: 0 35 | AllowAllArgumentsOnNextLine: true 36 | AllowAllParametersOfDeclarationOnNextLine: true 37 | AllowShortBlocksOnASingleLine: Empty 38 | AllowShortCaseLabelsOnASingleLine: true 39 | AllowShortFunctionsOnASingleLine: All 40 | AllowShortIfStatementsOnASingleLine: OnlyFirstIf 41 | AllowShortLoopsOnASingleLine: true 42 | AllowShortLambdasOnASingleLine: All 43 | AlwaysBreakAfterDefinitionReturnType: None 44 | AlwaysBreakAfterReturnType: None 45 | BinPackArguments: true 46 | BinPackParameters: true 47 | BraceWrapping: 48 | AfterClass: false 49 | AfterControlStatement: MultiLine 50 | AfterEnum: false 51 | AfterFunction: false 52 | AfterNamespace: false 53 | AfterObjCDeclaration: false 54 | AfterStruct: false 55 | AfterUnion: false 56 | AfterExternBlock: false 57 | BeforeCatch: false 58 | BeforeElse: false 59 | IndentBraces: false 60 | AfterCaseLabel: false 61 | BeforeLambdaBody: false 62 | BeforeWhile: false 63 | SplitEmptyFunction: false 64 | SplitEmptyRecord: false 65 | SplitEmptyNamespace: false 66 | BreakBeforeBinaryOperators: None 67 | BreakBeforeBraces: Attach 68 | BreakBeforeInheritanceComma: false 69 | BreakBeforeTernaryOperators: false 70 | BreakConstructorInitializersBeforeComma: false 71 | BreakConstructorInitializers: BeforeColon 72 | BreakAfterJavaFieldAnnotations: false 73 | BreakStringLiterals: true 74 | ColumnLimit: 0 75 | CommentPragmas: "^ IWYU pragma:" 76 | CompactNamespaces: false 77 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 78 | ConstructorInitializerIndentWidth: 4 79 | ContinuationIndentWidth: 4 80 | Cpp11BracedListStyle: true 81 | DerivePointerAlignment: false 82 | DisableFormat: false 83 | ExperimentalAutoDetectBinPacking: true 84 | FixNamespaceComments: true 85 | ForEachMacros: 86 | - foreach 87 | - Q_FOREACH 88 | - BOOST_FOREACH 89 | IncludeCategories: 90 | - Regex: ^" 91 | Priority: 1 92 | - Regex: ^ 93 | Priority: 2 94 | - Regex: ^<.*\.h> 95 | Priority: 4 96 | - Regex: ^< 97 | Priority: 3 98 | - Regex: .\* 99 | Priority: 5 100 | IncludeIsMainRegex: ([-_](test|unittest))?$ 101 | IndentAccessModifiers: false 102 | IndentCaseLabels: true 103 | IndentCaseBlocks: false 104 | IndentExternBlock: AfterExternBlock 105 | IndentGotoLabels: true 106 | IndentPPDirectives: None 107 | IndentRequiresClause: false 108 | IndentWidth: 4 109 | IndentWrappedFunctionNames: false 110 | InsertBraces: false 111 | InsertNewlineAtEOF: true 112 | InsertTrailingCommas: None 113 | JavaScriptQuotes: Leave 114 | JavaScriptWrapImports: true 115 | KeepEmptyLinesAtTheStartOfBlocks: false 116 | MacroBlockBegin: "" 117 | MacroBlockEnd: "" 118 | MaxEmptyLinesToKeep: 1 119 | NamespaceIndentation: None 120 | ObjCBlockIndentWidth: 4 121 | ObjCSpaceAfterProperty: false 122 | ObjCSpaceBeforeProtocolList: false 123 | PenaltyBreakBeforeFirstCallParameter: 1 124 | PenaltyBreakComment: 300 125 | PenaltyBreakFirstLessLess: 120 126 | PenaltyBreakString: 1000 127 | PenaltyExcessCharacter: 1000000 128 | PenaltyReturnTypeOnItsOwnLine: 200 129 | PointerAlignment: Left 130 | ReflowComments: false 131 | SortIncludes: CaseSensitive 132 | SpaceAfterCStyleCast: true 133 | SpaceAfterTemplateKeyword: true 134 | SpaceBeforeAssignmentOperators: true 135 | SpaceBeforeParens: ControlStatements 136 | SpaceBeforeParensOptions: 137 | AfterControlStatements: true 138 | AfterForeachMacros: true 139 | AfterFunctionDefinitionName: false 140 | AfterFunctionDeclarationName: false 141 | AfterIfMacros: true 142 | AfterOverloadedOperator: false 143 | AfterRequiresInClause: false 144 | AfterRequiresInExpression: false 145 | BeforeNonEmptyParentheses: false 146 | SpaceInEmptyParentheses: false 147 | SpacesBeforeTrailingComments: 1 148 | SpacesInAngles: Leave 149 | SpacesInContainerLiterals: false 150 | SpacesInCStyleCastParentheses: false 151 | SpacesInParentheses: false 152 | SpacesInSquareBrackets: false 153 | Standard: Latest 154 | TabWidth: 4 155 | UseTab: Never 156 | -------------------------------------------------------------------------------- /.github/actions/do-build/action.yml: -------------------------------------------------------------------------------- 1 | name: "Build AnyBlob" 2 | description: "Build AnyBlob using either clang or gcc" 3 | inputs: 4 | clang-toolchain: 5 | required: false 6 | description: "Build with clang and use libcxx as the standard library" 7 | default: false 8 | 9 | runs: 10 | using: "composite" 11 | steps: 12 | # CMake configuration option 1 - GCC toolchain 13 | - name: Configure CMake 14 | if: ${{ inputs.clang-toolchain == 'false' }} 15 | run: | 16 | cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} 17 | shell: bash 18 | 19 | # CMake configuration option 2 - Clang toolchain 20 | - name: Configure CMake 21 | if: ${{ inputs.clang-toolchain == 'true' }} 22 | # Explicitly use clang-15. The runner comes with clang-13, 14, and 15. 23 | # However, only clang-15 and up support `-Wno-unqualified-std-cast-call`. 24 | run: | 25 | sudo apt install libc++-15-dev libc++abi-15-dev 26 | cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ 27 | -DCMAKE_C_COMPILER=clang-15 \ 28 | -DCMAKE_CXX_COMPILER=clang++-15 \ 29 | -DANYBLOB_LIBCXX_COMPAT=1 30 | shell: bash 31 | 32 | # Build library and test binaries 33 | - name: Build 34 | run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} 35 | shell: bash 36 | -------------------------------------------------------------------------------- /.github/actions/install-deps/action.yml: -------------------------------------------------------------------------------- 1 | name: "Install Dependencies" 2 | description: "Installs dynamic libraries required by AnyBlob" 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | # Install dependencies 8 | - name: Install dependencies 9 | run: sudo apt update && sudo apt install liburing-dev openssl libssl-dev libjemalloc-dev lld && sudo sysctl -w vm.mmap_rnd_bits=28 10 | shell: bash 11 | 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "master", "codeql" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | analyze: 11 | name: Analyze (${{ matrix.language }}) 12 | runs-on: 'ubuntu-latest' 13 | timeout-minutes: 30 14 | permissions: 15 | security-events: write 16 | actions: read 17 | contents: read 18 | 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | include: 23 | - language: c-cpp 24 | build-mode: autobuild 25 | - language: python 26 | build-mode: none 27 | steps: 28 | - name: Checkout repository 29 | uses: actions/checkout@v4 30 | with: 31 | submodules: true 32 | 33 | - name: Initialize CodeQL 34 | uses: github/codeql-action/init@v3 35 | with: 36 | languages: ${{ matrix.language }} 37 | build-mode: ${{ matrix.build-mode }} 38 | 39 | - name: Perform CodeQL Analysis 40 | uses: github/codeql-action/analyze@v3 41 | with: 42 | category: "/language:${{matrix.language}}" 43 | -------------------------------------------------------------------------------- /.github/workflows/integration-tests.yml: -------------------------------------------------------------------------------- 1 | name: MinIO Integration Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - dev 8 | pull_request: 9 | branches: 10 | - master 11 | - dev 12 | 13 | env: 14 | BUILD_TYPE: Debug 15 | # False ASAN positives for alloc_dealloc_mismatch when running with clang 16 | # https://github.com/llvm/llvm-project/issues/59432 17 | ASAN_OPTIONS: alloc_dealloc_mismatch=0 18 | 19 | jobs: 20 | integration-test: 21 | name: Integration Tests (Clang ${{ matrix.clang-toolchain }}) 22 | # Run on ubuntu 23 | runs-on: ubuntu-latest 24 | # Use both the GCC and Clang Toolchain 25 | strategy: 26 | matrix: 27 | clang-toolchain: [true, false] 28 | 29 | # Define the steps 30 | steps: 31 | # Install MinIO Docker 32 | - name: Install MinIO Docker 33 | run: | 34 | docker run -d -p 9000:9000 --name minio \ 35 | -e "MINIO_ACCESS_KEY=test" -e "MINIO_SECRET_KEY=testtest" \ 36 | -v /tmp/buckets:/buckets -v /tmp/.minio:/root/.minio \ 37 | minio/minio server /buckets 38 | 39 | export AWS_ACCESS_KEY_ID=test 40 | export AWS_SECRET_ACCESS_KEY=testtest 41 | export AWS_EC2_METADATA_DISABLED=true 42 | 43 | aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test 44 | 45 | # Checkout repo 46 | - uses: actions/checkout@v3 47 | with: 48 | submodules: true 49 | 50 | # Use action to install dependencies 51 | - name: Install Dependencies 52 | uses: ./.github/actions/install-deps 53 | 54 | # Use action to build 55 | - name: Build AnyBlob 56 | uses: ./.github/actions/do-build 57 | with: 58 | clang-toolchain: ${{ matrix.clang-toolchain }} 59 | 60 | # Run the integration test 61 | - name: Integration Test 62 | working-directory: ${{github.workspace}}/build 63 | run: | 64 | export AWS_S3_BUCKET=test 65 | export AWS_S3_REGION=us-east-1 66 | export AWS_S3_ENDPOINT=127.0.0.1:9000 67 | export AWS_S3_ACCESS_KEY=test 68 | export AWS_S3_SECRET_ACCESS_KEY=testtest 69 | 70 | ./integration 71 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yml: -------------------------------------------------------------------------------- 1 | name: Unit Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - dev 8 | pull_request: 9 | branches: 10 | - master 11 | - dev 12 | 13 | env: 14 | BUILD_TYPE: Debug 15 | # False ASAN positives for alloc_dealloc_mismatch when running with clang 16 | # https://github.com/llvm/llvm-project/issues/59432 17 | ASAN_OPTIONS: alloc_dealloc_mismatch=0 18 | 19 | jobs: 20 | unit-test: 21 | name: Unit Tests (Clang ${{ matrix.clang-toolchain }}) 22 | # Run on ubuntu 23 | runs-on: ubuntu-latest 24 | # Use both the GCC and Clang Toolchain 25 | strategy: 26 | matrix: 27 | clang-toolchain: [true, false] 28 | 29 | # Define the steps 30 | steps: 31 | # Checkout repo 32 | - uses: actions/checkout@v3 33 | with: 34 | submodules: true 35 | 36 | # Use action to install dependencies 37 | - name: Install Dependencies 38 | uses: ./.github/actions/install-deps 39 | 40 | # Use action to build 41 | - name: Build AnyBlob 42 | uses: ./.github/actions/do-build 43 | with: 44 | clang-toolchain: ${{ matrix.clang-toolchain }} 45 | 46 | # Run the unit tests 47 | - name: Unit Test 48 | working-directory: ${{github.workspace}}/build 49 | run: ./tester 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | compile_commands.json 2 | .ccls-cache/ 3 | .ccls_cache/ 4 | build/ 5 | bin/ 6 | include/gen/ 7 | src/gen/ 8 | *.d 9 | *.slo 10 | *.lo 11 | *.o 12 | *.obj 13 | *.gch 14 | *.pch 15 | *.so 16 | *.lai 17 | *.la 18 | *.a 19 | *.lib 20 | *.out 21 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "thirdparty/perfevent"] 2 | path = thirdparty/perfevent 3 | url = https://github.com/viktorleis/perfevent 4 | [submodule "thirdparty/catch2"] 5 | path = thirdparty/catch2 6 | url = https://github.com/catchorg/Catch2 7 | branch = v2.13.10 8 | [submodule "example/benchmark/thirdparty/perfevent"] 9 | path = example/benchmark/thirdparty/perfevent 10 | url = https://github.com/viktorleis/perfevent 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AnyBlob - Universal Cloud Object Storage Download Manager 2 | 3 | In this repository, we present AnyBlob. 4 | AnyBlob is a universal download manager that allows to retrieve and upload objects to different cloud object stores. 5 | Our download manager uses less CPU resources than cloud-vendor provided libraries while retaining maximum throughput performance. 6 | AnyBlob leverages IO\_uring for superior performance per core. 7 | For experimental results, please visit our research paper at [PVLDB 16](https://www.vldb.org/pvldb/vol16/p2769-durner.pdf). 8 | 9 | ## Building AnyBlob 10 | 11 | AnyBlob relies on some third-party libraries: 12 | - uring 13 | - openssl 14 | - jemalloc 15 | 16 | For Ubuntu 22.04+ the following command installs the third-party libraries: 17 | ``` 18 | sudo apt update && sudo apt install liburing-dev openssl libssl-dev libjemalloc-dev lld g++ cmake 19 | ``` 20 | 21 | For building, use the following commands: 22 | ``` 23 | git clone --recursive https://github.com/durner/AnyBlob 24 | mkdir -p build/Release 25 | cd build/Release 26 | cmake -DCMAKE_BUILD_TYPE=Release ../.. 27 | make -j16 28 | ``` 29 | 30 | ## Using AnyBlob 31 | 32 | AnyBlob is a modern C++ library that can be linked to your program. 33 | We provide two examples how to use AnyBlob in your project. 34 | Please find a simple example that does only download a single object from AWS, and a more sophisticated example that runs download benchmarks. 35 | There, you can also find information on how to easily integrate the building process into existing CMake projects for an automated compilation and static linkage of AnyBlob. 36 | The anyblob.cmake should be used in your project to integrate this repository as external project. 37 | 38 | ## Contribution 39 | 40 | If you have bug fixes or improvement, please do not hesitate to open a new merge request. 41 | For coverage testing you can simply `make coverage` and open the coverage report found in the build directory at coverage/index.html. 42 | 43 | ## Cite this work 44 | 45 | If you are using AnyBlob in your scientific work, please cite: 46 | 47 | ``` 48 | Exploiting Cloud Object Storage for High-Performance Analytics 49 | Dominik Durner, Viktor Leis, and Thomas Neumann 50 | PVLDB 16 (11), 2023, 49th International Conference on Very Large Data Bases 51 | ``` 52 | -------------------------------------------------------------------------------- /anyblob.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # AnyBlob Include CMake 3 | # 4 | # This should be used to include AnyBlob as static library for your project 5 | # --------------------------------------------------------------------------- 6 | 7 | Include(ExternalProject) 8 | 9 | # --------------------------------------------------------------------------- 10 | # Get AnyBlob 11 | # --------------------------------------------------------------------------- 12 | 13 | ExternalProject_Add(anyblob 14 | GIT_REPOSITORY https://github.com/durner/AnyBlob.git 15 | GIT_TAG "master" 16 | PREFIX "thirdparty/anyblob" 17 | INSTALL_COMMAND "" 18 | CMAKE_ARGS 19 | -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 20 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 21 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 22 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 23 | ) 24 | 25 | # --------------------------------------------------------------------------- 26 | # Get SOURCE and BINARY DIR 27 | # --------------------------------------------------------------------------- 28 | 29 | ExternalProject_Get_Property(anyblob SOURCE_DIR) 30 | ExternalProject_Get_Property(anyblob BINARY_DIR) 31 | 32 | set(ANYBLOB_INCLUDE_DIR ${SOURCE_DIR}/include) 33 | file(MAKE_DIRECTORY ${ANYBLOB_INCLUDE_DIR}) 34 | 35 | # --------------------------------------------------------------------------- 36 | # Configure OpenSSL, Threads, and Uring 37 | # --------------------------------------------------------------------------- 38 | 39 | find_package(Threads REQUIRED) 40 | find_package(OpenSSL REQUIRED) 41 | 42 | find_path(LIBURING_INCLUDE_DIR NAMES liburing.h) 43 | mark_as_advanced(LIBURING_INCLUDE_DIR) 44 | find_library(LIBURING_LIBRARY NAMES uring) 45 | mark_as_advanced(LIBURING_LIBRARY) 46 | include(FindPackageHandleStandardArgs) 47 | FIND_PACKAGE_HANDLE_STANDARD_ARGS( 48 | LIBURING 49 | REQUIRED_VARS LIBURING_LIBRARY LIBURING_INCLUDE_DIR) 50 | if(LIBURING_FOUND) 51 | set(LIBURING_LIBRARIES ${LIBURING_LIBRARY}) 52 | set(LIBURING_INCLUDE_DIRS ${LIBURING_INCLUDE_DIR}) 53 | endif() 54 | 55 | # --------------------------------------------------------------------------- 56 | # Build Library with dependencies 57 | # --------------------------------------------------------------------------- 58 | 59 | add_library(AnyBlob STATIC IMPORTED) 60 | set_property(TARGET AnyBlob PROPERTY IMPORTED_LOCATION ${BINARY_DIR}/libAnyBlob.a) 61 | set_property(TARGET AnyBlob APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ANYBLOB_INCLUDE_DIR}) 62 | add_dependencies(AnyBlob anyblob) 63 | target_link_libraries(AnyBlob INTERFACE OpenSSL::SSL Threads::Threads ${LIBURING_LIBRARY} jemalloc) 64 | 65 | -------------------------------------------------------------------------------- /data/generate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | 4 | def read_csv(input_file, columns_to_include, vendor): 5 | output_data = [] 6 | 7 | with open(input_file, 'r') as csvfile: 8 | reader = csv.DictReader(csvfile) 9 | for row in reader: 10 | selected_data = [row[column] for column in columns_to_include] 11 | # Name 12 | selected_data[0] = "\"" + selected_data[0] + "\"" 13 | # Memory 14 | try: 15 | selected_data[1] = float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[1])) 16 | except ValueError: 17 | selected_data[1] = 0 18 | # vCPU 19 | try: 20 | selected_data[2] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[2][:4]))) 21 | except ValueError: 22 | selected_data[2] = 0 23 | # Network 24 | try: 25 | selected_data[3] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[3]))) 26 | except ValueError: 27 | selected_data[3] = 0 28 | 29 | # Specifics 30 | if vendor != "Azure": 31 | selected_data[3] *= 1000 32 | 33 | if vendor == "GCP": 34 | try: 35 | selected_data[4] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[4]))) 36 | except ValueError: 37 | selected_data[4] = 0 38 | if (selected_data[4] > selected_data[3]): 39 | selected_data[3] = selected_data[4] 40 | selected_data.pop() 41 | 42 | # Network default 43 | if selected_data[3] == 0: 44 | selected_data[3] = 1000 45 | 46 | output_data.append(selected_data) 47 | return output_data 48 | 49 | def write_to_file(output_data, output_file): 50 | with open(output_file, 'a') as f: 51 | f.write("{\n") 52 | for idx, data in enumerate(output_data): 53 | f.write("{ ") 54 | for lidx, value in enumerate(data): 55 | f.write(str(value)) 56 | if lidx < len(data) - 1: 57 | f.write(", ") 58 | else: 59 | f.write(" }") 60 | if idx < len(output_data) - 1: 61 | f.write(",\n") 62 | else: 63 | f.write(" };\n") 64 | 65 | def write_header(vendor, output_file): 66 | with open(output_file, 'w') as f: 67 | f.write("#include \"cloud/" + vendor.lower() + "_instances.hpp\"\n") 68 | f.write("//---------------------------------------------------------------------------\n") 69 | f.write("// AnyBlob - Universal Cloud Object Storage Library\n") 70 | f.write("// Dominik Durner, 2023\n") 71 | f.write("//\n") 72 | f.write("// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.\n") 73 | f.write("// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.\n") 74 | f.write("// SPDX-License-Identifier: MPL-2.0\n") 75 | f.write("//---------------------------------------------------------------------------\n") 76 | f.write("namespace anyblob {\n") 77 | f.write("namespace cloud {\n") 78 | f.write("//---------------------------------------------------------------------------\n") 79 | f.write("using namespace std;\n") 80 | f.write("//---------------------------------------------------------------------------\n") 81 | f.write("vector<" + vendor + "Instance> " + vendor + "Instance::getInstanceDetails()\n") 82 | f.write("// Gets a vector of instance type infos\n") 83 | f.write("{\n") 84 | f.write(" vector<" + vendor + "Instance> instances =\n") 85 | 86 | def write_footer(output_file): 87 | with open(output_file, 'a') as f: 88 | f.write("return instances;\n") 89 | f.write("}\n") 90 | f.write("//---------------------------------------------------------------------------\n") 91 | f.write("}; // namespace cloud\n") 92 | f.write("}; // namespace anyblob\n") 93 | 94 | def main(): 95 | vendors = ["AWS", "Azure", "GCP"] 96 | columns = { 97 | "AWS": ["API Name", "Instance Memory", "vCPUs", "Network Performance"], 98 | "Azure": ["Size", "Memory", "vC", "Network Bandwidth"], 99 | "GCP": ["name", "memoryGiB", "vCpus", "bandwidth", "tier1"], 100 | } 101 | for vendor in vendors: 102 | output_data = read_csv(vendor.lower()+".csv", columns.get(vendor), vendor) 103 | output_file = "../src/cloud/" + vendor.lower() + '_instances.cpp' 104 | write_header(vendor, output_file) 105 | write_to_file(output_data, output_file) 106 | write_footer(output_file) 107 | os.system("clang-format -i " + output_file) 108 | 109 | if __name__ == "__main__": 110 | main() 111 | -------------------------------------------------------------------------------- /data/sources.txt: -------------------------------------------------------------------------------- 1 | GCP: 2 | wget https://gcloud-compute.com/machine-types-regions.csv -O gcp.csv 3 | sed -i -n -e '1p' -e '/us-east1,/p' gcp.csv 4 | 5 | AWS: 6 | Export csv from website 7 | https://instances.vantage.sh/?region=us-east-2 8 | 9 | Azure: 10 | Get data with local scraper.py, add new URLs from azure doc for upcoming types 11 | -------------------------------------------------------------------------------- /example/benchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.8) 2 | 3 | project(AnyBlobBenchmark) 4 | 5 | # --------------------------------------------------------------------------- 6 | # Environment 7 | # --------------------------------------------------------------------------- 8 | 9 | if (NOT UNIX) 10 | message(STATUS FATAL_ERROR "Unsupported OS") 11 | endif (NOT UNIX) 12 | 13 | set(CMAKE_CXX_STANDARD 20) 14 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 15 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 16 | 17 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -Wall -fsanitize=undefined") 18 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -march=native -Wall") 19 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -march=native -Wall") 20 | add_compile_options(-Wall -Wextra -Werror) 21 | 22 | # --------------------------------------------------------------------------- 23 | # AnyBlob 24 | # --------------------------------------------------------------------------- 25 | 26 | include("${PROJECT_SOURCE_DIR}/thirdparty/anyblob.cmake") 27 | 28 | # --------------------------------------------------------------------------- 29 | # AWS 30 | # --------------------------------------------------------------------------- 31 | 32 | include("${PROJECT_SOURCE_DIR}/thirdparty/awssdk.cmake") 33 | 34 | # --------------------------------------------------------------------------- 35 | # Includes 36 | # --------------------------------------------------------------------------- 37 | 38 | include_directories(SYSTEM ${ANYBLOB_INCLUDE_DIR}) 39 | include("${PROJECT_SOURCE_DIR}/include/local.cmake") 40 | include_directories( 41 | ${PROJECT_SOURCE_DIR}/include 42 | ${PROJECT_SOURCE_DIR}/thirdparty 43 | ) 44 | 45 | # --------------------------------------------------------------------------- 46 | # Sources 47 | # --------------------------------------------------------------------------- 48 | 49 | include("${PROJECT_SOURCE_DIR}/src/local.cmake") 50 | 51 | # --------------------------------------------------------------------------- 52 | # Executable 53 | # --------------------------------------------------------------------------- 54 | 55 | add_executable(AnyBlobBenchmark src/main.cpp ${SRC_CPP} ${INCLUDE_HPP}) 56 | target_link_libraries(AnyBlobBenchmark AnyBlob ${AWS_LINK_LIBRARIES} jemalloc dl) 57 | -------------------------------------------------------------------------------- /example/benchmark/README.md: -------------------------------------------------------------------------------- 1 | # AnyBlobBenchmark 2 | This directory contains the AnyBlob benchmarking tools, and also includes all third-parties (e.g., AWS SDK) to run our experiments of the AnyBlob paper. 3 | 4 | # Validate Experiments 5 | 6 | ## AnyBlob Experiments 7 | 8 | To validate our findings you need to set-up an AWS storage account to create randomly generated data objects. 9 | Further, set up an instance with IAM Storage permissions and copy the files to this node. 10 | For throughput experiments, we recommend the c5n.18xlarge, but a cheaper instance such as the c5n.xlarge can be used for latency experiments. 11 | 12 | With the create.sh script from the scripts subdirectory, you can create random objects with different sizes. 13 | All .sh files within the script folder need some small adaptations to include your random object bucket and also the result bucket to write back the experimental results. 14 | 15 | The raw data of our Figures were created from multiple scripts. This list contains the mapping between individual scripts and Section / Figures. 16 | 17 | - sizes_latency.sh for Figure 2 (Section 2.3) 18 | - latency.sh for Figure 3 (Section 2.3) 19 | - latency_sparse.sh for Figure 4 (Section 2.3) 20 | - throughput_sparse.sh for Figures 5, 6 & 7 (Section 2.4) 21 | - sizes.sh for Figure 8 (Section 2.5) 22 | - https.sh for Figure 9 (Section 2.6) 23 | - model.sh for Figure 10 (Section 2.8) and Figure 12 (Section 3.4) 24 | 25 | In the following, we show the steps to compile our benchmarking binary. 26 | 27 | sudo apt update && sudo apt install liburing-dev openssl libssl-dev libjemalloc-dev g++ lld cmake libcurlpp-dev zlib1g-dev libcurl4-openssl-dev 28 | git clone --recursive https://github.com/durner/AnyBlob 29 | cd AnyBlob/example/benchmark 30 | mkdir -p build/Release 31 | cd build/Release 32 | cmake ../.. -DCMAKE_BUILD_TYPE=Release 33 | make -j16 34 | 35 | ## Database Experiments 36 | 37 | All our DBMS related experiments and our binary of our proprietary system are shared in https://gitlab.db.in.tum.de/durner/cloud-storage-analytics. 38 | -------------------------------------------------------------------------------- /example/benchmark/include/benchmark/bandwidth.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2021 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | namespace benchmark { 17 | class Bandwidth { 18 | public: 19 | /// Different bandwith systems 20 | enum class Systems : uint8_t { 21 | Uring, 22 | S3, 23 | S3Crt 24 | }; 25 | /// Settings for the bandwidth benchmark 26 | struct Settings { 27 | /// Number of network sockets/threads 28 | uint32_t concurrentThreads; 29 | /// Number of simultaneous requests per socket 30 | uint32_t concurrentRequests; 31 | /// Total number of requests 32 | uint64_t requests; 33 | /// The chunksize 34 | uint64_t chunkSize = 64u * 1024; 35 | /// Choose random file from xxx.bin (xxx files from 0 to this) 36 | uint32_t blobFiles; 37 | /// The number of iterations 38 | uint32_t iterations = 1; 39 | /// The filepath 40 | std::string filePath; 41 | /// The report filename 42 | std::string report; 43 | /// The resolver 44 | std::string resolver = "throughput"; 45 | /// The account / client email 46 | std::string account; 47 | /// The rsa key file 48 | std::string rsaKeyFile; 49 | /// The systems 50 | std::vector systems; 51 | /// HTTPS 52 | bool https = false; 53 | /// Encryption at rest 54 | bool encryption = false; 55 | /// Test upload 56 | bool testUpload = false; 57 | }; 58 | 59 | /// Benchmark runner 60 | static void run(const Settings& benchmarkSettings, const std::string& uri); 61 | /// Uring benchmark runner 62 | static void runUring(const Settings& benchmarkSettings, const std::string& uri); 63 | /// S3 benchmark runner 64 | template 65 | static void runS3(const Settings& benchmarkSettings, const std::string& uri); 66 | /// Timestamp util 67 | static inline size_t systemClockToMys(const std::chrono::steady_clock::time_point& t) { 68 | return std::chrono::duration_cast(t.time_since_epoch()).count(); 69 | } 70 | }; 71 | //--------------------------------------------------------------------------- 72 | }; // namespace benchmark 73 | }; // namespace anyblob 74 | -------------------------------------------------------------------------------- /example/benchmark/include/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE INCLUDE_HPP "include/*.hpp") 6 | -------------------------------------------------------------------------------- /example/benchmark/scripts/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | SIZES=($(( 2**0 )) $(( 2**10 )) $(( 2**19 )) $(( 2**20 )) $(( 2**21 )) $(( 2**22 )) $(( 2**23 )) $(( 2**24 )) $(( 2**25 ))) 7 | CONTAINER=anyblob-data 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | else 11 | CONTAINER=anyblob-data-${REGION} 12 | fi 13 | #------------------------------------------------------ 14 | aws s3api create-bucket --bucket $CONTAINER --region $REGION --create-bucket-configuration LocationConstraint=$REGION 15 | #------------------------------------------------------ 16 | mkdir data 17 | mkdir data/enc 18 | cd data 19 | for size in ${SIZES[@]} 20 | do 21 | for n in {1..8192} 22 | do 23 | openssl rand -out ${n}.bin $size 24 | openssl enc -aes-256-cbc -K "3031323334353637383930313233343536373839303132333435363738393031" -iv "30313233343536373839303132333435" -in ${n}.bin -out enc/${n}.bin 25 | done 26 | aws s3 sync ./ s3://${CONTAINER}/${size}/ 27 | for n in {1..8192} 28 | do 29 | rm ${n}.bin 30 | rm enc/${n}.bin 31 | done 32 | done 33 | #------------------------------------------------------ 34 | -------------------------------------------------------------------------------- /example/benchmark/scripts/https.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | runs = range(0, 2) 4 | bucket = os.environ['CONTAINER'] 5 | region = os.environ['REGION'] 6 | 7 | files = ["16777216/"] 8 | numberOfFiles = 4096 9 | requestsScale = 256 10 | rquestsMax = 50000 11 | systems = ["s3", "uring"] 12 | threadsUring = [12, 16, 20, 24] 13 | threadsS3 = [64, 128, 192, 256] 14 | throughputGoal = [] 15 | concurrentRequestsS3Crt = [] 16 | concurrentRequestsS3 = [256, 512] 17 | concurrentRequestsUring = [6, 8, 14, 16, 18, 20] 18 | https = [0, 1] 19 | encryption = [0, 1] 20 | iterations = 3 21 | 22 | for r in runs: 23 | for system in systems: 24 | threadOrThroughput = [] 25 | conccurency = [] 26 | if system == "uring": 27 | threadOrThroughput = threadsUring 28 | conccurency = concurrentRequestsUring 29 | elif system == "s3crt": 30 | threadOrThroughput = throughputGoal 31 | conccurency = concurrentRequestsS3Crt 32 | elif system == "s3": 33 | threadOrThroughput = threadsS3 34 | conccurency = concurrentRequestsS3 35 | 36 | for f in files: 37 | for c in conccurency: 38 | for t in threadOrThroughput: 39 | for h in https: 40 | for e in encryption: 41 | dirpath = os.path.dirname(os.path.realpath(__file__)) 42 | binPath = "timeout 600 " + dirpath + '/../build/Release/' 43 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 44 | requests = requestsScale 45 | if system == "uring": 46 | requests = min(requestsScale * c * t, rquestsMax) 47 | else: 48 | requests = min(requestsScale * c, rquestsMax) 49 | 50 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -n " + str(numberOfFiles) 51 | if e == 1: 52 | if system != "uring": 53 | continue 54 | else: 55 | binPath = binPath + " -f enc/" + f 56 | else: 57 | binPath = binPath + " -f " + f 58 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -h " + str(h) + " -x " + str(e) + " -o sizes_" + str(r) + ".csv" 59 | 60 | print(binPath) 61 | os.system(binPath) 62 | -------------------------------------------------------------------------------- /example/benchmark/scripts/https.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/https.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/https/${timestamp}/ --recursive 24 | -------------------------------------------------------------------------------- /example/benchmark/scripts/latency.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 2) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = "16777216/" 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = ["uring", "s3crt", "s3"] 15 | threadsUring = [1] 16 | threadsS3 = [1] 17 | throughputGoal = [1] 18 | concurrentRequestsS3Crt = [1] 19 | concurrentRequestsS3 = [1] 20 | concurrentRequestsUring = [1] 21 | dnsresolver = ["throughput", "aws"] 22 | iterations = 3 23 | 24 | for r in runs: 25 | for system in systems: 26 | threadOrThroughput = [] 27 | conccurency = [] 28 | if system == "uring": 29 | threadOrThroughput = threadsUring 30 | conccurency = concurrentRequestsUring 31 | elif system == "s3crt": 32 | threadOrThroughput = throughputGoal 33 | conccurency = concurrentRequestsS3Crt 34 | elif system == "s3": 35 | threadOrThroughput = threadsS3 36 | conccurency = concurrentRequestsS3 37 | 38 | 39 | for c in conccurency: 40 | for t in threadOrThroughput: 41 | for d in dnsresolver: 42 | if system != "uring" and d != "throughput": 43 | continue 44 | if system != "uring" and c < t: 45 | continue 46 | 47 | dirpath = os.path.dirname(os.path.realpath(__file__)) 48 | binPath = "timeout 600 " + dirpath + '/../build/Release/' 49 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 50 | requests = requestsScale 51 | if system == "uring": 52 | requests = min(requestsScale * c * t, rquestsMax) 53 | else: 54 | requests = min(requestsScale * c, rquestsMax) 55 | 56 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + files + " -n " + str(numberOfFiles) 57 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o benchmark_" + str(r) + ".csv" 58 | 59 | print(binPath) 60 | os.system(binPath) 61 | -------------------------------------------------------------------------------- /example/benchmark/scripts/latency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | while : 17 | do 18 | python3 ${SCRIPT_DIR}/latency.py 19 | #------------------------------------------------------ 20 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 21 | mkdir ${SCRIPT_DIR}/results 22 | mv *.csv ${SCRIPT_DIR}/results 23 | mv *.csv.summary ${SCRIPT_DIR}/results 24 | timestamp=$(date +%s) 25 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/latency/${timestamp}/ --recursive 26 | done 27 | 28 | -------------------------------------------------------------------------------- /example/benchmark/scripts/latency_sparse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/latency.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/latency_sparse/${timestamp}/ --recursive 24 | -------------------------------------------------------------------------------- /example/benchmark/scripts/model.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 1) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = "16777216/" 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = ["uring", "s3crt", "s3"] 15 | threadsUring = [1, 4, 8, 10, 12, 14] 16 | threadsS3 = [1, 8, 32, 64, 128, 192, 256] 17 | throughputGoal = [1, 10, 25, 50, 100] 18 | concurrentRequestsS3Crt = [1, 128, 256, 512] 19 | concurrentRequestsS3 = [1, 256, 512] 20 | concurrentRequestsUring = [1, 16, 18, 20] 21 | dnsresolver = ["throughput", "aws"] 22 | iterations = 3 23 | 24 | for r in runs: 25 | for system in systems: 26 | threadOrThroughput = [] 27 | conccurency = [] 28 | if system == "uring": 29 | threadOrThroughput = threadsUring 30 | conccurency = concurrentRequestsUring 31 | elif system == "s3crt": 32 | threadOrThroughput = throughputGoal 33 | conccurency = concurrentRequestsS3Crt 34 | elif system == "s3": 35 | threadOrThroughput = threadsS3 36 | conccurency = concurrentRequestsS3 37 | 38 | 39 | for c in conccurency: 40 | for t in threadOrThroughput: 41 | for d in dnsresolver: 42 | if system != "uring" and d != "throughput": 43 | continue 44 | if system != "uring" and c < t: 45 | continue 46 | 47 | dirpath = os.path.dirname(os.path.realpath(__file__)) 48 | binPath = "timeout 600 " + dirpath + '/../build/Release/' 49 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 50 | requests = requestsScale 51 | if system == "uring": 52 | requests = min(requestsScale * c * t, rquestsMax) 53 | else: 54 | requests = min(requestsScale * c, rquestsMax) 55 | 56 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + files + " -n " + str(numberOfFiles) 57 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o model_" + str(r) + ".csv" 58 | 59 | print(binPath) 60 | os.system(binPath) 61 | -------------------------------------------------------------------------------- /example/benchmark/scripts/model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/model.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/model/${timestamp}/ --recursive 24 | 25 | -------------------------------------------------------------------------------- /example/benchmark/scripts/sizes.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 2) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = ["524288/", "1048576/", "2097152/", "4194304/", "8388608/", "16777216/", "33554432/"] 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = ["uring", "s3crt"] 15 | threadsUring = [1, 12] 16 | threadsS3 = [] 17 | throughputGoal = [100] 18 | concurrentRequestsS3Crt = [1, 512] 19 | concurrentRequestsS3 = [] 20 | concurrentRequestsUring = [1, 18] 21 | dnsresolver = ["throughput", "aws"] 22 | iterations = 3 23 | 24 | for r in runs: 25 | for system in systems: 26 | threadOrThroughput = [] 27 | conccurency = [] 28 | if system == "uring": 29 | threadOrThroughput = threadsUring 30 | conccurency = concurrentRequestsUring 31 | elif system == "s3crt": 32 | threadOrThroughput = throughputGoal 33 | conccurency = concurrentRequestsS3Crt 34 | elif system == "s3": 35 | threadOrThroughput = threadsS3 36 | conccurency = concurrentRequestsS3 37 | 38 | for f in files: 39 | for c in conccurency: 40 | for t in threadOrThroughput: 41 | for d in dnsresolver: 42 | if system != "uring" and d != "throughput": 43 | continue 44 | if system != "uring" and c < t: 45 | continue 46 | 47 | dirpath = os.path.dirname(os.path.realpath(__file__)) 48 | binPath = "timeout 600 " + dirpath + '/../build/Release/' 49 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 50 | requests = requestsScale 51 | if system == "uring": 52 | requests = min(requestsScale * c * t, rquestsMax) 53 | else: 54 | requests = min(requestsScale * c, rquestsMax) 55 | 56 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + f + " -n " + str(numberOfFiles) 57 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o sizes_" + str(r) + ".csv" 58 | 59 | print(binPath) 60 | os.system(binPath) 61 | -------------------------------------------------------------------------------- /example/benchmark/scripts/sizes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/sizes.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/sizes/${timestamp}/ --recursive 24 | -------------------------------------------------------------------------------- /example/benchmark/scripts/sizes_latency.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 2) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = ["1/", "1024/", "524288/", "1048576/", "2097152/", "4194304/", "8388608/", "16777216/", "33554432/"] 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = ["uring", "s3"] 15 | threadsUring = [1] 16 | threadsS3 = [1] 17 | throughputGoal = [] 18 | concurrentRequestsS3Crt = [] 19 | concurrentRequestsS3 = [1] 20 | concurrentRequestsUring = [1] 21 | dnsresolver = ["throughput", "aws"] 22 | iterations = 20 23 | 24 | for r in runs: 25 | for system in systems: 26 | threadOrThroughput = [] 27 | conccurency = [] 28 | if system == "uring": 29 | threadOrThroughput = threadsUring 30 | conccurency = concurrentRequestsUring 31 | elif system == "s3crt": 32 | threadOrThroughput = throughputGoal 33 | conccurency = concurrentRequestsS3Crt 34 | elif system == "s3": 35 | threadOrThroughput = threadsS3 36 | conccurency = concurrentRequestsS3 37 | 38 | for f in files: 39 | for c in conccurency: 40 | for t in threadOrThroughput: 41 | for d in dnsresolver: 42 | if system != "uring" and d != "throughput": 43 | continue 44 | if system != "uring" and c < t: 45 | continue 46 | 47 | dirpath = os.path.dirname(os.path.realpath(__file__)) 48 | binPath = "timeout 6000 " + dirpath + '/../build/Release/' 49 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 50 | requests = requestsScale 51 | if system == "uring": 52 | requests = min(requestsScale * c * t, rquestsMax) 53 | else: 54 | requests = min(requestsScale * c, rquestsMax) 55 | 56 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + f + " -n " + str(numberOfFiles) 57 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o sizes_" + str(r) + ".csv" 58 | 59 | print(binPath) 60 | os.system(binPath) 61 | -------------------------------------------------------------------------------- /example/benchmark/scripts/sizes_latency.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/sizes_latency.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/sizes_latency/${timestamp}/ --recursive 24 | -------------------------------------------------------------------------------- /example/benchmark/scripts/throughput.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 2) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = "16777216/" 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = ["uring", "s3crt", "s3"] 15 | threadsUring = [10, 12, 14] 16 | threadsS3 = [64, 128] 17 | throughputGoal = [100] 18 | concurrentRequestsS3Crt = [128, 256, 512] 19 | concurrentRequestsS3 = [128, 256, 512] 20 | concurrentRequestsUring = [14, 16, 18, 20] 21 | dnsresolver = ["throughput", "aws"] 22 | iterations = 3 23 | 24 | for r in runs: 25 | for system in systems: 26 | threadOrThroughput = [] 27 | conccurency = [] 28 | if system == "uring": 29 | threadOrThroughput = threadsUring 30 | conccurency = concurrentRequestsUring 31 | elif system == "s3crt": 32 | threadOrThroughput = throughputGoal 33 | conccurency = concurrentRequestsS3Crt 34 | elif system == "s3": 35 | threadOrThroughput = threadsS3 36 | conccurency = concurrentRequestsS3 37 | 38 | 39 | for c in conccurency: 40 | for t in threadOrThroughput: 41 | for d in dnsresolver: 42 | if system != "uring" and d != "throughput": 43 | continue 44 | if system != "uring" and c < t: 45 | continue 46 | 47 | dirpath = os.path.dirname(os.path.realpath(__file__)) 48 | binPath = "timeout 600 " + dirpath + '/../build/Release/' 49 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 50 | requests = requestsScale 51 | if system == "uring": 52 | requests = min(requestsScale * c * t, rquestsMax) 53 | else: 54 | requests = min(requestsScale * c, rquestsMax) 55 | 56 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + files + " -n " + str(numberOfFiles) 57 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o throughput_" + str(r) + ".csv" 58 | 59 | print(binPath) 60 | os.system(binPath) 61 | -------------------------------------------------------------------------------- /example/benchmark/scripts/throughput.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER=anyblob-data 7 | export CONTAINER_RESULTS=anyblob-results 8 | if [[ -z "${REGION}" ]]; then 9 | export REGION=eu-central-1 10 | fi 11 | if [[ "${REGION}" != "eu-central-1" ]]; then 12 | export CONTAINER=anyblob-data-${REGION} 13 | export CONTAINER_RESULTS=anyblob-results-${REGION} 14 | fi 15 | #------------------------------------------------------ 16 | python3 ${SCRIPT_DIR}/throughput.py 17 | #------------------------------------------------------ 18 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 19 | mkdir ${SCRIPT_DIR}/results 20 | mv *.csv ${SCRIPT_DIR}/results 21 | mv *.csv.summary ${SCRIPT_DIR}/results 22 | timestamp=$(date +%s) 23 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/throughput/${timestamp}/ --recursive 24 | -------------------------------------------------------------------------------- /example/benchmark/scripts/throughput_sparse.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from datetime import datetime 4 | import multiprocessing 5 | 6 | runs = range(0, 2) 7 | bucket = os.environ['CONTAINER'] 8 | region = os.environ['REGION'] 9 | 10 | files = "16777216/" 11 | numberOfFiles = 4096 12 | requestsScale = 256 13 | rquestsMax = 50000 14 | systems = [] 15 | threadsUring = [12] 16 | threadsS3 = [128] 17 | throughputGoal = [100] 18 | concurrentRequestsS3Crt = [256] 19 | concurrentRequestsS3 = [256] 20 | concurrentRequestsUring = [20] 21 | dnsresolver = ["throughput"] 22 | iterations = 10 23 | 24 | print(bucket) 25 | 26 | if "s3crt" in bucket: 27 | systems = ["s3crt"] 28 | elif "s3" in bucket: 29 | systems = ["s3"] 30 | elif "throughput" in bucket: 31 | systems = ["uring"] 32 | elif "uring" in bucket: 33 | systems = ["uring"] 34 | dnsresolver = ["aws"] 35 | 36 | for r in runs: 37 | for system in systems: 38 | threadOrThroughput = [] 39 | conccurency = [] 40 | if system == "uring": 41 | threadOrThroughput = threadsUring 42 | conccurency = concurrentRequestsUring 43 | elif system == "s3crt": 44 | threadOrThroughput = throughputGoal 45 | conccurency = concurrentRequestsS3Crt 46 | elif system == "s3": 47 | threadOrThroughput = threadsS3 48 | conccurency = concurrentRequestsS3 49 | 50 | 51 | for c in conccurency: 52 | for t in threadOrThroughput: 53 | for d in dnsresolver: 54 | if system != "uring" and d != "throughput": 55 | continue 56 | if system != "uring" and c < t: 57 | continue 58 | 59 | dirpath = os.path.dirname(os.path.realpath(__file__)) 60 | binPath = "timeout 6000 " + dirpath + '/../build/Release/' 61 | binPath = binPath + "AnyBlobBenchmark aws bandwidth -i " + str(iterations) 62 | requests = requestsScale 63 | if system == "uring": 64 | requests = min(requestsScale * c * t, rquestsMax) 65 | else: 66 | requests = min(requestsScale * c, rquestsMax) 67 | 68 | binPath = binPath + " -b " + bucket + " -r " + region + " -l " + str(requests) + " -f " + files + " -n " + str(numberOfFiles) 69 | binPath = binPath + " -c " + str(c) + " -t " + str(t) + " -a " + system + " -d " + d + " -o throughput_sparse_" + str(r) + ".csv" 70 | 71 | print(binPath) 72 | os.system(binPath) 73 | -------------------------------------------------------------------------------- /example/benchmark/scripts/throughput_sparse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------ 3 | # Export Settings 4 | #------------------------------------------------------ 5 | SCRIPT_DIR="$(dirname $(readlink -f $0))" 6 | export CONTAINER_RESULTS=anyblob-results 7 | if [[ -z "${CONTAINER}" ]]; then 8 | export CONTAINER=anyblob-data 9 | fi 10 | if [[ -z "${REGION}" ]]; then 11 | export REGION=eu-central-1 12 | fi 13 | if [[ "${REGION}" != "eu-central-1" ]]; then 14 | export CONTAINER_RESULTS=anyblob-results-${REGION} 15 | fi 16 | #------------------------------------------------------ 17 | python3 ${SCRIPT_DIR}/throughput_sparse.py 18 | #------------------------------------------------------ 19 | aws s3api create-bucket --bucket $CONTAINER_RESULTS --region $REGION --create-bucket-configuration LocationConstraint=$REGION 20 | mkdir ${SCRIPT_DIR}/results 21 | mv *.csv ${SCRIPT_DIR}/results 22 | mv *.csv.summary ${SCRIPT_DIR}/results 23 | timestamp=$(date +%s) 24 | aws s3 cp ${SCRIPT_DIR}/results s3://${CONTAINER_RESULTS}/throughput_sparse/${timestamp}/ --recursive 25 | -------------------------------------------------------------------------------- /example/benchmark/src/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE SRC_CPP src/*.cpp) 6 | list(REMOVE_ITEM SRC_CPP ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp) 7 | 8 | 9 | -------------------------------------------------------------------------------- /example/benchmark/thirdparty/anyblob.cmake: -------------------------------------------------------------------------------- 1 | ../../../anyblob.cmake -------------------------------------------------------------------------------- /example/benchmark/thirdparty/awssdk.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # AWS SDK 3 | # --------------------------------------------------------------------------- 4 | Include(ExternalProject) 5 | 6 | set(AWS_REQUIRED_LIBS 7 | aws-cpp-sdk-s3-crt 8 | aws-cpp-sdk-s3 9 | aws-cpp-sdk-core 10 | aws-cpp-sdk-kms 11 | aws-cpp-sdk-s3-encryption 12 | aws-crt-cpp 13 | aws-c-s3 14 | aws-c-auth 15 | aws-c-http 16 | aws-c-io 17 | aws-c-event-stream 18 | aws-c-cal 19 | aws-c-compression 20 | aws-c-mqtt 21 | aws-c-sdkutils 22 | s2n 23 | aws-checksums 24 | aws-c-common 25 | ) 26 | 27 | set(AWS_INSTALL_DIR "thirdparty/awssdk/install") 28 | set(AWS_BUILD_BYPRODUCTS "") 29 | foreach(LIBNAME ${AWS_REQUIRED_LIBS}) 30 | list(APPEND AWS_BUILD_BYPRODUCTS ${AWS_INSTALL_DIR}/lib/lib${LIBNAME}.a) 31 | endforeach() 32 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations -Wno-stringop-truncation -Wno-nonnull") 33 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=deprecated-declarations -Wno-stringop-truncation -Wno-nonnull") 34 | 35 | ExternalProject_Add(awssdk 36 | GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git 37 | GIT_TAG "1.11.31" 38 | PREFIX "thirdparty/awssdk" 39 | INSTALL_DIR ${AWS_INSTALL_DIR} 40 | CMAKE_ARGS 41 | -DBUILD_ONLY=core\\$s3\\$s3-crt\\$s3-encryption 42 | -DBUILD_SHARED_LIBS:BOOL=OFF 43 | -DBUILD_STATIC_LIBS:BOOL=ON 44 | -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON 45 | -DENABLE_TESTING:BOOL=OFF 46 | -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 47 | -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/${AWS_INSTALL_DIR} 48 | -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} 49 | -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} 50 | -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} 51 | BUILD_BYPRODUCTS ${AWS_BUILD_BYPRODUCTS} 52 | ) 53 | 54 | ExternalProject_Get_Property(awssdk INSTALL_DIR) 55 | set(AWS_INCLUDE_DIR ${INSTALL_DIR}/include) 56 | file(MAKE_DIRECTORY ${AWS_INCLUDE_DIR}) 57 | 58 | 59 | foreach(LIBNAME ${AWS_REQUIRED_LIBS}) 60 | add_library(${LIBNAME} STATIC IMPORTED) 61 | set_property(TARGET ${LIBNAME} PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/lib${LIBNAME}.a) 62 | set_property(TARGET ${LIBNAME} APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${AWS_INCLUDE_DIR}) 63 | add_dependencies(${LIBNAME} awssdk) 64 | endforeach() 65 | 66 | # Load OpenSSL as a static lib 67 | # set(OPENSSL_USE_STATIC_LIBS TRUE) 68 | find_package(OpenSSL REQUIRED) 69 | find_package(CURL REQUIRED) 70 | 71 | set(AWS_LINK_LIBRARIES ${AWS_REQUIRED_LIBS} OpenSSL::Crypto ${CURL_LIBRARY} dl) 72 | -------------------------------------------------------------------------------- /example/simple/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.8) 2 | 3 | project(SimpleAnyBlob) 4 | 5 | # --------------------------------------------------------------------------- 6 | # Environment 7 | # --------------------------------------------------------------------------- 8 | 9 | if (NOT UNIX) 10 | message(STATUS FATAL_ERROR "Unsupported OS") 11 | endif (NOT UNIX) 12 | 13 | set(CMAKE_CXX_STANDARD 20) 14 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 15 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 16 | 17 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -Wall -fsanitize=undefined") 18 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fsanitize=address -march=native -Wall") 19 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -march=native -Wall") 20 | add_compile_options(-Wall -Wextra -Werror) 21 | 22 | # --------------------------------------------------------------------------- 23 | # AnyBlob 24 | # --------------------------------------------------------------------------- 25 | 26 | include("${PROJECT_SOURCE_DIR}/anyblob.cmake") 27 | 28 | # --------------------------------------------------------------------------- 29 | # Includes 30 | # --------------------------------------------------------------------------- 31 | 32 | include_directories(SYSTEM ${ANYBLOB_INCLUDE_DIR}) 33 | 34 | # --------------------------------------------------------------------------- 35 | # Executable 36 | # --------------------------------------------------------------------------- 37 | 38 | add_executable(SimpleAnyBlob main.cpp ${INCLUDE_HPP}) 39 | target_link_libraries(SimpleAnyBlob AnyBlob) 40 | -------------------------------------------------------------------------------- /example/simple/README.md: -------------------------------------------------------------------------------- 1 | # SimpleAnyBlob 2 | This directory contains a simple AnyBlob retrieval executable for objects. 3 | 4 | ## Build 5 | 6 | sudo apt update && sudo apt install liburing-dev openssl libssl-dev libjemalloc-dev g++ lld cmake 7 | git clone --recursive https://github.com/durner/AnyBlob 8 | cd AnyBlob/example/simple 9 | mkdir -p build/Release 10 | cd build/Release 11 | cmake ../.. -DCMAKE_BUILD_TYPE=Release 12 | make -j16 13 | 14 | -------------------------------------------------------------------------------- /example/simple/anyblob.cmake: -------------------------------------------------------------------------------- 1 | ../../anyblob.cmake -------------------------------------------------------------------------------- /example/simple/main.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/provider.hpp" 2 | #include "network/tasked_send_receiver.hpp" 3 | #include "network/transaction.hpp" 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2022 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | using namespace std; 15 | //--------------------------------------------------------------------------- 16 | int main(int /*argc*/, char** /*argv*/) { 17 | // The file to be downloaded 18 | auto bucketName = "s3://anyblob:eu-central-1"; 19 | auto fileName = "anyblob/anyblob.txt"; 20 | 21 | // Create a new task group 22 | anyblob::network::TaskedSendReceiverGroup group; 23 | 24 | // Create an AnyBlob scheduler object for the group 25 | auto sendReceiverHandle = group.getHandle(); 26 | 27 | // Create the provider for the corresponding filename 28 | bool https = false; 29 | auto provider = anyblob::cloud::Provider::makeProvider(bucketName, https, "", "", &sendReceiverHandle); 30 | 31 | // Optionally init the specialized aws cache 32 | // provider->initCache(sendReceiverHandle); 33 | 34 | // Update the concurrency according to instance settings 35 | auto config = provider->getConfig(sendReceiverHandle); 36 | group.setConfig(config); 37 | 38 | // Create the get request 39 | anyblob::network::Transaction getTxn(provider.get()); 40 | getTxn.getObjectRequest(fileName); 41 | 42 | // Create optionally more requests within a transaction 43 | auto requests = 64u; 44 | for (auto i = 0u; i < requests; i++) 45 | getTxn.getObjectRequest(fileName); 46 | 47 | // Retrieve the request synchronously with the scheduler object on this thread 48 | getTxn.processSync(sendReceiverHandle); 49 | 50 | // Get and print the result 51 | for (const auto& it : getTxn) { 52 | // Check if the request was successful 53 | if (!it.success()) { 54 | cout << "Request was not successful!" << endl; 55 | continue; 56 | } 57 | // Simple string_view interface 58 | cout << it.getResult() << endl; 59 | 60 | // Advanced raw interface 61 | // Note that the data lies in the data buffer but after the offset to skip the HTTP header 62 | // Note that the size is already without the header, so the full request has size + offset length 63 | string_view rawDataString(reinterpret_cast(it.getData()) + it.getOffset(), it.getSize()); 64 | cout << rawDataString << endl; 65 | } 66 | 67 | return 0; 68 | } 69 | //--------------------------------------------------------------------------- 70 | -------------------------------------------------------------------------------- /include/cloud/aws_cache.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/cache.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | //--------------------------------------------------------------------------- 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the AWS cache logic 17 | class AWSCache : public network::Cache { 18 | /// The good mtu cache 19 | std::unordered_map _mtuCache; 20 | 21 | public: 22 | /// The constructor 23 | AWSCache(); 24 | /// The address resolving 25 | virtual std::unique_ptr resolve(std::string hostname, unsigned port, bool tls) override; 26 | /// The destructor 27 | virtual ~AWSCache() = default; 28 | }; 29 | //--------------------------------------------------------------------------- 30 | }; // namespace cloud 31 | //--------------------------------------------------------------------------- 32 | }; // namespace anyblob 33 | -------------------------------------------------------------------------------- /include/cloud/aws_instances.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the AWS instances 17 | struct AWSInstance : public Provider::Instance { 18 | /// Gets a vector of instance type infos 19 | [[nodiscard]] static std::vector getInstanceDetails(); 20 | }; 21 | //--------------------------------------------------------------------------- 22 | }; // namespace cloud 23 | }; // namespace anyblob 24 | -------------------------------------------------------------------------------- /include/cloud/aws_signer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_request.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2021 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the AWS S3 Signing Logic 17 | /// It follows the v4 docu: https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html 18 | class AWSSigner { 19 | public: 20 | struct StringToSign { 21 | /// The canonical request 22 | network::HttpRequest& request; 23 | /// The region 24 | std::string region; 25 | /// The service 26 | std::string service; 27 | /// The request sha 28 | std::string requestSHA; 29 | /// The signed headers 30 | std::string signedHeaders; 31 | /// The payload hash 32 | std::string payloadHash; 33 | }; 34 | 35 | /// Creates the canonical request from the input 36 | static void encodeCanonicalRequest(network::HttpRequest& request, StringToSign& stringToSign, const uint8_t* bodyData = nullptr, uint64_t bodyLength = 0); 37 | /// Calculates the signature 38 | [[nodiscard]] static std::string createSignedRequest(const std::string& keyId, const std::string& secret, const StringToSign& stringToSign); 39 | 40 | private: 41 | /// Creates the string to sogn 42 | [[nodiscard]] static std::string createStringToSign(const StringToSign& stringToSign); 43 | }; 44 | //--------------------------------------------------------------------------- 45 | }; // namespace cloud 46 | }; // namespace anyblob 47 | -------------------------------------------------------------------------------- /include/cloud/azure.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/azure_instances.hpp" 3 | #include "cloud/provider.hpp" 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2022 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | //--------------------------------------------------------------------------- 16 | namespace network { 17 | class TaskedSendReceiver; 18 | }; // namespace network 19 | //--------------------------------------------------------------------------- 20 | namespace cloud { 21 | //--------------------------------------------------------------------------- 22 | namespace test { 23 | class AzureTester; 24 | }; // namespace test 25 | //--------------------------------------------------------------------------- 26 | /// Implements the Azure logic 27 | class Azure : public Provider { 28 | public: 29 | /// The settings for azure requests 30 | struct Settings { 31 | /// The container name 32 | std::string container; 33 | /// The port 34 | uint32_t port; 35 | }; 36 | 37 | /// The secret 38 | struct Secret { 39 | /// The IAM user 40 | std::string accountName; 41 | /// The private key 42 | std::string privateKey; 43 | }; 44 | 45 | /// The fake XMS timestamp 46 | const char* fakeXMSTimestamp = "Fri, 01 Jan 2100 00:00:00 GMT"; 47 | 48 | private: 49 | /// The settings 50 | Settings _settings; 51 | /// The secret 52 | std::unique_ptr _secret; 53 | 54 | /// Inits key from file 55 | void initKey(); 56 | 57 | public: 58 | /// Get instance details 59 | [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiverHandle& sendReceiver) override; 60 | /// Get the region of the instance 61 | [[nodiscard]] static std::string getRegion(network::TaskedSendReceiverHandle& sendReceiver); 62 | 63 | /// The constructor 64 | Azure(const RemoteInfo& info, const std::string& clientEmail, const std::string& key) : _settings({info.bucket, info.port}) { 65 | assert(info.provider == Provider::CloudService::Azure); 66 | _type = info.provider; 67 | _secret = std::make_unique(); 68 | _secret->accountName = clientEmail; 69 | _secret->privateKey = key; 70 | initKey(); 71 | } 72 | 73 | private: 74 | /// Get the settings 75 | [[nodiscard]] inline Settings getSettings() { return _settings; } 76 | 77 | /// Builds the http request for downloading a blob or listing the directory 78 | [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; 79 | /// Builds the http request for putting objects without the object data itself 80 | [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const override; 81 | // Builds the http request for deleting an objects 82 | [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; 83 | 84 | /// Get the address of the server 85 | [[nodiscard]] std::string getAddress() const override; 86 | /// Get the port of the server 87 | [[nodiscard]] uint32_t getPort() const override; 88 | 89 | /// Builds the info http request 90 | [[nodiscard]] static std::unique_ptr> downloadInstanceInfo(); 91 | /// Get the IAM address 92 | [[nodiscard]] static constexpr std::string_view getIAMAddress() { 93 | return "169.254.169.254"; 94 | } 95 | /// Get the port of the IAM server 96 | [[nodiscard]] static constexpr uint32_t getIAMPort() { 97 | return 80; 98 | } 99 | 100 | friend Provider; 101 | friend test::AzureTester; 102 | }; 103 | //--------------------------------------------------------------------------- 104 | } // namespace cloud 105 | } // namespace anyblob 106 | -------------------------------------------------------------------------------- /include/cloud/azure_instances.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the Azure instances 17 | struct AzureInstance : public Provider::Instance { 18 | /// Gets a vector of instance type infos 19 | [[nodiscard]] static std::vector getInstanceDetails(); 20 | }; 21 | //--------------------------------------------------------------------------- 22 | }; // namespace cloud 23 | }; // namespace anyblob 24 | -------------------------------------------------------------------------------- /include/cloud/azure_signer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_request.hpp" 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2022 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace cloud { 16 | //--------------------------------------------------------------------------- 17 | /// Implements the Azure Signing Logic 18 | /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually 19 | class AzureSigner { 20 | public: 21 | /// Builds the signed url 22 | [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request); 23 | }; 24 | //--------------------------------------------------------------------------- 25 | }; // namespace cloud 26 | }; // namespace anyblob 27 | -------------------------------------------------------------------------------- /include/cloud/gcp_instances.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the GCP instances 17 | struct GCPInstance : public Provider::Instance { 18 | /// Gets a vector of instance type infos 19 | [[nodiscard]] static std::vector getInstanceDetails(); 20 | }; 21 | //--------------------------------------------------------------------------- 22 | }; // namespace cloud 23 | }; // namespace anyblob 24 | -------------------------------------------------------------------------------- /include/cloud/gcp_signer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_request.hpp" 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2022 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace cloud { 16 | //--------------------------------------------------------------------------- 17 | /// Implements the GCP Signing Logic 18 | /// It follows the v4 docu: https://cloud.google.com/storage/docs/access-control/signing-urls-manually 19 | class GCPSigner { 20 | public: 21 | struct StringToSign { 22 | /// The region 23 | std::string region; 24 | /// The service 25 | std::string service; 26 | /// The signed headers 27 | std::string signedHeaders; 28 | }; 29 | 30 | /// Builds the signed url 31 | [[nodiscard]] static std::string createSignedRequest(const std::string& serviceAccountEmail, const std::string& privateRSA, network::HttpRequest& request, StringToSign& stringToSign); 32 | }; 33 | //--------------------------------------------------------------------------- 34 | }; // namespace cloud 35 | }; // namespace anyblob 36 | -------------------------------------------------------------------------------- /include/cloud/http.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2024 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | //--------------------------------------------------------------------------- 15 | namespace network { 16 | class TaskedSendReceiver; 17 | }; // namespace network 18 | //--------------------------------------------------------------------------- 19 | namespace cloud { 20 | //--------------------------------------------------------------------------- 21 | /// Implements a simple http request logic 22 | class HTTP : public Provider { 23 | public: 24 | /// The settings for azure requests 25 | struct Settings { 26 | /// The container name 27 | std::string hostname; 28 | /// The port 29 | uint32_t port; 30 | }; 31 | 32 | private: 33 | /// The settings 34 | Settings _settings; 35 | 36 | public: 37 | /// The constructor 38 | HTTP(const RemoteInfo& info) : _settings({info.endpoint, info.port}) { 39 | assert(info.provider == Provider::CloudService::HTTP || info.provider == Provider::CloudService::HTTPS); 40 | _type = info.provider; 41 | } 42 | 43 | private: 44 | /// Get the settings 45 | [[nodiscard]] inline Settings getSettings() { return _settings; } 46 | 47 | /// Builds the http request for downloading a blob or listing the directory 48 | [[nodiscard]] std::unique_ptr> getRequest(const std::string& filePath, const std::pair& range) const override; 49 | /// Builds the http request for putting objects without the object data itself 50 | [[nodiscard]] std::unique_ptr> putRequest(const std::string& filePath, std::string_view object) const override; 51 | // Builds the http request for deleting an objects 52 | [[nodiscard]] std::unique_ptr> deleteRequest(const std::string& filePath) const override; 53 | 54 | /// Get the address of the server 55 | [[nodiscard]] std::string getAddress() const override; 56 | /// Get the port of the server 57 | [[nodiscard]] uint32_t getPort() const override; 58 | /// Get the instance details 59 | [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiverHandle& sendReceiver) override; 60 | 61 | friend Provider; 62 | }; 63 | //--------------------------------------------------------------------------- 64 | } // namespace cloud 65 | } // namespace anyblob 66 | -------------------------------------------------------------------------------- /include/cloud/ibm.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/aws.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | //--------------------------------------------------------------------------- 13 | namespace network { 14 | class TaskedSendReceiver; 15 | }; // namespace network 16 | //--------------------------------------------------------------------------- 17 | namespace cloud { 18 | //--------------------------------------------------------------------------- 19 | /// Implements the IBM logic using the AWS S3 compatibility API 20 | class IBM : public AWS { 21 | public: 22 | /// The constructor 23 | explicit IBM(const RemoteInfo& info) : AWS(info) { 24 | assert(info.provider == Provider::CloudService::IBM); 25 | // COS requires path-style bucket URLs - similar to MinIO, thus we use the endpoint setting 26 | _settings.endpoint = getAddress(); 27 | } 28 | /// The custom endpoint constructor 29 | IBM(const RemoteInfo& info, const std::string& keyId, const std::string& key) : AWS(info, keyId, key) { 30 | // COS requires path-style bucket URLs - similar to MinIO, thus we use the endpoint setting 31 | _settings.endpoint = getAddress(); 32 | } 33 | /// Get the address of the server 34 | [[nodiscard]] std::string getAddress() const override; 35 | /// Get the instance details 36 | [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiverHandle& sendReceiver) override; 37 | 38 | friend Provider; 39 | }; 40 | //--------------------------------------------------------------------------- 41 | } // namespace cloud 42 | } // namespace anyblob 43 | -------------------------------------------------------------------------------- /include/cloud/ibm_instances.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the IBM instances 17 | struct IBMInstance : public Provider::Instance { 18 | /// Gets a vector of instance type infos 19 | [[nodiscard]] static std::vector getInstanceDetails(); 20 | }; 21 | //--------------------------------------------------------------------------- 22 | }; // namespace cloud 23 | }; // namespace anyblob 24 | -------------------------------------------------------------------------------- /include/cloud/minio.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/aws.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | //--------------------------------------------------------------------------- 13 | namespace network { 14 | class TaskedSendReceiver; 15 | }; // namespace network 16 | //--------------------------------------------------------------------------- 17 | namespace cloud { 18 | //--------------------------------------------------------------------------- 19 | /// Implements the MinIO logic using the AWS S3 compatibility API 20 | class MinIO : public AWS { 21 | public: 22 | /// The constructor 23 | explicit MinIO(const RemoteInfo& info) : AWS(info) { 24 | assert(info.provider == Provider::CloudService::MinIO); 25 | } 26 | /// The custom endpoint constructor 27 | MinIO(const RemoteInfo& info, const std::string& keyId, const std::string& key) : AWS(info, keyId, key) {} 28 | /// Get the address of the server 29 | [[nodiscard]] std::string getAddress() const override; 30 | /// Get the instance details 31 | [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiverHandle& sendReceiver) override; 32 | /// Set the upload split size 33 | constexpr void setMultipartUploadSize(uint64_t size) { _multipartUploadSize = size; } 34 | 35 | friend Provider; 36 | }; 37 | //--------------------------------------------------------------------------- 38 | } // namespace cloud 39 | } // namespace anyblob 40 | -------------------------------------------------------------------------------- /include/cloud/oracle.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/aws.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | //--------------------------------------------------------------------------- 13 | namespace network { 14 | class TaskedSendReceiver; 15 | }; // namespace network 16 | //--------------------------------------------------------------------------- 17 | namespace cloud { 18 | //--------------------------------------------------------------------------- 19 | /// Implements the Oracle logic using the AWS S3 compatibility API 20 | class Oracle : public AWS { 21 | public: 22 | /// The constructor 23 | explicit Oracle(const RemoteInfo& info) : AWS(info) { 24 | assert(info.provider == Provider::CloudService::Oracle); 25 | } 26 | /// The custom endpoint constructor 27 | Oracle(const RemoteInfo& info, const std::string& keyId, const std::string& key) : AWS(info, keyId, key) {} 28 | /// Get the address of the server 29 | [[nodiscard]] std::string getAddress() const override; 30 | /// Get the instance details 31 | [[nodiscard]] Provider::Instance getInstanceDetails(network::TaskedSendReceiverHandle& sendReceiver) override; 32 | 33 | friend Provider; 34 | }; 35 | //--------------------------------------------------------------------------- 36 | } // namespace cloud 37 | } // namespace anyblob 38 | -------------------------------------------------------------------------------- /include/cloud/oracle_instances.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace cloud { 15 | //--------------------------------------------------------------------------- 16 | /// Implements the Oracle instances 17 | struct OracleInstance : public Provider::Instance { 18 | /// Gets a vector of instance type infos 19 | [[nodiscard]] static std::vector getInstanceDetails(); 20 | }; 21 | //--------------------------------------------------------------------------- 22 | }; // namespace cloud 23 | }; // namespace anyblob 24 | -------------------------------------------------------------------------------- /include/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE INCLUDE_HPP "include/*.hpp") 6 | -------------------------------------------------------------------------------- /include/network/cache.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/tls_connection.hpp" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | //--------------------------------------------------------------------------- 12 | // AnyBlob - Universal Cloud Object Storage Library 13 | // Dominik Durner, 2022 14 | // 15 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 16 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 17 | // SPDX-License-Identifier: MPL-2.0 18 | //--------------------------------------------------------------------------- 19 | namespace anyblob { 20 | //--------------------------------------------------------------------------- 21 | namespace network { 22 | //--------------------------------------------------------------------------- 23 | class IOUringSocket; 24 | //--------------------------------------------------------------------------- 25 | /// The addr resolver and cacher, which is not thread safe 26 | class Cache { 27 | public: 28 | /// The dns entry 29 | struct DnsEntry { 30 | /// The addr 31 | std::unique_ptr addr; 32 | /// The cache priority 33 | int cachePriority = 0; 34 | 35 | /// The constructor 36 | DnsEntry(std::unique_ptr address, int cachePriority = 0) : addr(move(address)), cachePriority(cachePriority) {} 37 | }; 38 | 39 | /// The fd socket entry 40 | struct SocketEntry { 41 | /// The DnsEntry 42 | std::unique_ptr dns; 43 | /// The optional tls connection 44 | std::unique_ptr tls; 45 | /// The timestamp 46 | size_t timestamp; 47 | /// The fd 48 | int32_t fd; 49 | /// The port 50 | unsigned port; 51 | /// The hostname 52 | std::string hostname; 53 | 54 | /// The constructor 55 | SocketEntry(std::string hostname, unsigned port); 56 | }; 57 | 58 | protected: 59 | /// The cache, uses hostname as key, multimap traversals same keys in the insertion order 60 | std::multimap> _cache; 61 | /// The fifo deletion of sockets to help reduce open fds (ulimit -n issue) 62 | std::map _fifo; 63 | /// The timestamp counter for deletion 64 | size_t _timestamp = 0; 65 | /// The default priority 66 | int _defaultPriority = 8; 67 | 68 | public: 69 | /// The address resolving 70 | virtual std::unique_ptr resolve(std::string hostname, unsigned port, bool tls); 71 | /// Start the timing and advance to the next cache bucket 72 | virtual void startSocket(int /*fd*/) {} 73 | /// Stops the socket and either closes the connection or cashes it 74 | virtual void stopSocket(std::unique_ptr socketEntry, uint64_t bytes, unsigned cachedEntries, bool reuseSocket); 75 | /// Shutdown of the socket and clears the same addresses 76 | virtual void shutdownSocket(std::unique_ptr socketEntry, unsigned cachedEntries); 77 | /// The destructor 78 | virtual ~Cache(); 79 | 80 | /// Get the tld 81 | static std::string_view tld(std::string_view domain); 82 | 83 | friend IOUringSocket; 84 | }; 85 | //--------------------------------------------------------------------------- 86 | }; // namespace network 87 | //--------------------------------------------------------------------------- 88 | }; // namespace anyblob 89 | -------------------------------------------------------------------------------- /include/network/config.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2023 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace network { 14 | //--------------------------------------------------------------------------- 15 | /// Config for number of retriever threads and bandwidth 16 | struct Config { 17 | /// Default concurrent requests to achieve coreThroughput (based on AWS experiments) 18 | static constexpr uint64_t defaultCoreConcurrency = 20; 19 | /// Default throughput per core in Mbit/s (based on AWS experiments) 20 | /// Per-object bandwidth: 8,000 Mbits / 20 Requests = 400 Mbits / Requests = 50 MiBs / Request 21 | /// Total requests example: 100,000 Mbits / 400 Mbits = 250 Requests 22 | static constexpr uint64_t defaultCoreThroughput = 8000; 23 | 24 | /// Throughput per core in Mbit/s 25 | uint64_t coreThroughput; 26 | /// Concurrent requests to achieve coreThroughput 27 | unsigned coreConcurreny; 28 | /// The network performance in Mbit/s 29 | uint64_t network; 30 | 31 | /// Get the network bandwidth 32 | constexpr auto bandwidth() const { return network; } 33 | /// Get the number of requests per core 34 | constexpr auto coreRequests() const { return coreConcurreny; } 35 | /// Get the number of retriever threads to saturate bandwidth 36 | constexpr auto retrievers() const { return (network + coreThroughput - 1) / coreThroughput; } 37 | /// Get the total outstanding requests 38 | constexpr auto totalRequests() const { return retrievers() * coreRequests(); } 39 | }; 40 | //--------------------------------------------------------------------------- 41 | } // namespace network 42 | } // namespace anyblob 43 | -------------------------------------------------------------------------------- /include/network/connection_manager.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/cache.hpp" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | //--------------------------------------------------------------------------- 10 | // AnyBlob - Universal Cloud Object Storage Library 11 | // Dominik Durner, 2024 12 | // 13 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 14 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 15 | // SPDX-License-Identifier: MPL-2.0 16 | //--------------------------------------------------------------------------- 17 | namespace anyblob { 18 | namespace network { 19 | //--------------------------------------------------------------------------- 20 | class TLSConnection; 21 | class TLSContext; 22 | struct DnsEntry; 23 | //--------------------------------------------------------------------------- 24 | // This class acts as the connection enabler, closer, and main 25 | // cache for sockets and their optional tls connection. 26 | // We further add the DNS resolution to the connection manager. 27 | class ConnectionManager { 28 | public: 29 | /// The tcp settings 30 | struct TCPSettings { 31 | /// flag for nonBlocking 32 | int nonBlocking = 1; 33 | /// flag for noDelay 34 | int noDelay = 0; 35 | /// flag for recv no wait 36 | int recvNoWait = 0; 37 | /// flag for keepAlive 38 | int keepAlive = 1; 39 | /// time for tcp keepIdle 40 | int keepIdle = 1; 41 | /// time for tcp keepIntvl 42 | int keepIntvl = 1; 43 | /// probe count 44 | int keepCnt = 1; 45 | /// recv buffer for tcp 46 | int recvBuffer = 0; 47 | /// Maximum segment size 48 | int mss = 0; 49 | /// Reuse port 50 | int reusePorts = 0; 51 | /// Lingering of tcp packets 52 | int linger = 1; 53 | /// The timeout in usec 54 | int timeout = 500 * 1000; 55 | /// Reuse sockets 56 | int reuse = 1; 57 | /// The kernel timeout parameter 58 | __kernel_timespec kernelTimeout; 59 | 60 | TCPSettings() { 61 | kernelTimeout.tv_sec = 0; 62 | kernelTimeout.tv_nsec = timeout * 1000; 63 | } 64 | }; 65 | 66 | private: 67 | /// The socket wrapper 68 | std::unique_ptr _socketWrapper; 69 | /// The active sockets 70 | std::unordered_map> _fdSockets; 71 | /// Cache 72 | std::unordered_map> _cache; 73 | /// The tls context 74 | std::unique_ptr _context; 75 | 76 | /// The counter of current connection managers 77 | static std::atomic _activeConnectionManagers; 78 | /// The maximum number of cached fds, TODO: access ulimit for this 79 | constexpr static unsigned _maxCachedFds = 512; 80 | 81 | public: 82 | /// The constructor 83 | explicit ConnectionManager(unsigned uringEntries); 84 | /// The destructor 85 | ~ConnectionManager(); 86 | 87 | /// Creates a new socket connection 88 | [[nodiscard]] int32_t connect(std::string hostname, uint32_t port, bool tls, const TCPSettings& tcpSettings, int retryLimit = 0); 89 | /// Disconnects the socket 90 | void disconnect(int32_t fd, const TCPSettings* tcpSettings = nullptr, uint64_t bytes = 0, bool forceShutdown = false); 91 | 92 | /// Add domain-specific cache 93 | void addCache(const std::string& hostname, std::unique_ptr cache); 94 | /// Checks for a timeout 95 | bool checkTimeout(int fd, const TCPSettings& settings); 96 | 97 | /// Get the socket 98 | IOUringSocket& getSocketConnection() { 99 | assert(_socketWrapper); 100 | return *_socketWrapper.get(); 101 | } 102 | 103 | /// Get the tls connection of the fd 104 | TLSConnection* getTLSConnection(int32_t fd); 105 | }; 106 | //--------------------------------------------------------------------------- 107 | }; // namespace network 108 | }; // namespace anyblob 109 | -------------------------------------------------------------------------------- /include/network/http_helper.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_response.hpp" 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2021 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace network { 16 | //--------------------------------------------------------------------------- 17 | /// Implements an helper to resolve http requests 18 | class HttpHelper { 19 | public: 20 | /// The encoding 21 | enum class Encoding : uint8_t { 22 | Unknown, 23 | ContentLength, 24 | ChunkedEncoding 25 | }; 26 | 27 | struct Info { 28 | /// The response header 29 | HttpResponse response; 30 | /// The maximum length 31 | uint64_t length; 32 | /// The header length 33 | uint32_t headerLength; 34 | /// The encoding 35 | Encoding encoding; 36 | }; 37 | 38 | private: 39 | /// Detect the protocol 40 | [[nodiscard]] static Info detect(std::string_view s); 41 | 42 | public: 43 | /// Retrieve the content without http meta info, note that this changes data 44 | [[nodiscard]] static std::string_view retrieveContent(const uint8_t* data, uint64_t length, std::unique_ptr& info); 45 | /// Detect end / content 46 | [[nodiscard]] static bool finished(const uint8_t* data, uint64_t length, std::unique_ptr& info); 47 | }; 48 | //--------------------------------------------------------------------------- 49 | } // namespace network 50 | } // namespace anyblob 51 | -------------------------------------------------------------------------------- /include/network/http_message.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/message_task.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace network { 14 | //--------------------------------------------------------------------------- 15 | /// Implements a http message roundtrip 16 | struct HTTPMessage : public MessageTask { 17 | /// HTTP info header 18 | std::unique_ptr info; 19 | 20 | /// The constructor 21 | HTTPMessage(OriginalMessage* sendingMessage, ConnectionManager::TCPSettings& tcpSettings, uint32_t chunkSize); 22 | /// The destructor 23 | ~HTTPMessage() override = default; 24 | /// The message excecute callback 25 | MessageState execute(ConnectionManager& connectionManager) override; 26 | /// Reset for restart 27 | void reset(ConnectionManager& connectionManager, bool aborted); 28 | }; 29 | //--------------------------------------------------------------------------- 30 | } // namespace network 31 | } // namespace anyblob 32 | -------------------------------------------------------------------------------- /include/network/http_request.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2024 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | //--------------------------------------------------------------------------- 17 | namespace utils { 18 | template 19 | class DataVector; 20 | } 21 | //--------------------------------------------------------------------------- 22 | namespace network { 23 | //--------------------------------------------------------------------------- 24 | /// Implements an helper to serialize and deserialize http requests 25 | struct HttpRequest { 26 | /// The method class 27 | enum class Method : uint8_t { 28 | GET, 29 | PUT, 30 | POST, 31 | DELETE 32 | }; 33 | enum class Type : uint8_t { 34 | HTTP_1_0, 35 | HTTP_1_1 36 | }; 37 | /// The queries - need to be RFC 3986 conform 38 | std::map queries; 39 | /// The headers - need to be without trailing and leading whitespaces 40 | std::map headers; 41 | /// The method 42 | Method method; 43 | /// The type 44 | Type type; 45 | /// The path - needs to be RFC 3986 conform 46 | std::string path; 47 | 48 | /// Get the request method 49 | static constexpr auto getRequestMethod(const Method& method) { 50 | switch (method) { 51 | case Method::GET: return "GET"; 52 | case Method::PUT: return "PUT"; 53 | case Method::POST: return "POST"; 54 | case Method::DELETE: return "DELETE"; 55 | default: return ""; 56 | } 57 | } 58 | /// Get the request type 59 | static constexpr auto getRequestType(const Type& type) { 60 | switch (type) { 61 | case Type::HTTP_1_0: return "HTTP/1.0"; 62 | case Type::HTTP_1_1: return "HTTP/1.1"; 63 | default: return ""; 64 | } 65 | } 66 | /// Serialize the request 67 | [[nodiscard]] static std::unique_ptr> serialize(const HttpRequest& request); 68 | /// Deserialize the request 69 | [[nodiscard]] static HttpRequest deserialize(std::string_view data); 70 | }; 71 | //--------------------------------------------------------------------------- 72 | } // namespace network 73 | } // namespace anyblob 74 | -------------------------------------------------------------------------------- /include/network/http_response.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2024 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | //--------------------------------------------------------------------------- 17 | namespace utils { 18 | template 19 | class DataVector; 20 | } 21 | //--------------------------------------------------------------------------- 22 | namespace network { 23 | //--------------------------------------------------------------------------- 24 | /// Implements an helper to deserialize http responses 25 | struct HttpResponse { 26 | /// Important error codes 27 | enum class Code : uint8_t { 28 | OK_200, 29 | CREATED_201, 30 | NO_CONTENT_204, 31 | PARTIAL_CONTENT_206, 32 | BAD_REQUEST_400, 33 | UNAUTHORIZED_401, 34 | FORBIDDEN_403, 35 | NOT_FOUND_404, 36 | CONFLICT_409, 37 | LENGTH_REQUIRED_411, 38 | RANGE_NOT_SATISFIABLE_416, 39 | TOO_MANY_REQUESTS_429, 40 | INTERNAL_SERVER_ERROR_500, 41 | SERVICE_UNAVAILABLE_503, 42 | SLOW_DOWN_503, 43 | UNKNOWN = 255 44 | }; 45 | enum class Type : uint8_t { 46 | HTTP_1_0, 47 | HTTP_1_1 48 | }; 49 | /// The headers - need to be without trailing and leading whitespaces 50 | std::map headers; 51 | /// The method 52 | Code code; 53 | /// The type 54 | Type type; 55 | 56 | /// Get the request method 57 | static constexpr std::string_view getResponseCode(const Code& code) noexcept { 58 | switch (code) { 59 | case Code::OK_200: return "200 OK"; 60 | case Code::CREATED_201: return "201 Created"; 61 | case Code::NO_CONTENT_204: return "204 No Content"; 62 | case Code::PARTIAL_CONTENT_206: return "206 Partial Content"; 63 | case Code::BAD_REQUEST_400: return "400 Bad Request"; 64 | case Code::UNAUTHORIZED_401: return "401 Unauthorized"; 65 | case Code::FORBIDDEN_403: return "403 Forbidden"; 66 | case Code::NOT_FOUND_404: return "404 Not Found"; 67 | case Code::CONFLICT_409: return "409 Conflict"; 68 | case Code::LENGTH_REQUIRED_411: return "411 Length Required"; 69 | case Code::RANGE_NOT_SATISFIABLE_416: return "416 Range Not Satisfiable"; 70 | case Code::TOO_MANY_REQUESTS_429: return "429 Too Many Requests"; 71 | case Code::INTERNAL_SERVER_ERROR_500: return "500 Internal Server Error"; 72 | case Code::SERVICE_UNAVAILABLE_503: return "503 Service Unavailable"; 73 | case Code::SLOW_DOWN_503: return "503 Slow Down"; 74 | default: return "UNKNOWN"; 75 | } 76 | } 77 | /// Get the request type 78 | static constexpr std::string_view getResponseType(const Type& type) noexcept { 79 | switch (type) { 80 | case Type::HTTP_1_0: return "HTTP/1.0"; 81 | case Type::HTTP_1_1: return "HTTP/1.1"; 82 | default: return "UNKNOWN"; 83 | } 84 | } 85 | /// Check for successful operation 2xx operations 86 | static constexpr auto checkSuccess(const Code& code) { 87 | return (code == Code::OK_200 || code == Code::CREATED_201 || code == Code::NO_CONTENT_204 || code == Code::PARTIAL_CONTENT_206); 88 | } 89 | /// Check if the result has no content 90 | static constexpr auto withoutContent(const Code& code) { 91 | return code == Code::NO_CONTENT_204; 92 | } 93 | /// Deserialize the response 94 | [[nodiscard]] static HttpResponse deserialize(std::string_view data); 95 | }; 96 | //--------------------------------------------------------------------------- 97 | } // namespace network 98 | } // namespace anyblob 99 | -------------------------------------------------------------------------------- /include/network/https_message.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_message.hpp" 3 | #include "network/tls_connection.hpp" 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2023 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace network { 15 | //--------------------------------------------------------------------------- 16 | /// Implements a https message roundtrip 17 | struct HTTPSMessage : public HTTPMessage { 18 | /// The tls layer 19 | TLSConnection* tlsLayer; 20 | /// The fd 21 | int32_t fd; 22 | 23 | /// The constructor 24 | HTTPSMessage(OriginalMessage* sendingMessage, ConnectionManager::TCPSettings& tcpSettings, uint32_t chunksize); 25 | /// The destructor 26 | ~HTTPSMessage() override = default; 27 | /// The message excecute callback 28 | MessageState execute(ConnectionManager& connectionManager) override; 29 | /// Reset for restart 30 | void reset(ConnectionManager& socket, bool aborted); 31 | }; 32 | //--------------------------------------------------------------------------- 33 | } // namespace network 34 | } // namespace anyblob 35 | -------------------------------------------------------------------------------- /include/network/io_uring_socket.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2021 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace network { 16 | //--------------------------------------------------------------------------- 17 | struct MessageTask; 18 | //--------------------------------------------------------------------------- 19 | /// This class opens a TCP connection to a server and exchanges messages 20 | /// with the new io_uring library for minimizing syscalls and unnecessary 21 | /// kernel overhead 22 | class IOUringSocket { 23 | public: 24 | /// Request message event type 25 | enum class EventType : uint32_t { 26 | read = 0, 27 | write = 1 28 | }; 29 | 30 | /// The request message 31 | struct Request { 32 | union Data { 33 | /// The recv data 34 | uint8_t* data; 35 | /// The send data 36 | const uint8_t* cdata; 37 | }; 38 | /// The data 39 | Data data; 40 | /// The length 41 | int64_t length; 42 | /// The file descriptor 43 | int32_t fd; 44 | /// Specifies the event 45 | EventType event; 46 | /// The associated message task 47 | MessageTask* messageTask; 48 | }; 49 | 50 | private: 51 | /// The uring buffer 52 | struct io_uring _uring; 53 | /// The event id for the uring 54 | int _eventId; 55 | 56 | public: 57 | /// The IO Uring Socket Constructor 58 | explicit IOUringSocket(uint32_t entries, int32_t flags = 0); 59 | /// The destructor 60 | ~IOUringSocket(); 61 | 62 | /// Prepare a submission (sqe) send 63 | io_uring_sqe* send_prep(const Request* req, int32_t msg_flags = 0, uint8_t flags = 0); 64 | /// Prepare a submission (sqe) recv 65 | io_uring_sqe* recv_prep(Request* req, int32_t msg_flags = 0, uint8_t flags = 0); 66 | /// Prepare a submission (sqe) send with timeout 67 | io_uring_sqe* send_prep_to(const Request* req, __kernel_timespec* timeout, int32_t msg_flags = 0, uint8_t flags = 0); 68 | /// Prepare a submission (sqe) recv with timeout 69 | io_uring_sqe* recv_prep_to(Request* req, __kernel_timespec* timeout, int32_t msg_flags = 0, uint8_t flags = 0); 70 | 71 | /// Submits queue and gets all completion (cqe) event and mark them as seen; return the SQE attached requests 72 | uint32_t submitCompleteAll(uint32_t events, std::vector& completions); 73 | /// Get a completion (cqe) event and mark it as seen; return the SQE attached Request 74 | [[nodiscard]] Request* complete(); 75 | /// Get a completion (cqe) event if it is available and mark it as seen; return the SQE attached Request if available else nullptr 76 | Request* peek(); 77 | /// Get a completion (cqe) event 78 | [[nodiscard]] io_uring_cqe* completion(); 79 | /// Mark a completion (cqe) event seen to allow for new completions in the kernel 80 | void seen(io_uring_cqe* cqe); 81 | /// Wait for a new cqe event arriving 82 | void wait(); 83 | 84 | /// Submit uring to the kernel and return the number of submitted entries 85 | int32_t submit(); 86 | }; 87 | //--------------------------------------------------------------------------- 88 | } // namespace network 89 | } // namespace anyblob 90 | -------------------------------------------------------------------------------- /include/network/message_result.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/http_helper.hpp" 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2023 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace utils { 16 | //--------------------------------------------------------------------------- 17 | template 18 | class DataVector; 19 | //--------------------------------------------------------------------------- 20 | } // namespace utils 21 | //--------------------------------------------------------------------------- 22 | namespace network { 23 | //--------------------------------------------------------------------------- 24 | struct OriginalMessage; 25 | struct HTTPMessage; 26 | struct HTTPSMessage; 27 | class TLSConnection; 28 | class Transaction; 29 | //--------------------------------------------------------------------------- 30 | /// Current status of the message 31 | enum class MessageState : uint8_t { 32 | Init, 33 | TLSHandshake, 34 | InitSending, 35 | Sending, 36 | InitReceiving, 37 | Receiving, 38 | TLSShutdown, 39 | Finished, 40 | Aborted, 41 | Cancelled 42 | }; 43 | //--------------------------------------------------------------------------- 44 | /// The failure codes 45 | enum class MessageFailureCode : uint16_t { 46 | /// Socket creation error 47 | Socket = 1, 48 | /// Empty request error 49 | Empty = 1 << 1, 50 | /// Timeout passed 51 | Timeout = 1 << 2, 52 | /// Send syscall error 53 | Send = 1 << 3, 54 | /// Recv syscall error 55 | Recv = 1 << 4, 56 | /// HTTP header error 57 | HTTP = 1 << 5, 58 | /// TLS error 59 | TLS = 1 << 6 60 | }; 61 | //--------------------------------------------------------------------------- 62 | /// The result class 63 | class MessageResult { 64 | protected: 65 | /// The data 66 | std::unique_ptr> dataVector; 67 | /// The http response header info 68 | std::unique_ptr response; 69 | /// The error response 70 | const MessageResult* originError; 71 | /// The failure code 72 | uint16_t failureCode; 73 | /// The state 74 | std::atomic state; 75 | 76 | public: 77 | /// The default constructor 78 | MessageResult(); 79 | /// The constructor with buffer input 80 | MessageResult(uint8_t* data, uint64_t size); 81 | /// The constructor with buffer input 82 | explicit MessageResult(utils::DataVector* dataVector); 83 | 84 | /// Get the result 85 | [[nodiscard]] const std::string_view getResult() const; 86 | /// Get the result 87 | [[nodiscard]] std::string_view getResult(); 88 | /// Get the const data 89 | [[nodiscard]] const uint8_t* getData() const; 90 | /// Get the data 91 | [[nodiscard]] uint8_t* getData(); 92 | /// Transfer owenership 93 | [[nodiscard]] std::unique_ptr moveData(); 94 | /// Get the size 95 | [[nodiscard]] uint64_t getSize() const; 96 | /// Get the offset 97 | [[nodiscard]] uint64_t getOffset() const; 98 | /// Get the state 99 | [[nodiscard]] MessageState getState() const; 100 | /// Get the failure code 101 | [[nodiscard]] uint16_t getFailureCode() const; 102 | /// Get the error response (incl. header) 103 | [[nodiscard]] std::string_view getErrorResponse() const; 104 | /// Get the error response code 105 | [[nodiscard]] std::string_view getResponseCode() const; 106 | /// Is the data owned by this object 107 | [[nodiscard]] bool owned() const; 108 | /// Was the request successful 109 | [[nodiscard]] bool success() const; 110 | 111 | /// Get the data vector reference 112 | [[nodiscard]] utils::DataVector& getDataVector(); 113 | /// Transfer owenership of data vector 114 | [[nodiscard]] std::unique_ptr> moveDataVector(); 115 | 116 | /// Define the friend message and message tasks 117 | friend HTTPMessage; 118 | friend HTTPSMessage; 119 | friend OriginalMessage; 120 | friend TLSConnection; 121 | friend Transaction; 122 | }; 123 | //--------------------------------------------------------------------------- 124 | } // namespace network 125 | } // namespace anyblob 126 | -------------------------------------------------------------------------------- /include/network/message_task.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "cloud/provider.hpp" 3 | #include "network/connection_manager.hpp" 4 | #include "network/http_helper.hpp" 5 | #include "network/io_uring_socket.hpp" 6 | #include "network/original_message.hpp" 7 | #include "utils/data_vector.hpp" 8 | #include 9 | #include 10 | #include 11 | //--------------------------------------------------------------------------- 12 | // AnyBlob - Universal Cloud Object Storage Library 13 | // Dominik Durner, 2022 14 | // 15 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 16 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 17 | // SPDX-License-Identifier: MPL-2.0 18 | //--------------------------------------------------------------------------- 19 | namespace anyblob { 20 | //--------------------------------------------------------------------------- 21 | namespace utils { 22 | class Timer; 23 | }; // namespace utils 24 | //--------------------------------------------------------------------------- 25 | namespace network { 26 | class TLSContext; 27 | class ConnectionManager; 28 | //--------------------------------------------------------------------------- 29 | /// This implements a message task 30 | /// After each execute invocation a new request was added to the uring queue and requires submission 31 | struct MessageTask { 32 | /// Type of the message task 33 | enum class Type : uint8_t { 34 | HTTP, 35 | HTTPS 36 | }; 37 | 38 | /// Original sending message 39 | OriginalMessage* originalMessage; 40 | /// The TCP Settings 41 | ConnectionManager::TCPSettings& tcpSettings; 42 | /// Send message 43 | std::unique_ptr request; 44 | /// The reduced offset in the send buffers 45 | int64_t sendBufferOffset; 46 | /// The reduced offset in the receive buffers 47 | int64_t receiveBufferOffset; 48 | /// The chunksize 49 | uint32_t chunkSize; 50 | /// The failures 51 | uint16_t failures; 52 | /// The message task class 53 | Type type; 54 | 55 | /// The failure limit 56 | static constexpr uint16_t failuresMax = 32; 57 | 58 | /// The pure virtual callback 59 | virtual MessageState execute(ConnectionManager& connectionManager) = 0; 60 | /// The pure virtual destuctor 61 | virtual ~MessageTask(){}; 62 | 63 | /// Builds the message task according to the sending message 64 | template 65 | static std::unique_ptr buildMessageTask(OriginalMessage* sendingMessage, Args&&... args) { 66 | std::string_view s(reinterpret_cast(sendingMessage->message->data()), sendingMessage->message->size()); 67 | if (s.find("HTTP") != std::string_view::npos && sendingMessage->provider.getPort() == 443) { 68 | return std::make_unique(sendingMessage, std::forward(args)...); 69 | } else if (s.find("HTTP") != std::string_view::npos) { 70 | return std::make_unique(sendingMessage, std::forward(args)...); 71 | } 72 | return nullptr; 73 | } 74 | 75 | protected: 76 | /// The constructor 77 | MessageTask(OriginalMessage* sendingMessage, ConnectionManager::TCPSettings& tcpSettings, uint32_t chunkSize); 78 | }; 79 | //--------------------------------------------------------------------------- 80 | } // namespace network 81 | } // namespace anyblob 82 | -------------------------------------------------------------------------------- /include/network/original_message.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "network/message_result.hpp" 3 | #include "utils/data_vector.hpp" 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2021 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | //--------------------------------------------------------------------------- 17 | namespace utils { 18 | class Timer; 19 | }; // namespace utils 20 | //--------------------------------------------------------------------------- 21 | namespace cloud { 22 | class Provider; 23 | }; // namespace cloud 24 | //--------------------------------------------------------------------------- 25 | namespace network { 26 | //--------------------------------------------------------------------------- 27 | /// This is the original request message struct 28 | struct OriginalMessage { 29 | /// The message 30 | std::unique_ptr> message; 31 | /// The provider that knows the address, the port and, optionally signed the messsage 32 | cloud::Provider& provider; 33 | /// The result 34 | MessageResult result; 35 | 36 | /// If it is a put request store the additional data 37 | /// The raw data ptr for put requests 38 | const uint8_t* putData; 39 | /// The length 40 | uint64_t putLength; 41 | 42 | /// Optional trace info 43 | uint64_t traceId; 44 | 45 | /// The constructor 46 | OriginalMessage(std::unique_ptr> message, cloud::Provider& provider, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : message(std::move(message)), provider(provider), result(receiveBuffer, bufferSize), putData(nullptr), putLength(), traceId(traceId) {} 47 | 48 | /// The destructor 49 | virtual ~OriginalMessage() = default; 50 | 51 | /// Add the put request data to the message 52 | constexpr void setPutRequestData(const uint8_t* data, uint64_t length) { 53 | this->putData = data; 54 | this->putLength = length; 55 | } 56 | 57 | /// Set the result vector and transfer ownership 58 | inline void setResultVector(utils::DataVector* dataVector) { 59 | result.dataVector = std::unique_ptr>(dataVector); 60 | } 61 | 62 | /// Callback required 63 | virtual inline bool requiresFinish() { return false; } 64 | 65 | /// Callback 66 | virtual inline void finish() {} 67 | }; 68 | //--------------------------------------------------------------------------- 69 | /// The callback original message 70 | template 71 | struct OriginalCallbackMessage : public OriginalMessage { 72 | /// The callback 73 | Callback callback; 74 | 75 | /// The constructor 76 | OriginalCallbackMessage(Callback&& callback, std::unique_ptr> message, cloud::Provider& provider, uint8_t* receiveBuffer = nullptr, uint64_t bufferSize = 0, uint64_t traceId = 0) : OriginalMessage(std::move(message), provider, receiveBuffer, bufferSize, traceId), callback(std::forward(callback)) {} 77 | 78 | /// The destructor 79 | virtual ~OriginalCallbackMessage() override = default; 80 | 81 | /// Override if callback required 82 | bool inline requiresFinish() override { return true; } 83 | 84 | /// Callback 85 | void inline finish() override { 86 | callback(result); 87 | } 88 | }; 89 | //--------------------------------------------------------------------------- 90 | } // namespace network 91 | } // namespace anyblob 92 | -------------------------------------------------------------------------------- /include/network/throughput_cache.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | // The `ThroughputCache` depends on gnu stdlib associative containers that are 3 | // not supported by libcxx. We remove the throughput cache in its entirety when 4 | // building with libcxx. 5 | #ifndef ANYBLOB_LIBCXX_COMPAT 6 | #include "network/cache.hpp" 7 | #include 8 | #include 9 | #include 10 | #include 11 | //--------------------------------------------------------------------------- 12 | // AnyBlob - Universal Cloud Object Storage Library 13 | // Dominik Durner, 2022 14 | // 15 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 16 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 17 | // SPDX-License-Identifier: MPL-2.0 18 | //--------------------------------------------------------------------------- 19 | namespace anyblob { 20 | //--------------------------------------------------------------------------- 21 | namespace network { 22 | //--------------------------------------------------------------------------- 23 | /// Implements the throughput-based cache logic 24 | class ThroughputCache : public network::Cache { 25 | /// Order statistic tree 26 | typedef __gnu_pbds::tree< 27 | double, 28 | __gnu_pbds::null_type, 29 | std::greater, 30 | __gnu_pbds::rb_tree_tag, 31 | __gnu_pbds::tree_order_statistics_node_update> 32 | map_t; 33 | /// The order statistic tree 34 | map_t _throughputTree; 35 | /// The seconds vector as stable ring buffer 36 | std::vector _throughput; 37 | /// The map between fd and start time 38 | std::unordered_map _fdMap; 39 | /// The seconds vector iterator 40 | uint64_t _throughputIterator; 41 | /// The maximum history 42 | const unsigned _maxHistory = 128; 43 | 44 | public: 45 | /// The constructor 46 | explicit ThroughputCache(); 47 | /// Start the timing and advance to the next cache bucket 48 | virtual void startSocket(int fd) override; 49 | /// Stops the socket and either closes the connection or cashes it 50 | virtual void stopSocket(std::unique_ptr socketEntry, uint64_t bytes, unsigned cachedEntries, bool reuseSocket) override; 51 | /// The destructor 52 | virtual ~ThroughputCache() = default; 53 | }; 54 | //--------------------------------------------------------------------------- 55 | }; // namespace network 56 | //--------------------------------------------------------------------------- 57 | }; // namespace anyblob 58 | #endif // ANYBLOB_LIBCXX_COMPAT 59 | -------------------------------------------------------------------------------- /include/network/tls_connection.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2023 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace network { 15 | //--------------------------------------------------------------------------- 16 | struct HTTPSMessage; 17 | class TLSContext; 18 | class ConnectionManager; 19 | //--------------------------------------------------------------------------- 20 | /// The TLS Interface 21 | //--------------------------------------------------------------------------- 22 | /* anyblob | OpenSSL 23 | * | | 24 | * -------> SSL_read / SSL_write / SSL_connect 25 | * | /\ || 26 | * | || \/ 27 | * | internalBio 28 | * | networkBio 29 | * | || /\ 30 | * | \/ || 31 | * -------< recv / send 32 | * | | 33 | * | | 34 | * socket 35 | * Adopted from https://www.openssl.org/docs/man3.1/man3/BIO_new_bio_pair.html 36 | */ 37 | //--------------------------------------------------------------------------- 38 | class TLSConnection { 39 | public: 40 | /// The progress of the TLS 41 | enum class Progress : uint16_t { 42 | Init, 43 | SendingInit, 44 | Sending, 45 | ReceivingInit, 46 | Receiving, 47 | Progress, 48 | Finished, 49 | Aborted 50 | }; 51 | 52 | private: 53 | /// The state 54 | struct State { 55 | /// Bytes wanted to write from internal bio (used for send) 56 | size_t internalBioWrite; 57 | /// Bytes read from network bio (used for send) 58 | int64_t networkBioRead; 59 | /// Bytes written to socket (used for send) 60 | size_t socketWrite; 61 | /// Bytes wanted to read from internal bio (used for recv) 62 | size_t internalBioRead; 63 | /// Bytes written to network bio (used for recv) 64 | int64_t networkBioWrite; 65 | /// Bytes read fromsocket (used for recv) 66 | size_t socketRead; 67 | /// The progress 68 | Progress progress; 69 | 70 | /// Resets the statistics 71 | inline void reset() { 72 | internalBioWrite = 0; 73 | networkBioRead = 0; 74 | socketWrite = 0; 75 | internalBioRead = 0; 76 | networkBioWrite = 0; 77 | socketRead = 0; 78 | } 79 | }; 80 | /// The corresponding 81 | HTTPSMessage* _message; 82 | /// The SSL context 83 | TLSContext& _context; 84 | /// The SSL connection 85 | SSL* _ssl; 86 | /// The internal buffer used for communicating with SSL 87 | BIO* _internalBio; 88 | /// The external buffer used for communicating with the socket 89 | BIO* _networkBio; 90 | /// The buffer 91 | std::unique_ptr _buffer; 92 | /// The state 93 | State _state; 94 | /// Already conencted 95 | bool _connected; 96 | 97 | public: 98 | /// The constructor 99 | TLSConnection(TLSContext& context); 100 | 101 | /// The destructor 102 | ~TLSConnection(); 103 | 104 | // Initialze SSL 105 | [[nodiscard]] bool init(HTTPSMessage* message); 106 | // Initialze SSL 107 | void destroy(); 108 | /// Get the SSL/TLS context 109 | [[nodiscard]] inline TLSContext& getContext() const { return _context; } 110 | 111 | /// Recv a TLS encrypted message 112 | [[nodiscard]] Progress recv(ConnectionManager& connectionManager, char* buffer, int64_t bufferLength, int64_t& resultLength); 113 | /// Send a TLS encrypted message 114 | [[nodiscard]] Progress send(ConnectionManager& connectionManager, const char* buffer, int64_t bufferLength, int64_t& resultLength); 115 | /// SSL/TLS connect 116 | [[nodiscard]] Progress connect(ConnectionManager& connectionManager); 117 | /// SSL/TLS shutdown 118 | [[nodiscard]] Progress shutdown(ConnectionManager& connectionManager, bool failedOnce = false); 119 | 120 | private: 121 | /// Helper function that handles the SSL_op calls 122 | template 123 | Progress operationHelper(ConnectionManager& connectionManager, F&& func, int64_t& result); 124 | /// The processing of the shadow tls layer 125 | Progress process(ConnectionManager& connectionManager); 126 | }; 127 | //--------------------------------------------------------------------------- 128 | } // namespace network 129 | } // namespace anyblob 130 | -------------------------------------------------------------------------------- /include/network/tls_context.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2023 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | namespace network { 17 | //--------------------------------------------------------------------------- 18 | class TLSConnection; 19 | //--------------------------------------------------------------------------- 20 | // Although the tls context can be used safe in multi-threading enviornments, 21 | // we allow only one context per thread to avoid locking. 22 | // This simplifies also the caching of sessions. 23 | class TLSContext { 24 | /// The ssl context 25 | SSL_CTX* _ctx; 26 | /// The cache size as power of 2 27 | static constexpr uint8_t cachePower = 8; 28 | /// The cache mask 29 | static constexpr uint64_t cacheMask = (~0ull) >> (64 - cachePower); 30 | /// The session cache 31 | std::array, 1ull << cachePower> _sessionCache; 32 | 33 | public: 34 | /// The constructor 35 | TLSContext(); 36 | /// The destructor 37 | ~TLSContext(); 38 | 39 | /// Caches the SSL session 40 | bool cacheSession(int fd, SSL* ssl); 41 | /// Drops the SSL session 42 | bool dropSession(int fd); 43 | /// Reuses a SSL session 44 | bool reuseSession(int fd, SSL* ssl); 45 | 46 | /// Init the OpenSSL algos and errors 47 | static void initOpenSSL(); 48 | 49 | friend TLSConnection; 50 | }; 51 | //--------------------------------------------------------------------------- 52 | } // namespace network 53 | } // namespace anyblob 54 | -------------------------------------------------------------------------------- /include/utils/data_vector.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2021 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | namespace utils { 17 | //--------------------------------------------------------------------------- 18 | /// Minimal version of a vector that only allwos to store raw data 19 | template 20 | class DataVector { 21 | private: 22 | /// Current capacity 23 | uint64_t _capacity; 24 | /// Current size 25 | uint64_t _size; 26 | /// The data 27 | std::unique_ptr _dataOwned; 28 | /// The data not owned 29 | T* _data; 30 | 31 | public: 32 | /// Constructor 33 | constexpr DataVector() : _capacity(0), _size(0), _data(nullptr) {} 34 | 35 | /// Constructor with size 36 | explicit constexpr DataVector(uint64_t cap) : _capacity(0), _size(0), _data(nullptr) { 37 | resize(cap); 38 | } 39 | 40 | /// Copy constructor 41 | constexpr DataVector(DataVector& rhs) : _capacity(0), _size(0), _data(nullptr) { 42 | reserve(rhs._capacity); 43 | _size = rhs._size; 44 | std::memcpy(data(), rhs.data(), size() * sizeof(T)); 45 | } 46 | 47 | /// Constructor from other pointers 48 | constexpr DataVector(const T* start, const T* end) : _capacity(0), _size(0), _data(nullptr) { 49 | assert(end - start >= 0); 50 | resize(static_cast(end - start)); 51 | std::memcpy(data(), start, size() * sizeof(T)); 52 | } 53 | 54 | /// Constructor unowned, map to other pointer, capacity given in number of T elements 55 | constexpr DataVector(T* ptr, uint64_t capacity) : _capacity(capacity * sizeof(T)), _size(0) { 56 | _data = ptr; 57 | } 58 | 59 | /// Get the data 60 | [[nodiscard]] constexpr T* data() { 61 | return _data; 62 | } 63 | 64 | /// Get the data 65 | [[nodiscard]] constexpr const T* cdata() const { 66 | return _data; 67 | } 68 | 69 | /// Get the size 70 | [[nodiscard]] constexpr uint64_t size() const { 71 | return _size; 72 | } 73 | 74 | /// Get the capacity 75 | [[nodiscard]] constexpr uint64_t capacity() const { 76 | return _capacity; 77 | } 78 | 79 | /// Clear the size 80 | constexpr void clear() { 81 | _size = 0; 82 | } 83 | 84 | /// Is the data owned 85 | [[nodiscard]] constexpr bool owned() { 86 | return _dataOwned || !_capacity; 87 | } 88 | 89 | /// Increase the capacity 90 | constexpr void reserve(uint64_t cap) { 91 | if (_capacity < cap) { 92 | if (!_dataOwned && _capacity) 93 | throw std::runtime_error("Pointer not owned, thus size is fixed!"); 94 | auto swap = std::unique_ptr(new T[cap]()); 95 | if (_data && _size) 96 | std::memcpy(swap.get(), data(), _size * sizeof(T)); 97 | _dataOwned.swap(swap); 98 | _data = _dataOwned.get(); 99 | _capacity = cap; 100 | } 101 | } 102 | 103 | /// Change the number of elements 104 | constexpr void resize(uint64_t size) { 105 | if (size > _capacity) { 106 | reserve(size); 107 | } 108 | _size = size; 109 | } 110 | 111 | /// Transfer the ownership of the data 112 | [[nodiscard]] constexpr std::unique_ptr transferBuffer() { 113 | return move(_dataOwned); 114 | } 115 | }; 116 | //--------------------------------------------------------------------------- 117 | }; // namespace utils 118 | }; // namespace anyblob 119 | -------------------------------------------------------------------------------- /include/utils/timer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "utils/load_tracker.hpp" 3 | #include 4 | #include 5 | #include 6 | #include 7 | //--------------------------------------------------------------------------- 8 | // AnyBlob - Universal Cloud Object Storage Library 9 | // Dominik Durner, 2021 10 | // 11 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 12 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 13 | // SPDX-License-Identifier: MPL-2.0 14 | //--------------------------------------------------------------------------- 15 | namespace anyblob { 16 | namespace utils { 17 | //--------------------------------------------------------------------------- 18 | /// This is a timing helper 19 | struct TimingHelper { 20 | /// The file size 21 | uint64_t size; 22 | /// The start time 23 | std::chrono::steady_clock::time_point start; 24 | /// The first recieve time 25 | std::chrono::steady_clock::time_point recieve; 26 | /// The finish time 27 | std::chrono::steady_clock::time_point finish; 28 | }; 29 | //--------------------------------------------------------------------------- 30 | class Timer { 31 | public: 32 | enum Steps : uint64_t { 33 | Overall, 34 | Upload, 35 | Download, 36 | Submit, 37 | Request, 38 | Response, 39 | Buffer, 40 | BufferCapacity, 41 | BufferResize, 42 | Finishing, 43 | Waiting, 44 | Queue, 45 | SocketCreation, 46 | BufferCapacitySend 47 | }; 48 | 49 | /// Timer Guard 50 | class TimerGuard { 51 | /// The guarded step 52 | Steps step; 53 | /// The current timer 54 | Timer* timer; 55 | 56 | public: 57 | /// Constructor 58 | TimerGuard(Steps step, Timer* timer) : step(step), timer(timer) { 59 | if (timer) 60 | timer->start(step); 61 | } 62 | /// Destructor 63 | ~TimerGuard() { 64 | if (timer) 65 | timer->stop(step); 66 | } 67 | }; 68 | 69 | private: 70 | /// Currently active timer 71 | std::unordered_map _currentTimer; 72 | /// The load tracker 73 | std::unordered_map> _loadTracker; 74 | /// Total time in ns 75 | std::unordered_map _totalTimer; 76 | /// Total time in ns 77 | std::unordered_map>>> _totalLoad; 78 | /// The outstream 79 | std::ostream* _outStream; 80 | /// Is the header printed 81 | bool _writeHeader; 82 | /// Header info 83 | std::string _headerInfo; 84 | /// Content info 85 | std::string _contentInfo; 86 | /// Vector of timer values 87 | std::vector> timings; 88 | 89 | public: 90 | /// Default constructor 91 | Timer() = default; 92 | /// Constructor 93 | Timer(std::ostream* outStream, bool writeHeader = true) : _outStream(outStream), _writeHeader(writeHeader) {} 94 | /// Default copy constructor 95 | Timer(const Timer&) = default; 96 | /// Move Assignment - Timer and Merge 97 | Timer& operator=(Timer&& rhs); 98 | /// Destructor 99 | ~Timer() { 100 | if (_outStream) 101 | printResult(*_outStream); 102 | } 103 | /// Sets the ostream 104 | void setOutStream(std::ostream* outStream, bool writeHeader = true) { 105 | _outStream = outStream; 106 | _writeHeader = writeHeader; 107 | } 108 | /// Reserve timings 109 | void reserveTimings(uint64_t cap) { 110 | timings.reserve(cap); 111 | } 112 | /// Set info settings 113 | void setInfo(std::string headerInfo, std::string contentInfo); 114 | /// Starts the timer for step s 115 | void start(Steps s); 116 | /// Stop the timer and adds to totalTimer for step s 117 | void stop(Steps s); 118 | /// Prints the result 119 | void printResult(std::ostream& s); 120 | }; 121 | //--------------------------------------------------------------------------- 122 | }; // namespace utils 123 | }; // namespace anyblob 124 | -------------------------------------------------------------------------------- /include/utils/utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace utils { 14 | //--------------------------------------------------------------------------- 15 | #ifndef NDEBUG 16 | #define verify(expression) assert(expression) 17 | #else 18 | #define verify(expression) ((void) (expression)) 19 | #endif 20 | //--------------------------------------------------------------------------- 21 | /// Encode url special characters in %HEX 22 | std::string encodeUrlParameters(const std::string& encode); 23 | /// Encode everything from binary representation to hex 24 | std::string hexEncode(const uint8_t* input, uint64_t length, bool upper = false); 25 | /// Encode everything from binary representation to base64 26 | std::string base64Encode(const uint8_t* input, uint64_t length); 27 | /// Decodes from base64 to raw string 28 | std::pair, uint64_t> base64Decode(const uint8_t* input, uint64_t length); 29 | /// Build sha256 of the data encoded as hex 30 | std::string sha256Encode(const uint8_t* data, uint64_t length); 31 | /// Build md5 of the data 32 | std::string md5Encode(const uint8_t* data, uint64_t length); 33 | /// Sing with hmac and return sha256 encoded signature 34 | std::pair, uint64_t> hmacSign(const uint8_t* keyData, uint64_t keyLength, const uint8_t* msgData, uint64_t msgLength); 35 | /// Sing with rsa and return sha256 encoded signature 36 | std::pair, uint64_t> rsaSign(const uint8_t* keyData, uint64_t keyLength, const uint8_t* msgData, uint64_t msgLength); 37 | /// Decrypt with AES-256-CBC 38 | uint64_t aesDecrypt(const unsigned char* key, const unsigned char* iv, const uint8_t* encData, uint64_t encLength, uint8_t* plainData); 39 | /// Encrypt with AES-256-CBC 40 | uint64_t aesEncrypt(const unsigned char* key, const unsigned char* iv, const uint8_t* plainData, uint64_t plainLength, uint8_t* encData); 41 | //--------------------------------------------------------------------------- 42 | }; // namespace utils 43 | }; // namespace anyblob 44 | -------------------------------------------------------------------------------- /src/cloud/aws_cache.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/aws_cache.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | //--------------------------------------------------------------------------- 9 | // AnyBlob - Universal Cloud Object Storage Library 10 | // Dominik Durner, 2021 11 | // 12 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 13 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | // SPDX-License-Identifier: MPL-2.0 15 | //--------------------------------------------------------------------------- 16 | namespace anyblob { 17 | namespace cloud { 18 | //--------------------------------------------------------------------------- 19 | using namespace std; 20 | //--------------------------------------------------------------------------- 21 | AWSCache::AWSCache() : Cache(), _mtuCache() 22 | // Constructor 23 | { 24 | } 25 | //--------------------------------------------------------------------------- 26 | unique_ptr AWSCache::resolve(string hostname, unsigned port, bool tls) 27 | // Resolve the request 28 | { 29 | for (auto it = _cache.find(hostname); it != _cache.end();) { 30 | // loosely clean up the multimap cache 31 | if (!it->second->dns) { 32 | _fifo.erase(it->second->timestamp); 33 | it = _cache.erase(it); 34 | continue; 35 | } 36 | if (it->second->port == port && ((tls && it->second->tls.get()) || (!tls && !it->second->tls.get()))) { 37 | auto socketEntry = move(it->second); 38 | socketEntry->dns->cachePriority--; 39 | _fifo.erase(socketEntry->timestamp); 40 | _cache.erase(it); 41 | return socketEntry; 42 | } 43 | it++; 44 | } 45 | struct addrinfo hints = {}; 46 | memset(&hints, 0, sizeof hints); 47 | hints.ai_family = AF_INET; 48 | hints.ai_socktype = SOCK_STREAM; 49 | hints.ai_protocol = IPPROTO_TCP; 50 | 51 | addrinfo* temp; 52 | char port_str[16] = {}; 53 | sprintf(port_str, "%d", port); 54 | if (getaddrinfo(hostname.c_str(), port_str, &hints, &temp) != 0) { 55 | throw runtime_error("hostname getaddrinfo error"); 56 | } 57 | auto socketEntry = make_unique(hostname, port); 58 | socketEntry->dns = make_unique(unique_ptr(temp, &freeaddrinfo), _defaultPriority); 59 | 60 | if (!Cache::tld(hostname).compare("amazonaws.com")) { 61 | struct sockaddr_in* p = reinterpret_cast(socketEntry->dns->addr->ai_addr); 62 | auto ipAsInt = p->sin_addr.s_addr; 63 | auto it = _mtuCache.find(ipAsInt); 64 | if (it != _mtuCache.end()) { 65 | if (it->second) 66 | socketEntry->dns->cachePriority = numeric_limits::max(); 67 | } else { 68 | char ipv4[INET_ADDRSTRLEN]; 69 | inet_ntop(AF_INET, &p->sin_addr, ipv4, INET_ADDRSTRLEN); 70 | string cmd = "timeout 0.01 ping -s 1473 -D " + string(ipv4) + " -c 1 >>/dev/null 2>>/dev/null"; 71 | auto res = system(cmd.c_str()); 72 | if (!res) { 73 | _mtuCache.emplace(ipAsInt, true); 74 | socketEntry->dns->cachePriority = numeric_limits::max(); 75 | } else { 76 | _mtuCache.emplace(ipAsInt, false); 77 | } 78 | } 79 | } 80 | return socketEntry; 81 | } 82 | //--------------------------------------------------------------------------- 83 | }; // namespace cloud 84 | //--------------------------------------------------------------------------- 85 | }; // namespace anyblob 86 | -------------------------------------------------------------------------------- /src/cloud/http.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/http.hpp" 2 | #include "network/http_request.hpp" 3 | #include "utils/data_vector.hpp" 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2024 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace cloud { 16 | //--------------------------------------------------------------------------- 17 | using namespace std; 18 | //--------------------------------------------------------------------------- 19 | unique_ptr> HTTP::getRequest(const string& filePath, const pair& range) const 20 | // Builds the http request for downloading a blob 21 | { 22 | network::HttpRequest request; 23 | request.method = network::HttpRequest::Method::GET; 24 | request.type = network::HttpRequest::Type::HTTP_1_1; 25 | request.path = "/" + filePath; 26 | 27 | request.headers.emplace("Host", getAddress()); 28 | if (range.first != range.second) { 29 | stringstream rangeString; 30 | rangeString << "bytes=" << range.first << "-" << range.second; 31 | request.headers.emplace("Range", rangeString.str()); 32 | } 33 | 34 | string httpHeader = network::HttpRequest::getRequestMethod(request.method); 35 | httpHeader += " " + request.path + " "; 36 | httpHeader += network::HttpRequest::getRequestType(request.type); 37 | httpHeader += "\r\n"; 38 | for (const auto& h : request.headers) 39 | httpHeader += h.first + ": " + h.second + "\r\n"; 40 | httpHeader += "\r\n"; 41 | 42 | return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); 43 | } 44 | //--------------------------------------------------------------------------- 45 | unique_ptr> HTTP::putRequest(const string& filePath, string_view object) const 46 | // Builds the http request for putting objects without the object data itself 47 | { 48 | network::HttpRequest request; 49 | request.method = network::HttpRequest::Method::PUT; 50 | request.type = network::HttpRequest::Type::HTTP_1_1; 51 | request.path = "/" + filePath; 52 | auto bodyLength = object.size(); 53 | 54 | request.headers.emplace("Host", getAddress()); 55 | request.headers.emplace("Content-Length", to_string(bodyLength)); 56 | 57 | string httpHeader = network::HttpRequest::getRequestMethod(request.method); 58 | httpHeader += " " + request.path + " "; 59 | httpHeader += network::HttpRequest::getRequestType(request.type); 60 | httpHeader += "\r\n"; 61 | for (const auto& h : request.headers) 62 | httpHeader += h.first + ": " + h.second + "\r\n"; 63 | httpHeader += "\r\n"; 64 | 65 | return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); 66 | } 67 | //--------------------------------------------------------------------------- 68 | unique_ptr> HTTP::deleteRequest(const string& filePath) const 69 | // Builds the http request for deleting objects 70 | { 71 | network::HttpRequest request; 72 | request.method = network::HttpRequest::Method::DELETE; 73 | request.type = network::HttpRequest::Type::HTTP_1_1; 74 | request.path = "/" + filePath; 75 | 76 | request.headers.emplace("Host", getAddress()); 77 | 78 | string httpHeader = network::HttpRequest::getRequestMethod(request.method); 79 | httpHeader += " " + request.path + " "; 80 | httpHeader += network::HttpRequest::getRequestType(request.type); 81 | httpHeader += "\r\n"; 82 | for (const auto& h : request.headers) 83 | httpHeader += h.first + ": " + h.second + "\r\n"; 84 | httpHeader += "\r\n"; 85 | 86 | return make_unique>(reinterpret_cast(httpHeader.data()), reinterpret_cast(httpHeader.data() + httpHeader.size())); 87 | } 88 | //--------------------------------------------------------------------------- 89 | uint32_t HTTP::getPort() const 90 | // Gets the port of Azure on http 91 | { 92 | return _settings.port; 93 | } 94 | //--------------------------------------------------------------------------- 95 | string HTTP::getAddress() const 96 | // Gets the address of Azure 97 | { 98 | return _settings.hostname; 99 | } 100 | //--------------------------------------------------------------------------- 101 | Provider::Instance HTTP::getInstanceDetails(network::TaskedSendReceiverHandle& /*sendReceiver*/) 102 | // No real information for HTTP 103 | { 104 | return Instance{"http", 0, 0, 0}; 105 | } 106 | //--------------------------------------------------------------------------- 107 | } // namespace cloud 108 | } // namespace anyblob 109 | -------------------------------------------------------------------------------- /src/cloud/ibm.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/ibm.hpp" 2 | #include "cloud/ibm_instances.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2021 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace cloud { 14 | //--------------------------------------------------------------------------- 15 | using namespace std; 16 | //--------------------------------------------------------------------------- 17 | Provider::Instance IBM::getInstanceDetails(network::TaskedSendReceiverHandle& /*sendReceiver*/) 18 | // IBM instance info 19 | { 20 | // TODO: add instances and retrieve VM information 21 | return IBMInstance{"ibm", 0, 0, 0}; 22 | } 23 | //--------------------------------------------------------------------------- 24 | string IBM::getAddress() const 25 | // Gets the address of IBM COS 26 | { 27 | return "s3." + _settings.region + ".cloud-object-storage.appdomain.cloud"; 28 | } 29 | //--------------------------------------------------------------------------- 30 | } // namespace cloud 31 | } // namespace anyblob 32 | -------------------------------------------------------------------------------- /src/cloud/ibm_instances.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/ibm_instances.hpp" 2 | //--------------------------------------------------------------------------- 3 | // AnyBlob - Universal Cloud Object Storage Library 4 | // Dominik Durner, 2022 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 7 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // SPDX-License-Identifier: MPL-2.0 9 | //--------------------------------------------------------------------------- 10 | namespace anyblob { 11 | namespace cloud { 12 | //--------------------------------------------------------------------------- 13 | using namespace std; 14 | //--------------------------------------------------------------------------- 15 | vector IBMInstance::getInstanceDetails() 16 | // Gets a vector of instance type infos 17 | { 18 | // TODO: add instances 19 | vector instances = {}; 20 | return instances; 21 | } 22 | //--------------------------------------------------------------------------- 23 | }; // namespace cloud 24 | }; // namespace anyblob 25 | -------------------------------------------------------------------------------- /src/cloud/minio.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/minio.hpp" 2 | #include 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace cloud { 13 | //--------------------------------------------------------------------------- 14 | using namespace std; 15 | //--------------------------------------------------------------------------- 16 | Provider::Instance MinIO::getInstanceDetails(network::TaskedSendReceiverHandle& /*sendReceiver*/) 17 | // No real information for MinIO 18 | { 19 | return AWSInstance{"minio", 0, 0, 0}; 20 | } 21 | //--------------------------------------------------------------------------- 22 | string MinIO::getAddress() const 23 | // Gets the address of MinIO 24 | { 25 | // MinIO does not support virtual-hosted adresses, thus we use path-style requests 26 | assert(!_settings.endpoint.empty()); 27 | return _settings.endpoint; 28 | } 29 | //--------------------------------------------------------------------------- 30 | } // namespace cloud 31 | } // namespace anyblob 32 | -------------------------------------------------------------------------------- /src/cloud/oracle.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/oracle.hpp" 2 | #include "cloud/oracle_instances.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2021 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace cloud { 14 | //--------------------------------------------------------------------------- 15 | using namespace std; 16 | //--------------------------------------------------------------------------- 17 | Provider::Instance Oracle::getInstanceDetails(network::TaskedSendReceiverHandle& /*sendReceiver*/) 18 | // Get the instance details 19 | { 20 | // TODO: add instances and retrieve shape information 21 | return OracleInstance{"oracle", 0, 0, 0}; 22 | } 23 | //--------------------------------------------------------------------------- 24 | string Oracle::getAddress() const 25 | // Gets the address of the Oracle Cloud Storage 26 | { 27 | return _settings.bucket + ".compat.objectstorage." + _settings.region + ".oraclecloud.com"; 28 | } 29 | //--------------------------------------------------------------------------- 30 | } // namespace cloud 31 | } // namespace anyblob 32 | -------------------------------------------------------------------------------- /src/cloud/oracle_instances.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/oracle_instances.hpp" 2 | //--------------------------------------------------------------------------- 3 | // AnyBlob - Universal Cloud Object Storage Library 4 | // Dominik Durner, 2022 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 7 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // SPDX-License-Identifier: MPL-2.0 9 | //--------------------------------------------------------------------------- 10 | namespace anyblob { 11 | namespace cloud { 12 | //--------------------------------------------------------------------------- 13 | using namespace std; 14 | //--------------------------------------------------------------------------- 15 | vector OracleInstance::getInstanceDetails() 16 | // Gets a vector of instance type infos 17 | { 18 | // TODO: add instances 19 | vector instances = {}; 20 | return instances; 21 | } 22 | //--------------------------------------------------------------------------- 23 | }; // namespace cloud 24 | }; // namespace anyblob 25 | -------------------------------------------------------------------------------- /src/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE SRC_CPP src/*.cpp) 6 | list(REMOVE_ITEM SRC_CPP ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp) 7 | -------------------------------------------------------------------------------- /src/network/http_helper.cpp: -------------------------------------------------------------------------------- 1 | #include "network/http_helper.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | //--------------------------------------------------------------------------- 9 | // AnyBlob - Universal Cloud Object Storage Library 10 | // Dominik Durner, 2021 11 | // 12 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 13 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | // SPDX-License-Identifier: MPL-2.0 15 | //--------------------------------------------------------------------------- 16 | namespace anyblob { 17 | namespace network { 18 | //--------------------------------------------------------------------------- 19 | using namespace std; 20 | //--------------------------------------------------------------------------- 21 | HttpHelper::Info HttpHelper::detect(string_view header) 22 | // Detect the protocol 23 | { 24 | Info info; 25 | info.response = HttpResponse::deserialize(header); 26 | 27 | static constexpr string_view transferEncoding = "Transfer-Encoding"; 28 | static constexpr string_view chunkedEncoding = "chunked"; 29 | static constexpr string_view contentLength = "Content-Length"; 30 | static constexpr string_view headerEnd = "\r\n\r\n"; 31 | 32 | info.encoding = Encoding::Unknown; 33 | for (auto& keyValue : info.response.headers) { 34 | if (transferEncoding == keyValue.first && chunkedEncoding == keyValue.second) { 35 | info.encoding = Encoding::ChunkedEncoding; 36 | auto end = header.find(headerEnd); 37 | assert(end != string_view::npos); 38 | info.headerLength = static_cast(end) + static_cast(headerEnd.length()); 39 | } else if (contentLength == keyValue.first) { 40 | info.encoding = Encoding::ContentLength; 41 | from_chars(keyValue.second.data(), keyValue.second.data() + keyValue.second.size(), info.length); 42 | auto end = header.find(headerEnd); 43 | assert(end != string_view::npos); 44 | info.headerLength = static_cast(end) + static_cast(headerEnd.length()); 45 | } 46 | } 47 | 48 | if (info.encoding == Encoding::Unknown && !HttpResponse::withoutContent(info.response.code)) 49 | throw runtime_error("Unsupported HTTP encoding protocol"); 50 | 51 | if (HttpResponse::withoutContent(info.response.code)) 52 | info.headerLength = static_cast(header.length()); 53 | 54 | return info; 55 | } 56 | //--------------------------------------------------------------------------- 57 | string_view HttpHelper::retrieveContent(const uint8_t* data, uint64_t length, unique_ptr& info) 58 | // Retrieve the content without http meta info, note that this changes data 59 | { 60 | string_view sv(reinterpret_cast(data), length); 61 | if (!info) 62 | info = make_unique(detect(sv)); 63 | if (info->encoding == Encoding::ContentLength) 64 | return string_view(reinterpret_cast(data) + info->headerLength, info->length); 65 | return {}; 66 | } 67 | //--------------------------------------------------------------------------- 68 | bool HttpHelper::finished(const uint8_t* data, uint64_t length, unique_ptr& info) 69 | // Detect end / content 70 | { 71 | if (!info) { 72 | string_view sv(reinterpret_cast(data), length); 73 | info = make_unique(detect(sv)); 74 | } 75 | if (info && HttpResponse::withoutContent(info->response.code)) 76 | return true; 77 | switch (info->encoding) { 78 | case Encoding::ContentLength: 79 | return length >= info->headerLength + info->length; 80 | case Encoding::ChunkedEncoding: { 81 | string_view sv(reinterpret_cast(data), length); 82 | info->length = sv.find("0\r\n\r\n"sv); 83 | bool ret = info->length != sv.npos; 84 | if (ret) 85 | info->length -= info->headerLength; 86 | return ret; 87 | } 88 | default: { 89 | info = nullptr; 90 | throw runtime_error("Unsupported HTTP transfer protocol"); 91 | } 92 | } 93 | } 94 | //--------------------------------------------------------------------------- 95 | } // namespace network 96 | } // namespace anyblob 97 | -------------------------------------------------------------------------------- /src/network/http_response.cpp: -------------------------------------------------------------------------------- 1 | #include "network/http_response.hpp" 2 | #include "utils/data_vector.hpp" 3 | #include "utils/utils.hpp" 4 | #include 5 | #include 6 | #include 7 | #include 8 | //--------------------------------------------------------------------------- 9 | // AnyBlob - Universal Cloud Object Storage Library 10 | // Dominik Durner, 2024 11 | // 12 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 13 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | // SPDX-License-Identifier: MPL-2.0 15 | //--------------------------------------------------------------------------- 16 | namespace anyblob { 17 | namespace network { 18 | //--------------------------------------------------------------------------- 19 | using namespace std; 20 | //--------------------------------------------------------------------------- 21 | HttpResponse HttpResponse::deserialize(string_view data) 22 | // Deserialize the http header 23 | { 24 | static constexpr string_view strHttp1_0 = "HTTP/1.0"; 25 | static constexpr string_view strHttp1_1 = "HTTP/1.1"; 26 | static constexpr string_view strNewline = "\r\n"; 27 | static constexpr string_view strHeaderSeperator = ": "; 28 | 29 | HttpResponse response; 30 | 31 | string_view line; 32 | auto firstLine = true; 33 | while (true) { 34 | auto pos = data.find(strNewline); 35 | if (pos == data.npos) 36 | throw runtime_error("Invalid HttpResponse: Incomplete header!"); 37 | 38 | line = data.substr(0, pos); 39 | data = data.substr(pos + strNewline.size()); 40 | if (line.empty()) { 41 | if (!firstLine) 42 | break; 43 | else 44 | throw runtime_error("Invalid HttpResponse: Missing first line!"); 45 | } 46 | if (firstLine) { 47 | firstLine = false; 48 | // the http type 49 | if (line.starts_with(strHttp1_0)) { 50 | response.type = Type::HTTP_1_0; 51 | } else if (line.starts_with(strHttp1_1)) { 52 | response.type = Type::HTTP_1_1; 53 | } else { 54 | throw runtime_error("Invalid HttpResponse: Needs to be a HTTP type 1.0 or 1.1!"); 55 | } 56 | 57 | string_view httpType = getResponseType(response.type); 58 | line = line.substr(httpType.size() + 1); 59 | 60 | // the response type 61 | response.code = Code::UNKNOWN; 62 | for (auto code = static_cast(Code::OK_200); code <= static_cast(Code::SLOW_DOWN_503); code++) { 63 | const string_view responseCode = getResponseCode(static_cast(code)); 64 | if (line.starts_with(responseCode)) { 65 | response.code = static_cast(code); 66 | } 67 | } 68 | } else { 69 | // headers 70 | auto keyPos = line.find(strHeaderSeperator); 71 | string_view key, value = ""; 72 | if (keyPos == line.npos) { 73 | throw runtime_error("Invalid HttpResponse: Headers need key and value!"); 74 | } else { 75 | key = line.substr(0, keyPos); 76 | value = line.substr(keyPos + strHeaderSeperator.size()); 77 | } 78 | response.headers.emplace(key, value); 79 | } 80 | } 81 | 82 | return response; 83 | } 84 | //--------------------------------------------------------------------------- 85 | } // namespace network 86 | } // namespace anyblob 87 | -------------------------------------------------------------------------------- /src/network/message_task.cpp: -------------------------------------------------------------------------------- 1 | #include "network/message_task.hpp" 2 | //--------------------------------------------------------------------------- 3 | // AnyBlob - Universal Cloud Object Storage Library 4 | // Dominik Durner, 2022 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 7 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // SPDX-License-Identifier: MPL-2.0 9 | //--------------------------------------------------------------------------- 10 | namespace anyblob { 11 | namespace network { 12 | //--------------------------------------------------------------------------- 13 | using namespace std; 14 | //--------------------------------------------------------------------------- 15 | MessageTask::MessageTask(OriginalMessage* message, ConnectionManager::TCPSettings& tcpSettings, uint32_t chunkSize) : originalMessage(message), tcpSettings(tcpSettings), sendBufferOffset(0), receiveBufferOffset(0), chunkSize(chunkSize), failures(0) 16 | // The constructor 17 | { 18 | } 19 | //--------------------------------------------------------------------------- 20 | } // namespace network 21 | } // namespace anyblob 22 | -------------------------------------------------------------------------------- /src/network/throughput_cache.cpp: -------------------------------------------------------------------------------- 1 | #ifndef ANYBLOB_LIBCXX_COMPAT 2 | #include "network/throughput_cache.hpp" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | //--------------------------------------------------------------------------- 10 | // AnyBlob - Universal Cloud Object Storage Library 11 | // Dominik Durner, 2022 12 | // 13 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 14 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 15 | // SPDX-License-Identifier: MPL-2.0 16 | //--------------------------------------------------------------------------- 17 | namespace anyblob { 18 | namespace network { 19 | //--------------------------------------------------------------------------- 20 | using namespace std; 21 | //--------------------------------------------------------------------------- 22 | ThroughputCache::ThroughputCache() : Cache(), _throughputTree(), _throughput(), _throughputIterator() 23 | // Constructor 24 | { 25 | _defaultPriority = 2; 26 | _throughput.resize(_maxHistory); 27 | } 28 | //--------------------------------------------------------------------------- 29 | void ThroughputCache::startSocket(int fd) 30 | // Start a socket 31 | { 32 | _fdMap.emplace(fd, chrono::steady_clock::now()); 33 | } 34 | //--------------------------------------------------------------------------- 35 | void ThroughputCache::stopSocket(std::unique_ptr socketEntry, uint64_t bytes, unsigned cachedEntries, bool reuseSocket) 36 | // Stop a socket 37 | { 38 | auto now = chrono::steady_clock::now(); 39 | auto it = _fdMap.find(socketEntry->fd); 40 | if (it != _fdMap.end()) { 41 | auto timeNs = chrono::duration_cast(now - it->second).count(); 42 | auto throughput = static_cast(bytes) / chrono::duration(timeNs).count(); 43 | auto curThroughput = _throughputIterator++; 44 | auto del = _throughput[curThroughput % _maxHistory]; 45 | _throughput[curThroughput % _maxHistory] = throughput; 46 | _throughputTree.erase(del); 47 | _throughputTree.insert(throughput); 48 | auto maxElem = _throughputIterator < _maxHistory ? _throughputIterator : _maxHistory; 49 | bool possible = true; 50 | if (maxElem > 3) { 51 | auto percentile = _throughputTree.find_by_order(maxElem / 3); 52 | if (percentile.m_p_nd->m_value <= throughput) 53 | socketEntry->dns->cachePriority += 1; 54 | else 55 | possible = false; 56 | } 57 | if (possible && maxElem > 6) { 58 | auto percentile = _throughputTree.find_by_order(maxElem / 6); 59 | if (percentile.m_p_nd->m_value <= throughput) 60 | socketEntry->dns->cachePriority += 2; 61 | } 62 | } 63 | Cache::stopSocket(move(socketEntry), bytes, cachedEntries, reuseSocket); 64 | } 65 | //--------------------------------------------------------------------------- 66 | }; // namespace network 67 | //--------------------------------------------------------------------------- 68 | }; // namespace anyblob 69 | #endif // ANYBLOB_LIBCXX_COMPAT 70 | -------------------------------------------------------------------------------- /src/network/tls_context.cpp: -------------------------------------------------------------------------------- 1 | #include "network/tls_context.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | //--------------------------------------------------------------------------- 7 | // AnyBlob - Universal Cloud Object Storage Library 8 | // Dominik Durner, 2023 9 | // 10 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 11 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | // SPDX-License-Identifier: MPL-2.0 13 | //--------------------------------------------------------------------------- 14 | namespace anyblob { 15 | namespace network { 16 | //--------------------------------------------------------------------------- 17 | using namespace std; 18 | //--------------------------------------------------------------------------- 19 | TLSContext::TLSContext() : _sessionCache() 20 | // Construct the TLS Context 21 | { 22 | // Set to TLS 23 | auto method = TLS_client_method(); 24 | 25 | // Set up the context 26 | _ctx = SSL_CTX_new(method); 27 | 28 | // Enable session cache 29 | SSL_CTX_set_session_cache_mode(_ctx, SSL_SESS_CACHE_CLIENT); 30 | } 31 | //--------------------------------------------------------------------------- 32 | TLSContext::~TLSContext() 33 | // The desturctor 34 | { 35 | // Remove all sessions 36 | for (uint64_t i = 0ull; i < (1ull << cachePower); i++) { 37 | if (_sessionCache[i].first && _sessionCache[i].second) { 38 | SSL_SESSION_free(_sessionCache[i].second); 39 | } 40 | } 41 | 42 | // Destroy context 43 | if (_ctx) 44 | SSL_CTX_free(_ctx); 45 | } 46 | //--------------------------------------------------------------------------- 47 | void TLSContext::initOpenSSL() 48 | // Inits the openssl algos 49 | { 50 | // Load algos 51 | OpenSSL_add_ssl_algorithms(); 52 | SSL_load_error_strings(); 53 | } 54 | //--------------------------------------------------------------------------- 55 | bool TLSContext::cacheSession(int fd, SSL* ssl) 56 | // Caches the SSL session 57 | { 58 | // Is the session already cached? 59 | if (SSL_session_reused(ssl)) 60 | return false; 61 | 62 | // Get the IP address for caching 63 | struct sockaddr_in addr; 64 | socklen_t addrLen = sizeof(struct sockaddr_in); 65 | if (!getpeername(fd, reinterpret_cast(&addr), &addrLen)) { 66 | if (_sessionCache[addr.sin_addr.s_addr & cacheMask].first) { 67 | SSL_SESSION_free(_sessionCache[addr.sin_addr.s_addr & cacheMask].second); 68 | } 69 | _sessionCache[addr.sin_addr.s_addr & cacheMask] = pair(addr.sin_addr.s_addr, SSL_get1_session(ssl)); 70 | return true; 71 | } 72 | return false; 73 | } 74 | //--------------------------------------------------------------------------- 75 | bool TLSContext::dropSession(int fd) 76 | // Drop the SSL session from cache 77 | { 78 | struct sockaddr_in addr; 79 | socklen_t addrLen = sizeof(struct sockaddr_in); 80 | if (!getpeername(fd, reinterpret_cast(&addr), &addrLen)) { 81 | if (_sessionCache[addr.sin_addr.s_addr & cacheMask].first) { 82 | auto session = _sessionCache[addr.sin_addr.s_addr & cacheMask].second; 83 | _sessionCache[addr.sin_addr.s_addr & cacheMask].first = 0; 84 | SSL_SESSION_free(session); 85 | } 86 | return true; 87 | } 88 | return false; 89 | } 90 | //--------------------------------------------------------------------------- 91 | bool TLSContext::reuseSession(int fd, SSL* ssl) 92 | // Reuses the SSL session 93 | { 94 | struct sockaddr_in addr; 95 | socklen_t addrLen = sizeof(struct sockaddr_in); 96 | if (!getpeername(fd, reinterpret_cast(&addr), &addrLen)) { 97 | auto& pair = _sessionCache[addr.sin_addr.s_addr & cacheMask]; 98 | if (pair.first == addr.sin_addr.s_addr) { 99 | SSL_set_session(ssl, pair.second); 100 | return true; 101 | } 102 | } 103 | return false; 104 | } 105 | //--------------------------------------------------------------------------- 106 | } // namespace network 107 | } // namespace anyblob 108 | -------------------------------------------------------------------------------- /src/network/transaction.cpp: -------------------------------------------------------------------------------- 1 | #include "network/transaction.hpp" 2 | #include "network/original_message.hpp" 3 | #include "network/tasked_send_receiver.hpp" 4 | #include "utils/data_vector.hpp" 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2023 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace network { 15 | //--------------------------------------------------------------------------- 16 | using namespace std; 17 | //--------------------------------------------------------------------------- 18 | void Transaction::processSync(TaskedSendReceiverHandle& sendReceiverHandle) 19 | // Processes the request messages 20 | { 21 | assert(sendReceiverHandle.has()); 22 | do { 23 | // send the original request message 24 | for (; _messageCounter < _messages.size(); _messageCounter++) { 25 | sendReceiverHandle.sendSync(_messages[_messageCounter].get()); 26 | } 27 | 28 | for (auto& multipart : _multipartUploads) { 29 | if (multipart.state == MultipartUpload::State::Sending) { 30 | for (auto i = 0ull; i < multipart.eTags.size(); i++) 31 | sendReceiverHandle.sendSync(multipart.messages[i].get()); 32 | multipart.state = MultipartUpload::State::Default; 33 | } else if (multipart.state == MultipartUpload::State::Validating) { 34 | sendReceiverHandle.sendSync(multipart.messages[multipart.eTags.size()].get()); 35 | multipart.state = MultipartUpload::State::Default; 36 | } 37 | } 38 | 39 | // do the download work 40 | sendReceiverHandle.processSync(); 41 | } while (_multipartUploads.size() != _completedMultiparts); 42 | } 43 | //--------------------------------------------------------------------------- 44 | bool Transaction::processAsync(TaskedSendReceiverGroup& group) 45 | // Sends the request messages to the task group 46 | { 47 | // send the original request message 48 | vector submissions; 49 | auto multiPartSize = 0ull; 50 | for (auto& multipart : _multipartUploads) { 51 | multiPartSize += multipart.messages.size(); 52 | } 53 | submissions.reserve(_messages.size() + multiPartSize); 54 | for (; _messageCounter < _messages.size(); _messageCounter++) { 55 | submissions.emplace_back(_messages[_messageCounter].get()); 56 | } 57 | for (auto& multipart : _multipartUploads) { 58 | if (multipart.state == MultipartUpload::State::Sending) { 59 | for (auto i = 0ull; i < multipart.eTags.size(); i++) 60 | submissions.emplace_back(multipart.messages[i].get()); 61 | } else if (multipart.state == MultipartUpload::State::Validating) { 62 | submissions.emplace_back(multipart.messages[multipart.eTags.size()].get()); 63 | } 64 | } 65 | auto success = group.send(submissions); 66 | if (!success) [[unlikely]] 67 | return success; 68 | for (auto& multipart : _multipartUploads) { 69 | if (multipart.state == MultipartUpload::State::Sending || multipart.state == MultipartUpload::State::Validating) { 70 | multipart.state = MultipartUpload::State::Default; 71 | } 72 | } 73 | return success; 74 | } 75 | //--------------------------------------------------------------------------- 76 | Transaction::Iterator::reference Transaction::Iterator::operator*() const 77 | // Reference 78 | { 79 | return (*it)->result; 80 | } 81 | //--------------------------------------------------------------------------- 82 | Transaction::Iterator::pointer Transaction::Iterator::operator->() const 83 | // Pointer 84 | { 85 | return &(*it)->result; 86 | } 87 | //--------------------------------------------------------------------------- 88 | Transaction::ConstIterator::reference Transaction::ConstIterator::operator*() const 89 | // Reference 90 | { 91 | return (*it)->result; 92 | } 93 | //--------------------------------------------------------------------------- 94 | Transaction::ConstIterator::pointer Transaction::ConstIterator::operator->() const 95 | // Pointer 96 | { 97 | return &(*it)->result; 98 | } 99 | //--------------------------------------------------------------------------- 100 | }; // namespace network 101 | }; // namespace anyblob 102 | -------------------------------------------------------------------------------- /src/utils/timer.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/timer.hpp" 2 | #include 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace utils { 13 | //--------------------------------------------------------------------------- 14 | using namespace std; 15 | //--------------------------------------------------------------------------- 16 | Timer& Timer::operator=(Timer&& rhs) 17 | // Starts timer for step s 18 | { 19 | for (auto it : _totalTimer) { 20 | auto res = rhs._totalTimer.find(it.first); 21 | if (res != rhs._totalTimer.end()) { 22 | it.second += res->second; 23 | rhs._totalTimer.erase(res); 24 | } 25 | } 26 | for (auto it : rhs._totalTimer) { 27 | _totalTimer.insert(it); 28 | } 29 | return *this; 30 | } 31 | //--------------------------------------------------------------------------- 32 | void Timer::start(Steps s) 33 | // Starts timer for step s 34 | { 35 | _currentTimer[s] = chrono::steady_clock::now(); 36 | if (_totalLoad.find(s) == _totalLoad.end()) { 37 | auto vec = make_unique>>(); 38 | vec->emplace_back(make_unique()); 39 | _totalLoad.insert({s, move(vec)}); 40 | } 41 | _loadTracker[s] = make_unique(*_totalLoad.find(s)->second.get()); 42 | } 43 | //--------------------------------------------------------------------------- 44 | void Timer::stop(Steps s) 45 | // Stops timer for step s 46 | { 47 | _loadTracker[s].reset(); 48 | auto time = chrono::duration_cast(chrono::steady_clock::now() - _currentTimer[s]).count(); 49 | _totalTimer[s] += static_cast(time); 50 | } 51 | //--------------------------------------------------------------------------- 52 | void Timer::printResult(ostream& s) 53 | // Print results 54 | { 55 | // The reverse steps 56 | string reverseSteps[] = {"Overall", "Upload", "Download", "Submit", "Request", "Response", "Buffer", "BufferCapacity", "BufferResize", "Finishing", "Waiting", "Queue", "SocketCreation", "BufferCapacitySend"}; 57 | 58 | if (_writeHeader) 59 | s << "Step,Pct,Time,CPUUserTime,CPUSysTime,CPUElapsedTime,CPUActiveAllProcesses,CPUIdleAllProcesses" << _headerInfo << endl; 60 | uint64_t total = 0; 61 | for (auto it : _totalTimer) { 62 | total += it.second; 63 | } 64 | for (auto it : _totalTimer) { 65 | s << reverseSteps[it.first] << "," << static_cast(it.second) / static_cast(total) << "," << static_cast(it.second) / (1000 * 1000); 66 | auto elem = _totalLoad.find(it.first); 67 | if (elem != _totalLoad.end()) 68 | s << "," << elem->second->at(0)->userTime << "," << elem->second->at(0)->sysTime << "," << elem->second->at(0)->elapsedTime << "," << elem->second->at(0)->activeTimeAllProcesses << "," << elem->second->at(0)->idleTimeAllProcesses; 69 | else 70 | s << ",-1,-1,-1,-1,-1"; 71 | s << _contentInfo << endl; 72 | } 73 | } 74 | //--------------------------------------------------------------------------- 75 | void Timer::setInfo(string headerInfo, string contentInfo) 76 | // Sets the info 77 | { 78 | this->_headerInfo = headerInfo; 79 | this->_contentInfo = contentInfo; 80 | } 81 | //--------------------------------------------------------------------------- 82 | }; // namespace utils 83 | }; // namespace anyblob 84 | -------------------------------------------------------------------------------- /test/integration/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE INTEGRATION_CPP test/integration/*.cpp) 6 | list(REMOVE_ITEM INTEGRATION_CPP ${CMAKE_CURRENT_SOURCE_DIR}/test/integration/test.cpp) 7 | -------------------------------------------------------------------------------- /test/integration/test.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_MAIN 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | -------------------------------------------------------------------------------- /test/unit/cloud/azure_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/azure.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | #include "cloud/azure_instances.hpp" 4 | #include "cloud/azure_signer.hpp" 5 | #include "utils/data_vector.hpp" 6 | #include 7 | #include 8 | #include 9 | #include 10 | //--------------------------------------------------------------------------- 11 | // AnyBlob - Universal Cloud Object Storage Library 12 | // Dominik Durner, 2022 13 | // 14 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 15 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 16 | // SPDX-License-Identifier: MPL-2.0 17 | //--------------------------------------------------------------------------- 18 | namespace anyblob { 19 | namespace cloud { 20 | namespace test { 21 | //--------------------------------------------------------------------------- 22 | using namespace std; 23 | //--------------------------------------------------------------------------- 24 | // Helper to test private methods 25 | class AzureTester { 26 | public: 27 | void test() { 28 | Provider::testEnviornment = true; 29 | 30 | auto provider = Provider::makeProvider("azure://test/", false, "test", ""); 31 | Azure& azure = *static_cast(provider.get()); 32 | REQUIRE(!azure.getIAMAddress().compare("169.254.169.254")); 33 | REQUIRE(azure.getIAMPort() == 80); 34 | 35 | auto dv = azure.downloadInstanceInfo(); 36 | string resultString = "GET /metadata/instance?api-version=2021-02-01 HTTP/1.1\r\nHost: 169.254.169.254\r\nMetadata: true\r\n\r\n"; 37 | REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); 38 | 39 | auto p = pair(numeric_limits::max(), numeric_limits::max()); 40 | dv = azure.getRequest("a/b/c.d", p); 41 | resultString = "GET /test/a/b/c.d HTTP/1.1\r\nAuthorization: SharedKey test:uhjLcL68dDerTH3WiZ3Zuk0tm3WX+hdmMktg8cYJ74w=\r\nHost: test.blob.core.windows.net\r\nx-ms-date: "; 42 | resultString += azure.fakeXMSTimestamp; 43 | resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n"; 44 | REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); 45 | 46 | utils::DataVector putData(10); 47 | dv = azure.putRequest("a/b/c.d", string_view(reinterpret_cast(putData.data()), putData.size())); 48 | resultString = "PUT /test/a/b/c.d HTTP/1.1\r\nAuthorization: SharedKey test:AiWIKIaUYFV5UOGADs2R+/C8jQu0pW0+lrWV1IfW7Lc=\r\nContent-Length: 10\r\nHost: test.blob.core.windows.net\r\nx-ms-blob-type: BlockBlob\r\nx-ms-date: "; 49 | resultString += azure.fakeXMSTimestamp; 50 | resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n"; 51 | REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); 52 | 53 | dv = azure.deleteRequest("a/b/c.d"); 54 | resultString = "DELETE /test/a/b/c.d HTTP/1.1\r\nAuthorization: SharedKey test:nuGDW7QRI5/DB5Xt9vET/YEmipJ4UGjn64h4A+BFaL0=\r\nHost: test.blob.core.windows.net\r\nx-ms-date: "; 55 | resultString += azure.fakeXMSTimestamp; 56 | resultString += "\r\nx-ms-version: 2015-02-21\r\n\r\n"; 57 | REQUIRE(string_view(reinterpret_cast(dv->data()), dv->size()) == resultString); 58 | 59 | Provider::testEnviornment = false; 60 | } 61 | }; 62 | //--------------------------------------------------------------------------- 63 | TEST_CASE("azure") { 64 | AzureTester tester; 65 | tester.test(); 66 | } 67 | //--------------------------------------------------------------------------- 68 | } // namespace test 69 | } // namespace cloud 70 | } // namespace anyblob 71 | -------------------------------------------------------------------------------- /test/unit/cloud/provider_test.cpp: -------------------------------------------------------------------------------- 1 | #include "cloud/provider.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace cloud { 13 | namespace test { 14 | //--------------------------------------------------------------------------- 15 | TEST_CASE("provider") { 16 | REQUIRE(Provider::isRemoteFile("s3://a/b/c")); 17 | REQUIRE(!Provider::isRemoteFile("a/b/c")); 18 | REQUIRE(Provider::getRemoteParentDirectory("s3://bucket/b/c") == "b/"); 19 | auto info = Provider::getRemoteInfo("s3://x:y/b"); 20 | REQUIRE(info.bucket == "x"); 21 | REQUIRE(info.region == "y"); 22 | info = Provider::getRemoteInfo("s3://x/b"); 23 | REQUIRE(info.bucket == "x"); 24 | REQUIRE(info.region == ""); 25 | } 26 | //--------------------------------------------------------------------------- 27 | } // namespace test 28 | } // namespace cloud 29 | } // namespace anyblob 30 | -------------------------------------------------------------------------------- /test/unit/local.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | # Files 3 | # --------------------------------------------------------------------------- 4 | 5 | file(GLOB_RECURSE TEST_CPP test/unit/*.cpp) 6 | list(REMOVE_ITEM TEST_CPP ${CMAKE_CURRENT_SOURCE_DIR}/test/unit/test.cpp) 7 | -------------------------------------------------------------------------------- /test/unit/network/http_helper_test.cpp: -------------------------------------------------------------------------------- 1 | #include "network/http_helper.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace network { 13 | namespace test { 14 | //--------------------------------------------------------------------------- 15 | TEST_CASE("http_helper") { 16 | } 17 | //--------------------------------------------------------------------------- 18 | } // namespace test 19 | } // namespace network 20 | } // namespace anyblob 21 | -------------------------------------------------------------------------------- /test/unit/network/http_request_test.cpp: -------------------------------------------------------------------------------- 1 | #include "network/http_request.hpp" 2 | #include "utils/data_vector.hpp" 3 | #include "catch2/single_include/catch2/catch.hpp" 4 | #include 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2022 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace network { 15 | namespace test { 16 | //--------------------------------------------------------------------------- 17 | TEST_CASE("http_request") { 18 | network::HttpRequest request; 19 | 20 | request.method = network::HttpRequest::Method::GET; 21 | request.path = "/test"; 22 | request.type = network::HttpRequest::Type::HTTP_1_1; 23 | request.queries.emplace("key", "value"); 24 | request.queries.emplace("key2", "value2"); 25 | request.headers.emplace("Authorization", "test"); 26 | request.headers.emplace("Timestamp", "2024-02-18 00:00:00"); 27 | 28 | auto serialize = network::HttpRequest::serialize(request); 29 | auto serializeView = std::string_view(reinterpret_cast(serialize->data()), serialize->size()); 30 | 31 | auto deserializedRequest = network::HttpRequest::deserialize(serializeView); 32 | 33 | auto serializeAgain = network::HttpRequest::serialize(deserializedRequest); 34 | auto serializeAgainView = std::string_view(reinterpret_cast(serializeAgain->data()), serializeAgain->size()); 35 | 36 | REQUIRE(serializeView == serializeAgainView); 37 | } 38 | //--------------------------------------------------------------------------- 39 | } // namespace test 40 | } // namespace network 41 | } // namespace anyblob 42 | -------------------------------------------------------------------------------- /test/unit/network/io_uring_socket_test.cpp: -------------------------------------------------------------------------------- 1 | #include "catch2/single_include/catch2/catch.hpp" 2 | #include "cloud/aws_cache.hpp" 3 | #include "network/connection_manager.hpp" 4 | #include "perfevent/PerfEvent.hpp" 5 | //--------------------------------------------------------------------------- 6 | // AnyBlob - Universal Cloud Object Storage Library 7 | // Dominik Durner, 2021 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 10 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | // SPDX-License-Identifier: MPL-2.0 12 | //--------------------------------------------------------------------------- 13 | namespace anyblob { 14 | namespace network { 15 | namespace test { 16 | //--------------------------------------------------------------------------- 17 | TEST_CASE("io_uring_socket") { 18 | PerfEventBlock e; 19 | ConnectionManager io (10); 20 | ConnectionManager::TCPSettings tcpSettings; 21 | REQUIRE(io.connect("db.in.tum.de", 80, false, tcpSettings) > 0); 22 | } 23 | //--------------------------------------------------------------------------- 24 | } // namespace test 25 | } // namespace network 26 | } // namespace anyblob 27 | -------------------------------------------------------------------------------- /test/unit/network/messages_test.cpp: -------------------------------------------------------------------------------- 1 | #include "catch2/single_include/catch2/catch.hpp" 2 | #include "network/original_message.hpp" 3 | #include "utils/data_vector.hpp" 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace network { 14 | namespace test { 15 | //--------------------------------------------------------------------------- 16 | TEST_CASE("messages") { 17 | } 18 | //--------------------------------------------------------------------------- 19 | } // namespace test 20 | } // namespace network 21 | } // namespace anyblob 22 | -------------------------------------------------------------------------------- /test/unit/network/resolver_test.cpp: -------------------------------------------------------------------------------- 1 | #include "network/cache.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace network { 13 | namespace test { 14 | //--------------------------------------------------------------------------- 15 | TEST_CASE("cache") { 16 | } 17 | //--------------------------------------------------------------------------- 18 | } // namespace test 19 | } // namespace network 20 | } // namespace anyblob 21 | -------------------------------------------------------------------------------- /test/unit/network/send_receiver_test.cpp: -------------------------------------------------------------------------------- 1 | #include "catch2/single_include/catch2/catch.hpp" 2 | #include "cloud/http.hpp" 3 | #include "cloud/provider.hpp" 4 | #include "network/original_message.hpp" 5 | #include "network/tasked_send_receiver.hpp" 6 | #include "perfevent/PerfEvent.hpp" 7 | #include "utils/data_vector.hpp" 8 | #include 9 | //--------------------------------------------------------------------------- 10 | // AnyBlob - Universal Cloud Object Storage Library 11 | // Dominik Durner, 2021 12 | // 13 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 14 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 15 | // SPDX-License-Identifier: MPL-2.0 16 | //--------------------------------------------------------------------------- 17 | #ifndef NDEBUG 18 | #define verify(expression) assert(expression) 19 | #else 20 | #define verify(expression) ((void) (expression)) 21 | #endif 22 | //--------------------------------------------------------------------------- 23 | namespace anyblob { 24 | namespace network { 25 | namespace test { 26 | //--------------------------------------------------------------------------- 27 | using namespace std; 28 | //--------------------------------------------------------------------------- 29 | TEST_CASE("send_receiver") { 30 | PerfEventBlock e; 31 | auto concurrency = 8u; 32 | auto requests = 64u; 33 | 34 | TaskedSendReceiverGroup group; 35 | group.setConcurrentRequests(concurrency); 36 | 37 | auto provider = cloud::Provider::makeProvider("http://db.cs.tum.edu"); 38 | auto tlsProvider = cloud::Provider::makeProvider("https://db.cs.tum.edu"); 39 | 40 | vector> msgs; 41 | for (auto i = 0u; i < requests; i++) { 42 | auto range = std::pair(0, 0); 43 | string file = ""; 44 | msgs.emplace_back(new OriginalMessage{provider->getRequest(file, range), *provider}); 45 | verify(group.send(msgs.back().get())); 46 | 47 | msgs.emplace_back(new OriginalMessage{tlsProvider->getRequest(file, range), *tlsProvider}); 48 | verify(group.send(msgs.back().get())); 49 | } 50 | 51 | group.process(true); 52 | 53 | unique_ptr currentMessage; 54 | size_t size; 55 | for (auto i = 0u; i < msgs.size();) { 56 | auto& original = msgs[i]; 57 | switch (original->result.getState()) { 58 | case MessageState::Finished: { 59 | auto& message = original->result; 60 | if (i > 0) { 61 | // skip header 62 | size_t skip = 1024; 63 | REQUIRE(size > skip); 64 | REQUIRE(!strncmp(reinterpret_cast(currentMessage.get()) + skip, reinterpret_cast(message.getData()) + skip, size - skip)); 65 | } 66 | currentMessage = message.moveData(); 67 | size = message.getOffset() + message.getSize(); 68 | ++i; 69 | break; 70 | } 71 | default: 72 | REQUIRE(original->result.getState() != MessageState::Aborted); 73 | } 74 | } 75 | } 76 | //--------------------------------------------------------------------------- 77 | } // namespace test 78 | } // namespace network 79 | } // namespace anyblob 80 | -------------------------------------------------------------------------------- /test/unit/network/tasked_send_receiver_test.cpp: -------------------------------------------------------------------------------- 1 | #include "network/tasked_send_receiver.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace network { 13 | namespace test { 14 | //--------------------------------------------------------------------------- 15 | TEST_CASE("tasked_send_receiver") { 16 | } 17 | //--------------------------------------------------------------------------- 18 | } // namespace test 19 | } // namespace network 20 | } // namespace anyblob 21 | -------------------------------------------------------------------------------- /test/unit/test.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_MAIN 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2021 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | -------------------------------------------------------------------------------- /test/unit/utils/data_vector_test.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/data_vector.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | //--------------------------------------------------------------------------- 4 | // AnyBlob - Universal Cloud Object Storage Library 5 | // Dominik Durner, 2022 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 8 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | // SPDX-License-Identifier: MPL-2.0 10 | //--------------------------------------------------------------------------- 11 | namespace anyblob { 12 | namespace utils { 13 | namespace test { 14 | //--------------------------------------------------------------------------- 15 | TEST_CASE("data_vector") { 16 | DataVector dv; 17 | dv.reserve(1); 18 | *dv.data() = 42; 19 | REQUIRE(dv.size() == 0); 20 | dv.resize(1); 21 | REQUIRE(*dv.cdata() == 42); 22 | REQUIRE(dv.capacity() == 1); 23 | dv.resize(2); 24 | *(dv.data() + 1) = 43; 25 | REQUIRE(dv.size() == 2); 26 | REQUIRE(dv.capacity() == 2); 27 | REQUIRE(*dv.cdata() == 42); 28 | REQUIRE(*(dv.cdata() + 1) == 43); 29 | auto dv2 = dv; 30 | REQUIRE(dv2.size() == 2); 31 | REQUIRE(dv2.capacity() == 2); 32 | } 33 | //--------------------------------------------------------------------------- 34 | } // namespace test 35 | } // namespace utils 36 | } // namespace anyblob 37 | -------------------------------------------------------------------------------- /test/unit/utils/ring_buffer_test.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/ring_buffer.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace utils { 14 | namespace test { 15 | //--------------------------------------------------------------------------- 16 | TEST_CASE("ring_buffer") { 17 | RingBuffer rb(2); 18 | REQUIRE(rb.insert(1) == 0); 19 | REQUIRE(rb.insert(2) == 1); 20 | REQUIRE(rb.insert(2) == ~0ull); 21 | REQUIRE(rb.consume().value() == 1); 22 | REQUIRE(rb.consume().value() == 2); 23 | REQUIRE(!rb.consume().has_value()); 24 | uint64_t a[] = {3, 4}; 25 | REQUIRE(rb.insertAll(a) == 2); 26 | REQUIRE(rb.consume().value() == 3); 27 | REQUIRE(rb.consume().value() == 4); 28 | REQUIRE(rb.empty()); 29 | } 30 | //--------------------------------------------------------------------------- 31 | TEST_CASE("single_threaded_insert_multi_threaded_consume") { 32 | RingBuffer rb(1000); 33 | for (auto i = 0u; i < 1000u; i++) { 34 | REQUIRE(rb.insert(i) != ~0ull); 35 | } 36 | std::vector threads; 37 | for (auto i = 0u; i < 10u; i++) { 38 | threads.push_back(std::thread([&] { 39 | for (int j = 0; j < 100; j++) { 40 | REQUIRE(rb.consume().value() < 1000ul); 41 | } 42 | })); 43 | } 44 | for (auto& th : threads) { 45 | th.join(); 46 | } 47 | REQUIRE(!rb.consume().has_value()); 48 | } 49 | //--------------------------------------------------------------------------- 50 | TEST_CASE("multi_threaded_ring_buffer") { 51 | RingBuffer rb(1000); 52 | std::vector threads; 53 | for (auto i = 0u; i < 10u; i++) { 54 | threads.push_back(std::thread([&] { 55 | for (auto j = 0u; j < 100u; j++) { 56 | REQUIRE(rb.insert(j) != ~0ull); 57 | } 58 | })); 59 | } 60 | for (auto& th : threads) { 61 | th.join(); 62 | } 63 | for (auto i = 0u; i < 1000u; i++) { 64 | REQUIRE(rb.consume().value() < 100ul); 65 | } 66 | REQUIRE(!rb.consume().has_value()); 67 | } 68 | //--------------------------------------------------------------------------- 69 | TEST_CASE("multi_threaded_ring_buffer_multi_threaded_consume") { 70 | RingBuffer rb(1000); 71 | std::vector threads; 72 | for (int i = 0; i < 10; i++) { 73 | threads.push_back(std::thread([&] { 74 | for (auto j = 0u; j < 100u; j++) { 75 | REQUIRE(rb.insert(j) != ~0ull); 76 | } 77 | })); 78 | } 79 | for (auto& th : threads) { 80 | th.join(); 81 | } 82 | // Consume multi-threaded 83 | threads.clear(); 84 | for (int i = 0; i < 10; i++) { 85 | threads.push_back(std::thread([&] { 86 | for (auto j = 0u; j < 100u; j++) { 87 | REQUIRE(rb.consume().value() < 100ul); 88 | } 89 | })); 90 | } 91 | for (auto& th : threads) { 92 | th.join(); 93 | } 94 | REQUIRE(!rb.consume().has_value()); 95 | } 96 | //--------------------------------------------------------------------------- 97 | } // namespace test 98 | } // namespace utils 99 | } // namespace anyblob 100 | -------------------------------------------------------------------------------- /test/unit/utils/unordered_map_test.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/unordered_map.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace utils { 14 | namespace test { 15 | //--------------------------------------------------------------------------- 16 | TEST_CASE("unordered_map") { 17 | UnorderedMap ht(10); 18 | REQUIRE(ht.buckets() == 10); 19 | auto ret = ht.insert(0, 0); 20 | REQUIRE(ret->first == 0); 21 | REQUIRE(ret->second == 0); 22 | ht.push(1, 1); 23 | REQUIRE(ht.size() == 2); 24 | 25 | REQUIRE(ht.find(0) != ht.end()); 26 | REQUIRE(ht.find(2) == ht.end()); 27 | 28 | REQUIRE(ht.erase(ht.find(0))); 29 | REQUIRE(ht.size() == 1); 30 | } 31 | //--------------------------------------------------------------------------- 32 | TEST_CASE("unordered_map_multi_threaded") { 33 | // Insert 1000 elements in 10 threads 34 | UnorderedMap ht(128); 35 | std::thread t[10]; 36 | for (int i = 0; i < 10; i++) { 37 | t[i] = std::thread([&ht, i]() { 38 | for (int j = 0; j < 10; j++) { 39 | std::ignore = ht.insert(i * 10 + j, i * 10 + j); 40 | } 41 | }); 42 | } 43 | for (int i = 0; i < 10; i++) { 44 | t[i].join(); 45 | } 46 | // Find the elements multu-threaded 47 | for (int i = 0; i < 10; i++) { 48 | t[i] = std::thread([&ht, i]() { 49 | for (int j = 0; j < 10; j++) { 50 | REQUIRE(ht.find(i * 10 + j) != ht.end()); 51 | } 52 | }); 53 | } 54 | for (int i = 0; i < 10; i++) { 55 | t[i].join(); 56 | } 57 | } 58 | //--------------------------------------------------------------------------- 59 | TEST_CASE("unordered_map_multi_threaded_delete") { 60 | UnorderedMap ht(128); 61 | // Insert 1000 elements in 10 threads 62 | std::thread t[10]; 63 | for (int i = 0; i < 10; i++) { 64 | t[i] = std::thread([&ht, i]() { 65 | for (int j = 0; j < 10; j++) { 66 | std::ignore = ht.insert(i * 10 + j, i * 10 + j); 67 | } 68 | }); 69 | } 70 | for (int i = 0; i < 10; i++) { 71 | t[i].join(); 72 | } 73 | // Delete the elements multi-threaded 74 | for (int i = 0; i < 10; i++) { 75 | t[i] = std::thread([&ht, i]() { 76 | for (int j = 0; j < 10; j++) { 77 | REQUIRE(ht.erase(i * 10 + j)); 78 | } 79 | }); 80 | } 81 | for (int i = 0; i < 10; i++) { 82 | t[i].join(); 83 | } 84 | } 85 | //--------------------------------------------------------------------------- 86 | } // namespace test 87 | } // namespace utils 88 | } // namespace anyblob 89 | -------------------------------------------------------------------------------- /test/unit/utils/utils_test.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/utils.hpp" 2 | #include "catch2/single_include/catch2/catch.hpp" 3 | #include 4 | //--------------------------------------------------------------------------- 5 | // AnyBlob - Universal Cloud Object Storage Library 6 | // Dominik Durner, 2022 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. 9 | // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | // SPDX-License-Identifier: MPL-2.0 11 | //--------------------------------------------------------------------------- 12 | namespace anyblob { 13 | namespace utils { 14 | namespace test { 15 | //--------------------------------------------------------------------------- 16 | using namespace std; 17 | //--------------------------------------------------------------------------- 18 | TEST_CASE("utils") { 19 | constexpr unsigned char key[] = "01234567890123456789012345678901"; 20 | constexpr unsigned char iv[] = "0123456789012345"; 21 | string plain = "AnyBlob - Universal Cloud Object Storage Library"; 22 | uint8_t buffer[512]; 23 | auto len = utils::aesEncrypt(key, iv, reinterpret_cast(plain.data()), plain.length(), buffer); 24 | uint8_t result[512]; 25 | len = utils::aesDecrypt(key, iv, buffer, len, result); 26 | string_view res(reinterpret_cast(result), len); 27 | REQUIRE(!plain.compare(res)); 28 | } 29 | //--------------------------------------------------------------------------- 30 | } // namespace test 31 | } // namespace utils 32 | } // namespace anyblob 33 | --------------------------------------------------------------------------------