├── .gitignore ├── libs3 ├── .clang-format ├── README.md ├── LICENSE ├── CMakeLists.txt ├── include │ └── libs3 │ │ ├── request_context.h │ │ ├── response_headers_handler.h │ │ ├── simplexml.h │ │ ├── error_parser.h │ │ ├── util.h │ │ ├── string_buffer.h │ │ └── request.h ├── TODO ├── src │ ├── util.c │ ├── service.c │ ├── simplexml.c │ ├── response_headers_handler.c │ ├── request_context.c │ └── error_parser.c └── COPYING-LGPLv3 ├── unit_tests ├── unit_tests_list.json ├── cmake │ ├── utils.cmake │ └── test_config │ │ └── irods_s3_transport.cmake ├── src │ ├── catch2_compat_include │ │ └── catch2 │ │ │ └── catch_all.hpp │ └── main.cpp ├── create_unit_test_files.sh └── CMakeLists.txt ├── .github └── workflows │ ├── linter-irods-clang-format.yml │ └── linter-irods-clang-tidy.yml ├── packaging ├── postinst ├── s3_prc_upload_issues_2260_2261.py ├── s3plugin_lib.py ├── cleanup_buckets.py ├── test_irods_resource_plugin_s3_fujifilm.py ├── test_irods_resource_plugin_s3_ceph.py ├── test_irods_resource_plugin_s3_gcs.py ├── test_irods_resource_plugin_s3_minio.py └── test_irods_resource_plugin_s3.py ├── LICENSE ├── s3_transport ├── include │ └── irods │ │ └── private │ │ └── s3_transport │ │ ├── types.hpp │ │ ├── lock_and_wait_strategy.hpp │ │ ├── logging_category.hpp │ │ ├── circular_buffer.hpp │ │ ├── multipart_shared_data.hpp │ │ ├── managed_shared_memory_object.hpp │ │ └── util.hpp └── CMakeLists.txt ├── s3_resource ├── CMakeLists.txt └── include │ └── irods │ └── private │ └── s3_resource │ ├── multipart_shared_data.hpp │ ├── s3_plugin_logging_category.hpp │ ├── s3_operations.hpp │ └── s3_resource.hpp ├── irods_consortium_continuous_integration_build_hook.py ├── .clang-format ├── CMakeLists.txt ├── irods_consortium_continuous_integration_test_hook.py └── COPYING-LGPLv3 /.gitignore: -------------------------------------------------------------------------------- 1 | .dir-locals.el 2 | tags 3 | -------------------------------------------------------------------------------- /libs3/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | UseTab: AlignWithSpaces 3 | -------------------------------------------------------------------------------- /unit_tests/unit_tests_list.json: -------------------------------------------------------------------------------- 1 | [ 2 | "irods_s3_transport" 3 | ] 4 | -------------------------------------------------------------------------------- /.github/workflows/linter-irods-clang-format.yml: -------------------------------------------------------------------------------- 1 | name: Check Formatting 2 | 3 | on: pull_request 4 | 5 | defaults: 6 | run: 7 | shell: bash 8 | 9 | jobs: 10 | clang-format: 11 | uses: irods/irods_reusable_github_workflows/.github/workflows/linter-irods-clang-format.yml@main 12 | -------------------------------------------------------------------------------- /.github/workflows/linter-irods-clang-tidy.yml: -------------------------------------------------------------------------------- 1 | name: Run Static Analysis Checks 2 | 3 | on: pull_request 4 | 5 | defaults: 6 | run: 7 | shell: bash 8 | 9 | jobs: 10 | clang-tidy: 11 | uses: irods/irods_reusable_github_workflows/.github/workflows/linter-irods-clang-tidy.yml@main 12 | with: 13 | install_irods_development_package: true 14 | -------------------------------------------------------------------------------- /unit_tests/cmake/utils.cmake: -------------------------------------------------------------------------------- 1 | # utils.cmake 2 | # ~~~~~~~~~~~ 3 | # Defines helper functions and other utilities for testing. 4 | 5 | function(unset_irods_test_variables) 6 | unset(IRODS_TEST_TARGET) 7 | unset(IRODS_TEST_SOURCE_FILES) 8 | unset(IRODS_TEST_PROVIDES_MAIN) 9 | unset(IRODS_TEST_INCLUDE_PATH) 10 | unset(IRODS_TEST_LINK_LIBRARIES) 11 | unset(IRODS_TEST_LINK_OBJLIBRARIES) 12 | endfunction() 13 | -------------------------------------------------------------------------------- /packaging/postinst: -------------------------------------------------------------------------------- 1 | if [ -f /etc/irods/service_account.config ] ; then 2 | . /etc/irods/service_account.config 3 | else 4 | IRODS_SERVICE_ACCOUNT_NAME=`stat --format "%U" /var/lib/irods` 5 | IRODS_SERVICE_GROUP_NAME=`stat --format "%G" /var/lib/irods` 6 | fi 7 | 8 | 9 | chown $IRODS_SERVICE_ACCOUNT_NAME:$IRODS_SERVICE_GROUP_NAME /var/lib/irods 10 | chown $IRODS_SERVICE_ACCOUNT_NAME:$IRODS_SERVICE_GROUP_NAME /var/lib/irods/scripts 11 | chown $IRODS_SERVICE_ACCOUNT_NAME:$IRODS_SERVICE_GROUP_NAME /var/lib/irods/scripts/irods 12 | chown $IRODS_SERVICE_ACCOUNT_NAME:$IRODS_SERVICE_GROUP_NAME /var/lib/irods/scripts/irods/test 13 | chown $IRODS_SERVICE_ACCOUNT_NAME:$IRODS_SERVICE_GROUP_NAME /var/lib/irods/scripts/irods/test/test_irods_resource_plugin_s3.py 14 | -------------------------------------------------------------------------------- /unit_tests/cmake/test_config/irods_s3_transport.cmake: -------------------------------------------------------------------------------- 1 | set(IRODS_TEST_TARGET irods_s3_transport) 2 | 3 | set(IRODS_TEST_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp" 4 | "${CMAKE_CURRENT_SOURCE_DIR}/src/test_s3_transport.cpp") 5 | 6 | set(IRODS_TEST_PROVIDES_MAIN TRUE) 7 | 8 | set(IRODS_TEST_LINK_OBJLIBRARIES s3_transport_obj) 9 | 10 | set(IRODS_TEST_INCLUDE_PATH ${IRODS_EXTERNALS_FULLPATH_BOOST}/include) 11 | 12 | set(IRODS_TEST_LINK_LIBRARIES fmt::fmt 13 | ${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_system.so 14 | ${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_filesystem.so 15 | ${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_thread.so) 16 | -------------------------------------------------------------------------------- /unit_tests/src/catch2_compat_include/catch2/catch_all.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_CATCH2_V2_SHIM_HPP 2 | #define IRODS_CATCH2_V2_SHIM_HPP 3 | 4 | #include 5 | 6 | // NOLINTBEGIN(misc-unused-alias-decls) 7 | 8 | namespace Catch 9 | { 10 | namespace Matchers 11 | { 12 | template 13 | constexpr auto ContainsSubstring(Args&&... args) -> decltype(Contains(std::forward(args)...)) 14 | { 15 | return Contains(std::forward(args)...); 16 | } 17 | 18 | using Impl::MatcherBase; 19 | } //namespace Matchers 20 | 21 | namespace clara { } 22 | namespace Clara = clara; 23 | } //namespace Catch 24 | 25 | // NOLINTEND(misc-unused-alias-decls) 26 | 27 | #endif // IRODS_CATCH2_V2_SHIM_HPP 28 | -------------------------------------------------------------------------------- /packaging/s3_prc_upload_issues_2260_2261.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Upload a file via PRC 4 | 5 | import sys 6 | 7 | # To remove the potential name conflict with the irods package in scripts.A 8 | directory_to_remove = '/var/lib/irods/scripts/irods/test' 9 | if directory_to_remove in sys.path: 10 | sys.path.remove(directory_to_remove) 11 | 12 | from irods.session import iRODSSession 13 | 14 | if __name__ == "__main__": 15 | 16 | # just allow this to throw an exception if the arguments are not provided 17 | user = sys.argv[1] 18 | password = sys.argv[2] 19 | zone = sys.argv[3] 20 | file = sys.argv[4] 21 | file_logical_path = sys.argv[5] 22 | 23 | # put the file via PRC 24 | with iRODSSession(host='localhost', port=1247, user=user, password=password, zone=zone) as session: 25 | session.data_objects.put(file, file_logical_path) 26 | -------------------------------------------------------------------------------- /libs3/README.md: -------------------------------------------------------------------------------- 1 | This directory contains code from the libs3 library originally written by [Bryan Ischo](https://github.com/bji). 2 | 3 | The [original libs3 project](https://github.com/bji/libs3) has been [more-or-less abandoned](https://github.com/bji/libs3/pull/70#issuecomment-414755533) and as of November 2023, the [last upstream commit](https://github.com/bji/libs3/commit/287e4bee6fd430ffb52604049de80a27a77ff6b4) was in April 2019. For a time, iRODS maintained [its own fork](https://github.com/irods/libs3) for use with the S3 resource plugin. However, given that it seems unlikely that Ischo will resume development or maintenance of the library, the decision was made to absorb the libs3 code into the S3 resource plugin repository to ease maintenance. At time of absorption, the [last commit in the iRODS-maintained fork](https://github.com/irods/libs3/commit/e8457e0905dd9e502416b9c6400762ab440bb29e) is dated August 2023. 4 | 5 | The libs3 library is free software. See the file LICENSE for copying permission. 6 | -------------------------------------------------------------------------------- /unit_tests/create_unit_test_files.sh: -------------------------------------------------------------------------------- 1 | usage() { 2 | echo "./create_unit_test_files.sh " 3 | } 4 | 5 | if [[ $# != 1 ]]; then 6 | usage 7 | exit 1 8 | fi 9 | 10 | build_directory=$1 11 | 12 | if [[ ! -f $build_directory ]]; then 13 | mkdir $build_directory 14 | fi 15 | 16 | wget -O tmp http://www.gutenberg.org/files/2600/2600-0.txt 17 | 18 | # create zero file 19 | touch $build_directory/zero_file 20 | 21 | # create small file 22 | cp tmp ${build_directory}/small_file 23 | truncate --size 148 ${build_directory}/small_file 24 | 25 | # create medium file 26 | for i in {1..4}; do 27 | cat tmp >> ${build_directory}/medium_file 28 | done 29 | 30 | # create large file 31 | cp tmp $build_directory 32 | for i in {1..7}; do 33 | cat ${build_directory}/tmp >> ${build_directory}/tmp2 34 | cat ${build_directory}/tmp >> ${build_directory}/tmp2 35 | mv ${build_directory}/tmp2 ${build_directory}/tmp 36 | done 37 | mv ${build_directory}/tmp ${build_directory}/large_file 38 | 39 | rm tmp 40 | -------------------------------------------------------------------------------- /packaging/s3plugin_lib.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | 4 | def remove_if_exists(localfile): 5 | if os.path.exists(localfile): 6 | os.unlink(localfile) 7 | 8 | def make_arbitrary_file(f_name, f_size, buffer_size=32*1024*1024): 9 | # do not care about true randomness 10 | random.seed(5) 11 | bytes_written = 0 12 | buffer = buffer_size * [0x78] # 'x' - bytearray() below appears to require int instead 13 | # of char which was valid in python2 14 | with open(f_name, "wb") as out: 15 | 16 | while bytes_written < f_size: 17 | 18 | if f_size - bytes_written < buffer_size: 19 | to_write = f_size - bytes_written 20 | buffer = to_write * [0x78] # 'x' 21 | else: 22 | to_write = buffer_size 23 | 24 | current_char = random.randrange(256) 25 | 26 | # just write some random byte each 1024 chars 27 | for i in range(0, to_write, 1024): 28 | buffer[i] = current_char 29 | current_char = random.randrange(256) 30 | buffer[len(buffer)-1] = random.randrange(256) 31 | 32 | out.write(bytearray(buffer)) 33 | 34 | bytes_written += to_write 35 | -------------------------------------------------------------------------------- /libs3/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2008 Bryan Ischo 2 | 3 | libs3 is free software: you can redistribute it and/or modify it under the 4 | terms of the GNU Lesser General Public License as published by the Free 5 | Software Foundation, version 3 or above of the License. 6 | 7 | You can also redistribute it and/or modify it under the terms of the 8 | GNU General Public License as published by the Free Software Foundation, 9 | version 2 or above of the License. 10 | 11 | In addition, as a special exception, the copyright holders give 12 | permission to link the code of this library and its programs with the 13 | OpenSSL library, and distribute linked combinations including the two. 14 | 15 | libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 16 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 | details. 19 | 20 | You should have received a copy of the GNU Lesser General Public License 21 | version 3 along with libs3, in a file named COPYING-LGPLv3. If not, see 22 | . 23 | 24 | You should also have received a copy of the GNU General Public License 25 | version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 26 | . 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023, The University of North Carolina at Chapel Hill 2 | 3 | The iRODS S3 Resource Plugin is free software: you can redistribute it and/or 4 | modify it under the terms of the GNU Lesser General Public License as 5 | published by the Free Software Foundation, version 3 or above of the License. 6 | 7 | You can also redistribute it and/or modify it under the terms of the GNU 8 | General Public License as published by the Free Software Foundation, version 2 9 | or above of the License. 10 | 11 | In addition, as a special exception, the copyright holders give permission to 12 | link the code of this library and its programs with the OpenSSL library, and 13 | distribute linked combinations including the two. 14 | 15 | The iRODS S3 Resource Plugin is distributed in the hope that it will be 16 | useful, but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 18 | General Public License for more details. 19 | 20 | You should have received a copy of the GNU Lesser General Public License 21 | version 3 along with the iRODS S3 Resource Plugin, in a file named 22 | COPYING-LGPLv3. If not, see . 23 | 24 | You should also have received a copy of the GNU General Public License version 25 | 2 along with the iRODS S3 Resource Plugin, in a file named COPYING-GPLv2. If 26 | not, see . 27 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/types.hpp: -------------------------------------------------------------------------------- 1 | #ifndef S3_TRANSPORT_TYPES_HPP 2 | #define S3_TRANSPORT_TYPES_HPP 3 | 4 | #include "irods/private/s3_transport/circular_buffer.hpp" 5 | #include "libs3/libs3.h" 6 | 7 | namespace irods::experimental::io::s3_transport 8 | { 9 | 10 | struct libs3_types 11 | { 12 | using status = S3Status; 13 | const static status status_ok = status::S3StatusOK; 14 | const static status status_request_timeout = status::S3StatusErrorRequestTimeout; 15 | const static status status_error_unknown = status::S3StatusErrorUnknown; 16 | using bucket_context = S3BucketContext; 17 | using char_type = char; 18 | using buffer_type = char_type*; 19 | using error_details = S3ErrorDetails; 20 | using response_properties = S3ResponseProperties; 21 | }; 22 | 23 | enum class error_codes 24 | { 25 | SUCCESS, 26 | OUT_OF_DISK_SPACE, 27 | BAD_ALLOC, 28 | BYTES_TRANSFERRED_MISMATCH, 29 | INITIATE_MULTIPART_UPLOAD_ERROR, 30 | COMPLETE_MULTIPART_UPLOAD_ERROR, 31 | UPLOAD_FILE_ERROR, 32 | DOWNLOAD_FILE_ERROR, 33 | UPLOAD_PART_TIMEOUT 34 | }; 35 | 36 | enum class cache_file_download_status 37 | { 38 | NOT_STARTED, 39 | STARTED, 40 | SUCCESS, 41 | FAILED 42 | }; 43 | 44 | } // irods::experimental::io::s3_transport 45 | 46 | #endif // S3_TRANSPORT_TYPES_HPP 47 | 48 | -------------------------------------------------------------------------------- /unit_tests/src/main.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include 3 | 4 | extern std::string keyfile; 5 | extern std::string hostname; 6 | extern std::string bucket_name; 7 | 8 | int main( int argc, char* argv[] ) 9 | { 10 | Catch::Session session; // There must be exactly one instance 11 | 12 | // writing to session.configData() here sets defaults 13 | // this is the preferred way to set them 14 | 15 | // Build a new parser on top of Catch's 16 | using namespace Catch::Clara; 17 | auto cli 18 | = session.cli() // Get Catch's composite command line parser 19 | | Opt( hostname, "hostname" ) 20 | ["--hostname"] 21 | ("the S3 host (default: s3.amazonaws.com)") 22 | | Opt( keyfile, "keyfile" ) 23 | ["--keyfile"] 24 | ("the file holding the access key and secret access key"); 25 | 26 | // Now pass the new composite back to Catch so it uses that 27 | session.cli( cli ); 28 | 29 | int returnCode = session.applyCommandLine( argc, argv ); 30 | if( returnCode != 0 ) // Indicates a command line error 31 | return returnCode; 32 | 33 | // writing to session.configData() or session.Config() here 34 | // overrides command line args 35 | // only do this if you know you need to 36 | 37 | int numFailed = session.run(); 38 | 39 | // numFailed is clamped to 255 as some unices only use the lower 8 bits. 40 | // This clamping has already been applied, so just return it here 41 | // You can also do any post run clean-up here 42 | return numFailed; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /libs3/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library( 2 | libs3_obj 3 | OBJECT 4 | "${CMAKE_CURRENT_SOURCE_DIR}/src/bucket.c" 5 | "${CMAKE_CURRENT_SOURCE_DIR}/src/bucket_metadata.c" 6 | "${CMAKE_CURRENT_SOURCE_DIR}/src/error_parser.c" 7 | "${CMAKE_CURRENT_SOURCE_DIR}/src/general.c" 8 | "${CMAKE_CURRENT_SOURCE_DIR}/src/multipart.c" 9 | "${CMAKE_CURRENT_SOURCE_DIR}/src/object.c" 10 | "${CMAKE_CURRENT_SOURCE_DIR}/src/request.c" 11 | "${CMAKE_CURRENT_SOURCE_DIR}/src/request_context.c" 12 | "${CMAKE_CURRENT_SOURCE_DIR}/src/response_headers_handler.c" 13 | "${CMAKE_CURRENT_SOURCE_DIR}/src/service.c" 14 | "${CMAKE_CURRENT_SOURCE_DIR}/src/service_access_logging.c" 15 | "${CMAKE_CURRENT_SOURCE_DIR}/src/simplexml.c" 16 | "${CMAKE_CURRENT_SOURCE_DIR}/src/util.c" 17 | ) 18 | target_link_libraries( 19 | libs3_obj 20 | PUBLIC 21 | CURL::libcurl 22 | LibXml2::LibXml2 23 | OpenSSL::Crypto 24 | ) 25 | target_include_directories( 26 | libs3_obj 27 | PUBLIC 28 | "$" 29 | ) 30 | target_compile_definitions( 31 | libs3_obj 32 | PRIVATE 33 | LIBS3_VER_MAJOR="4" 34 | LIBS3_VER_MINOR="1" 35 | __STRICT_ANSI__ 36 | _ISOC99_SOURCE 37 | _POSIX_C_SOURCE=200112L 38 | OPENSSL_API_COMPAT=10100 39 | ) 40 | target_compile_options( 41 | libs3_obj 42 | PRIVATE 43 | -Wshadow 44 | -Wno-unused-function # due to OPENSSL_NO_DEPRECATED_1_1_0 45 | ) 46 | set_target_properties(libs3_obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE) 47 | set_target_properties(libs3_obj PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE TRUE) 48 | 49 | set_target_properties(libs3_obj PROPERTIES EXCLUDE_FROM_ALL TRUE) 50 | -------------------------------------------------------------------------------- /s3_resource/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(ObjectTargetHelpers) 2 | 3 | add_library( 4 | s3_resource_obj 5 | OBJECT 6 | "${CMAKE_CURRENT_SOURCE_DIR}/src/s3_resource.cpp" 7 | "${CMAKE_CURRENT_SOURCE_DIR}/src/s3_operations.cpp" 8 | ) 9 | target_link_objects( 10 | s3_resource_obj 11 | PUBLIC 12 | libs3_obj 13 | s3_transport_obj 14 | ) 15 | target_link_libraries( 16 | s3_resource_obj 17 | PUBLIC 18 | irods_common 19 | irods_server 20 | fmt::fmt 21 | Threads::Threads 22 | PRIVATE 23 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_filesystem.so" 24 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_thread.so" 25 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_system.so" 26 | ) 27 | target_include_directories( 28 | s3_resource_obj 29 | PUBLIC 30 | "$" 31 | PRIVATE 32 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/include" 33 | ) 34 | target_compile_definitions( 35 | s3_resource_obj 36 | PRIVATE 37 | IRODS_QUERY_ENABLE_SERVER_SIDE_API 38 | IRODS_ENABLE_SYSLOG 39 | ${IRODS_COMPILE_DEFINITIONS} 40 | ${IRODS_COMPILE_DEFINITIONS_PRIVATE} 41 | BOOST_SYSTEM_NO_DEPRECATED 42 | ) 43 | 44 | include(CheckCXXCompilerFlag) 45 | 46 | # due to boost headers 47 | check_cxx_compiler_flag(-Wno-error=deprecated-copy CMAKE_CXX_COMPILER_W_DEPRECATED_COPY) 48 | if (CMAKE_CXX_COMPILER_W_DEPRECATED_COPY) 49 | target_compile_options(s3_resource_obj PRIVATE $<$:-Wno-error=deprecated-copy>) 50 | endif() 51 | 52 | set_target_properties(s3_resource_obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE) 53 | set_target_properties(s3_resource_obj PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE TRUE) 54 | 55 | set_target_properties(s3_resource_obj PROPERTIES EXCLUDE_FROM_ALL TRUE) 56 | -------------------------------------------------------------------------------- /s3_resource/include/irods/private/s3_resource/multipart_shared_data.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP 2 | #define IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | namespace irods_s3 18 | { 19 | namespace interprocess_types 20 | { 21 | namespace bi = boost::interprocess; 22 | using segment_manager = bi::managed_shared_memory::segment_manager; 23 | using void_allocator = boost::container::scoped_allocator_adaptor >; 24 | } 25 | 26 | // data that needs to be shared among different processes 27 | struct multipart_shared_data 28 | { 29 | explicit multipart_shared_data(const interprocess_types::void_allocator &allocator) 30 | : threads_remaining_to_close{0} 31 | , number_of_threads{0} 32 | , ref_count{0} 33 | {} 34 | 35 | bool can_delete() { 36 | return threads_remaining_to_close == 0; 37 | } 38 | 39 | int threads_remaining_to_close; 40 | int number_of_threads; 41 | int ref_count; 42 | }; // struct multipart_shared_data 43 | } // namespace irods_s3 44 | 45 | #endif // IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP 46 | -------------------------------------------------------------------------------- /s3_transport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(CheckLibraryExists) 2 | include(ObjectTargetHelpers) 3 | 4 | check_library_exists("rt" "shm_open" "" LIBRT_HAS_SHM_OPEN) 5 | 6 | add_library( 7 | s3_transport_obj 8 | OBJECT 9 | "${CMAKE_CURRENT_SOURCE_DIR}/src/s3_transport.cpp" 10 | ) 11 | target_link_objects( 12 | s3_transport_obj 13 | PUBLIC 14 | libs3_obj 15 | ) 16 | target_link_libraries( 17 | s3_transport_obj 18 | PUBLIC 19 | irods_common 20 | irods_server 21 | nlohmann_json::nlohmann_json 22 | fmt::fmt 23 | PRIVATE 24 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_filesystem.so" 25 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/lib/libboost_system.so" 26 | ) 27 | if (LIBRT_HAS_SHM_OPEN) 28 | target_link_libraries( 29 | s3_transport_obj 30 | PUBLIC 31 | Threads::Threads 32 | rt 33 | ) 34 | endif() 35 | target_include_directories( 36 | s3_transport_obj 37 | PUBLIC 38 | "$" 39 | PRIVATE 40 | "${IRODS_EXTERNALS_FULLPATH_BOOST}/include" 41 | ) 42 | target_compile_definitions( 43 | s3_transport_obj 44 | PRIVATE 45 | IRODS_ENABLE_SYSLOG 46 | ${IRODS_COMPILE_DEFINITIONS} 47 | ${IRODS_COMPILE_DEFINITIONS_PRIVATE} 48 | BOOST_SYSTEM_NO_DEPRECATED 49 | ) 50 | 51 | target_compile_options(s3_transport_obj PUBLIC -Wno-unused-parameter) 52 | 53 | include(CheckCXXCompilerFlag) 54 | 55 | # due to boost headers 56 | check_cxx_compiler_flag(-Wno-error=deprecated-copy CMAKE_CXX_COMPILER_W_DEPRECATED_COPY) 57 | if (CMAKE_CXX_COMPILER_W_DEPRECATED_COPY) 58 | target_compile_options(s3_transport_obj PRIVATE $<$:-Wno-error=deprecated-copy>) 59 | endif() 60 | 61 | set_target_properties(s3_transport_obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE) 62 | set_target_properties(s3_transport_obj PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE TRUE) 63 | 64 | set_target_properties(s3_transport_obj PROPERTIES EXCLUDE_FROM_ALL TRUE) 65 | -------------------------------------------------------------------------------- /libs3/include/libs3/request_context.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * request_context.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef REQUEST_CONTEXT_H 34 | #define REQUEST_CONTEXT_H 35 | 36 | #include 37 | #include "libs3/libs3.h" 38 | 39 | typedef enum 40 | { 41 | S3CurlModeMultiPerform, 42 | S3CurlModeMultiSocket, 43 | } S3CurlMode; 44 | 45 | struct S3RequestContext 46 | { 47 | CURLM* curlm; 48 | S3CurlMode curl_mode; 49 | 50 | int verifyPeerSet; 51 | long verifyPeer; 52 | 53 | struct Request* requests; 54 | 55 | S3SetupCurlCallback setupCurlCallback; 56 | void* setupCurlCallbackData; 57 | }; 58 | 59 | #endif /* REQUEST_CONTEXT_H */ 60 | -------------------------------------------------------------------------------- /packaging/cleanup_buckets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import argparse 4 | import os 5 | import boto3 6 | from botocore.client import Config 7 | 8 | 9 | ci_keys_file='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 10 | s3region = 'us-east-1' 11 | 12 | # arg parsing 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument('-c', '--configure', action="store_true", help='set up aws configuration') 15 | 16 | args = parser.parse_args() 17 | 18 | def set_up_aws_config_dir(keys_file): 19 | # read access keys from keypair file 20 | with open(keys_file) as f: 21 | aws_access_key_id = f.readline().rstrip() 22 | aws_secret_access_key = f.readline().rstrip() 23 | 24 | # make config dir 25 | aws_cfg_dir_path = os.path.join(os.path.expanduser('~'), '.aws') 26 | try: 27 | os.makedirs(aws_cfg_dir_path) 28 | except OSError: 29 | if not os.path.isdir(aws_cfg_dir_path): 30 | raise 31 | 32 | # make config file 33 | with open(os.path.join(aws_cfg_dir_path, 'config'), 'w') as cfg_file: 34 | cfg_file.write('[default]\n') 35 | cfg_file.write('region=' + s3region + '\n') 36 | 37 | # make credentials file 38 | with open(os.path.join(aws_cfg_dir_path, 'credentials'), 'w') as cred_file: 39 | cred_file.write('[default]\n') 40 | cred_file.write('aws_access_key_id = ' + aws_access_key_id + '\n') 41 | cred_file.write('aws_secret_access_key = ' + aws_secret_access_key + '\n') 42 | 43 | 44 | # set up config files if needed 45 | if args.configure: 46 | set_up_aws_config_dir(ci_keys_file) 47 | 48 | s3 = boto3.resource('s3', config=Config(signature_version='s3v4')) 49 | 50 | # cleanup buckets 51 | for bucket in s3.buckets.all(): 52 | if bucket.name.startswith('irods-ci-ubuntu') or bucket.name.startswith('irods-ci-centos') or bucket.name.startswith('irods-ci-opensuse'): 53 | for obj in bucket.objects.all(): 54 | obj.delete() 55 | print('deleting {bucket.name}'.format(**locals())) 56 | bucket.delete() 57 | -------------------------------------------------------------------------------- /s3_resource/include/irods/private/s3_resource/s3_plugin_logging_category.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_S3_RESOURCE_PLUGIN_LOGGING_CATEGORY_HPP 2 | #define IRODS_S3_RESOURCE_PLUGIN_LOGGING_CATEGORY_HPP 3 | 4 | #include "libs3/libs3.h" 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | 12 | // 1. Declare the custom category tag. 13 | // This structure does not need to define a body. 14 | // This tag allows the logger to locate data specific to the new category. 15 | struct s3_plugin_logging_category; 16 | 17 | // 2. Specialize the logger configuration for the new category. 18 | // This also defines the default configuration for the new category. 19 | namespace irods::experimental 20 | { 21 | 22 | template <> 23 | class log::logger_config 24 | { 25 | // This defines the name that will appear in the log under the "log_category" key. 26 | // The "log_category" key defines where the message originated. Try to use a name 27 | // that makes it easy for administrators to determine what produced the message. 28 | static constexpr const char* name = "s3_plugin_logging_category"; 29 | 30 | // This is required since the fields above are private. 31 | // This allows the logger to access and modify the configuration. 32 | friend class logger; 33 | 34 | // This is the current log level for the category. This also represents the initial 35 | // log level. Use the "set_level()" function to adjust the level. 36 | static inline log::level level = log::level::info; 37 | 38 | public: 39 | 40 | static auto get_level() -> log::level 41 | { 42 | return level; 43 | } 44 | 45 | }; 46 | } // namespace irods::experimental 47 | 48 | template <> 49 | struct fmt::formatter : fmt::formatter> 50 | { 51 | constexpr auto format(const S3Status& e, format_context& ctx) const 52 | { 53 | return fmt::formatter>::format( 54 | static_cast>(e), ctx); 55 | } 56 | }; 57 | 58 | #endif //IRODS_S3_RESOURCE_PLUGIN_LOGGING_CATEGORY_HPP 59 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/lock_and_wait_strategy.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_LOCK_AND_WAIT_STRATEGY_HPP 2 | #define IRODS_LOCK_AND_WAIT_STRATEGY_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace irods { 9 | namespace experimental { 10 | 11 | class timeout_exception : public std::exception 12 | { 13 | virtual const char* what() const throw() 14 | { 15 | return "Timeout waiting for lock"; 16 | } 17 | }; 18 | 19 | class lock_and_wait_strategy { 20 | public: 21 | using wait_predicate = std::function; 22 | using the_work = std::function; 23 | virtual void operator()(wait_predicate, the_work) = 0; 24 | virtual ~lock_and_wait_strategy() {}; 25 | }; 26 | 27 | class do_not_lock_and_wait : public lock_and_wait_strategy { 28 | public: 29 | void operator()(wait_predicate, the_work w) { 30 | w(); 31 | } 32 | }; 33 | 34 | class lock_and_wait : public lock_and_wait_strategy { 35 | public: 36 | void operator()(wait_predicate p, the_work w) { 37 | { 38 | std::unique_lock lk(cv_mutex); 39 | cv.wait(lk, p); 40 | w(); 41 | } 42 | cv.notify_all(); 43 | } 44 | 45 | private: 46 | std::condition_variable cv; 47 | std::mutex cv_mutex; 48 | }; 49 | 50 | class lock_and_wait_with_timeout : public lock_and_wait_strategy { 51 | 52 | public: 53 | 54 | explicit lock_and_wait_with_timeout(int timeout_sec) 55 | : timeout_seconds(timeout_sec) 56 | {} 57 | 58 | void operator()(wait_predicate p, the_work w) { 59 | bool wait_until_ret = false; 60 | { 61 | std::unique_lock lk(cv_mutex); 62 | auto now = std::chrono::system_clock::now(); 63 | wait_until_ret = cv.wait_until(lk, now + std::chrono::seconds(timeout_seconds), p); 64 | if (wait_until_ret) { 65 | // predicate met 66 | w(); 67 | } 68 | } 69 | cv.notify_all(); 70 | 71 | if (!wait_until_ret) { 72 | throw timeout_exception(); 73 | } 74 | } 75 | 76 | private: 77 | std::condition_variable cv; 78 | std::mutex cv_mutex; 79 | int timeout_seconds; 80 | }; 81 | 82 | } // namespace experimental 83 | } // namespace irods 84 | 85 | 86 | 87 | #endif // IRODS_LOCK_AND_WAIT_STRATEGY_HPP 88 | -------------------------------------------------------------------------------- /libs3/include/libs3/response_headers_handler.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * response_headers_handler.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef RESPONSE_HEADERS_HANDLER_H 34 | #define RESPONSE_HEADERS_HANDLER_H 35 | 36 | #include "libs3/libs3.h" 37 | #include "libs3/string_buffer.h" 38 | #include "libs3/util.h" 39 | 40 | typedef struct ResponseHeadersHandler 41 | { 42 | // The structure to pass to the headers callback. This is filled in by 43 | // the ResponseHeadersHandler from the headers added to it. 44 | S3ResponseProperties responseProperties; 45 | 46 | // Set to 1 after the done call has been made 47 | int done; 48 | 49 | // copied into here. We allow 128 bytes for each header, plus \0 term. 50 | string_multibuffer(responsePropertyStrings, 5 * 129); 51 | 52 | // responseproperties.metaHeaders strings get copied into here 53 | string_multibuffer(responseMetaDataStrings, COMPACTED_METADATA_BUFFER_SIZE); 54 | 55 | // Response meta data 56 | S3NameValue responseMetaData[S3_MAX_METADATA_COUNT]; 57 | } ResponseHeadersHandler; 58 | 59 | void response_headers_handler_initialize(ResponseHeadersHandler* handler); 60 | 61 | void response_headers_handler_add(ResponseHeadersHandler* handler, char* data, int dataLen); 62 | 63 | void response_headers_handler_done(ResponseHeadersHandler* handler, CURL* curl); 64 | 65 | #endif /* RESPONSE_HEADERS_HANDLER_H */ 66 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/logging_category.hpp: -------------------------------------------------------------------------------- 1 | #ifndef S3_TRANSPORT_LOGGING_CATEGORY_HPP 2 | #define S3_TRANSPORT_LOGGING_CATEGORY_HPP 3 | 4 | #include "libs3/libs3.h" 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | 12 | // 1. Declare the custom category tag. 13 | // This structure does not need to define a body. 14 | // This tag allows the logger to locate data specific to the new category. 15 | struct s3_transport_logging_category; 16 | 17 | // 2. Specialize the logger configuration for the new category. 18 | // This also defines the default configuration for the new category. 19 | namespace irods::experimental 20 | { 21 | template <> 22 | class log::logger_config 23 | { 24 | // This defines the name that will appear in the log under the "log_category" key. 25 | // The "log_category" key defines where the message originated. Try to use a name 26 | // that makes it easy for administrators to determine what produced the message. 27 | static constexpr const char* name = "s3_transport_logging_category"; 28 | 29 | // This is the current log level for the category. This also represents the initial 30 | // log level. Use the "set_level()" function to adjust the level. 31 | static inline log::level level = log::level::info; 32 | 33 | // This is required since the fields above are private. 34 | // This allows the logger to access and modify the configuration. 35 | friend class logger; 36 | }; 37 | } // namespace irods::experimental 38 | 39 | template <> 40 | struct fmt::formatter : fmt::formatter> 41 | { 42 | constexpr auto format(const S3Protocol& e, format_context& ctx) const 43 | { 44 | return fmt::formatter>::format( 45 | static_cast>(e), ctx); 46 | } 47 | }; 48 | 49 | template <> 50 | struct fmt::formatter : fmt::formatter> 51 | { 52 | constexpr auto format(const S3UriStyle& e, format_context& ctx) const 53 | { 54 | return fmt::formatter>::format( 55 | static_cast>(e), ctx); 56 | } 57 | }; 58 | 59 | template <> 60 | struct fmt::formatter : fmt::formatter> 61 | { 62 | constexpr auto format(const S3STSDate& e, format_context& ctx) const 63 | { 64 | return fmt::formatter>::format( 65 | static_cast>(e), ctx); 66 | } 67 | }; 68 | 69 | #endif //S3_TRANSPORT_LOGGING_CATEGORY_HPP 70 | -------------------------------------------------------------------------------- /libs3/include/libs3/simplexml.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * simplexml.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef SIMPLEXML_H 34 | #define SIMPLEXML_H 35 | 36 | #include "libs3/libs3.h" 37 | 38 | // Simple XML callback. 39 | // 40 | // elementPath: is the full "path" of the element; i.e. 41 | // data would have 'data' in the element 42 | // foo/bar/baz. 43 | // 44 | // Return of anything other than S3StatusOK causes the calling 45 | // simplexml_add() function to immediately stop and return the status. 46 | // 47 | // data is passed in as 0 on end of element 48 | typedef S3Status(SimpleXmlCallback)(const char* elementPath, const char* data, int dataLen, void* callbackData); 49 | 50 | typedef struct SimpleXml 51 | { 52 | void* xmlParser; 53 | 54 | SimpleXmlCallback* callback; 55 | 56 | void* callbackData; 57 | 58 | char elementPath[512]; 59 | 60 | int elementPathLen; 61 | 62 | S3Status status; 63 | } SimpleXml; 64 | 65 | // Simple XML parsing 66 | // ---------------------------------------------------------------------------- 67 | 68 | // Always call this, even if the simplexml doesn't end up being used 69 | void simplexml_initialize(SimpleXml* simpleXml, SimpleXmlCallback* callback, void* callbackData); 70 | 71 | S3Status simplexml_add(SimpleXml* simpleXml, const char* data, int dataLen); 72 | 73 | // Always call this 74 | void simplexml_deinitialize(SimpleXml* simpleXml); 75 | 76 | #endif /* SIMPLEXML_H */ 77 | -------------------------------------------------------------------------------- /packaging/test_irods_resource_plugin_s3_fujifilm.py: -------------------------------------------------------------------------------- 1 | from .resource_suite_s3_nocache import Test_S3_NoCache_Base 2 | from .resource_suite_s3_cache import Test_S3_Cache_Base 3 | 4 | import sys 5 | import unittest 6 | 7 | class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): 8 | def __init__(self, *args, **kwargs): 9 | """Set up the test.""" 10 | self.keypairfile='/var/lib/irods/fujifilm.keypair' 11 | self.archive_naming_policy='decoupled' 12 | self.s3stsdate='' 13 | self.s3region='us-east4' 14 | self.s3endPoint = self.read_endpoint('/var/lib/irods/fujifilm_endpoint.txt') 15 | self.s3sse = 0 # server side encryption 16 | self.static_bucket_name = 'irods-testing' # fuji does not allow deletion of buckets so force static bucket use 17 | super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) 18 | 19 | class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): 20 | 21 | def __init__(self, *args, **kwargs): 22 | """Set up the test.""" 23 | self.keypairfile='/var/lib/irods/fujifilm.keypair' 24 | self.s3region='us-east4' 25 | self.s3endPoint = self.read_endpoint('/var/lib/irods/fujifilm_endpoint.txt') 26 | self.s3EnableMPU=1 27 | self.static_bucket_name = 'irods-testing' # fuji does not allow deletion of buckets so force static bucket use 28 | self.s3DisableCopyObject=1 29 | super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) 30 | 31 | class Test_S3_NoCache_Decoupled(Test_S3_NoCache_Base, unittest.TestCase): 32 | def __init__(self, *args, **kwargs): 33 | """Set up the test.""" 34 | self.keypairfile='/var/lib/irods/fujifilm.keypair' 35 | self.s3region='us-east4' 36 | self.s3endPoint = self.read_endpoint('/var/lib/irods/fujifilm_endpoint.txt') 37 | self.s3EnableMPU=0 38 | self.archive_naming_policy = 'decoupled' 39 | self.static_bucket_name = 'irods-testing' # fuji does not allow deletion of buckets so force static bucket use 40 | self.s3DisableCopyObject=1 41 | super(Test_S3_NoCache_Decoupled, self).__init__(*args, **kwargs) 42 | 43 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 44 | def test_iget_with_stale_replica(self): 45 | pass 46 | 47 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 48 | def test_irepl_with_purgec(self): 49 | pass 50 | 51 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 52 | def test_put_get_small_file_in_repl_node(self): 53 | pass 54 | 55 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 56 | def test_put_get_large_file_in_repl_node(self): 57 | pass 58 | 59 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 60 | def test_s3_in_replication_node__issues_2102_2122(self): 61 | pass 62 | -------------------------------------------------------------------------------- /packaging/test_irods_resource_plugin_s3_ceph.py: -------------------------------------------------------------------------------- 1 | from .resource_suite_s3_nocache import Test_S3_NoCache_Base 2 | from .resource_suite_s3_cache import Test_S3_Cache_Base 3 | 4 | import sys 5 | import unittest 6 | 7 | class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): 8 | def __init__(self, *args, **kwargs): 9 | """Set up the test.""" 10 | self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' 11 | self.archive_naming_policy='decoupled' 12 | self.s3stsdate='' 13 | self.s3region='us-east-1' 14 | self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') 15 | self.s3sse = 0 # server side encryption 16 | super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) 17 | 18 | class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): 19 | 20 | def __init__(self, *args, **kwargs): 21 | """Set up the test.""" 22 | self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' 23 | self.s3region='us-east-1' 24 | self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') 25 | self.s3EnableMPU=1 26 | super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) 27 | 28 | class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_Base, unittest.TestCase): 29 | def __init__(self, *args, **kwargs): 30 | """Set up the test.""" 31 | self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' 32 | self.s3region='us-east-1' 33 | self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') 34 | self.s3EnableMPU=0 35 | super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) 36 | 37 | 38 | class Test_S3_NoCache_Decoupled(Test_S3_NoCache_Base, unittest.TestCase): 39 | def __init__(self, *args, **kwargs): 40 | """Set up the test.""" 41 | self.keypairfile='/var/lib/irods/ceph_s3_key.keypair' 42 | self.s3region='us-east-1' 43 | self.s3endPoint = self.read_endpoint('/var/lib/irods/ceph_endpoint.txt') 44 | self.s3EnableMPU=0 45 | self.archive_naming_policy = 'decoupled' 46 | super(Test_S3_NoCache_Decoupled, self).__init__(*args, **kwargs) 47 | 48 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 49 | def test_iget_with_stale_replica(self): # formerly known as 'dirty' 50 | pass 51 | 52 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 53 | def test_irepl_with_purgec(self): 54 | pass 55 | 56 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 57 | def test_put_get_small_file_in_repl_node(self): 58 | pass 59 | 60 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 61 | def test_put_get_large_file_in_repl_node(self): 62 | pass 63 | 64 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 65 | def test_s3_in_replication_node__issues_2102_2122(self): 66 | pass 67 | -------------------------------------------------------------------------------- /packaging/test_irods_resource_plugin_s3_gcs.py: -------------------------------------------------------------------------------- 1 | from .resource_suite_s3_nocache import Test_S3_NoCache_Base 2 | from .resource_suite_s3_cache import Test_S3_Cache_Base 3 | 4 | import sys 5 | import unittest 6 | 7 | class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): 8 | def __init__(self, *args, **kwargs): 9 | """Set up the test.""" 10 | self.keypairfile='/var/lib/irods/google-gcs.keypair' 11 | self.archive_naming_policy='decoupled' 12 | self.s3stsdate='' 13 | self.s3region='us-east4' 14 | self.s3endPoint = self.read_endpoint('/var/lib/irods/google_endpoint.txt') 15 | self.s3sse = 0 # server side encryption 16 | super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) 17 | 18 | class Test_S3_NoCache_V4(Test_S3_NoCache_Base, unittest.TestCase): 19 | 20 | def __init__(self, *args, **kwargs): 21 | """Set up the test.""" 22 | self.keypairfile='/var/lib/irods/google-gcs.keypair' 23 | self.s3region='us-east4' 24 | self.s3endPoint = self.read_endpoint('/var/lib/irods/google_endpoint.txt') 25 | self.s3EnableMPU=1 26 | self.s3DisableCopyObject=1 27 | super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) 28 | 29 | class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_Base, unittest.TestCase): 30 | def __init__(self, *args, **kwargs): 31 | """Set up the test.""" 32 | self.keypairfile='/var/lib/irods/google-gcs.keypair' 33 | self.s3region='us-east4' 34 | self.s3endPoint = self.read_endpoint('/var/lib/irods/google_endpoint.txt') 35 | self.s3EnableMPU=0 36 | super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) 37 | 38 | 39 | class Test_S3_NoCache_Decoupled(Test_S3_NoCache_Base, unittest.TestCase): 40 | def __init__(self, *args, **kwargs): 41 | """Set up the test.""" 42 | self.keypairfile='/var/lib/irods/google-gcs.keypair' 43 | self.s3region='us-east4' 44 | self.s3endPoint = self.read_endpoint('/var/lib/irods/google_endpoint.txt') 45 | self.s3EnableMPU=0 46 | self.archive_naming_policy = 'decoupled' 47 | super(Test_S3_NoCache_Decoupled, self).__init__(*args, **kwargs) 48 | 49 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 50 | def test_iget_with_stale_replica(self): # formerly known as 'dirty' 51 | pass 52 | 53 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 54 | def test_irepl_with_purgec(self): 55 | pass 56 | 57 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 58 | def test_put_get_small_file_in_repl_node(self): 59 | pass 60 | 61 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 62 | def test_put_get_large_file_in_repl_node(self): 63 | pass 64 | 65 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 66 | def test_s3_in_replication_node__issues_2102_2122(self): 67 | pass 68 | -------------------------------------------------------------------------------- /libs3/include/libs3/error_parser.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * error_parser.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef ERROR_PARSER_H 34 | #define ERROR_PARSER_H 35 | 36 | #include "libs3/libs3.h" 37 | #include "libs3/simplexml.h" 38 | #include "libs3/string_buffer.h" 39 | 40 | #define EXTRA_DETAILS_SIZE 8 41 | 42 | typedef struct ErrorParser 43 | { 44 | // This is the S3ErrorDetails that this ErrorParser fills in from the 45 | // data that it parses 46 | S3ErrorDetails s3ErrorDetails; 47 | 48 | // This is the error XML parser 49 | SimpleXml errorXmlParser; 50 | 51 | // Set to 1 after the first call to add 52 | int errorXmlParserInitialized; 53 | 54 | // Used to buffer the S3 Error Code as it is read in 55 | string_buffer(code, 1024); 56 | 57 | // Used to buffer the S3 Error Message as it is read in 58 | string_buffer(message, 1024); 59 | 60 | // Used to buffer the S3 Error Resource as it is read in 61 | string_buffer(resource, 1024); 62 | 63 | // Used to buffer the S3 Error Further Details as it is read in 64 | string_buffer(furtherDetails, 1024); 65 | 66 | // The extra details; we support up to EXTRA_DETAILS_SIZE of them 67 | S3NameValue extraDetails[EXTRA_DETAILS_SIZE]; 68 | 69 | // This is the buffer from which the names and values used in extraDetails 70 | // are allocated 71 | string_multibuffer(extraDetailsNamesValues, EXTRA_DETAILS_SIZE * 1024); 72 | } ErrorParser; 73 | 74 | // Always call this 75 | void error_parser_initialize(ErrorParser* errorParser); 76 | 77 | S3Status error_parser_add(ErrorParser* errorParser, char* buffer, int bufferSize); 78 | 79 | void error_parser_convert_status(ErrorParser* errorParser, S3Status* status); 80 | 81 | // Always call this 82 | void error_parser_deinitialize(ErrorParser* errorParser); 83 | 84 | #endif /* ERROR_PARSER_H */ 85 | -------------------------------------------------------------------------------- /libs3/TODO: -------------------------------------------------------------------------------- 1 | * Implement functions for generating form stuff for posting to s3 2 | 3 | * Write s3 man page 4 | 5 | === Object Versioning === 6 | 7 | - Enabling versioning is just a PUT request to a specific object name with a canned XML content (3 hours) 8 | 9 | - Suspending versioning is another similar PUT request (1 hour) 10 | 11 | - Get versioning state is a simple request, requires some simple XML parsing to represent the result (2 hours) 12 | 13 | - Support reporting the version ID in PUT, POST, or COPY request results (1 hour) 14 | 15 | - List all object versions uses a GET request for a specific sub-resource (2 hours) 16 | 17 | - Include support for "truncated" object versions list (1 hour) 18 | 19 | - Support versionId in GET and HEAD requests (2 hours) 20 | 21 | 22 | === Reduced Redundancy Storage === 23 | 24 | - Support adding a "storage class" header to define storage class in PUT request (2 hours) 25 | 26 | - Documentation note that S3StatusErrorMethodNotAllowed is returned when there has been a loss of a reduced redundancy object (1 hour) 27 | 28 | - Consider adding an API for changing the storage class of an object (even though this is already covered by doing a PUT with the appropriate user-controlled parameters) (1 hour) 29 | 30 | 31 | === Bucket Policies === 32 | 33 | - Support parsing/deparsing of bucket policies (JSON parsing!) into structured representation - 26 hours 34 | 35 | - Set/delete/query bucket policies - 12 hours 36 | 37 | 38 | === Notifications === 39 | 40 | - Structurally represent and provide an API for setting/deleting/querying: 16 hours 41 | 42 | 43 | === IAM === 44 | 45 | This is Amazon's "user" management API; not part of S3 although IAM users can be referenced in S3 Bucket Policies. Not intending to support IAM account management since the creation and management of users is not S3 functionality. 46 | 47 | 48 | === Response Header API Support === 49 | 50 | - Allows HTTP response headers to be specified on a per-object basis 51 | 52 | - Full support: 5 hours 53 | 54 | 55 | === Support for Hosting static websites in Amazon S3 === 56 | 57 | - Structured representation of website routing rules: 8 hours 58 | 59 | - Support the "website" bucket resource and structured interactions with it: 16 hours 60 | 61 | 62 | === Multipart Upload Copy === 63 | 64 | - 10 hours 65 | 66 | 67 | === Temporary Security Credentials === 68 | 69 | - ??? 70 | 71 | 72 | === Server-Side Encryption === 73 | 74 | - Just have to set an additional header in some of the requests 75 | 76 | - 4 hours 77 | 78 | 79 | === Multi-Object Delete === 80 | 81 | - POST /?delete -- 4 hours 82 | 83 | 84 | === MFA Authentication === 85 | 86 | (part of Bucket Policy) 87 | 88 | 89 | === Cost Allocation Tagging === 90 | 91 | - Looks like just a simple free-form string value that can be associated with buckets 92 | 93 | - May need to implement special logging for querying by cost allocation tag 94 | 95 | - Need to find the REST API documentation for this 96 | 97 | - Likely to take ~8 hours to support 98 | 99 | 100 | === Cross-Origin Resource Sharing === 101 | 102 | - Support /?cors resource: 16 hours 103 | 104 | 105 | === Support for Archiving Data to Amazon Glacier === 106 | 107 | - Already included in Object Lifecycle Management 108 | 109 | 110 | === Root domain support for website hosting === 111 | 112 | - Already included in static website support stuff 113 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/circular_buffer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_RING_BUFFER_HPP 2 | #define IRODS_RING_BUFFER_HPP 3 | 4 | #include 5 | #include "irods/private/s3_transport/lock_and_wait_strategy.hpp" 6 | #include 7 | 8 | namespace irods { 9 | namespace experimental { 10 | 11 | // ring buffer with protection for overwrites 12 | template 13 | class circular_buffer { 14 | 15 | public: 16 | 17 | explicit circular_buffer( 18 | const std::size_t capacity, 19 | std::unique_ptr lws = std::make_unique()) 20 | : cb_{capacity} 21 | , lws_{std::move(lws)} 22 | { 23 | } 24 | 25 | explicit circular_buffer( 26 | const std::size_t capacity, 27 | int timeout) 28 | : circular_buffer(capacity, std::make_unique(timeout)) 29 | { 30 | } 31 | 32 | void pop_front(T& entry) 33 | { 34 | (*lws_)([this] { return 0 < cb_.size(); }, 35 | [this, &entry] { 36 | auto iter = cb_.begin(); 37 | entry = *iter; 38 | cb_.pop_front(); 39 | } ); 40 | } 41 | 42 | // erase n items from front of the queue 43 | void pop_front(std::size_t n) 44 | { 45 | (*lws_)([this, n] { return n <= cb_.size(); }, 46 | [this, n] { cb_.erase_begin(n); } ); 47 | } 48 | 49 | // peek item at offset from beginning without removing from queue 50 | void peek(std::size_t offset, T& entry) 51 | { 52 | (*lws_)([this, offset] { return offset < cb_.size(); }, 53 | [this, offset, &entry] { 54 | auto iter = cb_.begin(); 55 | entry = *(iter + offset); 56 | } ); 57 | } 58 | 59 | // peek n items starting at offset (from beginning) into array without removing from buffer 60 | // precondition: array is large enough to hold n items 61 | void peek(off_t offset, std::size_t n, T array[]) 62 | { 63 | auto length = offset + n; 64 | (*lws_)([this, length] { return length <= cb_.size(); }, 65 | [this, offset, n, &array] { 66 | auto iter = cb_.begin() + offset; 67 | std::copy(iter, iter + n, array); 68 | } ); 69 | } 70 | 71 | template 72 | std::int64_t push_back(iter begin, iter end) 73 | { 74 | // push what you can, return the number pushed 75 | std::int64_t insertion_count = 0; 76 | (*lws_)([this] { return cb_.size() < cb_.capacity(); }, 77 | [this, begin, end, &insertion_count] { 78 | 79 | auto distance = static_cast(std::distance(begin, end)); 80 | auto empty_space = cb_.capacity() - cb_.size(); 81 | insertion_count = ( empty_space < distance ? empty_space : distance ); 82 | cb_.insert(cb_.end(), begin, begin + insertion_count ); 83 | 84 | } ); 85 | 86 | return insertion_count; 87 | 88 | } 89 | 90 | void push_back(const T& entry) 91 | { 92 | (*lws_)([this] { return cb_.size() < cb_.capacity(); }, 93 | [this, &entry] { cb_.push_back(entry); } ); 94 | } 95 | 96 | private: 97 | 98 | boost::circular_buffer cb_; 99 | std::unique_ptr lws_; 100 | 101 | }; // class circular_buffer 102 | 103 | } // namespace experimental 104 | } // namespace irods 105 | #endif 106 | -------------------------------------------------------------------------------- /libs3/include/libs3/util.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * util.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef UTIL_H 34 | #define UTIL_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include "libs3/libs3.h" 40 | 41 | // acl groups 42 | #define ACS_URL "http://acs.amazonaws.com/groups/" 43 | 44 | #define ACS_GROUP_ALL_USERS ACS_URL "global/AllUsers" 45 | #define ACS_GROUP_AWS_USERS ACS_URL "global/AuthenticatedUsers" 46 | #define ACS_GROUP_LOG_DELIVERY ACS_URL "s3/LogDelivery" 47 | 48 | // Derived from S3 documentation 49 | 50 | // This is the maximum number of bytes needed in a "compacted meta header" 51 | // buffer, which is a buffer storing all of the compacted meta headers. 52 | #define COMPACTED_METADATA_BUFFER_SIZE (S3_MAX_METADATA_COUNT * sizeof(S3_METADATA_HEADER_NAME_PREFIX "n: v")) 53 | 54 | // Maximum url encoded key size; since every single character could require 55 | // URL encoding, it's 3 times the size of a key (since each url encoded 56 | // character takes 3 characters: %NN) 57 | #define MAX_URLENCODED_KEY_SIZE (3 * S3_MAX_KEY_SIZE) 58 | 59 | // This is the maximum size of a URI that could be passed to S3: 60 | // https://s3.amazonaws.com/${BUCKET}/${KEY}?acl 61 | // 255 is the maximum bucket length 62 | #define MAX_URI_SIZE \ 63 | ((sizeof("https:///") - 1) + S3_MAX_HOSTNAME_SIZE + 255 + 1 + MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent") - 1) + \ 64 | 1) 65 | 66 | // Maximum size of a canonicalized resource 67 | #define MAX_CANONICALIZED_RESOURCE_SIZE (1 + 255 + 1 + MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent") - 1) + 1) 68 | 69 | // fujifilm has 64 byte keys. Just for additional safety making this 128 bytes. 70 | #define MAX_ACCESS_KEY_ID_LENGTH 128 71 | 72 | // Maximum length of a credential string 73 | // ///s3/aws4_request 74 | #define MAX_CREDENTIAL_SIZE (MAX_ACCESS_KEY_ID_LENGTH + 1) + 8 + 1 + 32 + sizeof("/s3/aws4_request") 75 | 76 | // Utilities ----------------------------------------------------------------- 77 | 78 | // URL-encodes a string from [src] into [dest]. [dest] must have at least 79 | // 3x the number of characters that [source] has. At most [maxSrcSize] bytes 80 | // from [src] are encoded; if more are present in [src], 0 is returned from 81 | // urlEncode, else nonzero is returned. 82 | int urlEncode(char* dest, const char* src, int maxSrcSize, int encodeSlash); 83 | 84 | // Returns < 0 on failure >= 0 on success 85 | int64_t parseIso8601Time(const char* str); 86 | 87 | uint64_t parseUnsignedInt(const char* str); 88 | 89 | // Because Windows seems to be missing isblank(), use our own; it's a very 90 | // easy function to write in any case 91 | int is_blank(char c); 92 | 93 | #endif /* UTIL_H */ 94 | -------------------------------------------------------------------------------- /unit_tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(IRODS_UNIT_TESTS_BUILD NO CACHE BOOL "Build unit tests") 2 | set(IRODS_UNIT_TESTS_REPORTING_STYLE "junit" CACHE STRING "The style of output used for unit test reporting [console, compact, junit, xml]") 3 | set(IRODS_UNIT_TESTS_REPORT_DIR "${CMAKE_CURRENT_BINARY_DIR}/reports" CACHE STRING "The directory of the unit test reports") 4 | set(IRODS_UNIT_TESTS_REPORT_FILENAME_PREFIX "" CACHE STRING "The filename prefix of the unit test report") 5 | set(IRODS_UNIT_TESTS_REPORT_FILENAME_SUFFIX ".xml" CACHE STRING "The filename suffix of the unit test report") 6 | 7 | if (DEFINED IRODS_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH_INIT) 8 | set(DEFAULT_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH ${IRODS_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH_INIT}) 9 | else() 10 | set(DEFAULT_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH NO) 11 | endif() 12 | set(IRODS_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH "${DEFAULT_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH}" CACHE BOOL "Build unit tests with install RPATH/RUNPATH") 13 | 14 | if (NOT IRODS_UNIT_TESTS_BUILD) 15 | return() 16 | endif() 17 | 18 | find_package(Catch2 3.4) 19 | if (NOT Catch2_FOUND) 20 | find_package(Catch2 2.13.2 REQUIRED) 21 | endif() 22 | 23 | include(Catch) 24 | 25 | # create phony target to build all unit tests 26 | add_custom_target(all-unit_tests) 27 | 28 | # Enable CTest support. 29 | enable_testing() 30 | 31 | # Include helper functions and other utilities. 32 | include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake") 33 | 34 | # Each file in the ./cmake/test_config directory defines variables for a specific test. 35 | # New tests should be added to this list. 36 | set( 37 | IRODS_PLUGIN_UNIT_TESTS 38 | s3_transport 39 | ) 40 | 41 | foreach(test IN LISTS IRODS_PLUGIN_UNIT_TESTS) 42 | unset_irods_test_variables() 43 | 44 | include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/test_config/irods_${test}.cmake") 45 | add_executable( 46 | ${IRODS_TEST_TARGET} 47 | ${IRODS_TEST_SOURCE_FILES} 48 | ) 49 | if (Catch2_VERSION VERSION_LESS "3.0.0") 50 | target_sources( 51 | ${IRODS_TEST_TARGET} 52 | PRIVATE 53 | "${CMAKE_CURRENT_SOURCE_DIR}/src/catch2_compat_include/catch2/catch_all.hpp" 54 | ) 55 | target_link_libraries( 56 | ${IRODS_TEST_TARGET} 57 | PRIVATE 58 | Catch2::Catch2 59 | ) 60 | target_include_directories( 61 | ${IRODS_TEST_TARGET} 62 | PRIVATE 63 | "${CMAKE_CURRENT_SOURCE_DIR}/src/catch2_compat_include" 64 | ) 65 | if (NOT IRODS_TEST_PROVIDES_MAIN) 66 | target_sources( 67 | ${IRODS_TEST_TARGET} 68 | PRIVATE 69 | "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp" 70 | ) 71 | endif() 72 | else() 73 | if (NOT IRODS_TEST_PROVIDES_MAIN) 74 | target_link_libraries( 75 | ${IRODS_TEST_TARGET} 76 | PRIVATE 77 | Catch2::Catch2WithMain 78 | ) 79 | else() 80 | target_link_libraries( 81 | ${IRODS_TEST_TARGET} 82 | PRIVATE 83 | Catch2::Catch2 84 | ) 85 | endif() 86 | endif() 87 | target_compile_definitions(${IRODS_TEST_TARGET} PRIVATE ${IRODS_COMPILE_DEFINITIONS_PRIVATE}) 88 | if (DEFINED IRODS_TEST_LINK_OBJLIBRARIES) 89 | target_link_objects(${IRODS_TEST_TARGET} PRIVATE ${IRODS_TEST_LINK_OBJLIBRARIES}) 90 | endif() 91 | target_link_libraries(${IRODS_TEST_TARGET} PRIVATE ${IRODS_TEST_LINK_LIBRARIES}) 92 | if (DEFINED IRODS_TEST_INCLUDE_PATH) 93 | target_include_directories(${IRODS_TEST_TARGET} PRIVATE ${IRODS_TEST_INCLUDE_PATH}) 94 | endif() 95 | set_property(TARGET ${IRODS_TEST_TARGET} PROPERTY BUILD_WITH_INSTALL_RPATH ${IRODS_UNIT_TESTS_BUILD_WITH_INSTALL_RPATH}) 96 | if (IRODS_UNIT_TESTS_ENABLE_ALL) 97 | target_compile_definitions(${IRODS_TEST_TARGET} PRIVATE IRODS_ENABLE_ALL_UNIT_TESTS) 98 | endif() 99 | 100 | add_dependencies(all-unit_tests ${IRODS_TEST_TARGET}) 101 | install( 102 | TARGETS ${IRODS_TEST_TARGET} 103 | RUNTIME 104 | DESTINATION "${IRODS_HOME_DIRECTORY}/unit_tests" 105 | ) 106 | 107 | catch_discover_tests( 108 | ${IRODS_TEST_TARGET} 109 | REPORTER "${IRODS_UNIT_TESTS_REPORTING_STYLE}" 110 | OUTPUT_DIR "${IRODS_UNIT_TESTS_REPORT_DIR}" 111 | OUTPUT_PREFIX "${IRODS_UNIT_TESTS_REPORT_FILENAME_PREFIX}" 112 | OUTPUT_SUFFIX "${IRODS_UNIT_TESTS_REPORT_FILENAME_SUFFIX}" 113 | ) 114 | endforeach() 115 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/multipart_shared_data.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_S3_TRANSPORT_MULTIPART_SHARED_DATA_HPP 2 | #define IRODS_S3_TRANSPORT_MULTIPART_SHARED_DATA_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #pragma GCC diagnostic push 10 | #pragma GCC diagnostic ignored "-Wunused-but-set-variable" 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #pragma GCC diagnostic pop 17 | 18 | #include "irods/private/s3_transport/types.hpp" 19 | 20 | #include 21 | 22 | namespace irods::experimental::io::s3_transport::shared_data 23 | { 24 | 25 | namespace interprocess_types 26 | { 27 | 28 | namespace bi = boost::interprocess; 29 | 30 | using segment_manager = bi::managed_shared_memory::segment_manager; 31 | using void_allocator = boost::container::scoped_allocator_adaptor 32 | >; 33 | using int_allocator = bi::allocator; 34 | using char_allocator = bi::allocator; 35 | using shm_int_vector = bi::vector; 36 | using shm_char_string = bi::basic_string, 37 | char_allocator>; 38 | using char_string_allocator = bi::allocator; 39 | using shm_string_vector = bi::vector; 40 | } 41 | 42 | // data that needs to be shared among different processes 43 | struct multipart_shared_data 44 | { 45 | using interprocess_recursive_mutex = boost::interprocess::interprocess_recursive_mutex; 46 | using error_codes = irods::experimental::io::s3_transport::error_codes; 47 | 48 | explicit multipart_shared_data(const interprocess_types::void_allocator &allocator) 49 | : threads_remaining_to_close{0} 50 | , done_initiate_multipart{false} 51 | , upload_id{allocator} 52 | , etags{allocator} 53 | , last_error_code{error_codes::SUCCESS} 54 | , cache_file_download_progress{cache_file_download_status::NOT_STARTED} 55 | , ref_count{0} 56 | , existing_object_size{-1} 57 | , circular_buffer_read_timeout{false} 58 | , file_open_counter{0} 59 | , cache_file_flushed{false} 60 | , know_number_of_threads{true} 61 | , first_open_has_trunc_flag{false} 62 | {} 63 | 64 | bool can_delete() { 65 | return know_number_of_threads 66 | ? threads_remaining_to_close == 0 67 | : file_open_counter == 0; 68 | } 69 | 70 | int threads_remaining_to_close; 71 | bool done_initiate_multipart; 72 | interprocess_types::shm_char_string upload_id; 73 | interprocess_types::shm_string_vector etags; 74 | error_codes last_error_code; 75 | cache_file_download_status cache_file_download_progress; 76 | int ref_count; 77 | std::int64_t existing_object_size; 78 | bool circular_buffer_read_timeout; 79 | int file_open_counter; 80 | bool cache_file_flushed; 81 | bool know_number_of_threads; 82 | 83 | // this is set so that multiple processes that are used to write to the file don't download the file 84 | // to cache if the trunc flag is not set. 85 | bool first_open_has_trunc_flag; 86 | }; 87 | 88 | } 89 | 90 | #if FMT_VERSION >= 100000 && FMT_VERSION < 110000 91 | template 92 | struct fmt::formatter> 93 | : fmt::formatter> 94 | { 95 | constexpr auto format(const boost::interprocess::basic_string& _str, 96 | format_context& ctx) const 97 | { 98 | return fmt::formatter>::format( 99 | static_cast>(_str), ctx); 100 | } 101 | }; 102 | #endif 103 | 104 | #endif // IRODS_S3_TRANSPORT_MULTIPART_SHARED_DATA_HPP 105 | -------------------------------------------------------------------------------- /packaging/test_irods_resource_plugin_s3_minio.py: -------------------------------------------------------------------------------- 1 | from .resource_suite_s3_nocache import Test_S3_NoCache_Base 2 | from .resource_suite_s3_nocache import Test_S3_NoCache_Large_File_Tests_Base 3 | from .resource_suite_s3_nocache import Test_S3_NoCache_MPU_Disabled_Base 4 | from .resource_suite_s3_cache import Test_S3_Cache_Base 5 | 6 | import psutil 7 | import sys 8 | import unittest 9 | 10 | class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): 11 | def __init__(self, *args, **kwargs): 12 | """Set up the test.""" 13 | self.proto = 'HTTP' 14 | self.keypairfile='/var/lib/irods/minio.keypair' 15 | self.archive_naming_policy='decoupled' 16 | self.s3stsdate='' 17 | self.s3region='us-east-1' 18 | self.s3endPoint = 'localhost:9000' 19 | self.s3sse = 0 # server side encryption 20 | super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) 21 | 22 | class Test_Compound_With_S3_Resource_EU_Central_1(Test_S3_Cache_Base, unittest.TestCase): 23 | ''' 24 | This also tests signature V4 with the x-amz-date header. 25 | ''' 26 | def __init__(self, *args, **kwargs): 27 | """Set up the test.""" 28 | self.proto = 'HTTP' 29 | self.keypairfile='/var/lib/irods/minio.keypair' 30 | self.s3stsdate='' 31 | self.s3region='eu-central-1' 32 | self.s3endPoint='localhost:9001' 33 | super(Test_Compound_With_S3_Resource_EU_Central_1, self).__init__(*args, **kwargs) 34 | 35 | 36 | class Test_S3_NoCache_V4(Test_S3_NoCache_Large_File_Tests_Base, unittest.TestCase): 37 | 38 | def __init__(self, *args, **kwargs): 39 | """Set up the test.""" 40 | self.proto = 'HTTP' 41 | self.keypairfile='/var/lib/irods/minio.keypair' 42 | self.s3region='us-east-1' 43 | self.s3endPoint = 'localhost:9000' 44 | self.s3EnableMPU=1 45 | super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) 46 | 47 | # issue 2024 48 | @unittest.skip("File removal too slow with MinIO") 49 | def test_put_get_file_greater_than_8GiB_two_threads(self): 50 | Test_S3_NoCache_Large_File_Tests_Base.test_put_get_file_greater_than_8GiB_two_threads(self) 51 | 52 | # issue 2024 53 | @unittest.skipIf(psutil.disk_usage('/').free < 4 * (4*1024*1024*1024 + 2), "not enough free space for four 4 GiB files (upload, download, and two on-disk MinIO)") 54 | def test_put_get_file_greater_than_4GiB_one_thread(self): 55 | Test_S3_NoCache_Large_File_Tests_Base.test_put_get_file_greater_than_4GiB_one_thread(self) 56 | 57 | class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_MPU_Disabled_Base, unittest.TestCase): 58 | def __init__(self, *args, **kwargs): 59 | """Set up the test.""" 60 | self.proto = 'HTTP' 61 | self.keypairfile='/var/lib/irods/minio.keypair' 62 | self.s3region='us-east-1' 63 | self.s3endPoint = 'localhost:9000' 64 | self.s3EnableMPU=0 65 | super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) 66 | 67 | class Test_S3_NoCache_Decoupled(Test_S3_NoCache_Base, unittest.TestCase): 68 | def __init__(self, *args, **kwargs): 69 | """Set up the test.""" 70 | self.proto = 'HTTP' 71 | self.keypairfile='/var/lib/irods/minio.keypair' 72 | self.s3region='us-east-1' 73 | self.s3endPoint = 'localhost:9000' 74 | self.s3EnableMPU=1 75 | self.archive_naming_policy = 'decoupled' 76 | super(Test_S3_NoCache_Decoupled, self).__init__(*args, **kwargs) 77 | 78 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 79 | def test_iget_with_stale_replica(self): # formerly known as 'dirty' 80 | pass 81 | 82 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 83 | def test_irepl_with_purgec(self): 84 | pass 85 | 86 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 87 | def test_put_get_small_file_in_repl_node(self): 88 | pass 89 | 90 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 91 | def test_put_get_large_file_in_repl_node(self): 92 | pass 93 | 94 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 95 | def test_s3_in_replication_node__issues_2102_2122(self): 96 | pass 97 | 98 | class Test_S3_NoCache_EU_Central_1(Test_S3_NoCache_Base, unittest.TestCase): 99 | ''' 100 | This also tests signature V4 with the x-amz-date header. 101 | ''' 102 | def __init__(self, *args, **kwargs): 103 | """Set up the test.""" 104 | self.proto = 'HTTP' 105 | self.keypairfile='/var/lib/irods/minio.keypair' 106 | self.s3region='eu-central-1' 107 | self.s3endPoint='localhost:9001' 108 | self.s3EnableMPU=1 109 | super(Test_S3_NoCache_EU_Central_1, self).__init__(*args, **kwargs) 110 | -------------------------------------------------------------------------------- /libs3/src/util.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * util.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include 35 | #include "libs3/util.h" 36 | 37 | // Convenience utility for making the code look nicer. Tests a string 38 | // against a format; only the characters specified in the format are 39 | // checked (i.e. if the string is longer than the format, the string still 40 | // checks out ok). Format characters are: 41 | // d - is a digit 42 | // anything else - is that character 43 | // Returns nonzero the string checks out, zero if it does not. 44 | static int checkString(const char* str, const char* format) 45 | { 46 | while (*format) { 47 | if (*format == 'd') { 48 | if (!isdigit(*str)) { 49 | return 0; 50 | } 51 | } 52 | else if (*str != *format) { 53 | return 0; 54 | } 55 | str++, format++; 56 | } 57 | 58 | return 1; 59 | } 60 | 61 | /* 62 | * Encode rules: 63 | * 1. Every byte except: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~' 64 | * 2. The space must be encoded as "%20" (and not as "+") 65 | * 3. Letters in the hexadecimal value must be uppercase, for example "%1A" 66 | * 4. Encode the forward slash character, '/', everywhere except in the object key name 67 | */ 68 | int urlEncode(char* dest, const char* src, int maxSrcSize, int encodeSlash) 69 | { 70 | static const char* hex = "0123456789ABCDEF"; 71 | 72 | int len = 0; 73 | 74 | if (src) 75 | while (*src) { 76 | if (++len > maxSrcSize) { 77 | *dest = 0; 78 | return 0; 79 | } 80 | unsigned char c = *src; 81 | if (isalnum(c) || (c == '-') || (c == '_') || (c == '.') || (c == '~') || (c == '/' && !encodeSlash)) { 82 | *dest++ = c; 83 | } 84 | else { 85 | *dest++ = '%'; 86 | *dest++ = hex[c >> 4]; 87 | *dest++ = hex[c & 15]; 88 | } 89 | src++; 90 | } 91 | 92 | *dest = 0; 93 | 94 | return 1; 95 | } 96 | 97 | int64_t parseIso8601Time(const char* str) 98 | { 99 | // Check to make sure that it has a valid format 100 | if (!checkString(str, "dddd-dd-ddTdd:dd:dd")) { 101 | return -1; 102 | } 103 | 104 | #define nextnum() (((*str - '0') * 10) + (*(str + 1) - '0')) 105 | 106 | // Convert it 107 | struct tm stm; 108 | memset(&stm, 0, sizeof(stm)); 109 | 110 | stm.tm_year = (nextnum() - 19) * 100; 111 | str += 2; 112 | stm.tm_year += nextnum(); 113 | str += 3; 114 | 115 | stm.tm_mon = nextnum() - 1; 116 | str += 3; 117 | 118 | stm.tm_mday = nextnum(); 119 | str += 3; 120 | 121 | stm.tm_hour = nextnum(); 122 | str += 3; 123 | 124 | stm.tm_min = nextnum(); 125 | str += 3; 126 | 127 | stm.tm_sec = nextnum(); 128 | str += 2; 129 | 130 | stm.tm_isdst = -1; 131 | 132 | int64_t ret = mktime(&stm); 133 | 134 | // Skip the millis 135 | 136 | if (*str == '.') { 137 | str++; 138 | while (isdigit(*str)) { 139 | str++; 140 | } 141 | } 142 | 143 | if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) { 144 | int sign = (*str++ == '-') ? -1 : 1; 145 | int hours = nextnum(); 146 | str += 3; 147 | int minutes = nextnum(); 148 | ret += (-sign * (((hours * 60) + minutes) * 60)); 149 | } 150 | // Else it should be Z to be a conformant time string, but we just assume 151 | // that it is rather than enforcing that 152 | 153 | return ret; 154 | } 155 | 156 | uint64_t parseUnsignedInt(const char* str) 157 | { 158 | // Skip whitespace 159 | while (is_blank(*str)) { 160 | str++; 161 | } 162 | 163 | uint64_t ret = 0; 164 | 165 | while (isdigit(*str)) { 166 | ret *= 10; 167 | ret += (*str++ - '0'); 168 | } 169 | 170 | return ret; 171 | } 172 | 173 | int is_blank(char c) 174 | { 175 | return ((c == ' ') || (c == '\t')); 176 | } 177 | -------------------------------------------------------------------------------- /irods_consortium_continuous_integration_build_hook.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import glob 4 | import multiprocessing 5 | import optparse 6 | import os 7 | import sys 8 | import tempfile 9 | 10 | import irods_python_ci_utilities 11 | 12 | def update_local_package_repositories(): 13 | # Updating via dnf or yum actually upgrades packages, so don't do anything in those cases (for now). 14 | dispatch_map = { 15 | 'Ubuntu': ['sudo', 'apt-get', 'update'], 16 | 'Centos': None, 17 | 'Centos linux': None, 18 | 'Almalinux': None, 19 | 'Rocky linux': None, 20 | 'Opensuse ': None, 21 | 'Debian gnu_linux': ['sudo', 'apt-get', 'update'] 22 | } 23 | try: 24 | cmd = dispatch_map[irods_python_ci_utilities.get_distribution()] 25 | if cmd: 26 | irods_python_ci_utilities.subprocess_get_output(cmd, check_rc=True) 27 | except KeyError: 28 | irods_python_ci_utilities.raise_not_implemented_for_distribution() 29 | 30 | def install_building_dependencies(externals_directory): 31 | # The externals_list needs to include all dependencies, not the minimum set required for this plugin. If custom 32 | # externals are being supplied via externals_directory, only the externals packages which exist in that directory 33 | # will be installed. 34 | externals_list = [ 35 | 'irods-externals-boost1.81.0-2', 36 | 'irods-externals-clang16.0.6-0', 37 | 'irods-externals-nanodbc2.13.0-3' 38 | ] 39 | if externals_directory == 'None' or externals_directory is None: 40 | irods_python_ci_utilities.install_irods_core_dev_repository() 41 | irods_python_ci_utilities.install_os_packages(externals_list) 42 | else: 43 | # Make sure the local package repositories are up to date so package dependencies can also be installed. 44 | update_local_package_repositories() 45 | package_suffix = irods_python_ci_utilities.get_package_suffix() 46 | os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(externals_directory) 47 | externals = [] 48 | for irods_externals in externals_list: 49 | externals.append(glob.glob(os.path.join(os_specific_directory, irods_externals + '*.{0}'.format(package_suffix)))[0]) 50 | irods_python_ci_utilities.install_os_packages_from_files(externals) 51 | install_os_specific_dependencies() 52 | 53 | def install_os_specific_dependencies_apt(): 54 | update_local_package_repositories() 55 | irods_python_ci_utilities.install_os_packages(['cmake', 'make', 'libssl-dev', 'libxml2-dev', 'libcurl4-gnutls-dev', 'gcc']) 56 | 57 | def install_os_specific_dependencies_yum(): 58 | irods_python_ci_utilities.install_os_packages(['cmake', 'make', 'gcc', 'openssl-devel', 'libxml2-devel', 'curl-devel']) 59 | 60 | def install_os_specific_dependencies(): 61 | dispatch_map = { 62 | 'Ubuntu': install_os_specific_dependencies_apt, 63 | 'Centos': install_os_specific_dependencies_yum, 64 | 'Centos linux': install_os_specific_dependencies_yum, 65 | 'Almalinux': install_os_specific_dependencies_yum, 66 | 'Rocky linux': install_os_specific_dependencies_yum, 67 | 'Opensuse ': install_os_specific_dependencies_yum, 68 | 'Debian gnu_linux': install_os_specific_dependencies_apt 69 | } 70 | try: 71 | return dispatch_map[irods_python_ci_utilities.get_distribution()]() 72 | except KeyError: 73 | irods_python_ci_utilities.raise_not_implemented_for_distribution() 74 | 75 | def copy_output_packages(build_directory, output_root_directory): 76 | irods_python_ci_utilities.gather_files_satisfying_predicate( 77 | build_directory, 78 | irods_python_ci_utilities.append_os_specific_directory(output_root_directory), 79 | lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix())) 80 | 81 | def main(build_directory, output_root_directory, irods_packages_root_directory, externals_directory): 82 | install_building_dependencies(externals_directory) 83 | if irods_packages_root_directory: 84 | irods_python_ci_utilities.install_irods_dev_and_runtime_packages(irods_packages_root_directory) 85 | build_directory = os.path.abspath(build_directory or tempfile.mkdtemp(prefix='irods_s3_plugin_build_directory')) 86 | irods_python_ci_utilities.subprocess_get_output(['cmake', os.path.dirname(os.path.realpath(__file__))], check_rc=True, cwd=build_directory) 87 | irods_python_ci_utilities.subprocess_get_output(['make', '-j', str(multiprocessing.cpu_count()), 'package'], check_rc=True, cwd=build_directory) 88 | if output_root_directory: 89 | copy_output_packages(build_directory, output_root_directory) 90 | 91 | if __name__ == '__main__': 92 | parser = optparse.OptionParser() 93 | parser.add_option('--build_directory') 94 | parser.add_option('--output_root_directory') 95 | parser.add_option('--irods_packages_root_directory') 96 | parser.add_option('--externals_packages_directory') 97 | options, _ = parser.parse_args() 98 | 99 | main(options.build_directory, 100 | options.output_root_directory, 101 | options.irods_packages_root_directory, 102 | options.externals_packages_directory) 103 | -------------------------------------------------------------------------------- /s3_resource/include/irods/private/s3_resource/s3_operations.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IRODS_S3_RESOURCE_OPERATIONS_HPP 2 | #define IRODS_S3_RESOURCE_OPERATIONS_HPP 3 | 4 | // =-=-=-=-=-=-=- 5 | // irods includes 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace irods_s3 { 15 | 16 | // =-=-=-=-=-=-=- 17 | // interface for file registration 18 | irods::error s3_registered_operation( irods::plugin_context& _ctx); 19 | 20 | // =-=-=-=-=-=-=- 21 | // interface for file unregistration 22 | irods::error s3_unregistered_operation( irods::plugin_context& _ctx); 23 | 24 | // =-=-=-=-=-=-=- 25 | // interface for file modification 26 | irods::error s3_modified_operation( irods::plugin_context& _ctx); 27 | 28 | // =-=-=-=-=-=-=- 29 | // interface for POSIX create 30 | irods::error s3_file_create_operation( irods::plugin_context& _ctx); 31 | 32 | // =-=-=-=-=-=-=- 33 | // interface for POSIX Open 34 | irods::error s3_file_open_operation( irods::plugin_context& _ctx); 35 | 36 | // =-=-=-=-=-=-=- 37 | // interface for POSIX Read 38 | irods::error s3_file_read_operation( irods::plugin_context& _ctx, void* _buf, const int _len ); 39 | 40 | // =-=-=-=-=-=-=- 41 | // interface for POSIX Write 42 | irods::error s3_file_write_operation( irods::plugin_context& _ctx, 43 | const void* _buf, 44 | const int _len ); 45 | // =-=-=-=-=-=-=- 46 | // interface for POSIX Close 47 | irods::error s3_file_close_operation( irods::plugin_context& _ctx ); 48 | 49 | // =-=-=-=-=-=-=- 50 | // interface for POSIX Unlink 51 | irods::error s3_file_unlink_operation( irods::plugin_context& _ctx); 52 | 53 | // =-=-=-=-=-=-=- 54 | // interface for POSIX Stat 55 | irods::error s3_file_stat_operation( irods::plugin_context& _ctx, struct stat* _statbuf ); 56 | 57 | // =-=-=-=-=-=-=- 58 | // interface for POSIX Fstat 59 | irods::error s3FileFstatPlugin( irods::plugin_context& _ctx, struct stat* _statbuf ); 60 | 61 | // =-=-=-=-=-=-=- 62 | // interface for POSIX lseek 63 | irods::error s3_file_lseek_operation( irods::plugin_context& _ctx, const long long _offset, const int _whence ); 64 | 65 | // =-=-=-=-=-=-=- 66 | // interface for POSIX mkdir 67 | irods::error s3_file_mkdir_operation( irods::plugin_context& _ctx ); 68 | 69 | // =-=-=-=-=-=-=- 70 | // interface for POSIX mkdir 71 | irods::error s3_rmdir_operation( irods::plugin_context& _ctx ); 72 | 73 | // =-=-=-=-=-=-=- 74 | // interface for POSIX opendir 75 | irods::error s3_opendir_operation( irods::plugin_context& _ctx ); 76 | 77 | // =-=-=-=-=-=-=- 78 | // interface for POSIX closedir 79 | irods::error s3_closedir_operation( irods::plugin_context& _ctx); 80 | 81 | // =-=-=-=-=-=-=- 82 | // interface for POSIX readdir 83 | irods::error s3_readdir_operation( irods::plugin_context& _ctx, struct rodsDirent** _dirent_ptr ); 84 | 85 | // =-=-=-=-=-=-=- 86 | // interface for POSIX rename 87 | irods::error s3_file_rename_operation( irods::plugin_context& _ctx, const char* _new_file_name ); 88 | 89 | // =-=-=-=-=-=-=- 90 | // interface for POSIX truncate 91 | irods::error s3FileTruncatePlugin( irods::plugin_context& _ctx ); 92 | 93 | 94 | // interface to determine free space on a device given a path 95 | irods::error s3_get_fs_freespace_operation( irods::plugin_context& _ctx ); 96 | 97 | irods::error s3FileCopyPlugin( int mode, const char *srcFileName, const char *destFileName); 98 | 99 | // =-=-=-=-=-=-=- 100 | // s3StageToCache - This routine is for testing the TEST_STAGE_FILE_TYPE. 101 | // Just copy the file from filename to cacheFilename. optionalInfo info 102 | // is not used. 103 | irods::error s3_stage_to_cache_operation( irods::plugin_context& _ctx, const char* _cache_file_name ); 104 | 105 | // =-=-=-=-=-=-=- 106 | // s3SyncToArch - This routine is for testing the TEST_STAGE_FILE_TYPE. 107 | // Just copy the file from cacheFilename to filename. optionalInfo info 108 | // is not used. 109 | irods::error s3_sync_to_arch_operation( irods::plugin_context& _ctx, const char* _cache_file_name ); 110 | 111 | // =-=-=-=-=-=-=- 112 | // used to allow the resource to determine which host 113 | // should provide the requested operation 114 | irods::error s3_resolve_resc_hier_operation( irods::plugin_context& _ctx, 115 | const std::string* _opr, const std::string* _curr_host, 116 | irods::hierarchy_parser* _out_parser, float* _out_vote ); 117 | 118 | // =-=-=-=-=-=-=- 119 | // code which would rebalance the resource, S3 does not rebalance. 120 | irods::error s3_rebalance_operation( irods::plugin_context& _ctx ); 121 | 122 | irods::error s3_notify_operation( irods::plugin_context& _ctx, const std::string* str ); 123 | 124 | irods::error s3_read_checksum_from_storage_device(irods::plugin_context& _ctx, 125 | const std::string* checksum_scheme, 126 | std::string* returned_checksum); 127 | } 128 | 129 | #endif 130 | -------------------------------------------------------------------------------- /libs3/include/libs3/string_buffer.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * string_buffer.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef STRING_BUFFER_H 34 | #define STRING_BUFFER_H 35 | 36 | #include 37 | 38 | // Declare a string_buffer with the given name of the given maximum length 39 | #define string_buffer(name, len) \ 40 | char name[len + 1]; \ 41 | int name##Len 42 | 43 | // Initialize a string_buffer 44 | #define string_buffer_initialize(sb) \ 45 | do { \ 46 | sb[0] = 0; \ 47 | sb##Len = 0; \ 48 | } \ 49 | while (0) 50 | 51 | // Append [len] bytes of [str] to [sb], setting [all_fit] to 1 if it fit, and 52 | // 0 if it did not 53 | #define string_buffer_append(sb, str, len, all_fit) \ 54 | do { \ 55 | sb##Len += snprintf(&(sb[sb##Len]), sizeof(sb) - sb##Len - 1, "%.*s", (int) (len), str); \ 56 | if (sb##Len > (int) (sizeof(sb) - 1)) { \ 57 | sb##Len = sizeof(sb) - 1; \ 58 | all_fit = 0; \ 59 | } \ 60 | else { \ 61 | all_fit = 1; \ 62 | } \ 63 | } \ 64 | while (0) 65 | 66 | // Declare a string multibuffer with the given name of the given maximum size 67 | #define string_multibuffer(name, size) \ 68 | char name[size]; \ 69 | int name##Size 70 | 71 | // Initialize a string_multibuffer 72 | #define string_multibuffer_initialize(smb) \ 73 | do { \ 74 | smb##Size = 0; \ 75 | } \ 76 | while (0) 77 | 78 | // Evaluates to the current string within the string_multibuffer 79 | #define string_multibuffer_current(smb) &(smb[smb##Size]) 80 | 81 | // Adds a new string to the string_multibuffer 82 | #define string_multibuffer_add(smb, str, len, all_fit) \ 83 | do { \ 84 | smb##Size += (snprintf(&(smb[smb##Size]), sizeof(smb) - smb##Size, "%.*s", (int) (len), str) + 1); \ 85 | if (smb##Size > (int) sizeof(smb)) { \ 86 | smb##Size = sizeof(smb); \ 87 | all_fit = 0; \ 88 | } \ 89 | else { \ 90 | all_fit = 1; \ 91 | } \ 92 | } \ 93 | while (0) 94 | 95 | // Appends to the current string in the string_multibuffer. There must be a 96 | // current string, meaning that string_multibuffer_add must have been called 97 | // at least once for this string_multibuffer. 98 | #define string_multibuffer_append(smb, str, len, all_fit) \ 99 | do { \ 100 | smb##Size--; \ 101 | string_multibuffer_add(smb, str, len, all_fit); \ 102 | } \ 103 | while (0) 104 | 105 | #endif /* STRING_BUFFER_H */ 106 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | DisableFormat: false 3 | 4 | Language: Cpp 5 | Standard: c++20 6 | 7 | ColumnLimit: 120 8 | UseCRLF: false 9 | 10 | UseTab: Never 11 | TabWidth: 4 12 | IndentWidth: 4 13 | ConstructorInitializerIndentWidth: 4 14 | 15 | IndentPPDirectives: AfterHash 16 | PPIndentWidth: 2 17 | 18 | IndentAccessModifiers: false 19 | AccessModifierOffset: -2 20 | 21 | AlignAfterOpenBracket: Align 22 | #AlignAfterOpenBracket: BlockIndent 23 | AlignArrayOfStructures: None 24 | AlignConsecutiveAssignments: None 25 | AlignConsecutiveBitFields: Consecutive 26 | AlignConsecutiveDeclarations: None 27 | AlignConsecutiveMacros: AcrossEmptyLinesAndComments 28 | AlignEscapedNewlines: Left 29 | AlignOperands: Align 30 | AlignTrailingComments: true 31 | 32 | AllowAllArgumentsOnNextLine: true 33 | AllowAllParametersOfDeclarationOnNextLine: false 34 | AllowAllConstructorInitializersOnNextLine: false 35 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 36 | 37 | AllowShortBlocksOnASingleLine: Empty 38 | AllowShortCaseLabelsOnASingleLine: false 39 | AllowShortEnumsOnASingleLine: false 40 | AllowShortFunctionsOnASingleLine: None 41 | AllowShortIfStatementsOnASingleLine: Never 42 | AllowShortLambdasOnASingleLine: All 43 | AllowShortLoopsOnASingleLine: false 44 | 45 | IndentCaseBlocks: false 46 | IndentCaseLabels: true 47 | IndentExternBlock: AfterExternBlock 48 | IndentGotoLabels: true 49 | IndentRequires: true 50 | #IndentRequiresClause: true 51 | IndentWrappedFunctionNames: false 52 | LambdaBodyIndentation: Signature 53 | NamespaceIndentation: All 54 | 55 | AlwaysBreakAfterReturnType: None 56 | AlwaysBreakAfterDefinitionReturnType: None 57 | AlwaysBreakBeforeMultilineStrings: false 58 | AlwaysBreakTemplateDeclarations: Yes 59 | 60 | BinPackArguments: false 61 | BinPackParameters: false 62 | 63 | BreakBeforeBinaryOperators: None 64 | BreakBeforeConceptDeclarations: true 65 | BreakBeforeInheritanceComma: true 66 | BreakBeforeTernaryOperators: true 67 | BreakConstructorInitializers: BeforeComma 68 | BreakInheritanceList: BeforeComma 69 | BreakStringLiterals: true 70 | BreakBeforeBraces: Custom 71 | BraceWrapping: 72 | AfterCaseLabel: false 73 | AfterClass: true 74 | AfterControlStatement: MultiLine 75 | AfterEnum: true 76 | AfterExternBlock: false 77 | AfterFunction: true 78 | AfterNamespace: true 79 | AfterStruct: true 80 | AfterUnion: true 81 | BeforeCatch: true 82 | BeforeElse: true 83 | BeforeLambdaBody: false 84 | BeforeWhile: true 85 | IndentBraces: false 86 | SplitEmptyFunction: true 87 | SplitEmptyNamespace: true 88 | SplitEmptyRecord: true 89 | 90 | #InsertBraces: false 91 | #RemoveBracesLLVM: false 92 | 93 | DeriveLineEnding: true 94 | DerivePointerAlignment: false 95 | ExperimentalAutoDetectBinPacking: false 96 | 97 | SpaceAfterCStyleCast: true 98 | SpaceAfterLogicalNot: false 99 | SpaceAfterTemplateKeyword: true 100 | SpaceAroundPointerQualifiers: Default 101 | SpaceBeforeAssignmentOperators: true 102 | SpaceBeforeCaseColon: false 103 | SpaceBeforeCpp11BracedList: false 104 | SpaceBeforeCtorInitializerColon: true 105 | SpaceBeforeInheritanceColon: true 106 | SpaceBeforeParens: ControlStatements 107 | SpaceBeforeRangeBasedForLoopColon: true 108 | SpaceBeforeSquareBrackets: false 109 | SpaceInEmptyBlock: false 110 | SpaceInEmptyParentheses: false 111 | SpacesBeforeTrailingComments: 1 112 | SpacesInAngles: Never 113 | SpacesInConditionalStatement: false 114 | SpacesInContainerLiterals: false 115 | SpacesInCStyleCastParentheses: false 116 | SpacesInParentheses: false 117 | SpacesInSquareBrackets: false 118 | SpacesInLineCommentPrefix: 119 | Minimum: 0 120 | Maximum: -1 121 | 122 | BitFieldColonSpacing: Both 123 | Cpp11BracedListStyle: true 124 | 125 | FixNamespaceComments: true 126 | ShortNamespaceLines: 0 127 | 128 | SortIncludes: Never 129 | SortUsingDeclarations: true 130 | 131 | PointerAlignment: Left 132 | ReferenceAlignment: Pointer 133 | 134 | EmptyLineAfterAccessModifier: Never 135 | EmptyLineBeforeAccessModifier: LogicalBlock 136 | KeepEmptyLinesAtTheStartOfBlocks: false 137 | MaxEmptyLinesToKeep: 1 138 | #SeparateDefinitionBlocks: Always 139 | 140 | AttributeMacros: 141 | - __capability 142 | - __extension__ 143 | - __const__ 144 | - __volatile__ 145 | - __signed__ 146 | - __inline__ 147 | - BOOST_RESTRICT 148 | - BOOST_FORCEINLINE 149 | - BOOST_NOINLINE 150 | - BOOST_NORETURN 151 | - BOOST_ATTRIBUTE_UNUSED 152 | - BOOST_ATTRIBUTE_NODISCARD 153 | - BOOST_ATTRIBUTE_NO_UNIQUE_ADDRESS 154 | - BOOST_MAY_ALIAS 155 | - BOOST_NO_MAY_ALIAS 156 | - BOOST_SYMBOL_VISIBLE 157 | - BOOST_SYMBOL_EXPORT 158 | - BOOST_SYMBOL_IMPORT 159 | - BOOST_INLINE_VARIABLE 160 | - BOOST_INLINE_CONSTEXPR 161 | - BOOST_NOEXCEPT_OR_NOTHROW 162 | - BOOST_FINAL 163 | - BOOST_OVERRIDE 164 | ForEachMacros: 165 | - foreach 166 | - BOOST_FOREACH 167 | - BOOST_REVERSE_FOREACH 168 | IfMacros: 169 | - CATCH_CHECKED_IF 170 | - CATCH_CHECKED_ELSE 171 | - CHECKED_IF 172 | - CHECKED_ELSE 173 | - BOOST_IF_CONSTEXPR 174 | StatementMacros: 175 | - BOOST_PRAGMA_MESSAGE 176 | - BOOST_HEADER_DEPRECATED 177 | WhitespaceSensitiveMacros: 178 | - STRINGIZE 179 | - PP_STRINGIZE 180 | - BOOST_STRINGIZE 181 | - BOOST_PP_STRINGIZE 182 | - BOOST_PP_WSTRINGIZE 183 | - CMAKE_STRINGIFY 184 | - CMAKE_TOSTRING 185 | 186 | PenaltyBreakAssignment: 2 187 | PenaltyBreakBeforeFirstCallParameter: 0 188 | PenaltyBreakComment: 300 189 | PenaltyBreakFirstLessLess: 120 190 | #PenaltyBreakOpenParenthesis: 0 191 | PenaltyBreakString: 1000 192 | PenaltyBreakTemplateDeclaration: 10 193 | PenaltyExcessCharacter: 1000000 194 | PenaltyIndentedWhitespace: 0 195 | PenaltyReturnTypeOnItsOwnLine: 200 196 | 197 | CommentPragmas: '^ (IWYU pragma:|(NOLINT(NEXTLINE|BEGIN|END)?)|clang-format)' 198 | CompactNamespaces: false 199 | IncludeBlocks: Preserve 200 | InsertTrailingCommas: None 201 | ReflowComments: true 202 | #RequiresClausePosition: OwnLine 203 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/managed_shared_memory_object.hpp: -------------------------------------------------------------------------------- 1 | #ifndef S3_TRANSPORT_MANAGED_SHARED_MEMORY_OBJECT_HPP 2 | #define S3_TRANSPORT_MANAGED_SHARED_MEMORY_OBJECT_HPP 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "irods/private/s3_transport/logging_category.hpp" 13 | 14 | namespace irods::experimental::interprocess 15 | { 16 | 17 | namespace shared_memory 18 | { 19 | namespace bi = boost::interprocess; 20 | namespace bc = boost::container; 21 | 22 | namespace log = irods::experimental::log; 23 | using logger = log::logger; 24 | 25 | using segment_manager = bi::managed_shared_memory::segment_manager; 26 | using void_allocator = bc::scoped_allocator_adaptor>; 27 | 28 | template 29 | class named_shared_memory_object 30 | { 31 | 32 | private: 33 | 34 | struct ipc_object 35 | { 36 | template 37 | ipc_object(void_allocator&& alloc_inst, time_t access_time, 38 | Args&& ... args) 39 | : thing(alloc_inst, std::forward(args)...) 40 | , last_access_time_in_seconds(access_time) 41 | {} 42 | 43 | // T must have ref_count and can_delete() 44 | T thing; 45 | 46 | time_t last_access_time_in_seconds; 47 | bi::interprocess_recursive_mutex access_mutex; 48 | 49 | }; 50 | 51 | public: 52 | 53 | named_shared_memory_object(const named_shared_memory_object&) = delete; 54 | auto operator=(const named_shared_memory_object&) -> 55 | named_shared_memory_object& = delete; 56 | 57 | template 58 | named_shared_memory_object(const std::string& shm_name, 59 | time_t shared_memory_timeout_in_seconds, 60 | std::uint64_t shm_size, 61 | Args&& ...args) 62 | 63 | : shm_name_{shm_name} 64 | , shm_size_{shm_size} 65 | , shm_{bi::open_or_create, shm_name_.c_str(), shm_size_} 66 | , alloc_inst_{shm_.get_segment_manager()} 67 | 68 | { 69 | const time_t now = time(0); 70 | 71 | bi::named_mutex create_delete_reset_mutex(bi::open_or_create, shm_name_.c_str()); 72 | bi::scoped_lock lk{create_delete_reset_mutex}; 73 | 74 | object_ = shm_.find_or_construct(SHARED_DATA_NAME.c_str()) 75 | ( static_cast(shm_.get_segment_manager()), now, 76 | std::forward(args)...); 77 | 78 | (object_->thing.ref_count)++; 79 | 80 | const bool shmem_has_expired = now - 81 | object_->last_access_time_in_seconds 82 | > shared_memory_timeout_in_seconds; 83 | 84 | if (shmem_has_expired) { 85 | 86 | logger::debug("{}:{} ({}) SHMEM_HAS_EXPIRED", __FILE__, __LINE__, __func__); 87 | 88 | // rebuild shmem object 89 | shm_.destroy(SHARED_DATA_NAME.c_str()); 90 | object_ = shm_.find_or_construct(SHARED_DATA_NAME.c_str()) 91 | ( static_cast(shm_.get_segment_manager()), now, 92 | std::forward(args)...); 93 | 94 | object_->thing.ref_count = 1; 95 | } 96 | object_->last_access_time_in_seconds = now; 97 | } 98 | 99 | ~named_shared_memory_object() 100 | { 101 | { 102 | bi::named_mutex create_delete_reset_mutex(bi::open_or_create, shm_name_.c_str()); 103 | bi::scoped_lock lk{create_delete_reset_mutex}; 104 | 105 | (object_->thing.ref_count)--; 106 | 107 | bool can_delete = object_->thing.can_delete(); 108 | 109 | if (object_->thing.ref_count == 0 && can_delete) { 110 | 111 | object_->thing.~T(); 112 | object_ = nullptr; 113 | if (!bi::shared_memory_object::remove(shm_name_.c_str())) { 114 | logger::error("{}:{} ({}) removal of shared memory object [{}] failed", __FILE__, __LINE__, __func__, shm_name_); 115 | } 116 | if (!bi::named_mutex::remove(shm_name_.c_str())) { 117 | logger::error("{}:{} ({}) removal of mutex for shared memory object [{}] failed", __FILE__, __LINE__, __func__, shm_name_); 118 | } 119 | } 120 | } 121 | } 122 | 123 | template 124 | auto atomic_exec(Function _func) const 125 | { 126 | bi::scoped_lock lk{object_->access_mutex}; 127 | object_->last_access_time_in_seconds = time(0); 128 | return _func(object_->thing); 129 | } 130 | 131 | template 132 | auto exec(Function _func) const 133 | { 134 | object_->last_access_time_in_seconds = time(0); 135 | return _func(object_->thing); 136 | } 137 | 138 | void_allocator& get_allocator() { 139 | return alloc_inst_; 140 | } 141 | 142 | auto get_free_memory() { 143 | return shm_.get_free_memory(); 144 | } 145 | 146 | private: 147 | 148 | const std::string shm_name_; 149 | const std::uint64_t shm_size_; 150 | bi::managed_shared_memory shm_; 151 | void_allocator alloc_inst_; 152 | 153 | ipc_object* object_; 154 | 155 | const std::string SHARED_DATA_NAME{"SharedData"}; 156 | 157 | }; // class shared_memory_object 158 | 159 | } // namespace shared_memory 160 | } // namespace irods::experimental::ipc 161 | 162 | #endif // S3_TRANSPORT_MANAGED_SHARED_MEMORY_OBJECT_HPP 163 | -------------------------------------------------------------------------------- /s3_transport/include/irods/private/s3_transport/util.hpp: -------------------------------------------------------------------------------- 1 | #ifndef S3_TRANSPORT_UTIL_HPP 2 | #define S3_TRANSPORT_UTIL_HPP 3 | 4 | #include "irods/private/s3_transport/circular_buffer.hpp" 5 | 6 | // iRODS includes 7 | #include 8 | #include 9 | 10 | // misc includes 11 | #include 12 | #include "libs3/libs3.h" 13 | 14 | // stdlib and misc includes 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | // boost includes 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | // local includes 41 | #include "irods/private/s3_transport/multipart_shared_data.hpp" 42 | 43 | #include "irods/private/s3_transport/types.hpp" 44 | 45 | namespace irods::experimental::io::s3_transport 46 | { 47 | 48 | struct constants 49 | { 50 | 51 | static const std::int64_t MAXIMUM_NUMBER_ETAGS_PER_UPLOAD{10000}; 52 | static const std::int64_t BYTES_PER_ETAG{112}; // 80 bytes for every string added, 32 bytes for the vector size, 53 | // determined by testing 54 | static const std::int64_t UPLOAD_ID_SIZE{128}; 55 | 56 | // See https://groups.google.com/g/boost-list/c/5ADnEPYg-ho for an explanation 57 | // of why the 100*sizeof(void*) is used below. Essentially, the shared memory 58 | // must have enough space for the memory algorithm and reserved area but there is 59 | // no way of knowing the size for these. It is stated that 100*sizeof(void*) would 60 | // be enough. 61 | static constexpr std::int64_t MAX_S3_SHMEM_SIZE{100*sizeof(void*) + sizeof(shared_data::multipart_shared_data) + 62 | MAXIMUM_NUMBER_ETAGS_PER_UPLOAD * (BYTES_PER_ETAG + 1) + 63 | UPLOAD_ID_SIZE + 1}; 64 | 65 | static const int DEFAULT_SHARED_MEMORY_TIMEOUT_IN_SECONDS{900}; 66 | inline static const std::string SHARED_MEMORY_KEY_PREFIX{"irods_s3_transport-shm-"}; 67 | }; 68 | 69 | void print_bucket_context( const libs3_types::bucket_context& bucket_context ); 70 | 71 | void store_and_log_status( libs3_types::status status, 72 | const libs3_types::error_details *error, 73 | const std::string& function, 74 | const libs3_types::bucket_context& saved_bucket_context, 75 | libs3_types::status& pStatus, 76 | std::uint64_t thread_id = 0); 77 | // Sleep between _s / 2 and _s seconds. 78 | // The random addition ensures that threads don't all cluster up and retry 79 | // at the same time (dogpile effect) 80 | void s3_sleep( int _s ); 81 | 82 | // Returns timestamp in usec for delta-t comparisons 83 | auto get_time_in_microseconds() -> std::uint64_t; 84 | 85 | struct upload_manager 86 | { 87 | explicit upload_manager(libs3_types::bucket_context& _saved_bucket_context) 88 | : saved_bucket_context{_saved_bucket_context} 89 | , xml{""} 90 | , remaining{0} 91 | , offset{0} 92 | , shared_memory_timeout_in_seconds{60} 93 | { 94 | } 95 | 96 | libs3_types::bucket_context& saved_bucket_context; /* To enable more detailed error messages */ 97 | 98 | /* Below used for the upload completion command, need to send in XML */ 99 | std::string xml; 100 | 101 | std::int64_t remaining; 102 | std::int64_t offset; 103 | libs3_types::status status; /* status returned by libs3 */ 104 | std::string object_key; 105 | std::string shmem_key; 106 | time_t shared_memory_timeout_in_seconds; 107 | }; 108 | 109 | struct data_for_write_callback 110 | { 111 | data_for_write_callback(libs3_types::bucket_context& _saved_bucket_context, 112 | irods::experimental::circular_buffer& _circular_buffer) 113 | : offset{0} 114 | , circular_buffer{_circular_buffer} 115 | , content_length{0} 116 | , bytes_written{0} 117 | , saved_bucket_context{_saved_bucket_context} 118 | , thread_identifier{0} 119 | {} 120 | 121 | libs3_types::char_type *buffer; 122 | std::int64_t offset; 123 | 124 | irods::experimental::circular_buffer& 125 | circular_buffer; 126 | 127 | std::int64_t content_length; 128 | std::int64_t bytes_written; 129 | libs3_types::status status; 130 | 131 | libs3_types::bucket_context& 132 | saved_bucket_context; // To enable more detailed error messages 133 | std::uint64_t thread_identifier; 134 | }; 135 | 136 | struct data_for_head_callback 137 | { 138 | explicit data_for_head_callback(libs3_types::bucket_context& _bucket_context) 139 | : last_modified{0} 140 | , content_length{0} 141 | , x_amz_storage_class{} // for glacier 142 | , x_amz_restore{} // for glacier 143 | , status{libs3_types::status_ok} 144 | , bucket_context{_bucket_context} 145 | {} 146 | 147 | time_t last_modified; 148 | std::int64_t content_length; 149 | std::string x_amz_storage_class; 150 | std::string x_amz_restore; 151 | libs3_types::status status; 152 | libs3_types::bucket_context& bucket_context; 153 | }; 154 | 155 | } // irods::experimental::io::s3_transport 156 | 157 | #endif // S3_TRANSPORT_UTIL_HPP 158 | -------------------------------------------------------------------------------- /libs3/include/libs3/request.h: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * request.h 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #ifndef REQUEST_H 34 | #define REQUEST_H 35 | 36 | #include "libs3/libs3.h" 37 | #include "libs3/error_parser.h" 38 | #include "libs3/response_headers_handler.h" 39 | #include "libs3/util.h" 40 | 41 | // Describes a type of HTTP request (these are our supported HTTP "verbs") 42 | typedef enum 43 | { 44 | HttpRequestTypeGET, 45 | HttpRequestTypeHEAD, 46 | HttpRequestTypePUT, 47 | HttpRequestTypeCOPY, 48 | HttpRequestTypeDELETE, 49 | HttpRequestTypePOST, 50 | HttpRequestTypeInvalid 51 | } HttpRequestType; 52 | 53 | // This completely describes a request. A RequestParams is not required to be 54 | // allocated from the heap and its lifetime is not assumed to extend beyond 55 | // the lifetime of the function to which it has been passed. 56 | typedef struct RequestParams 57 | { 58 | // Request type, affects the HTTP verb used 59 | HttpRequestType httpRequestType; 60 | 61 | // Bucket context for request 62 | S3BucketContext bucketContext; 63 | 64 | // Key, if any 65 | const char* key; 66 | 67 | // Query params - ready to append to URI (i.e. ?p1=v1?p2=v2) 68 | const char* queryParams; 69 | 70 | // sub resource, like ?acl, ?location, ?torrent, ?logging 71 | const char* subResource; 72 | 73 | // If this is a copy operation, this gives the source bucket 74 | const char* copySourceBucketName; 75 | 76 | // If this is a copy operation, this gives the source key 77 | const char* copySourceKey; 78 | 79 | // Get conditions 80 | const S3GetConditions* getConditions; 81 | 82 | // Start byte 83 | size_t startByte; 84 | 85 | // Byte count 86 | size_t byteCount; 87 | 88 | // Put properties 89 | const S3PutProperties* putProperties; 90 | 91 | // Callback to be made when headers are available. Might not be called. 92 | S3ResponsePropertiesCallback* propertiesCallback; 93 | 94 | // Callback to be made to supply data to send to S3. Might not be called. 95 | S3PutObjectDataCallback* toS3Callback; 96 | 97 | // Number of bytes total that readCallback will supply 98 | int64_t toS3CallbackTotalSize; 99 | 100 | // Callback to be made that supplies data read from S3. 101 | // Might not be called. 102 | S3GetObjectDataCallback* fromS3Callback; 103 | 104 | // Callback to be made when request is complete. This will *always* be 105 | // called. 106 | S3ResponseCompleteCallback* completeCallback; 107 | 108 | // Data passed to the callbacks 109 | void* callbackData; 110 | 111 | // Request timeout. If 0, no timeout will be enforced 112 | int timeoutMs; 113 | 114 | /** 115 | * This optional field sets the list of object attributes for GetObjectAttributes. 116 | **/ 117 | const char* xAmzObjectAttributes; 118 | } RequestParams; 119 | 120 | // This is the stuff associated with a request that needs to be on the heap 121 | // (and thus live while a curl_multi is in use). 122 | typedef struct Request 123 | { 124 | // These put the request on a doubly-linked list of requests in a 125 | // request context, *if* the request is in a request context (else these 126 | // will both be 0) 127 | struct Request *prev, *next; 128 | 129 | // The status of this Request, as will be reported to the user via the 130 | // complete callback 131 | S3Status status; 132 | 133 | // The HTTP code returned by the S3 server, if it is known. Would rather 134 | // not have to keep track of this but S3 doesn't always indicate its 135 | // errors the same way 136 | int httpResponseCode; 137 | 138 | // The HTTP headers to use for the curl request 139 | struct curl_slist* headers; 140 | 141 | // The CURL structure driving the request 142 | CURL* curl; 143 | 144 | // libcurl requires that the uri be stored outside of the curl handle 145 | char uri[MAX_URI_SIZE + 1]; 146 | 147 | // Callback to be made when headers are available. Might not be called. 148 | S3ResponsePropertiesCallback* propertiesCallback; 149 | 150 | // Callback to be made to supply data to send to S3. Might not be called. 151 | S3PutObjectDataCallback* toS3Callback; 152 | 153 | // Number of bytes total that readCallback has left to supply 154 | int64_t toS3CallbackBytesRemaining; 155 | 156 | // Callback to be made that supplies data read from S3. 157 | // Might not be called. 158 | S3GetObjectDataCallback* fromS3Callback; 159 | 160 | // Callback to be made when request is complete. This will *always* be 161 | // called. 162 | S3ResponseCompleteCallback* completeCallback; 163 | 164 | // Data passed to the callbacks 165 | void* callbackData; 166 | 167 | // Handler of response headers 168 | ResponseHeadersHandler responseHeadersHandler; 169 | 170 | // This is set to nonzero after the properties callback has been made 171 | int propertiesCallbackMade; 172 | 173 | // Parser of errors 174 | ErrorParser errorParser; 175 | } Request; 176 | 177 | // Request functions 178 | // ---------------------------------------------------------------------------- 179 | 180 | // Initialize the API 181 | S3Status request_api_initialize(const char* userAgentInfo, int flags, const char* hostName); 182 | 183 | // Deinitialize the API 184 | void request_api_deinitialize(); 185 | 186 | // Perform a request; if context is 0, performs the request immediately; 187 | // otherwise, sets it up to be performed by context. 188 | void request_perform(const RequestParams* params, S3RequestContext* context); 189 | 190 | // Called by the internal request code or internal request context code when a 191 | // curl has finished the request 192 | void request_finish(Request* request); 193 | 194 | // Convert a CURLE code to an S3Status 195 | S3Status request_curl_code_to_status(CURLcode code); 196 | 197 | #endif /* REQUEST_H */ 198 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12.0 FATAL_ERROR) 2 | # CURL::libcurl and LibXml2::LibXml2 targets 3 | 4 | set(IRODS_MINIMUM_VERSION "4.90.0") 5 | set(IRODS_MAXIMUM_VERSION "6.0.0") 6 | find_package(IRODS "${IRODS_MINIMUM_VERSION}...<${IRODS_MAXIMUM_VERSION}" REQUIRED) 7 | 8 | set(IRODS_PLUGIN_VERSION "5.0.1") 9 | 10 | set(IRODS_PACKAGE_REVISION "0") 11 | 12 | include(IrodsCXXCompiler) 13 | set(CMAKE_CXX_STANDARD ${IRODS_CXX_STANDARD}) 14 | set(CMAKE_EXE_LINKER_FLAGS_INIT "-Wl,--export-dynamic -Wl,--enable-new-dtags -Wl,--as-needed -Wl,-z,defs") 15 | set(CMAKE_MODULE_LINKER_FLAGS_INIT "-Wl,--enable-new-dtags -Wl,--as-needed -Wl,-z,defs") 16 | set(CMAKE_EXE_LINKER_FLAGS_RELEASE_INIT "-Wl,--gc-sections -Wl,-z,combreloc") 17 | set(CMAKE_MODULE_LINKER_FLAGS_RELEASE_INIT "-Wl,--gc-sections -Wl,-z,combreloc") 18 | include(IrodsRunpathDefaults) 19 | 20 | project(irods_resource_plugin-s3 21 | VERSION "${IRODS_PLUGIN_VERSION}" 22 | LANGUAGES C CXX) 23 | 24 | include("${IRODS_TARGETS_PATH}") 25 | 26 | include(GNUInstallDirs) 27 | 28 | if (NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) 29 | set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build {Debug, Release}." FORCE) 30 | message(STATUS "Setting unspecified CMAKE_BUILD_TYPE to '${CMAKE_BUILD_TYPE}'") 31 | endif() 32 | 33 | include(CheckCCompilerFlag) 34 | include(CheckCXXCompilerFlag) 35 | 36 | if (CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") 37 | set(IRODS_BUILD_WITH_WERROR_DEFAULT ON) 38 | else() 39 | set(IRODS_BUILD_WITH_WERROR_DEFAULT OFF) 40 | endif() 41 | 42 | set(IRODS_BUILD_WITH_WERROR ${IRODS_BUILD_WITH_WERROR_DEFAULT} CACHE BOOL "Choose whether to compile with -Werror.") 43 | 44 | if (IRODS_BUILD_WITH_WERROR) 45 | add_compile_options(-Werror) 46 | endif() 47 | 48 | add_compile_options(-Wall -Wextra) 49 | 50 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 51 | add_compile_options($<$:-fpermissive>) 52 | endif() 53 | 54 | # Variable length arrays can crash clang (#2156), so let's turn on warnings for them. 55 | # -Wvla is not enabled by -Wall or -Wextra, so we have to do it manually. 56 | check_c_compiler_flag(-Wvla can_use_wvla_c) 57 | check_cxx_compiler_flag(-Wvla can_use_wvla_cxx) 58 | if (can_use_wvla_c) 59 | add_compile_options($<$:-Wvla>) 60 | endif() 61 | if (can_use_wvla_cxx) 62 | add_compile_options($<$:-Wvla>) 63 | endif() 64 | 65 | if (NOT DEFINED THREADS_PREFER_PTHREAD_FLAG) 66 | set(THREADS_PREFER_PTHREAD_FLAG TRUE) 67 | endif() 68 | find_package(Threads REQUIRED) 69 | find_package(nlohmann_json "3.6.1" REQUIRED) 70 | find_package(fmt "8.1.1" REQUIRED) 71 | find_package(CURL REQUIRED) 72 | find_package(OpenSSL REQUIRED) 73 | find_package(LibXml2 REQUIRED) 74 | include(ObjectTargetHelpers) 75 | 76 | add_subdirectory(libs3) 77 | add_subdirectory(s3_transport) 78 | add_subdirectory(s3_resource) 79 | add_subdirectory(unit_tests) 80 | 81 | add_library(irods_s3_plugin MODULE) 82 | target_link_objects( 83 | irods_s3_plugin 84 | PRIVATE 85 | libs3_obj 86 | s3_transport_obj 87 | s3_resource_obj 88 | ) 89 | set_property(TARGET irods_s3_plugin PROPERTY OUTPUT_NAME s3) 90 | 91 | install( 92 | TARGETS 93 | irods_s3_plugin 94 | LIBRARY 95 | DESTINATION "${IRODS_PLUGINS_DIRECTORY}/resources" 96 | ) 97 | 98 | install( 99 | FILES 100 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3.py" 101 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_minio.py" 102 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_ceph.py" 103 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_gcs.py" 104 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/test_irods_resource_plugin_s3_fujifilm.py" 105 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/resource_suite_s3_nocache.py" 106 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/resource_suite_s3_cache.py" 107 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/s3plugin_lib.py" 108 | "${CMAKE_CURRENT_SOURCE_DIR}/packaging/s3_prc_upload_issues_2260_2261.py" 109 | DESTINATION "${IRODS_HOME_DIRECTORY}/scripts/irods/test" 110 | PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ 111 | ) 112 | 113 | if (NOT CPACK_GENERATOR) 114 | set(CPACK_GENERATOR ${IRODS_CPACK_GENERATOR} CACHE STRING "CPack generator to use, e.g. {DEB, RPM, TGZ}." FORCE) 115 | message(STATUS "Setting unspecified CPACK_GENERATOR to ${CPACK_GENERATOR}. This is the correct setting for normal builds.") 116 | endif() 117 | 118 | include(IrodsCPackCommon) 119 | 120 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}") 121 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}${IRODS_PLUGINS_DIRECTORY}") 122 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}${IRODS_PLUGINS_DIRECTORY}/resources") 123 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}${IRODS_HOME_DIRECTORY}/scripts") 124 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}${IRODS_HOME_DIRECTORY}/scripts/irods") 125 | list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX}${IRODS_HOME_DIRECTORY}/scripts/irods/test") 126 | 127 | set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) 128 | set(CPACK_COMPONENT_INCLUDE_TOPLEVEL_DIRECTORY OFF) 129 | set(CPACK_COMPONENTS_GROUPING IGNORE) 130 | set(CPACK_PACKAGE_VERSION ${IRODS_PLUGIN_VERSION}) 131 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The integrated Rule-Oriented Data System") 132 | 133 | set(CPACK_DEB_COMPONENT_INSTALL OFF) 134 | set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) 135 | set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION ON) 136 | set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_SOURCE_DIR}/packaging/postinst;") 137 | set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) 138 | 139 | set(CPACK_RPM_COMPONENT_INSTALL OFF) 140 | set(CPACK_RPM_PACKAGE_LICENSE "LGPL-3.0-or-later OR GPL-2.0-or-later WITH cryptsetup-OpenSSL-exception") 141 | set(CPACK_RPM_PACKAGE_AUTOREQ 0) 142 | set(CPACK_RPM_PACKAGE_AUTOPROV 0) 143 | set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/packaging/postinst") 144 | set(CPACK_RPM_FILE_NAME RPM-DEFAULT) 145 | 146 | set(CPACK_ARCHIVE_COMPONENT_INSTALL OFF) 147 | 148 | set(CPACK_DEBIAN_PACKAGE_NAME "irods-resource-plugin-s3") 149 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "irods-runtime (= ${IRODS_VERSION}), libxml2, libc6") 150 | 151 | get_filename_component(CURL_LIBRARY_REALPATH ${CURL_LIBRARY} REALPATH) 152 | get_filename_component(CURL_LIBRARY_REALNAME ${CURL_LIBRARY_REALPATH} NAME_WE) 153 | if (CURL_LIBRARY_REALNAME STREQUAL "libcurl-gnutls") 154 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libcurl3-gnutls") 155 | elseif (CURL_LIBRARY_REALNAME STREQUAL "libcurl-nss") 156 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libcurl3-nss") 157 | elseif (CURL_LIBRARY_REALNAME STREQUAL "libcurl") 158 | set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libcurl4") 159 | endif () 160 | 161 | set(CPACK_RPM_PACKAGE_NAME "irods-resource-plugin-s3") 162 | if (IRODS_LINUX_DISTRIBUTION_NAME STREQUAL "opensuse") 163 | set(CPACK_RPM_PACKAGE_REQUIRES "irods-runtime = ${IRODS_VERSION}, libcurl, libopenssl1_0_0") 164 | else() 165 | set(CPACK_RPM_PACKAGE_REQUIRES "irods-runtime = ${IRODS_VERSION}, libcurl, libxml2") 166 | endif() 167 | 168 | include(CPack) 169 | -------------------------------------------------------------------------------- /libs3/src/service.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * service.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "libs3/request.h" 38 | 39 | typedef struct XmlCallbackData 40 | { 41 | SimpleXml simpleXml; 42 | 43 | S3ResponsePropertiesCallback* responsePropertiesCallback; 44 | S3ListServiceCallback* listServiceCallback; 45 | S3ResponseCompleteCallback* responseCompleteCallback; 46 | void* callbackData; 47 | 48 | string_buffer(ownerId, 256); 49 | string_buffer(ownerDisplayName, 256); 50 | string_buffer(bucketName, 256); 51 | string_buffer(creationDate, 128); 52 | } XmlCallbackData; 53 | 54 | static S3Status xmlCallback(const char* elementPath, const char* data, int dataLen, void* callbackData) 55 | { 56 | XmlCallbackData* cbData = (XmlCallbackData*) callbackData; 57 | 58 | int fit; 59 | 60 | if (data) { 61 | if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/ID")) { 62 | string_buffer_append(cbData->ownerId, data, dataLen, fit); 63 | } 64 | else if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/DisplayName")) { 65 | string_buffer_append(cbData->ownerDisplayName, data, dataLen, fit); 66 | } 67 | else if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket/Name")) { 68 | string_buffer_append(cbData->bucketName, data, dataLen, fit); 69 | } 70 | else if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket/CreationDate")) { 71 | string_buffer_append(cbData->creationDate, data, dataLen, fit); 72 | } 73 | } 74 | else { 75 | if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket")) { 76 | // Parse date. Assume ISO-8601 date format. 77 | time_t creationDate = parseIso8601Time(cbData->creationDate); 78 | 79 | // Make the callback - a bucket just finished 80 | S3Status status = (*(cbData->listServiceCallback))( 81 | cbData->ownerId, cbData->ownerDisplayName, cbData->bucketName, creationDate, cbData->callbackData); 82 | 83 | string_buffer_initialize(cbData->bucketName); 84 | string_buffer_initialize(cbData->creationDate); 85 | 86 | return status; 87 | } 88 | } 89 | 90 | /* Avoid compiler error about variable set but not used */ 91 | (void) fit; 92 | 93 | return S3StatusOK; 94 | } 95 | 96 | static S3Status propertiesCallback(const S3ResponseProperties* responseProperties, void* callbackData) 97 | { 98 | XmlCallbackData* cbData = (XmlCallbackData*) callbackData; 99 | 100 | return (*(cbData->responsePropertiesCallback))(responseProperties, cbData->callbackData); 101 | } 102 | 103 | static S3Status dataCallback(int bufferSize, const char* buffer, void* callbackData) 104 | { 105 | XmlCallbackData* cbData = (XmlCallbackData*) callbackData; 106 | 107 | return simplexml_add(&(cbData->simpleXml), buffer, bufferSize); 108 | } 109 | 110 | static void completeCallback(S3Status requestStatus, const S3ErrorDetails* s3ErrorDetails, void* callbackData) 111 | { 112 | XmlCallbackData* cbData = (XmlCallbackData*) callbackData; 113 | 114 | (*(cbData->responseCompleteCallback))(requestStatus, s3ErrorDetails, cbData->callbackData); 115 | 116 | simplexml_deinitialize(&(cbData->simpleXml)); 117 | 118 | free(cbData); 119 | } 120 | 121 | void S3_list_service(S3Protocol protocol, 122 | S3STSDate stsDate, 123 | const char* accessKeyId, 124 | const char* secretAccessKey, 125 | const char* securityToken, 126 | const char* hostName, 127 | const char* authRegion, 128 | S3RequestContext* requestContext, 129 | int timeoutMs, 130 | const S3ListServiceHandler* handler, 131 | void* callbackData) 132 | { 133 | // Create and set up the callback data 134 | XmlCallbackData* data = (XmlCallbackData*) malloc(sizeof(XmlCallbackData)); 135 | if (!data) { 136 | (*(handler->responseHandler.completeCallback))(S3StatusOutOfMemory, 0, callbackData); 137 | return; 138 | } 139 | 140 | simplexml_initialize(&(data->simpleXml), &xmlCallback, data); 141 | 142 | data->responsePropertiesCallback = handler->responseHandler.propertiesCallback; 143 | data->listServiceCallback = handler->listServiceCallback; 144 | data->responseCompleteCallback = handler->responseHandler.completeCallback; 145 | data->callbackData = callbackData; 146 | 147 | string_buffer_initialize(data->ownerId); 148 | string_buffer_initialize(data->ownerDisplayName); 149 | string_buffer_initialize(data->bucketName); 150 | string_buffer_initialize(data->creationDate); 151 | 152 | // Set up the RequestParams 153 | RequestParams params = { 154 | HttpRequestTypeGET, // httpRequestType 155 | {hostName, // hostName 156 | 0, // bucketName 157 | protocol, // protocol 158 | S3UriStylePath, // uriStyle 159 | accessKeyId, // accessKeyId 160 | secretAccessKey, // secretAccessKey 161 | securityToken, // securityToken 162 | authRegion, // authRegion 163 | stsDate}, // stsDate 164 | 0, // key 165 | 0, // queryParams 166 | 0, // subResource 167 | 0, // copySourceBucketName 168 | 0, // copySourceKey 169 | 0, // getConditions 170 | 0, // startByte 171 | 0, // byteCount 172 | 0, // requestProperties 173 | &propertiesCallback, // propertiesCallback 174 | 0, // toS3Callback 175 | 0, // toS3CallbackTotalSize 176 | &dataCallback, // fromS3Callback 177 | &completeCallback, // completeCallback 178 | data, // callbackData 179 | timeoutMs, // timeoutMs 180 | 0 // xAmzObjectAttributes 181 | }; 182 | 183 | // Perform the request 184 | request_perform(¶ms, requestContext); 185 | } 186 | -------------------------------------------------------------------------------- /libs3/src/simplexml.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * simplexml.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include 35 | #include "libs3/simplexml.h" 36 | 37 | // Use libxml2 for parsing XML. XML is severely overused in modern 38 | // computing. It is useful for only a very small subset of tasks, but 39 | // software developers who don't know better and are afraid to go against the 40 | // grain use it for everything, and in most cases, it is completely 41 | // inappropriate. Usually, the document structure is severely under-specified 42 | // as well, as is the case with S3. We do our best by just caring about the 43 | // most important aspects of the S3 "XML document" responses: the elements and 44 | // their values. The SAX API (just about the lamest API ever devised and 45 | // proof that XML sucks - well, the real proof is how crappy all of the XML 46 | // parsing libraries are, including libxml2 - but I digress) is used here 47 | // because we don't need much from the parser and SAX is fast and low memory. 48 | // 49 | // Note that for simplicity we assume all ASCII here. No attempts are made to 50 | // detect non-ASCII sequences in utf-8 and convert them into ASCII in any way. 51 | // S3 appears to only use ASCII anyway. 52 | 53 | static xmlEntityPtr saxGetEntity(void* user_data, const xmlChar* name) 54 | { 55 | (void) user_data; 56 | 57 | return xmlGetPredefinedEntity(name); 58 | } 59 | 60 | static void saxStartElement(void* user_data, const xmlChar* nameUtf8, const xmlChar** attr) 61 | { 62 | (void) attr; 63 | 64 | SimpleXml* simpleXml = (SimpleXml*) user_data; 65 | 66 | if (simpleXml->status != S3StatusOK) { 67 | return; 68 | } 69 | 70 | // Assume that name has no non-ASCII in it 71 | char* name = (char*) nameUtf8; 72 | 73 | // Append the element to the element path 74 | int len = strlen(name); 75 | 76 | if ((simpleXml->elementPathLen + len + 1) >= (int) sizeof(simpleXml->elementPath)) { 77 | // Cannot handle this element, stop! 78 | simpleXml->status = S3StatusXmlParseFailure; 79 | return; 80 | } 81 | 82 | if (simpleXml->elementPathLen) { 83 | simpleXml->elementPath[simpleXml->elementPathLen++] = '/'; 84 | } 85 | strcpy(&(simpleXml->elementPath[simpleXml->elementPathLen]), name); 86 | simpleXml->elementPathLen += len; 87 | } 88 | 89 | static void saxEndElement(void* user_data, const xmlChar* name) 90 | { 91 | (void) name; 92 | 93 | SimpleXml* simpleXml = (SimpleXml*) user_data; 94 | 95 | if (simpleXml->status != S3StatusOK) { 96 | return; 97 | } 98 | 99 | // Call back with 0 data 100 | simpleXml->status = (*(simpleXml->callback))(simpleXml->elementPath, 0, 0, simpleXml->callbackData); 101 | 102 | while ((simpleXml->elementPathLen > 0) && (simpleXml->elementPath[simpleXml->elementPathLen] != '/')) { 103 | simpleXml->elementPathLen--; 104 | } 105 | 106 | simpleXml->elementPath[simpleXml->elementPathLen] = 0; 107 | } 108 | 109 | static void saxCharacters(void* user_data, const xmlChar* ch, int len) 110 | { 111 | SimpleXml* simpleXml = (SimpleXml*) user_data; 112 | 113 | if (simpleXml->status != S3StatusOK) { 114 | return; 115 | } 116 | 117 | simpleXml->status = (*(simpleXml->callback))(simpleXml->elementPath, (char*) ch, len, simpleXml->callbackData); 118 | } 119 | 120 | static void saxError(void* user_data, const char* msg, ...) 121 | { 122 | (void) msg; 123 | 124 | SimpleXml* simpleXml = (SimpleXml*) user_data; 125 | 126 | if (simpleXml->status != S3StatusOK) { 127 | return; 128 | } 129 | 130 | simpleXml->status = S3StatusXmlParseFailure; 131 | } 132 | 133 | static struct _xmlSAXHandler saxHandlerG = { 134 | 0, // internalSubsetSAXFunc 135 | 0, // isStandaloneSAXFunc 136 | 0, // hasInternalSubsetSAXFunc 137 | 0, // hasExternalSubsetSAXFunc 138 | 0, // resolveEntitySAXFunc 139 | &saxGetEntity, // getEntitySAXFunc 140 | 0, // entityDeclSAXFunc 141 | 0, // notationDeclSAXFunc 142 | 0, // attributeDeclSAXFunc 143 | 0, // elementDeclSAXFunc 144 | 0, // unparsedEntityDeclSAXFunc 145 | 0, // setDocumentLocatorSAXFunc 146 | 0, // startDocumentSAXFunc 147 | 0, // endDocumentSAXFunc 148 | &saxStartElement, // startElementSAXFunc 149 | &saxEndElement, // endElementSAXFunc 150 | 0, // referenceSAXFunc 151 | &saxCharacters, // charactersSAXFunc 152 | 0, // ignorableWhitespaceSAXFunc 153 | 0, // processingInstructionSAXFunc 154 | 0, // commentSAXFunc 155 | 0, // warningSAXFunc 156 | &saxError, // errorSAXFunc 157 | &saxError, // fatalErrorSAXFunc 158 | 0, // getParameterEntitySAXFunc 159 | &saxCharacters, // cdataBlockSAXFunc 160 | 0, // externalSubsetSAXFunc 161 | 0, // initialized 162 | 0, // _private 163 | 0, // startElementNsSAX2Func 164 | 0, // endElementNsSAX2Func 165 | 0 // xmlStructuredErrorFunc serror; 166 | }; 167 | 168 | void simplexml_initialize(SimpleXml* simpleXml, SimpleXmlCallback* callback, void* callbackData) 169 | { 170 | simpleXml->callback = callback; 171 | simpleXml->callbackData = callbackData; 172 | simpleXml->elementPathLen = 0; 173 | simpleXml->status = S3StatusOK; 174 | simpleXml->xmlParser = 0; 175 | } 176 | 177 | void simplexml_deinitialize(SimpleXml* simpleXml) 178 | { 179 | if (simpleXml->xmlParser) { 180 | xmlFreeParserCtxt(simpleXml->xmlParser); 181 | } 182 | } 183 | 184 | S3Status simplexml_add(SimpleXml* simpleXml, const char* data, int dataLen) 185 | { 186 | if (!simpleXml->xmlParser && (!(simpleXml->xmlParser = xmlCreatePushParserCtxt(&saxHandlerG, simpleXml, 0, 0, 0)))) 187 | { 188 | return S3StatusInternalError; 189 | } 190 | 191 | if (xmlParseChunk((xmlParserCtxtPtr) simpleXml->xmlParser, data, dataLen, 0)) { 192 | return S3StatusXmlParseFailure; 193 | } 194 | 195 | return simpleXml->status; 196 | } 197 | -------------------------------------------------------------------------------- /irods_consortium_continuous_integration_test_hook.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import glob 4 | import optparse 5 | import os 6 | import random 7 | import shutil 8 | import stat 9 | import string 10 | import subprocess 11 | import time 12 | import platform 13 | import distro 14 | import logging 15 | import sys 16 | 17 | import irods_python_ci_utilities 18 | 19 | def get_package_type(): 20 | log = logging.getLogger(__name__) 21 | distro_id = distro.id() 22 | log.debug('linux distribution detected: {0}'.format(distro_id)) 23 | if distro_id in ['debian', 'ubuntu']: 24 | pt = 'deb' 25 | elif distro_id in ['rocky', 'almalinux', 'centos', 'rhel', 'scientific', 'opensuse', 'sles']: 26 | pt = 'rpm' 27 | else: 28 | if platform.mac_ver()[0] != '': 29 | pt = 'osxpkg' 30 | else: 31 | pt = 'not_detected' 32 | log.debug('package type detected: {0}'.format(pt)) 33 | return pt 34 | 35 | def install_test_prerequisites(): 36 | distro_major_version = int(distro.major_version()) 37 | ub24_or_later = (distro.id() == "ubuntu" and distro_major_version >= 24) 38 | deb13_or_later = (distro.id() == 'debian' and distro_major_version >= 13) 39 | if not any([ub24_or_later, deb13_or_later]): 40 | irods_python_ci_utilities.subprocess_get_output(['sudo', 'python3', '-m', 'pip', 'install', '--upgrade', 'pip>=20.3.4'], check_rc=True) 41 | irods_python_ci_utilities.subprocess_get_output(['sudo', 'python3', '-m', 'pip', 'install', 'boto3', '--upgrade'], check_rc=True) 42 | 43 | # Minio 7.1.17 imports the annontations module which only exists in Python 3.7 and beyond. 44 | # For OS which default to Python 3.6, we have to install the previous version of Minio to avoid 45 | # compatibility issues. The --upgrade flag is ignored if "minio_version" results in a non-empty string. 46 | minio_version = '==7.1.16' if sys.hexversion < 0x030700F0 else '' 47 | irods_python_ci_utilities.subprocess_get_output(['sudo', 'python3', '-m', 'pip', 'install', 'minio' + minio_version, '--upgrade'], check_rc=True) 48 | 49 | # install PRC 50 | irods_python_ci_utilities.subprocess_get_output(['python3', '-m', 'pip', 'install', 'python-irodsclient'], check_rc=True) 51 | 52 | 53 | def download_and_start_minio_server(): 54 | path_to_minio = os.path.abspath('minio') 55 | 56 | # Only download the executable if it's not already here 57 | if not os.path.isfile(path_to_minio): 58 | if get_package_type() == 'deb': 59 | minio_package_file = 'minio_20220526054841.0.0_amd64.deb' 60 | subprocess.check_output(['wget', '-q', '--no-check-certificate', 'https://dl.min.io/server/minio/release/linux-amd64/archive/{}'.format(minio_package_file)]) 61 | subprocess.check_output(['dpkg', '-i', minio_package_file]) 62 | elif get_package_type() == 'rpm': 63 | minio_package_file = 'minio-20220526054841.0.0.x86_64.rpm' 64 | subprocess.check_output(['wget', '-q', '--no-check-certificate', 'https://dl.min.io/server/minio/release/linux-amd64/archive/{}'.format(minio_package_file)]) 65 | subprocess.check_output(['rpm', '--force', '-i', minio_package_file]) 66 | else: 67 | raise Exception('Unknown or invalid OS type. Can not install minio.') 68 | 69 | path_to_minio = 'minio' 70 | 71 | root_username = ''.join(random.choice(string.ascii_letters) for i in list(range(10))) 72 | root_password = ''.join(random.choice(string.ascii_letters) for i in list(range(10))) 73 | keypair_path = '/var/lib/irods/minio.keypair' 74 | 75 | with open(keypair_path, 'w') as f: 76 | f.write('%s\n' % root_username) 77 | f.write('%s\n' % root_password) 78 | 79 | shutil.chown(keypair_path, user='irods', group='irods') 80 | 81 | os.environ['MINIO_ROOT_USER'] = root_username 82 | os.environ['MINIO_ROOT_PASSWORD'] = root_password 83 | os.environ['MINIO_CI_CD'] = '1' 84 | 85 | minio_region_name_key = 'MINIO_REGION_NAME' 86 | 87 | proc_infos = [ 88 | { 89 | 'address': '9000', 90 | 'console_address': '9002', 91 | minio_region_name_key: None 92 | }, 93 | { 94 | 'address': '9001', 95 | 'console_address': '9003', 96 | minio_region_name_key: 'eu-central-1' 97 | } 98 | ] 99 | 100 | procs = list() 101 | 102 | for p in proc_infos: 103 | if p[minio_region_name_key] is not None: 104 | os.environ[minio_region_name_key] = p[minio_region_name_key] 105 | elif minio_region_name_key in os.environ: 106 | del os.environ[minio_region_name_key] 107 | 108 | procs.append(subprocess.Popen([path_to_minio, 'server', 109 | '--address', ':' + p["address"], 110 | '--console-address', ':' + p["console_address"], 111 | '/data_%s' % p[minio_region_name_key]])) 112 | 113 | return procs 114 | 115 | 116 | def main(): 117 | parser = optparse.OptionParser() 118 | parser.add_option('--output_root_directory') 119 | parser.add_option('--built_packages_root_directory') 120 | parser.add_option('--test', metavar='dotted name') 121 | parser.add_option('--skip-setup', action='store_false', dest='do_setup', default=True) 122 | parser.add_option('--teardown-minio', action='store_true', dest='do_teardown', default=False) 123 | options, _ = parser.parse_args() 124 | 125 | if not options.do_setup and options.do_teardown: 126 | # TODO: if minio-client can shut down the server, this will not be true 127 | print('--skip-setup and --teardown-minio are incompatible') 128 | exit(1) 129 | 130 | built_packages_root_directory = options.built_packages_root_directory 131 | package_suffix = irods_python_ci_utilities.get_package_suffix() 132 | os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) 133 | 134 | if options.do_setup: 135 | irods_python_ci_utilities.install_os_packages_from_files( 136 | glob.glob(os.path.join(os_specific_directory, 137 | f'irods-resource-plugin-s3*.{package_suffix}') 138 | ) 139 | ) 140 | 141 | install_test_prerequisites() 142 | 143 | minio_processes = download_and_start_minio_server() 144 | 145 | test = options.test or 'test_irods_resource_plugin_s3_minio' 146 | 147 | try: 148 | test_output_file = 'log/test_output.log' 149 | irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 150 | f'python3 scripts/run_tests.py --xml_output --run_s {test} 2>&1 | tee {test_output_file}; exit $PIPESTATUS'], 151 | check_rc=True) 152 | 153 | if options.do_teardown: 154 | # TODO: investigate mc admin service stop 155 | # Get minio client 156 | # wget https://dl.min.io/client/mc/release/linux-amd64/mc 157 | # chmod +x mc 158 | # sudo cp mc /usr/bin 159 | for m in minio_processes: 160 | m.terminate() 161 | 162 | finally: 163 | output_root_directory = options.output_root_directory 164 | if output_root_directory: 165 | irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) 166 | shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) 167 | shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports')) 168 | 169 | 170 | if __name__ == '__main__': 171 | main() 172 | -------------------------------------------------------------------------------- /COPYING-LGPLv3: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /libs3/COPYING-LGPLv3: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /packaging/test_irods_resource_plugin_s3.py: -------------------------------------------------------------------------------- 1 | 2 | import datetime 3 | import os 4 | import platform 5 | import random 6 | import re 7 | import shutil 8 | import string 9 | import subprocess 10 | import urllib3 11 | 12 | from .resource_suite_s3_nocache import Test_S3_NoCache_Base 13 | from .resource_suite_s3_nocache import Test_S3_NoCache_Large_File_Tests_Base 14 | from .resource_suite_s3_nocache import Test_S3_NoCache_Glacier_Base 15 | from .resource_suite_s3_nocache import Test_S3_NoCache_MPU_Disabled_Base 16 | from .resource_suite_s3_nocache import Test_S3_NoCache_Decoupled_Base 17 | from .resource_suite_s3_cache import Test_S3_Cache_Base 18 | from .resource_suite_s3_cache import Test_S3_Cache_Glacier_Base 19 | 20 | import sys 21 | if sys.version_info >= (2,7): 22 | import unittest 23 | else: 24 | import unittest2 as unittest 25 | 26 | from .. import lib 27 | from . import session 28 | from ..configuration import IrodsConfig 29 | from .resource_suite import ResourceSuite 30 | from .test_chunkydevtest import ChunkyDevTest 31 | 32 | 33 | class Test_Compound_With_S3_Resource(Test_S3_Cache_Base, unittest.TestCase): 34 | def __init__(self, *args, **kwargs): 35 | """Set up the test.""" 36 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 37 | self.archive_naming_policy='decoupled' 38 | self.s3stsdate='' 39 | self.s3region='us-east-1' 40 | self.s3endPoint='s3.amazonaws.com' 41 | self.s3sse = 0 # server side encryption 42 | super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs) 43 | 44 | class Test_Compound_With_S3_Resource_EU_Central_1(Test_S3_Cache_Base, unittest.TestCase): 45 | ''' 46 | This also tests signature V4 with the x-amz-date header. 47 | ''' 48 | def __init__(self, *args, **kwargs): 49 | """Set up the test.""" 50 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 51 | self.archive_naming_policy='decoupled' 52 | self.s3stsdate='' 53 | self.s3region='eu-central-1' 54 | self.s3endPoint='s3.eu-central-1.amazonaws.com' 55 | super(Test_Compound_With_S3_Resource_EU_Central_1, self).__init__(*args, **kwargs) 56 | 57 | class Test_Compound_With_S3_Resource_STSDate_Header(Test_S3_Cache_Base, unittest.TestCase): 58 | def __init__(self, *args, **kwargs): 59 | """Set up the test.""" 60 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 61 | self.archive_naming_policy='decoupled' 62 | self.s3stsdate='date' 63 | self.s3region='us-east-1' 64 | self.s3endPoint='s3.amazonaws.com' 65 | super(Test_Compound_With_S3_Resource_STSDate_Header, self).__init__(*args, **kwargs) 66 | 67 | class Test_Compound_With_S3_Resource_STSDate_Header_V4(Test_S3_Cache_Base, unittest.TestCase): 68 | def __init__(self, *args, **kwargs): 69 | """Set up the test.""" 70 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 71 | self.archive_naming_policy='decoupled' 72 | self.s3stsdate='date' 73 | self.s3region='us-east-1' 74 | self.s3endPoint='s3.amazonaws.com' 75 | super(Test_Compound_With_S3_Resource_STSDate_Header_V4, self).__init__(*args, **kwargs) 76 | 77 | class Test_Compound_With_S3_Resource_V4_SSE(Test_S3_Cache_Base, unittest.TestCase): 78 | def __init__(self, *args, **kwargs): 79 | """Set up the test.""" 80 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 81 | self.archive_naming_policy='decoupled' 82 | self.s3stsdate='' 83 | self.s3sse=1 84 | self.s3region='us-east-1' 85 | self.s3endPoint='s3.amazonaws.com' 86 | super(Test_Compound_With_S3_Resource_V4_SSE, self).__init__(*args, **kwargs) 87 | 88 | class Test_S3_NoCache_V4(Test_S3_NoCache_Large_File_Tests_Base, unittest.TestCase): 89 | 90 | def __init__(self, *args, **kwargs): 91 | """Set up the test.""" 92 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 93 | self.s3region='us-east-1' 94 | self.s3endPoint='s3.amazonaws.com' 95 | self.s3EnableMPU=1 96 | super(Test_S3_NoCache_V4, self).__init__(*args, **kwargs) 97 | 98 | class Test_S3_NoCache_EU_Central_1(Test_S3_NoCache_Base, unittest.TestCase): 99 | ''' 100 | This also tests signature V4 with the x-amz-date header. 101 | ''' 102 | def __init__(self, *args, **kwargs): 103 | """Set up the test.""" 104 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 105 | self.s3region='eu-central-1' 106 | self.s3endPoint='s3.eu-central-1.amazonaws.com' 107 | self.s3EnableMPU=1 108 | super(Test_S3_NoCache_EU_Central_1, self).__init__(*args, **kwargs) 109 | 110 | class Test_S3_NoCache_MPU_Disabled(Test_S3_NoCache_MPU_Disabled_Base, unittest.TestCase): 111 | def __init__(self, *args, **kwargs): 112 | """Set up the test.""" 113 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 114 | self.s3region='us-east-1' 115 | self.s3endPoint='s3.amazonaws.com' 116 | self.s3EnableMPU=0 117 | super(Test_S3_NoCache_MPU_Disabled, self).__init__(*args, **kwargs) 118 | 119 | class Test_S3_NoCache_SSE(Test_S3_NoCache_Base, unittest.TestCase): 120 | def __init__(self, *args, **kwargs): 121 | """Set up the test.""" 122 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 123 | self.s3region='us-east-1' 124 | self.s3endPoint='s3.amazonaws.com' 125 | self.s3sse=1 126 | self.s3EnableMPU=1 127 | super(Test_S3_NoCache_SSE, self).__init__(*args, **kwargs) 128 | 129 | class Test_S3_NoCache_Decoupled(Test_S3_NoCache_Decoupled_Base, unittest.TestCase): 130 | def __init__(self, *args, **kwargs): 131 | """Set up the test.""" 132 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 133 | self.s3region='us-east-1' 134 | self.s3endPoint='s3.amazonaws.com' 135 | self.s3EnableMPU=0 136 | self.archive_naming_policy = 'decoupled' 137 | super(Test_S3_NoCache_Decoupled, self).__init__(*args, **kwargs) 138 | 139 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 140 | def test_iget_with_stale_replica(self): # formerly known as 'dirty' 141 | pass 142 | 143 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 144 | def test_irepl_with_purgec(self): 145 | pass 146 | 147 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 148 | def test_put_get_small_file_in_repl_node(self): 149 | pass 150 | 151 | @unittest.skipIf(True, 'test does not work in decoupled because we are using same bucket for multiple resources') 152 | def test_put_get_large_file_in_repl_node(self): 153 | pass 154 | 155 | @unittest.skip('test does not work in decoupled because we are using same bucket for multiple resources') 156 | def test_s3_in_replication_node__issues_2102_2122(self): 157 | pass 158 | 159 | class Test_S3_NoCache_Glacier(Test_S3_NoCache_Glacier_Base, unittest.TestCase): 160 | 161 | def __init__(self, *args, **kwargs): 162 | """Set up the test.""" 163 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 164 | self.s3region='us-east-1' 165 | self.s3endPoint='s3.amazonaws.com' 166 | self.s3EnableMPU=1 167 | super(Test_S3_NoCache_Glacier, self).__init__(*args, **kwargs) 168 | 169 | class Test_S3_Cache_Glacier(Test_S3_Cache_Glacier_Base, unittest.TestCase): 170 | 171 | def __init__(self, *args, **kwargs): 172 | """Set up the test.""" 173 | self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair' 174 | self.s3region='us-east-1' 175 | self.s3endPoint='s3.amazonaws.com' 176 | self.s3EnableMPU=1 177 | self.s3stsdate='' 178 | super(Test_S3_Cache_Glacier, self).__init__(*args, **kwargs) 179 | -------------------------------------------------------------------------------- /libs3/src/response_headers_handler.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * response_headers_handler.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include 35 | #include 36 | #include "libs3/response_headers_handler.h" 37 | 38 | void response_headers_handler_initialize(ResponseHeadersHandler* handler) 39 | { 40 | handler->responseProperties.requestId = 0; 41 | handler->responseProperties.requestId2 = 0; 42 | handler->responseProperties.contentType = 0; 43 | handler->responseProperties.contentLength = 0; 44 | handler->responseProperties.server = 0; 45 | handler->responseProperties.eTag = 0; 46 | handler->responseProperties.lastModified = -1; 47 | handler->responseProperties.metaDataCount = 0; 48 | handler->responseProperties.metaData = 0; 49 | handler->responseProperties.usesServerSideEncryption = 0; 50 | handler->done = 0; 51 | handler->responseProperties.xAmzStorageClass = 0; 52 | handler->responseProperties.xAmzRestore = 0; 53 | string_multibuffer_initialize(handler->responsePropertyStrings); 54 | string_multibuffer_initialize(handler->responseMetaDataStrings); 55 | } 56 | 57 | void response_headers_handler_add(ResponseHeadersHandler* handler, char* header, int len) 58 | { 59 | S3ResponseProperties* responseProperties = &(handler->responseProperties); 60 | char* end = &(header[len]); 61 | 62 | // Curl might call back the header function after the body has been 63 | // received, for 'chunked encoded' contents. We don't handle this as of 64 | // yet, and it's not clear that it would ever be useful. 65 | if (handler->done) { 66 | return; 67 | } 68 | 69 | // If we've already filled up the response headers, ignore this data. 70 | // This sucks, but it shouldn't happen - S3 should not be sending back 71 | // really long headers. 72 | if (handler->responsePropertyStringsSize == (sizeof(handler->responsePropertyStrings) - 1)) { 73 | return; 74 | } 75 | 76 | // It should not be possible to have a header line less than 3 long 77 | if (len < 3) { 78 | return; 79 | } 80 | 81 | // Skip whitespace at beginning of header; there never should be any, 82 | // but just to be safe 83 | while (is_blank(*header)) { 84 | header++; 85 | } 86 | 87 | // The header must end in \r\n, so skip back over it, and also over any 88 | // trailing whitespace 89 | end -= 3; 90 | while ((end > header) && is_blank(*end)) { 91 | end--; 92 | } 93 | if (!is_blank(*end)) { 94 | end++; 95 | } 96 | 97 | if (end == header) { 98 | // totally bogus 99 | return; 100 | } 101 | 102 | *end = 0; 103 | 104 | // Find the colon to split the header up 105 | char* c = header; 106 | while (*c && (*c != ':')) { 107 | c++; 108 | } 109 | 110 | int namelen = c - header; 111 | 112 | // Now walk c past the colon 113 | c++; 114 | // Now skip whitespace to the beginning of the value 115 | while (is_blank(*c)) { 116 | c++; 117 | } 118 | 119 | int valuelen = (end - c) + 1, fit; 120 | 121 | if (!strncasecmp(header, "x-amz-request-id", namelen)) { 122 | responseProperties->requestId = string_multibuffer_current(handler->responsePropertyStrings); 123 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 124 | } 125 | else if (!strncasecmp(header, "x-amz-id-2", namelen)) { 126 | responseProperties->requestId2 = string_multibuffer_current(handler->responsePropertyStrings); 127 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 128 | } 129 | else if (!strncasecmp(header, "x-amz-storage-class", namelen) || 130 | !strncasecmp(header, "x-goog-storage-class", namelen)) 131 | { 132 | responseProperties->xAmzStorageClass = string_multibuffer_current(handler->responsePropertyStrings); 133 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 134 | } 135 | else if (!strncasecmp(header, "x-amz-restore", namelen) || !strncasecmp(header, "x-goog-restore", namelen)) { 136 | responseProperties->xAmzRestore = string_multibuffer_current(handler->responsePropertyStrings); 137 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 138 | } 139 | else if (!strncasecmp(header, "Content-Type", namelen)) { 140 | responseProperties->contentType = string_multibuffer_current(handler->responsePropertyStrings); 141 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 142 | } 143 | else if (!strncasecmp(header, "Content-Length", namelen)) { 144 | handler->responseProperties.contentLength = 0; 145 | while (*c) { 146 | handler->responseProperties.contentLength *= 10; 147 | handler->responseProperties.contentLength += (*c++ - '0'); 148 | } 149 | } 150 | else if (!strncasecmp(header, "Server", namelen)) { 151 | responseProperties->server = string_multibuffer_current(handler->responsePropertyStrings); 152 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 153 | } 154 | else if ((!strncasecmp(header, "ETag", namelen)) || (!strncasecmp(header, "Etag", namelen))) 155 | { // some servers reply with Etag header 156 | responseProperties->eTag = string_multibuffer_current(handler->responsePropertyStrings); 157 | string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); 158 | } 159 | else if (!strncasecmp(header, S3_METADATA_HEADER_NAME_PREFIX, sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1)) { 160 | // Make sure there is room for another x-amz-meta header 161 | if (handler->responseProperties.metaDataCount == sizeof(handler->responseMetaData)) { 162 | return; 163 | } 164 | // Copy the name in 165 | char* metaName = &(header[sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1]); 166 | int metaNameLen = (namelen - (sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1)); 167 | char* copiedName = string_multibuffer_current(handler->responseMetaDataStrings); 168 | string_multibuffer_add(handler->responseMetaDataStrings, metaName, metaNameLen, fit); 169 | if (!fit) { 170 | return; 171 | } 172 | 173 | // Copy the value in 174 | char* copiedValue = string_multibuffer_current(handler->responseMetaDataStrings); 175 | string_multibuffer_add(handler->responseMetaDataStrings, c, valuelen, fit); 176 | if (!fit) { 177 | return; 178 | } 179 | 180 | if (!handler->responseProperties.metaDataCount) { 181 | handler->responseProperties.metaData = handler->responseMetaData; 182 | } 183 | 184 | S3NameValue* metaHeader = &(handler->responseMetaData[handler->responseProperties.metaDataCount++]); 185 | metaHeader->name = copiedName; 186 | metaHeader->value = copiedValue; 187 | } 188 | else if (!strncasecmp(header, "x-amz-server-side-encryption", namelen)) { 189 | if (!strncmp(c, "AES256", sizeof("AES256") - 1)) { 190 | responseProperties->usesServerSideEncryption = 1; 191 | } 192 | // Ignore other values - only AES256 is expected, anything else is 193 | // assumed to be "None" or some other value indicating no server-side 194 | // encryption 195 | } 196 | } 197 | 198 | void response_headers_handler_done(ResponseHeadersHandler* handler, CURL* curl) 199 | { 200 | // Now get the last modification time from curl, since it's easiest to let 201 | // curl parse it 202 | time_t lastModified; 203 | if (curl_easy_getinfo(curl, CURLINFO_FILETIME, &lastModified) == CURLE_OK) { 204 | handler->responseProperties.lastModified = lastModified; 205 | } 206 | 207 | handler->done = 1; 208 | } 209 | -------------------------------------------------------------------------------- /libs3/src/request_context.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * request_context.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include 35 | #include 36 | #include "libs3/request.h" 37 | #include "libs3/request_context.h" 38 | 39 | S3Status S3_create_request_context_ex(S3RequestContext** requestContextReturn, 40 | CURLM* curlm, 41 | S3SetupCurlCallback setupCurlCallback, 42 | void* setupCurlCallbackData) 43 | { 44 | *requestContextReturn = (S3RequestContext*) malloc(sizeof(S3RequestContext)); 45 | 46 | if (!*requestContextReturn) { 47 | return S3StatusOutOfMemory; 48 | } 49 | 50 | if (curlm) { 51 | (*requestContextReturn)->curlm = curlm; 52 | (*requestContextReturn)->curl_mode = S3CurlModeMultiSocket; 53 | } 54 | else { 55 | if (!((*requestContextReturn)->curlm = curl_multi_init())) { 56 | free(*requestContextReturn); 57 | return S3StatusOutOfMemory; 58 | } 59 | 60 | (*requestContextReturn)->curl_mode = S3CurlModeMultiPerform; 61 | } 62 | 63 | (*requestContextReturn)->requests = 0; 64 | (*requestContextReturn)->verifyPeer = 0; 65 | (*requestContextReturn)->verifyPeerSet = 0; 66 | (*requestContextReturn)->setupCurlCallback = setupCurlCallback; 67 | (*requestContextReturn)->setupCurlCallbackData = setupCurlCallbackData; 68 | 69 | return S3StatusOK; 70 | } 71 | 72 | S3Status S3_create_request_context(S3RequestContext** requestContextReturn) 73 | { 74 | return S3_create_request_context_ex(requestContextReturn, NULL, NULL, NULL); 75 | } 76 | 77 | void S3_destroy_request_context(S3RequestContext* requestContext) 78 | { 79 | // For each request in the context, remove curl handle, call back its done 80 | // method with 'interrupted' status 81 | Request *r = requestContext->requests, *rFirst = r; 82 | 83 | if (r) 84 | do { 85 | r->status = S3StatusInterrupted; 86 | // remove easy handle from a multi session 87 | curl_multi_remove_handle(requestContext->curlm, r->curl); 88 | Request* rNext = r->next; 89 | request_finish(r); 90 | r = rNext; 91 | } 92 | while (r != rFirst); 93 | 94 | if (requestContext->curl_mode == S3CurlModeMultiPerform) 95 | curl_multi_cleanup(requestContext->curlm); 96 | 97 | free(requestContext); 98 | } 99 | 100 | S3Status S3_runall_request_context(S3RequestContext* requestContext) 101 | { 102 | int requestsRemaining; 103 | do { 104 | fd_set readfds, writefds, exceptfds; 105 | FD_ZERO(&readfds); 106 | FD_ZERO(&writefds); 107 | FD_ZERO(&exceptfds); 108 | int maxfd; 109 | S3Status status = S3_get_request_context_fdsets(requestContext, &readfds, &writefds, &exceptfds, &maxfd); 110 | if (status != S3StatusOK) { 111 | return status; 112 | } 113 | // curl will return -1 if it hasn't even created any fds yet because 114 | // none of the connections have started yet. In this case, don't 115 | // do the select at all, because it will wait forever; instead, just 116 | // skip it and go straight to running the underlying CURL handles 117 | if (maxfd != -1) { 118 | int64_t timeout = S3_get_request_context_timeout(requestContext); 119 | struct timeval tv = {timeout / 1000, (timeout % 1000) * 1000}; 120 | select(maxfd + 1, &readfds, &writefds, &exceptfds, (timeout == -1) ? 0 : &tv); 121 | } 122 | status = S3_runonce_request_context(requestContext, &requestsRemaining); 123 | if (status != S3StatusOK) { 124 | return status; 125 | } 126 | } 127 | while (requestsRemaining); 128 | 129 | return S3StatusOK; 130 | } 131 | 132 | static S3Status process_request_context(S3RequestContext* requestContext, int* retry) 133 | { 134 | CURLMsg* msg; 135 | int junk; 136 | 137 | *retry = 0; 138 | 139 | while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) { 140 | if (msg->msg != CURLMSG_DONE) { 141 | return S3StatusInternalError; 142 | } 143 | Request* request; 144 | if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**) (char*) &request) != CURLE_OK) { 145 | return S3StatusInternalError; 146 | } 147 | // Remove the request from the list of requests 148 | if (request->prev == request->next) { 149 | // It was the only one on the list 150 | requestContext->requests = 0; 151 | } 152 | else { 153 | // It doesn't matter what the order of them are, so just in 154 | // case request was at the head of the list, put the one after 155 | // request to the head of the list 156 | requestContext->requests = request->next; 157 | request->prev->next = request->next; 158 | request->next->prev = request->prev; 159 | } 160 | if ((msg->data.result != CURLE_OK) && (request->status == S3StatusOK)) { 161 | request->status = request_curl_code_to_status(msg->data.result); 162 | } 163 | if (curl_multi_remove_handle(requestContext->curlm, msg->easy_handle) != CURLM_OK) { 164 | return S3StatusInternalError; 165 | } 166 | // Finish the request, ensuring that all callbacks have been made, 167 | // and also releases the request 168 | request_finish(request); 169 | // Now, since a callback was made, there may be new requests 170 | // queued up to be performed immediately, so do so 171 | *retry = 1; 172 | } 173 | 174 | return S3StatusOK; 175 | } 176 | 177 | S3Status S3_runonce_request_context(S3RequestContext* requestContext, int* requestsRemainingReturn) 178 | { 179 | S3Status s3_status; 180 | CURLMcode status; 181 | int retry; 182 | 183 | do { 184 | status = curl_multi_perform(requestContext->curlm, requestsRemainingReturn); 185 | 186 | switch (status) { 187 | case CURLM_OK: 188 | case CURLM_CALL_MULTI_PERFORM: 189 | break; 190 | case CURLM_OUT_OF_MEMORY: 191 | return S3StatusOutOfMemory; 192 | default: 193 | return S3StatusInternalError; 194 | } 195 | 196 | s3_status = process_request_context(requestContext, &retry); 197 | } 198 | while (s3_status == S3StatusOK && (status == CURLM_CALL_MULTI_PERFORM || retry)); 199 | 200 | return s3_status; 201 | } 202 | 203 | S3Status S3_process_request_context(S3RequestContext* requestContext) 204 | { 205 | int retry; 206 | /* In curl_multi_socket_action mode any new requests created during 207 | the following call will have already started associated socket 208 | operations, so no need to retry here */ 209 | return process_request_context(requestContext, &retry); 210 | } 211 | 212 | S3Status S3_get_request_context_fdsets(S3RequestContext* requestContext, 213 | fd_set* readFdSet, 214 | fd_set* writeFdSet, 215 | fd_set* exceptFdSet, 216 | int* maxFd) 217 | { 218 | return ((curl_multi_fdset(requestContext->curlm, readFdSet, writeFdSet, exceptFdSet, maxFd) == CURLM_OK) 219 | ? S3StatusOK 220 | : S3StatusInternalError); 221 | } 222 | 223 | int64_t S3_get_request_context_timeout(S3RequestContext* requestContext) 224 | { 225 | long timeout; 226 | 227 | if (curl_multi_timeout(requestContext->curlm, &timeout) != CURLM_OK) { 228 | timeout = 0; 229 | } 230 | 231 | return timeout; 232 | } 233 | 234 | void S3_set_request_context_verify_peer(S3RequestContext* requestContext, int verifyPeer) 235 | { 236 | requestContext->verifyPeerSet = 1; 237 | requestContext->verifyPeer = (verifyPeer != 0); 238 | } 239 | -------------------------------------------------------------------------------- /libs3/src/error_parser.c: -------------------------------------------------------------------------------- 1 | /** ************************************************************************** 2 | * error_parser.c 3 | * 4 | * Copyright 2008 Bryan Ischo 5 | * 6 | * This file is part of libs3. 7 | * 8 | * libs3 is free software: you can redistribute it and/or modify it under the 9 | * terms of the GNU Lesser General Public License as published by the Free 10 | * Software Foundation, version 3 or above of the License. You can also 11 | * redistribute and/or modify it under the terms of the GNU General Public 12 | * License, version 2 or above of the License. 13 | * 14 | * In addition, as a special exception, the copyright holders give 15 | * permission to link the code of this library and its programs with the 16 | * OpenSSL library, and distribute linked combinations including the two. 17 | * 18 | * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY 19 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21 | * details. 22 | * 23 | * You should have received a copy of the GNU Lesser General Public License 24 | * version 3 along with libs3, in a file named COPYING. If not, see 25 | * . 26 | * 27 | * You should also have received a copy of the GNU General Public License 28 | * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see 29 | * . 30 | * 31 | ************************************************************************** **/ 32 | 33 | #include 34 | #include "libs3/error_parser.h" 35 | 36 | static S3Status errorXmlCallback(const char* elementPath, const char* data, int dataLen, void* callbackData) 37 | { 38 | // We ignore end of element callbacks because we don't care about them 39 | if (!data) { 40 | return S3StatusOK; 41 | } 42 | 43 | ErrorParser* errorParser = (ErrorParser*) callbackData; 44 | 45 | int fit; 46 | 47 | if (!strcmp(elementPath, "Error")) { 48 | // Ignore, this is the Error element itself, we only care about subs 49 | } 50 | else if (!strcmp(elementPath, "Error/Code")) { 51 | string_buffer_append(errorParser->code, data, dataLen, fit); 52 | } 53 | else if (!strcmp(elementPath, "Error/Message")) { 54 | string_buffer_append(errorParser->message, data, dataLen, fit); 55 | errorParser->s3ErrorDetails.message = errorParser->message; 56 | } 57 | else if (!strcmp(elementPath, "Error/Resource")) { 58 | string_buffer_append(errorParser->resource, data, dataLen, fit); 59 | errorParser->s3ErrorDetails.resource = errorParser->resource; 60 | } 61 | else if (!strcmp(elementPath, "Error/FurtherDetails")) { 62 | string_buffer_append(errorParser->furtherDetails, data, dataLen, fit); 63 | errorParser->s3ErrorDetails.furtherDetails = errorParser->furtherDetails; 64 | } 65 | else { 66 | if (strncmp(elementPath, "Error/", sizeof("Error/") - 1)) { 67 | // If for some weird reason it's not within the Error element, 68 | // ignore it 69 | return S3StatusOK; 70 | } 71 | // It's an unknown error element. See if it matches the most 72 | // recent error element. 73 | const char* elementName = &(elementPath[sizeof("Error/") - 1]); 74 | if (errorParser->s3ErrorDetails.extraDetailsCount && 75 | !strcmp(elementName, 76 | errorParser->s3ErrorDetails.extraDetails[errorParser->s3ErrorDetails.extraDetailsCount - 1].name)) 77 | { 78 | // Append the value 79 | string_multibuffer_append(errorParser->extraDetailsNamesValues, data, dataLen, fit); 80 | // If it didn't fit, remove this extra 81 | if (!fit) { 82 | errorParser->s3ErrorDetails.extraDetailsCount--; 83 | } 84 | return S3StatusOK; 85 | } 86 | // OK, must add another unknown error element, if it will fit. 87 | if (errorParser->s3ErrorDetails.extraDetailsCount == sizeof(errorParser->extraDetails)) { 88 | // Won't fit. Ignore this one. 89 | return S3StatusOK; 90 | } 91 | // Copy in the name and value 92 | char* name = string_multibuffer_current(errorParser->extraDetailsNamesValues); 93 | int nameLen = strlen(elementName); 94 | string_multibuffer_add(errorParser->extraDetailsNamesValues, elementName, nameLen, fit); 95 | if (!fit) { 96 | // Name didn't fit; ignore this one. 97 | return S3StatusOK; 98 | } 99 | char* value = string_multibuffer_current(errorParser->extraDetailsNamesValues); 100 | string_multibuffer_add(errorParser->extraDetailsNamesValues, data, dataLen, fit); 101 | if (!fit) { 102 | // Value didn't fit; ignore this one. 103 | return S3StatusOK; 104 | } 105 | S3NameValue* nv = &(errorParser->extraDetails[errorParser->s3ErrorDetails.extraDetailsCount++]); 106 | nv->name = name; 107 | nv->value = value; 108 | } 109 | 110 | return S3StatusOK; 111 | } 112 | 113 | void error_parser_initialize(ErrorParser* errorParser) 114 | { 115 | errorParser->s3ErrorDetails.message = 0; 116 | errorParser->s3ErrorDetails.resource = 0; 117 | errorParser->s3ErrorDetails.furtherDetails = 0; 118 | errorParser->s3ErrorDetails.extraDetailsCount = 0; 119 | errorParser->s3ErrorDetails.extraDetails = errorParser->extraDetails; 120 | errorParser->errorXmlParserInitialized = 0; 121 | string_buffer_initialize(errorParser->code); 122 | string_buffer_initialize(errorParser->message); 123 | string_buffer_initialize(errorParser->resource); 124 | string_buffer_initialize(errorParser->furtherDetails); 125 | string_multibuffer_initialize(errorParser->extraDetailsNamesValues); 126 | } 127 | 128 | S3Status error_parser_add(ErrorParser* errorParser, char* buffer, int bufferSize) 129 | { 130 | if (!errorParser->errorXmlParserInitialized) { 131 | simplexml_initialize(&(errorParser->errorXmlParser), &errorXmlCallback, errorParser); 132 | errorParser->errorXmlParserInitialized = 1; 133 | } 134 | 135 | return simplexml_add(&(errorParser->errorXmlParser), buffer, bufferSize); 136 | } 137 | 138 | void error_parser_convert_status(ErrorParser* errorParser, S3Status* status) 139 | { 140 | // Convert the error status string into a code 141 | if (!errorParser->codeLen) { 142 | return; 143 | } 144 | 145 | #define HANDLE_CODE(name) \ 146 | do { \ 147 | if (!strcmp(errorParser->code, #name)) { \ 148 | *status = S3StatusError##name; \ 149 | goto code_set; \ 150 | } \ 151 | } \ 152 | while (0) 153 | 154 | HANDLE_CODE(AccessDenied); 155 | HANDLE_CODE(AccountProblem); 156 | HANDLE_CODE(AmbiguousGrantByEmailAddress); 157 | HANDLE_CODE(BadDigest); 158 | HANDLE_CODE(BucketAlreadyExists); 159 | HANDLE_CODE(BucketAlreadyOwnedByYou); 160 | HANDLE_CODE(BucketNotEmpty); 161 | HANDLE_CODE(CredentialsNotSupported); 162 | HANDLE_CODE(CrossLocationLoggingProhibited); 163 | HANDLE_CODE(EntityTooSmall); 164 | HANDLE_CODE(EntityTooLarge); 165 | HANDLE_CODE(ExpiredToken); 166 | HANDLE_CODE(IllegalVersioningConfigurationException); 167 | HANDLE_CODE(IncompleteBody); 168 | HANDLE_CODE(IncorrectNumberOfFilesInPostRequest); 169 | HANDLE_CODE(InlineDataTooLarge); 170 | HANDLE_CODE(InternalError); 171 | HANDLE_CODE(InvalidAccessKeyId); 172 | HANDLE_CODE(InvalidAddressingHeader); 173 | HANDLE_CODE(InvalidArgument); 174 | HANDLE_CODE(InvalidBucketName); 175 | HANDLE_CODE(InvalidBucketState); 176 | HANDLE_CODE(InvalidDigest); 177 | HANDLE_CODE(InvalidEncryptionAlgorithmError); 178 | HANDLE_CODE(InvalidLocationConstraint); 179 | HANDLE_CODE(InvalidObjectState); 180 | HANDLE_CODE(InvalidPart); 181 | HANDLE_CODE(InvalidPartOrder); 182 | HANDLE_CODE(InvalidPayer); 183 | HANDLE_CODE(InvalidPolicyDocument); 184 | HANDLE_CODE(InvalidRange); 185 | HANDLE_CODE(InvalidRequest); 186 | HANDLE_CODE(InvalidSecurity); 187 | HANDLE_CODE(InvalidSOAPRequest); 188 | HANDLE_CODE(InvalidStorageClass); 189 | HANDLE_CODE(InvalidTargetBucketForLogging); 190 | HANDLE_CODE(InvalidToken); 191 | HANDLE_CODE(InvalidURI); 192 | HANDLE_CODE(KeyTooLong); 193 | HANDLE_CODE(MalformedACLError); 194 | HANDLE_CODE(MalformedPOSTRequest); 195 | HANDLE_CODE(MalformedXML); 196 | HANDLE_CODE(MaxMessageLengthExceeded); 197 | HANDLE_CODE(MaxPostPreDataLengthExceededError); 198 | HANDLE_CODE(MetadataTooLarge); 199 | HANDLE_CODE(MethodNotAllowed); 200 | HANDLE_CODE(MissingAttachment); 201 | HANDLE_CODE(MissingContentLength); 202 | HANDLE_CODE(MissingRequestBodyError); 203 | HANDLE_CODE(MissingSecurityElement); 204 | HANDLE_CODE(MissingSecurityHeader); 205 | HANDLE_CODE(NoLoggingStatusForKey); 206 | HANDLE_CODE(NoSuchBucket); 207 | HANDLE_CODE(NoSuchKey); 208 | HANDLE_CODE(NoSuchLifecycleConfiguration); 209 | HANDLE_CODE(NoSuchUpload); 210 | HANDLE_CODE(NoSuchVersion); 211 | HANDLE_CODE(NotImplemented); 212 | HANDLE_CODE(NotSignedUp); 213 | HANDLE_CODE(NoSuchBucketPolicy); 214 | HANDLE_CODE(OperationAborted); 215 | HANDLE_CODE(PermanentRedirect); 216 | HANDLE_CODE(PreconditionFailed); 217 | HANDLE_CODE(Redirect); 218 | HANDLE_CODE(RestoreAlreadyInProgress); 219 | HANDLE_CODE(RequestIsNotMultiPartContent); 220 | HANDLE_CODE(RequestTimeout); 221 | HANDLE_CODE(RequestTimeTooSkewed); 222 | HANDLE_CODE(RequestTorrentOfBucketError); 223 | HANDLE_CODE(SignatureDoesNotMatch); 224 | HANDLE_CODE(ServiceUnavailable); 225 | HANDLE_CODE(SlowDown); 226 | HANDLE_CODE(TemporaryRedirect); 227 | HANDLE_CODE(TokenRefreshRequired); 228 | HANDLE_CODE(TooManyBuckets); 229 | HANDLE_CODE(UnexpectedContent); 230 | HANDLE_CODE(UnresolvableGrantByEmailAddress); 231 | HANDLE_CODE(UserKeyMustBeSpecified); 232 | HANDLE_CODE(QuotaExceeded); 233 | *status = S3StatusErrorUnknown; 234 | 235 | code_set: 236 | 237 | return; 238 | } 239 | 240 | // Always call this 241 | void error_parser_deinitialize(ErrorParser* errorParser) 242 | { 243 | if (errorParser->errorXmlParserInitialized) { 244 | simplexml_deinitialize(&(errorParser->errorXmlParser)); 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /s3_resource/include/irods/private/s3_resource/s3_resource.hpp: -------------------------------------------------------------------------------- 1 | /* -*- mode: c++; fill-column: 132; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | 3 | #ifndef IRODS_S3_RESOURCE_HPP 4 | #define IRODS_S3_RESOURCE_HPP 5 | 6 | // Needed to support pread with > 2GB offsets 7 | #define _USE_FILE_OFFSET64 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "libs3/libs3.h" 16 | 17 | #define S3_AUTH_FILE "s3Auth" 18 | #define ARCHIVE_NAMING_POLICY_KW "ARCHIVE_NAMING_POLICY" 19 | #define CONSISTENT_NAMING "consistent" 20 | #define DECOUPLED_NAMING "decoupled" 21 | 22 | // For s3PutCopyFile to identify the real source type 23 | typedef enum { S3_PUTFILE, S3_COPYOBJECT } s3_putcopy; 24 | 25 | extern const std::string s3_default_hostname; 26 | extern const std::string s3_default_hostname_vector; 27 | extern const std::string s3_hostname_index; 28 | extern const std::string host_mode; 29 | extern const std::string s3_auth_file; 30 | extern const std::string s3_key_id; 31 | extern const std::string s3_access_key; 32 | extern const std::string s3_retry_count; 33 | extern const std::string s3_wait_time_sec; 34 | extern const std::string s3_proto; 35 | extern const std::string s3_stsdate; 36 | extern const std::string s3_max_upload_size; 37 | extern const std::string s3_enable_mpu; 38 | extern const std::string s3_mpu_chunk; 39 | extern const std::string s3_mpu_threads; 40 | extern const std::string s3_enable_md5; 41 | extern const std::string s3_server_encrypt; 42 | extern const std::string s3_region_name; 43 | extern const std::string REPL_POLICY_KEY; 44 | extern const std::string REPL_POLICY_VAL; 45 | extern const std::string s3_cache_dir; 46 | extern const std::string s3_circular_buffer_size; 47 | extern const std::string s3_circular_buffer_timeout_seconds; // timeout for read or write to circular buffer 48 | extern const std::string s3_uri_request_style; // either "path" or "virtual_hosted" - default "path" 49 | extern const std::string s3_number_of_threads; // to save number of threads 50 | extern const std::size_t S3_DEFAULT_RETRY_WAIT_SECONDS; 51 | extern const std::size_t S3_DEFAULT_MAX_RETRY_WAIT_SECONDS; 52 | extern const std::size_t S3_DEFAULT_RETRY_COUNT; 53 | extern const int S3_DEFAULT_CIRCULAR_BUFFER_SIZE; 54 | extern const unsigned int S3_DEFAULT_CIRCULAR_BUFFER_TIMEOUT_SECONDS; 55 | extern const unsigned int S3_DEFAULT_NON_DATA_TRANSFER_TIMEOUT_SECONDS; 56 | 57 | std::string s3GetHostname(irods::plugin_property_map& _prop_map); 58 | std::int64_t s3GetMPUChunksize(irods::plugin_property_map& _prop_map); 59 | ssize_t s3GetMPUThreads(irods::plugin_property_map& _prop_map); 60 | bool s3GetEnableMultiPartUpload (irods::plugin_property_map& _prop_map); 61 | S3UriStyle s3_get_uri_request_style(irods::plugin_property_map& _prop_map); 62 | std::string get_region_name(irods::plugin_property_map& _prop_map); 63 | bool s3GetServerEncrypt (irods::plugin_property_map& _prop_map); 64 | std::string get_cache_directory(irods::plugin_property_map& _prop_map); 65 | 66 | std::size_t get_retry_wait_time_sec(irods::plugin_property_map& _prop_map); 67 | std::size_t get_max_retry_wait_time_sec(irods::plugin_property_map& _prop_map); 68 | std::size_t get_retry_count(irods::plugin_property_map& _prop_map); 69 | unsigned int get_non_data_transfer_timeout_seconds(irods::plugin_property_map& _prop_map); 70 | unsigned int s3_get_restoration_days(irods::plugin_property_map& _prop_map); 71 | std::string s3_get_restoration_tier(irods::plugin_property_map& _prop_map); 72 | std::string s3_get_storage_class_from_configuration(irods::plugin_property_map& _prop_map); 73 | bool s3_direct_checksum_read_enabled(irods::plugin_property_map& _prop_map); 74 | 75 | void StoreAndLogStatus(S3Status status, const S3ErrorDetails *error, 76 | const char *function, const S3BucketContext *pCtx, S3Status *pStatus, 77 | bool ignore_not_found_error = false); 78 | 79 | 80 | typedef struct S3Auth { 81 | char accessKeyId[MAX_NAME_LEN]; 82 | char secretAccessKey[MAX_NAME_LEN]; 83 | } s3Auth_t; 84 | 85 | typedef struct s3Stat 86 | { 87 | char key[MAX_NAME_LEN]; 88 | rodsLong_t size; 89 | time_t lastModified; 90 | } s3Stat_t; 91 | 92 | typedef struct callback_data 93 | { 94 | callback_data() 95 | : fd{0} 96 | , offset{0} 97 | , contentLength{0} 98 | , originalContentLength{0} 99 | , status{S3Status::S3StatusOK} 100 | , keyCount{0} 101 | , s3Stat{} 102 | , pCtx{nullptr} 103 | , prop_map_ptr{nullptr} 104 | , x_amz_storage_class{} // for glacier 105 | , x_amz_restore{} // for glacier 106 | {} 107 | int fd; 108 | std::int64_t offset; /* For multiple upload */ 109 | rodsLong_t contentLength, originalContentLength; 110 | S3Status status; 111 | int keyCount; 112 | s3Stat_t s3Stat; /* should be a pointer if keyCount > 1 */ 113 | 114 | S3BucketContext *pCtx; /* To enable more detailed error messages */ 115 | irods::plugin_property_map *prop_map_ptr; 116 | std::string x_amz_storage_class; 117 | std::string x_amz_restore; 118 | } callback_data_t; 119 | 120 | typedef struct upload_manager 121 | { 122 | char *upload_id; /* Returned from S3 on MP begin */ 123 | char **etags; /* Each upload part's MD5 */ 124 | 125 | /* Below used for the upload completion command, need to send in XML */ 126 | char *xml; 127 | std::int64_t remaining; 128 | std::int64_t offset; 129 | 130 | S3BucketContext *pCtx; /* To enable more detailed error messages */ 131 | 132 | S3Status status; 133 | } upload_manager_t; 134 | 135 | typedef struct multipart_data 136 | { 137 | multipart_data() 138 | : seq{0} 139 | , mode{0} 140 | , pSrcCtx{nullptr} 141 | , srcKey{nullptr} 142 | , put_object_data{} 143 | , manager{nullptr} 144 | , status{S3Status::S3StatusOK} 145 | , server_encrypt{false} 146 | {} 147 | int seq; /* Sequence number, i.e. which part */ 148 | int mode; /* PUT or COPY */ 149 | S3BucketContext *pSrcCtx; /* Source bucket context, ignored in a PUT */ 150 | const char *srcKey; /* Source key, ignored in a PUT */ 151 | callback_data put_object_data; /* File being uploaded */ 152 | upload_manager_t *manager; /* To update w/the MD5 returned */ 153 | 154 | S3Status status; 155 | bool server_encrypt; 156 | } multipart_data_t; 157 | 158 | typedef struct multirange_data 159 | { 160 | multirange_data() 161 | : seq{0} 162 | , get_object_data{} 163 | , status{S3Status::S3StatusOK} 164 | , pCtx{nullptr} 165 | , prop_map_ptr{nullptr} 166 | {} 167 | int seq; 168 | callback_data get_object_data; 169 | S3Status status; 170 | 171 | S3BucketContext *pCtx; /* To enable more detailed error messages */ 172 | irods::plugin_property_map *prop_map_ptr; 173 | } multirange_data_t; 174 | 175 | // Sleep between _s/2 and _s seconds. 176 | // The random addition ensures that threads don't all cluster up and retry 177 | // at the same time (dogpile effect) 178 | void s3_sleep( int _s ); 179 | 180 | void responseCompleteCallback( 181 | S3Status status, 182 | const S3ErrorDetails *error, 183 | void *callbackData); 184 | 185 | void responseCompleteCallbackIgnoreLoggingNotFound( 186 | S3Status status, 187 | const S3ErrorDetails *error, 188 | void *callbackData); 189 | 190 | S3Status responsePropertiesCallback( 191 | const S3ResponseProperties *properties, 192 | void *callbackData); 193 | 194 | // Utility functions 195 | irods::error parseS3Path ( 196 | const std::string& _s3ObjName, 197 | std::string& _bucket, 198 | std::string& _key, 199 | irods::plugin_property_map& _prop_map ); 200 | 201 | irods::error s3Init ( irods::plugin_property_map& _prop_map ); 202 | irods::error s3InitPerOperation ( irods::plugin_property_map& _prop_map ); 203 | 204 | S3Protocol s3GetProto( irods::plugin_property_map& _prop_map); 205 | 206 | S3STSDate s3GetSTSDate( irods::plugin_property_map& _prop_map); 207 | 208 | std::int64_t s3GetMaxUploadSizeMB (irods::plugin_property_map& _prop_map); 209 | 210 | bool s3_copyobject_disabled(irods::plugin_property_map& _prop_map); 211 | 212 | irods::error s3GetFile( 213 | const std::string& _filename, 214 | const std::string& _s3ObjName, 215 | rodsLong_t _fileSize, 216 | const std::string& _key_id, 217 | const std::string& _access_key, 218 | irods::plugin_property_map& _prop_map ); 219 | 220 | irods::error s3PutCopyFile( 221 | const s3_putcopy _mode, 222 | const std::string& _filename, 223 | const std::string& _s3ObjName, 224 | rodsLong_t _fileSize, 225 | const std::string& _key_id, 226 | const std::string& _access_key, 227 | irods::plugin_property_map& _prop_map); 228 | 229 | /// @brief Function to copy the specified src file to the specified dest file 230 | irods::error s3CopyFile( 231 | irods::plugin_context& _src_ctx, 232 | const std::string& _src_file, 233 | const std::string& _dest_file, 234 | const std::string& _key_id, 235 | const std::string& _access_key, 236 | const S3Protocol _proto, 237 | const S3STSDate _stsDate, 238 | const S3UriStyle _s3_uri_style); 239 | 240 | // =-=-=-=-=-=-=- 241 | /// @brief Checks the basic operation parameters and updates the physical path in the file object 242 | irods::error s3CheckParams(irods::plugin_context& _ctx ); 243 | 244 | 245 | std::tuple get_modes_from_properties(irods::plugin_property_map& _prop_map_); 246 | 247 | std::string get_resource_name(irods::plugin_property_map& _prop_map); 248 | 249 | bool determine_unlink_for_repl_policy( 250 | rsComm_t* _comm, 251 | const std::string& _logical_path, 252 | const std::string& _vault_path); 253 | 254 | // =-=-=-=-=-=-=- 255 | // redirect_get - code to determine redirection for get operation 256 | irods::error s3RedirectCreate( 257 | irods::plugin_property_map& _prop_map, 258 | irods::file_object& _file_obj, 259 | const std::string& _resc_name, 260 | const std::string& _curr_host, 261 | float& _out_vote ); 262 | 263 | // =-=-=-=-=-=-=- 264 | // redirect_get - code to determine redirection for get operation 265 | irods::error s3RedirectOpen( 266 | rsComm_t* _comm, 267 | irods::plugin_property_map& _prop_map, 268 | irods::file_object_ptr _file_obj, 269 | const std::string& _resc_name, 270 | const std::string& _curr_host, 271 | float& _out_vote ); 272 | 273 | irods::error s3GetAuthCredentials( 274 | irods::plugin_property_map& _prop_map, 275 | std::string& _rtn_key_id, 276 | std::string& _rtn_access_key); 277 | 278 | #endif 279 | --------------------------------------------------------------------------------