├── debian ├── compat ├── rules └── control ├── ubic ├── init.d │ └── mediastorage-proxy ├── service │ └── mediastorage-proxy.ini └── perl5 │ └── MDSProxy.pm ├── src ├── ns_settings.cpp ├── loggers.cpp ├── ranges.hpp ├── remove.hpp ├── delete.hpp ├── error.hpp ├── magic_provider.hpp ├── lookup_result.hpp ├── upload.hpp ├── deferred_function.hpp ├── cdn_cache.hpp ├── ns_settings.hpp ├── hex.hpp ├── data_container.cpp ├── couple_iterator.hpp ├── handystats.hpp ├── write_retrier.hpp ├── lookuper.hpp ├── download_info.hpp ├── loggers.hpp ├── lookup_result.cpp ├── timer.hpp ├── remove.cpp ├── utils.hpp ├── upload_simple.hpp ├── lookuper.cpp ├── buffered_writer.hpp ├── ranges.cpp ├── upload_multipart.hpp ├── delete.cpp ├── writer.hpp ├── utils.cpp ├── data_container.hpp ├── write_retrier.cpp ├── get.hpp ├── cdn_cache.cpp ├── expected.hpp ├── proxy.hpp ├── upload.cpp ├── buffered_writer.cpp ├── download_info.cpp ├── upload_simple.cpp ├── writer.cpp └── upload_multipart.cpp ├── example ├── config.json └── config_stats.json └── CMakeLists.txt /debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /ubic/init.d/mediastorage-proxy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use Ubic::Run; 4 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | include /usr/share/cdbs/1/class/cmake.mk 4 | include /usr/share/cdbs/1/rules/debhelper.mk 5 | 6 | DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_SKIP_RPATH=OFF 7 | 8 | DEB_DH_STRIP_ARGS := --dbg-package=mediastorage-proxy 9 | -------------------------------------------------------------------------------- /ubic/service/mediastorage-proxy.ini: -------------------------------------------------------------------------------- 1 | module = Ubic::Service::MDSProxy 2 | [options] 3 | user = root 4 | rlimit_nofile = 65536 5 | #rlimit_stack = 123 6 | rlimit_core = -1 7 | #conf_file = /etc/elliptics/mediastorage-proxy.conf 8 | #log_dir = /var/log/mdst 9 | #run_dir = /var/run/mediastorage 10 | -------------------------------------------------------------------------------- /src/ns_settings.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "ns_settings.hpp" 21 | 22 | const elliptics::ns_settings_t & 23 | elliptics::ns_settings(const mastermind::namespace_state_t &ns_state) { 24 | return static_cast(ns_state.settings().user_settings()); 25 | } 26 | 27 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: mediastorage-proxy 2 | Section: net 3 | Priority: optional 4 | Maintainer: Artem Sokolov 5 | Build-Depends: debhelper (>=5), 6 | cmake (>= 2.6), 7 | cdbs, 8 | libthevoid3-dev (>= 3.1.0), 9 | libswarm3-dev (>= 3.1.0), 10 | libmastermind-dev (>= 8.34), 11 | elliptics-dev (>= 2.26.3.33-6), 12 | cocaine-framework-native (>= 0.11), cocaine-framework-native (<< 0.12), 13 | cocaine-framework-native-dev (>= 0.11), cocaine-framework-native-dev (<< 0.12), 14 | libcocaine-core2 (>= 0.11), libcocaine-core2 (<< 0.12), 15 | libcocaine-dev (>= 0.11), libcocaine-dev (<< 0.12), 16 | libboost-system-dev, 17 | libboost-dev, 18 | libmsgpack-dev, 19 | libglib2.0-0, 20 | libglib2.0-dev, 21 | libmagic-dev, 22 | libcrypto++-dev, 23 | libyandex-utils-http, 24 | libcurl4-openssl-dev, 25 | handystats (>= 1.11), 26 | libssl-dev, 27 | Standards-Version: 3.9.1 28 | 29 | Package: mediastorage-proxy 30 | Architecture: any 31 | Depends: ${shlibs:Depends}, libyandex-ubic-shared-perl, libfcgi-procmanager-perl, libfcgi-perl, libfcgi-client-perl, libbsd-resource-perl, liblwp-protocol-http-socketunixalt-perl, handystats (>= 1.11), ubic, 32 | Description: Proxy with elliptics support based on thevoid 33 | 34 | -------------------------------------------------------------------------------- /src/loggers.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "loggers.hpp" 21 | 22 | ioremap::swarm::logger 23 | copy_logger(const ioremap::swarm::logger &logger) { 24 | return ioremap::swarm::logger{logger, blackhole::log::attributes_t()}; 25 | } 26 | 27 | shared_logger_t 28 | make_shared_logger(const ioremap::swarm::logger &logger) { 29 | typedef ioremap::swarm::logger logger_t; 30 | return std::make_shared(logger_t{logger, blackhole::log::attributes_t()}); 31 | } 32 | 33 | -------------------------------------------------------------------------------- /src/ranges.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__RANGES_HPP 21 | #define SRC__RANGES_HPP 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace elliptics { 29 | 30 | struct range_t { 31 | size_t offset; 32 | size_t size; 33 | }; 34 | 35 | typedef std::list ranges_t; 36 | 37 | boost::optional parse_range_header(const std::string &header, size_t total_size); 38 | 39 | } // namespace elliptics 40 | 41 | #endif /* SRC__RANGES_HPP */ 42 | -------------------------------------------------------------------------------- /example/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoints": [ 3 | "unix:/tmp/thevoid/mediastorage-proxy.sock", 4 | "0.0.0.0:8082" 5 | ], 6 | "backlog": 128, 7 | "threads": 2, 8 | "buffer_size": 65536, 9 | "daemon": { 10 | "monitor-port": 20001 11 | }, 12 | "application" : { 13 | "elliptics-log" : { 14 | "path" : "/tmp/log/mediastorage/elliptics.log", 15 | "level" : 2 16 | }, 17 | "proxy-log" : { 18 | "path" : "/tmp/log/mediastorage/proxy.log", 19 | "level" : 2 20 | }, 21 | "mastermind-log" : { 22 | "path" : "/tmp/log/mediastorage/mastermind.log", 23 | "level" : 2 24 | }, 25 | "timeouts" : { 26 | "wait" : 5, 27 | "check" : 60 28 | }, 29 | "cfg-flags" : 4, 30 | "remotes" : [ 31 | "elliptics-storage1.net:1025:2", 32 | "elliptics-storage2.net:1025:2", 33 | "slliptics-storage3.net:1025:2", 34 | "elliptics-storage4.net:1025:2" 35 | ], 36 | "mastermind" : { 37 | "nodes" : [ 38 | { 39 | "host" : "cocaine-runtime1.net", 40 | "port" : 10053 41 | }, 42 | { 43 | "host" : "cocaine-runtime2.net", 44 | "port" : 10053 45 | } 46 | ], 47 | "group-info-update-period" : 10 48 | }, 49 | "die-limit" : 1, 50 | "eblob-style-path" : true, 51 | "direction-bit-num" : 16, 52 | "base-port" : 1024, 53 | "namespaces" : { 54 | "default" : { 55 | "groups-count" : 2, 56 | "success-copies-num" : "any" 57 | }, 58 | "some" : { 59 | "groups-count" : 2, 60 | "success-copies-num" : "all" 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/remove.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__REMOVE__HPP 21 | #define MDS_PROXY__SRC__REMOVE__HPP 22 | 23 | #include "loggers.hpp" 24 | #include "expected.hpp" 25 | 26 | #include 27 | 28 | #include 29 | 30 | namespace elliptics { 31 | 32 | class remove_result_t { 33 | public: 34 | remove_result_t(bool has_bad_response_, bool not_found_); 35 | 36 | bool 37 | is_failed() const; 38 | 39 | bool 40 | is_successful() const; 41 | 42 | bool 43 | key_was_not_found() const; 44 | 45 | private: 46 | bool has_bad_response; 47 | bool not_found; 48 | 49 | }; 50 | 51 | void 52 | remove(shared_logger_t shared_logger 53 | , ioremap::elliptics::session session 54 | , std::string key 55 | , util::expected::callback_t next); 56 | 57 | } // namespace elliptics 58 | 59 | #endif /* MDS_PROXY__SRC__REMOVE__HPP */ 60 | 61 | -------------------------------------------------------------------------------- /src/delete.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__DELETE__HPP 21 | #define MDS_PROXY__SRC__DELETE__HPP 22 | 23 | #include "proxy.hpp" 24 | #include "remove.hpp" 25 | 26 | namespace elliptics { 27 | 28 | struct req_delete 29 | : public ioremap::thevoid::simple_request_stream 30 | , public std::enable_shared_from_this 31 | { 32 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 33 | void on_lookup(const ioremap::elliptics::sync_lookup_result &slr, const ioremap::elliptics::error_info &error); 34 | void on_finished(util::expected result); 35 | 36 | private: 37 | std::string url_str; 38 | ioremap::elliptics::key key; 39 | boost::optional session; 40 | size_t total_size; 41 | }; 42 | 43 | } // namespace elliptics 44 | 45 | #endif /* MDS_PROXY__SRC__DELETE__HPP */ 46 | 47 | -------------------------------------------------------------------------------- /src/error.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__ERROR__HPP 21 | #define MDS_PROXY__SRC__ERROR__HPP 22 | 23 | #include "loggers.hpp" 24 | 25 | #include 26 | 27 | namespace elliptics { 28 | 29 | class proxy_error : public std::runtime_error 30 | { 31 | public: 32 | proxy_error(const std::string &message) 33 | : std::runtime_error(message) 34 | {} 35 | }; 36 | 37 | class http_error : public proxy_error 38 | { 39 | public: 40 | http_error(int http_status_, const std::string &message) 41 | : proxy_error(message) 42 | , m_http_status(http_status_) 43 | {} 44 | 45 | int 46 | http_status() const { 47 | return m_http_status; 48 | } 49 | 50 | bool 51 | is_server_error() const { 52 | return m_http_status >= 500 && m_http_status <= 599; 53 | } 54 | 55 | 56 | private: 57 | int m_http_status; 58 | }; 59 | 60 | } // namespace elliptics 61 | 62 | #endif /* MDS_PROXY__SRC__ERROR__HPP */ 63 | 64 | -------------------------------------------------------------------------------- /src/magic_provider.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MAGIC_PROVIDER_HPP 21 | #define MAGIC_PROVIDER_HPP 22 | 23 | #include 24 | 25 | #include 26 | 27 | namespace elliptics { 28 | 29 | class magic_provider : private boost::noncopyable { 30 | 31 | public: 32 | magic_provider () { 33 | magic_ = magic_open(MAGIC_MIME_TYPE); 34 | magic_load(magic_, 0); 35 | } 36 | 37 | ~magic_provider() { 38 | magic_close(magic_); 39 | } 40 | 41 | public: 42 | std::string type(const char *data, const size_t size) { 43 | const char *result(magic_buffer(magic_, data, size)); 44 | 45 | if (result) { 46 | return result; 47 | } 48 | 49 | return "application/octet-stream"; 50 | } 51 | 52 | std::string type(const std::string &content) { 53 | return type(content.data(), content.size()); 54 | } 55 | 56 | private: 57 | magic_t magic_; 58 | 59 | }; 60 | 61 | } // namespace elliptics 62 | 63 | #endif /* MAGIC_PROVIDER_HPP */ 64 | -------------------------------------------------------------------------------- /src/lookup_result.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef _ELLIPTICS_LOOKUP_RESULT_HPP_ 21 | #define _ELLIPTICS_LOOKUP_RESULT_HPP_ 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace elliptics { 29 | 30 | class lookup_result { 31 | public: 32 | lookup_result(const ioremap::elliptics::lookup_result_entry &entry, std::string sign_port); 33 | 34 | const std::string &host() const; 35 | uint16_t port() const; 36 | int group() const; 37 | int status() const; 38 | const std::string &addr() const; 39 | const std::string &path() const; 40 | const std::string &full_path() const; 41 | 42 | private: 43 | ioremap::elliptics::lookup_result_entry m_entry; 44 | std::string m_sign_port; 45 | 46 | mutable boost::optional m_host; 47 | mutable boost::optional m_port; 48 | mutable boost::optional m_addr; 49 | mutable boost::optional m_path; 50 | mutable boost::optional m_full_path; 51 | }; 52 | 53 | } // namespace elliptics 54 | 55 | #endif /* _ELLIPTICS_LOOKUP_RESULT_HPP_ */ 56 | -------------------------------------------------------------------------------- /example/config_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoints": [ 3 | "unix:/tmp/thevoid/mediastorage-proxy.sock", 4 | "0.0.0.0:8082" 5 | ], 6 | "backlog": 128, 7 | "threads": 2, 8 | "buffer_size": 65536, 9 | "daemon": { 10 | "monitor-port": 20001 11 | }, 12 | "application" : { 13 | "elliptics-log" : { 14 | "path" : "/tmp/log/mediastorage/elliptics.log", 15 | "level" : 2 16 | }, 17 | "proxy-log" : { 18 | "path" : "/tmp/log/mediastorage/proxy.log", 19 | "level" : 2 20 | }, 21 | "mastermind-log" : { 22 | "path" : "/tmp/log/mediastorage/mastermind.log", 23 | "level" : 2 24 | }, 25 | "timeouts" : { 26 | "wait" : 5, 27 | "check" : 60 28 | }, 29 | "cfg-flags" : 4, 30 | "remotes" : [ 31 | "elliptics-storage1.net:1025:2", 32 | "elliptics-storage2.net:1025:2", 33 | "slliptics-storage3.net:1025:2", 34 | "elliptics-storage4.net:1025:2" 35 | ], 36 | "mastermind" : { 37 | "nodes" : [ 38 | { 39 | "host" : "cocaine-runtime1.net", 40 | "port" : 10053 41 | }, 42 | { 43 | "host" : "cocaine-runtime2.net", 44 | "port" : 10053 45 | } 46 | ], 47 | "group-info-update-period" : 10 48 | }, 49 | "handystats": { 50 | "enable": true, 51 | "dump-interval": 500, 52 | "defaults": { 53 | "moving-interval": 60000, 54 | "tags": [], 55 | "rate-unit": "s" 56 | }, 57 | "mds.{get,upload}": { 58 | "tags": ["value", "rate"] 59 | }, 60 | "mds.{get,upload}.reply.[1-5]{xx,[0-9][0-9]}": { 61 | "tags": ["value", "rate"] 62 | }, 63 | "mds.{get,upload}{,.reply}.time": { 64 | "histogram-bins": 30, 65 | "idle-timeout": 30000, 66 | "tags": ["moving-avg", "quantile"] 67 | } 68 | }, 69 | "die-limit" : 1, 70 | "eblob-style-path" : true, 71 | "direction-bit-num" : 16, 72 | "base-port" : 1024, 73 | "namespaces" : { 74 | "default" : { 75 | "groups-count" : 2, 76 | "success-copies-num" : "any" 77 | }, 78 | "some" : { 79 | "success-copies-num" : "all" 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/upload.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__UPLOAD__HPP 21 | #define MDS_PROXY__SRC__UPLOAD__HPP 22 | 23 | #include "utils.hpp" 24 | #include "proxy.hpp" 25 | #include "loggers.hpp" 26 | #include "couple_iterator.hpp" 27 | 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | namespace elliptics { 37 | 38 | class upload_t 39 | : public ioremap::thevoid::request_stream 40 | , public std::enable_shared_from_this 41 | { 42 | public: 43 | void 44 | on_headers(ioremap::thevoid::http_request &&http_request); 45 | 46 | size_t 47 | on_data(const boost::asio::const_buffer &buffer); 48 | 49 | void 50 | on_close(const boost::system::error_code &error); 51 | 52 | private: 53 | boost::optional 54 | create_couple_iterator(const ioremap::thevoid::http_request &http_request 55 | , const mastermind::namespace_state_t &ns_state, size_t total_size); 56 | 57 | std::shared_ptr request_stream; 58 | }; 59 | 60 | } // namespace elliptics 61 | 62 | #endif /* MDS_PROXY__SRC__UPLOAD__HPP */ 63 | 64 | -------------------------------------------------------------------------------- /src/deferred_function.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__DEFERRED_FUNCTION__HPP 21 | #define MDS_PROXY__SRC__DEFERRED_FUNCTION__HPP 22 | 23 | #include 24 | #include 25 | 26 | 27 | namespace elliptics { 28 | 29 | class deferred_function_t { 30 | public: 31 | typedef std::function function_t; 32 | 33 | deferred_function_t(function_t function_ = function_t(), int counter_ = 1) 34 | : function(std::move(function_)) 35 | , counter(counter_) 36 | { 37 | } 38 | 39 | void 40 | defer(int count = 1) { 41 | counter += count; 42 | } 43 | 44 | bool 45 | operator ()() { 46 | if (--counter) { 47 | return false; 48 | } 49 | 50 | if (!function) { 51 | return false; 52 | } 53 | 54 | function(); 55 | 56 | reset(); 57 | 58 | return true; 59 | } 60 | 61 | void 62 | reset(function_t function_ = function_t(), int counter_ = 1) { 63 | function = std::move(function_); 64 | counter = counter_; 65 | } 66 | 67 | private: 68 | function_t function; 69 | std::atomic counter; 70 | }; 71 | 72 | } // namespace elliptics 73 | 74 | 75 | #endif /* MDS_PROXY__SRC__DEFERRED_FUNCTION__HPP */ 76 | 77 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | set(TARGET mediastorage-proxy) 3 | 4 | set(CMAKE_CXX_FLAGS "-Wall -Wextra -pedantic -std=c++0x") 5 | 6 | find_package(PkgConfig) 7 | pkg_check_modules(glib-2.0 glib-2.0) 8 | find_path(GLIB_INCLUDE_DIR glib.h HINTS ${glib-2.0_INCLUDE_DIRS}) 9 | find_path(GLIBCONFIG_INCLUDE_DIR glibconfig.h HINTS ${glib-2.0_INCLUDE_DIRS}) 10 | find_library(GLIB_LIBRARY glib-2.0 HINTS ${glib-2.0_INCLUDE_DIRS}) 11 | 12 | set(REQUIRED_LIBRARIES 13 | thevoid 14 | elliptics_cpp 15 | mastermind 16 | boost_system 17 | boost_thread 18 | swarm 19 | msgpack 20 | magic 21 | ${GLIB_LIBRARY} 22 | crypto++ 23 | crypto 24 | curl 25 | handystats 26 | kora-util 27 | ) 28 | 29 | set(SOURCE 30 | ${PROJECT_SOURCE_DIR}/src/proxy.cpp 31 | ${PROJECT_SOURCE_DIR}/src/writer.cpp 32 | ${PROJECT_SOURCE_DIR}/src/write_retrier.cpp 33 | ${PROJECT_SOURCE_DIR}/src/buffered_writer.cpp 34 | ${PROJECT_SOURCE_DIR}/src/remove.cpp 35 | ${PROJECT_SOURCE_DIR}/src/upload.cpp 36 | ${PROJECT_SOURCE_DIR}/src/upload_simple.cpp 37 | ${PROJECT_SOURCE_DIR}/src/upload_multipart.cpp 38 | ${PROJECT_SOURCE_DIR}/src/lookuper.cpp 39 | ${PROJECT_SOURCE_DIR}/src/get.cpp 40 | ${PROJECT_SOURCE_DIR}/src/delete.cpp 41 | ${PROJECT_SOURCE_DIR}/src/download_info.cpp 42 | ${PROJECT_SOURCE_DIR}/src/lookup_result.cpp 43 | ${PROJECT_SOURCE_DIR}/src/data_container.cpp 44 | ${PROJECT_SOURCE_DIR}/src/ranges.cpp 45 | ${PROJECT_SOURCE_DIR}/src/cdn_cache.cpp 46 | ${PROJECT_SOURCE_DIR}/src/utils.cpp 47 | ${PROJECT_SOURCE_DIR}/src/loggers.cpp 48 | ${PROJECT_SOURCE_DIR}/src/ns_settings.cpp 49 | ) 50 | 51 | include_directories(BEFORE ${PROJECT_SOURCE_DIR}/include) 52 | include_directories(BEFORE ${GLIB_INCLUDE_DIR}) 53 | include_directories(BEFORE ${GLIBCONFIG_INCLUDE_DIR}) 54 | 55 | add_executable(${TARGET} ${SOURCE}) 56 | 57 | target_link_libraries(${TARGET} ${REQUIRED_LIBRARIES}) 58 | 59 | install(TARGETS ${TARGET} DESTINATION bin/) 60 | install(FILES ubic/init.d/mediastorage-proxy DESTINATION /etc/init.d) 61 | install(FILES ubic/service/mediastorage-proxy.ini DESTINATION /etc/ubic/service) 62 | install(FILES ubic/perl5/MDSProxy.pm DESTINATION /usr/share/perl5/Ubic/Service) 63 | -------------------------------------------------------------------------------- /src/cdn_cache.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__CDN_CACHE__HPP 21 | #define MDS_PROXY__SRC__CDN_CACHE__HPP 22 | 23 | #include "loggers.hpp" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | namespace elliptics { 33 | 34 | class cdn_cache_t { 35 | public: 36 | struct config_t { 37 | std::string url; 38 | int timeout; 39 | int update_period; 40 | std::string cache_path; 41 | }; 42 | 43 | cdn_cache_t(ioremap::swarm::logger bh_logger_, config_t config_); 44 | ~cdn_cache_t(); 45 | 46 | bool 47 | check_host(const std::string &host); 48 | 49 | void 50 | cache_force_update(); 51 | 52 | private: 53 | typedef std::shared_ptr> cache_ptr_t; 54 | 55 | ioremap::swarm::logger & 56 | logger(); 57 | 58 | void 59 | serialize() const; 60 | 61 | void 62 | deserialize(); 63 | 64 | cache_ptr_t 65 | parse_cache(const std::string &raw_data) const; 66 | 67 | cache_ptr_t 68 | copy_cache() const; 69 | 70 | void 71 | set_cache(cache_ptr_t cache_ptr_); 72 | 73 | void 74 | update_cache(); 75 | 76 | void 77 | background_loop(); 78 | 79 | ioremap::swarm::logger bh_logger; 80 | 81 | config_t config; 82 | 83 | mutable std::mutex cache_mutex; 84 | cache_ptr_t cache_ptr; 85 | 86 | std::mutex background_updater_mutex; 87 | std::thread background_updater; 88 | std::condition_variable background_updater_cv; 89 | bool work_is_done; 90 | }; 91 | 92 | } // namespace elliptics 93 | 94 | #endif /* MDS_PROXY__SRC__CDN_CACHE__HPP */ 95 | 96 | -------------------------------------------------------------------------------- /src/ns_settings.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__NS_SETTINGS__HPP 21 | #define MDS_PROXY__SRC__NS_SETTINGS__HPP 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | namespace elliptics { 32 | 33 | struct ns_settings_t 34 | : public mastermind::namespace_state_t::user_settings_t { 35 | 36 | ns_settings_t() 37 | : redirect_content_length_threshold(-1) 38 | , add_orig_path_query_arg(false) 39 | , can_choose_couple_to_upload(false) 40 | , multipart_content_length_threshold(0) 41 | , custom_expiration_time(false) 42 | , success_copies_num(-1) 43 | , check_for_update(true) 44 | {} 45 | 46 | std::string name; 47 | ioremap::elliptics::result_checker result_checker; 48 | 49 | std::string auth_key_for_write; 50 | std::string auth_key_for_read; 51 | 52 | std::vector static_couple; 53 | 54 | std::string sign_token; 55 | std::string sign_path_prefix; 56 | std::string sign_port; 57 | 58 | std::chrono::seconds redirect_expire_time; 59 | int64_t redirect_content_length_threshold; 60 | std::vector redirect_query_args; 61 | bool add_orig_path_query_arg; 62 | 63 | bool can_choose_couple_to_upload; 64 | int64_t multipart_content_length_threshold; 65 | bool custom_expiration_time; 66 | 67 | int success_copies_num; 68 | 69 | bool check_for_update; 70 | }; 71 | 72 | const ns_settings_t & 73 | ns_settings(const mastermind::namespace_state_t &ns_state); 74 | 75 | } // namespace elliptics 76 | 77 | #endif /* MDS_PROXY__SRC__NS_SETTINGS__HPP */ 78 | 79 | -------------------------------------------------------------------------------- /src/hex.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__HEX__HPP 21 | #define MDS_PROXY__SRC__HEX__HPP 22 | 23 | #include 24 | #include 25 | 26 | namespace elliptics { 27 | 28 | template 29 | typename std::enable_if::value, OutputIterator>::type 30 | hex_one(T val, OutputIterator out) { 31 | const size_t num_hex_digits = 2 * sizeof(T); 32 | char res[num_hex_digits]; 33 | char *p = res + num_hex_digits; 34 | 35 | for (size_t i = 0; i != num_hex_digits; ++i, val >>= 4) { 36 | *--p = "0123456789abcdef"[val & 0x0F]; 37 | } 38 | 39 | return std::copy(res, res + num_hex_digits, out); 40 | } 41 | 42 | template 43 | OutputSequence 44 | hex_one(T val) { 45 | OutputSequence output; 46 | output.reserve(2 * sizeof(T)); 47 | 48 | hex_one(val, std::back_inserter(output)); 49 | 50 | return output; 51 | } 52 | 53 | template 54 | OutputIterator 55 | hex(InputIterator it, InputIterator end, OutputIterator out) { 56 | 57 | for (; it != end; ++it) { 58 | out = hex_one(*it, out); 59 | } 60 | 61 | return out; 62 | } 63 | 64 | template 65 | OutputSequence 66 | hex(const InputSequence &input) { 67 | OutputSequence output; 68 | output.reserve(input.size() * (2 * sizeof(typename InputSequence::value_type))); 69 | 70 | hex(std::begin(input), std::end(input), std::back_inserter(output)); 71 | 72 | return output; 73 | } 74 | 75 | template 76 | Sequence 77 | hex(const Sequence &input) { 78 | return hex(input); 79 | } 80 | 81 | } // namespace elliptics 82 | 83 | #endif /* MDS_PROXY__SRC__HEX__HPP */ 84 | 85 | -------------------------------------------------------------------------------- /src/data_container.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "data_container.hpp" 21 | 22 | namespace elliptics { 23 | ioremap::elliptics::data_pointer data_container_t::pack(const data_container_t &ds) { 24 | ioremap::elliptics::data_buffer data; 25 | 26 | if (!ds.embeds.empty()) { 27 | for (auto it = ds.embeds.begin(); it != ds.embeds.end(); ++it) { 28 | data.write(bswap(it->second.header)); 29 | data.write((const char *)it->second.data_pointer.data(), it->second.data_pointer.size()); 30 | } 31 | embed_t::header_t h; 32 | h.size = ds.data.size(); 33 | h.type = DNET_FCGI_EMBED_DATA; 34 | h.flags = 0; 35 | data.write(bswap(h)); 36 | } 37 | 38 | data.write((const char *)ds.data.data(), ds.data.size()); 39 | return std::move(data); 40 | } 41 | 42 | data_container_t data_container_t::unpack(ioremap::elliptics::data_pointer data_pointer, bool embeded) { 43 | elliptics::data_container_t ds; 44 | 45 | if (embeded) { 46 | embed_t::header_t h; 47 | 48 | h = bswap(*data_pointer.data()); 49 | data_pointer = data_pointer.skip(); 50 | 51 | while (h.type != DNET_FCGI_EMBED_DATA) { 52 | embed_t e; 53 | e.header = h; 54 | e.data_pointer = data_pointer.slice(0, e.header.size); 55 | data_pointer = data_pointer.skip(e.header.size); 56 | ds.embeds.insert(std::make_pair(e.header.type, e)); 57 | 58 | h = bswap(*data_pointer.data()); 59 | data_pointer = data_pointer.skip(); 60 | } 61 | } 62 | 63 | ds.data = data_pointer; 64 | return ds; 65 | } 66 | 67 | data_container_t::embed_t::header_t data_container_t::bswap(const data_container_t::embed_t::header_t &header) { 68 | embed_t::header_t res; 69 | res.size = dnet_bswap64(header.size); 70 | res.flags = dnet_bswap32(header.flags); 71 | res.type = dnet_bswap32(header.type); 72 | return res; 73 | } 74 | 75 | } // namespace elliptics 76 | -------------------------------------------------------------------------------- /src/couple_iterator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__COUPLE_ITERATOR__HPP 21 | #define MDS_PROXY__SRC__COUPLE_ITERATOR__HPP 22 | 23 | #include 24 | #include 25 | 26 | namespace elliptics { 27 | 28 | class couple_iterator_t { 29 | public: 30 | static 31 | mastermind::couple_info_t 32 | create_couple_info(mastermind::groups_t groups) { 33 | mastermind::couple_info_t result; 34 | result.id = *std::min_element(groups.begin(), groups.end()); 35 | result.groups = std::move(groups); 36 | return result; 37 | } 38 | 39 | couple_iterator_t(mastermind::couple_sequence_t couple_sequence_) 40 | : couple_sequence(std::move(couple_sequence_)) 41 | , iter(couple_sequence.begin()) 42 | { 43 | } 44 | 45 | couple_iterator_t(mastermind::couple_info_t single_couple_info_) 46 | : iter(couple_sequence.end()) 47 | , single_couple_info(std::move(single_couple_info_)) 48 | { 49 | } 50 | 51 | couple_iterator_t(mastermind::groups_t groups) 52 | : iter(couple_sequence.end()) 53 | , single_couple_info(create_couple_info(std::move(groups))) 54 | { 55 | } 56 | 57 | bool 58 | has_next() const { 59 | return iter != couple_sequence.end() || single_couple_info.groups.size(); 60 | } 61 | 62 | mastermind::couple_info_t 63 | next() { 64 | if (!has_next()) { 65 | throw std::runtime_error("null couple iterator"); 66 | } 67 | 68 | if (single_couple_info.groups.size()) { 69 | auto result = std::move(single_couple_info); 70 | single_couple_info.groups.clear(); 71 | return result; 72 | } 73 | 74 | auto result = *iter++; 75 | 76 | return result; 77 | } 78 | 79 | private: 80 | mastermind::couple_sequence_t couple_sequence; 81 | mastermind::couple_sequence_t::const_iterator iter; 82 | 83 | mastermind::couple_info_t single_couple_info; 84 | 85 | }; 86 | 87 | } // namespace elliptics 88 | 89 | #endif /* MDS_PROXY__SRC__COUPLE_ITERATOR__HPP */ 90 | 91 | -------------------------------------------------------------------------------- /src/handystats.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__HANDYSTATS_HPP 21 | #define SRC__HANDYSTATS_HPP 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | // mds.REQUEST (counter) 29 | // - total number of requests 30 | // - request rate 31 | // mds.REQUEST.time (timer) 32 | // - total time spent on processing of successful (2xx) request 33 | // - quantiles (25%, 50%, 75%, 90%, 95%) 34 | // mds.REQUEST.reply.CODE (counter) 35 | // - total number of response codes (e.g., 404) and groups (e.g., 2xx) for request 36 | // - response code rate for request 37 | // mds.REQUEST.reply.time (timer) 38 | // - time spent between request and successful (2xx) response code reply 39 | // - quantiles (25%, 50%, 75%, 90%, 95%) 40 | 41 | 42 | namespace elliptics { 43 | 44 | // REQUEST 45 | 46 | inline void MDS_REQUEST_START(const std::string& method, const uint64_t& instance_id) { 47 | HANDY_COUNTER_INCREMENT(("mds.%s", method.c_str())); 48 | 49 | HANDY_TIMER_START(("mds.%s.time", method.c_str()), instance_id); 50 | 51 | HANDY_TIMER_START(("mds.%s.reply.time", method.c_str()), instance_id); 52 | } 53 | 54 | inline void MDS_REQUEST_STOP(const std::string& method, const uint64_t& instance_id) { 55 | HANDY_TIMER_STOP(("mds.%s.time", method.c_str()), instance_id); 56 | } 57 | 58 | inline void MDS_REQUEST_DISCARD(const std::string& method, const uint64_t& instance_id) { 59 | HANDY_TIMER_DISCARD(("mds.%s.time", method.c_str()), instance_id); 60 | } 61 | 62 | 63 | // REPLY 64 | 65 | inline void MDS_REQUEST_REPLY(const std::string& method, const int& code, const uint64_t& instance_id) { 66 | HANDY_COUNTER_INCREMENT(("mds.%s.reply.%d", method.c_str(), code)); 67 | 68 | HANDY_COUNTER_INCREMENT(("mds.%s.reply.%dxx", method.c_str(), code / 100)); 69 | 70 | if (code / 100 != 2) { 71 | HANDY_TIMER_DISCARD(("mds.%s.time", method.c_str()), instance_id); 72 | 73 | HANDY_TIMER_DISCARD(("mds.%s.reply.time", method.c_str()), instance_id); 74 | } 75 | else { 76 | HANDY_TIMER_STOP(("mds.%s.reply.time", method.c_str()), instance_id); 77 | } 78 | } 79 | 80 | } // namespace elliptics 81 | 82 | #endif // SRC__HANDYSTATS_HPP 83 | -------------------------------------------------------------------------------- /src/write_retrier.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__WRITE_RETRIER__HPP 21 | #define MDS_PROXY__SRC__WRITE_RETRIER__HPP 22 | 23 | #include "deferred_function.hpp" 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | namespace elliptics { 34 | 35 | class write_retrier 36 | : public std::enable_shared_from_this 37 | { 38 | public: 39 | typedef 40 | std::function 42 | command_t; 43 | 44 | write_retrier( 45 | ioremap::swarm::logger bh_logger_ 46 | , ioremap::elliptics::session session_ 47 | , command_t command_ 48 | , size_t success_copies_num_ 49 | , size_t limit_of_attempts_ 50 | , double scale_retry_timeout_ 51 | , ioremap::elliptics::async_write_result::handler promise 52 | ); 53 | 54 | void 55 | start(); 56 | 57 | private: 58 | ioremap::swarm::logger & 59 | logger(); 60 | 61 | void 62 | try_group(ioremap::elliptics::session group_session, size_t number_of_attempts); 63 | 64 | void 65 | on_finished(ioremap::elliptics::session group_session, size_t number_of_attempts 66 | , const ioremap::elliptics::sync_write_result &entries 67 | , const ioremap::elliptics::error_info &error_info); 68 | 69 | void 70 | init_error(const ioremap::elliptics::error_info &error_info_); 71 | 72 | void 73 | complete(); 74 | 75 | ioremap::swarm::logger bh_logger; 76 | ioremap::elliptics::session session; 77 | command_t command; 78 | size_t success_copies_num; 79 | size_t limit_of_attempts; 80 | double scale_retry_timeout; 81 | ioremap::elliptics::async_write_result::handler promise; 82 | 83 | std::mutex error_info_mutex; 84 | ioremap::elliptics::error_info error_info; 85 | deferred_function_t complete_once; 86 | 87 | }; 88 | 89 | ioremap::elliptics::async_write_result 90 | try_write( 91 | ioremap::swarm::logger bh_logger 92 | , ioremap::elliptics::session session 93 | , write_retrier::command_t command 94 | , size_t success_copies_num 95 | , size_t limit_of_attempts 96 | , double scale_retry_timeout 97 | ); 98 | 99 | } // namespace elliptics 100 | 101 | #endif /* MDS_PROXY__SRC__WRITE_RETRIER__HPP */ 102 | 103 | -------------------------------------------------------------------------------- /src/lookuper.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__LOOKUPER__HPP 21 | #define MDS_PROXY__SRC__LOOKUPER__HPP 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | namespace elliptics { 33 | 34 | class parallel_lookuper_t 35 | : public std::enable_shared_from_this 36 | { 37 | public: 38 | typedef ioremap::elliptics::sync_lookup_result entries_t; 39 | typedef ioremap::elliptics::error_info error_info_t; 40 | 41 | struct result_t { 42 | entries_t entries; 43 | error_info_t error_info; 44 | }; 45 | 46 | parallel_lookuper_t( 47 | ioremap::swarm::logger bh_logger_ 48 | , ioremap::elliptics::session session_ 49 | , std::string key_ 50 | ); 51 | 52 | void 53 | start(); 54 | 55 | ioremap::elliptics::async_lookup_result 56 | next_lookup_result(); 57 | 58 | size_t 59 | total_size() const; 60 | 61 | size_t 62 | results_left() const; 63 | 64 | #if 0 65 | ioremap::elliptics::async_lookup_result 66 | get_group(const ioremap::elliptics::lookup_result_entry &entry); 67 | #endif 68 | 69 | private: 70 | typedef std::mutex mutex_t; 71 | typedef std::unique_lock lock_guard_t; 72 | 73 | ioremap::swarm::logger & 74 | logger(); 75 | 76 | void 77 | on_lookup(const ioremap::elliptics::sync_lookup_result &entries 78 | , const ioremap::elliptics::error_info &error_info); 79 | 80 | void 81 | process_promise(ioremap::elliptics::async_lookup_result::handler &promise 82 | , const result_t &result); 83 | 84 | void 85 | process_promise(ioremap::elliptics::async_lookup_result::handler &promise); 86 | 87 | ioremap::swarm::logger bh_logger; 88 | ioremap::elliptics::session session; 89 | std::string key; 90 | 91 | mutable mutex_t results_mutex; 92 | std::list results; 93 | std::list promises; 94 | size_t groups_to_handle; 95 | 96 | }; 97 | 98 | typedef std::shared_ptr parallel_lookuper_ptr_t; 99 | 100 | parallel_lookuper_ptr_t 101 | make_parallel_lookuper( 102 | ioremap::swarm::logger bh_logger 103 | , ioremap::elliptics::session session 104 | , std::string key 105 | ); 106 | 107 | } // namespace elliptics 108 | 109 | #endif /* MDS_PROXY__SRC__LOOKUPER__HPP */ 110 | 111 | -------------------------------------------------------------------------------- /src/download_info.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__DOWNLOAD_INFO__HPP 21 | #define MDS_PROXY__SRC__DOWNLOAD_INFO__HPP 22 | 23 | #include "proxy.hpp" 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | 30 | namespace elliptics { 31 | 32 | class download_info_t 33 | : public ioremap::thevoid::simple_request_stream 34 | , public std::enable_shared_from_this 35 | { 36 | public: 37 | download_info_t(const std::string &handler_name_); 38 | 39 | void 40 | on_request(const ioremap::thevoid::http_request &req 41 | , const boost::asio::const_buffer &buffer); 42 | 43 | void 44 | on_finished(const ioremap::elliptics::sync_lookup_result &slr 45 | , const ioremap::elliptics::error_info &error); 46 | 47 | private: 48 | mastermind::namespace_state_t 49 | get_namespace_state(const std::string &path, const std::string &handler); 50 | 51 | void 52 | check_signature(); 53 | 54 | void 55 | check_query_args(); 56 | 57 | std::tuple, ioremap::elliptics::key> 58 | prepare_session(const mastermind::namespace_state_t &ns_state); 59 | 60 | void 61 | process_get(ioremap::elliptics::session session, const ioremap::elliptics::key key); 62 | 63 | void 64 | send_response(std::tuple res); 65 | 66 | std::string 67 | xml_response(std::tuple res); 68 | 69 | kora::dynamic_t 70 | json_response_impl(std::tuple res); 71 | 72 | std::string 73 | json_response(std::tuple res); 74 | 75 | std::string 76 | jsonp_response(std::tuple res); 77 | 78 | mastermind::namespace_state_t ns_state; 79 | std::string x_regional_host; 80 | std::string handler_name; 81 | boost::optional expiration_time; 82 | }; 83 | 84 | class download_info_1_t : public download_info_t { 85 | public: 86 | download_info_1_t(); 87 | static const std::string handler_name; 88 | }; 89 | 90 | class download_info_2_t : public download_info_t { 91 | public: 92 | download_info_2_t(); 93 | static const std::string handler_name; 94 | }; 95 | 96 | } // namespace elliptics 97 | 98 | #endif /* MDS_PROXY__SRC__DOWNLOAD_INFO__HPP */ 99 | 100 | -------------------------------------------------------------------------------- /src/loggers.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__LOGGERS_HPP 21 | #define SRC__LOGGERS_HPP 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #define MDS_LOG(verb, ...) BH_LOG(logger(), verb, __VA_ARGS__) 29 | #define MDS_LOG_ERROR(...) MDS_LOG(SWARM_LOG_ERROR, __VA_ARGS__) 30 | #define MDS_LOG_WARNING(...) MDS_LOG(SWARM_LOG_WARNING, __VA_ARGS__) 31 | #define MDS_LOG_INFO(...) MDS_LOG(SWARM_LOG_INFO, __VA_ARGS__) 32 | #define MDS_LOG_NOTICE(...) MDS_LOG(SWARM_LOG_NOTICE, __VA_ARGS__) 33 | #define MDS_LOG_DEBUG(...) MDS_LOG(SWARM_LOG_DEBUG, __VA_ARGS__) 34 | 35 | typedef std::shared_ptr shared_logger_t; 36 | 37 | ioremap::swarm::logger 38 | copy_logger(const ioremap::swarm::logger &logger); 39 | 40 | shared_logger_t 41 | make_shared_logger(const ioremap::swarm::logger &logger); 42 | 43 | class cocaine_logger_t : public cocaine::framework::logger_t { 44 | public: 45 | cocaine_logger_t(ioremap::swarm::logger logger_) 46 | : logger(std::move(logger_)) 47 | {} 48 | 49 | cocaine_logger_t(cocaine_logger_t &&other) 50 | : logger(std::move(other.logger)) 51 | {} 52 | 53 | void emit(cocaine::logging::priorities priority, const std::string& message) { 54 | //int lvl = level(priority); 55 | //m_logger.log(lvl, "%s", message.c_str()); 56 | BH_LOG(logger, level(priority), message); 57 | } 58 | 59 | cocaine::logging::priorities verbosity() const { 60 | using namespace cocaine::logging; 61 | return priorities::debug; 62 | /*switch(m_logger.log().verbosity()) { 63 | case blackhole::defaults::severity::error: 64 | return priorities::error; 65 | case blackhole::defaults::severity::info: 66 | return priorities::info; 67 | case blackhole::defaults::severity::notice: 68 | return priorities::info; 69 | default: 70 | return priorities::debug; 71 | }*/ 72 | } 73 | 74 | private: 75 | blackhole::defaults::severity level(cocaine::logging::priorities priority) { 76 | using namespace cocaine::logging; 77 | switch(priority) { 78 | case priorities::ignore: 79 | return blackhole::defaults::severity::error; 80 | case priorities::error: 81 | return blackhole::defaults::severity::error; 82 | case priorities::warning: 83 | return blackhole::defaults::severity::warning; 84 | case priorities::info: 85 | return blackhole::defaults::severity::info; 86 | default: 87 | return blackhole::defaults::severity::debug; 88 | } 89 | } 90 | 91 | ioremap::swarm::logger logger; 92 | }; 93 | 94 | #endif /* SRC__LOGGERS_HPP */ 95 | -------------------------------------------------------------------------------- /src/lookup_result.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "lookup_result.hpp" 21 | 22 | #include 23 | 24 | #include 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | namespace { 32 | 33 | static inline char* 34 | file_backend_get_dir(const unsigned char *id, uint64_t bit_num, char *dst) { 35 | char *res = dnet_dump_id_len_raw(id, (bit_num + 7) / 8, dst); 36 | 37 | if (res) { 38 | res[bit_num / 4] = '\0'; 39 | } 40 | 41 | return res; 42 | } 43 | 44 | } // namespace 45 | 46 | namespace elliptics { 47 | 48 | lookup_result::lookup_result(const ioremap::elliptics::lookup_result_entry &entry, std::string sign_port) 49 | : m_entry(entry) 50 | , m_sign_port(sign_port) 51 | { 52 | } 53 | 54 | const std::string &lookup_result::host() const 55 | { 56 | if (!m_host) { 57 | char hbuf[NI_MAXHOST]; 58 | memset(hbuf, 0, NI_MAXHOST); 59 | struct dnet_addr *addr = m_entry.storage_address(); 60 | 61 | if (getnameinfo((const sockaddr*)addr, addr->addr_len, hbuf, sizeof(hbuf), NULL, 0, 0) != 0) { 62 | throw std::runtime_error("can not make dns lookup"); 63 | } 64 | 65 | std::ostringstream oss; 66 | oss << hbuf; 67 | if (!m_sign_port.empty()) { 68 | oss << ':' << m_sign_port; 69 | } 70 | 71 | m_host.reset(oss.str()); 72 | } 73 | 74 | return *m_host; 75 | } 76 | 77 | uint16_t lookup_result::port() const 78 | { 79 | if (!m_port) { 80 | struct dnet_addr *addr = m_entry.storage_address(); 81 | m_port.reset(dnet_addr_port(addr)); 82 | } 83 | 84 | return *m_port; 85 | } 86 | 87 | int lookup_result::group() const 88 | { 89 | return m_entry.command()->id.group_id; 90 | } 91 | 92 | int lookup_result::status() const 93 | { 94 | return m_entry.command()->status; 95 | } 96 | 97 | const std::string &lookup_result::addr() const 98 | { 99 | if (!m_addr) { 100 | char addr_dst[512]; 101 | struct dnet_addr *addr = m_entry.storage_address(); 102 | dnet_addr_string_raw(addr, addr_dst, sizeof (addr_dst) - 1); 103 | std::string tmp(addr_dst); 104 | m_addr.reset(addr_dst); 105 | } 106 | 107 | return *m_addr; 108 | } 109 | 110 | const std::string &lookup_result::path() const 111 | { 112 | if (!m_path) { 113 | std::string p; 114 | struct dnet_file_info *info = m_entry.file_info(); 115 | p = m_entry.file_path(); 116 | std::ostringstream oss; 117 | oss 118 | << p << ':' << info->offset 119 | << ':' << info->size; 120 | m_path.reset(oss.str()); 121 | } 122 | 123 | return *m_path; 124 | } 125 | 126 | const std::string &lookup_result::full_path() const 127 | { 128 | if (!m_full_path) { 129 | struct dnet_file_info *info = m_entry.file_info(); 130 | m_full_path.reset((char *)(info + 1)); 131 | } 132 | 133 | return *m_full_path; 134 | } 135 | 136 | } // namespace elliptics 137 | -------------------------------------------------------------------------------- /src/timer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__TIMER__HPP 21 | #define MDS_PROXY__SRC__TIMER__HPP 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace util { 29 | 30 | namespace detail { 31 | 32 | template 33 | struct timer_traits; 34 | 35 | template <> 36 | struct timer_traits { 37 | static 38 | std::string 39 | suffix() { return "s"; } 40 | }; 41 | 42 | template <> 43 | struct timer_traits { 44 | static 45 | std::string 46 | suffix() { return "ms"; } 47 | }; 48 | 49 | template <> 50 | struct timer_traits { 51 | static 52 | std::string 53 | suffix() { return "us"; } 54 | }; 55 | 56 | template <> 57 | struct timer_traits { 58 | static 59 | std::string 60 | suffix() { return "ns"; } 61 | }; 62 | 63 | } // namespace detail 64 | 65 | template 66 | class timer { 67 | public: 68 | typedef Clock clock_type; 69 | typedef typename clock_type::time_point time_point_type; 70 | 71 | timer() { 72 | reset(); 73 | } 74 | 75 | void 76 | reset() { 77 | time_point = clock_type::now(); 78 | } 79 | 80 | template 81 | typename Duration::rep 82 | get() const { 83 | return std::chrono::duration_cast( 84 | clock_type::now() - time_point).count(); 85 | } 86 | 87 | template 88 | std::string 89 | str() const { 90 | return boost::lexical_cast(get()) 91 | + detail::timer_traits::suffix(); 92 | } 93 | 94 | /* 95 | * Specialize method get() and method str() for certain durations. 96 | * Methods will have a form get_xxx() and str_xxx() with 'xxx' that determines 97 | * certain duration. For instance the suffix 'ms' means milliseconds. 98 | */ 99 | #define SPECIALIZE_FOR_DURATION(duration, suffix) \ 100 | duration::rep \ 101 | get_##suffix() const { \ 102 | return get(); \ 103 | } \ 104 | \ 105 | std::string \ 106 | str_##suffix() const { \ 107 | return str(); \ 108 | } 109 | 110 | SPECIALIZE_FOR_DURATION(std::chrono::seconds, s) 111 | SPECIALIZE_FOR_DURATION(std::chrono::milliseconds, ms) 112 | SPECIALIZE_FOR_DURATION(std::chrono::microseconds, us) 113 | SPECIALIZE_FOR_DURATION(std::chrono::nanoseconds, ns) 114 | 115 | #undef SPECIALIZE_FOR_DURATION 116 | 117 | private: 118 | time_point_type time_point; 119 | }; 120 | 121 | typedef timer system_timer_t; 122 | //typedef timer steady_timer_t; 123 | typedef timer high_resolution_timer_t; 124 | 125 | typedef system_timer_t timer_t; 126 | 127 | } // namespace util 128 | 129 | #endif /* MDS_PROXY__SRC__TIMER__HPP */ 130 | 131 | -------------------------------------------------------------------------------- /src/remove.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "remove.hpp" 21 | 22 | #include "timer.hpp" 23 | #include "utils.hpp" 24 | 25 | #define logger() *shared_logger 26 | 27 | elliptics::remove_result_t::remove_result_t(bool has_bad_response_, bool not_found_) 28 | : has_bad_response(has_bad_response_) 29 | , not_found(not_found_) 30 | {} 31 | 32 | bool 33 | elliptics::remove_result_t::is_failed() const { 34 | return has_bad_response; 35 | } 36 | 37 | bool 38 | elliptics::remove_result_t::is_successful() const { 39 | return !has_bad_response; 40 | } 41 | 42 | bool 43 | elliptics::remove_result_t::key_was_not_found() const { 44 | return not_found; 45 | } 46 | 47 | namespace { 48 | 49 | void 50 | remove_was_done(shared_logger_t shared_logger 51 | , const ioremap::elliptics::sync_remove_result &entries 52 | , const ioremap::elliptics::error_info &error_info 53 | , const std::string &key, util::timer_t timer, size_t groups_count 54 | , util::expected::callback_t next) { 55 | { 56 | using elliptics::operator <<; 57 | 58 | std::ostringstream oss; 59 | oss << "remove was done: key=" << key << "; spent-time=" << timer.str_ms() 60 | << "; " << error_info; 61 | auto msg = oss.str(); 62 | 63 | if (error_info) { 64 | MDS_LOG_ERROR("%s", msg.c_str()); 65 | } else { 66 | MDS_LOG_INFO("%s", msg.c_str()); 67 | } 68 | } 69 | 70 | bool has_bad_response = false; 71 | size_t enoent_count = 0; 72 | 73 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 74 | auto status = it->status(); 75 | 76 | if (status != 0) { 77 | if (status != -ENOENT) { 78 | has_bad_response = true; 79 | } else { 80 | enoent_count += 1; 81 | } 82 | } 83 | } 84 | 85 | // The reason for this check: ELL-250 86 | if (entries.size() != groups_count) { 87 | has_bad_response = true; 88 | } 89 | 90 | elliptics::remove_result_t remove_result{has_bad_response, enoent_count == entries.size()}; 91 | next(remove_result); 92 | } 93 | 94 | } // namespace 95 | 96 | void 97 | elliptics::remove(shared_logger_t shared_logger 98 | , ioremap::elliptics::session session 99 | , std::string key 100 | , util::expected::callback_t next) { 101 | { 102 | std::ostringstream oss; 103 | oss << "remove: key=\"" << key << "\"; groups=" << session.get_groups(); 104 | auto msg = oss.str(); 105 | MDS_LOG_INFO("%s", msg.c_str()); 106 | } 107 | 108 | util::timer_t timer; 109 | 110 | session = session.clone(); 111 | session.set_filter(ioremap::elliptics::filters::all_with_ack); 112 | 113 | auto future = session.remove(key); 114 | 115 | auto next_ = std::bind(remove_was_done, std::move(shared_logger) 116 | , std::placeholders::_1, std::placeholders::_2 117 | , key, timer, session.get_groups().size(), std::move(next)); 118 | 119 | future.connect(next_); 120 | } 121 | 122 | -------------------------------------------------------------------------------- /src/utils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__UTILS__HPP 21 | #define MDS_PROXY__SRC__UTILS__HPP 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | namespace elliptics { 43 | 44 | typedef int group_t; 45 | typedef std::vector groups_t; 46 | 47 | typedef std::vector couple_t; 48 | 49 | template 50 | std::ostream & 51 | operator << (std::ostream &stream, const std::vector &vector) { 52 | stream << '['; 53 | 54 | { 55 | // The reason for declarations before for-loop is compiler' bug (gcc <= 4.6): 56 | // 'variable ‘auto begin’ with ‘auto’ type used in its own initializer' 57 | // in statement: 58 | // auto begin = vector.begin(), it = begin; 59 | auto begin = vector.begin(), end = vector.end(); 60 | for (auto it = begin; it != end; ++it) { 61 | if (it != begin) { 62 | stream << ", "; 63 | } 64 | 65 | stream << *it; 66 | } 67 | } 68 | 69 | stream << ']'; 70 | 71 | return stream; 72 | } 73 | 74 | std::ostream & 75 | operator << (std::ostream &stream, const ioremap::elliptics::error_info &error_info); 76 | 77 | template 78 | std::shared_ptr 79 | make_request_stream(Server *server 80 | , const std::shared_ptr &reply 81 | , Args &&...args) { 82 | auto request_stream = std::make_shared(std::forward(args)...); 83 | request_stream->set_server(server); 84 | request_stream->initialize(reply); 85 | return request_stream; 86 | } 87 | 88 | std::string 89 | encode_for_xml(const std::string &string); 90 | 91 | std::string 92 | url_encode(const std::string &string); 93 | 94 | struct file_location_t { 95 | std::string host; 96 | std::string path; 97 | }; 98 | 99 | file_location_t 100 | make_file_location(const ioremap::elliptics::sync_lookup_result &slr 101 | , const mastermind::namespace_state_t &ns_state); 102 | 103 | std::string 104 | make_signature_ts(boost::optional optional_expiration_time 105 | , const mastermind::namespace_state_t &ns_state); 106 | 107 | std::string 108 | make_signature_message(const file_location_t &file_location, const std::string &ts 109 | , const std::vector> &args 110 | = std::vector>{}); 111 | 112 | std::string 113 | make_signature(const std::string &message, const std::string &token); 114 | 115 | } // namespace elliptics 116 | 117 | #endif /* MDS_PROXY__SRC__UTILS__HPP */ 118 | 119 | -------------------------------------------------------------------------------- /src/upload_simple.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__UPLOAD_SIMPLE__HPP 21 | #define MDS_PROXY__SRC__UPLOAD_SIMPLE__HPP 22 | 23 | #include "upload.hpp" 24 | #include "writer.hpp" 25 | #include "couple_iterator.hpp" 26 | #include "expected.hpp" 27 | #include "buffered_writer.hpp" 28 | #include "deferred_function.hpp" 29 | #include "remove.hpp" 30 | 31 | #include 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace elliptics { 39 | 40 | struct upload_simple_t 41 | : public ioremap::thevoid::buffered_request_stream 42 | , public std::enable_shared_from_this 43 | { 44 | upload_simple_t(mastermind::namespace_state_t ns_state_ 45 | , couple_iterator_t couple_iterator_, std::string filename_); 46 | 47 | void 48 | on_request(const ioremap::thevoid::http_request &http_request); 49 | 50 | void 51 | on_chunk(const boost::asio::const_buffer &buffer, unsigned int flags); 52 | 53 | void 54 | on_error(const boost::system::error_code &error_code); 55 | 56 | void 57 | on_write_is_done(const std::error_code &error_code); 58 | 59 | void 60 | send_result(); 61 | 62 | void 63 | headers_are_sent(const std::string &res_str, const boost::system::error_code &error_code); 64 | 65 | void 66 | data_is_sent(const boost::system::error_code &error_code); 67 | 68 | void 69 | fallback(); 70 | 71 | void 72 | remove(const util::expected::callback_t next); 73 | 74 | private: 75 | enum class internal_error_errc { 76 | none 77 | , general_error 78 | , insufficient_storage 79 | }; 80 | 81 | void 82 | get_next_couple_info(util::expected::callback_t next); 83 | 84 | void 85 | process_couple_info(mastermind::couple_info_t couple_info_); 86 | 87 | void 88 | process_chunk(ioremap::elliptics::data_pointer chunk); 89 | 90 | void 91 | process_chunk_write_error(const std::error_code &error_code); 92 | 93 | std::shared_ptr 94 | make_writer(const groups_t &groups); 95 | 96 | void 97 | update_internal_error(internal_error_errc errc); 98 | 99 | void 100 | send_error(); 101 | 102 | void 103 | send_error(internal_error_errc errc); 104 | 105 | mastermind::namespace_state_t ns_state; 106 | couple_iterator_t couple_iterator; 107 | std::string filename; 108 | std::string key; 109 | 110 | std::shared_ptr writer; 111 | ioremap::elliptics::data_pointer data_pointer; 112 | 113 | deferred_function_t deferred_fallback; 114 | 115 | boost::optional lookup_session; 116 | boost::optional write_session; 117 | 118 | size_t offset; 119 | mastermind::couple_info_t couple_info; 120 | bool can_retry_couple; 121 | 122 | size_t attempt_to_choose_a_couple; 123 | 124 | internal_error_errc internal_error; 125 | }; 126 | 127 | } // namespace elliptics 128 | 129 | #endif /* MDS_PROXY__SRC__UPLOAD_SIMPLE__HPP */ 130 | 131 | -------------------------------------------------------------------------------- /src/lookuper.cpp: -------------------------------------------------------------------------------- 1 | #include "lookuper.hpp" 2 | #include "loggers.hpp" 3 | 4 | elliptics::parallel_lookuper_t::parallel_lookuper_t( 5 | ioremap::swarm::logger bh_logger_ 6 | , ioremap::elliptics::session session_ 7 | , std::string key_ 8 | ) 9 | : bh_logger(std::move(bh_logger_)) 10 | , session(session_.clone()) 11 | , key(std::move(key_)) 12 | , groups_to_handle(0) 13 | { 14 | } 15 | 16 | void 17 | elliptics::parallel_lookuper_t::start() { 18 | const auto &groups = session.get_groups(); 19 | groups_to_handle = groups.size(); 20 | 21 | auto self = shared_from_this(); 22 | auto callback = [this, self] ( 23 | const ioremap::elliptics::sync_lookup_result &entries 24 | , const ioremap::elliptics::error_info &error_info) { 25 | on_lookup(entries, error_info); 26 | }; 27 | 28 | for (auto it = groups.begin(), end = groups.end(); it != end; ++it) { 29 | auto group_session = session.clone(); 30 | group_session.set_filter(ioremap::elliptics::filters::all_with_ack); 31 | group_session.set_groups({*it}); 32 | auto future = group_session.lookup(key); 33 | future.connect(callback); 34 | } 35 | } 36 | 37 | ioremap::elliptics::async_lookup_result 38 | elliptics::parallel_lookuper_t::next_lookup_result() { 39 | lock_guard_t lock_guard(results_mutex); 40 | 41 | ioremap::elliptics::async_lookup_result future(session); 42 | ioremap::elliptics::async_lookup_result::handler promise(future); 43 | 44 | if (!results.empty()) { 45 | auto result = std::move(results.front()); 46 | results.pop_front(); 47 | 48 | lock_guard.unlock(); 49 | process_promise(promise, result); 50 | return future; 51 | } 52 | 53 | if (groups_to_handle - promises.size()) { 54 | promises.emplace_back(std::move(promise)); 55 | return future; 56 | } 57 | 58 | lock_guard.unlock(); 59 | process_promise(promise); 60 | return future; 61 | } 62 | 63 | size_t 64 | elliptics::parallel_lookuper_t::total_size() const { 65 | return session.get_groups().size(); 66 | } 67 | 68 | size_t 69 | elliptics::parallel_lookuper_t::results_left() const { 70 | lock_guard_t lock_guard(results_mutex); 71 | return groups_to_handle + results.size(); 72 | } 73 | 74 | ioremap::swarm::logger & 75 | elliptics::parallel_lookuper_t::logger() { 76 | return bh_logger; 77 | } 78 | 79 | void 80 | elliptics::parallel_lookuper_t::on_lookup(const ioremap::elliptics::sync_lookup_result &entries 81 | , const ioremap::elliptics::error_info &error_info) { 82 | lock_guard_t lock_guard(results_mutex); 83 | 84 | groups_to_handle -= 1; 85 | result_t result{entries, error_info}; 86 | 87 | if (!promises.empty()) { 88 | auto promise = std::move(promises.front()); 89 | promises.pop_front(); 90 | 91 | lock_guard.unlock(); 92 | process_promise(promise, result); 93 | return; 94 | } 95 | 96 | results.emplace_back(std::move(result)); 97 | } 98 | 99 | void 100 | elliptics::parallel_lookuper_t::process_promise( 101 | ioremap::elliptics::async_lookup_result::handler &promise 102 | , const result_t &result) { 103 | auto &entries = result.entries; 104 | 105 | promise.set_total(entries.size()); 106 | 107 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 108 | promise.process(*it); 109 | } 110 | 111 | promise.complete(result.error_info); 112 | } 113 | 114 | void 115 | elliptics::parallel_lookuper_t::process_promise( 116 | ioremap::elliptics::async_lookup_result::handler &promise) { 117 | promise.complete(ioremap::elliptics::error_info(EPIPE, "There is no enough groups")); 118 | } 119 | 120 | elliptics::parallel_lookuper_ptr_t 121 | elliptics::make_parallel_lookuper( 122 | ioremap::swarm::logger bh_logger 123 | , ioremap::elliptics::session session 124 | , std::string key 125 | ) { 126 | auto parallel_lookuper = std::make_shared(std::move(bh_logger) 127 | , std::move(session), std::move(key)); 128 | parallel_lookuper->start(); 129 | return parallel_lookuper; 130 | } 131 | 132 | -------------------------------------------------------------------------------- /src/buffered_writer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__BUFFERED_WRITER__HPP 21 | #define MDS_PROXY__SRC__BUFFERED_WRITER__HPP 22 | 23 | #include "writer.hpp" 24 | 25 | #include 26 | 27 | namespace elliptics { 28 | 29 | enum class buffered_writer_errc { 30 | success 31 | , interrupted 32 | , unexpected_event 33 | }; 34 | 35 | const std::error_category & 36 | buffered_writer_category(); 37 | 38 | std::error_code 39 | make_error_code(buffered_writer_errc e); 40 | 41 | std::error_condition 42 | make_error_condition(buffered_writer_errc e); 43 | 44 | class buffered_writer_error : public std::system_error 45 | { 46 | public: 47 | buffered_writer_error(buffered_writer_errc e, const std::string &message = ""); 48 | }; 49 | 50 | class buffered_writer_t : public std::enable_shared_from_this 51 | { 52 | public: 53 | typedef std::function callback_t; 54 | typedef std::shared_ptr writer_ptr_t; 55 | 56 | buffered_writer_t(ioremap::swarm::logger bh_logger_, std::string key_, size_t chunk_size_); 57 | 58 | void 59 | append(const char *data, size_t size); 60 | 61 | void 62 | write(const ioremap::elliptics::session &session, size_t commit_coef 63 | , size_t success_copies_num, size_t limit_of_middle_chunk_attempts 64 | , double scale_retry_timeout, callback_t next); 65 | 66 | void 67 | interrupt(); 68 | 69 | const std::string & 70 | get_key() const; 71 | 72 | bool 73 | is_finished() const; 74 | 75 | bool 76 | is_completed() const; 77 | 78 | bool 79 | is_failed() const; 80 | 81 | bool 82 | is_interrupted() const; 83 | 84 | const writer_ptr_t & 85 | get_writer() const; 86 | 87 | const writer_t::result_t & 88 | get_result() const; 89 | 90 | private: 91 | enum class state_tag { 92 | appending 93 | , writing 94 | , interrupting 95 | , completed 96 | , failed 97 | , interrupted 98 | }; 99 | 100 | typedef std::mutex mutex_t; 101 | typedef std::unique_lock lock_guard_t; 102 | typedef std::vector buffer_t; 103 | 104 | ioremap::swarm::logger & 105 | logger(); 106 | 107 | void 108 | append_impl(const char *data, size_t size); 109 | 110 | void 111 | write_impl(lock_guard_t &lock_guard 112 | , const ioremap::elliptics::session &session, size_t commit_coef 113 | , size_t success_copies_num, size_t limit_of_middle_chunk_attempts 114 | , double scale_retry_timeout, callback_t next); 115 | 116 | void 117 | write_chunk(lock_guard_t &lock_guard, callback_t next); 118 | 119 | void 120 | on_chunk_wrote(const std::error_code &error_code, callback_t next); 121 | 122 | state_tag state; 123 | mutable mutex_t state_mutex; 124 | 125 | ioremap::swarm::logger bh_logger; 126 | 127 | std::string key; 128 | size_t chunk_size; 129 | 130 | std::list buffers; 131 | 132 | size_t total_size; 133 | 134 | writer_ptr_t writer; 135 | writer_t::result_t result; 136 | }; 137 | 138 | } // namespace elliptics 139 | 140 | namespace std { 141 | 142 | template <> 143 | struct is_error_code_enum 144 | : public true_type 145 | {}; 146 | 147 | } // namespace std 148 | 149 | #endif /* MDS_PROXY__SRC__BUFFERED_WRITER__HPP */ 150 | 151 | -------------------------------------------------------------------------------- /src/ranges.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "ranges.hpp" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace elliptics { 30 | 31 | struct range_impl_t { 32 | enum types_tag { 33 | prefix, 34 | part, 35 | suffix 36 | }; 37 | 38 | types_tag type; 39 | size_t first; 40 | size_t last; 41 | size_t size; 42 | 43 | range_impl_t() 44 | : type(part) 45 | , first(0) 46 | , last(-1) 47 | {} 48 | }; 49 | 50 | size_t parse_pos(const char *&s) { 51 | char *end = 0; 52 | size_t res = strtoul(s, &end, 10); 53 | if (end == s) { 54 | throw std::runtime_error("Header is malformed"); 55 | } 56 | s = end; 57 | return res; 58 | } 59 | 60 | range_impl_t parse_range(const char *&s) { 61 | range_impl_t range; 62 | if (*s == '-') { 63 | range.type = range_impl_t::suffix; 64 | range.last = parse_pos(++s); 65 | } else { 66 | range.first = parse_pos(s); 67 | if (*s != '-') { 68 | throw std::runtime_error("Header is malformed"); 69 | } 70 | ++s; 71 | if (isdigit(*s)) { 72 | range.last = parse_pos(s); 73 | range.type = range_impl_t::part; 74 | } else { 75 | range.type = range_impl_t::prefix; 76 | } 77 | } 78 | return range; 79 | } 80 | 81 | std::vector parse_header_impl(const char *s, size_t total_size) { 82 | std::vector res; 83 | if (strncmp(s, "bytes=", sizeof("bytes=") - 1)) { 84 | throw std::runtime_error("Header is malformed"); 85 | } 86 | s += sizeof("bytes=") - 1; 87 | size_t last_byte = total_size - 1; 88 | while (*s) { 89 | while (isspace(*s)) ++s; 90 | auto range = parse_range(s); 91 | 92 | if (range.type != range_impl_t::suffix) { 93 | if (range.type == range_impl_t::part && range.first > range.last) { 94 | throw std::runtime_error("Header is malformed"); 95 | } 96 | 97 | if (range.type == range_impl_t::prefix || range.last >= total_size) { 98 | range.last = last_byte; 99 | } 100 | 101 | if (range.first > last_byte) { 102 | continue; 103 | } 104 | } else { 105 | if (total_size < range.last) { 106 | range.last = total_size; 107 | } 108 | } 109 | 110 | res.push_back(range); 111 | 112 | while (isspace(*s)) ++s; 113 | if (*s && *s != ',') { 114 | throw std::runtime_error("Header is malformed"); 115 | } 116 | if (*s) ++s; 117 | } 118 | return res; 119 | } 120 | 121 | boost::optional parse_range_header(const std::string &header, size_t total_size) { 122 | try { 123 | auto res_impl = parse_header_impl(header.c_str(), total_size); 124 | if (res_impl.empty()) { 125 | return boost::none; 126 | } 127 | ranges_t res; 128 | 129 | for (auto it = res_impl.begin(), end = res_impl.end(); it != end; ++it) { 130 | range_t range; 131 | if (it->type != range_impl_t::suffix) { 132 | range.offset = it->first; 133 | range.size = it->last - it->first + 1; 134 | } else { 135 | range.offset = total_size - it->last; 136 | range.size = it->last; 137 | } 138 | res.push_back(range); 139 | } 140 | 141 | return res; 142 | } catch (...) { 143 | return boost::none; 144 | } 145 | } 146 | 147 | } // namespace elliptics 148 | 149 | -------------------------------------------------------------------------------- /ubic/perl5/MDSProxy.pm: -------------------------------------------------------------------------------- 1 | package Ubic::Service::MDSProxy; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use Ubic::Daemon qw(:all); 7 | use Ubic::Result qw(result); 8 | ###use Ubic::Service::Shared::Ulimits; 9 | use Ubic::Service::Shared::Dirs; 10 | use parent qw(Ubic::Service::SimpleDaemon); 11 | 12 | use Params::Validate qw(:all); 13 | use JSON qw( decode_json from_json ); 14 | use LWP::UserAgent; 15 | use HTTP::Request::Common; 16 | use LWP::Protocol::http::SocketUnixAlt; 17 | 18 | my %opt2arg = (); 19 | for my $arg (qw(configuration server:announce server:announce-interval)) 20 | { 21 | my $opt = $arg; 22 | $opt2arg{$opt} = $arg; 23 | } 24 | 25 | use vars qw($params); 26 | 27 | sub new { 28 | my $class = shift; 29 | 30 | $params = validate(@_, { 31 | user => { type => SCALAR, default => "root", optional => 1 }, 32 | log_dir => { type => SCALAR, default => "/var/log/mdst", optional => 1 }, 33 | run_dir => { type => SCALAR, default => "/var/run/mediastorage", optional => 1 }, 34 | conf_file => { type => SCALAR, default => "/etc/elliptics/mediastorage-proxy.conf", optional => 1 }, 35 | rlimit_nofile => { type => SCALAR, 36 | regex => qr/^\d+$/, 37 | optional => 1, 38 | }, 39 | rlimit_core => { type => SCALAR, 40 | regex => qr/^(\-?\d+|unlimited)$/, 41 | optional => 1 }, 42 | rlimit_stack => { type => SCALAR, 43 | regex => qr/^\d+$/, 44 | optional => 1 }, 45 | }); 46 | 47 | # 48 | # check ulimits 49 | # 50 | my $ulimits; 51 | if (defined $params->{rlimit_nofile}) { $ulimits->{"RLIMIT_NOFILE"} = $params->{rlimit_nofile} }; 52 | if (defined $params->{rlimit_core}) { $ulimits->{"RLIMIT_CORE"} = $params->{rlimit_core} }; 53 | if (defined $params->{rlimit_stack}) { $ulimits->{"RLIMIT_STACK"} = $params->{rlimit_stack} }; 54 | 55 | my $bin = [ 56 | '/usr/bin/mediastorage-proxy -c '.$params->{conf_file} 57 | ]; 58 | 59 | my $daemon_user = $params->{user}; 60 | return $class->SUPER::new({ 61 | bin => $bin, 62 | user => 'root', 63 | ulimit => $ulimits || {}, 64 | daemon_user => $daemon_user, 65 | ubic_log => $params->{log_dir}.'/ubic.log', 66 | stdout => $params->{log_dir}.'/stdout.log', 67 | stderr => $params->{log_dir}.'/stderr.log', 68 | auto_start => 1, 69 | }); 70 | } 71 | 72 | sub start_impl { 73 | my $self = shift; 74 | local $/; 75 | open(my $fh, '<'.$params->{conf_file}); 76 | my $json_fh = <$fh> ; 77 | my $json = decode_json($json_fh); 78 | foreach my $socket (@{$json->{endpoints}}) { 79 | $socket =~ s/^unix://; 80 | if ( -S "$socket" ) { 81 | unlink ($socket); 82 | } 83 | } 84 | 85 | Ubic::Service::Shared::Dirs::directory_checker( $params->{log_dir}, $params->{user}); 86 | Ubic::Service::Shared::Dirs::directory_checker( $params->{run_dir}, $params->{user}); 87 | 88 | $self->SUPER::start_impl(@_); 89 | } 90 | 91 | 92 | sub status_impl { 93 | my $self = shift; 94 | my $status = $self->SUPER::status_impl(); 95 | 96 | return $status if $status->status ne 'running'; 97 | 98 | local $/; 99 | open(my $fh, '<'.$params->{conf_file}); 100 | my $json_fh = <$fh> ; 101 | my $json = decode_json($json_fh); 102 | #print "js: ".$json->{endpoints}[0]."\n"; 103 | my $socket_file = $json->{endpoints}[0]; 104 | 105 | LWP::Protocol::implementor( http => 'LWP::Protocol::http::SocketUnixAlt' ); 106 | my $ua = LWP::UserAgent->new; 107 | $socket_file =~ s/^.*?\/(.*)/$1/; #strip first '/' and 'unix:' from path 108 | my $resp = $ua->request(GET "http:$socket_file//ping"); 109 | #print "rc:'".$resp->code."'\n"; 110 | 111 | if ( !defined $resp->code || ( $resp->code ne 200) ) { 112 | my $res = ''; 113 | if (defined $resp->code) { $res = $resp->code; } 114 | return result("broken", "http through socket $socket_file didn't return 200 (result: '$res')"); 115 | }; 116 | 117 | return "running"; 118 | } 119 | 120 | sub timeout_options { 121 | my $self = shift; 122 | { start => { step => 1, trials => 10 } }; 123 | } 124 | 125 | sub reload { 126 | my ( $self ) = @_; 127 | my $status = check_daemon($self->pidfile) or die result('not running'); 128 | kill HUP => $status->pid; 129 | 130 | return 'reloaded'; 131 | } 132 | 133 | 1; 134 | 135 | -------------------------------------------------------------------------------- /src/upload_multipart.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__UPLOAD_MULTIPART__HPP 21 | #define MDS_PROXY__SRC__UPLOAD_MULTIPART__HPP 22 | 23 | #include "upload.hpp" 24 | #include "writer.hpp" 25 | #include "couple_iterator.hpp" 26 | #include "expected.hpp" 27 | #include "buffered_writer.hpp" 28 | #include "deferred_function.hpp" 29 | #include "remove.hpp" 30 | 31 | #include 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | namespace elliptics { 39 | 40 | struct upload_multipart_t 41 | : public ioremap::thevoid::request_stream 42 | , public std::enable_shared_from_this 43 | { 44 | upload_multipart_t(mastermind::namespace_state_t ns_state_, couple_t couple_); 45 | 46 | void 47 | on_headers(ioremap::thevoid::http_request &&http_request_); 48 | 49 | size_t 50 | on_data(const boost::asio::const_buffer &buffer); 51 | 52 | void 53 | on_close(const boost::system::error_code &error); 54 | 55 | private: 56 | 57 | enum class multipart_state_tag { 58 | init, headers, body, after_body, end 59 | }; 60 | 61 | class multipart_context_t { 62 | public: 63 | typedef std::vector buffer_t; 64 | typedef buffer_t::const_iterator const_iterator; 65 | 66 | multipart_context_t(); 67 | 68 | const_iterator 69 | begin() const; 70 | 71 | const_iterator 72 | end() const; 73 | 74 | size_t 75 | size() const; 76 | 77 | void 78 | append(const char *data, size_t size); 79 | 80 | void 81 | skip(size_t size); 82 | 83 | void 84 | trim(); 85 | 86 | void 87 | interrupt(bool is_error_); 88 | 89 | bool 90 | interrupted() const; 91 | 92 | bool 93 | error() const; 94 | 95 | multipart_state_tag state; 96 | 97 | private: 98 | void 99 | reset(); 100 | 101 | buffer_t buffer; 102 | buffer_t::const_iterator iterator; 103 | 104 | bool need_data; 105 | bool is_error; 106 | bool is_interrupted; 107 | } multipart_context; 108 | 109 | enum class error_type_tag { 110 | none 111 | , insufficient_storage 112 | , internal 113 | , multipart 114 | , client 115 | }; 116 | 117 | void sm_init(); 118 | void sm_headers(); 119 | void sm_body(); 120 | void sm_after_body(); 121 | void sm_end(); 122 | 123 | void start_writing(); 124 | 125 | void 126 | on_writer_is_finished(const std::error_code &error_code); 127 | 128 | void 129 | set_error(error_type_tag e); 130 | 131 | bool 132 | is_error(); 133 | 134 | error_type_tag 135 | get_error(); 136 | 137 | void 138 | interrupt_writers(error_type_tag e); 139 | 140 | void 141 | interrupt_writers(); 142 | 143 | void 144 | on_writers_are_finished(); 145 | 146 | void 147 | send_result(); 148 | 149 | void 150 | headers_are_sent(const std::string &res_str, const boost::system::error_code &error_code); 151 | 152 | void 153 | data_is_sent(const boost::system::error_code &error_code); 154 | 155 | void 156 | remove_files(); 157 | 158 | void 159 | on_removed(util::expected result); 160 | 161 | void 162 | send_error(); 163 | 164 | deferred_function_t interrupt_writers_once; 165 | deferred_function_t join_upload_tasks; 166 | deferred_function_t join_remove_tasks; 167 | 168 | error_type_tag error_type; 169 | std::mutex error_type_mutex; 170 | 171 | ioremap::thevoid::http_request http_request; 172 | mastermind::namespace_state_t ns_state; 173 | couple_t couple; 174 | int couple_id; 175 | 176 | std::string boundary; 177 | 178 | std::shared_ptr buffered_writer; 179 | std::string current_filename; 180 | 181 | std::mutex buffered_writers_mutex; 182 | std::map> buffered_writers; 183 | std::map results; 184 | }; 185 | 186 | } // namespace elliptics 187 | 188 | #endif /* MDS_PROXY__SRC__UPLOAD_MULTIPART__HPP */ 189 | 190 | -------------------------------------------------------------------------------- /src/delete.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "proxy.hpp" 21 | 22 | #include "delete.hpp" 23 | 24 | namespace elliptics { 25 | void req_delete::on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer) { 26 | (void) buffer; 27 | 28 | try { 29 | MDS_LOG_INFO("Delete: handle request: %s", req.url().path().c_str()); 30 | mastermind::namespace_state_t ns_state; 31 | url_str = req.url().path(); 32 | try { 33 | ns_state = server()->get_namespace_state(url_str, "/delete"); 34 | 35 | // The method runs in thevoid's io-loop, therefore proxy's dtor cannot run in this moment 36 | // Hence session can be safely used without any check 37 | auto &&prep_session = server()->prepare_session(url_str, ns_state); 38 | session = std::get<0>(prep_session); 39 | session->set_trace_bit(req.trace_bit()); 40 | session->set_trace_id(req.request_id()); 41 | key = std::get<1>(prep_session); 42 | } catch (const std::exception &ex) { 43 | MDS_LOG_INFO("Delete: request = \"%s\", err = \"%s\"", url_str.c_str(), ex.what()); 44 | send_reply(400); 45 | return; 46 | } 47 | 48 | if (!server()->check_basic_auth(ns_state.name() 49 | , ns_settings(ns_state).auth_key_for_write 50 | , req.headers().get("Authorization"))) { 51 | auto token = server()->get_auth_token(req.headers().get("Authorization")); 52 | MDS_LOG_INFO("%s: invalid token \"%s\"", url_str.c_str() 53 | , token.empty() ? "" : token.c_str()); 54 | ioremap::thevoid::http_response reply; 55 | ioremap::swarm::http_headers headers; 56 | 57 | reply.set_code(401); 58 | headers.add("WWW-Authenticate", std::string("Basic realm=\"") + ns_state.name() + "\""); 59 | headers.add("Content-Length", "0"); 60 | reply.set_headers(headers); 61 | send_reply(std::move(reply)); 62 | return; 63 | } 64 | 65 | if (session->state_num() < server()->die_limit()) { 66 | throw std::runtime_error("Too low number of existing states"); 67 | } 68 | 69 | session->set_timeout(server()->timeout.lookup); 70 | session->set_filter(ioremap::elliptics::filters::positive); 71 | 72 | if (ns_settings(ns_state).check_for_update) { 73 | session->set_cflags(session->get_cflags() | DNET_FLAGS_NOLOCK); 74 | } 75 | 76 | auto alr = session->quorum_lookup(key); 77 | alr.connect(wrap(std::bind(&req_delete::on_lookup, 78 | shared_from_this(), std::placeholders::_1, std::placeholders::_2))); 79 | } catch (const std::exception &ex) { 80 | MDS_LOG_ERROR("Delete request=\"%s\" error: %s" 81 | , url_str.c_str(), ex.what()); 82 | send_reply(500); 83 | } catch (...) { 84 | MDS_LOG_ERROR("Delete request=\"%s\" error: unknown" 85 | , url_str.c_str()); 86 | send_reply(500); 87 | } 88 | } 89 | 90 | void req_delete::on_lookup(const ioremap::elliptics::sync_lookup_result &slr, const ioremap::elliptics::error_info &error) { 91 | 92 | if (error) { 93 | MDS_LOG_ERROR("Delete request=\"%s\" lookup error: %s" 94 | , url_str.c_str(), error.message().c_str()); 95 | send_reply(error.code() == -ENOENT ? 404 : 500); 96 | return; 97 | } 98 | 99 | session->set_cflags(session->get_cflags() & ~DNET_FLAGS_NOLOCK); 100 | 101 | const auto &entry = slr.front(); 102 | total_size = entry.file_info()->size; 103 | 104 | session->set_timeout(server()->timeout.remove); 105 | 106 | MDS_LOG_DEBUG("Delete %s: data size %d" 107 | , url_str.c_str(), static_cast(total_size)); 108 | 109 | auto next = std::bind(&req_delete::on_finished, shared_from_this(), std::placeholders::_1); 110 | elliptics::remove(make_shared_logger(logger()), *session, key.remote(), std::move(next)); 111 | } 112 | 113 | void req_delete::on_finished(util::expected result) { 114 | try { 115 | auto remove_result = result.get(); 116 | 117 | if (remove_result.is_failed()) { 118 | send_reply(500); 119 | return; 120 | } 121 | 122 | if (remove_result.key_was_not_found()) { 123 | send_reply(404); 124 | return; 125 | } 126 | 127 | send_reply(200); 128 | } catch (const std::exception &ex) { 129 | MDS_LOG_ERROR("remove error: %s", ex.what()); 130 | send_reply(500); 131 | } 132 | } 133 | 134 | } // namespace elliptics 135 | 136 | -------------------------------------------------------------------------------- /src/writer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__UPLOADER__HPP 21 | #define MDS_PROXY__SRC__UPLOADER__HPP 22 | 23 | #include "loggers.hpp" 24 | #include "expected.hpp" 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace elliptics { 38 | 39 | enum class writer_errc { 40 | success 41 | , unexpected_event 42 | , incorrect_size 43 | , internal 44 | , insufficient_storage 45 | }; 46 | 47 | const std::error_category & 48 | writer_category(); 49 | 50 | std::error_code 51 | make_error_code(writer_errc e); 52 | 53 | std::error_condition 54 | make_error_condition(writer_errc e); 55 | 56 | class writer_error : public std::system_error 57 | { 58 | public: 59 | writer_error(writer_errc e, const std::string &message = ""); 60 | }; 61 | 62 | class writer_t : public std::enable_shared_from_this 63 | { 64 | public: 65 | typedef std::function callback_t; 66 | 67 | struct entry_info_t { 68 | std::string address; 69 | std::string path; 70 | int group; 71 | }; 72 | 73 | typedef std::vector entries_info_t; 74 | 75 | struct result_t { 76 | std::string id; 77 | std::string key; 78 | size_t total_size; 79 | entries_info_t entries_info; 80 | }; 81 | 82 | writer_t(ioremap::swarm::logger bh_logger_ 83 | , const ioremap::elliptics::session &session_, std::string key_ 84 | , size_t total_size_, size_t offset_, size_t commit_coef_, size_t success_copies_num_ 85 | , size_t limit_of_attempts_ = 1, double scale_retry_timeout_ = 1 86 | ); 87 | 88 | void 89 | write(const ioremap::elliptics::data_pointer &data_pointer, callback_t next); 90 | 91 | result_t 92 | get_result() const; 93 | 94 | const entries_info_t & 95 | get_entries_info() const; 96 | 97 | bool 98 | is_finished() const; 99 | 100 | bool 101 | is_committed() const; 102 | 103 | bool 104 | is_failed() const; 105 | 106 | size_t 107 | get_total_size() const; 108 | 109 | const std::string & 110 | get_key() const; 111 | 112 | std::string 113 | get_id() const; 114 | 115 | private: 116 | enum class state_tag { 117 | waiting 118 | , writing 119 | , committing 120 | , committed 121 | , failed 122 | }; 123 | 124 | typedef std::mutex mutex_t; 125 | typedef std::unique_lock lock_guard_t; 126 | 127 | ioremap::swarm::logger & 128 | logger(); 129 | 130 | void 131 | log_chunk(const std::string &write_type, size_t chunk_size); 132 | 133 | void 134 | update_groups(const ioremap::elliptics::sync_write_result &entries); 135 | 136 | void 137 | set_result(const ioremap::elliptics::sync_write_result &entries); 138 | 139 | bool 140 | write_is_good(const ioremap::elliptics::error_info &error_info); 141 | 142 | ioremap::elliptics::async_write_result 143 | write_impl(const ioremap::elliptics::data_pointer &data_pointer); 144 | 145 | elliptics::writer_errc 146 | choose_errc_for_client(const ioremap::elliptics::sync_write_result &entries); 147 | 148 | void 149 | on_data_wrote(const ioremap::elliptics::sync_write_result &entries 150 | , const ioremap::elliptics::error_info &error_info 151 | , callback_t next); 152 | 153 | state_tag state; 154 | mutable mutex_t state_mutex; 155 | writer_errc errc_for_client; 156 | 157 | ioremap::swarm::logger bh_logger; 158 | 159 | ioremap::elliptics::session session; 160 | ioremap::elliptics::key key; 161 | 162 | size_t total_size; 163 | size_t offset; 164 | size_t commit_coef; 165 | size_t success_copies_num; 166 | 167 | size_t limit_of_attempts; 168 | double scale_retry_timeout; 169 | 170 | size_t written_size; 171 | std::vector bad_groups; 172 | 173 | std::chrono::system_clock::time_point start_time; 174 | 175 | entries_info_t entries_info; 176 | }; 177 | 178 | void 179 | can_be_written(shared_logger_t shared_logger 180 | , ioremap::elliptics::session session 181 | , std::string key 182 | , mastermind::namespace_state_t ns_state 183 | , util::expected::callback_t next); 184 | 185 | } // namespace elliptics 186 | 187 | namespace std { 188 | 189 | template <> 190 | struct is_error_code_enum 191 | : public true_type 192 | {}; 193 | 194 | } // namespace std 195 | 196 | #endif /* MDS_PROXY__SRC__UPLOADER__HPP */ 197 | 198 | -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "utils.hpp" 21 | #include "ns_settings.hpp" 22 | #include "lookup_result.hpp" 23 | #include "hex.hpp" 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | std::ostream & 32 | elliptics::operator << (std::ostream &stream, const ioremap::elliptics::error_info &error_info) { 33 | stream << "status=\"" << (error_info ? "bad" : "ok") << "\"; description=\""; 34 | 35 | if (error_info) { 36 | stream << error_info.message() << "\"; "; 37 | } else { 38 | stream << "success\"; "; 39 | } 40 | 41 | return stream; 42 | } 43 | 44 | std::string 45 | elliptics::encode_for_xml(const std::string &string) { 46 | std::ostringstream oss; 47 | 48 | std::string::size_type pos = 0; 49 | 50 | do { 51 | auto pos2 = string.find_first_of("\"\'<>&", pos); 52 | oss << string.substr(pos, pos2 - pos); 53 | pos = pos2; 54 | 55 | if (pos != std::string::npos) { 56 | switch (string[pos]) { 57 | case '\"': oss << """; break; 58 | case '\'': oss << "'"; break; 59 | case '<': oss << "<"; break; 60 | case '>': oss << ">"; break; 61 | case '&': oss << "&"; break; 62 | } 63 | pos += 1; 64 | } 65 | } while (pos != std::string::npos && pos < string.size()); 66 | 67 | return oss.str(); 68 | } 69 | 70 | std::string 71 | elliptics::url_encode(const std::string &string) { 72 | std::string result; 73 | result.reserve(3 * string.size()); 74 | auto output = std::back_inserter(result); 75 | 76 | for (auto it = string.begin(), end = string.end(); it != end; ++it) { 77 | char symbol = *it; 78 | 79 | if (isalnum(symbol)) { 80 | *output++ = symbol; 81 | continue; 82 | } 83 | 84 | switch (symbol) { 85 | case '-': case '_': case '.': case '!': case '~': 86 | case '*': case '(': case ')': case '\'': 87 | *output++ = symbol; 88 | break; 89 | default: 90 | *output++ = '%'; 91 | output = hex_one(symbol, output); 92 | } 93 | } 94 | 95 | return result; 96 | } 97 | 98 | elliptics::file_location_t 99 | elliptics::make_file_location(const ioremap::elliptics::sync_lookup_result &slr 100 | , const mastermind::namespace_state_t &ns_state) { 101 | const auto &path_prefix = ns_settings(ns_state).sign_path_prefix; 102 | 103 | for (auto it = slr.begin(); it != slr.end(); ++it) { 104 | if (it->error()) { 105 | continue; 106 | } 107 | 108 | lookup_result entry(*it, ns_settings(ns_state).sign_port); 109 | 110 | if (entry.path().substr(0, path_prefix.size()) != path_prefix) { 111 | throw std::runtime_error{ 112 | "path_prefix does not match: prefix=" + path_prefix + "; path=" + entry.path()}; 113 | } 114 | 115 | file_location_t file_location; 116 | 117 | file_location.host = entry.host(); 118 | file_location.path = '/' + ns_state.name() + '/' + entry.path().substr(path_prefix.size()); 119 | 120 | return file_location; 121 | } 122 | 123 | throw std::runtime_error{ 124 | "cannot determine file location: there is no good lookup result entry"}; 125 | } 126 | 127 | std::string 128 | elliptics::make_signature_ts(boost::optional opt_expiration_time 129 | , const mastermind::namespace_state_t &ns_state) { 130 | using namespace std::chrono; 131 | 132 | auto now = system_clock::now().time_since_epoch(); 133 | auto expiration_time = opt_expiration_time.get_value_or( 134 | ns_settings(ns_state).redirect_expire_time); 135 | 136 | auto ts = duration_cast(now + expiration_time).count(); 137 | 138 | return hex_one(ts); 139 | } 140 | 141 | std::string 142 | elliptics::make_signature_message(const file_location_t &file_location, const std::string &ts 143 | , const std::vector> &args) { 144 | std::ostringstream oss; 145 | 146 | oss << file_location.host << file_location.path << '/' << ts; 147 | 148 | for (auto it = args.begin(), end = args.end(); it != end; ++it) { 149 | oss << '&' << std::get<0>(*it) << '=' << std::get<1>(*it); 150 | } 151 | 152 | return oss.str(); 153 | } 154 | 155 | std::string 156 | elliptics::make_signature(const std::string &message, const std::string &token) { 157 | using namespace CryptoPP; 158 | 159 | HMAC hmac((const byte *)token.data(), token.size()); 160 | hmac.Update((const byte *)message.data(), message.size()); 161 | std::vector res(hmac.DigestSize()); 162 | hmac.Final(res.data()); 163 | 164 | return hex(res); 165 | } 166 | 167 | -------------------------------------------------------------------------------- /src/data_container.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef INCLUDE__ELLIPTICS_FASTCGI__DATA_CONTAINER_HPP 21 | #define INCLUDE__ELLIPTICS_FASTCGI__DATA_CONTAINER_HPP 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | #include 34 | 35 | namespace elliptics { 36 | 37 | template 38 | void read(ioremap::elliptics::data_pointer &data_pointer, T&ob) { 39 | ob = *data_pointer.data(); 40 | data_pointer = data_pointer.skip(); 41 | } 42 | 43 | template 44 | struct type_traits_base { 45 | typedef T type; 46 | static ioremap::elliptics::data_pointer convert(const type &ob) { 47 | ioremap::elliptics::data_buffer data_buffer; 48 | msgpack::pack(data_buffer, ob); 49 | return std::move(data_buffer); 50 | } 51 | 52 | static type convert(ioremap::elliptics::data_pointer data_pointer) { 53 | type res; 54 | msgpack::unpacked unpacked; 55 | msgpack::unpack(&unpacked, (const char *)data_pointer.data(), data_pointer.size()); 56 | unpacked.get().convert(&res); 57 | return res; 58 | } 59 | }; 60 | 61 | template 62 | struct type_traits; 63 | 64 | enum DNET_COMMON_EMBED_TYPES { 65 | DNET_FCGI_EMBED_DATA = 1 66 | , DNET_FCGI_EMBED_TIMESTAMP = 2 67 | }; 68 | 69 | template<> struct type_traits : type_traits_base {}; 70 | template<> struct type_traits : type_traits_base { 71 | static ioremap::elliptics::data_pointer convert(const type &ob) { 72 | ioremap::elliptics::data_buffer data_buffer(sizeof(type)); 73 | type t; 74 | t.tv_sec = dnet_bswap64(ob.tv_sec); 75 | t.tv_nsec = dnet_bswap64(ob.tv_nsec); 76 | data_buffer.write(t.tv_sec); 77 | data_buffer.write(t.tv_nsec); 78 | return std::move(data_buffer); 79 | } 80 | 81 | static type convert(ioremap::elliptics::data_pointer data_pointer) { 82 | type res; 83 | 84 | read(data_pointer, res.tv_sec); 85 | read(data_pointer, res.tv_nsec); 86 | 87 | res.tv_sec = dnet_bswap64(res.tv_sec); 88 | res.tv_nsec = dnet_bswap64(res.tv_nsec); 89 | 90 | return res; 91 | } 92 | }; 93 | 94 | class data_container_t { 95 | public: 96 | data_container_t() {} 97 | 98 | data_container_t(const std::string &message) 99 | : data(std::move(ioremap::elliptics::data_buffer(message.data(), message.size()))) 100 | { 101 | } 102 | 103 | data_container_t(const ioremap::elliptics::data_pointer &data_pointer) 104 | : data(data_pointer) 105 | { 106 | } 107 | 108 | data_container_t(const data_container_t &ds) 109 | : data(ds.data) 110 | , embeds(ds.embeds) 111 | {} 112 | 113 | data_container_t(data_container_t &&ds) 114 | : data(std::move(ds.data)) 115 | , embeds(std::move(ds.embeds)) 116 | {} 117 | 118 | data_container_t &operator = (data_container_t &&ds) { 119 | data = std::move(ds.data); 120 | embeds = std::move(ds.embeds); 121 | return *this; 122 | } 123 | 124 | template 125 | boost::optional::type> get() const { 126 | auto it = embeds.find(type); 127 | if (it == embeds.end()) 128 | return boost::none; 129 | return type_traits::convert(it->second.data_pointer); 130 | } 131 | 132 | template 133 | void set(const typename type_traits::type &ob) { 134 | embed_t e(type_traits::convert(ob), type, 0); 135 | embeds.insert(std::make_pair(type, e)); 136 | } 137 | 138 | size_t embeds_count() const { 139 | return embeds.size(); 140 | } 141 | 142 | static ioremap::elliptics::data_pointer pack(const data_container_t &ds); 143 | static data_container_t unpack(ioremap::elliptics::data_pointer data_pointer, bool embeded = false); 144 | 145 | ioremap::elliptics::data_pointer data; 146 | 147 | private: 148 | struct embed_t { 149 | struct header_t { 150 | uint64_t size; 151 | uint32_t type; 152 | uint32_t flags; 153 | }; 154 | 155 | embed_t() {} 156 | 157 | embed_t(const ioremap::elliptics::data_pointer &data_pointer, uint32_t type, uint32_t flags) { 158 | this->data_pointer = data_pointer; 159 | header.type = type; 160 | header.flags = flags; 161 | header.size = data_pointer.size(); 162 | } 163 | 164 | header_t header; 165 | ioremap::elliptics::data_pointer data_pointer; 166 | }; 167 | 168 | static embed_t::header_t bswap(const embed_t::header_t &header); 169 | 170 | std::map embeds; 171 | }; 172 | } // namespace elliptcis 173 | 174 | #endif /* INCLUDE__ELLIPTICS_FASTCGI__DATA_CONTAINER_HPP */ 175 | -------------------------------------------------------------------------------- /src/write_retrier.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "write_retrier.hpp" 21 | #include "loggers.hpp" 22 | 23 | #include 24 | 25 | elliptics::write_retrier::write_retrier( 26 | ioremap::swarm::logger bh_logger_ 27 | , ioremap::elliptics::session session_ 28 | , command_t command_ 29 | , size_t success_copies_num_ 30 | , size_t limit_of_attempts_ 31 | , double scale_retry_timeout_ 32 | , ioremap::elliptics::async_write_result::handler promise_ 33 | ) 34 | : bh_logger(std::move(bh_logger_)) 35 | , session(std::move(session_)) 36 | , command(std::move(command_)) 37 | , success_copies_num(success_copies_num_) 38 | , limit_of_attempts(limit_of_attempts_) 39 | , scale_retry_timeout(scale_retry_timeout_) 40 | , promise(std::move(promise_)) 41 | , complete_once([this] { complete(); }) 42 | { 43 | session.set_error_handler(ioremap::elliptics::error_handlers::none); 44 | } 45 | 46 | void 47 | elliptics::write_retrier::start() { 48 | auto groups = session.get_groups(); 49 | promise.set_total(groups.size()); 50 | 51 | for (auto it = groups.begin(), end = groups.end(); it != end; ++it) { 52 | auto group_session = session.clone(); 53 | group_session.set_groups({*it}); 54 | 55 | complete_once.defer(); 56 | try_group(std::move(group_session), 0); 57 | } 58 | 59 | complete_once(); 60 | } 61 | 62 | ioremap::swarm::logger & 63 | elliptics::write_retrier::logger() { 64 | return bh_logger; 65 | } 66 | 67 | void 68 | elliptics::write_retrier::try_group(ioremap::elliptics::session group_session 69 | , size_t number_of_attempts) { 70 | auto self = shared_from_this(); 71 | 72 | auto callback = [this, self, group_session, number_of_attempts] ( 73 | const ioremap::elliptics::sync_write_result &entries 74 | , const ioremap::elliptics::error_info &error_info) { 75 | on_finished(std::move(group_session), number_of_attempts, entries, error_info); 76 | }; 77 | 78 | std::ostringstream oss; 79 | oss << "write session: group=" << group_session.get_groups()[0] 80 | << "; attempt=" << number_of_attempts + 1 << ";"; 81 | auto msg = oss.str(); 82 | MDS_LOG_INFO("%s", msg.c_str()); 83 | 84 | command(group_session).connect(callback); 85 | } 86 | 87 | void 88 | elliptics::write_retrier::on_finished(ioremap::elliptics::session group_session, size_t number_of_attempts 89 | , const ioremap::elliptics::sync_write_result &entries 90 | , const ioremap::elliptics::error_info &error_info) { 91 | std::ostringstream oss; 92 | oss << "write session is finished: group=" << group_session.get_groups()[0] 93 | << "; attempt=" << number_of_attempts + 1 << "; status="; 94 | 95 | if (!error_info) { 96 | oss << "\"ok\"; description=\"success\""; 97 | } else { 98 | oss << "\"bad\"; description=\"" << error_info.message() << "\""; 99 | } 100 | 101 | number_of_attempts += 1; 102 | 103 | bool process_entries = true; 104 | 105 | switch (error_info.code()) { 106 | case -ETIMEDOUT: 107 | group_session.set_timeout(scale_retry_timeout * group_session.get_timeout()); 108 | case -EINTR: 109 | case -EAGAIN: 110 | case -ENOMEM: 111 | case -EBUSY: 112 | case -EINVAL: 113 | case -EMFILE: 114 | process_entries = false; 115 | break; 116 | } 117 | 118 | if (number_of_attempts == limit_of_attempts) { 119 | process_entries = true; 120 | } 121 | 122 | if (process_entries) { 123 | oss << "; decision=\"process result\""; 124 | auto msg = oss.str(); 125 | MDS_LOG_INFO(msg.c_str()); 126 | 127 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 128 | promise.process(*it); 129 | } 130 | 131 | if (error_info) { 132 | init_error(error_info); 133 | } 134 | 135 | complete_once(); 136 | return; 137 | } 138 | 139 | oss << "; decision=\"try again\""; 140 | auto msg = oss.str(); 141 | MDS_LOG_INFO(msg.c_str()); 142 | try_group(std::move(group_session), number_of_attempts); 143 | } 144 | 145 | void 146 | elliptics::write_retrier::init_error(const ioremap::elliptics::error_info &error_info_) { 147 | std::lock_guard lock_guard(error_info_mutex); 148 | 149 | if (!error_info) { 150 | error_info = error_info_; 151 | } 152 | } 153 | 154 | void 155 | elliptics::write_retrier::complete() { 156 | promise.complete(error_info); 157 | } 158 | 159 | ioremap::elliptics::async_write_result 160 | elliptics::try_write( 161 | ioremap::swarm::logger bh_logger 162 | , ioremap::elliptics::session session 163 | , write_retrier::command_t command 164 | , size_t success_copies_num 165 | , size_t limit_of_attempts 166 | , double scale_retry_timeout 167 | ) { 168 | ioremap::elliptics::async_write_result future(session); 169 | ioremap::elliptics::async_write_result::handler promise(future); 170 | 171 | std::make_shared(std::move(bh_logger), session.clone(), std::move(command) 172 | , success_copies_num, limit_of_attempts, scale_retry_timeout 173 | , std::move(promise))->start(); 174 | 175 | return future; 176 | } 177 | 178 | -------------------------------------------------------------------------------- /src/get.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__GET__HPP 21 | #define SRC__GET__HPP 22 | 23 | #include 24 | #include 25 | 26 | namespace boost { 27 | namespace asio { 28 | 29 | const_buffers_1 30 | buffer(const ioremap::elliptics::data_pointer &data_pointer); 31 | 32 | } // namespace asio 33 | } // namespace boost 34 | 35 | #include "proxy.hpp" 36 | 37 | #include "ranges.hpp" 38 | #include "lookuper.hpp" 39 | #include "timer.hpp" 40 | 41 | #include 42 | 43 | #include 44 | 45 | #include 46 | #include 47 | 48 | namespace elliptics { 49 | 50 | namespace ie = ioremap::elliptics; 51 | 52 | struct req_get 53 | : public ioremap::thevoid::simple_request_stream 54 | , public std::enable_shared_from_this 55 | { 56 | void 57 | on_request(const ioremap::thevoid::http_request &http_request 58 | , const boost::asio::const_buffer &const_buffer); 59 | 60 | private: 61 | enum class redirect_arg_tag { 62 | none 63 | , client_want_redirect 64 | }; 65 | 66 | groups_t 67 | get_cached_groups(); 68 | 69 | void 70 | find_first_group(std::function on_result 71 | , std::function on_error); 72 | 73 | void 74 | next_first_group_is_found(const ie::sync_lookup_result &entries 75 | , const ie::error_info &error_info 76 | , std::function on_result 77 | , std::function on_error); 78 | 79 | void 80 | find_other_group(std::function on_result 81 | , std::function on_error); 82 | 83 | void 84 | next_other_group_is_found(const ie::sync_lookup_result &entries 85 | , const ie::error_info &error_info 86 | , std::function on_result 87 | , std::function on_error); 88 | 89 | void 90 | all_groups_were_processed(std::function on_error); 91 | 92 | bool 93 | check_lookup_result_entry(const ie::lookup_result_entry &entry); 94 | 95 | bool 96 | lookup_result_entries_are_equal(const ie::lookup_result_entry &lhs 97 | , const ie::lookup_result_entry &rhs); 98 | 99 | void 100 | process_group_info(const ie::lookup_result_entry &entry); 101 | 102 | void 103 | set_csum_type(const ie::lookup_result_entry &entry); 104 | 105 | void 106 | read_chunk(size_t offset, size_t size 107 | , std::function on_result 108 | , std::function on_error); 109 | 110 | void 111 | read_chunk_is_finished( 112 | const ie::sync_read_result &entries 113 | , const ie::error_info &error_info 114 | , util::timer_t timer 115 | , size_t offset, size_t size 116 | , std::function on_result 117 | , std::function on_error); 118 | 119 | void 120 | send_chunk(ie::data_pointer data_pointer 121 | , std::function on_result 122 | , std::function on_error); 123 | 124 | void 125 | send_chunk_is_finished(const boost::system::error_code &error_code 126 | , util::timer_t timer 127 | , std::function on_result 128 | , std::function on_error); 129 | 130 | void 131 | read_and_send_chunk(size_t offset, size_t size 132 | , std::function on_result 133 | , std::function on_error); 134 | 135 | void 136 | read_and_send_range(size_t offset, size_t size 137 | , std::function on_result 138 | , std::function on_error); 139 | void 140 | read_and_send_ranges(ranges_t ranges, std::list ranges_headers 141 | , std::function on_result 142 | , std::function on_error); 143 | 144 | void 145 | process_whole_file(); 146 | 147 | void 148 | process_range(size_t offset, size_t size); 149 | 150 | void 151 | process_ranges(ranges_t ranges, std::list boundaries); 152 | 153 | void 154 | detect_content_type(const ie::read_result_entry &entry); 155 | 156 | std::tuple process_precondition_headers(const time_t timestamp, const size_t size); 157 | 158 | redirect_arg_tag 159 | get_redirect_arg(); 160 | 161 | std::vector> 162 | get_redirect_query_args(); 163 | 164 | bool try_to_redirect_request(const ie::sync_lookup_result &slr, const size_t size); 165 | void start_reading(const size_t size, bool send_whole_file); 166 | 167 | size_t 168 | total_size(); 169 | 170 | void 171 | on_error(); 172 | 173 | void 174 | request_is_finished(); 175 | 176 | ie::session 177 | get_session(); 178 | 179 | ioremap::thevoid::http_response prospect_http_response; 180 | 181 | boost::optional m_session; 182 | mastermind::namespace_state_t ns_state; 183 | std::string key; 184 | parallel_lookuper_ptr_t parallel_lookuper_ptr; 185 | boost::optional lookup_result_entry_opt; 186 | 187 | bool m_first_chunk; 188 | bool with_chunked_csum; 189 | bool headers_were_sent; 190 | bool some_data_were_sent; 191 | bool has_internal_storage_error; 192 | 193 | groups_t cached_groups; 194 | std::vector bad_groups; 195 | 196 | boost::optional expiration_time; 197 | }; 198 | 199 | } // namespace elliptics 200 | 201 | #endif /* SRC__GET__HPP */ 202 | 203 | -------------------------------------------------------------------------------- /src/cdn_cache.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "cdn_cache.hpp" 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace elliptics { 30 | 31 | cdn_cache_t::cdn_cache_t(ioremap::swarm::logger bh_logger_, config_t config_) 32 | : bh_logger(std::move(bh_logger_)) 33 | , config(std::move(config_)) 34 | , work_is_done(false) 35 | { 36 | deserialize(); 37 | 38 | if (!config.url.empty()) { 39 | MDS_LOG_INFO("starting background thread"); 40 | background_updater = std::thread(std::bind(&cdn_cache_t::background_loop, this)); 41 | } 42 | } 43 | 44 | cdn_cache_t::~cdn_cache_t() { 45 | if (background_updater.joinable()) { 46 | MDS_LOG_INFO("stopping cdn cache updater"); 47 | { 48 | std::lock_guard lock(background_updater_mutex); 49 | (void) lock; 50 | 51 | work_is_done = true; 52 | background_updater_cv.notify_one(); 53 | } 54 | 55 | MDS_LOG_INFO("joining background thread"); 56 | background_updater.join(); 57 | } 58 | } 59 | 60 | bool 61 | cdn_cache_t::check_host(const std::string &host) { 62 | auto local_cache = copy_cache(); 63 | 64 | if (!local_cache) { 65 | return false; 66 | } 67 | 68 | auto it = local_cache->find(host); 69 | 70 | bool host_was_found = true; 71 | 72 | if (it == local_cache->end()) { 73 | host_was_found = false; 74 | } 75 | 76 | MDS_LOG_INFO("regional host \"%s\" %s in the cdn cache", host.c_str() 77 | , (host_was_found ? "was found" : "was not found")); 78 | 79 | return host_was_found; 80 | } 81 | 82 | void 83 | cdn_cache_t::cache_force_update() { 84 | std::lock_guard lock(background_updater_mutex); 85 | (void) lock; 86 | 87 | update_cache(); 88 | } 89 | 90 | ioremap::swarm::logger & 91 | cdn_cache_t::logger() { 92 | return bh_logger; 93 | } 94 | 95 | void 96 | cdn_cache_t::serialize() const { 97 | if (config.cache_path.empty()) { 98 | return; 99 | } 100 | 101 | auto local_cache = copy_cache(); 102 | 103 | std::ostringstream oss; 104 | 105 | for (auto it = local_cache->begin(), end = local_cache->end(); it != end; ++it) { 106 | oss << *it << std::endl; 107 | } 108 | 109 | auto raw_data = oss.str(); 110 | std::ofstream output(config.cache_path.c_str()); 111 | std::copy(raw_data.begin(), raw_data.end(), std::ostreambuf_iterator(output)); 112 | } 113 | 114 | void 115 | cdn_cache_t::deserialize() { 116 | if (config.cache_path.empty()) { 117 | return; 118 | } 119 | 120 | std::string raw_data; 121 | 122 | { 123 | typedef std::istreambuf_iterator input_iterator_t; 124 | std::ifstream input(config.cache_path.c_str()); 125 | raw_data.assign(input_iterator_t(input), input_iterator_t()); 126 | } 127 | 128 | set_cache(parse_cache(raw_data)); 129 | } 130 | 131 | cdn_cache_t::cache_ptr_t 132 | cdn_cache_t::parse_cache(const std::string &raw_data) const { 133 | auto local_cache = std::make_shared(); 134 | std::string::size_type pos = 0; 135 | 136 | while (pos != std::string::npos) { 137 | auto new_pos = raw_data.find('\n', pos); 138 | auto size = new_pos == std::string::npos ? std::string::npos : new_pos - pos; 139 | 140 | { 141 | auto row = raw_data.substr(pos, size); 142 | 143 | if (!row.empty()) { 144 | local_cache->insert(std::move(row)); 145 | } 146 | } 147 | 148 | pos = new_pos == std::string::npos ? std::string::npos : new_pos + 1; 149 | } 150 | 151 | return local_cache; 152 | } 153 | 154 | cdn_cache_t::cache_ptr_t 155 | cdn_cache_t::copy_cache() const { 156 | std::lock_guard lock(cache_mutex); 157 | (void) lock; 158 | 159 | return cache_ptr; 160 | } 161 | 162 | void 163 | cdn_cache_t::set_cache(cache_ptr_t cache_ptr_) { 164 | { 165 | std::lock_guard lock(cache_mutex); 166 | (void) lock; 167 | cache_ptr.swap(cache_ptr_); 168 | } 169 | } 170 | 171 | void 172 | cdn_cache_t::update_cache() { 173 | MDS_LOG_INFO("downloading new data"); 174 | 175 | std::stringstream res; 176 | std::string raw_data; 177 | 178 | try { 179 | yandex::common::curl_wrapper curl(config.url); 180 | curl.header("Expect", ""); 181 | curl.timeout(config.timeout); 182 | 183 | auto status = curl.perform(res); 184 | res.str().swap(raw_data); 185 | 186 | if (status != 200) { 187 | MDS_LOG_ERROR("cannot download data: status=%d", static_cast(status)); 188 | MDS_LOG_DEBUG("cannot download data: body=\"%s\"", raw_data.c_str()); 189 | return; 190 | } 191 | } catch (const std::exception &ex) { 192 | return; 193 | } 194 | 195 | set_cache(parse_cache(raw_data)); 196 | serialize(); 197 | } 198 | 199 | void 200 | cdn_cache_t::background_loop() { 201 | std::unique_lock lock(background_updater_mutex); 202 | #if __GNUC__ == 4 && __GNUC_MINOR__ >= 6 203 | auto no_timeout = std::cv_status::no_timeout; 204 | auto timeout = std::cv_status::timeout; 205 | #else 206 | bool no_timeout = true; 207 | bool timeout = false; 208 | #endif 209 | 210 | MDS_LOG_INFO("background loop update period is %ds, source url is %s" 211 | , config.update_period, config.url.c_str()); 212 | 213 | do { 214 | MDS_LOG_INFO("backgroung loop begin"); 215 | update_cache(); 216 | MDS_LOG_INFO("backgroung loop end"); 217 | 218 | auto tm = timeout; 219 | 220 | do { 221 | tm = background_updater_cv.wait_for(lock, std::chrono::seconds(config.update_period)); 222 | } while (tm == no_timeout && work_is_done == false); 223 | } while (work_is_done == false); 224 | } 225 | 226 | } // namespace elliptics 227 | 228 | -------------------------------------------------------------------------------- /src/expected.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef MDS_PROXY__SRC__EXPECTED__HPP 21 | #define MDS_PROXY__SRC__EXPECTED__HPP 22 | 23 | #include 24 | #include 25 | 26 | namespace util { 27 | 28 | namespace detail { 29 | 30 | struct expected_with_exception { 31 | std::exception_ptr exception_ptr; 32 | }; 33 | 34 | } // namespace detail 35 | 36 | template 37 | class expected { 38 | public: 39 | typedef std::function)> callback_t; 40 | 41 | template 42 | static 43 | expected 44 | from_exception(const E &exception) { 45 | if (typeid(exception) != typeid(E)) { 46 | throw std::runtime_error("slicing detected"); 47 | } 48 | 49 | return from_exception(std::make_exception_ptr(exception)); 50 | } 51 | 52 | static 53 | expected 54 | from_exception(std::exception_ptr exception_ptr) { 55 | expected result; 56 | result.value_is_set = false; 57 | new (&result.exception_ptr) std::exception_ptr(std::move(exception_ptr)); 58 | return result; 59 | } 60 | 61 | static 62 | expected 63 | from_current_exception() { 64 | return from_exception(std::current_exception()); 65 | } 66 | 67 | template 68 | static 69 | expected from_function(F function) { 70 | try { 71 | return expected(function()); 72 | } catch (...) { 73 | return from_current_exception(); 74 | } 75 | } 76 | 77 | expected(const T &value_) 78 | : value_is_set(true) 79 | , value(value_) 80 | {} 81 | 82 | expected(T &&value_) 83 | : value_is_set(true) 84 | , value(std::move(value_)) 85 | {} 86 | 87 | expected(const expected &that) 88 | : value_is_set(that.value_is_set) 89 | { 90 | if (value_is_set) { 91 | new (&value) T(that.value); 92 | } else { 93 | new (&exception_ptr) std::exception_ptr(that.exception_ptr); 94 | } 95 | } 96 | 97 | expected(expected &&that) 98 | : value_is_set(that.value_is_set) 99 | { 100 | if (value_is_set) { 101 | new (&value) T(std::move(that.value)); 102 | } else { 103 | new (&exception_ptr) std::exception_ptr(std::move(that.exception_ptr)); 104 | } 105 | } 106 | 107 | expected(detail::expected_with_exception &&that) 108 | : value_is_set(false) 109 | , exception_ptr(std::move(that.exception_ptr)) 110 | {} 111 | 112 | ~expected() { 113 | using std::exception_ptr; 114 | 115 | if (value_is_set) { 116 | value.~T(); 117 | } else { 118 | this->exception_ptr.~exception_ptr(); 119 | } 120 | } 121 | 122 | 123 | void 124 | swap(expected &rhs) { 125 | if (value_is_set) { 126 | if (rhs.value_is_set) { 127 | using std::swap; 128 | swap(value, rhs.value); 129 | } else { 130 | auto tmp = std::move(rhs.exception_ptr); 131 | new (&rhs.value) T(std::move(value)); 132 | new (&exception_ptr) std::exception_ptr(tmp); 133 | std::swap(value_is_set, rhs.value_is_set); 134 | } 135 | } else { 136 | if (rhs.value_is_set) { 137 | rhs.swap(*this); 138 | } else { 139 | exception_ptr.swap(rhs.exception_ptr); 140 | } 141 | } 142 | } 143 | 144 | bool 145 | has_value() const { 146 | return value_is_set; 147 | } 148 | 149 | T & 150 | get() { 151 | if (!value_is_set) { 152 | std::rethrow_exception(exception_ptr); 153 | } 154 | 155 | return value; 156 | } 157 | 158 | const T & 159 | get() const { 160 | if (!value_is_set) { 161 | std::rethrow_exception(exception_ptr); 162 | } 163 | 164 | return value; 165 | } 166 | 167 | private: 168 | expected() 169 | {} 170 | 171 | bool value_is_set; 172 | union { 173 | T value; 174 | std::exception_ptr exception_ptr; 175 | }; 176 | 177 | }; 178 | 179 | template <> 180 | class expected { 181 | public: 182 | typedef std::function)> callback_t; 183 | 184 | template 185 | static 186 | expected 187 | from_exception(const E &exception) { 188 | if (typeid(exception) != typeid(E)) { 189 | throw std::runtime_error("slicing detected"); 190 | } 191 | 192 | return from_exception(std::make_exception_ptr(exception)); 193 | } 194 | 195 | static 196 | expected 197 | from_exception(std::exception_ptr exception_ptr) { 198 | expected result; 199 | result.exception_ptr = std::exception_ptr(std::move(exception_ptr)); 200 | return result; 201 | } 202 | 203 | static 204 | expected 205 | from_current_exception() { 206 | return from_exception(std::current_exception()); 207 | } 208 | 209 | template 210 | static 211 | expected from_function(F function) { 212 | try { 213 | return expected(function()); 214 | } catch (...) { 215 | return from_current_exception(); 216 | } 217 | } 218 | 219 | expected() 220 | {} 221 | 222 | expected(const expected &that) 223 | : exception_ptr(that.exception_ptr) 224 | {} 225 | 226 | expected(expected &&that) 227 | : exception_ptr(std::move(that.exception_ptr)) 228 | {} 229 | 230 | expected(detail::expected_with_exception &&that) 231 | : exception_ptr(std::move(that.exception_ptr)) 232 | {} 233 | 234 | ~expected() { 235 | } 236 | 237 | 238 | void 239 | swap(expected &rhs) { 240 | exception_ptr.swap(rhs.exception_ptr); 241 | } 242 | 243 | bool 244 | has_value() const { 245 | return !exception_ptr; 246 | } 247 | 248 | void 249 | get() { 250 | if (exception_ptr) { 251 | std::rethrow_exception(exception_ptr); 252 | } 253 | } 254 | 255 | private: 256 | std::exception_ptr exception_ptr; 257 | 258 | }; 259 | 260 | template 261 | detail::expected_with_exception 262 | expected_from_exception(Args &&...args) { 263 | return detail::expected_with_exception{ 264 | std::make_exception_ptr(E(std::forward(args)...)) 265 | }; 266 | } 267 | 268 | } // namespace util 269 | 270 | #endif /* MDS_PROXY__SRC__EXPECTED__HPP */ 271 | 272 | -------------------------------------------------------------------------------- /src/proxy.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__PROXY_HPP 21 | #define SRC__PROXY_HPP 22 | 23 | #include "lookup_result.hpp" 24 | #include "loggers.hpp" 25 | #include "magic_provider.hpp" 26 | #include "utils.hpp" 27 | #include "cdn_cache.hpp" 28 | #include "ns_settings.hpp" 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | 37 | #include 38 | 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | 47 | #if __GNUC__ == 4 && __GNUC_MINOR__ >= 6 48 | #include 49 | #else 50 | #include 51 | #endif 52 | 53 | namespace elliptics { 54 | 55 | template 56 | T get_arg(const ioremap::swarm::url_query &query_list, const std::string &name, const T &default_value = T()) { 57 | auto &&arg = query_list.item_value(name); 58 | return arg ? boost::lexical_cast(*arg) : default_value; 59 | } 60 | 61 | std::string id_str(const ioremap::elliptics::key &key, ioremap::elliptics::session sess); 62 | 63 | class proxy : public ioremap::thevoid::server 64 | { 65 | public: 66 | ~proxy(); 67 | 68 | bool initialize(const rapidjson::Value &config); 69 | 70 | struct req_ping 71 | : public ioremap::thevoid::simple_request_stream 72 | , public std::enable_shared_from_this 73 | { 74 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 75 | }; 76 | 77 | struct req_cache 78 | : public ioremap::thevoid::simple_request_stream 79 | , public std::enable_shared_from_this 80 | { 81 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 82 | }; 83 | 84 | struct req_cache_update 85 | : public ioremap::thevoid::simple_request_stream 86 | , public std::enable_shared_from_this 87 | { 88 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 89 | }; 90 | 91 | struct req_statistics 92 | : public ioremap::thevoid::simple_request_stream 93 | , public std::enable_shared_from_this 94 | { 95 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 96 | }; 97 | 98 | struct req_stats 99 | : public ioremap::thevoid::simple_request_stream 100 | , public std::enable_shared_from_this 101 | { 102 | void on_request(const ioremap::thevoid::http_request &req, const boost::asio::const_buffer &buffer); 103 | }; 104 | 105 | protected: 106 | public: 107 | template 108 | void register_handler(const std::string &name, bool exact_match); 109 | 110 | ioremap::elliptics::node generate_node(const rapidjson::Value &config, int &timeout_def); 111 | std::shared_ptr generate_mastermind(const rapidjson::Value &config); 112 | std::shared_ptr generate_cdn_cache(const rapidjson::Value &config); 113 | 114 | boost::optional 115 | get_session(); 116 | 117 | boost::optional 118 | read_session(const ioremap::thevoid::http_request &http_request, const couple_t &couple); 119 | 120 | boost::optional 121 | write_session(const ioremap::thevoid::http_request &http_request, const couple_t &couple); 122 | 123 | boost::optional 124 | remove_session(const ioremap::thevoid::http_request &http_request, const couple_t &couple); 125 | 126 | boost::optional 127 | lookup_session(const ioremap::thevoid::http_request &http_request, const couple_t &couple); 128 | 129 | ioremap::elliptics::session 130 | setup_session(ioremap::elliptics::session session 131 | , const ioremap::thevoid::http_request &http_request, const couple_t &couple); 132 | 133 | mastermind::namespace_state_t 134 | get_namespace_state(const std::string &script, const std::string &handler); 135 | 136 | mastermind::namespace_state_t 137 | get_namespace_state(const std::string &name); 138 | 139 | int die_limit() const; 140 | 141 | std::tuple 142 | get_file_info(const ioremap::thevoid::http_request &req); 143 | 144 | std::vector 145 | get_groups(const mastermind::namespace_state_t &ns_state, int group); 146 | 147 | std::tuple, ioremap::elliptics::key> 148 | prepare_session(const std::string &url, const mastermind::namespace_state_t &ns_state); 149 | 150 | std::vector 151 | groups_for_upload(const mastermind::namespace_state_t &ns_state, uint64_t size); 152 | 153 | std::shared_ptr &mastermind(); 154 | std::string get_auth_token(const boost::optional &auth_header); 155 | bool check_basic_auth(const std::string &ns, const std::string &auth_key, const boost::optional &auth_header); 156 | 157 | file_location_t 158 | get_file_location(const ioremap::elliptics::sync_lookup_result &slr 159 | , const mastermind::namespace_state_t &ns_state 160 | , const std::string &x_regional_host); 161 | 162 | std::tuple 163 | generate_signature_for_elliptics_file(const ioremap::elliptics::sync_lookup_result &slr 164 | , const std::string &x_regional_host, const mastermind::namespace_state_t &ns_state 165 | , boost::optional optional_expiration_time); 166 | 167 | void 168 | update_elliptics_remotes(); 169 | 170 | void cache_update_callback(); 171 | 172 | mastermind::namespace_state_t::user_settings_ptr_t 173 | settings_factory(const std::string &name, const kora::config_t &config); 174 | 175 | private: 176 | public: 177 | std::mutex elliptics_node_mutex; 178 | std::mutex elliptics_session_mutex; 179 | boost::optional m_elliptics_node; 180 | boost::optional m_elliptics_session; 181 | 182 | boost::optional elliptics_read_session; 183 | boost::optional elliptics_write_session; 184 | boost::optional elliptics_remove_session; 185 | boost::optional elliptics_lookup_session; 186 | 187 | int m_die_limit; 188 | int m_write_chunk_size; 189 | int m_read_chunk_size; 190 | std::shared_ptr m_mastermind; 191 | std::shared_ptr cdn_cache; 192 | boost::thread_specific_ptr m_magic; 193 | 194 | // write retries 195 | size_t limit_of_middle_chunk_attempts; 196 | double scale_retry_timeout; 197 | 198 | struct { 199 | int def; 200 | int read; 201 | int write; 202 | int lookup; 203 | int remove; 204 | } timeout; 205 | 206 | struct { 207 | int data_flow_rate; 208 | } timeout_coef; 209 | 210 | struct { 211 | std::string name; 212 | std::string value; 213 | 214 | std::set handlers; 215 | } header_protector; 216 | }; 217 | 218 | template 219 | void proxy::register_handler(const std::string &name, bool exact_match) { 220 | options opts; 221 | if (exact_match) { 222 | options::exact_match('/' + name)(&opts); 223 | } else { 224 | options::prefix_match('/' + name)(&opts); 225 | } 226 | 227 | if (header_protector.handlers.count(name)) { 228 | options::header(header_protector.name, header_protector.value)(&opts); 229 | } 230 | 231 | base_server::on(std::move(opts), std::make_shared>(this)); 232 | } 233 | 234 | } // namespace elliptics 235 | 236 | #endif /* SRC__PROXY_HPP */ 237 | -------------------------------------------------------------------------------- /src/upload.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "proxy.hpp" 21 | #include "data_container.hpp" 22 | #include "lookup_result.hpp" 23 | 24 | #include "upload.hpp" 25 | #include "upload_simple.hpp" 26 | #include "upload_multipart.hpp" 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | namespace elliptics { 45 | 46 | void 47 | upload_t::on_headers(ioremap::thevoid::http_request &&http_request) { 48 | size_t total_size = 0; 49 | 50 | if (const auto &arg = http_request.headers().content_length()) { 51 | total_size = *arg; 52 | } else { 53 | MDS_LOG_INFO("missing Content-Length"); 54 | reply()->send_error(ioremap::swarm::http_response::bad_request); 55 | return; 56 | } 57 | 58 | if (total_size == 0) { 59 | MDS_LOG_INFO("Content-Length must be greater than zero"); 60 | reply()->send_error(ioremap::swarm::http_response::bad_request); 61 | return; 62 | } 63 | 64 | MDS_LOG_INFO("body size: %lu", total_size); 65 | 66 | { 67 | std::ostringstream oss; 68 | const auto &headers = http_request.headers().all(); 69 | oss << "Headers:" << std::endl; 70 | for (auto it = headers.begin(); it != headers.end(); ++it) { 71 | oss << it->first << ": " << it->second << std::endl; 72 | } 73 | MDS_LOG_DEBUG("%s", oss.str().c_str()); 74 | } 75 | 76 | std::tuple file_info; 77 | 78 | try { 79 | file_info = server()->get_file_info(http_request); 80 | } catch (const std::exception &ex) { 81 | MDS_LOG_ERROR("cannot parse file info: %s", ex.what()); 82 | reply()->send_error(ioremap::swarm::http_response::bad_request); 83 | return; 84 | } 85 | 86 | auto ns_state = std::get<1>(file_info); 87 | 88 | if (!ns_state) { 89 | MDS_LOG_INFO("cannot determine a namespace"); 90 | reply()->send_error(ioremap::swarm::http_response::bad_request); 91 | return; 92 | } 93 | 94 | if (ns_state.statistics().ns_is_full()) { 95 | MDS_LOG_INFO("namespace is marked as full"); 96 | reply()->send_error(ioremap::swarm::http_response::insufficient_storage); 97 | return; 98 | } 99 | 100 | { 101 | if (!server()->check_basic_auth(ns_state.name() 102 | , ns_settings(ns_state).auth_key_for_write 103 | , http_request.headers().get("Authorization"))) { 104 | auto token = server()->get_auth_token(http_request.headers().get("Authorization")); 105 | MDS_LOG_INFO("invalid token \"%s\"", token.empty() ? "" : token.c_str()); 106 | 107 | ioremap::thevoid::http_response reply; 108 | ioremap::swarm::http_headers headers; 109 | 110 | reply.set_code(401); 111 | headers.add("WWW-Authenticate", std::string("Basic realm=\"") + ns_state.name() + "\""); 112 | headers.set_content_length(0); 113 | headers.set_keep_alive(false); 114 | reply.set_headers(headers); 115 | send_reply(std::move(reply)); 116 | return; 117 | } 118 | } 119 | 120 | auto couple_iterator = create_couple_iterator(http_request, ns_state, total_size); 121 | 122 | if (!couple_iterator) { 123 | return; 124 | } 125 | 126 | if (auto content_type_opt = http_request.headers().content_type()) { 127 | int res = content_type_opt->compare(0, sizeof("multipart/form-data;") - 1 128 | , "multipart/form-data;"); 129 | 130 | if (!res) { 131 | auto size = ns_settings(ns_state).multipart_content_length_threshold; 132 | 133 | if (size != -1 && static_cast(size) < total_size) { 134 | MDS_LOG_INFO( 135 | "client tries to upload multipart with total_size=%d" 136 | ", but multipart_content_length_threshold=%d" 137 | , static_cast(total_size), static_cast(size)); 138 | reply()->send_error(ioremap::swarm::http_response::forbidden); 139 | return; 140 | } 141 | 142 | request_stream = make_request_stream(server(), reply() 143 | , std::move(ns_state), couple_iterator->next().groups); 144 | } 145 | } 146 | 147 | if (!request_stream) { 148 | request_stream = make_request_stream(server(), reply() 149 | , std::move(ns_state), *couple_iterator 150 | , std::move(std::get<0>(file_info))); 151 | } 152 | 153 | request_stream->on_headers(std::move(http_request)); 154 | } 155 | 156 | size_t 157 | upload_t::on_data(const boost::asio::const_buffer &buffer) { 158 | return request_stream->on_data(buffer); 159 | } 160 | 161 | void 162 | upload_t::on_close(const boost::system::error_code &error) { 163 | request_stream->on_close(error); 164 | } 165 | 166 | } // elliptics 167 | 168 | boost::optional 169 | elliptics::upload_t::create_couple_iterator(const ioremap::thevoid::http_request &http_request 170 | , const mastermind::namespace_state_t &ns_state, size_t total_size) { 171 | if (auto arg = http_request.url().query().item_value("couple_id")) { 172 | if (!ns_settings(ns_state).can_choose_couple_to_upload) { 173 | MDS_LOG_INFO("client wants to choose couple by himself, but you forbade that"); 174 | reply()->send_error(ioremap::swarm::http_response::forbidden); 175 | return boost::none; 176 | } 177 | 178 | int couple_id = 0; 179 | 180 | try { 181 | couple_id = boost::lexical_cast(*arg); 182 | } catch (...) { 183 | MDS_LOG_INFO("couple_id is malformed: \"%s\"", arg->c_str()); 184 | reply()->send_error(ioremap::swarm::http_response::bad_request); 185 | return boost::none; 186 | } 187 | 188 | auto couple = ns_state.couples().get_couple_groups(couple_id); 189 | 190 | if (couple.empty()) { 191 | MDS_LOG_INFO("cannot obtain couple by couple_id: %d", couple_id); 192 | reply()->send_error(ioremap::swarm::http_response::bad_request); 193 | return boost::none; 194 | } 195 | 196 | if (couple_id != *std::min_element(couple.begin(), couple.end())) { 197 | MDS_LOG_INFO("client tried to use no minimum group as couple_id: %d", couple_id); 198 | reply()->send_error(ioremap::swarm::http_response::bad_request); 199 | return boost::none; 200 | } 201 | 202 | auto space = ns_state.couples().free_effective_space(couple_id) 203 | + ns_state.couples().free_reserved_space(couple_id); 204 | 205 | if (space < total_size) { 206 | MDS_LOG_ERROR("client chose a couple with not enough space: couple_id=%d", couple_id); 207 | reply()->send_error(ioremap::swarm::http_response::insufficient_storage); 208 | return boost::none; 209 | } 210 | 211 | { 212 | std::ostringstream oss; 213 | oss << couple; 214 | auto couple_str = oss.str(); 215 | MDS_LOG_INFO("use couple chosen by client: %s", couple_str.c_str()); 216 | } 217 | 218 | return couple_iterator_t(couple); 219 | } else { 220 | try { 221 | if (!ns_settings(ns_state).static_couple.empty()) { 222 | return couple_iterator_t(ns_settings(ns_state).static_couple); 223 | } 224 | 225 | return couple_iterator_t(ns_state.weights().couple_sequence(total_size)); 226 | } catch (const mastermind::not_enough_memory_error &e) { 227 | MDS_LOG_ERROR("cannot obtain any couple size=%d namespace=%s : %s" 228 | , static_cast(ns_state.settings().groups_count()) 229 | , ns_state.name().c_str(), e.code().message().c_str()); 230 | reply()->send_error(ioremap::swarm::http_response::insufficient_storage); 231 | return boost::none; 232 | } catch (const std::system_error &e) { 233 | MDS_LOG_ERROR("cannot obtain any couple size=%d namespace=%s : %s" 234 | , static_cast(ns_state.settings().groups_count()) 235 | , ns_state.name().c_str(), e.code().message().c_str()); 236 | reply()->send_error(ioremap::swarm::http_response::internal_server_error); 237 | return boost::none; 238 | } 239 | } 240 | } 241 | 242 | -------------------------------------------------------------------------------- /src/buffered_writer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "buffered_writer.hpp" 21 | #include "loggers.hpp" 22 | 23 | class error_category_t 24 | : public std::error_category 25 | { 26 | public: 27 | const char * 28 | name() const noexcept { 29 | return "buffered writer error category"; 30 | } 31 | 32 | std::string 33 | message(int ev) const { 34 | switch (static_cast(ev)) { 35 | case elliptics::buffered_writer_errc::success: 36 | return "success"; 37 | case elliptics::buffered_writer_errc::interrupted: 38 | return "writing was interrupted"; 39 | case elliptics::buffered_writer_errc::unexpected_event: 40 | return "unexpected event"; 41 | default: 42 | return "unknown error"; 43 | } 44 | } 45 | }; 46 | 47 | const std::error_category & 48 | elliptics::buffered_writer_category() { 49 | const static error_category_t instance; 50 | return instance; 51 | } 52 | 53 | std::error_code 54 | elliptics::make_error_code(buffered_writer_errc e) { 55 | return std::error_code(static_cast(e), buffered_writer_category()); 56 | } 57 | 58 | std::error_condition 59 | elliptics::make_error_condition(buffered_writer_errc e) { 60 | return std::error_condition(static_cast(e), buffered_writer_category()); 61 | } 62 | 63 | elliptics::buffered_writer_error::buffered_writer_error(buffered_writer_errc e 64 | , const std::string &message) 65 | : std::system_error(make_error_code(e), message) 66 | { 67 | } 68 | 69 | elliptics::buffered_writer_t::buffered_writer_t(ioremap::swarm::logger bh_logger_, 70 | std::string key_, size_t chunk_size_) 71 | : state(state_tag::appending) 72 | , bh_logger(std::move(bh_logger_)) 73 | , key(std::move(key_)) 74 | , chunk_size(chunk_size_) 75 | , total_size(0) 76 | { 77 | } 78 | 79 | void 80 | elliptics::buffered_writer_t::append(const char *data, size_t size) { 81 | lock_guard_t lock_guard(state_mutex); 82 | (void) lock_guard; 83 | 84 | switch (state) { 85 | case state_tag::appending: 86 | append_impl(data, size); 87 | break; 88 | case state_tag::writing: 89 | case state_tag::interrupting: 90 | case state_tag::completed: 91 | case state_tag::failed: 92 | case state_tag::interrupted: 93 | throw buffered_writer_error(buffered_writer_errc::unexpected_event); 94 | } 95 | } 96 | 97 | void 98 | elliptics::buffered_writer_t::write(const ioremap::elliptics::session &session, size_t commit_coef 99 | , size_t success_copies_num, size_t limit_of_middle_chunk_attempts 100 | , double scale_retry_timeout, callback_t next) { 101 | lock_guard_t lock_guard(state_mutex); 102 | 103 | switch (state) { 104 | case state_tag::appending: 105 | state = state_tag::writing; 106 | write_impl(lock_guard, session, commit_coef, success_copies_num 107 | , limit_of_middle_chunk_attempts, scale_retry_timeout, std::move(next)); 108 | break; 109 | case state_tag::interrupted: 110 | buffers.clear(); 111 | result = writer->get_result(); 112 | writer.reset(); 113 | 114 | lock_guard.unlock(); 115 | next(buffered_writer_errc::interrupted); 116 | lock_guard.lock(); 117 | break; 118 | case state_tag::writing: 119 | case state_tag::interrupting: 120 | case state_tag::completed: 121 | case state_tag::failed: 122 | throw buffered_writer_error(buffered_writer_errc::unexpected_event); 123 | } 124 | } 125 | 126 | void 127 | elliptics::buffered_writer_t::interrupt() { 128 | lock_guard_t lock_guard(state_mutex); 129 | (void) lock_guard; 130 | 131 | switch (state) { 132 | case state_tag::appending: 133 | state = state_tag::interrupted; 134 | buffers.clear(); 135 | break; 136 | case state_tag::writing: 137 | state = state_tag::interrupting; 138 | break; 139 | case state_tag::interrupting: 140 | case state_tag::completed: 141 | case state_tag::failed: 142 | case state_tag::interrupted: 143 | // nothing to do 144 | break; 145 | } 146 | } 147 | 148 | const std::string & 149 | elliptics::buffered_writer_t::get_key() const { 150 | return key; 151 | } 152 | 153 | bool 154 | elliptics::buffered_writer_t::is_finished() const { 155 | return is_completed() || is_failed() || is_interrupted(); 156 | } 157 | 158 | bool 159 | elliptics::buffered_writer_t::is_completed() const { 160 | lock_guard_t lock_guard(state_mutex); 161 | (void) lock_guard; 162 | 163 | return state_tag::completed == state; 164 | } 165 | 166 | bool 167 | elliptics::buffered_writer_t::is_failed() const { 168 | lock_guard_t lock_guard(state_mutex); 169 | (void) lock_guard; 170 | 171 | return state_tag::failed == state; 172 | } 173 | 174 | bool 175 | elliptics::buffered_writer_t::is_interrupted() const { 176 | lock_guard_t lock_guard(state_mutex); 177 | (void) lock_guard; 178 | 179 | return state_tag::interrupted == state; 180 | } 181 | 182 | const elliptics::writer_t::result_t & 183 | elliptics::buffered_writer_t::get_result() const { 184 | return result; 185 | } 186 | 187 | ioremap::swarm::logger & 188 | elliptics::buffered_writer_t::logger() { 189 | return bh_logger; 190 | } 191 | 192 | void 193 | elliptics::buffered_writer_t::append_impl(const char *data, size_t size) { 194 | total_size += size; 195 | 196 | { 197 | size_t buffers_size = 0; 198 | 199 | if (!buffers.empty()) { 200 | buffers_size += (buffers.size() - 1) * chunk_size; 201 | buffers_size += buffers.back().size(); 202 | } 203 | MDS_LOG_DEBUG("buffer append: key=%s append-size=%llu buffer-size=%llu total-size=%llu" 204 | , key.c_str(), size, buffers_size, total_size); 205 | } 206 | 207 | while (size != 0) { 208 | if (buffers.empty() || buffers.back().size() == chunk_size) { 209 | buffer_t buffer; 210 | buffer.reserve(chunk_size); 211 | 212 | buffers.emplace_back(std::move(buffer)); 213 | } 214 | 215 | auto &buffer = buffers.back(); 216 | auto buffer_size = buffer.size(); 217 | 218 | auto part_size = std::min(size, chunk_size - buffer_size); 219 | buffer.insert(buffer.end(), data, data + part_size); 220 | 221 | data += part_size; 222 | size -= part_size; 223 | } 224 | } 225 | 226 | void 227 | elliptics::buffered_writer_t::write_impl(lock_guard_t &lock_guard 228 | , const ioremap::elliptics::session &session 229 | , size_t commit_coef, size_t success_copies_num, size_t limit_of_middle_chunk_attempts 230 | , double scale_retry_timeout, callback_t next) { 231 | writer = std::make_shared( 232 | ioremap::swarm::logger(logger(), blackhole::log::attributes_t()), session, get_key() 233 | , total_size, 0, commit_coef, success_copies_num 234 | , limit_of_middle_chunk_attempts, scale_retry_timeout); 235 | 236 | write_chunk(lock_guard, std::move(next)); 237 | } 238 | 239 | void 240 | elliptics::buffered_writer_t::write_chunk(lock_guard_t &lock_guard, callback_t next) { 241 | auto buffer = std::move(buffers.front()); 242 | buffers.pop_front(); 243 | 244 | auto self = shared_from_this(); 245 | auto next_ = [this, self, next] (const std::error_code &error_code) { 246 | on_chunk_wrote(error_code, std::move(next)); 247 | }; 248 | 249 | auto chunk = ioremap::elliptics::data_pointer::copy(buffer.data(), buffer.size()); 250 | 251 | lock_guard.unlock(); 252 | writer->write(std::move(chunk), std::move(next_)); 253 | lock_guard.lock(); 254 | } 255 | 256 | void 257 | elliptics::buffered_writer_t::on_chunk_wrote(const std::error_code &error_code, callback_t next) { 258 | lock_guard_t lock_guard(state_mutex); 259 | 260 | switch (state) { 261 | case state_tag::writing: 262 | if (error_code) { 263 | state = state_tag::failed; 264 | result = writer->get_result(); 265 | writer.reset(); 266 | 267 | lock_guard.unlock(); 268 | next(error_code); 269 | lock_guard.lock(); 270 | break; 271 | } 272 | 273 | if (buffers.empty()) { 274 | state = state_tag::completed; 275 | result = writer->get_result(); 276 | writer.reset(); 277 | 278 | lock_guard.unlock(); 279 | next(buffered_writer_errc::success); 280 | lock_guard.lock(); 281 | break; 282 | } 283 | 284 | write_chunk(lock_guard, std::move(next)); 285 | break; 286 | case state_tag::interrupting: 287 | state = state_tag::interrupted; 288 | buffers.clear(); 289 | result = writer->get_result(); 290 | writer.reset(); 291 | 292 | lock_guard.unlock(); 293 | next(buffered_writer_errc::interrupted); 294 | lock_guard.lock(); 295 | break; 296 | case state_tag::appending: 297 | case state_tag::completed: 298 | case state_tag::failed: 299 | case state_tag::interrupted: 300 | throw buffered_writer_error(buffered_writer_errc::unexpected_event); 301 | } 302 | } 303 | 304 | -------------------------------------------------------------------------------- /src/download_info.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "download_info.hpp" 21 | #include "error.hpp" 22 | 23 | const std::string elliptics::download_info_1_t::handler_name = "downloadinfo"; 24 | const std::string elliptics::download_info_2_t::handler_name = "download-info"; 25 | 26 | elliptics::download_info_1_t::download_info_1_t() 27 | : download_info_t(handler_name) 28 | {} 29 | 30 | elliptics::download_info_2_t::download_info_2_t() 31 | : download_info_t(handler_name) 32 | {} 33 | 34 | elliptics::download_info_t::download_info_t(const std::string &handler_name_) 35 | : handler_name('/' + handler_name_) 36 | {} 37 | 38 | void 39 | elliptics::download_info_t::on_request(const ioremap::thevoid::http_request &req 40 | , const boost::asio::const_buffer &buffer) { 41 | (void) buffer; 42 | 43 | MDS_LOG_INFO("Download info: handle request: %s", req.url().path().c_str()); 44 | 45 | try { 46 | ns_state = get_namespace_state(req.url().path(), handler_name); 47 | check_signature(); 48 | check_query_args(); 49 | 50 | boost::optional session; 51 | boost::optional key; 52 | 53 | // The method runs in thevoid's io-loop, therefore proxy's dtor cannot run 54 | // in this moment. Hence session can be safely used without any check. 55 | std::tie(session, key) = prepare_session(ns_state); 56 | 57 | if (ns_settings(ns_state).check_for_update) { 58 | session->set_cflags(session->get_cflags() | DNET_FLAGS_NOLOCK); 59 | } 60 | 61 | { 62 | const auto &headers = req.headers(); 63 | if (const auto &xrh = headers.get("X-Regional-Host")) { 64 | x_regional_host = *xrh; 65 | } 66 | } 67 | 68 | if (req.method() == "GET") { 69 | process_get(*session, *key); 70 | } else { 71 | throw http_error(405, "Method " + req.method() + " is not allowed"); 72 | } 73 | } catch (const http_error &ex) { 74 | std::ostringstream oss; 75 | oss 76 | << "http_error: http_status = " << ex.http_status() 77 | << " ; description = " << ex.what(); 78 | auto msg = oss.str(); 79 | 80 | if (ex.is_server_error()) { 81 | MDS_LOG_ERROR("%s", msg.c_str()); 82 | } else { 83 | MDS_LOG_INFO("%s", msg.c_str()); 84 | } 85 | 86 | send_reply(ex.http_status()); 87 | } catch (const std::exception &ex) { 88 | MDS_LOG_ERROR("uncaughted exception: http_status = 500 ; description = %s", ex.what()); 89 | send_reply(500); 90 | } 91 | } 92 | 93 | void 94 | elliptics::download_info_t::on_finished(const ioremap::elliptics::sync_lookup_result &slr 95 | , const ioremap::elliptics::error_info &error) { 96 | MDS_LOG_DEBUG("Download info: prepare response"); 97 | 98 | try { 99 | if (error) { 100 | auto http_status = (error.code() == -ENOENT ? 404 : 500); 101 | throw http_error(http_status, error.message()); 102 | } 103 | 104 | auto res = server()->generate_signature_for_elliptics_file(slr, x_regional_host 105 | , ns_state, expiration_time); 106 | 107 | send_response(std::move(res)); 108 | } catch (const http_error &ex) { 109 | std::ostringstream oss; 110 | oss 111 | << "http_error: http_status = " << ex.http_status() 112 | << " ; description = " << ex.what(); 113 | auto msg = oss.str(); 114 | 115 | if (ex.is_server_error()) { 116 | MDS_LOG_ERROR("%s", msg.c_str()); 117 | } else { 118 | MDS_LOG_INFO("%s", msg.c_str()); 119 | } 120 | 121 | send_reply(ex.http_status()); 122 | } catch (const std::exception &ex) { 123 | MDS_LOG_ERROR("uncaughted exception: http_status = 500 ; description = %s", ex.what()); 124 | send_reply(500); 125 | } 126 | } 127 | 128 | mastermind::namespace_state_t 129 | elliptics::download_info_t::get_namespace_state(const std::string &path 130 | , const std::string &handler) { 131 | try { 132 | return server()->get_namespace_state(path, handler); 133 | } catch (const std::exception &ex) { 134 | throw http_error(400, ex.what()); 135 | } 136 | } 137 | 138 | void 139 | elliptics::download_info_t::check_signature() { 140 | if (ns_settings(ns_state).sign_token.empty()) { 141 | throw http_error(403, "cannot generate downloadinfo xml without signature-token"); 142 | } 143 | } 144 | 145 | void 146 | elliptics::download_info_t::check_query_args() { 147 | const auto &query = request().url().query(); 148 | 149 | { 150 | auto format = get_arg(query, "format", "xml"); 151 | 152 | if (format != "xml" && format != "json" && format != "jsonp") { 153 | throw http_error(400, "unknown format=" + format); 154 | } 155 | } 156 | 157 | if (query.has_item("expiration-time")) { 158 | if (!ns_settings(ns_state).custom_expiration_time) { 159 | throw http_error(403, "using of expiration-time is prohibited"); 160 | } 161 | 162 | auto expiration_time_str = *query.item_value("expiration-time"); 163 | 164 | try { 165 | expiration_time = std::chrono::seconds( 166 | boost::lexical_cast(expiration_time_str)); 167 | } catch (const std::exception &ex) { 168 | throw http_error(400, std::string("cannot parse expiration-time: ") + ex.what()); 169 | } 170 | } 171 | } 172 | 173 | std::tuple, ioremap::elliptics::key> 174 | elliptics::download_info_t::prepare_session(const mastermind::namespace_state_t &ns_state) { 175 | try { 176 | auto prep_session = server()->prepare_session(request().url().path(), ns_state); 177 | 178 | if (std::get<0>(prep_session)->get_groups().empty()) { 179 | throw proxy_error("session was obtained without groups"); 180 | } 181 | 182 | std::get<0>(prep_session)->set_trace_bit(request().trace_bit()); 183 | std::get<0>(prep_session)->set_trace_id(request().request_id()); 184 | 185 | std::get<0>(prep_session)->set_filter(ioremap::elliptics::filters::all); 186 | std::get<0>(prep_session)->set_timeout(server()->timeout.lookup); 187 | 188 | return prep_session; 189 | } catch (const std::exception &ex) { 190 | throw http_error(400, ex.what()); 191 | } 192 | } 193 | 194 | void 195 | elliptics::download_info_t::process_get(ioremap::elliptics::session session 196 | , const ioremap::elliptics::key key) { 197 | MDS_LOG_DEBUG("Download info: looking up"); 198 | auto alr = session.quorum_lookup(key); 199 | 200 | alr.connect(wrap(std::bind(&download_info_t::on_finished, shared_from_this() 201 | , std::placeholders::_1, std::placeholders::_2))); 202 | } 203 | 204 | void 205 | elliptics::download_info_t::send_response( 206 | std::tuple res) { 207 | ioremap::thevoid::http_response reply; 208 | ioremap::swarm::http_headers headers; 209 | reply.set_code(200); 210 | std::string body; 211 | 212 | auto format = get_arg(request().url().query(), "format", "xml"); 213 | 214 | if (format == "xml") { 215 | headers.set_content_type("text/xml"); 216 | body = xml_response(std::move(res)); 217 | } else if (format == "json") { 218 | headers.set_content_type("application/json"); 219 | body = json_response(std::move(res)); 220 | } else if (format == "jsonp") { 221 | headers.set_content_type("application/javascript"); 222 | body = jsonp_response(std::move(res)); 223 | } 224 | 225 | headers.set_content_length(body.size()); 226 | reply.set_headers(headers); 227 | send_reply(std::move(reply), std::move(body)); 228 | } 229 | 230 | std::string 231 | elliptics::download_info_t::xml_response( 232 | std::tuple res) { 233 | std::stringstream oss; 234 | oss << ""; 235 | oss << ""; 236 | oss << "" << std::get<0>(res) << ""; 237 | oss << "" << std::get<1>(res) << ""; 238 | oss << "" << std::get<2>(res) << ""; 239 | oss << "-1"; 240 | oss << "" << std::get<3>(res) << ""; 241 | oss << ""; 242 | return oss.str(); 243 | } 244 | 245 | kora::dynamic_t 246 | elliptics::download_info_t::json_response_impl( 247 | std::tuple res) { 248 | auto dynamic = kora::dynamic_t::empty_object; 249 | auto &object = dynamic.as_object(); 250 | object["host"] = std::get<0>(res); 251 | object["path"] = std::get<1>(res); 252 | object["ts"] = std::get<2>(res); 253 | object["s"] = std::get<3>(res); 254 | return dynamic; 255 | } 256 | 257 | std::string 258 | elliptics::download_info_t::json_response( 259 | std::tuple res) { 260 | return kora::to_pretty_json(json_response_impl(std::move(res))); 261 | } 262 | 263 | std::string 264 | elliptics::download_info_t::jsonp_response( 265 | std::tuple res) { 266 | std::ostringstream oss; 267 | oss << get_arg(request().url().query(), "callback", ""); 268 | oss << "(" << json_response_impl(std::move(res)) << ")"; 269 | return oss.str(); 270 | } 271 | 272 | -------------------------------------------------------------------------------- /src/upload_simple.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "upload_simple.hpp" 21 | 22 | namespace elliptics { 23 | 24 | upload_simple_t::upload_simple_t(mastermind::namespace_state_t ns_state_ 25 | , couple_iterator_t couple_iterator_, std::string filename_) 26 | : ns_state(std::move(ns_state_)) 27 | , couple_iterator(std::move(couple_iterator_)) 28 | , filename(std::move(filename_)) 29 | , key(ns_state.name() + '.' + filename) 30 | , deferred_fallback([this] { fallback(); }) 31 | , can_retry_couple(true) 32 | , attempt_to_choose_a_couple(0) 33 | , internal_error(internal_error_errc::none) 34 | { 35 | } 36 | 37 | void 38 | upload_simple_t::on_request(const ioremap::thevoid::http_request &http_request) { 39 | set_chunk_size(server()->m_write_chunk_size); 40 | 41 | // The method runs in thevoid's io-loop, therefore proxy's dtor cannot run in this moment 42 | // Hence write_session can be safely used without any check 43 | lookup_session = *server()->lookup_session(http_request, {}); 44 | write_session = *server()->write_session(http_request, {}); 45 | 46 | offset = get_arg(http_request.url().query(), "offset", 0); 47 | 48 | auto self = shared_from_this(); 49 | auto next = [this, self] (util::expected result) { 50 | try { 51 | process_couple_info(std::move(result.get())); 52 | try_next_chunk(); 53 | } catch (const std::exception &ex) { 54 | MDS_LOG_INFO("cannot obtain couple: %s", ex.what()); 55 | send_error(internal_error_errc::general_error); 56 | } 57 | }; 58 | 59 | get_next_couple_info(std::move(next)); 60 | } 61 | 62 | void 63 | upload_simple_t::on_chunk(const boost::asio::const_buffer &buffer, unsigned int flags) { 64 | (void) flags; 65 | 66 | const char *buffer_data = boost::asio::buffer_cast(buffer); 67 | const size_t buffer_size = boost::asio::buffer_size(buffer); 68 | 69 | ioremap::elliptics::data_pointer chunk; 70 | 71 | chunk = ioremap::elliptics::data_pointer::copy(buffer_data, buffer_size); 72 | 73 | if (can_retry_couple) { 74 | data_pointer = chunk; 75 | } 76 | 77 | process_chunk(std::move(chunk)); 78 | } 79 | 80 | // The on_error call means an error occurred during working with socket (either read or write). 81 | // The close method is used as a part of send_headers callback. 82 | // That means on_error will not be called if socket write error occurrs. 83 | // Thus, only socket read error should be handled. 84 | void 85 | upload_simple_t::on_error(const boost::system::error_code &error_code) { 86 | MDS_LOG_ERROR("error during reading request: %s", error_code.message().c_str()); 87 | deferred_fallback(); 88 | } 89 | 90 | void 91 | upload_simple_t::on_write_is_done(const std::error_code &error_code) { 92 | if (error_code) { 93 | process_chunk_write_error(error_code); 94 | return; 95 | } 96 | 97 | can_retry_couple = false; 98 | data_pointer = ioremap::elliptics::data_pointer(); 99 | 100 | // Fallback will be executed if it is called twice: here and in on_error. 101 | if (deferred_fallback()) { 102 | return; 103 | } 104 | 105 | if (!writer->is_committed()) { 106 | try_next_chunk(); 107 | return; 108 | } 109 | 110 | send_result(); 111 | 112 | // Release writer to break cyclic links 113 | writer.reset(); 114 | } 115 | 116 | void 117 | upload_simple_t::send_result() { 118 | const auto &result = writer->get_result(); 119 | 120 | std::ostringstream oss; 121 | oss 122 | << "\n" 123 | << "\n"; 134 | 135 | const auto &entries_info = result.entries_info; 136 | 137 | for (auto it = entries_info.begin(), end = entries_info.end(); it != end; ++it) { 138 | oss 139 | << "address << "\"" 141 | << " path=\"" << it->path << "\"" 142 | << " group=\"" << it->group << "\"" 143 | << " status=\"0\"/>\n"; 144 | } 145 | 146 | oss 147 | << "" << entries_info.size() << "\n" 148 | << ""; 149 | 150 | auto res_str = oss.str(); 151 | 152 | ioremap::thevoid::http_response reply; 153 | ioremap::swarm::http_headers headers; 154 | 155 | reply.set_code(200); 156 | headers.set_content_length(res_str.size()); 157 | headers.set_content_type("text/xml"); 158 | reply.set_headers(headers); 159 | 160 | send_headers(std::move(reply) 161 | , std::bind(&upload_simple_t::headers_are_sent, shared_from_this() 162 | , res_str, std::placeholders::_1)); 163 | } 164 | 165 | void 166 | upload_simple_t::headers_are_sent(const std::string &res_str 167 | , const boost::system::error_code &error_code) { 168 | if (error_code) { 169 | MDS_LOG_ERROR("cannot send headers: %s", error_code.message().c_str()); 170 | fallback(); 171 | return; 172 | } 173 | 174 | MDS_LOG_INFO("headers are sent"); 175 | 176 | send_data(std::move(res_str) 177 | , std::bind(&upload_simple_t::data_is_sent, shared_from_this() 178 | , std::placeholders::_1)); 179 | } 180 | 181 | void 182 | upload_simple_t::data_is_sent(const boost::system::error_code &error_code) { 183 | if (error_code) { 184 | MDS_LOG_ERROR("cannot send data: %s", error_code.message().c_str()); 185 | fallback(); 186 | return; 187 | } 188 | 189 | MDS_LOG_INFO("data is sent"); 190 | 191 | close(boost::system::error_code()); 192 | } 193 | 194 | void 195 | upload_simple_t::fallback() { 196 | close(boost::system::error_code()); 197 | remove([] (util::expected) {}); 198 | } 199 | 200 | void 201 | upload_simple_t::remove(const util::expected::callback_t next) { 202 | if (auto session = server()->remove_session(request(), couple_info.groups)) { 203 | elliptics::remove(make_shared_logger(logger()), *session, key, std::move(next)); 204 | return; 205 | } 206 | 207 | next(util::expected_from_exception("remove-session is uninitialized")); 208 | } 209 | 210 | } // namespace elliptics 211 | 212 | void 213 | elliptics::upload_simple_t::get_next_couple_info( 214 | util::expected::callback_t next) { 215 | if (!couple_iterator.has_next()) { 216 | next(util::expected_from_exception( 217 | "there is no couple to process upload")); 218 | return; 219 | } 220 | 221 | auto self = shared_from_this(); 222 | auto couple_info = couple_iterator.next(); 223 | 224 | ++attempt_to_choose_a_couple; 225 | { 226 | std::ostringstream oss; 227 | oss << "process request with couple=" << couple_info.groups 228 | << "; attempt=" << attempt_to_choose_a_couple; 229 | auto msg = oss.str(); 230 | MDS_LOG_INFO("%s", msg.c_str()); 231 | } 232 | 233 | auto next_ = [this, self, couple_info, next] (util::expected result) { 234 | try { 235 | if (result.get()) { 236 | MDS_LOG_INFO("key can be written"); 237 | next(couple_info); 238 | return; 239 | } 240 | 241 | MDS_LOG_INFO("key cannot be written"); 242 | std::ostringstream oss; 243 | oss << "\n\n"; 244 | 245 | oss << ""; 246 | if (ns_settings(ns_state).static_couple.empty()) { 247 | oss << couple_info.id << '/'; 248 | } 249 | oss << encode_for_xml(filename); 250 | 251 | oss << "\n"; 252 | auto body = oss.str(); 253 | 254 | ioremap::thevoid::http_response reply; 255 | ioremap::swarm::http_headers headers; 256 | 257 | reply.set_code(403); 258 | headers.set_content_length(body.size()); 259 | headers.set_content_type("text/xml"); 260 | headers.set_keep_alive(false); 261 | reply.set_headers(headers); 262 | 263 | send_reply(std::move(reply), std::move(body)); 264 | return; 265 | } catch (const std::exception &ex) { 266 | MDS_LOG_ERROR("cannot check key for update: %s", ex.what()); 267 | get_next_couple_info(std::move(next)); 268 | } 269 | }; 270 | 271 | auto session = lookup_session->clone(); 272 | session.set_groups(couple_info.groups); 273 | 274 | can_be_written( 275 | make_shared_logger(logger()) 276 | , std::move(session), key, ns_state 277 | , std::move(next_)); 278 | } 279 | 280 | std::shared_ptr 281 | elliptics::upload_simple_t::make_writer(const groups_t &groups) { 282 | auto session = write_session->clone(); 283 | session.set_groups(groups); 284 | 285 | return std::make_shared( 286 | copy_logger(logger()) 287 | , session, key 288 | , *request().headers().content_length(), offset 289 | , server()->timeout_coef.data_flow_rate , ns_settings(ns_state).success_copies_num 290 | , server()->limit_of_middle_chunk_attempts 291 | , server()->scale_retry_timeout 292 | ); 293 | } 294 | 295 | void 296 | elliptics::upload_simple_t::process_couple_info(mastermind::couple_info_t couple_info_) { 297 | couple_info = std::move(couple_info_); 298 | 299 | lookup_session->set_groups(couple_info.groups); 300 | write_session->set_groups(couple_info.groups); 301 | 302 | writer = make_writer(couple_info.groups); 303 | } 304 | 305 | void 306 | elliptics::upload_simple_t::process_chunk(ioremap::elliptics::data_pointer chunk) { 307 | // There are two parallel activities: 308 | // 1. Reading client request 309 | // 2. Writing data into elliptics 310 | // Errors which could happen in second part are handled by writer object (involving removing of 311 | // needless file). But errors which could happen during reading request should be handled by 312 | // this object. 313 | // When such error is ocurred the on_error method will be called. But chunk writing can process 314 | // in this moment, so we cannot remove file and need to wait until writing is finished. 315 | // Otherwise if chunk writing does not process we have to initiate file removing in on_error 316 | // method. 317 | // To solve this problem deferred call of fallback method is used. The method will be executed 318 | // only on second call. The method is deferred by one call before each chunk writing, is called 319 | // after each chunk writing is finished and is called in on_error. 320 | deferred_fallback.defer(); 321 | 322 | auto self = shared_from_this(); 323 | auto next = [this, self] (const std::error_code &error_code) { 324 | on_write_is_done(error_code); 325 | }; 326 | 327 | writer->write(chunk, std::move(next)); 328 | } 329 | 330 | void 331 | elliptics::upload_simple_t::process_chunk_write_error(const std::error_code &error_code) { 332 | if (error_code == make_error_code(writer_errc::insufficient_storage)) { 333 | update_internal_error(internal_error_errc::insufficient_storage); 334 | } else { 335 | update_internal_error(internal_error_errc::general_error); 336 | } 337 | 338 | ns_state.weights().set_feedback(couple_info.id 339 | , mastermind::namespace_state_t::weights_t::feedback_tag::temporary_unavailable); 340 | 341 | writer.reset(); 342 | 343 | auto self = shared_from_this(); 344 | auto next = [this, self] (util::expected result) { 345 | // The remove result does not affect handler's flow 346 | (void) result; 347 | 348 | if (!can_retry_couple) { 349 | MDS_LOG_ERROR("could not write file into storage"); 350 | send_error(); 351 | return; 352 | } 353 | 354 | auto next = [this, self] (util::expected result) { 355 | try { 356 | process_couple_info(std::move(result.get())); 357 | process_chunk(data_pointer); 358 | } catch (const std::exception &ex) { 359 | MDS_LOG_INFO("cannot obtain couple: %s", ex.what()); 360 | send_error(internal_error_errc::general_error); 361 | } 362 | }; 363 | 364 | get_next_couple_info(std::move(next)); 365 | }; 366 | 367 | remove(std::move(next)); 368 | } 369 | 370 | void 371 | elliptics::upload_simple_t::update_internal_error(internal_error_errc errc) { 372 | if (static_cast(internal_error) < static_cast(errc)) { 373 | internal_error = errc; 374 | } 375 | } 376 | 377 | void 378 | elliptics::upload_simple_t::send_error() { 379 | switch(internal_error) { 380 | case internal_error_errc::none: 381 | throw std::runtime_error("cannot send 5xx error code because there is no error"); 382 | case internal_error_errc::general_error: 383 | reply()->send_error(ioremap::swarm::http_response::internal_server_error); 384 | break; 385 | case internal_error_errc::insufficient_storage: 386 | reply()->send_error(ioremap::swarm::http_response::insufficient_storage); 387 | break; 388 | } 389 | } 390 | 391 | void 392 | elliptics::upload_simple_t::send_error(internal_error_errc errc) { 393 | update_internal_error(errc); 394 | send_error(); 395 | } 396 | 397 | -------------------------------------------------------------------------------- /src/writer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "writer.hpp" 21 | 22 | #include "loggers.hpp" 23 | #include "utils.hpp" 24 | #include "lookup_result.hpp" 25 | #include "write_retrier.hpp" 26 | #include "proxy.hpp" 27 | 28 | class error_category_t 29 | : public std::error_category 30 | { 31 | public: 32 | const char * 33 | name() const noexcept { 34 | return "writer error category"; 35 | } 36 | 37 | std::string 38 | message(int ev) const { 39 | switch (static_cast(ev)) { 40 | case elliptics::writer_errc::success: 41 | return "success"; 42 | case elliptics::writer_errc::unexpected_event: 43 | return "unexpected event"; 44 | case elliptics::writer_errc::incorrect_size: 45 | return "incorrect size"; 46 | case elliptics::writer_errc::internal: 47 | return "internal error"; 48 | case elliptics::writer_errc::insufficient_storage: 49 | return "insufficient storage"; 50 | default: 51 | return "unknown error"; 52 | } 53 | } 54 | }; 55 | 56 | const std::error_category & 57 | elliptics::writer_category() { 58 | const static error_category_t instance; 59 | return instance; 60 | } 61 | 62 | std::error_code 63 | elliptics::make_error_code(writer_errc e) { 64 | return std::error_code(static_cast(e), writer_category()); 65 | } 66 | 67 | std::error_condition 68 | elliptics::make_error_condition(writer_errc e) { 69 | return std::error_condition(static_cast(e), writer_category()); 70 | } 71 | 72 | elliptics::writer_error::writer_error(writer_errc e, const std::string &message) 73 | : std::system_error(make_error_code(e), message) 74 | { 75 | } 76 | 77 | elliptics::writer_t::writer_t(ioremap::swarm::logger bh_logger_ 78 | , const ioremap::elliptics::session &session_, std::string key_ 79 | , size_t total_size_, size_t offset_, size_t commit_coef_, size_t success_copies_num_ 80 | , size_t limit_of_attempts_, double scale_retry_timeout_ 81 | ) 82 | : state(state_tag::waiting) 83 | , errc_for_client(writer_errc::success) 84 | , bh_logger(std::move(bh_logger_)) 85 | , session(session_.clone()) 86 | , key(std::move(key_)) 87 | , total_size(total_size_) 88 | , offset(offset_) 89 | , commit_coef(commit_coef_) 90 | , success_copies_num(success_copies_num_) 91 | , limit_of_attempts(limit_of_attempts_) 92 | , scale_retry_timeout(scale_retry_timeout_) 93 | , written_size(0) 94 | , start_time(std::chrono::system_clock::now()) 95 | { 96 | session.set_filter(ioremap::elliptics::filters::all_with_ack); 97 | session.set_checker(ioremap::elliptics::checkers::at_least_one); 98 | 99 | key.transform(session); 100 | key.set_id(key.id()); 101 | 102 | { 103 | std::ostringstream oss; 104 | oss 105 | << "writing starts:" 106 | << " key=" << key.remote() 107 | // The only reason to print elliptics key is ioremap::elliptics::key::transform 108 | // method doesn't log this information 109 | << " elliptics-key=" << key.to_string() 110 | << " offset=" << offset 111 | << " total-size=" << total_size 112 | << " groups=" << session.get_groups() 113 | << " success-copiens-num=" << success_copies_num; 114 | 115 | auto msg = oss.str(); 116 | 117 | MDS_LOG_INFO("%s", msg.c_str()); 118 | } 119 | } 120 | 121 | void 122 | elliptics::writer_t::write(const ioremap::elliptics::data_pointer &data_pointer 123 | , callback_t next) { 124 | // We need to prolong the lifetime of the shared state here to be sure it is alive 125 | // till the end of the function. 126 | // The reason of being uncertain is calling async_result.connect(next_) in this function below. 127 | // In that call user's callback may be called in-place, and the only external shared pointer to 128 | // the shared state may be destroyed there. 129 | auto self = shared_from_this(); 130 | 131 | lock_guard_t lock_guard(state_mutex); 132 | (void) lock_guard; 133 | 134 | switch (state) { 135 | case state_tag::waiting: { 136 | size_t future_size = data_pointer.size() + written_size; 137 | 138 | if (future_size > total_size) { 139 | throw writer_error(writer_errc::incorrect_size); 140 | } 141 | 142 | if (written_size == 0 && data_pointer.size() == total_size) { 143 | log_chunk("simple", data_pointer.size()); 144 | auto async_result = session.write_data(key, data_pointer, offset); 145 | written_size = data_pointer.size(); 146 | 147 | // Actually state should be changed immediately before return 148 | // But connect can call callback synchronously 149 | state = state_tag::committing; 150 | 151 | auto next_ = std::bind(&writer_t::on_data_wrote, shared_from_this() 152 | , std::placeholders::_1, std::placeholders::_2, std::move(next)); 153 | 154 | lock_guard.unlock(); 155 | async_result.connect(next_); 156 | lock_guard.lock(); 157 | return; 158 | } 159 | 160 | auto async_result = write_impl(data_pointer); 161 | written_size += data_pointer.size(); 162 | offset += data_pointer.size(); 163 | 164 | auto next_ = std::bind(&writer_t::on_data_wrote, shared_from_this() 165 | , std::placeholders::_1, std::placeholders::_2, std::move(next)); 166 | 167 | lock_guard.unlock(); 168 | async_result.connect(next_); 169 | lock_guard.lock(); 170 | break; 171 | } 172 | case state_tag::writing: 173 | case state_tag::committing: 174 | case state_tag::committed: 175 | case state_tag::failed: 176 | throw writer_error(writer_errc::unexpected_event); 177 | } 178 | } 179 | 180 | elliptics::writer_t::result_t 181 | elliptics::writer_t::get_result() const { 182 | result_t result; 183 | 184 | result.id = get_id(); 185 | result.key = get_key(); 186 | result.total_size = get_total_size(); 187 | result.entries_info = get_entries_info(); 188 | 189 | return result; 190 | } 191 | 192 | const elliptics::writer_t::entries_info_t & 193 | elliptics::writer_t::get_entries_info() const { 194 | lock_guard_t lock_guard(state_mutex); 195 | (void) lock_guard; 196 | 197 | switch (state) { 198 | case state_tag::committed: 199 | case state_tag::waiting: 200 | case state_tag::failed: 201 | return entries_info; 202 | case state_tag::writing: 203 | case state_tag::committing: 204 | // Default is needed only for avoding compile warning: 205 | // 'control reaches end of non-void function' 206 | default: 207 | throw writer_error(writer_errc::unexpected_event); 208 | } 209 | } 210 | 211 | bool 212 | elliptics::writer_t::is_finished() const { 213 | lock_guard_t lock_guard(state_mutex); 214 | (void) lock_guard; 215 | 216 | return state == state_tag::committed 217 | || state == state_tag::failed; 218 | } 219 | 220 | bool 221 | elliptics::writer_t::is_committed() const { 222 | lock_guard_t lock_guard(state_mutex); 223 | (void) lock_guard; 224 | 225 | return state == state_tag::committed; 226 | } 227 | 228 | bool 229 | elliptics::writer_t::is_failed() const { 230 | lock_guard_t lock_guard(state_mutex); 231 | (void) lock_guard; 232 | 233 | return state == state_tag::failed; 234 | } 235 | 236 | size_t 237 | elliptics::writer_t::get_total_size() const { 238 | return total_size; 239 | } 240 | 241 | const std::string & 242 | elliptics::writer_t::get_key() const { 243 | return key.remote(); 244 | } 245 | 246 | std::string 247 | elliptics::writer_t::get_id() const { 248 | return key.to_string(); 249 | } 250 | 251 | ioremap::swarm::logger & 252 | elliptics::writer_t::logger() { 253 | return bh_logger; 254 | } 255 | 256 | void 257 | elliptics::writer_t::log_chunk( 258 | const std::string &write_type, size_t chunk_size) { 259 | std::ostringstream oss; 260 | oss 261 | << "write chunk:" 262 | << " key=" << key.remote() 263 | << " total-size=" << total_size 264 | << " offset=" << offset 265 | << " chunk-size=" << chunk_size 266 | << " data-left=" << (total_size - offset) 267 | << " write-type=" << write_type; 268 | 269 | auto msg = oss.str(); 270 | 271 | MDS_LOG_INFO("%s", msg.c_str()); 272 | } 273 | 274 | void 275 | elliptics::writer_t::update_groups( 276 | const ioremap::elliptics::sync_write_result &entries) { 277 | std::vector good_groups; 278 | 279 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 280 | auto &entry = *it; 281 | 282 | int group_id = entry.command()->id.group_id; 283 | 284 | if (entry.status() == 0) { 285 | good_groups.emplace_back(group_id); 286 | } else { 287 | bad_groups.emplace_back(group_id); 288 | } 289 | } 290 | 291 | session.set_groups(good_groups); 292 | 293 | { 294 | std::ostringstream oss; 295 | oss 296 | << "writing of chunk is finished:" 297 | << " key=" << key.remote() 298 | << " good-groups=" << good_groups 299 | << " bad-groups=" << bad_groups; 300 | 301 | auto msg = oss.str(); 302 | 303 | MDS_LOG_INFO("%s", msg.c_str()); 304 | } 305 | } 306 | 307 | void 308 | elliptics::writer_t::set_result( 309 | const ioremap::elliptics::sync_write_result &entries) { 310 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 311 | lookup_result pl(*it, ""); 312 | if (pl.status() != 0) { 313 | continue; 314 | } 315 | 316 | entry_info_t entry_info; 317 | 318 | entry_info.address = pl.addr(); 319 | entry_info.path = pl.full_path(); 320 | entry_info.group = pl.group(); 321 | 322 | entries_info.emplace_back(std::move(entry_info)); 323 | } 324 | } 325 | 326 | bool 327 | elliptics::writer_t::write_is_good( 328 | const ioremap::elliptics::error_info &error_info) { 329 | return !error_info && session.get_groups().size() >= success_copies_num; 330 | } 331 | 332 | ioremap::elliptics::async_write_result 333 | elliptics::writer_t::write_impl( 334 | const ioremap::elliptics::data_pointer &data_pointer) { 335 | if (written_size == 0) { 336 | log_chunk("prepare", data_pointer.size()); 337 | state = state_tag::writing; 338 | return session.write_prepare(key, data_pointer, offset, total_size); 339 | } else { 340 | size_t future_size = written_size + data_pointer.size(); 341 | 342 | if (future_size > total_size) { 343 | throw writer_error(writer_errc::incorrect_size); 344 | } 345 | 346 | if (future_size == total_size) { 347 | if (commit_coef) { 348 | session.set_timeout(session.get_timeout() + total_size / commit_coef); 349 | } 350 | log_chunk("commit", data_pointer.size()); 351 | state = state_tag::committing; 352 | return session.write_commit(key, data_pointer, offset, future_size); 353 | } else { 354 | log_chunk("plain", data_pointer.size()); 355 | state = state_tag::writing; 356 | 357 | auto key = this->key; 358 | auto offset = this->offset; 359 | auto command = [key, data_pointer, offset] (ioremap::elliptics::session session) 360 | -> ioremap::elliptics::async_write_result { 361 | return session.write_plain(key, data_pointer, offset); 362 | }; 363 | 364 | return try_write(ioremap::swarm::logger(logger(), blackhole::log::attributes_t()) 365 | , session, command, success_copies_num, limit_of_attempts, scale_retry_timeout); 366 | } 367 | } 368 | } 369 | 370 | elliptics::writer_errc 371 | elliptics::writer_t::choose_errc_for_client(const ioremap::elliptics::sync_write_result &entries) { 372 | bool is_insufficient_storage = false; 373 | 374 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 375 | auto status = it->status(); 376 | 377 | switch (status) { 378 | case -ENOSPC: 379 | is_insufficient_storage = true; 380 | break; 381 | } 382 | 383 | } 384 | 385 | if (is_insufficient_storage) { 386 | return writer_errc::insufficient_storage; 387 | } 388 | 389 | return writer_errc::internal; 390 | } 391 | 392 | void 393 | elliptics::writer_t::on_data_wrote( 394 | const ioremap::elliptics::sync_write_result &entries 395 | , const ioremap::elliptics::error_info &error_info 396 | , callback_t next) { 397 | #define LOG_RESULT(VERBOSITY, STATUS) \ 398 | do { \ 399 | auto spent_time = std::chrono::duration_cast( \ 400 | std::chrono::system_clock::now() - start_time).count(); \ 401 | \ 402 | std::ostringstream oss; \ 403 | oss \ 404 | << "writing is finished:" \ 405 | << " key=" << key.remote() \ 406 | << " spent-time=" << spent_time << "ms" \ 407 | << " status=" << STATUS \ 408 | << " wrote into groups " << session.get_groups() \ 409 | << " failed to write into groups " << bad_groups; \ 410 | \ 411 | auto msg = oss.str(); \ 412 | MDS_LOG_##VERBOSITY("%s", msg.c_str()); \ 413 | } while (0) 414 | 415 | lock_guard_t lock_guard(state_mutex); 416 | 417 | switch (state) { 418 | case state_tag::writing: 419 | case state_tag::committing: { 420 | update_groups(entries); 421 | 422 | if (write_is_good(error_info)) { 423 | if (state == state_tag::committing) { 424 | LOG_RESULT(INFO, "ok"); 425 | set_result(entries); 426 | state = state_tag::committed; 427 | } else { 428 | state = state_tag::waiting; 429 | } 430 | 431 | lock_guard.unlock(); 432 | next(make_error_code(writer_errc::success)); 433 | lock_guard.lock(); 434 | return; 435 | } 436 | 437 | LOG_RESULT(ERROR, "bad"); 438 | 439 | state = state_tag::failed; 440 | 441 | lock_guard.unlock(); 442 | next(make_error_code(choose_errc_for_client(entries))); 443 | lock_guard.lock(); 444 | break; 445 | } 446 | case state_tag::waiting: 447 | case state_tag::committed: 448 | case state_tag::failed: 449 | throw writer_error(writer_errc::unexpected_event); 450 | } 451 | 452 | #undef LOG_RESULT 453 | } 454 | 455 | #define logger() *shared_logger 456 | 457 | namespace detail { 458 | 459 | void 460 | can_be_written_on_lookup(shared_logger_t shared_logger 461 | , const ioremap::elliptics::sync_lookup_result &entries 462 | , const ioremap::elliptics::error_info &error_info 463 | , util::expected::callback_t next) { 464 | (void) error_info; 465 | 466 | for (auto it = entries.begin(), end = entries.end(); it != end; ++it) { 467 | const int group_id = it->command()->id.group_id; 468 | 469 | if (it->status() == -ENOENT) { 470 | continue; 471 | } 472 | 473 | if (it->status() == 0) { 474 | MDS_LOG_INFO("key was found in group=%d", group_id); 475 | next(false); 476 | return; 477 | } 478 | 479 | const auto entry_error_info = it->error(); 480 | MDS_LOG_ERROR("cannot check group=%d: %s", group_id , entry_error_info.message().c_str()); 481 | 482 | next(util::expected_from_exception("cannot check some group")); 483 | return; 484 | } 485 | 486 | MDS_LOG_INFO("key was not found in any group"); 487 | next(true); 488 | } 489 | 490 | } // namespace detail 491 | 492 | void 493 | elliptics::can_be_written(shared_logger_t shared_logger 494 | , ioremap::elliptics::session session 495 | , std::string key 496 | , mastermind::namespace_state_t ns_state 497 | , util::expected::callback_t next) { 498 | 499 | { 500 | std::ostringstream oss; 501 | oss << "check for update couple " << session.get_groups(); 502 | auto msg = oss.str(); 503 | MDS_LOG_INFO("%s", msg.c_str()); 504 | } 505 | 506 | if (!ns_settings(ns_state).check_for_update) { 507 | MDS_LOG_INFO("check for update is disabled for the namespace"); 508 | next(true); 509 | return; 510 | } 511 | 512 | session = session.clone(); 513 | session.set_cflags(session.get_cflags() | DNET_FLAGS_NOLOCK); 514 | session.set_filter(ioremap::elliptics::filters::all); 515 | 516 | auto future = session.parallel_lookup(key); 517 | 518 | auto next_ = std::bind(&detail::can_be_written_on_lookup, shared_logger 519 | , std::placeholders::_1, std::placeholders::_2 520 | , std::move(next)); 521 | 522 | future.connect(next_); 523 | } 524 | 525 | -------------------------------------------------------------------------------- /src/upload_multipart.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Mediastorage-proxy is a HTTP proxy for mediastorage based on elliptics 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "upload_multipart.hpp" 21 | 22 | namespace elliptics { 23 | 24 | upload_multipart_t::multipart_context_t::multipart_context_t() 25 | : state(multipart_state_tag::init) 26 | , is_error(false) 27 | , is_interrupted(false) 28 | { 29 | reset(); 30 | } 31 | 32 | upload_multipart_t::multipart_context_t::const_iterator 33 | upload_multipart_t::multipart_context_t::begin() const { 34 | return iterator; 35 | } 36 | 37 | upload_multipart_t::multipart_context_t::const_iterator 38 | upload_multipart_t::multipart_context_t::end() const { 39 | return buffer.end(); 40 | } 41 | 42 | size_t 43 | upload_multipart_t::multipart_context_t::size() const { 44 | return end() - begin(); 45 | } 46 | 47 | void 48 | upload_multipart_t::multipart_context_t::append(const char *data, size_t size) { 49 | buffer.insert(buffer.end(), data, data + size); 50 | reset(); 51 | } 52 | 53 | void 54 | upload_multipart_t::multipart_context_t::skip(size_t size) { 55 | iterator += size; 56 | } 57 | 58 | void 59 | upload_multipart_t::multipart_context_t::trim() { 60 | buffer_t buffer_(begin(), end()); 61 | buffer.swap(buffer_); 62 | reset(); 63 | } 64 | 65 | void 66 | upload_multipart_t::multipart_context_t::interrupt(bool is_error_) { 67 | is_interrupted = true; 68 | is_error = is_error_; 69 | } 70 | 71 | bool 72 | upload_multipart_t::multipart_context_t::interrupted() const { 73 | return is_interrupted; 74 | } 75 | 76 | bool 77 | upload_multipart_t::multipart_context_t::error() const { 78 | return is_error; 79 | } 80 | 81 | void 82 | upload_multipart_t::multipart_context_t::reset() { 83 | iterator = buffer.begin(); 84 | is_interrupted = false; 85 | } 86 | 87 | upload_multipart_t::upload_multipart_t(mastermind::namespace_state_t ns_state_, couple_t couple_) 88 | : interrupt_writers_once([this] { interrupt_writers(); }) 89 | , join_upload_tasks([this] { on_writers_are_finished(); }) 90 | , join_remove_tasks([this] { send_error(); }) 91 | , error_type(error_type_tag::none) 92 | , ns_state(std::move(ns_state_)) 93 | , couple(std::move(couple_)) 94 | , couple_id(*std::min_element(couple.begin(), couple.end())) 95 | { 96 | } 97 | 98 | void 99 | upload_multipart_t::on_headers(ioremap::thevoid::http_request &&http_request_) { 100 | http_request = std::move(http_request_); 101 | 102 | if (const auto &arg = http_request.headers().content_type()) { 103 | const auto &type = *arg; 104 | auto pos = type.find("boundary="); 105 | if (pos == std::string::npos) { 106 | MDS_LOG_INFO("boundary is missing"); 107 | reply()->send_error(ioremap::swarm::http_response::bad_request); 108 | return; 109 | } 110 | pos += sizeof("boundary=") - 1; 111 | boundary = std::string("--") + type.substr(pos); 112 | } else { 113 | MDS_LOG_INFO("Cannot process request without content-type"); 114 | reply()->send_error(ioremap::swarm::http_response::bad_request); 115 | return; 116 | } 117 | 118 | } 119 | 120 | size_t 121 | upload_multipart_t::on_data(const boost::asio::const_buffer &buffer) { 122 | const char *buffer_data = boost::asio::buffer_cast(buffer); 123 | const size_t buffer_size = boost::asio::buffer_size(buffer); 124 | 125 | multipart_context.append(buffer_data, buffer_size); 126 | 127 | do { 128 | switch (multipart_context.state) { 129 | case multipart_state_tag::init: 130 | sm_init(); 131 | break; 132 | case multipart_state_tag::headers: 133 | sm_headers(); 134 | break; 135 | case multipart_state_tag::body: 136 | sm_body(); 137 | break; 138 | case multipart_state_tag::after_body: 139 | sm_after_body(); 140 | break; 141 | case multipart_state_tag::end: 142 | sm_end(); 143 | break; 144 | } 145 | } while (!multipart_context.interrupted()); 146 | 147 | { 148 | if (multipart_context.error()) { 149 | interrupt_writers(error_type_tag::multipart); 150 | } 151 | 152 | // If multipart_context.state is equal to end, the join was already called in this task. 153 | if (is_error() && multipart_state_tag::end != multipart_context.state) { 154 | buffered_writer.reset(); 155 | join_upload_tasks(); 156 | return 0; 157 | } 158 | } 159 | 160 | 161 | multipart_context.trim(); 162 | 163 | return buffer_size; 164 | } 165 | 166 | void 167 | upload_multipart_t::on_close(const boost::system::error_code &error) { 168 | if (error) { 169 | interrupt_writers(error_type_tag::client); 170 | // Multipart parser is not finished if reading request error is ocurred. 171 | // That means on_data will not be called anymore and we need to join parser task here. 172 | join_upload_tasks(); 173 | } 174 | } 175 | 176 | void 177 | upload_multipart_t::sm_init() { 178 | const std::string BOUNDARY_RN_STRING = boundary + "\r\n"; 179 | 180 | if (multipart_context.size() < BOUNDARY_RN_STRING.size()) { 181 | multipart_context.interrupt(false); 182 | return; 183 | } 184 | 185 | if (!std::equal(multipart_context.begin(), multipart_context.begin() + BOUNDARY_RN_STRING.size() 186 | , BOUNDARY_RN_STRING.begin())) { 187 | MDS_LOG_INFO("incorrect body: trace 1"); 188 | multipart_context.interrupt(true); 189 | return; 190 | } 191 | 192 | multipart_context.state = multipart_state_tag::headers; 193 | multipart_context.skip(BOUNDARY_RN_STRING.size()); 194 | } 195 | 196 | 197 | void 198 | upload_multipart_t::sm_headers() { 199 | static const std::string RNRN_STRING = "\r\n\r\n"; 200 | 201 | auto headers_end = std::search(multipart_context.begin(), multipart_context.end() 202 | , RNRN_STRING.begin(), RNRN_STRING.end()); 203 | 204 | if (headers_end == multipart_context.end()) { 205 | multipart_context.interrupt(false); 206 | return; 207 | } 208 | 209 | std::string headers(multipart_context.begin(), headers_end); 210 | multipart_context.skip(headers.size() + RNRN_STRING.size()); 211 | 212 | auto CD_pos = headers.find("Content-Disposition"); 213 | 214 | if (CD_pos == std::string::npos) { 215 | MDS_LOG_INFO("incorrect body: trace 2"); 216 | multipart_context.interrupt(true); 217 | return; 218 | } 219 | 220 | auto name_pos = headers.find("name=", CD_pos); 221 | auto return_pos = headers.find("\r\n", CD_pos); 222 | 223 | if (return_pos < name_pos) { 224 | MDS_LOG_INFO("incorrect body: trace 3"); 225 | multipart_context.interrupt(true); 226 | return; 227 | } 228 | 229 | auto name_begin = headers.find('\"', name_pos) + 1; 230 | auto name_end = headers.find('\"', name_begin); 231 | 232 | if (name_end - name_begin == 0) { 233 | MDS_LOG_INFO("incorrect body: trace 4"); 234 | multipart_context.interrupt(true); 235 | return; 236 | } 237 | 238 | auto name = headers.substr(name_begin, name_end - name_begin); 239 | 240 | { 241 | auto pos = name.find_first_not_of('/'); 242 | if (pos == std::string::npos) { 243 | MDS_LOG_INFO("incorrect body: part name consists only of \'/\'"); 244 | multipart_context.interrupt(true); 245 | return; 246 | } 247 | 248 | if (pos != 0) { 249 | MDS_LOG_INFO("cut %d \'/\' from the begin of the name \'%s\'" 250 | , static_cast(pos), name.c_str()); 251 | 252 | name = name.substr(pos); 253 | } 254 | } 255 | 256 | current_filename = name; 257 | 258 | buffered_writer = std::make_shared( 259 | ioremap::swarm::logger(logger(), blackhole::log::attributes_t()) 260 | , ns_state.name() + '.' + name, server()->m_write_chunk_size); 261 | 262 | multipart_context.state = multipart_state_tag::body; 263 | } 264 | 265 | 266 | void 267 | upload_multipart_t::sm_body() { 268 | const std::string RN_BOUNDARY_STRING = "\r\n" + boundary; 269 | 270 | if (multipart_context.size() < RN_BOUNDARY_STRING.size() + 1) { 271 | multipart_context.interrupt(false); 272 | return; 273 | } 274 | 275 | bool boundary_found = true; 276 | 277 | auto boundary_it = std::search(multipart_context.begin(), multipart_context.end() 278 | , RN_BOUNDARY_STRING.begin(), RN_BOUNDARY_STRING.end()); 279 | 280 | if (boundary_it == multipart_context.end()) { 281 | boundary_it = multipart_context.begin() + 282 | (multipart_context.size() - RN_BOUNDARY_STRING.size()); 283 | 284 | boundary_found = false; 285 | } 286 | 287 | auto size = boundary_it - multipart_context.begin(); 288 | buffered_writer->append(&*multipart_context.begin(), size); 289 | multipart_context.skip(size); 290 | 291 | if (boundary_found) { 292 | multipart_context.skip(RN_BOUNDARY_STRING.size()); 293 | multipart_context.state = multipart_state_tag::after_body; 294 | 295 | start_writing(); 296 | } else { 297 | multipart_context.interrupt(false); 298 | } 299 | } 300 | 301 | 302 | void 303 | upload_multipart_t::sm_after_body() { 304 | static const std::string MINUS_PREFIX_STRING = "--"; 305 | static const std::string RN_STRING = "\r\n"; 306 | 307 | if (multipart_context.size() < 2) { 308 | multipart_context.interrupt(false); 309 | return; 310 | } 311 | 312 | if (!std::equal(multipart_context.begin() 313 | , multipart_context.begin() + MINUS_PREFIX_STRING.size() 314 | , MINUS_PREFIX_STRING.begin())) { 315 | if (!std::equal(multipart_context.begin() 316 | , multipart_context.begin() + RN_STRING.size() 317 | , RN_STRING.begin())) { 318 | MDS_LOG_INFO("incorrect body: trace 5"); 319 | multipart_context.interrupt(true); 320 | return; 321 | } else { 322 | multipart_context.state = multipart_state_tag::headers; 323 | multipart_context.skip(RN_STRING.size()); 324 | return; 325 | } 326 | } 327 | 328 | multipart_context.state = multipart_state_tag::end; 329 | } 330 | 331 | void 332 | upload_multipart_t::sm_end() { 333 | multipart_context.interrupt(false); 334 | join_upload_tasks(); 335 | } 336 | 337 | void 338 | upload_multipart_t::start_writing() { 339 | { 340 | std::lock_guard lock(buffered_writers_mutex); 341 | (void) lock; 342 | 343 | if (is_error()) { 344 | multipart_context.interrupt(false); 345 | return; 346 | } 347 | 348 | join_upload_tasks.defer(); 349 | 350 | buffered_writers.insert(std::make_pair(current_filename, buffered_writer)); 351 | } 352 | 353 | auto self = shared_from_this(); 354 | auto next = [this, self] (const std::error_code &error_code) { 355 | on_writer_is_finished(error_code); 356 | }; 357 | 358 | // The method runs in thevoid's io-loop, therefore proxy's dtor cannot run in this moment 359 | // Hence write_session can be safely used without any check 360 | buffered_writer->write(*server()->write_session(http_request, couple) 361 | , server()->timeout_coef.data_flow_rate 362 | , ns_settings(ns_state).success_copies_num 363 | , server()->limit_of_middle_chunk_attempts 364 | , server()->scale_retry_timeout 365 | , std::move(next)); 366 | 367 | buffered_writer.reset(); 368 | } 369 | 370 | void 371 | upload_multipart_t::on_writer_is_finished(const std::error_code &error_code) { 372 | if (error_code) { 373 | const auto interrupted_error = make_error_code(buffered_writer_errc::interrupted); 374 | 375 | if (error_code != interrupted_error) { 376 | if (error_code == make_error_code(writer_errc::insufficient_storage)) { 377 | interrupt_writers(error_type_tag::insufficient_storage); 378 | } else { 379 | interrupt_writers(error_type_tag::internal); 380 | } 381 | } 382 | } 383 | 384 | join_upload_tasks(); 385 | } 386 | 387 | void 388 | upload_multipart_t::set_error(error_type_tag e) { 389 | if (error_type_tag::none == e) { 390 | throw std::runtime_error("unexpected error type"); 391 | } 392 | 393 | std::lock_guard lock_guard(error_type_mutex); 394 | (void) lock_guard; 395 | 396 | // Errors have priorities: 397 | // 1. Client error means there is no reason to send response 398 | // 2. Insufficient Storage error means we should send 507 399 | // 3. Internal error means we should send 500 400 | // 4. Multipart error means we should send 400 401 | switch (error_type) { 402 | case error_type_tag::none: 403 | error_type = e; 404 | break; 405 | case error_type_tag::insufficient_storage: 406 | if (error_type_tag::client == e) { 407 | error_type = e; 408 | } 409 | break; 410 | case error_type_tag::internal: 411 | if (error_type_tag::client == e || error_type_tag::insufficient_storage == e) { 412 | error_type = e; 413 | } 414 | break; 415 | case error_type_tag::multipart: 416 | error_type = e; 417 | break; 418 | case error_type_tag::client: 419 | // nothing to do 420 | break; 421 | } 422 | } 423 | 424 | bool 425 | upload_multipart_t::is_error() { 426 | return error_type_tag::none != get_error(); 427 | } 428 | 429 | upload_multipart_t::error_type_tag 430 | upload_multipart_t::get_error() { 431 | std::lock_guard lock_guard(error_type_mutex); 432 | (void) lock_guard; 433 | 434 | return error_type; 435 | } 436 | 437 | void 438 | upload_multipart_t::interrupt_writers(error_type_tag e) { 439 | set_error(e); 440 | interrupt_writers_once(); 441 | } 442 | 443 | void 444 | upload_multipart_t::interrupt_writers() { 445 | std::lock_guard lock_guard(buffered_writers_mutex); 446 | (void) lock_guard; 447 | 448 | MDS_LOG_INFO("interrupt writers"); 449 | for (auto it = buffered_writers.begin(), end = buffered_writers.end(); it != end; ++it) { 450 | it->second->interrupt(); 451 | } 452 | } 453 | 454 | void 455 | upload_multipart_t::on_writers_are_finished() { 456 | for (auto it = buffered_writers.begin(), end = buffered_writers.end(); it != end; ++it) { 457 | results.insert(std::make_pair(it->first, it->second->get_result())); 458 | } 459 | 460 | buffered_writers.clear(); 461 | 462 | if (is_error()) { 463 | remove_files(); 464 | return; 465 | } 466 | 467 | send_result(); 468 | } 469 | 470 | void 471 | upload_multipart_t::send_result() { 472 | MDS_LOG_INFO("send result"); 473 | 474 | std::ostringstream oss; 475 | 476 | oss << "\n"; 477 | 478 | oss 479 | << "\n"; 481 | 482 | for (auto it = results.begin(), end = results.end(); it != end; ++it) { 483 | 484 | oss 485 | << " second.key) 486 | << "\" id=\"" << it->second.id 487 | << "\" groups=\"" << ns_state.settings().groups_count() 488 | << "\" size=\"" << it->second.total_size 489 | << "\" key=\""; 490 | 491 | if (ns_settings(ns_state).static_couple.empty()) { 492 | oss << couple_id << '/'; 493 | } 494 | 495 | oss << encode_for_xml(it->first) << "\">\n"; 496 | 497 | const auto &entries_info = it->second.entries_info; 498 | 499 | for (auto it = entries_info.begin(), end = entries_info.end(); it != end; ++it) { 500 | oss 501 | << " address << "\"" 503 | << " path=\"" << it->path << "\"" 504 | << " group=\"" << it->group << "\"" 505 | << " status=\"0\"/>\n"; 506 | } 507 | 508 | oss 509 | << " " << entries_info.size() << "\n" 510 | << " \n"; 511 | } 512 | 513 | oss << ""; 514 | 515 | auto res_str = oss.str(); 516 | 517 | ioremap::thevoid::http_response reply; 518 | ioremap::swarm::http_headers headers; 519 | 520 | reply.set_code(200); 521 | headers.set_content_length(res_str.size()); 522 | headers.set_content_type("text/xml"); 523 | reply.set_headers(headers); 524 | 525 | send_headers(std::move(reply) 526 | , std::bind(&upload_multipart_t::headers_are_sent, shared_from_this() 527 | , res_str, std::placeholders::_1)); 528 | } 529 | 530 | void 531 | upload_multipart_t::headers_are_sent(const std::string &res_str 532 | , const boost::system::error_code &error_code) { 533 | if (error_code) { 534 | MDS_LOG_ERROR("cannot send headers: %s", error_code.message().c_str()); 535 | set_error(error_type_tag::client); 536 | remove_files(); 537 | return; 538 | } 539 | 540 | MDS_LOG_INFO("headers are sent"); 541 | 542 | send_data(std::move(res_str) 543 | , std::bind(&upload_multipart_t::data_is_sent, shared_from_this() 544 | , std::placeholders::_1)); 545 | } 546 | 547 | void 548 | upload_multipart_t::data_is_sent(const boost::system::error_code &error_code) { 549 | if (error_code) { 550 | MDS_LOG_ERROR("cannot send data: %s", error_code.message().c_str()); 551 | set_error(error_type_tag::client); 552 | remove_files(); 553 | return; 554 | } 555 | 556 | MDS_LOG_INFO("data is sent"); 557 | 558 | close(boost::system::error_code()); 559 | } 560 | 561 | void 562 | upload_multipart_t::remove_files() { 563 | if (auto session = server()->remove_session(http_request, couple)) { 564 | MDS_LOG_INFO("removing uploaded files"); 565 | 566 | auto shared_logger = make_shared_logger(logger()); 567 | auto next = std::bind(&upload_multipart_t::on_removed, shared_from_this() 568 | , std::placeholders::_1); 569 | 570 | for (auto it = results.begin(), end = results.end(); it != end; ++it) { 571 | join_remove_tasks.defer(); 572 | elliptics::remove(shared_logger, *session, it->second.key, next); 573 | } 574 | 575 | join_remove_tasks(); 576 | } else { 577 | MDS_LOG_ERROR("cannot remove files of failed request: remove-session is uninitialized"); 578 | } 579 | } 580 | 581 | void 582 | upload_multipart_t::on_removed(util::expected result) { 583 | // The remove result does not affect handler's flow 584 | (void) result; 585 | 586 | join_remove_tasks(); 587 | } 588 | 589 | void 590 | upload_multipart_t::send_error() { 591 | auto err = get_error(); 592 | 593 | MDS_LOG_INFO("send error: %s", static_cast(err)); 594 | 595 | switch (get_error()) { 596 | case error_type_tag::none: 597 | throw std::runtime_error("unexpected error type: none"); 598 | case error_type_tag::insufficient_storage: 599 | reply()->send_error(ioremap::swarm::http_response::insufficient_storage); 600 | break; 601 | case error_type_tag::internal: 602 | reply()->send_error(ioremap::swarm::http_response::internal_server_error); 603 | break; 604 | case error_type_tag::multipart: 605 | reply()->send_error(ioremap::swarm::http_response::bad_request); 606 | break; 607 | case error_type_tag::client: 608 | close(boost::system::error_code()); 609 | break; 610 | default: 611 | throw std::runtime_error("unexpected error type: " 612 | + boost::lexical_cast(static_cast(err))); 613 | } 614 | } 615 | 616 | } // namespace elliptics 617 | 618 | --------------------------------------------------------------------------------