├── .github └── FUNDING.yml ├── .gitignore ├── .gitmodules ├── .travis.yml ├── .ycm_extra_conf.py ├── CMakeLists.txt ├── Colors.cmake ├── DownloadProject.CMakeLists.cmake.in ├── DownloadProject.cmake ├── LICENSE ├── README.md ├── deps └── .gitignore ├── scripts └── install-boost.sh ├── src ├── ArgParser.cpp ├── ArgParser.hpp ├── CMakeLists.txt ├── Downloader.cpp ├── Downloader.hpp ├── main.cpp ├── toNetworkUris.cpp └── toNetworkUris.hpp └── test ├── CMakeLists.txt ├── Downloader_test.cpp └── toNetworkUris_test.cpp /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: pmalek 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | build 31 | compile_commands.json 32 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/uri"] 2 | path = deps/uri 3 | url = https://github.com/reBass/uri 4 | [submodule "deps/Beast"] 5 | path = deps/Beast 6 | url = https://github.com/vinniefalco/Beast.git 7 | [submodule "deps/googletest"] 8 | path = deps/googletest 9 | url = https://github.com/google/googletest.git 10 | [submodule "deps/rapidcheck"] 11 | path = deps/rapidcheck 12 | url = https://github.com/emil-e/rapidcheck.git 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | sudo: false 3 | 4 | env: 5 | global: 6 | # Maintenance note: to move to a new version 7 | # of boost, update both BOOST_ROOT and BOOST_URL. 8 | # Note that for simplicity, BOOST_ROOT's final 9 | # namepart must match the folder name internal 10 | # to boost's .tar.gz. 11 | - BOOST_ROOT=$HOME/boost_1_62_0 12 | - BOOST_URL='http://downloads.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.bz2?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fboost%2Ffiles%2Fboost%2F1.62.0%2F&ts=1479653651&use_mirror=netix' 13 | 14 | 15 | addons: 16 | apt: 17 | sources: &apt_sources 18 | - ubuntu-toolchain-r-test 19 | - george-edison55-precise-backports 20 | - llvm-toolchain-precise-3.5 21 | - llvm-toolchain-precise-3.6 22 | - llvm-toolchain-precise-3.7 23 | - llvm-toolchain-precise-3.8 24 | 25 | matrix: 26 | include: 27 | - compiler: gcc 28 | addons: 29 | apt: 30 | sources: *apt_sources 31 | packages: ["gcc-4.9", "g++-4.9", "cmake", "cmake-data"] 32 | env: 33 | - COMPILER=g++-4.9 34 | - compiler: gcc 35 | addons: 36 | apt: 37 | sources: *apt_sources 38 | packages: ["gcc-5", "g++-5", "cmake", "cmake-data"] 39 | env: 40 | - COMPILER=g++-5 41 | - compiler: gcc 42 | addons: 43 | apt: 44 | sources: *apt_sources 45 | packages: ["gcc-6", "g++-6", "cmake", "cmake-data"] 46 | env: 47 | - COMPILER=g++-6 48 | - compiler: gcc 49 | addons: 50 | apt: 51 | sources: *apt_sources 52 | packages: ["gcc-7", "g++-7", "cmake", "cmake-data"] 53 | env: 54 | - COMPILER=g++-7 55 | - compiler: clang 56 | addons: 57 | apt: 58 | sources: *apt_sources 59 | packages: ["clang-3.6", "cmake", "cmake-data"] 60 | env: 61 | - COMPILER=clang++-3.6 62 | - compiler: clang 63 | addons: 64 | apt: 65 | sources: *apt_sources 66 | packages: ["clang-3.7", "cmake", "cmake-data"] 67 | env: 68 | - COMPILER=clang++-3.7 69 | 70 | cache: 71 | directories: 72 | - $BOOST_ROOT 73 | 74 | before_install: 75 | - scripts/install-boost.sh 76 | - export CXX=$COMPILER 77 | - $CXX -v 78 | - git submodule update --init --recursive 79 | 80 | script: 81 | - mkdir build && cd build && cmake -DCMAKE_VERBOSE_MAKEFILE=1 .. 82 | - cmake --build . 83 | - ./wget http://www.google.com 84 | - ./wget http://kernel.ubuntu.com/\~kernel-ppa/mainline/v4.0-vivid/BUILT 85 | -------------------------------------------------------------------------------- /.ycm_extra_conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import ycm_core 3 | 4 | from clang_helpers import PrepareClangFlags 5 | 6 | def DirectoryOfThisScript(): 7 | return os.path.dirname(os.path.abspath(__file__)) 8 | 9 | # This is the single most important line in this script. Everything else is just nice to have but 10 | # not strictly necessary. 11 | compilation_database_folder = DirectoryOfThisScript() 12 | 13 | # This provides a safe fall-back if no compilation commands are available. You could also add a 14 | # includes relative to your project directory, for example. 15 | flags = [ 16 | '-Wall', 17 | '-Wextra', 18 | '-std=c++14', 19 | '-x', 'c++', 20 | '-isystem', '/usr/local/include', 21 | '-isystem', '/usr/include', 22 | '-I.', 23 | ] 24 | 25 | if compilation_database_folder: 26 | database = ycm_core.CompilationDatabase(compilation_database_folder) 27 | else: 28 | database = None 29 | 30 | SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] 31 | 32 | def MakeRelativePathsInFlagsAbsolute( flags, working_directory ): 33 | if not working_directory: 34 | return list( flags ) 35 | new_flags = [] 36 | make_next_absolute = False 37 | path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ] 38 | for flag in flags: 39 | new_flag = flag 40 | 41 | if make_next_absolute: 42 | make_next_absolute = False 43 | if not flag.startswith( '/' ): 44 | new_flag = os.path.join( working_directory, flag ) 45 | 46 | for path_flag in path_flags: 47 | if flag == path_flag: 48 | make_next_absolute = True 49 | break 50 | 51 | if flag.startswith( path_flag ): 52 | path = flag[ len( path_flag ): ] 53 | new_flag = path_flag + os.path.join( working_directory, path ) 54 | break 55 | 56 | if new_flag: 57 | new_flags.append( new_flag ) 58 | return new_flags 59 | 60 | 61 | def IsHeaderFile( filename ): 62 | extension = os.path.splitext( filename )[ 1 ] 63 | return extension in [ '.h', '.hxx', '.hpp', '.hh' ] 64 | 65 | 66 | def GetCompilationInfoForFile( filename ): 67 | # The compilation_commands.json file generated by CMake does not have entries 68 | # for header files. So we do our best by asking the db for flags for a 69 | # corresponding source file, if any. If one exists, the flags for that file 70 | # should be good enough. 71 | if IsHeaderFile( filename ): 72 | basename = os.path.splitext( filename )[ 0 ] 73 | for extension in SOURCE_EXTENSIONS: 74 | replacement_file = basename + extension 75 | if os.path.exists( replacement_file ): 76 | compilation_info = database.GetCompilationInfoForFile( 77 | replacement_file ) 78 | if compilation_info.compiler_flags_: 79 | return compilation_info 80 | return None 81 | return database.GetCompilationInfoForFile( filename ) 82 | 83 | 84 | def FlagsForFile( filename, **kwargs ): 85 | if database: 86 | # Bear in mind that compilation_info.compiler_flags_ does NOT return a 87 | # python list, but a "list-like" StringVec object 88 | compilation_info = GetCompilationInfoForFile( filename ) 89 | if not compilation_info: 90 | return None 91 | 92 | final_flags = MakeRelativePathsInFlagsAbsolute( 93 | compilation_info.compiler_flags_, 94 | compilation_info.compiler_working_dir_ ) 95 | 96 | else: 97 | relative_to = DirectoryOfThisScript() 98 | final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to ) 99 | 100 | return { 101 | 'flags': final_flags, 102 | 'do_cache': True 103 | } 104 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | project(wget C CXX) # C is necessary for find_package(Threads) for some reason 4 | 5 | option(WGET_BUILD_TESTS "Build wget's unit tests" ON) 6 | 7 | if(NOT CMAKE_BUILD_TYPE) 8 | set(CMAKE_BUILD_TYPE "Debug" CACHE STRING 9 | "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) 10 | endif() 11 | message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") 12 | 13 | find_package(OpenSSL REQUIRED) 14 | 15 | # Checking boost ... 16 | find_package(Boost 1.58.0 REQUIRED COMPONENTS system program_options) 17 | # ... and pthreads 18 | set(THREADS_PREFER_PTHREAD_FLAG ON) 19 | find_package(Threads) 20 | find_package(OpenSSL REQUIRED) 21 | 22 | ########################################################### 23 | 24 | include(Colors.cmake) 25 | 26 | greenMessage("Handling vinniefalco/Beast...") 27 | add_subdirectory(deps/Beast EXCLUDE_FROM_ALL) 28 | 29 | greenMessage("Handling cpp-netlib/uri...") 30 | set(Uri_FULL_WARNINGS OFF CACHE BOOL "") 31 | set(Uri_WARNINGS_AS_ERRORS OFF CACHE BOOL "") 32 | set(Uri_BUILD_TESTS OFF CACHE BOOL "") 33 | set(Uri_BUILD_DOCS OFF CACHE BOOL "") 34 | set(Uri_DISABLE_LIBCXX ON CACHE BOOL "") 35 | add_subdirectory(deps/uri EXCLUDE_FROM_ALL) 36 | 37 | ########################################################### 38 | 39 | greenMessage("wget...") 40 | set(CMAKE_CXX_STANDARD 14) 41 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 42 | set(COMMON_CXX_FLAGS "-Wuninitialized -Wpedantic -Wextra -Wall -Wsign-conversion") 43 | 44 | if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) 45 | message("Compiling with gcc...") 46 | set(COMMON_CXX_FLAGS "${COMMON_CXX_FLAGS} -Werror=suggest-override") 47 | elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 48 | message("Compiling with clang...") 49 | endif() 50 | 51 | ########################################################### 52 | # copmilation databse for YouCompleteMe 53 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 54 | IF( EXISTS "${CMAKE_BINARY_DIR}/compile_commands.json" ) 55 | EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different 56 | ${CMAKE_BINARY_DIR}/compile_commands.json 57 | ${CMAKE_SOURCE_DIR}/compile_commands.json 58 | ) 59 | ENDIF() 60 | ########################################################### 61 | 62 | add_subdirectory(src/) 63 | 64 | ########################################################### 65 | # tests 66 | if(WGET_BUILD_TESTS) 67 | enable_testing() 68 | add_subdirectory(test) 69 | endif(WGET_BUILD_TESTS) 70 | -------------------------------------------------------------------------------- /Colors.cmake: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | string(ASCII 27 Esc) 4 | set(ColourReset "${Esc}[m") 5 | set(ColourBold "${Esc}[1m") 6 | set(Red "${Esc}[31m") 7 | set(Green "${Esc}[32m") 8 | set(Yellow "${Esc}[33m") 9 | set(Blue "${Esc}[34m") 10 | set(Magenta "${Esc}[35m") 11 | set(Cyan "${Esc}[36m") 12 | set(White "${Esc}[37m") 13 | set(BoldRed "${Esc}[1;31m") 14 | set(BoldGreen "${Esc}[1;32m") 15 | set(BoldYellow "${Esc}[1;33m") 16 | set(BoldBlue "${Esc}[1;34m") 17 | set(BoldMagenta "${Esc}[1;35m") 18 | set(BoldCyan "${Esc}[1;36m") 19 | set(BoldWhite "${Esc}[1;37m") 20 | 21 | function(greenMessage message) 22 | message(STATUS "${Green}${message}${ColourReset}") 23 | endfunction(greenMessage) 24 | -------------------------------------------------------------------------------- /DownloadProject.CMakeLists.cmake.in: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | project(${DL_ARGS_PROJ}-download NONE) 4 | 5 | include(ExternalProject) 6 | ExternalProject_Add(${DL_ARGS_PROJ}-download 7 | ${DL_ARGS_UNPARSED_ARGUMENTS} 8 | SOURCE_DIR "${DL_ARGS_SOURCE_DIR}" 9 | BINARY_DIR "${DL_ARGS_BINARY_DIR}" 10 | CONFIGURE_COMMAND "" 11 | BUILD_COMMAND "" 12 | INSTALL_COMMAND "" 13 | TEST_COMMAND "" 14 | ) 15 | -------------------------------------------------------------------------------- /DownloadProject.cmake: -------------------------------------------------------------------------------- 1 | # MODULE: DownloadProject 2 | # 3 | # PROVIDES: 4 | # download_project( PROJ projectName 5 | # [PREFIX prefixDir] 6 | # [DOWNLOAD_DIR downloadDir] 7 | # [SOURCE_DIR srcDir] 8 | # [BINARY_DIR binDir] 9 | # [QUIET] 10 | # ... 11 | # ) 12 | # 13 | # Provides the ability to download and unpack a tarball, zip file, git repository, 14 | # etc. at configure time (i.e. when the cmake command is run). How the downloaded 15 | # and unpacked contents are used is up to the caller, but the motivating case is 16 | # to download source code which can then be included directly in the build with 17 | # add_subdirectory() after the call to download_project(). Source and build 18 | # directories are set up with this in mind. 19 | # 20 | # The PROJ argument is required. The projectName value will be used to construct 21 | # the following variables upon exit (obviously replace projectName with its actual 22 | # value): 23 | # 24 | # projectName_SOURCE_DIR 25 | # projectName_BINARY_DIR 26 | # 27 | # The SOURCE_DIR and BINARY_DIR arguments are optional and would not typically 28 | # need to be provided. They can be specified if you want the downloaded source 29 | # and build directories to be located in a specific place. The contents of 30 | # projectName_SOURCE_DIR and projectName_BINARY_DIR will be populated with the 31 | # locations used whether you provide SOURCE_DIR/BINARY_DIR or not. 32 | # 33 | # The DOWNLOAD_DIR argument does not normally need to be set. It controls the 34 | # location of the temporary CMake build used to perform the download. 35 | # 36 | # The PREFIX argument can be provided to change the base location of the default 37 | # values of DOWNLOAD_DIR, SOURCE_DIR and BINARY_DIR. If all of those three arguments 38 | # are provided, then PREFIX will have no effect. The default value for PREFIX is 39 | # CMAKE_BINARY_DIR. 40 | # 41 | # The QUIET option can be given if you do not want to show the output associated 42 | # with downloading the specified project. 43 | # 44 | # In addition to the above, any other options are passed through unmodified to 45 | # ExternalProject_Add() to perform the actual download, patch and update steps. 46 | # The following ExternalProject_Add() options are explicitly prohibited (they 47 | # are reserved for use by the download_project() command): 48 | # 49 | # CONFIGURE_COMMAND 50 | # BUILD_COMMAND 51 | # INSTALL_COMMAND 52 | # TEST_COMMAND 53 | # 54 | # Only those ExternalProject_Add() arguments which relate to downloading, patching 55 | # and updating of the project sources are intended to be used. Also note that at 56 | # least one set of download-related arguments are required. 57 | # 58 | # If using CMake 3.2 or later, the UPDATE_DISCONNECTED option can be used to 59 | # prevent a check at the remote end for changes every time CMake is run 60 | # after the first successful download. See the documentation of the ExternalProject 61 | # module for more information. It is likely you will want to use this option if it 62 | # is available to you. 63 | # 64 | # EXAMPLE USAGE: 65 | # 66 | # include(download_project.cmake) 67 | # download_project(PROJ googletest 68 | # GIT_REPOSITORY https://github.com/google/googletest.git 69 | # GIT_TAG master 70 | # UPDATE_DISCONNECTED 1 71 | # QUIET 72 | # ) 73 | # 74 | # add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR}) 75 | # 76 | #======================================================================================== 77 | 78 | 79 | set(_DownloadProjectDir "${CMAKE_CURRENT_LIST_DIR}") 80 | 81 | include(CMakeParseArguments) 82 | 83 | function(download_project) 84 | 85 | set(options QUIET) 86 | set(oneValueArgs 87 | PROJ 88 | PREFIX 89 | DOWNLOAD_DIR 90 | SOURCE_DIR 91 | BINARY_DIR 92 | # Prevent the following from being passed through 93 | CONFIGURE_COMMAND 94 | BUILD_COMMAND 95 | INSTALL_COMMAND 96 | TEST_COMMAND 97 | ) 98 | set(multiValueArgs "") 99 | 100 | cmake_parse_arguments(DL_ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) 101 | 102 | # Hide output if requested 103 | if (DL_ARGS_QUIET) 104 | set(OUTPUT_QUIET "OUTPUT_QUIET") 105 | else() 106 | unset(OUTPUT_QUIET) 107 | message(STATUS "Downloading/updating ${DL_ARGS_PROJ}") 108 | endif() 109 | 110 | # Set up where we will put our temporary CMakeLists.txt file and also 111 | # the base point below which the default source and binary dirs will be 112 | if (NOT DL_ARGS_PREFIX) 113 | set(DL_ARGS_PREFIX "${CMAKE_BINARY_DIR}") 114 | endif() 115 | if (NOT DL_ARGS_DOWNLOAD_DIR) 116 | set(DL_ARGS_DOWNLOAD_DIR "${DL_ARGS_PREFIX}/${DL_ARGS_PROJ}-download") 117 | endif() 118 | 119 | # Ensure the caller can know where to find the source and build directories 120 | if (NOT DL_ARGS_SOURCE_DIR) 121 | set(DL_ARGS_SOURCE_DIR "${DL_ARGS_PREFIX}/${DL_ARGS_PROJ}-src") 122 | endif() 123 | if (NOT DL_ARGS_BINARY_DIR) 124 | set(DL_ARGS_BINARY_DIR "${DL_ARGS_PREFIX}/${DL_ARGS_PROJ}-build") 125 | endif() 126 | set(${DL_ARGS_PROJ}_SOURCE_DIR "${DL_ARGS_SOURCE_DIR}" PARENT_SCOPE) 127 | set(${DL_ARGS_PROJ}_BINARY_DIR "${DL_ARGS_BINARY_DIR}" PARENT_SCOPE) 128 | 129 | # Create and build a separate CMake project to carry out the download. 130 | # If we've already previously done these steps, they will not cause 131 | # anything to be updated, so extra rebuilds of the project won't occur. 132 | configure_file("${_DownloadProjectDir}/DownloadProject.CMakeLists.cmake.in" 133 | "${DL_ARGS_DOWNLOAD_DIR}/CMakeLists.txt") 134 | execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . 135 | ${OUTPUT_QUIET} 136 | WORKING_DIRECTORY "${DL_ARGS_DOWNLOAD_DIR}" 137 | ) 138 | execute_process(COMMAND ${CMAKE_COMMAND} --build . 139 | ${OUTPUT_QUIET} 140 | WORKING_DIRECTORY "${DL_ARGS_DOWNLOAD_DIR}" 141 | ) 142 | 143 | endfunction() 144 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Patryk Małek 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## HTTP file downloader build with [boost::asio](http://www.boost.org/doc/libs/1_62_0/doc/html/boost_asio.html) and [beast](https://github.com/vinniefalco/Beast) 2 | 3 | [![Build Status](https://travis-ci.org/pmalek/wget.svg?branch=master)](https://travis-ci.org/pmalek/wget) 4 | 5 | ### How to build 6 | 7 | ``` 8 | git clone https://github.com/pmalek/wget.git && \ 9 | cd wget && git submodule update --init --recursive --depth 1 && \ 10 | mkdir ../build && cd ../build && cmake ../wget && cmake --build . 11 | ``` 12 | -------------------------------------------------------------------------------- /deps/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pmalek/wget/c10e7e47a14e54555f3d13582663bd81a597a4b0/deps/.gitignore -------------------------------------------------------------------------------- /scripts/install-boost.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Assumptions: 3 | # 1) BOOST_ROOT and BOOST_URL are already defined, 4 | # and contain valid values. 5 | # 2) The last namepart of BOOST_ROOT matches the 6 | # folder name internal to boost's .tar.bz2 7 | set -eu 8 | if [ ! -d "$BOOST_ROOT/lib" ] 9 | then 10 | wget --no-check-certificate $BOOST_URL -O /tmp/boost.tar.bz2 11 | cd `dirname $BOOST_ROOT` 12 | rm -fr ${BOOST_ROOT} 13 | tar -xf /tmp/boost.tar.bz2 14 | 15 | params="define=_GLIBCXX_USE_CXX11_ABI=0 \ 16 | --with-program_options --with-system --with-coroutine --with-filesystem" 17 | cd $BOOST_ROOT && \ 18 | ./bootstrap.sh --prefix=$BOOST_ROOT && \ 19 | ./b2 -d1 $params && \ 20 | ./b2 -d0 $params install 21 | else 22 | echo "Using cached boost at $BOOST_ROOT" 23 | fi 24 | 25 | -------------------------------------------------------------------------------- /src/ArgParser.cpp: -------------------------------------------------------------------------------- 1 | #include "ArgParser.hpp" 2 | 3 | #include 4 | 5 | namespace po = boost::program_options; 6 | 7 | ArgParser::ArgParser(int argc, const char** argv) : argc(argc), argv(argv) {} 8 | 9 | bool ArgParser::parse() { 10 | po::options_description desc("Allowed options"); 11 | desc.add_options()("help", "produce help message")( 12 | "files", 13 | po::value>()->required()->multitoken(), 14 | "list of files to be downloaded"); 15 | 16 | po::positional_options_description pos; 17 | pos.add("files", -1); 18 | 19 | try { 20 | po::store(po::command_line_parser(argc, argv).options(desc).positional(pos).run(), vm); 21 | po::notify(vm); 22 | } catch(const po::required_option& e) { 23 | if(!vm.count("help")) std::cerr << e.what() << '\n' << '\n'; 24 | std::cout << desc << "\n"; 25 | return false; 26 | } catch(const po::error& e) { 27 | std::cerr << "Couldn't parse command line arguments properly:\n"; 28 | std::cerr << e.what() << '\n' << '\n'; 29 | std::cerr << desc << '\n'; 30 | return false; 31 | } 32 | 33 | if(vm.count("help")) { 34 | std::cout << desc << "\n"; 35 | return false; 36 | } 37 | 38 | return true; 39 | } 40 | 41 | std::vector ArgParser::get(const std::string& arg_name) const { 42 | return vm[arg_name].as>(); 43 | } 44 | -------------------------------------------------------------------------------- /src/ArgParser.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | struct ArgParser { 4 | ArgParser(int argc, const char** argv); 5 | ArgParser(const ArgParser&) = delete; 6 | ArgParser& operator=(const ArgParser&) = delete; 7 | 8 | bool parse(); 9 | 10 | std::vector get(const std::string& arg_name) const; 11 | 12 | private: 13 | int argc; 14 | const char** argv; 15 | boost::program_options::variables_map vm; 16 | }; 17 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.2) 2 | 3 | set(wget_src_list Downloader.cpp 4 | ArgParser.cpp 5 | toNetworkUris.cpp 6 | ) 7 | set(wget_src_list_with_main main.cpp ${wget_src_list}) 8 | set_source_files_properties(${wget_src_list_with_main} PROPERTIES COMPILE_FLAGS "${COMMON_CXX_FLAGS}") 9 | 10 | ################################################################# 11 | add_library(wget_src OBJECT ${wget_src_list}) 12 | 13 | ################################################################# 14 | add_executable(${CMAKE_PROJECT_NAME} main.cpp $) 15 | set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) 16 | target_link_libraries(${CMAKE_PROJECT_NAME} network-uri ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES}) 17 | 18 | ################################################################# 19 | foreach(target wget_src ${CMAKE_PROJECT_NAME}) 20 | target_include_directories(${target} SYSTEM PRIVATE ${CMAKE_SOURCE_DIR}/deps/Beast/include 21 | ${CMAKE_SOURCE_DIR}/deps/uri/include 22 | ${Boost_INCLUDE_DIRS}) 23 | endforeach(target) 24 | -------------------------------------------------------------------------------- /src/Downloader.cpp: -------------------------------------------------------------------------------- 1 | #include "Downloader.hpp" 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | using namespace boost::asio; 13 | using namespace boost::asio::ip; 14 | 15 | Downloader::Downloader(boost::asio::io_service& ioservice) 16 | : ioservice(ioservice), resolv(ioservice) {} 17 | 18 | Downloader::future_type Downloader::download_async(const std::string& url) { 19 | std::promise promise; 20 | auto future = promise.get_future(); 21 | 22 | download_async(url, std::move(promise)); 23 | 24 | return future; 25 | } 26 | 27 | void Downloader::download_async(const std::string& url, 28 | std::promise&& promise) { 29 | auto state = std::make_shared(std::move(promise), 30 | boost::asio::ip::tcp::socket{ioservice}); 31 | try { 32 | state->uri = network::uri{url}; 33 | } catch(...) { 34 | state->promise.set_exception(std::current_exception()); 35 | } 36 | 37 | download_async(state); 38 | } 39 | 40 | void Downloader::download_async(state_ptr state) { 41 | ip::tcp::resolver::query query(state->uri.host().to_string(), 42 | state->uri.scheme().to_string()); 43 | 44 | resolv.async_resolve( 45 | query, 46 | [this, state](const boost::system::error_code& ec, 47 | ip::tcp::resolver::iterator it) { on_resolve(state, ec, it); }); 48 | } 49 | 50 | void Downloader::on_resolve(state_ptr state, 51 | const boost::system::error_code& ec, 52 | tcp::resolver::iterator it) { 53 | if(ec) { 54 | state->promise.set_exception( 55 | std::make_exception_ptr(boost::system::system_error(ec))); 56 | return; 57 | } 58 | 59 | state->socket.async_connect(*it, 60 | [this, state](const boost::system::error_code& ec) { 61 | this->on_connect(state, ec); 62 | }); 63 | } 64 | 65 | void Downloader::on_connect(state_ptr state, const boost::system::error_code& ec) { 66 | if(ec) { 67 | state->promise.set_exception( 68 | std::make_exception_ptr(boost::system::system_error(ec))); 69 | return; 70 | } 71 | 72 | beast::http::request req; 73 | req.method = "GET"; 74 | req.url = state->uri.path().empty() ? "/" : state->uri.path().to_string(); 75 | req.version = 11; 76 | req.fields.insert("Host", state->uri.host().to_string()); 77 | req.fields.insert("User-Agent", "Beast"); 78 | beast::http::prepare(req); 79 | 80 | if(state->uri.scheme().to_string() == "https") { 81 | boost::asio::ssl::context ctx{boost::asio::ssl::context::tlsv12}; 82 | state->ssl_stream = 83 | std::make_unique>( 84 | state->socket, ctx); 85 | state->ssl_stream->set_verify_mode(boost::asio::ssl::verify_fail_if_no_peer_cert); 86 | try { 87 | state->ssl_stream->handshake(boost::asio::ssl::stream_base::client); 88 | } catch(...) { 89 | state->promise.set_exception(std::current_exception()); 90 | return; 91 | } 92 | 93 | beast::http::async_write(*state->ssl_stream, 94 | std::move(req), 95 | [this, state](const boost::system::error_code& ec) { 96 | this->on_request_sent(state, ec); 97 | }); 98 | } else { 99 | beast::http::async_write(state->socket, 100 | std::move(req), 101 | [this, state](const boost::system::error_code& ec) { 102 | this->on_request_sent(state, ec); 103 | }); 104 | } 105 | } 106 | 107 | void Downloader::on_request_sent(state_ptr state, 108 | const boost::system::error_code& ec) { 109 | if(ec) { 110 | state->promise.set_exception( 111 | std::make_exception_ptr(boost::system::system_error(ec))); 112 | return; 113 | } 114 | 115 | state->response = 116 | std::make_unique>(); 117 | state->streambuf = std::make_unique(); 118 | 119 | if(state->ssl_stream) { 120 | beast::http::async_read(*state->ssl_stream, 121 | *state->streambuf, 122 | *state->response, 123 | [this, state](const boost::system::error_code& ec) { 124 | this->on_read(state, ec); 125 | }); 126 | 127 | } else { 128 | beast::http::async_read(state->socket, 129 | *state->streambuf, 130 | *state->response, 131 | [this, state](const boost::system::error_code& ec) { 132 | this->on_read(state, ec); 133 | }); 134 | } 135 | } 136 | 137 | void Downloader::on_read(state_ptr state, const boost::system::error_code& ec) { 138 | if(ec) { 139 | state->promise.set_exception( 140 | std::make_exception_ptr(boost::system::system_error(ec))); 141 | return; 142 | } 143 | 144 | if((state->response->status == 301 || state->response->status == 302) 145 | && state->response->fields.exists("Location")) { 146 | download_async(state->response->fields["Location"].to_string(), 147 | std::move(state->promise)); 148 | return; 149 | } 150 | 151 | state->promise.set_value(std::move(*state->response)); 152 | } 153 | -------------------------------------------------------------------------------- /src/Downloader.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | class Downloader { 14 | public: 15 | using response_type = beast::http::response; 16 | using future_type = std::future; 17 | 18 | explicit Downloader(boost::asio::io_service &ioservice); 19 | future_type download_async(const std::string &url); 20 | 21 | private: 22 | struct State { 23 | std::promise promise; 24 | network::uri uri; 25 | boost::asio::ip::tcp::socket socket; 26 | std::unique_ptr> 27 | ssl_stream; 28 | std::unique_ptr> response; 29 | std::unique_ptr streambuf; 30 | 31 | State(std::promise &&promise, 32 | boost::asio::ip::tcp::socket &&socket) 33 | : promise{std::move(promise)}, socket(std::move(socket)) {} 34 | }; 35 | using state_ptr = std::shared_ptr; 36 | 37 | void download_async(const std::string &url, std::promise &&promise); 38 | void download_async(state_ptr state); 39 | 40 | void on_resolve(state_ptr state, 41 | const boost::system::error_code &ec, 42 | boost::asio::ip::tcp::resolver::iterator iterator); 43 | void on_connect(state_ptr state, const boost::system::error_code &ec); 44 | void on_request_sent(state_ptr state, const boost::system::error_code &ec); 45 | void on_read(state_ptr state, const boost::system::error_code &ec); 46 | 47 | boost::asio::io_service &ioservice; 48 | boost::asio::ip::tcp::resolver resolv; 49 | }; 50 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "ArgParser.hpp" 7 | #include "Downloader.hpp" 8 | 9 | int main(int argc, const char** argv) { 10 | ArgParser a(argc, argv); 11 | if(!a.parse()) return 1; 12 | 13 | boost::asio::io_service io_service; 14 | 15 | auto urls = a.get("files"); 16 | Downloader d(io_service); 17 | 18 | std::vector> downloads(urls.size()); 19 | 20 | std::transform(std::begin(urls), std::end(urls), std::begin(downloads), [&](auto& url) { 21 | return std::make_tuple(url, std::move(d.download_async(url))); 22 | }); 23 | 24 | std::thread asio_thread{[&]() { io_service.run(); }}; 25 | 26 | for(auto& tuple : downloads) { 27 | auto& url = std::get<0>(tuple); 28 | auto& download = std::get<1>(tuple); 29 | 30 | std::cout << url << "\n===\n"; 31 | try { 32 | auto response = download.get(); 33 | 34 | std::cout << "Received status code: " << response.status << '\n'; 35 | 36 | for(auto pair : response.fields) { 37 | std::cout << pair.first << " : " << pair.second << '\n'; 38 | } 39 | } catch(boost::system::system_error& e) { 40 | std::cout << "Error (" << e.code() << "): " << e.what() << "\n"; 41 | } catch(std::exception& e) { 42 | std::cout << "Error: " << e.what() << "\n"; 43 | } 44 | std::cout << "\n"; 45 | } 46 | 47 | asio_thread.join(); 48 | } 49 | -------------------------------------------------------------------------------- /src/toNetworkUris.cpp: -------------------------------------------------------------------------------- 1 | #include "toNetworkUris.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | namespace utils { 10 | std::vector toNetworkUris(const std::vector& urls) { 11 | using namespace boost::adaptors; 12 | std::vector v; 13 | boost::copy(urls | transformed([](const auto& url) { return network::uri{url}; }), std::back_inserter(v)); 14 | return v; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/toNetworkUris.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | namespace utils { 6 | std::vector toNetworkUris(const std::vector& urls); } 7 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #set(GMOCK_ROOT ${CMAKE_SOURCE_DIR}/deps/googletest/googlemock/) 2 | #set(GTEST_ROOT ${CMAKE_SOURCE_DIR}/deps/googletest/googletest/) 3 | #find_package(GTest REQUIRED) 4 | 5 | #add_subdirectory(${CMAKE_SOURCE_DIR}/deps/googletest/googletest/ gtest/) 6 | add_subdirectory(${CMAKE_SOURCE_DIR}/deps/googletest/googlemock/ gmock/) 7 | add_subdirectory(${CMAKE_SOURCE_DIR}/deps/rapidcheck/ rapidcheck/) 8 | 9 | file(GLOB TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) 10 | 11 | foreach(testSrc ${TEST_SRCS}) 12 | #Extract the filename without an extension (NAME_WE) 13 | 14 | get_filename_component(testName ${testSrc} NAME_WE) 15 | 16 | #Add compile target 17 | add_executable(${testName} $ ${testSrc}) 18 | 19 | target_include_directories(${testName} PRIVATE ${CMAKE_SOURCE_DIR}/src/ 20 | ${CMAKE_SOURCE_DIR}/deps/Beast/include/ 21 | ${CMAKE_SOURCE_DIR}/deps/googletest/googletest/include/ 22 | ${CMAKE_SOURCE_DIR}/deps/rapidcheck/extras/gtest/include/ 23 | ${CMAKE_SOURCE_DIR}/deps/uri/include) 24 | 25 | #link to Boost libraries AND your targets and dependencies 26 | target_link_libraries(${testName} gmock_main rapidcheck network-uri ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES}) 27 | 28 | #I like to move testing binaries into a test directory 29 | set_target_properties(${testName} PROPERTIES 30 | RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test) 31 | 32 | #Finally add it to test execution - 33 | #Notice the WORKING_DIRECTORY and COMMAND 34 | add_test(NAME ${testName} 35 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test 36 | COMMAND ${testName}) 37 | endforeach(testSrc) 38 | 39 | -------------------------------------------------------------------------------- /test/Downloader_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include "Downloader.hpp" 8 | 9 | TEST(DownloaderTests, constructorDoesntThrow){ 10 | boost::asio::io_service ioservice; 11 | EXPECT_NO_THROW(Downloader d{ioservice}); 12 | } 13 | 14 | TEST(DownloaderTests, passingIncorrectURLThrows){ 15 | boost::asio::io_service ioservice; 16 | Downloader d{ioservice}; 17 | auto fut = d.download_async("url"); 18 | EXPECT_THROW(fut.get(), std::system_error); 19 | } 20 | -------------------------------------------------------------------------------- /test/toNetworkUris_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "toNetworkUris.hpp" 13 | 14 | namespace { 15 | const std::string HTTP_WWW = "http://www."; 16 | const std::string HTTPS_WWW = "https://www."; 17 | } 18 | 19 | RC_GTEST_PROP(toNetworkUrisTest, DoesntThrowForCorrectURLs, (std::string url)) { 20 | // TODO 21 | //if(!url.empty()) { 22 | //if(boost::starts_with(url, HTTP_WWW) || boost::starts_with(url, HTTPS_WWW)) { 23 | //EXPECT_NO_THROW(utils::toNetworkUris({url})); 24 | //} else { 25 | //EXPECT_THROW(utils::toNetworkUris({url}), network::uri_syntax_error) << "Failed URL '" << url << "'" 26 | //<< std::endl; 27 | //} 28 | //} 29 | } 30 | 31 | // TEST(toNetworkUrisTests, XXX){ 32 | // std::vector urls{"http://www.google.com"}; 33 | // EXPECT_NO_THROW(utils::toNetworkUris(urls)); 34 | //} 35 | 36 | // TEST(toNetworkUrisTests, XXX2){ 37 | // std::vector urls{"httpx://www.google.com"}; 38 | // EXPECT_THROW(utils::toNetworkUris(urls), std::logic_error); 39 | //} 40 | 41 | // TEST(toNetworkUrisTests, XXX3){ 42 | // std::vector urls{"google.com"}; 43 | // EXPECT_THROW(utils::toNetworkUris(urls), std::logic_error); 44 | //} 45 | --------------------------------------------------------------------------------