├── .clang-format ├── .condarc ├── .github ├── linters_env.yml └── workflows │ ├── linters.yml │ ├── main.yml │ └── static_build.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CMakeLists.txt ├── LICENSE ├── README.md ├── docs └── assets │ └── logo.png ├── environment-static-dev-win.yml ├── environment-static-dev.yml ├── environment.yml ├── include └── powerloader │ ├── context.hpp │ ├── curl.hpp │ ├── download_target.hpp │ ├── downloader.hpp │ ├── enums.hpp │ ├── errors.hpp │ ├── export.hpp │ ├── fastest_mirror.hpp │ ├── fileio.hpp │ ├── mirror.hpp │ ├── mirrorid.hpp │ ├── mirrors │ ├── oci.hpp │ └── s3.hpp │ ├── powerloader.hpp │ ├── url.hpp │ └── utils.hpp ├── powerloaderConfig.cmake.in ├── src ├── cli │ └── main.cpp ├── compression.cpp ├── compression.hpp ├── context.cpp ├── curl.cpp ├── curl_internal.hpp ├── download_target.cpp ├── downloader.cpp ├── fastest_mirror.cpp ├── mirror.cpp ├── mirrors │ ├── oci.cpp │ └── s3.cpp ├── python │ ├── CMakeLists.txt │ └── main.cpp ├── target.cpp ├── target.hpp ├── uploader │ ├── multipart_upload.cpp │ ├── oci_upload.cpp │ └── s3_upload.cpp ├── url.cpp ├── utils.cpp ├── zck.cpp └── zck.hpp ├── test.py ├── test ├── CMakeLists.txt ├── conda_mock │ ├── __init__.py │ ├── conda_mock.py │ ├── config.py │ └── static │ │ ├── packages │ │ └── .gitignore │ │ └── zchunk │ │ ├── lorem.txt.x3.zck │ │ └── lorem.txt.zck ├── fixtures.py ├── helpers.py ├── local_static_mirrors.yml ├── mirrors.yml ├── ocitemplate.yml ├── passwd_format_one.yml ├── remote_mirrors.yml ├── s3template.yml ├── server.py ├── test_compression.cpp ├── test_fileio.cpp ├── test_main.cpp ├── test_oci_registry.py ├── test_other.py ├── test_s3.cpp ├── test_s3main_branch.py ├── test_s3mock.py ├── test_url.cpp └── test_utility.cpp └── testdata ├── f1.txt └── f1.txt.zst /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Mozilla 2 | AccessModifierOffset: '-4' 3 | AlignAfterOpenBracket: Align 4 | AlignEscapedNewlinesLeft: 'false' 5 | AllowAllParametersOfDeclarationOnNextLine: 'true' 6 | AllowShortBlocksOnASingleLine: 'false' 7 | AllowShortCaseLabelsOnASingleLine: 'false' 8 | AllowShortFunctionsOnASingleLine: 'false' 9 | AllowShortIfStatementsOnASingleLine: 'false' 10 | AllowShortLoopsOnASingleLine: 'false' 11 | AlwaysBreakTemplateDeclarations: 'true' 12 | SpaceAfterTemplateKeyword: 'true' 13 | BreakBeforeBinaryOperators: All 14 | BreakBeforeBraces: Allman 15 | BreakBeforeTernaryOperators: 'true' 16 | BreakConstructorInitializersBeforeComma: 'true' 17 | BreakStringLiterals: 'false' 18 | ColumnLimit: '100' 19 | ConstructorInitializerAllOnOneLineOrOnePerLine: 'false' 20 | ConstructorInitializerIndentWidth: '4' 21 | ContinuationIndentWidth: '4' 22 | Cpp11BracedListStyle: 'false' 23 | DerivePointerAlignment: 'false' 24 | DisableFormat: 'false' 25 | ExperimentalAutoDetectBinPacking: 'true' 26 | IndentCaseLabels: 'true' 27 | IndentWidth: '4' 28 | IndentWrappedFunctionNames: 'false' 29 | JavaScriptQuotes: Single 30 | KeepEmptyLinesAtTheStartOfBlocks: 'false' 31 | Language: Cpp 32 | MaxEmptyLinesToKeep: '2' 33 | NamespaceIndentation: All 34 | ObjCBlockIndentWidth: '4' 35 | ObjCSpaceAfterProperty: 'false' 36 | ObjCSpaceBeforeProtocolList: 'false' 37 | PointerAlignment: Left 38 | ReflowComments: 'true' 39 | SortIncludes: 'false' 40 | SpaceAfterCStyleCast: 'true' 41 | SpaceBeforeAssignmentOperators: 'true' 42 | SpaceBeforeParens: ControlStatements 43 | SpaceInEmptyParentheses: 'false' 44 | SpacesBeforeTrailingComments: '2' 45 | SpacesInAngles: 'false' 46 | SpacesInCStyleCastParentheses: 'false' 47 | SpacesInContainerLiterals: 'false' 48 | SpacesInParentheses: 'false' 49 | SpacesInSquareBrackets: 'false' 50 | Standard: Cpp11 51 | TabWidth: '4' 52 | UseTab: Never 53 | -------------------------------------------------------------------------------- /.condarc: -------------------------------------------------------------------------------- 1 | 2 | experimental_sat_error_message: true 3 | -------------------------------------------------------------------------------- /.github/linters_env.yml: -------------------------------------------------------------------------------- 1 | name: lintersenv 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - clang-tools=11.1.0 6 | - python 3.8 7 | - pre-commit 8 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters (Python, C++) 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - main 8 | pull_request: 9 | branches: 10 | - master 11 | - main 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: install mamba 19 | uses: mamba-org/provision-with-micromamba@main 20 | with: 21 | environment-file: .github/linters_env.yml 22 | condarc-file: .condarc 23 | - name: Run all linters 24 | shell: bash -l {0} 25 | run: | 26 | pre-commit run --all-files --verbose --show-diff-on-failure 27 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI shared 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | 10 | defaults: 11 | run: 12 | shell: bash -l {0} 13 | 14 | jobs: 15 | test: 16 | runs-on: ${{ matrix.os }} 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | os: [ubuntu-latest, macos-latest, windows-latest] 21 | 22 | name: test 23 | steps: 24 | - uses: actions/checkout@v2 25 | 26 | - name: install mamba 27 | uses: mamba-org/provision-with-micromamba@main 28 | with: 29 | environment-file: environment.yml 30 | condarc-file: .condarc 31 | 32 | - name: configure powerloader build 33 | if: runner.os != 'Windows' 34 | shell: bash -l {0} 35 | run: | 36 | mkdir build; cd build 37 | export USE_ZCHUNK=ON 38 | cmake .. \ 39 | -GNinja \ 40 | -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ 41 | -DWITH_ZCHUNK=$USE_ZCHUNK \ 42 | -DBUILD_SHARED=ON \ 43 | -DENABLE_TESTS=ON \ 44 | -DBUILD_EXE=ON 45 | 46 | - name: configure powerloader build 47 | if: runner.os == 'Windows' 48 | shell: cmd /C CALL {0} 49 | run: | 50 | call activate powerloader 51 | echo %CONDA_PREFIX% 52 | echo %PATH% 53 | mkdir build 54 | cd build 55 | cmake .. -DCMAKE_PREFIX_PATH=%CONDA_PREFIX%\Library ^ 56 | -DENABLE_TESTS=ON ^ 57 | -DWITH_ZCHUNK=OFF ^ 58 | -DBUILD_SHARED=ON ^ 59 | -DBUILD_EXE=ON ^ 60 | -G "Ninja" 61 | 62 | - name: build powerloader 63 | if: runner.os != 'Windows' 64 | shell: bash -l {0} 65 | run: | 66 | cd build 67 | ninja 68 | 69 | - name: build powerloader 70 | if: runner.os == 'Windows' 71 | shell: cmd /C CALL {0} 72 | run: | 73 | call activate powerloader 74 | cd build 75 | ninja 76 | 77 | - name: run powerloader tests 78 | if: runner.os != 'Windows' 79 | shell: bash -l {0} 80 | run: | 81 | cd build 82 | ninja test 83 | 84 | - name: run powerloader tests 85 | if: runner.os == 'Windows' 86 | shell: cmd /C CALL {0} 87 | run: | 88 | call activate powerloader 89 | cd build 90 | ninja test 91 | 92 | - name: Download test packages 93 | shell: bash -l -eo pipefail {0} 94 | if: runner.os != 'Windows' 95 | run: | 96 | server_path="test/conda_mock/static/packages" 97 | build/powerloader download -f test/remote_mirrors.yml -d $server_path 98 | 99 | - name: Setup minio and oras 100 | env: 101 | AWS_S3_BUCKET: testbucket 102 | AWS_ACCESS_KEY_ID: minioadmin 103 | AWS_SECRET_ACCESS_KEY: minioadmin 104 | AWS_S3_ENDPOINT: http://127.0.0.1:9000 105 | AWS_EC2_METADATA_DISABLED: true 106 | if: runner.os == 'Linux' 107 | run: | 108 | sudo docker run -d -p 9000:9000 --name minio \ 109 | -e MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID \ 110 | -e MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY \ 111 | -v /tmp/data:/data \ 112 | -v /tmp/config:/root/.minio \ 113 | minio/minio server /data 114 | 115 | # Populate minIO 116 | # https://docs.min.io/docs/aws-cli-with-minio.html 117 | server_path="test/conda_mock/static/packages" 118 | aws --endpoint-url $AWS_S3_ENDPOINT s3 mb s3://testbucket 119 | aws --endpoint-url $AWS_S3_ENDPOINT \ 120 | s3 cp $server_path/xtensor-0.23.9-hc021e02_1.tar.bz2 s3://testbucket/ 121 | 122 | # Create mock_artifact file 123 | mkdir test/tmp 124 | echo "artifact" > test/tmp/mock_artifact 125 | 126 | - name: Python based tests that require secrets 127 | shell: bash -l -eo pipefail {0} 128 | if: runner.os == 'Linux' && github.event_name != 'pull_request' 129 | env: 130 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 131 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 132 | AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} 133 | GHA_PAT: ${{ secrets.GITHUB_TOKEN }} 134 | GHA_USER: mamba-org 135 | run: | 136 | # Run tests 137 | pytest ./test/test_s3main_branch.py 138 | 139 | - name: Run python based tests 140 | shell: bash -l -eo pipefail {0} 141 | env: 142 | AWS_S3_BUCKET: testbucket 143 | AWS_ACCESS_KEY_ID: minioadmin 144 | AWS_SECRET_ACCESS_KEY: minioadmin 145 | AWS_DEFAULT_REGION: eu-central-1 146 | AWS_S3_ENDPOINT: http://127.0.0.1:9000 147 | AWS_EC2_METADATA_DISABLED: true 148 | if: runner.os == 'Linux' 149 | run: | 150 | pytest ./test/test_other.py 151 | pytest ./test/test_oci_registry.py 152 | pytest ./test/test_s3mock.py 153 | 154 | - name: Run python based tests for OSX 155 | shell: bash -l -eo pipefail {0} 156 | if: runner.os == 'macOS' 157 | run: | 158 | pytest ./test/test_other.py 159 | pytest ./test/test_oci_registry.py 160 | -------------------------------------------------------------------------------- /.github/workflows/static_build.yml: -------------------------------------------------------------------------------- 1 | name: CI static 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | defaults: 12 | run: 13 | shell: bash -l {0} 14 | 15 | jobs: 16 | test: 17 | runs-on: ${{ matrix.os }} 18 | strategy: 19 | matrix: 20 | os: [ubuntu-latest, macos-latest] 21 | steps: 22 | - uses: actions/checkout@v3 23 | 24 | - name: create build environment 25 | uses: mamba-org/provision-with-micromamba@main 26 | with: 27 | environment-file: environment-static-dev.yml 28 | condarc-file: .condarc 29 | 30 | - name: configure powerloader build 31 | run: | 32 | mkdir build; cd build 33 | export USE_ZCHUNK=ON 34 | cmake .. \ 35 | -GNinja \ 36 | -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ 37 | -DWITH_ZCHUNK=$USE_ZCHUNK \ 38 | -DBUILD_STATIC=ON \ 39 | -DBUILD_EXE=ON 40 | 41 | - name: build powerloader 42 | run: | 43 | cd build 44 | ninja 45 | 46 | - name: run powerloader tests 47 | run: | 48 | cd build 49 | ninja test 50 | 51 | test_win: 52 | runs-on: windows-2019 53 | steps: 54 | - uses: actions/checkout@v3 55 | 56 | - name: create build environment 57 | uses: mamba-org/provision-with-micromamba@main 58 | with: 59 | environment-file: environment-static-dev-win.yml 60 | environment-name: build_env 61 | cache-env: true 62 | condarc-file: .condarc 63 | 64 | - name: fix up vcpkg recipes 65 | shell: python 66 | run: | 67 | # See https://github.com/microsoft/vcpkg/pull/28919 68 | import os 69 | from pathlib import Path 70 | vcpkg_root = Path(os.environ["MAMBA_ROOT_PREFIX"]) / "envs" / "build_env" / "Library" / "share" / "vcpkg" 71 | f = vcpkg_root / "scripts" / "cmake" / "vcpkg_acquire_msys.cmake" 72 | text = f.read_text() 73 | text = text.replace("b309799e5a9d248ef66eaf11a0bd21bf4e8b9bd5c677c627ec83fa760ce9f0b54ddf1b62cbb436e641fbbde71e3b61cb71ff541d866f8ca7717a3a0dbeb00ebf", 74 | "a202ddaefa93d8a4b15431dc514e3a6200c47275c5a0027c09cc32b28bc079b1b9a93d5ef65adafdc9aba5f76a42f3303b1492106ddf72e67f1801ebfe6d02cc") 75 | text = text.replace("https://repo.msys2.org/msys/x86_64/libtool-2.4.6-9-x86_64.pkg.tar.xz", "https://repo.msys2.org/msys/x86_64/libtool-2.4.7-3-x86_64.pkg.tar.zst") 76 | f.write_text(text) 77 | 78 | - name: build static dependencies 79 | shell: cmd /C CALL {0} 80 | run: | 81 | call micromamba activate build_env 82 | SET VCPKG_BUILD_TYPE=release && vcpkg install "libarchive[bzip2,lz4,lzma,lzo,openssl,zstd]" --triplet x64-windows-static 83 | if %errorlevel% neq 0 exit /b %errorlevel% 84 | vcpkg install curl --triplet x64-windows-static 85 | if %errorlevel% neq 0 exit /b %errorlevel% 86 | set CMAKE_PREFIX_PATH=%VCPKG_ROOT%\installed\x64-windows-static\;%CMAKE_PREFIX_PATH% 87 | if %errorlevel% neq 0 exit /b %errorlevel% 88 | 89 | - name: configure powerloader build 90 | shell: cmd /C CALL {0} 91 | run: | 92 | call micromamba activate build_env 93 | mkdir build 94 | cd build 95 | cmake .. ^ 96 | -D CMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ 97 | -D CMAKE_PREFIX_PATH="%VCPKG_ROOT%\installed\x64-windows-static;%CMAKE_PREFIX_PATH%" ^ 98 | -D WITH_ZCHUNK=ON ^ 99 | -D BUILD_STATIC=ON ^ 100 | -D BUILD_EXE=ON ^ 101 | -G "Ninja" 102 | 103 | - name: build powerloader 104 | shell: cmd /C CALL {0} 105 | run: | 106 | call micromamba activate build_env 107 | cd build 108 | ninja 109 | 110 | - name: run powerloader test 111 | shell: cmd /C CALL {0} 112 | run: | 113 | call micromamba activate build_env 114 | cd build 115 | ninja test 116 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /build-* 3 | 4 | /.pytest_cache 5 | *.pyc 6 | *.log 7 | *.PID 8 | 9 | .vscode 10 | .DS_Store 11 | .cache 12 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: '^(test/conda_mock/static|testdata/)' 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.4.0 5 | hooks: 6 | - id: trailing-whitespace 7 | - id: end-of-file-fixer 8 | - id: fix-encoding-pragma 9 | args: [--remove] 10 | - id: check-yaml 11 | exclude: tests 12 | - id: check-toml 13 | - id: check-json 14 | - id: check-merge-conflict 15 | - id: pretty-format-json 16 | args: [--autofix] 17 | - id: debug-statements 18 | language_version: python3 19 | - repo: https://github.com/pre-commit/mirrors-clang-format 20 | rev: v15.0.6 21 | hooks: 22 | - id: clang-format 23 | exclude: ".json$" 24 | args: [--style=file] 25 | - repo: https://github.com/psf/black 26 | rev: 22.12.0 27 | hooks: 28 | - id: black 29 | args: [--safe, --quiet] 30 | # - repo: https://gitlab.com/pycqa/flake8 31 | # rev: 3.8.3 32 | # hooks: 33 | # - id: flake8 34 | # exclude: tests/data 35 | # language_version: python3 36 | # additional_dependencies: 37 | # - flake8-typing-imports==1.9.0 38 | # - flake8-builtins==1.5.3 39 | # - flake8-bugbear==20.1.4 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 QuantStack and the Mamba contributors. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!WARNING] 2 | > Powerloader features have been implemented in mamba directly. Powerloader is archived, no further public 3 | > updates will be made. 4 | 5 | 6 | 7 | ![](docs/assets/logo.png) 8 | 9 | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/powerloader.svg)](https://github.com/conda-forge/powerloader-feedstock) 10 | 11 | ---- 12 | 13 | # The POWERLOADER 14 | 15 | This is a tool to download large files. This is to be used in `mamba` and potentially other package managers. It's a port of `librepo`, but extends it in several ways as well as being cross-platform (it should support Windows as well). 16 | 17 | Current features are: 18 | 19 | - Mirror support and automatic mirror selection 20 | - Native OCI registry and S3 bucket support 21 | - Resumable downloads 22 | - zchunk support for delta-downloads 23 | 24 | In the future this might be directly integrated into the `mamba` codebase -- or live seperately. 25 | 26 | ### Try it out 27 | 28 | Install dependencies (remove `zchunk` on Windows or where it's not available): 29 | 30 | `mamba env create -n powerloader -f environment.yml` 31 | 32 | Then you can run 33 | 34 | ``` 35 | conda activate test 36 | 37 | mkdir build; cd build 38 | cmake .. -GNinja 39 | ninja 40 | 41 | ./powerloader --help 42 | ``` 43 | 44 | ### Uploading files 45 | 46 | The following uplaods the xtensor-0.24.0.tar.bz2 file to the xtensor:0.24.0 name/tag on ghcr.io. 47 | The file will appear under the user authenticated by the GHA_TOKEN. 48 | 49 | `powerloader upload xtensor-0.24.0.tar.bz2:xtensor:0.24.0 -m oci://ghcr.io` 50 | -------------------------------------------------------------------------------- /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/docs/assets/logo.png -------------------------------------------------------------------------------- /environment-static-dev-win.yml: -------------------------------------------------------------------------------- 1 | name: powerloader-dev 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | # libpowerloader-static 6 | - vs2019_win-64 7 | - ninja 8 | - vcpkg 9 | - curl 10 | - cpp-expected 11 | - spdlog 12 | - nlohmann_json 13 | - zchunk-static 14 | # powerloader 15 | - cli11 16 | - yaml-cpp 17 | # Tests 18 | - doctest 19 | -------------------------------------------------------------------------------- /environment-static-dev.yml: -------------------------------------------------------------------------------- 1 | name: powerloader-dev 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | # libpowerloader-static 6 | - cmake 7 | - ninja 8 | - cxx-compiler 9 | - libcurl-static >=7.88.1 10 | - openssl 11 | - libopenssl-static 12 | - libnghttp2-static 13 | - zstd-static 14 | - libssh2-static 15 | - krb5-static 16 | - zchunk-static 17 | - cpp-expected 18 | - spdlog 19 | - nlohmann_json 20 | # powerloader 21 | - cli11 22 | - yaml-cpp 23 | # Tests 24 | - doctest 25 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: powerloader 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | # libpowerloader 6 | - cmake 7 | - ninja 8 | - cxx-compiler 9 | - libcurl >=7.88.1 10 | - openssl 11 | - nlohmann_json 12 | - zchunk 13 | - spdlog 14 | - cpp-expected 15 | # powerloader 16 | - cli11 17 | - yaml-cpp 18 | # Tests 19 | - doctest 20 | - pytest 21 | - pyyaml 22 | - python 23 | - pybind11 24 | - requests 25 | - pytest-xprocess 26 | -------------------------------------------------------------------------------- /include/powerloader/context.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_CONTEXT_HPP 2 | #define POWERLOADER_CONTEXT_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace powerloader 18 | { 19 | namespace fs = std::filesystem; 20 | 21 | class Context; 22 | class Mirror; 23 | 24 | using mirror_set 25 | = std::vector>; // TODO: replace by std::flat_set once available. 26 | using mirror_map_base 27 | = std::map; // TODO: replace by std::flat_map once available. 28 | 29 | namespace details 30 | { 31 | POWERLOADER_API 32 | bool already_exists(const MirrorID& id, const mirror_set& mirrors); 33 | POWERLOADER_API 34 | bool is_every_mirror_unique_per_host(const mirror_map_base& mirrors); 35 | } 36 | 37 | // Registry of (host name -> list of mirrors) which guarantee that every list 38 | // of mirror have a unique set of mirrors (no duplicates). 39 | class POWERLOADER_API mirror_map_type : private mirror_map_base 40 | { 41 | public: 42 | using mirror_map_base::clear; 43 | using mirror_map_base::empty; 44 | using mirror_map_base::mirror_map_base; 45 | using mirror_map_base::size; 46 | 47 | // Get a list of unique mirorrs if existing for the provided host name, or an empty list 48 | // otherwise. 49 | mirror_set get_mirrors(std::string_view mirror_name) const; 50 | 51 | // Returns a copy of this container's values in the shape of a map. 52 | mirror_map_base as_map() const 53 | { 54 | return *this; 55 | } 56 | 57 | std::string to_string() const; 58 | 59 | // Returns true if there are registered mirrors stored here, false if none are. 60 | bool has_mirrors(std::string_view mirror_name) const; 61 | 62 | // Creates, stores and return a new instance of `MirrorType` created with `args` IFF no 63 | // other mirror is already registed with the same id for the specified host, returns null 64 | // otherwise. 65 | template 66 | auto create_unique_mirror(const std::string& mirror_name, 67 | const Context& ctx, 68 | Args&&... args) -> std::shared_ptr 69 | { 70 | static_assert(std::is_base_of_v); 71 | 72 | const auto new_id = MirrorType::id(args...); 73 | auto& mirrors = (*this)[mirror_name]; 74 | if (details::already_exists(new_id, mirrors)) 75 | return {}; 76 | 77 | auto mirror = std::make_shared(ctx, std::forward(args)...); 78 | mirrors.push_back(mirror); 79 | return mirror; 80 | } 81 | 82 | // Stores a provided Mirror IFF no other mirror is already registed with the same id for the 83 | // specified host. Returns true if the mirror has been stored, false otherwise. 84 | bool add_unique_mirror(std::string_view mirror_name, std::shared_ptr mirror); 85 | 86 | // Reset the whole mapping to a new set of host -> mirrors values. 87 | // Without arguments, this clears all values. 88 | // Every `mirror_set` in `new_values` must have no duplicates mirrors for that set, 89 | // otherwise this will throw a `std::invalid_argument` exception. 90 | void reset(mirror_map_base new_values = {}); 91 | }; 92 | 93 | using proxy_map_type = std::map; 94 | 95 | // Options provided when starting a powerloader context. 96 | struct ContextOptions 97 | { 98 | // If set, specifies which SSL backend to use with CURL. 99 | std::optional ssl_backend; 100 | }; 101 | 102 | class POWERLOADER_API Context 103 | { 104 | public: 105 | bool offline = false; 106 | int verbosity = 0; 107 | bool adaptive_mirror_sorting = true; 108 | 109 | // ssl options 110 | bool disable_ssl = false; 111 | bool ssl_no_revoke = false; 112 | fs::path ssl_ca_info; 113 | int ssl_backend = -1; 114 | // Sets the ca info of curl to nullptr 115 | // instead of the default value. 116 | bool ssl_no_default_ca_info = false; 117 | 118 | bool validate_checksum = true; 119 | 120 | long connect_timeout = 30L; 121 | long low_speed_time = 30L; 122 | long low_speed_limit = 1000L; 123 | 124 | long max_speed_limit = -1L; 125 | long max_parallel_downloads = 5L; 126 | long max_downloads_per_mirror = -1L; 127 | 128 | // This can improve throughput significantly 129 | // see https://github.com/curl/curl/issues/9601 130 | long transfer_buffersize = 100 * 1024; 131 | 132 | bool preserve_filetime = true; 133 | bool ftp_use_seepsv = true; 134 | 135 | fs::path cache_dir; 136 | std::size_t retry_backoff_factor = 2; 137 | std::size_t max_resume_count = 3; 138 | std::chrono::steady_clock::duration retry_default_timeout = std::chrono::seconds(2); 139 | 140 | mirror_map_type mirror_map; 141 | proxy_map_type proxy_map; 142 | 143 | std::vector additional_httpheaders; 144 | 145 | void set_verbosity(int v); 146 | void set_log_level(spdlog::level::level_enum); 147 | 148 | // Throws if another instance already exists: there can only be one at any time! 149 | Context(ContextOptions options = {}); 150 | ~Context(); 151 | 152 | Context(const Context&) = delete; 153 | Context& operator=(const Context&) = delete; 154 | Context(Context&&) = delete; 155 | Context& operator=(Context&&) = delete; 156 | 157 | private: 158 | struct Impl; 159 | std::unique_ptr impl; // Private implementation details 160 | }; 161 | 162 | } 163 | 164 | #endif 165 | -------------------------------------------------------------------------------- /include/powerloader/curl.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_CURL_HPP 2 | #define POWERLOADER_CURL_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | extern "C" 17 | { 18 | #include 19 | } 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace powerloader 26 | { 27 | class Context; 28 | class CURLHandle; 29 | using proxy_map_type = std::map; 30 | 31 | enum class ssl_backend_t 32 | { 33 | none = CURLSSLBACKEND_NONE, 34 | openssl = CURLSSLBACKEND_OPENSSL, 35 | gnutls = CURLSSLBACKEND_GNUTLS, 36 | nss = CURLSSLBACKEND_NSS, 37 | gskit = CURLSSLBACKEND_GSKIT, 38 | // polarssl = CURLSSLBACKEND_POLARSSL /* deprecated by curl */, 39 | wolfssl = CURLSSLBACKEND_WOLFSSL, 40 | schannel = CURLSSLBACKEND_SCHANNEL, 41 | securetransport = CURLSSLBACKEND_SECURETRANSPORT, 42 | // axtls = CURLSSLBACKEND_AXTLS, /* deprecated by curl */ 43 | mbedtls = CURLSSLBACKEND_MBEDTLS, 44 | // mesalink = CURLSSLBACKEND_MESALINK, /* deprecated by curl */ 45 | bearssl = CURLSSLBACKEND_BEARSSL, 46 | rustls = CURLSSLBACKEND_RUSTLS, 47 | }; 48 | 49 | class POWERLOADER_API curl_error : public std::runtime_error 50 | { 51 | public: 52 | curl_error(const std::string& what = "download error", bool serious = false); 53 | bool is_serious() const; 54 | 55 | private: 56 | bool m_serious; 57 | }; 58 | 59 | struct POWERLOADER_API Response 60 | { 61 | std::map headers; 62 | 63 | curl_off_t average_speed = -1; 64 | curl_off_t downloaded_size = -1; 65 | long http_status = 0; 66 | std::string effective_url; 67 | 68 | bool ok() const; 69 | 70 | tl::expected get_header(const std::string& header) const; 71 | 72 | void fill_values(CURLHandle& handle); 73 | 74 | // These are only working _if_ you are filling the content (e.g. by using the default 75 | // `h.perform() method) 76 | std::optional content; 77 | nlohmann::json json() const; 78 | }; 79 | 80 | // TODO: rename this, try to not expose it 81 | POWERLOADER_API CURL* get_handle(const Context& ctx); 82 | 83 | class POWERLOADER_API CURLHandle 84 | { 85 | public: 86 | using end_callback_type = std::function; 87 | explicit CURLHandle(const Context& ctx); 88 | CURLHandle(const Context& ctx, const std::string& url); 89 | ~CURLHandle(); 90 | 91 | CURLHandle& url(const std::string& url, const proxy_map_type& proxies); 92 | CURLHandle& accept_encoding(); 93 | CURLHandle& user_agent(const std::string& user_agent); 94 | 95 | Response perform(); 96 | void finalize_transfer(); 97 | // TODO: should be private? 98 | void finalize_transfer(Response& response); 99 | 100 | template 101 | tl::expected getinfo(CURLINFO option); 102 | 103 | // TODO: why do we need to expose these three methods 104 | CURL* handle(); 105 | operator CURL*(); // TODO: consider making this `explicit` or remove it 106 | CURL* ptr() const; 107 | 108 | CURLHandle& add_header(const std::string& header); 109 | CURLHandle& add_headers(const std::vector& headers); 110 | CURLHandle& reset_headers(); 111 | 112 | template 113 | CURLHandle& setopt(CURLoption opt, const T& val); 114 | 115 | void set_default_callbacks(); 116 | CURLHandle& set_end_callback(end_callback_type func); 117 | 118 | CURLHandle& upload(std::ifstream& stream); 119 | CURLHandle& upload(std::istringstream& stream); 120 | 121 | CURLHandle(CURLHandle&& rhs); 122 | CURLHandle& operator=(CURLHandle&& rhs); 123 | 124 | private: 125 | void init_handle(const Context& ctx); 126 | 127 | CURL* m_handle; 128 | curl_slist* p_headers = nullptr; 129 | char errorbuffer[CURL_ERROR_SIZE]; 130 | 131 | std::unique_ptr response; 132 | end_callback_type end_callback; 133 | }; 134 | 135 | // TODO: restrict the possible implementations in the cpp file 136 | template 137 | CURLHandle& CURLHandle::setopt(CURLoption opt, const T& val) 138 | { 139 | CURLcode ok; 140 | if constexpr (std::is_same()) 141 | { 142 | ok = curl_easy_setopt(m_handle, opt, val.c_str()); 143 | } 144 | else if constexpr (std::is_same()) 145 | { 146 | ok = curl_easy_setopt(m_handle, opt, val ? 1L : 0L); 147 | } 148 | else 149 | { 150 | ok = curl_easy_setopt(m_handle, opt, val); 151 | } 152 | if (ok != CURLE_OK) 153 | { 154 | throw curl_error( 155 | fmt::format("curl: curl_easy_setopt failed {}", curl_easy_strerror(ok))); 156 | } 157 | return *this; 158 | } 159 | 160 | std::optional proxy_match(const proxy_map_type& ctx, const std::string& url); 161 | } 162 | 163 | #endif 164 | -------------------------------------------------------------------------------- /include/powerloader/download_target.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_DOWNLOAD_TARGET_HPP 2 | #define POWERLOADER_DOWNLOAD_TARGET_HPP 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace powerloader 15 | { 16 | 17 | struct zck_target; 18 | 19 | struct POWERLOADER_API CacheControl 20 | { 21 | std::string etag; 22 | std::string cache_control; 23 | std::string last_modified; 24 | }; 25 | 26 | enum CompressionType 27 | { 28 | NONE, 29 | ZSTD, 30 | }; 31 | 32 | class POWERLOADER_API DownloadTarget 33 | { 34 | public: 35 | /** Called when a transfer is done (use transfer status to check 36 | * if successful or failed). 37 | * @param clientp Pointer to user data. 38 | * @param status Transfer status 39 | * @param msg Error message or NULL. 40 | * @return See LrCbReturnCode codes 41 | */ 42 | using end_callback_t = std::function; 43 | using progress_callback_t = std::function; 44 | 45 | DownloadTarget(const std::string& path, 46 | const std::string& mirror_name, 47 | const fs::path& destination); 48 | 49 | ~DownloadTarget(); 50 | 51 | DownloadTarget(const DownloadTarget&) = delete; 52 | DownloadTarget& operator=(const DownloadTarget&) = delete; 53 | DownloadTarget(DownloadTarget&&) = delete; 54 | DownloadTarget& operator=(DownloadTarget&&) = delete; 55 | 56 | 57 | // Creates a `DownloadTarget` given an URL. 58 | // When the url is a regular url (with "://"), mirrors will be added to the `Context` for 59 | // it's host if not already existing. 60 | // @param target_url The complete url to interpret. 61 | // @param destination_path Name or path of the resulting local file. 62 | // @param destination_dir Path to the directory where the resulting file should be stored. 63 | // @param hostname_override If provided, this base url will be used instead of the one from 64 | // the target url. 65 | static std::shared_ptr from_url(Context& ctx, 66 | const std::string& target_url, 67 | const fs::path& destination_path, 68 | const fs::path& destination_dir, 69 | std::optional hostname_override 70 | = std::nullopt); 71 | 72 | void set_cache_options(const CacheControl& cache_control); 73 | void add_handle_options(CURLHandle& handle); 74 | 75 | bool validate_checksum(const fs::path& path); 76 | bool already_downloaded(); 77 | 78 | void set_error(DownloaderError err) 79 | { 80 | m_error = std::move(err); 81 | } 82 | 83 | /// Returns a DownloadError if there was a falure at download or none if no error was set so 84 | /// far. 85 | std::optional get_error() const noexcept 86 | { 87 | return m_error; 88 | } 89 | 90 | bool resume() const noexcept 91 | { 92 | return m_resume; 93 | } 94 | 95 | void set_resume(bool new_value) noexcept 96 | { 97 | m_resume = new_value; 98 | } 99 | 100 | bool no_cache() const noexcept 101 | { 102 | return m_no_cache; 103 | } 104 | 105 | const std::string& mirror_name() const noexcept 106 | { 107 | return m_mirror_name; 108 | } 109 | 110 | const std::string& path() const noexcept 111 | { 112 | return m_path; 113 | } 114 | 115 | const std::filesystem::path& destination_path() const noexcept 116 | { 117 | return m_destination_path; 118 | } 119 | 120 | std::size_t byterange_start() const noexcept 121 | { 122 | return m_byterange_start; 123 | } 124 | 125 | std::size_t byterange_end() const noexcept 126 | { 127 | return m_byterange_end; 128 | } 129 | 130 | std::uintmax_t expected_size() const noexcept 131 | { 132 | return m_expected_size; 133 | } 134 | 135 | // TOOD: check SOC (why is this modifed outside) 136 | void set_expected_size(std::uintmax_t value) 137 | { 138 | // TODO: add checks? 139 | m_expected_size = value; 140 | } 141 | 142 | std::uintmax_t orig_size() const noexcept 143 | { 144 | return m_orig_size; 145 | } 146 | 147 | const std::string& range() const noexcept 148 | { 149 | return m_range; 150 | } 151 | 152 | // TODO: fix SOC with zchunk's code modifying this outside 153 | std::string& range() noexcept 154 | { 155 | return m_range; 156 | } 157 | 158 | bool is_zchunk() const noexcept 159 | { 160 | return m_is_zchunk; 161 | } 162 | 163 | const zck_target& zck() const 164 | { 165 | if (!is_zchunk()) // TODO: REVIEW: should this be an assert? 166 | throw std::invalid_argument("attempted to access zchunk data but there is none"); 167 | return *m_p_zck; 168 | } 169 | 170 | // TODO: ownership/access issue: mostly modified outside 171 | zck_target& zck() 172 | { 173 | if (!is_zchunk()) // TODO: REVIEW: should this be an assert? 174 | throw std::invalid_argument("attempted to access zchunk data but there is none"); 175 | return *m_p_zck; 176 | } 177 | 178 | // TODO: rewrite outfile's ownership handling and processing to avoid sharing it with other 179 | // types's implementations 180 | void set_outfile(std::unique_ptr new_outfile) 181 | { 182 | m_outfile = std::move(new_outfile); 183 | } 184 | 185 | // TODO: rewrite outfile's ownership handling and processing to avoid sharing it with other 186 | // types's implementations 187 | std::unique_ptr& outfile() noexcept 188 | { 189 | return m_outfile; 190 | } 191 | 192 | const std::unique_ptr& outfile() const noexcept 193 | { 194 | return m_outfile; 195 | } 196 | 197 | const progress_callback_t& progress_callback() const 198 | { 199 | return m_progress_callback; 200 | } 201 | 202 | progress_callback_t set_progress_callback(progress_callback_t callback) 203 | { 204 | return std::exchange(m_progress_callback, callback); 205 | } 206 | 207 | const end_callback_t& end_callback() const 208 | { 209 | return m_end_callback; 210 | } 211 | 212 | end_callback_t set_end_callback(end_callback_t callback) 213 | { 214 | return std::exchange(m_end_callback, callback); 215 | } 216 | 217 | std::shared_ptr set_mirror_to_use(std::shared_ptr mirror) 218 | { 219 | return std::exchange(m_used_mirror, mirror); 220 | } 221 | 222 | std::shared_ptr used_mirror() const noexcept 223 | { 224 | return m_used_mirror; 225 | } 226 | 227 | const std::string& effective_url() const noexcept 228 | { 229 | return m_effective_url; 230 | } 231 | 232 | void set_effective_url(const std::string& new_url) 233 | { 234 | // TODO: add some checks here 235 | m_effective_url = new_url; 236 | } 237 | 238 | const std::vector& checksums() const 239 | { 240 | return m_checksums; 241 | } 242 | 243 | // TODO: consider making the whole set of checksums one value set when needed. 244 | void add_checksum(Checksum value) 245 | { 246 | m_checksums.push_back(value); 247 | } 248 | 249 | // only request HEAD info, to check wether a URL exists or not 250 | DownloadTarget& set_head_only(bool new_value) noexcept 251 | { 252 | m_head_only = new_value; 253 | return *this; 254 | } 255 | 256 | // if this is set, activate CURLOPT_NOBODY 257 | bool head_only() const noexcept 258 | { 259 | return m_head_only; 260 | } 261 | 262 | CompressionType compression() const noexcept 263 | { 264 | return m_compression_type; 265 | } 266 | 267 | DownloadTarget& set_compression_type(CompressionType new_value) noexcept 268 | { 269 | m_compression_type = new_value; 270 | return *this; 271 | } 272 | 273 | private: 274 | bool m_is_zchunk = false; 275 | bool m_resume = true; 276 | bool m_no_cache = false; 277 | bool m_head_only = false; 278 | 279 | std::string m_complete_url; 280 | std::string m_path; 281 | std::string m_mirror_name; 282 | std::unique_ptr m_outfile; 283 | 284 | fs::path m_destination_path; 285 | 286 | std::size_t m_byterange_start = 0; 287 | std::size_t m_byterange_end = 0; 288 | std::string m_range; 289 | std::uintmax_t m_expected_size = 0; 290 | std::uintmax_t m_orig_size = 0; 291 | 292 | progress_callback_t m_progress_callback; 293 | end_callback_t m_end_callback; 294 | 295 | // these are available checksums for the entire file 296 | std::vector m_checksums; 297 | 298 | std::shared_ptr m_used_mirror; 299 | std::string m_effective_url; 300 | std::optional m_error; 301 | 302 | std::unique_ptr m_p_zck; 303 | 304 | CacheControl m_cache_control; 305 | 306 | CompressionType m_compression_type = CompressionType::NONE; 307 | }; 308 | 309 | } 310 | 311 | #endif 312 | -------------------------------------------------------------------------------- /include/powerloader/downloader.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_DOWNLOADER_HPP 2 | #define POWERLOADER_DOWNLOADER_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | extern "C" 15 | { 16 | #ifndef _WIN32 17 | #include 18 | #endif 19 | #include 20 | } 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | namespace powerloader 32 | { 33 | namespace fs = std::filesystem; 34 | 35 | class Context; 36 | class Target; 37 | 38 | struct DownloadOptions 39 | { 40 | // Extracts zchunk files which have been downloaded if true. 41 | bool extract_zchunk_files = true; 42 | bool failfast = false; 43 | bool allow_failure = false; 44 | }; 45 | 46 | class POWERLOADER_API Downloader 47 | { 48 | public: 49 | explicit Downloader(const Context& ctx); 50 | ~Downloader(); 51 | 52 | // Adds a target to be downloaded when calling `Downloader::download()`. 53 | // Must not be called after `Downloader::download()` have been called. 54 | void add(const std::shared_ptr& dl_target); 55 | 56 | // Proceed to download the targets previously specified using `Downloader::add(target)`. 57 | // After calling this fonction, no other operations are valid except destroying this object. 58 | bool download(DownloadOptions options = {}); 59 | 60 | Downloader(const Downloader&) = delete; 61 | Downloader& operator=(const Downloader&) = delete; 62 | 63 | Downloader(Downloader&&) = delete; 64 | Downloader& operator=(Downloader&&) = delete; 65 | 66 | private: 67 | /** Check the finished transfer 68 | * Evaluate CURL return code and status code of protocol if needed. 69 | * @param serious_error Serious error is an error that isn't fatal, 70 | * but mirror that generate it should be penalized. 71 | * E.g.: Connection timeout - a mirror we are unable 72 | * to connect at is pretty useless for us, but 73 | * this could be only temporary state. 74 | * No fatal but also no good. 75 | * @param fatal_error An error that cannot be recovered - e.g. 76 | * we cannot write to a socket, we cannot write 77 | * data to disk, bad function argument, ... 78 | */ 79 | tl::expected check_finished_transfer_status(CURLMsg* msg, 80 | Target* target); 81 | 82 | bool is_max_mirrors_unlimited(); 83 | 84 | tl::expected, DownloaderError> select_suitable_mirror( 85 | Target* target); 86 | 87 | tl::expected, DownloaderError> select_next_target( 88 | bool allow_failure); 89 | 90 | bool prepare_next_transfer(bool* candidate_found, bool allow_failure); 91 | 92 | bool prepare_next_transfers(bool allow_failure); 93 | 94 | /** 95 | * @brief Returns whether the download can be retried, using the same URL in 96 | * case of base_url or full path, or using another mirror in case of using 97 | * mirrors. 98 | * 99 | * @param complete_path_or_base_url determine type of download - mirrors or 100 | * base_url/fullpath 101 | * @return Return true when another chance to download is allowed. 102 | */ 103 | bool can_retry_download(int num_of_tried_mirrors, const std::string& url); 104 | bool check_msgs(bool failfast, bool allow_failure); 105 | bool set_max_speeds_to_transfers(); 106 | 107 | void extract_zchunk_files(); 108 | 109 | CURLM* multi_handle; 110 | const Context& ctx; 111 | 112 | std::vector m_targets; 113 | std::vector m_running_transfers; 114 | 115 | int allowed_mirror_failures = 3; 116 | int max_mirrors_to_try = -1; 117 | std::size_t max_parallel_connections = 5; 118 | }; 119 | 120 | } 121 | 122 | #endif 123 | -------------------------------------------------------------------------------- /include/powerloader/enums.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_ENUMS_HPP 2 | #define POWERLOADER_ENUMS_HPP 3 | 4 | #define PARTEXT ".pdpart" 5 | #define EMPTY_SHA "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 6 | 7 | namespace powerloader 8 | { 9 | enum class Protocol 10 | { 11 | kOTHER, 12 | kFILE, 13 | kHTTP, 14 | kFTP, 15 | // Want: S3, OCI 16 | }; 17 | 18 | enum class DownloadState 19 | { 20 | // The target is waiting to be processed. 21 | kWAITING, 22 | // The target (or mirror) is running preparation requests (e.g. for auth) 23 | kPREPARATION, 24 | // The transfer is running. 25 | kRUNNING, 26 | // The transfer is successfully finished. 27 | kFINISHED, 28 | // The transfer is finished without success. 29 | kFAILED, 30 | }; 31 | 32 | enum class HeaderCbState 33 | { 34 | // Default state 35 | kDEFAULT, 36 | // HTTP headers with OK state 37 | kHTTP_STATE_OK, 38 | // Download was interrupted (e.g. Content-Length doesn't match 39 | // expected size etc.) 40 | kINTERRUPTED, 41 | // All headers which we were looking for are already found 42 | kDONE 43 | }; 44 | 45 | /** Enum with zchunk file status */ 46 | enum class ZckState 47 | { 48 | // The zchunk file is waiting to download the header lead if header_size & hash not 49 | // available 50 | kHEADER_LEAD, 51 | // The zchunk file is waiting to check whether the header is available locally. 52 | kHEADER_CK, 53 | // The zchunk file is waiting to download the header 54 | kHEADER, 55 | // The zchunk file is waiting to check what chunks are available locally 56 | kBODY_CK, 57 | // The zchunk file is waiting for its body to be downloaded. 58 | kBODY, 59 | // The zchunk file is finished being downloaded. 60 | kFINISHED 61 | }; 62 | 63 | enum class CbReturnCode 64 | { 65 | kOK = 0, 66 | kABORT, 67 | kERROR 68 | }; 69 | 70 | enum class TransferStatus 71 | { 72 | kSUCCESSFUL, 73 | kALREADYEXISTS, 74 | kERROR, 75 | }; 76 | 77 | enum class ChecksumType 78 | { 79 | kSHA1, 80 | kSHA256, 81 | kMD5 82 | }; 83 | 84 | struct Checksum 85 | { 86 | ChecksumType type; 87 | std::string checksum; 88 | }; 89 | } 90 | 91 | #endif 92 | -------------------------------------------------------------------------------- /include/powerloader/errors.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_ERRORS_HPP 2 | #define POWERLOADER_ERRORS_HPP 3 | 4 | #include 5 | 6 | namespace powerloader 7 | { 8 | /** Librepo return/error codes 9 | */ 10 | enum class ErrorCode 11 | { 12 | // everything is ok 13 | PD_OK, 14 | // bad function argument 15 | PD_BADFUNCARG, 16 | // bad argument of the option 17 | PD_BADOPTARG, 18 | // library doesn't know the option 19 | PD_UNKNOWNOPT, 20 | // cURL doesn't know the option. Too old curl version? 21 | PD_CURLSETOPT, 22 | // Result object is not clean 23 | PD_ADYUSEDRESULT, 24 | // Result doesn't contain all what is needed 25 | PD_INCOMPLETERESULT, 26 | // cannot duplicate curl handle 27 | PD_CURLDUP, 28 | // cURL error 29 | PD_CURL, 30 | // cURL multi handle error 31 | PD_CURLM, 32 | // HTTP or FTP returned status code which do not represent success 33 | // (file doesn't exists, etc.) 34 | PD_BADSTATUS, 35 | // some error that should be temporary and next try could work 36 | // (HTTP status codes 500, 502-504, operation timeout, ...) 37 | PD_TEMPORARYERR, 38 | // URL is not a local address 39 | PD_NOTLOCAL, 40 | // cannot create a directory in output dir (ady exists?) 41 | PD_CANNOTCREATEDIR, 42 | // input output error 43 | PD_IO, 44 | // bad mirrorlist/metalink file (metalink doesn't contain needed 45 | // file, mirrorlist doesn't contain urls, ..) 46 | PD_MIRRORS, 47 | // bad checksum 48 | PD_BADCHECKSUM, 49 | // no usable URL found 50 | PD_NOURL, 51 | // cannot create tmp directory 52 | PD_CANNOTCREATETMP, 53 | // unknown type of checksum is needed for verification 54 | PD_UNKNOWNCHECKSUM, 55 | // bad URL specified 56 | PD_BADURL, 57 | // Download was interrupted by signal. 58 | // Only if LRO_INTERRUPTIBLE option is enabled. 59 | PD_INTERRUPTED, 60 | // sigaction error 61 | PD_SIGACTION, 62 | // File ady exists and checksum is ok.*/ 63 | PD_ADYDOWNLOADED, 64 | // The download wasn't or cannot be finished. 65 | PD_UNFINISHED, 66 | // select() call failed. 67 | PD_SELECT, 68 | // OpenSSL library related error. 69 | PD_OPENSSL, 70 | // Cannot allocate more memory 71 | PD_MEMORY, 72 | // Interrupted by user cb 73 | PD_CBINTERRUPTED, 74 | // File operation error (operation not permitted, filename too long, no memory available, 75 | // bad file descriptor, ...) 76 | PD_FILE, 77 | // Zchunk error (error reading zchunk file, ...) 78 | PD_ZCK, 79 | // (xx) unknown error - sentinel of error codes enum 80 | PD_UNKNOWNERROR, 81 | }; 82 | 83 | enum ErrorLevel 84 | { 85 | INFO, 86 | SERIOUS, 87 | FATAL 88 | }; 89 | 90 | struct DownloaderError 91 | { 92 | ErrorLevel level; 93 | ErrorCode code; 94 | std::string reason; 95 | 96 | bool is_serious() const noexcept 97 | { 98 | return (level == ErrorLevel::SERIOUS || level == ErrorLevel::FATAL); 99 | } 100 | 101 | bool is_fatal() const noexcept 102 | { 103 | return level == ErrorLevel::FATAL; 104 | } 105 | 106 | void log() const 107 | { 108 | switch (level) 109 | { 110 | case ErrorLevel::FATAL: 111 | spdlog::critical(reason); 112 | break; 113 | case ErrorLevel::SERIOUS: 114 | spdlog::error(reason); 115 | break; 116 | default: 117 | spdlog::warn(reason); 118 | } 119 | } 120 | }; 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /include/powerloader/export.hpp: -------------------------------------------------------------------------------- 1 | 2 | #ifndef POWERLOADER_API_HPP 3 | #define POWERLOADER_API_HPP 4 | 5 | // clang-format off 6 | #ifdef POWERLOADER_STATIC 7 | // As a static library: no symbol import/export. 8 | # define POWERLOADER_API 9 | #else 10 | // As a shared library: export symbols on build, import symbols on use. 11 | # ifdef POWERLOADER_EXPORTS 12 | // We are building this library 13 | # ifdef _MSC_VER 14 | # define POWERLOADER_API __declspec(dllexport) 15 | # else 16 | # define POWERLOADER_API __attribute__((__visibility__("default"))) 17 | # endif 18 | # else 19 | // We are using this library 20 | # ifdef _MSC_VER 21 | # define POWERLOADER_API __declspec(dllimport) 22 | # else 23 | # define POWERLOADER_API // Symbol import is implicit on non-msvc compilers. 24 | # endif 25 | # endif 26 | #endif 27 | // clang-format on 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /include/powerloader/fastest_mirror.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_FASTEST_MIRROR_HPP 2 | #define POWERLOADER_FASTEST_MIRROR_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | namespace powerloader 10 | { 11 | class Context; 12 | 13 | POWERLOADER_API tl::expected, std::string> fastest_mirror( 14 | const Context& ctx, const std::vector& urls); 15 | } 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /include/powerloader/fileio.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_FILEIO_HPP 2 | #define POWERLOADER_FILEIO_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #ifndef _WIN32 10 | #include 11 | #include 12 | #else 13 | #include 14 | #endif 15 | 16 | 17 | namespace powerloader 18 | { 19 | namespace fs = std::filesystem; 20 | 21 | class FileIO 22 | { 23 | private: 24 | FILE* m_fs = nullptr; 25 | fs::path m_path; 26 | 27 | public: 28 | #ifdef _WIN32 29 | constexpr static wchar_t append_update_binary[] = L"ab+"; 30 | constexpr static wchar_t read_update_binary[] = L"rb+"; 31 | constexpr static wchar_t write_update_binary[] = L"wb+"; 32 | constexpr static wchar_t write_binary[] = L"wb"; 33 | constexpr static wchar_t read_binary[] = L"rb"; 34 | #else 35 | constexpr static char append_update_binary[] = "ab+"; 36 | constexpr static char read_update_binary[] = "rb+"; 37 | constexpr static char write_update_binary[] = "wb+"; 38 | constexpr static char write_binary[] = "wb"; 39 | constexpr static char read_binary[] = "rb"; 40 | #endif 41 | 42 | FileIO() = default; 43 | 44 | #ifdef _WIN32 45 | explicit FileIO(const fs::path& file_path, 46 | const wchar_t* mode, 47 | std::error_code& ec) noexcept 48 | : m_path(file_path) 49 | { 50 | m_fs = ::_wfsopen(file_path.wstring().c_str(), mode, _SH_DENYNO); 51 | if (!m_fs) 52 | { 53 | ec.assign(GetLastError(), std::generic_category()); 54 | spdlog::error("Could not open file: {}", ec.message()); 55 | } 56 | } 57 | #else 58 | explicit FileIO(const fs::path& file_path, const char* mode, std::error_code& ec) noexcept 59 | : m_path(file_path) 60 | { 61 | m_fs = ::fopen(file_path.c_str(), mode); 62 | if (m_fs) 63 | { 64 | ec.clear(); 65 | } 66 | else 67 | { 68 | ec.assign(errno, std::generic_category()); 69 | spdlog::error("Could not open file: {}", ec.message()); 70 | } 71 | } 72 | #endif 73 | 74 | ~FileIO() 75 | { 76 | if (m_fs) 77 | { 78 | std::error_code ec; 79 | close(ec); 80 | if (ec) 81 | { 82 | spdlog::error("Error: {}", ec.message()); 83 | } 84 | } 85 | } 86 | 87 | int fd() const noexcept 88 | { 89 | #ifndef _WIN32 90 | return ::fileno(m_fs); 91 | #else 92 | return ::_fileno(m_fs); 93 | #endif 94 | } 95 | 96 | std::streamoff seek(int offset, int origin) const noexcept 97 | { 98 | return ::fseek(m_fs, offset, origin); 99 | } 100 | 101 | std::streamoff seek(unsigned int offset, int origin) const noexcept 102 | { 103 | return this->seek(static_cast(offset), origin); 104 | } 105 | 106 | std::streamoff seek(long offset, int origin) const noexcept 107 | { 108 | return ::fseek(m_fs, offset, origin); 109 | } 110 | 111 | std::streamoff seek(unsigned long offset, int origin) const noexcept 112 | { 113 | #ifdef _WIN32 114 | return ::_fseeki64(m_fs, static_cast(offset), origin); 115 | #else 116 | assert(offset < LLONG_MAX); 117 | return ::fseek(m_fs, offset, origin); 118 | #endif 119 | } 120 | 121 | std::streamoff seek(long long offset, int origin) const noexcept 122 | { 123 | #ifdef _WIN32 124 | return ::_fseeki64(m_fs, offset, origin); 125 | #else 126 | return ::fseek(m_fs, offset, origin); 127 | #endif 128 | } 129 | 130 | bool open() 131 | { 132 | return m_fs != nullptr; 133 | } 134 | 135 | std::streamoff tell() const 136 | { 137 | return ::ftell(m_fs); 138 | } 139 | 140 | std::streamoff seek(unsigned long long offset, int origin) const noexcept 141 | { 142 | assert(offset < LLONG_MAX); 143 | return this->seek(static_cast(offset), origin); 144 | } 145 | 146 | int eof() const noexcept 147 | { 148 | return ::feof(m_fs); 149 | } 150 | 151 | std::size_t read(void* buffer, 152 | std::size_t element_size, 153 | std::size_t element_count) const noexcept 154 | { 155 | assert(m_fs); 156 | return ::fread(buffer, element_size, element_count, m_fs); 157 | } 158 | 159 | std::size_t write(const void* buffer, 160 | std::size_t element_size, 161 | std::size_t element_count) const noexcept 162 | { 163 | return ::fwrite(buffer, element_size, element_count, m_fs); 164 | } 165 | 166 | std::streamoff put(int c) const noexcept 167 | { 168 | return ::fputc(c, m_fs); 169 | } 170 | 171 | void truncate(std::streamoff length, std::error_code& ec) const noexcept 172 | { 173 | #ifdef _WIN32 174 | return fs::resize_file(m_path, length, ec); 175 | #else 176 | ec.clear(); 177 | if (::ftruncate(fd(), length) != 0) 178 | { 179 | ec.assign(errno, std::generic_category()); 180 | } 181 | #endif 182 | } 183 | 184 | void flush() 185 | { 186 | ::fflush(m_fs); 187 | } 188 | 189 | const fs::path& path() const 190 | { 191 | return m_path; 192 | } 193 | 194 | bool copy_from(const FileIO& other) 195 | { 196 | constexpr std::size_t bufsize = 2048; 197 | char buf[bufsize]; 198 | std::size_t size; 199 | 200 | this->seek(0, SEEK_SET); 201 | other.seek(0, SEEK_SET); 202 | 203 | while ((size = other.read(buf, 1, bufsize)) > 0) 204 | { 205 | if (this->write(buf, 1, size) == SIZE_MAX) 206 | { 207 | return false; 208 | } 209 | } 210 | this->flush(); 211 | return size != std::size_t(0); 212 | } 213 | 214 | bool replace_from(const FileIO& other) 215 | { 216 | std::error_code ec; 217 | truncate(0, ec); 218 | if (copy_from(other) == true) 219 | { 220 | other.seek(0, SEEK_END); 221 | truncate(other.tell(), ec); 222 | } 223 | else 224 | { 225 | return false; 226 | } 227 | this->flush(); 228 | this->seek(0, SEEK_SET); 229 | other.seek(0, SEEK_SET); 230 | return !!ec; 231 | } 232 | 233 | 234 | void close(std::error_code& ec) noexcept 235 | { 236 | if (!m_fs) 237 | { 238 | ec.clear(); 239 | return; 240 | } 241 | if (::fclose(m_fs) == 0) 242 | { 243 | ec.clear(); 244 | m_fs = nullptr; 245 | } 246 | else 247 | { 248 | ec.assign(errno, std::generic_category()); 249 | } 250 | } 251 | 252 | int error() 253 | { 254 | return ::ferror(m_fs); 255 | } 256 | }; 257 | } 258 | 259 | #endif 260 | -------------------------------------------------------------------------------- /include/powerloader/mirror.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_MIRROR_HPP 2 | #define POWERLOADER_MIRROR_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace powerloader 16 | { 17 | class Target; 18 | class Context; 19 | 20 | enum class MirrorState 21 | { 22 | WAITING, 23 | AUTHENTICATING, 24 | READY, 25 | RETRY_DELAY, 26 | AUTHENTICATION_FAILED, 27 | FAILED 28 | }; 29 | 30 | struct MirrorStats 31 | { 32 | // Maximum number of allowed parallel connections to this mirror. -1 means no 33 | // limit. Dynamically adjusted(decreased) if no fatal(temporary) error will 34 | // occur. 35 | long allowed_parallel_connections = -1; 36 | 37 | // The maximum number of tried parallel connections to this mirror 38 | // (including unsuccessful). 39 | int max_tried_parallel_connections = 0; 40 | 41 | // How many transfers from this mirror are currently in progress. 42 | int running_transfers = 0; 43 | 44 | // How many transfers was finished successfully from the mirror. 45 | int successful_transfers = 0; 46 | 47 | // How many transfers failed. 48 | int failed_transfers = 0; 49 | 50 | // Maximum ranges supported in a single request. This will be automatically 51 | // adjusted when mirrors respond with 200 to a range request 52 | int max_ranges = 256; 53 | 54 | // Returns the total count of finished transfered. 55 | int count_finished_transfers() const 56 | { 57 | return successful_transfers + failed_transfers; 58 | } 59 | }; 60 | 61 | // mirrors should be dict -> urls mapping 62 | class POWERLOADER_API Mirror 63 | { 64 | public: 65 | Mirror(const MirrorID& id, const Context& ctx, const std::string& url); 66 | 67 | virtual ~Mirror(); 68 | 69 | Mirror(const Mirror&) = delete; 70 | Mirror& operator=(const Mirror&) = delete; 71 | Mirror(Mirror&&) = delete; 72 | Mirror& operator=(Mirror&&) = delete; 73 | 74 | // Identifier used to compare mirror instances. 75 | const MirrorID& id() const 76 | { 77 | return m_id; 78 | } 79 | 80 | // URL of the mirror 81 | const std::string& url() const 82 | { 83 | return m_url; 84 | } 85 | 86 | // Protocol of mirror (can be detected from URL) 87 | Protocol protocol() const 88 | { 89 | return m_protocol; 90 | } 91 | 92 | // Statistics about this mirror. 93 | // TODO: consider returning by copy for concurrent safety... (like "capturing" the stats at 94 | // a given moment - but they might change while observing) 95 | const MirrorStats& stats() const 96 | { 97 | return m_stats; 98 | } 99 | 100 | void change_max_ranges(int new_value); 101 | 102 | std::chrono::system_clock::time_point next_retry() const 103 | { 104 | return m_next_retry; 105 | } 106 | 107 | // Return mirror rank or -1.0 if the rank cannot be determined 108 | // (e.g. when is too early) 109 | // Rank is currently just success rate for the mirror 110 | double rank() const; 111 | 112 | bool need_wait_for_retry() const; 113 | bool has_running_transfers() const; 114 | 115 | // Maximum number of allowed parallel connections to this mirror. -1 means no 116 | // limit. Dynamically adjusted(decreased) if no fatal(temporary) error will 117 | // occur. 118 | void set_allowed_parallel_connections(int max_allowed_parallel_connections); 119 | 120 | void increase_running_transfers(); 121 | 122 | bool is_parallel_connections_limited_and_reached() const; 123 | 124 | void update_statistics(bool transfer_success); 125 | 126 | 127 | // TODO: protected: then make this apply protection-against-change to these 128 | 129 | virtual bool prepare(Target* target); 130 | virtual bool prepare(const std::string& path, CURLHandle& handle); 131 | 132 | virtual bool needs_preparation(Target* target) const; 133 | virtual bool authenticate(CURLHandle& handle, const std::string& path); 134 | 135 | virtual std::vector get_auth_headers(const std::string& path) const; 136 | 137 | // virtual void add_extra_headers(Target* target) { return; }; 138 | virtual std::string format_url(Target* target) const; 139 | 140 | // TODO: use operator<=> instead once C++20 is enabled. 141 | [[nodiscard]] friend bool operator<(const Mirror& left, const Mirror& right) 142 | { 143 | return left.id() < right.id(); 144 | } 145 | [[nodiscard]] friend bool operator==(const Mirror& left, const Mirror& right) 146 | { 147 | return left.id() == right.id(); 148 | } 149 | 150 | static MirrorID id(const std::string& url); 151 | 152 | private: 153 | const MirrorID m_id; 154 | const std::string m_url; 155 | 156 | Protocol m_protocol = Protocol::kHTTP; 157 | MirrorState m_state = MirrorState::READY; 158 | 159 | std::chrono::steady_clock::time_point m_next_allowed_retry; 160 | 161 | MirrorStats m_stats; 162 | 163 | // retry & backoff values 164 | std::chrono::system_clock::time_point m_next_retry; 165 | 166 | // first retry should wait for how many seconds? 167 | std::chrono::system_clock::duration m_retry_wait_seconds = std::chrono::milliseconds(200); 168 | 169 | // backoff factor for retry 170 | std::size_t m_retry_backoff_factor = 2; 171 | 172 | // count number of retries (this is not the same as failed transfers, as mutiple 173 | // transfers can be started at the same time, but should all be retried only once) 174 | std::size_t m_retry_counter = 0; 175 | }; 176 | 177 | class POWERLOADER_API HTTPMirror : public Mirror 178 | { 179 | public: 180 | HTTPMirror(const Context& ctx, const std::string& url); 181 | 182 | static MirrorID id(const std::string& url); 183 | 184 | void set_auth(const std::string& user, const std::string& password); 185 | 186 | bool authenticate(CURLHandle& handle, const std::string& path) override; 187 | 188 | private: 189 | std::string m_auth_user; 190 | std::string m_auth_password; 191 | }; 192 | 193 | bool sort_mirrors(std::vector>& mirrors, 194 | const std::shared_ptr& mirror, 195 | bool success, 196 | bool serious); 197 | 198 | } 199 | 200 | #endif 201 | -------------------------------------------------------------------------------- /include/powerloader/mirrorid.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_MIRRORID_HPP 2 | #define POWERLOADER_MIRRORID_HPP 3 | 4 | #include 5 | #include 6 | 7 | namespace powerloader 8 | { 9 | 10 | // Identifies a Mirror and is used to compare Mirrors. 11 | class MirrorID 12 | { 13 | std::string value; 14 | 15 | public: 16 | MirrorID() = default; 17 | MirrorID(const MirrorID&) = default; 18 | MirrorID& operator=(const MirrorID&) = default; 19 | MirrorID(MirrorID&&) = default; 20 | MirrorID& operator=(MirrorID&&) = default; 21 | 22 | explicit MirrorID(const std::string& v) 23 | : value(v) 24 | { 25 | } 26 | 27 | std::string to_string() const 28 | { 29 | return fmt::format("MirrorID <{}>", value); 30 | } 31 | 32 | // TODO: use operator<=> instead once C++20 is enabled. 33 | [[nodiscard]] friend bool operator<(const MirrorID& left, const MirrorID& right) 34 | { 35 | return left.value < right.value; 36 | } 37 | [[nodiscard]] friend bool operator==(const MirrorID& left, const MirrorID& right) 38 | { 39 | return left.value == right.value; 40 | } 41 | }; 42 | } 43 | 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /include/powerloader/mirrors/oci.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_OCI_HPP 2 | #define POWERLOADER_OCI_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | namespace powerloader 11 | { 12 | class POWERLOADER_API OCIMirror : public Mirror 13 | { 14 | public: 15 | using split_function_type 16 | = std::function(const std::string&)>; 17 | 18 | OCIMirror(const Context& ctx, const std::string& host, const std::string& repo_prefix); 19 | OCIMirror(const Context& ctx, 20 | const std::string& host, 21 | const std::string& repo_prefix, 22 | const std::string& scope, 23 | const std::string& username, 24 | const std::string& password); 25 | ~OCIMirror(); 26 | 27 | 28 | std::string get_repo(const std::string& repo) const; 29 | std::string get_auth_url(const std::string& repo, const std::string& scope) const; 30 | std::string get_manifest_url(const std::string& repo, const std::string& reference) const; 31 | std::string get_preupload_url(const std::string& repo) const; 32 | 33 | std::vector get_auth_headers(const std::string& path) const override; 34 | 35 | // authenticate per target, and authentication state 36 | // is also dependent on each target unfortunately?! 37 | bool prepare(const std::string& path, CURLHandle& handle) override; 38 | bool need_auth() const; 39 | bool needs_preparation(Target* target) const override; 40 | 41 | // void add_extra_headers(Target* target); 42 | std::string format_url(Target* target) const override; 43 | 44 | template 45 | static MirrorID id(const std::string& host, 46 | const std::string& repo_prefix, 47 | [[maybe_unused]] Args&&... args) 48 | { 49 | return MirrorID(fmt::format("OCIMirror[{}/{}]", host, repo_prefix)); 50 | } 51 | 52 | void set_fn_tag_split_function( 53 | const split_function_type& func); // TODO: review, looks like a hack 54 | 55 | private: 56 | struct AuthCallbackData 57 | { 58 | OCIMirror* self; 59 | Target* target; 60 | Response response; 61 | std::string sha256sum, token, buffer; 62 | }; 63 | 64 | 65 | std::map> m_path_cb_map; 66 | std::string m_repo_prefix; 67 | std::string m_scope; 68 | std::string m_username; 69 | std::string m_password; 70 | split_function_type m_split_func; 71 | 72 | // we copy over the proxy map from the context, otherwise we can't set new 73 | // proxy options for each curl handle 74 | proxy_map_type m_proxy_map; 75 | 76 | std::pair split_path_tag(const std::string& path) const; 77 | 78 | AuthCallbackData* get_data(Target* target) const; 79 | 80 | 81 | // upload specific functions 82 | std::string get_digest(const fs::path& p) const; 83 | }; 84 | 85 | struct POWERLOADER_API OCILayer 86 | { 87 | std::string mime_type; 88 | 89 | // The OCI Layer can either contain a file or string contents 90 | std::optional file; 91 | std::optional contents; 92 | 93 | // sha256 digest and size computed in the constructor 94 | std::string digest; 95 | std::size_t size; 96 | 97 | // optional annotations that can be added to each layer or config 98 | std::optional annotations; 99 | 100 | static OCILayer from_file(const std::string& mime_type, 101 | const fs::path& file, 102 | const std::optional& annotations = std::nullopt); 103 | 104 | static OCILayer from_string(const std::string& mime_type, 105 | const std::string& content, 106 | const std::optional& annotations 107 | = std::nullopt); 108 | 109 | Response upload(const Context& ctx, 110 | const OCIMirror& mirror, 111 | const std::string& reference) const; 112 | 113 | nlohmann::json to_json() const; 114 | 115 | private: 116 | OCILayer(const std::string& mime_type, 117 | const std::optional& path, 118 | const std::optional& content, 119 | const std::optional& annotations = std::nullopt); 120 | }; 121 | 122 | POWERLOADER_API 123 | Response oci_upload(const Context& ctx, 124 | OCIMirror& mirror, 125 | const std::string& reference, 126 | const std::string& tag, 127 | const std::vector& layers, 128 | const std::optional& config = std::nullopt); 129 | 130 | } 131 | 132 | #endif 133 | -------------------------------------------------------------------------------- /include/powerloader/mirrors/s3.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_S3_HPP 2 | #define POWERLOADER_S3_HPP 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace powerloader 11 | { 12 | 13 | class Target; 14 | 15 | struct POWERLOADER_API S3CanonicalRequest 16 | { 17 | std::string http_verb; 18 | std::string resource; 19 | std::string bucket_url; 20 | std::map headers; 21 | std::map query_string; 22 | std::string hashed_payload; 23 | 24 | std::chrono::system_clock::time_point date; 25 | 26 | S3CanonicalRequest(const std::string& http_verb, 27 | const URLHandler& uh, 28 | const std::string& sha256sum = ""); 29 | 30 | void init_default_headers(); 31 | std::string get_signed_headers(); 32 | std::string canonical_request(); 33 | 34 | std::string string_to_sign(const std::string& region, const std::string& service); 35 | }; 36 | 37 | // https://gist.github.com/mmaday/c82743b1683ce4d27bfa6615b3ba2332 38 | class POWERLOADER_API S3Mirror : public Mirror 39 | { 40 | public: 41 | S3Mirror(const Context& ctx, 42 | const std::string& bucket_url, 43 | const std::string& region, 44 | const std::string& aws_access_key, 45 | const std::string& aws_secret_key); 46 | 47 | ~S3Mirror(); 48 | 49 | template 50 | static MirrorID id(const std::string& bucket_url, 51 | const std::string& region, 52 | [[maybe_unused]] Args&&... args) 53 | { 54 | return MirrorID(fmt::format("S3Mirror[{}/{}]", bucket_url, region)); 55 | } 56 | 57 | std::vector get_auth_headers(const std::string& path) const override; 58 | std::vector get_auth_headers(S3CanonicalRequest& request) const; 59 | 60 | private: 61 | std::string bucket_url; 62 | std::string region = "eu-central-1"; 63 | std::string aws_access_key_id = "AKIAIOSFODNN7EXAMPLE"; 64 | std::string aws_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; 65 | 66 | bool authenticate(CURLHandle& handle, const std::string& path) override; 67 | std::string format_url(Target* target) const override; 68 | bool needs_preparation(Target* target) const override; 69 | bool prepare(Target* target) override; 70 | }; 71 | 72 | 73 | POWERLOADER_API 74 | std::string s3_calculate_signature(const std::chrono::system_clock::time_point& request_date, 75 | const std::string& secret, 76 | const std::string& region, 77 | const std::string& service, 78 | const std::string& string_to_sign); 79 | 80 | POWERLOADER_API 81 | Response s3_upload(const Context& ctx, 82 | S3Mirror& mirror, 83 | const std::string& path, 84 | const fs::path& file); 85 | } 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /include/powerloader/powerloader.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021, QuantStack and Mamba Contributors 2 | // 3 | // Distributed under the terms of the BSD 3-Clause License. 4 | // 5 | // The full license is in the file LICENSE, distributed with this software. 6 | 7 | #ifndef POWERLOADER_API_HPP 8 | #define POWERLOADER_API_HPP 9 | 10 | /* 11 | // clang-format off 12 | #ifdef POWERLOADER_STATIC 13 | // As a static library: no symbol import/export. 14 | # define POWERLOADER_API 15 | #else 16 | // As a shared library: export symbols on build, import symbols on use. 17 | # ifdef POWERLOADER_EXPORTS 18 | // We are building this library 19 | # ifdef _MSC_VER 20 | # define POWERLOADER_API __declspec(dllexport) 21 | # else 22 | # define POWERLOADER_API __attribute__((__visibility__("default"))) 23 | # endif 24 | # else 25 | // We are using this library 26 | # ifdef _MSC_VER 27 | # define POWERLOADER_API __declspec(dllimport) 28 | # else 29 | # define POWERLOADER_API // Symbol import is implicit on non-msvc compilers. 30 | # endif 31 | # endif 32 | #endif 33 | // clang-format on*/ 34 | 35 | // Project version 36 | #define POWERLOADER_VERSION_MAJOR 0 37 | #define POWERLOADER_VERSION_MINOR 6 38 | #define POWERLOADER_VERSION_PATCH 0 39 | 40 | // Binary version 41 | #define POWERLOADER_BINARY_CURRENT 0 42 | #define POWERLOADER_BINARY_REVISION 0 43 | #define POWERLOADER_BINARY_AGE 1 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /include/powerloader/url.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, QuantStack and Mamba Contributors 2 | // 3 | // Distributed under the terms of the BSD 3-Clause License. 4 | // 5 | // The full license is in the file LICENSE, distributed with this software. 6 | 7 | #ifndef POWERLOADER_URL_HPP 8 | #define POWERLOADER_URL_HPP 9 | 10 | extern "C" 11 | { 12 | #include 13 | } 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | // typedef enum { 23 | // CURLUE_OK, 24 | // CURLUE_BAD_HANDLE, /* 1 */ 25 | // CURLUE_BAD_PARTPOINTER, /* 2 */ 26 | // CURLUE_MALFORMED_INPUT, /* 3 */ 27 | // CURLUE_BAD_PORT_NUMBER, /* 4 */ 28 | // CURLUE_UNSUPPORTED_SCHEME, /* 5 */ 29 | // CURLUE_URLDECODE, /* 6 */ 30 | // CURLUE_OUT_OF_MEMORY, /* 7 */ 31 | // CURLUE_USER_NOT_ALLOWED, /* 8 */ 32 | // CURLUE_UNKNOWN_PART, /* 9 */ 33 | // CURLUE_NO_SCHEME, /* 10 */ 34 | // CURLUE_NO_USER, /* 11 */ 35 | // CURLUE_NO_PASSWORD, /* 12 */ 36 | // CURLUE_NO_OPTIONS, /* 13 */ 37 | // CURLUE_NO_HOST, /* 14 */ 38 | // CURLUE_NO_PORT, /* 15 */ 39 | // CURLUE_NO_QUERY, /* 16 */ 40 | // CURLUE_NO_FRAGMENT /* 17 */ 41 | // } CURLUcode; 42 | 43 | namespace powerloader 44 | { 45 | POWERLOADER_API bool has_scheme(const std::string& url); 46 | 47 | POWERLOADER_API bool compare_cleaned_url(const std::string& url1, const std::string& url2); 48 | 49 | POWERLOADER_API bool is_path(const std::string& input); 50 | POWERLOADER_API std::string path_to_url(const std::string& path); 51 | 52 | template 53 | std::string join_url(const S& s, const Args&... args); 54 | 55 | POWERLOADER_API std::string unc_url(const std::string& url); 56 | POWERLOADER_API std::string encode_url(const std::string& url); 57 | POWERLOADER_API std::string decode_url(const std::string& url); 58 | // Only returns a cache name without extension 59 | POWERLOADER_API std::string cache_name_from_url(const std::string& url); 60 | 61 | class POWERLOADER_API URLHandler 62 | { 63 | public: 64 | URLHandler(const std::string& url = ""); 65 | ~URLHandler(); 66 | 67 | URLHandler(const URLHandler&); 68 | URLHandler& operator=(const URLHandler&); 69 | 70 | URLHandler(URLHandler&&); 71 | URLHandler& operator=(URLHandler&&); 72 | 73 | std::string url(bool strip_scheme = false) const; 74 | std::string url_without_path() const; 75 | 76 | std::string scheme() const; 77 | std::string host() const; 78 | std::string path() const; 79 | std::string port() const; 80 | 81 | std::string query() const; 82 | std::string fragment() const; 83 | std::string options() const; 84 | 85 | std::string auth() const; 86 | std::string user() const; 87 | std::string password() const; 88 | std::string zoneid() const; 89 | 90 | URLHandler& set_scheme(const std::string& scheme); 91 | URLHandler& set_host(const std::string& host); 92 | URLHandler& set_path(const std::string& path); 93 | URLHandler& set_port(const std::string& port); 94 | 95 | URLHandler& set_query(const std::string& query); 96 | URLHandler& set_fragment(const std::string& fragment); 97 | URLHandler& set_options(const std::string& options); 98 | 99 | URLHandler& set_user(const std::string& user); 100 | URLHandler& set_password(const std::string& password); 101 | URLHandler& set_zoneid(const std::string& zoneid); 102 | 103 | private: 104 | std::string get_part(CURLUPart part) const; 105 | void set_part(CURLUPart part, const std::string& s); 106 | 107 | std::string m_url; 108 | CURLU* m_handle; 109 | bool m_has_scheme; 110 | }; 111 | 112 | namespace detail 113 | { 114 | inline std::string join_url_impl(std::string& s) 115 | { 116 | return s; 117 | } 118 | 119 | template 120 | inline std::string join_url_impl(std::string& s1, const S& s2, const Args&... args) 121 | { 122 | if (!s2.empty()) 123 | { 124 | s1 += '/' + s2; 125 | } 126 | return join_url_impl(s1, args...); 127 | } 128 | 129 | template 130 | inline std::string join_url_impl(std::string& s1, const char* s2, const Args&... args) 131 | { 132 | s1 += '/'; 133 | s1 += s2; 134 | return join_url_impl(s1, args...); 135 | } 136 | } // namespace detail 137 | 138 | inline std::string join_url() 139 | { 140 | return ""; 141 | } 142 | 143 | template 144 | inline std::string join_url(const S& s, const Args&... args) 145 | { 146 | std::string res = s; 147 | return detail::join_url_impl(res, args...); 148 | } 149 | } // namespace powerloader 150 | 151 | #endif 152 | -------------------------------------------------------------------------------- /include/powerloader/utils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_UTILS_HPP 2 | #define POWERLOADER_UTILS_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | namespace powerloader 18 | { 19 | namespace fs = std::filesystem; 20 | 21 | POWERLOADER_API bool is_sig_interrupted(); 22 | POWERLOADER_API bool starts_with(const std::string_view& str, const std::string_view& prefix); 23 | POWERLOADER_API bool ends_with(const std::string_view& str, const std::string_view& suffix); 24 | 25 | template 26 | inline std::vector hex_to_bytes(const B& buffer, std::size_t size) noexcept 27 | { 28 | std::vector res; 29 | if (size % 2 != 0) 30 | return res; 31 | 32 | std::string extract; 33 | for (auto pos = buffer.cbegin(); pos < buffer.cend(); pos += 2) 34 | { 35 | extract.assign(pos, pos + 2); 36 | res.push_back(std::stoi(extract, nullptr, 16)); 37 | } 38 | return res; 39 | } 40 | 41 | template 42 | inline std::vector hex_to_bytes(const B& buffer) noexcept 43 | { 44 | return hex_to_bytes(buffer, buffer.size()); 45 | } 46 | 47 | template 48 | inline std::string hex_string(const B& buffer, std::size_t size) 49 | { 50 | std::ostringstream oss; 51 | oss << std::hex; 52 | for (std::size_t i = 0; i < size; ++i) 53 | { 54 | oss << std::setw(2) << std::setfill('0') << static_cast(buffer[i]); 55 | } 56 | return oss.str(); 57 | } 58 | 59 | template 60 | inline std::string hex_string(const B& buffer) 61 | { 62 | return hex_string(buffer, buffer.size()); 63 | } 64 | 65 | POWERLOADER_API std::string sha256(const std::string& str) noexcept; 66 | 67 | class POWERLOADER_API download_error : public std::runtime_error 68 | { 69 | public: 70 | download_error(const std::string& what = "download error", bool serious = false) 71 | : std::runtime_error(what) 72 | , m_serious(serious) 73 | { 74 | } 75 | bool m_serious; 76 | }; 77 | 78 | class fatal_download_error : public std::runtime_error 79 | { 80 | public: 81 | fatal_download_error(const std::string& what = "fatal download error") 82 | : std::runtime_error(what) 83 | { 84 | } 85 | }; 86 | 87 | POWERLOADER_API std::string string_transform(const std::string_view& input, 88 | int (*functor)(int)); 89 | POWERLOADER_API std::string to_upper(const std::string_view& input); 90 | POWERLOADER_API std::string to_lower(const std::string_view& input); 91 | POWERLOADER_API bool contains(const std::string_view& str, const std::string_view& sub_str); 92 | 93 | POWERLOADER_API std::string sha256sum(const fs::path& path); 94 | POWERLOADER_API std::string md5sum(const fs::path& path); 95 | 96 | POWERLOADER_API std::pair parse_header( 97 | const std::string_view& header); 98 | POWERLOADER_API std::string get_env(const char* var); 99 | POWERLOADER_API std::string get_env(const char* var, const std::string& default_value); 100 | 101 | POWERLOADER_API 102 | std::vector split(const std::string_view& input, 103 | const std::string_view& sep, 104 | std::size_t max_split = SIZE_MAX); 105 | 106 | POWERLOADER_API 107 | std::vector rsplit(const std::string_view& input, 108 | const std::string_view& sep, 109 | std::size_t max_split); 110 | 111 | 112 | template 113 | inline void replace_all_impl(S& data, const S& search, const S& replace) 114 | { 115 | std::size_t pos = data.find(search); 116 | while (pos != std::string::npos) 117 | { 118 | data.replace(pos, search.size(), replace); 119 | pos = data.find(search, pos + replace.size()); 120 | } 121 | } 122 | 123 | POWERLOADER_API 124 | void replace_all(std::string& data, const std::string& search, const std::string& replace); 125 | 126 | POWERLOADER_API 127 | void replace_all(std::wstring& data, const std::wstring& search, const std::wstring& replace); 128 | 129 | // Removes duplicate values (compared using `==`) from a sequence container. 130 | // This will change the order of the elements. 131 | // Returns the new end iterator for the container. 132 | template // TODO: use a concept once C++20 is available 133 | auto erase_duplicates(SequenceContainer&& container) 134 | { 135 | // TODO: use ranges once c++20 is available 136 | std::stable_sort(begin(container), end(container)); 137 | auto new_end = std::unique(begin(container), end(container)); 138 | return container.erase(new_end, container.end()); 139 | } 140 | 141 | } 142 | 143 | #endif 144 | -------------------------------------------------------------------------------- /powerloaderConfig.cmake.in: -------------------------------------------------------------------------------- 1 | ############################################################################ 2 | # Copyright (c) 2019, QuantStack and Mamba Contributors # 3 | # # 4 | # Distributed under the terms of the BSD 3-Clause License. # 5 | # # 6 | # The full license is in the file LICENSE, distributed with this software. # 7 | ############################################################################ 8 | 9 | # powerloader cmake module 10 | # This module sets the following variables in your project:: 11 | # 12 | # libpowerloader_FOUND - true if powerloader found on the system 13 | # libpowerloader_INCLUDE_DIRS - the directory containing powerloader headers 14 | # libpowerloader_LIBRARY - the library for dynamic linking 15 | # libpowerloader_STATIC_LIBRARY - the library for static linking 16 | # libpowerloader_FULL_STATIC_LIBRARY - the library for static linking, incl. static deps 17 | 18 | @PACKAGE_INIT@ 19 | 20 | set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR};${CMAKE_MODULE_PATH}") 21 | 22 | @POWERLOADER_CONFIG_CODE@ 23 | 24 | # TODO: when all the dependencies are packaged correclty, 25 | # find dep or dep-static depending on the available target 26 | # in @PROJECT_NAME@Targets.cmake 27 | 28 | include(CMakeFindDependencyMacro) 29 | find_dependency(CURL) 30 | find_dependency(OpenSSL) 31 | find_dependency(CLI11) 32 | find_dependency(yaml-cpp) 33 | find_dependency(tl-expected) 34 | find_dependency(spdlog) 35 | 36 | 37 | if(NOT TARGET libpowerloader AND NOT TARGET libpowerloader-static) 38 | include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") 39 | 40 | if (TARGET libpowerloader-static) 41 | get_target_property(@PROJECT_NAME@_INCLUDE_DIR libpowerloader-static INTERFACE_INCLUDE_DIRECTORIES) 42 | get_target_property(@PROJECT_NAME@_STATIC_LIBRARY libpowerloader-static LOCATION) 43 | endif () 44 | 45 | if (TARGET libpowerloader) 46 | get_target_property(@PROJECT_NAME@_INCLUDE_DIR libpowerloader INTERFACE_INCLUDE_DIRECTORIES) 47 | get_target_property(@PROJECT_NAME@_LIBRARY libpowerloader LOCATION) 48 | endif () 49 | endif() 50 | -------------------------------------------------------------------------------- /src/compression.cpp: -------------------------------------------------------------------------------- 1 | #include "compression.hpp" 2 | 3 | namespace powerloader 4 | { 5 | size_t ZstdStream::write(char* in, size_t size) 6 | { 7 | ZSTD_inBuffer input = { in, size, 0 }; 8 | ZSTD_outBuffer output = { buffer, BUFFER_SIZE, 0 }; 9 | 10 | while (input.pos < input.size) 11 | { 12 | auto ret = ZSTD_decompressStream(stream, &output, &input); 13 | if (ZSTD_isError(ret)) 14 | { 15 | spdlog::error("ZSTD decompression error: {}", ZSTD_getErrorName(ret)); 16 | return size + 1; 17 | } 18 | if (output.pos > 0) 19 | { 20 | size_t wcb_res = m_write_callback(buffer, 1, output.pos, m_write_callback_data); 21 | if (wcb_res != output.pos) 22 | { 23 | return size + 1; 24 | } 25 | output.pos = 0; 26 | } 27 | } 28 | return size; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/compression.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_COMPRESSION_HPP 2 | #define POWERLOADER_COMPRESSION_HPP 3 | 4 | #ifdef WITH_ZSTD 5 | 6 | #include 7 | 8 | namespace powerloader 9 | { 10 | #include 11 | #include 12 | 13 | template 14 | using curl_write_callback_t = size_t (*)(char* ptr, size_t size, size_t nmemb, T* self); 15 | 16 | class ZstdStream 17 | { 18 | public: 19 | constexpr static size_t BUFFER_SIZE = 256000; 20 | 21 | template 22 | ZstdStream(curl_write_callback_t callback, T* write_callback_data) 23 | : stream(ZSTD_createDCtx()) 24 | , m_write_callback((curl_write_callback) callback) 25 | , m_write_callback_data(write_callback_data) 26 | { 27 | ZSTD_initDStream(stream); 28 | } 29 | 30 | ~ZstdStream() 31 | { 32 | ZSTD_freeDCtx(stream); 33 | } 34 | 35 | size_t write(char* in, size_t size); 36 | 37 | static size_t write_callback(char* ptr, size_t size, size_t nmemb, void* self) 38 | { 39 | return static_cast(self)->write(ptr, size * nmemb); 40 | } 41 | 42 | private: 43 | ZSTD_DCtx* stream; 44 | char buffer[BUFFER_SIZE]; 45 | 46 | // original curl callback 47 | curl_write_callback m_write_callback; 48 | void* m_write_callback_data; 49 | }; 50 | } 51 | 52 | #endif 53 | 54 | #endif // POWERLOADER_COMPRESSION_HPP 55 | -------------------------------------------------------------------------------- /src/context.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #ifdef WITH_ZCHUNK 11 | extern "C" 12 | { 13 | #include 14 | } 15 | #endif 16 | 17 | #include 18 | 19 | #include "./curl_internal.hpp" 20 | 21 | 22 | namespace powerloader 23 | { 24 | struct Context::Impl 25 | { 26 | std::optional curl_setup; 27 | }; 28 | 29 | static std::atomic is_context_alive{ false }; 30 | 31 | Context::Context(ContextOptions options) 32 | : impl(new Impl) 33 | { 34 | bool expected = false; 35 | if (!is_context_alive.compare_exchange_strong(expected, true)) 36 | throw std::runtime_error( 37 | "powerloader::Context created more than once - instance must be unique"); 38 | 39 | if (options.ssl_backend) 40 | { 41 | impl->curl_setup.emplace(options.ssl_backend.value()); 42 | } 43 | 44 | cache_dir = fs::absolute(fs::path(".pdcache")); 45 | if (!fs::exists(cache_dir)) 46 | { 47 | fs::create_directories(cache_dir); 48 | } 49 | set_verbosity(0); 50 | } 51 | 52 | Context::~Context() 53 | { 54 | is_context_alive = false; 55 | } 56 | 57 | void Context::set_verbosity(int v) 58 | { 59 | verbosity = v; 60 | if (v > 2) 61 | { 62 | spdlog::set_level(spdlog::level::warn); 63 | } 64 | else if (v > 0) 65 | { 66 | #ifdef WITH_ZCHUNK 67 | zck_set_log_level(ZCK_LOG_DEBUG); 68 | #endif 69 | spdlog::set_level(spdlog::level::debug); 70 | } 71 | else 72 | { 73 | spdlog::set_level(spdlog::level::off); 74 | } 75 | } 76 | 77 | 78 | void Context::set_log_level(spdlog::level::level_enum log_level) 79 | { 80 | spdlog::set_level(log_level); 81 | #ifdef WITH_ZCHUNK 82 | if (log_level <= spdlog::level::debug) 83 | { 84 | zck_set_log_level(ZCK_LOG_DEBUG); 85 | } 86 | #endif 87 | } 88 | 89 | std::string mirror_map_type::to_string() const 90 | { 91 | std::string result; 92 | for (const auto& [mirror_name, mirrors] : *this) 93 | { 94 | result += mirror_name + ": ["; 95 | for (const auto& mirror : mirrors) 96 | { 97 | result += mirror->id().to_string() + ", "; 98 | } 99 | result += "]\n"; 100 | } 101 | return result; 102 | } 103 | 104 | mirror_set mirror_map_type::get_mirrors(std::string_view host_name) const 105 | { 106 | auto find_it = find(std::string(host_name)); 107 | if (find_it == end()) 108 | return {}; 109 | 110 | return find_it->second; 111 | } 112 | 113 | // Returns true if there are registered mirrors stored here, false if none are. 114 | bool mirror_map_type::has_mirrors(std::string_view host_name) const 115 | { 116 | auto find_it = find(std::string(host_name)); 117 | return find_it != end() && !find_it->second.empty(); 118 | } 119 | 120 | bool mirror_map_type::add_unique_mirror(std::string_view host_name, 121 | std::shared_ptr mirror) 122 | { 123 | auto find_it = find(std::string(host_name)); 124 | if (find_it != end()) 125 | { 126 | auto& mirrors = find_it->second; 127 | if (details::already_exists(mirror->id(), mirrors)) 128 | return false; 129 | mirrors.push_back(std::move(mirror)); 130 | } 131 | else 132 | { 133 | (*this)[std::string(host_name)] = { std::move(mirror) }; 134 | } 135 | return true; 136 | } 137 | 138 | void mirror_map_type::reset(mirror_map_base new_values) 139 | { 140 | if (!details::is_every_mirror_unique_per_host(new_values)) 141 | throw std::invalid_argument("mirror map must have unique mirrors per host name"); 142 | static_cast(*this) = std::move(new_values); 143 | } 144 | 145 | namespace details 146 | { 147 | bool already_exists(const MirrorID& id, const mirror_set& mirrors) 148 | { 149 | for (auto&& mirror : mirrors) 150 | if (mirror->id() == id) 151 | return true; 152 | return false; 153 | } 154 | 155 | bool is_every_mirror_unique_per_host(const mirror_map_base& mirrors) 156 | { 157 | for (const auto& slot : mirrors) 158 | { 159 | std::set mirrors_ids; // TODO: replace by flat_set once available. 160 | for (const auto& mirror : slot.second) 161 | { 162 | auto [_, success] = mirrors_ids.insert(mirror->id()); 163 | if (!success) 164 | return false; 165 | } 166 | } 167 | return true; 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/curl_internal.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_SRC_CURL_INTERNAL_HPP 2 | #define POWERLOADER_SRC_CURL_INTERNAL_HPP 3 | 4 | #include 5 | 6 | #include 7 | 8 | namespace powerloader::details 9 | { 10 | // Scoped initialization and termination of CURL. 11 | // This should never have more than one instance live at any time, 12 | // this object's constructor will throw an `std::runtime_error` if it's the case. 13 | class CURLSetup final 14 | { 15 | public: 16 | explicit CURLSetup(const ssl_backend_t& ssl_backend); 17 | ~CURLSetup(); 18 | 19 | CURLSetup(CURLSetup&&) = delete; 20 | CURLSetup& operator=(CURLSetup&&) = delete; 21 | 22 | CURLSetup(const CURLSetup&) = delete; 23 | CURLSetup& operator=(const CURLSetup&) = delete; 24 | }; 25 | } 26 | #endif 27 | -------------------------------------------------------------------------------- /src/download_target.cpp: -------------------------------------------------------------------------------- 1 | #include "powerloader/context.hpp" 2 | #include 3 | 4 | #ifdef WITH_ZCHUNK 5 | #include "zck.hpp" 6 | #endif 7 | 8 | namespace powerloader 9 | { 10 | 11 | #ifndef WITH_ZCHUNK 12 | struct zck_target 13 | { 14 | }; 15 | #endif 16 | 17 | DownloadTarget::DownloadTarget(const std::string& path, 18 | const std::string& mirror_name, 19 | const fs::path& destination) 20 | : m_is_zchunk(ends_with(path, ".zck")) 21 | , m_path(path) 22 | , m_mirror_name(mirror_name) 23 | , m_destination_path(destination) 24 | { 25 | #if WITH_ZCHUNK 26 | if (m_is_zchunk) 27 | { 28 | m_p_zck = std::make_unique(); 29 | m_p_zck->zck_cache_file = m_destination_path; 30 | } 31 | #endif 32 | } 33 | 34 | std::shared_ptr DownloadTarget::from_url( 35 | Context& ctx, 36 | const std::string& target_url, 37 | const fs::path& destination_path, 38 | const fs::path& destination_dir, 39 | std::optional hostname_override) 40 | { 41 | if (contains(target_url, "://")) 42 | { 43 | // even when we get a regular URL like `http://test.com/download.tar.gz` 44 | // we want to create a "mirror" for `http://test.com` to make sure we correctly 45 | // retry and wait on mirror failures 46 | URLHandler uh{ target_url }; 47 | if (uh.scheme() == "file") 48 | { 49 | ctx.mirror_map.create_unique_mirror("[file]", ctx, "file://"); 50 | return std::make_shared(uh.path(), "[file]", destination_path); 51 | } 52 | 53 | const std::string url = uh.url(); 54 | const std::string host = hostname_override ? hostname_override.value() : uh.host(); 55 | const std::string path = uh.path(); 56 | const std::string mirror_url = uh.url_without_path(); 57 | const fs::path dst = destination_path.empty() ? fs::path{ rsplit(path, "/", 1).back() } 58 | : destination_path; 59 | 60 | ctx.mirror_map.create_unique_mirror(host, ctx, mirror_url); 61 | return std::make_shared(path.substr(1, std::string::npos), host, dst); 62 | } 63 | else 64 | { 65 | const std::vector parts = split(target_url, ":"); 66 | if (parts.size() != 2) 67 | { 68 | throw std::runtime_error("Not the correct number of : in the url"); 69 | } 70 | const auto mirror = hostname_override ? hostname_override.value() : parts[0]; 71 | const auto path = parts[1]; 72 | 73 | fs::path dst = destination_path.empty() ? fs::path{ rsplit(path, "/", 1).back() } 74 | : destination_path; 75 | 76 | if (!destination_dir.empty()) 77 | dst = destination_dir / dst; 78 | 79 | return std::make_shared(path, mirror, dst); 80 | } 81 | } 82 | 83 | DownloadTarget::~DownloadTarget() = default; 84 | 85 | bool DownloadTarget::validate_checksum(const fs::path& path) 86 | { 87 | if (m_checksums.empty()) 88 | return false; 89 | 90 | auto findchecksum = [&](const ChecksumType& t) -> Checksum* 91 | { 92 | for (auto& cs : m_checksums) 93 | { 94 | if (cs.type == t) 95 | return &cs; 96 | } 97 | return nullptr; 98 | }; 99 | 100 | Checksum* cs; 101 | if ((cs = findchecksum(ChecksumType::kSHA256))) 102 | { 103 | auto sum = sha256sum(path); 104 | 105 | if (sum != cs->checksum) 106 | { 107 | spdlog::error("SHA256 sum of downloaded file is wrong.\nIs {}. Should be {}", 108 | sum, 109 | cs->checksum); 110 | return false; 111 | } 112 | return true; 113 | } 114 | else if ((cs = findchecksum(ChecksumType::kSHA1))) 115 | { 116 | spdlog::error("Checking SHA1 sum not implemented!"); 117 | return false; 118 | } 119 | else if ((cs = findchecksum(ChecksumType::kMD5))) 120 | { 121 | spdlog::info("Checking MD5 sum"); 122 | auto sum = md5sum(path); 123 | if (sum != cs->checksum) 124 | { 125 | spdlog::error( 126 | "MD5 sum of downloaded file is wrong.\nIs {}. Should be {}", sum, cs->checksum); 127 | return false; 128 | } 129 | return true; 130 | } 131 | return false; 132 | } 133 | 134 | bool DownloadTarget::already_downloaded() 135 | { 136 | if (m_checksums.empty()) 137 | return false; 138 | return fs::exists(m_destination_path) && validate_checksum(m_destination_path); 139 | } 140 | 141 | void DownloadTarget::set_cache_options(const CacheControl& cache_control) 142 | { 143 | m_cache_control = cache_control; 144 | } 145 | 146 | void DownloadTarget::add_handle_options(CURLHandle& handle) 147 | { 148 | auto to_header = [](const std::string& key, const std::string& value) 149 | { return std::string(key + ": " + value); }; 150 | 151 | if (m_cache_control.etag.size()) 152 | { 153 | handle.add_header(to_header("If-None-Match", m_cache_control.etag)); 154 | } 155 | if (m_cache_control.last_modified.size()) 156 | { 157 | handle.add_header(to_header("If-Modified-Since", m_cache_control.last_modified)); 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/fastest_mirror.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | namespace powerloader 10 | { 11 | namespace detail 12 | { 13 | struct InternalMirror 14 | { 15 | std::string url; 16 | CURLHandle handle; 17 | curl_off_t plain_connect_time; 18 | }; 19 | 20 | long HALF_OF_SECOND_IN_MICROS = 500000; 21 | 22 | tl::expected, std::string> fastestmirror_perform( 23 | std::vector& mirrors, std::size_t length_of_measurement); 24 | } 25 | 26 | tl::expected, std::string> fastest_mirror( 27 | const Context& ctx, const std::vector& urls) 28 | { 29 | std::vector check_mirrors; 30 | for (const std::string& u : urls) 31 | { 32 | CURLHandle handle(ctx, u); 33 | handle.setopt(CURLOPT_CONNECT_ONLY, 1L); 34 | check_mirrors.push_back(detail::InternalMirror{ u, std::move(handle), -1 }); 35 | } 36 | return fastestmirror_perform(check_mirrors, 1000000); 37 | } 38 | 39 | namespace detail 40 | { 41 | tl::expected, std::string> fastestmirror_perform( 42 | std::vector& mirrors, std::size_t length_of_measurement) 43 | { 44 | if (mirrors.size() == 0) 45 | return {}; 46 | 47 | CURLM* multihandle = curl_multi_init(); 48 | if (!multihandle) 49 | { 50 | return tl::unexpected(std::string("curl_multi_init() error")); 51 | } 52 | 53 | // Add curl easy handles to multi handle 54 | std::size_t handles_added = 0; 55 | for (auto& el : mirrors) 56 | { 57 | if (el.handle) 58 | { 59 | curl_multi_add_handle(multihandle, el.handle); 60 | handles_added++; 61 | spdlog::info("Checking URL: {}", el.url); 62 | } 63 | } 64 | 65 | if (handles_added == 0) 66 | { 67 | curl_multi_cleanup(multihandle); 68 | return {}; 69 | } 70 | 71 | // cb(cbdata, LR_FMSTAGE_DETECTION, (void *) &handles_added); 72 | 73 | int still_running; 74 | // _cleanup_timer_destroy_ GTimer *timer = g_timer_new(); 75 | // g_timer_start(timer); 76 | using time_point_t = std::chrono::steady_clock::time_point; 77 | time_point_t tend, tbegin = std::chrono::steady_clock::now(); 78 | std::size_t elapsed_micros = 0; 79 | do 80 | { 81 | timeval timeout; 82 | int rc; 83 | CURLMcode cm_rc; 84 | int maxfd = -1; 85 | long curl_timeout = -1; 86 | fd_set fdread, fdwrite, fdexcep; 87 | 88 | FD_ZERO(&fdread); 89 | FD_ZERO(&fdwrite); 90 | FD_ZERO(&fdexcep); 91 | 92 | // Set suitable timeout to play around with 93 | timeout.tv_sec = 0; 94 | timeout.tv_usec = detail::HALF_OF_SECOND_IN_MICROS; 95 | 96 | cm_rc = curl_multi_timeout(multihandle, &curl_timeout); 97 | if (cm_rc != CURLM_OK) 98 | { 99 | spdlog::error("fastestmirror: CURL multi failed"); 100 | curl_multi_cleanup(multihandle); 101 | return tl::unexpected( 102 | fmt::format("curl_multi_timeout() error: {}", curl_multi_strerror(cm_rc))); 103 | } 104 | 105 | // Set timeout to a reasonable value 106 | if (curl_timeout >= 0) 107 | { 108 | timeout.tv_sec = curl_timeout / 1000; 109 | if (timeout.tv_sec >= 1) 110 | { 111 | timeout.tv_sec = 0; 112 | timeout.tv_usec = detail::HALF_OF_SECOND_IN_MICROS; 113 | } 114 | else 115 | { 116 | timeout.tv_usec = (curl_timeout % 1000) * 1000; 117 | if (timeout.tv_usec > detail::HALF_OF_SECOND_IN_MICROS) 118 | timeout.tv_usec = detail::HALF_OF_SECOND_IN_MICROS; 119 | } 120 | } 121 | 122 | // Get file descriptors from the transfers 123 | cm_rc = curl_multi_fdset(multihandle, &fdread, &fdwrite, &fdexcep, &maxfd); 124 | if (cm_rc != CURLM_OK) 125 | { 126 | spdlog::error("fastestmirror: CURL fd set error."); 127 | curl_multi_cleanup(multihandle); 128 | return tl::unexpected( 129 | fmt::format("curl_multi_fdset() error: {}", curl_multi_strerror(cm_rc))); 130 | } 131 | 132 | rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); 133 | if (rc < 0) 134 | { 135 | if (errno == EINTR) 136 | { 137 | spdlog::debug("fastestmirror: select() interrupted by signal"); 138 | } 139 | else 140 | { 141 | curl_multi_cleanup(multihandle); 142 | return tl::unexpected( 143 | fmt::format("select() error: %s", std::strerror(errno))); 144 | } 145 | } 146 | 147 | curl_multi_perform(multihandle, &still_running); 148 | 149 | // Break loop after some reasonable amount of time 150 | tend = std::chrono::steady_clock::now(); 151 | elapsed_micros 152 | = std::chrono::duration_cast(tend - tbegin).count(); 153 | } while (still_running && elapsed_micros < length_of_measurement); 154 | 155 | // Remove curl easy handles from multi handle and calculate plain_connect_time 156 | for (auto& el : mirrors) 157 | { 158 | // Remove handle 159 | curl_multi_remove_handle(multihandle, el.handle); 160 | 161 | // Calculate plain_connect_time 162 | auto effective_url = el.handle.getinfo(CURLINFO_EFFECTIVE_URL); 163 | 164 | if (!effective_url) 165 | { 166 | // No effective url is most likely an error 167 | el.plain_connect_time = std::numeric_limits::max(); 168 | } 169 | else if (starts_with(effective_url.value(), "file:")) 170 | { 171 | // Local directories are considered to be the best mirrors 172 | el.plain_connect_time = 0; 173 | } 174 | else 175 | { 176 | // Get connect time 177 | curl_off_t namelookup_time 178 | = el.handle.getinfo(CURLINFO_NAMELOOKUP_TIME_T).value_or(0); 179 | curl_off_t connect_time 180 | = el.handle.getinfo(CURLINFO_CONNECT_TIME_T).value_or(0); 181 | 182 | if (connect_time == 0) 183 | { 184 | // Zero connect time is most likely an error 185 | el.plain_connect_time = std::numeric_limits::max(); 186 | } 187 | else 188 | { 189 | el.plain_connect_time = connect_time - namelookup_time; 190 | } 191 | } 192 | } 193 | 194 | curl_multi_cleanup(multihandle); 195 | 196 | for (auto& el : mirrors) 197 | { 198 | spdlog::info("Mirror: {} -> {}", el.url, el.plain_connect_time); 199 | } 200 | 201 | // sort 202 | std::sort(mirrors.begin(), 203 | mirrors.end(), 204 | [](detail::InternalMirror& m1, detail::InternalMirror& m2) 205 | { return m1.plain_connect_time < m2.plain_connect_time; }); 206 | 207 | std::vector sorted_urls(mirrors.size()); 208 | for (auto& m : mirrors) 209 | { 210 | sorted_urls.push_back(m.url); 211 | } 212 | 213 | return sorted_urls; 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/mirror.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include "powerloader/mirrorid.hpp" 9 | #include "target.hpp" 10 | 11 | 12 | namespace powerloader 13 | { 14 | namespace 15 | { 16 | std::string strip_trailing_slash(const std::string& s) 17 | { 18 | if (s.size() > 0 && s.back() == '/' && s != "file://") 19 | { 20 | return s.substr(0, s.size() - 1); 21 | } 22 | return s; 23 | } 24 | } 25 | 26 | MirrorID Mirror::id(const std::string& url) 27 | { 28 | return MirrorID(fmt::format("Mirror[{}]", url)); 29 | } 30 | 31 | Mirror::Mirror(const MirrorID& id, const Context& ctx, const std::string& url) 32 | : m_id(id) 33 | , m_url(strip_trailing_slash(url)) 34 | { 35 | if (ctx.max_downloads_per_mirror > 0) 36 | { 37 | m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror; 38 | } 39 | } 40 | 41 | Mirror::~Mirror() = default; 42 | 43 | void Mirror::change_max_ranges(int new_value) 44 | { 45 | // TODO: add some checks here. 46 | m_stats.max_ranges = new_value; 47 | } 48 | 49 | bool Mirror::need_wait_for_retry() const 50 | { 51 | return m_retry_counter != 0 && m_next_retry > std::chrono::system_clock::now(); 52 | } 53 | 54 | bool Mirror::has_running_transfers() const 55 | { 56 | return m_stats.running_transfers > 0; 57 | } 58 | 59 | void Mirror::set_allowed_parallel_connections(int max_allowed_parallel_connections) 60 | { 61 | m_stats.allowed_parallel_connections = max_allowed_parallel_connections; 62 | } 63 | 64 | void Mirror::increase_running_transfers() 65 | { 66 | m_stats.running_transfers++; 67 | if (m_stats.max_tried_parallel_connections < m_stats.running_transfers) 68 | { 69 | m_stats.max_tried_parallel_connections = m_stats.running_transfers; 70 | } 71 | } 72 | 73 | bool Mirror::is_parallel_connections_limited_and_reached() const 74 | { 75 | return m_stats.allowed_parallel_connections != -1 76 | && m_stats.running_transfers >= m_stats.allowed_parallel_connections; 77 | } 78 | 79 | void Mirror::update_statistics(bool transfer_success) 80 | { 81 | m_stats.running_transfers--; 82 | if (transfer_success) 83 | { 84 | m_stats.successful_transfers++; 85 | } 86 | else 87 | { 88 | m_stats.failed_transfers++; 89 | if (m_stats.failed_transfers == 1 || m_next_retry < std::chrono::system_clock::now()) 90 | { 91 | m_retry_counter++; 92 | m_retry_wait_seconds = m_retry_wait_seconds * m_retry_backoff_factor; 93 | m_next_retry = std::chrono::system_clock::now() + m_retry_wait_seconds; 94 | } 95 | } 96 | } 97 | 98 | double Mirror::rank() const 99 | { 100 | double rank = -1.0; 101 | 102 | const int finished_transfers = m_stats.count_finished_transfers(); 103 | 104 | if (finished_transfers < 3) 105 | return rank; // Do not judge too early 106 | 107 | rank = m_stats.successful_transfers / static_cast(finished_transfers); 108 | 109 | return rank; 110 | } 111 | 112 | bool Mirror::prepare(Target*) 113 | { 114 | m_state = MirrorState::READY; 115 | return true; 116 | } 117 | 118 | bool Mirror::prepare(const std::string&, CURLHandle&) 119 | { 120 | m_state = MirrorState::READY; 121 | return true; 122 | } 123 | 124 | bool Mirror::needs_preparation(Target*) const 125 | { 126 | return false; 127 | } 128 | 129 | bool Mirror::authenticate(CURLHandle&, const std::string&) 130 | { 131 | return true; 132 | } 133 | 134 | std::vector Mirror::get_auth_headers(const std::string&) const 135 | { 136 | return {}; 137 | } 138 | 139 | std::string Mirror::format_url(Target* target) const 140 | { 141 | return join_url(m_url, target->target().path()); 142 | } 143 | 144 | /** Sort mirrors. Penalize the error ones. 145 | * In fact only move the current finished mirror forward or backward 146 | * by one position. 147 | * @param mirrors GSList of mirrors (order of list elements won't be 148 | * changed, only data pointers) 149 | * @param mirror Mirror of just finished transfer 150 | * @param success Was download from the mirror successful 151 | * @param serious If success is FALSE, serious mean that error was serious 152 | * (like connection timeout), and the mirror should be 153 | * penalized more that usual. 154 | */ 155 | bool sort_mirrors(std::vector>& mirrors, 156 | const std::shared_ptr& mirror, 157 | bool success, 158 | bool serious) 159 | { 160 | assert(mirror); 161 | 162 | if (mirrors.size() == 1) 163 | return true; 164 | 165 | auto it = std::find(mirrors.begin(), mirrors.end(), mirror); 166 | 167 | // no penalization, mirror is already last 168 | if (!success && (it + 1) == mirrors.end()) 169 | return true; 170 | 171 | // Bonus not needed - Mirror is already the first one 172 | if (success && it == mirrors.begin()) 173 | return true; 174 | 175 | // Serious errors 176 | if (serious && mirror->stats().successful_transfers == 0) 177 | { 178 | // Mirror that encounter a serious error and has no successful 179 | // transfers should be moved at the end of the list 180 | // (such mirror is probably down/broken/buggy) 181 | 182 | // TODO should we really _swap_ here or rather move that one mirror down and shuffle all 183 | // others?! 184 | std::iter_swap(it, mirrors.end() - 1); 185 | spdlog::info("Mirror {} was moved to the end", mirror->url()); 186 | return true; 187 | } 188 | 189 | // Calculate ranks 190 | double rank_cur = mirror->rank(); 191 | // Too early to judge 192 | if (rank_cur < 0.0) 193 | return true; 194 | 195 | if (!success) 196 | { 197 | // Penalize 198 | double rank_next = (*(it + 1))->rank(); 199 | if (rank_next < 0.0 || rank_next > rank_cur) 200 | { 201 | std::iter_swap(it, it + 1); 202 | spdlog::info("Mirror {} was penalized", mirror->url()); 203 | } 204 | } 205 | else 206 | { 207 | // Bonus 208 | double rank_prev = (*(it - 1))->rank(); 209 | if (rank_prev < rank_cur) 210 | { 211 | std::iter_swap(it, it - 1); 212 | spdlog::info("Mirror {} was awarded", mirror->url()); 213 | } 214 | } 215 | 216 | return true; 217 | } 218 | 219 | HTTPMirror::HTTPMirror(const Context& ctx, const std::string& url) 220 | : Mirror(HTTPMirror::id(url), ctx, url) 221 | { 222 | } 223 | 224 | MirrorID HTTPMirror::id(const std::string& url) 225 | { 226 | return MirrorID{ fmt::format("HTTPMirror[{}]", url) }; 227 | } 228 | 229 | bool HTTPMirror::authenticate(CURLHandle& handle, const std::string& path) 230 | { 231 | if (!m_auth_password.empty()) 232 | { 233 | spdlog::warn( 234 | "Setting HTTP authentication for {} to {}:{}", path, m_auth_user, m_auth_password); 235 | handle.setopt(CURLOPT_USERNAME, m_auth_user.c_str()); 236 | handle.setopt(CURLOPT_PASSWORD, m_auth_password.c_str()); 237 | } 238 | return true; 239 | } 240 | 241 | void HTTPMirror::set_auth(const std::string& user, const std::string& password) 242 | { 243 | m_auth_user = user; 244 | m_auth_password = password; 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/mirrors/oci.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include "target.hpp" 5 | 6 | namespace powerloader 7 | { 8 | // OCI Mirror: 9 | // When knowing the SHA256 we can directly get to the blob 10 | // When we do not know the SHA256 sum, we need to find the `latest` or some 11 | // other blob 12 | 13 | // OCI upload process 14 | // 4 steps: 15 | // - first get auth token with push rights 16 | // - then 17 | 18 | // This is what an OCI manifest (index) looks like: 19 | // { 20 | // "schemaVersion": 2, 21 | // "config": { 22 | // "mediaType": "application/vnd.unknown.config.v1+json", 23 | // "digest": 24 | // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 25 | // "size": 0 26 | // }, 27 | // "layers": [ 28 | // { 29 | // "mediaType": "application/vnd.unknown.layer.v1+txt", 30 | // "digest": 31 | // "sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4", 32 | // "size": 13, 33 | // "annotations": { 34 | // "org.opencontainers.image.title": "artifact.txt" 35 | // } 36 | // } 37 | // ] 38 | // } 39 | 40 | OCIMirror::OCIMirror(const Context& ctx, 41 | const std::string& host, 42 | const std::string& repo_prefix) 43 | : Mirror(OCIMirror::id(host, repo_prefix), ctx, host) 44 | , m_repo_prefix(repo_prefix) 45 | , m_scope("pull") 46 | , m_proxy_map(ctx.proxy_map) 47 | { 48 | } 49 | 50 | OCIMirror::OCIMirror(const Context& ctx, 51 | const std::string& host, 52 | const std::string& repo_prefix, 53 | const std::string& scope, 54 | const std::string& username, 55 | const std::string& password) 56 | : Mirror(OCIMirror::id(host, repo_prefix), ctx, host) 57 | , m_repo_prefix(repo_prefix) 58 | , m_scope(scope) 59 | , m_username(username) 60 | , m_password(password) 61 | , m_proxy_map(ctx.proxy_map) 62 | { 63 | } 64 | 65 | OCIMirror::~OCIMirror() = default; 66 | 67 | void OCIMirror::set_fn_tag_split_function(const split_function_type& func) 68 | { 69 | m_split_func = func; 70 | } 71 | 72 | std::pair OCIMirror::split_path_tag(const std::string& path) const 73 | { 74 | std::string split_path, split_tag; 75 | if (m_split_func) 76 | { 77 | std::tie(split_path, split_tag) = m_split_func(path); 78 | } 79 | else 80 | { 81 | split_path = path; 82 | split_tag = "latest"; 83 | } 84 | return std::make_pair(split_path, split_tag); 85 | } 86 | 87 | std::string OCIMirror::get_repo(const std::string& repo) const 88 | { 89 | if (!m_repo_prefix.empty()) 90 | return fmt::format("{}/{}", m_repo_prefix, repo); 91 | else 92 | return repo; 93 | } 94 | 95 | std::string OCIMirror::get_auth_url(const std::string& repo, const std::string& scope) const 96 | { 97 | return fmt::format("{}/token?scope=repository:{}:{}", this->url(), get_repo(repo), scope); 98 | } 99 | 100 | std::string OCIMirror::get_manifest_url(const std::string& repo, 101 | const std::string& reference) const 102 | { 103 | return fmt::format("{}/v2/{}/manifests/{}", this->url(), get_repo(repo), reference); 104 | } 105 | 106 | std::string OCIMirror::get_preupload_url(const std::string& repo) const 107 | { 108 | return fmt::format("{}/v2/{}/blobs/uploads/", this->url(), get_repo(repo)); 109 | } 110 | 111 | OCIMirror::AuthCallbackData* OCIMirror::get_data(Target* target) const 112 | { 113 | auto [split_path, _] = split_path_tag(target->target().path()); 114 | auto it = m_path_cb_map.find(split_path); 115 | if (it != m_path_cb_map.end()) 116 | { 117 | return it->second.get(); 118 | } 119 | return nullptr; 120 | } 121 | 122 | std::vector OCIMirror::get_auth_headers(const std::string& path) const 123 | { 124 | if (m_username.empty() && m_password.empty()) 125 | return {}; 126 | auto [split_path, _] = split_path_tag(path); 127 | auto& data = m_path_cb_map.at(split_path); 128 | return { fmt::format("Authorization: Bearer {}", data->token) }; 129 | } 130 | 131 | bool OCIMirror::prepare(const std::string& path, CURLHandle& handle) 132 | { 133 | auto [split_path, split_tag] = split_path_tag(path); 134 | 135 | auto it = m_path_cb_map.find(split_path); 136 | if (it == m_path_cb_map.end()) 137 | { 138 | m_path_cb_map[split_path].reset(new AuthCallbackData); 139 | auto data = m_path_cb_map[split_path].get(); 140 | data->self = this; 141 | } 142 | 143 | auto& cbdata = m_path_cb_map[split_path]; 144 | 145 | if (cbdata->token.empty() && need_auth()) 146 | { 147 | std::string auth_url = get_auth_url(split_path, m_scope); 148 | handle.url(auth_url, m_proxy_map); 149 | 150 | handle.set_default_callbacks(); 151 | 152 | if (!m_username.empty()) 153 | { 154 | handle.setopt(CURLOPT_USERNAME, m_username.c_str()); 155 | } 156 | if (!m_password.empty()) 157 | { 158 | handle.setopt(CURLOPT_PASSWORD, m_password.c_str()); 159 | } 160 | 161 | auto end_callback = [&cbdata](const Response& response) 162 | { 163 | if (!response.ok()) 164 | return CbReturnCode::kERROR; 165 | 166 | auto j = response.json(); 167 | if (j.contains("token")) 168 | { 169 | cbdata->token = j["token"].get(); 170 | return CbReturnCode::kOK; 171 | } 172 | return CbReturnCode::kERROR; 173 | }; 174 | 175 | handle.set_end_callback(end_callback); 176 | } 177 | else 178 | { 179 | handle.set_default_callbacks(); 180 | 181 | std::string manifest_url = get_manifest_url(split_path, split_tag); 182 | 183 | handle.url(manifest_url, m_proxy_map) 184 | .add_headers(get_auth_headers(path)) 185 | .add_header("Accept: application/vnd.oci.image.manifest.v1+json"); 186 | 187 | auto finalize_manifest_callback = [&cbdata](const Response& response) 188 | { 189 | if (!response.ok()) 190 | return CbReturnCode::kERROR; 191 | auto j = response.json(); 192 | 193 | if (j.contains("layers")) 194 | { 195 | std::string digest = j["layers"][0]["digest"]; 196 | 197 | assert(starts_with(digest, "sha256:")); 198 | 199 | // For some reason target->target isn't available here? 200 | // cbdata->target().target->checksums.push_back( 201 | // Checksum{ChecksumType::kSHA256, digest.substr(sizeof("sha256:") - 1)} 202 | // ); 203 | 204 | cbdata->sha256sum = digest.substr(sizeof("sha256:") - 1); 205 | return CbReturnCode::kOK; 206 | } 207 | 208 | return CbReturnCode::kERROR; 209 | }; 210 | 211 | handle.set_end_callback(finalize_manifest_callback); 212 | } 213 | return true; 214 | } 215 | 216 | bool OCIMirror::need_auth() const 217 | { 218 | return m_username.size() && m_password.size(); 219 | } 220 | 221 | bool OCIMirror::needs_preparation(Target* target) const 222 | { 223 | auto* data = get_data(target); 224 | if ((!data || (data && data->token.empty())) && need_auth()) 225 | return true; 226 | 227 | if (data && !data->sha256sum.empty()) 228 | return false; 229 | 230 | const auto& checksums = target->target().checksums(); 231 | if (std::none_of(checksums.begin(), 232 | checksums.end(), 233 | [](auto& ck) { return ck.type == ChecksumType::kSHA256; })) 234 | return true; 235 | 236 | return false; 237 | } 238 | 239 | std::string OCIMirror::format_url(Target* target) const 240 | { 241 | const std::string* checksum = nullptr; 242 | 243 | for (const auto& ck : target->target().checksums()) // TODO: replace by std::find? 244 | { 245 | if (ck.type == ChecksumType::kSHA256) 246 | checksum = &ck.checksum; 247 | } 248 | 249 | if (!checksum) 250 | { 251 | auto* data = get_data(target); 252 | checksum = &data->sha256sum; 253 | } 254 | auto [split_path, split_tag] = split_path_tag(target->target().path()); 255 | // https://ghcr.io/v2/wolfv/artifact/blobs/sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4 256 | return fmt::format( 257 | "{}/v2/{}/blobs/sha256:{}", this->url(), get_repo(split_path), *checksum); 258 | } 259 | 260 | std::string OCIMirror::get_digest(const fs::path& p) const 261 | { 262 | return fmt::format("sha256:{}", sha256sum(p)); 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /src/mirrors/s3.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | extern "C" 4 | { 5 | #include 6 | #include 7 | #include 8 | } 9 | 10 | #include 11 | #include 12 | #include 13 | #include "target.hpp" 14 | 15 | namespace powerloader 16 | { 17 | std::string get_yyyymmdd(const std::chrono::system_clock::time_point& t) 18 | { 19 | static constexpr std::size_t yyyymmddlength = sizeof("YYYYMMDD"); 20 | std::time_t t_date = std::chrono::system_clock::to_time_t(t); 21 | char yyyymmdd[yyyymmddlength]; 22 | strftime(yyyymmdd, yyyymmddlength, "%Y%m%d", gmtime(&t_date)); 23 | return yyyymmdd; 24 | } 25 | 26 | std::string get_iso8601(const std::chrono::system_clock::time_point& t) 27 | { 28 | static constexpr std::size_t iso8601length = sizeof("YYYYMMDDTHHMMSSZ"); 29 | std::time_t t_date = std::chrono::system_clock::to_time_t(t); 30 | char iso8601[iso8601length]; 31 | std::strftime(iso8601, iso8601length, "%Y%m%dT%H%M%SZ", std::gmtime(&t_date)); 32 | return iso8601; 33 | } 34 | 35 | /********************** 36 | * S3CanonicalRequest * 37 | **********************/ 38 | 39 | S3CanonicalRequest::S3CanonicalRequest(const std::string& lhttp_verb, 40 | const URLHandler& uh, 41 | const std::string& sha256sum) 42 | : http_verb(lhttp_verb) 43 | , hashed_payload(sha256sum.empty() ? EMPTY_SHA : sha256sum) 44 | , date(std::chrono::system_clock::now()) 45 | { 46 | bucket_url = uh.url_without_path(); 47 | resource = uh.path(); 48 | if (resource.size() >= 1 && resource[0] == '/') 49 | { 50 | resource = resource.substr(1, std::string::npos); 51 | } 52 | 53 | init_default_headers(); 54 | } 55 | 56 | void S3CanonicalRequest::init_default_headers() 57 | { 58 | URLHandler uh(bucket_url); 59 | headers["x-amz-date"] = get_iso8601(date); 60 | // if (s3_session_token != "") 61 | // headers["x-amz-security-token"] = s3_session_token; 62 | headers["x-amz-content-sha256"] = hashed_payload; 63 | headers["Host"] = uh.host(); 64 | headers["Content-Type"] = "application/octet-stream"; 65 | } 66 | 67 | std::string S3CanonicalRequest::get_signed_headers() 68 | { 69 | std::stringstream signed_headers; 70 | for (auto it = headers.begin(); it != headers.end(); ++it) 71 | signed_headers << (it == headers.begin() ? "" : ";") << to_lower(it->first); 72 | return signed_headers.str(); 73 | } 74 | 75 | std::string S3CanonicalRequest::canonical_request() 76 | { 77 | std::stringstream canonical_headers, signed_headers; 78 | for (auto it = headers.begin(); it != headers.end(); ++it) 79 | { 80 | canonical_headers << to_lower(it->first) << ":" << it->second << "\n"; 81 | } 82 | 83 | std::stringstream ss; 84 | ss << http_verb << "\n" 85 | << "/" << resource << "\n" 86 | << "" 87 | << "\n" // canonical query string 88 | << canonical_headers.str() << "\n" 89 | << get_signed_headers() << "\n" 90 | << hashed_payload; 91 | 92 | return ss.str(); 93 | } 94 | 95 | std::string S3CanonicalRequest::string_to_sign(const std::string& region, 96 | const std::string& service) 97 | { 98 | std::stringstream ss; 99 | ss << "AWS4-HMAC-SHA256\n" 100 | << get_iso8601(date) << "\n" 101 | << get_yyyymmdd(date) << "/" << region << "/" << service << "/aws4_request\n" 102 | << sha256(canonical_request()); 103 | return ss.str(); 104 | } 105 | 106 | /*************** 107 | * S3Mirror * 108 | ***************/ 109 | 110 | S3Mirror::S3Mirror(const Context& ctx, 111 | const std::string& lbucket_url, 112 | const std::string& lregion, 113 | const std::string& aws_access_key, 114 | const std::string& aws_secret_key) 115 | : Mirror(S3Mirror::id(lbucket_url, lregion), ctx, lbucket_url) 116 | , bucket_url(lbucket_url) 117 | , region(lregion) 118 | , aws_access_key_id(aws_access_key) 119 | , aws_secret_access_key(aws_secret_key) 120 | { 121 | if (bucket_url.back() == '/') 122 | this->bucket_url = this->bucket_url.substr(0, this->bucket_url.size() - 1); 123 | } 124 | 125 | S3Mirror::~S3Mirror() = default; 126 | 127 | bool S3Mirror::authenticate(CURLHandle&, const std::string&) 128 | { 129 | return true; 130 | } 131 | 132 | std::string S3Mirror::format_url(Target* target) const 133 | { 134 | return fmt::format("{}/{}", bucket_url, target->target().path()); 135 | } 136 | 137 | bool S3Mirror::needs_preparation(Target*) const 138 | { 139 | return false; 140 | } 141 | 142 | bool S3Mirror::prepare(Target*) 143 | { 144 | return true; 145 | } 146 | 147 | std::string s3_calculate_signature(const std::chrono::system_clock::time_point& request_date, 148 | const std::string& secret, 149 | const std::string& region, 150 | const std::string& service, 151 | const std::string& string_to_sign) 152 | { 153 | std::string yyyymmdd = get_yyyymmdd(request_date); 154 | 155 | const std::string key1{ "AWS4" + secret }; 156 | 157 | unsigned int DateKeyLen = 0; 158 | unsigned char* DateKey = HMAC(EVP_sha256(), 159 | key1.c_str(), 160 | static_cast(key1.size()), 161 | reinterpret_cast(yyyymmdd.c_str()), 162 | yyyymmdd.size(), 163 | NULL, 164 | &DateKeyLen); 165 | 166 | unsigned int DateRegionKeyLen = 0; 167 | unsigned char* DateRegionKey = HMAC(EVP_sha256(), 168 | DateKey, 169 | DateKeyLen, 170 | reinterpret_cast(region.c_str()), 171 | region.size(), 172 | NULL, 173 | &DateRegionKeyLen); 174 | 175 | ; 176 | unsigned int DateRegionServiceKeyLen = 0; 177 | unsigned char* DateRegionServiceKey 178 | = HMAC(EVP_sha256(), 179 | DateRegionKey, 180 | DateRegionKeyLen, 181 | reinterpret_cast(service.c_str()), 182 | service.size(), 183 | NULL, 184 | &DateRegionServiceKeyLen); 185 | 186 | const std::string AWS4_REQUEST{ "aws4_request" }; 187 | unsigned int SigningKeyLen = 0; 188 | unsigned char* SigningKey 189 | = HMAC(EVP_sha256(), 190 | DateRegionServiceKey, 191 | DateRegionServiceKeyLen, 192 | reinterpret_cast(AWS4_REQUEST.c_str()), 193 | AWS4_REQUEST.size(), 194 | NULL, 195 | &SigningKeyLen); 196 | 197 | unsigned int SignatureLen = 0; 198 | unsigned char* Signature 199 | = HMAC(EVP_sha256(), 200 | SigningKey, 201 | SigningKeyLen, 202 | reinterpret_cast(string_to_sign.c_str()), 203 | string_to_sign.size(), 204 | NULL, 205 | &SignatureLen); 206 | 207 | return hex_string(Signature, SHA256_DIGEST_LENGTH); 208 | } 209 | 210 | std::vector S3Mirror::get_auth_headers(S3CanonicalRequest& request) const 211 | { 212 | std::vector headers; 213 | 214 | const std::string signature = s3_calculate_signature(request.date, 215 | aws_secret_access_key, 216 | region, 217 | "s3", 218 | request.string_to_sign(region, "s3")); 219 | 220 | std::stringstream authorization_header; 221 | authorization_header << "AWS4-HMAC-SHA256 Credential=" << aws_access_key_id << "/" 222 | << get_yyyymmdd(request.date) << "/" << region << "/s3/aws4_request, " 223 | << "SignedHeaders=" << request.get_signed_headers() << ", " 224 | << "Signature=" << signature; 225 | 226 | for (auto& [key, header] : request.headers) 227 | { 228 | // if (key == "x-amz-content-sha256") continue; 229 | headers.push_back(fmt::format("{}: {}", key, header)); 230 | } 231 | headers.push_back(fmt::format("Authorization: {}", authorization_header.str())); 232 | return headers; 233 | } 234 | 235 | std::vector S3Mirror::get_auth_headers(const std::string& path) const 236 | { 237 | URLHandler uh(fmt::format("{}/{}", bucket_url, path)); 238 | S3CanonicalRequest req_data("GET", uh); 239 | return get_auth_headers(req_data); 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(pybind11) 2 | pybind11_add_module(pypowerloader main.cpp) 3 | target_link_libraries(pypowerloader PRIVATE libpowerloader) 4 | -------------------------------------------------------------------------------- /src/python/main.cpp: -------------------------------------------------------------------------------- 1 | #include "powerloader/context.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | namespace py = pybind11; 10 | 11 | PYBIND11_MODULE(pypowerloader, m) 12 | { 13 | using namespace powerloader; 14 | 15 | m.def("hello_world", []() { std::cout << "Hello world!" << std::endl; }); 16 | 17 | py::class_(m, "Path") 18 | .def(py::init()) 19 | .def("__str__", [](fs::path& self) -> std::string { return self.string(); }) 20 | .def("__repr__", 21 | [](fs::path& self) -> std::string 22 | { return std::string("fs::path[") + self.u8string() + "]"; }); 23 | py::implicitly_convertible(); 24 | 25 | py::class_>(m, "DownloadTarget") 26 | .def(py::init()) 27 | .def_property("progress_callback", 28 | &DownloadTarget::progress_callback, 29 | &DownloadTarget::set_progress_callback); 30 | 31 | py::class_(m, "MirrorID").def(py::init()); 32 | 33 | py::class_>(m, "Mirror") 34 | .def(py::init()); 35 | 36 | py::class_(m, "MirrorMap") 37 | .def(py::init<>()) 38 | .def("get_mirrors", &mirror_map_type::get_mirrors) 39 | .def("add_unique_mirror", &mirror_map_type::get_mirrors) 40 | .def("as_dict", [](const mirror_map_type& value) { return value.as_map(); }); 41 | 42 | py::class_>(m, "Context") 43 | .def(py::init([] { return std::make_unique(); })) 44 | .def_readwrite("verbosity", &Context::verbosity) 45 | .def_readwrite("mirror_map", &Context::mirror_map); 46 | 47 | py::class_(m, "Downloader") 48 | .def(py::init()) 49 | .def("download", &Downloader::download) 50 | .def("add", &Downloader::add); 51 | } 52 | -------------------------------------------------------------------------------- /src/target.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_TARGET_HPP 2 | #define POWERLOADER_TARGET_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "compression.hpp" 17 | 18 | namespace powerloader 19 | { 20 | namespace fs = std::filesystem; 21 | 22 | class Target 23 | { 24 | public: 25 | /** Header callback for CURL handles. 26 | * It parses HTTP and FTP headers and try to find length of the content 27 | * (file size of the target). If the size is different then the expected 28 | * size, then the transfer is interrupted. 29 | * This callback is used only if the expected size is specified. 30 | */ 31 | static std::size_t header_callback(char* buffer, 32 | std::size_t size, 33 | std::size_t nitems, 34 | Target* self); 35 | static std::size_t write_callback(char* buffer, 36 | std::size_t size, 37 | std::size_t nitems, 38 | Target* self); 39 | 40 | Target(const Context& ctx, 41 | std::shared_ptr dl_target, 42 | mirror_set mirrors = {}); 43 | 44 | ~Target(); 45 | 46 | CbReturnCode call_end_callback(TransferStatus status); 47 | 48 | 49 | // Mark the target as failed and returns the value returned by the end callback 50 | CbReturnCode set_failed(DownloaderError error); 51 | 52 | // Completes transfer that finished successfully. 53 | void finalize_transfer(const std::string& effective_url); 54 | 55 | bool set_retrying(); 56 | 57 | // Changes the state of this target IFF the downloaded file already exists (however it was 58 | // produced). 59 | void check_if_already_finished(); 60 | 61 | // Forces max speed (provided by context) for already prepared Target. 62 | // Requirement: `prepare_for_transfer()` must have been called successfully before calling 63 | // this function. 64 | void set_to_max_speed(); 65 | 66 | void reset(); 67 | void reset_response(); 68 | 69 | void prepare_for_transfer(CURLM* multi_handle, 70 | const std::string& full_url, 71 | Protocol protocol); 72 | 73 | void flush_target_file(); 74 | 75 | tl::expected finish_transfer(const std::string& effective_url); 76 | 77 | void complete_mirror_usage(bool was_success, 78 | const tl::expected& result); 79 | 80 | bool can_retry_transfer_with_fewer_connections() const; 81 | void lower_mirror_parallel_connections(); 82 | 83 | void assert_target() const noexcept 84 | { 85 | if (!m_target) 86 | { 87 | spdlog::critical("No target set for target with mirror: {}::{}", 88 | target().mirror_name(), 89 | target().path()); 90 | } 91 | } 92 | 93 | const DownloadTarget& target() const noexcept 94 | { 95 | assert_target(); 96 | return *m_target; 97 | } 98 | 99 | // TODO: don't allow external code from manipulating this internal target. 100 | DownloadTarget& target() noexcept 101 | { 102 | assert_target(); 103 | return *m_target; 104 | } 105 | 106 | void assert_mirror() const noexcept 107 | { 108 | if (!m_mirror) 109 | { 110 | spdlog::critical("No mirror set for target with mirror: {}::{}", 111 | target().mirror_name(), 112 | target().path()); 113 | } 114 | } 115 | 116 | // TODO: don't expose ownership 117 | const std::shared_ptr& mirror() const noexcept 118 | { 119 | assert_mirror(); 120 | return m_mirror; 121 | } 122 | 123 | // TODO: don't expose ownership 124 | std::shared_ptr& mirror() noexcept 125 | { 126 | assert_mirror(); 127 | return m_mirror; 128 | } 129 | 130 | void change_mirror(std::shared_ptr mirror); 131 | 132 | const fs::path temp_file() const noexcept 133 | { 134 | return m_temp_file; 135 | } 136 | 137 | bool writecb_required_range_written() const noexcept 138 | { 139 | return m_writecb_required_range_written; 140 | } 141 | 142 | HeaderCbState headercb_state() const noexcept 143 | { 144 | return m_headercb_state; 145 | } 146 | 147 | const std::string& headercb_interrupt_reason() const noexcept 148 | { 149 | return m_headercb_interrupt_reason; 150 | } 151 | 152 | int range_fail() const noexcept 153 | { 154 | return m_range_fail; 155 | } 156 | 157 | void reset_range_fail() noexcept 158 | { 159 | m_range_fail = false; 160 | } 161 | 162 | std::size_t retries() const noexcept 163 | { 164 | return m_retries; 165 | } 166 | 167 | DownloadState state() const noexcept 168 | { 169 | return m_state; 170 | } 171 | 172 | bool failed() const noexcept 173 | { 174 | return m_state == DownloadState::kFAILED; 175 | } 176 | 177 | ZckState zck_state() const noexcept 178 | { 179 | return m_zck_state; 180 | } 181 | 182 | // TODO: refactor to avoid state being changed directly from outisde. 183 | void set_zck_state(ZckState new_state) noexcept 184 | { 185 | m_zck_state = new_state; 186 | } 187 | 188 | const auto& errorbuffer() const noexcept 189 | { 190 | return m_errorbuffer; 191 | } 192 | 193 | const auto& mirrors() const noexcept 194 | { 195 | return m_mirrors; 196 | } 197 | 198 | const auto& tried_mirrors() const noexcept 199 | { 200 | return m_tried_mirrors; 201 | } 202 | 203 | CURLHandle* curl_handle() const noexcept 204 | { 205 | return m_curl_handle.get(); 206 | } 207 | 208 | private: 209 | std::shared_ptr m_target; 210 | fs::path m_temp_file; 211 | std::string m_url_stub; 212 | 213 | bool m_resume = false; 214 | std::size_t m_resume_count = 0; 215 | std::ptrdiff_t m_original_offset; 216 | 217 | // internal stuff 218 | std::size_t m_retries = 0; 219 | 220 | DownloadState m_state = DownloadState::kWAITING; 221 | 222 | // mirror list (or should we have a failure callback) 223 | std::shared_ptr m_mirror; 224 | std::vector> m_mirrors; 225 | std::set> m_tried_mirrors; 226 | std::shared_ptr m_used_mirror; 227 | 228 | HeaderCbState m_headercb_state; 229 | std::string m_headercb_interrupt_reason; 230 | std::size_t m_writecb_received; 231 | bool m_writecb_required_range_written; 232 | 233 | char m_errorbuffer[CURL_ERROR_SIZE] = {}; 234 | 235 | std::unique_ptr m_curl_handle; 236 | Protocol m_protocol; 237 | 238 | Response m_response; 239 | 240 | bool m_range_fail = false; 241 | ZckState m_zck_state; 242 | 243 | const Context& m_ctx; 244 | 245 | bool zck_running() const; 246 | 247 | void reset_file(TransferStatus status); 248 | 249 | static int progress_callback(Target* ptr, 250 | curl_off_t total_to_download, 251 | curl_off_t now_downloaded, 252 | curl_off_t total_to_upload, 253 | curl_off_t now_uploaded); 254 | 255 | bool truncate_transfer_file(); 256 | 257 | void open_target_file(); 258 | 259 | bool check_filesize(); 260 | bool check_checksums(); 261 | 262 | friend std::size_t zckwritecb(char* buffer, size_t size, size_t nitems, Target* self); 263 | friend std::size_t zckheadercb(char* buffer, 264 | std::size_t size, 265 | std::size_t nitems, 266 | Target* self); 267 | 268 | #ifdef WITH_ZSTD 269 | std::unique_ptr m_zstd_stream; 270 | #endif 271 | }; 272 | } 273 | 274 | #endif 275 | -------------------------------------------------------------------------------- /src/uploader/multipart_upload.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void 4 | add_multipart_upload(CURLHandle& target, 5 | const std::vector& files, 6 | const std::map& extra_fields) 7 | { 8 | curl_mimepart* part; 9 | curl_mime* mime; 10 | mime = curl_mime_init(target.handle()) 11 | 12 | for (const auto& f : files) 13 | { 14 | part = curl_mime_addpart(mime); 15 | curl_mime_filedata(part, f.c_str()); 16 | curl_mime_name(part, "files"); 17 | } 18 | 19 | for (const auto& [k, v] : extra_fields) 20 | { 21 | part = curl_mime_addpart(mime); 22 | curl_mime_name(part, k.c_str()); 23 | curl_mime_data(part, v.c_str(), CURL_ZERO_TERMINATED); 24 | } 25 | 26 | curl_easy_setopt(target, CURLOPT_MIMEPOST, mime); 27 | } 28 | -------------------------------------------------------------------------------- /src/uploader/oci_upload.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | 9 | namespace powerloader 10 | { 11 | std::string format_upload_url(const std::string& mirror_url, 12 | const std::string& temp_upload_location, 13 | const std::string& digest) 14 | { 15 | std::string upload_url; 16 | if (contains(temp_upload_location, "://")) 17 | { 18 | upload_url = fmt::format("{}", temp_upload_location); 19 | } 20 | else 21 | { 22 | upload_url = fmt::format("{}{}", mirror_url, temp_upload_location); 23 | } 24 | if (contains(upload_url, "?")) 25 | { 26 | upload_url = fmt::format("{}&digest={}", upload_url, digest); 27 | } 28 | else 29 | { 30 | upload_url = fmt::format("{}?digest={}", upload_url, digest); 31 | } 32 | return upload_url; 33 | } 34 | 35 | std::string create_manifest(const OCILayer& config, const std::vector& layers) 36 | { 37 | std::stringstream ss; 38 | nlohmann::json j; 39 | j["schemaVersion"] = 2; 40 | 41 | j["config"] = config.to_json(); 42 | 43 | j["layers"] = nlohmann::json::array(); 44 | 45 | for (auto& layer : layers) 46 | { 47 | j["layers"].push_back(layer.to_json()); 48 | } 49 | 50 | return j.dump(4); 51 | } 52 | 53 | OCILayer OCILayer::from_file(const std::string& mime_type, 54 | const fs::path& file, 55 | const std::optional& annotations) 56 | { 57 | return OCILayer(mime_type, file, std::nullopt, annotations); 58 | } 59 | 60 | OCILayer OCILayer::from_string(const std::string& mime_type, 61 | const std::string& content, 62 | const std::optional& annotations) 63 | { 64 | return OCILayer(mime_type, std::nullopt, content, annotations); 65 | } 66 | 67 | OCILayer::OCILayer(const std::string& lmime_type, 68 | const std::optional& path, 69 | const std::optional& content, 70 | const std::optional& lannotations) 71 | : mime_type(lmime_type) 72 | , file(path) 73 | , contents(content) 74 | , annotations(lannotations) 75 | { 76 | if (file) 77 | { 78 | digest = fmt::format("sha256:{}", sha256sum(file.value())); 79 | size = fs::file_size(file.value()); 80 | } 81 | else 82 | { 83 | digest = fmt::format("sha256:{}", sha256(contents.value())); 84 | size = contents.value().size(); 85 | } 86 | } 87 | 88 | 89 | Response OCILayer::upload(const Context& ctx, 90 | const OCIMirror& mirror, 91 | const std::string& reference) const 92 | { 93 | std::string preupload_url = mirror.get_preupload_url(reference); 94 | auto response = CURLHandle(ctx, preupload_url) 95 | .setopt(CURLOPT_CUSTOMREQUEST, "POST") 96 | .add_headers(mirror.get_auth_headers(reference)) 97 | .perform(); 98 | 99 | if (!response.ok()) 100 | return response; 101 | 102 | std::string temp_upload_location = response.headers.at("location"); 103 | 104 | auto upload_url = format_upload_url(mirror.url(), temp_upload_location, digest); 105 | 106 | spdlog::info("Uploading digest {}", digest); 107 | spdlog::info("Upload url: {}", upload_url); 108 | 109 | CURLHandle chandle(ctx, upload_url); 110 | // for uploading we always use application/octet-stream. The proper mimetypes 111 | // are defined in the manifest 112 | chandle.setopt(CURLOPT_UPLOAD, 1L) 113 | .add_headers(mirror.get_auth_headers(reference)) 114 | .add_header(fmt::format("Content-Type: application/octet-stream", mime_type)); 115 | 116 | if (file) 117 | { 118 | std::ifstream ufile(file.value(), std::ios::in | std::ios::binary); 119 | chandle.upload(ufile); 120 | return chandle.perform(); 121 | } 122 | else 123 | { 124 | std::istringstream config_stream(contents.value()); 125 | chandle.upload(config_stream); 126 | return chandle.perform(); 127 | } 128 | } 129 | 130 | nlohmann::json OCILayer::to_json() const 131 | { 132 | auto json_layer = nlohmann::json::object(); 133 | json_layer["mediaType"] = mime_type; 134 | json_layer["size"] = size; 135 | json_layer["digest"] = digest; 136 | if (annotations) 137 | { 138 | json_layer["annotations"] = annotations.value(); 139 | } 140 | return json_layer; 141 | } 142 | 143 | Response oci_upload(const Context& ctx, 144 | OCIMirror& mirror, 145 | const std::string& reference, 146 | const std::string& tag, 147 | const std::vector& layers, 148 | const std::optional& config) 149 | { 150 | // default is a empty json object 151 | OCILayer default_config 152 | = OCILayer::from_string("application/vnd.unknown.config.v1+json", std::string("{}")); 153 | OCILayer oci_layer_config = config.value_or(default_config); 154 | 155 | CURLHandle auth_handle{ ctx }; 156 | if (mirror.need_auth() && mirror.prepare(reference, auth_handle)) 157 | { 158 | auto auth_res = auth_handle.perform(); 159 | if (!auth_res.ok()) 160 | { 161 | spdlog::error("Could not authenticate to OCI Registry"); 162 | return auth_res; 163 | } 164 | } 165 | 166 | for (auto& layer : layers) 167 | { 168 | auto upload_res = layer.upload(ctx, mirror, reference); 169 | 170 | if (!upload_res.ok()) 171 | return upload_res; 172 | } 173 | 174 | // Upload the config, too 175 | { 176 | auto upload_res = oci_layer_config.upload(ctx, mirror, reference); 177 | if (!upload_res.ok()) 178 | return upload_res; 179 | } 180 | 181 | // Now we need to upload the manifest for OCI servers 182 | std::string manifest_url = mirror.get_manifest_url(reference, tag); 183 | std::string manifest = create_manifest(oci_layer_config, layers); 184 | 185 | spdlog::info("Manifest: {}", manifest); 186 | std::istringstream manifest_stream(manifest); 187 | 188 | CURLHandle mhandle(ctx, manifest_url); 189 | mhandle.add_headers(mirror.get_auth_headers(reference)) 190 | .add_header("Content-Type: application/vnd.oci.image.manifest.v1+json") 191 | .upload(manifest_stream); 192 | 193 | Response result = mhandle.perform(); 194 | spdlog::info("Uploaded {} layers to {}:{}", layers.size(), mirror.get_repo(reference), tag); 195 | return result; 196 | } 197 | 198 | } 199 | -------------------------------------------------------------------------------- /src/uploader/s3_upload.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | namespace powerloader 7 | { 8 | Response s3_upload(const Context& ctx, 9 | S3Mirror& mirror, 10 | const std::string& path, 11 | const fs::path& file) 12 | { 13 | std::string digest = sha256sum(file); 14 | std::size_t fsize = fs::file_size(file); 15 | 16 | URLHandler uh(fmt::format("{}/{}", mirror.url(), path)); 17 | 18 | S3CanonicalRequest request("PUT", uh, digest); 19 | request.hashed_payload = digest; 20 | 21 | // this is enough to make a file completely public 22 | // request.headers["x-amz-acl"] = "public-read"; 23 | 24 | CURLHandle uploadrequest(ctx, uh.url()); 25 | 26 | std::ifstream ufile(file, std::ios::in | std::ios::binary); 27 | 28 | uploadrequest.setopt(CURLOPT_INFILESIZE_LARGE, fsize) 29 | .add_headers(mirror.get_auth_headers(request)) 30 | .upload(ufile); 31 | 32 | return uploadrequest.perform(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace powerloader 5 | { 6 | bool is_sig_interrupted() 7 | { 8 | return false; 9 | } 10 | 11 | bool starts_with(const std::string_view& str, const std::string_view& prefix) 12 | { 13 | return str.size() >= prefix.size() && 0 == str.compare(0, prefix.size(), prefix); 14 | } 15 | 16 | bool ends_with(const std::string_view& str, const std::string_view& suffix) 17 | { 18 | return str.size() >= suffix.size() 19 | && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); 20 | } 21 | 22 | std::string sha256(const std::string& str) noexcept 23 | { 24 | unsigned char hash[32]; 25 | 26 | EVP_MD_CTX* mdctx; 27 | mdctx = EVP_MD_CTX_create(); 28 | EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL); 29 | EVP_DigestUpdate(mdctx, str.c_str(), str.size()); 30 | EVP_DigestFinal_ex(mdctx, hash, nullptr); 31 | EVP_MD_CTX_destroy(mdctx); 32 | 33 | return hex_string(hash, 32); 34 | } 35 | 36 | std::string string_transform(const std::string_view& input, int (*functor)(int)) 37 | { 38 | std::string res(input); 39 | std::transform( 40 | res.begin(), res.end(), res.begin(), [&](unsigned char c) { return functor(c); }); 41 | return res; 42 | } 43 | 44 | std::string to_upper(const std::string_view& input) 45 | { 46 | return string_transform(input, std::toupper); 47 | } 48 | 49 | std::string to_lower(const std::string_view& input) 50 | { 51 | return string_transform(input, std::tolower); 52 | } 53 | 54 | bool contains(const std::string_view& str, const std::string_view& sub_str) 55 | { 56 | return str.find(sub_str) != std::string::npos; 57 | } 58 | 59 | std::string sha256sum(const fs::path& path) 60 | { 61 | unsigned char hash[32]; 62 | EVP_MD_CTX* mdctx; 63 | mdctx = EVP_MD_CTX_create(); 64 | EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL); 65 | 66 | std::ifstream infile(path, std::ios::binary); 67 | constexpr std::size_t BUFSIZE = 32768; 68 | std::vector buffer(BUFSIZE); 69 | 70 | while (infile) 71 | { 72 | infile.read(buffer.data(), BUFSIZE); 73 | size_t count = infile.gcount(); 74 | if (!count) 75 | break; 76 | EVP_DigestUpdate(mdctx, buffer.data(), count); 77 | } 78 | 79 | EVP_DigestFinal_ex(mdctx, hash, nullptr); 80 | EVP_MD_CTX_destroy(mdctx); 81 | 82 | return hex_string(hash, 32); 83 | } 84 | 85 | std::string md5sum(const fs::path& path) 86 | { 87 | unsigned char hash[16]; 88 | 89 | EVP_MD_CTX* mdctx; 90 | mdctx = EVP_MD_CTX_create(); 91 | EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); 92 | 93 | std::ifstream infile(path, std::ios::binary); 94 | constexpr std::size_t BUFSIZE = 32768; 95 | std::vector buffer(BUFSIZE); 96 | 97 | while (infile) 98 | { 99 | infile.read(buffer.data(), BUFSIZE); 100 | size_t count = infile.gcount(); 101 | if (!count) 102 | break; 103 | EVP_DigestUpdate(mdctx, buffer.data(), count); 104 | } 105 | 106 | EVP_DigestFinal_ex(mdctx, hash, nullptr); 107 | EVP_MD_CTX_destroy(mdctx); 108 | 109 | return hex_string(hash, 16); 110 | } 111 | 112 | std::pair parse_header(const std::string_view& header) 113 | { 114 | auto colon_idx = header.find(':'); 115 | if (colon_idx != std::string_view::npos) 116 | { 117 | std::string_view key, value; 118 | key = header.substr(0, colon_idx); 119 | colon_idx++; 120 | // remove spaces 121 | while (std::isspace(header[colon_idx])) 122 | { 123 | ++colon_idx; 124 | } 125 | 126 | // remove \r\n header ending 127 | value = header.substr(colon_idx, header.size() - colon_idx - 2); 128 | // http headers are case insensitive! 129 | std::string lkey = to_lower(key); 130 | 131 | return std::make_pair(lkey, std::string(value)); 132 | } 133 | return std::make_pair(std::string(), std::string(header)); 134 | } 135 | 136 | std::string get_env(const char* var) 137 | { 138 | const char* val = getenv(var); 139 | if (!val) 140 | { 141 | throw std::runtime_error(std::string("Could not find env var: ") + var); 142 | } 143 | return val; 144 | } 145 | 146 | std::string get_env(const char* var, const std::string& default_value) 147 | { 148 | const char* val = getenv(var); 149 | if (!val) 150 | { 151 | return default_value; 152 | } 153 | return val; 154 | } 155 | 156 | std::vector split(const std::string_view& input, 157 | const std::string_view& sep, 158 | std::size_t max_split) 159 | { 160 | std::vector result; 161 | std::size_t i = 0, j = 0, len = input.size(), n = sep.size(); 162 | 163 | while (i + n <= len) 164 | { 165 | if (input[i] == sep[0] && input.substr(i, n) == sep) 166 | { 167 | if (max_split-- <= 0) 168 | break; 169 | result.emplace_back(input.substr(j, i - j)); 170 | i = j = i + n; 171 | } 172 | else 173 | { 174 | i++; 175 | } 176 | } 177 | result.emplace_back(input.substr(j, len - j)); 178 | return result; 179 | } 180 | 181 | std::vector rsplit(const std::string_view& input, 182 | const std::string_view& sep, 183 | std::size_t max_split) 184 | { 185 | if (max_split == SIZE_MAX) 186 | return split(input, sep, max_split); 187 | 188 | std::vector result; 189 | 190 | std::ptrdiff_t i, j, len = static_cast(input.size()), 191 | n = static_cast(sep.size()); 192 | i = j = len; 193 | 194 | while (i >= n) 195 | { 196 | if (input[i - 1] == sep[n - 1] && input.substr(i - n, n) == sep) 197 | { 198 | if (max_split-- <= 0) 199 | { 200 | break; 201 | } 202 | result.emplace_back(input.substr(i, j - i)); 203 | i = j = i - n; 204 | } 205 | else 206 | { 207 | i--; 208 | } 209 | } 210 | result.emplace_back(input.substr(0, j)); 211 | std::reverse(result.begin(), result.end()); 212 | 213 | return result; 214 | } 215 | 216 | void replace_all(std::string& data, const std::string& search, const std::string& replace) 217 | { 218 | replace_all_impl(data, search, replace); 219 | } 220 | 221 | void replace_all(std::wstring& data, const std::wstring& search, const std::wstring& replace) 222 | { 223 | replace_all_impl(data, search, replace); 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /src/zck.hpp: -------------------------------------------------------------------------------- 1 | #ifndef POWERLOADER_ZCK_HPP 2 | #define POWERLOADER_ZCK_HPP 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include "target.hpp" 9 | 10 | extern "C" 11 | { 12 | #include 13 | #ifndef _WIN32 14 | #include 15 | #else 16 | #include 17 | #endif 18 | #include 19 | } 20 | 21 | namespace powerloader 22 | { 23 | struct zchunk_error : public std::runtime_error 24 | { 25 | zchunk_error(const std::string& what = "zchunk error") 26 | : std::runtime_error(what) 27 | { 28 | } 29 | }; 30 | 31 | struct zck_target 32 | { 33 | // Zchunk download context 34 | zckDL* zck_dl = nullptr; 35 | 36 | // Zchunk header size 37 | std::ptrdiff_t zck_header_size = -1; 38 | std::unique_ptr zck_header_checksum; 39 | 40 | fs::path zck_cache_file; 41 | 42 | // Total to download in zchunk file 43 | std::uint64_t total_to_download = 0; 44 | 45 | // Amount already downloaded in zchunk file 46 | std::uint64_t downloaded = 0; 47 | }; 48 | 49 | bool zck_read_lead(Target& target); 50 | 51 | zckCtx* zck_init_read(const DownloadTarget& target, int fd); 52 | zckCtx* zck_init_read(const Target& target); 53 | 54 | bool zck_valid_header(const DownloadTarget& target, int fd); 55 | bool zck_valid_header(const Target& target); 56 | 57 | bool check_zck(Target& target); 58 | 59 | bool zck_extract(const fs::path& source, const fs::path& dst, bool validate); 60 | 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append("./build/src/python/") 4 | 5 | import pypowerloader 6 | 7 | pypowerloader.hello_world() 8 | 9 | path = "linux-64/python-3.9.7-hb7a2778_1_cpython.tar.bz2" 10 | baseurl = "https://conda.anaconda.org/conda-forge" 11 | filename = "python3.9_test" 12 | downTarg = pypowerloader.DownloadTarget(path, baseurl, filename) 13 | 14 | 15 | def progress(total, done): 16 | print(f"Total {total}, done {done}") 17 | return 0 18 | 19 | 20 | downTarg.progress_callback = progress 21 | 22 | con = pypowerloader.Context() 23 | 24 | dl = pypowerloader.Downloader(con) 25 | dl.add(downTarg) 26 | 27 | # dl.download() 28 | mirror = pypowerloader.Mirror(con, baseurl) 29 | 30 | print("mirror_map1: " + str(con.mirror_map.as_dict())) 31 | con.mirror_map.reset({"conda-forge": [mirror], "test": []}) 32 | print("mirror_map2: " + str(con.mirror_map.as_dict())) 33 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(doctest) 2 | 3 | include_directories(${GTEST_INCLUDE_DIRS} SYSTEM) 4 | 5 | set(TEST_SRCS 6 | test_main.cpp 7 | test_s3.cpp 8 | test_fileio.cpp 9 | test_url.cpp 10 | test_compression.cpp 11 | test_utility.cpp 12 | ) 13 | 14 | add_executable(test_powerloader ${TEST_SRCS}) 15 | if (WIN32 AND BUILD_STATIC) 16 | set_target_properties(test_powerloader PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") 17 | endif() 18 | 19 | target_link_libraries( 20 | test_powerloader 21 | PRIVATE 22 | ${powerloader_dependency} 23 | doctest::doctest 24 | ) 25 | set_property(TARGET test_powerloader PROPERTY CXX_STANDARD 17) 26 | 27 | add_custom_target(test 28 | COMMAND test_powerloader 29 | DEPENDS test_powerloader 30 | WORKING_DIRECTORY $ 31 | ) 32 | -------------------------------------------------------------------------------- /test/conda_mock/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/test/conda_mock/__init__.py -------------------------------------------------------------------------------- /test/conda_mock/conda_mock.py: -------------------------------------------------------------------------------- 1 | from http.server import BaseHTTPRequestHandler, SimpleHTTPRequestHandler, HTTPServer 2 | import os, sys, time, re, json 3 | import hashlib, base64 4 | 5 | from .config import AUTH_USER, AUTH_PASS 6 | 7 | 8 | def file_path(path): 9 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), path) 10 | 11 | 12 | failure_count = 0 13 | prev_headers = [] 14 | BYTE_RANGE_RE = re.compile(r"bytes=(\d+)-(\d+)?$") 15 | 16 | 17 | def parse_byte_range(byte_range): 18 | """Returns the two numbers in 'bytes=123-456' or throws ValueError. 19 | 20 | The last number or both numbers may be None. 21 | """ 22 | if byte_range.strip() == "": 23 | return None, None 24 | 25 | m = BYTE_RANGE_RE.match(byte_range) 26 | if not m: 27 | raise ValueError("Invalid byte range %s" % byte_range) 28 | 29 | first, last = [x and int(x) for x in m.groups()] 30 | if last and last < first: 31 | raise ValueError("Invalid byte range %s" % byte_range) 32 | return first, last 33 | 34 | 35 | def conda_mock_handler(port, pkgs, err_type, username, pwd): 36 | class CondaMockHandler(BaseHTTPRequestHandler): 37 | _port, _pkgs, _err_type = port, pkgs, err_type 38 | _username, _pwd = username, pwd 39 | count_thresh = 3 40 | number_of_servers = 4 41 | 42 | def return_bad_request(self): 43 | self.send_response(400) 44 | self.end_headers() 45 | 46 | def return_not_found(self): 47 | self.send_response(404) 48 | self.end_headers() 49 | 50 | def return_server_error_counts(self): 51 | global failure_count 52 | failure_count += 1 53 | if failure_count < self.count_thresh: 54 | self.send_response(500) 55 | self.end_headers() 56 | else: 57 | failure_count = 0 58 | self.serve_static() 59 | 60 | def reset_failure_count(self): 61 | global failure_count 62 | failure_count = 0 63 | self.send_response(200) 64 | self.end_headers() 65 | 66 | def return_ok_with_message(self, message, content_type="text/html"): 67 | if content_type == "text/html": 68 | message = bytes(message, "utf8") 69 | self.send_response(200) 70 | self.send_header("Content-type", content_type) 71 | self.send_header("Content-Length", str(len(message))) 72 | self.end_headers() 73 | self.wfile.write(message) 74 | 75 | def parse_path(self, test_prefix="", keyword_expected=False): 76 | keyword, path = "", self.path[len(test_prefix) :] 77 | if keyword_expected: 78 | keyword, path = path.split("/", 1) 79 | # Strip arguments 80 | if "?" in path: 81 | path = path[: path.find("?")] 82 | if keyword_expected: 83 | return keyword, path 84 | return path 85 | 86 | def serve_harm_checksum(self): 87 | """Append two newlines to content of a file (from the static dir) with 88 | specified keyword in the filename. If the filename doesn't contain 89 | the keyword, content of the file is returnen unchanged.""" 90 | keyword, path = self.parse_path("", keyword_expected=True) 91 | return self.serve_file(path, harm_keyword=keyword) 92 | 93 | def serve_range_data(self, data, content_type): 94 | first, last = self.range 95 | print(f"serving {first} -> {last}") 96 | if first >= len(data): 97 | self.send_error(416, "Requested Range Not Satisfiable") 98 | return None 99 | 100 | self.send_response(206) 101 | self.send_header("Accept-Ranges", "bytes") 102 | if last is None or last >= len(data): 103 | last = len(data) - 1 104 | response_length = last - first + 1 105 | self.send_header("Content-type", content_type) 106 | self.send_header( 107 | "Content-Range", "bytes %s-%s/%s" % (first, last, len(data)) 108 | ) 109 | self.send_header("Content-Length", str(response_length)) 110 | # self.send_header('Last-Modified', self.date_time_string(fs.st_mtime)) 111 | self.end_headers() 112 | self.wfile.write(data[first : last + 1]) 113 | 114 | def clear_prev_headers(self): 115 | global prev_headers 116 | prev_headers = [] 117 | return self.return_ok_with_message("OK") 118 | 119 | def serve_prev_headers(self): 120 | if not prev_headers: 121 | self.return_ok_with_message( 122 | json.dumps(None).encode("utf-8"), "application/json" 123 | ) 124 | 125 | res = [] 126 | for el in prev_headers: 127 | d = {} 128 | for k in el.keys(): 129 | d[k] = el[k] 130 | res.append(d) 131 | self.return_ok_with_message( 132 | json.dumps(res).encode("utf-8"), "application/json" 133 | ) 134 | 135 | def serve_file(self, path, harm_keyword=None): 136 | global prev_headers 137 | prev_headers.append(self.headers) 138 | 139 | if "Range" not in self.headers: 140 | self.range = None 141 | else: 142 | try: 143 | self.range = parse_byte_range(self.headers["Range"]) 144 | except ValueError as e: 145 | self.send_error(400, "Invalid byte range") 146 | return None 147 | first, last = self.range 148 | 149 | if "static/" not in path: 150 | # Support changing only files from static directory 151 | return self.return_bad_request() 152 | 153 | path = path[path.find("static/") :] 154 | try: 155 | with open(file_path(path), "rb") as f: 156 | data = f.read() 157 | if harm_keyword is not None and harm_keyword in os.path.basename( 158 | file_path(path) 159 | ): 160 | data += b"\n\n" 161 | if self.range: 162 | return self.serve_range_data(data, "application/octet-stream") 163 | return self.return_ok_with_message(data, "application/octet-stream") 164 | except IOError: 165 | # File probably doesn't exist or we can't read it 166 | return self.return_not_found() 167 | 168 | def select_error(self, err_type): 169 | # possible errors = 404, boken, lazy 170 | if err_type == "404": 171 | return self.return_not_found() 172 | elif err_type == "broken": 173 | return self.serve_harm_checksum() 174 | elif err_type == "lazy": 175 | return self.return_server_error_counts() 176 | path = self.parse_path() 177 | return self.serve_file(path) 178 | 179 | def get_filename(self): 180 | filename = self.path.split("/")[-1] 181 | return filename 182 | 183 | def serve_static(self): 184 | if self.get_filename() in pkgs: 185 | return self.select_error(err_type) 186 | else: 187 | path = self.parse_path() 188 | return self.serve_file(path) 189 | 190 | def do_HEAD(self): 191 | self.send_response(200) 192 | self.send_header("Content-type", "text/html") 193 | self.end_headers() 194 | 195 | def do_AUTHHEAD(self): 196 | self.send_response(401) 197 | self.send_header("WWW-Authenticate", 'Basic realm="Test"') 198 | self.send_header("Content-type", "text/html") 199 | self.end_headers() 200 | 201 | def get_main(self): 202 | if self.path.startswith("/prev_headers"): 203 | return self.serve_prev_headers() 204 | if self.path.startswith("/clear_prev_headers"): 205 | return self.clear_prev_headers() 206 | 207 | if self.path.startswith("/broken_counts/static/"): 208 | return self.return_server_error_counts() 209 | 210 | if self.path.startswith("/reset_broken_count"): 211 | return self.reset_failure_count() 212 | 213 | if self.path.startswith("/harm_checksum/static/"): 214 | return self.serve_harm_checksum() 215 | 216 | return self.serve_static() 217 | 218 | def do_GET(self): 219 | """ 220 | Add specific hooks if needed 221 | :return: 222 | """ 223 | # "user:passwort" # os.environ["TESTPWD"] 224 | key = username + ":" + pwd 225 | 226 | if key == ":": 227 | # Workaround, because we don't support empty usernames and passwords 228 | self.get_main() 229 | else: 230 | key = base64.b64encode(bytes(key, "utf-8")).decode("ascii") 231 | """ Present frontpage with user authentication. """ 232 | auth_header = self.headers.get("Authorization", "") 233 | 234 | if not auth_header: 235 | self.do_AUTHHEAD() 236 | self.wfile.write(b"no auth header received") 237 | return True 238 | elif auth_header == "Basic " + key: 239 | # SimpleHTTPRequestHandler.do_GET(self) 240 | return self.get_main() 241 | else: 242 | self.do_AUTHHEAD() 243 | self.wfile.write(auth_header.encode("ascii")) 244 | self.wfile.write(b"not authenticated") 245 | return True 246 | 247 | return CondaMockHandler 248 | -------------------------------------------------------------------------------- /test/conda_mock/config.py: -------------------------------------------------------------------------------- 1 | AUTH_USER = "admin" 2 | AUTH_PASS = "secret" 3 | -------------------------------------------------------------------------------- /test/conda_mock/static/packages/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/test/conda_mock/static/packages/.gitignore -------------------------------------------------------------------------------- /test/conda_mock/static/zchunk/lorem.txt.x3.zck: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/test/conda_mock/static/zchunk/lorem.txt.x3.zck -------------------------------------------------------------------------------- /test/conda_mock/static/zchunk/lorem.txt.zck: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/test/conda_mock/static/zchunk/lorem.txt.zck -------------------------------------------------------------------------------- /test/fixtures.py: -------------------------------------------------------------------------------- 1 | import sys, socket, pytest, py, pathlib 2 | from urllib.request import urlopen 3 | import platform, datetime 4 | import shutil, subprocess 5 | import os, time, json 6 | import shutil 7 | 8 | from helpers import * 9 | 10 | 11 | def proj_root(cwd=os.getcwd()): 12 | proj_root = cwd 13 | if not Path(proj_root).exists(): 14 | print("POWERLOADER NOT FOUND!") 15 | return Path(proj_root) 16 | 17 | 18 | def get_powerloader_binary(proj_root=proj_root()): 19 | env_var = os.environ.get("POWERLOADER_EXE") 20 | if env_var: 21 | return env_var 22 | else: 23 | if platform.system() == "Windows": 24 | return Path(proj_root) / "build" / "powerloader.exe" 25 | else: 26 | return Path(proj_root) / "build" / "powerloader" 27 | 28 | 29 | @pytest.fixture 30 | def get_proj_root(cwd=os.getcwd()): 31 | return proj_root(cwd) 32 | 33 | 34 | @pytest.fixture 35 | def powerloader_binary(get_proj_root): 36 | return get_powerloader_binary(get_proj_root) 37 | 38 | 39 | def no_docker(): 40 | return shutil.which("docker") is None 41 | 42 | 43 | @pytest.fixture 44 | def file(get_proj_root, name="xtensor-0.24.0-hc021e02_0.tar.bz2"): 45 | file_map = {} 46 | file_map["name"] = name 47 | file_map["location"] = Path(get_proj_root) 48 | file_map["server"] = file_map["location"] / "server.py" 49 | file_map["url"] = ( 50 | "https://beta.mamba.pm/get/conda-forge/osx-arm64/" + file_map["name"] 51 | ) 52 | file_map["size"] = 185929 53 | file_map["test_path"] = file_map["location"] / Path("test") 54 | file_map["tmp_path"] = file_map["test_path"] / Path("tmp") 55 | file_map["output_path"] = file_map["tmp_path"] / file_map["name"] 56 | file_map["output_path_pdpart"] = file_map["tmp_path"] / Path( 57 | str(file_map["name"]) + ".pdpart" 58 | ) 59 | file_map["mirrors"] = file_map["test_path"] / Path("mirrors.yml") 60 | file_map["local_mirrors"] = file_map["test_path"] / Path("local_static_mirrors.yml") 61 | file_map["authentication"] = file_map["test_path"] / Path("passwd_format_one.yml") 62 | file_map["s3_server"] = "s3://powerloadertestbucket.s3.eu-central-1.amazonaws.com" 63 | file_map["s3_mock_server"] = "s3://127.0.0.1:9000" 64 | file_map["s3_yml_template"] = file_map["test_path"] / Path("s3template.yml") 65 | file_map["s3_bucketname"] = Path("testbucket") 66 | file_map["tmp_yml"] = file_map["tmp_path"] / Path("tmp.yml") 67 | file_map["xtensor_path"] = file_map["test_path"] / Path( 68 | "conda_mock/static/packages/xtensor-0.23.9-hc021e02_1.tar.bz2" 69 | ) 70 | file_map["oci_template"] = file_map["test_path"] / Path("ocitemplate.yml") 71 | file_map["oci_upload_location"] = "oci://ghcr.io" 72 | file_map["name_on_server"] = "artifact" 73 | file_map["tag"] = "1.0" 74 | file_map["username"] = "wolfv" 75 | 76 | try: 77 | os.mkdir(file_map["tmp_path"]) 78 | except OSError: 79 | print("Creation of the directory %s failed" % file_map["tmp_path"]) 80 | else: 81 | print("Successfully created the directory %s " % file_map["tmp_path"]) 82 | 83 | yield file_map 84 | shutil.rmtree(file_map["tmp_path"]) 85 | 86 | 87 | @pytest.fixture 88 | def checksums(): 89 | cksums = { 90 | "xtensor-0.24.0-hc021e02_0.tar.bz2": "e785d6770ea5e69275c920cb1a6385bf22876e83fe5183a011d53fe705b21980", 91 | "python-3.9.7-hb7a2778_1_cpython.tar.bz2": "6971e6721bbf774a152de720f055d8f9b51439742a09c134698a57a4ed7304ba", 92 | "xtensor-0.23.10-hd62202e_0.tar.bz2": "e47ed847659b646c20d4e3e6162ebc11a53ecfe565928bea4f6c7110333241d5", 93 | "xtensor-0.23.10-h4bd325d_0.tar.bz2": "6440497a44cc09fa43fd6606c2461e52fb3cb3f980e7fe949e332c6a468f024a", 94 | "xtensor-0.23.10-h2acdbc0_0.tar.bz2": "6cfa43e528c21cff3a73b30c48eb04d0332224bd51471a307eea05737c0488d9", 95 | "xtensor-0.23.10-hc021e02_0.tar.bz2": "c21c3cea6517c2f968548b82008a8f418a5d9f47a41ce1cb796574b5f1bdbb67", 96 | "xtensor-0.23.10-h940c156_0.tar.bz2": "cc6a113c98012ee9dbbf695a5ce0d8a4230de8342194766e1259d660d1859f6f", 97 | "xtensor-0.23.9-h4bd325d_1.tar.bz2": "419098106d6c5233f374ec383be6d673d24000c72cf62ca7c56916853b7bdf4f", 98 | "xtensor-0.23.9-hd62202e_1.tar.bz2": "58c515f6be3aa1cef8cd047068751cab92f7cc525e9d62672e009b104d06f9de", 99 | "xtensor-0.23.9-h2acdbc0_1.tar.bz2": "70f65c25f8a8c3879923cd01cffc32c603d7675e7657fa9ca265f5565b9203fd", 100 | "xtensor-0.23.9-hc021e02_1.tar.bz2": "404a2e4664a1cbf94f5f98deaf568b267b7474c4e1267deb367b8c758fe71ed2", 101 | "boa-0.8.1.tar.gz": "b824237d80155efd97b79469534d602637b40a2a27c4f71417d5e6977238ff74", 102 | "artifact": "c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4", 103 | "mock_artifact": "5b3513f580c8397212ff2c8f459c199efc0c90e4354a5f3533adf0a3fff3a530", 104 | } 105 | return cksums 106 | 107 | 108 | @pytest.fixture 109 | def mock_server_404(xprocess, checksums): 110 | port = 5001 111 | pkgs = get_pkgs(port, checksums) 112 | yield from mock_server(xprocess, "m1", port, pkgs, error_type="404") 113 | 114 | 115 | @pytest.fixture 116 | def mock_server_lazy(xprocess, checksums): 117 | port = 5002 118 | pkgs = get_pkgs(port, checksums) 119 | yield from mock_server(xprocess, "m2", port, pkgs, error_type="lazy") 120 | 121 | 122 | @pytest.fixture 123 | def mock_server_broken(xprocess, checksums): 124 | port = 5003 125 | pkgs = get_pkgs(port, checksums) 126 | yield from mock_server(xprocess, "m3", port, pkgs, error_type="broken") 127 | 128 | 129 | @pytest.fixture 130 | def mock_server_working(xprocess, checksums): 131 | port = 5004 132 | pkgs = {} 133 | yield from mock_server(xprocess, "m4", port, pkgs, error_type=None) 134 | 135 | 136 | @pytest.fixture 137 | def mock_server_password(xprocess, checksums): 138 | port = 5005 139 | pkgs = {} 140 | yield from mock_server( 141 | xprocess, "m5", port, pkgs, error_type=None, uname="user", pwd="secret" 142 | ) 143 | 144 | 145 | @pytest.fixture 146 | def mirrors_with_names(file): 147 | return add_names(file, target="mirrors") 148 | 149 | 150 | @pytest.fixture 151 | def sparse_mirrors_with_names(file): 152 | return add_names(file, target="local_mirrors") 153 | -------------------------------------------------------------------------------- /test/helpers.py: -------------------------------------------------------------------------------- 1 | import platform, glob, datetime, hashlib, subprocess 2 | import shutil, yaml, copy, math 3 | from xprocess import ProcessStarter 4 | from urllib.request import urlopen 5 | import sys, socket, pathlib 6 | from pathlib import Path 7 | import json, os 8 | import requests 9 | 10 | 11 | def mock_server(xprocess, name, port, pkgs, error_type, uname=None, pwd=None): 12 | curdir = pathlib.Path(__file__).parent 13 | print("Starting mock_server") 14 | authenticate = (uname is not None) and (pwd is not None) 15 | 16 | class Starter(ProcessStarter): 17 | 18 | pattern = "Server started!" 19 | terminate_on_interrupt = True 20 | 21 | args = [ 22 | sys.executable, 23 | "-u", 24 | curdir / "server.py", 25 | "-p", 26 | str(port), 27 | "-e", 28 | error_type, 29 | "--pkgs", 30 | pkgs, 31 | ] 32 | 33 | if authenticate: 34 | args.extend(["-u", uname, "--pwd", pwd]) 35 | 36 | def startup_check(self): 37 | s = socket.socket() 38 | address = "localhost" 39 | error = False 40 | try: 41 | s.connect((address, port)) 42 | except Exception as e: 43 | print( 44 | "something's wrong with %s:%d. Exception is %s" % (address, port, e) 45 | ) 46 | error = True 47 | finally: 48 | s.close() 49 | 50 | return not error 51 | 52 | logfile = xprocess.ensure(name, Starter) 53 | 54 | if authenticate: 55 | yield f"http://{uname}:{pwd}@localhost:{port}" 56 | else: 57 | yield f"http://localhost:{port}" 58 | 59 | xprocess.getinfo(name).terminate() 60 | 61 | 62 | def generate_random_file(path, size): 63 | with open(path, "wb") as fout: 64 | fout.write(os.urandom(size)) 65 | 66 | 67 | def get_pkgs(port, checksums, num_servers=3): 68 | files = list(checksums.keys()) 69 | section, increment = port % num_servers, len(files) / num_servers 70 | lb = math.floor(section * increment) 71 | ub = math.ceil((section + 1) * increment) 72 | lb = max(lb, 0) 73 | ub = min(ub, len(files) - 1) 74 | return set(files[lb:ub]) 75 | 76 | 77 | def yml_content(path): 78 | with open(path, "r") as stream: 79 | try: 80 | return yaml.safe_load(stream) 81 | except yaml.YAMLError as exc: 82 | print(exc) 83 | 84 | 85 | def add_names(file, target): 86 | yml_cont = yml_content(file[target]) 87 | names = [] 88 | for target in yml_cont["targets"]: 89 | names.append(Path(target.split("/")[-1])) 90 | content = copy.deepcopy(yml_cont) 91 | content["names"] = names 92 | return content 93 | 94 | 95 | def path_to_name(path): 96 | return Path(path).name 97 | 98 | 99 | def ifnone(var): 100 | return (var == None) or (var == "") 101 | 102 | 103 | def get_files(file): 104 | return glob.glob(str(file["tmp_path"]) + "/*") 105 | 106 | 107 | def remove_all(file): 108 | Path(file["output_path"]).unlink(missing_ok=True) 109 | Path(file["output_path_pdpart"]).unlink(missing_ok=True) 110 | 111 | for fle in get_files(file): 112 | (file["tmp_path"] / Path(fle)).unlink() 113 | 114 | 115 | def calculate_sha256(file): 116 | with open(file, "rb") as f: 117 | b = f.read() 118 | readable_hash = hashlib.sha256(b).hexdigest() 119 | return readable_hash 120 | 121 | 122 | def unique_filename(with_txt=False): 123 | if with_txt == False: 124 | return Path(str(platform.system()).lower().replace("_", "") + "test") 125 | else: 126 | return Path(str(platform.system()).lower().replace("_", "") + "test.txt") 127 | 128 | 129 | def generate_unique_file(file, with_txt=False): 130 | upload_path = str(file["tmp_path"] / unique_filename(with_txt)) 131 | with open(upload_path, "w+") as f: 132 | f.write("Content: " + str(datetime.datetime.now())) 133 | f.close() 134 | return upload_path 135 | 136 | 137 | def filter_broken(file_list, pdp): 138 | broken = [] 139 | for file in file_list: 140 | if file.endswith(pdp): 141 | broken.append(file) 142 | return broken 143 | 144 | 145 | def gha_credentials_exist(): 146 | user = not ( 147 | (os.environ.get("GHA_USER") is None) or (os.environ.get("GHA_USER") == "") 148 | ) 149 | pwd = not ((os.environ.get("GHA_PAT") is None) or (os.environ.get("GHA_PAT") == "")) 150 | return user and pwd 151 | 152 | 153 | def upload_s3_file(powerloader_binary, up_path, server, plain_http=False): 154 | command = [powerloader_binary, "upload", up_path] 155 | if plain_http != False: 156 | command.extend(["-k", "--plain-http"]) 157 | command.append("-m") 158 | command.append(server) 159 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 160 | out, err = proc.communicate() 161 | assert err == "".encode("utf-8") 162 | assert proc.returncode == 0 163 | 164 | 165 | def oci_path_resolver(file, tag=None, name_on_server=None, username=None): 166 | if tag == None: 167 | t = file["tag"] 168 | else: 169 | t = tag 170 | 171 | if name_on_server == None: 172 | nos = file["name_on_server"] 173 | else: 174 | nos = name_on_server 175 | 176 | if username == None: 177 | un = file["username"] 178 | else: 179 | un = username 180 | return t, nos, un 181 | 182 | 183 | def generate_s3_download_yml(file, server, filename): 184 | aws_template = yml_content(file["s3_yml_template"]) 185 | aws_template["targets"] = [ 186 | aws_template["targets"][0].replace("__filename__", filename) 187 | ] 188 | aws_template["mirrors"]["s3test"][0]["url"] = aws_template["mirrors"]["s3test"][0][ 189 | "url" 190 | ].replace("__server__", server) 191 | 192 | with open(str(file["tmp_yml"]), "w") as outfile: 193 | yaml.dump(aws_template, outfile, default_flow_style=False) 194 | 195 | 196 | def download_s3_file(powerloader_binary, file, plain_http=False): 197 | command = [powerloader_binary, "download", "-f", str(file["tmp_yml"])] 198 | if plain_http != False: 199 | command.extend(["-k", "--plain-http"]) 200 | command.extend(["-d", str(file["tmp_path"])]) 201 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 202 | out, err = proc.communicate() 203 | assert err == "".encode("utf-8") 204 | assert proc.returncode == 0 205 | 206 | 207 | def get_prev_headers(mock_server_working, n_headers=1): 208 | with urlopen(f"{mock_server_working}/prev_headers") as fi: 209 | x = json.loads(fi.read().decode("utf-8")) 210 | if not x: 211 | return x 212 | if n_headers == 1: 213 | return x[-1] 214 | else: 215 | return x[-n_headers:] 216 | 217 | 218 | def clear_prev_headers(mock_server_working): 219 | urlopen(f"{mock_server_working}/clear_prev_headers") 220 | 221 | 222 | def get_percentage(delta_size): 223 | dsize_list = delta_size.decode("utf-8").split(" ") 224 | of_idx = [i for i, val in enumerate(dsize_list) if val == "of"] 225 | portion_bytes = ( 226 | 100 * float(dsize_list[of_idx[0] - 1]) / float(dsize_list[of_idx[0] + 1]) 227 | ) 228 | portion_chunks = ( 229 | 100 * float(dsize_list[of_idx[1] - 1]) / float(dsize_list[of_idx[1] + 1]) 230 | ) 231 | return portion_bytes, portion_chunks, dsize_list[of_idx[1] + 1] 232 | 233 | 234 | def env_vars_absent(): 235 | user_absent = os.environ.get("GHA_USER") == None or os.environ.get("GHA_USER") == "" 236 | passwd_absent = os.environ.get("GHA_PAT") == None or os.environ.get("GHA_PAT") == "" 237 | 238 | return user_absent and passwd_absent 239 | -------------------------------------------------------------------------------- /test/local_static_mirrors.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | conda-forge: 3 | - http://localhost:5001 4 | - http://localhost:5002 5 | - http://localhost:5003 6 | - http://localhost:5004 7 | - http://user:secret@localhost:5005 8 | 9 | targets: 10 | - conda-forge:static/packages/xtensor-0.23.10-hd62202e_0.tar.bz2 11 | - conda-forge:static/packages/xtensor-0.23.10-h4bd325d_0.tar.bz2 12 | - conda-forge:static/packages/xtensor-0.23.10-h2acdbc0_0.tar.bz2 13 | - conda-forge:static/packages/xtensor-0.23.10-hc021e02_0.tar.bz2 14 | - conda-forge:static/packages/xtensor-0.23.10-h940c156_0.tar.bz2 15 | - conda-forge:static/packages/xtensor-0.23.9-h4bd325d_1.tar.bz2 16 | - conda-forge:static/packages/xtensor-0.23.9-hd62202e_1.tar.bz2 17 | - conda-forge:static/packages/xtensor-0.23.9-h2acdbc0_1.tar.bz2 18 | - conda-forge:static/packages/xtensor-0.23.9-hc021e02_1.tar.bz2 19 | - conda-forge:static/packages/python-3.9.7-hb7a2778_1_cpython.tar.bz2 20 | -------------------------------------------------------------------------------- /test/mirrors.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | conda-forge: 3 | - http://localhost:5004 4 | 5 | targets: 6 | - conda-forge:/static/packages/python-3.9.7-hb7a2778_1_cpython.tar.bz2 7 | - conda-forge:/static/packages/xtensor-0.23.10-hd62202e_0.tar.bz2 8 | - conda-forge:/static/packages/xtensor-0.23.10-h4bd325d_0.tar.bz2 9 | -------------------------------------------------------------------------------- /test/ocitemplate.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | ocitest: 3 | - oci://ghcr.io/__username__ 4 | 5 | targets: 6 | - ocitest:__filename__ 7 | -------------------------------------------------------------------------------- /test/passwd_format_one.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | conda-forge: 3 | - http://user:secret@localhost:5005 4 | 5 | targets: 6 | - conda-forge:static/packages/xtensor-0.23.10-hd62202e_0.tar.bz2 7 | - conda-forge:static/packages/xtensor-0.23.10-h4bd325d_0.tar.bz2 8 | - conda-forge:static/packages/xtensor-0.23.10-h2acdbc0_0.tar.bz2 9 | - conda-forge:static/packages/xtensor-0.23.10-hc021e02_0.tar.bz2 10 | - conda-forge:static/packages/xtensor-0.23.10-h940c156_0.tar.bz2 11 | - conda-forge:static/packages/xtensor-0.23.9-h4bd325d_1.tar.bz2 12 | - conda-forge:static/packages/xtensor-0.23.9-hd62202e_1.tar.bz2 13 | - conda-forge:static/packages/xtensor-0.23.9-h2acdbc0_1.tar.bz2 14 | - conda-forge:static/packages/xtensor-0.23.9-hc021e02_1.tar.bz2 15 | - conda-forge:static/packages/python-3.9.7-hb7a2778_1_cpython.tar.bz2 16 | -------------------------------------------------------------------------------- /test/remote_mirrors.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | conda-forge: 3 | - https://conda.anaconda.org/conda-forge 4 | - https://repo.mamba.pm/conda-forge 5 | 6 | targets: 7 | - conda-forge:linux-64/python-3.9.7-hb7a2778_1_cpython.tar.bz2 8 | - conda-forge:linux-aarch64/xtensor-0.23.10-hd62202e_0.tar.bz2 9 | - conda-forge:linux-64/xtensor-0.23.10-h4bd325d_0.tar.bz2 10 | - conda-forge:linux-ppc64le/xtensor-0.23.10-h2acdbc0_0.tar.bz2 11 | - conda-forge:osx-arm64/xtensor-0.23.10-hc021e02_0.tar.bz2 12 | - conda-forge:osx-64/xtensor-0.23.10-h940c156_0.tar.bz2 13 | - conda-forge:linux-64/xtensor-0.23.9-h4bd325d_1.tar.bz2 14 | - conda-forge:linux-aarch64/xtensor-0.23.9-hd62202e_1.tar.bz2 15 | - conda-forge:linux-ppc64le/xtensor-0.23.9-h2acdbc0_1.tar.bz2 16 | - conda-forge:osx-arm64/xtensor-0.23.9-hc021e02_1.tar.bz2 17 | - conda-forge:osx-arm64/xtensor-0.24.0-hc021e02_0.tar.bz2 18 | -------------------------------------------------------------------------------- /test/s3template.yml: -------------------------------------------------------------------------------- 1 | mirrors: 2 | s3test: 3 | - url: __server__ 4 | user: env:AWS_ACCESS_KEY_ID 5 | password: env:AWS_SECRET_ACCESS_KEY 6 | region: env:AWS_DEFAULT_REGION 7 | 8 | targets: 9 | - s3test:__filename__ 10 | -------------------------------------------------------------------------------- /test/server.py: -------------------------------------------------------------------------------- 1 | from http.server import HTTPServer 2 | from argparse import ArgumentParser 3 | 4 | # from optparse import OptionParser 5 | 6 | try: 7 | from conda_mock.conda_mock import conda_mock_handler 8 | except (ValueError, ImportError): 9 | from .conda_mock.conda_mock import conda_mock_handler 10 | 11 | 12 | def start_server(port, broken, err_type, username, pwd, host="127.0.0.1"): 13 | handler = conda_mock_handler(port, broken, err_type, username, pwd) 14 | print(f"Starting server with {port} on {host}") 15 | print(f"Missing packages: {broken}\n") 16 | print("Server started!") 17 | with HTTPServer((host, port), handler) as server: 18 | server.serve_forever() 19 | print("ended") 20 | 21 | 22 | if __name__ == "__main__": 23 | parser = ArgumentParser() 24 | 25 | parser.add_argument("-p", "--port", default=5555, type=int) 26 | parser.add_argument("-n", "--host", default="127.0.0.1") 27 | parser.add_argument("-e", "--error_type", default="404") 28 | parser.add_argument("-u", "--username", default="") 29 | parser.add_argument("--pwd", default="") 30 | parser.add_argument( 31 | "--pkgs", metavar="N", type=str, nargs="+", help="broken pkgs", default=[] 32 | ) 33 | 34 | args = parser.parse_args() 35 | start_server( 36 | args.port, set(args.pkgs), args.error_type, args.username, args.pwd, args.host 37 | ) 38 | -------------------------------------------------------------------------------- /test/test_compression.cpp: -------------------------------------------------------------------------------- 1 | #ifdef WITH_ZSTD 2 | #include 3 | 4 | #include "powerloader/downloader.hpp" 5 | #include "powerloader/mirrors/oci.hpp" 6 | #include "powerloader/mirrors/s3.hpp" 7 | 8 | using namespace powerloader; 9 | 10 | TEST_SUITE("mirror_id") 11 | { 12 | TEST_CASE("create_mirror_id") 13 | { 14 | // test that this does not segfault 15 | std::string arg = "test"; 16 | const auto new_id = Mirror::id(arg); 17 | CHECK_EQ(new_id.to_string(), "MirrorID "); 18 | 19 | std::string arg2 = "test2"; 20 | const auto new_id2 = OCIMirror::id(arg, arg2); 21 | CHECK_EQ(new_id2.to_string(), "MirrorID "); 22 | 23 | const auto new_id3 = S3Mirror::id(arg, arg2); 24 | CHECK_EQ(new_id3.to_string(), "MirrorID "); 25 | } 26 | } 27 | 28 | TEST_SUITE("compression") 29 | { 30 | TEST_CASE("download") 31 | { 32 | fs::path filename = fs::canonical("../testdata/f1.txt.zst"); 33 | if (fs::exists("out_zst.txt")) 34 | { 35 | fs::remove("out_zst.txt"); 36 | } 37 | 38 | Context ctx; 39 | std::string file_url = path_to_url(filename.string()); 40 | auto target = DownloadTarget::from_url(ctx, file_url, "out_zst.txt", ""); 41 | 42 | target->set_compression_type(CompressionType::ZSTD); 43 | target->add_checksum( 44 | { ChecksumType::kSHA256, 45 | "06fa557926742aad170074b1ce955014a4213e960e8f09f07fa23371100dd18e" }); 46 | 47 | Downloader downloader(ctx); 48 | downloader.add(target); 49 | downloader.download(); 50 | 51 | CHECK_FALSE(target->get_error().has_value()); 52 | CHECK(fs::exists("out_zst.txt")); 53 | 54 | std::ifstream ifs("out_zst.txt"); 55 | std::string content((std::istreambuf_iterator(ifs)), 56 | (std::istreambuf_iterator())); 57 | std::string beginning = "Lorem ipsum dolor sit amet, consetetur"; 58 | CHECK_EQ(content.substr(0, beginning.size()), beginning); 59 | 60 | std::ifstream orig_fs("../testdata/f1.txt"); 61 | 62 | std::string orig_content((std::istreambuf_iterator(orig_fs)), 63 | (std::istreambuf_iterator())); 64 | 65 | CHECK_EQ(content, orig_content); 66 | } 67 | } 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /test/test_fileio.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | using namespace powerloader; 7 | 8 | TEST_SUITE("fileio") 9 | { 10 | TEST_CASE("open") 11 | { 12 | std::error_code ec; 13 | FileIO f("test.txt", FileIO::write_update_binary, ec); 14 | 15 | CHECK_FALSE(ec); 16 | f.write("test", 1, 4); 17 | f.close(ec); 18 | CHECK_FALSE(ec); 19 | } 20 | 21 | TEST_CASE("truncate_empty") 22 | { 23 | std::error_code ec; 24 | FileIO f("empty.txt", FileIO::append_update_binary, ec); 25 | CHECK_FALSE(ec); 26 | CHECK(fs::exists("empty.txt")); 27 | f.truncate(0, ec); 28 | CHECK_FALSE(ec); 29 | } 30 | 31 | TEST_CASE("replace_from") 32 | { 33 | std::error_code ec; 34 | { 35 | std::ofstream f1("x1.txt"), f2("x2.txt"); 36 | f1 << "Hello world this is file number 1"; 37 | f2 << "File 2"; 38 | } 39 | { 40 | FileIO f1("x1.txt", FileIO::append_update_binary, ec); 41 | FileIO f2("x2.txt", FileIO::read_update_binary, ec); 42 | f1.replace_from(f2); 43 | 44 | f1.seek(0, SEEK_END); 45 | CHECK_EQ(f1.tell(), 6); 46 | } 47 | { 48 | std::ifstream f1("x1.txt"); 49 | std::stringstream buffer; 50 | buffer << f1.rdbuf(); 51 | CHECK_EQ(buffer.str(), "File 2"); 52 | } 53 | { 54 | std::ofstream f1("x1.txt", std::ios::trunc), f2("x2.txt", std::ios::trunc); 55 | f1 << "Hello world this is file number 1"; 56 | f2 << "File 2"; 57 | } 58 | { 59 | FileIO f1("x1.txt", FileIO::read_update_binary, ec); 60 | FileIO f2("x2.txt", FileIO::append_update_binary, ec); 61 | f2.replace_from(f1); 62 | f2.seek(0, SEEK_END); 63 | CHECK_EQ(f2.tell(), 33); 64 | } 65 | { 66 | std::ifstream f2("x2.txt"); 67 | std::stringstream buffer; 68 | buffer << f2.rdbuf(); 69 | CHECK_EQ(buffer.str(), "Hello world this is file number 1"); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/test_main.cpp: -------------------------------------------------------------------------------- 1 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN 2 | #include "doctest/doctest.h" 3 | -------------------------------------------------------------------------------- /test/test_oci_registry.py: -------------------------------------------------------------------------------- 1 | from fixtures import * 2 | from xprocess import ProcessStarter 3 | 4 | 5 | def get_oci_path(name_on_server, tag): 6 | newname = name_on_server + "-" + tag 7 | newpath = file["tmp_path"] / Path(newname) 8 | return newname, newpath 9 | 10 | 11 | def generate_oci_download_yml(name_on_server, tag, server, tmp_folder): 12 | download_name = f"{name_on_server}-{tag}" 13 | oci = { 14 | "mirrors": {"ocitest": [f"oci://{server.split('://')[1]}"]}, 15 | "targets": [f"ocitest:{download_name}"], 16 | } 17 | 18 | tmp_yaml = tmp_folder / Path("tmp.yml") 19 | with open(str(tmp_yaml), "w") as outfile: 20 | yaml.dump(oci, outfile, default_flow_style=False) 21 | return tmp_yaml 22 | 23 | 24 | def download_oci_file(tmp_yaml, tmp_folder, server, plain_http=False): 25 | server = f"oci://{server.split('://')[1]}" 26 | plb = get_powerloader_binary() 27 | command = [ 28 | plb, 29 | "download", 30 | "-f", 31 | str(tmp_yaml), 32 | "-d", 33 | str(tmp_folder), 34 | ] 35 | 36 | if plain_http != False: 37 | command.extend(["-k", "-v", "--plain-http"]) 38 | 39 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 40 | out, err = proc.communicate() 41 | 42 | print(out.decode("utf-8"), err.decode("utf-8")) 43 | err_ok = server.split("://")[1] in str(out.decode("utf-8")) 44 | assert (err == "".encode("utf-8")) or err_ok 45 | assert proc.returncode == 0 46 | 47 | 48 | def oci_check_present(uploc, srvname, tag, expect=True): 49 | if gha_credentials_exist(): 50 | # Github doesn't support `/v2/_catalog` yet 51 | # https://github.community/t/ghcr-io-docker-http-api/130121/3 52 | pass 53 | else: 54 | oci_file_presence(uploc, srvname, tag, expect) 55 | if expect == True: 56 | path = uploc.replace("oci://", "http://") 57 | path += "/v2/" + srvname + "/tags/list" 58 | tags = requests.get(path).json() 59 | assert tag in set(tags["tags"]) 60 | 61 | 62 | def oci_file_presence(uploc, srvname, tag, expect): 63 | path = uploc.replace("oci://", "http://") + "/v2/_catalog" 64 | repos = requests.get(path).json() 65 | print(repos) 66 | assert (srvname in set(repos["repositories"])) == expect 67 | 68 | 69 | def upload_oci(upload_path, tag, server, plain_http=False): 70 | server = f"oci://{server.split('://')[1]}" 71 | 72 | plb = get_powerloader_binary() 73 | srv_name = path_to_name(upload_path) 74 | command = [ 75 | plb, 76 | "upload", 77 | f"{upload_path}:{srv_name}:{tag}", 78 | "-m", 79 | server, 80 | ] 81 | print("Uploading ", upload_path, srv_name, tag) 82 | if plain_http != False: 83 | command.extend(["-k", "-v", "--plain-http"]) 84 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 85 | out, err = proc.communicate() 86 | 87 | oci_check_present(server, srv_name, tag, expect=True) 88 | 89 | assert "error" not in str(out.decode("utf-8")) 90 | err_ok = server.split("://")[1] in str(out.decode("utf-8")) 91 | assert (err == "".encode("utf-8")) or err_ok 92 | assert proc.returncode == 0 93 | return srv_name 94 | 95 | 96 | def mock_oci_registry_starter(xprocess, name, port): 97 | curdir = pathlib.Path(__file__).parent 98 | print("Starting mock_server") 99 | 100 | class Starter(ProcessStarter): 101 | 102 | pattern = "listening on" 103 | terminate_on_interrupt = True 104 | 105 | args = [ 106 | "docker", 107 | "run", 108 | "-p", 109 | f"{port}:5000", 110 | "--rm", 111 | f"--name={name}", 112 | "registry:2", 113 | ] 114 | 115 | def startup_check(self): 116 | s = socket.socket() 117 | error = False 118 | try: 119 | s.connect(("localhost", port)) 120 | except Exception as e: 121 | print( 122 | "something's wrong with %s:%d. Exception is %s" % (address, port, e) 123 | ) 124 | error = True 125 | finally: 126 | s.close() 127 | return not error 128 | 129 | logfile = xprocess.ensure(name, Starter) 130 | 131 | yield f"http://localhost:{port}" 132 | 133 | xprocess.getinfo(name).terminate() 134 | 135 | 136 | @pytest.fixture 137 | def mock_oci_registry(xprocess): 138 | yield from mock_oci_registry_starter(xprocess, "mock_oci_registry", 5123) 139 | 140 | 141 | @pytest.fixture 142 | def temp_txt_file(tmp_path): 143 | p = tmp_path / "testfile" 144 | p.write_text("Content: " + str(datetime.datetime.now())) 145 | 146 | return (p, calculate_sha256(p)) 147 | 148 | 149 | @pytest.fixture 150 | def clean_env(): 151 | pat = os.environ.get("GHA_PAT") 152 | user = os.environ.get("GHA_USER") 153 | 154 | if pat: 155 | del os.environ["GHA_PAT"] 156 | if user: 157 | del os.environ["GHA_USER"] 158 | 159 | yield 160 | 161 | if pat: 162 | os.environ["GHA_PAT"] = pat 163 | if user: 164 | os.environ["GHA_USER"] = user 165 | 166 | 167 | skip_no_docker = pytest.mark.skipif(no_docker(), reason="No docker installed") 168 | 169 | 170 | class TestOCImock: 171 | 172 | # Upload a file 173 | @skip_no_docker 174 | def test_upload(self, mock_oci_registry, temp_txt_file, clean_env): 175 | tag = "1.0" 176 | name_on_server = upload_oci( 177 | temp_txt_file[0], tag, mock_oci_registry, plain_http=True 178 | ) 179 | 180 | # Download a file that's always there... 181 | # def test_download_permanent( 182 | # self, file, powerloader_binary, checksums, mock_oci_registry 183 | # ): 184 | # tag, name_on_server, username = oci_path_resolver( 185 | # file, username="", name_on_server=file["name_on_mock_server"] 186 | # ) 187 | # Path(get_oci_path(file, name_on_server, tag)[1]).unlink(missing_ok=True) 188 | # newpath, tmp_yaml = generate_oci_download_yml( 189 | # file, tag, name_on_server, username, local=True 190 | # ) 191 | # download_oci_file(powerloader_binary, tmp_yaml, file, plain_http=True) 192 | # assert checksums[file["name_on_mock_server"]] == calculate_sha256(newpath) 193 | 194 | # Upload a file and download it again 195 | @skip_no_docker 196 | def test_upload_and_download( 197 | self, temp_txt_file, powerloader_binary, mock_oci_registry, clean_env 198 | ): 199 | # Upload 200 | temp_folder = temp_txt_file[0].parent 201 | tag = "1.123" 202 | name_on_server = upload_oci( 203 | temp_txt_file[0], tag, mock_oci_registry, plain_http=True 204 | ) 205 | temp_txt_file[0].unlink() 206 | 207 | # Download 208 | tmp_yaml = generate_oci_download_yml( 209 | name_on_server, tag, mock_oci_registry, temp_folder 210 | ) 211 | dl_folder = temp_folder / "dl" 212 | dl_folder.mkdir() 213 | 214 | print(dl_folder) 215 | 216 | download_oci_file(tmp_yaml, dl_folder, mock_oci_registry, plain_http=True) 217 | assert temp_txt_file[1] == calculate_sha256( 218 | dl_folder / f"{temp_txt_file[0].name}-{tag}" 219 | ) 220 | 221 | @pytest.mark.skipif(not os.environ.get("GHA_PAT"), reason="No GHA_PAT set") 222 | def test_upload_ghcr(self, file): 223 | upload_path = generate_unique_file(file) 224 | hash_before_upload = calculate_sha256(upload_path) 225 | tag = "12.24" 226 | name_on_server = upload_oci(upload_path, tag, "https://ghcr.io") 227 | 228 | # def test_download_permanent(self, file, checksums): 229 | # tag, name_on_server, username = oci_path_resolver(file) 230 | # Path(get_oci_path(file, name_on_server, tag)[1]).unlink(missing_ok=True) 231 | # newpath, tmp_yaml = generate_oci_download_yml( 232 | # file, tag, name_on_server, username 233 | # ) 234 | # download_oci_file(powerloader_binary, tmp_yaml, file) 235 | # assert checksums[file["name_on_server"]] == calculate_sha256(newpath) 236 | 237 | def set_username(self): 238 | username = "" 239 | if gha_credentials_exist(): 240 | username = os.environ.get("GHA_USER") 241 | else: 242 | username = "mamba-org" # GHA_PAT is only available on the main branch 243 | os.environ["GHA_USER"] = username # GHA_USER must also be set 244 | return username 245 | 246 | # Download a file that's always there... 247 | @pytest.mark.skipif(not os.environ.get("GHA_PAT"), reason="No GHA_PAT set") 248 | def test_upload_and_download_ghcr(self, temp_txt_file): 249 | username = self.set_username() 250 | temp_folder = temp_txt_file[0].parent 251 | tag = "24.21" 252 | name_on_server = upload_oci(temp_txt_file[0], tag, "https://ghcr.io") 253 | temp_txt_file[0].unlink() 254 | 255 | tmp_yaml = generate_oci_download_yml( 256 | name_on_server, 257 | tag, 258 | f"https://ghcr.io/{username}", 259 | temp_folder, 260 | ) 261 | dl_folder = temp_folder / "dl" 262 | dl_folder.mkdir() 263 | 264 | download_oci_file(tmp_yaml, dl_folder, f"https://ghcr.io/{username}") 265 | 266 | assert temp_txt_file[1] == calculate_sha256( 267 | dl_folder / f"{temp_txt_file[0].name}-{tag}" 268 | ) 269 | 270 | # TODO: Delete OCI from server 271 | # Need to figure out what the package id is 272 | # Delete: https://stackoverflow.com/questions/59103177/how-to-delete-remove-unlink-unversion-a-package-from-the-github-package-registry 273 | # https://github.com/actions/delete-package-versions 274 | -------------------------------------------------------------------------------- /test/test_s3.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | using namespace powerloader; 6 | 7 | TEST_SUITE("s3") 8 | { 9 | TEST_CASE("signdata") 10 | { 11 | const auto p0 = std::chrono::time_point{}; 12 | 13 | const auto s = s3_calculate_signature( 14 | p0, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "eu", "s3", "thisisateststring"); 15 | 16 | CHECK_EQ(s, "85ae731ab003e28b9d40bedf8f10967f43025942de2bae7dc99679c50a194457"); 17 | 18 | const auto s2 = s3_calculate_signature( 19 | p0, "wXalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "eu", "s3", "thisisateststring"); 20 | 21 | CHECK_NE(s, s2); 22 | // s3mirror_signdata.add_extra_headers(nullptr); 23 | } 24 | } 25 | // TODO: add tests actually usuing S3Mirror 26 | -------------------------------------------------------------------------------- /test/test_s3main_branch.py: -------------------------------------------------------------------------------- 1 | from fixtures import * 2 | 3 | 4 | class TestS3Server: 5 | @pytest.mark.skipif( 6 | ifnone("AWS_SECRET_ACCESS_KEY") 7 | or ifnone("AWS_ACCESS_KEY_ID") 8 | or ifnone("AWS_DEFAULT_REGION"), 9 | reason="Environment variable(s) not defined", 10 | ) 11 | def test_s3_server(self, file, powerloader_binary): 12 | remove_all(file) 13 | upload_path = generate_unique_file(file) 14 | hash_before_upload = calculate_sha256(upload_path) 15 | up_path = upload_path + ":" + path_to_name(upload_path) 16 | upload_s3_file( 17 | powerloader_binary, up_path, server=file["s3_server"], plain_http=False 18 | ) 19 | Path(upload_path).unlink() 20 | generate_s3_download_yml(file, file["s3_server"], path_to_name(upload_path)) 21 | download_s3_file(powerloader_binary, file) 22 | assert hash_before_upload == calculate_sha256(upload_path) 23 | -------------------------------------------------------------------------------- /test/test_s3mock.py: -------------------------------------------------------------------------------- 1 | from fixtures import * 2 | 3 | 4 | class TestS3Mock: 5 | @pytest.mark.skipif( 6 | ifnone("AWS_SECRET_ACCESS_KEY") 7 | or ifnone("AWS_ACCESS_KEY_ID") 8 | or ifnone("AWS_DEFAULT_REGION"), 9 | reason="Environment variable(s) not defined", 10 | ) 11 | def test_s3_mock(self, file, powerloader_binary): 12 | remove_all(file) 13 | upload_path = generate_unique_file(file) 14 | hash_before_upload = calculate_sha256(upload_path) 15 | up_path = ( 16 | upload_path 17 | + ":" 18 | + str(file["s3_bucketname"] / Path(path_to_name(upload_path))) 19 | ) 20 | upload_s3_file( 21 | powerloader_binary, up_path, server=file["s3_mock_server"], plain_http=True 22 | ) 23 | Path(upload_path).unlink() 24 | filename = str(file["s3_bucketname"]) + "/" + path_to_name(upload_path) 25 | generate_s3_download_yml(file, file["s3_mock_server"], filename) 26 | download_s3_file(powerloader_binary, file, plain_http=True) 27 | assert hash_before_upload == calculate_sha256(upload_path) 28 | 29 | @pytest.mark.skipif( 30 | ifnone("AWS_SECRET_ACCESS_KEY") 31 | or ifnone("AWS_ACCESS_KEY_ID") 32 | or ifnone("AWS_DEFAULT_REGION"), 33 | reason="Environment variable(s) not defined", 34 | ) 35 | def test_s3_mock_mod_txt(self, file, powerloader_binary): 36 | remove_all(file) 37 | upload_path = generate_unique_file(file, with_txt=True) 38 | hash_before_upload = calculate_sha256(upload_path) 39 | up_path = ( 40 | upload_path 41 | + ":" 42 | + str(file["s3_bucketname"] / Path(path_to_name(upload_path))) 43 | ) 44 | upload_s3_file( 45 | powerloader_binary, up_path, server=file["s3_mock_server"], plain_http=True 46 | ) 47 | Path(upload_path).unlink() 48 | filename = str(file["s3_bucketname"]) + "/" + path_to_name(upload_path) 49 | generate_s3_download_yml(file, file["s3_mock_server"], filename) 50 | download_s3_file(powerloader_binary, file, plain_http=True) 51 | hash_after_upload = calculate_sha256(upload_path) 52 | assert hash_before_upload == hash_after_upload 53 | 54 | @pytest.mark.skipif( 55 | ifnone("AWS_SECRET_ACCESS_KEY") 56 | or ifnone("AWS_ACCESS_KEY_ID") 57 | or ifnone("AWS_DEFAULT_REGION"), 58 | reason="Environment variable(s) not defined", 59 | ) 60 | def test_s3_mock_yml_mod_loc(self, file, powerloader_binary): 61 | remove_all(file) 62 | upload_path = generate_unique_file(file) 63 | hash_before_upload = calculate_sha256(upload_path) 64 | up_path = ( 65 | upload_path 66 | + ":" 67 | + str(file["s3_bucketname"] / Path(path_to_name(upload_path))) 68 | ) 69 | upload_s3_file( 70 | powerloader_binary, up_path, server=file["s3_mock_server"], plain_http=True 71 | ) 72 | Path(upload_path).unlink() 73 | server = file["s3_mock_server"] + "/" + str(file["s3_bucketname"]) 74 | generate_s3_download_yml(file, server, path_to_name(upload_path)) 75 | download_s3_file(powerloader_binary, file, plain_http=True) 76 | assert hash_before_upload == calculate_sha256(upload_path) 77 | 78 | @pytest.mark.skipif( 79 | ifnone("AWS_SECRET_ACCESS_KEY") 80 | or ifnone("AWS_ACCESS_KEY_ID") 81 | or ifnone("AWS_DEFAULT_REGION"), 82 | reason="Environment variable(s) not defined", 83 | ) 84 | def test_yml_s3_mock_mirror(self, file, checksums, powerloader_binary): 85 | remove_all(file) 86 | filename = str(file["s3_bucketname"]) + "/" + path_to_name(file["xtensor_path"]) 87 | generate_s3_download_yml(file, file["s3_mock_server"], filename) 88 | download_s3_file(powerloader_binary, file, plain_http=True) 89 | Path(file["tmp_yml"]).unlink() 90 | 91 | for fp in get_files(file): 92 | assert calculate_sha256(fp) == checksums[str(path_to_name(fp))] 93 | -------------------------------------------------------------------------------- /test/test_utility.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | TEST_SUITE("utility") 6 | { 7 | TEST_CASE("erase_duplicates") 8 | { 9 | auto values = std::vector{ "a", "a", "a", "a", "b", "b", "c", "d", "d", "d" }; 10 | const auto expected_values = std::vector{ "a", "b", "c", "d" }; 11 | 12 | auto new_end = powerloader::erase_duplicates(values); 13 | CHECK_EQ(new_end, values.end()); 14 | CHECK_EQ(values, expected_values); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /testdata/f1.txt.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mamba-org/powerloader/e2a45eb5a24f5e9e83e4e6d5305a0fa46f21e6d0/testdata/f1.txt.zst --------------------------------------------------------------------------------