├── .clang-format ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yaml │ ├── config.yml │ └── feature-request.yaml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── cmake.yml │ ├── project-checks.yml │ ├── release.yml │ └── release │ ├── Dockerfile │ └── build.sh ├── .gitignore ├── .gitmodules ├── CMake ├── Findaio.cmake ├── Findcurl.cmake ├── Finde2fs.cmake ├── Findopenssl.cmake ├── Findphoton.cmake ├── Findrapidjson.cmake ├── Findtcmu.cmake └── pack.cmake ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── MAINTAINERS ├── README.md ├── baselayers ├── CMakeLists.txt └── ext4_64.tar.gz ├── docs ├── .nojekyll ├── README.md ├── assets │ ├── Scaling_up.jpg │ ├── batch_code_startup_latency.png │ ├── cold_startup_latency.png │ ├── overlaybd_logo.svg │ ├── startup_latency_with_prefetch.png │ ├── time_launch_app.png │ ├── time_pull_image.png │ └── warm_startup_latency.png ├── cache.md ├── index.html ├── lsmt_lookup.md └── specs │ ├── lsmt.md │ └── zfile.md └── src ├── CMakeLists.txt ├── bk_download.cpp ├── bk_download.h ├── config.h ├── example_config ├── cred.json ├── overlaybd-registryv2.json ├── overlaybd-tcmu.service ├── overlaybd.json ├── redis.obd.config.json └── stream-conv.yaml ├── exporter_handler.h ├── exporter_server.h ├── image_file.cpp ├── image_file.h ├── image_service.cpp ├── image_service.h ├── main.cpp ├── metrics_fs.h ├── overlaybd ├── CMakeLists.txt ├── base64.h ├── cache │ ├── CMakeLists.txt │ ├── cache.cpp │ ├── cache.h │ ├── cached_fs.cpp │ ├── download_cache │ │ ├── CMakeLists.txt │ │ └── download_cache.cpp │ ├── forwardcfs.h │ ├── full_file_cache │ │ ├── CMakeLists.txt │ │ ├── cache_pool.cpp │ │ ├── cache_pool.h │ │ ├── cache_store.cpp │ │ └── cache_store.h │ ├── gzip_cache │ │ ├── CMakeLists.txt │ │ ├── cached_fs.cpp │ │ └── cached_fs.h │ ├── ocf_cache │ │ ├── CMakeLists.txt │ │ ├── ease_bindings │ │ │ ├── ctx.cpp │ │ │ ├── ctx.h │ │ │ ├── env │ │ │ │ ├── ocf_env.cpp │ │ │ │ ├── ocf_env.h │ │ │ │ ├── ocf_env_headers.h │ │ │ │ ├── ocf_env_list.h │ │ │ │ ├── utils_mpool.cpp │ │ │ │ └── utils_mpool.h │ │ │ ├── provider.cpp │ │ │ ├── provider.h │ │ │ ├── queue.cpp │ │ │ ├── queue.h │ │ │ ├── volume.cpp │ │ │ └── volume.h │ │ ├── include │ │ │ └── ocf │ │ ├── ocf_cache.cpp │ │ ├── ocf_namespace.cpp │ │ ├── ocf_namespace.h │ │ └── test │ │ │ ├── CMakeLists.txt │ │ │ ├── flags.conf │ │ │ └── ocf_perf_test.cpp │ ├── policy │ │ └── lru.h │ ├── pool_store.h │ ├── store.cpp │ └── test │ │ ├── CMakeLists.txt │ │ ├── cache_test.cpp │ │ └── random_generator.h ├── config_util.h ├── gzindex │ ├── CMakeLists.txt │ ├── gzfile.cpp │ ├── gzfile.h │ ├── gzfile_index.h │ ├── gzip_index_create.cpp │ └── test │ │ ├── CMakeLists.txt │ │ └── test.cpp ├── gzip │ ├── CMakeLists.txt │ ├── gz.cpp │ └── gz.h ├── lsmt │ ├── CMakeLists.txt │ ├── file.cpp │ ├── file.h │ ├── format_spec.md │ ├── index.cpp │ ├── index.h │ └── test │ │ ├── CMakeLists.txt │ │ ├── lsmt-filetest.h │ │ └── test.cpp ├── registryfs │ ├── CMakeLists.txt │ ├── registryfs.cpp │ ├── registryfs.h │ └── registryfs_v2.cpp ├── stream_convertor │ ├── CMakeLists.txt │ ├── config.h │ ├── config_utils.h │ └── stream_conv.cpp ├── tar │ ├── CMakeLists.txt │ ├── erofs │ │ ├── CMakeLists.txt │ │ ├── erofs_common.cpp │ │ ├── erofs_common.h │ │ ├── erofs_fs.cpp │ │ ├── erofs_fs.h │ │ ├── liberofs.cpp │ │ ├── liberofs.h │ │ └── test │ │ │ ├── CMakeLists.txt │ │ │ ├── erofs_simple.cpp │ │ │ ├── erofs_stress.cpp │ │ │ ├── erofs_stress_base.cpp │ │ │ └── erofs_stress_base.h │ ├── header.cpp │ ├── libtar.cpp │ ├── libtar.h │ ├── tar_file.cpp │ ├── tar_file.h │ ├── test │ │ ├── CMakeLists.txt │ │ └── test.cpp │ └── whiteout.cpp ├── zfile │ ├── CMakeLists.txt │ ├── README.md │ ├── compressor.cpp │ ├── compressor.h │ ├── crc32 │ │ ├── crc32c.cpp │ │ └── crc32c.h │ ├── format_spec.md │ ├── lz4 │ │ ├── lz4-qat.c │ │ ├── lz4-qat.h │ │ ├── lz4.c │ │ ├── lz4.h │ │ └── test.c │ ├── test │ │ ├── CMakeLists.txt │ │ └── test.cpp │ ├── thirdparty │ │ └── CMakeLists.txt │ ├── zfile.cpp │ └── zfile.h └── zstd │ ├── CMakeLists.txt │ ├── zstdfile.cpp │ └── zstdfile.h ├── prefetch.cpp ├── prefetch.h ├── switch_file.cpp ├── switch_file.h ├── test ├── CMakeLists.txt ├── image_service_test.cpp ├── simple_credsrv_test.cpp └── trace_test.cpp ├── textexporter.h ├── tools ├── CLI11.hpp ├── CMakeLists.txt ├── comm_func.cpp ├── comm_func.h ├── overlaybd-apply.cpp ├── overlaybd-commit.cpp ├── overlaybd-create.cpp ├── overlaybd-merge.cpp ├── overlaybd-zfile.cpp ├── sha256file.cpp ├── sha256file.h └── turboOCI-apply.cpp └── version.h /.clang-format: -------------------------------------------------------------------------------- 1 | Language: Cpp 2 | BasedOnStyle: LLVM 3 | IndentWidth: 4 4 | AccessModifierOffset: -4 5 | SortIncludes: false 6 | ColumnLimit: 100 7 | AllowShortFunctionsOnASingleLine: None 8 | AlwaysBreakTemplateDeclarations: true 9 | BreakStringLiterals: false 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yaml: -------------------------------------------------------------------------------- 1 | # Copyright The Overlaybd Authors 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Bug Report 16 | description: File a bug report 17 | labels: [bug] 18 | body: 19 | - type: markdown 20 | id: preface 21 | attributes: 22 | value: | 23 | Thank you for reporting bugs to Overlaybd! 24 | - type: textarea 25 | id: environment 26 | validations: 27 | required: true 28 | attributes: 29 | label: What happened in your environment? 30 | description: "Please also attach logs here." 31 | - type: textarea 32 | id: expect 33 | attributes: 34 | label: "What did you expect to happen?" 35 | - type: textarea 36 | id: reproduce 37 | validations: 38 | required: true 39 | attributes: 40 | label: "How can we reproduce it?" 41 | description: "Please tell us your environment information as minimally and precisely as possible." 42 | - type: textarea 43 | id: version 44 | validations: 45 | required: true 46 | attributes: 47 | label: What is the version of your Overlaybd? 48 | description: "You can find the released versions from https://github.com/containerd/overlaybd/releases." 49 | - type: input 50 | id: os 51 | validations: 52 | required: true 53 | attributes: 54 | label: What is your OS environment? 55 | description: "e.g. Ubuntu 22.04" 56 | - type: checkboxes 57 | id: idea 58 | attributes: 59 | label: "Are you willing to submit PRs to fix it?" 60 | description: "This is absolutely not required, but we are happy to guide you in the contribution process 61 | especially when you already have a good proposal or understanding of how to implement it. Join us at https://slack.cncf.io/ and choose #overlaybd channel." 62 | options: 63 | - label: Yes, I am willing to fix it. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | # Copyright The Overlaybd Authors 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | blank_issues_enabled: true 16 | contact_links: 17 | - name: Ask a question or request support in the community 18 | url: https://github.com/containerd/overlaybd/discussions/ 19 | about: Ask a question or request support for using Overlaybd -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yaml: -------------------------------------------------------------------------------- 1 | # Copyright The Overlaybd Authors 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Feature Request 16 | description: File a feature request 17 | labels: [enhancement] 18 | body: 19 | - type: markdown 20 | id: preface 21 | attributes: 22 | value: "Thank you for submitting new features for Overlaybd." 23 | - type: input 24 | id: version 25 | attributes: 26 | label: "What is the version of your Overlaybd" 27 | description: "You can find the released versions from https://github.com/containerd/overlaybd/releases." 28 | - type: textarea 29 | id: description 30 | attributes: 31 | label: "What would you like to be added?" 32 | validations: 33 | required: true 34 | - type: textarea 35 | id: solution 36 | attributes: 37 | label: "Why is this needed for Overlaybd?" 38 | description: "Please describe your user story or scenario." 39 | validations: 40 | required: true 41 | - type: checkboxes 42 | id: idea 43 | attributes: 44 | label: "Are you willing to submit PRs to contribute to this feature?" 45 | description: "This is absolutely not required, but we are happy to guide you in the contribution process 46 | especially when you already have a good proposal or understanding of how to implement it. Join us at https://slack.cncf.io/ and choose #overlaybd channel." 47 | options: 48 | - label: Yes, I am willing to implement it. -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **What this PR does / why we need it**: 2 | 3 | **Which issue(s) this PR fixes** *(optional, in `fixes #(, fixes #, ...)` format, will close the issue(s) when PR gets merged)*: 4 | Fixes # 5 | 6 | **Please check the following list**: 7 | - [ ] Does the affected code have corresponding tests, e.g. unit test, E2E test? 8 | - [ ] Does this change require a documentation update? 9 | - [ ] Does this introduce breaking changes that would require an announcement or bumping the major version? 10 | - [ ] Do all new files have an appropriate license header? 11 | 12 | 13 | -------------------------------------------------------------------------------- /.github/workflows/project-checks.yml: -------------------------------------------------------------------------------- 1 | name: Project Checks 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | 8 | jobs: 9 | # 10 | # Project checker 11 | # 12 | # based on https://github.com/containerd/project-checks/blob/main/action.yml 13 | project: 14 | name: Project Checks 15 | runs-on: ubuntu-22.04 16 | timeout-minutes: 5 17 | 18 | steps: 19 | - uses: actions/setup-go@v3 20 | with: 21 | go-version: 1.19 22 | 23 | - uses: actions/checkout@v3 24 | with: 25 | path: src/github.com/containerd/overlaybd 26 | fetch-depth: 100 27 | 28 | - name: set env 29 | shell: bash 30 | run: | 31 | echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV 32 | echo "${{ github.workspace }}/bin" >> $GITHUB_PATH 33 | 34 | - name: Install dependencies 35 | shell: bash 36 | env: 37 | GO111MODULE: on 38 | run: | 39 | echo "::group::🚧 Get dependencies" 40 | go install -v github.com/vbatts/git-validation@latest 41 | go install -v github.com/containerd/ltag@latest 42 | echo "::endgroup::" 43 | 44 | - name: DCO Checks 45 | shell: bash 46 | working-directory: src/github.com/containerd/overlaybd 47 | env: 48 | GITHUB_COMMIT_URL: ${{ github.event.pull_request.commits_url }} 49 | REPO_ACCESS_TOKEN: "" 50 | DCO_VERBOSITY: "-v" 51 | DCO_RANGE: "" 52 | run: | 53 | echo "::group::👮 DCO checks" 54 | set -x 55 | if [[ ! -z "${REPO_ACCESS_TOKEN}" ]]; then 56 | HEADERS=(-H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${REPO_ACCESS_TOKEN}") 57 | else 58 | HEADERS=(-H "Accept: application/vnd.github+json") 59 | fi 60 | if [ -z "${GITHUB_COMMIT_URL}" ]; then 61 | DCO_RANGE=$(jq -r '.after + "..HEAD"' ${GITHUB_EVENT_PATH}) 62 | else 63 | DCO_RANGE=$(curl "${HEADERS[@]}" ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha + "..HEAD"') 64 | fi 65 | git-validation ${DCO_VERBOSITY} ${DCO_RANGE} -run DCO,short-subject,dangling-whitespace 66 | echo "::endgroup::" 67 | 68 | - name: Validate file headers 69 | shell: bash 70 | working-directory: src/github.com/containerd/overlaybd 71 | run: | 72 | echo "::group::📚 File headers" 73 | ltag -t "script/validate/template" --excludes "vendor contrib" --check -v 74 | echo "::endgroup::" 75 | -------------------------------------------------------------------------------- /.github/workflows/release/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright The containerd Authors. 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG BUILD_IMAGE 16 | FROM ${BUILD_IMAGE} AS builder 17 | WORKDIR /src 18 | COPY . . 19 | ARG OS 20 | ARG RELEASE_VERSION 21 | ARG RELEASE_NO 22 | ARG COMMIT_ID 23 | RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${OS} ${RELEASE_VERSION} ${RELEASE_NO} ${COMMIT_ID} 24 | 25 | FROM scratch AS release 26 | COPY --from=builder /src/build/overlaybd-*.* / 27 | -------------------------------------------------------------------------------- /.github/workflows/release/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # Copyright The containerd Authors. 4 | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | OS=${1} 19 | PACKAGE_VERSION=${2} 20 | RELEASE_NO=${3} 21 | COMMIT_ID=${4} 22 | ARCH=`uname -m` 23 | BUILD_TYPE="Release" 24 | COMPILER="" 25 | PACKAGE_RELEASE="" 26 | CMAKE="cmake" 27 | CPACK="cpack" 28 | 29 | 30 | # Install Dependencies 31 | if [[ ${OS} =~ "ubuntu" ]]; then 32 | export DEBIAN_FRONTEND="noninteractive" 33 | export TZ="Etc/UTC" 34 | apt-get update -y 35 | apt-get install -y libgflags-dev libcurl4-openssl-dev libssl-dev libaio-dev libnl-3-dev libnl-genl-3-dev rpm wget make g++ git dpkg-dev sudo pkg-config 36 | apt-get install -y uuid-dev libjson-c-dev libkmod-dev libsystemd-dev autoconf automake libtool libpci-dev nasm libzstd-dev libext2fs-dev zlib1g-dev 37 | 38 | DISTRO=${OS/:/1~} 39 | PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}" 40 | elif [[ ${OS} =~ "centos" ]]; then 41 | if [[ ${OS} == "centos:7" ]]; then 42 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo 43 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo 44 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo 45 | yum clean all 46 | rm -rf /var/cache/yum 47 | yum -y update 48 | 49 | yum install -y centos-release-scl 50 | 51 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo 52 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo 53 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo 54 | yum clean all 55 | rm -rf /var/cache/yum 56 | yum -y update 57 | 58 | yum install -y devtoolset-7-gcc-c++ 59 | export PATH="/opt/rh/devtoolset-7/root/usr/bin:$PATH" 60 | PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.el7" 61 | COMPILER="-DCMAKE_C_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/gcc -DCMAKE_CXX_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/g++" 62 | /opt/rh/devtoolset-7/root/usr/bin/gcc --version 63 | /opt/rh/devtoolset-7/root/usr/bin/g++ --version 64 | elif [[ ${OS} == "centos:8" ]]; then 65 | rm -rf /etc/yum.repos.d/* && curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo 66 | 67 | yum install -y gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ 68 | COMPILER="-DCMAKE_C_COMPILER=/opt/rh/gcc-toolset-9/root/usr/bin/gcc -DCMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-9/root/usr/bin/g++" 69 | /opt/rh/gcc-toolset-9/root/usr/bin/gcc --version 70 | /opt/rh/gcc-toolset-9/root/usr/bin/g++ --version 71 | 72 | PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.el8" 73 | fi 74 | 75 | yum install -y epel-release libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel 76 | yum install -y rpm-build make git wget sudo autoconf automake libtool 77 | yum install --skip-broken -y libzstd-static gcc gcc-c++ binutils libzstd-devel 78 | elif [[ ${OS} =~ "mariner" ]]; then 79 | yum install -y libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel glibc-devel libzstd-devel binutils ca-certificates-microsoft build-essential 80 | yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool 81 | 82 | DISTRO=${OS/:/.} 83 | PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}" 84 | elif [[ ${OS} =~ "azurelinux" ]]; then 85 | tdnf update -y 86 | tdnf install -y libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel glibc-devel libzstd-devel binutils ca-certificates-microsoft build-essential 87 | tdnf install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool 88 | 89 | DISTRO=${OS/:/.} 90 | PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}" 91 | fi 92 | 93 | if [[ ${ARCH} == "x86_64" ]]; then 94 | wget --no-check-certificate https://cmake.org/files/v3.15/cmake-3.15.0-Linux-x86_64.tar.gz 95 | tar -zxf cmake-3.15.0-Linux-x86_64.tar.gz -C /usr/local/ 96 | export PATH="/usr/local/cmake-3.15.0-Linux-x86_64/bin:$PATH" 97 | else 98 | wget --no-check-certificate https://cmake.org/files/v3.20/cmake-3.20.6-linux-aarch64.tar.gz 99 | tar -zxf cmake-3.20.6-linux-aarch64.tar.gz -C /usr/local/ 100 | export PATH="/usr/local/cmake-3.20.6-linux-aarch64/bin:$PATH" 101 | fi 102 | 103 | # Build 104 | mkdir build 105 | cd build 106 | ${CMAKE} .. -DOBD_VER="overlaybd/${COMMIT_ID}" -DPACKAGE_VERSION=${PACKAGE_VERSION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER} 107 | make -j8 108 | ${CPACK} --verbose 109 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | external 2 | build 3 | releases 4 | .vscode 5 | .idea -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/overlaybd/cache/ocf_cache/ocf"] 2 | path = src/overlaybd/cache/ocf_cache/ocf 3 | url = https://github.com/Open-CAS/ocf 4 | branch = v21.6.4 5 | [submodule "src/overlaybd/zfile/thirdparty/DML"] 6 | path = src/overlaybd/zfile/thirdparty/DML 7 | url = https://github.com/intel/DML.git 8 | [submodule "src/overlaybd/zfile/thirdparty/isa-l"] 9 | path = src/overlaybd/zfile/thirdparty/isa-l 10 | url = https://github.com/intel/isa-l.git 11 | -------------------------------------------------------------------------------- /CMake/Findaio.cmake: -------------------------------------------------------------------------------- 1 | find_path(AIO_INCLUDE_DIR libaio.h) 2 | 3 | find_library(AIO_LIBRARIES aio) 4 | 5 | include(FindPackageHandleStandardArgs) 6 | find_package_handle_standard_args(aio DEFAULT_MSG AIO_LIBRARIES AIO_INCLUDE_DIR) 7 | 8 | mark_as_advanced(AIO_INCLUDE_DIR AIO_LIBRARIES) -------------------------------------------------------------------------------- /CMake/Findcurl.cmake: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | 3 | if(${BUILD_CURL_FROM_SOURCE}) 4 | message("Add and build standalone libcurl") 5 | include(FetchContent) 6 | FetchContent_Declare( 7 | curl_bundle 8 | GIT_REPOSITORY https://github.com/curl/curl.git 9 | GIT_TAG curl-7_42_1 10 | GIT_PROGRESS 1) 11 | 12 | FetchContent_GetProperties(curl_bundle) 13 | 14 | # In libcurl, CMakeLists build static lib is broken add build command via 15 | # make 16 | if(NOT TARGET libcurl_static_build) 17 | if (NOT curl_bundle_POPULATED) 18 | FetchContent_Populate(curl_bundle) 19 | endif() 20 | find_package(openssl) 21 | add_custom_command( 22 | OUTPUT ${curl_bundle_BINARY_DIR}/lib/libcurl.a 23 | WORKING_DIRECTORY ${curl_bundle_SOURCE_DIR} 24 | COMMAND 25 | export CC=${CMAKE_C_COMPILER} && 26 | export CXX=${CMAKE_CXX_COMPILER} && 27 | export LD=${CMAKE_LINKER} && 28 | export CFLAGS=-fPIC && 29 | export LIBS=-ldl && 30 | autoreconf -i && sh configure --with-ssl="${OPENSSL_ROOT_DIR}" 31 | --without-libssh2 --enable-static --enable-shared=no --enable-optimize 32 | --disable-manual --without-libidn 33 | --disable-ftp --disable-file --disable-ldap --disable-ldaps 34 | --disable-rtsp --disable-dict --disable-telnet --disable-tftp 35 | --disable-pop3 --disable-imap --disable-smb --disable-smtp 36 | --disable-gopher --without-nghttp2 --enable-http 37 | --with-pic=PIC 38 | --prefix="${curl_bundle_BINARY_DIR}" && make -j 8 && make install) 39 | add_custom_target(libcurl_static_build 40 | DEPENDS ${curl_bundle_BINARY_DIR}/lib/libcurl.a) 41 | add_dependencies(libcurl_static_build openssl102_static_build) 42 | make_directory(${curl_bundle_BINARY_DIR}/include) 43 | endif() 44 | 45 | set(CURL_FOUND yes) 46 | set(CURL_LIBRARY ${curl_bundle_BINARY_DIR}/lib/libcurl.a) 47 | set(CURL_THIRDPARTY_DEPS OpenSSL::SSL OpenSSL::Crypto z) 48 | set(CURL_LIBRARIES ${CURL_LIBRARY} ${CURL_THIRDPARTY_DEPS}) 49 | set(CURL_INCLUDE_DIR ${curl_bundle_BINARY_DIR}/include) 50 | set(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIR}) 51 | set(CURL_VERSION_STRING 7.42.1) 52 | 53 | # Use libcurl static lib instead of cmake defined shared lib 54 | if(NOT TARGET CURL::libcurl) 55 | add_library(CURL::libcurl UNKNOWN IMPORTED) 56 | endif() 57 | add_dependencies(CURL::libcurl libcurl_static_build) 58 | message("${CURL_LIBRARY}") 59 | set_target_properties( 60 | CURL::libcurl 61 | PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" 62 | IMPORTED_LOCATION "${CURL_LIBRARY}" 63 | INTERFACE_INCLUDE_DIRECTORIES "${CURL_INCLUDE_DIRS}" 64 | INTERFACE_LINK_LIBRARIES "${CURL_THIRDPARTY_DEPS}") 65 | else() 66 | include(${CMAKE_ROOT}/Modules/FindCURL.cmake) 67 | endif() 68 | -------------------------------------------------------------------------------- /CMake/Finde2fs.cmake: -------------------------------------------------------------------------------- 1 | if(NOT ORIGIN_EXT2FS) 2 | message("Add and build standalone libext2fs") 3 | include(FetchContent) 4 | FetchContent_Declare( 5 | e2fsprogs 6 | GIT_REPOSITORY https://github.com/data-accelerator/e2fsprogs.git 7 | GIT_TAG b4cf6c751196a12b1df9a269d8e0b516b99fe6a7 8 | ) 9 | FetchContent_GetProperties(e2fsprogs) 10 | 11 | if(NOT TARGET libext2fs_build) 12 | if (NOT e2fsprogs_POPULATED) 13 | FetchContent_Populate(e2fsprogs) 14 | endif() 15 | set(LIBEXT2FS_INSTALL_DIR ${e2fsprogs_SOURCE_DIR}/build/libext2fs CACHE STRING "") 16 | 17 | add_custom_command( 18 | OUTPUT ${LIBEXT2FS_INSTALL_DIR}/lib 19 | WORKING_DIRECTORY ${e2fsprogs_SOURCE_DIR} 20 | COMMAND chmod 755 build.sh && ./build.sh 21 | ) 22 | add_custom_target(libext2fs_build DEPENDS ${LIBEXT2FS_INSTALL_DIR}/lib) 23 | endif() 24 | 25 | set(E2FS_FOUND yes) 26 | set(E2FS_LIBRARY ${LIBEXT2FS_INSTALL_DIR}/lib/libext2fs.so) 27 | set(E2FS_LIBRARIES ${E2FS_LIBRARY}) 28 | set(E2FS_INCLUDE_DIR ${LIBEXT2FS_INSTALL_DIR}/include) 29 | set(E2FS_INCLUDE_DIRS ${E2FS_INCLUDE_DIR}) 30 | 31 | if(NOT TARGET libext2fs) 32 | add_library(libext2fs UNKNOWN IMPORTED) 33 | endif() 34 | add_dependencies(libext2fs libext2fs_build) 35 | 36 | else() 37 | find_path(E2FS_INCLUDE_DIRS ext2fs/ext2fs.h) 38 | find_library(E2FS_LIBRARIES ext2fs) 39 | endif() 40 | 41 | find_package_handle_standard_args(e2fs DEFAULT_MSG E2FS_LIBRARIES E2FS_INCLUDE_DIRS) 42 | 43 | mark_as_advanced(E2FS_INCLUDE_DIRS E2FS_LIBRARIES) 44 | -------------------------------------------------------------------------------- /CMake/Findopenssl.cmake: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | 3 | if(${BUILD_CURL_FROM_SOURCE}) 4 | message("Add and build standalone libopenssl") 5 | include(FetchContent) 6 | 7 | # make openssl into bundle 8 | FetchContent_Declare( 9 | openssl102 10 | GIT_REPOSITORY https://github.com/openssl/openssl.git 11 | GIT_TAG OpenSSL_1_0_2-stable 12 | GIT_PROGRESS 1) 13 | 14 | FetchContent_GetProperties(openssl102) 15 | 16 | if(NOT TARGET openssl102_static_build) 17 | if(NOT openssl102_POPULATED) 18 | FetchContent_Populate(openssl102) 19 | endif() 20 | add_custom_command( 21 | OUTPUT ${openssl102_BINARY_DIR}/lib/libssl.a 22 | WORKING_DIRECTORY ${openssl102_SOURCE_DIR} 23 | COMMAND 24 | sh config -fPIC no-unit-test no-shared 25 | --openssldir="${openssl102_BINARY_DIR}" 26 | --prefix="${openssl102_BINARY_DIR}" && make depend -j && make 27 | -j 8 && make install) 28 | add_custom_target(openssl102_static_build 29 | DEPENDS ${openssl102_BINARY_DIR}/lib/libssl.a) 30 | make_directory(${openssl102_BINARY_DIR}/include) 31 | endif() 32 | 33 | set(OPENSSL_FOUND yes) 34 | set(OPENSSL_ROOT_DIR ${openssl102_BINARY_DIR}) 35 | set(OPENSSL_INCLUDE_DIR ${OPENSSL_ROOT_DIR}/include) 36 | set(OPENSSL_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) 37 | set(OPENSSL_SSL_LIBRARY ${OPENSSL_ROOT_DIR}/lib/libssl.a) 38 | set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY}) 39 | set(OPENSSL_CRYPTO_LIBRARY ${OPENSSL_ROOT_DIR}/lib/libcrypto.a) 40 | set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY}) 41 | set(OPENSSL_LINK_DIR ${OPENSSL_ROOT_DIR}/lib) 42 | set(OPENSSL_LINK_DIRS ${OPENSSL_LINK_DIR}) 43 | 44 | if(NOT TARGET OpenSSL::SSL) 45 | add_library(OpenSSL::SSL STATIC IMPORTED) 46 | add_dependencies(OpenSSL::SSL openssl102_static_build) 47 | set_target_properties( 48 | OpenSSL::SSL 49 | PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" 50 | IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}" 51 | INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIRS}" 52 | INTERFACE_LINK_LIBRARIES "${OPENSSL_SSL_LIBRARY}") 53 | endif() 54 | 55 | if(NOT TARGET OpenSSL::Crypto) 56 | add_library(OpenSSL::Crypto STATIC IMPORTED) 57 | add_dependencies(OpenSSL::Crypto openssl102_static_build) 58 | set_target_properties( 59 | OpenSSL::Crypto 60 | PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" 61 | IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}" 62 | INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIRS}" 63 | INTERFACE_LINK_LIBRARIES "${OPENSSL_CRYPTO_LIBRARY}") 64 | endif() 65 | else() 66 | include(${CMAKE_ROOT}/Modules/FindOpenSSL.cmake) 67 | endif() 68 | -------------------------------------------------------------------------------- /CMake/Findphoton.cmake: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | set(FETCHCONTENT_QUIET false) 3 | set(PHOTON_ENABLE_EXTFS ON) 4 | 5 | FetchContent_Declare( 6 | photon 7 | GIT_REPOSITORY https://github.com/alibaba/PhotonLibOS.git 8 | GIT_TAG v0.6.17 9 | ) 10 | 11 | if(BUILD_TESTING) 12 | set(BUILD_TESTING 0) 13 | FetchContent_MakeAvailable(photon) 14 | set(BUILD_TESTING 1) 15 | else() 16 | FetchContent_MakeAvailable(photon) 17 | endif() 18 | 19 | if (BUILD_CURL_FROM_SOURCE) 20 | find_package(openssl REQUIRED) 21 | find_package(curl REQUIRED) 22 | add_dependencies(photon_obj CURL::libcurl OpenSSL::SSL OpenSSL::Crypto) 23 | endif() 24 | 25 | if(NOT ORIGIN_EXT2FS) 26 | add_dependencies(photon_obj libext2fs) 27 | endif() 28 | 29 | set(PHOTON_INCLUDE_DIR ${photon_SOURCE_DIR}/include/) 30 | -------------------------------------------------------------------------------- /CMake/Findrapidjson.cmake: -------------------------------------------------------------------------------- 1 | FetchContent_Declare( 2 | rapidjson 3 | GIT_REPOSITORY https://github.com/Tencent/rapidjson.git 4 | GIT_TAG 80b6d1c83402a5785c486603c5611923159d0894 5 | GIT_SUBMODULES "" 6 | ) 7 | FetchContent_GetProperties(rapidjson) 8 | if (NOT rapidjson_POPULATED) 9 | FetchContent_Populate(rapidjson) 10 | endif() 11 | 12 | add_definitions("-DRAPIDJSON_HAS_STDSTRING=1") -------------------------------------------------------------------------------- /CMake/Findtcmu.cmake: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | set(FETCHCONTENT_QUIET false) 3 | 4 | FetchContent_Declare( 5 | tcmu 6 | GIT_REPOSITORY https://github.com/data-accelerator/photon-libtcmu.git 7 | GIT_TAG 813fd65361bb2f348726b9c41478a44211847614 8 | ) 9 | 10 | if(BUILD_TESTING) 11 | set(BUILD_TESTING 0) 12 | FetchContent_MakeAvailable(tcmu) 13 | set(BUILD_TESTING 1) 14 | else() 15 | FetchContent_MakeAvailable(tcmu) 16 | endif() 17 | set(TCMU_INCLUDE_DIR ${tcmu_SOURCE_DIR}/) 18 | -------------------------------------------------------------------------------- /CMake/pack.cmake: -------------------------------------------------------------------------------- 1 | set(CPACK_PACKAGE_VERSION ${PACKAGE_VERSION}) 2 | set(CPACK_GENERATOR "RPM;DEB") 3 | set(CPACK_PACKAGE_NAME ${PROJECT_NAME}) 4 | if (PACKAGE_RELEASE) 5 | set(CPACK_PACKAGE_RELEASE ${PACKAGE_RELEASE}) 6 | else() 7 | set(CPACK_PACKAGE_RELEASE "1") 8 | endif() 9 | 10 | set(CPACK_PACKAGE_CONTACT "The overlaybd authors") 11 | set(CPACK_PACKAGE_VENDOR "Overlaybd") 12 | set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) 13 | set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}.${CMAKE_SYSTEM_PROCESSOR}") 14 | 15 | set(CPACK_RPM_PACKAGE_LICENSE "Apache v2.0") 16 | set(CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE}) 17 | set(CPACK_RPM_PACKAGE_SUMMARY "Overlaybd backtore for tcmu") 18 | set(CPACK_RPM_PACKAGE_DESCRIPTION ${CPACK_RPM_PACKAGE_SUMMARY}) 19 | 20 | set(CPACK_DEBIAN_PACKAGE_VERSION "${PACKAGE_VERSION}-${PACKAGE_RELEASE}") 21 | 22 | 23 | set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) 24 | 25 | include(CPack) 26 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14) 2 | 3 | project( 4 | overlaybd 5 | LANGUAGES C CXX 6 | ) 7 | enable_language(C) 8 | 9 | set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") 10 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -DNDEBUG -g") 11 | 12 | if (NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) AND NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) AND NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL arm64)) 13 | message(FATAL_ERROR "Unknown CPU architecture ${CMAKE_SYSTEM_PROCESSOR}") 14 | endif () 15 | 16 | option(OBD_VER "Overlaybd version" "overlaybd/0.0.0-undefined") 17 | 18 | set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") 19 | set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") 20 | 21 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake") 22 | 23 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fpic -Wall -Werror=sign-compare") 24 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare -DOVERLAYBD_VER=${OBD_VER}") 25 | 26 | if ((CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) OR (CMAKE_SYSTEM_PROCESSOR STREQUAL arm64)) 27 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc -fsigned-char -fno-stack-protector -fomit-frame-pointer") 28 | endif () 29 | 30 | set(CMAKE_CXX_STANDARD_LIBRARIES "-static-libgcc ${CMAKE_CXX_STANDARD_LIBRARIES}") 31 | find_library(STATIC_LIBSTDC++ libstdc++.a PATHS /usr/lib/gcc/*/*) 32 | if(NOT ${STATIC_LIBSTDC++} STREQUAL "STATIC_LIBSTDC++-NOTFOUND") 33 | set(CMAKE_CXX_STANDARD_LIBRARIES "-static-libstdc++ ${CMAKE_CXX_STANDARD_LIBRARIES}") 34 | endif() 35 | 36 | find_library(LIBZSTD libzstd.a PATHS /usr/lib/x86_64-linux-gnu/* /usr/lib64/*) 37 | if(${LIBZSTD} STREQUAL "LIBZSTD-NOTFOUND") 38 | message("libzstd.a not found, try to find shared library") 39 | find_library(LIBZSTD libzstd.so PATHS /usr/lib/x86_64-linux-gnu/* /usr/lib64/*) 40 | endif() 41 | 42 | set(CMAKE_CXX_STANDARD 14) 43 | set(CMAKE_CXX_STANDARD_REQUIRED on) 44 | set(ENABLE_MIMIC_VDSO off) 45 | 46 | option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off) 47 | option(ORIGIN_EXT2FS "Use original libext2fs" off) 48 | find_package(photon REQUIRED) 49 | find_package(tcmu REQUIRED) 50 | find_package(yaml-cpp) 51 | if (NOT yaml-cpp_FOUND) 52 | FetchContent_Declare( 53 | yaml-cpp 54 | GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git 55 | GIT_TAG 0.8.0 56 | ) 57 | FetchContent_MakeAvailable(yaml-cpp) 58 | endif() 59 | 60 | 61 | if(BUILD_TESTING) 62 | enable_testing() 63 | include(CTest) 64 | endif() 65 | 66 | add_subdirectory(src) 67 | 68 | add_subdirectory(baselayers) 69 | 70 | include(CMake/pack.cmake) 71 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions should be made via pull requests. 4 | Pull requests will be reviewed by one or more maintainers and merged when acceptable. 5 | Every contribution is very appreciated!!! 6 | 7 | ## Commit Style 8 | 9 | The seven rules of a great Git commit message: 10 | 11 | * Separate the subject from body with a blank line 12 | * Limit the subject line to 50 characters 13 | * Capitalize the subject line 14 | * Do not end the subject line with a period 15 | * Use the imperative mood in the subject line 16 | * Wrap the body at 72 characters 17 | * Use the body to explain what and why vs. how 18 | 19 | Read more on [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/). 20 | 21 | ## Sign your work 22 | 23 | The sign-off is a simple line at the end of the explanation for the patch. 24 | Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. 25 | The rules are pretty simple: if you can certify the below (from developercertificate.org): 26 | 27 | ``` 28 | Developer Certificate of Origin 29 | Version 1.1 30 | 31 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 32 | 660 York Street, Suite 102, 33 | San Francisco, CA 94110 USA 34 | 35 | Everyone is permitted to copy and distribute verbatim copies of this 36 | license document, but changing it is not allowed. 37 | 38 | Developer's Certificate of Origin 1.1 39 | 40 | By making a contribution to this project, I certify that: 41 | 42 | (a) The contribution was created in whole or in part by me and I 43 | have the right to submit it under the open source license 44 | indicated in the file; or 45 | 46 | (b) The contribution is based upon previous work that, to the best 47 | of my knowledge, is covered under an appropriate open source 48 | license and I have the right under that license to submit that 49 | work with modifications, whether created in whole or in part 50 | by me, under the same open source license (unless I am 51 | permitted to submit under a different license), as indicated 52 | in the file; or 53 | 54 | (c) The contribution was provided directly to me by some other 55 | person who certified (a), (b) or (c) and I have not modified 56 | it. 57 | 58 | (d) I understand and agree that this project and the contribution 59 | are public and that a record of the contribution (including all 60 | personal information I submit with it, including my sign-off) is 61 | maintained indefinitely and may be redistributed consistent with 62 | this project or the open source license(s) involved. 63 | ``` 64 | 65 | Then you just add a line to every git commit message: 66 | 67 | ``` 68 | Signed-off-by: Joe Smith 69 | ``` 70 | 71 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 72 | 73 | If you set your `user.name` and `user.email` git configs, you can sign your commit automatically with `git commit -s`. -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | # overlaybd maintainers 2 | # 3 | # As a containerd sub-project, containerd maintainers are also included from https://github.com/containerd/project/blob/main/MAINTAINERS. 4 | # See https://github.com/containerd/project/blob/main/GOVERNANCE.md for description of maintainer role 5 | # 6 | # COMMITTERS 7 | # GitHub ID, Name, Email address 8 | "liulanzheng", "Lanzheng Liu", "lanzheng.liulz@alibaba-inc.com" 9 | "bigvan", "Yifan Yuan", "tuji.yyf@alibaba-inc.com" 10 | "salvete", "Hongzhen Luo", "hongzhen@linux.alibaba.com" 11 | 12 | # REVIEWERS 13 | # GitHub ID, Name, Email address 14 | "beef9999", "Bob Chen", "beef9999@qq.com" 15 | "hhb584520", "Haibin Huang", "haibin.huang@intel.com" 16 | "hsiangkao", "Gao Xiang", "hsiangkao@linux.alibaba.com" 17 | -------------------------------------------------------------------------------- /baselayers/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_custom_command( 2 | OUTPUT ${EXECUTABLE_OUTPUT_PATH}/ext4_64 3 | COMMAND tar -zxvf ${CMAKE_CURRENT_LIST_DIR}/ext4_64.tar.gz -C ${EXECUTABLE_OUTPUT_PATH} 4 | VERBATIM 5 | ) 6 | 7 | add_custom_target(baselayer ALL DEPENDS ${EXECUTABLE_OUTPUT_PATH}/ext4_64) 8 | 9 | install(FILES ${EXECUTABLE_OUTPUT_PATH}/ext4_64 DESTINATION /opt/overlaybd/baselayers) -------------------------------------------------------------------------------- /baselayers/ext4_64.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/baselayers/ext4_64.tar.gz -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/.nojekyll -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Overlaybd 2 | 3 | 4 | 5 | Overlaybd (overlay block device) is a novel layering block-level image format, which is design for container, secure container and applicable to virtual machine. And it is an open-source implementation of paper [DADI: Block-Level Image Service for Agile and Elastic Application Deployment. USENIX ATC'20](https://www.usenix.org/conference/atc20/presentation/li-huiba). 6 | 7 | 8 | 9 | [Scaling up Without Slowing Down: Accelerating Pod Start Time. KubeCon+CloudNativeCon Europe 2024](https://youtu.be/RJ6Lt9bVNTw) 10 | 11 | Overlaybd was first proposed by Alibaba Cloud and widely used in Alibaba cloud services. It became a sub-project of containerd in 2021. 12 | 13 | As an image format, overlaybd has 2 core component: 14 | 15 | - Overlaybd is a layering image format, provideing a merged view of a sequence of block-based layers as a virtual block device. [SPEC](https://containerd.github.io/overlaybd/#/specs/lsmt.md) 16 | 17 | - Zfile is a compression file format which support seekalbe online decompression. [SPEC](https://containerd.github.io/overlaybd/#/specs/zfile.md) 18 | 19 | 20 | # Components 21 | 22 | ## Overlaybd service 23 | 24 | [GitHub](https://github.com/containerd/overlaybd) 25 | 26 | Sub-project of containerd, contains the storage service of overlaybd image format, provideing a merged view of a sequence of block-based layers as a virtual block device. 27 | Now this service contains an implementation of overlaybd based on [TCMU](https://www.kernel.org/doc/Documentation/target/tcmu-design.txt), and will provide an implementation based on [ublk](https://docs.kernel.org/block/ublk.html) in the future. 28 | 29 | This service is based on [PhotonLibOS](https://github.com/alibaba/PhotonLibOS), which is a high-efficiency LibOS framework. 30 | 31 | The LBA lookup algorithm employs a linearized B+ tree and AVX-512 to optimize performance, significantly accelerating search speed up to 10X. [Lookup Performance](https://github.com/containerd/overlaybd/blob/main/docs/lsmt_lookup.md) 32 | 33 | ## Accelerated container image 34 | 35 | [GitHub](https://github.com/containerd/accelerated-container-image) 36 | [Getting started](https://github.com/containerd/accelerated-container-image/blob/main/docs/QUICKSTART.md) 37 | 38 | Sub-project of containerd, which is a solution of remote container image by fetching image data on-demand without downloading and unpacking the whole image before the container starts. This repositry contains a containerd snapshotter and image conversion tools for overlaybd. 39 | 40 | ## P2P data distribution 41 | 42 | [GitHub](https://github.com/data-accelerator/dadi-p2proxy) 43 | 44 | Use p2p protocol to speed up HTTP file download for registry in large-scale clusters. 45 | 46 |
47 | 48 | # Key features 49 | 50 | ## High Performace 51 | 52 | Overlaybd is a block-device-based image format, which has much lower complexity than filesystem-based implementations. For example, cross-layer hardlink and non-copy commands like chown are very complex for filesystem-based image without copying up, but is natively supported by overlaybd. 53 | 54 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | ## High Reliability 69 | 70 | Overlaybd outputs virtual block devices through TCMU, which is a linux kernel module and widely supported in most operation systems. 71 | Overlaybd backstore can recover from failures or crashes, which is difficult for FUSE-based image formats. 72 | 73 | 74 | # Community 75 | 76 | For sync communication catch us in the #overlaybd slack channels on Cloud Native Computing Foundation's (CNCF) slack - cloud-native.slack.com. Everyone is welcome to join and chat. [Get Invite to CNCF slack.](https://communityinviter.com/apps/cloud-native/cncf) -------------------------------------------------------------------------------- /docs/assets/Scaling_up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/Scaling_up.jpg -------------------------------------------------------------------------------- /docs/assets/batch_code_startup_latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/batch_code_startup_latency.png -------------------------------------------------------------------------------- /docs/assets/cold_startup_latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/cold_startup_latency.png -------------------------------------------------------------------------------- /docs/assets/overlaybd_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /docs/assets/startup_latency_with_prefetch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/startup_latency_with_prefetch.png -------------------------------------------------------------------------------- /docs/assets/time_launch_app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/time_launch_app.png -------------------------------------------------------------------------------- /docs/assets/time_pull_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/time_pull_image.png -------------------------------------------------------------------------------- /docs/assets/warm_startup_latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containerd/overlaybd/222aa97867b314bb9f466ca69d4f70fa8dcb449b/docs/assets/warm_startup_latency.png -------------------------------------------------------------------------------- /docs/cache.md: -------------------------------------------------------------------------------- 1 | # Overlaybd File Cache 2 | 3 | Cache is used to accelerate remote file access. In overlaybd's own terminology, we name the remote files to `source` as well as the local cached files to `media`, and the word `cache` refers to the whole caching library that provides a POSIX compliant filesystem API. 4 | 5 | ## Operations 6 | 7 | Typically, a caching library should support these operations: 8 | 9 | * Read 10 | * Write 11 | * Query (Check if data is already cached in media) 12 | * Evict 13 | * Refill / Prefetch (Cache data in advance) 14 | * Flush 15 | * ... 16 | 17 | In the container image scenario, however, the layer blobs are immutable for the most of the time. So we may simplify our goal by trying to design a read-only file cache. Some of the operations will be no longer needed then. 18 | 19 | ## Cache implementations 20 | 21 | Currently there are two supported caches in overlaybd: 22 | 23 | * full file cache 24 | * ocf cache 25 | 26 | ### full file cache 27 | 28 | The full file cache implementation is based on a very simple idea: fill a duplicated file in disk and keep its size grow as long as we still can (not evicted). We managed to do this by leveraging many kernel features such as [`sparse files`](https://en.wikipedia.org/wiki/Sparse_file) and [`fiemap`](https://www.kernel.org/doc/html/latest/filesystems/fiemap.html). The first one is able to reduce cache usage, because containers would normally not require the whole image file content to start. The second one provides us a portable way to manage metadata (Query). 29 | 30 | ### ocf cache 31 | 32 | The ocf cache is built on [Intel Open CAS Framework](https://open-cas.github.io/). This open-source framework is a high performance block storage caching meta-library written in C. We have implemented a read-only filesystem on top of this block driver with modern C++, and reshaped it to a new lib. 33 | 34 | Ocf cache has solved many old issues that came along with full file cache, for instance, lacking good support to heterogeneous filesystems such as xfs and tmpfs, getting low performance if eviction happened, or the annoying bugs when src files is even larger than the entire cache media. Besides, the new flexible infrastructure makes it easier to adopt overlaybd's native coroutine-scheduling mechanism and perhaps some fresh I/O engines (io_uring) in the future, comparing to those heavy-weight caching systems. 35 | 36 | ## Cache configurations 37 | 38 | Edit `/etc/overlaybd/overlaybd.json`, add the following line. The default value of `cacheType` is "file". 39 | 40 | ``` 41 | "cacheType": "ocf", 42 | ``` 43 | 44 | ## TODO 45 | 46 | We are still working with the OCF community to improve ocf cache. Any contribution is welcome. -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Overlaybd 6 | 7 | 8 | 9 | 10 | 15 | 16 | 17 |
18 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /docs/lsmt_lookup.md: -------------------------------------------------------------------------------- 1 | # Lookup Algorithm in LSMT 2 | 3 | ## Description 4 | 5 | LBA lookup in LSMT can be abstracted as a segment search problem, searching within a sorted set of non-overlapping intervals. Previously, we used binary search via std::lower_bound. Now, we've adopted a linearized B+ tree combined with AVX-512, which better exploits CPU cache efficiency and delivers over a 10X speedup in lookup performance. Even in environments without AVX-512 support, using a loop optimized with bitmask still yields significant performance gains. 6 | 7 | 8 | ## Performance 9 | 10 | | segment count | b+tree + avx512 | b+tree + loop + bitmask | lower bound | 11 | |---------------|-----------------|---------------|-------------| 12 | | 1k | 220 M/s | 42.2 M/s | 18.3 M/s | 13 | | 10k | 160 M/s | 30.7 M/s | 12.8 M/s | 14 | | 100k | 108 M/s | 21.8 M/s | 8.6 M/s | 15 | | 1M | 57.4 M/s | 15.2 M/s | 5.6 M/s | -------------------------------------------------------------------------------- /docs/specs/lsmt.md: -------------------------------------------------------------------------------- 1 | ../../src/overlaybd/lsmt/format_spec.md -------------------------------------------------------------------------------- /docs/specs/zfile.md: -------------------------------------------------------------------------------- 1 | ../../src/overlaybd/zfile/format_spec.md -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(curl REQUIRED) 2 | find_package(openssl REQUIRED) 3 | find_package(aio REQUIRED) 4 | find_package(rapidjson REQUIRED) 5 | 6 | link_libraries(rt pthread resolv) 7 | 8 | add_subdirectory(overlaybd) 9 | 10 | add_library(overlaybd_image_lib 11 | image_file.cpp 12 | image_service.cpp 13 | switch_file.cpp 14 | bk_download.cpp 15 | prefetch.cpp 16 | tools/sha256file.cpp 17 | tools/comm_func.cpp 18 | ) 19 | target_include_directories(overlaybd_image_lib PUBLIC 20 | ${CURL_INCLUDE_DIRS} 21 | ${OPENSSL_INCLUDE_DIR} 22 | ${rapidjson_SOURCE_DIR}/include 23 | ${PHOTON_INCLUDE_DIR} 24 | ) 25 | 26 | target_link_libraries(overlaybd_image_lib 27 | photon_static 28 | overlaybd_lib 29 | ${CURL_LIBRARIES} 30 | ${OPENSSL_SSL_LIBRARY} 31 | ${OPENSSL_CRYPTO_LIBRARY} 32 | ${AIO_LIBRARIES} 33 | ) 34 | 35 | add_executable(overlaybd-tcmu 36 | main.cpp 37 | ) 38 | target_include_directories(overlaybd-tcmu PUBLIC 39 | ${TCMU_INCLUDE_DIR} 40 | ${CURL_INCLUDE_DIRS} 41 | ${OPENSSL_INCLUDE_DIR} 42 | ${rapidjson_SOURCE_DIR}/include 43 | ${PHOTON_INCLUDE_DIR} 44 | ) 45 | target_link_libraries(overlaybd-tcmu 46 | photon_static 47 | overlaybd_lib 48 | overlaybd_image_lib 49 | tcmu_static 50 | ${CURL_LIBRARIES} 51 | ${OPENSSL_SSL_LIBRARY} 52 | ${OPENSSL_CRYPTO_LIBRARY} 53 | ${AIO_LIBRARIES} 54 | ) 55 | 56 | install(TARGETS overlaybd-tcmu DESTINATION /opt/overlaybd/bin) 57 | install(FILES example_config/overlaybd-tcmu.service DESTINATION /opt/overlaybd/) 58 | install(FILES example_config/overlaybd.json DESTINATION /etc/overlaybd/) 59 | install(FILES example_config/cred.json DESTINATION /opt/overlaybd/) 60 | if (NOT ORIGIN_EXT2FS) 61 | install(DIRECTORY ${LIBEXT2FS_INSTALL_DIR}/lib DESTINATION /opt/overlaybd/ USE_SOURCE_PERMISSIONS) 62 | endif() 63 | 64 | add_subdirectory(tools) 65 | if (BUILD_TESTING) 66 | add_subdirectory(test) 67 | endif () 68 | -------------------------------------------------------------------------------- /src/bk_download.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | class ImageFile; 23 | class ISwitchFile; 24 | 25 | namespace BKDL { 26 | 27 | static std::string DOWNLOAD_TMP_NAME = ".download"; 28 | 29 | bool check_downloaded(const std::string &dir); 30 | 31 | class BkDownload { 32 | public: 33 | std::string dir; 34 | uint32_t try_cnt; 35 | 36 | bool download(); 37 | bool lock_file(); 38 | void unlock_file(); 39 | 40 | BkDownload() = delete; 41 | ~BkDownload() { 42 | unlock_file(); 43 | delete src_file; 44 | } 45 | BkDownload(ISwitchFile *sw_file, photon::fs::IFile *src_file, size_t file_size, 46 | const std::string &dir, const std::string &digest, const std::string &url, 47 | int &running, int32_t limit_MB_ps, int32_t try_cnt, uint32_t bs) 48 | : dir(dir), try_cnt(try_cnt), sw_file(sw_file), src_file(src_file), 49 | file_size(file_size), digest(digest), url(url), running(running), 50 | limit_MB_ps(limit_MB_ps), block_size(bs) { 51 | } 52 | 53 | private: 54 | void switch_to_local_file(); 55 | bool download_blob(); 56 | bool download_done(); 57 | 58 | ISwitchFile *sw_file = nullptr; 59 | photon::fs::IFile *src_file = nullptr; 60 | size_t file_size; 61 | std::string digest; 62 | std::string url; 63 | int &running; 64 | int32_t limit_MB_ps; 65 | uint32_t block_size; 66 | bool force_download = false; 67 | }; 68 | 69 | void bk_download_proc(std::list &, uint64_t, int &); 70 | 71 | } // namespace BKDL -------------------------------------------------------------------------------- /src/example_config/cred.json: -------------------------------------------------------------------------------- 1 | { 2 | "auths": { 3 | "hub.docker.com": { 4 | "username": "username", 5 | "password": "password" 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /src/example_config/overlaybd-registryv2.json: -------------------------------------------------------------------------------- 1 | { 2 | "logConfig": { 3 | "logLevel": 1, 4 | "logPath": "/var/log/overlaybd.log" 5 | }, 6 | "cacheConfig": { 7 | "cacheType": "file", 8 | "cacheDir": "/opt/overlaybd/registry_cache", 9 | "cacheSizeGB": 4 10 | }, 11 | "credentialConfig": { 12 | "mode": "file", 13 | "path": "/opt/overlaybd/cred.json" 14 | }, 15 | "ioEngine": 0, 16 | "download": { 17 | "enable": true, 18 | "delay": 600, 19 | "delayExtra": 30, 20 | "maxMBps": 100 21 | }, 22 | "p2pConfig": { 23 | "enable": false, 24 | "address": "localhost:19145/dadip2p" 25 | }, 26 | "enableAudit": true, 27 | "auditPath": "/var/log/overlaybd-audit.log", 28 | "registryFsVersion": "v2" 29 | } 30 | -------------------------------------------------------------------------------- /src/example_config/overlaybd-tcmu.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=overlaybd-tcmu service 3 | After=network.target local-fs.target 4 | Before=shutdown.target 5 | DefaultDependencies=no 6 | Conflicts=shutdown.target 7 | 8 | [Service] 9 | LimitNOFILE=1048576 10 | LimitCORE=infinity 11 | Type=simple 12 | 13 | ExecStartPre=/sbin/modprobe target_core_user 14 | 15 | ExecStart=/opt/overlaybd/bin/overlaybd-tcmu 16 | 17 | GuessMainPID=no 18 | Restart=always 19 | RestartSec=1s 20 | KillMode=process 21 | OOMScoreAdjust=-999 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /src/example_config/overlaybd.json: -------------------------------------------------------------------------------- 1 | { 2 | "logConfig": { 3 | "logLevel": 1, 4 | "logPath": "/var/log/overlaybd.log" 5 | }, 6 | "cacheConfig": { 7 | "cacheType": "file", 8 | "cacheDir": "/opt/overlaybd/registry_cache", 9 | "cacheSizeGB": 4 10 | }, 11 | "gzipCacheConfig": { 12 | "enable": true, 13 | "cacheDir": "/opt/overlaybd/gzip_cache", 14 | "cacheSizeGB": 4 15 | }, 16 | "credentialConfig": { 17 | "mode": "file", 18 | "path": "/opt/overlaybd/cred.json" 19 | }, 20 | "ioEngine": 0, 21 | "download": { 22 | "enable": true, 23 | "delay": 600, 24 | "delayExtra": 30, 25 | "maxMBps": 100 26 | }, 27 | "p2pConfig": { 28 | "enable": false, 29 | "address": "localhost:19145/dadip2p" 30 | }, 31 | "exporterConfig": { 32 | "enable": false, 33 | "uriPrefix": "/metrics", 34 | "port": 9863, 35 | "updateInterval": 60000000 36 | }, 37 | "enableAudit": true, 38 | "auditPath": "/var/log/overlaybd-audit.log", 39 | "registryFsVersion": "v2" 40 | } 41 | -------------------------------------------------------------------------------- /src/example_config/redis.obd.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "repoBlobUrl": "https://registry-1.docker.io/v2/overlaybd/redis/blobs", 3 | "lowers": [ 4 | { 5 | "digest": "sha256:a8b5fca80efae55088290f3da8110d7742de55c2a378d5ab53226a483f390e21", 6 | "size": 4739584, 7 | "dir": "/var/lib/overlaybd/test/1" 8 | }, 9 | { 10 | "digest": "sha256:87763befd4f3289905d709bd03c969db43e512502be7e1132b625bdef487d01f", 11 | "size": 43458048, 12 | "dir": "/var/lib/overlaybd/test/2" 13 | }, 14 | { 15 | "digest": "sha256:5bf55fa8550c47a1054c7a02138b9f79b5f574f040b1e444ad717d320d3afc67", 16 | "size": 25600, 17 | "dir": "/var/lib/overlaybd/test/3" 18 | }, 19 | { 20 | "digest": "sha256:62a999219eb529a2403f2b5849869d3253bf1014721293333f7f66be54308b94", 21 | "size": 2610688, 22 | "dir": "/var/lib/overlaybd/test/4" 23 | }, 24 | { 25 | "digest": "sha256:f2d33f598db59a8a4fcb490764cdfca3157ec6a742870378154cbef93acefce9", 26 | "size": 17303040, 27 | "dir": "/var/lib/overlaybd/test/5" 28 | }, 29 | { 30 | "digest": "sha256:8d77203e222f30ab4b8ba2e232fd9d71880dd80f6f24fa18e45d1d578e40eb57", 31 | "size": 8192, 32 | "dir": "/var/lib/overlaybd/test/6" 33 | }, 34 | { 35 | "digest": "sha256:8bdb50d0eb5ec766ba235c06ac8c8a6f44ab1beeed756efa532e73b79786e36a", 36 | "size": 11776, 37 | "dir": "/var/lib/overlaybd/test/7" 38 | } 39 | ], 40 | "upper": {}, 41 | "resultFile": "/var/lib/overlaybd/init-debug.log" 42 | } -------------------------------------------------------------------------------- /src/example_config/stream-conv.yaml: -------------------------------------------------------------------------------- 1 | globalConfig: 2 | workDir: /tmp/stream_conv 3 | udsAddr: /var/run/stream_conv.sock 4 | # httpAddr: 127.0.0.1 5 | httpPort: 9101 6 | reusePort: true 7 | 8 | logConfig: 9 | level: 1 10 | mode: stdout 11 | rotateNum: 3 12 | limitSizeMB: 10 13 | path: /var/log/overlaybd/stream_convertor.log 14 | -------------------------------------------------------------------------------- /src/exporter_handler.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "textexporter.h" 25 | 26 | namespace ExposeMetrics { 27 | 28 | #define EXPOSE_PHOTON_METRICLIST(name, type) \ 29 | std::vector> va_##name; \ 30 | void add_##name(const char* tag, type& metric) { \ 31 | va_##name.emplace_back(std::make_pair(tag, &metric)); \ 32 | } 33 | 34 | #define EXPOSE_TEMPLATE(name, ...) static auto name = PROMMETRIC(__VA_ARGS__) 35 | 36 | #define LOOP_APPEND_METRIC(ret, name) \ 37 | if (!va_##name.empty()) { \ 38 | ret.append(name.help_str()).append("\n"); \ 39 | ret.append(name.type_str()).append("\n"); \ 40 | for (auto x : va_##name) { \ 41 | ret.append(name.render(x.second->val(), x.first)).append("\n"); \ 42 | } \ 43 | ret.append("\n"); \ 44 | } 45 | 46 | struct ExposeRender : public photon::net::http::HTTPHandler { 47 | EXPOSE_PHOTON_METRICLIST(throughput, Metric::QPSCounter); 48 | EXPOSE_PHOTON_METRICLIST(qps, Metric::QPSCounter); 49 | EXPOSE_PHOTON_METRICLIST(latency, Metric::MaxLatencyCounter); 50 | EXPOSE_PHOTON_METRICLIST(count, Metric::AddCounter); 51 | EXPOSE_PHOTON_METRICLIST(cache, Metric::ValueCounter); 52 | 53 | template 54 | ExposeRender(Args&&... args) {} 55 | 56 | std::string render() { 57 | EXPOSE_TEMPLATE(alive, OverlayBD_Alive : gauge{node}); 58 | EXPOSE_TEMPLATE(throughput, OverlayBD_Read_Throughtput 59 | : gauge{node, type, mode} #Bytes / sec); 60 | EXPOSE_TEMPLATE(qps, OverlayBD_QPS : gauge{node, type, mode}); 61 | EXPOSE_TEMPLATE(latency, OverlayBD_MaxLatency 62 | : gauge{node, type, mode} #us); 63 | EXPOSE_TEMPLATE(count, OverlayBD_Count : gauge{node, type} #Bytes); 64 | std::string ret(alive.help_str()); 65 | ret.append("\n") 66 | .append(alive.type_str()) 67 | .append("\n") 68 | .append(alive.render(1)) 69 | .append("\n\n"); 70 | LOOP_APPEND_METRIC(ret, throughput); 71 | LOOP_APPEND_METRIC(ret, qps); 72 | LOOP_APPEND_METRIC(ret, latency); 73 | LOOP_APPEND_METRIC(ret, count); 74 | return ret; 75 | } 76 | 77 | int handle_request(photon::net::http::Request& req, 78 | photon::net::http::Response& resp, 79 | std::string_view) override { 80 | auto body = render(); 81 | resp.set_result(200); 82 | resp.keep_alive(true); 83 | resp.headers.insert("Content-Type", "text/plain; version=0.0.4"); 84 | resp.headers.content_length(body.length()); 85 | ssize_t len = 0; 86 | len = resp.write((void*)body.data(), body.length()); 87 | if (len == (ssize_t)body.length()) { 88 | return 0; 89 | } else { 90 | LOG_ERRNO_RETURN(0, -1, "Failed to write exporter response"); 91 | } 92 | } 93 | }; 94 | 95 | #undef LOOP_APPEND_METRIC 96 | #undef EXPOSE_PHOTON_METRICLIST 97 | }; // namespace ExposeMetrics -------------------------------------------------------------------------------- /src/exporter_server.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "config.h" 26 | #include "exporter_handler.h" 27 | #include "metrics_fs.h" 28 | 29 | class OverlayBDMetric { 30 | public: 31 | MetricMeta pread, download; 32 | 33 | ExposeMetrics::ExposeRender exporter; 34 | 35 | OverlayBDMetric() { 36 | exporter.add_throughput("pread", pread.throughput); 37 | exporter.add_latency("pread", pread.latency); 38 | exporter.add_qps("pread", pread.qps); 39 | exporter.add_count("pread", pread.total); 40 | exporter.add_throughput("download", download.throughput); 41 | exporter.add_latency("download", download.latency); 42 | exporter.add_qps("download", download.qps); 43 | exporter.add_count("download", download.total); 44 | } 45 | }; 46 | 47 | struct ExporterServer { 48 | photon::net::http::HTTPServer *httpserver = nullptr; 49 | photon::net::ISocketServer *tcpserver = nullptr; 50 | 51 | bool ready = false; 52 | 53 | ExporterServer(ImageConfigNS::GlobalConfig &config, 54 | OverlayBDMetric *metrics) { 55 | tcpserver = photon::net::new_tcp_socket_server(); 56 | tcpserver->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1); 57 | if (tcpserver->bind(config.exporterConfig().port()) < 0) 58 | LOG_ERRNO_RETURN(0, , "Failed to bind exporter port `", 59 | config.exporterConfig().port()); 60 | if (tcpserver->listen() < 0) 61 | LOG_ERRNO_RETURN(0, , "Failed to listen exporter port `", 62 | config.exporterConfig().port()); 63 | httpserver = photon::net::http::new_http_server(); 64 | httpserver->add_handler(&metrics->exporter, false, 65 | config.exporterConfig().uriPrefix()); 66 | tcpserver->set_handler(httpserver->get_connection_handler()); 67 | tcpserver->start_loop(); 68 | ready = true; 69 | } 70 | 71 | ~ExporterServer() { 72 | delete tcpserver; 73 | delete httpserver; 74 | } 75 | }; -------------------------------------------------------------------------------- /src/image_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include "image_service.h" 29 | #include "bk_download.h" 30 | #include "config.h" 31 | #include "image_service.h" 32 | #include "prefetch.h" 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include "overlaybd/lsmt/file.h" 38 | 39 | static std::string COMMIT_FILE_NAME = "overlaybd.commit"; 40 | static std::string SEALED_FILE_NAME = "overlaybd.sealed"; 41 | 42 | class ImageFile : public photon::fs::ForwardFile { 43 | public: 44 | ImageFile(ImageConfigNS::ImageConfig &_conf, ImageService &is) 45 | : ForwardFile(nullptr), image_service(is), m_lower_file(nullptr) { 46 | conf.CopyFrom(_conf, conf.GetAllocator()); 47 | m_exception = ""; 48 | m_status = init_image_file(); 49 | if (m_status == 1) { 50 | struct stat st; 51 | fstat(&st); 52 | LOG_INFO("new imageFile, bs: `, size: `", block_size, size); 53 | } 54 | } 55 | 56 | ~ImageFile() { 57 | m_status = -1; 58 | if (dl_thread_jh != nullptr) 59 | photon::thread_join(dl_thread_jh); 60 | delete m_prefetcher; 61 | if (m_file) { 62 | m_file->close(); 63 | delete m_file; 64 | } 65 | if (m_lower_file) delete m_lower_file; 66 | if (m_upper_file) delete m_upper_file; 67 | } 68 | 69 | int fstat(struct stat *buf) override { 70 | int ret = m_file->fstat(buf); 71 | block_size = buf->st_blksize; 72 | size = buf->st_size; 73 | if (block_size == 0) 74 | block_size = 512; 75 | num_lbas = size / block_size; 76 | return ret; 77 | } 78 | 79 | ssize_t pwritev(const struct iovec *iov, int iovcnt, off_t offset) override { 80 | if (read_only) { 81 | LOG_ERROR_RETURN(EROFS, -1, "writing read only file"); 82 | } 83 | return m_file->pwritev(iov, iovcnt, offset); 84 | } 85 | 86 | ssize_t preadv(const struct iovec *iov, int iovcnt, off_t offset) override { 87 | return m_file->preadv(iov, iovcnt, offset); 88 | } 89 | 90 | int fdatasync() override { 91 | return m_file->fdatasync(); 92 | } 93 | 94 | int fallocate(int mode, off_t offset, off_t len) override { 95 | return m_file->fallocate(mode, offset, len); 96 | } 97 | 98 | void set_auth_failed(); 99 | int open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer, int index); 100 | 101 | std::string m_exception; 102 | int m_status = 0; // 0: not started, 1: running, -1 exit 103 | 104 | size_t size; 105 | uint64_t num_lbas; 106 | uint32_t block_size; 107 | bool read_only = false; 108 | 109 | // a merged view after stack all layers. 110 | IFile* get_base() { 111 | return m_file; 112 | } 113 | 114 | int compact(IFile *as); 115 | 116 | private: 117 | Prefetcher *m_prefetcher = nullptr; 118 | ImageConfigNS::ImageConfig conf; 119 | std::list dl_list; 120 | photon::join_handle *dl_thread_jh = nullptr; 121 | ImageService &image_service; 122 | photon::fs::IFile *m_lower_file = nullptr; 123 | photon::fs::IFile *m_upper_file = nullptr; 124 | 125 | int init_image_file(); 126 | template void set_failed(const Ts&...xs); 127 | LSMT::IFileRO *open_lowers(std::vector &, bool &); 128 | LSMT::IFileRW *open_upper(ImageConfigNS::UpperConfig &); 129 | 130 | IFile *open_localfile(ImageConfigNS::LayerConfig &layer, std::string &opened); 131 | IFile *__open_ro_file(const std::string &); 132 | IFile *__open_ro_target_file(const std::string &); 133 | IFile *__open_ro_remote(const std::string &dir, const std::string &, const uint64_t, int); 134 | IFile *__open_ro_target_remote(const std::string &dir, const std::string &, const uint64_t, int); 135 | 136 | // size_t seek_data(off_t begin, off_t end); 137 | void start_bk_dl_thread(); 138 | }; 139 | -------------------------------------------------------------------------------- /src/image_service.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include "config.h" 21 | #include "exporter_server.h" 22 | #include "overlaybd/cache/gzip_cache/cached_fs.h" 23 | #include 24 | #include 25 | 26 | using namespace photon::fs; 27 | 28 | 29 | struct GlobalFs { 30 | IFileSystem *underlay_registryfs = nullptr; 31 | IFileSystem *remote_fs = nullptr; 32 | IFileSystem *srcfs = nullptr; 33 | IFileSystem *cached_fs = nullptr; 34 | Cache::GzipCachedFs *gzcache_fs = nullptr; 35 | 36 | // ocf cache only 37 | IFile *media_file = nullptr; 38 | IFileSystem *namespace_fs = nullptr; 39 | IOAlloc *io_alloc = nullptr; 40 | }; 41 | 42 | struct ImageAuthResponse : public ConfigUtils::Config { 43 | APPCFG_CLASS 44 | 45 | APPCFG_PARA(traceId, std::string, ""); 46 | APPCFG_PARA(success, bool, false); 47 | APPCFG_PARA(data, ImageConfigNS::AuthConfig); 48 | }; 49 | 50 | struct ImageFile; 51 | 52 | class ImageService { 53 | public: 54 | ImageService(const char *config_path = nullptr); 55 | ~ImageService(); 56 | int init(); 57 | ImageFile *create_image_file(const char *image_config_path); 58 | // bool enable_acceleration(GlobalFs *global_fs, ImageConfigNS::P2PConfig conf); 59 | bool enable_acceleration(); 60 | 61 | 62 | ImageConfigNS::GlobalConfig global_conf; 63 | struct GlobalFs global_fs; 64 | std::unique_ptr metrics; 65 | ExporterServer *exporter = nullptr; 66 | 67 | private: 68 | int read_global_config_and_set(); 69 | std::pair reload_auth(const char *remote_path); 70 | void set_result_file(std::string &filename, std::string &data); 71 | std::string m_config_path; 72 | }; 73 | 74 | ImageService *create_image_service(const char *config_path = nullptr); 75 | 76 | int load_cred_from_file(const std::string path, const std::string &remote_path, 77 | std::string &username, std::string &password); 78 | 79 | void destroy(); -------------------------------------------------------------------------------- /src/metrics_fs.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | struct MetricMeta { 23 | Metric::MaxLatencyCounter latency; 24 | Metric::QPSCounter throughput; 25 | Metric::QPSCounter qps; 26 | Metric::AddCounter total; 27 | Metric::AddCounter interval; 28 | 29 | MetricMeta() {} 30 | }; 31 | 32 | class MetricFile : public photon::fs::ForwardFile_Ownership { 33 | public: 34 | MetricMeta *metrics; 35 | 36 | MetricFile(photon::fs::IFile *file, MetricMeta *metricMeta) 37 | : photon::fs::ForwardFile_Ownership(file, true), metrics(metricMeta) {} 38 | 39 | __attribute__((always_inline)) void mark_metrics(ssize_t ret) { 40 | if (ret > 0) { 41 | metrics->throughput.put(ret); 42 | metrics->total.add(ret); 43 | metrics->interval.add(ret); 44 | } 45 | } 46 | 47 | virtual ssize_t pread(void *buf, size_t cnt, off_t offset) override { 48 | metrics->qps.put(); 49 | SCOPE_LATENCY(metrics->latency); 50 | auto ret = m_file->pread(buf, cnt, offset); 51 | mark_metrics(ret); 52 | return ret; 53 | } 54 | 55 | virtual ssize_t preadv(const struct iovec *iovec, int iovcnt, 56 | off_t offset) override { 57 | metrics->qps.put(); 58 | SCOPE_LATENCY(metrics->latency); 59 | auto ret = m_file->preadv(iovec, iovcnt, offset); 60 | mark_metrics(ret); 61 | return ret; 62 | } 63 | 64 | virtual ssize_t preadv2(const struct iovec *iovec, int iovcnt, off_t offset, 65 | int flags) override { 66 | metrics->qps.put(); 67 | SCOPE_LATENCY(metrics->latency); 68 | auto ret = m_file->preadv2(iovec, iovcnt, offset, flags); 69 | mark_metrics(ret); 70 | return ret; 71 | } 72 | }; 73 | 74 | class MetricFS : public photon::fs::ForwardFS_Ownership { 75 | public: 76 | MetricMeta *metrics; 77 | 78 | MetricFS(photon::fs::IFileSystem *fs, MetricMeta *metricMeta) 79 | : photon::fs::ForwardFS_Ownership(fs, true), metrics(metricMeta) {} 80 | 81 | virtual photon::fs::IFile *open(const char *fn, int flags) override { 82 | auto file = m_fs->open(fn, flags); 83 | if (!file) return nullptr; 84 | return new MetricFile(file, metrics); 85 | } 86 | 87 | virtual photon::fs::IFile *open(const char *fn, int flags, 88 | mode_t mode) override { 89 | auto file = m_fs->open(fn, flags, mode); 90 | if (!file) return nullptr; 91 | return new MetricFile(file, metrics); 92 | } 93 | }; 94 | -------------------------------------------------------------------------------- /src/overlaybd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(registryfs) 2 | add_subdirectory(lsmt) 3 | add_subdirectory(zfile) 4 | add_subdirectory(zstd) 5 | add_subdirectory(cache) 6 | add_subdirectory(tar) 7 | add_subdirectory(gzip) 8 | add_subdirectory(gzindex) 9 | add_subdirectory(stream_convertor) 10 | 11 | add_library(overlaybd_lib INTERFACE) 12 | target_include_directories(overlaybd_lib INTERFACE 13 | ${PHOTON_INCLUDE_DIR} 14 | ) 15 | target_link_libraries(overlaybd_lib INTERFACE 16 | photon_static 17 | registryfs_lib 18 | lsmt_lib 19 | zfile_lib 20 | zstd_lib 21 | cache_lib 22 | tar_lib 23 | gzip_lib 24 | gzindex_lib 25 | ) 26 | -------------------------------------------------------------------------------- /src/overlaybd/base64.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef _BASE64_H_ 4 | #define _BASE64_H_ 5 | 6 | #include 7 | #include 8 | typedef unsigned char BYTE; 9 | 10 | static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 11 | "abcdefghijklmnopqrstuvwxyz" 12 | "0123456789+/"; 13 | 14 | static inline bool is_base64(BYTE c) { 15 | return (isalnum(c) || (c == '+') || (c == '/')); 16 | } 17 | 18 | static inline std::string base64_encode(BYTE const *buf, unsigned int bufLen) { 19 | std::string ret; 20 | int i = 0; 21 | int j = 0; 22 | BYTE char_array_3[3]; 23 | BYTE char_array_4[4]; 24 | 25 | while (bufLen--) { 26 | char_array_3[i++] = *(buf++); 27 | if (i == 3) { 28 | char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; 29 | char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); 30 | char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); 31 | char_array_4[3] = char_array_3[2] & 0x3f; 32 | 33 | for (i = 0; (i < 4); i++) 34 | ret += base64_chars[char_array_4[i]]; 35 | i = 0; 36 | } 37 | } 38 | 39 | if (i) { 40 | for (j = i; j < 3; j++) 41 | char_array_3[j] = '\0'; 42 | 43 | char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; 44 | char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); 45 | char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); 46 | char_array_4[3] = char_array_3[2] & 0x3f; 47 | 48 | for (j = 0; (j < i + 1); j++) 49 | ret += base64_chars[char_array_4[j]]; 50 | 51 | while ((i++ < 3)) 52 | ret += '='; 53 | } 54 | 55 | return ret; 56 | } 57 | 58 | static inline std::string base64_decode(std::string const &encoded_string) { 59 | int in_len = encoded_string.size(); 60 | int i = 0; 61 | int j = 0; 62 | int in_ = 0; 63 | BYTE char_array_4[4], char_array_3[3]; 64 | std::string ret = ""; 65 | 66 | while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { 67 | char_array_4[i++] = encoded_string[in_]; 68 | in_++; 69 | if (i == 4) { 70 | for (i = 0; i < 4; i++) 71 | char_array_4[i] = base64_chars.find(char_array_4[i]); 72 | 73 | char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); 74 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 75 | char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; 76 | 77 | for (i = 0; (i < 3); i++) 78 | ret += char_array_3[i]; 79 | i = 0; 80 | } 81 | } 82 | 83 | if (i) { 84 | for (j = i; j < 4; j++) 85 | char_array_4[j] = 0; 86 | 87 | for (j = 0; j < 4; j++) 88 | char_array_4[j] = base64_chars.find(char_array_4[j]); 89 | 90 | char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); 91 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 92 | char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; 93 | 94 | for (j = 0; (j < i - 1); j++) 95 | ret += char_array_3[j]; 96 | } 97 | 98 | return ret; 99 | } 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /src/overlaybd/cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(full_file_cache) 2 | add_subdirectory(ocf_cache) 3 | add_subdirectory(download_cache) 4 | add_subdirectory(gzip_cache) 5 | 6 | file(GLOB SRC_CACHE "*.cpp") 7 | 8 | add_library(cache_lib STATIC ${SRC_CACHE}) 9 | target_link_libraries(cache_lib 10 | photon_static 11 | full_file_cache_lib 12 | ocf_cache_lib 13 | download_cache_lib 14 | gzip_cache_lib 15 | ) 16 | target_include_directories(cache_lib PUBLIC 17 | ${PHOTON_INCLUDE_DIR} 18 | ) 19 | 20 | if(BUILD_TESTING) 21 | add_subdirectory(test) 22 | endif() 23 | -------------------------------------------------------------------------------- /src/overlaybd/cache/cache.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #include "cache.h" 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "full_file_cache/cache_pool.h" 25 | 26 | namespace FileSystem { 27 | using namespace photon::fs; 28 | ICachedFileSystem *new_full_file_cached_fs(IFileSystem *srcFs, IFileSystem *mediaFs, 29 | uint64_t refillUnit, uint64_t capacityInGB, 30 | uint64_t periodInUs, uint64_t diskAvailInBytes, 31 | IOAlloc *allocator, int quotaDirLevel, 32 | CacheFnTransFunc fn_trans_func) { 33 | if (refillUnit % 4096 != 0 || !is_power_of_2(refillUnit)) { 34 | LOG_ERROR_RETURN(EINVAL, nullptr, "refill Unit need to be aligned to 4KB and power of 2") 35 | } 36 | if (!allocator) { 37 | allocator = new IOAlloc; 38 | } 39 | Cache::FileCachePool *pool = nullptr; 40 | pool = 41 | new ::Cache::FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit); 42 | pool->Init(); 43 | return new_cached_fs(srcFs, pool, 4096, allocator, fn_trans_func); 44 | } 45 | 46 | using OC = ObjectCache; 47 | ICachePool::ICachePool(uint32_t pool_size, uint32_t max_refilling, uint32_t refilling_threshold) 48 | : m_stores(new OC(10UL * 1000 * 1000)), m_max_refilling(max_refilling), 49 | m_refilling_threshold(refilling_threshold) { 50 | if (pool_size != 0) { 51 | m_thread_pool = photon::new_thread_pool(pool_size, 128 * 1024UL); 52 | m_vcpu = photon::get_vcpu(); 53 | }; 54 | } 55 | 56 | #define cast(x) static_cast(x) 57 | ICachePool::~ICachePool() { 58 | stores_clear(); 59 | delete cast(m_stores); 60 | } 61 | 62 | void ICachePool::stores_clear() { 63 | if (m_thread_pool) { 64 | auto pool = static_cast(m_thread_pool); 65 | m_thread_pool = nullptr; 66 | photon::delete_thread_pool(pool); 67 | } 68 | cast(m_stores)->clear(); 69 | } 70 | 71 | ICacheStore *ICachePool::open(std::string_view filename, int flags, mode_t mode) { 72 | char store_name[4096]; 73 | std::string x(filename); 74 | auto len = this->fn_trans_func(filename, store_name, sizeof(store_name)); 75 | std::string_view store_sv = len ? std::string_view(store_name, len) : filename; 76 | auto ctor = [&]() -> ICacheStore * { 77 | auto cache_store = this->do_open(store_sv, flags, mode); 78 | if (nullptr == cache_store) { 79 | LOG_ERRNO_RETURN(0, nullptr, "fileCachePool_ open file failed, name : `", 80 | filename.data()); 81 | } 82 | auto it = cast(m_stores)->find(store_sv); 83 | std::string_view map_key = (*it)->key(); 84 | cache_store->set_store_key(map_key); 85 | cache_store->set_src_name(filename); 86 | cache_store->set_pool(this); 87 | struct stat st; 88 | SET_STRUCT_STAT(&st); 89 | st.st_size = -1; 90 | if (cache_store->fstat(&st) == 0) { 91 | cache_store->set_cached_size(st.st_size); 92 | cache_store->set_actual_size(st.st_size); 93 | } 94 | cache_store->set_open_flags(flags); 95 | return cache_store; 96 | }; 97 | auto store = cast(m_stores)->acquire(store_sv, ctor); 98 | if (store) { 99 | auto cnt = store->ref_.fetch_add(1, std::memory_order_relaxed); 100 | if (cnt) 101 | cast(m_stores)->release(store_sv); 102 | } 103 | return store; 104 | } 105 | 106 | void ICachePool::set_trans_func(CacheFnTransFunc fn_trans_func) { 107 | this->fn_trans_func = fn_trans_func; 108 | } 109 | 110 | int ICachePool::store_release(ICacheStore *store) { 111 | return cast(m_stores)->release(store->get_store_key()); 112 | } 113 | } // namespace FileSystem 114 | -------------------------------------------------------------------------------- /src/overlaybd/cache/download_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SRC_DOWNLOADCACHE "*.cpp") 2 | 3 | add_library(download_cache_lib STATIC ${SRC_DOWNLOADCACHE}) 4 | target_include_directories(download_cache_lib PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ) 7 | -------------------------------------------------------------------------------- /src/overlaybd/cache/forwardcfs.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | #include "cache.h" 18 | #include "photon/fs/forwardfs.h" 19 | 20 | namespace FileSystem { 21 | template 22 | class ForwardCachedFileBase : public IForwardCachedFile { 23 | protected: 24 | using Base = IForwardCachedFile; 25 | using Base::Base; 26 | virtual photon::fs::IFile *get_source() override { 27 | return Base::m_file->get_source(); 28 | } 29 | virtual int set_source(photon::fs::IFile *src) override { 30 | return Base::m_file->set_source(src); 31 | } 32 | virtual ICacheStore *get_store() override { 33 | return Base::m_file->get_store(); 34 | } 35 | virtual int query(off_t offset, size_t count) override { 36 | return Base::m_file->query(offset, count); 37 | } 38 | }; 39 | using ForwardCachedFile = ForwardCachedFileBase>; 40 | using ForwardCachedFile_Ownership = 41 | ForwardCachedFileBase>; 42 | 43 | template 44 | class ForwardCachedFSBase : public IForwardCachedFS { 45 | protected: 46 | using Base = IForwardCachedFS; 47 | using Base::Base; 48 | virtual photon::fs::IFileSystem *get_source() override { 49 | return Base::m_fs->get_source(); 50 | } 51 | virtual int set_source(photon::fs::IFileSystem *src) override { 52 | return Base::m_fs->set_source(src); 53 | } 54 | virtual ICachePool *get_pool() override { 55 | return Base::m_fs->get_pool(); 56 | } 57 | virtual int set_pool(ICachePool *pool) override { 58 | return Base::m_fs->set_pool(pool); 59 | } 60 | }; 61 | using ForwardCachedFS = ForwardCachedFSBase>; 62 | using ForwardCachedFS_Ownership = 63 | ForwardCachedFSBase>; 64 | } // namespace FileSystem 65 | -------------------------------------------------------------------------------- /src/overlaybd/cache/full_file_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SRC_FULLFILECACHE "*.cpp") 2 | 3 | add_library(full_file_cache_lib STATIC ${SRC_FULLFILECACHE}) 4 | target_include_directories(full_file_cache_lib PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ) 7 | -------------------------------------------------------------------------------- /src/overlaybd/cache/full_file_cache/cache_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include "../policy/lru.h" 28 | #include "../pool_store.h" 29 | 30 | #include 31 | 32 | namespace Cache { 33 | 34 | class FileCachePool : public FileSystem::ICachePool { 35 | public: 36 | FileCachePool(photon::fs::IFileSystem *mediaFs, uint64_t capacityInGB, uint64_t periodInUs, 37 | uint64_t diskAvailInBytes, uint64_t refillUnit); 38 | ~FileCachePool(); 39 | 40 | static const uint64_t kDiskBlockSize = 512; // stat(2) 41 | static const uint64_t kDeleteDelayInUs = 1000; 42 | static const uint32_t kWaterMarkRatio = 90; 43 | 44 | void Init(); 45 | 46 | // pathname must begin with '/' 47 | FileSystem::ICacheStore *do_open(std::string_view pathname, int flags, mode_t mode) override; 48 | 49 | int set_quota(std::string_view pathname, size_t quota) override; 50 | int stat(FileSystem::CacheStat *stat, 51 | std::string_view pathname = std::string_view(nullptr, 0)) override; 52 | 53 | int evict(std::string_view filename) override; 54 | int evict(size_t size = 0) override; 55 | int rename(std::string_view oldname, std::string_view newname) override; 56 | 57 | struct LruEntry { 58 | LruEntry(uint32_t lruIt, int openCnt, uint64_t fileSize) 59 | : lruIter(lruIt), openCount(openCnt), size(fileSize), truncate_done(false) { 60 | } 61 | ~LruEntry() = default; 62 | uint32_t lruIter; 63 | int openCount; 64 | uint64_t size; 65 | photon::rwlock rw_lock_; 66 | bool truncate_done; 67 | }; 68 | 69 | // Normally, fileIndex(std::map) always keep growing, so its iterators always 70 | // keep valid, iterator will be erased when file be unlinked in period of eviction, 71 | // but on that time corresponding CachedFile had been destructed, so nobody hold 72 | // erased iterator. 73 | typedef map_string_key> FileNameMap; 74 | 75 | bool isFull(); 76 | void removeOpenFile(FileNameMap::iterator iter); 77 | void forceRecycle(); 78 | void updateLru(FileNameMap::iterator iter); 79 | uint64_t updateSpace(FileNameMap::iterator iter, uint64_t size); 80 | 81 | protected: 82 | photon::fs::IFile *openMedia(std::string_view name, int flags, int mode); 83 | 84 | static uint64_t timerHandler(void *data); 85 | virtual void eviction(); 86 | uint64_t calcWaterMark(uint64_t capacity, uint64_t maxFreeSpace); 87 | 88 | photon::fs::IFileSystem *mediaFs_; // owned by current class 89 | uint64_t capacityInGB_; 90 | uint64_t periodInUs_; 91 | uint64_t diskAvailInBytes_; 92 | size_t refillUnit_; 93 | int64_t totalUsed_; 94 | int64_t riskMark_; 95 | uint64_t waterMark_; 96 | 97 | photon::Timer *timer_; 98 | bool running_; 99 | bool exit_; 100 | 101 | bool isFull_; 102 | 103 | virtual bool afterFtrucate(FileNameMap::iterator iter); 104 | 105 | int traverseDir(const std::string &root); 106 | virtual int insertFile(std::string_view file); 107 | 108 | typedef FileSystem::LRU LRUContainer; 109 | LRUContainer lru_; 110 | // filename -> lruEntry 111 | FileNameMap fileIndex_; 112 | }; 113 | 114 | } // namespace Cache 115 | -------------------------------------------------------------------------------- /src/overlaybd/cache/full_file_cache/cache_store.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include "cache_pool.h" 22 | 23 | namespace Cache { 24 | 25 | class FileCachePool; 26 | 27 | class FileCacheStore : public FileSystem::ICacheStore { 28 | public: 29 | typedef FileCachePool::FileNameMap::iterator FileIterator; 30 | FileCacheStore(FileSystem::ICachePool *cachePool, photon::fs::IFile *localFile, 31 | size_t refillUnit, FileIterator iterator); 32 | ~FileCacheStore(); 33 | 34 | try_preadv_result try_preadv2(const struct iovec *iov, int iovcnt, off_t offset, int flags) override; 35 | 36 | ssize_t do_preadv2(const struct iovec *iov, int iovcnt, off_t offset, int flags) override; 37 | 38 | ssize_t do_pwritev2(const struct iovec *iov, int iovcnt, off_t offset, int flags) override; 39 | 40 | int set_quota(size_t quota) override; 41 | int stat(FileSystem::CacheStat *stat) override; 42 | int evict(off_t offset, size_t count = -1) override; 43 | 44 | std::pair queryRefillRange(off_t offset, size_t size) override; 45 | 46 | int fstat(struct stat *buf) override; 47 | 48 | protected: 49 | bool cacheIsFull(); 50 | 51 | struct ReadRequest { 52 | off_t offset; 53 | size_t size; 54 | }; 55 | 56 | // merge from first extent to last extent(or encounter hole), 57 | // because fiemap could return multiple continuous extents even though no any hole. 58 | std::pair getFirstMergedExtents(struct fiemap *fie); 59 | 60 | std::pair getLastMergedExtents(struct fiemap *fie); 61 | 62 | std::pair getHoleFromCacheHitResult(off_t offset, size_t alignSize, 63 | struct fiemap *fie); 64 | 65 | FileCachePool *cachePool_; // owned by extern class 66 | photon::fs::IFile *localFile_; // owned by current class 67 | size_t refillUnit_; 68 | FileIterator iterator_; 69 | RangeLock rangeLock_; 70 | 71 | ssize_t do_pwritev(const struct iovec *iov, int iovcnt, off_t offset); 72 | }; 73 | 74 | } // namespace Cache 75 | -------------------------------------------------------------------------------- /src/overlaybd/cache/gzip_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SRC_FRONTEND "*.cpp") 2 | 3 | add_library(gzip_cache_lib STATIC ${SRC_FRONTEND}) 4 | target_link_libraries(gzip_cache_lib gzindex_lib) 5 | target_include_directories(gzip_cache_lib PUBLIC 6 | ${PHOTON_INCLUDE_DIR} 7 | ) -------------------------------------------------------------------------------- /src/overlaybd/cache/gzip_cache/cached_fs.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #include "cached_fs.h" 17 | #include "../full_file_cache/cache_pool.h" 18 | #include "../cache.h" 19 | #include 20 | 21 | namespace Cache { 22 | 23 | class GzipCachedFsImpl : public GzipCachedFs { 24 | public: 25 | GzipCachedFsImpl(FileSystem::ICachePool *pool, size_t page_size, 26 | size_t refill_unit, IOAlloc *io_alloc) 27 | : pool_(pool), page_size_(page_size), 28 | refill_unit_(refill_unit), io_alloc_(io_alloc) { 29 | } 30 | ~GzipCachedFsImpl() { 31 | delete pool_; 32 | } 33 | 34 | photon::fs::IFile *open_cached_gzip_file(photon::fs::IFile *file, const char *file_name) { 35 | if (!file) { 36 | LOG_ERRNO_RETURN(0, nullptr, "Open source gzfile failed"); 37 | } 38 | estring fn = file_name; 39 | if (fn[0] != '/') { 40 | fn = estring().appends("/", fn); 41 | } 42 | auto cache_store = pool_->open(fn, O_RDWR | O_CREAT, 0644); 43 | if (cache_store == nullptr) { 44 | delete file; 45 | LOG_ERRNO_RETURN(0, nullptr, "file cache pool open file failed, name : `", file_name); 46 | } 47 | cache_store->set_src_file(file); 48 | cache_store->set_allocator(io_alloc_); 49 | cache_store->set_page_size(page_size_); 50 | auto ret = FileSystem::new_cached_file(cache_store, page_size_, nullptr); 51 | if (ret == nullptr) { // if create file is failed 52 | // file and cache_store must be release, or will leak 53 | delete file; 54 | cache_store->release(); 55 | } 56 | return ret; 57 | } 58 | private: 59 | FileSystem::ICachePool *pool_; 60 | size_t page_size_; 61 | size_t refill_unit_; 62 | IOAlloc *io_alloc_; 63 | }; 64 | 65 | GzipCachedFs *new_gzip_cached_fs(photon::fs::IFileSystem *mediaFs, uint64_t refillUnit, 66 | uint64_t capacityInGB, uint64_t periodInUs, 67 | uint64_t diskAvailInBytes, IOAlloc *allocator) { 68 | if (refillUnit % 4096 != 0) { 69 | LOG_ERROR_RETURN(EINVAL, nullptr, "refill Unit need to be aligned to 4KB") 70 | } 71 | if (!allocator) { 72 | allocator = new IOAlloc; 73 | } 74 | FileCachePool *pool = nullptr; 75 | pool = new FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit); 76 | pool->Init(); 77 | return new GzipCachedFsImpl(pool, 4096, refillUnit, allocator); 78 | } 79 | } // namespace Cache 80 | -------------------------------------------------------------------------------- /src/overlaybd/cache/gzip_cache/cached_fs.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #include 17 | #include 18 | namespace Cache { 19 | 20 | class GzipCachedFs { 21 | public: 22 | virtual ~GzipCachedFs() {} 23 | virtual photon::fs::IFile *open_cached_gzip_file(photon::fs::IFile *file, const char *file_name) = 0; 24 | }; 25 | GzipCachedFs *new_gzip_cached_fs(photon::fs::IFileSystem *mediaFs, uint64_t refillUnit, 26 | uint64_t capacityInGB, uint64_t periodInUs, 27 | uint64_t diskAvailInBytes, IOAlloc *allocator); 28 | } // namespace Cache 29 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(BUILD_TESTING) 2 | add_subdirectory(test) 3 | endif() 4 | 5 | # ocf_env_lib 6 | file(GLOB_RECURSE src_ocf_env ease_bindings/env/*.cpp) 7 | add_library(ocf_env_lib STATIC ${src_ocf_env}) 8 | target_include_directories(ocf_env_lib PUBLIC include/ ${PHOTON_INCLUDE_DIR}) 9 | 10 | # ocf_lib 11 | file(GLOB_RECURSE src_ocf ocf/src/*.c) 12 | add_library(ocf_lib STATIC ${src_ocf}) 13 | target_include_directories(ocf_lib PUBLIC include/ ease_bindings/env/) 14 | target_link_libraries(ocf_lib ocf_env_lib z) 15 | target_compile_options(ocf_lib PRIVATE -Wno-sign-compare) 16 | 17 | # ocf_cache_lib 18 | file(GLOB src_ocf_cache ocf_cache.cpp ocf_namespace.cpp ease_bindings/*.cpp) 19 | add_library(ocf_cache_lib STATIC ${src_ocf_cache}) 20 | target_include_directories(ocf_cache_lib PUBLIC include/ ease_bindings/env/ ${PHOTON_INCLUDE_DIR}) 21 | target_link_libraries(ocf_cache_lib ocf_lib photon_static) -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/ctx.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include "../ocf_namespace.h" 9 | 10 | extern "C" { 11 | #include 12 | } 13 | 14 | #define ROUND_UP(N, S) ((((N) + (S)-1) / (S)) * (S)) 15 | #define ROUND_DOWN(N, S) ((N) & ~((S)-1)) 16 | 17 | class ease_ocf_provider; 18 | 19 | /* Src file context */ 20 | struct OcfSrcFileCtx { 21 | OcfSrcFileCtx(photon::fs::IFile *src_file_, const OcfNamespace::NsInfo &ns_info_, 22 | ease_ocf_provider *provider_, const estring &path_) 23 | : src_file(src_file_), ns_info(ns_info_), provider(provider_), path(path_) { 24 | } 25 | 26 | ~OcfSrcFileCtx() { 27 | delete src_file; 28 | } 29 | 30 | photon::fs::IFile *src_file; 31 | OcfNamespace::NsInfo ns_info; 32 | ease_ocf_provider *provider; 33 | estring path; 34 | }; 35 | 36 | /* IO data */ 37 | struct ease_ocf_io_data { 38 | ease_ocf_io_data() = default; 39 | 40 | ease_ocf_io_data(iovec *iovs_, int iovcnt_, uint32_t size_, off_t blk_addr_, 41 | OcfSrcFileCtx *ctx_, bool prefetch_) 42 | : iovs(iovs_), iovcnt(iovcnt_), size(size_), blk_addr(blk_addr_), ctx(ctx_), 43 | prefetch(prefetch_) { 44 | } 45 | 46 | /* basic IO related */ 47 | iovec *iovs = nullptr; 48 | int iovcnt = 0; 49 | uint32_t size = 0; 50 | uint32_t seek = 0; 51 | 52 | /* ease related */ 53 | off_t blk_addr = -1; 54 | OcfSrcFileCtx *ctx = nullptr; 55 | int err_no = 0; 56 | photon::semaphore sem; 57 | bool prefetch = false; 58 | }; 59 | 60 | struct ease_ocf_config { 61 | /* Initial cache configuration */ 62 | ocf_mngt_cache_config cache; 63 | 64 | /* Cache device config */ 65 | ocf_mngt_cache_device_config device; 66 | 67 | /* Core initial config */ 68 | ocf_mngt_core_config core; 69 | }; 70 | 71 | struct ease_ocf_queue { 72 | ocf_queue_t mngt_queue; 73 | ocf_queue_t io_queue; 74 | }; 75 | 76 | /* Context config */ 77 | const ocf_ctx_config *get_context_config(); 78 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/env/ocf_env_headers.h: -------------------------------------------------------------------------------- 1 | #ifndef __OCF_ENV_HEADERS_H__ 2 | #define __OCF_ENV_HEADERS_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | /* TODO: Move prefix printing to context logger. */ 9 | #define OCF_LOGO "OCF" 10 | #define OCF_PREFIX_SHORT "[" OCF_LOGO "] " 11 | #define OCF_PREFIX_LONG "Open CAS Framework" 12 | 13 | #define OCF_VERSION_MAIN 20 14 | #define OCF_VERSION_MAJOR 3 15 | #define OCF_VERSION_MINOR 0 16 | 17 | #endif /* __OCF_ENV_HEADERS_H__ */ 18 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/env/ocf_env_list.h: -------------------------------------------------------------------------------- 1 | #ifndef __OCF_ENV_LIST__ 2 | #define __OCF_ENV_LIST__ 3 | 4 | #define LIST_POISON1 ((void *)0x101) 5 | #define LIST_POISON2 ((void *)0x202) 6 | 7 | /** 8 | * List entry structure mimicking linux kernel based one. 9 | */ 10 | struct list_head { 11 | struct list_head *next; 12 | struct list_head *prev; 13 | }; 14 | 15 | /** 16 | * start an empty list 17 | */ 18 | #define INIT_LIST_HEAD(l) { (l)->prev = l; (l)->next = l; } 19 | 20 | /** 21 | * Add item to list head. 22 | * @param it list entry to be added 23 | * @param l1 list main node (head) 24 | */ 25 | static inline void list_add(struct list_head *it, struct list_head *l1) 26 | { 27 | it->prev = l1; 28 | it->next = l1->next; 29 | 30 | l1->next->prev = it; 31 | l1->next = it; 32 | } 33 | 34 | /** 35 | * Add item it to tail. 36 | * @param it list entry to be added 37 | * @param l1 list main node (head) 38 | */ 39 | static inline void list_add_tail(struct list_head *it, struct list_head *l1) 40 | { 41 | it->prev = l1->prev; 42 | it->next = l1; 43 | 44 | l1->prev->next = it; 45 | l1->prev = it; 46 | } 47 | 48 | /** 49 | * check if a list is empty (return true) 50 | * @param l1 list main node (head) 51 | */ 52 | static inline int list_empty(struct list_head *l1) 53 | { 54 | return l1->next == l1; 55 | } 56 | 57 | /** 58 | * delete an entry from a list 59 | * @param it list entry to be deleted 60 | */ 61 | static inline void list_del(struct list_head *it) 62 | { 63 | it->next->prev = it->prev; 64 | it->prev->next = it->next; 65 | } 66 | 67 | /** 68 | * Move element to list head. 69 | * @param it list entry to be moved 70 | * @param l1 list main node (head) 71 | */ 72 | static inline void list_move(struct list_head *it, struct list_head *l1) 73 | { 74 | list_del(it); 75 | list_add(it, l1); 76 | } 77 | 78 | /** 79 | * Move element to list tail. 80 | * @param it list entry to be moved 81 | * @param l1 list main node (head) 82 | */ 83 | static inline void list_move_tail(struct list_head *it, struct list_head *l1) 84 | { 85 | list_del(it); 86 | list_add_tail(it, l1); 87 | } 88 | 89 | /** 90 | * Extract an entry. 91 | * @param list_head_i list head item, from which entry is extracted 92 | * @param item_type type (struct) of list entry 93 | * @param field_name name of list_head field within item_type 94 | */ 95 | #define list_entry(list_head_i, item_type, field_name) \ 96 | (item_type *)(((void*)(list_head_i)) - offsetof(item_type, field_name)) 97 | 98 | #define list_first_entry(list_head_i, item_type, field_name) \ 99 | list_entry((list_head_i)->next, item_type, field_name) 100 | 101 | /** 102 | * @param iterator uninitialized list_head pointer, to be used as iterator 103 | * @param plist list head (main node) 104 | */ 105 | #define list_for_each(iterator, plist) \ 106 | for (iterator = (plist)->next; \ 107 | (iterator)->next != (plist)->next; \ 108 | iterator = (iterator)->next) 109 | 110 | /** 111 | * Safe version of list_for_each which works even if entries are deleted during 112 | * loop. 113 | * @param iterator uninitialized list_head pointer, to be used as iterator 114 | * @param q another uninitialized list_head, used as helper 115 | * @param plist list head (main node) 116 | */ 117 | /* 118 | * Algorithm handles situation, where q is deleted. 119 | * consider in example 3 element list with header h: 120 | * 121 | * h -> 1 -> 2 -> 3 -> 122 | *1. i q 123 | * 124 | *2. i q 125 | * 126 | *3. q i 127 | */ 128 | #define list_for_each_safe(iterator, q, plist) \ 129 | for (iterator = (q = (plist)->next->next)->prev; \ 130 | (q) != (plist)->next; \ 131 | iterator = (q = (q)->next)->prev) 132 | 133 | #define _list_entry_helper(item, head, field_name) \ 134 | list_entry(head, typeof(*item), field_name) 135 | 136 | /** 137 | * Iterate over list entries. 138 | * @param list pointer to list item (iterator) 139 | * @param plist pointer to list_head item 140 | * @param field_name name of list_head field in list entry 141 | */ 142 | #define list_for_each_entry(item, plist, field_name) \ 143 | for (item = _list_entry_helper(item, (plist)->next, field_name); \ 144 | _list_entry_helper(item, (item)->field_name.next, field_name) !=\ 145 | _list_entry_helper(item, (plist)->next, field_name); \ 146 | item = _list_entry_helper(item, (item)->field_name.next, field_name)) 147 | 148 | /** 149 | * Safe version of list_for_each_entry which works even if entries are deleted 150 | * during loop. 151 | * @param list pointer to list item (iterator) 152 | * @param q another pointer to list item, used as helper 153 | * @param plist pointer to list_head item 154 | * @param field_name name of list_head field in list entry 155 | */ 156 | #define list_for_each_entry_safe(item, q, plist, field_name) \ 157 | for (item = _list_entry_helper(item, (plist)->next, field_name), \ 158 | q = _list_entry_helper(item, (item)->field_name.next, field_name); \ 159 | _list_entry_helper(item, (item)->field_name.next, field_name) != \ 160 | _list_entry_helper(item, (plist)->next, field_name); \ 161 | item = q, q = _list_entry_helper(q, (q)->field_name.next, field_name)) 162 | 163 | #endif // __OCF_ENV_LIST__ 164 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/env/utils_mpool.cpp: -------------------------------------------------------------------------------- 1 | #ifdef __cplusplus 2 | extern "C" { 3 | #include "ocf_env.h" 4 | #include "utils_mpool.h" 5 | } 6 | #else 7 | #include "ocf_env.h" 8 | #include "utils_mpool.h" 9 | #endif 10 | 11 | struct env_mpool { 12 | int mpool_max; 13 | /*!< Max mpool allocation order */ 14 | 15 | env_allocator *allocator[env_mpool_max]; 16 | /*!< OS handle to memory pool */ 17 | 18 | uint32_t hdr_size; 19 | /*!< Data header size (constant allocation part) */ 20 | 21 | uint32_t elem_size; 22 | /*!< Per element size increment (variable allocation part) */ 23 | 24 | bool fallback; 25 | /*!< Should mpool fallback to vmalloc */ 26 | 27 | int flags; 28 | /*!< Allocation flags */ 29 | }; 30 | 31 | struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, 32 | int flags, int mpool_max, bool fallback, 33 | const uint32_t limits[env_mpool_max], 34 | const char *name_perfix, 35 | bool zero) 36 | { 37 | uint32_t i; 38 | char name[MPOOL_ALLOCATOR_NAME_MAX] = { '\0' }; 39 | int result; 40 | struct env_mpool *mpool; 41 | size_t size; 42 | 43 | mpool = (env_mpool *)env_zalloc(sizeof(*mpool), ENV_MEM_NORMAL); 44 | if (!mpool) 45 | return NULL; 46 | 47 | mpool->flags = flags; 48 | mpool->fallback = fallback; 49 | mpool->mpool_max = mpool_max; 50 | mpool->hdr_size = hdr_size; 51 | mpool->elem_size = elem_size; 52 | 53 | for (i = 0; i < (uint32_t) min(env_mpool_max, mpool_max + 1); i++) { 54 | result = snprintf(name, sizeof(name), "%s_%u", name_perfix, 55 | (1 << i)); 56 | if (result < 0 || result >= (int) sizeof(name)) 57 | goto err; 58 | 59 | size = hdr_size + (elem_size * (1 << i)); 60 | 61 | mpool->allocator[i] = env_allocator_create_extended( 62 | size, name, limits ? limits[i] : -1, 63 | zero); 64 | 65 | if (!mpool->allocator[i]) 66 | goto err; 67 | } 68 | 69 | return mpool; 70 | 71 | err: 72 | env_mpool_destroy(mpool); 73 | return NULL; 74 | } 75 | 76 | void env_mpool_destroy(struct env_mpool *mallocator) 77 | { 78 | if (mallocator) { 79 | uint32_t i; 80 | 81 | for (i = 0; i < env_mpool_max; i++) 82 | if (mallocator->allocator[i]) 83 | env_allocator_destroy(mallocator->allocator[i]); 84 | 85 | env_free(mallocator); 86 | } 87 | } 88 | 89 | static env_allocator *env_mpool_get_allocator( 90 | struct env_mpool *mallocator, uint32_t count) 91 | { 92 | unsigned int idx; 93 | 94 | if (unlikely(count == 0)) 95 | return nullptr; 96 | 97 | idx = 31 - __builtin_clz(count); 98 | 99 | if (__builtin_ffs(count) <= idx) 100 | idx++; 101 | 102 | if (idx >= env_mpool_max || idx > (unsigned int) mallocator->mpool_max) 103 | return NULL; 104 | 105 | return mallocator->allocator[idx]; 106 | } 107 | 108 | void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags) 109 | { 110 | void *items = NULL; 111 | env_allocator *allocator; 112 | size_t size = mpool->hdr_size + (mpool->elem_size * count); 113 | 114 | allocator = env_mpool_get_allocator(mpool, count); 115 | 116 | if (allocator) { 117 | items = env_allocator_new(allocator); 118 | } else if(mpool->fallback) { 119 | items = env_zalloc(size, 0); 120 | } 121 | 122 | #ifdef ZERO_OR_NULL_PTR 123 | if (ZERO_OR_NULL_PTR(items)) 124 | return NULL; 125 | #endif 126 | 127 | return items; 128 | } 129 | 130 | void *env_mpool_new(struct env_mpool *mpool, uint32_t count) 131 | { 132 | return env_mpool_new_f(mpool, count, mpool->flags); 133 | } 134 | 135 | bool env_mpool_del(struct env_mpool *mpool, 136 | void *items, uint32_t count) 137 | { 138 | env_allocator *allocator; 139 | 140 | allocator = env_mpool_get_allocator(mpool, count); 141 | 142 | if (allocator) 143 | env_allocator_del(allocator, items); 144 | else if (mpool->fallback) 145 | env_free(items); 146 | else 147 | return false; 148 | 149 | return true; 150 | } 151 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/env/utils_mpool.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_MPOOL_H_ 2 | #define UTILS_MPOOL_H_ 3 | 4 | #include 5 | #include 6 | 7 | #define MPOOL_ALLOCATOR_NAME_MAX 128 8 | 9 | enum { 10 | env_mpool_1, 11 | env_mpool_2, 12 | env_mpool_4, 13 | env_mpool_8, 14 | env_mpool_16, 15 | env_mpool_32, 16 | env_mpool_64, 17 | env_mpool_128, 18 | 19 | env_mpool_max 20 | }; 21 | 22 | struct env_mpool; 23 | 24 | /** 25 | * @brief Create CAS memory pool 26 | * 27 | * @param hdr_size size of constant allocation part 28 | * @param elem_size size increment for each element 29 | * @param flags Allocation flags 30 | * @param mpool_max Maximal allocator size (power of two) 31 | * @param fallback Should allocations fall back to vmalloc if allocator fails 32 | * @param limits Array of rpool preallocation limits per each mpool allocation 33 | * order or NULL if defaults are to be used. Array should have 34 | * mpool_max elements 35 | * @param name_prefix Format name prefix 36 | * 37 | * @return CAS memory pool 38 | */ 39 | struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, 40 | int flags, int mpool_max, bool fallback, 41 | const uint32_t limits[env_mpool_max], 42 | const char *name_perfix, bool zero); 43 | 44 | /** 45 | * @brief Destroy existing memory pool 46 | * 47 | * @param mpool memory pool 48 | */ 49 | void env_mpool_destroy(struct env_mpool *mpool); 50 | 51 | /** 52 | * @brief Allocate new items of memory pool 53 | * 54 | * @note Allocation based on ATOMIC memory pool and this function can be called 55 | * when IRQ disable 56 | * 57 | * @param mpool CAS memory pool reference 58 | * @param count Count of elements to be allocated 59 | * 60 | * @return Pointer to the new items 61 | */ 62 | void *env_mpool_new(struct env_mpool *mpool, uint32_t count); 63 | 64 | /** 65 | * @brief Allocate new items of memory pool with specified allocation flag 66 | * 67 | * @param mpool CAS memory pool reference 68 | * @param count Count of elements to be allocated 69 | * @param flags Kernel allocation falgs 70 | * 71 | * @return Pointer to the new items 72 | */ 73 | void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags); 74 | 75 | /** 76 | * @brief Free existing items of memory pool 77 | * 78 | * @param mpool CAS memory pool reference 79 | * @param items Items to be freed 80 | * @param count - Count of elements to be free 81 | * 82 | * @return Allocation was freed 83 | */ 84 | bool env_mpool_del(struct env_mpool *mpool, void *items, uint32_t count); 85 | 86 | #endif /* UTILS_MPOOL_H_ */ 87 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/provider.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "ctx.h" 7 | #include "queue.h" 8 | #include "volume.h" 9 | 10 | class ease_ocf_provider { 11 | public: 12 | ease_ocf_provider(ease_ocf_volume_params *params, size_t prefetch_unit) 13 | : m_volume_params(params), m_prefetch_unit(prefetch_unit) { 14 | } 15 | 16 | int start(bool reload_media); 17 | 18 | int stop(); 19 | 20 | /** 21 | * @brief Caller should guarantee that offset + count does not exceed the EOF. 22 | * @return On success, count is returned. On error, other value is returned. 23 | */ 24 | ssize_t ocf_pread(void *buf, size_t count, off_t offset, size_t blk_addr, OcfSrcFileCtx *ctx, 25 | bool prefetch = false); 26 | 27 | size_t prefetch_unit() const { 28 | return m_prefetch_unit; 29 | } 30 | 31 | static const size_t SectorSize; 32 | 33 | private: 34 | static constexpr const char *CACHE_NAME = "Ease Cache"; 35 | static constexpr const char *CORE_NAME = "Ease Core"; 36 | static constexpr const char *CACHE_UUID = "cache"; 37 | static constexpr const char *CORE_UUID = "core"; 38 | 39 | /* Main control context */ 40 | ocf_ctx_t m_ctx = nullptr; 41 | 42 | /* Cache and Core objects */ 43 | ocf_cache_t m_cache = nullptr; 44 | ocf_core_t m_core = nullptr; 45 | 46 | /* Configurations */ 47 | ease_ocf_config m_cfg = {}; 48 | 49 | /* Queue */ 50 | ease_ocf_queue *m_queue = nullptr; 51 | 52 | /* Volume parameters */ 53 | ease_ocf_volume_params *m_volume_params; // owned by external class 54 | 55 | size_t m_prefetch_unit; 56 | 57 | /* 58 | * | | | | 59 | * | | | | 60 | * |< lower_padding >| |< mid_count >| |< upper_padding >| 61 | * ----|-----------------------|-----[sectors ...]-----|-----------------------|---- 62 | * | | | | 63 | * lower_bound | | upper_bound 64 | * offset offset + count 65 | */ 66 | struct alignment { 67 | off_t lower_bound; 68 | size_t lower_padding; 69 | size_t mid_count; 70 | size_t upper_padding; 71 | off_t upper_bound; 72 | }; 73 | 74 | static void prepare_aligned_iov(size_t count, off_t offset, alignment &a, IOVector &iov, 75 | const void *buf, void *padding_buf); 76 | 77 | static void copy_aligned_iov(size_t count, alignment &a, IOVector &iov, void *buf); 78 | }; -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/queue.cpp: -------------------------------------------------------------------------------- 1 | #include "queue.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | static void *run(void *args) { 8 | auto queue = (ocf_queue_t)args; 9 | ocf_queue_run(queue); 10 | return nullptr; 11 | } 12 | 13 | /* Pooled queue kicker */ 14 | class QueueKicker { 15 | public: 16 | explicit QueueKicker(ocf_queue_t queue) : m_queue(queue) { 17 | } 18 | QueueKicker(ocf_queue_t queue, size_t vcpu_num, int ev_engine, int io_engine, int mode) : m_queue(queue) { 19 | work_pool = new photon::WorkPool(vcpu_num, ev_engine, io_engine, mode); 20 | } 21 | ~QueueKicker() { 22 | delete work_pool; 23 | } 24 | 25 | inline void kick() { 26 | if (work_pool) { 27 | work_pool->async_call(new auto([this](){ run(m_queue); })); 28 | } else { 29 | photon::thread_create(run, m_queue); 30 | } 31 | } 32 | 33 | private: 34 | /* associated OCF queue */ 35 | ocf_queue_t m_queue; 36 | /* thread pool */ 37 | photon::WorkPool* work_pool = nullptr; 38 | }; 39 | 40 | int init_queues(ocf_queue_t mngt_queue, ocf_queue_t io_queue) { 41 | auto mngt_queue_kicker = new QueueKicker(mngt_queue, 2, 0, 0, 64); 42 | auto io_queue_kicker = new QueueKicker(io_queue, 4, photon::INIT_EVENT_EPOLL, photon::INIT_IO_LIBCURL, 64); 43 | 44 | ocf_queue_set_priv(mngt_queue, mngt_queue_kicker); 45 | ocf_queue_set_priv(io_queue, io_queue_kicker); 46 | return 0; 47 | } 48 | 49 | /* Callback for OCF to kick the queue thread */ 50 | static void queue_thread_kick(ocf_queue_t q) { 51 | auto qk = (QueueKicker *)ocf_queue_get_priv(q); 52 | qk->kick(); 53 | } 54 | 55 | /* Callback for OCF to stop the queue thread */ 56 | static void queue_thread_stop(ocf_queue_t q) { 57 | auto qk = (QueueKicker *)ocf_queue_get_priv(q); 58 | delete qk; 59 | } 60 | 61 | /* Queue ops */ 62 | static const ocf_queue_ops queue_ops = { 63 | .kick = queue_thread_kick, 64 | .kick_sync = nullptr, 65 | .stop = queue_thread_stop, 66 | }; 67 | 68 | const ocf_queue_ops *get_queue_ops() { 69 | return &queue_ops; 70 | } 71 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/queue.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | extern "C" { 4 | #include 5 | } 6 | 7 | int init_queues(ocf_queue_t mngt_queue, ocf_queue_t io_queue); 8 | 9 | const ocf_queue_ops *get_queue_ops(); 10 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ease_bindings/volume.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | extern "C" { 6 | #include 7 | } 8 | 9 | #define EASE_OCF_VOLUME_TYPE 1 10 | 11 | struct ease_ocf_volume_params { 12 | size_t blk_size; 13 | size_t media_size; 14 | photon::fs::IFile *media_file; 15 | bool enable_logging; 16 | }; 17 | 18 | int volume_init(ocf_ctx_t ocf_ctx); 19 | 20 | void volume_cleanup(ocf_ctx_t ocf_ctx); 21 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/include/ocf: -------------------------------------------------------------------------------- 1 | ../ocf/inc -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/ocf_namespace.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class OcfNamespace : public Object { 11 | public: 12 | explicit OcfNamespace(size_t blk_size) : m_blk_size(blk_size) { 13 | } 14 | 15 | /** 16 | * @brief Validate parameters, and load some metadata into memory 17 | */ 18 | virtual int init() = 0; 19 | 20 | /** NsInfo indicates a file's starting offset within its filesystem's address space, and its 21 | * size */ 22 | struct NsInfo { 23 | off_t blk_idx; 24 | size_t file_size; 25 | }; 26 | 27 | /** 28 | * @brief Locate a source file in namespace 29 | * @param[in] file_path 30 | * @param[in] src_file 31 | * @param[out] info 32 | * @retval 0 for success 33 | */ 34 | virtual int locate_file(const estring &file_path, photon::fs::IFile *src_file, 35 | NsInfo &info) = 0; 36 | 37 | size_t block_size() const { 38 | return m_blk_size; 39 | } 40 | 41 | protected: 42 | size_t m_blk_size; 43 | }; 44 | 45 | OcfNamespace *new_ocf_namespace_on_fs(size_t blk_size, photon::fs::IFileSystem *fs); 46 | 47 | OcfNamespace *new_ocf_namespace_on_rocksdb(size_t blk_size); -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | find_package(curl REQUIRED) 5 | 6 | add_executable(ocf_perf_test ocf_perf_test.cpp) 7 | target_include_directories( 8 | ocf_perf_test PUBLIC 9 | ${CURL_INCLUDE_DIRS} 10 | ${PHOTON_INCLUDE_DIR} 11 | ) 12 | target_link_libraries( 13 | ocf_perf_test 14 | gflags pthread ${CURL_LIBRARIES} 15 | photon_static overlaybd_lib 16 | ) 17 | 18 | add_test( 19 | NAME ocf_perf_test 20 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/ocf_perf_test --ut_pass=true 21 | ) 22 | -------------------------------------------------------------------------------- /src/overlaybd/cache/ocf_cache/test/flags.conf: -------------------------------------------------------------------------------- 1 | --multi_files_test=false 2 | --cache_type=ocf 3 | --page_size=8192 4 | --io_engine=1 5 | --concurrency=32 6 | --media_file_size_gb=2 7 | --media_file=/root/cache-bench/media 8 | --ocf_prefetch_unit=0 9 | 10 | --random_read=true 11 | --src_file=/root/cache-bench/src 12 | --dst_file= 13 | --total_requests=1000000 14 | 15 | --num_files=40 16 | --file_size_mb=50 -------------------------------------------------------------------------------- /src/overlaybd/cache/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | add_executable(cache_test cache_test.cpp) 8 | target_include_directories(cache_test PUBLIC ${PHOTON_INCLUDE_DIR}) 9 | target_link_libraries(cache_test gtest gtest_main gflags pthread photon_static overlaybd_lib) 10 | 11 | add_test( 12 | NAME cache_test 13 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/cache_test 14 | ) -------------------------------------------------------------------------------- /src/overlaybd/cache/test/random_generator.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | namespace Cache { 23 | 24 | /* DataTypes for random generator */ 25 | template 26 | class RandomValueGen { 27 | public: 28 | RandomValueGen() {} 29 | virtual ~RandomValueGen() {} 30 | virtual T next() = 0; 31 | }; 32 | 33 | // an uniform int random generator that produces inclusive-inclusive value range 34 | class UniformInt32RandomGen: public RandomValueGen { 35 | public: 36 | UniformInt32RandomGen() = default; 37 | UniformInt32RandomGen(uint32_t start, uint32_t end, int seed = 1213) 38 | : gen_(seed), dis_(start, end) {} 39 | uint32_t next() override { return dis_(gen_); } 40 | void seed(std::mt19937::result_type val) { gen_.seed(val); } 41 | 42 | private: 43 | std::mt19937 gen_; 44 | std::uniform_int_distribution dis_; 45 | }; 46 | 47 | class UniformCharRandomGen : public RandomValueGen { 48 | public: 49 | UniformCharRandomGen(uint32_t start, uint32_t end, int seed = 1213) 50 | : gen_(seed), dis_(start, end) {} 51 | unsigned char next() { return dis_(gen_); } 52 | 53 | private: 54 | std::mt19937 gen_; 55 | std::uniform_int_distribution dis_; 56 | }; 57 | 58 | } // namespace Cache 59 | -------------------------------------------------------------------------------- /src/overlaybd/gzindex/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_TAR "*.cpp") 2 | add_library(gzindex_lib STATIC ${SOURCE_TAR}) 3 | 4 | target_include_directories(gzindex_lib PUBLIC ${PHOTON_INCLUDE_DIR}) 5 | target_link_libraries(gzindex_lib photon_static) 6 | 7 | if(BUILD_TESTING) 8 | add_subdirectory(test) 9 | endif() 10 | -------------------------------------------------------------------------------- /src/overlaybd/gzindex/gzfile.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #include "photon/fs/filesystem.h" 19 | #include "gzfile_index.h" 20 | 21 | 22 | extern photon::fs::IFile* new_gzfile(photon::fs::IFile* gzip_file, photon::fs::IFile* index, bool ownership = false); 23 | 24 | //chunksize: 25 | //1MB: 1048576 26 | //2MB: 2097152 27 | //3MB: 3145728 28 | //4MB: 4194304 29 | 30 | //dict_compress_algo: 31 | //0: don't compress dictionary 32 | //1: compress dictionary with zlib 33 | 34 | //dict_compress_level: 35 | //0: no compression 36 | //1: best speed 37 | //9: best compression 38 | extern int create_gz_index(photon::fs::IFile* gzip_file, const char *index_file_path, 39 | off_t chunk_size=GZ_CHUNK_SIZE, int dict_compress_algo=GZ_DICT_COMPERSS_ALGO, int dict_compress_level=GZ_COMPRESS_LEVEL); 40 | 41 | bool is_gzfile(photon::fs::IFile* file); 42 | -------------------------------------------------------------------------------- /src/overlaybd/gzindex/gzfile_index.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "photon/common/checksum/crc32c.h" 25 | #include "photon/fs/filesystem.h" 26 | 27 | #define GZ_CHUNK_SIZE 1048576 28 | #define GZ_DICT_COMPERSS_ALGO 1 29 | #define GZ_COMPRESS_LEVEL 6 30 | 31 | #define WINSIZE 32768U 32 | #define DEFLATE_BLOCK_UNCOMPRESS_MAX_SIZE 65536U 33 | #define GZFILE_INDEX_MAGIC "ddgzidx" 34 | 35 | struct IndexFileHeader { 36 | char magic[8]; 37 | uint8_t major_version; 38 | uint8_t minor_version; 39 | uint8_t dict_compress_algo; 40 | int8_t dict_compress_level; 41 | uint8_t flag; 42 | int32_t span; 43 | int32_t window; 44 | int32_t index_size; 45 | int64_t index_num; 46 | int64_t gzip_file_size; 47 | int64_t index_file_size; 48 | int64_t uncompress_file_size; 49 | char reserve[256]; 50 | int64_t index_start; 51 | int64_t index_area_len; 52 | uint32_t crc; 53 | uint32_t cal_crc() { return crc32c(this, sizeof(IndexFileHeader) - sizeof(crc));} 54 | std::string to_str() { 55 | std::stringstream ss; 56 | ss << "magic:" << magic 57 | <<",major_version:"<(major_version) 58 | <<",minor_version:"<(minor_version) 59 | <<",dict_compress_algo:"<(dict_compress_algo) 60 | <<",dict_compress_level:"<(dict_compress_level) 61 | <<",flag:" <(flag) 62 | <<",span:" < INDEX; 87 | 88 | class IndexFilterRecorder; 89 | IndexFilterRecorder *new_index_filter(IndexFileHeader *h, INDEX *index, photon::fs::IFile *save_as); 90 | void delete_index_filter(IndexFilterRecorder *&); 91 | 92 | int init_index_header(photon::fs::IFile* src, IndexFileHeader &h, off_t span, int dict_compress_algo, int dict_compress_level); 93 | 94 | int create_index_entry(z_stream strm, IndexFilterRecorder *filter, off_t en_pos, off_t de_pos, unsigned char *window); 95 | 96 | int save_index_to_file(IndexFileHeader &h, INDEX& index, photon::fs::IFile *index_file, ssize_t gzip_file_size = -1); 97 | -------------------------------------------------------------------------------- /src/overlaybd/gzindex/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | add_executable(gzindex_test test.cpp) 8 | target_include_directories(gzindex_test PUBLIC ${PHOTON_INCLUDE_DIR}) 9 | target_link_libraries(gzindex_test gtest gtest_main gflags pthread photon_static 10 | gzindex_lib gzip_lib cache_lib checksum_lib) 11 | 12 | add_test( 13 | NAME gzindex_test 14 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/gzindex_test 15 | ) 16 | -------------------------------------------------------------------------------- /src/overlaybd/gzip/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_GZIP "*.cpp") 2 | 3 | add_library(gzip_lib STATIC ${SOURCE_GZIP}) 4 | target_include_directories(gzip_lib PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ) 7 | target_link_libraries(gzip_lib photon_static checksum_lib) 8 | 9 | # if(BUILD_TESTING) 10 | # add_subdirectory(test) 11 | # endif() 12 | -------------------------------------------------------------------------------- /src/overlaybd/gzip/gz.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | class IGzFile : public photon::fs::VirtualReadOnlyFile { 25 | public : 26 | // return full filename of gzip index 27 | virtual std::string save_index() = 0; 28 | virtual std::string sha256_checksum() = 0; 29 | }; 30 | 31 | photon::fs::IFile* open_gzfile_adaptor(const char *path); 32 | IGzFile* open_gzstream_file(IStream *sock, ssize_t st_size, 33 | bool save_index = true, const char *uid = nullptr, const char *workdir = nullptr); 34 | -------------------------------------------------------------------------------- /src/overlaybd/lsmt/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_LSMT "*.cpp") 2 | 3 | add_library(lsmt_lib STATIC ${SOURCE_LSMT}) 4 | target_include_directories(lsmt_lib PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ) 7 | 8 | if(BUILD_TESTING) 9 | add_subdirectory(test) 10 | endif() 11 | -------------------------------------------------------------------------------- /src/overlaybd/lsmt/format_spec.md: -------------------------------------------------------------------------------- 1 | # Overlaybd layer blob format 2 | ## Overview 3 | Each layer blob consists of 4 sections, namely header, data, index and trailer, 4 | as described below. 5 | 6 | | Section | Size (bytes) | Description | 7 | | :---: | :----: | :--- | 8 | | header | 4096 | file header | 9 | | data | variable | raw data (over) written in the layer | 10 | | index | variable | a table that associates logical block addressing (LBA) with raw data | 11 | | trailer | 4096 | file trailer (similar to header) | 12 | 13 | ## header 14 | The format of header is described as below. All fields are little-endian. 15 | 16 | | Field | Offset (bytes) | Size (bytes) | Description | 17 | | :---: | :----: | :----: | :--- | 18 | | magic0 | 0 | 8 | "LSMT\0\1\2" (and an implicit '\0') | 19 | | magic1 | 8 | 16 | 65 7E 63 D2, 94 44 08 4C, A2 D2 C8 EC, 4F CF AE 8A | 20 | | size | 24 | uint32_t | size of the header struct (390), excluding the tail padding | 21 | | flags | 28 | uint32_t | bits for flags* (see later for details) | 22 | | index_offset | 32 | uint64_t | index offset | 23 | | index_size | 40 | uint64_t | index size | 24 | | virtual_size | 48 | uint64_t | virtual size of the image that users see | 25 | | uuid | 56 | 37 | a string that identifies this blob | 26 | | parent_uuid | 93 | 37 | a string that identifies the parent (previous) blob | 27 | | from | 130 | uint8_t | deprecated | 28 | | to | 131 | uint8_t | deprecated | 29 | | version | 132 | uint8_t | version of this blob | 30 | | sub_version | 133 | uint8_t | sub-version of this blob | 31 | | user_tag | 134 | 256 | commit message (user-defined text) | 32 | | reserved | 390 | 3706 | reserved space for future use (offset 390 ~ 4095), should be 0 | 33 | 34 | **flags:** 35 | 36 | | Field | Offset (bits) | Description | 37 | | :---: | :----: | :--- | 38 | | is_header | 0 | header (1) or trailer (0) | 39 | | type | 1 | this is a data file (1) or index file (0) | 40 | | sealed | 2 | this file is sealed (1) or not (0) | 41 | | gc_layer | 3 | this is a gc layer (1) or normal layer (0) | 42 | | sparse_rw | 4 | this is a sparse rw layer | 43 | | info_valid | 5 | information validity of the fields *after* flags (they were initially invalid (0) after creation; and readers must resort to trailer when they meet such headers) | 44 | | reserved | 6~31 | reserved for future use; must be 0s | 45 | 46 | 47 | ## raw data 48 | This section do not have a perticular format. The data stored in this section is 49 | supposed to be referenced by the index entries. And it's possible for this section 50 | to contain garbage data blocks that are not referenced by the any entry of the 51 | index. 52 | 53 | 54 | ## index 55 | The index section is a table that associates logical block addressing (LBA) with 56 | raw data. Its format is simply a sorted array of record entries. Each entry is 57 | a 128-bit struct defined as below: 58 | 59 | | Field | Offset (bits) | Size (bits) | Type | Description | 60 | | :---: | :----: | :----: | :----: | :--- | 61 | | offset | 0 | 50 | uint64_t | logical block addressing (in unit of 512-byte sector) | 62 | | length | 50 | 14 | uint32_t | length of the mapping (in unit of 512-byte sector) | 63 | | moffset | 64 | 55 | uint64_t | mapped block addressing in the blob (in unit of 512-byte sector) | 64 | | zeroed | 119 | 1 | bool | whether the block is all zero (without actual mapping) | 65 | | tag | 120 | 8 | uint8_t | runtime usage only, should be 0 on-disk | 66 | 67 | ## trailer 68 | An updated edition of header, in the same format. Trailer is useful in 69 | append-only storage during creation of the blob. Use trailer whenever 70 | possible. -------------------------------------------------------------------------------- /src/overlaybd/lsmt/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | add_executable(lsmt_test test.cpp) 8 | target_include_directories(lsmt_test PUBLIC ${PHOTON_INCLUDE_DIR}) 9 | target_link_libraries(lsmt_test gtest gtest_main gflags pthread photon_static overlaybd_lib) 10 | 11 | add_test( 12 | NAME lsmt_test 13 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/lsmt_test 14 | ) 15 | 16 | -------------------------------------------------------------------------------- /src/overlaybd/registryfs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_REGISTRYFS "*.cpp") 2 | 3 | find_package(curl REQUIRED) 4 | 5 | add_library(registryfs_lib STATIC ${SOURCE_REGISTRYFS}) 6 | target_include_directories(registryfs_lib PUBLIC 7 | ${CURL_INCLUDE_DIRS} 8 | ${rapidjson_SOURCE_DIR}/include 9 | ${PHOTON_INCLUDE_DIR} 10 | ) 11 | -------------------------------------------------------------------------------- /src/overlaybd/registryfs/registryfs.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | class RegistryFS : public photon::fs::IFileSystem { 24 | public: 25 | virtual int setAccelerateAddress(const char* addr = "") = 0; 26 | }; 27 | 28 | using PasswordCB = Delegate, const char *>; 29 | 30 | extern "C" { 31 | photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback, 32 | const char *caFile = nullptr, 33 | uint64_t timeout = -1, 34 | const char *cert_file = nullptr, 35 | const char *key_file = nullptr, 36 | const char *__ = nullptr); 37 | 38 | photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, 39 | const char *caFile = nullptr, 40 | uint64_t timeout = -1, 41 | const char *cert_file = nullptr, 42 | const char *key_file = nullptr, 43 | const char *customized_ua = nullptr); 44 | 45 | photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile, 46 | const std::string &upload_url, 47 | const std::string &username, 48 | const std::string &password, 49 | uint64_t timeout, 50 | ssize_t upload_bs = -1, 51 | const char *cert_file = nullptr, 52 | const char *key_file = nullptr); 53 | 54 | int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest); 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/overlaybd/stream_convertor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_SERV "*.cpp") 2 | 3 | add_executable(overlaybd-streamConv ${SOURCE_SERV}) 4 | target_include_directories(overlaybd-streamConv PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ${rapidjson_SOURCE_DIR}/include 7 | ) 8 | target_link_libraries(overlaybd-streamConv 9 | photon_static 10 | gzip_lib 11 | gzindex_lib 12 | tar_lib 13 | yaml-cpp 14 | ) 15 | 16 | # if(BUILD_TESTING) 17 | # add_subdirectory(test) 18 | # endif() 19 | -------------------------------------------------------------------------------- /src/overlaybd/stream_convertor/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include "config_utils.h" 21 | 22 | namespace App { 23 | 24 | struct LogConfigPara : public App::ConfigGroup { 25 | APPCFG_CLASS 26 | APPCFG_PARA(level, uint32_t, 1); 27 | APPCFG_PARA(path, std::string, "/var/log/overlaybd/stream-convertor.log"); 28 | APPCFG_PARA(limitSizeMB, uint32_t, 10); 29 | APPCFG_PARA(rotateNum, int, 3); 30 | APPCFG_PARA(mode, std::string, "stdout"); 31 | }; 32 | 33 | struct GlobalConfigPara : public App::ConfigGroup { 34 | APPCFG_CLASS; 35 | APPCFG_PARA(udsAddr, std::string, ""); 36 | APPCFG_PARA(httpAddr, std::string, "127.0.0.1"); 37 | APPCFG_PARA(httpPort, int, 9101); 38 | APPCFG_PARA(reusePort, bool, true); 39 | 40 | APPCFG_PARA(workDir, std::string, "/tmp/stream_conv"); 41 | //APPCFG_PARA(ServerConfig, ServerConfigPara); 42 | APPCFG_PARA(logConfig, LogConfigPara); 43 | }; 44 | 45 | struct AppConfig : public App::ConfigGroup { 46 | APPCFG_CLASS 47 | 48 | APPCFG_PARA(globalConfig, GlobalConfigPara); 49 | }; 50 | 51 | } // namespace ImageConfigNS 52 | -------------------------------------------------------------------------------- /src/overlaybd/stream_convertor/config_utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace App { 10 | 11 | struct ConfigGroup : public YAML::Node { 12 | ConfigGroup() = default; 13 | ConfigGroup(const YAML::Node &node) : YAML::Node(node){}; 14 | ConfigGroup(const std::string &filename) { 15 | parseYAML(std::move(filename)); 16 | } 17 | 18 | void parseYAML(const std::string &fn) { 19 | Clone(YAML::LoadFile(fn)); 20 | } 21 | 22 | static size_t charfilter(char *dst, const char *src, char extract, size_t maxlen = 256UL) { 23 | size_t i; 24 | for (i = 0; (*src) && i < maxlen; i++, src++) { 25 | while (*src == '-') { 26 | src++; 27 | } 28 | if (!(*src)) 29 | break; 30 | dst[i] = *(src); 31 | } 32 | dst[i] = 0; 33 | return i; 34 | } 35 | }; 36 | 37 | #define APPCFG_PARA(ParaName, ParaType, ...) \ 38 | inline ParaType ParaName() const { \ 39 | return operator[](#ParaName).as(__VA_ARGS__); \ 40 | } 41 | 42 | #define APPCFG_CLASS using ConfigGroup::ConfigGroup; 43 | 44 | // merge two yaml nodes 45 | // generate new node with full data 46 | static YAML::Node mergeConfig(const YAML::Node &lhs, const YAML::Node &rhs) { 47 | // if one of nodes is not type of map 48 | // just return rhs 49 | if (lhs.Type() != YAML::NodeType::Map || rhs.Type() != YAML::NodeType::Map) 50 | return YAML::Clone(rhs); 51 | // both are map, merge two maps 52 | YAML::Node ret = YAML::Clone(lhs); 53 | for (auto &node : rhs) { 54 | auto key = node.first.as(); 55 | if (ret[key].IsDefined()) { 56 | // if key exists in lhs, merge recursivily 57 | ret[key] = mergeConfig(lhs[key], node.second); 58 | } else { 59 | // just add as new key 60 | ret[key] = Clone(node.second); 61 | } 62 | } 63 | return ret; 64 | } 65 | 66 | } // namespace App 67 | 68 | namespace YAML { 69 | 70 | template 71 | struct convert { 72 | template 73 | static Node encode(const T &rhs) { 74 | return rhs; 75 | } 76 | 77 | template 78 | static bool decode(const Node &node, T &rhs) { 79 | rhs = T(node); 80 | return true; 81 | } 82 | }; 83 | 84 | } // namespace YAML -------------------------------------------------------------------------------- /src/overlaybd/tar/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_TAR "*.cpp") 2 | 3 | add_library(tar_lib STATIC ${SOURCE_TAR}) 4 | target_include_directories(tar_lib PUBLIC 5 | ${PHOTON_INCLUDE_DIR} 6 | ) 7 | 8 | if(BUILD_TESTING) 9 | add_subdirectory(test) 10 | endif() 11 | 12 | add_subdirectory(erofs) 13 | target_link_libraries(tar_lib PRIVATE erofs_lib) -------------------------------------------------------------------------------- /src/overlaybd/tar/erofs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(FetchContent) 2 | 3 | FetchContent_Declare( 4 | erofs-utils 5 | GIT_REPOSITORY https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git 6 | GIT_TAG eec6f7a2755dfccc8f655aa37cf6f26db9164e60 7 | ) 8 | 9 | FetchContent_MakeAvailable(erofs-utils) 10 | 11 | execute_process( 12 | COMMAND ./autogen.sh 13 | WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} 14 | ) 15 | execute_process( 16 | COMMAND ./configure --disable-lz4 --disable-lzma --without-libzstd --without-uuid --disable-multithreading 17 | WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} 18 | ) 19 | execute_process( 20 | COMMAND make 21 | WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} 22 | ) 23 | 24 | set(EROFS_LIB_INCLUDE_DIR "${erofs-utils_SOURCE_DIR}/include/" CACHE PATH "erofs-utils include path.") 25 | set(EROFS_CONFIG_FILE "${erofs-utils_SOURCE_DIR}/config.h" CACHE PATH "erofs-utils config file.") 26 | set(EROFS_LIB_STATIC "${erofs-utils_SOURCE_DIR}/lib/.libs/liberofs.a" CACHE PATH "erofs-utils static lib.") 27 | 28 | file(GLOB EROFS_SOURCE "*.cpp") 29 | 30 | add_library(erofs_lib STATIC ${EROFS_SOURCE}) 31 | 32 | target_include_directories(erofs_lib PRIVATE 33 | ${PHOTON_INCLUDE_DIR} 34 | ) 35 | 36 | target_include_directories(erofs_lib PRIVATE 37 | ${EROFS_LIB_INCLUDE_DIR} 38 | ) 39 | 40 | target_compile_options(erofs_lib PRIVATE "-include${EROFS_CONFIG_FILE}") 41 | target_link_libraries(erofs_lib PRIVATE ${EROFS_LIB_STATIC}) 42 | 43 | if(BUILD_TESTING) 44 | add_subdirectory(test) 45 | endif() 46 | -------------------------------------------------------------------------------- /src/overlaybd/tar/erofs/erofs_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // common contents used internally by erofs 18 | 19 | #pragma once 20 | 21 | #include "erofs/io.h" 22 | #include 23 | #include 24 | #include 25 | 26 | /* block-related definitions */ 27 | #define SECTOR_SIZE 512ULL 28 | #define SECTOR_BITS 9 29 | 30 | /* address alignment operation */ 31 | #define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1))) 32 | #define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1)) 33 | #define erofs_min(a, b) (a) < (b) ? (a) : (b) 34 | 35 | #define EROFS_ROOT_XATTR_SZ (16 * 1024) 36 | 37 | #define EROFS_UNIMPLEMENTED 1 38 | 39 | #define EROFS_UNIMPLEMENTED_FUNC(ret_type, cls, func, ret) \ 40 | ret_type cls::func { \ 41 | return ret; \ 42 | } 43 | 44 | struct liberofs_inmem_sector { 45 | char data[SECTOR_SIZE]; 46 | }; 47 | 48 | /* 49 | * Internal cache of EROFS, used to accelerate 50 | * the read and write operations of an IFile. 51 | */ 52 | class ErofsCache { 53 | public: 54 | ErofsCache(photon::fs::IFile *file, unsigned long int capacity): 55 | file(file), capacity(capacity) 56 | {} 57 | ~ErofsCache() {} 58 | ssize_t write_sector(u64 addr, char *buf); 59 | ssize_t read_sector(u64 addr, char *buf); 60 | int flush(); 61 | public: 62 | photon::fs::IFile *file; 63 | long unsigned int capacity; 64 | std::mapcaches; 65 | std::set dirty; 66 | }; 67 | 68 | /* 69 | * Encapsulation of IFile by liberofs, 70 | * including I/O operations and ErofsCache. 71 | */ 72 | struct liberofs_file { 73 | struct erofs_vfops ops; 74 | photon::fs::IFile *file; 75 | ErofsCache *cache; 76 | }; 77 | 78 | /* helper functions for reading and writing photon files */ 79 | extern ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len, 80 | ErofsCache *cache); 81 | extern ssize_t erofs_write_photon_file(const void *buf, u64 offset, 82 | size_t len, ErofsCache *cache); 83 | 84 | /* I/O controllers for target */ 85 | extern ssize_t erofs_target_pread(struct erofs_vfile *vf, void *buf, 86 | u64 offset, size_t len); 87 | extern ssize_t erofs_target_pwrite(struct erofs_vfile *vf, const void *buf, 88 | u64 offset, size_t len); 89 | extern int erofs_target_fsync(struct erofs_vfile *vf); 90 | extern int erofs_target_fallocate(struct erofs_vfile *vf, u64 offset, 91 | size_t len, bool pad); 92 | extern int erofs_target_ftruncate(struct erofs_vfile *vf, u64 length); 93 | extern ssize_t erofs_target_read(struct erofs_vfile *vf, void *buf, size_t len); 94 | extern off_t erofs_target_lseek(struct erofs_vfile *vf, u64 offset, int whence); 95 | 96 | /* I/O controllers for source */ 97 | extern ssize_t erofs_source_pread(struct erofs_vfile *vf, void *buf, u64 offset, 98 | size_t len); 99 | extern ssize_t erofs_source_pwrite(struct erofs_vfile *vf, const void *buf, 100 | u64 offset, size_t len); 101 | extern int erofs_source_fsync(struct erofs_vfile *vf); 102 | extern int erofs_source_fallocate(struct erofs_vfile *vf, 103 | u64 offset, size_t len, bool pad); 104 | extern int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length); 105 | extern ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf, 106 | size_t bytes); 107 | extern off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, 108 | int whence); 109 | -------------------------------------------------------------------------------- /src/overlaybd/tar/erofs/erofs_fs.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | class ErofsFileSystem: public photon::fs::IFileSystem { 26 | public: 27 | ErofsFileSystem(photon::fs::IFile *imgfile, uint64_t blksize); 28 | ~ErofsFileSystem(); 29 | photon::fs::IFile* open(const char *pathname, int flags); 30 | photon::fs::IFile* open(const char *pathname, int flags, mode_t mode); 31 | photon::fs::IFile* creat(const char *pathname, mode_t mode); 32 | int mkdir(const char *pathname, mode_t mode); 33 | int rmdir(const char *pathname); 34 | int symlink(const char *oldname, const char *newname); 35 | ssize_t readlink(const char *path, char *buf, size_t bufsiz); 36 | int link(const char *oldname, const char *newname); 37 | int rename(const char *oldname, const char *newname); 38 | int unlink(const char *filename); 39 | int chmod(const char *pathname, mode_t mode); 40 | int chown(const char *pathname, uid_t owner, gid_t group); 41 | int lchown(const char *pathname, uid_t owner, gid_t group); 42 | int statfs(const char *path, struct statfs *buf); 43 | int statvfs(const char *path, struct statvfs *buf); 44 | int stat(const char *path, struct stat *buf); 45 | int lstat(const char *path, struct stat *buf); 46 | int access(const char *pathname, int mode); 47 | int truncate(const char *path, off_t length); 48 | int utime(const char *path, const struct utimbuf *file_times); 49 | int utimes(const char *path, const struct timeval times[2]); 50 | int lutimes(const char *path, const struct timeval times[2]); 51 | int mknod(const char *path, mode_t mode, dev_t dev); 52 | int syncfs(); 53 | photon::fs::DIR* opendir(const char *name); 54 | private: 55 | struct ErofsFileSystemInt; 56 | struct ErofsFileSystemInt *fs_private; 57 | 58 | friend class ErofsFile; 59 | }; 60 | 61 | class ErofsFile: public photon::fs::VirtualReadOnlyFile, public photon::fs::IFileXAttr { 62 | public: 63 | ErofsFile(ErofsFileSystem *fs); 64 | ~ErofsFile(); 65 | photon::fs::IFileSystem *filesystem(); 66 | int fstat(struct stat *buf); 67 | int fiemap(struct photon::fs::fiemap *map); 68 | ssize_t pread(void *buf, size_t count, off_t offset); 69 | ssize_t fgetxattr(const char *name, void *value, size_t size); 70 | ssize_t flistxattr(char *list, size_t size); 71 | int fsetxattr(const char *name, const void *value, size_t size, int flags); 72 | int fremovexattr(const char *name); 73 | private: 74 | ErofsFileSystem *fs; 75 | struct ErofsFileInt; 76 | ErofsFileInt *file_private; 77 | 78 | friend class ErofsFileSystem; 79 | }; 80 | 81 | class ErofsDir: public photon::fs::DIR { 82 | public: 83 | std::vector<::dirent> m_dirs; 84 | ::dirent *direntp = nullptr; 85 | long loc; 86 | ErofsDir(std::vector<::dirent> &dirs); 87 | ~ErofsDir(); 88 | int closedir(); 89 | dirent *get(); 90 | int next(); 91 | void rewinddir(); 92 | void seekdir(long loc); 93 | long telldir(); 94 | }; 95 | 96 | bool erofs_check_fs(const photon::fs::IFile *imgfile); 97 | photon::fs::IFileSystem *erofs_create_fs(photon::fs::IFile *imgfile, uint64_t blksz); 98 | -------------------------------------------------------------------------------- /src/overlaybd/tar/erofs/liberofs.h: -------------------------------------------------------------------------------- 1 | #ifndef TAREROFS_INTERFACE_H 2 | #define TAREROFS_INTERFACE_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class LibErofs { 9 | public: 10 | LibErofs(photon::fs::IFile *target, uint64_t blksize, bool import_tar_headers = false); 11 | ~LibErofs(); 12 | int extract_tar(photon::fs::IFile *source, bool meta_only, bool first_layer); 13 | private: 14 | photon::fs::IFile * target= nullptr; /* output file */ 15 | uint64_t blksize; 16 | bool ddtaridx; 17 | }; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/overlaybd/tar/erofs/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | # erofs simple test 8 | add_executable(erofs_simple_test erofs_simple.cpp) 9 | target_include_directories(erofs_simple_test PUBLIC ${PHOTON_INCLUDE_DIR}) 10 | target_link_libraries(erofs_simple_test gtest gtest_main pthread photon_static 11 | tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib overlaybd_image_lib) 12 | 13 | target_include_directories(erofs_simple_test PUBLIC 14 | ${PHOTON_INCLUDE_DIR} 15 | ${rapidjson_SOURCE_DIR}/include 16 | ) 17 | 18 | add_test( 19 | NAME erofs_simple_test 20 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_simple_test 21 | ) 22 | 23 | # erofs stress test 24 | add_executable(erofs_stress_test erofs_stress.cpp erofs_stress_base.cpp) 25 | target_include_directories(erofs_stress_test PUBLIC ${PHOTON_INCLUDE_DIR}) 26 | target_link_libraries(erofs_stress_test gtest gtest_main pthread photon_static 27 | tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib overlaybd_image_lib) 28 | 29 | target_include_directories(erofs_stress_test PUBLIC 30 | ${PHOTON_INCLUDE_DIR} 31 | ${rapidjson_SOURCE_DIR}/include 32 | ) 33 | 34 | add_test( 35 | NAME erofs_stress_test 36 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/erofs_stress_test 37 | ) 38 | -------------------------------------------------------------------------------- /src/overlaybd/tar/tar_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #include 19 | 20 | // tarfile and tarfs are not complete implementation of tar. 21 | // only for overlaybd remote blob, which stores one blob file with tar header and trailer. 22 | // used to skip header for file I/O. 23 | 24 | photon::fs::IFileSystem *new_tar_fs_adaptor(photon::fs::IFileSystem *fs); 25 | 26 | int is_tar_file(photon::fs::IFile *file); 27 | photon::fs::IFile *new_tar_file_adaptor(photon::fs::IFile *file); 28 | -------------------------------------------------------------------------------- /src/overlaybd/tar/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | add_executable(untar_test test.cpp) 8 | target_include_directories(untar_test PUBLIC ${PHOTON_INCLUDE_DIR}) 9 | target_link_libraries(untar_test gtest gtest_main pthread photon_static 10 | tar_lib lsmt_lib gzip_lib gzindex_lib checksum_lib) 11 | 12 | add_test( 13 | NAME untar_test 14 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/untar_test 15 | ) 16 | -------------------------------------------------------------------------------- /src/overlaybd/tar/whiteout.cpp: -------------------------------------------------------------------------------- 1 | #include "libtar.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | // whiteoutPrefix prefix means file is a whiteout. If this is followed by a 17 | // filename this means that file has been removed from the base layer. 18 | // See https://github.com/opencontainers/image-spec/blob/main/layer.md#whiteouts 19 | const std::string whiteoutPrefix = ".wh."; 20 | 21 | // whiteoutMetaPrefix prefix means whiteout has a special meaning and is not 22 | // for removing an actual file. Normally these files are excluded from exported 23 | // archives. 24 | const std::string whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix; 25 | 26 | // whiteoutOpaqueDir file means directory has been made opaque - meaning 27 | // readdir calls to this directory do not follow to lower layers. 28 | const std::string whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq"; 29 | 30 | const std::string paxSchilyXattr = "SCHILY.xattr."; 31 | 32 | int UnTar::mkdir_hier(const std::string_view &dir) { 33 | struct stat s; 34 | std::string path = remove_last_slash(dir); 35 | if (fs->lstat(path.c_str(), &s) == 0) { 36 | if (S_ISDIR(s.st_mode)) { 37 | return 0; 38 | } else { 39 | errno = ENOTDIR; 40 | // LOG_ERROR("mkdir ` failed, `", path.c_str(), strerror(errno)); 41 | return -1; 42 | } 43 | } 44 | 45 | return photon::fs::mkdir_recursive(dir, fs, 0755); 46 | } 47 | 48 | int UnTar::remove_all(const std::string &path, bool rmdir) { 49 | if (fs == nullptr || path.empty()) { 50 | LOG_ERROR("remove_all ` failed, fs is null or path is empty", path); 51 | return -1; 52 | } 53 | struct stat statBuf; 54 | if (fs->lstat(path.c_str(), &statBuf) == 0) { // get stat 55 | if (S_ISDIR(statBuf.st_mode) == 0) { // not dir 56 | if (unpackedPaths.find(path) == unpackedPaths.end()) { 57 | fs->unlink(path.c_str()); 58 | } 59 | return 0; 60 | } 61 | } else { 62 | LOG_ERRNO_RETURN(0, -1, "get path ` stat failed", path); 63 | } 64 | 65 | auto dirs = fs->opendir(path.c_str()); 66 | if (dirs == nullptr) { 67 | LOG_ERRNO_RETURN(0, -1, "open dir ` failed", path); 68 | } 69 | dirent *dirInfo; 70 | while ((dirInfo = dirs->get()) != nullptr) { 71 | if (strcmp(dirInfo->d_name, ".") != 0 && strcmp(dirInfo->d_name, "..") != 0) { 72 | remove_all(path + "/" + dirInfo->d_name); 73 | } 74 | dirs->next(); 75 | } 76 | 77 | fs->closedir(dirs); 78 | if (rmdir && unpackedPaths.find(path) == unpackedPaths.end()) { 79 | fs->rmdir(path.c_str()); 80 | } 81 | 82 | return 0; 83 | } 84 | 85 | // 1 whiteout done 86 | // 0 not whiteout 87 | // -1 error 88 | int UnTar::convert_whiteout(const char *filename) { 89 | photon::fs::Path p(filename); 90 | auto dir = std::string(p.dirname()); 91 | auto base = p.basename(); 92 | if (base == whiteoutOpaqueDir) { 93 | struct stat buf; 94 | std::string path = remove_last_slash(dir); 95 | if (fs->lstat(path.c_str(), &buf) < 0) { 96 | LOG_ERRNO_RETURN(0, -1, "failed to lstat ", VALUE(dir)); 97 | } 98 | 99 | remove_all(path, false); 100 | return 1; 101 | } 102 | if (base.substr(0, whiteoutPrefix.size()) == whiteoutPrefix) { 103 | auto opath = dir + std::string(base.data() + whiteoutPrefix.size()); 104 | remove_all(opath); 105 | return 1; 106 | } 107 | return 0; 108 | } 109 | -------------------------------------------------------------------------------- /src/overlaybd/zfile/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_ZFILE "*.cpp") 2 | file(GLOB SOURCE_LZ4 "lz4/*.c") 3 | file(GLOB SOURCE_CRC32 "crc32/crc32c.cpp") 4 | 5 | set (CMAKE_CXX_STANDARD 17) 6 | add_library(crc32_lib STATIC ${SOURCE_CRC32}) 7 | target_include_directories(crc32_lib PUBLIC 8 | ${PHOTON_INCLUDE_DIR} 9 | ) 10 | 11 | if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) 12 | target_compile_options(crc32_lib PUBLIC -msse4.2 -mcrc32) 13 | else() 14 | if (NOT CMAKE_CXX_FLAGS MATCHES "-march=|-mcpu=") 15 | check_cxx_compiler_flag(-mcpu=native COMPILER_HAS_NATIVE_FLAG) 16 | if (COMPILER_HAS_NATIVE_FLAG) 17 | target_compile_options(crc32_lib PRIVATE -mcpu=native) 18 | else () 19 | target_compile_options(crc32_lib PRIVATE -mcpu=generic+crc) 20 | endif () 21 | endif () 22 | endif() 23 | 24 | if(ENABLE_DSA OR ENABLE_ISAL) 25 | add_subdirectory(thirdparty) 26 | add_dependencies(crc32_lib thirdparty_lib) 27 | target_link_directories(crc32_lib PUBLIC ${LIBRARY_OUTPUT_PATH}) 28 | target_include_directories(crc32_lib PUBLIC ${LIBRARY_OUTPUT_PATH}/include) 29 | if(ENABLE_DSA) 30 | target_link_libraries(crc32_lib -lpci -ldmlhl -ldl) 31 | target_compile_definitions(crc32_lib PUBLIC -DENABLE_DSA) 32 | endif() 33 | if(ENABLE_ISAL) 34 | target_compile_options(crc32_lib PUBLIC -mavx512f) 35 | target_compile_definitions(crc32_lib PUBLIC -DENABLE_ISAL) 36 | target_link_libraries(crc32_lib -lisal) 37 | endif() 38 | endif() 39 | set (CMAKE_CXX_STANDARD 14) 40 | 41 | add_library(zfile_lib STATIC ${SOURCE_ZFILE} ${SOURCE_LZ4}) 42 | target_link_libraries(zfile_lib photon_static crc32_lib ${LIBZSTD}) 43 | 44 | if (ENABLE_QAT) 45 | target_compile_definitions(zfile_lib PUBLIC -DENABLE_QAT) 46 | target_link_libraries(zfile_lib -lpthread -lpci) 47 | #target_link_libraries(zfile_lib -lqat_s -lusdm_drv_s -lpthread -lpci) 48 | endif() 49 | 50 | if (BUILD_TESTING) 51 | add_subdirectory(test) 52 | endif () 53 | -------------------------------------------------------------------------------- /src/overlaybd/zfile/README.md: -------------------------------------------------------------------------------- 1 | # CRC 2 | 3 | ## The default order for CRC 4 | If multiple features are included, the default order is: 5 | 1. DSA 6 | 2. AVX512 7 | 3. SSE4.2 8 | 4. software 9 | 10 | ## DSA 11 | 12 | ### Introduction 13 | 14 | Intel® DSA is a high-performance data copy and transformation accelerator that will be 15 | integrated into future Intel® processors, targeted for optimizing streaming data movement 16 | and transformation operations common with applications for high-performance storage, 17 | networking, persistent memory, and various data processing applications. 18 | 19 | Intel® DSA replaces the Intel® QuickData Technology, which is a part of Intel® I/O 20 | Acceleration Technology. 21 | 22 | The goal is to provide higher overall system performance for data mover and transformation 23 | operations while freeing up CPU cycles for higher-level functions. Intel® DSA enables 24 | high-performance data mover capability to/from volatile memory, persistent memory, 25 | memory-mapped I/O, and through a Non-Transparent Bridge (NTB) device to/from remote volatile 26 | and persistent memory on another node in a cluster. Enumeration and configuration are done 27 | with a PCI Express* compatible programming interface to the Operating System (OS) and can be 28 | controlled through a device driver. 29 | 30 | Besides the basic data mover operations, Intel® DSA supports a set of transformation 31 | operations on memory. For example: 32 | 33 | Generate and test CRC checksum, or Data Integrity Field (DIF) to support storage and 34 | networking applications. 35 | Memory Compare and delta generate/merge to support VM migration, VM Fast check-pointing, and 36 | software-managed memory deduplication usages. 37 | 38 | We did a performance comparison test, and the QPS calculated by CRC were 960124 and 7977434 39 | respectively in the case of 4K size without and with DSA. About a nine-times improvement 40 | using DSA in performance. 41 | 42 | ### Check DSA hardware 43 | ```bash 44 | $ lspci -nn | grep 8086:0b25 45 | ``` 46 | 47 | If get any information, which mean the system have DSA hardware. 48 | 49 | Function check_dsa() is used to ensure that system have DSA hardware. 50 | 51 | ### Dependencies 52 | Ubuntu 20.04 : apt-get install -y libpci-dev 53 | CentOS 8.4 : yum install -y pciutils-devel 54 | 55 | ### Unit test for DSA Calculate CRC 56 | ```bash 57 | $ cmake -DENABLE_DSA=1 -DBUILD_TESTING=1 .. 58 | $ make -j 59 | $./output/zfile_test 60 | ``` 61 | 62 | ## AVX512 63 | 64 | ### Introduction 65 | 66 | Here we use Intel(R) Intelligent Storage Acceleration Library for AVX512. 67 | 68 | Intel(R) Intelligent Storage Acceleration Library is a collection of optimized low-level functions targeting storage applications. ISA-L includes: 69 | 70 | Erasure codes - Fast block Reed-Solomon type erasure codes for any encode/decode matrix in GF(2^8). 71 | CRC - Fast implementations of cyclic redundancy check. Six different polynomials supported. 72 | iscsi32, ieee32, t10dif, ecma64, iso64, jones64. 73 | Raid - calculate and operate on XOR and P+Q parity found in common RAID implementations. 74 | Compression - Fast deflate-compatible data compression. 75 | De-compression - Fast inflate-compatible data compression. 76 | igzip - A command line application like gzip, accelerated with ISA-L. 77 | 78 | ### Check if the system have AVX512 79 | 80 | __builtin_cpu_supports("avx512f") is used to check if the system have AVX512. 81 | 82 | ### Dependencies 83 | Ubuntu 20.04 : apt-get install -y nasm 84 | CentOS 8.4 : yum install -y nasm 85 | 86 | ### Unit test for DSA Calculate CRC 87 | ```bash 88 | $ cmake -DENABLE_ISAL=1 -DBUILD_TESTING=1 .. 89 | $ make -j 90 | $./output/zfile_test 91 | ``` 92 | 93 | # Compression 94 | 95 | ## QAT 96 | 97 | ### Introduction 98 | Accelerate encryption and compression to provide improved 99 | efficiency, scalability and performance – for data in motion or 100 | at rest, in-flight or in process. Intel QuickAssist Technology (Intel QAT) 101 | can save cycles, time and space of applications as varied as networking 102 | to enterprise, cloud to storage, content delivery to database. 103 | 104 | ### Check QAT hardware 105 | ```bash 106 | $ lspci -nn | grep 8086:4940 107 | ``` 108 | 109 | ### Dependencies 110 | Ubuntu 20.04 : apt-get install -y libpci-dev 111 | CentOS 8.4 : yum install -y pciutils-devel 112 | 113 | ### Unit test for QAT compression and decompression 114 | ```bash 115 | $ cmake -DENABLE_QAT=1 -DBUILD_TESTING=1 .. 116 | $ make -j 117 | $./output/zfile_test 118 | ``` -------------------------------------------------------------------------------- /src/overlaybd/zfile/compressor.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #ifndef __COMPRESSOR_H__ 17 | #define __COMPRESSOR_H__ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | namespace photon { 25 | namespace fs { 26 | class IFile; 27 | } 28 | } 29 | 30 | namespace ZFile { 31 | 32 | /* CompressOption will write into file */ 33 | class CompressOptions { 34 | public: 35 | const static uint8_t MINI_LZO = 0; 36 | const static uint8_t LZ4 = 1; 37 | const static uint8_t ZSTD = 2; 38 | const static uint32_t DEFAULT_BLOCK_SIZE = 4096; // 8192;//32768; 39 | 40 | uint32_t block_size = DEFAULT_BLOCK_SIZE; 41 | uint8_t algo = LZ4; // algorithm 42 | uint8_t level = 0; // compress level 43 | uint8_t use_dict = 0; 44 | uint8_t __padding_0 = 0; 45 | uint32_t reserved = 0; 46 | uint32_t dict_size = 0; 47 | uint8_t verify = 0; 48 | uint8_t __padding_1[7] = {0}; 49 | 50 | CompressOptions(uint8_t type = LZ4, uint32_t block_size = DEFAULT_BLOCK_SIZE, 51 | uint8_t verify = 0) 52 | : block_size(block_size), algo(type), verify(verify) { 53 | } 54 | }; 55 | static_assert(sizeof(CompressOptions) == 24, "sizeof(CompressOptions) != 24"); 56 | 57 | class CompressArgs { 58 | public: 59 | photon::fs::IFile *fdict = nullptr; 60 | std::unique_ptr dict_buf = nullptr; 61 | CompressOptions opt; 62 | bool overwrite_header; 63 | int workers; 64 | 65 | CompressArgs(const CompressOptions &opt, photon::fs::IFile *dict = nullptr, 66 | unsigned char *dict_buf = nullptr, bool overwrite_header = false, int workers = 1) 67 | : fdict(dict), dict_buf(dict_buf), opt(opt), overwrite_header(overwrite_header), workers(workers) { 68 | if (fdict || dict_buf) { 69 | this->opt.use_dict = 1; 70 | } 71 | }; 72 | }; 73 | 74 | class ICompressor { 75 | public: 76 | virtual ~ICompressor(){}; 77 | /* 78 | return compressed buffer size. 79 | return -1 when error occurred. 80 | */ 81 | virtual int compress(const unsigned char *src, size_t src_len, unsigned char *dst, 82 | size_t dst_len) = 0; 83 | /* 84 | return the number of batches in QAT compressing... 85 | */ 86 | virtual int nbatch() = 0; 87 | virtual int compress_batch(const unsigned char *src, size_t *src_chunk_len, unsigned char *dst, 88 | size_t dst_buffer_capacity, size_t *dst_chunk_len /* save result chunk length */, 89 | size_t nchunk) = 0; 90 | /* 91 | return decompressed buffer size. 92 | return -1 when error occurred. 93 | */ 94 | virtual int decompress(const unsigned char *src, size_t src_len, unsigned char *dst, 95 | size_t dst_len) = 0; 96 | 97 | virtual int decompress_batch(const unsigned char *src, size_t *src_chunk_len, unsigned char *dst, 98 | size_t dst_buffer_capacity, size_t *dst_chunk_len /* save result chunk length */, 99 | size_t nchunk) = 0; 100 | }; 101 | 102 | extern "C" ICompressor *create_compressor(const CompressArgs *args); 103 | } // namespace ZFile 104 | 105 | #endif -------------------------------------------------------------------------------- /src/overlaybd/zfile/crc32/crc32c.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | #pragma once 21 | 22 | namespace crc32 { 23 | 24 | extern uint32_t crc32c(const void *data, size_t nbytes); 25 | extern uint32_t crc32c(const std::string &text); 26 | 27 | extern uint32_t crc32c_extend(const void *data, size_t nbytes, uint32_t crc); 28 | extern uint32_t crc32c_extend(const std::string &text, uint32_t crc); 29 | 30 | namespace testing { 31 | extern uint32_t crc32c_slow(const void *data, size_t nbytes, uint32_t crc); 32 | extern uint32_t crc32c_fast(const void *data, size_t nbytes, uint32_t crc); 33 | } // namespace testing 34 | 35 | } // namespace crc32 36 | 37 | -------------------------------------------------------------------------------- /src/overlaybd/zfile/format_spec.md: -------------------------------------------------------------------------------- 1 | # ZFile format 2 | ## Overview 3 | ZFile is a generic compression format that realizes random read of the compressed 4 | file and online decompression, providing the users with an illusion of reading the 5 | original file. ZFile is not tied with overlaybd. Instead, it works with arbitary 6 | underlay file. 7 | 8 | 9 | | Section | Size (bytes) | Description | 10 | | :---: | :----: | :--- | 11 | | header | 512 | file header | 12 | | data | variable | compressed blocks of the original file | 13 | | dict | variable | optional dictionary to assist decompression | 14 | | index | variable | a jump table that stores the size of each compressed block, which can by easily transformed into offset of the block at runtime | 15 | | trailer | 512 | file trailer (similar to header) | 16 | 17 | ## header 18 | The format of header is described as below. All fields are little-endian. 19 | 20 | | Field | Offset (bytes) | Size (bytes) | Description | 21 | | :---: | :----: | :----: | :--- | 22 | | magic0 | 0 | 8 | "ZFile\0\1" (and an implicit '\0') | 23 | | magic1 | 8 | 16 | 74 75 6A 69, 2E 79 79 66, 40 41 6C 69, 62 61 62 61 | 24 | | size | 24 | uint32_t | size of the header structure, excluding the tail padding | 25 | | digest | 28 | uint32_t | checksum for the range 28-511 bytes in header | 26 | | flags | 32 | uint64_t | bits for flags* (see later for details) | 27 | | index_offset | 40 | uint64_t | index offset | 28 | | index_size | 48 | uint64_t | size of the index section, possibly compressed base on flags | 29 | | original_file_size | 56 | uint64_t | size of the orignal file before compression | 30 | | index_crc | 64 | uint32_t | checksum value of index | 31 | | reserved| 68 | 4 | reserved space, should be 0 | 32 | | block_size| 72 | uint32_t | size of each compression block | 33 | | algo | 76 | uint8_t | compression algorithm | 34 | | level | 77 | uint8_t | compression level | 35 | | use_dict| 78 | bool | whether use dictionary | 36 | | reserved| 79 | 5 | reserved space, should be 0 | 37 | | dict_size | 84 | uint32_t | size of the dictionary section, 0 for non-existence | 38 | | verify | 88 | bool | whether these exists a 4-byte CRC32 checksum following each compressed block | 39 | | reserved| 89 | 423 | reserved space for future use (offset 89 ~ 511), should be 0 | 40 | 41 | **flags:** 42 | 43 | | Field | Offset (bits) | Description | 44 | | :---: | :----: | :--- | 45 | | is_header | 0 | header (1) or trailer (0) | 46 | | type | 1 | this is a data file (1) or index file (0) | 47 | | sealed | 2 | this file is sealed (1) or not (0) | 48 | | info_valid | 3 | information validity of the fields *after* flags (they were initially invalid (0) after creation; and readers must resort to trailer when they meet such headers) | 49 | | digest | 4 | the digest of this header/trailer has been recorded in the digest field | 50 | | index_comperssion | 5 | whether the index has been compressed(1) or not(0) | 51 | | reserved | 6~63 | reserved for future use; must be 0s | 52 | 53 | 54 | ## index 55 | The index section is a table of (uint32_t) compressed size of each data block. 56 | The whole section may be compressed with the same compression algorithm and 57 | level. 58 | 59 | ## trailer 60 | An updated edition of header, in the same format. Trailer is useful in 61 | append-only storage during creation of the blob. Use trailer whenever 62 | possible. 63 | -------------------------------------------------------------------------------- /src/overlaybd/zfile/lz4/lz4-qat.c: -------------------------------------------------------------------------------- 1 | #include "lz4.h" 2 | #include "lz4-qat.h" 3 | #include 4 | 5 | #define sleeptime 100 6 | 7 | int gDebugParam = 1; 8 | 9 | 10 | int qat_init(LZ4_qat_param *pQat) { 11 | int32_t status = 0; 12 | 13 | return (int)status; 14 | } 15 | 16 | int qat_uninit(LZ4_qat_param *pQat) { 17 | int32_t status = 0; 18 | 19 | return status; 20 | } 21 | 22 | // compression operation. 23 | 24 | LZ4LIB_API int LZ4_compress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], 25 | size_t src_chunk_len[], unsigned char *compressed_data[], 26 | size_t dst_chunk_len[], size_t n) { 27 | int32_t status = 0; 28 | for (size_t i = 0; i < n; i++) { 29 | int ret = LZ4_compress_default((const char *)raw_data[i], (char *)compressed_data[i], 30 | src_chunk_len[i], 4096); 31 | dst_chunk_len[i] = ret; 32 | } 33 | return status; 34 | } 35 | 36 | LZ4LIB_API int LZ4_decompress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], 37 | size_t src_chunk_len[], unsigned char *decompressed_data[], 38 | size_t dst_chunk_len[], size_t n){ 39 | int32_t status = 0; 40 | for (size_t i = 0; i < n; i++) { 41 | int ret = LZ4_decompress_safe((const char *)raw_data[i], (char *)decompressed_data[i], 42 | src_chunk_len[i], 4096); 43 | dst_chunk_len[i] = ret; 44 | } 45 | return status; 46 | } 47 | -------------------------------------------------------------------------------- /src/overlaybd/zfile/lz4/lz4-qat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * LZ4 - Fast LZ compression algorithm 3 | * Header File 4 | * Copyright (C) 2011-present, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 homepage : http://www.lz4.org 33 | - LZ4 source repository : https://github.com/lz4/lz4 34 | */ 35 | #if defined(__cplusplus) 36 | extern "C" { 37 | #endif 38 | 39 | #ifndef LZ4_QAT_H_2983827168210 40 | #define LZ4_QAT_H_2983827168210 41 | 42 | #include 43 | #include 44 | /* --- Dependency --- */ 45 | 46 | #ifndef LZ4LIB_VISIBILITY 47 | #if defined(__GNUC__) && (__GNUC__ >= 4) 48 | #define LZ4LIB_VISIBILITY __attribute__((visibility("default"))) 49 | #else 50 | #define LZ4LIB_VISIBILITY 51 | #endif 52 | #endif 53 | #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT == 1) 54 | #define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY 55 | #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT == 1) 56 | #define LZ4LIB_API \ 57 | __declspec(dllimport) \ 58 | LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a \ 59 | function pointer load from the IAT and an indirect jump.*/ 60 | #else 61 | #define LZ4LIB_API LZ4LIB_VISIBILITY 62 | #endif 63 | 64 | typedef struct _LZ4_qat_param LZ4_qat_param; 65 | struct _LZ4_qat_param { 66 | 67 | }; 68 | 69 | /*-************************************ 70 | * Simple Functions 71 | **************************************/ 72 | /*! LZ4_compress_qat() : 73 | Compresses 'srcSize' bytes from buffer 'src' 74 | into already allocated 'dst' buffer of size 'dstCapacity'. 75 | Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). 76 | It also runs faster, so it's a recommended setting. 77 | If the function cannot compress 'src' into a more limited 'dst' budget, 78 | compression stops *immediately*, and the function result is zero. 79 | Note : as a consequence, 'dst' content is not valid. 80 | Note 2 : This function is protected against buffer overflow scenarios (never writes outside 81 | 'dst' buffer, nor read outside 'source' buffer). srcSize : max supported value is 82 | LZ4_MAX_INPUT_SIZE. dstCapacity : size of buffer 'dst' (which must be already allocated) return 83 | : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) 84 | or 0 if compression fails */ 85 | LZ4LIB_API int LZ4_compress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], 86 | size_t src_chunk_len[], unsigned char *compressed_data[], 87 | size_t dst_chunk_len[], size_t n); 88 | 89 | /*! LZ4_decompress_qat() : 90 | compressedSize : is the exact complete size of the compressed block. 91 | dstCapacity : is the size of destination buffer, which must be already allocated. 92 | return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) 93 | If destination buffer is not large enough, decoding will stop and output an error code 94 | (negative value). If the source stream is detected malformed, the function will stop decoding and 95 | return a negative result. This function is protected against malicious data packets. 96 | */ 97 | LZ4LIB_API int LZ4_decompress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], 98 | size_t src_chunk_len[], unsigned char *decompressed_data[], 99 | size_t dst_chunk_len[], size_t n); 100 | 101 | 102 | int qat_init(LZ4_qat_param *pQat); 103 | 104 | int qat_uninit(LZ4_qat_param *pQat); 105 | 106 | #endif /* LZ4_QAT_H_2983827168210 */ 107 | 108 | #if defined(__cplusplus) 109 | } 110 | #endif -------------------------------------------------------------------------------- /src/overlaybd/zfile/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | include_directories($ENV{GFLAGS}/include) 3 | link_directories($ENV{GFLAGS}/lib) 4 | 5 | include_directories($ENV{GTEST}/googletest/include) 6 | link_directories($ENV{GTEST}/lib) 7 | 8 | add_executable(zfile_test ./test.cpp) 9 | 10 | target_link_libraries(zfile_test gtest gtest_main gflags pthread photon_static overlaybd_lib) 11 | target_include_directories(zfile_test PUBLIC ${PHOTON_INCLUDE_DIR}) 12 | add_test( 13 | NAME zfile_test 14 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/zfile_test 15 | ) -------------------------------------------------------------------------------- /src/overlaybd/zfile/thirdparty/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_custom_target(thirdparty_lib) 2 | set(THIRDPARTY_PATH "${CMAKE_SOURCE_DIR}/src/overlaybd/zfile/thirdparty") 3 | 4 | if(ENABLE_DSA) 5 | add_custom_command(TARGET thirdparty_lib 6 | COMMAND cd ${THIRDPARTY_PATH}/DML && git checkout 5a2956362d7b1c65d8aee753938ae9bf5cf0b7fd 7 | COMMAND cd ${THIRDPARTY_PATH}/DML && rm -rf build && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=dmldir .. 8 | COMMAND cd ${THIRDPARTY_PATH}/DML/build && cmake --build . --target install 9 | COMMAND cd ${THIRDPARTY_PATH}/DML/build/dmldir && find ./ -name "libdml*.a" | xargs -i cp {} ${LIBRARY_OUTPUT_PATH}/; 10 | COMMAND cp -r ${THIRDPARTY_PATH}/DML/build/dmldir/include ${LIBRARY_OUTPUT_PATH}/ 11 | ) 12 | endif() 13 | 14 | if(ENABLE_ISAL) 15 | add_custom_command(TARGET thirdparty_lib 16 | COMMAND cd ${THIRDPARTY_PATH}/isa-l && git checkout ad8dce15c6d3f0c7f3d1b486d9c649ed39223b45 17 | COMMAND cd ${THIRDPARTY_PATH}/isa-l && ./autogen.sh 18 | COMMAND cd ${THIRDPARTY_PATH}/isa-l && ./configure 19 | COMMAND cd ${THIRDPARTY_PATH}/isa-l && make 20 | COMMAND cp -r ${THIRDPARTY_PATH}/isa-l/.libs/libisal.a ${LIBRARY_OUTPUT_PATH}/ 21 | COMMAND mkdir -p ${LIBRARY_OUTPUT_PATH}/include && cp ${THIRDPARTY_PATH}/isa-l/include/crc.h ${LIBRARY_OUTPUT_PATH}/include/ 22 | ) 23 | endif() -------------------------------------------------------------------------------- /src/overlaybd/zfile/zfile.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include "compressor.h" 19 | namespace ZFile { 20 | const static size_t MAX_READ_SIZE = 65536; // 64K 21 | 22 | extern "C" photon::fs::IFile *zfile_open_ro(photon::fs::IFile *file, bool verify = false, 23 | bool ownership = false); 24 | 25 | extern "C" int zfile_compress(photon::fs::IFile *src_file, photon::fs::IFile *dst_file, 26 | const CompressArgs *opt = nullptr); 27 | 28 | extern "C" int zfile_decompress(photon::fs::IFile *src_file, photon::fs::IFile *dst_file); 29 | 30 | extern "C" int zfile_validation_check(photon::fs::IFile *src_file); 31 | 32 | 33 | extern "C" photon::fs::IFile *new_zfile_builder(photon::fs::IFile *file, 34 | const CompressArgs *args = nullptr, 35 | bool ownership = false); 36 | 37 | // return 1 if file object is a zfile. 38 | // return 0 if file object is a normal file. 39 | // otherwise return -1. 40 | extern "C" int is_zfile(photon::fs::IFile *file); 41 | } // namespace ZFile 42 | -------------------------------------------------------------------------------- /src/overlaybd/zstd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB SOURCE_ZSTD "*.cpp") 2 | 3 | add_library(zstd_lib STATIC ${SOURCE_ZSTD}) 4 | 5 | target_include_directories(zstd_lib PUBLIC 6 | ${PHOTON_INCLUDE_DIR} 7 | ) 8 | 9 | target_link_libraries(zstd_lib photon_static ${LIBZSTD}) 10 | -------------------------------------------------------------------------------- /src/overlaybd/zstd/zstdfile.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "zstdfile.h" 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | class ZStdAdaptorFile : public photon::fs::VirtualReadOnlyFile { 28 | public: 29 | ZStdAdaptorFile(photon::fs::IFile* file, bool ownership) 30 | : m_file(file) 31 | , m_ownership(ownership) 32 | , m_stream(ZSTD_createDStream()) 33 | , m_buffer(ZSTD_DStreamInSize()) 34 | , m_input({m_buffer.data(), 0, 0}) { 35 | } 36 | 37 | ~ZStdAdaptorFile() { 38 | ZSTD_freeDStream(m_stream); 39 | if (m_ownership) { 40 | delete m_file; 41 | } 42 | } 43 | 44 | ssize_t read(void *buf, size_t count) override { 45 | ssize_t bytes_read = 0; 46 | ZSTD_outBuffer output = { buf, count, 0 }; 47 | while (output.pos < output.size) { 48 | // Output buffer is not filled, read more. 49 | if (m_input.pos == m_input.size) { 50 | // Reached the end of the input buffer, read more compressed data. 51 | ssize_t read_compressed_bytes = m_file->read(m_buffer.data(), m_buffer.size()); 52 | if (read_compressed_bytes == 0) { 53 | // EOF reached. 54 | return bytes_read; 55 | } 56 | if (read_compressed_bytes < 0 || read_compressed_bytes > (ssize_t) m_buffer.size()) { 57 | // Error reading file. 58 | LOG_ERRNO_RETURN(EIO, -1, "read buffer error"); 59 | } 60 | m_input.size = read_compressed_bytes; 61 | m_input.pos = 0; 62 | } 63 | const size_t prev_pos = output.pos; 64 | const size_t ret = ZSTD_decompressStream(m_stream, &output, &m_input); 65 | if (ZSTD_isError(ret)) { 66 | LOG_ERRNO_RETURN(EIO, -1, "failed to decompress zstd frame"); 67 | } 68 | bytes_read += output.pos - prev_pos; 69 | if (ret == 0) { 70 | // End of this ZSTD frame, set up for the next one (if there is one). 71 | ZSTD_initDStream(m_stream); 72 | } 73 | } 74 | return bytes_read; 75 | } 76 | 77 | int fstat(struct stat *buf) override { 78 | return m_file->fstat(buf); 79 | } 80 | 81 | UNIMPLEMENTED_POINTER(photon::fs::IFileSystem *filesystem() override); 82 | UNIMPLEMENTED(off_t lseek(off_t offset, int whence) override); 83 | UNIMPLEMENTED(ssize_t readv(const struct iovec *iov, int iovcnt) override); 84 | UNIMPLEMENTED(ssize_t preadv(const struct iovec *iov, int iovcnt, off_t offset) override); 85 | private: 86 | photon::fs::IFile* m_file; 87 | bool m_ownership; // whether we own m_file and should delete it. 88 | ZSTD_DStream* m_stream; 89 | 90 | std::vector m_buffer; 91 | ZSTD_inBuffer m_input; 92 | }; 93 | 94 | photon::fs::IFile *open_zstdfile_adaptor(photon::fs::IFile* file, bool ownership) { 95 | return new ZStdAdaptorFile(file, ownership); 96 | } 97 | 98 | const uint8_t ZSTD_MAGIC_HEADER[4] = {0x28, 0xB5, 0x2F, 0xFD}; 99 | 100 | bool is_zstdfile(photon::fs::IFile* file) { 101 | char buf[4] = {0}; 102 | ssize_t readn = file->read(buf, 4); 103 | file->lseek(0, 0); 104 | return readn == 4 && memcmp(buf, ZSTD_MAGIC_HEADER, 4) == 0; 105 | } 106 | -------------------------------------------------------------------------------- /src/overlaybd/zstd/zstdfile.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | // Opens a ZSTD compressed file and provides a view of the decompressed data. 22 | // Takes ownership of `file` when `ownership` is true (default). 23 | photon::fs::IFile* open_zstdfile_adaptor(photon::fs::IFile* file, bool ownership = true); 24 | 25 | bool is_zstdfile(photon::fs::IFile* file); 26 | -------------------------------------------------------------------------------- /src/prefetch.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace photon::fs; 23 | /* 24 | * Prefetcher provides a capability which can download remote data before the true I/O trigger. 25 | * It contains two modes: 26 | ==== static mode: replay the record I/O trace ==== 27 | * 1. Prefetcher supports `record` and `replay` operations on a specific container image. 28 | * It will persist R/W metadata of all layers into a trace file during container boot-up, 29 | * and then replay this file, in order to retrieve data in advance. 30 | * 31 | * 2. When starting recording, a lock file will be created. Delete it to stop recording. 32 | * 33 | * 3. After recording stopped, data will be dumped into trace file. A OK file will be created 34 | * to indicate dump finished. 35 | * 36 | * 4. In conclusion, the work modes are as follows: 37 | * trace file non-existent => Disabled 38 | * trace file exist but empty => Start Recording 39 | * lock file deleted or prefetcher destructed => Stop Recording 40 | * trace file exist and not empty => Replay 41 | * 42 | ==== dynamic mode: reload specified the data from a external file list ==== 43 | * overlaybd will read the filelist to prefetch from the value of 'recordTracePath' in config.v1.json 44 | * { 45 | * ... 46 | * "recordTracePath": "path/to/filelist", 47 | * ... 48 | *} 49 | * # cat /path/to/filelist 50 | * /absolute/path/to/fileA 51 | * /absolute/path/to/fileB 52 | * ... 53 | * /absolute/path/to/directory ## support but not recommend 54 | * 55 | */ 56 | class Prefetcher : public Object { 57 | public: 58 | enum class Mode { 59 | Disabled, 60 | Record, 61 | Replay, 62 | }; 63 | enum class TraceOp : char { READ = 'R', WRITE = 'W' }; 64 | 65 | virtual int record(TraceOp op, uint32_t layer_index, size_t count, off_t offset) = 0; 66 | 67 | virtual int replay(const IFile *imagefile = nullptr) = 0; 68 | 69 | // Prefetch file inherits ForwardFile, and it is the actual caller of `record` method. 70 | // The source file is supposed to have cache. 71 | virtual IFile *new_prefetch_file(IFile *src_file, uint32_t layer_index) = 0; 72 | 73 | static Mode detect_mode(const std::string &trace_file_path, size_t *file_size = nullptr); 74 | 75 | Mode get_mode() const { 76 | return m_mode; 77 | } 78 | 79 | protected: 80 | Mode m_mode; 81 | }; 82 | 83 | Prefetcher *new_prefetcher(const std::string &trace_file_path, int concurrency); 84 | Prefetcher *new_dynamic_prefetcher(const std::string &prefetch_list, int concurrency); 85 | -------------------------------------------------------------------------------- /src/switch_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | #include 18 | 19 | // switch to local file after background download finished, and audit for local file pread 20 | // operations. if initialized with local file, only audit for pread. 21 | class ISwitchFile : public photon::fs::IFile { 22 | public: 23 | virtual void set_switch_file(const char *filepath) = 0; 24 | }; 25 | 26 | extern "C" ISwitchFile *new_switch_file(photon::fs::IFile *source, bool local = false, 27 | const char *filepath = nullptr); 28 | 29 | -------------------------------------------------------------------------------- /src/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories($ENV{GFLAGS}/include) 2 | link_directories($ENV{GFLAGS}/lib) 3 | 4 | include_directories($ENV{GTEST}/googletest/include) 5 | link_directories($ENV{GTEST}/lib) 6 | 7 | add_executable(image_service_test image_service_test.cpp) 8 | target_include_directories(image_service_test PUBLIC 9 | ${PHOTON_INCLUDE_DIR} 10 | ${rapidjson_SOURCE_DIR}/include 11 | ) 12 | target_link_libraries(image_service_test gtest gtest_main gflags pthread photon_static overlaybd_lib overlaybd_image_lib) 13 | 14 | add_test( 15 | NAME image_service_test 16 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/image_service_test 17 | ) 18 | 19 | 20 | add_executable(simple_credsrv_test simple_credsrv_test.cpp) 21 | add_test( 22 | NAME simple_credsrv_test 23 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/simple_credsrv_test 24 | ) 25 | target_link_libraries(simple_credsrv_test 26 | overlaybd_image_lib 27 | photon_static 28 | rt 29 | resolv 30 | aio 31 | pthread 32 | gtest 33 | ) 34 | 35 | target_include_directories(simple_credsrv_test PUBLIC 36 | ${PHOTON_INCLUDE_DIR} 37 | ${rapidjson_SOURCE_DIR}/include 38 | $ENV{GTEST}/googletest/include 39 | ) 40 | 41 | add_executable(trace_test trace_test.cpp ../tools/comm_func.cpp) 42 | target_include_directories(trace_test PUBLIC 43 | ${PHOTON_INCLUDE_DIR} 44 | ${rapidjson_SOURCE_DIR}/include 45 | ) 46 | target_link_libraries(trace_test gtest gtest_main gflags pthread photon_static overlaybd_lib overlaybd_image_lib) 47 | 48 | add_test( 49 | NAME trace_test 50 | COMMAND ${EXECUTABLE_OUTPUT_PATH}/trace_test 51 | ) 52 | -------------------------------------------------------------------------------- /src/test/simple_credsrv_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "photon/common/alog.h" 4 | #include "photon/common/alog-stdstring.h" 5 | 6 | #include "photon/net/http/message.h" 7 | #include "photon/thread/thread.h" 8 | #include "photon/net/http/server.h" 9 | #include "photon/net/socket.h" 10 | #include "photon/photon.h" 11 | #include 12 | 13 | #include "photon/fs/localfs.h" 14 | #include "photon/io/fd-events.h" 15 | #include 16 | #include "../config.h" 17 | #include "../image_service.cpp" 18 | 19 | #include 20 | 21 | 22 | using namespace photon::net; 23 | using namespace photon::net::http; 24 | using namespace photon::fs; 25 | class SimpleAuthHandler : public HTTPHandler { 26 | public: 27 | IFile *m_file = nullptr; 28 | struct stat m_st; 29 | void FailedResp(Response &resp, int result = 404) { 30 | resp.set_result(result); 31 | resp.headers.content_length(0); 32 | resp.keep_alive(true); 33 | } 34 | 35 | virtual int handle_request(Request& req, Response& resp, std::string_view ) override { 36 | 37 | /* 38 | // for local test 39 | auto fn = "/opt/overlaybd/cred.json"; 40 | if (m_file == nullptr) { 41 | m_file = open_localfile_adaptor(fn, O_RDONLY); 42 | m_file->fstat(&m_st); 43 | } 44 | auto buf = new char[m_st.st_size]; 45 | auto ret_r = m_file->pread(buf, m_st.st_size, 0); 46 | */ 47 | 48 | auto msg = std::string(R"delimiter( 49 | { 50 | "success": true, 51 | "traceId": "trace_id", 52 | "data": { 53 | "auths": { 54 | "": { 55 | "username": "", 56 | "password": "" 57 | } 58 | } 59 | } 60 | })delimiter"); 61 | LOG_INFO("response: `", msg.c_str()); 62 | resp.set_result(200); 63 | resp.headers.content_length(msg.size()); 64 | resp.keep_alive(true); 65 | photon::thread_sleep(1); 66 | auto ret_w = resp.write((void*)msg.c_str(), msg.size()); 67 | if (ret_w != (ssize_t)msg.size()) { 68 | LOG_ERRNO_RETURN(0, -1, 69 | "send body failed, target: `, `", req.target(), VALUE(ret_w)); 70 | } else { 71 | LOG_DEBUG("Send response header success"); 72 | } 73 | LOG_DEBUG("send body done"); 74 | return 0; 75 | } 76 | }; 77 | 78 | TEST(auth, http_server) { 79 | auto tcpserver = photon::net::new_tcp_socket_server(); 80 | tcpserver->timeout(1000UL*1000); 81 | tcpserver->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1); 82 | tcpserver->bind(19876, IPAddr("127.0.0.1")); 83 | tcpserver->listen(); 84 | DEFER(delete tcpserver); 85 | auto server = new_http_server(); 86 | DEFER(delete server); 87 | SimpleAuthHandler h; 88 | server->add_handler(&h, false, "/auth"); 89 | tcpserver->set_handler(server->get_connection_handler()); 90 | tcpserver->start_loop(); 91 | photon::thread_sleep(1); 92 | std::string remote_path = "", user = "", passwd = ""; 93 | auto ret = load_cred_from_http("http://127.0.0.1:19876/auth", remote_path, user, passwd, 1); 94 | EXPECT_EQ(ret, -1); // expect timeout 95 | ret = load_cred_from_http("http://127.0.0.1:19876/auth", remote_path, user, passwd, 2); 96 | EXPECT_EQ(ret, 0); 97 | photon::thread_sleep(60); 98 | } 99 | 100 | int main(int argc, char** arg) { 101 | photon::init(); 102 | DEFER(photon::fini()); 103 | set_log_output_level(ALOG_DEBUG); 104 | ::testing::InitGoogleTest(&argc, arg); 105 | return RUN_ALL_TESTS(); 106 | } 107 | -------------------------------------------------------------------------------- /src/tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(overlaybd-commit overlaybd-commit.cpp) 2 | target_include_directories(overlaybd-commit PUBLIC ${PHOTON_INCLUDE_DIR}) 3 | target_link_libraries(overlaybd-commit photon_static overlaybd_image_lib) 4 | 5 | add_executable(overlaybd-merge overlaybd-merge.cpp) 6 | target_include_directories(overlaybd-merge PUBLIC ${PHOTON_INCLUDE_DIR}) 7 | target_link_libraries(overlaybd-merge photon_static overlaybd_image_lib) 8 | 9 | 10 | add_executable(overlaybd-create overlaybd-create.cpp) 11 | target_include_directories(overlaybd-create PUBLIC ${PHOTON_INCLUDE_DIR}) 12 | target_link_libraries(overlaybd-create photon_static overlaybd_lib) 13 | set_target_properties(overlaybd-create PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib") 14 | 15 | add_executable(overlaybd-zfile overlaybd-zfile.cpp) 16 | target_include_directories(overlaybd-zfile PUBLIC ${PHOTON_INCLUDE_DIR}) 17 | target_link_libraries(overlaybd-zfile photon_static overlaybd_lib) 18 | 19 | add_executable(overlaybd-apply overlaybd-apply.cpp) 20 | target_include_directories(overlaybd-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include) 21 | target_link_libraries(overlaybd-apply photon_static overlaybd_lib overlaybd_image_lib checksum_lib) 22 | set_target_properties(overlaybd-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib") 23 | 24 | add_executable(turboOCI-apply turboOCI-apply.cpp) 25 | target_include_directories(turboOCI-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include) 26 | target_link_libraries(turboOCI-apply photon_static overlaybd_lib overlaybd_image_lib) 27 | set_target_properties(turboOCI-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib") 28 | 29 | add_library(checksum_lib sha256file.cpp) 30 | target_include_directories(checksum_lib PUBLIC ${PHOTON_INCLUDE_DIR}) 31 | target_link_libraries(checksum_lib photon_static) 32 | 33 | install(TARGETS 34 | overlaybd-commit 35 | overlaybd-create 36 | overlaybd-zfile 37 | overlaybd-apply 38 | overlaybd-merge 39 | 40 | turboOCI-apply 41 | DESTINATION /opt/overlaybd/bin 42 | ) 43 | -------------------------------------------------------------------------------- /src/tools/comm_func.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "comm_func.h" 18 | #include "../overlaybd/tar/tar_file.h" 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "../overlaybd/registryfs/registryfs.h" 24 | #include "../image_service.h" 25 | #include "../image_file.h" 26 | #include "../overlaybd/tar/erofs/erofs_fs.h" 27 | 28 | 29 | using namespace std; 30 | using namespace photon::fs; 31 | 32 | IFile *open_file(const char *fn, int flags, mode_t mode, IFileSystem *fs) { 33 | IFile *file = nullptr; 34 | if (fs) { 35 | file = fs->open(fn, flags, mode); 36 | } else { 37 | file = open_localfile_adaptor(fn, flags, mode, 0); 38 | } 39 | if (!file) { 40 | fprintf(stderr, "failed to open file '%s', %d: %s\n", fn, errno, strerror(errno)); 41 | exit(-1); 42 | } 43 | return file; 44 | } 45 | 46 | int create_overlaybd(const std::string &srv_config, const std::string &dev_config, 47 | ImageService *&imgservice, photon::fs::IFile *&imgfile) { 48 | 49 | imgservice = create_image_service(srv_config.c_str()); 50 | if (imgservice == nullptr) { 51 | fprintf(stderr, "failed to create image service\n"); 52 | exit(-1); 53 | } 54 | imgfile = imgservice->create_image_file(dev_config.c_str()); 55 | if (imgfile == nullptr) { 56 | fprintf(stderr, "failed to create image file\n"); 57 | exit(-1); 58 | } 59 | return 0; 60 | }; 61 | 62 | photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, 63 | bool enable_buffer, const char* root){ 64 | if (mkfs) { 65 | if (make_extfs(imgfile) < 0) { 66 | fprintf(stderr, "mkfs failed, %s\n", strerror(errno)); 67 | exit(-1); 68 | } 69 | } 70 | // for now, buffer_file can't be used with turboOCI 71 | auto extfs = new_extfs(imgfile, enable_buffer); 72 | if (!extfs) { 73 | fprintf(stderr, "new extfs failed, %s\n", strerror(errno)); 74 | exit(-1); 75 | } 76 | auto target = new_subfs(extfs, root, true); 77 | if (!target) { 78 | fprintf(stderr, "new subfs failed, %s\n", strerror(errno)); 79 | exit(-1); 80 | } 81 | return target; 82 | } 83 | 84 | bool is_erofs_fs(const photon::fs::IFile *imgfile) 85 | { 86 | if (imgfile == nullptr) 87 | LOG_ERRNO_RETURN(-EINVAL, false, "Imgfile is nullptr"); 88 | return erofs_check_fs(imgfile); 89 | } 90 | 91 | photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t blksz) 92 | { 93 | return erofs_create_fs(imgfile, blksz); 94 | } 95 | IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, 96 | const string &upload_url, const string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB, 97 | const string &tls_key_path, const string &tls_cert_path 98 | ){ 99 | 100 | zfile_args->overwrite_header = false; 101 | LOG_INFO("upload to `", upload_url); 102 | std::string username, password; 103 | if (load_cred_from_file(cred_file_path, upload_url, username, password) < 0) { 104 | fprintf(stderr, "failed to read upload cred file\n"); 105 | exit(-1); 106 | } 107 | auto upload_builder = new_registry_uploader(src, upload_url, username, password, 108 | timeout_minute*60*1000*1000, upload_bs_KB*1024, tls_cert_path.c_str(), tls_key_path.c_str()); 109 | if (upload_builder == nullptr) { 110 | fprintf(stderr, "failed to init upload\n"); 111 | exit(-1); 112 | } 113 | return upload_builder; 114 | } 115 | -------------------------------------------------------------------------------- /src/tools/comm_func.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #pragma once 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include "../overlaybd/lsmt/file.h" 29 | #include "../overlaybd/zfile/zfile.h" 30 | #include "../image_service.h" 31 | #include "../image_file.h" 32 | #include 33 | #include 34 | #include 35 | #include "CLI11.hpp" 36 | 37 | 38 | int generate_option(CLI::App &app); 39 | photon::fs::IFile *open_file(const char *fn, int flags, mode_t mode = 0, photon::fs::IFileSystem *fs = nullptr); 40 | 41 | int create_overlaybd(const std::string &srv_config, const std::string &dev_config, 42 | ImageService *&, photon::fs::IFile *&); 43 | 44 | photon::fs::IFile *create_uploader(ZFile::CompressArgs *zfile_args, IFile *src, 45 | const std::string &upload_url, const std::string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB, 46 | const std::string &tls_key_path, const std::string &tls_cert_path); 47 | 48 | photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs, 49 | bool enable_buffer, const char* root); 50 | 51 | bool is_erofs_fs(const photon::fs::IFile *imgfile); 52 | photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t blksz); 53 | -------------------------------------------------------------------------------- /src/tools/overlaybd-create.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Overlaybd Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "../overlaybd/lsmt/file.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include "CLI11.hpp" 33 | 34 | using namespace std; 35 | using namespace photon::fs; 36 | 37 | IFile *open_file(const char *fn, int flags, mode_t mode = 0) { 38 | auto file = open_localfile_adaptor(fn, flags, mode, 0); 39 | if (!file) { 40 | fprintf(stderr, "failed to open file '%s', %d: %s\n", fn, errno, strerror(errno)); 41 | exit(-1); 42 | } 43 | return file; 44 | } 45 | 46 | 47 | 48 | int main(int argc, char **argv) { 49 | uint64_t vsize; 50 | string parent_uuid; 51 | bool sparse = false; 52 | std::string data_file_path, index_file_path, warp_index_path; 53 | bool build_turboOCI = false; 54 | bool build_fastoci = false; 55 | bool mkfs = false; 56 | bool raw = false; 57 | bool verbose = false; 58 | 59 | CLI::App app{"this is overlaybd-create"}; 60 | app.add_option("-u", parent_uuid, "parent uuid"); 61 | app.add_flag("-s", sparse, "create sparse RW layer")->default_val(false); 62 | app.add_flag("--turboOCI", build_turboOCI, "commit using turboOCI format")->default_val(false); 63 | app.add_flag("--fastoci", build_fastoci, "commit using turboOCI format(depracated)")->default_val(false); 64 | app.add_flag("--raw", raw, "create raw image")->default_val(false); 65 | app.add_flag("--mkfs", mkfs, "mkfs after create")->default_val(false); 66 | app.add_option("data_file", data_file_path, "data file path")->type_name("FILEPATH")->required(); 67 | app.add_option("index_file", index_file_path, "index file path")->type_name("FILEPATH")->required(); 68 | app.add_option("vsize", vsize, "virtual size(GB)")->type_name("INT")->check(CLI::NonNegativeNumber)->required(); 69 | app.add_flag("--verbose", verbose, "output debug info")->default_val(false); 70 | CLI11_PARSE(app, argc, argv); 71 | 72 | build_turboOCI = build_turboOCI || build_fastoci; 73 | 74 | set_log_output_level(verbose ? 0 : 1); 75 | photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); 76 | DEFER({photon::fini();}); 77 | 78 | vsize *= 1024 * 1024 * 1024; 79 | const auto flag = O_RDWR | O_EXCL | O_CREAT; 80 | const auto mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; 81 | IFile* fdata = open_file(data_file_path.c_str(), flag, mode); 82 | IFile* findex = open_file(index_file_path.c_str(), flag, mode); 83 | IFile* file = nullptr; 84 | 85 | if (raw) { 86 | file = fdata; 87 | file->ftruncate(vsize); 88 | } else if (build_turboOCI) { 89 | LSMT::WarpFileArgs args(findex, fdata, nullptr); 90 | args.virtual_size = vsize; 91 | file = LSMT::create_warpfile(args, false); 92 | } else { 93 | LSMT::LayerInfo args(fdata, findex); 94 | args.parent_uuid.parse(parent_uuid.c_str(), parent_uuid.size()); 95 | args.virtual_size = vsize; 96 | args.sparse_rw = sparse; 97 | file = LSMT::create_file_rw(args, false); 98 | } 99 | 100 | if (!file) { 101 | fprintf(stderr, "failed to create lsmt file object, possibly I/O error!\n"); 102 | exit(-1); 103 | } 104 | 105 | if (mkfs) { 106 | if (make_extfs(file) < 0) { 107 | fprintf(stderr, "mkfs failed, %s\n", strerror(errno)); 108 | exit(-1); 109 | } 110 | } 111 | 112 | if (file && file != fdata) 113 | delete file; 114 | delete fdata; 115 | delete findex; 116 | printf("overlaybd-create has created files SUCCESSFULLY\n"); 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /src/tools/sha256file.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "sha256file.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace photon::fs; 11 | using namespace std; 12 | 13 | class SHA256CheckedFile: public SHA256File { 14 | public: 15 | IFile *m_file; 16 | SHA256_CTX ctx = {0}; 17 | size_t total_read = 0; 18 | bool m_ownership = false; 19 | 20 | SHA256CheckedFile(IFile *file, bool ownership): m_file(file), m_ownership(ownership) { 21 | SHA256_Init(&ctx); 22 | } 23 | ~SHA256CheckedFile() { 24 | if (m_ownership) delete m_file; 25 | } 26 | virtual IFileSystem *filesystem() override { 27 | return nullptr; 28 | } 29 | ssize_t read(void *buf, size_t count) override { 30 | auto rc = m_file->read(buf, count); 31 | if (rc > 0 && SHA256_Update(&ctx, buf, rc) < 0) { 32 | LOG_ERROR("sha256 calculate error"); 33 | return -1; 34 | } 35 | return rc; 36 | } 37 | off_t lseek(off_t offset, int whence) override { 38 | return m_file->lseek(offset, whence); 39 | } 40 | virtual std::string sha256_checksum() override{ 41 | // read trailing data 42 | char buf[64*1024]; 43 | auto rc = m_file->read(buf, 64*1024); 44 | while (rc > 0) { 45 | // if (rc == 64*1024) { 46 | // LOG_WARN("too much trailing data"); 47 | // } 48 | if (rc > 0 && SHA256_Update(&ctx, buf, rc) < 0) { 49 | LOG_ERROR("sha256 calculate error"); 50 | return ""; 51 | } 52 | rc = m_file->read(buf, 64*1024); 53 | } 54 | // calc sha256 result 55 | unsigned char sha[32]; 56 | SHA256_Final(sha, &ctx); 57 | char res[SHA256_DIGEST_LENGTH * 2]; 58 | for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) 59 | sprintf(res + (i * 2), "%02x", sha[i]); 60 | return "sha256:" + std::string(res, SHA256_DIGEST_LENGTH * 2); 61 | } 62 | int fstat(struct stat *buf) override { 63 | return m_file->fstat(buf); 64 | } 65 | }; 66 | 67 | SHA256File *new_sha256_file(IFile *file, bool ownership = true) { 68 | return new SHA256CheckedFile(file, ownership); 69 | } 70 | 71 | string sha256sum(const char *fn) { 72 | constexpr size_t BUFFERSIZE = 65536; 73 | // auto file = open_localfile_adaptor(fn, O_RDONLY | O_DIRECT); 74 | // auto sha256file = new_sha256_file(file, true); 75 | // DEFER(delete sha256file); 76 | // return sha256file->sha256_checksum(); 77 | int fd = open(fn, O_RDONLY | O_DIRECT); 78 | if (fd < 0) { 79 | LOG_ERROR("failed to open `", fn); 80 | return ""; 81 | } 82 | DEFER(close(fd);); 83 | 84 | struct stat stat; 85 | if (::fstat(fd, &stat) < 0) { 86 | LOG_ERROR("failed to stat `", fn); 87 | return ""; 88 | } 89 | SHA256_CTX ctx = {0}; 90 | SHA256_Init(&ctx); 91 | __attribute__((aligned(ALIGNMENT_4K))) char buffer[65536]; 92 | unsigned char sha[32]; 93 | ssize_t recv = 0; 94 | for (off_t offset = 0; offset < stat.st_size; offset += BUFFERSIZE) { 95 | recv = pread(fd, &buffer, BUFFERSIZE, offset); 96 | if (recv < 0) { 97 | LOG_ERROR("io error: `", fn); 98 | return ""; 99 | } 100 | if (SHA256_Update(&ctx, buffer, recv) < 0) { 101 | LOG_ERROR("sha256 calculate error: `", fn); 102 | return ""; 103 | } 104 | } 105 | SHA256_Final(sha, &ctx); 106 | char res[SHA256_DIGEST_LENGTH * 2 + 1]; 107 | for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) 108 | sprintf(res + (i * 2), "%02x", sha[i]); 109 | return "sha256:" + std::string(res, SHA256_DIGEST_LENGTH * 2); 110 | } 111 | -------------------------------------------------------------------------------- /src/tools/sha256file.h: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | class SHA256File : public photon::fs::VirtualReadOnlyFile { 8 | public: 9 | virtual std::string sha256_checksum() = 0; 10 | }; 11 | 12 | SHA256File *new_sha256_file(photon::fs::IFile *file, bool ownership); 13 | 14 | std::string sha256sum(const char *fn); 15 | -------------------------------------------------------------------------------- /src/version.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define MACROTOSTR(x) #x 4 | #define PRINTMACRO(x) MACROTOSTR(x) 5 | static const char OVERLAYBD_VERSION[] = PRINTMACRO(OVERLAYBD_VER); 6 | --------------------------------------------------------------------------------