├── .dockerignore ├── .github ├── CODEOWNERS └── workflows │ ├── build-aws.yml │ └── build.yml ├── .gitignore ├── .gitlab-ci.yml ├── .gitlab ├── defaults.yml ├── shellcheck.yml └── sync-github.yml ├── .gitmodules ├── LICENSE ├── README.md └── scripts ├── aws-git-1.9.patch ├── build_all.sh ├── build_aws-sdk.sh ├── build_datamover.sh ├── build_dss-client.sh ├── build_dss-sdk.sh ├── build_gcc.sh ├── build_kernel.sh ├── build_minio.sh ├── build_mlnx-tools.sh ├── docker ├── DSS.Dockerfile ├── aws-sdk.Dockerfile ├── build_all.sh ├── build_aws-sdk.sh ├── build_datamover.sh ├── build_docker.sh ├── build_dss-client.sh ├── build_dss-sdk.sh ├── build_gcc.sh ├── build_kernel.sh ├── build_minio.sh ├── build_mlnx-tools.sh ├── gcc.Dockerfile ├── kernel.Dockerfile └── mlnx.Dockerfile ├── kernel_config ├── load_gcc.sh ├── stagemergeartifacts.sh └── utils.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | dss-ecosystem 4 | dss-minio 5 | dss-sdk 6 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default Owners 2 | * @OpenMPDK/dss-default-reviewers 3 | -------------------------------------------------------------------------------- /.github/workflows/build-aws.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | secrets: 4 | AWS_ACCESS_KEY_ID: 5 | description: AWS Access Key 6 | required: true 7 | AWS_SECRET_ACCESS_KEY: 8 | description: AWS Secret Key 9 | required: true 10 | AWS_ACCOUNT_ID: 11 | description: AWS Account ID 12 | required: true 13 | REPOSITORY_NAME: 14 | description: ECR Repository Name 15 | required: true 16 | 17 | inputs: 18 | project-name: 19 | description: Name of the CodeBuild Project 20 | required: false 21 | default: OpenMPDK_DSS 22 | type: string 23 | compute-type-override: 24 | description: Compute type for CodeBuild runtime 25 | required: false 26 | default: BUILD_GENERAL1_SMALL 27 | type: string 28 | component: 29 | description: Name of the component you want to build (for display only) 30 | required: true 31 | type: string 32 | buildspec: 33 | description: CodeBuild buildspec filename (inside ./buildspec/) - Derived from 'component' by default 34 | required: false 35 | type: string 36 | image: 37 | description: Custom image to use for CodeBuild runtime, if not the default dss-build image 38 | required: false 39 | type: string 40 | 41 | jobs: 42 | build-aws: 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Configure AWS Credentials 46 | uses: aws-actions/configure-aws-credentials@v2 47 | with: 48 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 49 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 50 | aws-region: us-west-1 51 | - name: Derive the branchname for ECR tag (replace '/' with '_') 52 | id: nice_branchname 53 | run: | 54 | # Use base_ref if PR 55 | if [[ "${{ github.ref_name }}" == *"/merge" ]] 56 | then 57 | BRANCH_NAME="${{ github.base_ref }}" 58 | # Use ref_name if merge 59 | else 60 | BRANCH_NAME="${{ github.ref_name }}" 61 | fi 62 | echo "NICE_BRANCHNAME=$(echo "$BRANCH_NAME" | tr '/' '_')" >> "$GITHUB_OUTPUT" 63 | shell: bash 64 | - name: "Execute ${{ inputs.component }} workflow on CodeBuild" 65 | uses: aws-actions/aws-codebuild-run-build@v1 66 | with: 67 | project-name: ${{ inputs.project-name }} 68 | compute-type-override: ${{ inputs.compute-type-override }} 69 | buildspec-override: "./buildspec/${{ inputs.buildspec != '' && inputs.buildspec || format('{0}.yml', inputs.component) }}" 70 | image-override: "${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-west-1.amazonaws.com/${{ secrets.REPOSITORY_NAME }}:${{ inputs.image != '' && inputs.image || format('dss-build_{0}', steps.nice_branchname.outputs.NICE_BRANCHNAME) }}" 71 | env-vars-for-codebuild: SONAR_SCANNER_URL 72 | env: 73 | SONAR_SCANNER_URL: ${{ vars.SONAR_SCANNER_URL }} 74 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | types: [opened, synchronize, reopened] 8 | jobs: 9 | shellcheck: 10 | name: Shellcheck 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Run ShellCheck 15 | uses: ludeeus/action-shellcheck@master 16 | with: 17 | scandir: './scripts' 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | go_* 2 | *.tgz 3 | .vscode 4 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - local: .gitlab/defaults.yml 3 | - local: .gitlab/shellcheck.yml 4 | - local: .gitlab/sync-github.yml 5 | 6 | stages: 7 | - lint 8 | - sync 9 | 10 | sync-github: 11 | variables: 12 | GITHUB_REPO: DSS 13 | -------------------------------------------------------------------------------- /.gitlab/defaults.yml: -------------------------------------------------------------------------------- 1 | .default_rules: 2 | merge_only: 3 | - if: $CI_PIPELINE_SOURCE == "merge_request_event" 4 | variables: 5 | BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME 6 | SONAR_BRANCH: '' 7 | - if: $CI_PIPELINE_SOURCE == "schedule" 8 | when: never 9 | - if: $CI_PIPELINE_SOURCE == "web" 10 | merge_and_push: 11 | - if: $CI_PIPELINE_SOURCE == "merge_request_event" 12 | variables: 13 | BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME 14 | SONAR_BRANCH: '' 15 | - if: $CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push" 16 | - if: $CI_COMMIT_BRANCH =~ /^(stable|feature)\/.*/ && $CI_PIPELINE_SOURCE == "push" 17 | - if: $CI_PIPELINE_SOURCE == "schedule" 18 | when: never 19 | - if: $CI_PIPELINE_SOURCE == "web" 20 | push_only: 21 | - if: $CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push" 22 | - if: $CI_COMMIT_BRANCH =~ /^(stable|feature)\/.*/ && $CI_PIPELINE_SOURCE == "push" 23 | scheduled_sync: 24 | - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "sync" 25 | 26 | variables: 27 | BRANCH_NAME: $CI_COMMIT_BRANCH 28 | SONAR_BRANCH: -Dsonar.branch.name=$CI_COMMIT_BRANCH 29 | 30 | default: 31 | image: 32 | name: dss-build_$BRANCH_NAME 33 | -------------------------------------------------------------------------------- /.gitlab/shellcheck.yml: -------------------------------------------------------------------------------- 1 | shellcheck: 2 | stage: lint 3 | image: koalaman/shellcheck-alpine:$SHELLCHECK_VER 4 | before_script: 5 | - apk update 6 | - apk add git xmlstarlet 7 | - shellcheck --version 8 | - | 9 | cat > checkstyle2junit.xslt < 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | Line 46 | 47 | : 48 | 49 | See https://www.shellcheck.net/wiki/ 50 | 51 | 52 | 53 | 54 | EOL 55 | script: 56 | - git ls-files --exclude='*.sh' --ignored -c -z | 57 | xargs -t --no-run-if-empty -0 shellcheck --format=checkstyle | 58 | xmlstarlet tr checkstyle2junit.xslt > 59 | shellcheck.xml 60 | artifacts: 61 | when: always 62 | reports: 63 | junit: shellcheck.xml 64 | variables: 65 | SHELLCHECK_VER: v0.9.0 66 | needs: [] 67 | rules: 68 | - !reference [.default_rules, merge_and_push] 69 | -------------------------------------------------------------------------------- /.gitlab/sync-github.yml: -------------------------------------------------------------------------------- 1 | sync-github: 2 | stage: sync 3 | image: 4 | name: alpine:3.19.1 5 | before_script: 6 | - apk add git 7 | - git config --global http.sslVerify false 8 | script: 9 | - mkdir $GITHUB_REPO && cd $GITHUB_REPO 10 | - git clone --branch $GITHUB_BRANCH https://github.com/$GITHUB_PROJECT/$GITHUB_REPO . 11 | - git remote rename origin old-origin 12 | - git remote add origin https://$CI_USERNAME:$CI_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git 13 | - git push origin $GITHUB_BRANCH:$GITLAB_BRANCH 14 | - git push -u origin --tags 15 | variables: 16 | GIT_STRATEGY: none 17 | GITHUB_PROJECT: OpenMPDK 18 | GITHUB_BRANCH: master 19 | GITLAB_BRANCH: master 20 | rules: 21 | - !reference [.default_rules, scheduled_sync] 22 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dss-ansible"] 2 | path = dss-ansible 3 | url = https://github.com/OpenMPDK/dss-ansible 4 | [submodule "dss-ecosystem"] 5 | path = dss-ecosystem 6 | url = https://github.com/OpenMPDK/dss-ecosystem 7 | [submodule "dss-minio"] 8 | path = dss-minio 9 | url = https://github.com/OpenMPDK/dss-minio 10 | [submodule "dss-sdk"] 11 | path = dss-sdk 12 | url = https://github.com/OpenMPDK/dss-sdk 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The Clear BSD License 2 | 3 | Copyright (c) 2022 Samsung Electronics Co., Ltd. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted (subject to the limitations in the disclaimer 8 | below) provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, 11 | this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 16 | contributors may be used to endorse or promote products derived from this 17 | software without specific prior written permission. 18 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 19 | THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 20 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 21 | NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 22 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 | OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 | OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DSS 2 | 3 | Disaggregated Storage Solution 4 | 5 | ## What is DSS 6 | 7 | DSS is a rack-scalable, very high read-bandwidth-optimized, Amazon S3-compatible object storage solution developed by Samsung. It utilizes a disaggregated architecture, enabling independent scaling of storage and compute. It features an end-to-end KV semantic communication stack, entirely eliminating the legacy software storage stack. All storage communication uses the NVMeOf-KV-RDMA protocol introduced and open sourced by Samsung. With zero-copy transfer, it achieves high end-to-end performance. The DSS client-side stack includes a high performance wrapper library for simple application integration. Applications utilizing the DSS client library eliminate the need for bucket semantics, key distribution and load balancing between server-side S3 endpoints. 8 | 9 | [![How to build, deploy, and use DSS software](https://img.youtube.com/vi/fpAFvLhTpqw/0.jpg)](https://youtu.be/fpAFvLhTpqw "How to build, deploy, and use DSS software") 10 | 11 | [How to build, deploy, and use DSS software](https://youtu.be/fpAFvLhTpqw) 12 | 13 | ## DSS Performance 14 | 15 | [S3 Benchmark](https://github.com/OpenMPDK/dss-ecosystem/tree/master/dss_s3benchmark) results: 16 | 17 | | | [v1.0.0](https://github.com/OpenMPDK/DSS/releases/tag/v1.0.0) | [v2.0.0](https://github.com/OpenMPDK/DSS/releases/tag/v2.0.0) (S3-over-RDMA) | [v3.0.0](https://github.com/OpenMPDK/DSS/releases/tag/v3.0.0) (Write Optimization) | 18 | |:----------:|:----:|:-------------------:|:-------------------------:| 19 | | PUT (GB/s) | 12 | 12 | 65 | 20 | | GET (GB/s) | 112 | 160 | 162 | 21 | 22 | Results are aggregated with the following specification: 23 | 24 | - 4-node Dell R7525 Cluster 25 | - 16x Samsung PM1733 3.84TB NVMe drives per Node 26 | - 4x Dual-port 100g Mellanox CX-5 NIC per node 27 | - Dual-socket AMD EPYC 7742 64-Core Processors 28 | - 1TB DIMM per Node 29 | 30 | ## Build DSS - Docker 31 | 32 | DSS is optimally built via Docker using the scripts documented below. 33 | 34 | ### Build All - Docker 35 | 36 | Build all of the DSS artifacts and its dependency artifacts using one script: 37 | 38 | ```bash 39 | ./scripts/docker/build_all.sh 40 | ``` 41 | 42 | ### Build Dependencies - Docker 43 | 44 | Optionally, build only the dependencies artifacts: 45 | 46 | ```bash 47 | ./scripts/docker/build_aws-sdk.sh 48 | ./scripts/docker/build_kernel.sh 49 | ./scripts/docker/build_mlnx-tools.sh 50 | ``` 51 | 52 | ### Build DSS Artifacts - Docker 53 | 54 | Optionally, build only the DSS artifacts: 55 | 56 | ```bash 57 | ./scripts/docker/build_dss-sdk.sh 58 | ./scripts/docker/build_minio.sh 59 | ./scripts/docker/build_dss-client.sh 60 | ./scripts/docker/build_datamover.sh 61 | ``` 62 | 63 | ## Build DSS 64 | 65 | Alternatively, DSS can be built natively, but all dependencies must be installed first. 66 | 67 | ### Prerequisites 68 | 69 | #### Operating system requirements 70 | 71 | DSS build and runtime is presently supported on CentOS 7.8. 72 | 73 | #### Note about CentOS 7 Deprecation 74 | 75 | [CentOS 7 has reached end-of-life.](https://www.redhat.com/en/topics/linux/centos-linux-eol#:~:text=Hat%20Enterprise%20Linux%3F-,Overview,can%20help%20ease%20your%20migration.) 76 | 77 | As such, the YUM repositories that enable dependency download on CentOS 7 are no longer available. 78 | 79 | However, you may work around this situation with the following steps: 80 | 81 | ```bash 82 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo 83 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo 84 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo 85 | ``` 86 | 87 | When installing some dependencies, this process may need to be repeated to ensure you have access to the archived dependencies. 88 | 89 | #### Build package dependencies 90 | 91 | Install the following packages / modules to build DSS and its external dependencies: 92 | 93 | ```bash 94 | sudo yum install epel-release centos-release-scl-rh -y 95 | sudo yum install bc bison boost-devel cmake cmake3 cppunit-devel CUnit-devel devtoolset-11 dpkg \ 96 | elfutils-libelf-devel flex gcc gcc-c++ git glibc-devel gmp-devel golang jemalloc-devel Judy-devel \ 97 | libaio-devel libcurl-devel libmpc-devel libuuid-devel make man-db meson mpfr-devel ncurses-devel \ 98 | numactl-devel openssl openssl-devel patch pulseaudio-libs-devel python3 python3-devel python3-pip \ 99 | rdma-core-devel redhat-lsb-core rpm-build \ 100 | snappy-devel tbb-devel wget zlib-devel -y 101 | sudo python3 -m pip install pybind11==2.11.1 gcovr==5.0 102 | ``` 103 | 104 | **NOTE: User-built AWS-SDK-CPP RPM must be installed on the build machine.** 105 | 106 | On initial build: 107 | 108 | 1. Build AWS-SDK-CPP: `./scripts/build_aws-sdk.sh` 109 | 2. Install the resulting AWS-SDK-CPP RPM: `sudo yum install ./dss-ansible/artifacts/aws-sdk-cpp-*.rpm -y` 110 | 3. Run the `build_all.sh` script: `./scripts/build_all.sh` 111 | 112 | Once the AWS RPM is installed, only the `build_all.sh` script needs to be run on subsequent builds. 113 | 114 | Dependency artifacts for kernel, aws-sdk-cpp, and mlnx-tools are staged under `rpmbuilder` and `workspace` directories of your home directory by default. By leaving them in-place, re-build of these upstream components will be skipped on subsequent builds. 115 | 116 | ### Optional: Build individual components 117 | 118 | DSS Dependency build scripts: 119 | 120 | - Build aws-sdk-cpp: `./scripts/build_aws-sdk.sh` 121 | - Build kernel: `./scripts/build_kernel.sh` 122 | - Build mlnx-tools: `./scripts/build_mlnx-tools.sh` 123 | - Install aws-sdk-cpp: `yum install dss-ansible/artifacts/aws-sdk-cpp-1.9-0.x86_64.rpm -y` 124 | 125 | DSS individual components: 126 | 127 | - Build dss-sdk: `./scripts/build_dss-sdk.sh` 128 | - Build dss-minio: `./scripts/build_minio.sh` 129 | - Build dss-client: `./scripts/build_dss-client.sh` 130 | - Build dss-datamover: `./scripts/build_datamover.sh` 131 | 132 | ## Deploy DSS 133 | 134 | See [dss-ansible README](https://github.com/OpenMPDK/dss-ansible/blob/master/README.md) 135 | 136 | ## Blogs and Papers 137 | 138 | [DSS: High I/O Bandwidth Disaggregated Object Storage System for AI Applications](https://www.researchgate.net/publication/358580692_DSS_High_IO_Bandwidth_Disaggregated_Object_Storage_System_for_AI_Applications) 139 | 140 | [High-Capacity SSDs for AI/ML using Disaggregated Storage Solution: Performance Test Results Show Promise](https://semiconductor.samsung.com/us/newsroom/tech-blog/high-capacity-ssds-for-ai-ml-using-disaggregated-storage-solution-performance-test-results-show-promise/) 141 | -------------------------------------------------------------------------------- /scripts/aws-git-1.9.patch: -------------------------------------------------------------------------------- 1 | --- aws-git-1.9/CMakeLists.txt 2022-12-06 13:37:22.790891000 -0800 2 | +++ aws-git-1.9.new/CMakeLists.txt 2022-12-06 13:56:46.023876000 -0800 3 | @@ -204,6 +204,7 @@ 4 | set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_PREV}) 5 | endif() 6 | else() 7 | + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/crt/aws-crt-cpp/crt/aws-c-common/cmake") 8 | include(AwsFindPackage) 9 | set(IN_SOURCE_BUILD OFF) 10 | endif() 11 | -------------------------------------------------------------------------------- /scripts/build_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Build aws-sdk for DSS build / runtime 34 | set -e 35 | 36 | # Load utility functions 37 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 38 | . "$SCRIPT_DIR/utils.sh" 39 | 40 | # Build dependency artificats 41 | "$SCRIPT_DIR/build_aws-sdk.sh" 42 | "$SCRIPT_DIR/build_kernel.sh" 43 | "$SCRIPT_DIR/build_mlnx-tools.sh" 44 | 45 | # # Build DSS 46 | "$SCRIPT_DIR/build_dss-sdk.sh" 47 | "$SCRIPT_DIR/build_minio.sh" 48 | "$SCRIPT_DIR/build_dss-client.sh" 49 | "$SCRIPT_DIR/build_datamover.sh" 50 | -------------------------------------------------------------------------------- /scripts/build_aws-sdk.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | set -e 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | source "$SCRIPT_DIR/utils.sh" 7 | 8 | GIT_AWS_RELEASE=1.9 9 | GIT_CHECKOUT_TAG="1.9.343-elbencho-tag" 10 | ARTIFACT='aws-sdk-cpp-*.rpm' 11 | 12 | # Check if aws-sdk-cpp RPM is already built and present in dss-ansible/artifacts dir 13 | CHECK_ARTIFACT=$(find "$ARTIFACTS_DIR/" -name "$ARTIFACT" | wc -l) 14 | 15 | # Skip build if RPM is present 16 | if [ "$CHECK_ARTIFACT" -gt 0 ] 17 | then 18 | echo "aws-sdk-cpp RPM is present in $ARTIFACTS_DIR" - Skipping... 19 | exit 0 20 | fi 21 | 22 | # Check if aws-sdk-cpp RPM is already installed 23 | set +e 24 | CHECK_RPM_INSTALLED=$(rpm -q aws-sdk-cpp) 25 | set -e 26 | 27 | # Abort if aws-sdk-cpp RPM is installed - must be uninstalled before build 28 | if [ "$CHECK_RPM_INSTALLED" != 'package aws-sdk-cpp is not installed' ] 29 | then 30 | echo "aws-sdk-cpp RPM is installed." 31 | echo "aws-sdk-cpp RPM must be uninstalled before rebuilding: \"sudo yum remove aws-sdk-cpp -y\"" 32 | exit 1 33 | fi 34 | 35 | echo "Preparing the environment and the spec file" 36 | mkdir -p "$RPMBUILD_DIR"/{SOURCES,BUILD,RPMS,SPECS} 37 | cp "$SCRIPT_DIR/aws-git-$GIT_AWS_RELEASE.patch" "$RPMBUILD_DIR/SOURCES/" 38 | AWS_SPEC_FILE="$RPMBUILD_DIR/SPECS/aws-$GIT_AWS_RELEASE.spec" 39 | 40 | # Create spec file for RPM build 41 | cat > "$AWS_SPEC_FILE" << EOF 42 | Name: aws-sdk-cpp 43 | Version: $GIT_AWS_RELEASE 44 | Release: 0 45 | Summary: Amazon Web Services SDK for C++ 46 | License: ASL 2.0 47 | URL: https://github.com/aws/%{name} 48 | Patch0: aws-git-$GIT_AWS_RELEASE.patch 49 | 50 | 51 | %define _unpackaged_files_terminate_build 0 52 | 53 | %description 54 | The Amazon Web Services (AWS) SDK for C++ provides a modern C++ interface for 55 | AWS. It is meant to be performant and fully functioning with low- and 56 | high-level SDKs, while minimizing dependencies and providing platform 57 | portability (Windows, OSX, Linux, and mobile). 58 | 59 | %prep 60 | rm -rf aws-sdk-cpp 61 | git clone --recursive -q https://github.com/breuner/aws-sdk-cpp.git --branch $GIT_CHECKOUT_TAG --single-branch 62 | cd aws-sdk-cpp 63 | 64 | %patch0 -p1 65 | 66 | %build 67 | # Compiling the code 68 | source /opt/rh/devtoolset-11/enable 69 | cd %{_builddir}/aws-sdk-cpp/crt/aws-crt-cpp 70 | cmake3 . -DBUILD_SHARED_LIBS=ON -DCPP_STANDARD=17 -DAUTORUN_UNIT_TESTS=OFF -DENABLE_TESTING=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBYO_CRYPTO=ON -DCMAKE_INSTALL_PREFIX=%{buildroot}/usr/local/ && make -j $(nproc) install 71 | cd ../.. 72 | cmake3 . -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=ON -DCPP_STANDARD=17 -DAUTORUN_UNIT_TESTS=OFF -DENABLE_TESTING=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBYO_CRYPTO=ON -DBUILD_DEPS=OFF -DCMAKE_INSTALL_PREFIX=%{buildroot}/usr/local/ && make -j $(nproc) install 73 | 74 | %files 75 | /usr/local/lib64/* 76 | /usr/local/include/* 77 | 78 | %clean 79 | rm -rf aws-sdk-cpp 80 | rm -rf %_buildrootdir/aws-sdk-cpp* 81 | 82 | EOF 83 | 84 | # Build AWS RPM 85 | echo -n "Building AWS RPM ...." 86 | if ! rpmbuild -bb "$AWS_SPEC_FILE"; 87 | then 88 | echo "[Failed]" 89 | exit 1 90 | fi 91 | echo "[Success]" 92 | 93 | find "$RPM_DIR" -name "$ARTIFACT" -exec cp {} "$ARTIFACTS_DIR/" \; 94 | -------------------------------------------------------------------------------- /scripts/build_datamover.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | set -e 34 | 35 | # Load utility functions 36 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 37 | . "$SCRIPT_DIR/utils.sh" 38 | 39 | # Check for submodules in update init recursive if missing 40 | checksubmodules 41 | 42 | pushd "$DATAMOVER_DIR" 43 | echo "Getting release string from $(basename "$DATAMOVER_DIR") repo" 44 | git fetch --tags 45 | RELEASESTRING=$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short HEAD) 46 | echo "Release: $RELEASESTRING" 47 | popd 48 | 49 | echo "Removing existing datamover tarball from dss-ansible artifacts directory" 50 | rm -f "$ARTIFACTS_DIR"/nkv-datamover-*.tgz 51 | 52 | echo "Creating datamover tarball in dss-ansible artifacts directory" 53 | git archive --format=tgz -19 --remote="$DSS_ECOSYSTEM_DIR" --output="$ARTIFACTS_DIR/nkv-datamover-$RELEASESTRING.tgz" HEAD:"$(basename "$DATAMOVER_DIR")" 54 | -------------------------------------------------------------------------------- /scripts/build_dss-client.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | set -e 34 | 35 | # Load utility functions 36 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 37 | . "$SCRIPT_DIR/utils.sh" 38 | 39 | # Check for submodules in update init recursive if missing 40 | checksubmodules 41 | 42 | # Build DSS Client Library 43 | "$DSS_CLIENT_DIR/scripts/build.sh" 44 | 45 | echo "Removing existing Client Library release tarball from dss-ansible artifacts directory" 46 | rm -f "${ARTIFACTS_DIR}"/dss_client-*.tgz 47 | 48 | echo "Copying Client Library release tarball to dss-ansible artifacts directory" 49 | cp "$DSS_CLIENT_DIR"/dss_client-*.tgz "$ARTIFACTS_DIR" 50 | -------------------------------------------------------------------------------- /scripts/build_dss-sdk.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Build nkv-sdk components: target, host, ufm, agent 34 | 35 | set -e 36 | 37 | # Load utility functions 38 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 39 | . "$SCRIPT_DIR/utils.sh" 40 | 41 | # Set build variables 42 | HOST_BUILD_MODE='kdd-samsung-remote' 43 | 44 | # Check for submodules in update init recursive if missing 45 | checksubmodules 46 | 47 | # Build dss-sdk 48 | "$DSS_SDK_DIR/scripts/build_all.sh" "$HOST_BUILD_MODE" 49 | 50 | # Set artifacts build directory paths 51 | target_build_dir="${DSS_DIR}/dss-sdk/df_out" 52 | host_build_dir="${DSS_DIR}/dss-sdk/host_out" 53 | 54 | echo "Removing existing artifacts from artifacts directory..." 55 | rm -f "${ARTIFACTS_DIR}"/nkv-target-*.tgz 56 | rm -f "${ARTIFACTS_DIR}"/nkv-sdk-bin-*.tgz 57 | 58 | echo "Copying artifacts to artifacts directory..." 59 | cp "${target_build_dir}"/nkv-target-*.tgz "${ARTIFACTS_DIR}" 60 | cp "${host_build_dir}"/nkv-sdk-bin-*.tgz "${ARTIFACTS_DIR}" 61 | -------------------------------------------------------------------------------- /scripts/build_gcc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Build GCC for DSS build / runtime 34 | set -e 35 | 36 | # Load utility functions 37 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 38 | . "$SCRIPT_DIR/utils.sh" 39 | 40 | # Set build variables 41 | GCC_REPO="https://github.com/BobSteagall/gcc-builder" 42 | GCC_REPONAME=$(basename "$GCC_REPO") 43 | GCC_BRANCH="gcc5" 44 | GCC_VERSION="5.1.0" 45 | GCC_DIR="$BUILD_STAGING_DIR/$GCC_REPONAME" 46 | 47 | # Check for submodules in update init recursive if missing 48 | checksubmodules 49 | 50 | # Create BUILD_STAGING_DIR if it doesn't exist 51 | if [ ! -d "$BUILD_STAGING_DIR" ] 52 | then 53 | mkdir "$BUILD_STAGING_DIR" 54 | fi 55 | 56 | # Clone gcc-builder repo 57 | pushd "$BUILD_STAGING_DIR" 58 | if [ ! -d "$GCC_REPONAME" ] 59 | then 60 | git clone --single-branch --branch "$GCC_BRANCH" "$GCC_REPO" 61 | fi 62 | popd 63 | 64 | # Check if GCC RPM already built 65 | CHECK_GCC_RPM=$(find "$GCC_DIR/packages" -name '*.rpm' | wc -l) 66 | 67 | # Only build GCC RPM if not already built 68 | if [ "$CHECK_GCC_RPM" == 0 ] 69 | then 70 | # Build GCC 71 | pushd "$GCC_DIR" 72 | sed -i "s/^\(export GCC_VERSION=\)5.X.0$/\1$GCC_VERSION/" gcc-build-vars.sh 73 | sed -i "s/^\(export GCC_BUILD_THREADS_ARG=\)'-j8'$/\1'-j$(nproc)'/" gcc-build-vars.sh 74 | sed -i "s/KEWB Computing Build/Samsung R\&D/" gcc-build-vars.sh 75 | sed -i "s/^\(export GCC_CUSTOM_BUILD_STR=\)kewb$/\1dss/" gcc-build-vars.sh 76 | sed -i "s/^\(Name: \)kewb-gcc/\1dss-gcc/" gcc.spec 77 | sed -i "s/^\(Vendor: \)KEWB Enterprises/\1Samsung R\&D/" gcc.spec 78 | sed -i "s/build by KEWB/build by Samsung R\&D/" gcc.spec 79 | sed -i "s/_build_name_fmt %%{NAME}-%%{RELEASE}/_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}/" make-gcc-rpm.sh 80 | ./clean-gcc.sh 81 | ./build-gcc.sh -T | tee build.log 82 | ./stage-gcc.sh 83 | ./make-gcc-rpm.sh -v 84 | popd 85 | else 86 | echo 'GCC RPM already built. Skipping...' 87 | fi 88 | 89 | echo 'Copying GCC RPM to Ansible artifacts...' 90 | find "$GCC_DIR/packages" -name '*.rpm' -exec cp {} "$ARTIFACTS_DIR" \; 91 | -------------------------------------------------------------------------------- /scripts/build_kernel.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Build Kernel for DSS runtime 34 | set -e 35 | 36 | # Load utility functions 37 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 38 | . "$SCRIPT_DIR/utils.sh" 39 | 40 | # Set build variables 41 | KERNEL_CONFIG="$SCRIPT_DIR/kernel_config" 42 | KERNEL_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/v5.x/linux-5.1.tar.gz" 43 | KERNEL_TGZ=$(basename $KERNEL_URL) 44 | KERNEL_NAME=$(basename $KERNEL_URL .tar.gz) 45 | KERNEL_DIR="$BUILD_STAGING_DIR/$KERNEL_NAME" 46 | 47 | # Check for submodules in update init recursive if missing 48 | checksubmodules 49 | 50 | # Create BUILD_STAGING_DIR if it doesn't exist 51 | if [ ! -d "$BUILD_STAGING_DIR" ] 52 | then 53 | mkdir "$BUILD_STAGING_DIR" 54 | fi 55 | 56 | # Download kernel source tarball 57 | pushd "$BUILD_STAGING_DIR" 58 | if [ ! -d "$KERNEL_NAME" ] 59 | then 60 | curl "$KERNEL_URL" --output "$KERNEL_TGZ" 61 | tar xvfz "./$KERNEL_TGZ" 62 | rm -f "./$KERNEL_TGZ" 63 | fi 64 | popd 65 | 66 | # Check if kernel RPMs already built 67 | CHECK_KERNEL_RPM=$(find "$RPM_DIR" -name 'kernel-*.rpm' | wc -l) 68 | 69 | # Only build kernel RPMs if not already built 70 | if [ "$CHECK_KERNEL_RPM" -lt 3 ] 71 | then 72 | # Build kernel 73 | pushd "$KERNEL_DIR" 74 | # Copy kernel config file 75 | cp "${KERNEL_CONFIG}" .config 76 | 77 | # Make kernel RPMs 78 | make clean 79 | make "-j$(nproc)" rpm-pkg 80 | popd 81 | else 82 | echo 'Kernel RPMs already built. Skipping...' 83 | fi 84 | 85 | echo 'Copying kernel RPMs to Ansible artifacts...' 86 | find "$RPM_DIR" -name 'kernel-*.rpm' -exec cp {} "$ARTIFACTS_DIR/" \; 87 | -------------------------------------------------------------------------------- /scripts/build_minio.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | set -e 34 | 35 | # Load utility functions 36 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 37 | . "$SCRIPT_DIR/utils.sh" 38 | 39 | # Check for submodules in update init recursive if missing 40 | checksubmodules 41 | 42 | # Build MinIO 43 | "$MINIO_DIR/build.sh" 44 | 45 | echo "Removing existing MinIO release tarball from dss-ansible artifacts directory" 46 | rm -f "${ARTIFACTS_DIR}"/dss-minio-bin-*.tgz 47 | 48 | echo "Copying MinIO release tarball to dss-ansible artifacts directory" 49 | cp "$MINIO_DIR"/dss-minio-bin-*.tgz "$ARTIFACTS_DIR" 50 | -------------------------------------------------------------------------------- /scripts/build_mlnx-tools.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Build mlnx-tools and mft for DSS runtime 34 | set -e 35 | 36 | # Load utility functions 37 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 38 | . "$SCRIPT_DIR/utils.sh" 39 | 40 | # Set build variables 41 | MLNX_STAGING_DIR="$HOME/workspace" 42 | MFT_URL='https://content.mellanox.com/MFT/mft-4.17.0-106-x86_64-rpm.tgz' 43 | MFT_FILENAME=$(basename $MFT_URL) 44 | MLNX_TOOLS_URL='https://github.com/Mellanox/mlnx-tools' 45 | MLNX_TOOLS_NAME=$(basename "$MLNX_TOOLS_URL") 46 | MLNX_TOOLS_BRANCH='mlnx_ofed' 47 | 48 | # Check for submodules in update init recursive if missing 49 | checksubmodules 50 | 51 | # Create GCC_STAGING_DIR if it doesn't exist 52 | if [ ! -d "$MLNX_STAGING_DIR" ] 53 | then 54 | mkdir "$MLNX_STAGING_DIR" 55 | fi 56 | 57 | # Download mft to Mellanox staging directory if not present 58 | if [ ! -f "$MLNX_STAGING_DIR/$MFT_FILENAME" ] 59 | then 60 | pushd "$MLNX_STAGING_DIR" 61 | curl -O "$MFT_URL" 62 | popd 63 | else 64 | echo "mft RPM already downloaded. Skipping..." 65 | fi 66 | 67 | # Extract mft if not present 68 | if [ ! -f "$MLNX_STAGING_DIR/${MFT_FILENAME%.*}" ] 69 | then 70 | pushd "$MLNX_STAGING_DIR" 71 | tar xvfz "$MFT_FILENAME" 72 | popd 73 | else 74 | echo "mft RPM already extracted. Skipping..." 75 | fi 76 | 77 | # Check if GCC RPM already built 78 | CHECK_MLNX_TOOLS_RPM=$(find "$RPM_DIR" -name 'mlnx-tools*.rpm' | wc -l) 79 | 80 | # Build mlnx-tools rpm if not present 81 | if [ "$CHECK_MLNX_TOOLS_RPM" == 0 ] 82 | then 83 | # Build mlnx-tools 84 | pushd "$MLNX_STAGING_DIR" 85 | if [ ! -d "$MLNX_TOOLS_NAME" ] 86 | then 87 | git clone --single-branch --branch "$MLNX_TOOLS_BRANCH" "$MLNX_TOOLS_URL" 88 | fi 89 | 90 | # Create source tarball 91 | TOOLSVERSION=$(grep -oP "Version: \K.+" "$MLNX_TOOLS_NAME"/mlnx-tools.spec) 92 | SRCTAR="$MLNX_TOOLS_NAME-$TOOLSVERSION.tar.gz" 93 | tar czvf "$SRCTAR" "$MLNX_TOOLS_NAME" --transform "s/$MLNX_TOOLS_NAME/$MLNX_TOOLS_NAME-$TOOLSVERSION/" 94 | mkdir -p "$RPMBUILD_DIR"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} 95 | mv "$SRCTAR" "$RPMBUILD_DIR/SOURCES" 96 | 97 | pushd "$MLNX_TOOLS_NAME" 98 | # Use python3 for CentOS 7 or newer - so resulting RPM works on both CentOS 7 and 8 (Stream) 99 | sed -i "s/^\(%global RHEL8 0%{?rhel} >= \)8/\17/" mlnx-tools.spec 100 | rpmbuild -ba mlnx-tools.spec 101 | popd 102 | popd 103 | else 104 | echo "mlnx-tools RPM already built. Skipping..." 105 | fi 106 | 107 | echo 'Copying mft RPM to Ansible artifacts...' 108 | find "$MLNX_STAGING_DIR/${MFT_FILENAME%.*}" -name "${MFT_FILENAME%-x86_64-rpm.*}*.rpm" -exec cp {} "$ARTIFACTS_DIR/" \; 109 | 110 | echo 'Copying mlnx-tools RPM to Ansible artifacts...' 111 | find "$RPM_DIR" -name 'mlnx-tools-*.rpm' -exec cp {} "$ARTIFACTS_DIR/" \; 112 | -------------------------------------------------------------------------------- /scripts/docker/DSS.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:centos7.8.2003 2 | ENV LANG en_US.UTF-8 3 | ENV LC_ALL en_US.UTF-8 4 | 5 | COPY dss-ansible/artifacts/aws-sdk-cpp-*.rpm ./ 6 | RUN set -eux && \ 7 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 8 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 9 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 10 | yum install -y \ 11 | epel-release \ 12 | centos-release-scl-rh && \ 13 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 14 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 15 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 16 | yum install -y \ 17 | bc \ 18 | bison \ 19 | boost-devel \ 20 | cmake \ 21 | cmake3 \ 22 | cppunit-devel \ 23 | CUnit-devel \ 24 | devtoolset-11 \ 25 | dpkg \ 26 | elfutils-libelf-devel \ 27 | flex \ 28 | gcc \ 29 | gcc-c++ \ 30 | git \ 31 | glibc-devel \ 32 | gmp-devel \ 33 | golang \ 34 | jemalloc-devel \ 35 | Judy-devel \ 36 | libaio-devel \ 37 | libcurl-devel \ 38 | libmpc-devel \ 39 | libuuid-devel \ 40 | man-db \ 41 | meson \ 42 | mpfr-devel \ 43 | ncurses-devel \ 44 | numactl-devel \ 45 | openssl-devel \ 46 | patch \ 47 | pulseaudio-libs-devel \ 48 | python3 \ 49 | python3-devel \ 50 | python3-pip \ 51 | rdma-core-devel \ 52 | redhat-lsb-core \ 53 | rpm-build \ 54 | snappy-devel \ 55 | tbb-devel \ 56 | wget \ 57 | zlib-devel \ 58 | /aws-sdk-cpp*.rpm && \ 59 | rm -f ./*.rpm && \ 60 | yum clean all && \ 61 | rm -rf /var/cache/yum && \ 62 | chmod -R 0777 /var/lib/rpm/ && \ 63 | mkdir /var/cache/yum && \ 64 | chmod 0777 /var/cache/yum && \ 65 | mkdir /.cache && \ 66 | chmod 0777 /.cache && \ 67 | python3 -m pip install --no-cache-dir --upgrade --no-compile --upgrade pip && \ 68 | python3 -m pip install --no-cache-dir --no-compile \ 69 | "ansible>=2.9,<2.10" \ 70 | ansible-lint==5.3.2 \ 71 | gcovr==5.0 \ 72 | pybind11==2.11.1 \ 73 | pycodestyle==2.8.0 \ 74 | shellcheck-py==0.8.0.3 \ 75 | yamllint==1.26.3 && \ 76 | find /usr/lib/ -name '__pycache__' -print0 | xargs -0 -n1 rm -rf && \ 77 | find /usr/lib/ -name '*.pyc' -print0 | xargs -0 -n1 rm -rf && \ 78 | git config --global user.email "docker@msl.lab" && \ 79 | git config --global user.name "Docker Build" && \ 80 | cp /root/.gitconfig / && \ 81 | wget https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip && \ 82 | unzip build-wrapper-linux-x86.zip && \ 83 | rm -rf build-wrapper-linux-x86.zip && \ 84 | wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.8.0.2856-linux.zip && \ 85 | unzip sonar-scanner-cli-4.8.0.2856-linux.zip && \ 86 | rm -rf sonar-scanner-cli-4.8.0.2856-linux.zip && \ 87 | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ 88 | unzip awscliv2.zip && \ 89 | ./aws/install && \ 90 | rm -rf ./aws 91 | ENV PATH="$PATH:$GEM_HOME/bin:/build-wrapper-linux-x86:/sonar-scanner-4.8.0.2856-linux/bin" 92 | ENV PYTHONWARNINGS=ignore::UserWarning 93 | COPY scripts/stagemergeartifacts.sh / 94 | WORKDIR /DSS 95 | -------------------------------------------------------------------------------- /scripts/docker/aws-sdk.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM centos:centos7.8.2003 4 | RUN set -eux && \ 5 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 6 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 7 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 8 | yum install -y \ 9 | epel-release \ 10 | centos-release-scl-rh && \ 11 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 12 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 13 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 14 | yum install -y \ 15 | boost-devel \ 16 | cmake3 \ 17 | devtoolset-11 \ 18 | git \ 19 | libcurl-devel \ 20 | openssl-devel \ 21 | rpm-build && \ 22 | yum clean all && \ 23 | rm -rf /var/cache/yum 24 | WORKDIR /aws-sdk 25 | -------------------------------------------------------------------------------- /scripts/docker/build_all.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # shellcheck disable=SC1091 3 | set -e 4 | 5 | # Load utility functions 6 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 7 | . "$DOCKER_DIR/../utils.sh" 8 | 9 | # DSS Dependencies 10 | "$DOCKER_DIR"/build_aws-sdk.sh 11 | "$DOCKER_DIR"/build_kernel.sh 12 | "$DOCKER_DIR"/build_mlnx-tools.sh 13 | 14 | # Build DSS 15 | "$DOCKER_DIR"/build_dss-sdk.sh 16 | "$DOCKER_DIR"/build_minio.sh 17 | "$DOCKER_DIR"/build_dss-client.sh 18 | "$DOCKER_DIR"/build_datamover.sh 19 | -------------------------------------------------------------------------------- /scripts/docker/build_aws-sdk.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build aws-sdk-cpp in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c aws-sdk -d aws-sdk -t awsbuild -a 'aws-sdk-cpp*.rpm' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_datamover.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build aws-sdk-cpp in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c datamover -d DSS -t dssbuild -a 'nkv-datamover-*.tgz' -u 7 | -------------------------------------------------------------------------------- /scripts/docker/build_docker.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # shellcheck disable=SC1091 3 | 4 | # Generic DSS Build Docker script 5 | set -e 6 | 7 | # Default values 8 | COMPONENT_NAME="" 9 | DOCKERFILE_NAME="" 10 | DOCKER_TAG="" 11 | ARTIFACT_BLOB="" 12 | 13 | # Parse command line arguments 14 | while getopts ":c:d:t:a:u" opt; do 15 | case $opt in 16 | c) COMPONENT_NAME="$OPTARG";; 17 | d) DOCKERFILE_NAME="$OPTARG";; 18 | t) DOCKER_TAG="$OPTARG";; 19 | a) ARTIFACT_BLOB="$OPTARG";; 20 | u) USERFLAG=true;; 21 | \?) echo "Invalid option: -$OPTARG" >&2;; 22 | esac 23 | done 24 | 25 | # Check if required arguments are present 26 | if [[ -z "$COMPONENT_NAME" || -z "$DOCKERFILE_NAME" || -z "$DOCKER_TAG" || -z "$ARTIFACT_BLOB" ]]; then 27 | echo "Usage: $0 -c COMPONENT_NAME -d DOCKERFILE_NAME -t DOCKER_TAG -a ARTIFACT_BLOB" [-u]>&2 28 | exit 1 29 | fi 30 | 31 | # Load utility functions 32 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 33 | . "$DOCKER_DIR/../utils.sh" 34 | 35 | # Check for submodules and update init recursive if missing 36 | checksubmodules 37 | 38 | # Read ARTIFACT_BLOB var as array 39 | IFS=' ' read -ra ARTIFACT_LIST <<< "$ARTIFACT_BLOB" 40 | 41 | # Check number of built artifacts 42 | CHECK_ARTIFACT=0 43 | for ARTIFACT in "${ARTIFACT_LIST[@]}" 44 | do 45 | CHECK_ARTIFACT=$((CHECK_ARTIFACT+$(find "$ARTIFACTS_DIR/" -name "$ARTIFACT" | wc -l))) 46 | done 47 | echo "Num. Artifacts: $CHECK_ARTIFACT" 48 | echo "Num. Artifacts expected: ${#ARTIFACT_LIST[@]}" 49 | 50 | # Only build if number of built artifacts does not match number of blobs 51 | if [ "$CHECK_ARTIFACT" != "${#ARTIFACT_LIST[@]}" ] 52 | then 53 | # Build Docker image 54 | DOCKERBUILDSTR="docker build -t $DOCKER_TAG -f $DOCKER_DIR/$DOCKERFILE_NAME.Dockerfile $DSS_DIR" 55 | echo "BUILDING: $DOCKERBUILDSTR" 56 | eval "$DOCKERBUILDSTR" 57 | 58 | # Check if the docker build should run as the current user 59 | USERSTR='' 60 | CHOWNSTR='' 61 | 62 | if [[ "$USERFLAG" == true ]] 63 | then 64 | USERSTR="-u $(id -u "${USER}"):$(id -g "${USER}")" 65 | echo "Running build in Docker with USERFLAG enabled" 66 | else 67 | for ARTIFACT in "${ARTIFACT_LIST[@]}" 68 | do 69 | CHOWNSTR="$CHOWNSTR && chown $(id -u "${USER}"):$(id -g "${USER}") dss-ansible/artifacts/$ARTIFACT" 70 | done 71 | fi 72 | 73 | # Build component with Docker container 74 | DOCKERRUNSTR="docker run --rm -v $DSS_DIR:/$DOCKERFILE_NAME $USERSTR $DOCKER_TAG sh -c \"./scripts/build_$COMPONENT_NAME.sh $CHOWNSTR\"" 75 | echo "RUNNING: $DOCKERRUNSTR" 76 | eval "$DOCKERRUNSTR" 77 | 78 | else 79 | echo 'Artifact already built. Skipping...' 80 | fi 81 | -------------------------------------------------------------------------------- /scripts/docker/build_dss-client.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build aws-sdk-cpp in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c dss-client -d DSS -t dssbuild -a 'dss_client-*.tgz' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_dss-sdk.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build dss-sdk in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c dss-sdk -d DSS -t dssbuild -a 'nkv-sdk-bin-*.tgz nkv-target-*.tgz' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_gcc.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build dss-gcc in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c gcc -d gcc -t gccbuild -a 'dss-gcc510-*.rpm' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_kernel.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build dss-gcc in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c kernel -d kernel -t kernelbuild -a 'kernel-5.1.0-*.rpm kernel-devel-5.1.0-*.rpm kernel-headers-5.1.0-*.rpm' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_minio.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build dss-gcc in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c minio -d DSS -t dssbuild -a 'dss-minio-bin-*' 7 | -------------------------------------------------------------------------------- /scripts/docker/build_mlnx-tools.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | # Build dss-gcc in Docker 5 | DOCKER_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | "$DOCKER_DIR"/build_docker.sh -c mlnx-tools -d mlnx -t mlnxbuild -a 'mft-*.rpm mlnx-tools-*.rpm' 7 | -------------------------------------------------------------------------------- /scripts/docker/gcc.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM centos:centos7.8.2003 4 | RUN set -eux && \ 5 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 6 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 7 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 8 | yum install -y \ 9 | epel-release && \ 10 | yum install -y \ 11 | gcc \ 12 | gcc-c++ \ 13 | git \ 14 | make \ 15 | redhat-lsb-core \ 16 | rpm-build \ 17 | wget \ 18 | zlib-devel && \ 19 | yum clean all && \ 20 | rm -rf /var/cache/yum 21 | WORKDIR /gcc 22 | -------------------------------------------------------------------------------- /scripts/docker/kernel.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM centos:centos7.8.2003 4 | RUN set -eux && \ 5 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 6 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 7 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 8 | yum install -y \ 9 | epel-release && \ 10 | yum install -y \ 11 | bc \ 12 | bison \ 13 | elfutils-libelf-devel \ 14 | flex \ 15 | gcc \ 16 | make \ 17 | openssl \ 18 | openssl-devel \ 19 | rpm-build && \ 20 | yum clean all && \ 21 | rm -rf /var/cache/yum 22 | WORKDIR /kernel 23 | -------------------------------------------------------------------------------- /scripts/docker/mlnx.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM centos:centos7.8.2003 4 | RUN set -eux && \ 5 | sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ 6 | sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ 7 | sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \ 8 | yum install -y \ 9 | git \ 10 | make \ 11 | python3-devel \ 12 | rpm-build && \ 13 | yum clean all && \ 14 | rm -rf /var/cache/yum 15 | WORKDIR /mlnx 16 | -------------------------------------------------------------------------------- /scripts/load_gcc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # shellcheck disable=SC1090,SC1091 3 | # The Clear BSD License 4 | # 5 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted (subject to the limitations in the disclaimer 10 | # below) provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, 13 | # this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 18 | # contributors may be used to endorse or promote products derived from this 19 | # software without specific prior written permission. 20 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 21 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 25 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Load GCC for DSS build an runtime environment 34 | set -e 35 | 36 | # Load utility functions 37 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 38 | . "$SCRIPT_DIR/utils.sh" 39 | 40 | # Minimum and Maximum GCC versions 41 | GCCMINVER=5.1.0 42 | GCCMAXVER=5.5.0 43 | 44 | # GCC Setenv scripts for CentOS 45 | GCCSETENV='/usr/local/bin/setenv-for-gcc510.sh' 46 | 47 | # Load GCC 5.1.0 paths if the GCCSETENV script is present 48 | if test -f "$GCCSETENV" 49 | then 50 | . $GCCSETENV 51 | fi 52 | 53 | # Check gcc version 54 | GCCVER=$(gcc --version | grep -oP '^gcc \([^)]+\) \K[^ ]+') 55 | 56 | # Validate GCC version is supported 57 | if testvercomp "$GCCVER" "$GCCMINVER" '<' || testvercomp "$GCCVER" "$GCCMAXVER" '>' 58 | then 59 | die "ERROR - Found GCC version: $GCCVER. Must be between $GCCMINVER and $GCCMAXVER." 60 | else 61 | echo "Supported GCC version found: $GCCVER" 62 | fi 63 | -------------------------------------------------------------------------------- /scripts/stagemergeartifacts.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | # stagemainartifacts.sh 4 | # 5 | # Identify newly-built artifacts, then stage them in a folder of the artifacts bucket. 6 | # This folder will match the branch name of the merge. 7 | # If artifacts exist in this folder matching a file glob, remove them first. 8 | # 9 | # This script will only execute when triggered by a GitHub / GitLab merge event. 10 | # More plainly, this is when a commit is pushed to a protected branch (after PR / MR). 11 | # This is determined by the presence of the substring 'refs/heads/' in a $GITHUB_REF var, 12 | # or when $CI_PIPELINE_SOURCE == "push". 13 | # 14 | # This script must be passed the following vars: 15 | # DSSS3URI: The S3 URI pointing to the artifacts bucket 16 | # DSSGLOBLIST: A list of fileglobs expected to be present after a successful build. 17 | # Note if any glob does not match, the script will fail and return non-zero. 18 | # 19 | 20 | # Read DSSGLOBLIST var as array 21 | IFS=' ' read -ra DSSGLOBLIST <<< "$DSSGLOBLIST" 22 | 23 | # Check if DSSS3URI is defined 24 | if [[ "$DSSS3URI" == '' ]] 25 | then 26 | echo "*** ERROR: DSSS3URI var not defined" 27 | exit 1 28 | fi 29 | 30 | # Check if DSSGLOBLIST var provided 31 | if [[ ${DSSGLOBLIST[0]} == '' ]] 32 | then 33 | echo "No artifact globs provided." 34 | exit 1 35 | fi 36 | 37 | # Check if build was triggered by merge. Otherwise skip (eg: skip PR or manual build) 38 | if [[ $GITHUB_REF != "refs/heads"* && $CI_PIPELINE_SOURCE != "push" ]] 39 | then 40 | echo "Not a merge. Skipping artifact rotation." 41 | exit 0 42 | fi 43 | 44 | # Derive the branch name 45 | if [[ $GITHUB_REF ]] 46 | then 47 | BRANCH_NAME=$(echo "$GITHUB_REF" | cut --complement -c 1-11) 48 | elif [[ $CI_COMMIT_BRANCH ]] 49 | then 50 | BRANCH_NAME=$CI_COMMIT_BRANCH 51 | else 52 | echo "*** ERROR - I could not derive the branch name." 53 | exit 1 54 | fi 55 | 56 | # Find local artifacts 57 | LOCALARTIFACTS=() 58 | for GLOB in "${DSSGLOBLIST[@]}" 59 | do 60 | mapfile -t GLOBFILES < <(find . -name "$GLOB") 61 | 62 | if [[ "${GLOBFILES[0]}" == '' ]] 63 | then 64 | echo "*** ERROR - I couldn't find a file matching glob: '$GLOB'! Terminating..." 65 | exit 1 66 | fi 67 | 68 | LOCALARTIFACTS+=("${GLOBFILES[@]}") 69 | done 70 | 71 | # Delete existing main artifacts 72 | for GLOB in "${DSSGLOBLIST[@]}" 73 | do 74 | set +e 75 | mapfile -t REMOTEARTIFACTS < <(aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} ls "$DSSS3URI/$BRANCH_NAME/" | grep -oP "${GLOB//\*/.*}") 76 | set -e 77 | 78 | if [[ "${REMOTEARTIFACTS[0]}" == '' ]] 79 | then 80 | echo "I couldn't find any existing main artifacts matching glob '$GLOB' in the bucket. Skipping deletion..." 81 | else 82 | for artifact in "${REMOTEARTIFACTS[@]}" 83 | do 84 | echo "Deleting existing object from artifacts bucket: $artifact" 85 | aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} rm "$DSSS3URI/$BRANCH_NAME/$artifact" --only-show-errors 86 | done 87 | fi 88 | done 89 | 90 | # Copy new main artifacts 91 | for file in "${LOCALARTIFACTS[@]}" 92 | do 93 | echo "Copying file to artifacts bucket: $file" 94 | aws s3 ${MINIO_HOST_URL:+--endpoint-url $MINIO_HOST_URL} cp "$file" "$DSSS3URI/$BRANCH_NAME/" --only-show-errors 95 | done 96 | -------------------------------------------------------------------------------- /scripts/utils.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # The Clear BSD License 3 | # 4 | # Copyright (c) 2022 Samsung Electronics Co., Ltd. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted (subject to the limitations in the disclaimer 9 | # below) provided that the following conditions are met: 10 | # 11 | # * Redistributions of source code must retain the above copyright notice, 12 | # this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # * Neither the name of Samsung Electronics Co., Ltd. nor the names of its 17 | # contributors may be used to endorse or promote products derived from this 18 | # software without specific prior written permission. 19 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 20 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 21 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 22 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 23 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 24 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 25 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 26 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 27 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 28 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 29 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 30 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | # Set path vars used by all build scripts 33 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 34 | DSS_DIR=$(realpath "$SCRIPT_DIR/..") 35 | export DSS_DIR 36 | export AWS_SPEC_FILE="$SCRIPT_DIR/aws-sdk-cpp.spec" 37 | export RPMBUILD_DIR="$HOME/rpmbuild" 38 | export RPM_DIR="$RPMBUILD_DIR/RPMS" 39 | export ANSIBLE_DIR="$DSS_DIR/dss-ansible" 40 | export ARTIFACTS_DIR="$ANSIBLE_DIR/artifacts" 41 | export DSS_ECOSYSTEM_DIR="$DSS_DIR/dss-ecosystem" 42 | export DSS_CLIENT_DIR="$DSS_ECOSYSTEM_DIR/dss_client" 43 | export DATAMOVER_DIR="$DSS_ECOSYSTEM_DIR/dss_datamover" 44 | export DSS_SDK_DIR="$DSS_DIR/dss-sdk" 45 | export BUILD_STAGING_DIR="$HOME/workspace" 46 | export MINIO_DIR="$DSS_DIR/dss-minio" 47 | 48 | set -e 49 | 50 | # Check submodules for git init and checkout recursive if not 51 | checksubmodules() 52 | { 53 | echo 'Checking submodules for init...' 54 | 55 | # Get list of repository paths from .gitmodules 56 | mapfile -t REPOS < <(grep -oP "path = \K.+" "$DSS_DIR/.gitmodules") 57 | 58 | for REPO in "${REPOS[@]}" 59 | do 60 | echo "Checking $REPO" 61 | if [ ! -e "$DSS_DIR/$REPO/.git" ] 62 | then 63 | pushd "$DSS_DIR" 64 | echo "$REPO not init" 65 | git submodule update --init --recursive 66 | break 67 | popd 68 | else 69 | echo "$REPO already checked out" 70 | fi 71 | done 72 | } 73 | 74 | # Print a message to console and return non-zero 75 | die() 76 | { 77 | echo "$*" 78 | exit 1 79 | } 80 | 81 | # Compare two version strings 82 | vercomp () { 83 | if [[ "$1" == "$2" ]] 84 | then 85 | return 0 86 | fi 87 | local IFS=. 88 | # shellcheck disable=SC2206 89 | local i ver1=($1) ver2=($2) 90 | # fill empty fields in ver1 with zeros 91 | for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) 92 | do 93 | ver1[i]=0 94 | done 95 | for ((i=0; i<${#ver1[@]}; i++)) 96 | do 97 | if [[ -z ${ver2[i]} ]] 98 | then 99 | # fill empty fields in ver2 with zeros 100 | ver2[i]=0 101 | fi 102 | if ((10#${ver1[i]} > 10#${ver2[i]})) 103 | then 104 | return 1 105 | fi 106 | if ((10#${ver1[i]} < 10#${ver2[i]})) 107 | then 108 | return 2 109 | fi 110 | done 111 | return 0 112 | } 113 | 114 | # Test two version strings 115 | testvercomp () { 116 | vercomp "$1" "$2" 117 | case $? in 118 | 0) op='=';; 119 | 1) op='>';; 120 | 2) op='<';; 121 | esac 122 | if [[ "$op" != "$3" ]] 123 | then 124 | return 1 125 | else 126 | return 0 127 | fi 128 | } 129 | --------------------------------------------------------------------------------