├── tests ├── conda_build_config.yaml ├── conda_build_config2.yaml ├── testcondabuild │ └── index.html ├── package_22_fix1.patch ├── package_22_fix2.patch ├── open-ce-licenses.template ├── test-meta1.yaml ├── open-ce-tests2.yaml ├── open-ce-tests4.yaml ├── test-conda-env2.yaml ├── open-ce-tests1.yaml ├── test-conda-env3.yaml ├── my_config.yaml ├── test-conda-env.yaml ├── open-ce-licenses.csv ├── open-ce-tests3.yaml ├── test-meta2.yaml ├── ffmpeg-env.yaml ├── openssl-env.yaml ├── test-env5.yaml ├── pytest.ini ├── test-env4.yaml ├── test-env2.yaml ├── test-env-invalid1.yaml ├── test-env3.yaml ├── test-open-ce-info-1.yaml ├── test-env1.yaml ├── test_env_test.py ├── README.md ├── get_graph_test.py ├── feedstock_tests │ ├── check_recipes.py │ ├── common.py │ └── check_build_numbers.py ├── validate_env_test.py ├── feedstock_tests_test.py ├── inputs_test.py ├── util_test.py ├── helpers.py └── get_licenses_test.py ├── MANIFEST.in ├── conda ├── config │ └── build-config.yaml └── recipe │ └── meta.yaml ├── codecov.yml ├── open_ce ├── __init__.py ├── opence_globals.py ├── test_env.py ├── build_utils │ └── open-ce-common-utils.sh ├── open-ce-builder ├── yaml_utils.py ├── validate_env.py ├── get_graph.py ├── images │ ├── performance-test │ │ ├── ppc │ │ │ └── p9 │ │ │ │ ├── Dockerfile-py310 │ │ │ │ ├── Dockerfile-py39 │ │ │ │ └── Dockerfile-py311 │ │ └── x86 │ │ │ └── p9 │ │ │ ├── Dockerfile-py310 │ │ │ ├── Dockerfile-py311 │ │ │ └── Dockerfile-py39 │ ├── opence-runtime │ │ ├── podman │ │ │ └── Dockerfile │ │ └── docker │ │ │ └── Dockerfile │ ├── builder │ │ ├── ubi9 │ │ │ ├── Dockerfile │ │ │ └── Dockerfile-p10 │ │ └── ubi8 │ │ │ ├── Dockerfile-s390x │ │ │ ├── Dockerfile │ │ │ └── Dockerfile-p10 │ └── builder-cuda │ │ ├── ubi9 │ │ └── Dockerfile.cuda-12.2 │ │ └── ubi8 │ │ ├── Dockerfile.cuda-12.2 │ │ └── Dockerfile.cuda-11.8 ├── constants.py ├── validate_config.py ├── graph.py ├── conda_env_file_generator.py ├── build_env.py ├── build_command.py ├── env_config.py ├── build_image.py ├── errors.py └── open-ce ├── .github └── workflows │ ├── publish_to_anaconda.yml │ ├── conda-recipe-test.yml │ ├── xgboost-build.yml │ └── builder-unit-test.yml ├── packaging_scripts └── create_conda_package.sh ├── .gitignore ├── setup.py ├── git_tools ├── apply_patches.py ├── tag_all_repos.py └── create_version_branch.py └── doc └── README.cuda_support.md /tests/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conda_build_config2.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/testcondabuild/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/package_22_fix1.patch: -------------------------------------------------------------------------------- 1 | some fix 1 2 | -------------------------------------------------------------------------------- /tests/package_22_fix2.patch: -------------------------------------------------------------------------------- 1 | some fix 2 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include open_ce/images/ * 4 | -------------------------------------------------------------------------------- /conda/config/build-config.yaml: -------------------------------------------------------------------------------- 1 | recipes: 2 | - name: open-ce-builder 3 | path: conda/recipe 4 | -------------------------------------------------------------------------------- /tests/open-ce-licenses.template: -------------------------------------------------------------------------------- 1 | First BSD License is: 2 | {{ licenseInfo["BSD-3-Clause"][0].name }} 3 | -------------------------------------------------------------------------------- /tests/test-meta1.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: test_package1 3 | version: 1.2.3 4 | 5 | build: 6 | number: 3 7 | -------------------------------------------------------------------------------- /tests/open-ce-tests2.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: Test 1 3 | command: "[ 1 -eq 2 ]" 4 | - name: Test 2 5 | command: "[ 1 -eq 1 ]" 6 | - name: Test 3 7 | command: "[ 1 -eq 3 ]" 8 | -------------------------------------------------------------------------------- /tests/open-ce-tests4.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: 3 | - Test 1 4 | command: "[ 1 -eq 2 ]" 5 | - name: Test 2 6 | command: "[ 1 -eq 1 ]" 7 | - name: Test 3 8 | command: "[ 1 -eq 3 ]" 9 | -------------------------------------------------------------------------------- /tests/test-conda-env2.yaml: -------------------------------------------------------------------------------- 1 | #open-ce-variant:py3.8-cuda-openmpi-10.2 2 | #Open-CE Version:open-ce-v1.0.0 3 | channels: 4 | - defaults 5 | dependencies: 6 | - pytest 6.2.2 7 | name: opence-conda-env-py3.8-cuda-openmpi 8 | -------------------------------------------------------------------------------- /tests/open-ce-tests1.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: Test 1 3 | command: echo Test 1 4 | {% if build_type=='cpu' %} 5 | - name: Test 2 6 | command: | 7 | echo Test 2a 8 | echo Test 2b 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/test-conda-env3.yaml: -------------------------------------------------------------------------------- 1 | #open-ce-variant:py3.8-cuda-openmpi-10.2 2 | channels: 3 | - defaults 4 | dependencies: 5 | - pytest 6.2.2 6 | - libopus 1.3.1 7 | - icu 58.2 8 | - libgcc-ng=8.2 9 | - libstdcxx-ng=8.2 10 | name: opence-conda-env-py3.8-cuda-openmpi 11 | -------------------------------------------------------------------------------- /tests/my_config.yaml: -------------------------------------------------------------------------------- 1 | recipes: 2 | {% set py_ver = python | float %} 3 | {% if py_ver < 4.0 %} 4 | - name : my_variant 5 | path: cuda_recipe_path #[build_type == 'cuda'] 6 | path: cpu_recipe_path #[build_type == 'cpu' ] 7 | resources: 8 | memory: 200Gi 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /tests/test-conda-env.yaml: -------------------------------------------------------------------------------- 1 | channels: 2 | - file://home/user/opence-work/condabuild 3 | - defaults 4 | dependencies: 5 | - package1 1.2.* 6 | - package2 1.2.* 7 | - package3 1.2.* 8 | - package4 1.2.* 9 | - package5 1.2.* 10 | - package6 1.2.* 11 | name: opence-conda-env-py3.8-cuda-openmpi 12 | -------------------------------------------------------------------------------- /tests/open-ce-licenses.csv: -------------------------------------------------------------------------------- 1 | pytest 6.2.2 https://github.com/pytest-dev/pytest/ MIT 2 | libopus 1.3.1 http://opus-codec.org/development/ BSD-3-Clause Copyright 2001-2011 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo 3 | -------------------------------------------------------------------------------- /tests/open-ce-tests3.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | {% if long %} 3 | - name: Test Long 4 | command: "[ 1 -eq 1 ]" 5 | {% endif %} 6 | {% if distributed %} 7 | - name: Test Distributed 8 | command: "[ 2 -eq 2 ]" 9 | {% endif %} 10 | {% if long and distributed %} 11 | - name: Test Long and Distributed 12 | command: "[ 3 -eq 3 ]" 13 | {% endif %} 14 | -------------------------------------------------------------------------------- /tests/test-meta2.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: test_package1 3 | version: 1.2.3 #[ python == '2.3' ] 4 | version: 2.3.4 #[ python == '2.4' ] 5 | 6 | build: 7 | number: 3 #[ python == '2.3' and build_type == 'cpu'] 8 | number: 4 #[ python == '2.3' and build_type == 'cuda'] 9 | number: 1 #[ python == '2.4' and build_type == 'cpu'] 10 | number: 3 #[ python == '2.4' and build_type == 'cuda'] 11 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | precision: 2 3 | round: nearest 4 | range: "70...95" 5 | 6 | comment: 7 | behavior: new # if a new commit is made, delete old comment and make a new one. 8 | require_changes: false # Always post a comment, even if coverage stayed the same. 9 | require_base: no # Always post, even without a base to compare. 10 | require_head: no # Always post, even without a head to compare. 11 | 12 | -------------------------------------------------------------------------------- /open_ce/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021, 2023. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | __version__ = "13.0.4" 19 | -------------------------------------------------------------------------------- /.github/workflows/publish_to_anaconda.yml: -------------------------------------------------------------------------------- 1 | name: Publish package to Anaconda.org on release 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build: 9 | name: Upload package 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: '0' 15 | - uses: conda-incubator/setup-miniconda@v2.0.0 16 | with: 17 | auto-update-conda: false 18 | python-version: "3.11" 19 | - name: Build and upload the conda package 20 | id: build_upload 21 | env: 22 | ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} 23 | run: | 24 | conda install -y conda-build anaconda-client 25 | conda config --set anaconda_upload yes 26 | export ANACONDA_API_TOKEN=$ANACONDA_TOKEN 27 | ./packaging_scripts/create_conda_package.sh 28 | -------------------------------------------------------------------------------- /tests/ffmpeg-env.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2023. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | builder_version: ">=11.0" 18 | channels: 19 | - defaults 20 | 21 | packages: 22 | - feedstock: ffmpeg-package 23 | -------------------------------------------------------------------------------- /tests/openssl-env.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2023. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | builder_version: ">=11.0" 18 | channels: 19 | - defaults 20 | 21 | packages: 22 | - feedstock: openssl-package 23 | -------------------------------------------------------------------------------- /tests/test-env5.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2022. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | imported_envs: 18 | - test-env1.yaml 19 | 20 | packages: 21 | - feedstock : package51 22 | - feedstock : package52 23 | 24 | -------------------------------------------------------------------------------- /.github/workflows/conda-recipe-test.yml: -------------------------------------------------------------------------------- 1 | name: Open-CE PR Tests for Conda recipe 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | jobs: 8 | required_tests: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | with: 13 | fetch-depth: '0' 14 | - uses: conda-incubator/setup-miniconda@v2.0.0 15 | with: 16 | auto-update-conda: false 17 | python-version: "3.10" 18 | - name: Build and install the conda package 19 | id: build_install 20 | run: | 21 | conda install -y conda-build 22 | sh ./packaging_scripts/create_conda_package.sh 23 | conda create -y -n test_opence python=3.11 24 | source /home/runner/.profile 25 | conda activate test_opence 26 | conda install -y open-ce-builder -c file://$(pwd)/open-ce-conda-pkg 27 | conda list open-ce-builder 28 | open-ce build env -h 29 | -------------------------------------------------------------------------------- /packaging_scripts/create_conda_package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | 18 | set -ex 19 | OUTPUT_FOLDER=$(pwd)/open-ce-conda-pkg 20 | 21 | mkdir -p ${OUTPUT_FOLDER} 22 | conda build --output-folder ${OUTPUT_FOLDER} conda/recipe 23 | -------------------------------------------------------------------------------- /tests/pytest.ini: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | [pytest] 18 | python_functions = test_* 19 | addopts = --cov-report xml --cov-report term-missing --cov=open_ce 20 | filterwarnings = 21 | ignore:.*imp module.* is deprecated:DeprecationWarning 22 | ignore:.*numpy version.* 23 | -------------------------------------------------------------------------------- /tests/test-env4.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2022. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | builder_version: ">=1" 18 | 19 | packages: 20 | - feedstock : https://github.com/AnacondaRecipes/package21-feedstock 21 | - feedstock : https://github.com/open-ce/libsndfile-feedstock 22 | 23 | git_tag_for_env: open-ce_v1.7.0 24 | -------------------------------------------------------------------------------- /.github/workflows/xgboost-build.yml: -------------------------------------------------------------------------------- 1 | name: Open-CE XGBoost Build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | python-version: ["3.11"] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - uses: conda-incubator/setup-miniconda@v2.0.0 21 | with: 22 | auto-update-conda: false 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | shell: bash -l {0} 26 | run: | 27 | pip install . 28 | - name: Build XGBoost 29 | shell: bash -l {0} 30 | # yamllint disable rule:line-length 31 | run: | 32 | open-ce build env xgboost-env \ 33 | --container_build \ 34 | --python_versions ${{ matrix.python-version }} \ 35 | --build_types cpu 36 | # yamllint enable rule:line-length 37 | -------------------------------------------------------------------------------- /open_ce/opence_globals.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2022. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | from open_ce.constants import DEFAULT_PPC_ARCH 20 | 21 | global PPC_ARCH_VARIANT # pylint: disable=global-at-module-level 22 | global DEFAULT_PYTHON_VERS # pylint: disable=global-at-module-level 23 | PPC_ARCH_VARIANT = DEFAULT_PPC_ARCH 24 | -------------------------------------------------------------------------------- /tests/test-env2.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | imported_envs: 18 | - test-env1.yaml 19 | - test-env5.yaml #[ppc_arch == 'p10'] 20 | channels: 21 | - defaults 22 | packages: 23 | - feedstock : package21 #[python == '2.1' or cudatoolkit=='9.2'] 24 | - feedstock : package22 25 | 26 | external_dependencies: 27 | - external_dep3=5.6.* 28 | -------------------------------------------------------------------------------- /tests/test-env-invalid1.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | chnnels: #<--- This key is misspelled, should cause an error. 18 | - https://anaconda.org/anaconda 19 | packages: 20 | - feedstock : package11 21 | - feedstock : package12 22 | - feedstock : package13 23 | - feedstock : package14 24 | - feedstock : package15 25 | - feedstock : package16 26 | -------------------------------------------------------------------------------- /open_ce/test_env.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | from open_ce import build_env 20 | 21 | COMMAND = 'env' 22 | DESCRIPTION = 'Test Open-CE Environment' 23 | ARGUMENTS = build_env.ARGUMENTS 24 | 25 | def test_env(args): 26 | '''Entry Function''' 27 | args.skip_build_packages = True 28 | args.run_tests = True 29 | build_env.build_env(args) 30 | 31 | ENTRY_FUNCTION = test_env 32 | -------------------------------------------------------------------------------- /open_ce/build_utils/open-ce-common-utils.sh: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2023. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | # This script can contain common functions needed by feedstocks during the builds. 18 | 19 | function cleanup_bazel { 20 | bazel clean --expunge && bazel shutdown 21 | if [[ $? -eq 0 ]]; then 22 | echo "bazel shutdown completed successfully" 23 | else 24 | echo "bazel shutdown failed, now trying to kill bazel process ID: $1" 25 | kill -9 $1 && sleep 20 && echo "Killed bazel process successfully" 26 | fi 27 | } 28 | -------------------------------------------------------------------------------- /tests/test-env3.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | builder_version: ">=1" 18 | 19 | packages: 20 | - feedstock : package21 #[python == '2.1'] 21 | - feedstock : package211 22 | patches: 23 | - non-existing.patch 24 | - feedstock : package22 25 | patches: 26 | - package_22_fix1.patch 27 | - https://raw.githubusercontent.com/cdeepali/open-ce-builder/main/tests/package_22_fix2.patch 28 | - feedstock : package222 29 | runtime_package : False 30 | 31 | external_dependencies: 32 | - external_dep3=5.6.* 33 | -------------------------------------------------------------------------------- /tests/test-open-ce-info-1.yaml: -------------------------------------------------------------------------------- 1 | third_party_packages: 2 | # Test downloading a TypeScript file 3 | - name: DefinitelyTyped.google.analytics 4 | license: MIT 5 | url: 6 | - https://raw.githubusercontent.com/DefinitelyTyped/DefinitelyTyped/ebc69904eb78f94030d5d517b42db20867f679c0/chai/chai.d.ts 7 | version: ebc69904eb78f94030d5d517b42db20867f679c0 8 | # Test downloading a zip file 9 | - name: google-test 10 | license: BSD-equivalent 11 | url: 12 | - https://github.com/google/googletest/archive/release-1.7.0.zip 13 | version: 1.7.0 14 | # Test duplicate entries 15 | - name: google-test 16 | license: BSD-equivalent 17 | url: 18 | - https://github.com/google/googletest/archive/release-1.7.0.zip 19 | version: 1.7.0 20 | # Test searching a repo that has a directory named LICENSE 21 | - name: kissfft 22 | license: BSD-equivalent 23 | url: 24 | - https://github.com/mborgerding/kissfft/archive/36dbc057604f00aacfc0288ddad57e3b21cfc1b8.tar.gz 25 | version: 36dbc057604f00aacfc0288ddad57e3b21cfc1b8 26 | # Test downloading a git repository that doesn't exist 27 | - name: bad_git_package 28 | license: NO_LICENSE 29 | url: 30 | - not_a_url.git 31 | version: 6.0.0 32 | # Test downloading a tar file that doesn't exist 33 | - name: bad_url 34 | license: NO_LICENSE 35 | url: 36 | - not_a_url.tar.gz 37 | version: 123 38 | copyright_string: "Not a real copy right" 39 | -------------------------------------------------------------------------------- /tests/test-env1.yaml: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | channels: 18 | - defaults 19 | packages: 20 | - feedstock : package11 21 | git_tag : package11-bug-fix 22 | recipe_path: package11_recipe_path 23 | - feedstock : https://myhost.com/myorg/package12-feedstock 24 | git_tag : abcde 25 | - feedstock : http://myhost.com/myorg/package13-feedstock.git 26 | git_tag : xyz 27 | - feedstock : git@myhost/myorg/package14-feedstock.git 28 | git_tag : pqr 29 | - feedstock : package15 30 | - feedstock : package16 31 | 32 | git_tag_for_env: release-v1.0 33 | 34 | external_dependencies: 35 | - external_dep1 36 | - external_dep2 5.2 37 | 38 | conda_build_configs: 39 | - conda_build_config2.yaml 40 | - https://raw.githubusercontent.com/open-ce/open-ce-builder/main/tests/conda_build_config.yaml 41 | -------------------------------------------------------------------------------- /conda/recipe/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set name = "open-ce-builder" %} 2 | {% set version = "13.0.4" %} 3 | 4 | package: 5 | name: {{ name }} 6 | version: {{ version }} 7 | 8 | source: 9 | - git_url: ../.. 10 | 11 | build: 12 | number: {{ GIT_DESCRIBE_NUMBER }} 13 | script: python setup.py install --single-version-externally-managed --record record.txt 14 | noarch: python 15 | 16 | requirements: 17 | build: 18 | - python 19 | - setuptools 75.* 20 | run: 21 | - conda-build 22 | - conda 23 | - networkx 24 | - pyyaml 25 | - requests 26 | - jinja2 27 | - python 28 | - git 29 | - matplotlib 3.7.* 30 | - junit-xml 31 | 32 | test: 33 | commands: 34 | - open-ce -h 35 | - open-ce-builder build env -h 36 | - open-ce build feedstock -h 37 | - open-ce build image -h 38 | imports: 39 | - open_ce 40 | - open_ce.build_env 41 | - open_ce.build_feedstock 42 | - open_ce.container_build 43 | - open_ce.build_image 44 | 45 | about: 46 | home: https://github.com/open-ce/open-ce 47 | license: Apache 2.0 48 | license_family: APACHE 49 | license_file: LICENSE 50 | summary: 'Open-CE tools for building feedstocks' 51 | description: | 52 | Open-CE tools package provides the common controlling collection of 53 | configuration and build scripts which are foundational to building 54 | the underlying software component packages (feedstocks) which comprise 55 | the greater Open-CE package ecosystem. 56 | doc_url: https://github.com/open-ce/open-ce-builder 57 | dev_url: https://github.com/open-ce/open-ce-builder 58 | 59 | extra: 60 | recipe-maintainers: 61 | - open-ce/open-ce-dev-team 62 | -------------------------------------------------------------------------------- /open_ce/open-ce-builder: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | 18 | """ 19 | This module simply calls open-ce's main function. It is essentially an alias for open-ce. 20 | """ 21 | 22 | import sys 23 | import os 24 | from importlib.util import spec_from_loader, module_from_spec 25 | from importlib.machinery import SourceFileLoader 26 | 27 | from open_ce.errors import OpenCEError 28 | 29 | spec = spec_from_loader("opence", SourceFileLoader("opence", 30 | os.path.join(os.path.dirname(os.path.abspath(__file__)), 'open-ce'))) 31 | opence = module_from_spec(spec) 32 | spec.loader.exec_module(opence) 33 | 34 | def _main(arg_strings=None): 35 | #pylint: disable=protected-access 36 | opence._main(arg_strings) 37 | 38 | if __name__ == '__main__': 39 | try: 40 | _main() 41 | except OpenCEError as err: 42 | print(err.msg, file=sys.stderr) 43 | sys.exit(1) 44 | 45 | sys.exit(0) 46 | -------------------------------------------------------------------------------- /open_ce/yaml_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import open_ce.utils 20 | 21 | open_ce.utils.check_if_package_exists('pyyaml') 22 | 23 | #pylint: disable=wrong-import-position, wrong-import-order 24 | import yaml 25 | 26 | # Determine if the yaml package contains the C loaders/dumpers. If so, use 27 | # the C versions. Otherwise, use the python versions. 28 | try: 29 | safe_loader = yaml.CSafeLoader 30 | dumper = yaml.CDumper 31 | # pylint: disable=bare-except 32 | except: 33 | safe_loader = yaml.SafeLoader 34 | dumper = yaml.Dumper 35 | 36 | def load(stream): 37 | """ 38 | Use pyyaml's safe loader. If available, the C based version of the loader will be used. 39 | """ 40 | return yaml.load(stream, Loader=safe_loader) 41 | 42 | def dump(data, stream=None, **kwds): 43 | """ 44 | Use pyyaml's dumper. If available, the C based version of the dumper will be used. 45 | """ 46 | return yaml.dump(data, stream, Dumper=dumper, **kwds) 47 | -------------------------------------------------------------------------------- /open_ce/validate_env.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | from open_ce import env_config 20 | from open_ce import utils 21 | from open_ce.inputs import Argument 22 | from open_ce import inputs 23 | from open_ce.errors import OpenCEError, Error 24 | 25 | COMMAND = 'env' 26 | 27 | DESCRIPTION = 'Lint Environment Files' 28 | 29 | ARGUMENTS = [Argument.ENV_FILE, 30 | Argument.GIT_LOCATION] + \ 31 | inputs.VARIANT_ARGS 32 | 33 | def validate_env(args): 34 | '''Entry Function''' 35 | variants = utils.make_variants(args.python_versions, args.build_types, 36 | args.mpi_types, args.cuda_versions) 37 | 38 | for variant in variants: 39 | try: 40 | env_config.load_env_config_files(args.env_config_file, [variant]) 41 | except OpenCEError as exc: 42 | raise OpenCEError(Error.VALIDATE_ENV, args.env_config_file, str(variant), exc.msg) from exc 43 | 44 | ENTRY_FUNCTION = validate_env 45 | -------------------------------------------------------------------------------- /tests/test_env_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | import pathlib 19 | from importlib.util import spec_from_loader, module_from_spec 20 | from importlib.machinery import SourceFileLoader 21 | 22 | test_dir = pathlib.Path(__file__).parent.absolute() 23 | 24 | spec = spec_from_loader("opence", SourceFileLoader("opence", os.path.join(test_dir, '..', 'open_ce', 'open-ce'))) 25 | opence = module_from_spec(spec) 26 | spec.loader.exec_module(opence) 27 | 28 | import open_ce.test_env as test_env 29 | 30 | def test_test_env(mocker): 31 | ''' 32 | This is a test for test_env. Since test_env is a wrapper for build_env, we are mainly relying 33 | on the tests for build_env. 34 | ''' 35 | def validate_build_env(args): 36 | assert args.skip_build_packages == True 37 | assert args.run_tests == True 38 | 39 | mocker.patch('open_ce.build_env.build_env', side_effect=validate_build_env) 40 | 41 | opence._main(["test", test_env.COMMAND, "some_env_file.yaml"]) 42 | -------------------------------------------------------------------------------- /open_ce/get_graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | 21 | from open_ce.inputs import Argument, ENV_BUILD_ARGS 22 | from open_ce import graph 23 | from open_ce import constants 24 | from open_ce.errors import log 25 | 26 | COMMAND = "graph" 27 | 28 | DESCRIPTION = 'Plot a dependency graph.' 29 | 30 | ARGUMENTS = ENV_BUILD_ARGS + \ 31 | [Argument.WIDTH, 32 | Argument.HEIGHT] 33 | 34 | def export_graph(args): 35 | '''Entry Function''' 36 | # Importing BuildTree is intentionally done here because it checks for the 37 | # existence of conda-build as BuildTree uses conda_build APIs. 38 | from open_ce.build_tree import construct_build_tree # pylint: disable=import-outside-toplevel 39 | 40 | build_tree = construct_build_tree(args) 41 | os.makedirs(args.output_folder, exist_ok=True) 42 | graph_output = os.path.join(args.output_folder, constants.DEFAULT_GRAPH_FILE) 43 | graph.export_image(build_tree._tree, graph_output, args.width, args.height) # pylint: disable=protected-access 44 | log.info("Build graph successfully output to: %s", graph_output) 45 | 46 | ENTRY_FUNCTION = export_graph 47 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Builder Tests 2 | 3 | ## Required Conda Packages 4 | 5 | The following packages are required to run the tests: 6 | 7 | * pytest 8 | * pytest-cov 9 | * pytest-mock 10 | 11 | ```shell 12 | $ conda install pytest pytest-cov pytest-mock 13 | ... 14 | ``` 15 | 16 | ## Running Tests 17 | 18 | Before running the tests, install the Open-CE pip package using 19 | following command - 20 | ```shell 21 | $ pip install -e . 22 | ... 23 | ``` 24 | 25 | `pytest.ini` contains configuration information. 26 | 27 | To execute the tests use the following command: 28 | 29 | ```shell 30 | $ pytest tests/ 31 | ================================================================================ test session starts ================================================================================= 32 | platform linux -- Python 3.7.7, pytest-6.0.1, py-1.9.0, pluggy-0.13.1 33 | rootdir: ~/git/open-ce 34 | plugins: cov-2.10.0, mock-3.2.0 35 | collected 10 items 36 | 37 | tests/build_feedstock_test.py ....... [ 70%] 38 | tests/util_test.py ... [100%] 39 | 40 | ----------- coverage: platform linux, python 3.7.7-final-0 ----------- 41 | Name Stmts Miss Cover Missing 42 | ---------------------------------------------------------- 43 | open_ce/__init__.py 0 0 100% 44 | open_ce/build_env.py 163 163 0% 2-332 45 | open_ce/build_feedstock.py 83 6 93% 173-178, 193 46 | open_ce/utils.py 4 0 100% 47 | ---------------------------------------------------------- 48 | TOTAL 250 169 32% 49 | 50 | ================================================================================= 10 passed in 0.59s ================================================================================= 51 | ``` 52 | -------------------------------------------------------------------------------- /tests/get_graph_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | import pathlib 19 | import tempfile 20 | from importlib.util import spec_from_loader, module_from_spec 21 | from importlib.machinery import SourceFileLoader 22 | 23 | test_dir = pathlib.Path(__file__).parent.absolute() 24 | 25 | spec = spec_from_loader("opence", SourceFileLoader("opence", os.path.join(test_dir, '..', 'open_ce', 'open-ce'))) 26 | opence = module_from_spec(spec) 27 | spec.loader.exec_module(opence) 28 | 29 | import open_ce.get_graph as get_graph 30 | import open_ce.constants as constants 31 | 32 | def test_get_graph(caplog): 33 | ''' 34 | This is a complete test of `get_graph`. 35 | ''' 36 | tmp_test = tempfile.TemporaryDirectory() 37 | output_folder = os.path.join(tmp_test.name, "output") 38 | opence._main(["get", get_graph.COMMAND, "xgboost-env.yaml", 39 | "--output_folder", output_folder, 40 | "--python_versions", "3.10", 41 | "--cuda_versions", "11.8", 42 | "--build_types", "cuda,cpu", 43 | "--repository_folder", os.path.join(tmp_test.name, "repos")]) 44 | 45 | assert "Build graph successfully output" in caplog.text 46 | 47 | output_file = os.path.join(output_folder, constants.DEFAULT_GRAPH_FILE) 48 | assert os.path.exists(output_file) 49 | 50 | tmp_test.cleanup() 51 | -------------------------------------------------------------------------------- /tests/feedstock_tests/check_recipes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # ***************************************************************** 4 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ***************************************************************** 18 | 19 | import sys 20 | import os 21 | import pathlib 22 | 23 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 24 | import open_ce.utils as utils # pylint: disable=wrong-import-position 25 | import open_ce.inputs as inputs # pylint: disable=wrong-import-position 26 | from common import get_configs, make_parser, check_recipes 27 | 28 | def main(arg_strings=None): 29 | ''' 30 | Entry function. 31 | ''' 32 | parser = make_parser() 33 | args = inputs.parse_args(parser, arg_strings) 34 | variants = utils.make_variants(args.python_versions, args.build_types, args.mpi_types, args.cuda_versions) 35 | 36 | check_result = True 37 | for variant in variants: 38 | main_build_config_data, main_config = get_configs(variant, args.conda_build_configs) 39 | if main_build_config_data["recipes"] is None: 40 | continue 41 | if not check_recipes(main_build_config_data, main_config, variant): 42 | check_result = False 43 | print("Recipe validation failed for variant '{}'.".format(variant)) 44 | 45 | assert check_result, "All recipes must be valid." 46 | 47 | if __name__ == '__main__': 48 | try: 49 | main() 50 | print("RECIPE VALIDATION SUCCESS") 51 | except Exception as exc: # pylint: disable=broad-except 52 | print("RECIPE VALIDATION ERROR: ", exc) 53 | sys.exit(1) 54 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/ppc/p9/Dockerfile-py310: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.10" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/ppc/p9/Dockerfile-py39: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.9" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/ppc/p9/Dockerfile-py311: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.11" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/x86/p9/Dockerfile-py310: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs glibc-devel psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.10" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/x86/p9/Dockerfile-py311: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs glibc-devel psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.11" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/images/performance-test/x86/p9/Dockerfile-py39: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ENV CICD_GROUP=cicd 10 | ARG GROUP_ID=1500 11 | ENV BUILD_USER=builder 12 | ARG BUILD_ID=1084 13 | 14 | RUN export ARCH="$(uname -m)" && \ 15 | yum repolist && yum install -y perf wget rsync openssh-clients diffutils procps binutils git-lfs glibc-devel psmisc && \ 16 | # Create CICD Group 17 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 18 | # Adduser Builder 19 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 20 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 21 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 22 | chmod +x /tmp/anaconda.sh && \ 23 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 24 | rm -f /tmp/anaconda.sh && \ 25 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 26 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 27 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 28 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 29 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 30 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 31 | $CONDA_HOME/bin/conda --version && \ 32 | mkdir -p $CONDA_HOME/conda-bld && \ 33 | mkdir -p $HOME/.cache && \ 34 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 35 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} 36 | 37 | USER ${BUILD_USER} 38 | RUN export PATH="${PATH}" && \ 39 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 40 | mkdir -p $HOME/.cache && \ 41 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "conda activate base" >> ${HOME}/.bashrc && \ 44 | echo "conda create -n opence python=3.9" >> ${HOME}/.bashrc && \ 45 | echo "conda activate opence" >> ${HOME}/.bashrc 46 | -------------------------------------------------------------------------------- /open_ce/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2022. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | import os 19 | 20 | DEFAULT_BUILD_TYPES = "cpu,cuda" 21 | SUPPORTED_BUILD_TYPES = DEFAULT_BUILD_TYPES 22 | DEFAULT_PYTHON_VERS = "3.11" 23 | SUPPORTED_PYTHON_VERS = "3.9,3.10,3.11" 24 | DEFAULT_MPI_TYPES = "openmpi" 25 | SUPPORTED_MPI_TYPES = "system,openmpi" 26 | DEFAULT_CUDA_VERS = "12.2" 27 | SUPPORTED_CUDA_VERS = "11.8,12.2" 28 | CONDA_BUILD_CONFIG_FILE = "conda_build_config.yaml" 29 | DEFAULT_CONDA_BUILD_CONFIG = os.path.abspath(os.path.join(os.getcwd(), CONDA_BUILD_CONFIG_FILE)) 30 | DEFAULT_GIT_LOCATION = "https://github.com/open-ce" 31 | DEFAULT_ENVS_REPO = "open-ce" 32 | SUPPORTED_GIT_PROTOCOLS = ["https:", "http:", "git@"] 33 | DEFAULT_RECIPE_CONFIG_FILE = "config/build-config.yaml" 34 | CONDA_ENV_FILENAME_PREFIX = "opence-conda-env-" 35 | DEFAULT_OUTPUT_FOLDER = "condabuild" 36 | DEFAULT_TEST_CONFIG_FILE = "tests/open-ce-tests.yaml" 37 | DEFAULT_GIT_TAG = None 38 | OPEN_CE_VARIANT = "open-ce-variant" 39 | DEFAULT_TEST_WORKING_DIRECTORY = "./" 40 | KNOWN_VARIANT_PACKAGES = ["python", "cudatoolkit"] 41 | DEFAULT_LICENSES_FILE = "licenses.csv" 42 | TMP_LICENSE_DIR = "tmp_license_src" 43 | OPEN_CE_INFO_FILE = "open-ce-info.yaml" 44 | CONTAINER_TOOLS = ["podman", "docker"] 45 | DEFAULT_CONTAINER_TOOL = next(filter(lambda tool: os.system(f"which {tool} &> /dev/null") 46 | == 0, CONTAINER_TOOLS), None) 47 | DEFAULT_PKG_FORMAT = "conda" # use .conda output format 48 | NUM_THREAD_POOL = 16 49 | OPEN_CE_VERSION_STRING = "Open-CE Version" 50 | DEFAULT_GRAPH_FILE = "graph.png" 51 | DEFAULT_TEST_RESULT_FILE = "test_results.xml" 52 | DEFAULT_PPC_ARCH = "p9" 53 | DEFAULT_GCC_HOME_DIR = "/opt/rh/gcc-toolset-12/root/usr" 54 | DEFAULT_GCC_13_HOME_DIR = "/opt/rh/gcc-toolset-13/root/usr" 55 | OPENSSL_ENV_FILE = "openssl-env.yaml" 56 | FFMPEG_ENV_FILE = "ffmpeg-env.yaml" 57 | -------------------------------------------------------------------------------- /open_ce/images/opence-runtime/podman/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ARG OPENCE_USER 10 | ARG LOCAL_CONDA_CHANNEL 11 | ARG CONDA_ENV_FILE 12 | ARG TARGET_DIR 13 | 14 | ENV OPENCE_GROUP=${OPENCE_USER} 15 | ARG GROUP_ID=1500 16 | ARG USER_ID=1084 17 | 18 | # GPUs 19 | # nvidia-container-runtime 20 | ENV NVIDIA_VISIBLE_DEVICES all 21 | ENV NVIDIA_DRIVER_CAPABILITIES compute,utility 22 | ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2" 23 | 24 | # Install basic requirements. 25 | RUN export ARCH="$(uname -m)" && \ 26 | yum repolist && yum install -y psmisc \ 27 | sudo openssh && \ 28 | groupadd --non-unique --gid ${GROUP_ID} ${OPENCE_GROUP} && \ 29 | # Adduser 30 | adduser -b /home/ --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 31 | --uid ${USER_ID} ${OPENCE_USER} && \ 32 | echo "${OPENCE_USER} ALL=NOPASSWD: ALL" > /etc/sudoers.d/user-${OPENCE_USER} && \ 33 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 34 | chmod +x /tmp/anaconda.sh && \ 35 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 36 | rm -f /tmp/anaconda.sh && \ 37 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} && \ 38 | chown -R ${OPENCE_USER}:${OPENCE_GROUP} ${CONDA_HOME} 39 | 40 | USER ${OPENCE_USER} 41 | RUN export PATH="${PATH}" && \ 42 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 43 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 44 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 45 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 46 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 47 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 48 | $CONDA_HOME/bin/conda --version && \ 49 | sudo rm -f /run/nologin && \ 50 | yes | ssh-keygen -f $HOME/.ssh/id_rsa -q -t rsa -N '' && cp $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys && \ 51 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 52 | echo "conda activate base" >> ${HOME}/.bashrc && \ 53 | mkdir -p ${TARGET_DIR} 54 | 55 | COPY --chown=${OPENCE_USER}:${OPENCE_GROUP} ${LOCAL_CONDA_CHANNEL} ${TARGET_DIR} 56 | 57 | RUN if [ -n "$CONDA_ENV_FILE" ]; then $CONDA_HOME/bin/conda env create -f ${TARGET_DIR}/${CONDA_ENV_FILE}; fi && \ 58 | rm -rf ${TARGET_DIR} && \ 59 | conda clean -y --all 60 | -------------------------------------------------------------------------------- /open_ce/images/opence-runtime/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi7/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | 9 | ARG OPENCE_USER 10 | ARG LOCAL_CONDA_CHANNEL 11 | ARG CONDA_ENV_FILE 12 | ARG TARGET_DIR 13 | 14 | ENV OPENCE_GROUP=${OPENCE_USER} 15 | ARG GROUP_ID=1500 16 | ARG USER_ID=1084 17 | 18 | # GPUs 19 | # nvidia-container-runtime 20 | ENV NVIDIA_VISIBLE_DEVICES all 21 | ENV NVIDIA_DRIVER_CAPABILITIES compute,utility 22 | ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2" 23 | 24 | # Install basic requirements. 25 | RUN export ARCH="$(uname -m)" && \ 26 | yum repolist && yum install -y psmisc \ 27 | sudo openssh && \ 28 | groupadd --non-unique --gid ${GROUP_ID} ${OPENCE_GROUP} && \ 29 | # Adduser 30 | adduser -b /home/ --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 31 | --uid ${USER_ID} ${OPENCE_USER} && \ 32 | echo "${OPENCE_USER} ALL=NOPASSWD: ALL" > /etc/sudoers.d/user-${OPENCE_USER} && \ 33 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 34 | chmod +x /tmp/anaconda.sh && \ 35 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 36 | rm -f /tmp/anaconda.sh && \ 37 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} && \ 38 | chown -R ${OPENCE_USER}:${OPENCE_GROUP} ${CONDA_HOME} 39 | 40 | USER ${OPENCE_USER} 41 | RUN export PATH="${PATH}" && \ 42 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 43 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 44 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 45 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 46 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 47 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 48 | $CONDA_HOME/bin/conda --version && \ 49 | sudo rm -f /run/nologin && \ 50 | yes | ssh-keygen -f $HOME/.ssh/id_rsa -q -t rsa -N '' && cp $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys && \ 51 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 52 | echo "conda activate base" >> ${HOME}/.bashrc && \ 53 | mkdir -p ${TARGET_DIR} 54 | 55 | COPY ${LOCAL_CONDA_CHANNEL} ${TARGET_DIR} 56 | 57 | RUN sudo chown -R ${OPENCE_USER}:${OPENCE_GROUP} ${TARGET_DIR} && \ 58 | if [ -n "$CONDA_ENV_FILE" ]; then $CONDA_HOME/bin/conda env create -f ${TARGET_DIR}/${CONDA_ENV_FILE}; fi && \ 59 | rm -rf ${TARGET_DIR} && \ 60 | conda clean -y --all 61 | -------------------------------------------------------------------------------- /open_ce/images/builder/ubi9/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/ubi:9.6 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | ENV OPENSSL_VER=3.* 9 | 10 | ENV CICD_GROUP=cicd 11 | ARG GROUP_ID=1500 12 | ENV BUILD_USER=builder 13 | ARG BUILD_ID=1084 14 | 15 | ARG LIMIT_BUILD_RESOURCES=0 16 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 17 | 18 | RUN export ARCH="$(uname -m)" && \ 19 | yum repolist && yum install -y rsync openssh-clients diffutils procps binutils git-lfs glibc-devel file psmisc libxcrypt-compat openssl-devel java-11-openjdk-devel && \ 20 | # Create CICD Group 21 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 22 | # Adduser Builder 23 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 24 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 25 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 26 | chmod +x /tmp/anaconda.sh && \ 27 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 28 | rm -f /tmp/anaconda.sh && \ 29 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 30 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 31 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 32 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 33 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 34 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 35 | $CONDA_HOME/bin/conda --version && \ 36 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 37 | mkdir -p $CONDA_HOME/conda-bld && \ 38 | mkdir -p $HOME/.cache && \ 39 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 40 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 41 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 42 | git config --global http.version HTTP/1.1 && \ 43 | git config --global http.postBuffer 157286400 44 | 45 | USER ${BUILD_USER} 46 | RUN export PATH="${PATH}" && \ 47 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 48 | mkdir -p $HOME/.cache && \ 49 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 50 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 51 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 52 | echo "conda activate base" >> ${HOME}/.bashrc && \ 53 | git config --global http.version HTTP/1.1 && \ 54 | git config --global http.postBuffer 157286400 55 | 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Open-CE Specific Ignores 2 | condabuild/ 3 | 4 | # VSCode Files 5 | .vscode/ 6 | *.code-workspace 7 | 8 | # Byte-compiled / optimized / DLL files 9 | *__pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | cover/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | db.sqlite3-journal 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | .pybuilder/ 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # IPython 89 | profile_default/ 90 | ipython_config.py 91 | 92 | # pyenv 93 | # For a library or package, you might want to ignore these files since the code is 94 | # intended to run in multiple environments; otherwise, check them in: 95 | # .python-version 96 | 97 | # pipenv 98 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 99 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 100 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 101 | # install all needed dependencies. 102 | #Pipfile.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | # ***************************************************************** 5 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ***************************************************************** 19 | """ 20 | import platform 21 | from setuptools import find_packages, setup 22 | import open_ce 23 | 24 | def get_readme(): 25 | with open("README.md") as f: 26 | return f.read() 27 | 28 | 29 | REQUIRED_PACKAGES = [ 30 | "pyyaml", 31 | "requests", 32 | "jinja2", 33 | "networkx", 34 | "junit-xml", 35 | ] 36 | 37 | # We are installing matplotlib only on x86. It is needed for one of the pytests (get_graph_test.py) 38 | # which is run in the PR check and it runs on x86. On ppc, matplotlib's pip package isn't working. 39 | 40 | if platform.machine() == 'x86_64': 41 | REQUIRED_PACKAGES.append("matplotlib==3.7.1") 42 | 43 | setup( 44 | name="open-ce-builder", 45 | version=open_ce.__version__, 46 | description="Open-CE tools for building feedstocks", 47 | long_description=get_readme(), 48 | url="https://github.com/open-ce/open-ce", 49 | author="Open-CE Dev Team", 50 | author_email="", 51 | packages = find_packages(), 52 | scripts = ["open_ce/open-ce", "open_ce/open-ce-builder", "open_ce/build_utils/open-ce-common-utils.sh"], 53 | include_package_data=True, 54 | package_data={ 55 | "open_ce": [ 56 | "open_ce/images", 57 | "doc", 58 | ], 59 | }, 60 | 61 | python_requires=">= 3.9", 62 | install_requires=REQUIRED_PACKAGES, 63 | tests_require=["pytest"], 64 | # PyPI package information. 65 | classifiers=[ 66 | "Development Status :: 5 - Production/Stable", 67 | "Intended Audience :: Developers", 68 | "Intended Audience :: Education", 69 | "Intended Audience :: Science/Research", 70 | "License :: OSI Approved :: Apache Software License", 71 | "Programming Language :: Python :: 3.9", 72 | "Programming Language :: Python :: 3.10", 73 | "Programming Language :: Python :: 3.11" 74 | "Topic :: Software Development :: Libraries :: Python Modules", 75 | "Topic :: Software Development :: Libraries", 76 | ], 77 | license="Apache 2.0", 78 | keywords="Machine learning, build tools", 79 | ) 80 | -------------------------------------------------------------------------------- /open_ce/images/builder/ubi8/Dockerfile-s390x: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.26.1 7 | ENV OPEN_CE_CONDA=4.14 8 | ENV OPENSSL_VER=3.* 9 | 10 | ENV CICD_GROUP=cicd 11 | ARG GROUP_ID=1500 12 | ENV BUILD_USER=builder 13 | ARG BUILD_ID=1084 14 | 15 | ARG LIMIT_BUILD_RESOURCES=0 16 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 17 | 18 | RUN export ARCH="$(uname -m)" && \ 19 | yum repolist && yum install -y rsync openssh-clients diffutils procps binutils git-lfs glibc-devel file psmisc openssl openssl-libs openssl-devel && \ 20 | # Create CICD Group 21 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 22 | # Adduser Builder 23 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 24 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 25 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 26 | chmod +x /tmp/anaconda.sh && \ 27 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 28 | rm -f /tmp/anaconda.sh && \ 29 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 30 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 31 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 32 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 33 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 34 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 35 | $CONDA_HOME/bin/conda config --system --add create_default_packages openssl=${OPENSSL_VER} && \ 36 | $CONDA_HOME/bin/conda --version && \ 37 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 38 | mkdir -p $CONDA_HOME/conda-bld && \ 39 | mkdir -p $HOME/.cache && \ 40 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 41 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 42 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 43 | git config --global http.version HTTP/1.1 && \ 44 | git config --global http.postBuffer 157286400 45 | 46 | USER ${BUILD_USER} 47 | RUN export PATH="${PATH}" && \ 48 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 49 | mkdir -p $HOME/.cache && \ 50 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 51 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 52 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 53 | echo "conda activate base" >> ${HOME}/.bashrc && \ 54 | git config --global http.version HTTP/1.1 && \ 55 | git config --global http.postBuffer 157286400 56 | 57 | -------------------------------------------------------------------------------- /open_ce/images/builder/ubi8/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | ENV OPENSSL_VER=3.* 9 | 10 | ENV CICD_GROUP=cicd 11 | ARG GROUP_ID=1500 12 | ENV BUILD_USER=builder 13 | ARG BUILD_ID=1084 14 | 15 | ARG CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=0 16 | ENV CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=${CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY} 17 | 18 | ARG LIMIT_BUILD_RESOURCES=0 19 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 20 | 21 | RUN export ARCH="$(uname -m)" && \ 22 | yum repolist && yum install -y rsync openssh-clients diffutils procps binutils git-lfs glibc-devel file psmisc openssl-devel && \ 23 | # Create CICD Group 24 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 25 | # Adduser Builder 26 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 27 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 28 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 29 | chmod +x /tmp/anaconda.sh && \ 30 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 31 | rm -f /tmp/anaconda.sh && \ 32 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 33 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 34 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 35 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 36 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 37 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 38 | $CONDA_HOME/bin/conda --version && \ 39 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 40 | mkdir -p $CONDA_HOME/conda-bld && \ 41 | mkdir -p $HOME/.cache && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 44 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 45 | git config --global http.version HTTP/1.1 && \ 46 | git config --global http.postBuffer 157286400 47 | 48 | USER ${BUILD_USER} 49 | RUN export PATH="${PATH}" && \ 50 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 51 | mkdir -p $HOME/.cache && \ 52 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 53 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 54 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 55 | echo "conda activate base" >> ${HOME}/.bashrc && \ 56 | git config --global http.version HTTP/1.1 && \ 57 | git config --global http.postBuffer 157286400 58 | 59 | -------------------------------------------------------------------------------- /open_ce/images/builder-cuda/ubi9/Dockerfile.cuda-12.2: -------------------------------------------------------------------------------- 1 | FROM docker.io/nvidia/cuda:12.2.2-devel-ubi9 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | ENV CUDA_HOME=/usr/local/cuda 6 | 7 | ENV OPEN_CE_CONDA_BUILD=3.22.0 8 | ENV OPEN_CE_CONDA=4.14 9 | ENV OPENSSL_VER=3.* 10 | 11 | ENV CICD_GROUP=cicd 12 | ARG GROUP_ID=1500 13 | ENV BUILD_USER=builder 14 | ARG BUILD_ID=1084 15 | 16 | ARG LIMIT_BUILD_RESOURCES=0 17 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 18 | 19 | # Remove preinclude system compilers 20 | RUN rpm -e --nodeps --verbose gcc gcc-c++ 21 | 22 | RUN export ARCH="$(uname -m)" && \ 23 | yum repolist && yum install -y rsync openssh-clients diffutils procps git-lfs file psmisc libxcrypt-compat openssl-devel && \ 24 | # Create CICD Group 25 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 26 | # Adduser Builder 27 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 28 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 29 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 30 | chmod +x /tmp/anaconda.sh && \ 31 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 32 | rm -f /tmp/anaconda.sh && \ 33 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 34 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 35 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 36 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 37 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 38 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 39 | $CONDA_HOME/bin/conda --version && \ 40 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 41 | mkdir -p $CONDA_HOME/conda-bld && \ 42 | mkdir -p $HOME/.cache && \ 43 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 44 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 45 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 46 | git config --global http.version HTTP/1.1 && \ 47 | git config --global http.postBuffer 157286400 48 | 49 | 50 | USER ${BUILD_USER} 51 | RUN export PATH="${PATH}" && \ 52 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 53 | mkdir -p $HOME/.cache && \ 54 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 55 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 56 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 57 | echo "conda activate base" >> ${HOME}/.bashrc && \ 58 | git config --global http.version HTTP/1.1 && \ 59 | git config --global http.postBuffer 157286400 60 | -------------------------------------------------------------------------------- /.github/workflows/builder-unit-test.yml: -------------------------------------------------------------------------------- 1 | name: Open-CE Builder Unit Tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | python-version: ["3.10", "3.11"] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | with: 21 | fetch-depth: 2 22 | - uses: conda-incubator/setup-miniconda@v2.0.0 23 | with: 24 | auto-update-conda: false 25 | python-version: ${{ matrix.python-version }} 26 | - name: Ensure open-ce can load without conda-build 27 | shell: bash -l {0} 28 | run: | 29 | pip install -e . 30 | conda install -y pyyaml jinja2 networkx 31 | open-ce -h 32 | - name: Install dependencies 33 | shell: bash -l {0} 34 | run: | 35 | conda install -y pylint=2.16.2 \ 36 | conda-build=24.9.0 \ 37 | networkx \ 38 | matplotlib=3.7.* \ 39 | junit-xml \ 40 | pytest=7.4.0 \ 41 | pytest-cov=4.0.0 \ 42 | pytest-mock=3.10.0 \ 43 | setuptools=65.6.3 44 | pip install yamllint==1.24.2 45 | - name: Lint with pylint 46 | shell: bash -l {0} 47 | run: | 48 | PYTHONPATH=./open_ce/:${PYTHONPATH} pylint --disable=unnecessary-lambda-assignment ./open_ce ./open_ce/open-ce 49 | - name: Lint git tools with pylint 50 | shell: bash -l {0} 51 | run: | 52 | PYTHONPATH=./open_ce/:${PYTHONPATH} pylint --disable=unnecessary-lambda-assignment ./git_tools/*.py 53 | - name: Lint all test files for warnings and errors only 54 | shell: bash -l {0} 55 | run: | 56 | PYTHONPATH=./open_ce/:./tests/:${PYTHONPATH} \ 57 | pylint \ 58 | --disable=R,C,protected-access,unnecessary-lambda-assignment \ 59 | ./tests/*.py \ 60 | ./tests/**/*.py 61 | - name: Lint config files 62 | shell: bash -l {0} 63 | run: | 64 | # Use yamllint to check the actual yaml files. 65 | yamllint -d "{extends: relaxed, rules: {line-length: {max: 120}}}" .github/workflows/*.yml 66 | - name: Test with pytest and Generate coverage report 67 | shell: bash -l {0} 68 | run: | 69 | pip install -e . 70 | pytest tests/ 71 | # - name: Upload coverage to Codecov 72 | # uses: codecov/codecov-action@v1 73 | #with: 74 | #file: ./coverage.xml 75 | #directory: ./ 76 | #flags: unittests 77 | #env_vars: OS,PYTHON 78 | #name: codecov-umbrella 79 | #fail_ci_if_error: true 80 | #path_to_write_report: ./codecov_report.gz 81 | -------------------------------------------------------------------------------- /open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-12.2: -------------------------------------------------------------------------------- 1 | FROM docker.io/nvidia/cuda:12.2.2-devel-ubi8 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | ENV CUDA_HOME=/usr/local/cuda 6 | 7 | ENV OPEN_CE_CONDA_BUILD=3.22.0 8 | ENV OPEN_CE_CONDA=4.14 9 | ENV OPENSSL_VER=3.* 10 | 11 | ENV CICD_GROUP=cicd 12 | ARG GROUP_ID=1500 13 | ENV BUILD_USER=builder 14 | ARG BUILD_ID=1084 15 | 16 | ARG CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=0 17 | ENV CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=${CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY} 18 | 19 | ARG LIMIT_BUILD_RESOURCES=0 20 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 21 | 22 | # Remove preinclude system compilers 23 | RUN rpm -e --nodeps --verbose gcc gcc-c++ 24 | 25 | RUN export ARCH="$(uname -m)" && \ 26 | yum repolist && yum install -y rsync openssh-clients diffutils procps git-lfs file psmisc openssl-devel && \ 27 | # Create CICD Group 28 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 29 | # Adduser Builder 30 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 31 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 32 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 33 | chmod +x /tmp/anaconda.sh && \ 34 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 35 | rm -f /tmp/anaconda.sh && \ 36 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 37 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 38 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 39 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 40 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 41 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 42 | $CONDA_HOME/bin/conda --version && \ 43 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 44 | mkdir -p $CONDA_HOME/conda-bld && \ 45 | mkdir -p $HOME/.cache && \ 46 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 47 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 48 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 49 | git config --global http.version HTTP/1.1 && \ 50 | git config --global http.postBuffer 157286400 51 | 52 | 53 | USER ${BUILD_USER} 54 | RUN export PATH="${PATH}" && \ 55 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 56 | mkdir -p $HOME/.cache && \ 57 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 58 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 59 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 60 | echo "conda activate base" >> ${HOME}/.bashrc && \ 61 | git config --global http.version HTTP/1.1 && \ 62 | git config --global http.postBuffer 157286400 63 | -------------------------------------------------------------------------------- /tests/feedstock_tests/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # ***************************************************************** 4 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ***************************************************************** 18 | 19 | import argparse 20 | import sys 21 | import os 22 | import pathlib 23 | 24 | import conda_build.api 25 | from conda_build.config import get_or_merge_config 26 | 27 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 28 | import open_ce.build_feedstock as build_feedstock # pylint: disable=wrong-import-position 29 | import open_ce.inputs as inputs # pylint: disable=wrong-import-position 30 | 31 | def make_parser(): 32 | ''' Parser for input arguments ''' 33 | arguments = [inputs.Argument.PYTHON_VERSIONS, inputs.Argument.BUILD_TYPES, inputs.Argument.MPI_TYPES, 34 | inputs.Argument.CUDA_VERSIONS, inputs.Argument.CONDA_BUILD_CONFIG] 35 | parser = argparse.ArgumentParser(arguments) 36 | for argument in arguments: 37 | argument(parser) 38 | return parser 39 | 40 | def get_build_numbers(build_config_data, config, variant): 41 | build_numbers = dict() 42 | for recipe in build_config_data["recipes"]: 43 | metas = conda_build.api.render(recipe['path'], 44 | config=config, 45 | variants=variant, 46 | bypass_env_check=True, 47 | finalize=True) 48 | for meta,_,_ in metas: 49 | build_numbers[meta.meta['package']['name']] = {"version" : meta.meta['package']['version'], 50 | "number" : meta.meta['build']['number']} 51 | return build_numbers 52 | 53 | def get_configs(variant, conda_build_config=None): 54 | build_config_data, _ = build_feedstock.load_package_config(variants=variant) 55 | config = get_or_merge_config(None, variant=variant) 56 | config.variant_config_files = conda_build_config if conda_build_config else [] 57 | config.verbose = False 58 | recipe_conda_build_config = build_feedstock.get_conda_build_config() 59 | if recipe_conda_build_config: 60 | config.variant_config_files.append(recipe_conda_build_config) 61 | return build_config_data, config 62 | 63 | def check_recipes(build_config_data, config, variant): 64 | return all(conda_build.api.check(recipe['path'], config=config, variants=variant) for recipe in build_config_data["recipes"]) 65 | -------------------------------------------------------------------------------- /open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-11.8: -------------------------------------------------------------------------------- 1 | FROM docker.io/nvidia/cuda:11.8.0-devel-ubi8 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | ENV CUDA_HOME=/usr/local/cuda 6 | 7 | ENV OPEN_CE_CONDA_BUILD=3.22.0 8 | ENV OPEN_CE_CONDA=4.14 9 | ENV OPENSSL_VER=3.* 10 | 11 | ENV CICD_GROUP=cicd 12 | ARG GROUP_ID=1500 13 | ENV BUILD_USER=builder 14 | ARG BUILD_ID=1084 15 | 16 | ARG CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=0 17 | ENV CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=${CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY} 18 | 19 | ARG LIMIT_BUILD_RESOURCES=0 20 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 21 | 22 | # Remove preinclude system compilers 23 | RUN rpm -e --nodeps --verbose gcc gcc-c++ 24 | 25 | RUN export ARCH="$(uname -m)" && \ 26 | yum repolist && yum install -y rsync openssh-clients diffutils procps git-lfs file psmisc openssl openssl-libs openssl-devel && \ 27 | # Create CICD Group 28 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 29 | # Adduser Builder 30 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 31 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 32 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 33 | chmod +x /tmp/anaconda.sh && \ 34 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 35 | rm -f /tmp/anaconda.sh && \ 36 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 37 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 38 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 39 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 40 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 41 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 42 | $CONDA_HOME/bin/conda config --system --add create_default_packages openssl=${OPENSSL_VER} && \ 43 | $CONDA_HOME/bin/conda --version && \ 44 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 45 | mkdir -p $CONDA_HOME/conda-bld && \ 46 | mkdir -p $HOME/.cache && \ 47 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 48 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 49 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 50 | git config --global http.version HTTP/1.1 && \ 51 | git config --global http.postBuffer 157286400 52 | 53 | 54 | USER ${BUILD_USER} 55 | RUN export PATH="${PATH}" && \ 56 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 57 | mkdir -p $HOME/.cache && \ 58 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 59 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 60 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 61 | echo "conda activate base" >> ${HOME}/.bashrc && \ 62 | git config --global http.version HTTP/1.1 && \ 63 | git config --global http.postBuffer 157286400 64 | 65 | -------------------------------------------------------------------------------- /open_ce/validate_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | from open_ce import utils 20 | from open_ce.inputs import ENV_BUILD_ARGS 21 | from open_ce.errors import OpenCEError, Error, log 22 | 23 | COMMAND = 'config' 24 | 25 | DESCRIPTION = 'Perform validation on a conda_build_config.yaml file.' 26 | 27 | ARGUMENTS = ENV_BUILD_ARGS 28 | 29 | def validate_config(args): 30 | '''Entry Function 31 | Validates a lits of Open-CE env files against a conda build config 32 | for a given set of variants. 33 | ''' 34 | # Importing BuildTree is intentionally done here because it checks for the 35 | # existence of conda-build as BuildTree uses conda_build APIs. 36 | from open_ce.build_tree import construct_build_tree # pylint: disable=import-outside-toplevel 37 | 38 | for env_file in list(args.env_config_file): #make a copy of the env_file list 39 | log.info('Validating %s for %s', args.conda_build_configs, env_file) 40 | try: 41 | args.env_config_file = [env_file] 42 | _ = construct_build_tree(args) 43 | except OpenCEError as err: 44 | raise OpenCEError(Error.VALIDATE_CONFIG, args.conda_build_configs, env_file, err.msg) from err 45 | log.info('Successfully validated %s for %s', args.conda_build_configs, env_file) 46 | 47 | def validate_build_tree(tree, external_deps, start_nodes=None): 48 | ''' 49 | Check a build tree for dependency compatability. 50 | ''' 51 | # Importing BuildTree is intentionally done here because it checks for the 52 | # existence of conda-build as BuildTree uses conda_build APIs. 53 | from open_ce import build_tree # pylint: disable=import-outside-toplevel 54 | 55 | packages = [package for recipe in build_tree.traverse_build_commands(tree, start_nodes) 56 | for package in recipe.packages] 57 | channels = {channel for recipe in build_tree.traverse_build_commands(tree, start_nodes) for channel in recipe.channels} 58 | env_channels = {channel for node in tree.nodes() for channel in node.channels} 59 | deps = build_tree.get_installable_packages(tree, external_deps, start_nodes, True) 60 | 61 | pkg_args = " ".join([f"\'{utils.generalize_version(dep)}\'" for dep in deps 62 | if not utils.remove_version(dep) in packages]) 63 | channel_args = " ".join({f"-c \'{channel}\'" for channel in channels.union(env_channels)}) 64 | 65 | cli = f"conda create --dry-run -n test_conda_dependencies {channel_args} {pkg_args}" 66 | ret_code, std_out, std_err = utils.run_command_capture(cli) 67 | if not ret_code: 68 | raise OpenCEError(Error.VALIDATE_BUILD_TREE, cli, std_out, std_err) 69 | 70 | ENTRY_FUNCTION = validate_config 71 | -------------------------------------------------------------------------------- /open_ce/graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import networkx as nx 20 | from open_ce.utils import check_if_package_exists 21 | 22 | class OpenCEGraph(nx.DiGraph): 23 | """ 24 | Wrapper class for the NetworkX DiGraph class. 25 | Improves functionality for objects that can be equal but different. 26 | """ 27 | def add_node(self, node_for_adding, **attr): 28 | """ 29 | Add a single node `node_for_adding` and update node attributes. 30 | """ 31 | existing_node = next((x for x in self.nodes() if x == node_for_adding), None) 32 | if existing_node: 33 | super().add_node(existing_node, **attr) 34 | else: 35 | super().add_node(node_for_adding, **attr) 36 | 37 | def add_nodes_from(self, nodes_for_adding, **attr): 38 | """ 39 | Add multiple nodes. 40 | """ 41 | for n in nodes_for_adding: 42 | if isinstance(n, tuple): 43 | n, ndict = n 44 | newdict = attr.copy() 45 | newdict.update(ndict) 46 | else: 47 | newdict = attr 48 | self.add_node(n, **newdict) 49 | 50 | def add_edge(self, u_of_edge, v_of_edge, **attr): 51 | """ 52 | Add an edge between u and v. 53 | """ 54 | existing_u = next((x for x in self.nodes() if x == u_of_edge), None) 55 | if existing_u is None: 56 | super().add_node(u_of_edge) 57 | existing_u = u_of_edge 58 | 59 | existing_v = next((x for x in self.nodes() if x == v_of_edge), None) 60 | if existing_v is None: 61 | super().add_node(v_of_edge) 62 | existing_v = v_of_edge 63 | 64 | super().add_edge(existing_u, existing_v, **attr) 65 | 66 | def add_edges_from(self, ebunch_to_add, **attr): 67 | """ 68 | Add all the edges in ebunch_to_add. 69 | """ 70 | for e in ebunch_to_add: 71 | ne = len(e) 72 | if ne == 3: 73 | u, v, dd = e 74 | elif ne == 2: 75 | u, v = e 76 | dd = {} 77 | else: 78 | raise nx.NetworkXError(f"Edge tuple {e} must be a 2-tuple or 3-tuple.") 79 | newdict = attr.copy() 80 | newdict.update(dd) 81 | self.add_edge(u, v, **newdict) 82 | 83 | def export_image(graph, output_file, x=50, y=50): 84 | ''' 85 | Export a plot of a graph using matplotlib. 86 | ''' 87 | check_if_package_exists('matplotlib') 88 | import matplotlib.pyplot as plt # pylint: disable=import-outside-toplevel 89 | f = plt.figure() 90 | f.set_figwidth(x) 91 | f.set_figheight(y) 92 | nx.draw_circular(graph, with_labels = True) 93 | plt.savefig(output_file) 94 | -------------------------------------------------------------------------------- /open_ce/images/builder/ubi8/Dockerfile-p10: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/ubi 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | ENV OPENSSL_VER=3.* 9 | 10 | ENV CICD_GROUP=cicd 11 | ARG GROUP_ID=1500 12 | ENV BUILD_USER=builder 13 | ARG BUILD_ID=1084 14 | ARG CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=0 15 | ENV CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY=${CONDA_ADD_PIP_AS_PYTHON_DEPENDENCY} 16 | ARG LIMIT_BUILD_RESOURCES=0 17 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 18 | 19 | RUN export ARCH="$(uname -m)" && \ 20 | yum install -y yum-utils rsync openssh-clients diffutils procps git-lfs gcc-toolset-12 gcc-toolset-13 glibc-devel file psmisc libX11-devel openssl openssl-libs openssl-devel && \ 21 | yum install -y libtirpc-devel && \ 22 | # Create CICD Group 23 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 24 | # Adduser Builder 25 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 26 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 27 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 28 | chmod +x /tmp/anaconda.sh && \ 29 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 30 | rm -f /tmp/anaconda.sh && \ 31 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 32 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 33 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 34 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 35 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 36 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 37 | $CONDA_HOME/bin/conda config --system --add create_default_packages openssl=${OPENSSL_VER} && \ 38 | $CONDA_HOME/bin/conda --version && \ 39 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 40 | mkdir -p $CONDA_HOME/conda-bld && \ 41 | mkdir -p $HOME/.cache && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 44 | export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ 45 | export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ 46 | export GCC_13_HOME="/opt/rh/gcc-toolset-13/root/usr" && \ 47 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 48 | echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ 49 | echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ 50 | echo "GCC_13_HOME="${GCC_13_HOME}"" >> ${HOME}/.bashrc && \ 51 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 52 | git config --global http.version HTTP/1.1 && \ 53 | git config --global http.postBuffer 157286400 54 | 55 | 56 | USER ${BUILD_USER} 57 | RUN export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ 58 | export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ 59 | export GCC_13_HOME="/opt/rh/gcc-toolset-13/root/usr" && \ 60 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 61 | echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ 62 | echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ 63 | echo "GCC_13_HOME="${GCC_13_HOME}"" >> ${HOME}/.bashrc && \ 64 | mkdir -p $HOME/.cache && \ 65 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 66 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 67 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 68 | echo "conda activate base" >> ${HOME}/.bashrc && \ 69 | git config --global http.version HTTP/1.1 && \ 70 | git config --global http.postBuffer 157286400 71 | 72 | -------------------------------------------------------------------------------- /open_ce/images/builder/ubi9/Dockerfile-p10: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/ubi:9.6 2 | 3 | ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} 4 | ENV PATH=$CONDA_HOME/bin:$PATH 5 | 6 | ENV OPEN_CE_CONDA_BUILD=3.22.0 7 | ENV OPEN_CE_CONDA=4.14 8 | ENV OPENSSL_VER=3.* 9 | 10 | ENV CICD_GROUP=cicd 11 | ARG GROUP_ID=1500 12 | ENV BUILD_USER=builder 13 | ARG BUILD_ID=1084 14 | 15 | ARG LIMIT_BUILD_RESOURCES=0 16 | ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} 17 | 18 | RUN export ARCH="$(uname -m)" && \ 19 | yum install -y yum-utils rsync openssh-clients diffutils procps git-lfs gcc-toolset-12 gcc-toolset-13 glibc-devel file libxcrypt-compat psmisc libX11-devel openssl openssl-libs openssl-devel java-11-openjdk-devel && \ 20 | yum-config-manager --set-enabled codeready-builder-for-rhel-9-ppc64le-rpms && \ 21 | yum install -y libtirpc-devel && \ 22 | # Create CICD Group 23 | groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ 24 | # Adduser Builder 25 | useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ 26 | --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ 27 | curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ 28 | chmod +x /tmp/anaconda.sh && \ 29 | /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ 30 | rm -f /tmp/anaconda.sh && \ 31 | $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ 32 | $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ 33 | $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ 34 | $CONDA_HOME/bin/conda config --system --set always_yes true && \ 35 | $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ 36 | $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ 37 | $CONDA_HOME/bin/conda config --system --add create_default_packages openssl=${OPENSSL_VER} && \ 38 | $CONDA_HOME/bin/conda --version && \ 39 | $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ 40 | mkdir -p $CONDA_HOME/conda-bld && \ 41 | mkdir -p $HOME/.cache && \ 42 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 43 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 44 | export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ 45 | export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ 46 | export GCC_13_HOME="/opt/rh/gcc-toolset-13/root/usr" && \ 47 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 48 | echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ 49 | echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ 50 | echo "GCC_13_HOME="${GCC_13_HOME}"" >> ${HOME}/.bashrc && \ 51 | chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ 52 | git config --global http.version HTTP/1.1 && \ 53 | git config --global http.postBuffer 157286400 54 | 55 | 56 | USER ${BUILD_USER} 57 | RUN export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ 58 | export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ 59 | export GCC_13_HOME="/opt/rh/gcc-toolset-13/root/usr" && \ 60 | echo "PATH="${PATH}"" >> ${HOME}/.profile && \ 61 | echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ 62 | echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ 63 | echo "GCC_13_HOME="${GCC_13_HOME}"" >> ${HOME}/.bashrc && \ 64 | mkdir -p $HOME/.cache && \ 65 | echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ 66 | echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ 67 | echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ 68 | echo "conda activate base" >> ${HOME}/.bashrc && \ 69 | git config --global http.version HTTP/1.1 && \ 70 | git config --global http.postBuffer 157286400 71 | 72 | -------------------------------------------------------------------------------- /tests/validate_env_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | import pathlib 19 | import pytest 20 | from importlib.util import spec_from_loader, module_from_spec 21 | from importlib.machinery import SourceFileLoader 22 | 23 | test_dir = pathlib.Path(__file__).parent.absolute() 24 | 25 | spec = spec_from_loader("opence", SourceFileLoader("opence", os.path.join(test_dir, '..', 'open_ce', 'open-ce'))) 26 | opence = module_from_spec(spec) 27 | spec.loader.exec_module(opence) 28 | 29 | import open_ce.validate_env as validate_env 30 | from open_ce.errors import OpenCEError 31 | 32 | def test_validate_env(): 33 | ''' 34 | Positive test for validate_env. 35 | ''' 36 | env_file = os.path.join(test_dir, 'test-env2.yaml') 37 | opence._main(["validate", validate_env.COMMAND, env_file]) 38 | 39 | def test_validate_env_negative(caplog): 40 | ''' 41 | Negative test for validate_env. 42 | ''' 43 | env_file = os.path.join(test_dir, 'test-env-invalid1.yaml') 44 | with pytest.raises(OpenCEError) as exc: 45 | opence._main(["validate", validate_env.COMMAND, env_file]) 46 | assert "Unexpected key chnnels was found in " in str(exc.value) 47 | assert "WARNING" in caplog.text 48 | assert "test-env-invalid1.yaml' does not provide 'builder_version'. Possible schema mismatch." in caplog.text 49 | 50 | def test_validate_env_wrong_external_deps(mocker,): 51 | ''' 52 | Test that validate env correctly handles invalid data for external dependencies. 53 | ''' 54 | unused_env_for_arg = os.path.join(test_dir, 'test-env-invalid1.yaml') 55 | env_file = { 'packages' : [{ 'feedstock' : 'test1' }], 'external_dependencies' : 'ext_dep' } 56 | mocker.patch('conda_build.metadata.MetaData.get_rendered_recipe_text', return_value=env_file) 57 | 58 | with pytest.raises(OpenCEError) as exc: 59 | opence._main(["validate", validate_env.COMMAND, unused_env_for_arg]) 60 | assert "ext_dep is not of expected type " in str(exc.value) 61 | 62 | def test_validate_env_dict_for_external_deps(mocker,): 63 | ''' 64 | Test that validate env correctly handles erroneously passing a dict for external dependencies. 65 | ''' 66 | unused_env_for_arg = os.path.join(test_dir, 'test-env-invalid1.yaml') 67 | env_file = { 'packages' : [{ 'feedstock' : 'test1' }], 'external_dependencies' : [{ 'feedstock' : 'ext_dep'}] } 68 | mocker.patch('conda_build.metadata.MetaData.get_rendered_recipe_text', return_value=env_file) 69 | 70 | with pytest.raises(OpenCEError) as exc: 71 | opence._main(["validate", validate_env.COMMAND, unused_env_for_arg]) 72 | assert "{'feedstock': 'ext_dep'} is not of expected type " in str(exc.value) 73 | 74 | def test_validate_env_schema_mismatch(mocker): 75 | ''' 76 | Positive test for validate_env. 77 | ''' 78 | mocker.patch('open_ce.conda_utils.version_matches_spec.__defaults__', ("0.1.0",)) 79 | env_file = os.path.join(test_dir, 'test-env3.yaml') 80 | with pytest.raises(OpenCEError) as exc: 81 | opence._main(["validate", validate_env.COMMAND, env_file]) 82 | assert "test-env3.yaml' expects to be built with Open-CE Builder [>=1]. But this version is " in str(exc.value) 83 | -------------------------------------------------------------------------------- /tests/feedstock_tests/check_build_numbers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # ***************************************************************** 4 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ***************************************************************** 18 | 19 | import sys 20 | import os 21 | import pathlib 22 | 23 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 24 | import open_ce.utils as utils # pylint: disable=wrong-import-position 25 | import open_ce.inputs as inputs # pylint: disable=wrong-import-position 26 | 27 | from common import get_configs, make_parser, get_build_numbers 28 | 29 | def main(arg_strings=None): 30 | ''' 31 | Entry function. 32 | ''' 33 | parser = make_parser() 34 | args = inputs.parse_args(parser, arg_strings) 35 | variants = utils.make_variants(args.python_versions, args.build_types, args.mpi_types, args.cuda_versions) 36 | 37 | pr_branch = utils.get_output("git log -1 --format='%H'") 38 | utils.run_and_log("git remote set-head origin -a") 39 | default_branch = utils.get_output("git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'") 40 | 41 | variant_build_results = dict() 42 | for variant in variants: 43 | utils.run_and_log("git checkout {}".format(pr_branch)) 44 | pr_build_config_data, pr_config = get_configs(variant, args.conda_build_configs) 45 | if pr_build_config_data["recipes"] is None: 46 | continue 47 | 48 | current_pr_build_numbers = get_build_numbers(pr_build_config_data, pr_config, variant) 49 | 50 | utils.run_and_log("git checkout {}".format(default_branch)) 51 | main_build_config_data, main_config = get_configs(variant, args.conda_build_configs) 52 | if main_build_config_data["recipes"] is None: 53 | continue 54 | main_build_numbers = get_build_numbers(main_build_config_data, main_config, variant) 55 | 56 | print("Build Info for Variant: {}".format(variant)) 57 | print("Current PR Build Info: {}".format(current_pr_build_numbers)) 58 | print("Main Branch Build Info: {}".format(main_build_numbers)) 59 | 60 | #No build numbers can go backwards without a version change. 61 | for package in main_build_numbers: 62 | if package in current_pr_build_numbers and current_pr_build_numbers[package]["version"] == main_build_numbers[package]["version"]: 63 | assert int(current_pr_build_numbers[package]["number"]) >= int(main_build_numbers[package]["number"]), "If the version doesn't change, the build number can't be reduced." 64 | 65 | #If packages are added or removed, don't require a version change 66 | if set(main_build_numbers.keys()) != set(current_pr_build_numbers.keys()): 67 | return 68 | 69 | #At least one package needs to increase the build number or change the version. 70 | checks = [current_pr_build_numbers[package]["version"] != main_build_numbers[package]["version"] or 71 | int(current_pr_build_numbers[package]["number"]) > int(main_build_numbers[package]["number"]) 72 | for package in main_build_numbers] 73 | variant_build_results[utils.variant_string(variant.get("python"), variant.get("build_type"), variant.get("mpi_type"), variant.get("cudatoolkit"))] = any(checks) 74 | assert any(variant_build_results.values()), "At least one package needs to increase the build number or change the version in at least one variant." 75 | 76 | if __name__ == '__main__': 77 | try: 78 | main() 79 | print("BUILD NUMBER SUCCESS") 80 | except Exception as exc: # pylint: disable=broad-except 81 | print("BUILD NUMBER ERROR: ", exc) 82 | sys.exit(1) 83 | -------------------------------------------------------------------------------- /tests/feedstock_tests_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | import pathlib 19 | import pytest 20 | import sys 21 | import shutil 22 | 23 | test_dir = pathlib.Path(__file__).parent.absolute() 24 | sys.path.append(os.path.join(test_dir, "feedstock_tests")) 25 | 26 | import check_build_numbers 27 | import check_recipes 28 | import open_ce.utils as utils 29 | 30 | repo_name = "test_git_repo" 31 | git_repo = os.path.join(os.getcwd(), repo_name + ".git") 32 | 33 | def create_git_repository(): 34 | ''' 35 | Create a local git repository that contains a conda recipe. 36 | The main branch will contain the meta.yaml file within test-meta1.yaml. 37 | The new branch will contain the meta.yaml file within test-meta2.yaml. 38 | ''' 39 | meta_path = os.path.join("recipe", "meta.yaml") 40 | 41 | remove_git_repo() 42 | 43 | os.mkdir(git_repo) 44 | utils.run_and_log("git init --bare {}".format(git_repo)) 45 | 46 | utils.run_and_log("git clone {}".format(git_repo)) 47 | os.chdir(repo_name) 48 | os.mkdir("recipe") 49 | shutil.copy(os.path.join(test_dir, "test-meta1.yaml"), meta_path) 50 | utils.run_and_log("git config user.email \"me@test_address.fake\"") 51 | utils.run_and_log("git config user.name \"Fake Name\"") 52 | utils.run_and_log("git add {}".format(meta_path)) 53 | utils.run_and_log("git commit -a -m \"Initial Commit\"") 54 | utils.run_and_log("git push origin HEAD") 55 | 56 | utils.run_and_log("git checkout -b new_test_branch") 57 | shutil.copy(os.path.join(test_dir, "test-meta2.yaml"), meta_path) 58 | utils.run_and_log("git commit -a -m \"Test branch commit\"") 59 | 60 | def remove_git_repo(): 61 | ''' 62 | Delete the test git repository. 63 | ''' 64 | shutil.rmtree(git_repo, ignore_errors=True) 65 | shutil.rmtree(repo_name, ignore_errors=True) 66 | 67 | def test_check_build_numbers(): 68 | ''' 69 | Tests the check_build_number script. 70 | ''' 71 | create_git_repository() 72 | 73 | # Verify that check passes when version is the same, and build number increases 74 | check_build_numbers.main(["--python_versions", "2.3", "--build_types", "cuda"]) 75 | 76 | utils.run_and_log("git checkout new_test_branch") 77 | # Verify that check fails when version is the same, and build number is the same. 78 | with pytest.raises(AssertionError) as exc: 79 | check_build_numbers.main(["--python_versions", "2.3", "--build_types", "cpu"]) 80 | assert "At least one package needs to increase the build number or change the version in at least one variant." in str(exc.value) 81 | utils.run_and_log("git checkout new_test_branch") 82 | # Verify that check passes when version increases and build number goes back to 1. 83 | check_build_numbers.main(["--python_versions", "2.4", "--build_types", "cpu"]) 84 | 85 | utils.run_and_log("git checkout new_test_branch") 86 | # Verify that check passes when version increase and build number stays the same. 87 | check_build_numbers.main(["--python_versions", "2.4", "--build_types", "cuda"]) 88 | 89 | os.chdir("..") 90 | remove_git_repo() 91 | 92 | def test_check_recipe(): 93 | ''' 94 | Tests the check_recipe script. 95 | ''' 96 | shutil.rmtree("recipe", ignore_errors=True) 97 | os.mkdir("recipe") 98 | meta_path = os.path.join("recipe", "meta.yaml") 99 | shutil.copy(os.path.join(test_dir, "test-meta2.yaml"), meta_path) 100 | 101 | check_recipes.main(["--python_versions", "2.3,2.4", "--build_types", "cpu,cuda"]) 102 | 103 | shutil.rmtree("recipe", ignore_errors=True) 104 | 105 | -------------------------------------------------------------------------------- /open_ce/conda_env_file_generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | 21 | from open_ce import constants 22 | 23 | #pylint: disable=too-few-public-methods 24 | class CondaEnvFileGenerator(): 25 | """ 26 | The CondaEnvData class holds all of the information needed to generate a conda 27 | environment file that can be used to create a conda environment from the built packages. 28 | """ 29 | 30 | def __init__(self, 31 | dependencies, 32 | channels 33 | ): 34 | self._dependency_set = dependencies 35 | self._channels = channels 36 | 37 | #pylint: disable=too-many-arguments 38 | def write_conda_env_file(self, 39 | variant_string, 40 | output_folder=None, 41 | env_file_prefix=constants.CONDA_ENV_FILENAME_PREFIX, 42 | path=constants.DEFAULT_OUTPUT_FOLDER, 43 | git_tag_for_env=constants.DEFAULT_GIT_TAG): 44 | """ 45 | This function writes conda environment files using the dependency dictionary 46 | created from all the buildcommands. 47 | 48 | It returns the path to the file that was written. 49 | """ 50 | #pylint: disable=import-outside-toplevel 51 | import open_ce.yaml_utils 52 | 53 | os.makedirs(path, exist_ok=True) 54 | 55 | conda_env_name = env_file_prefix + variant_string 56 | conda_env_file = conda_env_name + ".yaml" 57 | conda_env_file = os.path.join(path, conda_env_file) 58 | 59 | channels = _create_channels(self._channels, output_folder) 60 | 61 | # data = dict( 62 | # name = conda_env_name, 63 | # channels = channels, 64 | # dependencies = self._dependency_set, 65 | # ) 66 | data = { 67 | 'name': conda_env_name, 68 | 'channels': channels, 69 | 'dependencies': self._dependency_set, 70 | } 71 | if not git_tag_for_env: 72 | # git_tag_for_env will be None in the case open-ce's main branch is used. 73 | # And, main branch means the latest code, hence open-ce version string should be `latest`. 74 | git_tag_for_env = "latest" 75 | with open(conda_env_file, 'w', encoding='utf8') as outfile: 76 | outfile.write("#" + constants.OPEN_CE_VARIANT + ":" + variant_string + "\n") 77 | outfile.write("#" + constants.OPEN_CE_VERSION_STRING + ":" + git_tag_for_env + "\n" ) 78 | open_ce.yaml_utils.dump(data, outfile, default_flow_style=False) 79 | file_name = conda_env_file 80 | 81 | return file_name 82 | 83 | def get_variant_string(conda_env_file): 84 | """ 85 | Return the variant string from a conda environment file that was added by CondaEnvFileGenerator. 86 | If a variant string was not added to the conda environment file, None will be returned. 87 | """ 88 | with open(conda_env_file, 'r', encoding='utf8') as stream: 89 | first_line = stream.readline().strip()[1:] 90 | values = first_line.split(':') 91 | if values[0] == constants.OPEN_CE_VARIANT: 92 | return values[1] 93 | 94 | return None 95 | 96 | def get_channels(conda_env_file): 97 | """ 98 | Returns a list of the channels specified in the conda environment file. 99 | """ 100 | #pylint: disable=import-outside-toplevel 101 | import open_ce.yaml_utils 102 | 103 | with open(conda_env_file, 'r', encoding='utf8') as file_handle: 104 | env_info = open_ce.yaml_utils.load(file_handle) 105 | 106 | return env_info.get('channels') 107 | 108 | def _create_channels(channels, output_folder): 109 | result = [] 110 | 111 | result.append("file:/" + output_folder) 112 | if channels: 113 | result += channels 114 | result.append("defaults") 115 | 116 | return result 117 | -------------------------------------------------------------------------------- /tests/inputs_test.py: -------------------------------------------------------------------------------- 1 | 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2021, 2022. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | 18 | import os 19 | import pathlib 20 | import pytest 21 | from importlib.util import spec_from_loader, module_from_spec 22 | from importlib.machinery import SourceFileLoader 23 | 24 | test_dir = pathlib.Path(__file__).parent.absolute() 25 | 26 | spec = spec_from_loader("opence", SourceFileLoader("opence", os.path.join(test_dir, '..', 'open_ce', 'open-ce'))) 27 | opence = module_from_spec(spec) 28 | spec.loader.exec_module(opence) 29 | 30 | from open_ce.inputs import make_parser, _create_env_config_paths, Argument, _check_ppc_arch, parse_args 31 | from open_ce.errors import OpenCEError, Error 32 | 33 | def test_create_env_config_paths(mocker): 34 | ''' 35 | Test the _create_env_config_paths function. 36 | ''' 37 | mocker.patch('os.path.exists', return_value=0) 38 | envs_repo = "open-ce" 39 | 40 | parser = make_parser([Argument.ENV_FILE, Argument.GIT_LOCATION, Argument.GIT_TAG_FOR_ENV]) 41 | 42 | args = parser.parse_args(["test-env.yaml"]) 43 | _create_env_config_paths(args) 44 | assert args.env_config_file[0] == "https://raw.githubusercontent.com/open-ce/" + envs_repo + "/main/envs/test-env.yaml" 45 | 46 | args = parser.parse_args(["test-env"]) 47 | _create_env_config_paths(args) 48 | assert args.env_config_file[0] == "https://raw.githubusercontent.com/open-ce/" + envs_repo + "/main/envs/test-env.yaml" 49 | 50 | args = parser.parse_args(["test-env", "--git_tag_for_env", "my_tag"]) 51 | _create_env_config_paths(args) 52 | assert args.env_config_file[0] == "https://raw.githubusercontent.com/open-ce/" + envs_repo + "/my_tag/envs/test-env.yaml" 53 | 54 | args = parser.parse_args(["test-env", "--git_location", "https://github.com/my_org"]) 55 | _create_env_config_paths(args) 56 | assert args.env_config_file[0] == "https://raw.githubusercontent.com/my_org/" + envs_repo + "/main/envs/test-env.yaml" 57 | 58 | def test_check_ppc_arch_for_p9(mocker): 59 | ''' 60 | Test if needed environment variables are set if ppc_arch is p9 61 | ''' 62 | mocker.patch( 63 | 'os.path.exists', 64 | return_value=True 65 | ) 66 | if 'GCC_HOME' in os.environ: 67 | del os.environ["GCC_HOME"] 68 | 69 | parser = make_parser([Argument.ENV_FILE, Argument.PPC_ARCH]) 70 | 71 | args = parser.parse_args(["test-env.yaml", "--ppc_arch=p9"]) 72 | _check_ppc_arch(args) 73 | assert "GCC_HOME" not in os.environ 74 | 75 | def test_check_ppc_arch_for_p10(mocker): 76 | ''' 77 | Test if needed environment variables are set if ppc_arch is p10 78 | ''' 79 | mocker.patch( 80 | 'os.path.exists', 81 | return_value=True 82 | ) 83 | 84 | parser = make_parser([Argument.ENV_FILE, Argument.PPC_ARCH]) 85 | 86 | args = parser.parse_args(["test-env.yaml", "--ppc_arch=p10"]) 87 | _check_ppc_arch(args) 88 | assert "GCC_HOME" in os.environ 89 | del os.environ["GCC_HOME"] 90 | 91 | def test_check_ppc_arch_for_p10_container_build(mocker): 92 | ''' 93 | Test if needed environment variables are set if ppc_arch is p10 with container_build 94 | ''' 95 | mocker.patch( 96 | 'os.path.exists', 97 | return_value=True 98 | ) 99 | 100 | parser = make_parser([Argument.ENV_FILE, Argument.PPC_ARCH, Argument.CONTAINER_BUILD]) 101 | args_str = ["test-env.yaml", "--container_build", "--ppc_arch=p10"] 102 | parse_args(parser, args_str) 103 | assert "GCC_HOME" not in os.environ 104 | 105 | def test_check_ppc_arch_for_p10_with_no_gcc_path(mocker): 106 | ''' 107 | Test if GCC_HOME don't exist, an error is thrown 108 | ''' 109 | mocker.patch( 110 | 'os.path.exists', 111 | return_value=False 112 | ) 113 | 114 | parser = make_parser([Argument.ENV_FILE, Argument.PPC_ARCH]) 115 | 116 | args = parser.parse_args(["test-env.yaml", "--ppc_arch=p10"]) 117 | 118 | with pytest.raises(OpenCEError) as exc: 119 | _check_ppc_arch(args) 120 | assert Error.GCC12_13_COMPILER_NOT_FOUND.value[1] in str(exc.value) 121 | 122 | -------------------------------------------------------------------------------- /git_tools/apply_patches.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | # ***************************************************************** 4 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ***************************************************************** 18 | """ 19 | 20 | import os 21 | import sys 22 | import pathlib 23 | import git_utils 24 | 25 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 26 | from open_ce.inputs import make_parser as inputs_make_parser # pylint: disable=wrong-import-position 27 | from open_ce.utils import parse_arg_list # pylint: disable=wrong-import-position 28 | 29 | 30 | def make_parser(): 31 | ''' Parser input arguments ''' 32 | parser = inputs_make_parser([git_utils.Argument.PUBLIC_ACCESS_TOKEN, git_utils.Argument.REPO_DIR, 33 | git_utils.Argument.BRANCH, git_utils.Argument.ORG, git_utils.Argument.SKIPPED_REPOS, 34 | git_utils.Argument.REVIEWERS, git_utils.Argument.TEAM_REVIEWERS, 35 | git_utils.Argument.PARAMS], 36 | description = 'Apply patches to all repos in an organization.') 37 | 38 | parser.add_argument( 39 | '--commit-msg', 40 | type=str, 41 | default="Apply Patches.", 42 | help="""Commit message to use.""") 43 | 44 | parser.add_argument( 45 | '--pr-msg', 46 | type=str, 47 | default="Apply Patches.", 48 | help="""PR message to use.""") 49 | 50 | parser.add_argument( 51 | '--patches', 52 | type=str, 53 | default="", 54 | help="""Patches to aply to repos.""") 55 | 56 | return parser 57 | 58 | def _main(arg_strings=None): 59 | parser = make_parser() 60 | args = parser.parse_args(arg_strings) 61 | 62 | skipped_repos = parse_arg_list(args.skipped_repos) 63 | repos = git_utils.get_all_repos(args.github_org, args.pat) 64 | repos = [repo for repo in repos if repo["name"] not in skipped_repos] 65 | 66 | param_dict = {param.split(":")[0]: param.split(":")[1] for param in args.params} 67 | 68 | patches = [os.path.abspath(arg_file) for arg_file in parse_arg_list(args.patches)] 69 | for repo in repos: 70 | try: 71 | print("Beginning " + repo["name"] + "---------------------------") 72 | 73 | repo_path = os.path.abspath(os.path.join(args.repo_dir, repo["name"])) 74 | print("--->Making clone location: " + repo_path) 75 | os.makedirs(repo_path, exist_ok=True) 76 | print(f"--->Cloning {repo['name']}") 77 | git_utils.clone_repo(repo["ssh_url"], repo_path) 78 | head_branch = git_utils.get_current_branch(repo_path) 79 | git_utils.create_branch(repo_path, args.branch) 80 | 81 | for patch in patches: 82 | replaced_patch = git_utils.fill_in_params(patch, param_dict, default_branch=head_branch) 83 | print(f"--->Applying Patch {replaced_patch}") 84 | git_utils.apply_patch(repo_path, replaced_patch) 85 | 86 | print("--->Pushing Branch") 87 | git_utils.push_branch(repo_path, args.branch) 88 | 89 | print("--->Creating PR") 90 | created_pr = git_utils.create_pr(args.github_org, 91 | repo["name"], 92 | args.pat, 93 | args.commit_msg, 94 | args.pr_msg, 95 | args.branch, 96 | head_branch) 97 | 98 | print("--->Requesting PR Review") 99 | git_utils.request_pr_review(args.github_org, 100 | repo["name"], 101 | args.pat, 102 | created_pr["number"], 103 | parse_arg_list(args.reviewers), 104 | parse_arg_list(args.team_reviewers)) 105 | 106 | print("---------------------------" + "Finished " + repo["name"]) 107 | except Exception as exc:# pylint: disable=broad-except 108 | print(f"Error encountered when trying to patch {repo['name']}") 109 | print(exc) 110 | cont = git_utils.ask_for_input("Would you like to continue applying patches to other repos?") 111 | if cont.startswith("y"): 112 | continue 113 | raise 114 | 115 | if __name__ == '__main__': 116 | try: 117 | _main() 118 | sys.exit(0) 119 | except Exception as exception:# pylint: disable=broad-except 120 | print("Error: ", exception) 121 | sys.exit(1) 122 | -------------------------------------------------------------------------------- /open_ce/build_env.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2022. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | import sys 21 | 22 | from open_ce import build_feedstock 23 | from open_ce import container_build 24 | from open_ce import utils 25 | from open_ce.inputs import Argument, ENV_BUILD_ARGS 26 | from open_ce import test_feedstock 27 | from open_ce.errors import OpenCEError, Error, log 28 | 29 | COMMAND = "env" 30 | 31 | DESCRIPTION = 'Build conda environment as part of Open-CE' 32 | 33 | ARGUMENTS = ENV_BUILD_ARGS + \ 34 | [Argument.SKIP_BUILD_PACKAGES, 35 | Argument.FIPS, 36 | Argument.BUILD_FFMPEG, 37 | Argument.RUN_TESTS, 38 | Argument.CONTAINER_BUILD, 39 | Argument.TEST_LABELS, 40 | Argument.CONTAINER_BUILD_ARGS, 41 | Argument.CONTAINER_TOOL, 42 | Argument.CONDA_PKG_FORMAT, 43 | Argument.PPC_ARCH] 44 | 45 | def _run_tests(build_tree, test_labels, conda_env_files, output_folder): 46 | """ 47 | Run through all of the tests within a build tree for the given conda environment files. 48 | 49 | Args: 50 | build_tree (BuildTree): The build tree containing the tests 51 | conda_env_files (dict): A dictionary where the key is a variant string and the value 52 | is the name of a conda environment file. 53 | """ 54 | test_results = {} 55 | # Run test commands for each conda environment that was generated 56 | for variant_string, conda_env_file in conda_env_files.items(): 57 | test_feedstocks = build_tree.get_test_feedstocks(variant_string) 58 | if test_feedstocks: 59 | log.info("\n*** Running tests within the %s conda environment ***\n", os.path.basename(conda_env_file)) 60 | for feedstock in test_feedstocks: 61 | log.info("Running tests for %s", feedstock) 62 | test_result = test_feedstock.test_feedstock(conda_env_file, 63 | test_labels=test_labels, 64 | working_directory=feedstock) 65 | if feedstock not in test_results: 66 | test_results[feedstock] = test_result 67 | else: 68 | test_results[feedstock] += test_result 69 | test_feedstock.process_test_results(test_results, output_folder, test_labels) 70 | 71 | def build_env(args): 72 | '''Entry Function''' 73 | utils.check_conda_build_configs_exist(args.conda_build_configs) 74 | 75 | if args.container_build: 76 | if len(args.cuda_versions.split(',')) > 1: 77 | raise OpenCEError(Error.TOO_MANY_CUDA) 78 | container_build.build_with_container_tool(args, sys.argv) 79 | return 80 | 81 | # Importing BuildTree is intentionally done here because it checks for the 82 | # existence of conda-build as BuildTree uses conda_build APIs. 83 | from open_ce.build_tree import construct_build_tree # pylint: disable=import-outside-toplevel 84 | 85 | build_tree = construct_build_tree(args) 86 | # Generate conda environment files 87 | conda_env_files = build_tree.write_conda_env_files(output_folder=os.path.abspath(args.output_folder), 88 | path=os.path.abspath(args.output_folder)) 89 | log.info("Generated conda environment files from the selected build arguments: %s", conda_env_files.values()) 90 | log.info("One can use these environment files to create a conda" \ 91 | " environment using \"conda env create -f .\"") 92 | 93 | if not args.skip_build_packages: 94 | # Build each package in the packages list 95 | for build_command in build_tree: 96 | if not build_command.all_outputs_exist(args.output_folder): 97 | try: 98 | log.info("Building %s", build_command.recipe) 99 | build_feedstock.build_feedstock_from_command(build_command, 100 | output_folder=os.path.abspath(args.output_folder), 101 | pkg_format=args.conda_pkg_format) 102 | except OpenCEError as exc: 103 | raise OpenCEError(Error.BUILD_RECIPE, build_command.repository, exc.msg) from exc 104 | else: 105 | log.info("Skipping build of %s because it already exists.", build_command.recipe) 106 | 107 | if args.run_tests: 108 | _run_tests(build_tree, utils.parse_arg_list(args.test_labels), conda_env_files, os.path.abspath(args.output_folder)) 109 | 110 | ENTRY_FUNCTION = build_env 111 | -------------------------------------------------------------------------------- /open_ce/build_command.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | from open_ce import utils 21 | 22 | class BuildCommand(): 23 | """ 24 | The BuildCommand class holds all of the information needed to call the build_feedstock 25 | function a single time. 26 | """ 27 | #pylint: disable=too-many-instance-attributes,too-many-arguments,too-many-locals 28 | def __init__(self, 29 | recipe, 30 | repository, 31 | packages, 32 | version=None, 33 | recipe_path=None, 34 | runtime_package=True, 35 | output_files=None, 36 | python=None, 37 | build_type=None, 38 | mpi_type=None, 39 | cudatoolkit=None, 40 | run_dependencies=None, 41 | host_dependencies=None, 42 | build_dependencies=None, 43 | test_dependencies=None, 44 | channels=None, 45 | resources=None, 46 | conda_build_configs=None): 47 | self.recipe = recipe 48 | self.repository = repository 49 | self.packages = packages 50 | self.version = version 51 | self.recipe_path = recipe_path 52 | self.runtime_package = runtime_package 53 | self.output_files = set(output_files) if output_files else set() 54 | self.python = python 55 | self.build_type = build_type 56 | self.mpi_type = mpi_type 57 | self.cudatoolkit = cudatoolkit 58 | self.run_dependencies = run_dependencies 59 | self.host_dependencies = host_dependencies 60 | self.build_dependencies = build_dependencies 61 | self.test_dependencies = test_dependencies 62 | self.channels = channels if channels else [] 63 | self.conda_build_configs = conda_build_configs 64 | self.resources = resources 65 | 66 | def feedstock_args(self): 67 | """ 68 | Returns a list of strings that can be provided to the build_feedstock function to 69 | perform a build. 70 | """ 71 | build_args = ["--working_directory", self.repository] 72 | 73 | if self.channels: 74 | for channel in self.channels: 75 | build_args += ["--channels", channel] 76 | 77 | if self.python: 78 | build_args += ["--python_versions", self.python] 79 | if self.build_type: 80 | build_args += ["--build_types", self.build_type] 81 | if self.mpi_type: 82 | build_args += ["--mpi_types", self.mpi_type] 83 | if self.cudatoolkit: 84 | build_args += ["--cuda_versions", self.cudatoolkit] 85 | if self.conda_build_configs: 86 | build_args += ["--conda_build_configs", "\'" + ",".join(self.conda_build_configs) + "\'"] 87 | 88 | if self.recipe: 89 | build_args += ["--recipes", self.recipe] 90 | 91 | return build_args 92 | 93 | def all_outputs_exist(self, output_folder): 94 | """ 95 | Returns true if all of the output_files already exist. 96 | """ 97 | #pylint: disable=import-outside-toplevel 98 | from open_ce import conda_utils 99 | 100 | # Only check the output folder if it already exists and is a conda channel. 101 | channels = self.channels 102 | if os.path.exists(os.path.join(output_folder, 'channeldata.json')): 103 | channels = [os.path.abspath(output_folder)] + channels 104 | 105 | return all(conda_utils.output_file_exists(package, channels) 106 | for package in self.output_files) 107 | 108 | def name(self): 109 | """ 110 | Returns a name representing the Build Command 111 | """ 112 | result = self.recipe 113 | variant_string = utils.variant_string(self.python, self.build_type, self.mpi_type, self.cudatoolkit) 114 | if variant_string: 115 | result += "-" + variant_string 116 | 117 | result = result.replace(".", "-") 118 | result = result.replace("_", "-") 119 | return result 120 | 121 | def __repr__(self): 122 | return ",".join(self.output_files) 123 | 124 | def __str__(self): 125 | return self.name() 126 | 127 | def __hash__(self): 128 | return hash(self.__repr__()) 129 | 130 | def __eq__(self, other): 131 | if not isinstance(other, BuildCommand): 132 | return False 133 | return self.output_files == other.output_files 134 | 135 | def get_all_dependencies(self): 136 | ''' 137 | Return a union of all dependencies. 138 | ''' 139 | return self.run_dependencies.union(self.host_dependencies, self.build_dependencies, self.test_dependencies) 140 | -------------------------------------------------------------------------------- /tests/util_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import pathlib 18 | import errno 19 | import pytest 20 | import os 21 | 22 | test_dir = pathlib.Path(__file__).parent.absolute() 23 | 24 | import open_ce.utils as utils 25 | from open_ce.errors import OpenCEError 26 | 27 | def test_parse_arg_list_list_input(): 28 | ''' 29 | Parse arg list should return the input argument if it's already a list. 30 | ''' 31 | list_input = ["a", "b", "c"] 32 | assert list_input == utils.parse_arg_list(list_input) 33 | 34 | def test_parse_arg_list_small_string_input(): 35 | ''' 36 | Tests that parse_arg_list works for a simple case. 37 | ''' 38 | string_input = "a,b,c" 39 | list_output = ["a", "b", "c"] 40 | assert list_output == utils.parse_arg_list(string_input) 41 | 42 | def test_parse_arg_list_large_string_input(): 43 | ''' 44 | Test parse_arg_list with a more complicated input, including spaces. 45 | ''' 46 | string_input = "this,is a, big , test ," 47 | list_output = ["this", "is a", " big ", " test ", ""] 48 | assert list_output == utils.parse_arg_list(string_input) 49 | 50 | def test_cuda_level_supported(mocker): 51 | ''' 52 | Simple test for cuda_level_supported 53 | ''' 54 | #expected cuda version supported by the system 55 | cuda_version="10.2" 56 | mocker.patch('open_ce.utils.get_driver_cuda_level',return_value="10.2") 57 | assert utils.cuda_level_supported(cuda_version) == True 58 | 59 | #expected cuda version not supported by the system 60 | cuda_version="11.0" 61 | mocker.patch('open_ce.utils.get_driver_cuda_level',return_value="10.2") 62 | assert utils.cuda_level_supported(cuda_version) == False 63 | 64 | def test_get_driver_cuda_level(mocker): 65 | ''' 66 | Simple test for get_driver_cuda_level 67 | ''' 68 | mocker.patch('subprocess.check_output',return_value=bytes("CUDA Version: 10.2","utf-8")) 69 | assert utils.get_driver_cuda_level() == "10.2" 70 | 71 | def test_get_driver_cuda_level_failures(mocker): 72 | ''' 73 | Simple test for get_driver_cuda_level failure scenarios 74 | ''' 75 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.ENOENT,"" )) 76 | with pytest.raises(OpenCEError) as exc: 77 | utils.get_driver_cuda_level() 78 | assert "nvidia-smi command not found" in str(exc.value) 79 | 80 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.EPERM,"" )) 81 | with pytest.raises(OpenCEError) as exc: 82 | utils.get_driver_cuda_level() 83 | assert "nvidia-smi command unexpectedly failed" in str(exc.value) 84 | 85 | def test_get_driver_level(mocker): 86 | ''' 87 | Simple test for get_driver_level 88 | ''' 89 | mocker.patch('subprocess.check_output',return_value=bytes("Driver Version: 440.33.01","utf-8")) 90 | assert utils.get_driver_level() == "440.33.01" 91 | 92 | def test_get_driver_level_failures(mocker): 93 | ''' 94 | Simple test for get_driver_level failure scenarios 95 | ''' 96 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.ENOENT,"" )) 97 | with pytest.raises(OpenCEError) as exc: 98 | utils.get_driver_level() 99 | assert "nvidia-smi command not found" in str(exc.value) 100 | 101 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.EPERM,"" )) 102 | with pytest.raises(OpenCEError) as exc: 103 | utils.get_driver_level() 104 | assert "nvidia-smi command unexpectedly failed" in str(exc.value) 105 | 106 | def test_cuda_driver_installed(mocker): 107 | ''' 108 | Simple test for cuda_driver_installed 109 | ''' 110 | mocker.patch('subprocess.check_output',return_value=bytes("nvidia 123","utf-8")) 111 | assert utils.cuda_driver_installed() == True 112 | 113 | def test_cuda_driver_installed_failures(mocker): 114 | ''' 115 | Simple test for cuda_driver_installed failure scenarios 116 | ''' 117 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.ENOENT,"" )) 118 | with pytest.raises(OpenCEError) as exc: 119 | utils.cuda_driver_installed() 120 | assert "lsmod command not found" in str(exc.value) 121 | 122 | mocker.patch('subprocess.check_output',side_effect=OSError(errno.EPERM,"" )) 123 | with pytest.raises(OpenCEError) as exc: 124 | utils.cuda_driver_installed() 125 | assert "lsmod command unexpectedly failed" in str(exc.value) 126 | 127 | def test_get_branch_of_tag(mocker): 128 | ''' 129 | Simple tests for the get_up_to_date_branch 130 | ''' 131 | sample_output = "main\n remotes/origin/main\n* remotes/origin/r2.4.1 \n" 132 | mocker.patch('open_ce.utils.run_command_capture', side_effect=[(True, sample_output, ""), (False, sample_output, "")]) 133 | 134 | assert utils.get_branch_of_tag("mytag") == "remotes/origin/r2.4.1" 135 | 136 | assert utils.get_branch_of_tag("mytag") == "mytag" 137 | 138 | def test_get_container_tool_ver(mocker): 139 | ''' 140 | Simple test to check get_container_tool_ver 141 | ''' 142 | sample_output = "Client:\n Version: 1.2.3 \n Some Field: x.y.z\n" 143 | mocker.patch('open_ce.utils.get_output', return_value=sample_output ) 144 | 145 | assert utils.get_container_tool_ver("container_tool") == "1.2.3" 146 | 147 | def test_get_open_ce_version(): 148 | ''' 149 | Simple test to read open-ce version from the conda environment file 150 | ''' 151 | 152 | expected_version = "open-ce-v1.0.0" 153 | 154 | test_conda_env_file = os.path.join(test_dir,"test-conda-env2.yaml") 155 | assert utils.get_open_ce_version(test_conda_env_file) == expected_version 156 | 157 | test_conda_env_file = os.path.join(test_dir,"test-conda-env.yaml") 158 | assert utils.get_open_ce_version(test_conda_env_file) == "open-ce" 159 | 160 | -------------------------------------------------------------------------------- /open_ce/env_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | from enum import Enum, unique, auto 21 | 22 | from open_ce import utils 23 | from open_ce.errors import OpenCEError, Error, show_warning 24 | from open_ce import __version__ as open_ce_version 25 | 26 | @unique 27 | class Key(Enum): 28 | '''Enum for Env Config Keys''' 29 | builder_version = auto() 30 | imported_envs = auto() 31 | conda_build_configs = auto() 32 | channels = auto() 33 | packages = auto() 34 | git_tag_for_env = auto() 35 | git_tag = auto() 36 | feedstock = auto() 37 | recipes = auto() 38 | external_dependencies = auto() 39 | patches = auto() 40 | opence_env_file_path = auto() 41 | runtime_package = auto() 42 | recipe_path = auto() 43 | 44 | _PACKAGE_SCHEMA ={ 45 | Key.feedstock.name: utils.make_schema_type(str, True), 46 | Key.git_tag.name: utils.make_schema_type(str), 47 | Key.recipes.name: utils.make_schema_type([str]), 48 | Key.channels.name: utils.make_schema_type([str]), 49 | Key.patches.name: utils.make_schema_type([str]), 50 | Key.runtime_package.name: utils.make_schema_type(bool), 51 | Key.recipe_path.name: utils.make_schema_type(str) 52 | } 53 | 54 | _ENV_CONFIG_SCHEMA = { 55 | Key.builder_version.name: utils.make_schema_type(str), 56 | Key.imported_envs.name: utils.make_schema_type([str]), 57 | Key.channels.name: utils.make_schema_type([str]), 58 | Key.git_tag_for_env.name: utils.make_schema_type(str), 59 | Key.external_dependencies.name: utils.make_schema_type([str]), 60 | Key.conda_build_configs.name: utils.make_schema_type([str]), 61 | Key.packages.name: utils.make_schema_type([_PACKAGE_SCHEMA]) 62 | } 63 | 64 | def _validate_config_file(env_file, variants): 65 | '''Perform some validation on the environment file after loading it.''' 66 | # pylint: disable=import-outside-toplevel 67 | from open_ce import conda_utils 68 | 69 | try: 70 | original_env_file = env_file 71 | if utils.is_url(env_file): 72 | env_file = utils.download_file(env_file) 73 | 74 | # although version_check_obj below is a dict, pylint throws a false positive thinking its a list 75 | # so disabling pylint complaints. 76 | # pylint: disable=no-member, invalid-sequence-index 77 | # First, partially render yaml to validate builder version number. 78 | version_check_obj = conda_utils.render_yaml(env_file, permit_undefined_jinja=True) 79 | if Key.builder_version.name in version_check_obj.keys(): 80 | if not conda_utils.version_matches_spec(version_check_obj.get(Key.builder_version.name)): 81 | raise OpenCEError(Error.SCHEMA_VERSION_MISMATCH, 82 | original_env_file, 83 | version_check_obj.get(Key.builder_version.name), 84 | open_ce_version) 85 | 86 | meta_obj = None 87 | try: 88 | meta_obj = conda_utils.render_yaml(env_file, variants=variants, schema=_ENV_CONFIG_SCHEMA) 89 | if not (Key.packages.name in meta_obj.keys() or Key.imported_envs.name in meta_obj.keys()): 90 | raise OpenCEError(Error.CONFIG_CONTENT) 91 | meta_obj[Key.opence_env_file_path.name] = original_env_file 92 | except OpenCEError as exc: 93 | if Key.builder_version.name not in version_check_obj.keys(): 94 | show_warning(Error.SCHEMA_VERSION_NOT_FOUND, original_env_file, Key.builder_version.name) 95 | raise exc 96 | return meta_obj 97 | # pylint: enable=no-member, invalid-sequence-index 98 | except (Exception, SystemExit) as exc: #pylint: disable=broad-except 99 | raise OpenCEError(Error.ERROR, f"Error in {original_env_file}:\n {str(exc)}") from exc 100 | 101 | def load_env_config_files(config_files, variants, ignore_urls=False): 102 | ''' 103 | Load all of the environment config files, plus any that come from "imported_envs" 104 | within an environment config file. 105 | ''' 106 | if ignore_urls: 107 | env_config_files = [e for e in config_files if not utils.is_url(e)] 108 | env_config_files = [os.path.abspath(e) if not utils.is_url(e) else e for e in config_files] 109 | env_config_data_list = [] 110 | loaded_files = [] 111 | while env_config_files: 112 | for variant in variants: 113 | # Load the environment config files using conda-build's API. This will allow for the 114 | # filtering of text using selectors and jinja2 functions 115 | env = _validate_config_file(env_config_files[0], variant) 116 | 117 | # Examine all of the imported_envs items and determine if they still need to be loaded. 118 | new_config_files = [] 119 | imported_envs = env.get(Key.imported_envs.name, []) 120 | if not imported_envs: 121 | imported_envs = [] 122 | for imported_env in imported_envs: 123 | if ignore_urls and utils.is_url(imported_env): 124 | continue 125 | if not utils.is_url(imported_env): 126 | imported_env = utils.expanded_path(imported_env, relative_to=env_config_files[0]) 127 | if not imported_env in env_config_files and not imported_env in loaded_files: 128 | new_config_files += [imported_env] 129 | 130 | # If there are new files to load, add them to the env_conf_files list. 131 | # Otherwise, remove the current file from the env_conf_files list and 132 | # add its data to the env_config_data_list. 133 | if new_config_files: 134 | env_config_files = new_config_files + env_config_files 135 | else: 136 | env_config_data_list += [env] 137 | loaded_files += [env_config_files.pop(0)] 138 | 139 | return env_config_data_list 140 | -------------------------------------------------------------------------------- /open_ce/build_image.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import os 20 | import shutil 21 | from open_ce import utils, constants 22 | from open_ce import __version__ as open_ce_version 23 | from open_ce.inputs import Argument 24 | from open_ce.errors import OpenCEError, Error, show_warning, log 25 | 26 | COMMAND = 'image' 27 | DESCRIPTION = 'Run Open-CE tools within a container' 28 | ARGUMENTS = [Argument.LOCAL_CONDA_CHANNEL, 29 | Argument.CONDA_ENV_FILE, 30 | Argument.CONTAINER_BUILD_ARGS, 31 | Argument.CONTAINER_TOOL] 32 | 33 | OPEN_CE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".")) 34 | RUNTIME_IMAGE_NAME = "opence-runtime" 35 | RUNTIME_IMAGE_PATH = os.path.join(OPEN_CE_PATH, "images", 36 | RUNTIME_IMAGE_NAME) 37 | REPO_NAME = "open-ce" 38 | TEMP_FILES = "temp_dir" 39 | 40 | OPENCE_USER = "opence" 41 | LOCAL_CONDA_CHANNEL_IN_IMG = "opence-local-conda-channel" 42 | TARGET_DIR = f"/home/{OPENCE_USER}/{LOCAL_CONDA_CHANNEL_IN_IMG}" 43 | 44 | def build_image(local_conda_channel, conda_env_file, container_tool, image_version, container_build_args=""): 45 | """ 46 | Build a container image from the Dockerfile in RUNTIME_IMAGE_PATH. 47 | Returns a result code and the name of the new image. 48 | """ 49 | variant = os.path.splitext(conda_env_file)[0].replace(constants.CONDA_ENV_FILENAME_PREFIX, "", 1) 50 | variant = variant.replace("-runtime", "") 51 | image_name = REPO_NAME + ":" + image_version + "-" + variant 52 | # Docker version on ppc64le rhel7 doesn't allow Dockerfiles to be out of build context. 53 | # Hence, copying it in temp_dir inside the build context. This isn't needed with newer 54 | # docker versions or podman but to be consistent, doing this in all cases. 55 | dockerfile_path = os.path.join(local_conda_channel, TEMP_FILES, "Dockerfile") 56 | runtime_img_file = _get_runtime_image_file(container_tool) 57 | create_copy(runtime_img_file, dockerfile_path) 58 | 59 | build_cmd = container_tool + " build " 60 | build_cmd += "-f " + dockerfile_path + " " 61 | build_cmd += "-t " + image_name + " " 62 | build_cmd += "--build-arg OPENCE_USER=" + OPENCE_USER + " " 63 | build_cmd += "--build-arg LOCAL_CONDA_CHANNEL=" + "./ " 64 | build_cmd += "--build-arg CONDA_ENV_FILE=" + os.path.join(TEMP_FILES, conda_env_file) + " " 65 | build_cmd += "--build-arg TARGET_DIR=" + TARGET_DIR + " " 66 | build_cmd += container_build_args + " " 67 | build_cmd += local_conda_channel 68 | 69 | log.info("Container build command: %s", build_cmd) 70 | if os.system(build_cmd): 71 | raise OpenCEError(Error.BUILD_IMAGE, image_name) 72 | 73 | return image_name 74 | 75 | def _get_runtime_image_file(container_tool): 76 | tool_ver = utils.get_container_tool_ver(container_tool) 77 | image_file = os.path.join(RUNTIME_IMAGE_PATH, "docker/Dockerfile") 78 | if not tool_ver: 79 | show_warning(Error.CONTAINER_VERSION, container_tool) 80 | # If we couldn't get the version of the tool, should use the older docker 81 | # supported Dockerfile 82 | return image_file 83 | 84 | tool_ver = tool_ver.replace(".", "") 85 | tool_ver = tool_ver.replace("-dev", "") 86 | if (container_tool == "docker" and int(tool_ver) >= 1709) or \ 87 | (container_tool == "podman" and int(tool_ver) >= 200): 88 | # Use the newer docker/podman supported Dockerfile 89 | image_file = os.path.join(RUNTIME_IMAGE_PATH, "podman/Dockerfile") 90 | 91 | log.info("Dockerfile being used: %s", image_file) 92 | return image_file 93 | 94 | def build_runtime_container_image(args): 95 | """ 96 | Create a runtime image which will have a conda environment created 97 | using locally built conda packages and environment file. 98 | """ 99 | 100 | if not args.container_tool: 101 | raise OpenCEError(Error.NO_CONTAINER_TOOL_FOUND) 102 | 103 | local_conda_channel = os.path.abspath(args.local_conda_channel) 104 | if not os.path.exists(local_conda_channel): 105 | raise OpenCEError(Error.INCORRECT_INPUT_PATHS) 106 | 107 | os.makedirs(os.path.join(local_conda_channel, TEMP_FILES), exist_ok=True) 108 | 109 | for conda_env_file in utils.parse_arg_list(args.conda_env_files): 110 | conda_env_file = os.path.abspath(conda_env_file) 111 | if not os.path.exists(conda_env_file): 112 | raise OpenCEError(Error.INCORRECT_INPUT_PATHS) 113 | 114 | # Copy the conda environment file into the TEMP_FILES dir inside local 115 | # conda channel with a new name and modify it 116 | conda_env_runtime_filename = os.path.splitext(os.path.basename(conda_env_file))[0]+'-runtime.yaml' 117 | conda_env_runtime_file = os.path.join(local_conda_channel, TEMP_FILES, conda_env_runtime_filename) 118 | create_copy(conda_env_file, conda_env_runtime_file) 119 | utils.replace_conda_env_channels(conda_env_runtime_file, r'file:.*', f"file:/{TARGET_DIR}") 120 | 121 | image_version = utils.get_open_ce_version(conda_env_file) 122 | image_name = build_image(args.local_conda_channel, os.path.basename(conda_env_runtime_file), 123 | args.container_tool, image_version, args.container_build_args) 124 | log.info("Docker image with name %s is built successfully.", image_name) 125 | 126 | cleanup(local_conda_channel) 127 | 128 | def create_copy(src_file, dest_file): 129 | ''' 130 | Creates a copy of src_file to dest_file 131 | ''' 132 | try: 133 | shutil.copy(src_file, dest_file) 134 | except shutil.SameFileError: 135 | log.info("File '%s' already in local conda channel.", src_file) 136 | 137 | def cleanup(local_conda_channel): 138 | ''' 139 | Removes TEMP_FILES directory from the specified directory 140 | ''' 141 | try: 142 | shutil.rmtree(os.path.join(local_conda_channel, TEMP_FILES)) 143 | except OSError: 144 | show_warning(Error.TEMP_BUILD_IMAGE_FILES) 145 | 146 | 147 | ENTRY_FUNCTION = build_runtime_container_image 148 | -------------------------------------------------------------------------------- /open_ce/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | """ 18 | 19 | import sys 20 | import logging 21 | from enum import Enum, unique 22 | 23 | log = logging.getLogger("OPEN-CE") 24 | log.setLevel(logging.DEBUG) 25 | formatter = logging.Formatter('[%(name)s-%(levelname)s]%(message)s') 26 | log_out_handler = logging.StreamHandler(sys.stdout) 27 | log_out_handler.setLevel(logging.DEBUG) 28 | log_out_handler.addFilter(lambda record: record.levelno < logging.WARNING) 29 | log_out_handler.setFormatter(formatter) 30 | log_warn_handler = logging.StreamHandler(sys.stderr) 31 | log_warn_handler.setLevel(logging.WARNING) 32 | log_warn_handler.addFilter(lambda record: record.levelno < logging.ERROR) 33 | log_warn_handler.setFormatter(formatter) 34 | log_err_handler = logging.StreamHandler(sys.stderr) 35 | log_err_handler.setLevel(logging.ERROR) 36 | log.addHandler(log_out_handler) 37 | log.addHandler(log_warn_handler) 38 | log.addHandler(log_err_handler) 39 | 40 | 41 | @unique 42 | class Error(Enum): 43 | '''Enum for Error Messages''' 44 | ERROR = (0, "Unexpected Error: {}") 45 | CREATE_CONTAINER = (1, "Error creating container: \"{}\"") 46 | COPY_DIR_TO_CONTAINER = (2, "Error copying \"{}\" directory into container: \"{}\"") 47 | START_CONTAINER = (3, "Error starting container: \"{}\"") 48 | BUILD_IN_CONTAINER = (4, "Error executing build in container: \"{}\"") 49 | BUILD_IMAGE = (5, "Failure building image: \"{}\"") 50 | VALIDATE_ENV = (6, "Error validating \"{}\" for variant {}\n{}") 51 | VALIDATE_CONFIG = (7, "Error validating \"{}\" for \"{}\":\n{}") 52 | CONFIG_CONTENT = (8, "Content Error!:\n" 53 | "An environment file needs to specify packages or " 54 | "import another environment file.") 55 | CLONE_REPO = (9, "Unable to clone repository: {}") 56 | CREATE_BUILD_TREE = (10, "Error creating Build Tree\n{}") 57 | BUILD_RECIPE = (11, "Unable to build recipe: {}\n{}") 58 | CONFIG_FILE = (12, "Unable to open provided config file: {}") 59 | LOCAL_SRC_DIR = (13, "local_src_dir path \"{}\" specified doesn't exist") 60 | BUILD_TREE_CYCLE = (14, "Build dependencies should form a Directed Acyclic Graph.\n" 61 | "The following dependency cycles were detected in the build tree:\n{}") 62 | INCORRECT_INPUT_PATHS = (15, "Input paths specified don't exist") 63 | LOCAL_CHANNEL_NOT_IN_CONTEXT = (16, "Specified local conda channel directory is not" + 64 | " in the current build context. \n Either move the local conda channel" + 65 | " directory in the current directory or run the script from the path" + 66 | " which contains local conda channel directory.") 67 | VALIDATE_BUILD_TREE = (17, "Dependencies are not compatible.\nCommand:\n{}\nOutput:\n{}\nError:\n{}") 68 | INCOMPAT_CUDA = (18, "Driver level \"{}\" is not new enough to support cuda \"{}\"") 69 | UNSUPPORTED_CUDA = (19, "Cannot build using container image for cuda \"{}\" no Dockerfile currently exists") 70 | TOO_MANY_CUDA = (20, "Only one cuda version allowed to be built with container build at a time") 71 | FAILED_TESTS = (21, "There were {} test failures. The following tests failed: {}") 72 | CONDA_ENV_FILE_REQUIRED = (22, "The '--conda_env_file' argument is required.") 73 | PATCH_APPLICATION = (23, "Failed to apply patch {} on feedstock {}") 74 | GET_LICENSES = (24, "Error generating licenses file.\nCommand:\n{}\nOUTPUT:\n{}Error:\n{}") 75 | FILE_DOWNLOAD = (25, "Failed to download {} with error:\n{}") 76 | CONDA_BUILD_CONFIG_FILE_NOT_FOUND = (26, "Failed to locate conda_build_config file: {}.") 77 | NO_CONTAINER_TOOL_FOUND = (27, "No container tool found on the system.") 78 | CONDA_PACKAGE_INFO = (28, "Conda Package Info Failed.\nCommand:\n{}\nOutput:\n{}") 79 | REMOTE_PACKAGE_DEPENDENCIES = (29, "Failure getting remote dependencies for the following packages:\n{}\nError:\n{}") 80 | WARNING = (30, "Unexpected Warning: {}") 81 | SCHEMA_VERSION_NOT_FOUND = (31, "'{}' does not provide '{}'. Possible schema mismatch.") 82 | CONTAINER_VERSION = (32, "Could not retrieve version of {} container tool.") 83 | CONDA_BUILD_CONFIG_NOT_FOUND = (33, "No valid '{}' file was found. Some recipes may fail to build.") 84 | CONDA_IO_ERROR = (34, "IO error occurred while reading version information from conda environment file '{}'.") 85 | SCHEMA_VERSION_MISMATCH = (35, "Open-CE Env file '{}' expects to be built with Open-CE Builder [{}]. " + 86 | "But this version is '{}'.") 87 | PACKAGE_NOT_FOUND = (36, "Cannot find `{}`, please see https://github.com/open-ce/open-ce-builder#requirements" + 88 | " for a list of requirements.") 89 | TEMP_BUILD_IMAGE_FILES = (37, "Error removing temporary files created during build image.") 90 | UNABLE_DOWNLOAD_SOURCE = (38, "Unable to download source for '{}'.") 91 | UNABLE_CLONE_SOURCE = (39, "Unable to clone source for '{}'.") 92 | GCC12_13_COMPILER_NOT_FOUND = (40, "Please check if GCC12 and GCC13 are installed. If not, install" + 93 | " gcc-toolset-12 and gcc-toolset-13. Also, set environment variables" + 94 | " GCC_HOME and GCC_13_HOME to point to the installed location." + 95 | " For e.g. GCC_HOME=\"/opt/rh/gcc-toolset-11/root/usr\"") 96 | 97 | GIT_TAG_MISSING = (41, "git_tag attribute is missing for '{}'") 98 | FIPS_PACKAGES_NOT_BUILT = (42, "FIPS Compliant OpenSSL env failed to build") 99 | FFMPEG_PACKAGE_NOT_BUILT = (43, "ffmpeg env failed to build") 100 | 101 | class OpenCEError(Exception): 102 | """ 103 | Exception class for errors that occur in an Open-CE tool. 104 | """ 105 | def __init__(self, error, *additional_args, **kwargs): 106 | # When pickling, error can already be a string. 107 | if isinstance(error, str): 108 | msg = error 109 | else: 110 | msg = f"[OPEN-CE-ERROR]-{error.value[0]} {error.value[1].format(*additional_args)}" 111 | super().__init__(msg, **kwargs) 112 | self.msg = msg 113 | 114 | def show_warning(warning, *additional_args, **kwargs): 115 | """ 116 | Prints an Open-CE Warning. 117 | """ 118 | msg = f"-{warning.value[0]} {warning.value[1].format(*additional_args)}" 119 | log.warning(msg, **kwargs) 120 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | 19 | class Namespace: 20 | def __init__(self, **kwargs): 21 | self.__dict__.update(kwargs) 22 | 23 | def validate_cli(cli_string, expect=None, reject=None, ignore=None, possible_expect=None, retval=0): 24 | """ 25 | Used to mock os.system with the assumption that it is making a call to 'conda-build'. 26 | 27 | Args: 28 | cli_string: The placeholder argument for the system command. 29 | expect: A list of strings that must occur in the 'cli_string' arg. 30 | reject: A list of strings that cannot occur in the 'cli_string' arg. 31 | ignore: Don't validate the CLI_STRING if one of these strings is contained in it. 32 | possible_expect: A list of possible strings that the CLI may contain. CLI should contain 33 | at least one of these. 34 | retval: The mocked value to return from 'os.system'. 35 | Returns: 36 | retval 37 | """ 38 | if not expect: expect = [] 39 | if not reject: reject = [] 40 | if not ignore: ignore = [] 41 | if not possible_expect: possible_expect = [] 42 | 43 | if not any ({term in cli_string for term in ignore}): 44 | for term in expect: 45 | assert term in cli_string 46 | for term in reject: 47 | assert term not in cli_string 48 | if len(expect) == 0: 49 | assert any ({term in cli_string for term in possible_expect}) 50 | return retval 51 | return 0 52 | 53 | def validate_conda_build_args(recipe, expect_recipe=None, expect_config=None, expect_variants=None, reject_recipe=None, **kwargs): 54 | """ 55 | Used to mock `conda_build.api.build` 56 | 57 | Args: 58 | recipe: The placeholder argument for the conda_build.api.build 'recipe' arg. 59 | expect_recipe: A string that must occur in the 'recipe' arg. 60 | expect_config: A dict for the keys and values that must occur in the 'config' arg. 61 | expect_variants: A dict for the keys and values that must occur in the 'variants' arg. 62 | reject_recipe: A string that cannot occur in the 'recipe` arg. 63 | """ 64 | if not expect_recipe: expect_recipe = [] 65 | if not expect_config: expect_config = [] 66 | if not expect_variants: expect_variants = [] 67 | if not reject_recipe: reject_recipe = [] 68 | 69 | if expect_recipe: 70 | assert recipe in expect_recipe 71 | if reject_recipe: 72 | assert recipe not in reject_recipe 73 | 74 | if expect_config: 75 | config = kwargs['config'] 76 | for term, value in expect_config.items(): 77 | assert hasattr(config, term) 78 | assert getattr(config, term) == value 79 | if expect_variants: 80 | variants = kwargs['config'].variant 81 | for term, value in expect_variants.items(): 82 | assert term in variants 83 | assert variants.get(term) == value 84 | 85 | def validate_conda_debug_args(recipe, expect_activation_string, debug_output_id=None): 86 | 87 | """ 88 | Used to mock `conda_build.api.debug` 89 | Args: 90 | recipe: The placeholder argument for the conda_build.api.debug 'recipe' arg. 91 | debug_output_id: Output ID for which conda debug is invoked in case of multiple output recipe. 92 | """ 93 | # pylint: disable=W0613 94 | if debug_output_id: 95 | assert debug_output_id in expect_activation_string 96 | 97 | class DirTracker(object): 98 | def __init__(self, starting_dir=os.getcwd()): 99 | self.current_dir = starting_dir 100 | self.chdir_count = 0 101 | 102 | def validate_chdir(self, arg1, expected_dirs=None): 103 | """ 104 | Used to mock os.chdir. Each time a directory is changed, the global counter ch_dir is incremented, 105 | and each change is validated against the expected_dirs list. 106 | The current directory is tracked in `current_dir` and used by `mocked_getcwd`. 107 | 108 | Args: 109 | arg1: The placeholder argumentfor the chdir command. 110 | expected_dirs: The list of directories that are expected to be chdired during execution.. 111 | Returns: 112 | 0 113 | """ 114 | if expected_dirs and self.chdir_count < len(expected_dirs): 115 | assert arg1 == expected_dirs[self.chdir_count] 116 | self.current_dir = arg1 117 | self.chdir_count+=1 118 | return 0 119 | 120 | def mocked_getcwd(self): 121 | return self.current_dir 122 | 123 | def make_render_result(package_name, build_reqs=None, run_reqs=None, host_reqs=None, test_reqs=None, 124 | string=None): 125 | ''' 126 | Creates a YAML string that is a mocked result of `conda_build.api.render`. 127 | ''' 128 | if not build_reqs: build_reqs = [] 129 | if not run_reqs: run_reqs = [] 130 | if not host_reqs: host_reqs = [] 131 | if not test_reqs: test_reqs = [] 132 | if not string: string = '' 133 | 134 | retval = [(Namespace(meta={ 135 | 'package': {'name': package_name, 'version': '1.2.3'}, 136 | 'source': {'git_url': 'https://github.com/'+package_name+'.git', 'git_rev': 'v0.19.5', 'patches': []}, 137 | 'build': {'number': '1', 'string': 'py37_1'}, 138 | 'requirements': {'build': build_reqs, 'host': host_reqs, 'run': run_reqs + ["upstreamdep1 2.3","upstreamdep2 2"], 'run_constrained': []}, 139 | 'test': {'requires': test_reqs}, 140 | 'about': {'home': 'https://github.com/'+package_name+'.git', 'license_file': 'LICENSE', 'summary': package_name}, 141 | 'extra': {'final': True}}), 142 | True, 143 | None)] 144 | return retval 145 | 146 | def mock_renderer(path, package_deps): 147 | ''' 148 | Used to mock the `conda_build.api.render` function by extracting the package name from `path` 149 | and using that to get the dependencies from `package_deps`. 150 | ''' 151 | package = os.path.basename(path)[:-10] 152 | return make_render_result(package, package_deps[package]) 153 | 154 | def mock_get_output_file_paths(meta): 155 | return ["noarch/" + meta.meta['package']['name'] + "-" + meta.meta['package']['version'] + "-" + meta.meta['build']['string'] + ".tar.bz2"] 156 | -------------------------------------------------------------------------------- /open_ce/open-ce: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | # ***************************************************************** 4 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ***************************************************************** 18 | """ 19 | 20 | import sys 21 | from open_ce import inputs 22 | from open_ce.errors import OpenCEError, log 23 | from open_ce import build_env 24 | from open_ce import build_feedstock 25 | from open_ce import test_feedstock 26 | from open_ce import test_env 27 | from open_ce import validate_config 28 | from open_ce import validate_env 29 | from open_ce import build_image 30 | from open_ce import get_licenses 31 | from open_ce import get_graph 32 | from open_ce import __version__ as open_ce_version 33 | 34 | #pylint: disable=too-many-locals 35 | def make_parser(): 36 | ''' Parser for input arguments ''' 37 | parser = inputs.make_parser([inputs.Argument.VERSION], description = f'Open-CE Builder {open_ce_version}') 38 | 39 | subparsers = parser.add_subparsers(title='Open-CE Commands', 40 | description='Choose from the following commands to perform.') 41 | subparsers.required = True 42 | subparsers.dest = 'command' 43 | 44 | build_parser = inputs.add_subparser(subparsers, 'build', [], help="Build a feedstock, an image, or an environment.") 45 | build_sub_parsers = build_parser.add_subparsers(title='Build Commands', 46 | description='Valid Commands') 47 | build_sub_parsers.required = True 48 | build_sub_parsers.dest = 'sub_command' 49 | 50 | build_env_parser = inputs.add_subparser(build_sub_parsers, 51 | build_env.COMMAND, 52 | build_env.ARGUMENTS, 53 | help=build_env.DESCRIPTION) 54 | build_env_parser.set_defaults(func=build_env.ENTRY_FUNCTION) 55 | 56 | build_feedstock_parser = inputs.add_subparser(build_sub_parsers, 57 | build_feedstock.COMMAND, 58 | build_feedstock.ARGUMENTS, 59 | help=build_feedstock.DESCRIPTION) 60 | build_feedstock_parser.set_defaults(func=build_feedstock.ENTRY_FUNCTION) 61 | 62 | build_image_parser = inputs.add_subparser(build_sub_parsers, 63 | build_image.COMMAND, 64 | build_image.ARGUMENTS, 65 | help=build_image.DESCRIPTION) 66 | build_image_parser.set_defaults(func=build_image.ENTRY_FUNCTION) 67 | 68 | test_parser = inputs.add_subparser(subparsers, 'test', [], help="Run tests within a feedstock.") 69 | test_sub_parsers = test_parser.add_subparsers(title='Test Commands', 70 | description='Valid Commands') 71 | test_sub_parsers.required = True 72 | test_sub_parsers.dest = 'sub_command' 73 | 74 | 75 | test_feedstock_parser = inputs.add_subparser(test_sub_parsers, 76 | test_feedstock.COMMAND, 77 | test_feedstock.ARGUMENTS, 78 | help=test_feedstock.DESCRIPTION) 79 | test_feedstock_parser.set_defaults(func=test_feedstock.ENTRY_FUNCTION) 80 | 81 | test_env_parser = inputs.add_subparser(test_sub_parsers, 82 | test_env.COMMAND, 83 | test_env.ARGUMENTS, 84 | help=test_env.DESCRIPTION) 85 | test_env_parser.set_defaults(func=test_env.ENTRY_FUNCTION) 86 | 87 | validate_parser = inputs.add_subparser(subparsers, 'validate', [], 88 | help="Validate an env file or a conda_build_config file.") 89 | validate_sub_parsers = validate_parser.add_subparsers(title='Validate Commands', 90 | description='Valid Commands') 91 | validate_sub_parsers.required = True 92 | validate_sub_parsers.dest = 'sub_command' 93 | 94 | validate_config_parser = inputs.add_subparser(validate_sub_parsers, 95 | validate_config.COMMAND, 96 | validate_config.ARGUMENTS, 97 | help=validate_config.DESCRIPTION) 98 | validate_config_parser.set_defaults(func=validate_config.ENTRY_FUNCTION) 99 | 100 | validate_env_parser = inputs.add_subparser(validate_sub_parsers, 101 | validate_env.COMMAND, 102 | validate_env.ARGUMENTS, 103 | help=validate_env.DESCRIPTION) 104 | validate_env_parser.set_defaults(func=validate_env.ENTRY_FUNCTION) 105 | 106 | get_parser = inputs.add_subparser(subparsers, 'get', [], 107 | help="Get information about Open-CE builds.") 108 | get_sub_parsers = get_parser.add_subparsers(title='Get Commands', 109 | description='Valid Commands') 110 | get_sub_parsers.required = True 111 | get_sub_parsers.dest = 'sub_command' 112 | 113 | get_licenses_parser = inputs.add_subparser(get_sub_parsers, 114 | get_licenses.COMMAND, 115 | get_licenses.ARGUMENTS, 116 | help=get_licenses.DESCRIPTION) 117 | get_licenses_parser.set_defaults(func=get_licenses.ENTRY_FUNCTION) 118 | 119 | get_graph_parser = inputs.add_subparser(get_sub_parsers, 120 | get_graph.COMMAND, 121 | get_graph.ARGUMENTS, 122 | help=get_graph.DESCRIPTION) 123 | get_graph_parser.set_defaults(func=get_graph.ENTRY_FUNCTION) 124 | 125 | return parser 126 | 127 | def _main(arg_strings=None): 128 | parser = make_parser() 129 | if len(sys.argv) == 1: 130 | parser.print_help() 131 | return 132 | args = inputs.parse_args(parser, arg_strings) 133 | args.func(args) 134 | 135 | if __name__ == '__main__': 136 | try: 137 | _main() 138 | except OpenCEError as err: 139 | log.error(err.msg) 140 | sys.exit(1) 141 | sys.exit(0) 142 | -------------------------------------------------------------------------------- /git_tools/tag_all_repos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | 18 | """ 19 | ******************************************************************************* 20 | Script: tag_all_repos.py 21 | A script that can be used to create the same annotated git tag in all 22 | repos within an organization. 23 | 24 | To tag all of the feedstocks in the open-ce org with the tag `open-ce-v1.0.0`, 25 | the following command can be used: 26 | ./git_tools/tag_all_repos.py open-ce \\ 27 | --tag open-ce-v1.0.0 \\ 28 | --tag-msg "Open-CE Release Version 1.0.0" \\ 29 | --pat ${YOUR_PUBLIC_ACCESS_TOKEN} \\ 30 | --repo-dir ./repos \\ 31 | --branch main \\ 32 | --skipped_repos open-ce 33 | ******************************************************************************* 34 | """ 35 | 36 | import os 37 | import sys 38 | import pathlib 39 | import git_utils 40 | 41 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 42 | from open_ce.inputs import make_parser # pylint: disable=wrong-import-position 43 | from open_ce.utils import parse_arg_list # pylint: disable=wrong-import-position 44 | 45 | def _make_parser(): 46 | ''' Parser input arguments ''' 47 | parser = make_parser([git_utils.Argument.PUBLIC_ACCESS_TOKEN, git_utils.Argument.REPO_DIR, 48 | git_utils.Argument.BRANCH, git_utils.Argument.ORG, git_utils.Argument.SKIPPED_REPOS], 49 | description = """ 50 | Tag all repos in an organization using the following logic: 51 | 1- If the branch argument is passed in and that branch exists in the repo, the tag will be made at the tip of that branch. 52 | 2- Otherwise, if a previous-tag is passed, the tag will be made at the tip of the latest branch which contains that tag. 53 | 3- Finally, if none of these cases hold, the tag is made at the tip of the default branch. 54 | """) 55 | 56 | parser.add_argument( 57 | '--tag', 58 | type=str, 59 | required=True, 60 | help="""Tag to create.""") 61 | 62 | parser.add_argument( 63 | '--tag-msg', 64 | type=str, 65 | required=True, 66 | help="""Tag message to use.""") 67 | 68 | parser.add_argument( 69 | '--previous-tag', 70 | '--prev-tag', 71 | type=str, 72 | default=None, 73 | help="""Previous tag to find the branch to tag.""") 74 | 75 | return parser 76 | 77 | def tag_all_repos(github_org, tag, tag_msg, branch, repo_dir, pat, skipped_repos, prev_tag): # pylint: disable=too-many-arguments 78 | ''' 79 | Clones, then tags all repos with a given tag, and pushes back to remote. 80 | These steps are performed in separate loops to make debugging easier. 81 | ''' 82 | skipped_repos = parse_arg_list(skipped_repos) 83 | repos = git_utils.get_all_repos(github_org, pat) 84 | repos = [repo for repo in repos if repo["name"] in skipped_repos ] 85 | 86 | clone_repos(repos, branch, repo_dir, prev_tag) 87 | 88 | tag_repos(repos, tag, tag_msg, repo_dir) 89 | 90 | push = git_utils.ask_for_input("Would you like to push all tags to remote?") 91 | if not push.startswith("y"): 92 | return 93 | 94 | push_repos(repos, tag, repo_dir) 95 | 96 | def clone_repos(repos, branch, repo_dir, prev_tag): 97 | ''' 98 | Clones a list of repos. 99 | ''' 100 | print("---------------------------Cloning all Repos") 101 | for repo in repos: 102 | repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) 103 | print("--->Making clone location: " + repo_path) 104 | os.makedirs(repo_path, exist_ok=True) 105 | print(f"--->Cloning {repo['name']}") 106 | git_utils.clone_repo(repo["ssh_url"], repo_path) 107 | if branch and git_utils.branch_exists(repo_path, branch): 108 | print(f"--->Branch '{branch}' exists, checking it out.") 109 | git_utils.checkout(repo_path, branch) 110 | elif prev_tag: 111 | repo_branch = git_utils.get_tag_branch(repo_path, prev_tag) 112 | print(f"-->Repo branch is: {repo_branch}") 113 | if git_utils.branch_exists(repo_path, os.path.basename(repo_branch)): 114 | print(f"--->Checking out branch '{repo_branch}' which contains tag '{prev_tag}'.") 115 | git_utils.checkout(repo_path, repo_branch) 116 | 117 | def tag_repos(repos, tag, tag_msg, repo_dir): 118 | ''' 119 | Tags a list of repos. 120 | ''' 121 | print("---------------------------Tagging all Repos") 122 | for repo in repos: 123 | repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) 124 | print(f"--->Tagging {repo['name']}") 125 | git_utils.create_tag(repo_path, tag, tag_msg) 126 | 127 | def push_repos(repos, tag, repo_dir, continue_query=True): 128 | ''' 129 | Pushes a list of repos. 130 | ''' 131 | print("---------------------------Pushing all Repos") 132 | for repo in repos: 133 | try: 134 | repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) 135 | print(f"--->Pushing {repo['name']}") 136 | git_utils.push_branch(repo_path, tag) 137 | except Exception as ex:# pylint: disable=broad-except 138 | print(f"Error encountered when trying to push {repo['name']}") 139 | print(ex) 140 | if not continue_query: 141 | continue 142 | cont_tag = git_utils.ask_for_input("Would you like to continue tagging other repos?") 143 | if cont_tag.startswith("y"): 144 | continue 145 | raise 146 | 147 | def _main(arg_strings=None): 148 | parser = _make_parser() 149 | args = parser.parse_args(arg_strings) 150 | tag_all_repos(args.github_org, 151 | args.tag, 152 | args.tag_msg, 153 | args.branch, 154 | args.repo_dir, 155 | args.pat, 156 | args.skipped_repos, 157 | args.previous_tag) 158 | 159 | if __name__ == '__main__': 160 | try: 161 | _main() 162 | except Exception as exc:# pylint: disable=broad-except 163 | print("Error: ", exc) 164 | sys.exit(1) 165 | -------------------------------------------------------------------------------- /doc/README.cuda_support.md: -------------------------------------------------------------------------------- 1 | # Open-CE CUDA Support 2 | 3 | The Open-CE build scripts and feedstocks support CUDA from NVIDIA. In order 4 | for the build scripts to work, CUDA must be installed on the system on which 5 | the builds are taking place. This can be accomplished in two ways (see below). 6 | 7 | **In all cases, the `CUDA_HOME` environment variable must be set to the base 8 | directory where CUDA has been installed.** 9 | 10 | The standard CUDA installation location is in `/usr/local/cuda` which is typically 11 | a symbolic link to either `/usr/local/cuda-11.8` or `/usr/local/cuda-12.2`. Most of 12 | the code bases that are built in Open-CE are flexible enough to tolerate CUDA 13 | installations in non-standard locations, but not all of them. The reference table below 14 | includes the recipes that will not work when CUDA is installed elsewhere. 15 | 16 | | CUDA_HOME restrictions | 17 | |-----------------| 18 | | tensorflow-serving | 19 | 20 | ## Bare metal builds 21 | 22 | If you are using Open-CE on a bare metal system. Install the appropriate version 23 | of CUDA on the server that matches the version of Open-CE you are using. There 24 | is a reference table below. 25 | 26 | | Open-CE version | CUDA version | 27 | |-----------------|--------------| 28 | | 1.7 | 11.2 \| 11.4 | 29 | | 1.8 | 11.2 \| 11.4 | 30 | | 1.9 | 11.2 \| 11.8 | 31 | | 1.10 | 11.8 \| 12.2 | 32 | | 1.11 | 11.8 \| 12.2 | 33 | 34 | It's important to install CUDA correctly. This procedure is dependent on the system 35 | type and CUDA version. It's advised to follow NVIDIA's [official installation documentation](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) 36 | Note that NVLINK2-enabled POWER9 systems require a few [extra steps](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#power9-setup) 37 | 38 | ## Container builds 39 | 40 | It is recommended to run the Open-CE build scripts within a container. NVIDIA provides 41 | official CUDA base containers [on](https://hub.docker.com/r/nvidia/cuda) [dockerhub](https://hub.docker.com/r/nvidia/cuda-ppc64le), 42 | which are ideal for ensuring a properly installed CUDA environment. 43 | 44 | Open-CE provides a working set of images that will work for running CUDA enabled builds. 45 | These images use the official CUDA images referenced above as a base image. Dockerfiles for 46 | these images are located in the [images directory](../open_ce/images) 47 | To build these images manually, use the command below. 48 | 49 | ```shell 50 | docker build open-ce-builder/open_ce/images/build-cuda-x86_64 51 | ``` 52 | 53 | The Open-CE build image presets the `CUDA_HOME` environment variable to the appropriate location. 54 | 55 | ### Automatic container builds 56 | 57 | The [`open-ce build env`](README.open_ce_build.md#open-ce-build-env-sub-command) command supports the `--container_build` command line argument. 58 | This argument will automatically build the Open-CE CUDA-enabled build image and when combined 59 | with the `--build_types=cuda` command line argument, it will build CUDA support into all of the 60 | recipes included in the Open-CE environment file that are enabled to do so. 61 | 62 | --- 63 | 64 | ## Building with CUDA 65 | 66 | ### CUDA Build Type 67 | 68 | Both the [`open-ce build env`](README.open_ce_build.md#open-ce-build-env.md) and [`open-ce build feedstock`](README.build_feedstock-sub-command) scripts 69 | support the `--build_types=cuda` command line argument. This is required when CUDA support is desired in the build. 70 | This argument sets a parameter in the build environment which can then be referenced in the 71 | collected feedstocks and Open-CE environment files using a tag. For example, 72 | a PyTorch Open-CE envionment file could use the tag as: 73 | 74 | ```shell 75 | {% if build_type == 'cuda' %} 76 | imported_envs: 77 | - tensorrt-env.yaml 78 | {% endif %} 79 | 80 | packages: 81 | - feedstock : pytorch 82 | - feedstock : numactl 83 | {% if build_type == 'cuda' %} 84 | - feedstock : nccl 85 | - feedstock : magma 86 | - feedstock : cudnn 87 | {% endif %} 88 | - feedstock : torchtext 89 | - feedstock : onnx 90 | - feedstock : av 91 | - feedstock : torchvision 92 | ``` 93 | 94 | The tag can also be used per line shown in this example for the xgboost [meta.yaml](https://github.com/open-ce/xgboost-feedstock/blob/main/recipe/meta.yaml) 95 | 96 | ```shell 97 | - cudatoolkit {{ cudatoolkit }} #[build_type == 'cuda'] 98 | - nccl {{ nccl }} #[build_type == 'cuda'] 99 | ``` 100 | 101 | ### Specifying CUDA Version 102 | 103 | The `--cuda_versions` flag can be passed to `open-ce` to specify which version of CUDA to build conda packages for. 104 | 105 | ```shell 106 | open-ce build env --build_types cuda --cuda_versions 12.2 envs/opence-env.yaml 107 | ``` 108 | 109 | Only one CUDA version can be specified at a time. To build for both on the same system, the build must be run twice in one of the following ways: 110 | 111 | * Use the `container_build` option, which automatically handles the cuda version. 112 | * For bare metal builds: 113 | * install both CUDA versions locally 114 | * ensure the CUDA 11 driver is used 115 | * adjust CUDA_HOME appropriately for each build 116 | 117 | --- 118 | 119 | ## CUDA Runtime support 120 | 121 | A system-level CUDA installation is not required at runtime. Once packages are built, CUDA is pulled 122 | at the appropriate level from Anaconda's [Defaults channel](https://repo.anaconda.com/pkgs/). Systems 123 | with NVIDIA GPUs installed do still need the NVIDIA device driver installed. Drivers can be found on 124 | NVIDIA's [Driver Downloads](https://www.nvidia.com/Download/index.aspx) website. 125 | 126 | ## NVIDIA cuDNN 127 | 128 | The NVIDIA CUDA Deep Neural Network library (cuDNN) is a GPU-accelerated library of primitives for deep neural 129 | networks. Both PyTorch and TensorFlow make use of cuDNN and it is require for those frameworks CUDA support. 130 | Open-CE includes a build recipe for a cuDNN package and will automatically fetch the appropriate version 131 | for which the package being built. 132 | 133 | ## NVIDIA TensorRT 134 | 135 | NOTE: TensorRT has been disabled within Open-CE 1.3, so it is no longer necessary to install it when building 136 | Open-CE 1.3. 137 | 138 | NVIDIA TensorRT is an SDK for high-performance deep learning inference. It includes a deep learning inference 139 | optimizer and runtime that delivers low latency and high-throughput for deep learning inference applications. 140 | PyTorch, TensorFlow, and TensorFlow Serving include optional TensorRT support. Open-CE is unable to automatically 141 | fetch TensorRT at build time. The appropriate version of TensorRT must be downloaded from NVIDIA's [TensorRT website](https://developer.nvidia.com/nvidia-tensorrt-download) 142 | and saved in a directory called `local_files` adjacent to the `open-ce` repository. The version of TensorRT 143 | must match the version of CUDA. 144 | 145 | | CUDA version | TensorRT version | file type | 146 | |--------------|------------------|-----------| 147 | | 10.2 | 7.0.0.11 | tar.gz | 148 | | 11.0 | 7.2.* | tar.gz | 149 | 150 | Note that the Open-CE recipe for TensorRT also includes some [open source samples and parsers](https://github.com/nvidia/tensorrt). 151 | These are fetched automatically to match the version of TensorRT included in the version of Open-CE being used. 152 | 153 | ## NCCL 154 | 155 | NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for NVIDIA GPUs, 156 | implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. Pytorch and TensorFlow 157 | utilize NCCL when CUDA support is enabled. NCCL is now open source and fetched for build automatically. 158 | -------------------------------------------------------------------------------- /git_tools/create_version_branch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ***************************************************************** 3 | # (C) Copyright IBM Corp. 2020, 2021. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ***************************************************************** 17 | 18 | """ 19 | ******************************************************************************* 20 | Script: create_version_branch.py 21 | A script that can be used to create a version branch for an Open-CE feedstock. 22 | 23 | The user will pass in a link to a repository, plus an optional commit. After 24 | running the script, a new remote branch will be added to the repository with 25 | a name based on the version number of the packages within the feedstock. 26 | ******************************************************************************* 27 | """ 28 | 29 | import os 30 | import sys 31 | import shutil 32 | import pathlib 33 | import re 34 | import tempfile 35 | import git_utils 36 | 37 | sys.path.append(os.path.join(pathlib.Path(__file__).parent.absolute(), '..')) 38 | # pylint: disable=wrong-import-position 39 | from open_ce import inputs 40 | from open_ce import conda_utils 41 | from open_ce import build_feedstock 42 | from open_ce import utils 43 | 44 | def _make_parser(): 45 | ''' Parser input arguments ''' 46 | parser = inputs.make_parser([git_utils.Argument.REPO_DIR, inputs.Argument.CONDA_BUILD_CONFIG], 47 | description = 'Create a version branch for a feedstock.') 48 | 49 | parser.add_argument( 50 | '--repository', 51 | type=str, 52 | required=False, 53 | help="""URL to the git repository.""") 54 | 55 | parser.add_argument( 56 | '--commit', 57 | type=str, 58 | required=False, 59 | help="""Commit to branch from. If none is provided, the head of the repo will be used.""") 60 | 61 | parser.add_argument( 62 | '--branch_if_changed', 63 | action='store_true', 64 | required=False, 65 | help="""If there was a change in version form the previous commit to the current one, create a version branch.""") 66 | 67 | return parser 68 | 69 | def _get_repo_version(repo_path, variants, variant_config_files=None): 70 | ''' 71 | Get the version of the first package that appears in the recipes list for the feedstock. 72 | Copying to a temporary directory is to get around an unknown issue with conda's render 73 | function which seems to cache the file the first time it is read, even if the file is 74 | changed by checking out a new git commit. 75 | ''' 76 | saved_working_directory = os.getcwd() 77 | with tempfile.TemporaryDirectory() as renamed_path: 78 | new_repo_path = os.path.join(renamed_path, "repo") 79 | shutil.copytree(repo_path, new_repo_path) 80 | os.chdir(os.path.abspath(new_repo_path)) 81 | for variant in variants: 82 | build_config_data, _ = build_feedstock.load_package_config(variants=variant, permit_undefined_jinja=True) 83 | if build_config_data["recipes"]: 84 | for recipe in build_config_data["recipes"]: 85 | rendered_recipe = conda_utils.render_yaml(recipe["path"], 86 | variants=variant, 87 | variant_config_files=variant_config_files, 88 | permit_undefined_jinja=True) 89 | for meta, _, _ in rendered_recipe: 90 | package_version = meta.meta['package']['version'] 91 | if package_version and package_version != "None": 92 | os.chdir(saved_working_directory) 93 | return package_version, meta.meta['package']['name'] 94 | 95 | os.chdir(saved_working_directory) 96 | raise ValueError("Error: Unable to determine current version of the feedstock") 97 | 98 | def _get_repo_name(repo_url): 99 | if not repo_url.endswith(".git"): 100 | repo_url += ".git" 101 | return os.path.splitext(os.path.basename(repo_url))[0] 102 | 103 | def _versions_match(current_version, previous_version): 104 | if current_version == previous_version: 105 | return True 106 | version_regex = re.compile(r"^(.+\..+)\.(.+)$") 107 | current_match = version_regex.match(current_version) 108 | previous_match = version_regex.match(previous_version) 109 | if not current_match or not previous_match: 110 | return False 111 | if len(current_match.groups()) != 2 or len(previous_match.groups()) != 2: 112 | return False 113 | return current_match.groups()[0] == previous_match.groups()[0] 114 | 115 | def _create_version_branch(arg_strings=None):# pylint: disable=too-many-branches 116 | parser = _make_parser() 117 | args = parser.parse_args(arg_strings) 118 | 119 | if args.repository: 120 | repo_name = _get_repo_name(args.repository) 121 | repo_url = args.repository 122 | repo_path = os.path.abspath(os.path.join(args.repo_dir, repo_name)) 123 | print("--->Making clone location: " + repo_path) 124 | os.makedirs(repo_path, exist_ok=True) 125 | print(f"--->Cloning {repo_name}") 126 | git_utils.clone_repo(repo_url, repo_path) 127 | elif args.repo_dir: 128 | repo_path = args.repo_dir 129 | else: 130 | repo_path = "./" 131 | 132 | if args.commit: 133 | git_utils.checkout(repo_path, args.commit) 134 | current_commit = git_utils.get_current_branch(repo_path) 135 | config_file = None 136 | if args.conda_build_configs: 137 | config_file = os.path.abspath(args.conda_build_configs) 138 | try: 139 | git_utils.checkout(repo_path, "HEAD~") 140 | previous_version, _ = _get_repo_version(repo_path, utils.ALL_VARIANTS(), config_file) 141 | 142 | git_utils.checkout(repo_path, current_commit) 143 | current_version, _ = _get_repo_version(repo_path, utils.ALL_VARIANTS(), config_file) 144 | 145 | if args.branch_if_changed and _versions_match(current_version, previous_version): 146 | print("The version has not changed, no branch created.") 147 | else: 148 | if args.branch_if_changed: 149 | print("The version has changed, creating branch.") 150 | git_utils.checkout(repo_path, "HEAD~") 151 | branch_name = "r" + previous_version 152 | else: 153 | print("Creating branch.") 154 | branch_name = "r" + current_version 155 | 156 | if git_utils.branch_exists(repo_path, branch_name): 157 | print(f"The branch {branch_name} already exists.") 158 | else: 159 | git_utils.create_branch(repo_path, branch_name) 160 | git_utils.push_branch(repo_path, branch_name) 161 | 162 | if args.branch_if_changed: 163 | git_utils.checkout(repo_path, current_commit) 164 | 165 | if args.repository: 166 | shutil.rmtree(repo_path) 167 | except Exception as exc:# pylint: disable=broad-except 168 | if args.branch_if_changed: 169 | git_utils.checkout(repo_path, current_commit) 170 | raise exc 171 | 172 | if __name__ == '__main__': 173 | _create_version_branch() 174 | -------------------------------------------------------------------------------- /tests/get_licenses_test.py: -------------------------------------------------------------------------------- 1 | # ***************************************************************** 2 | # (C) Copyright IBM Corp. 2021. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ***************************************************************** 16 | 17 | import os 18 | import pathlib 19 | import pytest 20 | import shutil 21 | import sys 22 | from importlib.util import spec_from_loader, module_from_spec 23 | from importlib.machinery import SourceFileLoader 24 | 25 | test_dir = pathlib.Path(__file__).parent.absolute() 26 | 27 | spec = spec_from_loader("opence", SourceFileLoader("opence", os.path.join(test_dir, '..', 'open_ce', 'open-ce'))) 28 | opence = module_from_spec(spec) 29 | spec.loader.exec_module(opence) 30 | 31 | import open_ce.get_licenses as get_licenses 32 | import open_ce.constants as constants 33 | 34 | from open_ce.errors import OpenCEError 35 | 36 | def test_get_licenses(mocker): 37 | ''' 38 | This is a complete test of `get_licenses`. 39 | ''' 40 | mocker.patch("open_ce.get_licenses.show_warning", 41 | side_effect=(lambda warning, *additional_args, **kwargs: print(warning.value[1].format(*additional_args), file=sys.stderr, **kwargs))) 42 | 43 | output_folder = "get_licenses_output" 44 | template_file = "tests/open-ce-licenses.template" 45 | opence._main(["get", get_licenses.COMMAND, "--conda_env_file", "tests/test-conda-env3.yaml", "--output_folder", output_folder, "--template_files", template_file]) 46 | 47 | output_file = os.path.join(output_folder, constants.DEFAULT_LICENSES_FILE) 48 | assert os.path.exists(output_file) 49 | with open(output_file, encoding='utf8') as file_stream: 50 | license_contents = file_stream.read() 51 | 52 | print(license_contents) 53 | assert "pytest 6.2.2 https://github.com/pytest-dev/pytest/ MIT" in license_contents 54 | assert "libopus 1.3.1 http://opus-codec.org/development/ BSD-3-Clause Copyright 2001-2011 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo" in license_contents 55 | 56 | template_output_file = os.path.join(output_folder, os.path.splitext(os.path.basename(template_file))[0] + ".txt") 57 | assert os.path.exists(template_output_file) 58 | with open(template_output_file, encoding='utf8') as file_stream: 59 | template_contents = file_stream.read() 60 | 61 | print(template_contents) 62 | assert "libopus" in template_contents or "tzdata" in template_contents 63 | shutil.rmtree(output_folder) 64 | 65 | def test_get_licenses_licenses_file_arg(): 66 | ''' 67 | This tests that the --licenses_file argument works properly. 68 | ''' 69 | output_folder = "get_licenses_output" 70 | template_file = "tests/open-ce-licenses.template" 71 | opence._main(["get", get_licenses.COMMAND, "--licenses_file", "tests/open-ce-licenses.csv", "--output_folder", output_folder, "--template_files", template_file]) 72 | 73 | template_output_file = os.path.join(output_folder, os.path.splitext(os.path.basename(template_file))[0] + ".txt") 74 | assert os.path.exists(template_output_file) 75 | with open(template_output_file, encoding='utf8') as file_stream: 76 | template_contents = file_stream.read() 77 | 78 | print(template_contents) 79 | assert "libopus" in template_contents 80 | shutil.rmtree(output_folder) 81 | 82 | def test_get_licenses_failed_conda_create(mocker): 83 | ''' 84 | This tests that an exception is thrown when `conda env create` fails. 85 | ''' 86 | output_folder = "get_licenses_output" 87 | mocker.patch('open_ce.utils.run_command_capture', side_effect=[(False, "", "")]) 88 | 89 | with pytest.raises(OpenCEError) as err: 90 | opence._main(["get", get_licenses.COMMAND, "--conda_env_file", "tests/test-conda-env3.yaml", "--output_folder", output_folder]) 91 | 92 | assert "Error generating licenses file." in str(err.value) 93 | 94 | def test_get_licenses_failed_conda_remove(mocker): 95 | ''' 96 | This tests that an exception is thrown when `conda env remove` is called. 97 | ''' 98 | output_folder = "get_licenses_output" 99 | mocker.patch('open_ce.utils.run_command_capture', side_effect=[(True, "", ""), (False, "", "")]) 100 | mocker.patch('open_ce.get_licenses.LicenseGenerator._add_licenses_from_environment', return_value=[]) 101 | 102 | with pytest.raises(OpenCEError) as err: 103 | opence._main(["get", get_licenses.COMMAND, "--conda_env_file", "tests/test-conda-env3.yaml", "--output_folder", output_folder]) 104 | 105 | assert "Error generating licenses file." in str(err.value) 106 | 107 | def test_get_licenses_no_conda_env(): 108 | ''' 109 | This test ensures that an exception is thrown when no conda environment is provided. 110 | ''' 111 | with pytest.raises(OpenCEError) as err: 112 | opence._main(["get", get_licenses.COMMAND]) 113 | 114 | assert "The \'--conda_env_file\' argument is required." in str(err.value) 115 | 116 | def test_add_licenses_from_info_file(mocker, capfd): 117 | ''' 118 | This is a complete test of the add_licenses_from_info_file method. 119 | ''' 120 | mocker.patch("open_ce.get_licenses.show_warning", 121 | side_effect=(lambda warning, *additional_args, **kwargs: print(warning.value[1].format(*additional_args), file=sys.stderr, **kwargs))) 122 | output_folder = "get_licenses_output" 123 | gen = get_licenses.LicenseGenerator() 124 | license_info = get_licenses._get_info_file_packages(os.path.join("tests", "test-open-ce-info-1.yaml")) 125 | gen.add_licenses_from_info_files(license_info) 126 | 127 | captured = capfd.readouterr() 128 | assert "Unable to clone source for 'bad_git_package" in captured.err 129 | assert "Unable to download source for 'bad_url" in captured.err 130 | 131 | gen.write_licenses_file(output_folder) 132 | 133 | output_file = os.path.join(output_folder, constants.DEFAULT_LICENSES_FILE) 134 | assert os.path.exists(output_file) 135 | with open(output_file, encoding='utf8') as file_stream: 136 | license_contents = file_stream.read() 137 | 138 | print(license_contents) 139 | assert "DefinitelyTyped.google.analytics\tebc69904eb78f94030d5d517b42db20867f679c0\thttps://raw.githubusercontent.com/DefinitelyTyped/DefinitelyTyped/ebc69904eb78f94030d5d517b42db20867f679c0/chai/chai.d.ts\tMIT\tCopyright Jed Mao ,, Copyright Bart van der Schoor ,, Copyright Andrew Brown ,, Copyright Olivier Chevet " in license_contents 140 | assert "bad_url\t123\tnot_a_url.tar.gz\tNO_LICENSE\tNot a real copy right" in license_contents 141 | assert "google-test\t1.7.0\thttps://github.com/google/googletest/archive/release-1.7.0.zip\tBSD-equivalent\tCopyright 2008, Google Inc. All rights reserved." in license_contents 142 | assert "kissfft\t36dbc057604f00aacfc0288ddad57e3b21cfc1b8\thttps://github.com/mborgerding/kissfft/archive/36dbc057604f00aacfc0288ddad57e3b21cfc1b8.tar.gz\tBSD-equivalent\tCopyright (c) 2003-2010 Mark Borgerding . All rights reserved." in license_contents 143 | 144 | # Make sure google-test only appears once 145 | assert license_contents.count("google-test\t1.7.0") == 1 146 | shutil.rmtree(output_folder) 147 | shutil.rmtree(constants.TMP_LICENSE_DIR) 148 | 149 | def test_no_info_file(): 150 | ''' 151 | Test that an empty file is returned when an info file doesn't exist at the provided path. 152 | ''' 153 | output_folder = "get_licenses_output" 154 | gen = get_licenses.LicenseGenerator() 155 | license_info = get_licenses._get_info_file_packages(os.path.join("tests", "no-file.yaml")) 156 | gen.add_licenses_from_info_files(license_info) 157 | gen.write_licenses_file(output_folder) 158 | 159 | output_file = os.path.join(output_folder, constants.DEFAULT_LICENSES_FILE) 160 | assert os.path.exists(output_file) 161 | with open(output_file, encoding='utf8') as file_stream: 162 | license_contents = file_stream.read() 163 | 164 | assert license_contents == "" 165 | 166 | shutil.rmtree(output_folder) 167 | 168 | def test_clean_copyright_string(): 169 | assert get_licenses._clean_copyright_string("// Not a Copyright 2020 ") == "Copyright 2020" 170 | assert get_licenses._clean_copyright_string("// Not a Copyright 2020 ", primary=False) == "Not a Copyright 2020" 171 | assert get_licenses._clean_copyright_string("// -------------") == "" 172 | --------------------------------------------------------------------------------